
    bi                        d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZddlmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZm Z  dd
l!m"Z" ddl#m$Z$m%Z% g fdZ&d Z'de jP                  de)e*e*f   fdZ+de jP                  de,e-e*   e)e*e*f   f   fdZ.de jP                  de)e*e*f   fdZ/de jP                  de,e-e*   e)e*e*f   f   fdZ0	 dde jP                  de)e*e*f   de1de,e jP                  e)e*e*f   f   fdZ2de-e*   de)e*e*f   fdZ3dede jP                  de,e jP                  e)e*e
f   f   fdZ4d Z5 G d d      Z6y)     N)literal_eval)which)Any   )SageMakerConfig)DynamoBackendPrecisionTypeis_ccl_availableis_fp8_availableis_hpu_availableis_ipex_availableis_mlu_availableis_musa_availableis_npu_availableis_sdaa_availableis_torch_xla_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)get_free_portis_port_in_usemerge_dicts)compare_versions   )DistributedTypeSageMakerDistributedTypec                     |j                  |      \  }}t        |       j                         D ].  \  }}|t        |      j                         v s"t	        |||       0 |S )z4
    Filters out all `accelerate` specific args
    )parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalues          R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/accelerate/utils/launch.py_filter_argsr*   /   s`     )),7KHa4j&&( *
U$x.%%''Hc5)* O    c                      dD  cg c]  } t        |       s|  }} t        |      dk(  rt        d      |d   }t        j                  |dg      }d|v r|dddd	fS |d
dddfS c c} w )a  
    Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
    are: OpenMPI, Intel MPI, or MVAPICH.

    Returns: Program name and arg names for hostfile, num processes, and processes per node
    )mpirunmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPI
--hostfilez-nz
--npernodez	--bind-toz-fz-ppn )r   lenOSError
subprocesscheck_output)xmpi_appsmpi_appmpirun_versions       r)   _get_mpirun_argsr9   :   s     1=aE!H=H=
8}tuu qkG,,g{-CDNn$dL+EE dFB.. >s
   A#A#r"   current_envc                 D   d}t        |       D ]  }|j                  d      st        | |      }|$|dk(  r=t        |d         ||dz   <   t        |d         ||dz   <   t        |d         ||d	z   <   ft        t        | |            || |j	                          <    |S )
z.
    Setup the FP8 environment variables.
    ACCELERATE_fp8_fp8_override_linear_precisionr   FP8_OVERRIDE_FPROPr   FP8_OVERRIDE_DGRADr   FP8_OVERRIDE_WGRAD)r   
startswithgetattrstrupper)r"   r:   prefixargr(   s        r)   setup_fp8_envrH   R   s     FDz 	T>>&!D#&E 99ADU1XK)= =>ADU1XK)= =>ADU1XK)= =><?c@R<SK6(399;- 89	T r+   returnc                 	   g }| j                   r| j                  rt        d      t        | dd      }| j                  }| j
                  `t               \  }}}}}t        | dd      }	|r|rt        ||z        nd}
|||| j
                  ||
gz  }|r||t        |      gz  }|r|||	gz  }| j                   s<|j                  t        j                         | j                  r|j                  d       |j                  | j                         |j                  | j                         t        j                  j!                         }t        | j"                  xs | j$                        |d<   | j&                  rd	|d
<   | j(                  dk7  r| j(                  t+               r| j(                  |d<   nt-               r| j(                  |d<   nwt/               r| j(                  |d<   n]t1               r| j(                  |d<   nCt3               r| j(                  |d<   n)t5               r| j(                  |d<   n| j(                  |d<   |dkD  r&| j6                  J d       | j8                  J d       t;               rt        | dd      nd}||dkD  s|dkD  rQ| j6                  | j6                  nd|d<   | j8                  t        | j8                        nd|d<   t        |      |d<   |d   rd|d<   t        d      |d<   	 t=        | j>                  jA                               }t        |      |d#<   | j>                  jA                         d$k(  r!tE               stG        d%      tI        | |      }	 tK        | jL                  jO                               }|jP                  |d'<   | jR                  |d(<   t        | jT                        |d)<   t        | jV                        |d*<   t        | jX                        |d+<   t        | jZ                        |d,<   t]               r&t        | j^                        jA                         |d-<   | j`                  rd|d.<   ||fS # t        $ r= t        d | j>                  jA                          d!t=        jB                          d"      w xY w# t        $ r= t        d&| jL                  jO                          d!tK        jB                          d"      w xY w)/zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    0--module and --no_python cannot be used togethernum_processesNzbind-tosocket1z-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKMLU_VISIBLE_DEVICESSDAA_VISIBLE_DEVICESMUSA_VISIBLE_DEVICESASCEND_RT_VISIBLE_DEVICESHABANA_VISIBLE_MODULESCUDA_VISIBLE_DEVICESr   zFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.
mpirun_cclr   z	127.0.0.1MASTER_ADDR29500MASTER_PORTCCL_WORKER_COUNTzgranularity=fine,compact,1,0KMP_AFFINITYKMP_BLOCKTIMEUnknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONfp8rFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.Unknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMIC*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONOMP_NUM_THREADSACCELERATE_USE_IPEXACCELERATE_CPU_AFFINITY)1	no_pythonmodule
ValueErrorrC   num_machinesmpirun_hostfiler9   rD   appendsys
executabletraining_scriptextendtraining_script_argsosenvironcopycpuuse_cpudebuggpu_idsr   r   r   r   r   r   main_process_ipmain_process_portr
   r	   mixed_precisionlowerlistr   RuntimeErrorrH   r   dynamo_backendrE   r(   dynamo_modedynamo_use_fullgraphdynamo_use_dynamicdynamo_use_regional_compilationnum_cpu_threads_per_processr   ipexenable_cpu_affinity)r"   cmdrL   rs   mpi_app_namehostfile_argnum_proc_argproc_per_node_argbind_to_argbind_tonproc_per_noder:   ccl_worker_countr   r   s                  r)   prepare_simple_launcher_cmd_envr   d   s    C~~$++KLLD/48M$$L'ScSePlL2C[$	84?LQ]]l:;cf  
 	
 L#m"455CK))C>>

3>>";;JJtJJt##$JJt(()**//#K(+DHH,D(EK$%zz/5+,||u!9.2llK*+15K-. 26,,K./ 26,,K./7;||K3448LLK0126,,K./a##/ 	
T	
/ %%1 	
V	
1 :J9Kwt\15QR!ma&7L1<L=A=Q=Q=]T%9%9cnM"DHDZDZDfS)?)?%@lsM"*-.>*?&''(&DN#'*1vO$
'(<(<(B(B(DE 14O0DK,-!!#u,! E  $D+6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=%()I)I%JK!"-0^-A-A-C)*14-.=  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   >#Q .#R ARAS#c                    | j                   dk(  rt               | _         n| j                   d| _         | j                  }| j                  }| j                  }| j                   }|dkD  rxt        ||z        | _        t        |      | _        t        | j                        | _
        t        | dd      r!t        |      | _        t        |      | _        n/| d| | _        n"t        |      | _        |t        |      | _        |dk  xs t        | j                        dk(  }|r@t        |      r5|dk  r!d| _        t#        j$                  d	| d
       nt'        d| d      | j(                  r| j*                  rt-        d      | j(                  rd| _        n| j*                  rd| _        t.        j0                  j3                         }| j4                  rd|d<   t        | dd      }|dk7  rq| j6                  et9               r||d<   nUt;               r||d<   nEt=               r||d<   n5t?               r||d<   n%tA               r||d<   ntC               r||d<   n||d<   | jD                  jG                         }	 tI        |      }t        |      |d<   | jD                  jG                         dk(  r!tM               stO        d      tQ        | |      }	 tS        | jT                  jW                               }	|	jX                  |d <   | jZ                  |d!<   t        | j\                        |d"<   t        | j^                        |d#<   t        | j`                        |d$<   | jb                  r=d|d%<   | jd                  r| jf                  st-        d&      ti        | d'      rt        | jj                        nd(|d)<   t        | jl                        |d*<   t        | jn                        jG                         |d+<   t        | jp                        jG                         |d,<   t        | jr                        |d-<   | jt                  t        | jt                        |d.<   | jv                  t        | jv                        |d/<   | jx                  t        | jx                        |d0<   | jz                  t        | jz                        |d1<   t        | j|                        jG                         |d2<   t        | j~                        jG                         |d3<   t        | jd                        jG                         |d4<   t        | jf                        jG                         |d5<   t        | j                        jG                         |d6<   t        | d7d      t        | j                        |d8<   | j                  rd9}
d|d:<   t        | j                        ||
d;z   <   t        | j                        ||
d<z   <   t        | j                        ||
d=z   <   | j                  t        | j                        ||
d>z   <   | j                  t        | j                        ||
d?z   <   | j                  t        | j                        ||
d@z   <   | j                  t        | j                        ||
dAz   <   t        | j                        |dB<   | j                  rd(|dC<   | j                  s|S dD}
| j                  rd|dE<   t        | j                        ||
dFz   <   t        | j                        ||
dGz   <   t        | j                        ||
dHz   <   t        | j                        ||
dIz   <   | j                  dkD  rt        | j                        ||
dJz   <   |S # t,        $ r% t-        d| dtI        jJ                          d      w xY w# t,        $ r= t-        d| jT                  jW                          dtS        jJ                          d      w xY w)Kz_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    r   N<s  r   same_networkF:TPort `  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.3Tried to launch distributed communication on port `$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rK   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`fsdp_versionrN   FSDP_VERSIONFSDP_SHARDING_STRATEGYFSDP_RESHARD_AFTER_FORWARDFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESFSDP_ACTIVATION_CHECKPOINTINGfsdp_ignored_modulesFSDP_IGNORED_MODULESMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREE	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERrm   ro   PARALLELISM_CONFIG_!ACCELERATE_USE_PARALLELISM_CONFIGDP_REPLICATE_SIZETP_SIZECP_SIZEDP_SHARD_SIZECP_COMM_STRATEGY)Rr   r   rL   rs   r   rD   r   nnodesintmachine_rank	node_rankrC   master_addrmaster_portrdzv_endpointr   
standalonewarningswarnConnectionErrorrq   rp   rr   r{   r|   r}   r   r   r   r   r   r   r   r   r   r   r	   r   r   r   rH   r   r   rE   r(   r   r   r   r   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_stateshasattrr   fsdp_sharding_strategyfsdp_reshard_after_forwardfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetchfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsfsdp_activation_checkpointingr   use_megatron_lmmegatron_lm_tp_degreemegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizerr   r   use_parallelism_config$parallelism_config_dp_replicate_sizeparallelism_config_tp_sizeparallelism_config_cp_size parallelism_config_dp_shard_size#parallelism_config_cp_comm_strategy)r"   rL   rs   r   r   need_port_checkr:   r   r   r   rF   s              r)   prepare_multi_gpu_envr      s   
 "!.				'!&&&M$$L**O..a!-<"?@,'T../4/"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5K*+18K-. 29K./ 29K./7>K344;K0129K./**002Ou'8 14O0DK,-!!#u,! E  $D+6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=}}-3)*..t7S7Svww@Gn@]c$*;*;&<cfN# 14D4O4O0P,-478W8W4X4^4^4`01-01I1I-J-P-P-R)*-01I1I-J)*%%136t7Q7Q3RK/022>:=d>e>e:fK67&&2478S8S4TK01$$025d6O6O2PK.//243M3M/N/T/T/V+,.1$2K2K.L.R.R.T*+8;D<_<_8`8f8f8h4514T5Q5Q1R1X1X1Z-.7:4;];]7^7d7d7f344/6B25d6O6O2PK./4:01,/0J0J,KF[(),/0J0J,KF[()478Z8Z4[F001--98;D<^<^8_K!44500<;>t?d?d;eK!77811=<?@f@f<gK!88955A@CDDnDn@oK!<<=%()I)I%JK!"14-.&&"F"";A78478a8a4bF001*-d.M.M*NFY&'*-d.M.M*NFY&'03D4Y4Y0ZF_,-**Q.7:4;c;c7dK!334k  u9/9JJ[\i\n\n\p[qqrsttu  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   :^ #^8 .^58A_>c                    | j                   dk(  rt               | _         n| j                   d| _         | j                  }| j                  }| j                  }| j                   }d}| j
                  t        d   | _        |dkD  r| j
                  t        d   k7  rdg}|j                  dt        | j                        g       | j
                  dk(  r@t        ddd	      rt        d
      |j                  dt        | j                        dg       n'|j                  ddt        | j
                        g       | j                  '|j                  dt        | j                        g       nf| j                  '|j                  dt        | j                        g       n3|j                  dt        | j                  | j                  z        g       |r|j                  dt        |      g       |j                  dt        |      g       | j                  r| j                   rt        d      | j                  r|j#                  d       n| j                   r|j#                  d       |j#                  | j$                         |j                  | j&                         n|dkD  r| j
                  t        d   k(  rxt        ||z        | _        t        |      | _        t-        | j                        | _        t1        | dd      r!t        |      | _        t        |      | _        n/| d| | _        n"t        |      | _        |t        |      | _        |dk  xs t-        | j                        dk(  }|r@t9        |      r5|dk  r!d| _        t=        j>                  d| d       ntA        d| d      | j                  r| j                   rt        d      | j                  rd| _        n| j                   rd| _        tB        jD                  jG                         }| jH                  rd|d <   t1        | d!d"      }|d"k7  rq| jJ                  etM               r||d#<   nUtO               r||d$<   nEtQ               r||d%<   n5tS               r||d&<   n%tU               r||d'<   ntW               r||d(<   n||d)<   	 tY        | jZ                  j]                               }	ta        d-tB        jb                  je                  d,            |d-<   t        |	      |d.<   | jZ                  j]                         d/k(  r!tg               sti        d0      tk        | |      }t        | jl                        j]                         |d1<   d|d2<   | jn                  t        | jn                        |d3<   | jp                  t        | jp                        |d4<   | jr                  &t        | jr                        j]                         |d5<   | jt                  &t        | jt                        j]                         |d6<   | jv                  &t        | jv                        j]                         |d7<   | jx                  &t        | jx                        j]                         |d8<   | jz                  &t        | jz                        j]                         |d9<   | j|                  t        | j|                        |d:<   | j~                  rd;|d<<   | j                  t        | j                        |d=<   ||fS # t        $ r= t        d*| jZ                  j]                          d+tY        j^                          d,      w xY w)>zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    r   Nr   r   	deepspeedr/   nossh<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrK   z--modulez--no_pythonr   Fr   Tr   r   r   r   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   ra   rb   rc   
PYTHONPATHrd   re   rf   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILErN   ro   (ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Ar   r   rL   rs   r   deepspeed_multinode_launcherr   ry   rD   deepspeed_hostfiler   rr   r   deepspeed_exclusion_filterdeepspeed_inclusion_filterrq   rp   ru   rx   rz   r   r   r   r   rC   r   r   r   r   r   r   r   r   r{   r|   r}   r   r   r   r   r   r   r   r   r	   r   r   r   env_var_path_addpathabspathr   r   rH   'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_filer   deepspeed_moe_layer_cls_names)
r"   rL   rs   r   r   r   r   r:   r   r   s
             r)   prepare_deepspeed_cmd_envr  q  s   
 "!.				'!&&&M$$L**O..
C ((0,I!,L)aD==A^_`Aaam

L#d&=&=">?@,,7S(; !NOOJJs4+<+<'=zJKJJ)<T=^=^9_`a**6JJ778 ,,8JJ778 JJc$*<*<@Q@Q*Q&RSTJJ_)=>?

OS):%;<=;;4>>OPP[[JJz"^^JJ}%

4''(

4,,-		d??C`abCcc!-<"?@,'T../4/"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5K*+18K-. 29K./ 29K./7>K344;K0129K./
'(<(<(B(B(DE !1rwws?S TK03O0DK,-!!#u,! E  $D+614T5a5a1b1h1h1jK-..4K*+"9<T__9M56''3@CDDdDd@e<=)69$:P:P6Q6W6W6Y23$$0GJ4KhKhGiGoGoGqCD  ,CFtG`G`CaCgCgCi?@'9<T=Q=Q9R9X9X9Z56"".EHIdIdEeEkEkEmAB!!-:=d>X>X:Y6714-.))5BEdFhFhBi>?G  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

s   6#\ A]!podc                     | j                   dk(  r#t        d      r| j                  rd|d<   nd|d<   | j                  rd|d<   |r"| j                  | _        | j                  | _        | |fS )	zY
    Prepares and returns an environment with the correct TPU environment variables.
    bf16T)check_is_tpurN   XLA_DOWNCAST_BF16XLA_USE_BF16rP   rQ   )r   r   downcast_bf16r   tpu_vmvmtpu_nametpu)r"   r:   r  s      r)   prepare_tpur    sn     v%*@d*S/2K+,*-K'zz/5+,
++==r+   nargsc                 8   t        |       dk  ri S d }t        j                         }|j                  |       \  }}t	        |      D ]  \  }}|j                  d      sd }|dz   t        |      k  r"||dz      j                  d      rt        d      t        d      ||j                  ||       n|j                  ||        |j                  |       j                  j                         D 	ci c]  \  }}	||	dv rt        |	      n|	 c}	}S c c}	}w )	Nr   c                 f    	 t        |       } | dz  | k(  rt        |       S | S # t        $ r | cY S w xY w)Nr   )floatr   rr   )ss    r)   _infer_typez+_convert_nargs_to_dict.<locals>._infer_type(  s=    	aAAv{1vH 	H	s   " " 00)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)action)TrueFalse)r1   argparseArgumentParserr   	enumeraterB   rr   add_argument
parse_args__dict__r   r   )
r  r!  r#   r&   unknownindexargumentr$  r'   r(   s
             r)   _convert_nargs_to_dictr0  #  s6   
5zA~	 $$&F((/JAw$W- =x{+Fqy3w<'519%00=$ G  ! C  ~##H;#?##HV#<#=* !++E2;;AACC 	U.?%?l5!UJ  s   6Dsagemaker_configc                 	   t        d       | j                  t        j                  d<   | j                  | j                  t        j                  d<   n^|j
                  G|j                  ;|j
                  t        j                  d<   |j                  t        j                  d<   nt        d      t        j                  j                  |j                        }|sd}t        j                  j                  |j                        }|j                  d      st        d	| d
      t        d       t        |j                        }	 t!        |j"                  j%                               }	 t)        |j*                  j-                               }dt/        |      |j0                  |j2                  t/        |j4                        t/        |j6                        t/        |j8                        | j:                  j0                  d}|j"                  j%                         dk(  r!t=               st?        d      tA        ||      }d }| j:                  tB        jD                  k(  rddddiii}d }	| jF                  t        d| jF                   d       i }	tI        | jF                        5 }
tK        |
      D ]5  \  }}|dk(  r|jM                  d      }|d   jO                         |	|d   <   7 	 d d d        t        d|	        d }| jP                  t        d| jP                   d       g }tI        | jP                        5 }
tK        |
      D ]F  \  }}|dk(  r|jM                  d      }|d   |d   jO                         d}|jS                  |       H 	 d d d        t        d|        t        d        | jT                  ||| jV                  | jX                  | jZ                  | j\                  | j^                  | j`                  | jb                  d!||||d"}| jd                  tg        | jd                  |      }||	fS # t        $ r= t        d|j"                  j%                          dt!        j&                          d      w xY w# t        $ r= t        d|j*                  j-                          dt)        j&                          d      w xY w# 1 sw Y   xY w# 1 sw Y   JxY w)#Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerc   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersra   rb   rg   rP   )ACCELERATE_USE_SAGEMAKERrd   rh   ri   rj   rk   rl   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEre   rf   smdistributeddataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configdistributionhyperparametersenvironmentmetric_definitions)4printregionr{   r|   profileaws_access_key_idaws_secret_access_keyr2   r  dirnamerx   basenameendswithrr   r0  rz   r	   r   r   r   r   r   rE   rD   r(   r   r   r   r   distributed_typer   r   rH   r   DATA_PARALLELsagemaker_inputs_fileopenr)  splitstripsagemaker_metrics_fileru   r@  iam_role_namerD  rE  rF  rG  rs   ec2_instance_typeadditional_argsr   )r1  r"   rB  rA  rL  r   r   rM  rK  sagemaker_inputsfileilinelsagemaker_metricsmetric_dicts                   r)   prepare_sagemager_args_inputsrh  M  s    

45'7'>'>BJJ#$ +$4$<$<

=!				+0J0J0V*.*@*@

&'.2.H.H

*+uvv !5!56J
''""4#7#78K&ST_S``abcc	
34,T-F-FGO
'(<(<(B(B(DE
&t':':'@'@'BC %+&)/&:%3%9%9"&"2"2+.t/H/H+I),T-D-D)E69$:^:^6_1A1R1R1X1X	K !!#u,! E  $D+6L((,D,R,RR'.9d:K)LM --9./?/U/U.VV[\]"889 	6T$T? 646JJt$)*1 1&	6	6 	)*:);<= ..:/0@0W0W/XX]^_"99: 		6d$T? 646JJt$aDqTZZ\ "((56		6 	*+<*=>? 

%//"  .. 0 E E+;;&11)77*77);; %$*"/D$ ''3+;;TB!!!m  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

@	6 	6		6 		6s4   #P? ,#R 'AS AS?ARASSS(c                     t         j                  j                  | d      j                  d      D cg c]  }t	        |      dkD  s| }}|j                  t        |             dj                  |      S c c}w )z
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    r0   r   r   )r{   r|   getr[  r1   ru   rD   join)env_var_namepath_to_addppathss       r)   r  r    sa    
 

|R8>>sCR1s1vPQzQRER	LL[!"88E? Ss   A7A7c                       e Zd ZdZddZd Zy)PrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    c                 @    || _         t        |      | _        || _        y )N)launcherr   rW  r   )selfrs  rW  r   s       r)   __init__zPrepareForLaunch.__init__  s      /0@ A
r+   c                 t   | j                   rt        t        j                  j	                  d            }t        j                  j	                  d      }t
        j                  j                  d|t
        j                  j                  ||      |       n| j                  t        j                  t        j                  t        j                  t        j                  t        j                  t        j                   fv rt#        |      t        j                  d<   t        t        j                  j	                  dd            }t        t        j                  j	                  dd	            }t#        ||z  |z         t        j                  d
<   t#        d      t        j                  d<    | j$                  |  y )N
WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROCr   	NODE_RANKr   RANKFORK_LAUNCHED)r   r   r{   r|   rj  torchdistributedinit_process_group	FileStorerW  r   	MULTI_GPU	MULTI_MLU
MULTI_MUSA	MULTI_NPU	MULTI_XPU	MULTI_CPUrD   rs  )rt  r.  r"   r|  rdv_filenprocr   s          r)   __call__zPrepareForLaunch.__call__  s9   ::RZZ^^L9:Jzz~~&ABH00''11(JG%	 1  ""%%%%&&%%%%%%'
 
 (+5zBJJ|$

w23EBJJNN;:;I!$UY%6%>!?BJJv&)!f

?#tr+   N)NOF)__name__
__module____qualname____doc__ru  r   r+   r)   rq  rq    s    

r+   rq  )F)7r'  r{   r3   rv   r   astr   shutilr   typingr   r  commands.config.config_argsr   utilsr   r	   r
   r   r   r   r   r   r   r   r   r   utils.constantsr   utils.otherr   r   r   utils.versionsr   dataclassesr   r   r*   r9   	NamespacedictrD   rH   tupler   r   r   r  boolr  r0  rh  r  rq  r  r+   r)   <module>r     s    	  
      9    < D D - B -/ /0** c3h $d(*<*< dtCyRVWZ\_W_R`G`Aa dNc 2 2 ctCH~ cL[H$6$6 [5cDQTVYQYNAZ;[ [~ HM


+/S>@D
8tCH~-.('$s) 'S#X 'Ts"%s"-5-?-?s"
8tCH~-.s"l+ +r+   