
    biI                         d dl mZ d dlZd dlmc mZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d d	lmZmZ d
 ZddZddZd Zd Zd ZddZ	 ddZ d Z!d Z"d Z#e$dk(  r e"        yy)    )deepcopyN)AdamW)LambdaLR)
DataLoader)AcceleratorDataLoaderConfigurationGradientAccumulationPlugin)GradientState)RegressionDatasetRegressionModel)DistributedTypeset_seedc           	         t        | j                         |j                               D ]  \  }}|j                  s|sSt        j                  |j
                  |j
                  fi |du rCJ d| d|j
                   d|j
                   d       t        j                  |j
                  |j
                  fi |du rJ d| d|j
                   d|j
                   d        y )	NF7Gradients in sync when they should not be at iteration z:
model_a grad (z) == model_b grad ()T7Gradients not in sync when they should be at iteration z) != model_b grad ()zip
parametersrequires_gradtorchallclosegrad)model_amodel_bdid_step	iterationkwargsparam
grad_params          b/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/accelerate/test_utils/scripts/test_sync.pycheck_model_parametersr!      s&    !3!3!5w7I7I7KL z"">>%**jooHHEQ I)Tefkfpfpeq  rE  FP  FU  FU  EV  VW  XQ
 >>%**jooHHDP I)Tefkfpfpeq  rE  FP  FU  FU  EV  VW  XP    c                     | j                           | |      }t        j                  ||j                  |j                              }|s ||j
                  z  }|j                          y |j                  |       y N)trainFmse_losstodevicegradient_accumulation_stepsbackward)modelinputtargetacceleratordo_backwardoutputlosss          r    
step_modelr3   -   s[    	KKM5\F::ffii67D777T"r"   c                    t        d       t               }t        |      }t        d      }t	        |d      }|j                  | j                         |rRt        |j                         d      }t        |j                         d      }t        |d 	      }t        |d
 	      }|r| j                  ||      \  }}}}n| j                  ||      \  }}|r	||||fS |||fS )z3Returns everything needed to perform basic training*   P   length   
batch_sizeMbP?)paramslrc                     | dz  S Ng? epochs    r    <lambda>z$get_training_setup.<locals>.<lambda>C   s
    eTk r"   )	lr_lambdac                     | dz  S r@   rA   rB   s    r    rD   z$get_training_setup.<locals>.<lambda>D   s
    eTk r"   )r   r   r   r   r   r(   r)   r   r   r   prepare)	r/   schedr,   	ddp_modeldset
dataloaderoptddp_opt	ddp_scheds	            r    get_training_setuprO   8   s    RLEIB'DDR0J	HH[ 5++-$7y335$?(ABW0IJ	4?4G4G	SZ\egq4r1	7Iz + 3 3Iz J	:sE:y'9MM)Z''r"   c                 z   t        |       \  }}}t        t        |            j                         \  }}t	        d      D ]l  }| j                  ||f      \  }}|j                  | j                        |j                  | j                        }}t        ||||        |dz  dk(  r)| j                  |      5  t        ||||        d d d        nt        ||||        t        ||d|       t        |j                         |j                               D ]^  \  }	}
|	j                  st        j                  |	j                   |
j                         r>J d|	j                    d|
j                    d        t        j"                  d|z          |t        j$                  t'        |               }o y # 1 sw Y   xY w)	N      r   T7Gradients not in sync when they should be:
Model grad () != DDP grad (r   9  )rO   nextitervaluesrangegatherr(   r)   r3   no_syncr!   r   r   r   r   r   r   manual_seedrandpermlenr/   r,   rI   rK   	ddp_input
ddp_targetr   r-   r.   r   	ddp_params              r    test_noop_syncrc   O   s   #5k#B E9j j!1299;Iz1X >	#**Iz+BCv!3!34fii@R@R6Sv5%5q=A$$Y/ J9i[IJ J y)ZE 	uiyA #E$4$4$6	8L8L8N O 	E9&&>>%**inn= J5::,Vefoftfteuuvw=	 	$*+ennS^<=	5>J Js   7F11F:	c                    t        |       \  }}}t        t        |            j                         \  }}t	        d      D ]  }| j                  ||f      \  }}|j                  | j                        |j                  | j                        }}t        ||||        |dz  dk(  r)| j                  |      5  t        ||||        d d d        nt        ||||        t        |j                         |j                               D ]  \  }	}
|	j                  s|dz  dk(  rOt        j                  |	j                  |
j                        du rHJ d|	j                   d|
j                   d       t        j                  |	j                  |
j                        du rJ d	|	j                   d
|
j                   d        t        j                   d|z          |t        j"                  t%        |               } y # 1 sw Y   )xY w)NrQ   rR   r   F7Gradients in sync when they should not be:
Model grad () == DDP grad (r   TrS   rT   rU   )rO   rV   rW   rX   rY   rZ   r(   r)   r3   r[   r   r   r   r   r   r   r\   r]   r^   r_   s              r    test_distributed_syncrg   q   s   #5k#B E9j j!1299;Iz1X  >	#**Iz+BCv!3!34fii@R@R6Sv5%5q=A$$Y/ J9i[IJ J y)ZE !$E$4$4$6	8L8L8N O 	E9&&1}!~~ejj)..AUJ NuzzlZijsjxjxiyyz{J
 ~~ejj)..ATI NuzzlZijsjxjxiyyz{I	 	$*+ennS^<=	A >J Js   7G<<H	c                 "   t        |       \  }}}g }d}t        |      D ]  }t        t        |            j	                         \  }}| j                  ||f      \  }	}
|	j                  | j                        |
j                  | j                        }
}	t        ||	|
|        | j                  |      5   ||      }t        j                  ||j                  |j                              }|j                  |       d d d         t        |      D ]c  }||   }||dz
  k  r| j                  |       t        |j                         |j                               D ]`  \  }}|j                   st#        j$                  |j&                  |j&                        du r@J d|j&                   d|j&                   d        | j)                  |      5  | j                  |       d d d        t        |j                         |j                               D ]`  \  }}|j                   st#        j$                  |j&                  |j&                        du r@J d|j&                   d	|j&                   d        f y # 1 sw Y   cxY w# 1 sw Y   xY w)
NrQ      Fre   rf   r   TrS   rT   )rO   rY   rV   rW   rX   rZ   r(   r)   r3   r[   r&   r'   appendr+   r   r   r   r   r   r   trigger_sync_in_backward)r/   r,   rI   rK   lossesnum_iterationsr   r`   ra   r-   r.   
ddp_outputr2   r   rb   s                  r    "test_distributed_sync_multiple_fwdro      ss   #5k#B E9jFN>*  	 $T*%5 6 = = ?	: $**Iz+BCv!3!34fii@R@R6Sv 	5%5   + 	 "9-J::j*--
8I8I*JKDMM$	  	  " >* 	i ~))  & %((8(8(:I<P<P<R$S  y**~~ejj)..AUJ NuzzlZijsjxjxiyyz{J	 55i@ +$$T*+ %((8(8(:I<P<P<R$S  y**~~ejj)..ATI NuzzlZijsjxjxiyyz{I	-	  	 0+ +s   2A	I8J8J	J	c           
      v   t        d|      }t        | |      }t        ||      }t        |      \  }}}t	        |      D ]  \  }	}
|
j                         \  }}|j                  ||f      \  }}|j                  |j                        |j                  |j                        }}t        ||||d       |j                  |      5  t        ||||       d d d        t        |j                         |j                               D ]  \  }}|j                  s|	dz   dz  dk(  s|	t        |      dz
  k(  s|rRt        j                   |j"                  |j"                        du r^J d	|	 d
|j"                   d|j"                   d       t        j                   |j"                  |j"                        du rJ d|	 d
|j"                   d|j"                   d        t        j$                  d|	z          |t        j&                  t        |               } t)        j*                          y # 1 sw Y   YxY w)NrR   	num_stepssync_each_batchsplit_batchesdispatch_batchesdataloader_configgradient_accumulation_pluginFri   r   Tr   z:
Model grad (rT   r   r   rf   rU   )r	   r   r   rO   	enumeraterX   rZ   r(   r)   r3   
accumulater   r   r   r^   r   r   r   r\   r]   r
   _reset_state)ru   rv   rs   ry   rx   r/   r,   rI   rK   r   batchr`   ra   r-   r.   r   rb   s                    r    test_gradient_accumulationr~      sf   #=[j#k /m^no+%AK
 $6k#B E9j%j1 >	5 %	:#**Iz+BCv!3!34fii@R@R6Sv5%e<##I. 	Fy)ZE	F !$E$4$4$6	8L8L8N O 	E9&&Q!#q(i3z?Q;N.NSb~~ejj)..ATI Mi[Xghmhrhrgs  tC  DM  DR  DR  CS  ST  UI
 ~~ejj)..AUJ Mi[Xghmhrhrgs  tC  DM  DR  DR  CS  ST  UJ	 	$*+ennS^<=	9>:  +	F 	Fs   H..H8	c                    t        d|      }t        | |      }t        ||      }t        |d      \  }}}}	}
}}t	        |	      D ]  \  }}|j                         \  }}|j                  ||f      \  }}|j                  |j                        |j                  |j                        }}|j                          |
j                          t        ||||d       |j                          |dz   dz  dk(  s|dz   t        |	      k(  r=| r|j                          n*t        |j                        D ]  }|j                           |j                  |
      5  t        |
|||       |j                          |j                          d d d        |j                   d   d	   |j                   d   d	   k(  s.J d
|j                   d   d	    d|j                   d   d	    d       |dz   dz  dk(  xs |dz   t        |	      k(  }|j                  dkD  rt#        ||
|xs ||d       |r|j%                          |j%                          t'        j(                  d|z           t+        j,                          y # 1 sw Y   xY w)NrR   rq   rt   rw   TFri   r   r>   z:Learning rates found in each optimizer did not align
opt: z

DDP opt: 
r<   )rtolrU   )r	   r   r   rO   rz   rX   rZ   r(   r)   r%   r3   stepr^   rY   num_processesr{   param_groupsr!   	zero_gradr   r\   r
   r|   )ru   rv   rs   ry   rx   r/   r,   rL   rH   rK   rI   rM   rN   r   r}   r`   ra   r-   r.   _r   s                        r    1test_gradient_accumulation_with_opt_and_schedulerr      s    $>[j#k /m^no+%AK
 DVVacgCh@E3z9gy%j1 +,	5 %	:#**Iz+BCv!3!34fii@R@R6Sv5%e<
]a1$9q=S_*L

{889 !AJJL! ##I. 	y)ZELLNNN	 "4(G,@,@,CD,II 	
I#JZJZ[\J]^bJcIddopw  qE  qE  FG  qH  IM  qN  pO  OQ  R	
I  !mq(Q.WY]s:4V$$q("+O MMO 	$*+W+,X  3	 	s   /I33I<	c                     t               } t        d      }t        |d      }t        d      }t        |d      }| j                  ||      \  }}| j                  j
                  J t        |      D ]  \  }}t        | j                  j
                        t        |      k(  sJ |t        |      dz
  k  r| j                  j                  rJ |dk(  sbt        |      D ]t  \  }}t        | j                  j
                        t        |      k(  sJ |t        |      dz
  k  r| j                  j                  s[J | j                  j                  rtJ  | j                  j                  rJ  | j                  j
                  J y )Nr6   r7   r9   r:   `   ri   )
r   r   r   rG   gradient_stateactive_dataloaderrz   idr^   end_of_dataloader)r/   
first_dsetfirst_dataloadersecond_dsetsecond_dataloaderr   r   	batch_nums           r    test_dataloader_breakr   2  s   -K""-J!*<#2.K";2>*5*=*=>NPa*b''%%77???!"23 @	1+,,>>?2FVCWWWWs+,q00"11CCCCA~$-.?$@ LLIqk88JJKrRcOdddd 3'8#9A#==#.#=#=#O#OOO*99KKKKL --????@ %%77???r"   c                     t               } | j                  }|j                  dk(  rt        d       |j                  t
        j                  k7  r
t                |j                  t
        j                  k(  r%|j                  dk(  rt        d       t        |        |j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                   fv rJ|j                  dk(  rt        d       t#        |        |j                  dk(  rt        d       t%        |        |j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                   fv rGdD ]B  }dD ];  }dD ]4  }|j                  dk(  rt        dd| d	| d
| d       t'        |||       6 = D |j                  dk(  rt        dd       t)                |j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                  t
        j                   fv rNdD ]I  }dD ]B  }dD ];  }|s|s|s
|j                  dk(  rt        dd| d	| d
| d       t)        |||       = D K |j+                          y )Nr   zA**Test `accumulate` gradient accumulation with dataloader break**z'**Test NOOP `no_sync` context manager**z.**Test Distributed `no_sync` context manager**zE**Test Distributed `no_sync` context manager with multiple forwards**)TFz+**Test `accumulate` gradient accumulation, z`split_batches=z` and `dispatch_batches=z` and `sync_each_batch=z`**zH**Test `accumulate` gradient accumulation with optimizer and scheduler, zJ`split_batches=False`, `dispatch_batches=False`, `sync_each_batch=False`**)r   statelocal_process_indexprintdistributed_typer   XLAr   NOrc   	MULTI_GPU	MULTI_NPU	MULTI_MLU
MULTI_SDAA
MULTI_MUSA	MULTI_CPU	MULTI_HPUrg   ro   r~   r   destroy_process_group)r/   r   split_batchrv   rs   s        r    mainr   K  s   -KE  A%QR!4!44!3!33$$);<{#!!!!!!""""!!!!"  $$)BCk*$$)YZ*;7!!!!!!""""!!"  ) 	_K$1 _ '4 _O00A5I-k]:RScRdd{  }L  |M  MP  Q /{<Lo^__	_   A%VX	
 67!!!!!!""""!!"  ) 
	vK$1 	v '4 vO&/? 00A5f-k]:RScRdd{  }L  |M  MP  Q FkScetuv	v
	v 
!r"   c                     t                y r$   )r   )indexs    r    _mp_fnr     s    Fr"   __main__)T)F)FFF)%copyr   r   torch.nn.functionalnn
functionalr&   torch.optimr   torch.optim.lr_schedulerr   torch.utils.datar   accelerate.acceleratorr   r   r	   accelerate.stater
   accelerate.test_utilsr   r   accelerate.utilsr   r   r!   r3   rO   rc   rg   ro   r~   r   r   r   r   __name__rA   r"   r    <module>r      s         - ' c c * D 6 #(.>D%>P3l&!T BG7!t@2F"R
 zF r"   