
    bi^                        d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZ d d	lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%  e#       rd d
lm&Z' nd dlm'Z'  e        rdZ(dZ)ndZ(dZ)d'dZ*d Z+d Z,d Z-d Z.d Z/d Z0d Z1d Z2d Z3d Z4d Z5d Z6d Z7d'dZ8d'dZ9d Z:d Z;d  Z<d! Z=d" Z>d# Z?d$ Z@d% ZAeBd&k(  r eA        yy)(    N)deepcopy)Path)
DataLoaderDataset)Accelerator)SeedableRandomSamplerprepare_data_loader)AcceleratorState)RegressionDatasetare_the_same_tensors)DataLoaderConfigurationDistributedTypegathergather_objectis_bf16_availableis_datasets_availableis_fp16_availableis_hpu_availableis_ipex_availableis_pytest_availableis_xpu_availableset_seedsynchronize_rng_states)RegressionModel4XPU)RegressionModelgMbP?gư>c                 n    |r%t        || t        |             }t        | ||      S t        | |d|      S )zBCreates a dataloader that can also use the `SeedableRandomSampler`)	generatordata_sourcenum_samples)
batch_sizesamplerT)r    shuffler   )r   lenr   )	train_setr   r    use_seedable_samplerr!   s        d/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/accelerate/test_utils/scripts/test_script.pygenerate_baseline_dataloaderr'   ?   sA     (!I

 )
GLL)
DT]^^    c                 4    t        d| j                          y )NzPrinting from the main process printprocess_indexstates    r&   
print_mainr/   N       	+E,?,?+@
ABr(   c                 4    t        d| j                          y )Nz%Printing from the local main process )r+   local_process_indexr-   s    r&   print_local_mainr3   R   s    	1%2K2K1L
MNr(   c                 4    t        d| j                          y )NPrinting from the last process r*   r-   s    r&   
print_lastr6   V   r0   r(   c                 :    t        d| d| j                          y )NPrinting from process : r*   )r.   process_idxs     r&   print_onr;   Z   s    	";-r%2E2E1F
GHr(   c                  
   t               } | j                  }t        d      }| j                         5  | j                  r<t        j                  d       t        |d      5 }|j                  d       d d d        n&t        |d      5 }|j                  d       d d d        d d d        | j                          | j                  rt        |      5 }dj                  |j                               }d d d        	 j                  d      sJ d       |dkD  r|j                  d      sJ d       |j                  d      | j                  dz
  k(  s)J d	|j                  d
      dz    d| j                          	 | j                  r |j#                         r|j!                          | j                          t%        j&                         }t)        j*                  |      5   | j-                  t.              | j0                         d d d        |j3                         j5                         }| j                  r|dk(  s5J | d       |j3                         j5                         dk(  s
J | d       |j7                  d       |j9                  d       t)        j*                  |      5   | j;                  t<              | j0                         d d d        | j>                  r#|j3                         j5                         dk(  s%J |j3                         j5                         dk(  sJ |j7                  d       |j9                  d       t)        j*                  |      5   | jA                  tB              | j0                         d d d        | jD                  r=|j3                         j5                         d| j0                  j                  dz
   k(  s%J |j3                         j5                         dk(  sJ |j7                  d       |j9                  d       tG        |      D ]  }t)        j*                  |      5   | jI                  tJ        |      | j0                  |       d d d        | jL                  |k(  r3|j3                         j5                         d| d| jL                   k(  s%J |j3                         j5                         dk(  sJ |j7                  d       |j9                  d        y # 1 sw Y   axY w# 1 sw Y   nxY w# 1 sw Y   sxY w# 1 sw Y   1xY w# t        $ r |j!                           w xY w# 1 sw Y   HxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nzcheck_main_process_first.txt皙?za+zCurrently in the main process
zNow on another process
 zMain process was not first   zOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""r   z&Printing from the local main process 0r5   )r,   r8   r9   )'r   num_processesr   main_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin	readlines
startswithendswithcountAssertionErrorunlinkexistsioStringIO
contextlibredirect_stdouton_main_processr/   r.   getvaluerstriptruncateseekon_local_main_processr3   is_local_main_processon_last_processr6   is_last_processrange
on_processr;   r,   )acceleratorr@   pathftextresultr:   s          r&   process_execution_checkrd   ^   s   -K--M./D		'	'	) 4&&JJsOdD! ;Q9:; ; dD! 4Q2344 !!#""$Z 	*1771;;=)D	*		??#DEcGccEq }}%?@^B^^@::89[=V=VYZ=ZZ %djj1I&JQ&N%O|\g\u\u[vwZ ""t{{}!!#
A		#	#A	& C/##J/0A0ABCZZ\  "F"";;lxGk=ll;zz|""$*=vhf,==*JJqMFF1I		#	#A	& O;))*:;K<M<MNO((zz|""$(PPPPzz|""$***JJqMFF1I		#	#A	& C/##J/0A0ABC""zz|""$*I+J[J[JiJilmJmIn(oooozz|""$***JJqMFF1I]+ ''* 	hGK""8;"GHYHY[fg	h$$3::<&&(.D[MQST_TmTmSn,oooo::<&&(B...	

1	q	m; ;4 44 4	* 	*  	KKM	C CO OC C	h 	hs   .T S82TTT T9A?T, &U
'&U&U$=)U18T	=TT	
TTT),U
UU!$U.1U;	c                  b    t               } | j                  dk(  rt        d       t        |        y )Nr   zTesting, testing. 1, 2, 3.)r
   r2   r+   r-   s    r&   init_state_checkrf      s(    E  A%*+	%Lr(   c                     t               } t        dg       t        t        j                               sJ d       | j
                  t        j                  k(  r:t        dg       t        t        j                  j	                               s^J d       | j
                  t        j                  k(  r:t        dg       t        t        j                  j	                               sJ d       t        j                         }t        dg|       t        |j                               sJ d	       | j                  d
k(  rt        d       y y )Ntorchz*RNG states improperly synchronized on CPU.cudaz*RNG states improperly synchronized on GPU.xpuz*RNG states improperly synchronized on XPU.r   )r   z0RNG states improperly synchronized in generator.r   zAll rng are properly synched.)r
   r   r   rh   get_rng_statedistributed_typer   	MULTI_GPUri   	MULTI_XPUrj   	Generator	get_stater2   r+   )r.   r   s     r&   rng_sync_checkrq      s    EG9% 3 3 56d8dd6!:!::x(#EJJ$<$<$>?mAmm?			?#<#<	<w'#EII$;$;$=>l@ll>!IK=I>	 3 3 56j8jj6  A%-. &r(   c                     t               } d| j                  z  }t        t        |      d      }t	        || j
                  | j                  | j                  d      }g }|D ]  }|j                  t        |              t        j                  |      }t        j                  |j                         t        j                  d|      j                               sJ d       t        t        |      d      }t	        || j
                  | j                  | j                  dd      }g }|D ]  }|j                  t        |              t        j                  |      }t        j                  |j                         t        j                  d|      j                               sJ d       | j                  dk(  rt        d	       t        t        |      dd
      }t	        || j
                  | j                  | j                  d      }g }|D ]  }|j                  t        |              t        j                  |      j!                         }|j#                          |t%        t        |            k(  sJ d       t        t        |      dd
      }t	        || j
                  | j                  | j                  dd      }g }|D ]  }|j                  t        |              t        j                  |      j!                         }|j#                          |t%        t        |            k(  sJ d       | j&                  dk(  rt        d       y y )N       r    T)put_on_devicer   %Wrong non-shuffled dataloader result.)rv   split_batchesz Non-shuffled dataloader passing.r    r"   !Wrong shuffled dataloader result.zShuffled dataloader passing.r
   r@   r   r]   r	   devicer,   appendr   rh   catequalcpuarangelongr+   tolistsortlistr2   r.   lengthdlrc   batchs        r&   dl_preparation_checkr      s   E%%%%F	E&Ma	0B	Ru/B/BEDWDWgk	lBF %fUm$%YYvF;;vzz|U\\!V%<%A%A%CDmFmmD	E&Ma	0B	

B F %fUm$%YYvF;;vzz|U\\!V%<%A%A%CDmFmmDa01	E&Ma	>B	Ru/B/BEDWDWgk	lBF %fUm$%YYv%%'F
KKMT%-((M*MM(	E&Ma	>B	

B F %fUm$%YYv%%'F
KKMT%-((M*MM(  A%,- &r(   c            	         t               } d| j                  z  }t        t        |      d      }t	        || j
                  | j                  | j                  dd      }g }|D ]  }|j                  t        |              t        j                  |      }t        j                  |j                         t        j                  d|      j                               sJ d       t        t        |      d      }t	        || j
                  | j                  | j                  ddd      }g }|D ]  }|j                  t        |              t        j                  |      }t        j                  |j                         t        j                  d|      j                               sJ d       | j                  dk(  rt        d	       t        t        |      dd
      }t	        || j
                  | j                  | j                  dd      }g }|D ]  }|j                  t        |              t        j                  |      j!                         }|j#                          |t%        t        |            k(  sJ d       t        t        |      dd
      }t	        || j
                  | j                  | j                  ddd      }g }|D ]  }|j                  t        |              t        j                  |      j!                         }|j#                          |t%        t        |            k(  sJ d       | j&                  dk(  rt        d       y y )Nrs   rt   ru   T)rv   dispatch_batchesr   rw   )rv   rx   r   z(Non-shuffled central dataloader passing.ry   rz   z$Shuffled central dataloader passing.r{   r   s        r&   central_dl_preparation_checkr      s   E%%%%F	E&Ma	0B	
ELL%--u/B/BRVim
B F %fUm$%YYvF;;vzz|U\\!V%<%A%A%CDmFmmD	E&Ma	0B	

B F %fUm$%YYvF;;vzz|U\\!V%<%A%A%CDmFmmDa89	E&Ma	>B	
ELL%--u/B/BRVim
B F %fUm$%YYv%%'F
KKMT%-((M*MM(	E&Ma	>B	

B F %fUm$%YYv%%'F
KKMT%-((M*MM(  A%45 &r(   c                     t               }  G d dt              } G d d      } |t        d| j                  z              } |t	        |      d      }t        ||      }t        || j                  | j                  | j                        }t        |j                  d	      r(t        |j                  j                  |      sJ d
       y t        |j                  |      sJ d
       y )Nc                       e Zd Zd Zd Zd Zy)+custom_sampler_check.<locals>.CustomDatasetc                     || _         y Ndata)selfr   s     r&   __init__z4custom_sampler_check.<locals>.CustomDataset.__init__B  s	    DIr(   c                 ,    t        | j                        S r   )r#   r   r   s    r&   __len__z3custom_sampler_check.<locals>.CustomDataset.__len__E  s    tyy>!r(   c                      | j                   |   S r   r   )r   indexs     r&   __getitem__z7custom_sampler_check.<locals>.CustomDataset.__getitem__H  s    99U##r(   N)__name__
__module____qualname__r   r   r    r(   r&   CustomDatasetr   A  s    		"	$r(   r   c                   .    e Zd ZddededefdZd Zd Zy)	0custom_sampler_check.<locals>.CustomBatchSamplerdataset_lengthr    r"   c                 T    || _         t        j                  |      | _        || _        y r   )r    npr   
data_indexr"   )r   r   r    r"   s       r&   r   z9custom_sampler_check.<locals>.CustomBatchSampler.__init__L  s     (DO ii7DO"DLr(   c              3      K   t        |       }| j                  r*t        j                  j	                  | j
                        }n| j
                  }t        j                  ||      }|E d {    y 7 wr   )r#   r"   r   randompermutationr   array_split)r   num_batchesr   outputs       r&   __iter__z9custom_sampler_check.<locals>.CustomBatchSampler.__iter__Q  sP     d)K||		--doo>^^E;7Fs   A)A3+A1,A3c                 l    t        j                  t        | j                        | j                  z        S r   )mathceilr#   r   r    r   s    r&   r   z8custom_sampler_check.<locals>.CustomBatchSampler.__len__Z  s#    99S1DOOCDDr(   N)T)r   r   r   intboolr   r   r   r   r(   r&   CustomBatchSamplerr   K  s'    	#3 	#C 	#$ 	#
		Er(   r   rs   rt   ru   )batch_samplerr   z>Custom sampler was changed after calling `prepare_data_loader`)r
   r   r]   r@   r#   r   r	   r|   r,   hasattrr   
isinstance)r.   r   r   datasetr!   r   s         r&   custom_sampler_checkr   >  s    E$ $E E$ E"u':':":;<G W!<G	G7	3B	Ru/B/BEDWDW	XBr1"**88:LM 	
L	
M "**,>? 	
L	
?r(   c                     t        d       t        dd      } t        | dd      }t        d      }t	        |      }|j                  |      }g }t        d	      D ]  }|D ]  }|j                  |d
            t        j                  |      }t        d       |j                  d       g }t        d	      D ]  }|D ]  }|j                  |d
            t        j                  |      }t        j                  ||      sJ d       y )N*   
   r   seed   Try   r%   dataloader_config   xr   z;Did not obtain the same items with the same seed and epoch.)r   r   r   r   r   preparer]   r}   rh   r~   	set_epochallclose)r$   train_dlconfigr_   original_items_r   	new_itemss           r&   check_seedable_samplerr   l  s
   RL!"5I)4@H$$?F7K""8,HN1X . 	.E!!%*-	.. YY~.N RLqI1X ) 	)EU3Z(	)) 		)$I>>.)4s6ss4r(   c                  >   t        d       t        d      } t        |       }|j                  dkD  sJ d       t	        t        t        d            dd      }t        |d	      }|j                  j                  j                  }t        |t              sJ d
       y )Nr   Tr   r   r?   z)This test requires more than one process.r   ry   )
dataloaderr%   z:Sampler in BatchSamplerShard is not SeedableRandomSampler.)r   r   r   r@   r   r   r]   r	   r   r!   r   r   )r   r_   r   prepared_data_loadertarget_samplers        r&   -check_seedable_sampler_in_batch_sampler_shardr     s    RL$$?F7K$$q(U*UU(DrO4HJ.!
 *77EEMMNn&;< D<r(   c                  2   t        d       d} t        dd      }t        |dd      }t        d|       }t	        |      }|j                  |      }g }t        d	      D ]  }|D ]  }|j                  |d
            t        j                  |      }d|_
        t	        |      }|j                  |      }g }	t        d	      D ]  }|D ]  }|	j                  |d
            t        j                  |	      }	t        j                  ||	      rJ d       y )Nr   r   r   r   Try   )r%   	data_seedr   r   r   +   z1Obtained the same items with different data seed.)r   r   r   r   r   r   r]   r}   rh   r~   r   r   )
r   r$   r   r   r_   prepared_dlr   r   r   r   s
             r&   %check_seedable_sampler_with_data_seedr     s!   RLI!"5I)4@H$$)TF7K%%h/KN1X .  	.E!!%*-	.. YY~.N F7K%%h/KI1X )  	)EU3Z(	)) 		)$I~~ni8m:mm88r(   c                    t        d       |j                  d       t        | d      }t        ||||      }t	               }t
        j                  j                  |j                         d      }t        d      D ]q  }|D ]j  }	|j                           ||	d         }
t
        j                  j                  j                  |
|	d         }|j                          |j                          l s ||fS )Nr   r   r=   lrr   r   y)r   manual_seedr   r'   r   rh   optimSGD
parametersr]   	zero_gradnn
functionalmse_lossbackwardstep)r   r    r   r%   r$   r   model	optimizerepochr   r   losss               r&   mock_trainingr     s    RL"!b9I+Iy*NbcHE 0 0 2s;Iq  	EOO5:&F88&&//c
CDMMONN	 er(   c                    t               }t        j                         }d}|dz  |j                  z  }t	        |||j                  z  ||       \  }}t        |j                        sJ d       t        |j                        sJ d       t               }t        ||||       }t               }	t        j                  j                  |	j                         d      }
|j                  ||	|
      \  }}	}
t        d       |j!                  d       t#        d      D ]r  }|D ]k  }|	j%                           |	|d         }t        j&                  j(                  j+                  ||d	         }|j-                  |       |
j/                          m t |j1                  |	      j3                         }	t        j4                  j7                  |j                  |	j                  t8        t:        d
        t        j4                  j7                  |j                  |	j                  t8        t:        d        |j=                  d       t?        d|       }t        |      }t        ||||j                  z  |       }t               }	t        j                  j                  |	j                         d      }
|j                  ||	|
      \  }}	}
t        d       |j!                  d       t#        d      D ]r  }|D ]k  }|	j%                           |	|d         }t        j&                  j(                  j+                  ||d	         }|j-                  |       |
j/                          m t |j1                  |	      j3                         }	t        j4                  j7                  |j                  |	j                  t8        t:        d        t        j4                  j7                  |j                  |	j                  t8        t:        d        |j=                  d       t        j@                  jC                         rt=        d       t        jD                          t        d      }t        j&                  jG                  dd      }	|j                  |	      }	|j1                  |	d      }t        jH                  ddg      jK                  t        jL                  |jN                        } ||      }tQ               rt=        d       t        jD                          t?        |       }t        d|      }t        ||||       }t               }	t        j                  j                  |	j                         d      }
|j                  ||	|
      \  }}	}
t        d       |j!                  d       t#        d      D ]r  }|D ]k  }|	j%                           |	|d         }t        j&                  j(                  j+                  ||d	         }|j-                  |       |
j/                          m t |j1                  |	      j3                         }	t        j4                  j7                  |j                  |	j                  t8        t:        d        t        j4                  j7                  |j                  |	j                  t8        t:        d         tS               rtU               st=        d!       t        jD                          t?        |       }t        d|      }t        ||||       }t               }	t        j                  j                  |	j                         d      }
|j                  ||	|
      \  }}	}
t        d       |j!                  d       t#        d      D ]r  }|D ]k  }|	j%                           |	|d         }t        j&                  j(                  j+                  ||d	         }|j-                  |       |
j/                          m t |j1                  |	      j3                         }	t        j4                  j7                  |j                  |	j                  t8        t:        d"        t        j4                  j7                  |j                  |	j                  t8        t:        d#        tW               rt=        d$       t        jD                          t?        |       }t        dd|%      }t        ||||       }t               }	t        j                  j                  |	j                         d      }
|j                  ||	|
      \  }}	}
t        d       |j!                  d       t#        d      D ]r  }|D ]k  }|	j%                           |	|d         }t        j&                  j(                  j+                  ||d	         }|j-                  |       |
j/                          m t |j1                  |	      j3                         }	t        j4                  j7                  |j                  |	j                  t8        t:        d&        t        j4                  j7                  |j                  |	j                  t8        t:        d'        y y )(Nrt      z0Did not obtain the same model on both processes.r=   r   r   r   r   r   c                     d|  S Nz>Did not obtain the same model on CPU or distributed training.
r   msgs    r&   <lambda>z training_check.<locals>.<lambda>      YZ]Y^_ r(   )atolrtolr   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>  r   r(   zVTraining yielded the same results on one CPU or distributed setup with no batch split.T)rx   r%   r   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>  r   r(   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>  r   r(   zSTraining yielded the same results on one CPU or distributed setup with batch split.zKeep fp32 wrapper check.fp16)mixed_precisionr   )keep_fp32_wrapperr?   )dtyper|   zBF16 training check.r   bf16)r   r   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>D      ]^a]bc r(   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>K  r   r(   zFP16 training check.c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>j  r   r(   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>q  r   r(   zipex BF16 training check.)r   r   r   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>  r   r(   c                     d|  S r   r   r   s    r&   r   z training_check.<locals>.<lambda>  r   r(   ),r
   rh   ro   r@   r   r   abr   r'   r   r   r   r   r   r   r   r]   r   r   r   r   r   r   unwrap_modelr   testingassert_closeATOLRTOLr+   r   ri   is_available_reset_stateLinearTensortofloat16r|   r   r   r   r   )r%   r.   r   r    r   r$   	old_modelr_   r   r   r   r   r   r   r   r   model_with_fp32_wrapperinput_tensors                     r&   training_checkr    s   E!IJ!^e111F(e>Q>Q1QS\^rsIy	,`.``,	,`.``,-K+Iy*NbcHE 0 0 2s;I!,!4!4Xui!PHeYRL"1X  	EOO5:&F88&&//c
CD  &NN	 $$U+//1E	MM_   
MM_   no/dYmn0ABK+9j5+>+>>@TH E 0 0 2s;I!,!4!4Xui!PHeYRL"1X  	EOO5:&F88&&//c
CD  &NN	 $$U+//1E	MM_   
MM_   kl zz ()%%'!&91%##E*"-":":5TX":"Y ||QF+..U]];K]K].^(6 $%%%'3I]^!&L]^/	9jRfg!KKOOE$4$4$63O?	%0%8%85)%T"%b!q 	!A! !!uSz*xx**33FE#JG$$T* !	! ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
 #3#5$%%%'3I]^!&L]^/	9jRfg!KKOOE$4$4$63O?	%0%8%85)%T"%b!q 	!A! !!uSz*xx**33FE#JG$$T* !	! ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
 )*%%'3I]^!&dVgh/	9jRfg!KKOOE$4$4$63O?	%0%8%85)%T"%b!q 	!A! !!uSz*xx**33FE#JG$$T* !	! ((/335""KKGGc 	# 	
 	""KKGGc 	# 	
9 r(   c           	      `   t               }| j                  t        d|j                  z        D cg c]  }t	        |       c}      }|j                  |d      5 }t        |      dk(  s J d|j                   dt        |              	 d d d        | j                  t        d|j                  z  dz
        D cg c]  }t	        |       c}      }|j                  |d      5 }|j                  r.t        |      dk(  sNJ d|j                   dt        |              t        |      dk(  s J d	|j                   dt        |              d d d        |j                          | j                  t        d|j                  z  dz
        D cg c]  }t	        |       c}      }| j                  t        d|j                  z        D cg c]  }t	        |       c}      }||fD ]  }|d
   }|j                  |d      5 }|j                  dk(  r7t        |      t        |      k(  sNJ d|j                   dt        |              t        |      dk(  s J d|j                   dt        |              g }|D ]  }	|j                  |	        	 d d d        |j                          t              }
|
d t        |       D cg c]  }|d
   	 }}||k(  rJ d| d|         y c c}w # 1 sw Y   kxY wc c}w # 1 sw Y   xY wc c}w c c}w # 1 sw Y   xY wc c}w )Nr   )kFapply_padding4Each process did not have two items. Process index: 
; Length: r?   z;Last process did not receive a single item. Process index: zLOne of the intermediate processes did not receive two items. Process index: r  Tz9Single process did not receive all items. Process index: )Gathered results is incorrect. Expected: ; Got: )r
   	from_listr]   r@   dictsplit_between_processesr#   r,   r\   rG   r}   r   )datasets_Datasetr.   vr   resultsodd_data	even_dataexpected_outputresults_per_processrc   gathered_resultsrr   s                r&   $test_split_between_processes_datasetr)    so   E%%%EDWDW@W:X&YQtay&YZD		&	&t5	&	A 
W7|q  	
B5CVCVBWWabefmbnaop	
 

 %%%EDWDW@WZ[@[:\&]Qtay&]^D		&	&t5	&	A W  w<1$ MeNaNaMbblmpqxmylz{$ w<1$ ^_d_r_r^ss}  B  CJ  K  ~L  M$ 
))eAH[H[D[^_D_>`*a4!9*abH **uQI\I\E\?]+^!D1I+^_I9% ws)**4t*D 	3""a'7|s4y0 OPUPcPcOddnorszo{n|}0 7|q( J5K^K^J__ijmnujviwx( #%! 3#**623	3 	!()<="2;SY"?@Q!C&@@&(v,UVeUffmntmu*vv(-w+ 'Z
 

 '^  +b+^
	3 	3" AsC   K1/K6=L*A)LLLBLL+6L LL(	c                     t               } t        t        dd| j                  z              }| j	                  |      5 }t        |      dk(  s J d| j                   dt        |              	 d d d        | j                          t        t        dd| j                  z              }t        t        dd| j                  z  dz
              }||fD ]  }|}| j	                  |d      5 }t        j                  t        |      | j                  z        }t        |      |k(  s-J d| j                   d	| j                   dt        |              g }|D ]  }|j                  |        	 d d d        | j                          t              }	|	d t        |       }
||
k(  rJ d
| d|
         y # 1 sw Y   ExY w# 1 sw Y   TxY w)Nr   r   r  r  r?   Tr  Process ; did not get the correct number of item(s). Process index: r  r  )r
   r   r]   r@   r  r#   r,   rG   r   r   r|   r}   r   )r.   r   r"  r$  r#  r%  num_samples_per_devicer&  rc   r'  r   s              r&   !test_split_between_processes_listr.    s   EaU00012D		&	&t	, 
7|q  	
B5CVCVBWWabefmbnaop	
 
 
U1q5#6#6689IE!a%"5"55:;<H9% w**4t*D 		3%)YYs4y5;N;N/N%O"w<#99 5<<.(cdidwdwcx  yC  DG  HO  DP  CQ  R9 #%! 3#**623		3 	!()<=!+CI.&(v,UVeUffmntmu*vv('w
 
		3 		3s   /F1)BF>1F;>G	c                     t               } g d}g d}t        j                  g d      }| j                  dv r|||d}t	        |      }| j                  |      5 }| j                  dk(  r|d   |d   d d| j                  z   k(  sTJ | j                  dk(  r|d   |d   d	d  k(  s5J | j                  d
k(  r$|d   |d   dd  k(  sJ d|d   d    d|d           | j                  dk(  r|d   |d   d d| j                  z   k(  s@J | j                  dk(  r|d   |d   d	d  k(  s!J | j                  d
k(  r|d   |d   dd  k(  sJ | j                  dk(  rOt        j                  |d   |d   d d| j                  z         sJ d|d   d d| j                  z    d|d           | j                  dk(  r5t        j                  |d   |d   d	d        sZJ d|d   d	d   d|d           | j                  d
k(  r5t        j                  |d   |d   dd        sJ d|d   dd   d|d           d d d        | j                          y # 1 sw Y   xY w)N)r?   r   r   r            rt   )r  r  cdera   gh)r?   r   r   )r  r  r3  r   r  rt   r   r   r   z
Expected: 
, Actual: r  r3  z7Did not obtain expected values on process 0, expected `z`, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `)	r
   rh   tensorr@   r   r  r,   r   rG   )r.   r  r  r3  r   	data_copyr"  s          r&   (test_split_between_processes_nested_dictr<    s
   E A0A-.Ai'QQ'TN	**40 	G""a's|y~6PU=P=P8P'QQQQ$$)s|y~ab'9999$$)s|y~bc'::ujSVXZI[H\\fgnorgsft<uu:""a's|y~6PU=P=P8P'QQQQ$$)s|y~ab'9999$$)s|y~bc'::::""a'~~gclIcNC]Q%J]J]E]4^_ MdSViXrZ[_d_r_rZrNsMt  uB  CJ  KN  CO  BP  Q_ $$)~~gclIcN124FG MdSViXYXZm_\ijqrujviwxG $$)~~gclIcN234GH MdSViXZX[nM]]jkrsvkwjxyH/	6 
7	 	s   GIIc                     t               } | j                  dkD  rt        j                  g dg dg      j	                  | j
                        }| j                  |      5 }| j                  dk(  r2t        j                  g dg      j	                  | j
                        }n1t        j                  g dg      j	                  | j
                        }t        j                  j                  ||       d d d        | j                          t        j                  t        d| j                  z        D cg c]  }|g c}      j	                  | j
                        }t        j                  t        d| j                  z  dz
        D cg c]  }|g c}      j	                  | j
                        }||fD ]  }|j                         D cg c]  }t        j                  |       }}| j                  |d      5 }t        j                  t        |      | j                  z        }t        |      |k(  s-J d| j
                   d	| j                   d
t        |              g }	|D ]"  }
|	j!                  |
j	                  d             $ 	 d d d        | j                          t#        	      }|d t        |       }||k(  rJ d| d|         y # 1 sw Y   xY wc c}w c c}w c c}w # 1 sw Y   dxY w)Nr?   )r   r?   r   r   )r   r0  r1  r2  r   r   Tr  r+  r,  r  r   r  r  )r
   r@   rh   r:  r  r|   r  r,   r  r  rG   r]   r   r   r   r#   r}   r   )r.   r   r"  expectedir$  r#  r%  r-  r&  rc   r'  r   s                r&   #test_split_between_processes_tensorr@    s   EQ||\<89<<U\\J**40 	:G""a' <<7::5<<H <<7::5<<HMM&&w9	: 	!5U5H5H1H+IJaqcJKNNu||\I||%E4G4G0G!0K*LMQaSMNQQRWR^R^_HH% w48KKMBq5<<?BB**4t*D 	=%)YYs4y5;N;N/N%O"w<#99 5<<.(cdidwdwcx  yC  DG  HO  DP  CQ  R9 #%! =#**699U+;<=	= 	!()<=!+CI.&(v,UVeUffmntmu*vv(#w	: 	: KMB	= 	=s,   BK3
K#	
K(K->BK2K 2K;	c                     t               } | j                  dv rt        t        d            }t	        |      | j                  z  }t	        |      | j                  z  }| j                  |      5 }| j                  |k  r*t	        |      |dz   k(  s=J d|dz    dt	        |              t	        |      |k(  sJ d| dt	        |              d d d        | j                          y # 1 sw Y   xY w)N)r?   r   r   rt      r?   z2Each Process should have even elements. Expected: r9  )r
   r@   r   r]   r#   r  r,   rG   )r.   r   num_samples_per_process
num_extrasr"  s        r&   #test_split_between_processes_evenlyrE  3  s   El*E"I"%d)u/B/B"BY!4!44
**40 	G""Z/7|'>'BB HI`cdIdHeeopst{p|o}~B 7|'>> HI`Haaklopwlxkyz>	 
	 	s   .AC%%C.c                      t               } | j                         du sJ | j                  r| j                          | j                         du sJ | j                         du sJ y )NFT)r   check_triggerrB   set_trigger)r_   s    r&   test_triggerrI  E  si    -K$$&%/// ""! $$&$... $$&%///r(   c                     dd l } t        j                          t        j                  j                  dd      }t               }t        j                          | j                  t              5 }|j                  |       d d d        dt        j                  j                  d         v sJ dt        |j                  j                  d         v sJ y # 1 sw Y   VxY w)Nr   r?   z*`AcceleratorState` object has no attributez1This happens if `AcceleratorState._reset_state()`)pytestr
   r  rh   r   r  r   raisesAttributeErrorr   strvalueargs)rK  simple_modelr_   cms       r&   test_reinstantiated_staterS  V  s    !!#88??1a(L-K!!#	~	& *"L)*73rxx}}Q?O;PPPP>#bhhmmTUFVBWWWW* *s   ,CCc                     t               } | j                  }|j                  dk(  rt        d       t	                |j                          |j                  t        j                  k(  rt        j                  j                         }n|j                  }||j                  k(  r|j                  dk(  rt        d       t                |j                  dk(  rt        d       t                |j                  dk(  rt        d       t!                |j                  dk(  rt        d       t#                |j                  dk(  rt        d       t%                |j                  dk(  rt        d       t'               rdd	lm} t-        |       nt        d
       |j                  dk(  rt        d       t/                |j                  dk(  rt        d       t1                |j                  t        j2                  k7  r(t5                t7                t9                t;                |j                  dkD  r
t=                |j                  t        j>                  k(  ry |j                  dk(  rt        d       tA        d       tA        d       |j                  dk(  rt        d       tC                tE               r$|j                  dk(  rt        d       tG                |jI                          y )Nr   z**Initialization**z
**Test process execution**z+
**Test split between processes as a list**z+
**Test split between processes as a dict**z-
**Test split between processes as a tensor**z(
**Test split between processes evenly**z7
**Test split between processes as a datasets.Dataset**)r   z6Skipped because Hugging Face datasets is not availablez1
**Test random number generator synchronization**z 
**DataLoader integration test**r?   z
**Training integration test**Fr   Tz
**Breakpoint trigger test**z
**Test reinstantiated state**)%r   r.   r2   r+   rf   rG   rl   r   rm   rh   ri   device_countr@   r,   rd   r.  r<  r@  rE  r   datasetsr   r)  rq   r   XLAr   r   r   r   r   	DEEPSPEEDr  rI  r   rS  destroy_process_group)r_   r.   num_processes_per_noder   s       r&   mainr[  f  sH   -KE  A%"#	!:!::!&!8!8!:!&!4!4 !4!44!#01!!#@A)+!#@A02!#BC+-!#=>+-!#LM "<01ABJK  A%BC  A%12!4!44$& -/Q57 !:!::  A%/0.-  A%-.N$$)34!#	!r(   __main__)F)CrR   rP   r   rC   copyr   pathlibr   numpyr   rh   torch.utils.datar   r   
accelerater   accelerate.data_loaderr   r	   accelerate.stater
   accelerate.test_utilsr   r   accelerate.utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r	  r
  r'   r/   r3   r6   r;   rd   rf   rq   r   r   r   r   r   r   r   r  r)  r.  r<  r@  rE  rI  rS  r[  r   r   r(   r&   <module>rf     s  "  	       0 " M - I   $ L5DDDD_COCIFR/$9.x>6B+
\t4&n8$P
f-w`wB#LwD$0"X N"b zF r(   