
    bi                    :
   d dl mZ d dlZ ej                  d      Zd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZmZmZ d dl Z d dl!m"Z# d dl m$Z$m%Z%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2 d e-e      v r eed      Z eed      Zejf                  ji                  ejj                  dk(  d      Z6ejf                  jo                  d      Z8d Z9d Z: ed	      d        Z;d Z<d Z=d Z>d Z?ejf                  j                  dd d gg      ejf                  j                  d!ddg      d"               ZAejf                  j                  d#g d$      ejf                  j                  d%d&d'g      d(               ZBd) ZCd* ZDe8 ed	      d+               ZEejf                  j                  d,        ZGe8ejf                  j                  d-d. ej                  d/ejf                  j                  d0d1      2       ej                  d3ejf                  j                  d0d1      2      g      d4               ZJ ed	      d5        ZKd6 ZL ej                  d78      d9        ZNd: ZOd; ZPd< ZQd= ZRd> ZS ed	      d?        ZTd@ ZU ed	      dA        ZVejf                  j                  dBg dC      ejf                  j                  dDg dE      dF               ZWe8ejf                  j                  dBdGdH ej                  dIejf                  j                  dJK      2      g      ejf                  j                  dLddg      dM                      ZY ed	      dN        ZZdO Z[ ed	      dP        Z\ ed	      dQ        Z]dR Z^ ed	      dS        Z_dT Z` ed	      dU        ZadV Zb ed	      dW        ZcdX Zdejf                  jo                  d       ed	      dY               Ze ed	      dZ        Zfd[ Zgd\ Zhd] Ziejf                  j                  d^g d_      d`        Zjejf                  j                  dag db      dc        Zkejf                  j                  ejf                  j                  dddej                  j                         g      de               Zodf Zp ed	      dg        Zq ed	      dh        Zr ed	      di        Zs ed	      dj        Zt ed	      dk        Zuy)l    )annotationsNdistributed)partial)add)Version)ClientSchedulerPlugin
futures_ofwait)cleanup)client)clustercluster_fixturegen_clusterlooploop_in_threadpopensvarying)computedelayedpersist)compute_as_if_collectionget_scheduler)Delayed)get_named_argsget_scheduler_locktmpdirtmpfile)incshould_check_stateF)r!   win32zTThe teardown of distributed.utils_test.cluster_fixture fails on windows CI currentlyreasonzrignore:Running on a single-machine scheduler when a distributed client is active might lead to unexpected results.c                     ddl m}  y )Nr   r   )dask.distributedr   r&   s    V/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/tests/test_distributed.pytest_can_import_clientr)   ?   s    '    c                     ddl m}  y )Nr   dumps)dask.distributed.protocolr-   r,   s    r(   test_can_import_nested_thingsr/   C   s    /r*   Tc                   K    t        t              d      } | j                  |      }t        |       d {    |j                  |j
                  v s|j                  |j
                  v sJ y y 7 8wN   )r   r    r   r   keydata)cr   abxx2s         r(   test_persistr:   G   s_     QA	1B
r(NN66QVVrvv//// s   6A3A19A3c                   t        d      dz   }|dz   }|dz   } t        |dd|gd| dfddg      }t        |d   d   t              sJ t        |d   d   d   t              sJ t        |d   d   t              sJ d	g d
ddddgf}t	        | |k(  sJ t        ||g| ddgd      }|d   d   |u sJ |d   d   |u sJ |d   j	                         dk(  sJ |dd  ddgfk(  sJ y )Nr2         r6   r7      r   r6   r7      )r2   r=      )   r=   F)traverserB   )r   r   
isinstancer   r   )r5   r6   r7   resultsolress         r(   test_persist_nestedrH   P   s*   
QA	AA	AA1Aq!9-1vq1#>FfQing...fQinQ'111fQilG,,,#VQ
4CFs"""
1a&!Qe
4Cq6!9>>q6!9>>q6>>q   qr7q1#hr*   c                   t        j                  d      }t        j                  d      }|j                  dg di      } | j                  ||g      }|j	                  |      }|j
                  j                  |j                         |j                  ||gd             t        j                  t              5  |j	                  ddg      }d d d        y # 1 sw Y   y xY w)	Npandasdask.dataframer8   r2   r=      r   axisr2   r=   )pytestimportorskip	DataFramescatterfrom_delayedutils	assert_eqr   concatraises	TypeError)r5   pddddffuturesddfs         r(   !test_futures_to_delayed_dataframer_   c   s    			X	&B			-	.B	sI&	'BaiiR!G
//'
"CHHs{{}biiRqi&AB	y	! &ooq!f%& & &s   ;CC!c                0   t        j                  d      }t        j                  d      }|j                  dt        d      i      }|j	                  |d      }|j                  |j                               }|j                  j                  |||        y )NrJ   rK   r8      r=   npartitions	scheduler)	rP   rQ   rR   rangefrom_pandasrT   
to_delayedrU   rV   )r5   rZ   r[   r\   r^   s        r(   test_from_delayed_dataframeri   q   s|     
		X	&B			-	.B	sE"I&	'B
...
+C
//#..*
+CHHsB!,r*   c                j   t        j                  d      }t        j                  d      }d}|j                  t        |      t        |      d      }|j                  t        |      t        |      d      }|j	                  ||      dz   }|j	                  |d      dz   }|dz  }|dz  }|j                  |d	gd
d      }|j                          |j                         j                  d	      }	|j                  j                  |	|j                  |d	gd
      j                  d	      d       y )NrJ   rK   #   r8   y)r8   zrb   
   r<   r8   lefttasks)onhowshuffle_method)rr   rs   Fcheck_index)rP   rQ   rR   rf   rg   mergeheadr   sort_valuesrU   rV   )
r5   rZ   r[   sizedf1df2ddf1ddf2ddfmdfms
             r(   $test_fused_blockwise_dataframe_merger   }   s   			X	&B			-	.B
 D
,,U4[uT{;
<C
,,U4[uT{;
<C>>#4>025D>>#1>-2D2IC2IC::du&:IDIIK
,,.
$
$S
)C HHSYYsu&Y1==cBPU  r*   rr   r6   	broadcastc                   t        j                  d      }t        j                  d      }|j                  ddgdz  t        d      d      }|j                  ddgt        d      d      }|j	                  |d      }|j	                  |d      }|j                  ||||d	
      }	|	j                         }
|j                  j                  |
j                  d      |j                  |||      j                  d      d       y )NrJ   rK   r2   r=   r?   )r6   b_left)r6   b_rightrb   rq   )rr   r   rt   r6   )rr   Fru   )
rP   rQ   rR   rf   rg   rw   r   rU   rV   ry   )r5   rr   r   rZ   r[   pdflpdfrdfldfrr   r   s              r(   test_dataframe_broadcast_merger      s     
		X	&B			-	.B<<q!fqjE!H=>D<<q!fq:;D
..1.
-C
..1.
-C88Cy8QD
,,.CHH
t#//4  r*   computation)Nr   dask.computezscheduler, use_distributed)NT)syncFc                L  	 t        j                  d      	t        j                  d       G d dt              } | j                   |              	fd} | j                  |      }|j                         du sJ  | j                  d       }|r|dk(  sJ |       y	y	)
zvShould a collection use its default scheduler or the distributed
    scheduler when being computed within a task?
    rJ   rK   c                      e Zd Zd Zd Zy)<test_default_scheduler_on_worker.<locals>.UpdateGraphCounterc                   K   d|_         y w)Nr   _update_graph_count)selfre   s     r(   startzBtest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.start   s     ,-I)s   	c                .    |xj                   dz  c_         y r1   r   )r   re   argskwargss       r(   update_graphzItest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.update_graph   s    ))Q.)r*   N)__name__
__module____qualname__r   r    r*   r(   UpdateGraphCounterr      s    	.	/r*   r   c                 Z   d} j                  t        |       t        |       d      }j                  |d      }|j                         ydk(  rt	        j                  |       ydk(  r7t        |j                  |j                  t        |j                               yJ )	Nro   rl   r=   rb   rd   r   r   T)rR   rf   rg   r   daskr   	__class__list)rz   r\   r^   r   r[   rZ   re   s      r(   fooz-test_default_scheduler_on_worker.<locals>.foo   s    \\d%+>?nnRQn/KK)K,  N*LL	2  66$sxxchh9  5r*   Tc                    | j                   S Nr   )dask_schedulers    r(   <lambda>z2test_default_scheduler_on_worker.<locals>.<lambda>   s    ~AA r*   r=   N)rP   rQ   r	   register_pluginsubmitrE   run_on_scheduler)
r5   r   use_distributedre   r   r   rG   num_update_graphsr[   rZ   s
    ` `    @@r(    test_default_scheduler_on_workerr      s    * 
		X	&B			-	.B/_ / A(*+" !((3-C::<4***A &5!N=NN;!!r*   c                    g d} | j                   ||g      }t        j                  |      }t        |      ||z   k(  sJ y )NrL   )rS   dbrT   r   )r5   Lr]   r7   s       r(   test_futures_to_delayed_bagr      s?    AaiiAG
 A7a!er*   c           
        t        j                  d       t        j                  d      }ddlm} t        j                  d      }|j	                  d      } | j
                  ||g      }|j                  |D cg c]*  }|j                  ||j                  |j                        , c}d      } ||j                         |j                  ||gd             y c c}w )Nnumpy
dask.arrayr   rV   r<   )shapedtyperN   )rP   rQ   dask.array.utilsrV   arangerS   concatenaterT   r   r   r   )r5   darV   npr8   r]   fAs           r(   test_futures_to_delayed_arrayr      s    
 			\	*B*			W	%B
		!AaiiAG
CJKa!''	9KRS 	 	A aiik2>>1a&q>9: 	Ls   :/Cc                  K   t         j                  j                  d      5   t        t              d      j                         }d d d        t        j                  d       d {    |j                  rJ  t        t              d      j                  d      }t        j                  d       d {    |j                  rJ y # 1 sw Y   xY w7 o7 !w)Nr   rd   r2   {Gz?r=   )	r   configsetr   r    r   asynciosleeprq   r5   r   r6   r7   r8   s        r(   &test_local_get_with_distributed_activer      s      
6	* &GCLO##%&
--
ww;Q&1A
--
ww;w& & s;   !C$C C'C(AC7C8CCCCc                r    t        j                  d       t        j                  d       ddlm}  |        y )Nr   rJ   r   )test_to_hdf)rP   rQ    dask.dataframe.io.tests.test_hdfr   )r5   r   s     r(   test_to_hdf_distributedr     s&    
 
!<Mr*   rc   r2   r?   zHDF not multi-process safe)r$   strict)marksro   c                v    t        j                  d       t        j                  d       ddlm}  |d |        y )Nr   rJ   r   )test_to_hdf_schedulers)rP   rQ   r   r   )rc   r5   r   s      r(   !test_to_hdf_scheduler_distributedr     s-       
!G4-r*   c                F  K   t        j                  d      }t        j                  d      }|j                  g dg dd      }|j                  |d      }|j	                  dd	
      j                  dd      } | j                  |j                  d          d {   }	 | j                  |j                  d          d {   }
|j                  j                  |j                  |	|
g      |j                  ddgddgd      j                  d             y 7 ~7 Yw)NrJ   rK   )r2   r=   rM   r?   )r2   r   r2   r   rl   r=   rb   rm   F)sortcount)	split_outr   r2   )rP   rQ   rR   rg   groupbyaggr   
partitionsrU   rV   rW   	set_index)r5   r   r6   r7   rZ   r[   r\   r^   rE   agg0agg1s              r(   test_serializable_groupby_aggr   -  s     			X	&B			-	.B	L|<	=B
...
+C[[5[)--g-CF 6,,Q/00D6,,Q/00DHH
		4,
Aq6A/0::3? 10s%   BD!D&D!DAD!D!c                2   t        d      t        d      }} t        t              ||      } t        t              ||      } t        t              ||      } | j                  |      } t        t              |d      }|j                  d      dk(  sJ y )Nr2   r=   ro   dask.distributedrd      )r   r   r   r   )r5   r8   rm   xxyyxxyyxxyy2xxyy3s           r(   test_futures_in_graphr   ?  s    1:wqzqA	a	B	a	B73<BDAIIdOEGCL#E==#5=6:RRRRr*   function)scopec              #    K   t        j                  d      }	 |j                  j                  j	                         }|j                  j                  j                         }| 	 |j                  j                  j                          y # t        $ r | Y 5w xY w# t        $ r Y y w xY w# 	 |j                  j                  j                          w # t        $ r Y w w xY wxY ww)Nzarr)rP   rQ   corer   	_get_loop_get_executorAttributeErrorcleanup_resources)r5   zarr_lib_s      r(   r   r   K  s     ""6*H
MM((*MM,,.	MM002    			MM002 		sw   C&AB &$B 
C&BB- BB- 	B*'C&)B**C&-C#/$CC#	C C#C  C##C&c                `   t        j                  d       t        j                  d      }t               5 }|j                  dd      }|j	                  |       |j                  |      }|j                  |||        |j                  |j                  k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r   rM   rM   r2   r2   chunksrd   )rP   rQ   r   zerosto_zarr	from_zarrrV   r   )r5   r   r   dr6   a2s         r(   test_zarr_distributed_roundtripr   c  s    
 			\	*B	 %QHHVFH+			!\\!_
Qa(yyAHH$$$% % %s   A%B$$B-c                l   t        j                  d       t        j                  d      }t               5 }d}|j                  d|      }t	        |j
                        t	        d      k  r|j                  j                  |      }n|j                  j                  |d      }|j                  |j                  ||j                  |d	
      }|j                  |       |j                  |      }|j                  |||        |j                  |j                  k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   3.0.0.a0F	read_onlyr6   r   r   r   storemoderd   )rP   rQ   r   r   r   __version__storageDirectoryStore
LocalStore
open_arrayr   r   r   r   rV   r   )	r5   r   r   r   r   r6   r   rn   r   s	            r(   3test_zarr_distributed_with_explicit_directory_storer  o  s   
 			\	*B	 %QHHVFH+4##$wz'::++A.A''U';AOO''''  
 	
		!\\!_
Qa(yyAHH$$$#% % %s   C+D**D3c                   t        j                  d       t        j                  d      }d}|j                  d|      }t        |j                        t        d      k  r|j
                  j                         }n|j
                  j                  d      }|j                  |j                  ||j                  |d	
      }t        j                  t        d      5  |j                  |       d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   Fr   r6   r   distributed schedulermatch)rP   rQ   r   r   r  r  MemoryStorer  r   r   rX   RuntimeErrorr   )r5   r   r   r   r6   r   rn   s          r(   0test_zarr_distributed_with_explicit_memory_storer    s    
 			\	*BF
'At 7:#66LL$$&LL$$u$5gggg 	 	A 
|+B	C 			!  s   C88Dc                *   t        j                  d       t        j                  d      }d}|j                  d|      }|j                  ||      }t        j                  t
        d      5  |j                  |       d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r	  r
  )rP   rQ   ones
zeros_likerX   r  r   )r5   r   r   r   r6   rn   s         r(   #test_zarr_in_memory_distributed_errr    sx    
 			\	*BF
v&A&)A	|+B	C 			!  s   .B		Bc                     t        d              j                  |       dk(  sJ  | j                  fd      sJ y )Nc                      yr1   r   r   r*   r(   r   z.test_scheduler_equals_client.<locals>.<lambda>  s    r*   rd   r2   c                :    | j                  j                        S r   )storyr3   )r   r8   s    r(   r   z.test_scheduler_equals_client.<locals>.<lambda>  s    ^5I5I!%%5P r*   )r   r   r   )r5   r8   s    @r(   test_scheduler_equals_clientr    sB    	A99q9!Q&&&1PQQQr*   c                $  K    t        j                  t              d      } | j                  |       d {   }|j                  |j
                  v sJ |j                  s|j                  sJ t        d t        |      D              sJ y 7 Vw)Nr2   c              3  <   K   | ]  }|j                           y wr   )done).0r   s     r(   	<genexpr>ztest_await.<locals>.<genexpr>  s     /Aqvvx/s   )	r   r   r    r   r3   rq   r4   allr
   r   s        r(   
test_awaitr    ss     S!AaiilA55AGG66QVV/A//// 	s   5BBABc                 >    d } t        j                   |               y )Nc                    K    t        j                  t              d      } | dz   }|j                          d {   }t	        |j                         dk(  sJ y 7 wr1   )r   r   r    r   len)r8   rm   rn   s      r(   r   ztest_local_scheduler.<locals>.f  sM     DLLa E))+166{a s   8AA A)r   run)r   s    r(   test_local_schedulerr#    s      KKr*   c                L  K   t        j                  d       t        j                  d      }t        j                  d      }ddlm} t	        t        d      t        d      ddg      fd}d	 }|j                  d
d      }	t        j                  d      5  |	j                  ||j                  d|j                              }
d d d        t        j                  d      5  
j                  ||j                  d|j                              }d d d        t        j                  j                  d      5   | j                         d {   }d d d         ||j                  d
      dz         y # 1 sw Y   xY w# 1 sw Y   txY w7 @# 1 sw Y   ?xY ww)Nr   r   r   r   onetwor=   c                            | z  S r   r   )r8   scales    r(   flaky_doublez7test_annotations_blockwise_unpack.<locals>.flaky_double  s    w{r*   c                    d| z  S )Nr=   r   r8   s    r(   reliable_doublez:test_annotations_blockwise_unpack.<locals>.reliable_double  s    1ur*   ro   )r<   r   )retriesr   r   metaF)optimization__fuse__activeg      @)rP   rQ   r   rV   r   ZeroDivisionErrorr  r   annotate
map_blocksarrayfloat64r   r   r   )r5   r   r6   r7   r   r   rV   r)  r,  r8   rm   rn   r(  s               @r(   !test_annotations_blockwise_unpackr7    sb    
 			\	*B			W	%B* &u-/@/GANOE 	4 A 
q	! LLLBHHRrzzH,JLKL	q	! OLLrxx"**x/MLNO 
E	: !))A, as"#L LO O  s`   BF$/E>
F$(/F
(F$?FFF$F$>FF$
FF$FF!F$io)r  r   fullfuse)TFNc                   t        j                  d      }t        j                  d      }d}d}|dk(  r%|j                  ||      }|j                  |      }nU|dk(  r%|j                  ||      }|j                  |      }n+|dk(  r&|j	                  |d	|      }|j	                  |d	      }d
z  }d
z  }t
        j                  j                  d|i      5  |j                          t
        j                  j                  |j
                  |j                               }	t        |	t              |duk(  sJ |j                  |||        d d d        y # 1 sw Y   y xY w)Nr   r   )r<   r=   )ro   r?   r  r   r   r9  ro   r=   optimization.fuse.activeFrd   )rP   rQ   r  r   r9  r   r   r   r   r5  optimize__dask_keys__rD   dictrV   )
r5   r8  r:  r   r   r   r   darrnarrdsks
             r(   test_blockwise_array_creationrC    s:    
		W	%B			\	*BFE	V|wwuVw,wwu~	wxxfx-xx	vwwubw0wwub!AIDAID	4d;	< .jj!!$))T-?-?-AB#t$U):;;;
T41-. . .s   A4EEparquetcsvhdfr<   )rerunsfrom_futuresc                   t        j                  d      }t        j                  d      }|j                  g ddz  t        d      d      }|rU|j                  d d |j                  dd |j                  dd g} | j
                  |      }|j                  ||d   	      }	n|j                  |d
      }	|dk(  rJt        j                  d       |	j                  t        |             |j                  t        |            }
n|dk(  rU|	j                  t        |      d       |j                  t        j                  j                  t        |      d            }
ne|dk(  rUt        j                  d       t         |j                  d            }|	j!                  |d       |j#                  |d      }
nt%        d      |dg   dz   }|
dg   dz   }
y )NrJ   rK   rL   r<      rl   ro   r   r/  rM   rb   rD  pyarrowrE  F)index*rF  tablesh5z/data*unreachabler8   )rP   rQ   rR   rf   ilocrS   rT   rg   
to_parquetstrread_parquetto_csvread_csvospathjointo_hdfread_hdfAssertionError)r5   r   r8  rH  rZ   r[   r\   partsfutsddf0r^   fns               r(   test_blockwise_dataframe_iora    s    
		X	&B			-	.B	IMb	:	;B!bggamRWWR^<qyyt%(3~~ba~0	YI&F$ooc&k*	uCKu-kk"'',,s6{C89	uH%T"#B!kk"h']++	SERB
se*r/Cr*   c                l  K   t        j                  d      }t        j                  d      }|j                  g ddz  t        d      d      }|j	                  |d      }t
        j                  j                  d	d
i      5  t        j                  t        d      5  |j                  t        |             d d d        |j                  t        |            }	 | j                   | j                  |	             d {    t        j                  t        d      5  t        j                  |	       d d d        d d d        t
        j                  j                  d	d i      5  t        j                  t         d      5  |j                  t        |             d d d        t        j                  t         d      5  t        j                  	       d d d        d d d        y # 1 sw Y   1xY w7 # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   rxY w# 1 sw Y   ExY w# 1 sw Y   y xY ww)NrJ   rK   rL   r<   rJ  rl   rM   rb   zadmin.async-client-fallbackr   asynchronousr
  )rP   rQ   rR   rf   rg   r   r   r   warnsUserWarningrR  rS  rT  gatherr   rX   r  )
r5   r   r6   r7   r   rZ   r[   r\   r_  r^   s
             r(   test_client_compute_parquetrg  0  s    			X	&B			-	.B	IMb	:	;B>>"!>,D	7@	A \\+^< 	)OOCK(	) ooc&k*ahhyqyy~&&&\\+^< 	LL	 
7>	? ]]<~> 	)OOCK(	)]]<~> 	LL	 	) 	) 	'	 	 	) 	)	 	 s   BH4HG):AHG6H!G87H?)H4(H(H#H(HH( 	H4)G3	.	H8H	=HH	H4H	H(H%	!H((H1-H4c                B   t        j                  d      }t        j                  d      }|j                  dg ddz  i      }|j                  |d      d   }|dk  }t	        |      }|d	k(  sJ |j
                  |j                            j                         d	k(  sJ y )
NrJ   rK   r8   rL   r<   r=   rb   rM   rJ  )rP   rQ   rR   rg   r!  r8   r   sum)r5   rZ   r[   r\   seriesrE   
series_lens          r(   #test_blockwise_fusion_after_computerl  E  s     
		X	&B			-	.B 
sIM*	+B^^BA^.s3FaZF
 VJ44 !%%'2---r*   c           	     |  K   t        j                  d       t        j                  d      }t        j                  d      fd}|j                  |d|j                  d      dj	                  d      dj                        } | j
                  |j                         d	
       d{   }|dk(  sJ y7 w)zDTest pack/unpack of blockwise that includes a NumPy literal argumentr   r   c                X    t        |      j                  u sJ | j                  |      S r   typeuint16astyper8   dtr   s     r(   r`  z%test_blockwise_numpy_args.<locals>.fn_  '    Bx299$$$xx|r*   r8     *   Nr.  Foptimize_graphrP   rQ   	blockwiser  rq  r   ri  	r5   r   r6   r7   r   r`  arrrG   r   s	           @r(   test_blockwise_numpy_argsr~  X  s       			\	*B			W	%B ,,
CRYYr]D		  C 		#'')E:
:C$;; ;   B*B<-B:.B<c           
     |  K   t        j                  d       t        j                  d      }t        j                  d      dfd	}|j                  |d|j                  d      dj                  j	                  d            } | j
                  |j                         d	
       d{   }|dk(  sJ y7 w)zLTest pack/unpack of blockwise that includes a NumPy literal keyword argumentr   r   Nc                X    t        |      j                  u sJ | j                  |      S r   ro  rs  s     r(   r`  z'test_blockwise_numpy_kwargs.<locals>.fnq  ru  r*   r8   rv  rw  )r   rt  Frx  r   rz  r|  s	           @r(   test_blockwise_numpy_kwargsr  j  s       			\	*B			W	%B ,,r3s"))		RT,
VC		#'')E:
:C$;; ;r  c                   t        j                  d      }t        j                  d      }|j                  |j                  d            }|j                  |j	                  g d            }|j                         }||z  }t        j                  |      \  }||z  }|j	                  g d      }t        j                  j                  ddi      5  |j                         }	|j                         }
d d d        |j                  j                  	|       |j                  j                  
|       y # 1 sw Y   BxY w)Nr   r   rM   )y      $@       @y      @      y       @      ?)y                y      @      @y      0@       r<  F)rP   rQ   
from_arrayr   r5  conjr   r=  r   r   r   testingassert_equal)r5   r   r   uvcvr8   rm   expectedx_valuey_values              r(   %test_blockwise_different_optimizationr  z  s    
 
		W	%B			\	*B
biil#A
bhh89:A	
B	BAMM"ER	BAxx12H	4e<	= ))+))+ JJGX.JJGX.	 s   !D44D=c                  K   t        j                  d      }t        j                  d      }t        j                  d      }t        j                  d      }d } | j                  d d      }	 | j                  d d	      }
|j                  |d
|j	                  dd      d
|j                  dd      d
|	ddt        |
      }|j                  |j                  d|j                  d	      i      d	      }|j                  dd      }|d   j                         }|j                         |j                         z   } | j                  |d       d{   }|dk(  sJ y7 w)z9Check pack/unpack of a HLG that has every type of Layers!r   rJ   r   rK   c                    | |z   |z   |z   S r   r   )r8   rm   rn   	extra_args       r(   r   z&test_combo_of_layer_types.<locals>.add  s    1uqy9$$r*   c                    | S r   r   r+  s    r(   r   z+test_combo_of_layer_types.<locals>.<lambda>      1 r*   r=   c                    | S r   r   r+  s    r(   r   z+test_combo_of_layer_types.<locals>.<lambda>  r  r*   rM   r8   )rM   r2   r   NF)r   r   r  r6   rb   rq   )rt   rx     )rP   rQ   r   r{  r   r  intrg   rR   r   shuffleto_dask_arrayri  r   )r5   r   r6   r7   r   rZ   r   r[   r   rm   rn   r8   r\   rG   s                 r(   test_combo_of_layer_typesr    sL     
		W	%B			X	&B			\	*B			-	.B% 	a Aa A

d#
T"	 	 	A 
c299Q<%89q	IB	C	0B	C			 B
%%'BFFH
C		#e4
4C"99 5s   EE$E"E$c           
        t        j                  d      }t        j                  d      fd}j                  |j                  g d            }j                  |j                  g dg            }j	                  |d|d|d|j
                  d      } | j                  |d	
       j                  |||        y)z1Test a blockwise operation with concatenated axesr   r   c                2    j                  |g dg       | S )Nr   r2   r=   r   )r8   rm   r   s     r(   r   z%test_blockwise_concatenate.<locals>.f  s    
Q$r*   r  iijT)r   r   Frx  rd   N)rP   rQ   r  r5  r{  r   r   rV   )r5   r   r   r8   rm   rn   r   s         @r(   test_blockwise_concatenater    s    			W	%B			\	*B 	bhhy)*A
bhh	{+,A
						gg 	 		A AIIa&LLAL#r*   c                H  K   t        j                  d      }t        j                  d      }|j                  |j                  dt	        d      i      d      } | j
                  |j                  dd             d {   }|d   ddd	k(  sJ |d
   d
dd	k(  sJ y 7 w)NrJ   rK   r6   ro   r=   rb   c                    |S r   r   )r8   partition_infos     r(   r   z4test_map_partitions_partition_info.<locals>.<lambda>  s    . r*   r   )numberdivisionr2   r<   r   )rP   rQ   rg   rR   rf   r   map_partitions)r5   r   r6   r7   rZ   r[   r^   rG   s           r(   "test_map_partitions_partition_infor    s     			X	&B			-	.B
..sE"I&67Q.
GC		HI C q6q1111q6q1111	s   A>B" B  B"c                   t        j                  d      }t        j                  d      }t               5 \  }\  }}t        |d   |       5  |j	                  |j                  t        t        d      |j                  dddd	      
            d      }||j                  j                  t        d               j                         }|j                  j                  j                  d      |d<   |j                  j                  j                         |d<   |j!                  dgd      }|j#                          ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z.Copied from distributed (tests/test_client.py)rJ   rK   address)r   2   z
2020-01-01z
2020-09-01UTC)r   endperiodstz)uid
enter_timer<   rb      z
US/Central
local_timedayF)columnsrL  N)rP   rQ   r   r   rg   rR   r?  rf   
date_ranger  isinr   r  rt  
tz_convertday_name
categorizer   )r   rZ   r[   r   r6   r7   r^   s          r(   test_futures_in_subgraphsr    s4   			X	&B			-	.B	 kq&1a6!I,^#L nnLLb	!}}*bU  -     

 #'',,uRy)*224NN--88FL^^&&//1E
nneWEn:#     s$   EC2E>EE	EE$c                
  	
K   t        j                  d      	t        j                  d      
t        j                  d      }t        j                  d       t        j                  d      }t        j                  d       	
fd}|j                  d	      j	                         }|j                  d
d      j	                         } | j                  |j                  |||j                               d{    y7 w)z7Check that map_partitions can handle a dask array inputr   rJ   r   rK   zdask.datasetsz-roundtripping through arrays doesn't work yetc                h    t        | j                        sJ t        |j                        sJ | S r   )rD   rR   ndarray)r   r6   r   rZ   s     r(   r   z'test_map_partitions_da_input.<locals>.f  s0    !R\\***!RZZ(((r*   1d)freqr  r2   r   r/  N)	rP   rQ   xfail
timeseriesr   r  r   r  _meta)r5   r   r6   r7   r   datasetsr   r\   r}  r   rZ   s            @@r(   test_map_partitions_da_inputr    s      
		W	%B			X	&B			\	*B
()""?3H
LL@A
 
		$		'	/	/	1B
''$q'
!
)
)
+C
!))B%%a288%<
===s   C7D;D<Dc            	     l   t        j                  d      t        j                  d      t        j                  d       fdfd} t        j	                  dddd	id	d
d
d	      5 }t        j                  |d	      5   |         ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z[
    Check that map_partitions can handle a delayed
    partition of a dataframe input
    rJ   rK   z/map partitions can't deal with delayed properlyc                h    t        | j                        sJ t        |j                        sJ | S r   )rD   rR   )r   r6   rZ   s     r(   r   z'test_map_partitions_df_input.<locals>.f  s0    !R\\***!R\\***r*   c                 h   j                  j                  dt        d      i      d      } | j                         d   j	                         }j                  j                  dt        d      i      d      }|j                  dd      }|j                  ||d	
      j                          y )Nr6   ro   r2   rb   r   r7   rq   )rr   rt   F)r0  enforce_metadata)rg   rR   rf   rh   r   r  r  r   )item_dfr^   	merged_dfr[   r   rZ   s      r(   mainz*test_map_partitions_df_input.<locals>.main  s    ..sE"I.>!?Q.O  "1%--/NN2<<eBi0@#AqNQ	 %%W%E	  sU 	! 	

')r*   r   z:0	dashboardFr2   )scheduler_portdashboard_addressscheduler_kwargsrc  	n_workersnthreads	processes)rc  N)rP   rQ   r  r   LocalClusterr   )r  r   r[   r   rZ   s     @@@r(   test_map_partitions_df_inputr    s    
 
		X	&B			-	.B
LLBC

 
	!	!
 %u- 
" 
  
e< 	F	 	 	 s$   -B*BB*B'	#B**B3c           	     ,  K   t        j                  d      }t        j                  d      }t        j                  d       |j                  t        d      t        d      d      }d|j                  _        |j                  |d      }t               5 }d	| }	|j                  d
|	dddid      }
 | j                  |
       d {    |j                  j                  ||j                  d
|	d      d       d d d        y 7 <# 1 sw Y   y xY ww)NrJ   rK   
sqlalchemyro   r>   rL  r2   rb   z
sqlite:///testTechoF)rL  engine_kwargsr   )check_divisions)rP   rQ   rR   rf   rL  namerg   r   to_sqlr   rU   rV   read_sql_table)r5   r   r6   r7   rZ   r[   r\   r^   r   urirE   s              r(   test_to_sql_engine_kwargsr  1  s     
		X	&B			-	.B
%	E"IE"I6	7BBHHM
...
+C	 
a1#CtFE?E  
 aii
fc73! 	 	

 

 	 
 
s0   BD3D	D
3D=	DDDDc                z  
K   t        j                  d      }t        j                  d      } G d d      
d}|j                  dg|z  t        d|dz               }|j	                  |d	
      }|j                  d 
fdddt              }	 | j                  |	       d {   j                  dk(  sJ y 7 w)NrJ   rK   c                      e Zd Zd Zy)0test_non_recursive_df_reduce.<locals>.SomeObjectc                    || _         y r   )val)r   r  s     r(   __init__z9test_non_recursive_df_reduce.<locals>.SomeObject.__init__T  s	    DHr*   N)r   r   r   r  r   r*   r(   
SomeObjectr  S  s    	r*   r     r2   r=   )r4   rL  "   rb   c                    | S r   r   r+  s    r(   r   z.test_non_recursive_df_reduce.<locals>.<lambda>[  s     r*   c                L     | j                         j                               S r   )ri  )r8   r  s    r(   r   z.test_non_recursive_df_reduce.<locals>.<lambda>\  s    Jquuw{{}5 r*   Fzcommit-dataset)chunk	aggregatesplit_everytokenr0  )	rP   rQ   Seriesrf   rg   	reductionobjectr   r  )r5   r   r6   r7   rZ   r[   Nrj  dask_seriesrE   r  s             @r(   test_non_recursive_df_reducer  M  s      
		X	&B			-	.B  	AYYQC!G5AE?Y;F..R.8K""5 # F !))F##((C///#s   B!B;$B9%B;c                r   t        j                  d       t        j                  d       	 t        j                  j	                  ddd      j                         j                  dt        i      }|j                  dd	      }|j                          y # t        $ r t        j                  d
       Y y w xY w)NrJ   rK   
2000-01-01
2000-07-0112hr   r  r  	timestampT)sortedz(dd.set_index triggered a recursion error)rP   rQ   r   r  r  reset_indexrr  rS  r   r   RecursionErrorfail)r5   r^   s     r(   !test_set_index_no_resursion_errorr  e  s    
!
()	@MM$$<\PU$V[]V[#&' 	
 mmKm5 @>?@s   A(B B65B6c                     d} t        j                  t        |       5  t        d       d d d        t        j                  t        |       5  t        d       d d d        y # 1 sw Y   :xY w# 1 sw Y   y xY w)Nz	no Clientr
  r   rd   r   )rP   rX   r  r   )msgs    r(   -test_get_scheduler_without_distributed_raisesr  u  se    
C	|3	/ 4 234 
|3	/ /./ /4 4/ /s   A$A0$A-0A9c                p   t               | j                  k(  sJ t        j                  j	                  d      5  t               | j                  k7  sJ t        j                  j	                  d       5  t               | j                  k(  sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nthreadsrd   )r   getr   r   r   )r5   s    r(   7test_get_scheduler_with_distributed_active_reset_configr  ~  s    ?aee###	9	- ,!%%'''[[__t_, 	, ?aee+++	,, ,	, 	,, ,s#   :B,4B B, B)	%B,,B5zscheduler, expected_classes))N)SerializableLockr  AcquirerProxy)r  )r  r  r  )r  )r  r  r  c                   t        j                  d       t        j                  d       t        j                  dd      }t        j                  dd      }t        j                  dd	      }|j                  d
      }|j                  |dg      } |j                  dd      }t        |||f|      D ]-  \  }}	t        ||       }
|
j                  j                  |	k(  r-J  y )Nr   rJ   r   Requires dask.arrayr#   zdask.bagzRequires dask.bagrK   Requires dask.dataframed   r8   r  r  r=   rb   rd   )	rP   rQ   r  from_dask_arrayrf   zipr   r   r   )re   expected_classesr   r   r[   r@  r^   dbag
collectionr  rG   s              r(   test_get_scheduler_lockr    s      
!			\2G	HB			Z0C	DB			-6O	PB776?D


TC5

1C288CQ'D #S$$57G H 2
H yA}}%%1112r*   multiprocessing_method)spawnfork
forkserverc                   t        j                  d       t        j                  d       t        j                  dd      }t        j                  dd      }|j                  d      }|j                  |d	g
      }t	        j
                  dd      }t        j                  j                  d|i      5  |||fD ]5  }t        |d      }t        |t        j                  j                        r5J  	 d d d        y # 1 sw Y   y xY w)Nr   rJ   r   r
  r#   rK   r  r  r8   r  r  r=   rb   z)distributed.worker.multiprocessing-methodr   rd   )rP   rQ   r  r  r   rf   r   r   r   r   rD   r   lockLock)	r5   r  r   r[   r@  r^   r  r  rG   s	            r(   #test_get_scheduler_lock_distributedr    s      
!			\2G	HB			-6O	PB776?D


TC5

1C88CQ'D		46LM
 : d+ 	:J$Z=ICc;#3#3#8#8999	:: : :s   6:C>1C>>D
lock_paramc                B   t        j                  d       t        j                  d       t        j                  d       t        d      5 }t        j                  j                  ddd	      }|j                  t        |      d
|       ddd       y# 1 sw Y   yxY w)z_https://github.com/dask/dask/issues/9972 and
    https://github.com/dask/dask/issues/10315
    rJ   rK   rN  hd5)	extensionr  r  r  r  z/ds_*)r3   r  N)rP   rQ   r   r   r  r  rZ  rS  )r5   r  r   r^   s       r(   test_write_single_hdfr!    s     !
()
!	5	! 9Qmm&&\|RW&X

3q6wZ
89 9 9s   A BBc                
   | d   }t         j                  j                  d      5  t         j                  j	                         t         j
                  j                  k(  sJ t         j                  j                  d      5  t         j                  j	                         t         j                  j                  k(  sJ t        |d      }	 t         j                  j	                         t         j                  j                  k(  sJ 	 |j                          t        |d      }	 t         j                  j	                         |j                  k(  sJ 	 |j                          t         j                  j	                         t         j                  j                  k(  sJ t        |d      }	 t         j                  j	                         t         j                  j                  k(  sJ |j                         5  t         j                  j	                         }||j                  k(  sJ 	 d d d        t         j                  j	                         t         j                  j                  k(  sJ 	 |j                          t        |d      }t        |d      }	 |j                         5  t         j                  j	                         |j                  k(  sJ 	 d d d        t         j                  j	                         |j                  k(  sJ 	 |j                          |j                          t         j                  j	                         t         j                  j                  k(  sJ 	 d d d        t         j                  j	                         t         j
                  j                  k(  sJ t        |d      }	 t         j                  j	                         |j                  k(  sJ t         j                  j                  d      5  t         j                  j	                         t         j                  j                  k(  sJ |j                         5  t         j                  j	                         |j                  k(  sJ 	 d d d        d d d        |j                          	 d d d        y # |j                          w xY w# |j                          w xY w# 1 sw Y   xY w# |j                          w xY w# 1 sw Y   #xY w# |j                          |j                          w xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# |j                          w xY w# 1 sw Y   y xY w)Nr  r   rd   r  F)set_as_defaultT)r   r   r   baser   localget_syncthreadedr  r   close
as_current)r   	s_addressr   scclient2s        r(   5test_get_scheduler_default_client_config_interleavingr-    s    )I	6	* 7yy&&(DJJ,?,????[[__y_1 )	B99**,0A0AAAAIe<Fyy..0DMM4E4EEEEId;Fyy..0FJJ>>>99**,0A0AAAA Ie<Fyy..0DMM4E4EEEE&&( ,002B+++, yy..0DMM4E4EEEE Id;FYu=G '') D99224CCCDyy..0FJJ>>>99**,0A0AAAAS)	BV yy&&(DJJ,?,????	$7	99**,

:::95 Cyy..0DMM4E4EEEE&&( C99224

BBBCC
 LLNo7 7  , ,
 D D O)	B )	BdC CC C
 LLNo7 7s  AU$A	T*;R,T*&-SAT*-AS#80S)AS#-*T*T(.S85TAT*)AU$:AUAU.T7U
UU$,R>>T*ST*S S##S55T*8T=T"T''T**T4	/U$7U <UU	UU!!U$$U-c                   K   t        j                  dd      }|j                  d       }t        d |j                  D              rJ y w)Nr  ro   rb   c                    | dz  S )N   r   r+  s    r(   r   z*test_bag_groupby_default.<locals>.<lambda>  s
    QV r*   c              3  *   K   | ]  }d |d   v   yw)partdr   Nr   )r  ks     r(   r  z+test_bag_groupby_default.<locals>.<genexpr>  s     4q7ad?4s   )r   rf   r   anyr   )r5   r   r6   r7   b2s        r(   test_bag_groupby_defaultr6    s@     
"%A	
#	$B4BGG44444s   A	Ac                  K   t        j                  d       t        j                  d      }t        j                          t        j                          	 |j                  dd      } | j                  |dz         }g }t        j                  |      }|j                  t        |      D 	cg c]  }	t        j                  |	       c}	       t        j                  |      }
| d {   } | j                  |d d       }|j                  t        |      D 	cg c]  }	t        j                  |	       c}	       t        j                  |      }
| d {   }~~ |       J  |
       J t        d |D              sJ |j                  r*t        j                  d	       d {    |j                  r*t        j                          y c c}	w 7 c c}	w 7 7 4# t        j                          w xY ww)
Nr   r   r  )ra   r   r2   ra   c              3  .   K   | ]  } |       d u   y wr   r   )r  futs     r(   r  z<test_release_persisted_futures_without_gc.<locals>.<genexpr>"  s     8S35D=8s   r   )rP   rQ   gccollectdisabler   r   weakrefrefextendr
   r  rq   r   r   enable)r5   r   r6   r7   r   r8   rm   future_refscolyr9  colzrn   s               r(   )test_release_persisted_futures_without_gcrD    s}    
 			\	*BJJLJJLIIc%I(AIIa!e{{1~
1FGKK,FG{{1~GAIIaf
1FGKK,FG{{1~Gqv~~v~~8K8888gg--%%% gg 			! G G &
		sn   AG:AG! -G	!G! *G+1G! G8!G! GAG! *G+G! <G:G! G! G! !G77G:c                   K    | j                   d       d {   } t        j                  t              |      } | j                  |       d {   dk(  sJ y 7 D7 w)Nr2   r+  r=   )rS   r   r   r    r   )r5   r   r6   r7   r9  rE   s         r(   test_delayed_future_with_kwargsrF  )  sT     		!
CT\\#%F6""a'''  #s!   A!A9A!AA!A!c                  K   t        j                  d      }t        j                  d      }|j                  j                         j	                  ddd      j                  d      }|j                  |d      }d	}|j                  ||d
      } | j                  |       d {   }	|j                  |	|       y 7 w)Nr   r   r   r2   r  )ro   ro   )ro   r2   r   ))
r2   r2   r2   r2   r2   r2   r2   r2   r2   r2   )ro   p2p)r   method)
rP   rQ   randomdefault_rnguniformreshaper  rechunkr   rV   )
r5   r   r6   r7   r   r   r8   newr9   rE   s
             r(   test_fusion_barrier_taskrP  1  s     			W	%B			\	*B
		''1c2::8DA
a(A
C	Ac%	0B199R= FLL !s   B#B?%B=&B?c                   K    | j                   d       d {   }t        j                  |      } | j                  |       d {   }|dk(  sJ |j                  |j                  k(  sJ y 7 V7 )wr1   )rS   r   r   r   r3   )r5   r   r6   r7   r9  rE   rG   s          r(   test_delayed_futurerR  >  sc     		!
C\\#F		&!
!C!8O8::   	 
!s!   A3A/.A3A1(A31A3)v
__future__r   rP   rQ   r   r   r:  rW  sysr=  	functoolsr   operatorr   packaging.versionr   r   r	   r
   r   distributed.utils_testr   r   r5   r   r   r   r   r   r   r   r   r   dask.bagbagr   r   r   r   	dask.baser   r   dask.delayedr   
dask.utilsr   r   r   r   dask.utils_testr    markskipifplatform
pytestmarkfilterwarningsignore_sync_scheduler_warningr)   r/   r:   rH   r_   ri   r   parametrizer   r   r   r   r   xfail_with_pyarrow_stringsr   paramr  r   r   r   fixturer   r   r  r  r  r  r  r#  r7  rC  flakyra  rg  rl  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  skip_with_pyarrow_stringsr  r  r!  r-  r6  rD  rF  rP  rR  r   r*   r(   <module>rk     s
   " !f!!-0  	 	 
    % A A * .	 	 	   * * =   J J >+66+%@Kg%8G [[LLG	(   
 !' : :2! (0 D0 0&&	-2 cU|,tUm4 5 -&   )O )OX; D   '' ( 	++##+GPU#V	
 	++##+GPU#V	

. . D "	S j! ".	%%0*	R D0 0 D$ $>  !45. 6.6 U&++"3"31"3"=>	 $7 8 @ D (.& D " D /, D   F$2 D	2 	20 D> >&(V 2 D
 	

. D0 0.@ /, !22  ::$ &&k.>.>.C.C.E'FG	9 H '	9;| D5 5 D : D( ( D	 	 D! !r*   