
    bi1                    H   U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlZd d	lmZmZmZmZmZmZmZ d dlZd dlm Z! d d
l"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9m:Z: d dl;m<Z<m=Z=m>Z> e?dfe?dfe?dfdZ@deAd<    eB e?d            dz  ZC e&e@dd      ZDd ZEd ZFd ZGd ZHd ZId ZJej                  j                  ddd g      ej                  j                  d!d"d#g      d$               ZM ed%&       G d' d(             ZNej                  j                  d) eNd*+       eNd*d,-      gd.d/g0      ej                  j                  ddd g      ej                  j                  d!d"d#g      d1                      ZOd2 ZPd3 ZQd4 ZRd5 ZSd6 ZTd7 ZUej                  j                  d8eVeWg      d9        ZXd: ZYd; ZZd< Z[d= Z\d> Z]d? Z^d@ Z_dA Z`dB ZadC Zbej                  j                  dDd*d,g      dE        ZcdF ZddG ZedH ZfdI ZgdJ Zhej                  j                  dDg dK      dL        Ziej                  j                  dDd*dMg      dN        ZjdO ZkdP Zlej                  j                  dQeej                  dR g      dS        ZndT ZodU ZpdV ZqdW ZrdX ZsdY ZtdZ Zud[ Zvd\ Zwd] Zxd^ Zyd_ Zzd` Z{da Z|db Z}dc Z~dd Zde Zdf Zej                  j                  ej                  j                  ej                  j	                  dgh      di                      Zdj Zdk Zdl Zdm Zej                  j                  dnh      ej                  j                  ej                  j                  do                      Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zdx Zdy Zdz Zd{ Zd| Zd} Zd~efde	fdefgZej                  j                  de      d        Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z G d de!jL                        Zd Zd Zd Zd Zd Zd Zd Zd Zej                  j                  dDg d      dd       Zej                  j                  dg d      ej                  j                  dg d      d               Zej                  j                  dg d      d        Zd Zd Zd Zd Zd Zd Zd Zej                  j                  dg d      d        Zd Zd Zd Zej                  j                  dDg d      d        Z G d de      Zd Zej                  j                  dddg      d        Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdÄ ZdĄ Zdń Zej                  j                  ddd%g      dȄ        ZdɄ Zej                  j                  dg dˢ      d̄        Zej                  j                  dg dˢ      d̈́        Zd΄ Zy)    )annotationsN)BZ2File)Iterator)ProcessPoolExecutor)	dataclass)GzipFile)repeat)groupbyidentityjoinmergepluckuniquevalmap)ListTask)	Bagcollectfrom_delayedlazifylazify_task	partitionreducebyreifytotal_mem_usage	assert_eq)	Blockwise)Delayed)Graph)	filetextstmpdirtmpfile)add	hlg_layerinc   )xr   r)      )r)      r    dsk   r)   c                    | dz  dk(  S Nr,   r    r)   s    R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/bag/tests/test_bag.pyisevenr4   /       q5A:    c                    | dz  dk(  S )Nr,   r+   r1   r2   s    r3   isoddr8   3   r5   r6   c                 X    t         j                  dk(  sJ t         j                  dk(  sJ y )Nr)   r.   )bnamenpartitionsr1   r6   r3   test_Bagr=   7   s$    66S====Ar6   c                 l    t         j                         t        t        j	                               k(  sJ y N)r:   __dask_keys__sortedr-   keysr1   r6   r3   	test_keysrC   <   s"    ??sxxz 2222r6   c                     t         j                  t              j                         } | dg ddz  fdddgdz  fgk(  sJ y )NTr   r,      r.   Fr+   )r:   r
   r4   computeresults    r3   test_bag_groupby_pure_hashrJ   @   sB    YYv&&(FtY]+eaVaZ-@AAAAr6   c                     t         j                  d       j                         } t        |       dk(  sJ dddgdz  f| v sJ dg ddz  f| v sJ y )Nc                     t        |       rdS dS )Nevenodd)r4   r2   s    r3   <lambda>z.test_bag_groupby_normal_hash.<locals>.<lambda>H   s    6!9 % r6   r,   rN   r+   r.   rM   rE   )r:   r
   rG   lenrH   s    r3   test_bag_groupby_normal_hashrQ   F   s^    YY?@HHJFv;!Aq6A:&(((IM"f,,,r6   shuffledisktasks	schedulersynchronous	processesc                <   t         j                  j                  |      5  t        d      D cg c]  }d |f }}t	        j
                  |      j                  d |       }|j                         }t        |      dk(  sJ 	 d d d        y c c}w # 1 sw Y   y xY w)NrU   2   c                    | d   S Nr   r1   r2   s    r3   rO   z'test_bag_groupby_none.<locals>.<lambda>S   s
    AaD r6   rR   r+   )	daskconfigsetrangedbfrom_sequencer
   rG   rP   )rR   rU   iseqr:   rI   s         r3   test_bag_groupby_nonerf   N   s     
9	-  "'),Qay,,S!)).')J6{a	   ,   s   BBA	BBBT)frozenc                  &    e Zd ZU ded<   dZded<   y)KeyintfooNz
int | Nonebar)__name__
__module____qualname____annotations__rl   r1   r6   r3   ri   ri   X   s    	HCr6   ri   keyr+   rk   r,   )rk   rl   
none_fieldno_none_fields)idsc                :   t        d      D cg c]  }| |f }}t        j                  |      j                  d |      }t        j
                  j                  |      5  |j                         }d d d        t              dk(  sJ y c c}w # 1 sw Y   xY w)NrZ   c                    | d   S r\   r1   r2   s    r3   rO   z,test_bag_groupby_dataclass.<locals>.<lambda>k   s
    ! r6   r]   rY   r+   )	ra   rb   rc   r
   r^   r_   r`   rG   rP   )rq   rR   rU   rd   re   r:   rI   s          r3   test_bag_groupby_dataclassrx   ^   s     #2Y
'C8
'C
'
%%ng%FA	9	- v;!	 ( s   B"BBc            
     >
   t        j                  t        d      d      } t        j                  t        dd      d      }| j                         }|j                         }dd}t	        t        j
                  ||       t        t        ||                   t	        t        j
                  ||       t        t        ||                   t	        t        j
                  || |      t        t        |||                   t	        t        j
                  || d      |D cg c]  } ||d       c}       t	        t        j
                  |d| 	      |D cg c]  } |d|	       c}       t        ||      D cg c]  \  }} |||d
       }}}t	        t        j
                  || |d
      |       t        ||      D cg c]  \  }} ||d       }}}t	        t        j
                  || d      |       t        |      }|D cg c]  } |||d
       }}t	        t        j
                  || j                         |d
      |       |D cg c]  } |||d
       }}t	        t        j
                  ||| j                         d      |       |D cg c]  } |d||       }}t	        t        j
                  |d| j                         |      |       t        j                  d      }	t	        t        j
                  || |	      |D cg c]  } ||d       c}       t	        t        j
                  || |		      |D cg c]  } ||d	       c}       t        j                  t        d      d      }
t        j                  t              5  t        j
                  || |
       d d d        t        j                  t              5  t        j
                  || j                         dd       d d d        t        j                  t        d      d      }t        j                  t              5  t        j
                  || ||      j                          d d d        t        j                  t              5  t        j
                  || ||
      j                          d d d        y c c}w c c}w c c}}w c c}}w c c}w c c}w c c}w c c}w c c}w # 1 sw Y   4xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nd   
   r<      r+   r,   c                    | |z   |z   S r?   r1   ar:   cs      r3   myaddztest_bag_map.<locals>.myaddw       1uqyr6   r   r:   r:   r   r   r   r'   n   r+   r,   r.   )rb   rc   ra   rG   r   maplistzipsumr^   delayedpytestraises
ValueError)r:   b2r)   x2r   rd   jsolx_sumr   fewer_partsunequals               r3   test_bag_mapr   q   s   
s4A			%S/r	:B			A	B bffUAS] 34bffUa $s5!}"56bffUAr"DUAr):$;<bffUAr"1$=aU1b\$=>bffUB!$q&A!uR1~&AB.1!Rj
9FQ5a3
9C
9bffUAs+S1),Q
4v15c?
4C
4bffUA%s+FE-/
05!s#
0C
0bffUAEEGrS137-/
05es#
0C
0bffUB3/5/1
2!53%1%
2C
2bffUcQUUW3S9RAbffUAq!!#<QE!RL#<=bffUA#a%@eAn%@A ""5:1=K	z	" &
ua%& 
z	" %
uaeegq!$% uSzr:G	z	" 2
uaB'//12	z	" 4
ua7b)1134 4I %>&A
9 5 1 1 3 $=%@& &% %
2 24 4sf   !R?
S
?S		SSS#SS$
S)
$S.'S;(T(T.S8;TTTc            
        t        j                  t        d      d      } t        j                  t        dd      d      }| j                         }|j                         }dd}| j	                  |      j                         t        t	        ||            k(  sJ | j	                  ||      j                         t        t	        |||            k(  sJ | j	                  |d      j                         |D cg c]  } ||d       c}k(  sJ | j	                  |d      j                         |D cg c]  } ||d       c}k(  sJ | j	                  ||d      j                         t        ||      D cg c]  \  }} |||d       c}}k(  sJ t        |      }| j	                  || j                         d      j                         |D cg c]  } |||d       c}k(  sJ y c c}w c c}w c c}}w c c}w )	Nrz   r{   r|   r}   c                    | |z   |z   S r?   r1   r   s      r3   r   ztest_map_method.<locals>.myadd   r   r6   r   r   )r,   r.   )rb   rc   ra   rG   r   r   r   r   )r:   r   r)   r   r   rd   r   r   s           r3   test_map_methodr      s   
s4A			%S/r	:B			A	B 55<!T#eQ-%888855##%c%B.?)@@@@55##%)B1%2,)BBBB55"5%%'A+FqE!rN+FFFF55b5!))+&)!Rj0"QaB0    FE5525&..0RS4TQU1eR5H4TTTT *C+F0 5Us   9G/4G4<G9G?c            
        g d} t        j                  | d      }d	d}|j                  |      j                         | D cg c]  } || 	 c}k(  sJ |j                  |d      j                         | D cg c]
  } ||ddi c}k(  sJ |j	                  d      j                         }|j                  ||      j                         | D cg c]  } ||d|j                         i c}k(  sJ t        j                  d      }|j                  ||      j                         | D cg c]
  } ||ddi c}k(  sJ y c c}w c c}w c c}w c c}w )
N)r+   r,   )r.   rF   )r'      )      )	   r{   r,   r|   c                    | |z   |z   S r?   r1   r   s      r3   r   ztest_starmap.<locals>.myadd   r   r6   r{   r   r   r+   r   )rb   rc   starmaprG   r   maxr^   r   )datar:   r   r   
max_secondr   s         r3   test_starmapr      sC   4D
1-A 99U##%T)B%))BBBB99Ub9!))+/NAqB/NNNN!J99Uj9)113378./q)J&&()8    	RA99Ua9 ((*.M1ua2.MMMM *C/N8 /Ns   E9EE/Ec                    t         j                  t              } t        t        t        t         j                        D ci c]2  }| j                  |ft        t        t        t         j                  |fff4 c}      }| j                  |k(  sJ | j                  t         j                  t              j                  k(  sJ y c c}w r?   )
r:   filterr4   r   r-   ra   r<   r;   r   r^   )r   rd   expecteds      r3   test_filterr      s    	A 1==)	
 VVQK%&&1661+!>??	
H 66X66QXXf%*****	
s   7C
c                     d } t         j                  |       }t        |      ddgdz  k(  sJ |j                  t         j                  |       j                  k(  sJ y )Nc                    | dz  dk(  S r0   r1   r2   s    r3   rO   ztest_remove.<locals>.<lambda>   s    !a%1* r6   r+   r.   )r:   remover   r;   )fr   s     r3   test_remover      sN    A	A7q!fqj   66QXXa[%%%%%r6   c            	         t        t              t        t              k(  sJ t        t        j                  t                    t        t        t        dd            dz        k(  sJ y )Nr+   r   r.   )rA   r:   Lr   r&   r   ra   r1   r6   r3   	test_iterr      sH    !9q	!!!!%%*U1a[(9A(=!>>>>r6   funcc                    t        t        j                         | t              v sJ t        j                  d d  | t              v sJ d | t	        j
                  t        d                  v sJ y )Nr'   rc   )strr:   r<   r;   rb   rc   ra   )r   s    r3   	test_reprr      s[    q}}a(((66"1:a   d2#3#3E!H#=>>>>r6   c                 t   ddgddgd} t        | dd      }t        |j                  d            h d	k(  sJ t        |j                  d
            h dk(  sJ t        |j                  d
dg            h dk(  sJ |j                  d
dg      j                  |j                  d
dg      j                  k(  sJ y )Nr+   r{   )r,      )r.      )rF   (   )r(   r*   r)   r,   r   >   r+   r,   r.   rF   r+   >   r{   r   r   r   >   r{   r+   r   r,   r   r.   r   rF   )r   r`   r   r;   dr:   s     r3   
test_pluckr      s    W%'71CDAAsAAqwwqz?l***qwwqz?....qww1v#GGGG77Aq6?177Aq6?#7#7777r6   c                     t        j                  g d       t        j                  t         fd       t         j                  dd             g dk(  sJ  j                  dd       j                   j                  dd       j                  k(  sJ  j                  d      j                   j                  dd       j                  k7  sJ y )N)Hello Worldc                 8    t         j                  d            S r\   )r   r   r   s   r3   rO   z)test_pluck_with_default.<locals>.<lambda>   s    d1771:&6 r6   r   )HNW)rb   rc   r   r   
IndexErrorr   r   r;   r   s   @r3   test_pluck_with_defaultr      s    
/0A
MM*674 !%5555771d  AGGAt$4$9$9999771:??agga.33333r6   c                    t        j                  t        d            j                  d       } | j	                  d      \  }}}t        |      t        t        d            k(  sJ t        |      t        d      D cg c]  }|dz   	 c}k(  sJ |j                  | j	                  d      d   j                  k(  sJ |j                  |j                  k7  sJ y c c}w )Nrz   c                    | | dz   | dz   fS )Nr+   r,   r1   r2   s    r3   rO   ztest_unzip.<locals>.<lambda>   s    Aq1ua!e3D r6   r.   r,   r   )rb   rc   ra   r   unzipr   r;   )r:   onetwothreerd   s        r3   
test_unzipr      s    
s$(()DEAggajOCe9U3Z((((;%*5Q1q5555588qwwqz!})))))88sxx 6s   =Cc                    t         j                  t              } | j                         t	        t
              k(  sJ | j                  t         j                  t              j                  k(  sJ t         j                  t        d      }|j                  | j                  k7  sJ |j                         t	        t
              dt         j                  z  z   k(  sJ |j                  t         j                  t        d      j                  k(  sJ t        j                  t        d      d      } d }| j                  |t        j                  t                     }|j                         t        |       k(  sJ |j                  | j                  |t        j                  t                     j                  k(  sJ t        j                  d      }t        |j                  d d	      j                               t        d      k(  sJ t        j                  d
gdgdggd      }t        |j                  t        g       j                  d            h dk(  sJ y )Nr{   )initialr'   r.   r|   c                H    | j                         } | j                  |       | S r?   )copyr$   accr)   s     r3   binopztest_fold.<locals>.binop  s    hhj

r6   helloc                (    dj                  | |g      S )Nr   )r   r   r:   s     r3   rO   ztest_fold.<locals>.<lambda>  s    277Aq6? r6   r   r+   r,   syncrY   >   r+   r,   r.   )r:   foldr$   rG   r   r   rq   r<   rb   rc   ra   r`   union)r   c2r   r   es        r3   	test_foldr     s   	sA99;#a&   55AFF3KOO###	
R	 B66QUU??::<3q6B$6666666QVVCV,00000
qq1A
 	
ucii/A99;#a&   55AFF5#))SUF;?????
!Aqvv2Bv?GGIJcO    	1#sQCa8Aqvvc2v&...@AYNNNr6   c            
         d } t         j                  | t        j                  t               t              }t        |t              sJ t        |t        t        t        d                         y )Nc                (    | j                  |       | S r?   )r$   )totr)   s     r3   r   ztest_fold_bag.<locals>.binop$  s    

r6   )combiner   out_typer'   )	r:   r   r`   r   r   
isinstancer   r   ra   )r   r   s     r3   test_fold_bagr   #  sL     	
uciiEAaac%(m$%r6   c                    t        t        j                               g dk(  sJ t        j                         j                  t        j                         j                  k(  sJ dt        j                         j                  v sJ t        j                         j	                         j                         dk(  sJ t        j                  dgdz  d      } | j                  d       j                         j                         g k(  sJ y )Nr   r+   r,   r.   rF   distinctr'   r   rZ   r|   )	rA   r:   r   r;   countrG   rb   rc   r   bags    r3   test_distinctr   -  s    !**,?222::<

 1 1111*****::<'')Q...


A38
4C::d$$&..0B666r6   c                     dD  cg c]  } d| i }} t        j                  |d      }t        t        |d             }t	        |j                  d      |       t	        |j                  d       |       y c c} w )N)
r   r+   r,   r+   r,   r.   r,   r.   rF   r'   r   r.   r|   c                    | d   S Nr   r1   r2   s    r3   rO   z(test_distinct_with_key.<locals>.<lambda>9  s
    af r6   rq   c                    | d   S r   r1   r2   s    r3   rO   z(test_distinct_with_key.<locals>.<lambda>;  s
    3 r6   )rb   rc   r   r   r   r   )rd   re   r   r   s       r3   test_distinct_with_keyr   6  sm    ;
<C8
<C
<


3A
.CF3$456Hcllsl#X.cll/l0(;	 =s   A:c                    t         j                         } t        |       ddddddk(  sJ t         j                  d      }t        |      ddddddk(  sJ | j                  t         j                         j                  k(  sJ | j                  |j                  k7  sJ |j                  t         j                  d      j                  k(  sJ t	        j
                  t        d      d      }|j                  d       }|j                         }t        |      t        t        t        d      d	gdz              k(  sJ t	        j
                  g d
d      }|j                  d       j                  d      }t        |g        y )Nr.   r   r,   split_everyr   partition_sizec                    | dk  S Nr{   r1   r2   s    r3   rO   z"test_frequencies.<locals>.<lambda>H  s
    QV r6   r{   r+   r   r   r   r   rF   r|   )
r:   frequenciesdictr;   rb   rc   ra   r   r   r   )r   r   r   r   r   bag2s         r3   test_frequenciesr  >  s?   	A7!aA!4444	
1	%B81qQ1555566Q]]_)))))66RWW77ammm277777			%)A	6B	#	$B
A7d3uRy1#(34444


<Q
7C::d''A'6DdBr6   c                     t        j                  g d      } t        | j                  d      j	                               g dk(  sJ y )N)r   r:   r:   r:   r   r   T)sort))r:   r.   )r   r,   )r   r+   )rb   rc   r   r  rG   r   s    r3   test_frequencies_sortedr  P  s;    
78A4(00237UUUUr6   c                    t        t        j                  d            g dk(  sJ t        j                  dd       } t        |       g dk(  sJ t        j                  dd d      }t        |      g dk(  sJ | j                  |j                  k7  sJ t        j                  d      j                  t        j                  d      j                  k(  sJ y )	NrF   )rF   rF   rF   r.   c                    |  S r?   r1   r2   s    r3   rO   ztest_topk.<locals>.<lambda>W  s     r6   r   r   r   r   r+   c                    |  S r?   r1   r2   s    r3   rO   ztest_topk.<locals>.<lambda>Y  s    ! r6   r,   )rq   r   )r   r:   topkr;   )r   r   s     r3   	test_topkr  U  s    q	?l***	ql#A7l"""	
|	3B8|###66RWW66!9>>QVVAY^^+++r6   r<   c                <   t        j                  g d|       }t        |j                  dd            ddgk(  sJ t        |j                  dd            d	dgk(  sJ |j                  dd      j                  |j                  dd      j                  k(  sJ y )
Nr   r,   r   r.   r   r|   r,   r+   r   r   r  r   r  )rb   rc   r   r  r;   r<   r:   s     r3   test_topk_with_non_callable_keyr  _  s    
2LAqa !gv%6666qa !ff%555566!6  AFF1!F$4$9$9999r6   c                 ~    t        j                  g dd      } t        | j                  dd             ddgk(  sJ y )Nr  r,   r|   c                    |S r?   r1   r   s     r3   rO   z0test_topk_with_multiarg_lambda.<locals>.<lambda>i  s    1 r6   r   r   r  )rb   rc   r   r  r   s    r3   test_topk_with_multiarg_lambdar  g  s:    
2BAqn-.7F2CCCCr6   c                     t        t        j                  d             t        t        j                  t                    k(  sJ y )Nc                    | dz   S Nr+   r1   r2   s    r3   rO   ztest_lambdas.<locals>.<lambda>m  s
    A r6   )r   r:   r   r&   r1   r6   r3   test_lambdasr  l  s+    o&'4c
+;;;;r6   c                    t        t        j                               dk(  sJ t        t        j                               dk(  sJ t        t        j	                               dk(  sJ t        t        j                               dk(  sJ t        j                         j                         du sJ t        j                         j                         du sJ t        j                         j                  t        j                         j                  k(  sJ t        j                         j                  t        j                         j                  k7  sJ y )N   r   rF   r   TF)
rj   r:   r   r   r   minanyrG   allrq   r1   r6   r3   test_reductionsr!  p  s    qwwy>Rquuw<2quuw<1quuw<1557??$$$557??%%%557;;!%%'++%%%557;;!%%'++%%%r6   c                    t         j                         j                  j                  d      sJ t         j	                  t        t              j                  j                  d      sJ t        d t         j	                  t        t              j                  D              sJ t         j	                  t        t        d      j                  j                  d      sJ y )Nr   c              3  b   K   | ]'  }t        |t              xr |j                  d        ) yw)r   N)r   r   
startswith).0ks     r3   	<genexpr>z'test_reduction_names.<locals>.<genexpr>~  s-      78
1c2q||E22s   -/rk   )r;   )r:   r   r;   r$  	reductionr  r   r^   r1   r6   r3   test_reduction_namesr)  {  s    557<<""5)));;sC %%00777 <=KKS<Q<V<V    ;;sCe;,11<<UCCCr6   c                 |   t        j                  t        d            } | j                  t        t        d      }| j                  t        t        d      }| j                  t        t        d      }|j                         |j                         cxk(  r|j                         k(  sJ  J t        |j                        t        |j                        kD  sJ | j	                  d      }| j	                  d      }|j                         |j                         k(  sJ t        |j                        t        |j                        kD  sJ |j                  |j                  k7  sJ |j                  | j	                  d      j                  k(  sJ |j                  | j	                         j                  k7  sJ y )N   r,   r   r   r'   )	rb   rc   ra   r(  r   rG   rP   r^   rq   )r:   r   r   r   s       r3   test_tree_reductionsr,    sF   
r#A	C!,A	C!,A	C!,A99;!))+444444qvv;QVV$$$	!A	!A99;!))+%%%qvv;QVV$$$55AEE>>55AEEaE(,,,,,55AEEGKKr6   )r+   r.   rF   c                D   t        t        d            }t        j                  d|       }t        |j	                         t        |      t        |      z         t        |j                         t        |             t        |j                         t        |             y )Nr  r|   )r   ra   rb   r   meanr   rP   r   )r<   r   r:   s      r3   test_aggregationr/    sc    U2YA
-AaffhAQ(aeegs1vaggiQ r6   r{   c                (   t        j                  d      }t        t        d            }t	        j
                  ||       }t        |j                         |j                  |             t        |j                         |j                  |             y )Nnumpyrz   r|   )	r   importorskipr   ra   rb   rc   r   r.  std)r<   npr   r   s       r3   test_non_splittable_reductionsr5    sb    			W	%Bc
D
;7Aaffh&aeegrvvd|$r6   c                     t        t        j                         t        j                  d             t        t        j                               t        j                  d      k(  sJ y Ng       @)r   r:   r3  mathsqrtfloatr1   r6   r3   test_stdr;    s:    aeegtyy~&>TYYs^+++r6   c                     t        t        j                         d       t        t        j                               dk(  sJ y r7  )r   r:   varr:  r1   r6   r3   test_varr>    s*    aeegs>S   r6   	transformc                0    t        j                  | d      S )Nr+   r|   )rb   rc   r2   s    r3   rO   rO     s    B4D4DQTU4V r6   c                    | g d      }t         j                  |t        t              }t	        |t        t        t        g dt        t        t                                  t	        t         j                  |t              t        t        t        g dt        t        t                                  |j                  t         j                  |t        t              j                  k(  sJ y )Nr   )on_selfon_other)r:   r   r8   r4   r   r   r;   )r?  otherr   s      r3   	test_joinrE    s     i E	uef5Aad69eT!W=>?affUE"DeYtAw)O$PQ66QVVE56VBGGGGGr6   c            	        t         j                  t        t        dt        d      } t        t        t        t         j
                  dfdft        | j                  j                               v sJ t        |       t        t	        t        d t        d      j                               k(  sJ | j
                  t         j                  t        t        dt        d      j
                  k(  sJ t         j                  t        d       } t        |       t        t	        t        d t        d      j                               k(  sJ y )Nr   c                    | |z   S r?   r1   r   s     r3   rO   ztest_foldby.<locals>.<lambda>  
    q r6   c                    | |z   S r?   r1   r   s     r3   rO   ztest_foldby.<locals>.<lambda>  s
    a r6   c                    | |z   S r?   r1   r   s     r3   rO   ztest_foldby.<locals>.<lambda>  rH  r6   )r:   foldbyr4   r$   r   r;   r   r^   valuesr`   r   itemsr   s    r3   test_foldbyrN    s    	aa(AfcAFFA;2d166==?6KKKKq6S&*@!QGMMOPPPP66QXXfc1c15:::::	/0Aq6S&*@!QGMMOPPPPr6   c            
     l   t               } dD ]  }t        j                  t        d      |      }|j	                  t
        t              }| j                  |g       dD ]  }|j	                  t
        t        |      }|j	                  t
        t        d|      }|j	                  t
        t        dt        |      }|j	                  t
        t        dt        d|      }| j                  ||||g         t        j                  |       }	|	d   }
t        |	D cg c]  }||
k(  	 c}      sJ y c c}w )N)r+   r       rz   r|   )FNr,   r.   r   r   )r   rb   rc   ra   rK  r4   r$   extendr^   rG   r   )r-   nr:   r   mr   r   r   gresultsfirstrs               r3   test_foldby_tree_reductionrX    s   
&C 	%U3ZQ7HHVS!

A3$ 	%A!4AaQ7Aa!<AaaQ?AJJ1a|$	%		% ll3GAJEG,qU
,---,s   D1c                 t   t        t        j                  t                    g dk(  sJ t        j                  t              j                  t        j                  t              j                  k(  sJ t        j                  d       j                  t        j                  t              j                  k7  sJ y )N)r'   r'   r'   c                    t        |       dz   S r  )rP   r   s    r3   rO   z%test_map_partitions.<locals>.<lambda>  s    c!fqj r6   )r   r:   map_partitionsrP   r;   r1   r6   r3   test_map_partitionsr\    s      %&)333C %%)9)9#)>)C)CCCC0166!:J:J3:O:T:TTTTr6   c            	     ^   t        d      D  cg c]  } t        j                  dd       }} t        d      D  cg c]  } t        j                  dd       }} t        j                  |d      }t        j                  |d      }dd} ||d      }t        t        j                  ||d      |       t        |j                  |d      |       t        |j                  |d      |        |||      }t        t        j                  |||      |       t        |j                  ||      |       t        |j                  ||      |       |j                         j                  t              } ||t        t        |      t        |      z              }t        |j                  ||      |       t        |j                  ||      |       t        j                  |      }t        |j                  ||      |       t        |j                  ||      |       y c c} w c c} w )Nrz   ir{   r|   c                    t        |t              rt        |      n|}t        | |      D cg c]  \  }}t	        ||       c}}S c c}}w r?   )r   rj   r	   r   r   )r)   yr   r:   s       r3   maximumz0test_map_partitions_args_kwargs.<locals>.maximum  s:    #As+F1I(+Aq	2fq!Aq	222s   A)r_  r   )ra   randomrandintrb   rc   r   r[  r.  applyrj   r   rP   r^   r   )rd   r)   r_  dxdyr`  r   dy_means           r3   test_map_partitions_args_kwargsrg    s   ,1#J7qc	"7A7,1#J7qc	"7A7			!	,B			!	,B3 !r
Cbr2C8b2.4b,c2
!Q-CbR0#6b2.4b,c2ggiooc"G
!SQ#a&)
*Cb73S9b137ll7#Gb73S9b1377 	87s
   H%H*c                     t        t        j                  d d      j                  d      } | sJ t	        | t
              sJ y )Nc                    | S r?   r1   r2   s    r3   rO   z/test_map_partitions_blockwise.<locals>.<lambda>  s    1 r6   ztest-string)token)r%   r:   r[  r^   r   r   layers    r3   test_map_partitions_blockwiserm    sB     	M:??E L5eY'''r6   c            	         t        j                  t        d      d      } dt        t	        | j                  dd                  cxk  rdk  sJ  J y)	zH
    Number of randomly sampled elements are in the expected range.
      r'   r|   r{   g?*   i,  N)rb   rc   ra   rP   r   random_sampler   s    r3   test_random_sample_sizerr    sJ     	t!4A Db1239c99999r6   c                 @   t        j                  t        d      d      } t        j                  t
              5  | j                  d       ddd       t        j                  t
              5  | j                  d       ddd       y# 1 sw Y   =xY w# 1 sw Y   yxY w)zN
    Specifying probabilities outside the range [0, 1] raises ValueError.
    rZ   r'   r|   Ng?)rb   rc   ra   r   r   r   rq  r   s    r3   test_random_sample_prob_rangeru    sy     	r2A	z	" 		z	" 	   s   B-BBBc                     t        j                  t        d      d      } | j                  d      }t	        |      t	        |      k(  sJ y)zf
    Repeated computation of a defined random sampling operation
    generates identical results.
    rZ   r'   r|   g?Nrb   rc   ra   rq  r   r   s     r3   'test_random_sample_repeated_computationrx     s>    
 	r2A	A7d1gr6   c                 $   t        j                  t        d      d      } t        | j	                  d            t        | j	                  d            k7  sJ | j	                  d      j
                  | j	                  d      j
                  k7  sJ y)z
    Repeatedly defining a random sampling operation yields different results
    upon computation if no random seed is specified.
    rZ   r'   r|         ?N)rb   rc   ra   r   rq  r;   r   s    r3   (test_random_sample_different_definitionsr{  *  sp    
 	r2A$%aooc.B)CCCC??3$$(<(A(AAAAr6   c                     t        j                  t        d      d      } | j                  dd      }| j                  dd      }t	        |      t	        |      k(  sJ y)zF
    Sampling with fixed random seed generates identical results.
    rZ   r'   r|   rz  i  Nrw  r   s      r3   test_random_sample_random_stater}  4  sP     	r2A	T"A	T"A7d1gr6   c                    t        d t        t        d t        t        d t        t        g d                  } t        |       t        d t        t        d t        t        g d            k(  sJ t        d t        t        d t        t        g d            } t        |       | k(  sJ t        d t        t        d t        t        t        d t        t        d t        t        d                        } t        d t        t        d t        t        t        d t        t        d                  }t        |       |k(  sJ t        d t        t        t        d t        t        d t        t        t        d t        t        d t        t        d                        t        d t        t        d t        t        t        d t        t        d t        t        d                                    } t        |       t        d t        t        t        d t        t        t        d t        t        d            t        d t        t        t        d t        t        d                        k(  sJ y )Nr   r_  r+   r,   )	r   r   r   r   r&   r   r   r4   r   )taskr   s     r3   test_lazify_taskr  >  s   c4eT$S)-LMNDtT3T3Y0O PPPPeT$S)<=Dt$$$T3T$tD&&#/NOPD
 D%dCd4QT6U!VWHt(((T3T$tD&&RS7T%UV
 T3T$tD&&RS7T%UV	
D  tsCdFFA!>?sCdFFA!>?	
!   r6   c                 n   t         t        t        t        g dfff} t	        |       t         t        t        g dffk(  sJ t        t        t        g dff} t	        |       | k(  sJ t        t        t        t        t
        t        dffff}t        t        t        t
        t        dfff}t	        |      |k(  sJ y )Nr   r_  )r   r   r   r&   r   r   r4   )r  r   r:   s      r3   test_lazify_task_legacyr  g  s    %#sI./0DtsC&; <<<<Ci()Dt$$$	cEFFC#89:;A	cFFC012Aq>Qr6   c                    | S r?   r1   r2   s    r3   rO   rO   s  s    a r6   c                     t         t        t        t         t        t        dfffft
        dft
        dfd} t         t        t        t        t        dffft
        dft
        dfd}t        |       |k(  sJ y )Nr_  r)   )r)   r   r:   )r   r   r&   r   r4   r   r   r   s     r3   test_lazifyr  v  sl    c3(= >?@XX	A
 sC&&#!6783xqRUhWA!9>>r6   c                     t        t        j                  d            ddgk(  sJ t        j                  d      dk(  sJ t        t        j                  dd      t              sJ y )Nr,   r   r+   )r   r+   FrG   )r   r:   taker   r   r1   r6   r3   	test_taker    sS    q	?q!f$$$66!9affQf.444r6   c                 L   t        t        j                  dd            g dk(  sJ t        j                  dd      dk(  sJ t        j                  dd      dk(  sJ t        j                  t
              5  t        j                  dd	       d d d        y # 1 sw Y   y xY w)
Nr   r,   r|   )r   r+   r,   r.   rF   r   rt  r.   )r   r+   r,   r+   r'   )r   r:   r  r   r   r   r1   r6   r3   test_take_npartitionsr    s    qa()-????66!6$(::::66!6$	111	z	" !	qa ! ! !s   9BB#c                 6   t         j                  j                  d      5  t        j                  t
              5  t        j                  d       d d d        t        j                  t
              5  t        j                  d       d d d        t        j                  d      5 } t        j                  dd       t        j                  dd	
       d d d         rJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   pxY w# 1 sw Y   /xY w# 1 sw Y   y xY w)Nr   rY   rz   r   Trecordr,   r|   F)warn)
r^   r_   r`   r   warnsUserWarningr:   r  warningscatch_warningsr  s    r3   test_take_npartitions_warnr    s     
6	* 
\\+& 	FF3K	 \\+& 	FF1I	 $$D1 	"VFF1!F$FF15F!	" z6
 
	 		 		" 	"
 
sR   DC+!D2C7D&/DD+C4	0D7D 	<DD	DDc                 B    t        t        d g d      t              sJ y )Nc                    | S r?   r1   r2   s    r3   rO   z"test_map_is_lazy.<locals>.<lambda>  s    A r6   r   )r   r   r   r1   r6   r3   test_map_is_lazyr    s    c+y18<<<r6   c                 ^    t        t        t        j                               t              sJ y r?   )r   r  r:   r  r1   r6   r3   "test_can_use_dict_to_make_concreter    s    d1==?+T222r6   Hangs)reasonc                     t        j                  ddg      } | j                  dk(  sJ t        j                  d      }|j                  dk(  sJ d|j                  d      v sJ y )Nzhttp://google.comzhttp://github.comr,   z:http://raw.githubusercontent.com/dask/dask/main/README.rstr+   s   Dask
r{   )rb   from_urlr<   r  r   s     r3   test_from_urlr    sb     	(*=>?A==A
PQA==Ar
"""r6   c                    t        ddd      5 } t        j                  |       D ch c]  }|j                          c}t	        d      k(  sJ t        j                  d      D ch c]  }|j                          c}t	        d      k(  sJ 	 d d d        t        j                  t        d        y c c}w c c}w # 1 sw Y   /xY w)NzA
BzC
D)za1.logza2.logABCDza*.logc                 ,    t        j                  d      S )Nznon-existent-*-path)rb   	read_textr1   r6   r3   rO   z test_read_text.<locals>.<lambda>  s    bll3H&I r6   )r!   rb   r  stripr`   r   r   r   )fnslines     r3   test_read_textr    s    	f7	8 PC)+c):;

;s6{JJJ)+h)?@

@CKOOOP MM*IJ <@P Ps'   B=B3(B=&B8=B=3
B==Cc            	        t               5 } t        | d      5 }|j                  dt        j                  z   j                         dz         d d d        t        j                  | d      }t        j                  |       }t        |j                        dkD  sJ t        t        t        |j                  j                                     t        t        t        |j                  j                                     k(  sJ t        j                  | gd      }t        |      t        |      k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)NwbzHello, world!rz   	blocksizer'   )r#   openwriteoslinesepencoderb   r  rP   r^   r   r   r   r  )fnr   r:   r   r   s        r3   test_read_text_larger    s    	 	"b"d^ 	CqGG_rzz199;cAB	CLLs+LL166{QCQUU[[]+,Saeekkm5L0MMMMLL"-Aw$q'!!!	" 	"	C 	C	" 	"s#   E4D:C$E:E	?EEc                    t               5 } t        | d      5 }|j                  dt        j                  z   j                  d      dz         d d d        t        j                  | dd      }t        j                  | d      }t        |j                        dkD  sJ |j                  j                         j                  d       }|j                  j                         j                  d	       }t        |      t        |      k(  sJ t        j                  | gdd      }t        |      t        |      k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)
Nr  u	   你好！gb18030rz   )r  encodingr  r'   c                $    | j                  d      S Nutf-8r  r2   s    r3   rO   z)test_read_text_encoding.<locals>.<lambda>      AHHW,= r6   c                $    | j                  d      S r  r  r2   s    r3   rO   z)test_read_text_encoding.<locals>.<lambda>  r  r6   )r#   r  r  r  r  r  rb   r  rP   r^   r   r  r   r   )r  r   r:   r   b_encc_encr   s          r3   test_read_text_encodingr    s   	 "b"d^ 	HqGG[2::-55i@3FG	HLLsY?LLi0166{Q!!"=>!!"=>E{d5k)))LL"yAAw$q'!!!" "	H 	H" "s#   E5EC0EE	EEc                    t        d      5 } d}t        | d      }|j                  |       |j                          t	        j
                  t              5  t        j                  | dd       d d d        t        j                  | d       }|j                  dk(  sJ d	j                  |j                               |j                         k(  sJ 	 d d d        y # 1 sw Y   mxY w# 1 sw Y   y xY w)
Ngzsx  Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
r  rZ   
)r  linedelimiterr  r+   r   )r#   r   r  closer   r   r   rb   r  r<   r   rG   decode)r  r   r   r   s       r3   test_read_text_large_gzipr    s    	 5"'R			]]:& 	?LLr>	? LLt,}}!!!wwqyy{#t{{}4445 5	? 	?5 5s%   A	C&C.A"C&C#	C&&C/z(https://github.com/dask/dask/issues/6914c                    t        j                  d       d} t        j                  dt	        d            }|j                  d      | k(  sJ t        j                  g dt	        d            }|j                  d	k(  sJ y )
Ns3fs)z(total_bill,tip,sex,smoker,day,time,size
z"16.99,1.01,Female,No,Sun,Dinner,2
z 10.34,1.66,Male,No,Sun,Dinner,3
z21.01,3.5,Male,No,Sun,Dinner,3
z 23.68,3.31,Male,No,Sun,Dinner,2
zs3://tip-data/t*.gzT)anon)storage_optionsr'   )zs3://tip-data/tips.gzzs3://tip-data/tips.jsonzs3://tip-data/tips.csvr.   )r   r2  rb   r  r  r  r<   )	five_tipsr   r   s      r3   test_from_s3r    st     I 	*DdOLA66!9	!!! 	V$	A ==Ar6   c                     t        j                  g dd      } t        | j                        dk(  sJ t	        |       h dk(  sJ y )N)r+   r,   r.   rF   r'   r.   r|   >   r+   r,   r.   rF   r'   )rb   rc   rP   r^   r`   r   s    r3   test_from_sequencer    s<    
a8Aqvv;!q6_$$$r6   c                     t        t        d            } t        j                  |       }t	        |      t	        |       k(  sJ y )Ni  )r   ra   rb   rc   r`   )r   r:   s     r3   test_from_long_sequencer    s6    U4[A
Aq6SVr6   c                    t        j                  d       t        j                  d       t        j                  g       } | j                  dk(  sJ | j                  ddi      j                         }|j                  sJ d       y )Npandasdask.dataframer+   r   rj   metazDataFrame is not empty)r   r2  rb   rc   r<   to_dataframerG   empty)r:   dfs     r3   test_from_empty_sequencer    sp    
!
()
A==A	
c5\	*	2	2	4B88---8r6   c                 N   t         j                  t               } | j                  t         j                  dz  k(  sJ t        |       t        D ch c]  }t        D ]  }||f  c}}k(  sJ t        j                  g d      }t        j                  g d      }|j                  |      }t        |      dD ch c]  }dD ]  }||f  c}}k(  sJ |j                  | j                  k7  sJ |j                  |j                  |      j                  k(  sJ y c c}}w c c}}w )Nr,   )r+   r,   r.   rF   )r{   r   r   )r:   productr<   r`   r   rb   rc   r;   )r   rd   r   r)   r_  zs         r3   test_productr    s    	
1B>>Q]]A----r7q4!!4Q1v4v4444
&A
&A			!Aq6lILIqq!fIfIIII66RWW66QYYq\&&&&& 5
 Js   D;D!c            
     ^   t        j                         5 } t        t        t	        d      d|        t	        d      D ]U  }| j                  |      sJ t        t        t        || d            | j                  |      D cg c]  }||gf	 c}k(  rUJ  	 d d d        y c c}w # 1 sw Y   y xY w)Nr   r.   r   )partdPickler   r   ra   getrA   r   )prd   r   s      r3   test_partition_collectr  %  s    	 W1(E!Ha+q 	WA558O8'(Aq"56QUUSTX:VAs8:VVVV	WW W ;W	W Ws$   A*B#?B
B#B#B##B,c                 z   t         j                  t              } t        |       }|g dg dg dg dg ddk(  sJ | j                  t         j                  k(  sJ | j
                  t         j                  t              j
                  k(  sJ | j
                  t         j                  d       j
                  k7  sJ y )Nr   r   r   r+   r+   r+   r,   r,   r,   r.   r.   r.   rF   rF   rF   r   c                    | dz   S r  r1   r2   s    r3   rO   ztest_groupby.<locals>.<lambda>9  s
    Q r6   )r:   r
   r   r  r<   r;   )r   rI   s     r3   test_groupbyr  -  s    			(A!WF    ==AMM)))66QYYx(-----66QYY/44444r6   c                     t        j                  g dg dg dg      } t        | j                  d            }t	        t
        |      g dg dgg dgdk(  sJ y )Nr   )r+   rF   r   r,   r.   rF   r   r   )rb   rc   r  r
   r   rA   r:   rI   s     r3   test_groupby_with_indexerr  <  sL    
)Y	:;A!))A,F&&!)Y)?YK%PPPPr6   c                     t         j                  d d      } t        |       }|g dg dg dg dg dd	k(  sJ | j                  dk(  sJ y )
Nc                    | S r?   r1   r2   s    r3   rO   z7test_groupby_with_npartitions_changed.<locals>.<lambda>C  s     r6   r+   r|   r  r  r  r  r  r   )r:   r
   r  r<   )rI   result2s     r3   %test_groupby_with_npartitions_changedr  B  sY    YY{Y2F6lG    """r6   c                     ddl m}  t        j                  j	                  |       5  t
        j                  d d      j                          d d d        y # 1 sw Y   y xY w)Nr   r  rY   c                    | S r?   r1   r2   s    r3   rO   z2test_groupby_with_scheduler_func.<locals>.<lambda>T  s    A r6   r+   r|   )dask.threadedr  r^   r_   r`   r:   r
   rG   r  s    r3    test_groupby_with_scheduler_funcr  P  sE    !	3	' 8			+1	-5578 8 8s   'AA c                    t        j                  g d      } t        j                  g d      }t        j                  | |g      }t        |      g dk(  sJ |j                  t        j                  | |g      j                  k(  sJ y )Nr   rF   r'   r   r+   r,   r.   rF   r'   r   )rb   rc   concatr   r;   r   s      r3   test_concatr  W  sl    
#A
#A
		1a&A7((((66RYY1v&+++++r6   c                     t        j                  dgddgg      } t        | j                               g dk(  sJ | j                         j                  | j                         j                  k(  sJ y )Nr+   r,   r.   r   )rb   rc   r   flattenr;   r   s    r3   test_flattenr  _  s[    
1#1v'A			)))99;qyy{/////r6   c                     t        j                  ddg      } t        j                  ddg      }t        j                  | j                  t              |g      }t        |      g dk(  sJ y )Nr+   r,   rF   r'   )r,   r.   rF   r'   )rb   rc   r  r   r&   r   )r   r:   rI   s      r3   test_concat_after_mapr  e  sV    
!Q A
!Q AYYc
A'F<<'''r6   c                     t         j                  d       } t        | j                   }t	        |       t	        |      k(  sJ | j
                  |j
                  k(  sJ y )Nc                    | dz   S r  r1   r2   s    r3   rO   ztest_args.<locals>.<lambda>m  s
    A r6   )r:   r   r   _argsr   r<   )r   r   s     r3   	test_argsr  l  sI    	oAQWWA7d1g==AMM)))r6   c                    t        j                  d      } t        j                  d      }d }dgg ddgd}t        |dd	      }| j                  |j	                         d
dg      }|j                         }|j                  j                  ||j                  ddd      d       |j                  d
dg      }|j                  j                  ||d        |||       |j                  ddg      }|j                  j                  ||d        |||       |j                  d       }|j                         }|j                  j                  ||d        |||       |j                  |j                         j                  k(  sJ |ddgfD ];  }|j                  |      }|j                  j                  ||d        |||       = t        j                  t              5  |j                  d
dg|       d d d        |j                  d       }t        j                  t              5  |j                          d d d        |j                  d
      }|d
g   }|j                  |      }|j                  j                  ||d        |||       | j                  d
t        d      i      }t!        j"                  t        d      d      }t$        t&        fD ]J  }	|j)                  |	      j                  |      }|j                  j                  ||d        |||       L y # 1 sw Y   4xY w# 1 sw Y   xY w)Nr  r  c                n    t        fdt        j                  | j                          D              sJ y )Nc              3  l   K   | ]+  }|j                   j                   k(  j                          - y wr?   )dtypesr   )r%  r  r   s     r3   r'  z9test_to_dataframe.<locals>.check_parts.<locals>.<genexpr>y  s,      
/0QXX#((*
s   14)r   r^   rG   
to_delayed)r  r   s    `r3   check_partsz&test_to_dataframe.<locals>.check_partsx  s1     
48LL"--/4R
 
 	
 
r6   r   )r{   r   )rz   r}   ))testr   )r  r+   )r  r,   r  r.   r   r:   )columnsr   r+   r   F)check_index)r   i8)r:   r  r  c                0    t        t        ddg|             S )Nr   r:   )r  r   r2   s    r3   rO   z#test_to_dataframe.<locals>.<lambda>  s    S#sQ/0 r6   )r  r  c                    | d   dkD  S )Nr   r}   r1   r2   s    r3   rO   z#test_to_dataframe.<locals>.<lambda>  s    AcFSL r6   rz   r'   r|   )r   r2  r   	DataFramerG   r  utilsr   renamer   _namer   r   r   r   ra   rb   rc   itertupler[  )
pdddr  r-   r:   r   r  r  r   r   s
             r3   test_to_dataframer  t  s   			X	&B			-	.B

  r:@V
WCCA
,,qyy{S#J,
7C 
	BHHr3::AA.>:?US	
c
	+BHHr3E2C	
k;7	8BHHr3E2C 	
01A	
	BHHr3E2C88q~~'----- {K01 ^^^&
2s6B 
z	" 5	Sz45 
(	)B	z	" 
 	
A
se*C	
S	!BHHr3E2C ,,U3Z(
)C
s3AE] a --3-7
2s6B%5 5
 s   -L56M5L?Mr  bz2r   z
ext,myopenc                b   t        j                  g dd      }t               5 }|j                  t        j
                  j                  |d| z         d      }t        j                  |ddi t        j
                  j                  t        j
                  j                  |d	| z               sJ  |t        j
                  j                  |d	| z         d
      }|j                         }t        |d      r|j                         }d|v sJ |j                          d d d        y # 1 sw Y   y xY w)Nabc123xyzr,   r|   *.Fr  rU   r   1.rbr  r   )rb   rc   r"   to_textfilesr  pathr   r^   rG   existsreadhasattrr  r  )extmyopenr:   dirr   r   texts          r3   test_to_textfilesr-    s    
.A>A	 
SNN277<<TCZ8%NHa*6*ww~~bggll3s
;<<<277<<TCZ0$7vvx4";;=D}}		
 
 
s   C8D%%D.c                    g d} t        j                  | d      }t               5 }|j                  |       t        j                  t
        j                  j                  |d      d      j                  t              j                  t        j                        j                         }| |k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r:   r   r   r   r   rT  hrd   r   r&  lrS  rR  or     r|   *asciir  )rb   rc   r"   r$  r  r  r%  r   r   r   r  rG   )re   r:   dnouts       r3   /test_to_textfiles_name_function_preserves_orderr8    s    C$ 	"-A	 	R	r LLb#.ASXS^WY	 	 czz	 	 	s   BB<<Cc                     g d} t        j                  | d      }t               5 }|j                  |t               d d d        y # 1 sw Y   y xY w)Nr/  r3  r|   )name_function)rb   rc   r"   r$  r   )re   r   r6  s      r3   $test_to_textfiles_name_function_warnr;    sI    C$ 	"-A	 .R	r-. . .s   AAc            	        t        j                  g dd      } t        D ]  \  }}t               5 }| j	                  t
        j                  j                  |d|z         dd      }t        j                  |  t
        j                  j                  t
        j                  j                  |d|z               sJ  |t
        j                  j                  |d|z         d	      }|j                         }t        |d
      r|j                  d      }d|v sJ |j                          d d d         y # 1 sw Y   xY w)N)u   汽车u   苹果   天气r,   r|   r!  r  F)r  rG   r"  r#  r  r=  )rb   rc   ext_openr"   r$  r  r%  r   r^   rG   r&  r'  r(  r  r  )r:   r)  r*  r+  r   r   r,  s          r3   test_to_textfiles_encodingr?    s   
7QGA VX 	S$*-	5  A LL!77>>"'',,sD3J"?@@@rww||C4d;A668DtX&{{9-t###GGI	 		 	s   C7D44D>	c                    t        j                  g dd      } t               5 }t               5 }| j                  ||g       t        j
                  j                  |      sJ t        j
                  j                  |      sJ 	 d d d        d d d        t               5 }| j                  |       t        j
                  j                  |      sJ t        j
                  j                  t        j
                  j                  |d            sJ 	 d d d        t        j                  t              5  | j                  d       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   UxY w# 1 sw Y   y xY w)Nr  r,   r|   z0.partr'   )rb   rc   r#   r$  r  r%  r&  r"   r   r   r   	TypeError)Br   r:   dirnames       r3   test_to_textfiles_inputsrD    s"   
.A>A	 %aY 	%!NNAq6"77>>!$$$77>>!$$$	%% 
 ?W	www~~g&&&ww~~bggll7H=>>>?
 
y	! 	q 	% 	%% %? ?
 s<   EAEE A2E'4E3E	EE$'E03E<c                 (   t        j                  g dd      } t               5 }dD ]H  }| j                  |g|       t	        |      5 }|j                         }d d d        dd|rdnd	gk(  rHJ  	 d d d        y # 1 sw Y   #xY w# 1 sw Y   y xY w)
Nr   r+   r|   )FT)last_endlineza
zb
zc
r   )rb   rc   r#   r$  r  	readlines)r:   r  rF  r   rI   s        r3   test_to_textfiles_endlinesrH    s    
a8A	 Lb' 	LLNNB4lN;b 'Q'eU\EsKKKK		LL L' 'L Ls)   %B	A<B/B<BBBc                     t        j                  g dd       dt         j                        v sJ dt         j                        v sJ t	         j                  j                               g dk(  sJ t	         j                  j                  d            dd	gd
dgdd	ggk(  sJ t	         j                  j                  d            ddgk(  sJ t        j                  t         fd        j                  j                  d      j                   j                  j                  d      j                  k(  sJ  j                  j                  d      j                   j                  j                  d      j                  k7  sJ y )NAlice Smithz	Bob JonesCharlie Smithr,   r|   splitmatchzalice smithz	bob joneszcharlie smith AliceSmithBobJonesCharliez*SmithrK  rL  c                 0     j                   j                  S r?   )r   	sfohsofhfr   s   r3   rO   z'test_string_namespace.<locals>.<lambda>7  s    !%%// r6   z*John)rb   rc   r+  r   r   lowerrM  rN  r   r   AttributeErrorr;   r   s   @r3   test_string_namespacerZ  )  s@   
FTUVAc!%%j   c!%%j   "OOOOC !	'		G&   
 H%&=/*JJJJ
MM."9:55;;x %%X)>)C)CCCC55;;x %%W)=)B)BBBBr6   c                     t        j                  g dd      } t        | j                  j	                               g dk(  sJ y )NrJ  r,   r|   rO  )rb   rc   r   r   rX  r   s    r3   "test_string_namespace_with_unicoder\  <  s4    
FTUVA"OOOOr6   c                     t        j                  g dd      } t        | j                  j	                               ddgddgddggk(  sJ y )	NrJ  r,   r|   rQ  rR  rS  rT  rU  )rb   rc   r   r   rM  r   s    r3   test_str_empty_splitr^  A  sT    
FTUVA	'		G#   r6   c                     t        j                  g dg dgd      } d }| j                  |      }t        |      g dg dgk(  sJ y )Nr   r  r,   r|   c              3  (   K   | D ]	  }|dz     y wr  r1   )r   r)   s     r3   r   z*test_map_with_iterator_function.<locals>.fM  s      	Aa%K	   r  )r'   r   r   )rb   rc   r   r   )r:   r   r   s      r3   test_map_with_iterator_functionrb  J  sD    
)Y/Q?A 	
aA7y),,,,r6   c                     t        j                  g d      } | j                  d       j                         }t	        |t
              rJ y )Nr   c                    | dz   S r  r1   r2   s    r3   rO   z8test_ensure_compute_output_is_concrete.<locals>.<lambda>X  s
    QU r6   )rb   rc   r   rG   r   r   r  s     r3   &test_ensure_compute_output_is_concretere  V  s<    
#AUU?#++-F&(++++r6   c                      e Zd ZddZd Zy)
BagOfDictsNc                0    | j                  fd      S )Nc                (    | j                        S r?   r  )r   defaultrq   s    r3   rO   z BagOfDicts.get.<locals>.<lambda>^  s    !%%W"5 r6   r   )selfrq   rj  s    ``r3   r  zBagOfDicts.get]  s    xx566r6   c                4    fd}| j                  |      S )Nc                    | <   | S r?   r1   )r   rq   values    r3   setterzBagOfDicts.set.<locals>.settera  s    AcFHr6   rk  )rl  rq   ro  rp  s    `` r3   r`   zBagOfDicts.set`  s    	 xxr6   r?   )rm   rn   ro   r  r`   r1   r6   r3   rg  rg  \  s    7 r6   rg  c                    t        t        j                  dddiig      j                   } | j	                  d      j	                  d      j                         d   dk(  sJ | j	                  d      j                  dd      j                         d   dddk(  sJ t        | j	                  d      j	                  d      t               sJ y )Nr   r:   r   r   r   zEXTENSIBILITY!!!r:   r   )rg  rb   rc   r  r  rG   r`   r   )dictbags    r3   test_bag_class_extendrt  h  s    "**S3*,=+>?EEFG;;s$,,.q1S888;;s%78@@B1EJ    gkk#&**3/<<<r6   c                 @   dj                         } t               5 }t        |d      5 }|j                  |        d d d        t	        j
                  |      }|j                         d   | j                  d      k(  sJ 	 d d d        y # 1 sw Y   OxY w# 1 sw Y   y xY w)Nu   €r  r   r  )r  r#   r  r  rb   r  rG   r  )bin_datar  r   r   s       r3   
test_gh715rw  r  s     H	 :b"d^ 	qGGH	LLyy{1~!9999	: :	 	: :s"   BBABB	BBc                 v    t        j                  g d      j                  d       } | j                  d       y )Nr   c                    | dz   S r  r1   r   s    r3   rO   z1test_bag_compute_forward_kwargs.<locals>.<lambda>|  s
    !a% r6   r{   )bogus_keyword)rb   rc   r   rG   r2   s    r3   test_bag_compute_forward_kwargsr{  {  s+    
#''8AIIBIr6   c                    t        j                  g dd      } | j                  t              j	                         \  }} }t        d || |fD              sJ | j                         ddgk(  sJ t        j                  g dd      } | j                         j	                         }t        |t              sJ |j                         dk(  sJ y )Nr  r.   r|   c              3  <   K   | ]  }t        |t                y wr?   )r   r   )r%  r)   s     r3   r'  z"test_to_delayed.<locals>.<genexpr>  s     9!z!W%9s   rF   r'      )
rb   rc   r   r&   r
  r   rG   r   r   r   )r:   r   r   ts       r3   test_to_delayedr    s    
+;AeeCj##%GAq!91ay999999;1a&   
+;A	Aa!!!99;"r6   c                  
 d
ddl m 
fd}|j                  t        j                  j
                  d|       t        j                  g dd      }|j                  t              j                  t              j                  t              }|j                         \  }d
|j                          
dk(  sJ d
t        j                  g dd      }|j                  t              j                  t              j                  t              }|j                         \  }|j                         |j                         k7  sJ |j                  d	      \  }t        |j                        t        |j                        k(  sJ |j                         |j                         k(  sJ d
|j                         }
dk(  sJ d
|j                         }
d
k(  sJ ||k(  sJ |j                         }	|	j                         }|j                         |	j                         k(  sJ |	j                  d	      }t        |j                        t        |	j                        k(  sJ |j                         |	j                         k(  sJ d
|j                         |j                         k(  sJ 
d
k(  sJ d
|j                  t!        |       d      \  }
dk(  sJ y )Nr   )r   c                     dz   | i |S r  r1   )argskwargscallsr   s     r3   count_reifyz3test_to_delayed_optimize_graph.<locals>.count_reify  s    
d%f%%r6   r   r  r+   r|   Foptimize_graphr.   r  )dask.bag.corer   setattrr^   r   corerb   rc   r   r&   r
  rG   __dask_layers__r  r   r$  r   )r"   monkeypatchr  r:   r   r   d2rI   r  r)   r  r   s             @@r3   test_to_delayed_optimize_graphr    s[   E#&
 w<
+;A	
s		 	 	%B
--/CQEIIKA::E
+;A	
s		 	 	%B
--/CQ""4"4"6666===.DR=DM)))2#5#5#7777EYY[FA::EjjlGA::W
A	A!"3"3"5555	
U	+B=DL(((1#4#4#6666E99;"**,&&&A::E
//#f+u/
5CQA::r6   c                    ddl m}   | g d       | g d       | g d      }}}t        |||g      }|j                  t        |||g      j                  k(  sJ t	        |t
              sJ t        |      g dk(  sJ   | t              |      }t        j                  j                  |      }|j                         |j                         cxk(  rdk(  sJ  J y )Nr   r   r   r  )r   r   r   )	r+   r,   r.   rF   r'   r   r   r   r   r   )dask.delayedr   r   r;   r   r   r   r   rb   ItemrG   )r   r   r:   r   bb
asum_value	asum_items          r3   test_from_delayedr    s    $i ')"4gi6H!qA	q!Qi	 B77lAq!9-22222b#82222aJ$$Z0I9#4#4#6;!;;;;;r6   c                    ddl m}  d } | |d      }t        j                  t	        d      D cg c]
  } |d       c}      }t        j
                  |j                         |j                  d      j                         |j                  d      j                         j                         d	      d
k(  sJ y c c}w )Nr   r  c                &    d t        |       D        S )Nc              3  (   K   | ]
  }d ddgi  yw)
operationsr+   r,   Nr1   )r%  _s     r3   r'  zCtest_from_delayed_iterator.<locals>.lazy_records.<locals>.<genexpr>  s     911v&9ra  ra   )rR  s    r3   lazy_recordsz0test_from_delayed_iterator.<locals>.lazy_records  s    9a99r6   F)purer'   r  r   rY   )   r  rZ   )	r  r   rb   r   ra   rG   r   r   r  )r   r  delayed_recordsr  r   s        r3   test_from_delayed_iteratorr    s    $: l7O
//uQx@!?1-@
AC::				,%%'		,'')//1	
 
   As   B;c                     dD ]f  } t        j                  d|       }t        |j                        | k(  sJ |j                  | k(  sJ t        |      t        t        d            k(  rfJ  y )Nr+   r   r{      rz   r|   )rb   ra   rP   r^   r<   r   r  s     r3   
test_ranger    sd    % +HHSk2166{k)))}}+++Aw$uSz****	+r6   r  c                   t        j                  t        d|d      |       }t        j                  t        d|d      |       }t        j                  ||      }|j                  |j                  k(  sJ |j                  |j                  k(  sJ t        |      t        t        t        d|d      t        d|d                  k(  sJ y )Nr   r,   r|   r+   )rb   rc   ra   r   r<   r   )r<   hievensoddspairss        r3   test_zipr    s    U1b!_+FEE!ROEDFF5$E 1 1111 0 0000;$s5B?E!RODEEEEr6   nin)r+   r,   r         nout)r+   r,   r'   r+  r  c                   t        j                  t        d      |       }|j                  |      }|j                  |k(  sJ t        ||       t        j                  |j                  |j                               }t        |      sJ y )Nrz   r|   )
rb   rc   ra   repartitionr<   r   r^   r  r@   r   )r  r  r:   r   rU  s        r3   test_repartition_npartitionsr    sl     	s5A	$'A==D   aOhhqvvq01Gw<<r6   z	nin, nout)	r+   r+   )r,   r+   )r'   r+   r   )r,   r,   )r'   r,   )r+   r'   )r,   r'   )r'   r'   c                   t        j                  t        dd      |       }t        |j	                  t
              j                               }|j                  ||z        }|j                  |k\  sJ t        ||       y )Nr+   rz   r|   r   
rb   rc   ra   r   r[  r   rG   r  r<   r   )r  r  r:   	total_memr   s        r3   test_repartition_partition_sizer    sj     	q#C8AA$$_5==?@I	i4&79A==D   aOr6   c                 f   t        j                  t        dd      d      } t        | j	                  t
              j                               }| j                  |dz        }|j                  dk\  sJ t        | |       |j                  |dz        }|j                  dk\  sJ t        ||       y )Nr+   rz   r|   r,   r   r'   r  )r:   r  r   r   s       r3   (test_multiple_repartition_partition_sizer    s    
q#A6AA$$_5==?@I	i1n6A==AaO	i1n6A==AaOr6   c            
        t        j                  d      } t        j                  t	        d      D cg c]  }| j                  t	        d             c}d      }t        |j                  t              j                               }|dz  }|j                  |      }|j                  dk\  sJ t        ||       y c c}w )Nr1  rF   rz   r+   r|   r   )r   r2  rb   rc   ra   arrayr   r[  r   rG   r  r<   r   )r4  r  r:   r  new_partition_sizer   s         r3   .test_repartition_partition_size_complex_dtypesr    s    			W	%B
aA1"((5:.AqQAA$$_5==?@I"a	%78A==AaO Bs   !B=c                 X   t        j                  t        d      d      } | j                  d      }| j                  |j                  k7  sJ | j                  d      }| j                  |j                  k7  sJ |j                  |j                  k7  sJ | j                  d      }| |u sJ y )Nrz   r'   r|   r,   r   )rb   rc   ra   r  r;   )r:   r   r   s      r3   test_repartition_namesr  *  s    
s3A	aA66QVV	bA66QVV66QVV	aA6M6r6   c                     t        j                  t              5  t        j                  t        d            } | j                  dd       d d d        y # 1 sw Y   y xY w)Nr{   r'   5MiB)r<   r   )r   r   r   rb   rc   ra   r  r   s    r3   test_repartition_input_errorsr  7  sG    	z	" >uRy)Af=> > >s   2AAc                    g dddgg ddgg} t        |       D ci c]
  \  }}d|f| }}}t        j                  |dt        |             }|j	                  t
              }|j                  |j	                  t
              j                  k(  sJ |j                  |j	                  t
        d      j                  k7  sJ |j                         g dk(  sJ |j	                  t
        d      j                         g d	k(  sJ |j	                  t
              j                  t              j                         g d
k(  sJ t        j                  g dd      }|j	                  t
              j                         g dk(  sJ y c c}}w )Nr   rF   r'   r   r   r  rt  )r+   r.   r   r{   r  r~  r  )rt  r   r,   r'   r      r      )r,   rF   r   r  r3        r+   r|   )r+   r.   r   )	enumeraterb   r   rP   
accumulater$   r;   rG   r   r&   rc   )partsrd   r  r-   r:   rW  s         r3   test_accumulater  =  s?   AQF+E(1%(8
9fq!FA;>
9C
9
sFCJ'A	SA66Q\\#&+++++66Q\\#r*/////99;3333<<R ((*.JJJJ<<  %--/3LLLL
2A<<$$&)333 :s   Fc            	        t        j                  t        d      d      } | j                  d dd      }t	        j
                  |j                  |j                               }|D ];  }|D ]4  } || ust        t        d|            t        t        d|             z  s4J  = t        j                  t        d      d	      } | j                  d
 d      }t        |j                        dk  sJ t	        j
                  |j                  |j                               }|D ];  }|D ]4  } || ust        t        d|            t        t        d|             z  s4J  = t        j                  t        d      d      } | j                  d dd      }t	        j
                  |j                  |j                               }|D ];  }|D ]4  } || ust        t        d|            t        t        d|             z  s4J  = y )N   rF   r|   c                    | dz  S r   r1   r2   s    r3   rO   z$test_groupby_tasks.<locals>.<lambda>N  s
    a"f r6   rT   
max_branchrR   r   ro  rz   c                    | dz  S )N{   r1   r2   s    r3   rO   z$test_groupby_tasks.<locals>.<lambda>W  s
    a#g r6   r]   i'  iY  c                    | dz  S )Ni  r1   r2   s    r3   rO   z$test_groupby_tasks.<locals>.<lambda>a  s
    a$h r6      )
rb   rc   ra   r
   r^   r  r@   r`   r   rP   )r:   r7  
partitionsr   s       r3   test_groupby_tasksr  L  s   
s3A
))$G)
DC#((C$5$5$78J ? 	?AzuQ{+c%1+.>>>>	??
 	t#6A
))%w)
7Csxx=6!!!#((C$5$5$78J ? 	?AzuQ{+c%1+.>>>>	??
 	u37A
))&2w)
GC#((C$5$5$78J ? 	?AzuQ{+c%1+.>>>>	??r6   c                 6   t        j                  t        d      d      } d }d }t        | j	                  |dd      j
                        t        | j	                  |dd      j
                        k(  sJ t        | j	                  |dd      j
                        t        | j	                  |dd      j
                        k7  sJ t        | j	                  |dd      j
                        t        | j	                  |dd      j
                        k7  sJ y )	Nr  rF   r|   c                    | dz  S r   r1   r2   s    r3   rO   z*test_groupby_tasks_names.<locals>.<lambda>l  
    QV r6   c                    | dz  S )Nr   r1   r2   s    r3   rO   z*test_groupby_tasks_names.<locals>.<lambda>m  s
    a"f r6   rT   r  r,   )rb   rc   ra   r`   r
   r^   )r:   r   func2s      r3   test_groupby_tasks_namesr  j  s
   
s3ADEqyy!Wy=BBCs			$1g	6;;H    qyy!Wy=BBCs			$1g	6;;H    qyy!Wy=BBCs			%Aw	7<<H   r6   zsize,npartitions,groups))ro  r   rz   )i90     i  )rz   r+   rZ   c                    fd}t        j                  | |      j                  |d      }|j                  d      }t	        |      t        |t        |             k(  sJ y )Nc                    | z  S r?   r1   )r)   groupss    r3   rO   z&test_groupby_tasks_2.<locals>.<lambda>}  s    QZ r6   r|   rT   r]   r   rY   rb   ra   r
   rG   r  )sizer<   r  r   r:   rI   s     `   r3   test_groupby_tasks_2r  y  sW      D
;/77g7NAYYY(F<74t5555r6   c                     d } t        j                  dd      j                  | dd      }|j                  d	      }t	        |      t        | t        d            k(  sJ y )
Nc                    | dz  S r   r1   r2   s    r3   rO   z&test_groupby_tasks_3.<locals>.<lambda>  r  r6   r   r'   r|   rT   r,   )rR   r  r   rY   r  )r   r:   rI   s      r3   test_groupby_tasks_3r    sV    D
#++D'a+PAYYY(F<74r3333r6   c                 \   t               5 } t        j                  dd      j                  d       j	                  t
              }|j                  t        j                  j                  | d             t        t        j                  |             dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr'   r|   c                    | dk(  S r  r1   r2   s    r3   rO   z4test_to_textfiles_empty_partitions.<locals>.<lambda>  s
    Q r6   z*.txt)r"   rb   ra   r   r   r   r$  r  r%  r   rP   listdirr   s     r3   "test_to_textfiles_empty_partitionsr    s{    	 'QHHQA&--.>?CCCH	rww||Aw/02::a=!Q&&&' ' 's   BB""B+c                     t        j                  t        d      d      } t        | j	                  d       j                         d       t        | j	                  d       j                         d       y )Nr{   rz   r|   c                    | dz  dk(  S r0   r1   r2   s    r3   rO   z&test_reduction_empty.<locals>.<lambda>      Q! r6   r   c                    | dz  dk(  S r0   r1   r2   s    r3   rO   z&test_reduction_empty.<locals>.<lambda>  r  r6   r   )rb   rc   ra   r   r   r   r  r   s    r3   test_reduction_emptyr    sR    
r4Aahh+,002A6ahh+,002A6r6   )r+   r,   rF   c                   t        j                  g d|       j                  d       }t        |j	                  d      d       t        j
                  |j	                  d      |j                  d      d      }|dk(  sJ t        j                  t              5  t        j                  g d	|       }|j                  d       j	                  d      j                  d       d d d        y # 1 sw Y   y xY w)
Nr  r|   r,   r   r+   r   rY   r  r  )
rb   rc   r   r   r  rG   r   r   r   r   )r<   r:   valss      r3   test_reduction_empty_aggregater    s    
;?FFtLAaeee"A&::aeee*AEEaE,@FSD6>>	z	" D\{C	q)11F1CD D Ds   AC11C:c                      e Zd Zd Zd Zy)
StrictRealc                X    t        |t              sJ | j                  |j                  k(  S r?   r   r  realrl  rD  s     r3   __eq__zStrictReal.__eq__  %    %,,,yyEJJ&&r6   c                X    t        |t              sJ | j                  |j                  k7  S r?   r  r  s     r3   __ne__zStrictReal.__ne__  r  r6   N)rm   rn   ro   r  r  r1   r6   r3   r  r    s    ''r6   r  c                     t        j                  t        d      D  cg c]  } t        |        c} d      }t	        |j                  t        t              t        d             y c c} w )Nr{   r,   r   r   )rb   rc   ra   r  r   r   r   )r)   r:   s     r3   *test_reduction_with_non_comparable_objectsr    sE    
r;A*Q-;ANAaffS#
1. <s   A&	containerr  matrixc                   t        j                  d      | dk(  r!t        d      st        j                  d       | dk(  rj                  nj
                  }t        j                  t        d      D cg c]  } |dg       c}d	      }fd
}|j                  ||      j                  d      j                  dk(  sJ y c c}w )Nzscipy.sparser  	csr_arrayzscipy<1.11 has no sparrayr  rF   r   r,   r   c                *    j                  | |g      S r?   )vstack)r   r:   sps     r3   	sp_reducez6test_reduction_with_sparse_matrices.<locals>.sp_reduce  s    yy!Q  r6   r   rY   )rF   r+   )r   r2  r(  skip
csr_matrixr  rb   rc   ra   r   rG   shape)r   clsr)   r:   r  r  s        @r3   #test_reduction_with_sparse_matricesr    s    			^	,BGGB$</0$0"--bllC
E!H5q#qc(5aHA! 66)Y'//&/AGG6QQQ 6s   6Cc                 J    t        t        j                  g             g k(  sJ y r?   )r   rb   rc   r1   r6   r3   
test_emptyr    s       $%+++r6   c                 4   ddl m} m} t        j                  t        d            } | | |            }|j                         |j                         k(  sJ |j                         } | | |            }|j                         |j                         k(  sJ y )Nr   )dumpsloadsrz   )pickler  r  rb   rc   ra   rG   r   )r  r  r:   r   ss2s         r3   test_bag_picklabler    sr    #
s$A	uQxB99;"**,&&&	A	uQxB99;"**,&&&r6   c                     t        j                  ddig      j                  d      } | j                  d      }t	        |      dddigik(  sJ y )Nr   r+   r   rY   )rb   rc   r
   rG   r  r  s     r3   test_msgpack_unicoder    sO    
3($,,S1AYYY(F<Aaz?***r6   c                 P    d } t        j                  | g      }t        || g       y )Nc                      y r?   r1   r1   r6   r3   rO   z/test_bag_with_single_callable.<locals>.<lambda>  s    r6   )rb   rc   r   )r   r:   s     r3   test_bag_with_single_callabler    s%    A
!Aa!r6   c                    t        j                  dd      } | j                  t              }|j                  t              }|j	                  |j
                  |j                               |j
                  j                         j                         z  rJ |j	                  |j
                  |j                         |j                               t        fd|j                         D              sJ y )Nr{   r,   r|   )	fuse_keysc              3  &   K   | ]  }|v  
 y wr?   r1   )r%  r&  r-   s     r3   r'  z*test_optimize_fuse_keys.<locals>.<genexpr>  s     3AqCx3s   )	rb   ra   r   r&   __dask_optimize__r^   r@   rB   r   )r)   r_  r  r-   s      @r3   test_optimize_fuse_keysr    s    
#A	c
A	c
A


affaoo&7
8Cvv{{}sxxz)))


affaoo&71??CT

UC3!23333r6   c                     d gfd} fd}t        d |        idd      }|j                  |t              }t        |t        t	        d                   y )Nc               3  @   K   t        d      D ]  } | d<   |   y w)Nr{   r   r  )rd   currents    r3   partz&test_reductions_are_lazy.<locals>.part  s(     r 	AGAJG	s   c                .    d   dk(  sJ t        |       S r\   )r   )r#  r"  s    r3   r   z&test_reductions_are_lazy.<locals>.func  s    qzQ4yr6   )rk   r   rk   r+   r{   )r   r(  r   r   ra   )r#  r   r:   resr"  s       @r3   test_reductions_are_lazyr&    sM    fG
 	Z %+A
++dC
 Cc3uRy>"r6   c                     t        j                  dd      } | j                  d       }t        t        t        |            t        t        t        |            k(  sJ y )Nr{   rF   r|   c                    | dz  S )Nr.   r1   r2   s    r3   rO   z'test_repeated_groupby.<locals>.<lambda>  s
    AE r6   )rb   ra   r
   r   rP   r  r   s     r3   test_repeated_groupbyr)    sD    
#A			/"A#tAw6#tAw#7777r6   c           
        t        j                  dd      }t        d      5 }t        j                  j                  t        |       |      5  |j                  d       }|j                          t        d t        j                  t        |             D              sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr{   rF   r|   )temporary_directorypoolc                    | dz  S )Nr,   r1   r2   s    r3   rO   z*test_temporary_directory.<locals>.<lambda>	  s
    QU r6   c              3  >   K   | ]  }|j                  d         yw)z.partdN)endswith)r%  r  s     r3   r'  z+test_temporary_directory.<locals>.<genexpr>  s     Or{{8,Os   )rb   ra   r   r^   r_   r`   r   r
   rG   r  r  r  )r"   r:   r,  r   s       r3   test_temporary_directoryr0    s    
#A 
Q	 P4[[__V4_H 	P?+BJJLOrzz#f+7NOOOO	PP P	P 	PP Ps$   +B?AB3"B?3B<	8B??Cc                    t        j                  g       } t        | j                  t              j                         d       t        | j                  t              j                         d       t        | j                  t              j                         d       t        | j                  t              j                         d       y )NTF)	rb   rc   r   r   r&   r   r  r   r   r   s    r3   test_empty_bagr2    sx    
AaeeCjnn%aeeCjnn&aeeCjnn&aeeCj %(r6   c                    t        j                  g dd      } | j                  d      }|d   j                  d      sJ |d   j                  d      sJ t	        j
                  d       t	        j
                  d       y )	Nr  r,   r|   zfoo*r   foo0r+   foo1)rb   rc   r$  r/  r  r   )r:   pathss     r3   test_bag_pathsr7    sk    
.A>ANN6"E8V$$$8V$$$IIfIIfr6   c                     d } t        j                  g d      }t        |j                  | d      g d       t        |j                  | t	        j
                  d            g d       y )Nc                2    | D cg c]  }||z   	 c}S c c}w r?   r1   )r   r  r)   s      r3   
append_strz+test_map_partitions_arg.<locals>.append_str!  s    ()!A)))s   r   rk   )afoobfoocfoo)rb   rc   r   r[  r^   r   )r:  mybags     r3   test_map_partitions_argr?     sU    * _-Ee"":u57OPZe)<=?Wr6   c                    t        j                  g d      } t        | j                  t              j                               }dt        t        j                  j                  |      v sJ t        | j                  t              j                               t        | j                  t              j                               k7  sJ y )Nr   r&   )rb   rc   r  r   r&   __dask_graph__r^   r  	key_splitr`   r[  rr  s     r3   test_map_keynamesrC  ,  s    
#AQUU3Z&&()AC

,,a0000quuSz((*+s	,,.0   r6   c                     G d d      t        j                         fd} d }t        j                  t	        d      d      j                  |       j                  |      j                  |       j                  |      j                         }	 t        j                          |j                  d	       t        j                          y # t        j                          w xY w)
Nc                      e Zd Zd Zy)Ctest_map_releases_element_references_as_soon_as_possible.<locals>.Cc                    || _         y r?   rd   )rl  rd   s     r3   __init__zLtest_map_releases_element_references_as_soon_as_possible.<locals>.C.__init__F  s	    DFr6   N)rm   rn   ro   rI  r1   r6   r3   CrF  E  s    	r6   rJ  c                Z    t              dk(  sJ  |       }j                  |       |S r\   )rP   r$   )rd   r2  rJ  	in_memorys     r3   f_createzJtest_map_releases_element_references_as_soon_as_possible.<locals>.f_createL  s1    9~""" aDar6   c                     | j                   dz   S )Nrz   rH  )r2  s    r3   f_dropzHtest_map_releases_element_references_as_soon_as_possible.<locals>.f_dropV  s    ssSyr6   r,   r+   r|   r   rY   )weakrefWeakSetrb   rc   ra   r   r   gcdisablerG   enable)rM  rO  r:   rJ  rL  s      @@r3   8test_map_releases_element_references_as_soon_as_possiblerU  6  s     
 !I
 	qq1	X	V	X	V	 


			F	#
				s   &C C&c                    t        j                  d       t        j                  d      } | j                  dd      j                         d   }t	        j
                  |      }|j                         }t        |g d       y )Nr1  
dask.arrayr{   r'   )chunksr   )      ?rY  rY  rY  rY  )r   r2  onesr
  rb   r   rG   r   )daobjr   r:   s       r3   test_bagged_array_delayedr]  j  sd    
 			\	*B
''"Q'

*
*
,Q
/C
//#
CAa*+r6   c                    t        j                  ddgd      } | j                         | j                  fk(  sJ | j                  j
                  j                         | j                  hk(  sJ | j                  j                  | j                  t               ik(  sJ | j                         }|j                         |j                  fk(  sJ |j                  j
                  j                         | j                  |j                  hk(  sJ |j                  j                  | j                  t               |j                  | j                  hik(  sJ y )Nr+   r,   r|   )rb   rc   r  r;   r^   layersrB   dependenciesr`   r  rq   )r   rd   s     r3   test_dask_layersra  t  s    
!QQ/A166)+++66==AFF8+++6616635/111	A155(***66==AFFAEE?2226616635!%%!&&"BBBBr6   optimizeFc                H   t        j                  d       t        j                  d      }t        j                  j	                  |j                  d      j                         d         }|j                  d   }|j                  dd  dk(  sJ |j                  j                  j                         d|z   hk(  sJ |j                  j                  d|z   t               ik(  sJ |j                         d|z   fk(  sJ |j                  d      dz   }|j                  |       d   }t        j                  j	                  |      }|j                  |j                  k(  sJ |j                  |j                  u sJ |j                         |j                         k(  sJ |j                  |       }|j                         |j                         k(  sJ | sh|j                  |j                  u sJ t        j                  t        d	      5  t        j                  |j                  |j                         d d d        t        j                  t        d	      5  t        j                  |j                  |j                   fd
       d d d        y # 1 sw Y   [xY w# 1 sw Y   y xY w)Nr1  rW  r+   r   r   zdelayed-r  znot in)rN  rk   rk  )r   r2  rb   r  r   rZ  r
  rq   r^   r_  rB   r`  r`   r  r   r   r;   )rb  r[  rd   r;   arrr   backs          r3   test_dask_layers_to_delayedrf    s'     			\	*B
RWWQZ224Q78A558D55966==J$5#666666:#4ce"<<<<:#4"6666
''!*q.CnnHn5a8G
W%A55GKK66W\\!!!'"9"9";;;;<<x<0D!Q%6%6%8888yyCHH$$$]]:X6 	)GGDIItxx(	) 
z	2 4
388+U34 4	) 	)4 4s   +J.JJJ!c                    t        j                  d       t        j                  d       ddlm}  t	        j
                  ddddd	dgd	
      }t        j                  d      5  |j                  d       }|j                  d       }|j                  d       }d d d        t        j                        |j                  dz  k(  sJ |j                         }|j                  d      } | ||       y # 1 sw Y   \xY w)Nr  r  r   r   test1r+   )r;   v1test2r,   r|   Trr   c                ,    t        di | d| d   dz   iS )Nv2ri  r+   r1   r  r   s    r3   rO   z2test_to_dataframe_optimize_graph.<locals>.<lambda>      D515415 r6   c                ,    t        di | d| d   dz   iS )Nv3rl  r+   r1   rm  r   s    r3   rO   z2test_to_dataframe_optimize_graph.<locals>.<lambda>  rn  r6   c                ,    t        di | d| d   dz   iS )Nv4rp  r+   r1   rm  r   s    r3   rO   z2test_to_dataframe_optimize_graph.<locals>.<lambda>  rn  r6   rF   Fr  )r   r2  dask.dataframe.utilsr   rb   rc   r^   annotater   rP   r<   r  )assert_eq_dfr)   r_  r   r  s        r3    test_to_dataframe_optimize_graphrv    s    
!
()>
	#gQ%?@a	A
 
4	  7EE56EE56EE567 qvv;!--!++++ 	
A	
u	-BB7 7s   &7C88Dnworkers)rz      i  ro  c                    d}d}|| k  rdt        t        j                  t        |            j                        }|t        j                  t        d| dz              z  }|dk  sJ || k  rcy y )Nr   r+   r{   i N  )rP   rb   rc   ra   r^   r8  floorr   )rw  ntasksnitemss      r3   +test_default_partitioning_worker_saturationr}    si    
 FF
8
R%%eFm499:$**SHrM233 8
r6   c                    t        | d| z  t        dt        j                  | dz                    D ]:  }t	        t        j                  t        |      |       j                        | k\  r:J  y )Nr{   r+   r|   )ra   r   r8  rz  rP   rb   rc   r^   )rw  r|  s     r3   test_npartitions_saturationr    s`     "x-Q

8b=8Q1RS 
  vHEJJKxW	
W
r6   c                 (   t        j                  t        dd      d      } t        | j	                  t
              j                               }| j                  d       }t        |j	                  t
              j                               }||k(  sJ y)z)https://github.com/dask/dask/issues/10338r+   rz   r.   r|   c                    | S r?   r1   r2   s    r3   rO   z*test_map_total_mem_usage.<locals>.<lambda>  s     r6   N)rb   rc   ra   r   r[  r   rG   r   )r:   total_mem_br   total_mem_cs       r3   test_map_total_mem_usager    sr    
q#A6Aa&&7??ABK	kAa&&7??ABK+%%%r6   )ro  )
__future__r   rR  r8  r  ra  r  rP  r  r   collections.abcr   concurrent.futuresr   dataclassesr   gzipr   	itertoolsr	   r  r   tlzr
   r   r   r   r   r   r   r^   dask.bagr   rb   dask._task_specr   r   r  r   r   r   r   r   r   r   r   r   dask.bag.utilsr   dask.blockwiser   r  r   dask.typingr    
dask.utilsr!   r"   r#   dask.utils_testr$   r%   r&   ra   r-   rp   r   r   r:   r4   r8   r=   rC   rJ   rQ   markparametrizerf   ri   rx   r   r   r   r   r   r   r   reprr   r   r   r   r   r   r   r   r  r  r  r  r  r  r!  r)  r,  r/  r5  r;  r>  r   rE  rN  rX  r\  rg  rm  rr  ru  rx  r{  r}  r  r  r   r  r  r  r  r  r  slownetworkr  r  r  r  r  r  xfailr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r>  r-  r8  r;  r?  rD  rH  rZ  r\  r^  rb  re  rg  rt  rw  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rj   r  r  r  r  r  r  r  r  r&  r)  r0  r2  r7  r?  rC  rU  r]  ra  rf  rv  r}  r  r  r1   r6   r3   <module>r     s0   " 	  	     $ 2 !     E E E   &
 
 
 % $    1 1 / /
uajUAJOU OqNQS!
3B- VW$56}k&BC  D 7  $  
 	
 	QZ"#	'(   VW$56}k&BC D 714hU(N"
+&?
 #t-? .?84 O<&7<$V
, A/: 0:D
<&D * 	2! 3! B0% 1%,
!
 (DLL*VWHHQ."U8>(:B&R	 5! =3 !# "  #K
""5  DE   F2%.'W5Q#8,0(*:z 8ug.T
; x0 1@.0$"LC&P
	-,	  	 =: 
	0f< + 7F 8F  12!23 4 3 

	
>4?< R664'7 	2D 3D' '/
 w&9:R ;R,	'+	4#&8	P)	1h,C eT]34 44@4 %:;	 <	 %:;
 <
&r6   