
    biL                       d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZmZmZ d d	lm Z m!Z! d d
l"m#Z#m$Z$ 	 d dl%Z& ejP                  g dg ddg d       ejP                  g dg ddg d       ejP                  g dg ddg d      dZ) e$ddd ejT                  g d       ejP                               Z+ ejX                   ejZ                  e)j]                               g d      Z/d Z0d Z1e/je                         Z3d Z4d Z5d Z6d Z7d  Z8d! Z9d" Z:d# Z; ejP                   ejx                  g dd$z  d%&       ejx                  g d'd$z  d(&       ejz                  g d)d$z        j}                  d*       ejz                  g d+d$z         ejx                  g d,d$z         ejz                   ej~                  d-d./             ejz                   ej~                  d-d.d01             ejz                   ej                  d2d./            d3      ZAd4 ZBd5 ZCej                  j                  d6d7d8d9 ej                  d:ej                  j                  ;      g      d<        ZHej                  j                  d=g d>      d?        ZId@ ZJdA ZKerdBndCZLdD ZMdE ZNej                  j                  dFdG      ej                  j                  dHdG      dI               ZOdJ ZPdK ZQdL ZRdM ZSdN ZTej                  j                  dO        ZUdP ZVdQ ZWdR ZXdS ZYej                  j                  dTdU ej                  dVej                  j                  ;      g      dW        Z[ej                  j                  dTdU ej                  dVej                  j                  ;      g      dX        Z\ej                  j                  dTdU ej                  dVej                  j                  ;      g      dY        Z]dZ Z^ej                  j                  d[d\d]g      d^        Z_ej                  j                  dFd_d`g      da        Z`db Zadc Zbdd Zcde Zddf Zeej                  j                  dgefegehdh g      di        Zidj Zjdk Zkdl Zldm Zmej                  j                  dn        Zndo Zodp Zpej                  j                  dqddrg      ds        Zqdt Zrdu Zsej                  j                  dvg dw      ej                  j                  dxdydzdydzg ejz                  dydzg      g      ej                  j                  d{dd8g      d|                      Ztd} Zud~ Zvd Zwd Zxd Zyd Zzd Z{d Z|d Z}ej                  j                  dd_d`g      ej                  j                  dddddgg      ej                  j                  dddg      d                      Z~ej                  j                  ddU ej                  dVej                  j                  ;      g      ej                  j                  dddddgddgg      ej                  j                  dd_d`g      d                      Zej                  j                  dd_d`d`d_gd_d`gg      ej                  j                  dddgddgg      ej                  j                  dddg      d                      Zej                  j                  dddg      ej                  j                  dd_d`g      ej                  j                  dddddgg      ej                  j                  dd7dg      ej                  j                  d e ed            dgdz  z    e edd            z   dgdz   e edd            z   d e ed            dgdz  z    e e ed                  dg      d                                    Zd Zd Zej                  j                  dddgddgg      ej                  j                  dd7dg      d               Zd Zej                  j                  d6d7d$g      d        Zej                  j                  e&du d      ej                  j                  dddgdfddgdfd7dgdfd7dgdfd7dgdf e
dd7d7       e
ddd      gdf ej                  d       ej                  d      gdf ed       ed      gdf ej                  d       ej                  d      gdf edd        ed d      gdfg
      d               Zd Zy# e'$ r dZ&Y 	w xY w)    )annotationsN)ProcessPoolExecutor)datetime)Decimal)partial)compute_as_if_collection)PANDAS_GE_220assert_categorical_equaltm)maybe_buffered_partdpartitioning_index)	assert_eq	make_meta         r         abr   r   r   indexr         )r   r      r   r    r!   r   r!   	   )r   r    r$   r$   r$   r$   )xr   )r'   r   )r'   r   i8)r   parent_metar   r   r$   r$   c                .     | j                   |i |ddiS NforceTshuffledfargskwargss      \/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/dataframe/tests/test_shuffle.py<lambda>r5   %   s    :2::t+Rv+RT+R     c                .     | j                   |i |ddiS r,   r.   r0   s      r4   r5   r5   &   s    jbjj$&M&&M&M r6   c                   t        t        t        j                  |       }t        |t        j
                        sJ |j                  t        j                  k(  sJ t        |j                        j                  t        j                        sJ t        t        t        j                        j                  t        t        t        j                        j                  k(  sJ y )Nshuffle_method)shuffle_funcdr   
isinstancedd	DataFramenpartitionssetdask
issuperset_name)r:   ss     r4   test_shufflerF   *   s    QN;Aa&&&==AMM)))qvv;!!!&&)))133%%a)=)C)CCCCr6   c                 z    t        t        t        j                        j                  t        j                  k(  sJ y N)r/   r<   r   r@    r6   r4   test_default_partitionsrJ   3   s$    1acc?&&!--777r6   c                   t        j                  dt        j                  j                  d      i      }t	        j
                  |d      }t        ||j                  | dd      }|j                         }|j                  dk(  sJ t        |j                        j                  t        |j                              sJ t        |      t        |      k(  sJ t        |j                        t        |j                        k(  sJ t        t!        t"        |j$                  j'                                     t        t!        t"        |j$                  j'                                     k(  sJ y )Nr'   d   
   r@      r   r:   r@   
max_branchpdr?   nprandomr>   from_pandasr/   r'   computer@   rA   rB   rC   lenlistcolumnsmaptuplevaluestolistr:   r1   ddfrE   scs        r4   test_shuffle_npartitionsrb   7   s   	sBII,,S12	3B
..
,CSUU>rVWXA	
B==Bqvv;!!#chh-000r7c"g		?d2::....s5"))**,-.#c%AQAQAS6T2UUUUr6   c                   t        j                  dt        j                  j                  d      i      }t	        j
                  |d      }t        ||j                  | dd      }|j                         }|j                  dk(  sJ t        |j                        j                  t        |j                              sJ t        |      t        |      k(  sJ t        |j                        t        |j                        k(  sJ t        t!        t"        |j$                  j'                                     t        t!        t"        |j$                  j'                                     k(  sJ y )Nr'   rL      rN   r   r   rP   rR   r_   s        r4   ,test_shuffle_npartitions_lt_input_partitionsre   D   s   	sBII,,S12	3B
..
,CSUU>qUVWA	
B==Aqvv;!!#chh-000r7c"g		?d2::....s5"))**,-.#c%AQAQAS6T2UUUUr6   c                ~    ddl m}  |t        t        t        j                  |       t        t        d|              y )Nr   )list_eqr9   r   )dask.dataframe.tests.test_multirg   r/   r<   r   )r:   rg   s     r4   test_index_with_non_seriesri   Q   s)    7133~63~6r6   c                   t        t        t        dg   |       j                         }t        t        dg|       j                         }t        t        d|       j                         }t        |j                  j                               t        |j                  j                               k(  sJ t        |j                  j                               t        |j                  j                               k(  sJ y )Nr   r9   )r/   r<   rW   sortedr]   r^   )r:   res1res2res3s       r4   test_index_with_dataframero   Z   s    1ah~>FFHD1seN;CCED1c.9AACD$++$$&'6$++2D2D2F+GGGG$++$$&'6$++2D2D2F+GGGGr6   c                    t        j                  dg di      }t        j                  |d      }dD ]J  }t	        |d||       }t        |j                  d            t        |j                  d            k(  rJJ  y )Nr'   r   r   )r   r   r@   r:   sync	scheduler)rS   r?   r>   rV   r/   rX   rW   )r:   r1   r   ir   s        r4   ,test_shuffle_from_one_partition_to_one_otherrv   c   st    	sI&	'B
r1A TAs.I199v9./3qyy6y7R3SSSSTr6   c                F   t        j                  dg ddz  i      }t        j                  |d      }t	        ||j
                  d|       }t        t        j                  |j                  |j                               }|D ]  }|j                  |j                  k(  rJ  y )Nr'   r   rM   r   rN   r    rq   )
rS   r?   r>   rV   r/   r'   r	   rB   __dask_keys__rZ   )r:   r1   r`   rE   partsps         r4   test_shuffle_empty_partitionsr{   l   s    	sIN+	,B
..
+CSUU.IA$R\\1661??;LME &yyAII%%%&r6   r   int32dtype)Ng      @g      @float32)r   r   ccategory)r<   ef)TFT20130101r$   )periodsz
US/Easternr   tz2000)i32f32catobjbooldtdt_tztdc                    t        t        j                  d      } | dk  | dk\  z  j                         sJ t	        t        j                  |             dkD  sJ t        t        j                  d      t        t        j                  d      k(  j                         sJ t        t        dg   d      } | dk  | dk\  z  j                         sJ t	        t        j                  |             dkD  sJ t        t        g d   d      } d| k  | dk  z  j                         sJ t        t        j                  d      } | dk  | dk\  z  j                         sJ t	        t        j                  |             dkD  sJ y )Nr   r   r   r   )r   r   r   r   r   )r   df2r   allrX   rT   uniquer   )ress    r4   test_partitioning_indexr      sA   
SWWa
(C1W"'')))ryy~"""sww*.@!.LLQQSSS
S%\1
-C1W"'')))ryy~"""
S!78!
<C#X#'"'')))
SYY
*C1W"'')))ryy~"""r6   c                 \   t        j                  t        t        j                        g ddz  d      } | j
                  j                  d      | _        | j                         }|j
                  j                  j                  t        t        |j
                  j                  j                                    |_        t        | j
                  d      }t        |j
                  d      }||k(  j                         sJ t        | d      }t        |d      }||k(  j                         sJ y )Nr   r   r   r      r   r   r   )rS   r?   rY   stringascii_lettersr   astypecopyr   set_categoriesreversed
categoriesr   r   )r1   r   r   rm   s       r4   -test_partitioning_index_categorical_on_valuesr      s    	D!5!56\B=NO	PB44;;z"BD
'')CEEII$$T(355993G3G*H%IJCE
RTT1
%CceeQ'D4K
R
#Cc1%D4Kr6   r@   r   r   r      )marksc                P   g d}t        j                  t        j                  j                  d      t        j                  j                  d      dz  t        j                  j	                  |d      dt        j                  j                  d            }|j                  ddd      }t        j                  || 	      }t        |j                  d
      |j                  d
|             t        |j                  d      |j                  d|             t        |j                  d      |j                  d|             t        |j                  |j                        |j                  |j                  |             t        |j                  |j                  |j                  z         |j                  |j                  |j                  z   |             t        |j                  |j                  dz         |j                  |j                  dz   |             y )NalicebobrickyrL   皙?r'   yzr   Float64r   )r'   r   rN   r'   r9   r   r   r   )rS   r?   rT   rU   choicer   r>   rV   r   	set_indexr'   r   )r@   r:   namesr1   r`   s        r4   test_set_index_generalr      s    &E	!!#&!!#&#-!!%-	

 iis#
B 
2	3B
..
5Cbll3s>!RSbll3s>!RSbll3s>!RSbll244 #--n-"UV
RTTBDD[!ceeceemNC 
RTTAXceeai Wr6   string_dtype)zstring[python]string[pyarrow]objectc                   |dk(  rt        j                  d       g d}t        j                  t        j
                  j                  d      t        j
                  j                  |d      dt        j
                  j                  d            }|j                  d|i      }t        j                  |d	      }t        |j                  d      |j                  d| 
             y )Nr   pyarrowr   rL   r'   r   r   r   rM   rN   r9   )pytestimportorskiprS   r?   rT   rU   r   r   r>   rV   r   r   )r:   r   r   r1   r`   s        r4   test_set_index_stringr      s     ((I&%E	!!#&!!%-	
 iis#
B 
C&	'B
..
,Cbll3s>!RSr6   c                   t        j                  t        j                  j                  d      t        j                  j                  d      dz  dt        j                  j                  d            }t	        j
                  |d      }|j                  |j                  |       }||u sJ t        ||j                  |j                               y )NrL   r   r   r   r   rN   r9   )	rS   r?   rT   rU   r>   rV   r   r   r   )r:   r1   r   r   s       r4   test_set_index_self_indexr      s    	iis#"))*:*:3*?3*FGiis#
B
 	rq)A	AGGN;A6M6abhh'(r6   c                   | dk(  rt        j                  d       t        j                  t        j
                  j                  d      t        j
                  j                  d      dz  dt        j
                  j                  d            }t        j                  |d      }t        |j                  d	| 
      j                        t        |j                  d	| 
      j                        k(  sJ t        |j                  d	| 
      j                        t        |j                  d| 
      j                        k7  sJ t        |j                  d	d|       j                        t        |j                  d	d|       j                        k7  sJ t        |j                  d	d|       j                        t        |j                  d	d|       j                        k7  sJ y )Ndiskz/dsk names in disk shuffle are not deterministicrL   r   r   r   r   rN   r'   r9   r   )rQ   r:   r   T)dropr:   F)r   xfailrS   r?   rT   rU   r>   rV   rA   r   rB   )r:   r1   r`   s      r4   test_set_index_namesr      s   FG	iis#"))*:*:3*?3*FGiis#
B
 ..
+Cs}}S}@EEF#c.9>>K    s}}S}@EEF#c.9>>K    caGLL	S]]31^]LQQ	RS S S c^DII	S]]3U>]JOO	PQ Q Qr6   MEMc                    t         j                  j                  ddt        t        t
        dddt         d      }|j                  d| 	      }|j                  j                         j                  d
       y )Nr   2004valuenameid2h1r   )freqpartition_freqseedr   r9   rr   rs   )r>   demomake_timeseriesfloatstrintr   r   r   sumrW   )r:   r1   r   s      r4   test_set_index_2r      si    		 	 C02$x 
! 
B ,,vn,
=CIIMMOf-r6   c                L   t        j                  t        j                  j                  d      ddg      }t	        j
                  |d      }|j                  d| d|j                        }|j                  d      }t        ||       |j                  |j                  k(  sJ y )	N)rM   r   r'   r   rZ   r   rN   r   )r:   rQ   r@   )	rS   r?   rT   rU   r>   rV   r   r@   r   )r:   r1   r`   ddf2r   s        r4   test_set_index_3r     s    	bii&&w/#s	DB
..
+C==Nqcoo  D ,,s
Cc4s...r6   r   TFappendc                   t        j                  g dg dd      }t        j                  |d      }|j                  dkD  sJ |j                  d      }|j                  d      }|j                  sJ t        ||j                         d	       |j                  d| |
      }|j                  dd| |      }|j                  rJ t        ||d	       y)zv
    GH10333 - Allow setting index on existing partitions without
    computing new divisions and repartitioning.
    )r   r   r   r   r   r   r   r   r   r   )col1col2r   rN   r   r   Fsort_results)r   r   )sortr   r   N)	rS   r?   r>   rV   r@   r   known_divisionsr   
sort_index)r   r   r1   r`   	df_result
ddf_results         r4   test_set_index_no_sortr     s     
H	IB
..
+C??Q V$Iv&J%%%%j)..0uE
 V$v>IvEVLJ))))j)%8r6   c                    t        j                  g dg dd      }t        j                  |d      }|j	                  d      j                         }|j	                  d|       }t        ||d	       y )
N)r   r   r   r   r   )r$   r!   r   r   r   r   r   rN   r'   r9   Fr   )rS   r?   r>   rV   r   r   r   )r:   r1   r`   r   r   s        r4   test_shuffle_sortr   6  sZ    	O/B	CB
..
+C
,,s

&
&
(C==^=<DdCe,r6   c                   dd l }t               } |       }t        |j                   |j                        sJ t	        j
                  t	        j                  |            }|j                  rJ  |       }t        |j                   |j                        sJ t        |       } |       }t        |j                   |j                        sJ t        | j                               }t        |      dk(  sJ |d   j                  dk(  sJ |d   j                  | k(  sJ t	        j
                  t	        j                  |            }	|	j                  rJ |	j                  | k(  sJ y )Nr   )tempdirr   z.partd)partdr   r=   BufferpickleloadsdumpsbufferFilerY   iterdirrX   suffixparentr   )
tmp_pathr   r   p1f2p2f3p3contentsf4s
             r4   test_maybe_buffered_partdr   @  s    A	
Bbhh---	fll1o	&Byy=	Bbhh

+++	h	/B	Bbhh---H$$&'Hx=AA;)))A;)))	fll2&	'Byy=::!!!r6   c                    t        j                  dg dig d      } t        j                  | d      }d }t        j
                  j                  |      5  |j                  dg d	
      }d d d        j                  d	k(  sJ | j                  d      }t        ||       t        j                  t              5  |j                  dg d
       d d d        y # 1 sw Y   oxY w# 1 sw Y   y xY w)Nr'   )r   r   r   r   )rM   rd      (   r   r   rN   c                     t               rH   )	Exception)r2   r3   s     r4   throwz5test_set_index_with_explicit_divisions.<locals>.throw\  s
    kr6   rs   )r   r   r   	divisions)r   r   r   )rS   r?   r>   rV   rB   configrA   r   r  r   r   raises
ValueError)r1   r`   r  r   r   s        r4   &test_set_index_with_explicit_divisionsr
  W  s    	sL)1A	BB
..
+C 
5	) 7}}SI}67>>Y&&&
,,s
CdC 
z	" 0cY/0 07 70 0s   C=C(C%(C1c                     t        j                  dg di      } t        j                  | d      }t	        j
                  t              5  |j                  dg        d d d        y # 1 sw Y   y xY w)Nr'   r   r   rN   r  )rS   r?   r>   rV   r   r  r	  r   r1   r`   s     r4   #test_set_index_with_empty_divisionsr  k  sX    	sL)	*B
..
+C 
z	" )cR() ) )s   
A''A0c                     t        j                  g dt        d      d      } t        j                  | d      }|j                  dg d      }|j                  dk(  sJ t        |j                  d	      j                  d
d        ddgk(  sJ y )N)r   r   r   r   r   r    abdabdr   r   r   )r   r   r<   r  rr   rs   r<   )	rS   r?   rY   r>   rV   r   r  rW   r   )r1   r`   results      r4   test_set_index_divisions_2r  u  s    	.T(^D	EB
..Q
C]]3/]:F...066rs;<c
JJJr6   c            	     2   t        j                  t        j                  j                  d      t        j                  j                  d      dz  dt        j                  j                  d            } t	        j
                  | d      }|j                         }t        j                  d      }t        d|      5 }t        t        |d	
      }t        |j                  |t        d                  }d d d        t              dk(  sJ y # 1 sw Y   xY w)NrL   r   r   r   r   rN   spawnr!   r'   )r1   idxr   )rS   r?   rT   rU   r>   rV   clear_divisionsmpget_contextr   r   
_set_indexrA   r[   rangerX   )r1   r`   ctxpoolfuncdivisions_sets         r4   #test_set_index_consistent_divisionsr    s     
iis#"))*:*:3*?3*FGiis#
B ..
+C



C
..
!C	Q	$ 8zcs3DHHT5:678 }"""8 8s   =7DDc                8    |j                  |      j                  S rH   )r   r  )ru   r1   r  s      r4   r  r    s    <<&&&r6   c                    t        j                  t        j                  j                  |       t        j                  j                  |       d      S )Nr   )rS   r?   rT   rU   )ns    r4   	make_partr#    s4    <<bii..q1		8H8H8KLMMr6   c                    t        j                  t        d      t        d      d      }t        j                  |dd      }|j                  d|       }t        |j                        |j                  dz  k  sJ y )	NrL   r   rM   F)r@   r   r'   r9   r   )	rS   r?   r  r>   rV   r   rX   rB   r@   )r:   r1   r`   r   s       r4   "test_set_index_detects_sorted_datar%    s`    	E#JU3Z8	9B
..%
8C==^=<Dtyy>COOa////r6   c            
        t        j                  g d      } t        j                  | d      } g d}g }t	        t        |            D ]V  }t        |d |       }t        |||dz          }|j                  t        j                  d| || it	        ||                   X t        j                  |      j                         }|j                  d      j                  j                         j                  du sJ y )	N):    ra r'      li) r(      PG r)      /D     $;{lH r*  r+      r]:x r,      .     L7%     ` r-  r.  r/      XTwK     Vv;
M      2M r1  r2  r0      4Sj r3      g?     &q r4  r5      8^WK r6      Ceo     JMzl r7  r8      F2y| r9      zr r:      ud r;  r;      ^J;p r<  r<      j{t r=      R~%     4     dw' r>  r?  r@      @z}Z rA  rA  rA  nsunit)rM   $   :   r   	timestampr   T)rT   arrayrS   to_datetimer  rX   r   r   r?   r>   concatr  r   r   rW   is_monotonic_increasing)valsbreaksdfsru   lohir`   s          r4   test_set_index_sortsrQ    s    88;	
=D| >>$T*DF
C3v; R_AE"#

2<<d2bk :%B-PQ	R ))C.
(
(
*C==%++335MMQUUUUr6   enginepandascudfc                :   | dk(  rt        j                  d       t        j                  g dg ddg d      t        j                  g dg d	dg d
      t        j                  g dg ddg d      d}t	        j
                  t        j                  |j                               g d      }|j                         }|j                  dd      }|j                  dk(  sJ |j                  j                  dk(  sJ t        ||j                  d             |j                  |j                  d      }|j                  dk(  sJ |j                  j                  dk(  sJ t        ||j                  |j                               |j                  d      }|j                  j                  dk(  sJ t        ||j                  d             |j                  dg      }|j                  j                  dk(  sJ t        ||j                  dg             y )NrT  	dask_cudfr   )r   r   r    r   r   r   r   )r   r   r!   r"   r#   )r$   r   r!   r%   r&   r*   r   r   rN   )r   r   rS   r?   r>   repartitionrJ  r]   rW   r   r@   r   r   r   r   )rR  dskr<   fulld2d3d4d5s           r4   test_set_indexr^    s     	K( ,,YY?yQ,,YY?yQ,,YY?yQC 	ryy.=A99;D	
Sa	(B>>Q88==Cb$..%&	
QSSa	(B>>Q88==Cb$..()	
S	B88==Cb$..%&	
cU	B88==Cb$..#'(r6   c                p   | dk(  r*t        j                  d      }t        j                  d      }t        j                  g dg dd      }| dk(  r%j	                  |      }j                  |d      }nt        j                  |d      }|j                  d	d      }|j                  dk  sJ t        |j                        h d
k(  sJ |j                  dd      }|j                  d   dk(  sJ d|j                  d   cxk  r|j                  d   cxk  rdk  sJ  J |j                  d   dk(  sJ y )NrT  rV  )r   r   r   r   r   )      ?r   r   r   r   r   r   rN   r   r'   >   r   r   r   r   r   r`  r   g       @)r   r   rS   r?   rV   	from_cudfr>   r   r@   rA   r  )rR  rT  rV  r1   gdfr<   d1rZ  s           r4   test_set_index_interpolaterd    s&     ""6*''4		O2CD	EBr"3NN2q!	
Sa	(B>>Qr||	)))	
Sa	(B<<?c!!!a82<<?8S88888<<?c!!!r6   c           	        | dk(  r*t        j                  d      }t        j                  d      }t        t        t	        ddd            dz        }t        j                  dd|z  i      }| dk(  r%j                  |      }j                  |d      }nt        j                  |d      }|j                  dd      }t        d	 |j                  D              sJ y )
NrT  rV  r      rM   r   r'   rN   c              3  v   K   | ]1  }t        j                  t        |      t         j                         3 y wrH   )rT   
issubdtypetypeinteger).0r'   s     r4   	<genexpr>z1test_set_index_interpolate_int.<locals>.<genexpr>E  s#     Har}}T!Wbjj1Hs   79)r   r   rk   rY   r  rS   r?   rV   ra  r>   r   r   r  )rR  rT  rV  Lr1   rb  r<   rc  s           r4   test_set_index_interpolate_intrn  0  s      ""6*''4	tE!S"%&*+A	sAEl	#Br"3NN2q!	
Sb	)BH2<<HHHHr6   c                     t        j                  t        j                  g d            } t        j                  | d      } | j                         j                  dd      }|j                  dk(  sJ y )NArq  rq  rq  rq  rq  rq  rq  rq  Brr  rr  Cr   r   rN   r   r   )rS   r?   Indexr>   rV   reset_indexr   r@   )dataoutputs     r4   test_set_index_npartitionsrx  H  sc    <<hhM
D
 >>$A.D))'q)AF"""r6   rD  rB  usc                    t        j                  ddgddgddgddggd	d
g      }t        j                  |j                  |       |_        t	        j
                  |d      }|j                  d	      }t        ||j                  d	             y )Nl   9+wEG, r   l   !|IwEG, r   l   wwEG, r   l   a>hwEG, r   tsrankr   rC  rN   )rS   r?   rI  r{  r>   rV   r   r   )rD  r1   r`   s      r4   !test_set_index_datetime_precisionr}  T  s     
q!q!q!q!		
 v
B NN255t,BE
..
+C
--
Cc2<<%&r6   TFc                L   t        j                  t        d      g dg dd      }t        j                  |d      }t        |j                  d|       |j                  d|              t        |j                  d|       |j                  d|              t        |j                  d	|       |j                  d	|              t        |j                  |j                  |       |j                  |j                  |              t        |j                  |j                  |       |j                  |j                  |              t        |j                  |j                  |       |j                  |j                  |              t        j                  t        d      g dg dd
      }t        j                  |d      }t        |j                  d|       |j                  d|              t        |j                  d|       |j                  d|              y )N
ABAABBABAA)
r   r   r   r   r   r    r   r!   r$   rM   )
r   r   r   r   r   r   r   r   r   r   )rq  rr  rs  r   rq  r   rr  rs  )r   r   r   r   r   )
rS   r?   rY   r>   rV   r   r   rq  rr  rs  )r   pdfr`   s      r4   test_set_index_dropr  h  s   
,,l#0/	
C ..a
 CcmmCdm+S]]3T]-JKcmmCdm+S]]3T]-JKcmmCdm+S]]3T]-JKcmmCEEm-s}}SUU}/NOcmmCEEm-s}}SUU}/NOcmmCEEm-s}}SUU}/NO ,,L!.-	
C ..a
 CcmmADm)3===+FGcmmADm)3===+FGr6   c                    t        j                  g dg dd      } t        j                  | d      }d}t	        j
                  t              5 }|j                  ddg       d d d        |t        j                        v sJ t	        j
                  t              5 }|j                  ddgg       d d d        |t        |j                        v sJ t	        j
                  t              5 }|j                  dgg       d d d        |t        |j                        v sJ y # 1 sw Y   xY w# 1 sw Y   }xY w# 1 sw Y   ;xY w)N)r   r   r   r   r   r    r   )r   r    r   r   r   r   r   r   r   z1Dask dataframe does not yet support multi-indexesr   r   )
rS   r?   r>   rV   r   r  NotImplementedErrorr   r   r   )r1   r`   msgerrs       r4   (test_set_index_raises_error_on_bad_inputr    s   	18MN	OB
..Q
C
>C	*	+ "ssCj!"#cii.   	*	+ $sSzl#$#cii.   	*	+ sug#cii.   " "$ $ s$   D!D-+D9!D*-D69Ec                    t        j                  g dg dg dd      } t        j                  | dd      j	                         }|j
                  rJ |j                  dd	
      }|j
                  sJ t        |j                        j                  t        |j                              sJ dD ]  }t        |j                  d|      | j                  d|             t        |j                  |j                  d	|      | j                  | j                  |             t        |j                  |j                  dz   d	|      | j                  | j                  dz   |              y )Nr   rM   rd   rd   r  r   r   r   r   r   r   Fr   r'   Trk   r   r  )rk   r   r   )rS   r?   r>   rV   r  r   r   rA   rB   issubsetr   r'   )r1   r   r   r   s       r4   test_set_index_sorted_truer    s'   	L/?lS	TB
r15)99;A    	C%Aqvv;AFF,,, 
!++c+-r||Cd|/KLKKDtK4bll244dl6S	
 	KKa4K8LLL-	

r6   c                     t        j                  g dg dd      } t        j                  | d      }t	        |j                  dd      | j                  d             y )	Nr   )r   r   r   r   r   r   rN   r'   Tr  )rS   r?   r>   rV   r   r   r  s     r4   &test_set_index_sorted_single_partitionr    sE    	L|<	=B
..
+CcmmCm-r||C/@Ar6   c                 ^   t        j                  g dg dd      } t        j                  g dg dd      }t        j                  |       }t        j                  |      }t	        j
                  ||g|       }|j                  rJ |j                  dd      }|j                  d	k(  sJ y )
Nr   )r   r   r   r   )r   r   r   )metar   Tr  )r   r   r   )	rS   r?   rB   delayedr>   from_delayedr   r   r  )r   r   aabbr1   r   s         r4   "test_set_index_sorted_min_max_samer    s    
9956A
9956A	aB	aB	"b	*B!!!!
,,s4,
(C==I%%%r6   c            
     (   g d} t         t        t        d g}|D ]  }t        j                  | D cg c]  } ||      |d c}ddg      }t        j                  t        j                  |d      t        j                  ||j                  |j                  j                         kD     d      g      t        fd	t        j                        D              sJ t        j                  d      |j                  d            rJ  y c c}w )
Nr   c                0    t        j                  | d      S NrB  rC  rS   rI  r'   s    r4   r5   z0test_set_index_empty_partition.<locals>.<lambda>      R^^AD-I r6   r   r'   r   r   r   rN   c              3  p   K   | ]-  }j                  |      j                         j                   / y wrH   )get_partitionrW   empty)rk  rz   r`   s     r4   rl  z1test_set_index_empty_partition.<locals>.<genexpr>  s+     XA3$$Q'//177Xs   36)r   r   r   rS   r?   r>   rJ  rV   r   maxanyr  r@   r   r   )	test_vals
convertersconvru   r1   r`   s        @r4   test_set_index_empty_partitionr    s    Iuc#IJJ @\\-6747#7#s
 iirq1r"$$"34!D
 XsAWXXXXs+R\\#->???@7s   D
	converterc                0    t        j                  | d      S r  r  r  s    r4   r5   r5     r  r6   c                    g d}t        j                  |D cg c]  } | |      |d c}      }t        j                  |d      }|j                  dkD  sJ ||j
                  |j
                  j                         kD     j                  d      }||j
                  |j
                  j                         kD     j                  d      }t        ||d      sJ |j                  dk(  sJ t        d	 |j                  D              sJ y c c}w )
Nr   r   r   rN   r   r'   F
check_freqc              3  F   K   | ]  }t        j                  |        y wrH   )rS   isnull)rk  r<   s     r4   rl  z*test_set_index_on_empty.<locals>.<genexpr>  s     6ryy|6s   !)rS   r?   r>   rV   r@   r   r  r   r   r   r  )r  r  r'   r1   r`   actualexpecteds          r4   test_set_index_on_emptyr    s     I	IFqYq\2F	GB
..
+C??Q#$..s3F"$$#$..s3HVX%888"""6V%5%56666 Gs   Dc                 H   t        t        t        j                              } t        t        j                        }t	        j
                  |       t        j                  j                  j                  | d      }t        j                  t        j                  ||      dd      }t        j                  |d      j                  d      }t        |      t        |      k(  sJ t        j                  |j                   |      }t#        ||j%                                y )	NT)orderedr}   r   rq  rr  r   rN   rq  )rY   r   r   r   rU   r/   rS   apitypesCategoricalDtyper?   Categoricalr>   rV   r   rX   r  r   sort_values)orderr]   r~   r1   r  r  s         r4   test_set_index_categoricalr    s    &../0E&&&'F
NN6FFLL))%)>E	BNN6?aH	IB^^BA.88=Fv;#b'!!! v//u=IY	(=(=(?@r6   c                 R   t        j                  t        t        d            g dg dd      } t	        j
                  | d      }||j                  dk(     j                  dd	
      }| | j                  dk(     j                  d      }|j                  dk(  sJ t        ||       y )Nr!   )r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   )r   rv  r   r   r   Tr  )r`        @r  )
rS   r?   rY   r  r>   rV   r   r   r  r   r1   r`   r  r  s       r4   %test_set_index_with_empty_and_overlapr    s    	58n))

B ..Q
C!_&&s4&8F"$$!)}&&s+H...fhr6   c                    t        j                  g dg dg ddg d      } t        j                  | dd	      j	                         }|j
                  rJ |j                  d
      }t        ||d       |j
                  sJ y )Nr   r  r  r   )r   r   rM   rd   r   r   Fr  T)set_divisionscheck_divisions)rS   r?   r>   rV   r  r   compute_current_divisionsr   )r1   r   r   s      r4   test_compute_divisionsr    sx    	!1E
B 	r15)99;A    	##$#7AaE*r6   c                     t        j                  dt        t        d            i      } | d   dz  | d<   | d   j	                  t
              | d<   t        j                  | d      }|j                  d      }|j                  d      }|j                  d      j                          t        || j                  d             |j                  d      }t        || j                  d      j                  d             y )Nr   rM   r   r   r   rN   r   )rS   r?   rY   r  r   r   r>   rV   r   rW  r  rW   r   r  s     r4   test_empty_partitionsr    s    	sDrO,	-BgkBsGgnnS!BsG
..
+C
--
C
//a/
(Ca  "c2<<$%
--
Cc2<<$..s34r6   c                 T   t        j                  dt        j                  dd      i      } t        j                  dt        j                  dd      i      }t	        j
                  |d      }t	        j
                  | d      }t        j                  j                  d	d
d      5  |j                  |dd      }|j                         }d d d        | j                  |dd      }t        j                  j                  d      j                  d      |       y # 1 sw Y   SxY w)NKEYr   iP       r   rN   i  tasksrr   )zdataframe.shuffle.methodrt   innerhowonTr  )rS   r?   rT   aranger>   rV   rB   r  rA   mergerW   r   assert_frame_equalr  ru  )largesmalldd_leftdd_right	dd_mergedr  r  s          r4   test_gh_2730r  %  s    LL%1e!456ELL%2s!345EnnU2G~~e5H	gFS	T %MM(EMB	""$% {{5g%{8H&,,U3??T?JHU% %s   '%DD'c                 >   t        j                  t        j                  ddd      dd      } t        j                  | d      }t        j
                  dd	      t        j
                  d
d	      f}| j                  d      }|j                  d|      }t        ||j                        D ]C  \  }}|j                         |j                         k(  sJ |j                  |j                  k(  rCJ  t        ||d       t        ||j                  d      d       y )Nr      z
US/Centralr   r   r  r   z2000-01-01 00:00:00-0600r   z2000-01-12 00:00:00-0600rq  r  Fr  )rS   r?   
date_ranger>   rV   	Timestampr   zipr  	timetupler   r   )r1   r`   r  r   ddf_new_divts1ts2s          r4   test_set_index_timestampr  6  s    	BMM&"NUVW	XB
..Q
C
/LA
/LAI ,,s
C--y-9K	;#8#89  S}}#--/111vv  c;51c3==%%8r6   c                     t        j                  dt        j                  ddgd      d      } t        j                  | d      }| j                  d      }|j                  d      }t        ||       y )Nr   r   Int64r}   r   rN   r   )rS   r?   Seriesr>   rV   r   r   )r  r`   
pdf_resultr   s       r4   test_set_index_ea_dtyper  I  sZ    
,,QRYY1vW%EF
GC
..!
,Cs#Js#Jj*%r6   compressionZLibc                    t         j                  j                  d| i      5  t        d       d d d        y # 1 sw Y   y xY w)Ndataframe.shuffle.compressionr   )rB   r  rA   rF   r  s    r4   )test_disk_shuffle_with_compression_optionr  Q  s7     
9;G	H V  s	   7A c                 J   t         j                  j                  ddi      5  t        j                  t
        dj                  d            5  t        t        t        j                  d      j                          d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr  UNKOWN_COMPRESSION_ALGOzxNot able to import and load {} as compression algorithm.Please check if the library is installed and supported by Partd.matchr   r9   )rB   r  rA   r   r  ImportErrorformatr;   r<   r   rW   rI   r6   r4   *test_disk_shuffle_with_unknown_compressionr  X  s    	9;TU	V 
B]]SSYSY-T	
 		B ACC7??A		B
B 
B		B 		B
B 
Bs#   +B/B<BB	BB"c                 `    d }  | d       } | d      }t        |      t        |      kD  sJ y )Nc                D   t        j                  dt        t        d            i      }|d   dz  j	                  t
              |d<   t        j                  j                  d| i      5   t        dd              }|j                  d|i       | r%|j                  j                  j                  d      n|j                  j                  d      }t        |d	      5 }|j                         cd d d        cd d d        S # 1 sw Y   nxY w	 d d d        y # 1 sw Y   y xY w)
Nr   i'  {   r   r  F)r   r   r'   rb)rS   r?   rY   r  r   r   rB   r  rA   r   r   r   filenameopenread)r  df1r   r  r   s        r4   generate_raw_partd_filezKtest_disk_shuffle_check_actual_compression.<locals>.generate_raw_partd_filei  s    llCeEl!345HsN**3/C[[__={KL 	 A%UDACBIIsCj! 1<'',ARARSVAW  h%  vvx   	  	      	  	  	 s%   )A4DD -	D D		DDr  BZ2rX   )r  uncompressed_datacompressed_datas      r4   *test_disk_shuffle_check_actual_compressionr  g  s7      0DA-%@O !C$8888r6   ignore_index)NTFr  r   r   rQ   c                (   t         j                  j                  ddt        t        t
        dddt         d      }t        | t              r|| g   j                         }n||    j                         }|j                  | |||      }|j                  |||	      }t        ||| 
       |r5|dk(  r0|j                  j                  |j                  j                  k7  sJ y |j                  j                  |j                  j                  k(  sJ y )Nr   2001r   r   r   r   )r  r   r   r   )r:   r  rQ   )r:   r  check_indexr  )rB   datasets
timeseriesr   r   r   r   r=   r   r/   r   r   r~   )r  r  rQ   r:   df_inext_ondf_out_1df_out_2s           r4   test_dataframe_shuffle_on_argr  ~  s    MM$$s#62$x % E "ct!!#r!}}
%!	  H }}~L  H h</?A '1~~##u{{'8'8888~~##u{{'8'8888r6   c                     t        j                  g dt        d      d      } t        j                  | d      }|j                  dd      }|j                  |j                  	      }t        ||       y )
Nr   r   r   r   r   r   r    r   abcdabcdkeyr   r   rN   r  Tr  r  )	rS   r?   rY   r>   rV   r   rW  r  r   )rq  r   r   s      r4   test_set_index_overlapr    sZ    
5Z@PQRA
qa(A	E$'A	,AaOr6   c                 f   t        j                  t        j                  g dd            } t        j                  | d      }|j                         j                  d      j                  dd	      }| j                         j                  d      }t        ||       |j                  dk(  sJ y )
Nrp  r   r   r   r   rN   r!   Tr  )
rS   r?   rt  r>   rV   ru  rW  r   r   r@   r  s       r4   test_set_index_overlap_2r    s    	hhM

B ..
+C%%!%4>>wt>T  ~~))'2Hfh"""r6   c                     t        j                  g ddd      } t        j                  | d      }| j	                  d      }|j	                  dd      }t        ||       y )	N)r   r   r   r   r   r   r   r   abc)r{  r   r   rN   r{  Tr  )rS   r?   r>   rV   r   r   )r1   r`   r  r  s       r4   @test_set_index_overlap_does_not_drop_rows_when_divisions_overlapr    sP    	5F	GB
..
+C||D!H]]4]-Fhr6   c                 @   t         t         j                  dkD     j                  d      } | j                  d      }|dk(  sJ t	        | | d       t         t         j                  dkD     j                  d      } | j                  d      }|dk(  sJ t	        | | d       y )Nr   r   )r   r   r!   r$   Fr  r   )r   r   r   r$   )r<   r   r  r  r   )r   r  s     r4   ,test_compute_current_divisions_nan_partitionr    s    	!##'
s#A++C0I$$$aE* 	
!##'
s#A++C0I$$$aE*r6   c                    t        j                  g dt        d      d      } t        j                  | d      }t        j                  t        d      5  |j                  d      }|j                  d|	      }|j                  d
k(  sJ |j                  D cg c]  }t        |       c}ddgk(  sJ 	 d d d        y c c}w # 1 sw Y   y xY w)Nr  r  r  r   rN   "Partitions have overlapping valuesr  r  r  r   r   r   )rS   r?   rY   r>   rV   r   warnsUserWarningr  r   r  
partitionsrX   )rq  r   r  r   rz   s        r4   &test_compute_current_divisions_overlapr#    s    
5Z@PQRA
qa(A	k)M	N 8//6	KKK3{{i''' !-1A-!Q777	8 8 .	8 8s   ACC ,
C CCc                 L   t        j                  t        j                  g dd            } t        j                  | d      }|j                         j                  d      }t        j                  t        d	      5  |j                          d d d        y # 1 sw Y   y xY w)
Nrp  r   r  r   r   rN   r!   r  r  )rS   r?   rt  r>   rV   r  rW  r   r   r!  r  )rv  ddf1r   s      r4   (test_compute_current_divisions_overlap_2r&    s    <<hhM
D >>$A.D!--!-<D	k)M	N )&&() ) )s    BB#c                    t         t         j                  dkD     j                  d       t         t         j                  dkD     j                  dd       t         t         j                  dkD     j                  dd      } t        | |        y )Nr   r   r   Tr  )r<   r   r   r   )r   s    r4   test_set_index_nan_partitionr(    se    accAgJaccAgJT*	!##'
S.AaOr6   c                    g ddz  g ddz  ddgdz  d} t        j                  d	d
d      t        j                  d      z
  }t        j                  | |      }t	        j
                  |d      }|j                  j                  j                  d      }|j                  |      }t	        j
                  t        j                  | |j                  d            d      }t        ||       t        j                  d      }|j                  |j                  |z         }t	        j
                  t        j                  | ||z         d      }t        ||       t	        j
                  t        j                  |       d      }	|j                  |	j                        }
t	        j
                  t        j                  |       d      }t        |
|       y )Nr   r   rM   rd   r   r   AliceBobr    )r'   r   r   
2022-02-2216hr  startr   r   r   secondsr   rN   Dr   days)rS   r  	Timedeltar?   r>   rV   r   r   floorr   r   )r]   
date_indexr1   r`   	day_indexday_dfr  one_daynext_day_dfno_datesrange_dfs              r4   !test_set_index_with_dask_dt_indexr?    ss   AA% 1$F
 
R !J 
fJ	/B
..
+C 		""3'I]]9%F~~
V:#3#3C#89qH fh ll"G--		G 34K~~
V:#78aH k8$ ~~bll62BH}}X^^,H~~bll62BHh!r6   c            	        t        j                  ddd      t        j                  d      z
  } t        j                  d      }t        j                  g d	d
z  g ddz  ddgdz  | |z   | |dz  z   d|       }t	        j
                  |d
      }|j                  |j                  |z         }|j                  |j                  |z         }t        ||       y )Nr-  r.  r  r/  r   r1  r   r4  r   r   r*  r   r+  r,  r    r   )r'   r   r   rc  rZ  r   rN   )	rS   r  r6  r?   r>   rV   r   rZ  r   )datesr;  r1   r`   r   r  s         r4   (test_set_index_with_series_uses_fastpathrB    s    MM5"EI E ll"G	!!e$q('/'A+%	
 	
B ..
+C
--(
)C||BEEGO,Hc8r6   	ascendingbyr   r   nelemrM   r  c                   t         j                  j                  d       t        j                         }t        j
                  t        j                  |       d d d         |d<   t        j                  d| dz         |d<   t        j                  |d      }t        j                  j                  d	      5  |j                  ||
      }d d d        |j                  ||
      }t        j                  |dd       y # 1 sw Y   6xY w)Nr   r   rL   r   rM   rN   single-threadedrs   rD  rC  F)r  r   rT   rU   r   rS   r?   ascontiguousarrayr  r>   rV   rB   r  rA   r  r   rE  rD  rC  r1   r`   gotexpects          r4   test_sort_valuesrO  .  s     IINN1	B""299U#3DbD#9:BsGiiUS[)BsG
..
,C 
#4	5 :ooyo9:^^rY^7FLLf%eD: :s   8DD
backendr'   r   c                   | dk(  rt        j                  d       t        j                  t	        d      g ddz  ddgdz  d	      }t        j                  |d
      }| dk(  r|j                  |       }|j                  ||      }t
        j                  j                  |||d      }t        j                  ||d       y )NrT  rV  rM   r   r   r   dogr   r   rN   rI  r  )rD  rC  r:   Fr   )
r   r   rS   r?   r  r>   rV   
to_backendr  r   )rP  rD  rC  r  r`   rN  rM  s          r4   test_sort_values_tasks_backendrT  ?  s     &K(
,,Bio18JKC .."
-C&nnW%__i_8F
,,
"
"i # C LLf51r6   c                   t         j                  j                  d       t        j                         }t        j
                  t        j                  |       d d d         |d<   t        j                  d| dz         |d<   t        j                  |d      }t        j                  j                  d	      5  |j                  ||
      }d d d        |j                  ||
      }t        j                  |d       y # 1 sw Y   5xY w)Nr   rG  r   rL   r   r   rN   rH  rs   rI  Fr  rJ  rL  s          r4   !test_sort_values_single_partitionrV  U  s     IINN1	B""299U#3DbD#9:BsGiiUS[)BsG
..
+C 
#4	5 :ooyo9:^^rY^7FLLf%0: :s   8D  D	na_positionfirstlastnpartsr   rv  2   rL         rd   c                F   t        j                  |       }t        j                  ||      }t        j
                  j                  d      5  |j                  |||      }d d d        |j                  |||      }t        j                  |d       y # 1 sw Y   6xY w)NrN   rH  rs   )rD  rC  rW  Fr  	rS   r?   r>   rV   rB   r  rA   r  r   )	rv  rZ  rD  rC  rW  r1   r`   rM  rN  s	            r4   test_sort_values_with_nullsr`  f  s    $ 
d	B
..
0C 
#4	5 SooykoRS^^rYK^PFLLf%0S Ss   BB c                     t        j                  dg di      } t        j                  | d      }t	        j
                  t        d      5  |j                  dd       d d d        y # 1 sw Y   y xY w)	Nr   r   r   r   r   rN   z,na_position must be either 'first' or 'last'r  invalid)rD  rW  )rS   r?   r>   rV   r   r  r	  r  r  s     r4   test_shuffle_values_raisesrd    s^    	sI&	'B
..
+C	H
 7 	3I67 7 7s   A))A2c                 X   t        j                  dg di      } t        j                  | d      }t        j
                  j                  d      5  |j                  dgd      }d d d        t        j                  dg d	i      }t        j                  |d
       y # 1 sw Y   ;xY w)Nr   rb  r   rN   rH  rs   TrI  r   Fr  r_  )r1   r`   rM  rN  s       r4   test_shuffle_by_as_listrf    s    	sI&	'B
..
+C	#4	5 8oo#$o78\\3	*+FLLf%08 8s   B  B)c                l   t        j                  g ddz  g ddz  d      }t        j                  ||      }d }t        j
                  j                  d	      5  |j                  | d
   |d| i      }d d d        |j                  |       }t        j                  |d       y # 1 sw Y   4xY w)Nr   rd   r   r   r    r   r]  r   rN   c                *    | j                  |||      S )N)rC  rW  )r  )	partition
by_columnsrC  rW  r3   s        r4   r   z+test_sort_values_custom_function.<locals>.f  s!    $$) % 
 	
r6   rH  rs   r   rk  )rD  sort_functionsort_function_kwargs)rD  Fr  r_  )rD  rZ  r1   r`   r   rM  rN  s          r4    test_sort_values_custom_functionrn    s     
IN1BC	DB
..
0C
 
#4	5 
oo!uA\2<N  

 ^^r^"FLLf%0
 
s   B**B3c                 X   t        j                  g ddz  g ddz  d      } t        j                  | d      }t	        j
                  t        d	      5  |j                  d
ddg       d d d        t	        j
                  t        d	      5  |j                  d
dgdg       d d d        t        |j                  d
dg      | j                  d
dg             t        |j                  d
dgddg      | j                  d
dgddg             y # 1 sw Y   xY w# 1 sw Y   |xY w)Nr   rd   rh  r]  r   rM   rN   lengthr  r   TFrI  r   )	rS   r?   r>   rV   r   r  r	  r  r   r  s     r4   test_sort_values_bool_ascendingrq    s   	IN1BC	DB
..
,C 
z	2 934-89	z	2 9C:$8934&1
#$0 C:$?
3*u>9 99 9s   DD D D)c                   t         j                  j                  t        j                  dd      ddgt        j                  dd      ddgt        j                  d	d      d
dgt        j                  dd      ddgt        j                  dd      ddgt        j                  dd      ddgt        j                  dd      ddgt        j                  dd      ddgt        j                  dd      ddgt        j                  dd      ddgt        j                  dd      d d!ggg d"#      }t	        j
                  || $      }|j                  d%      }|j                  d%      }t        ||       y )&Nz2002-01-11 21:00:01+0000UTCr  i  g    @z2002-01-14 21:00:01+0000i  g    @z2002-01-15 21:00:01+0000i1  g    @z2002-01-23 21:00:01+0000iu  g    (@z2002-01-29 21:00:01+0000i  g     @z2002-02-01 21:00:01+0000i  g    @i  g    @ig  g    n@z2002-01-22 21:00:01+0000g   g    `#@z2002-01-30 21:00:01+0000iA  g    @@z2002-01-31 21:00:01+0000i/  g     {@)r   id1id2r   rN   r   )rS   r?   from_recordsr  r>   rV   r  r   )r@   r1   r`   r  r  s        r4   test_sort_values_timestamprx    sQ    
	"	"\\4?wO\\4?wO\\4?P\\4?wO\\4?wO\\4?vN\\4?wO\\4?wO\\4?gN\\4?P\\4?wO	
 ' 
# 
B  ..
5C__V$F~~f%Hfhr6   zNeed pyarrow)reasonzdata, dtyper      a   bzbinary[pyarrow]r   zint64[pyarrow]zfloat64[pyarrow]zuint64[pyarrow]i  i  r     zdate32[pyarrow]z
2022-01-01z
2023-01-02ztimestamp[ns][pyarrow]5z6.24
decimal128z1 dayz20 dayszduration[ns][pyarrow]ztime64[ns][pyarrow]c                D   |dk(  r)t        j                  t        j                  dd            }t        j                  dt        j
                  | |      d      }t        j                  |d      }|j                  d      }|j                  d      }t        ||       y )	Nr~  rM   r   r   r}   )r   	arrow_colrN   r  )
rS   
ArrowDtypepar~  r?   r  r>   rV   r   r   )rv  r~   r  r`   r  r   s         r4   test_set_index_pyarrow_dtyper    sy    . bmmB23
,,QRYYt5-IJ
KC
..!
,C{+J{+Jj*%r6   c                    t        j                  dgdggdz  dg      } t        t        d            | d<   t        j                  dd	gd
dgdd	gddgddggddg      }t	        j
                  | d      }t	        j
                  |d      }t        j                  t        t        j                  g t        d      d      }t        rddini } t	        j                  ||dd      j                  d      j                  d fd|i|} t        j                  | |dd      j                  d      j                  d fi |}t        ||d       y )NTFr[  rq  r   rL   rr  r   r   m   iY  r   _   r   rs  rM   rN   )r~   r   )r~   r   r   include_groupsouterr  c                    t        |       S rH   r   r1   s    r4   r5   z/test_shuffle_nulls_introduced.<locals>.<lambda>  
    #b' r6   r  c                    t        |       S rH   r   r  s    r4   r5   z/test_shuffle_nulls_introduced.<locals>.<lambda>  r  r6   )check_names)rS   r?   rY   r  r>   rV   r  r   rt  r   r
   r  groupbyapplyr   )r  r   r%  r   r  r  r  r  s           r4   test_shuffle_nulls_introducedr    sE   
,,(2-u
=CE#JCH
,,
Q#qC8b!Wr1g6c
C >>#2.D>>#1-D993bhhrC&HsSD2?&.RN	tS1		!	@(,	@0>	@ 	cw3/		!	5%3	5 
 fhE2r6   )
__future__r   multiprocessingr  r   rU   r   concurrent.futuresr   datetimer   r   decimalr   	functoolsr   numpyrT   rS  rS   r   rB   dask.dataframe	dataframer>   	dask.baser	   dask.dataframe._compatr
   r   r   dask.dataframe.shuffler   r   dask.dataframe.utilsr   r   r   r  r  r?   rX  rt  r  rW  rJ  r]   r<   r;   r/   rW   rY  rF   rJ   rb   re   ri   ro   rv   r{   rH  r  r   r  timedelta_ranger   r   r   markparametrizeparamslowr   r   r   r   r   r   r   r   r   r   r
  r  r  r  r  r#  r%  rQ  gpur^  rd  rn  rx  r}  r  r  r  r  r  r  r   r   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r&  r(  r?  rB  rO  rT  rV  rY   r  r   r`  rd  rf  rn  rq  rx  skipifr  r6  r  r  rI   r6   r4   <module>r     s
   "     2         . N N K 5
 bll;9Mbll;9Mbll;9M
 
T("((2t"4,",,. BNN9299SZZ\*L9R
Myy{D8
V
VHT& bllrxx	AW5rxx(1,I>ryy1,-44Z@ryy1,-,q01biij!<==2==Q<PQbii***61=>	#& Aq!\V\\"FKK4D4DEF8 CTT 
)Q2 T.	/ /=19 2 094-".0()K # # 'N0KV\ xfFKKOODE ) )F xfFKKOODE""4 xfFKKOODEII*	# $.' /'& $/H 0H<!$
(B&@& #uc#IJ77 A  "
5  V V 9&& v7 8B9. )<=4$D&>)B
C ay1 9 2 >
 9F#  +8
) "F, tUm4S3*562s),E - 7 5E ,&,,vV[[__EF S3*sCjABtUm42 5 C
2" tUUDMD%=&QRc
S#J782s),1 - 9 S1 &(9:tUm4S3*56Aq6*
 eBiD6B;.eBn1EE#U3_ 55	

 eBiD6A:-huRy)*	
	1 + 7 5 ;"171 c
S#J78Ar7+1 , 91$& A/  0 0 B$J~6
s&'
() Q!"
Q#$
Q"#
tQ
Dr2.	/1BCR\\,'l)CD$	
 #,	(,7
",,w
i!8	9;RS
r1+tAr{	#%:;!*&+ 7,&3w&  	Bs   (e5 5f ?f 