
    biF2                   8   d dl mZ d dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZmZmZ 	 d dlZd	 Zd
 Z d Z!d Z"d Z#d Z$d Z%ejL                  jO                  dddg      ejL                  jO                  dg d      d               Z(ejL                  jO                  dddg      ejL                  jO                  dg d      ejL                  jO                  dddg      d                      Z)d Z*d Z+d Z,d Z-d Z.d Z/ejL                  jO                  ddd g      d!        Z0d" Z1ejL                  jO                  d#d$d%g      d&        Z2ejL                  jO                  d#d$d%g      d'        Z3ejL                  jO                  d#d$d%g      d(        Z4d) Z5d* Z6ejL                  jO                  d+g d,      ejL                  jO                  d-ddg      d.               Z7ejL                  jO                  d+g d,      d/        Z8ejL                  jO                  d+d0d%g      d1        Z9d2 Z:ejL                  jO                  d3g d4      ejL                  jO                  d+d5 ejv                  d6ejL                  jy                  d78      9      g      ejL                  jO                  d: ejv                  d;ejL                  jy                  d<8      9       ejv                  d=ejL                  jz                  9      g      d>                      Z>d? Z?ejL                  j                  ejL                  jO                  d+g d,      d@               ZAejL                  j                  edA8      ejL                  j                  ejL                  jO                  d+g d,      dB                      ZCdC ZDejL                  j                  ejL                  jO                  d+g d,      dD               ZEejL                  jO                  dEi  eFdFG       eFdFH       eFdIdFgdJK       eFdLdIdFgdMN       eFdIdLgH       eFdLdOgH      g      dP        ZGdQ ZHejL                  jO                  dRddg      dS        ZIdT ZJejL                  jO                  d+dUd0g      ejL                  jO                  dRddg      dV               ZKdW ZLdX ZMdY ZNdZ ZOd[ ZPd\ ZQd] ZRd^ ZSd_ ZTd` ZUda ZVejL                  jO                  dbdc ejv                  dddejL                  jy                  exs e e	dd      k\  deeWdf      9      dgdh ejv                  dddejL                  jy                  exs e e	dd      k\  deeWdf      9      dig      dj        ZXdk ZYdl ZZdm Z[dn Z\do Z]dp Z^dq Z_dr Z`ejL                  jO                  d:d; ejv                  d=ejL                  jz                  9      g      ds        ZaejL                  jO                  dtddg      du        ZbejL                  jO                  dvdwdx ejv                  dyejL                  j                  edu dz8      9       ejv                  d{ejL                  j                  edu dz8      9      g      d|        Zcd} Zdd~ Zed Zfd Zgd ZhejL                  jO                  d+g d      ejL                  jO                  dddg      ejL                  jO                  dddg      d                      ZiejL                  jO                  d+d0d$g      d        ZjejL                  jO                  d+dUd%g      ejL                  jO                  dg d      ejL                  jO                  dg d      d                      ZkejL                  jO                  dg d      ejL                  jO                  dg d      ejL                  jO                  d+g d      d                      Zly# e$ r dZY w xY w)    )annotationsN)Version)
PY_VERSION)compute_as_if_collection)PANDAS_GE_210PANDAS_GE_220tm)concat)assert_divisions	assert_eq
check_metahas_known_categoriesc                    t        j                  dg dig d      } t        j                  | g d      }t        j                  dt	        d      ig d      }t        j                  |g d	      }|j                  |d
      }|j                  d   |j                  d   k(  sJ |j                  d   t        |j                  |j                  z         k(  sJ t        || j                  |             |j                  |d      }|j                  d   |j                  d   k(  sJ |j                  d   |j                  d   k(  sJ t        || j                  |d             |j                  |d      }|j                  d   dk(  sJ |j                  d   t        |j                  |j                  z         k(  sJ t        |j                         | j                  |d             |j                  |d      }|j                  d   dk(  sJ |j                  d   dk(  sJ t        |j                         | j                  |d             y )Nx                  r   r   r   r   r      indexr   r   r   yabcdefr   r   r   r   r      r   r   r   r    lefthowr   rightinnerr   outerr    )pd	DataFrameddrepartitionlistmerge	divisionsmaxr   joincompute)AaBbcs        Z/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/dataframe/tests/test_multi.py1test_merge_indexed_dataframe_to_indexed_dataframer9      s   
c-.6HIA
q)$A
c4>*2DEA
q,'A	vA;;q>Q[[^+++;;r?c!++";<<<<a	wA;;q>Q[[^+++;;r?akk"o---aw'(	wA;;q>Q;;r?c!++";<<<<aiik166!612	wA;;q>Q;;r?aaiik166!612    c                   t        | t        j                        r| j                  d      }n| }t        |t        j                        r|j                  d      }n|}t	        j
                  |j                  |j                         t        |t        j                        r]|j                  t        |j                              j                  }|j                  t        |j                              j                  }n4|j                         j                  }|j                         j                  }t        j                  j                  ||       y )Nsync)	scheduler)
isinstancer+   r*   r2   r	   assert_index_equalcolumnsr)   sort_valuesr-   values_compatassert_numpy_array_equal)aabbr4   r6   avbvs         r8   list_eqrI   8   s    "bll#JJJ("bll#JJJ(!))QYY/!R\\"]]4		?+22]]4		?+22]]_##]]_##JJ''B/r:   c                 X   t        j                  t        t        d            g dd      } t        j                  t        t        d            g dd      }t        j                  t        t        dd            g d	d
      }| j	                  |ddd      }|j	                  |ddd      }t        j                  | d      }t        j                  |d      }t        j                  |d      }|j	                  |ddd      }|j	                  |ddd      }	t        ||	       y )Nr   )A0A1A2A3A4A5)keyr3   r   B0B1B2B3)rQ   r5   r   r   C0C1C2C3)rQ   Cr'   _l_rr$   lsuffixrsuffixr   npartitionsr   )r)   r*   r-   ranger1   r+   from_pandasr   )
df1df2df3join_pdmulti_join_pdddf1ddf2ddf3join_ddmulti_join_dds
             r8   test_sequential_joinsrp   M   s    
,,U1X%IJC ,,tE!H~4LM
NC
,,tE!QK07OP
QChhsthDGLL'4LNM >>#1-D>>#1-D>>#1-Dii'4iFGLL7D$LOMm]+r:   c                 l   t        j                  dt        d      ig d      } t        j                  | d      }t        j                  dt        d      ig d	      }t        j                  |d
      }t        j
                  | |dd      }t        j
                  ||dd      }t        ||       y )Nleft_valabcdabcdabcd)r   r   r   	   
                        r   r   rb   	right_valxyzxyzxyzxyz)r   r   r   r   r   ru      rw               r   T
left_indexright_index)r)   r*   r-   r+   re   
merge_asofr   r3   r4   r5   r6   r\   r7   s         r8   test_merge_asof_indexedr   b   s    
	T*%&:	A 	qa(A
	d9o&9	A 	qa(A
at>A
at>AaOr:   c                 L   t        j                  g dg dd      } t        j                  | d      }t        j                  g dg dd      }t        j                  |d      }t        j                  | |d	      }t        j                  ||d	      }t        ||d
       y )Nr   r   ru   r4   r6   r7   r4   rr   r   rb   r   r   r   r   r   r4   r}   r4   onFcheck_indexr)   r*   r+   re   r   r   r   s         r8   test_merge_asof_on_basicr   t   sx    
:?CDA
qa(A
?IJA
qa(A
as#A
as#Aa&r:   c                    t        j                  g dg dd      } t        j                  | d      }t        j                  g dg dd      }t        j                  |d      }t	        j
                  t        d	      5  t        j                  ||d
d       d d d        t	        j
                  t        d	      5  t        j                  ||d
d       d d d        t	        j
                  t        d	      5  t        j                  ||d
dd       d d d        y # 1 sw Y   xY w# 1 sw Y   TxY w# 1 sw Y   y xY w)Nr   r   r   r   rb   r   r   combination of bothmatchr4   rr   )r   left_onr}   )r   right_on)r   r   r   r)   r*   r+   re   pytestraises
ValueErrorr   r3   r4   r5   r6   s       r8   'test_merge_asof_on_lefton_righton_errorr      s    
:?CDA
qa(A
?IJA
qa(A	z)>	? 8
asJ78	z)>	? :
as[9:	z)>	? N
asJMN N	8 8: :N Ns$   D ?D,<D8 D),D58Ec            	        t        j                  g dg dg dd      } t        j                  | d      }t        j                  g dg dg dd	      }t        j                  |d      }t	        j
                  t        d
      5  t        j                  ||ddd       d d d        t	        j
                  t        d
      5  t        j                  ||ddd       d d d        t	        j
                  t        d
      5  t        j                  ||dddd       d d d        y # 1 sw Y   xY w# 1 sw Y   UxY w# 1 sw Y   y xY w)Nr   )r   r   rt   r   )r4   r6   rr   r   rb   r   )r   r   rt   r      )r4   r6   r}   r   r   r4   r6   rr   )r   byleft_byr}   )r   r   right_by)r   r   r   r   r   r   s       r8   'test_merge_asof_by_leftby_rightby_errorr      s   
:I?STA
qa(A
$5OT	A 	qa(A	z)>	? @
assJ?@	z)>	? B
ass[AB	z)>	? V
assJUV V	@ @B BV Vs$   D)D5E)D25D>E
allow_exact_matchesTF	direction)backwardforwardnearestc                T   t        j                  g dg dd      }t        j                  |d      }t        j                  g dg dd      }t        j                  |d      }t        j                  ||d| |	      }t        j                  ||d| |	      }t        ||d
       y )Nr   r   r   r   rb   r   r   r4   )r   r   r   Fr   r   )r   r   r3   r4   r5   r6   r\   r7   s           r8   test_merge_asof_onr      s     	:?CDA
qa(A
?IJA
qa(A
	1*=	A 		1*=	A a&r:   unknown_divisionsc           
     p   t        j                  g dg ddg d      }t        j                  |d      }t        j                  dg d	ig d	      }t        j                  |d      }|r|j	                         }t        j
                  ||d
d| |      }t        j
                  ||d
d| |      }t        ||       dD ]  }	g dg dfg dg dfg dg dfg dg dffD ]  \  }
}t        j                  d
|
i|
D cg c]  }|dz  	 c}      }t        j                  ||	      }t        j                  d|i|      }t        j                  ||	      }|r|j	                         }t        j
                  ||d
d| |      }t        j
                  ||d
d| |      }t        ||         y c c}w )Nr   r   r   )ru   ry      r   r   rb   r}   )r   r   r   r   r4   T)r   r   r   r   r   r   r   )r   ru   r   ry   r   r   r   r   )r   r   ru   ru   r   r   )	r   r   r   rt   ru      rw   r   r   )r   ru   r   )r   r   r   r   r   r   rt   rt   ru   ru   r   r   ru   r6   )r)   r*   r+   re   clear_divisionsr   r   )r   r   r   r3   r4   r5   r6   r\   r7   npartsa1idx2r   s                r8   #test_merge_asof_left_on_right_indexr      s    	:?C<XA
qa(A
k<0EA
qa(A
		/	A 			/	A aO l+?+#%EFBC	
 	HB c2Yr.B!q2v.BCAqf5Ac4[5Aqf5A %%' $7#A  $7#A aO=	 /Cs   ?F3c                 P   t        j                  dg dig d      } t        j                  | d      }t        j                  dg dig d      }t        j                  |d      }t        j                  | |d	d	
      }t        j                  ||d	d	
      }t        ||       y )Nrr   r   r   r   r   rb   r}   r   Tr   r   r   s         r8   &test_merge_asof_indexed_two_partitionsr      s}    
j/2*EA
qa(A
k?3?KA
qa(A
at>A
at>AaOr:   c                 d   dD  cg c]  } t        j                  |        }} g d}g d}g d}dD  cg c]  } t        j                  |        }} g d}g d}g d}t        j                  ||||d	g d	
      }	t        j                  |	d      }
t        j                  ||||dg d
      }t        j
                  d |j                  dd |j                  dd gg d      }t        j                  ||	dd      }t        j                  ||
dd      }t        ||d       y c c} w c c} w )N2016-05-25 13:30:00.023r   z2016-05-25 13:30:00.030z2016-05-25 13:30:00.0412016-05-25 13:30:00.048z2016-05-25 13:30:00.049z2016-05-25 13:30:00.072z2016-05-25 13:30:00.075GOOGMSFTr   r   r   AAPLr   r        @皙I@g\(I@gQI@r   g(\X@r   gzGJ@=
ףp@g{GI@g=
ףpI@g      J@r   gq=
ףX@gףp=
@gp=
J@r   z2016-05-25 13:30:00.038r   r   r   r   r   r   r   r   r   r   g\(@g(\@g     X@K      d   r   r   timetickerbidaskr@   r   rb   r   r   pricequantityc                    | S N r   s    r8   <lambda>z'test_merge_asof_on_by.<locals>.<lambda>-      ! r:   r   r   r   r   r   r   r/   r   r   )r   r   Fr   )	r)   to_datetimer*   r+   re   from_mapilocr   r   dtimes_A	tickers_Abids_Aasks_Atimes_B	tickers_Bprices_Bquantities_Br3   r4   r5   r6   r\   r7   s                  r8   test_merge_asof_on_byr      s;   	
 	qG  QIHFHF
	 	q	G 	 9I4H+L
IfVL0	A 	qa(A
$		
 8	A 		
!affQqk"	A 	av(3A
av(3Aa&m 	s
   D(D-c            	        dD  cg c]  } t        j                  |        }} g d}g d}g d}dD  cg c]  } t        j                  |        }} g d}g d}g d}t        j                  ||||d	g d	
      }	t        j                  |	d      }
t        j                  ||||dg d
      }t        j
                  d |j                  dd |j                  dd gg d      }t        j                  ||	ddt        j                  d            }t        j                  ||
ddt        j                  d            }t        ||d       y c c} w c c} w )Nr   r   r   r   r   r   r   r   r   r   r   rb   r   c                    | S r   r   r   s    r8   r   z1test_merge_asof_on_by_tolerance.<locals>.<lambda>g  r   r:   r   r   r   r   r   r   r   2ms)r   r   	toleranceFr   
r)   r   r*   r+   re   r   r   r   	Timedeltar   r   s                  r8   test_merge_asof_on_by_tolerancer   7  sQ   	
 	qG  QIHFHF
	 	q	G 	 9I4H+L
IfVL0	A 	qa(A
$		
 8	A 		
!affQqk"	A 	av(bll5>QRA
av(bll5>QRAa&m 	s
   EEc            	        dD  cg c]  } t        j                  |        }} g d}g d}g d}dD  cg c]  } t        j                  |        }} g d}g d}g d}t        j                  ||||d	g d	
      }	t        j                  |	d      }
t        j                  ||||dg d
      }t        j
                  d |j                  dd |j                  dd gg d      }t        j                  ||	ddt        j                  d      d      }t        j                  ||
ddt        j                  d      d      }t        ||d       y c c} w c c} w )Nr   r   r   r   r   r   r   r   r   r   r   rb   r   c                    | S r   r   r   s    r8   r   zBtest_merge_asof_on_by_tolerance_no_exact_matches.<locals>.<lambda>  r   r:   r   r   r   r   r   r   r   10msF)r   r   r   r   r   r   r   s                  r8   0test_merge_asof_on_by_tolerance_no_exact_matchesr   q  si   	
 	qG  QIHFHF
	 	q	G 	 9I4H+L
IfVL0	A 	qa(A
$		
 8	A 		
!affQqk"	A 			,,v&!	A 			,,v&!	A a&I 	s
   EEc                    t        j                  g dg dd      } t        j                  | d      }t        j                  g dg dd      }t        j                  |d      }t        j                  ||d	
      }t        j                  t        d      5  |j                          d d d        y # 1 sw Y   y xY w)Nr   r   r   r   rb   )r   r   r   r   r   r   r   r4   r   z
right keysr   )	r)   r*   r+   re   r   r   r   r   r2   )r3   r4   r5   r6   results        r8   test_merge_asof_unsorted_raisesr     s    
:?CDA
qa(A
?IJA
qa(A]]1aC(F	z	6   s   B44B=c                 $   t        j                  t        t        d            t        t        d            d      } t	        j
                  | d      }| | j                  dk     j                         j                  ddi      }t	        j
                  |d      }t	        j                  |j                  d	      |j                  d	      d
d
      }t        j                  | j                  d	      |j                  d	      d
d
      }t        ||d       t	        j                  |j                  d	      |j                  d	      d
d
      }t        j                  |j                  d	      | j                  d	      d
d
      }t        ||d       t	        j                  |j                  d	      |j                  d	      d
d
      }t        j                  |j                  d	      |j                  d	      d
d
      }t        ||d       y )Nru   )	index_colgood_valr   rb   r   r   	empty_valr   r   Tr   Fr   )r)   r*   r-   rd   r+   re   r   copyrenamer   	set_indexr   )good_dfgood_ddempty_dfempty_dd	result_dd	result_dfs         r8   test_merge_asof_with_emptyr
    s   lleBid5QS9oVWGnnW!4G!!A%&++-44j+=V4W  ~~hA6H +&;'	I +&;'	I i6;'+&	I ;'+&	I i6;';'	I ;';'	I i6r:   zleft_col, right_col)	endofweek	timestamp)r  r  c                L   t        j                  | g ddg di      }t        j                  |g ddg di      }t        j                  ||| |      }t        j                  t        j                  |d	      t        j                  |d	      | |      }t        ||d
       y )Nr   r   r   r   r   r   GroupCol)  !  r  r  r  r  )r   r   r   r   GroupVal)r  r  r  r  r   r   r   rb   Fr   )r)   r*   r   r+   re   r   )left_col	right_colrf   rg   r	  r  s         r8   test_merge_asof_on_left_rightr    s     ,,(<	
C ,,	<=UV
WC c39MI 
s*
s*	I i6r:   c                 >   t        j                  t        t        j                  d      gdz  g d            } t        j                  | | d      }t        dd      D ]=  }t        j                  | |	      }t        j                  ||d      }t        ||       ? y )
Nz1-1-2020r   r   )tsfoor  )r"   r&   r   r   r   rb   )	r)   r*   dictr   r   rd   r+   re   r   )dfexpectedrc   ddfr   s        r8   (test_merge_asof_with_various_npartitionsr    s{    	dr~~j9:Q>IN	OB}}"B48HQ{ $nnR[9Cst<(F#	$r:   r1   r'   r(   c                   t        j                  g dt        d      dg d      }t        j                  |g d      }t        j                  dg dig d      }t        j                  |g d	      }t        j
                  ||gd
| d      }t        j                         5  t        j                  dt               t        j
                  ||g|       }t        ||       d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   r   )ru   ry   (   2   <   P   r   r!   r   F)axisr1   sortignorer1   )r)   r*   r-   r+   r,   r
   warningscatch_warningssimplefilterFutureWarningr   )r1   r3   r4   r5   r6   r  r   s          r8   test_indexed_concatr-    s    
 tH~6>P	A 	q)$A
c34<NOA
q,'Ayy!Qad?H		 	 	" $h6Aq6-&(#$ $ $s   $A C--C6c                &   t        j                  g dt        d      dg d      }t        j                  |d      }t        j                  g dt        d      dg d      }t        j                  |d      }t        j                  g dt        d      dg d      }t        j                  |d      }dd	i}||||f||||ffD ]G  \  }}	}
}t        j
                  |
|gfd
| i|}t        j
                  ||	gfd
| i|}t        ||       I y Nr   r   r   r   r   r    rt   ru   r   r   rv   r   zr&  Fr1   )r)   r*   r-   r+   re   r
   r   r1   pdf1rk   pdf2rl   pdf3rm   kwargsdd1dd2pd1pd2r  r   s                 r8   test_concatr<  1  s   << tH~6>PD >>$"D<< tH~6>TD >>$"D << tH~6>TD >>$"De_F 
tT4 	tT4  $S#s 99c3Z=d=f=C:;D;F;&(#$r:   c                   t        j                  g dt        d      dg d      }t        j                  |d      }t        j                  g dt        d      dg d      }t        j                  |d      }t        j                  g dt        d      dg d      }t        j                  |d      }dd	i}|j
                  |j
                  |j
                  |j
                  f|j
                  |j                  |j
                  |j                  ffD ]G  \  }}	}
}t        j                  |
|gfd
| i|}t        j                  ||	gfd
| i|}t        ||       I y r/  )	r)   r*   r-   r+   re   r   r2  r
   r   r3  s                 r8   test_concat_seriesr>  M  s2   << tH~6>PD >>$"D<< tH~6>TD >>$"D << tH~6>TD >>$"De_F 
(	( $S#s 99c3Z=d=f=C:;D;F;&(#$r:   c                    t        j                  g dt        d      dg d      } t        j                  | d      }t        j                  dt        d      ig d      }t        j                  |d      }t        t        |d               |d<   |j                  d	
      j                  j                         d	z
  |d<   ddi}t        j                  | |gfi |}t        j                  ||gfi |}t        ||       y )Nr   r   r   r   r   r   r0  r   r   )partition_countr&  F)r)   r*   r-   r+   re   rd   lenassignr@  cumsumr
   r   )r4  rk   r5  rl   r7  r  r   s          r8   &test_concat_with_operation_remains_hlgrD  i  s    << tH~6>PD >>$"D<<d8n-5KLD>>$"D c$s)n%DIA.>>EEG!KDIe_Fyy$00HYYd|.v.Ffhr:   c                    t        j                  dg did      } t        j                  g d      }t        j                  | |g      }t        j                  | d      }t        j                  |d      }t        j                  ||g      }t        ||       t        j                  g dgd      }t        j                  | |g      }t        j                  |d      }t        j                  ||g      }	|	j                  j                  j                  d      |	j                  _        t        ||	d	
       t        t        j                  ||g    g      t        j                  | | g    g             y )Nr4   )r      ,  int64dtyper   rb   r   )r@   rJ  F)check_dtype)	r)   r*   r
   r+   re   r   _metar   astype)
r  r  	df_concatr  	empty_ddf
ddf_concatempty_df_with_coldf_concat_with_colempty_ddf_with_colddf_concat_with_cols
             r8   test_concat_dataframe_emptyrU  }  s)   	sO,G	<B||Bg.H		2x.)I
..
+CxQ7IC+,Ji$R#gFB(9#:;(9qI))S*<$=>
 ':&?&?&E&E&L&LW&U# "55Ibiic"g'B2<)@Ar:   r$   )r'   r(   r"   r&   on_indexc           	        t        j                  t        t        j                  d      j                  t                    dz  t        t        j                  d            dz  d      }t        j                  t        j                  d      t        j                  d      d      }t        j                  |d      }t        j                  |d      }dg}|x}}|r$|j                  d      }|j                  d      }d }t        j                  d      5 }	t        j                  d       t        j                  |||| ||      }
t        d	 |	D              }d d d        t        
t         j                        r|
n|
j!                         }
|
j#                         j$                  j                         }|rs|rJ y y # 1 sw Y   gxY w)
Nr   r   r3   r5   r3   Trecordalways)r   r$   r   r   c              3  6   K   | ]  }d t        |      v   yw)z!merge column data type mismatchesN)str).0rs     r8   	<genexpr>z,test_merge_columns_dtypes.<locals>.<genexpr>  s     Sq8CFBSs   )r)   r*   r-   nparangerM  floatr+   re   r  r)  r*  r+  r.   anyr>   r2   isnarB   )r$   rV  rf   rg   r4   r6   r   r   r   rZ  r   warnedhas_nanss                r8   test_merge_columns_dtypesrh    so    ,,299Q<&&u-.2biil9Ka9OPC ,,RYYq\		!=
>C
sAA
sAA
B''JKKKK		 	 	- Th'qRSZ[
 SFSST "&",,7VV^^=MF{{}##'')H00T Ts   AG  G	c                   t        j                  g dg dd      }t        j                  |g d      }t        j                  g dg dd      }t        j                  |g d      }t	        t        j
                  ||d	d	| |
      t        j
                  ||d	d	|              t        j
                  ||d|       }t        |t        j
                  ||d|              t        d |j                  D              sJ t        t        j
                  ||dd| |      t        j
                  ||dd|              t        t        j
                  ||dd| d|      t        j
                  ||dd| d             t        t        j
                  ||| |      t        j
                  |||              t        t        j
                  ||| |      t        j
                  |||              t        t        j
                  ||| |      t        j
                  |||              t        t        j
                  ||| |      t        j
                  |||              t        t        j
                  ||d	d	| |
      t        j
                  ||d	d	|              t        t        j
                  ||d	d	| d|      t        j
                  ||d	d	| d             t        t        j
                  ||dd	| |      t        j
                  ||dd	|              t        t        j
                  ||dd	| d|      t        j
                  ||dd	| d             y )Nr   r  r   )r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   r   )r   r2  )r   r   r   T)r   r   r$   shuffle_method)r   r   r$   r   r   r$   c              3  $   K   | ]  }|d u  
 y wr   r   )r^  r   s     r8   r`  ztest_merge.<locals>.<genexpr>  s     3QqDy3s   r   r2  )r   r   r$   rj  )r   r   r$   )12)r   r   r$   suffixesrj  )r   r   r$   ro  r$   rj  r#   )r   r   r$   ro  rj  )r   r   r$   ro  )r   r   r$   rj  r   r   r$   )r   r   r$   ro  rj  )r   r   r$   ro  )	r)   r*   r+   r,   r   r.   rI   allr/   )r$   rj  r3   r4   r5   r6   r   s          r8   
test_mergers    s   
-4FGHA
q)$A
-4FGHA
q)$A
)	
 	A$DcB
 XXas,FFBHHQcs343&"2"23333
q#^	
 	AsSc:	 
)	
 	AsScJO 
A3~>ASV@W 
A3~>ASV@W 
A3~>ASV@W 
A3~>ASV@W 
)	
 	A$DcB
 
)	
 	A$DcJW 
q#4S	
 	As#>	 
)	
 	As#
Sr:   r&   c                   t        j                  g dg dd      }t        j                  g dg dd      }t        j                  |d      }t        j                  |d      }|j	                  |d|	      }|j	                  |d|	      }t        ||d
       |j                  d |j                        j                          y )N)r   r   r   r   r   r   r   r    )r4   val)r   r   r   r   )r   r   rv   rw   r   rb   r4   rk  Fr   c                    | S r   r   r   s    r8   r   z*test_merge_empty_left_df.<locals>.<lambda>4  s    A r:   )meta)	r)   r*   r+   re   r.   r   map_partitionsrL  r2   )rj  r$   r"   r&   dd_leftdd_rightmergedr  s           r8   test_merge_empty_left_dfr}  #  s     <<l<@ADLL|4DEFEnnTq1G~~e3H]]8]5Fzz%CSz1HfhE2 +FLL9AACr:   c                    t        j                  g dg dd      } t        j                  g dg dd      }t        j                  t        d      5  t        j                  | |d	       d d d        y # 1 sw Y   y xY w)
N)rK   rL   rM   rN   rR   rX  rW   )D0D1D2D3z1dask.dataframe.merge does not support how='cross'r   crossr#   )r)   r*   r   r   r   r+   r.   )r"   r&   s     r8   test_merge_how_raisesr  7  sq    <<))	
D LL))	
E 
M
 + 	u'*+ + +s   A66A?parts))r   r   )r   r   )r   r   leftsemileftantizleftanti is not supported yet)reason)marksenginepandasz,Pandas does not support leftsemi or leftanticudfc                   | dk(  r*t        j                  d      }t        j                  d      }t        j                  t	        j
                  dd      g dg dg dd	      }t        j                  g d
g dg dd      }| dk(  rOj                  |      }|j                  |      }j                  ||d         }|j                  ||d         }n4t        j                  ||d         }t        j                  ||d         }|j                  |d|      j                  dg      }	|j                  |d|      j                  dg      }
t        |
|	d       y )Nr  	dask_cudfe   j   )stop)JohnTomHarryRahulSakil)CalMumDelBanr  )iP  i@  i8 i`  i_ )emp_idnamecitysalary)i  i  i  i  i  )g   r  i   f   r  )DaskSparkr\   PythonR)skill_idr  
skill_namer   rb   r   r  rk  Fr   )r   importorskipr)   r*   ra  rb  re   	from_cudfr+   r.   rA   r   )r  r$   r  r  r  empskillsdd_emp	dd_skillsexpectr   s              r8   test_merge_tasks_semi_anti_cudfr  K  sH   0  ""6*''4	
,,ii#.>79		
C \\1/?	
F s#!!&)$$SeAh$?''E!H'E	q:NN6uQx@	YYv(Y4@@(LF\\)c\:FFzRFff%0r:   c                 6   t        j                  g dg dd      } t        j                  g dg dd      }t        j                  | d      }t        j                  |d      }|j	                  |dd	d
      }t        d |j                  D              rJ y )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r4   r6   r7   r   r   rb   r   r4   r   tasks)r   r   rj  c              3  *   K   | ]  }d |d   v   yw)partdr   Nr   r^  ks     r8   r`  z2test_merge_tasks_passes_through.<locals>.<genexpr>  s     4q7ad?4   )r)   r*   r+   re   r.   rd  dask)r4   r6   rE   rF   ccs        r8   test_merge_tasks_passes_throughr    s}    
07LMNA
07LMNA	q	)B	q	)B	"cC	HB4BGG44444r:   c                   t        j                  g dg dd      }t        j                  g dg dd      }t        j                  g dg ddt        d            }t        j                  g dg ddt        d            }|}t        j                  g dg ddt        d	            }|}t        j                  g d
g ddt        d	            }	t        j                  g dg ddt        d            }
t        j                  g dg ddt        d            }t        j                  g dg ddt        d            }t        j                  g dg ddt        d            }t        j                  g dg ddt        d            }t        j                  g dg ddt        d            }d }||f||f||f||	f|
|f||f||ffD ]|  \  }}dD ]p  \  }}t        j                  ||      }t        j                  ||      }t        t        j                  ||| dd|       |t        j                  ||| dd      |j                  j                               t        t        j                  ||| dd|       |t        j                  ||| dd      |j                  j                               t        t        j                  ||| dd|d       |t        j                  ||| ddd      |j                  j                               t        t        j                  ||| dd|d       |t        j                  ||| ddd      |j                  j                               t        |j                  || dd|       ||j                  || dd      |j                  j                               t        |j                  || dd|       ||j                  || dd      |j                  j                               t        t        j                  ||| dd|      t        j                  ||| dd             t        t        j                  ||| dd|      t        j                  ||| dd             t        t        j                  ||| dd|d       t        j                  ||| ddd!             t        t        j                  ||| dd|d       t        j                  ||| ddd!             t        t        j                  ||| dd|      t        j                  ||| dd             t        t        j                  ||| dd|      t        j                  ||| dd             t        |j                  || dd|      |j                  || dd             t        |j                  || dd|      |j                  || dd             t        |j                  || dd|      |j                  || dd             t        |j                  || dd|      |j                  || dd             s  y )"Nr  r  r  r  abcdefgr   )r   r   r    rt   r   r   r   r   abdg)rt   ru   r   r   )r   r   r   r   r   r   r   lmnopqr)r   r   r   r   abcdcdefghir   r   r   r   ru  fghic                v    t        |       dk(  r*| j                  | j                  j                  |            S | S Nr   rA  r  r   rM  outrJ  s     r8   	fix_indexz/test_merge_by_index_patterns.<locals>.fix_index  2     s8q===!1!1%!899
r:   )r   r   )r   r   )r   r   Tr$   r   r   rj  r$   r   r   )r$   r   r   rj  	indicator)r$   r   r   r  r4   r7   r$   r   r   rj  r$   r   r   r6   r   )r$   r   r   rj  r  )r$   r   r   r  )
r)   r*   r-   r+   re   r   r.   r   rJ  rI   )r$   rj  pdf1lpdf1rpdf2lpdf2rpdf3lpdf3rpdf4lpdf4rpdf5lpdf5rpdf6lpdf6rpdf7lpdf7rr  pdlpdrlpartrpartddlddrs                          r8   test_merge_by_index_patternsr    sa    LL4;PQRELL4;PQRELL#*?@YE LL#*?@YE ELL|,?tF|TEELL\B$v,WE LL#*?@YE LL|,?tF|TELL#*?@YE LL|,?tF|TELL#*?@YE LL|,?tF|TE 
						 fS
 ]	LE5
 ..e,C..e,C# $#1 HHS#34TRIIOO # $#1 HHS#34TRIIOO # $#1" HH#'$("& IIOO
, # $#1" HH#'$("& IIOO
. 		# $#1   IIcstINIIOO 		# $#1   IIcstINIIOO  #1 csC#F
  #1 csC#F
  #1" csC#QUV  #1" csC#QUV  #1 csC#F
  #1 csC#F
 		 #1   		#3c	B	 		 #1   		#3c	B	 		 #1   		#3c	B	 		 #1   		#3c	B	i]	fr:   zbreaks with pandas=2.1.0+c                	   d }t        j                  t        d      g ddt        d            }t        j                  t        d      g ddt        d            }|}t        j                  t        d      g ddt        d            }|}t        j                  t        d      g d	dt        d            }t        j                  t        d
      g ddt        d            }	t        j                  t        d      g d	dt        d            }
t        j                  t        d      g ddt        d            }t        j                  t        d      g d	dt        d            }t        j                  t        d      g ddt        d            }t        j                  t        d      g d	dt        d            }t        j                  t        d      g ddt        d            }t        j                  t        d      g d	dt        d            }||f||f||f|	|
f||f||f||ffD ]  \  }}dD ]  \  }}t        j                  ||      }t        j                  ||      }t        |j                  || |       ||j                  ||       |j                  j                               t        |j                  || |       ||j                  ||       |j                  j                               t        |j                  || dd|       ||j                  || dd      |j                  j                               t        |j                  || dd|       ||j                  || dd      |j                  j                               t        |j                  || ddd      |j                  || ddd             t        |j                  || ddd      |j                  || ddd             t        |j                  || dd      |j                  || dd             t        |j                  || dd      |j                  || dd             t        |j                  || dd      |j                  || dd             t        |j                  || dd      |j                  || dd               y ) Nc                v    t        |       dk(  r*| j                  | j                  j                  |            S | S r  r  r  s     r8   r  z.test_join_by_index_patterns.<locals>.fix_index  r  r:   r  r  r  r   r  gfedcbar  r  abcabceabdar  r  r  ababaabbccdaabbr  r  rp  r#   lr_  )r$   r`   ra   rj  r_   r4   )r$   r   r`   ra   r7   T)r$   r   r   )r$   r   r   )r)   r*   r-   r+   re   r   r1   r   rJ  rI   r.   )r$   rj  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                          r8   test_join_by_index_patternsr    si    LL9o$9:$y/E LL9o$9:$y/E ELL9o$9:$y/E ELLtF|,?tF|TELL9o$9:$y/E LLtF|,?tF|TE LL9o$9:$y/E LLtF|,?tF|TELL9o$9:$y/E LLtF|,?tF|TELL9o$9:$y/E LLtF|,?tF|TE 
						 MS 5 D	LE5..e,C..e,C#nE#((3C(0#))//B #nE#((3C(0#))//B
 #1   HHSc3HDciioo #1   HHSc3HDciioo" ##sCH##sCH
 ##sCH##sCH 		#3	F		#3	F 		#3	F		#3	F 		#34#	F		#34#	F 		#34#	F		#34#	FCD	Mr:   c                 D   t        j                  dg dig d      } t        j                  | d      }t        j                  dg dig d      }| j	                  |d	d
      }|j	                  |d	d
      }|j
                  |j
                  k(  sJ t        ||       y )Nr4   r   )r   r   r   r   r   rb   r6   )      ?g       @g      @r'   )r$   r   )r)   r*   r+   re   r1   r/   r   )r  r  right_dfr  actuals        r8    test_join_gives_proper_divisionsr  )  s    	sO,I	>B
..
+C||S/2/JHwwxWw5HXXhGX4Fs}},,,hr:   c                >   d }t        j                  t        d      t        d      g ddt        d            }t        j                  t        d      t        d      g ddt        d            }t        j                  t        d      t        d	      g ddt        d            }t        j                  t        d
      t        d      g ddt        d            }t        j                  t        d      t        d      g ddt        d            }t        j                  t        d      t        d      g ddt        d            }||f||f||ffD ]  \  }	}
dD ]  \  }}t        j                  |	|      }t        j                  |
|      }t        |j                  || |       ||	j                  |
|       |	j                  j                               t        |j                  || |       ||
j                  |	|       |
j                  j                               t        t        j                  ||| dd|       |t        j                  |	|
| dd      |	j                  j                               t        t        j                  ||| dd|       |t        j                  |
|	| dd      |
j                  j                               t        t        j                  ||| dd|      t        j                  |	|
| dd             t        t        j                  ||| dd|      t        j                  |	|
| dd             t        t        j                  ||| dd|      t        j                  |
|	| dd             t        t        j                  ||| dd|      t        j                  |
|	| dd             t        t        j                  ||| ddgddg|      t        j                  |	|
| ddgddg               y )Nc                v    t        |       dk(  r*| j                  | j                  j                  |            S | S r  r  r  s     r8   r  z1test_merge_by_multiple_columns.<locals>.fix_index:  s0    s8q===!1!1%!899
r:   
abcdefghij)
r   r   r   r   r   r   r   r    rt   ru   r   r   )
ru   rt   r    r   r   r   r   r   r   r   )r   ef
abcdeabcde
abcabcabca
edcbaedcba
aaabbbcccd
fghijklmno
aaaaaaaaaa
aaabbbccaa
abbbbbbbbb
ABCDEFGHIJr  rp  r#   Tr  r  r4   r   r  r  r6   r  )r)   r*   r-   r+   re   r   r1   r   rJ  r.   rI   )r$   rj  r  r  r  r  r  r  r  r  r  r  r  r  r  s                  r8   test_merge_by_multiple_columnsr  7  s    LLl#l#0	

 < E LLl#l#0	

 < E LLl#l#0	

 < E LLl#l#0	

 < E LLl#l#0	

 < E LLl#l#0	

 < E U^eU^eU^D dS4 c	LE5..e,C..e,C#nE#((3C(0#))//B #nE#((3C(0#))//B
 # $#1 HHS#34TRIIOO # $#1 HHS#34TRIIOO   #1 csC#F
  #1 csC#F
  #1 csC#F
  #1 csC#F
  #J!3Z#1 csS#J#sT
sc	dr:   r7  int)id_vars)
value_varsobjmyvar)r	  var_names1myval)r  r	  
value_names2c                   t        j                  t        d      dz  t        d      dz  t        d      dz  t        j                  j                  d      d      }t        r|j                  d	d	d
      }t        j                  |d      }t        t        j                  |fi | t        j                  |fi |        t         |j                  di |  |j                  di |        y )Nr  r   XYru   abcder   ry   )r
  r  r  r  zstring[pyarrow])r  r  r   )r)   r*   r-   ra  randomrandnparM  r+   re   rI   melt)r7  pdfr  s      r8   	test_meltr    s     ,,<!#t*r/w-!#99??2&		
C 
 jj 19JKL
..a
 CBGGC"6"BGGC$:6$:;HCHHv 26 23r:   c                    t        j                  g dt        d      dg d      } t        j                  | d      }t        j                  g dt        d	      d
      }|j                  |dd      }t        d |j                  D              sJ t        |j                  |dd      | j                  |dd             y )Nr   abdabdr   ru   ry   r   r!  r"  r#  r   r   rb   r   r  r1  r   r'   rk  c              3  *   K   | ]  }d |d   v  ywshuffler   Nr   r  s     r8   r`  z<test_cheap_inner_merge_with_pandas_object.<locals>.<genexpr>       6y!$6r  	r)   r*   r-   r+   re   r.   rr  r  rI   )r4   dar6   dcs       r8   )test_cheap_inner_merge_with_pandas_objectr$    s    
 tH~6>V	A 
q	)B
<d6l;<A	!	)B6bgg6666BHHQ3GH,aggaCWg.MNr:   flipc           	        t        j                  g dt        d      dg d      }t        j                  |d      }t        j                  g dt        d	      d
      }t        j                  |dd      }| r||fn||f}| r||fn||f}t        j
                  |ddd}t        d |j                  D              sJ t        |t        j
                  |ddd       y )Nr   r  r   r  r   r   rb   r   r  r1  r   Frc   r&  r   r'   rk  c              3  *   K   | ]  }d |d   v  ywr  r   r  s     r8   r`  z4test_cheap_single_partition_merge.<locals>.<genexpr>  r   r  r!  )r%  r4   rE   r6   rF   	pd_inputsinputsr  s           r8   !test_cheap_single_partition_merger+  
  s    
 tH~6>V	A 
q	)B
<d6l;<A	qu	5BAQFIb"XB8F	6cw	/B6bgg6666B)9:r:   c                    t        j                  g dt        d      dg d      } t        j                  | d      }t        j                  g dt        d	      d
      }t        j                  |dd      }|j                  |dd      }|j                  rJ t        |       |j                  |dd      }|j                  rJ t        |       y )Nr   r  r   r  r   r   rb   r   r  r1  r   Fr'  r   r'   rk  )r)   r*   r-   r+   re   r.   known_divisionsr   )r4   rE   r6   rF   r  s        r8   +test_cheap_single_partition_merge_divisionsr.    s    
 tH~6>V	A 
q	)B
<d6l;<A	qu	5BXXbSgX.F%%%%VXXbSgX.F%%%%Vr:   r"   c                6   t        j                  t        d      t        d      dt        d            }t	        j
                  |d      }t        j                  g dt        d      dt        d	            }t	        j
                  |d      }|r||fn||f}|r||fn||f}t	        j                  |d
d| d}t        j                  |d
d| d}	t        ||	       t	        j                  |dd
| d}t        j                  |dd
| d}	t        ||	       y )Nr    ababbddar1  r   r   rb   r   r  r   Tr   r   r   r$   rq  )r)   r*   rd   r-   r+   re   r.   r   )
r$   r%  r4   rE   r6   rF   r)  r*  r  r  s
             r8   +test_cheap_single_parition_merge_left_rightr2  /  s     	58$z*:;58LA	q	)B
<d6l;58LA	q	)BAQFIb"XB8FXXv$#FFxxtcsKHfhXXvs#FFxxCTsKHfhr:   c                    t        j                  g dt        d      dg d      } t        j                  | d      }t        j                  g dt        d	      d
      }t        j                  |dd      }|j                  |ddd      }| j                  |ddd      }|j                  j                  d      |_        |j                  rJ t        ||       |j                  |ddd      }|j                  | ddd      }|j                  j                  d      |_        |j                  rJ t        ||       y )Nr   r  r   r  r   r   rb   r   r  r1  r   Fr'  Tr   r'   r1  rH  )r   r   r$   )
r)   r*   r-   r+   re   r.   r   rM  r-  r   )r4   rE   r6   rF   r  r  s         r8   *test_cheap_single_partition_merge_on_indexr4  F  s   
 tH~6>V	A 
q	)B
<d6l;<A	qu	5BXXbTCWXEFwwqTCWwEH
 ^^**73HN%%%%fhXXbdCWXEFwwqdCWwEH^^**73HN%%%%fhr:   c                 r   t        j                  g dt        d      dddt        d            } t        j                  dd	gd
dt        d      dt        d            }t        j                  | d      }t        j
                  ||d      j                         }t        |j                        dk(  sJ y )Nr   abcr  r  )r3   r5   r\   DDCBAr   r   r   g      @barab)GHIr5   GHIBr   rb   r5   r   )r7  r\   r5   r3   r;  r<  r=  )	r)   r*   r-   r+   re   r.   r2   tupler@   )lhsrhsr  r|  s       r8   test_merge_maintains_columnsrB  b  s    
,,d5kC@$v,C ,,!f3Ud<d6lC ..!
,CXXc33'//1F $GGGGr:   c                <   t        j                  dg dig d      }t        j                  dg dig d      }t        j                  |dd      }t        j                  |d	
      }|j	                  |d|       }|j	                  |d      }t        ||       y )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   Fr'  r   rb   r'   rp  r#   )r)   r*   r+   re   r1   r   )rj  r4   r6   rE   rF   r   r  s          r8   "test_merge_index_without_divisionsrF  n  s~    
c?+?CA
c?+?CA	qu	5B	q	)BWWRW^WDFvvaWv%Hfhr:   c                 ,   t        j                  dt        j                  j	                  dd      i      } t        j                  dt        j                  j	                  dd      it        j                  j	                  dd            }t        j                  | d      }t        j                  |d      }t        j                  | |d	d
      }t        j                  ||d	dd      }t        ||       t        |j                        dk  sJ y )Nr   r   i  sizer   r   rb   r   Tr   r   r  )r   r   rj  i  )r)   r*   ra  r  randintr+   re   r.   rI   rA  r  )r4   r6   rE   rF   r7   r  s         r8   *test_half_indexed_dataframe_avoids_shufflerL  z  s    
c299,,St,<=>A
	bii#./ryy7H7HSV7H7W	A 
s	+B	q	)B
A$5A	"bTC	PBArNrww<#r:   c                 .   t        j                  dg dig d      } t        j                  dg dig d      }t        j                  | dd      }t        j                  |d	
      }t	        j
                  t              5  t        j                  ||d|j                         d d d        t	        j
                  t              5  t        j                  |||j                  |j                         d d d        y # 1 sw Y   YxY w# 1 sw Y   y xY w)Nr   rD  r   r   rE  r   Fr'  r   rb   r  )
r)   r*   r+   re   r   r   NotImplementedErrorr.   r   r   r4   r6   rE   rF   s       r8   &test_errors_for_merge_on_frame_columnsrP    s    
c?+?CA
c?+?CA	qu	5B	q	)B	*	+ 5
Rrtt45 
*	+ 6
R56 65 56 6s   $C?.D?DDc                 2   t        j                  g d      } t        j                  | dd      }t        j                  |gd      }t        |t        j                        sJ t        j                  |gd      }t        |t        j                        sJ y )Nr   r   Fr'  r   r%  r   )r)   Seriesr+   re   r
   r>   r*   )r4   rE   r7   s      r8   test_concat_one_seriesrT    sn    
		,A	qu	5B
		2$QAa###
		2$QAa&&&r:   c                 P   t        j                  g d      } t        j                  g d      }t        j                  | dd      j	                         }t        j                  |dd      j	                         }|j
                  rJ t        j                  t              5  t        t        j                  | |gd      t        j                  ||gd             d d d        t        j                  |dd      }t        j                  t              5  t        j                  ||gd      j                          d d d        t        j                  d	      5 }t        j                  ||gdd
       d d d        rJ y # 1 sw Y   xY w# 1 sw Y   SxY w# 1 sw Y   &xY w)Nr   r   r   r   r   r   Fr'  r   rR  TrY  )r%  ignore_unknown_divisions)r)   rS  r+   re   r   r-  r   warnsUserWarningr   r
   r   r   optimizer)  r*  )r4   r6   rE   rF   r  rZ  s         r8   test_concat_unknown_divisionsr[    sI   
		,A
		,A	qu	5	E	E	GB	qu	5	E	E	GB!!!!	k	" J"))QF+RYYBxa-HIJ 
qu	5B	z	" /
		2r(#,,./ 
	 	 	- C
		2r(TBC:vJ J/ /C Cs$   ";F(FFFFF%c                    t        j                  g d      } t        j                  g d      }t        j                  | dd      j	                         }t        j                  |dd      j	                         }t        j                  t              5  t        j                  t              5  t        j                  ||gd      j                          d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   rV  r   Fr'  r   rR  )r)   rS  r+   re   r   r   r   r   rX  rY  r
   r2   rO  s       r8   $test_concat_unknown_divisions_errorsr]    s    
		$%A
		,A	qu	5	E	E	GB	qu	5	E	E	GB	z	" 2\\+& 	2IIr2hQ'//1	22 2	2 	22 2s$   C3.(C'C3'C0	,C33C<c                    t        j                  t        j                  j	                  dd      t        d      t        d            } t        j                  t        j                  j	                  dd      t        d      t        d            }t        j                  t        j                  j	                  dd      t        d      t        d	            }t        j                  | d
      }t        j                  |d      }t        j                  |d
      }t        j                  | |gd      }t        j                  ||g      }|j                  |j                  d d |j                  z   k(  sJ |j                  |j                  |j                  z   k(  sJ t        ||       t        t        j                  ||gd      t        j                  | |g             t        j                  | ||gd      }t        j                  |||g      }|j                  |j                  d d |j                  d d z   |j                  z   k(  sJ |j                  |j                  |j                  z   |j                  z   k(  sJ t        ||       t        t        j                  |||gd      t        j                  | ||g             y )Nr   r   ABCDEr   r@   r   ABCFGghijklABCHImnopqrr   r   Fr&  r%   Tinterleave_partitions)r)   r*   ra  r  r  r-   r+   re   r
   r/   rc   r   )r4  r5  r6  rk   rl   rm   r  r   s           r8   test_concat3rh    sD   <<
		1tG}DND <<
		1tG}DND <<
		1tG}DND >>$"D>>$"D>>$"Dyy$E2HYYd|$Ft~~cr2T^^CCCC!1!1D4D4D!DDDDfh
		4,d;RYYd|=T yy$d+%8HYYdD)*FsdnnSb11DNNB   4+++d.>.>>   fh
		4t$DA
		4t$%r:   c            	     Z   t        j                  t        j                  j	                  dd      t        d      t        d            } t        j                  t        j                  j	                  dd      t        d      t        d            }t        j                  t        j                  j	                  dd      t        d	      t        d            }t        j                  | d
      }t        j                  |d      }t        j                  |d
      }||g||g||g||g||g||g||gg}|D ]  }|D cg c]  }|j                          }	}t        t        j                  |d      t        j                  |	d             t        t        j                  |dd      t        j                  |	dd              d}
t        j                  t              5 }t        j                  ||gdd       d d d        |
t        j                        v sJ y c c}w # 1 sw Y   (xY w)Nru   r   r_  r  r`  rv   fghijklmnopqrr   CDEXYZr   r   Trf  Fre  r'   r1   rg  )r1   r&  z!'join' must be 'inner' or 'outer'invalid)r)   r*   ra  r  r  r-   r+   re   r2   r   r
   r   r   r   r]  value)r4  r5  r6  rk   rl   rm   casescaser7   pdcasemsgerrs               r8   "test_concat4_interleave_partitionsrt    s   <<
		AWT,=OD <<
		AWT/=RD <<
		AXd?>SD >>$"D>>$"D>>$"D 
t	t	t	t	t	t	tE  	
'+,!!))+,,IId$76PU9V	
 	IIdEIIf77	
	
 .C	z	" Lc
		4,YdKL#cii.    -L Ls   ;HH!!H*c                 D
   t        j                  t        j                  j	                  dd      t        d      t        d            } t        j                  t        j                  j	                  dd      t        d      t        d            }t        j                  t        j                  j	                  dd      t        d      t        d            }t        j                  t        j                  j	                  dd      t        d	      t        d            }t        j                  t        j                  j	                  dd      t        d	      t        d
            }t        j                  | d      }t        j                  |d      }t        j                  |d      }t        j                  |d      }t        j                  |d      }	||g||g||g||	g||g||	g|	||g|	|g|j                  |j                  g|j                  |j                  g|j                  |	j                  g|j                  |j                  g|j                  |j                  g|j                  |	j                  g||j                  g|j                  |g|	|j                  |gg}
|
D ]  }|D cg c]  }|j                          }}t        t        j                  |d      t        j                  |d             t        t        j                  |dd      t        j                  |d             t        t        j                  |d      t        j                  |d             t        t        j                  |dd      t        j                  |dd              ||g||g| |g| j                  |j                  g|j                  |j                  g||j                  g|j                  |g|| |j                  gg}
|
D ]  }ddlm} |D cg c]   }t#        ||      r|j                         n|" }}t        t        j                  |d      t        j                  |             t        t        j                  |dd      t        j                  |d             t        t        j                  |d      t        j                  |d             t        t        j                  |dd      t        j                  |dd              y c c}w c c}w )Nr   r   r_  r  r`  r   FGHIJKr  FGHABfklmnopr   r   Trf  Fre  r'   rl  r(  r   rR  )r%  r1   r   )	FrameBase)r)   r*   ra  r  r  r-   r+   re   r3   Fr<  r5   r2   r   r
   $dask.dataframe.dask_expr._collectionry  r>   )r4  r5  r6  pdf4pdf5rk   rl   rm   ddf4ddf5ro  rp  r7   rq  ry  s                  r8   test_concat5r    s   <<
		1tG}DOD <<
		1tH~T)_D <<
		1tH~T)_D <<
		1tG}DOD <<
		1tG}DOD >>$"D>>$"D>>$"D>>$"D>>$"D 
t	t	t	t	t	t	tT	t							tvv		tvvt#E(  
'+,!!))+,,IId$7IIf5)	

 	IIdEIIf7+	

 	"))Dq)299V!+DEIId1IIf173	

, 
t	t	t			tvv		tTVV	E  
BJNOQAy!9!))+q@OO"))D=ryy?PQIIdEIIf7+	

 	"))Dq)299V!+DEIId1IIf173	

? -D Ps   -T%Tzknown, cat_index, divisions)TTFz3.12.0z>fails on pandas dev: https://github.com/dask/dask/issues/10558)r  r   strict)TFF)FTF)FFFc                    t        j                  t        d      t        j                  d      t        d      t        j                  dd      d      t        j                  t        d      t        j                  dd      t        d	      t        j                  ddd      d      t        j                  t        d
      t        j                  dd      t        d      t        j                  ddd      d      g}|D ]B  }|j
                  j                  d      |_        |j                  j                  d      |_        D r(|D cg c]  }|j                  |j                         }}|D cg c]  }t        j                  |d       }} sW|d   d   j                  j                         |d   d<   r/|d   j                  j                  j                         |d   _         fd}dD ])  } ||||      }	t        |	j
                        sJ t        |	j                         k(  sJ  ||D 
cg c]  }
|
j                   c}
|D 
cg c]  }
|
j                   c}
|      }	t        |	       k(  sJ r: ||D 
cg c]  }
|
j                   c}
|D 
cg c]  }
|
j                   c}
|      }	 ||d   ddg   g|dd  z   |d   ddg   g|dd  z   |      }	t!        |	d      rt        |	j
                        sJ t        |	j                         k(  r*J  y c c}w c c}w c c}
w c c}
w c c}
w c c}
w )Nxxxxxr   abcbcf8rI  )wr   r   r2  yyyyyru   abbbazzzzzr   bcbcccategoryr   r'  r   r   c                   t        ||      }t        j                   | |	      }t        ||       
rkt        t        j                  |j
                  |j                               }|D cg c]  }|j                  d d  c}D ]  }t        |j                  |        rt        |j                        
k(  sJ |S c c}w )Nr(  rl  r   )r
   r+   r   r   r*   r  __dask_keys__r   r   rL  r   r   )ddfsdfsr1   solresr  ip	cat_indexr/   knowns           r8   check_and_returnz1test_concat_categorical.<locals>.check_and_return  s    St$ii4yI#s,chh(9(9(;E +00QaffRaj0 )399a() 4SYY ?5 HHH
 1s   ,B>)r'   r(   r   r   r  )r)   r*   r-   ra  rb  r  rM  r   r  r+   re   cat
as_unknownr   r   r   hasattr)r  r  r/   framesr  r  dframesr  r1   r  r  s   ```        r8   test_concat_categoricalr  s  s   B 	']YYq\']YYq-		
 	']YYq"%']YYq"D1		
 	']YYr2&']YYr2T2		
#F4  'tt{{:&tt{{:&' /56",,rtt$66IOPAr~~aQY?PGP!!*S/--88:
3&qz//33>>@GAJ # 4w5#CEE***#CEE*e333 W55V7L7LdS#C(E111 "#9AACC#9;PAACC;PRVWC QZc
#$wqr{2AYSz"#fQRj0

 3$(<SUU(CCC#CEE*e333+4- 7P6  67L
 $:;Ps$   "L)8L. L3
L8L=
%Mc                    t        j                  g dd      } t        j                  ddgd      }t        j                  | d      j                  j                         j                  d      }t        j                  |d      j                  d      }t        | j                  d      |j                  d      g      }t        j                  ||g      }t        ||       y )Nr   r  rI  r4   r6   r   r3   )	r)   rS  r+   re   r  r  to_framer
   r   )r4   r6   r"  dbr  r   s         r8   $test_concat_categorical_mixed_simpler    s    
		/4A
		3*J/A	1		!	!	,	,	.	7	7	<B	1		&	&s	+Bqzz#

389HYYBx Ffhr:   c                    t        j                  ddgit        j                  dgd            } t        j                  ddgit        j                  dgd            }| j                  j	                  d	      j
                  j                  ddg      | d<   |j                  j	                  d	      j
                  j                  ddg      |d<   t        j                  | d
      }t        j                  |d
      }t        | j                  d d |j                  d d g      }|j                  j                  dk(  sJ t        j                  ||g      }t        j                  | |g      }t        ||       y )Nr   r4   z2015-03-24 00:00:16zdatetime64[ns]rI  r   r7   z2015-03-29 00:00:44r  r   r   zM8[ns])r)   r*   DatetimeIndexr   rM  r  set_categoriesr+   re   r
   r   r   rJ  r   )b2b3db2db3r   r  s         r8   test_concat_datetimeindexr    s=   		se 56>NO
B 
	se 56>NO
B
 ddkk*%))88#sDBsGddkk*%))88#sDBsG
..Q
C
..Q
CRWWRa["''"1+./F<<)))YYSz"Fyy"b"Hfhr:   c                    t        j                  t        d      5  |j                  |      }| j                  |      }t	        ||       d d d        |S # 1 sw Y   S xY w)Nzappend method is deprecatedr   )r   rX  r,  appendr   )dask_objdask_append
pandas_objpandas_appendr  r   s         r8   check_append_with_warningr    sW    	m+H	I $$$]3-&(#$
 M$
 Ms   /AAc                     t        j                  dg dig d      } t        j                  | d      }|j	                  d      }|j                  |d      }|j                  d	k(  sJ |j                          y )
Nr   )r   r   r   r   r   r   rb   r_  )ra   )r   r   )r)   r*   r+   re   r  r1   r/   r2   )r  r  rl   joineds       r8   test_singleton_divisionsr    se    	sI&i	8B
..
+C==DYYtSY)Fv%%%
NNr:   c                     t        j                  dg di      } t        j                  | d      j	                  d      }|j                  dd      }t        || j	                  d             y )Nr   r   r   r   r   r   rb   )r   r   T)r/   force)r)   r*   r+   re   r  r,   r   )r  r  rl   s      r8   #test_repartition_repeated_divisionsr    sU    	sL)	*B
..
+
5
5c
:C??V4?8DdBLL%&r:   c                    t        j                  dg di      } t        j                  ddgi      }t        j                  | d      j	                  d      }t        j                  |d      j	                  d      }|j
                  dk  sJ t        |      t        |       k(  sJ |j                  |ddd	      }| j	                  d      }|j	                  d      }|j                  |ddd	      }t        ||       y )
Nr   r  r   r   rb   r   r"   Tr  )	r)   r*   r+   re   r  rc   rA  r.   r   )rf   rg   rk   rl   r1sf1sf2r2s           r8   test_multi_duplicate_divisionsr    s    
,,\*
+C
,,aSz
"C>>#1-77<D>>#1-77<Dq   t9C   	Df4	HB
--
C
--
C	3Ft	FBb"r:   c            	     r   d} t        j                  g dg dd      }t        j                  |d      }t        j                  t        j                         d      }t	        d| dz         D ]G  }t        t        j                  |||j                  |k(     d	
      ||j                  |k(     dd       I y )Nr   )r3   r5   r\   r7  Erz  )r   r   r   r   r   r   )userclusterru   rb   r   r   r(   r#   F)r   check_divisions)r)   r*   r+   re   rd   r   r.   r  )
k_clustersr  r  r   s       r8   test_merge_outer_emptyr  1  s    J	/<NO
B 
	+B~~blln"=H1j1n% 
HHXr"**/2@rzzQ!		

r:   c            	        t        j                  dt        j                  ddgt        j                  t        j
                              i      } t        j                  dt        j                  ddgt        j                  t        j                              i      }t        j                  d      5 }t        j                  j                  | |dd       d d d        rJ y # 1 sw Y   xY w)Nr4   r   r   rI  TrY  )r)   r*   ra  arrayrJ  rH  longlongr)  r*  r+   multiwarn_dtype_mismatch)rf   rg   rZ  s      r8   test_dtype_equality_warningr  C  s    
,,RXXq!fBHHRXX4FGH
IC
,,RXXq!fBHHR[[4IJK
LC		 	 	- 9
$$S#sC89:v9 9s   5#C%%C.c                .   d}d}t        j                  t        j                  j	                  t        j
                  |            t        j                  j                  d|      d      }t        j                  t        j                  j	                  t        j
                  |            t        j                  j                  d|      d      }| dk(  rqt        j                  d      }t        j                  d      }|j                  |      }|j                  |      }|j                  ||      }|j                  ||      }n,t        j                  ||      }t        j                  ||      }|j                  d	g      j                         }	|j                  d
g      j                         }
t        |	|
gd      }|j                  d	g      j                         }|j                  d
g      j                         }t        j                          5  t        j"                  dt$               t        j                  ||gd      }d d d        t'        j)                         j+                         |j+                                y # 1 sw Y   @xY w)Nr   r   r   rH  r  r  r  r  r4   r7   r   rR  r'  )r)   r*   ra  r  permutationrb  rK  r   r  re   r  r+   groupbysumr
   r)  r*  r+  rY  r   r2   
sort_index)r  rI  rc   d1d2r  r  r8  r9  
grouped_d1
grouped_d2r  grouped_dd1grouped_dd2res_dds                  r8   test_groupby_concat_cudfr  M  s    DK	&&ryy7""3T"2	

B 
&&ryy7""3T"2	

B  ""6*''4	b!b!!!"k2!!"k2nnR-nnR-SE"&&(JSE"&&(J
*j)
2C++se$((*K++se$((*K		 	 	" ?h4K5A>? fnn))+S^^-=>	? ?s   4JJorderedc                   t        j                  dt        j                  g dg d|       i      }t        j                  |d      }t        j                  dt        j                  g dg d|       i      }t        j                  |d      }t        j
                  ||g      }|d   j                  d      |d<   t        j
                  ||gd	      }t        ||       y )
Nr   )r4   r6   r7   r4   r   )
categoriesr  r   )r7   r6   r4   r  T)ignore_order)r)   r*   Categoricalr+   re   r
   rM  r   )r  r4  rk   r5  rl   r  r   s          r8   test_concat_ignore_orderr  ~  s    <<$'	
D >>$"D<<OW	
D >>$"Dyy$&HSM((4HSMYYd|$7Ffhr:   rJ  Int64Float64zint64[pyarrow]z:Support for ArrowDtypes requires pyarrow and pandas>=1.5.0zfloat64[pyarrow]c                8   t        j                  g dg dt        d      d      }t        j                  g dg dt        d      d      }|d   j                  |       |d<   |d   j                  |       |d<   t	        j
                  |d      }t	        j
                  |d      }|j                  |d	      }|j                  |d	      }t        ||d
       t        j                  ||d	      }t	        j                  ||d	      }t        ||d
       y )Nr   )r   r   r   aabr   )r4   r  r  r4   r   rb   r   Fr   )r)   r*   r-   rM  r+   re   r.   r   )rJ  rf   rg   rk   rl   r  r  s          r8   test_nullable_types_merger    s    , ,,YYT%[I
JC
,,YYT%[I
JC3xu%CH3xu%CH>>#1-D>>#1-DYYssY#FZZZ%Fff%0XXc33'FXXdDS)Fff%0r:   c                 p   t        j                  g dg dd      } t        j                  dg dig d      }t        j                  | d      }t        j                  |d	      }|d
   j	                  d      |d
<   |j
                  j	                  d      |_        |j                         j                  |j                         d
d      }|j                  |d
d      }|j                  j                  dk(  sJ |j                         }|j                  j                  dk(  sJ t        ||      sJ y )N)r4   r4   r6   r6   )r   r   ru   ru   )join_colr4   r6   r  r   r   rb   r   r  r  r"   rk  )r)   r*   r+   re   rM  r   r2   r1   r  rJ  r   )r  rg   r  rl   r  actual_daskr  s          r8   test_categorical_joinr    s   	,	

B ,,\*2F
GC
..
+C>>#1-D*o,,Z8C
O"":.DJ{{}!!$,,.ZV!LH((4JF(;K%%333  "F??  J...Xv&&&r:   c                    t        j                  ddgt        j                  ddg      d      } t        j                  dt        j                  ddg      i      }t        j                  || dd	      }t	        j
                  | d
      }t	        j
                  |d
      }t	        j                  ||dd	      j                         }|j                  j                  dk(  sJ |j                  j                  dk(  sJ |j                  j                  dk(  sJ |j                  j                  dk(  sJ t        ||      sJ y Nr   r   r4   r6   rX  r\   Tr3   rJ  r   rb   r  rH  r)   r*   r  r.   r+   re   r2   r\   rJ  r5   r3   r   r   rf   rg   r  rk   rl   r  s         r8   5test_categorical_merge_with_columns_missing_from_leftr    s   
,,aV"..#s*DE
FC
,,R^^S#J78
9CxxSTC@H>>#1-D>>#1-DXXdDTC@HHJF88>>Z'''88>>Z'''88>>W$$$<<(((Xv&&&r:   c                    t        j                  t        j                  ddg      t        j                  ddg      d      } t        j                  dt        j                  ddg      i      }t        j                  || dd	      }t	        j
                  | d
      }t	        j
                  |d
      }t	        j                  ||dd	      j                         }|j                  j                  dk(  sJ |j                  j                  dk(  sJ |j                  j                  dk(  sJ |j                  j                  dk(  sJ t        ||      sJ y r  r  r  s         r8   Itest_categorical_merge_with_merge_column_cat_in_one_and_not_other_upcastsr    s
   
,,R^^QF3"..#s:TU
VC
,,R^^S#J78
9CxxSTC@H>>#1-D>>#1-DXXdDTC@HHJF88>>Z'''88>>Z'''88>>W$$$<<(((Xv&&&r:   c                    t        j                  g dg dd      } t        j                  g dg dd      }t        j                  | d      }|j                  j                  d      |d	<   t        j                  |d      }|j                  j                  d      |d	<   |j                  |d	
      }|j                  j                  dk(  sJ |j                         }|j                  j                  dk(  sJ y )Nr   r   r   r   )r   r   r   r   rX  )r   r   r   r   )r   r   r   r   )r3   r\   r   r  r3   r   )	r)   r*   r+   re   r3   rM  r.   rJ  r2   )r4   r6   rf   rg   r  r  s         r8   -test_categorical_merge_retains_category_dtyper    s    
<l;<A
<l;<A
..A
Cuu||J'CH
..A
Cuu||J'CH))CC)(K==*,,,  "F88>>Z'''r:   c                 $   t        j                  dg dig d      } t        j                  dg dig d      }t        j                  | d	      }|j	                  dd
i      }|j                  |      }t        | j                  |      |       y )Nr3   r   )svr  )datar   r5   )tr   r  )r  r  r_  r   rb   r  )r)   r*   r+   re   rM  r1   r   )rf   rg   rk   qs       r8   ?test_categorical_merge_does_not_raise_setting_with_copy_warningr  
	  sn    
,,S/2/
JC
,,S/2/
JC>>#1-D
**c:&
'C		#AchhsmQr:   )r'   r"   r&   rc   r|       baselgsmc           	        d}d}d}d}d}t        j                  t        j                  j	                  t        j
                  d      |      t        j
                  |      d      }t        j                  ||      }	t        j                  t        j                  j	                  t        j
                  d      |      t        j
                  |      d      }
t        j                  |
|      }|d	k(  r	|}|	}|
}|}n|
}|}|}|	}t        j                  ||d
| ||d      }t        j                  ||d
|       }|d
   j                  t        j                        |d
<   |d
   j                  t        j                        |d
<   |r|j                  |k(  sJ t        |j                         j                  d
      |j                  d
      d       y )Ni  rG  r   r   r  r   r   rb   r  r   r  )r   r$   rc   	broadcastrj  rk  Fr   )r)   r*   ra  r  choicerb  r+   re   r.   rM  int32rc   r   r2   rA   )r$   rc   r  size_lgsize_smnpartitions_lgnpartitions_smbroadcast_biasr  ddf_lgr  ddf_smr"   ddf_leftr&   	ddf_right	dd_result	pd_results                     r8   test_merge_tasks_large_to_smallr
  	  s    GGNNN	!!"))C.':7#	

B ^^BN;F	!!"))C.':7#	

B ^^BN;Ft|		 I u#6I s^**2884IcNs^**2884IcN$$333'',c"r:   c                   t        j                  t        j                  g dg ddg d      d      }t        j                  t        j                  g dg ddg d      d      t        j                  t        j                  g d	g d
dg d      d      g}t	        j
                  t              5 }|j                  ||        d d d        j                  d       y # 1 sw Y   xY w)Nr   )r   r   r   r  r   r   r   r   r   )r   r   r   )r   r   r    )r   r    rt   )r   r   r   )rt   rt   rt   r#   z-merge_multi only supports left or outer joins)	r+   re   r)   r*   r   r   r   r1   r   )r$   base_dfr  r  s       r8   ,test_pairwise_rejects_unsupported_join_typesr  T	  s    nn
995YGG 	LLyy9KQ	
 	LLyy9KQ	
	C 
z	" #aSc"#GG;<# #s   6C$$C-npartitions_baser   npartitions_otherc                   g }t        d      D ]P  }t        j                  | dg d| dg dig d      }t        j                  ||      }|j                  |       R t        j                  t        j                  g d      |      }|D ]  }|j                  || 	      } t        j                  t        j                  g d      |      }|j                  || 	      }t        ||       y )
Nru   r3   ru  r5   rV  r  r   r  r#   )rd   r)   r*   r+   re   r  r1   r   )	r$   r  r  dfs_to_merger  r  r  ddf_loopddf_pairwises	            r8   2test_pairwise_merge_results_in_identical_output_dfr  g	  s     L2Y 	!\\#Q#Q 
 nnR!23C 	! ~~bll;=MNH /==#=./ >>",,Y"?AQRL$$\s$;LlH%r:   npartitions_leftnpartitions_right)r"   r&   r(   c                   t        j                  t        j                  g dg ddg d      |      }t        j                  t        j                  g d      |      }|j	                  ||       }|j
                  j                  |j                  k(  sJ |j                  t        |j                        dz
  k(  sJ |j                  |j                         j                  k(  sJ t        |      t        |j                               k(  sJ t        |      t        |j                  t              j                               k(  sJ y )	Nru  rV  rX  r  r   r  r#   r   )r+   re   r)   r*   r1   expr_npartitionsrc   rA  r/   rZ  r2   r  ry  )r$   r  r  r  r  r  s         r8   $test_ensure_npartitions_properly_setr  	  s     
\2	
 	I ~~bll;=MNH
--	s-
+C88  COO333??c#--014444??clln88888s8s3;;=))))s8s3--c2::<====r:   )m
__future__r   r)  numpyra  r  r)   r   packaging.versionr   dask.dataframe	dataframer+   dask._compatibilityr   	dask.baser   dask.dataframe._compatr   r   r	   dask.dataframe.methodsr
   dask.dataframe.utilsr   r   r   r   pyarrowr  ImportErrorr9   rI   rp   r   r   r   r   markparametrizer   r   r   r   r   r   r   r
  r  r  r-  r<  r>  rD  rU  rh  rs  r}  r  paramxfailgpur  r  slowr  skipifr  r  r  r  r  r$  r+  r.  r2  r4  rB  rF  rL  rP  rT  r[  r]  rh  rt  r  AssertionErrorr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r   r:   r8   <module>r0     s   "     %  * . C C ) 
380*,*$	'NV  .u>&HI' J ?'  .u>&HI,udm<< = J ?<~7't7'tE'P	17h 68RS77.	$ '7!34$ 5$" '7!34$ 5$6 '7!34$ 5$6 (B2  CDdE]3 1 4 E 1F  CD` E`N ' 23D 4D&+( ":;	++##+J#K	
	 ++##E $ 	
 	V6;;??3#1	 <.#1L	5  CDN E Nb M*EF CD| E  G|~   CD_ E _D 
U9TuenI&t%44*O %/; 0;"&  12%/  0 3 * 8	H	 "6'(2%P&!R\
~ !++##@wx/@!@W%	 $ 	
	
 	++##@wx/@!@W%	 $ 	
	
 	5>K4?>K4\  2'$
$ xfFKKOODE+?+?\ T5M2  3 . ++$$d
S % 	
 	++$$d
S % 	
*1+*1$'2'"'"($	   :;R1$.8 / 2 <8v ' 23= 4=$  12+Y7,i8& 9 8 3&4 +Y7,i8 :;> < 9 8>_K  	Bs   ` ``