
    bi                       d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlZ ej                  d      Z ej                  d      Zd dlZd dlmZ d dlmZ d d	lmZmZ d dlZd d
lmZ d dlmZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z# d dl$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1m2Z2m3Z3 eD  cg c]  } |  c} dgz   Z4d Z5d Z6djo                         Z8e8js                  dd      Z:djo                         Z;djo                         Z<ddddZ=e=j}                         D ci c]  \  }}||js                  dd       c}}Z?dd d!dZ@e=fd"ZAd# ZBd$ZCd%ZDd&ZEeEjs                  dd      ZFej                  j                  d'ej                  e=fej                  e?fej                  e@fg      ZLeLd(        ZMeLd)        ZNeLd*        ZOeLd+        ZPeLd,        ZQeLd-        ZReLd.        ZSej                  j                  d/ej                  ej                  e=fej                  ej                  e?fg      d0        ZTej                  j                  d/ej                  ej                  e=fej                  ej                  e?fg      d1        ZUej                  j                  d/ej                  ej                  e=fej                  ej                  e?fg      d2        ZVej                  j                  d3ej                  ej                  e=eEfej                  ej                  e?eFfg      d4        ZWd5d6gd5d6ggZXd7d8gd7d8ggZYej                  j                  d9ej                  eXfej                  eYfg      d:        ZZej                  j                  d9ej                  eXfej                  eYfg      d;        Z[ej                  j                  d<ej                  ej                  e8dfej                  ej                  e:dfej                  ej                  e;d=fg      d>        Z\d? Z]ej                  j                  d@ej                  ej                  e8dAfej                  ej                  e:dBdCgfg      dD        Z^ej                  j                  d@ej                  ej                  e8dAfej                  ej                  e:dBdEgfg      dF        Z_ej                  j                  d/ej                  ej                  e=fej                  ej                  e?fg      dG        Z`ej                  j                  d/ej                  ej                  e=fej                  ej                  e?fg      dH        Zaej                  j                  dIej                  e=fej                  e?fg      dJ        Zbej                  j                  dIej                  e=fej                  e?fg      dK        Zcej                  j                  dIej                  e=fej                  e?fg      dL        Zdej                  j                  dIej                  e=fej                  e?fg      dM        Zeej                  j                  dIej                  e=fej                  e?fg      dN        ZfdO ZgdP ZhdQ ZidR ZjdS ZkdT ZldU ZmdV ZndW Zoej                  j                  ej                  j                  dXdYdZg      d[               Zqd\ Zrd] Zsd^ Ztej                  j                  d_dd`g      ej                  j                  dae4      db               Zuej                  j                  dc        Zwdd Zxej                  j                  deg df      dg        Zydh Zzdi Z{dj Z|dk Z}dl Z~dm Zdn Zdo Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zej                  j                  dxg dy      dz        Zd{ Zd| Zd} Zd~ Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zej                  j=                  d      d        Zej                  jA                   eejB                         ed      k(  d      d        Zd Zd Zd Zd Zd Zd Zd Zd Zej                  j                  dddg      d        Zej                  j                  dddddddgdddfddgdddfg      d        Zd Zej                  j                  ddBdgdgg dgg g g dBgg dBgg ed`      D cg c]  } e e|             c}g      d        Zd Zd Zd Zd Zd Zd Zej                  j                  dddg      d        Zyc c} w c c}}w c c}w )    )annotationsN)BytesIOStringIO)mockpandaszdask.dataframe)compr)Version)partition_allvalmap)compute_as_if_collection)
read_bytes)compress)flatten)PANDAS_GE_220PANDAS_GE_300tm)_infer_block_sizeauto_blocksize
block_maskpandas_read_texttext_blocks_to_pandas)	assert_eqget_string_dtypehas_known_categoriespyarrow_strings_enabled)filetext	filetextstmpdirtmpfilec                    dj                  t        t        j                  | j                         j	                  d                  S )N
)joinmapstrstripsplit)ss    [/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/dataframe/io/tests/test_csv.pynormalize_textr)   -   s+    99SAGGIOOD$9:;;    c                F    t         j                  j                  |       d   S )N   )ospathr&   )r.   s    r(   parse_filenamer/   1   s    77==q!!r*   z
name,amount
Alice,100
Bob,-200
Charlie,300
Dennis,400
Edith,-500
Frank,600
Alice,200
Frank,-200
Bob,600
Alice,400
Frank,200
Alice,300
Edith,600
,	z
name   amount
Alice    100
Bob     -200
Charlie  300
Dennis   400
Edith   -500
Frank    600
Alice    200
Frank   -200
Bob      600
Alice    400
Frank    200
Alice    300
Edith    600
a  
Date,Open,High,Low,Close,Volume,Adj Close
2015-08-28,198.50,199.839996,197.919998,199.240005,143298900,199.240005
2015-08-27,197.020004,199.419998,195.210007,199.160004,266244700,199.160004
2015-08-26,192.080002,194.789993,188.369995,194.679993,328058100,194.679993
2015-08-25,195.429993,195.449997,186.919998,187.229996,353966700,187.229996
2015-08-24,197.630005,197.630005,182.399994,189.550003,478672400,189.550003
2015-08-21,201.729996,203.940002,197.520004,197.630005,328271500,197.630005
2015-08-20,206.509995,208.289993,203.899994,204.009995,185865600,204.009995
2015-08-19,209.089996,210.009995,207.350006,208.279999,167316300,208.279999
2015-08-18,210.259995,210.679993,209.699997,209.929993,70043800,209.929993
s3   name,amount,id
Alice,100,1
Bob,200,2
Charlie,300,3
s   name,amount,id
s4   name,amount,id
Dennis,400,4
Edith,500,5
Frank,600,6
)2014-01-01.csv2014-01-02.csv2014-01-03.csv   ,   	sT       name  amount  id
   Alice     100   1
     Bob     200   2
 Charlie     300   3
s       name  amount  id
sT       name  amount  id
  Dennis     400   4
   Edith     500   5
   Frank     600   6
c                    t         j                  t        |       D cg c]'  }t         j                  t	        t
        |               ) c}      }|j                  t               t        t        d      }|S c c}w Nnameamountid)	pdconcatsortedread_csvr   	csv_filesastyper   int)
file_nameskdfs      r(   
read_filesrG      sV    	z@RS1BKK	! 56S	TB	,.#SI	JBI Ts   ,A7c           
         t         j                  t        |       D cg c]  } ||fi | c}      }|j                  t	               t
        t
        d      }|S c c}w r8   )r=   r>   r?   rB   r   rC   )rD   handlerkwargsnrF   s        r(   read_files_withrL      sN    	&2DEQGA((E	FB	,.#SI	JBI Fs   AsL   # some header lines
# that may be present
# in a data file
# before any datasB   # some footer lines
# that may be present
# at the end of the files   str, int, int
zreader,filesc                    |d   }t        | |di       }t        |j                        g dk(  sJ t        |      dk(  sJ |j                  j                         dk(  sJ y )Nr2   r*   r9         )r   listcolumnslenr<   sumreaderfilesbrF   s       r(   test_pandas_read_textrX      s]    A	&!S"	-B

7777r7a<<5599;)###r*   c                j    |d   }t        | |ddddgi      }t        |j                        ddgk(  sJ y )Nr2   r*   usecolsr:   r<   )r   rP   rQ   rT   s       r(   test_pandas_read_text_kwargsr[      sA    A	&!S9vtn*E	FB

~---r*   c                f    |d   }t        | |di ddi      }|j                  j                  dk(  sJ y )Nr2   r*   r;   float)r   r;   dtyperT   s       r(   $test_pandas_read_text_dtype_coercionr_      s:    A	&!S"x.A	BB99??g%%%r*   c                    |d   }|j                  dd      \  }}|dz   }t        | ||i       }t        |j                        g dk(  sJ t	        |      dk(  sJ |j
                  j                         dk(  sJ y )Nr2      
r,   r9   rN   rO   )r&   r   rP   rQ   rR   r<   rS   )rU   rV   rW   headerrF   s        r(   !test_pandas_read_text_with_headerrc      sz    Aq!IFAe^F	&!VR	0B

7777r7a<<5599;)###r*   c                2   t        |      D cg c]  }||   g
 }}i }t        | |d   di       }|d   j                  d      d   dz   }t        | ||||      }t	        |t
        j                        sJ t        |j                        g dk(  sJ t        | ||||      }t	        |t
        j                        sJ t        |d      sJ t               rt        |j                        dk(  sJ 	 t        |j                  j                         d       y c c}w )	Nr2   r*   ra   r   r9   daskrO   i4  )r?   r   r&   r   
isinstancedd	DataFramerP   rQ   hasattrr   rR   re   r   r;   rS   )	rU   rV   rE   blocksrJ   headrb   rF   valuess	            r(   !test_text_blocks_to_pandas_simplerm      s   "(-0QuQxj0F0FFE*:$;S"ED#$**51!4u<F	vvvtV	DBb",,'''

7777"6664HFfbll+++66"""$;$=3v{{q DD1biimmo@A 1s   Dc                   t        |      D cg c]  }||   	 }}|D cg c]  }|g }}dddgi}t        | |d   d|      }|d   j                  d      d   dz   }t        | ||||      }t	        |j
                        ddgk(  sJ |j                         }	|	j
                  |j
                  k(  j                         sJ y c c}w c c}w )NrZ   r:   r<   r2   r*   ra   r   )r?   r   r&   r   rP   rQ   computeall)
rU   rV   rE   rj   rW   rJ   rk   rb   rF   results
             r(   !test_text_blocks_to_pandas_kwargsrr      s     &u.1eAh.F.!"aqc"F"&$(FFE*:$;S&ID#$**51!4u<F	vvvtV	DB

~---ZZ\FNNbjj(--/// /"s
   C
Cc           	        t               }|d   j                  d      d   dz   }g }t        |      D ]P  }||   }|j                  d      }|j                  t	        d|      D cg c]  }dj                  |       c}       R t        | |||j                         i       }	t        |	j                         j                  d      |j                  d      d       |d	d
g   }
t        | |||
j                         dd	d
gi      }	t        |	j                         j                  d      |
j                  d      d       y c c}w )Nr2   ra   r      TdropFcheck_dtyper:   r<   rZ   )rG   r&   r?   appendr
   r"   r   rk   r   ro   reset_index)rU   rV   expectedrb   rj   rE   rW   linesbsrF   	expected2s              r(   "test_text_blocks_to_pandas_blockedr      s<   |H#$**51!4u<FFE] J!Ha0GH"uzz"~HIJ
 
vvvx}}	KB


  d +$' &$(I		 09vtn2M
B 


  d +4( Is   $E
zdd_read,pd_read,filesc                4   |j                         D ci c]  \  }}|t        dz   |z    }}}t        t        j                               }t	        |d      5   | d|      }t        |||      }t        ||d       d d d        y c c}}w # 1 sw Y   y xY w)Nra   rW   mode2014-01-*.csvskiprowsFrw   )itemscomment_headerrR   
splitlinesr   rL   r   dd_readpd_readrV   r:   contentskiprF   expected_dfs           r(   test_skiprowsr     s    
 JOWgT>E)G33WEW~((*+D	5s	# 6_t4%eWtD"ku56 6 X6 6s   B'BBc                   |j                         D ci c]#  \  }}|t        dz   |j                  ddd      z   % }}}t        |d      5   | dd      }t	        ||d      }t        ||d	
       d d d        y c c}}w # 1 sw Y   y xY w)Nra   s   # just some comment
r,   rW   r   r   #)commentFrw   r   r   replacer   rL   r   )r   r   rV   r:   r   rF   r   s          r(   test_commentr     s     #[[]	 D' 	n

//%!91
=> 	>E  
5s	# 6_c2%eWcB"ku56 66 6s   (A<'BBc                8   |j                         D ci c]  \  }}||dz   t        z    }}}t        t        j                               }t	        |d      5   | d|d      }t        |||d      }t        ||d       d d d        y c c}}w # 1 sw Y   y xY w)	Nra   rW   r   r   python)
skipfooterengineFrw   )r   comment_footerrR   r   r   rL   r   r   s           r(   test_skipfooterr     s    
 JOWgT7U?^33WEW~((*+D	5s	# 6_hG%eWhW"ku56 6 X6 6s   B
)BBzdd_read,pd_read,files,unitsc                *   |j                         D ci c]&  \  }}|t        dz   |j                  dd|z   d      z   ( }}}g d}t        |d      5   | d|      }t	        |||      }t        ||d	       d d d        y c c}}w # 1 sw Y   y xY w)
Nra   r,   )r   r,   rt   rN      rW   r   r   r   Frw   r   )	r   r   rV   unitsr:   r   r   rF   r   s	            r(   test_skiprows_as_listr   ,  s     #[[]D' 	~%uu}a(PPPE  D	5s	# 6_t4%eWtD"ku56 6
6 6s   +B'B		Bs   aa,bb
1,1.0
2,2.0s   10,20
30,40s   aa	bb
1	1.0
2	2.0s   10	20
30	40zreader,blocksc                     | t        |d   d         d      |d   d   j                  d      d   dz   }t        | ||i       }t        j                  |d      }t        fd|D              sJ y )Nr   rb   ra   sync	schedulerc              3     K   | ]9  }|j                   j                         j                   j                         k(   ; y wN)dtypesto_dict).0rF   rk   s     r(   	<genexpr>z&test_enforce_dtypes.<locals>.<genexpr>R  s0     Jryy  "dkk&9&9&;;Js   ?A)r   r&   r   re   ro   rp   )rU   rj   rb   dfsrk   s       @r(   test_enforce_dtypesr   J  sv     '&)A,'2DAYq\&q)E1F
b
AC
,,sf
-CJcJJJJr*   c                f   |d   |d   d   j                  dd      |d   d   gg} | t        |d   d         d      }|d   d   j                  d      d   dz   }t        | |||i d      }t	        j
                  t              5  t        j                  |d	
       d d d        y # 1 sw Y   y xY w)Nr   r,      a   Ar   ra   Tenforcer   r   )	r   r   r&   r   pytestraises
ValueErrorre   ro   )rU   rj   rk   rb   r   s        r(   test_enforce_columnsr   U  s    
 Qi&)A,..tT:F1IaLIJF'&)A,'2DAYq\&q)E1F
b$
OC	z	" ,SF+, , ,s   B''B0zdd_read,pd_read,text,sepz\s+c                   t        |      5 } | |dt        j                  |      }t        |j                        ddgk(  sJ |j                  d      j                  d      }t        | |||	             d d d        y # 1 sw Y   y xY w)
N   )	blocksizelineterminatorsepr:   r;   r   r   Tru   r   )r   r-   lineseprP   rQ   ro   rz   r   )r   r   textr   fnfrq   s          r(   test_read_csvr   g  s     
$ 02B"RZZSIAII68"4444V,88d8C&'"#./0 0 0s   A-BBc                 |   t        j                  dd       t        t              5 } t        j                  |       }t        j                  j                  ddi      5  t        j                  |       }d d d        |j                  ddi      }t        |d	       d d d        y # 1 sw Y   3xY w# 1 sw Y   y xY w)
NpyarrowzRequires pyarrow stringsreasonzdataframe.convert-stringTr:   zstring[pyarrow]Fcheck_index)r   importorskipr   csv_textr=   r@   re   configsetrg   rB   r   )r   rF   ddf
df_pyarrows       r(   #test_read_csv_convert_string_configr   x  s    
	*DE	(	 6r[[_[[__8$?@ 	"++b/C	"YY(9:;
*cu56 6	" 	"6 6s#   7B2B&4)B2&B/	+B22B;zdd_read,pd_read,text,skip   r,      c           	         ddg}t        |      5 } | |||      }t        | ||||             d d d        y # 1 sw Y   y xY w)Nr:   r;   r   names)r   r   r   r   r   r   r   r   actuals          r(   test_read_csv_large_skiprowsr     sQ     XE	$ C2d%8&'"t5ABC C Cs	   !:A   c           
        ddg}t        |      5 }t        j                  t        d      5   | |d||      j	                         }t        | ||||             d d d        t        j                  t              5  t        j                  t              5   | |d||       d d d        d d d        d d d        y # 1 sw Y   bxY w# 1 sw Y   &xY w# 1 sw Y   *xY w# 1 sw Y   y xY w)	Nr:   r;   zsample=blocksizematch   )r   r   r   r   r   )r   r   warnsUserWarningro   r   r   r   r   s          r(   .test_read_csv_skiprows_only_in_first_partitionr     s     XE	$ F2\\+-?@ 	GR3UKSSUFfgb4uEF	G \\+& 	Fz* Fb4uEF	FF F	G 	GF F	F 	FF FsR   C!0B=!C!=CC	$C,C!=C	C!	CCC	C!!C*c                    t               }t        |d      5   | d      }t        ||d       d} | |      } |t        ||               }t        ||d       d d d        y # 1 sw Y   y xY w)NrW   r   r   Frw   r2   )rG   r   r   r   )r   r   rV   r{   rF   r   r~   s          r(   test_read_csv_filesr     sm    
 |H	5s	# 4_%"hE2R[GE"I./	"iU34 4 4s   AA$$A-c                   t        |d      5  t        |      d d }t        |      } | |      }t        ||d       t	        j
                  t              5   | g        d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)NrW   r   rt   Frw   )r   r?   rG   r   r   r   r   )r   r   rV   subsetsolress         r(   test_read_csv_files_listr     s    
 
5s	# r" fo#s.]]:& 	BK	 	 	 s$   A	A=	A1 A=1A:	6A==Bzdd_read,filesc                    t        |d      5   | dddt        i      }|j                  j                         j	                         }d|v sJ d|vsJ d	|v sJ 	 d d d        y # 1 sw Y   y xY w)
NrW   r   r   Tr.   include_path_column
convertersr2   r3   r4   )r   r/   r.   ro   uniquer   rV   rF   	filenamess       r(   !test_read_csv_include_path_columnr     s     
5s	# 	- $/

 GGOO%,,.	9,,,y0009,,,	- 	- 	-   AA$$A-c                    t        |d      5   | dddt        i      }|j                  j                         j	                         }d|v sJ d|vsJ d|v sJ 	 d d d        y # 1 sw Y   y xY w)	NrW   r   r   filenamer   r2   r3   r4   )r   r/   r   ro   r   r   s       r(   (test_read_csv_include_path_column_as_strr     s     
5s	# 	- *"N3

 KK'')002	9,,,y0009,,,	- 	- 	-r   c                    t        |d      5  t        j                  t              5   | dd       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)NrW   r   r   r:   r   )r   r   r   r   )r   rV   s     r(   5test_read_csv_include_path_column_with_duplicate_namer     s\     
5s	# A]]:& 	AO@	AA A	A 	AA As!   AAAA		AAc                Z   t        |d      5   | dd      }|j                  j                  dk(  sJ t        |j                        sJ  | dd      }|j	                         }|j                  j                  dk(  sJ t        |j                        sJ 	 d d d        y # 1 sw Y   y xY w)NrW   r   r   Tr   category)r   r.   r^   r   ro   r   rV   rF   r   rq   s        r(   3test_read_csv_include_path_column_is_dtype_categoryr     s     
5s	# 1_$?ww}}
***#BGG,,,o4@{{  J...#FKK0001 1 1s   B	B!!B*c                   t        |d      5   | ddd      }|j                  dkD  sJ |j                  j                  dk(  sJ t	        |j                        sJ  | ddd      }|j                         }|j                  j                  dk(  sJ t	        |j                        sJ 	 d d d        y # 1 sw Y   y xY w)	NrW   r   r   10BT)r   r   rN   r   )r   npartitionsr.   r^   r   ro   r   s        r(   Ctest_read_csv_include_path_column_with_multiple_partitions_per_filer     s     
5s	# 	1_4P~~!!!ww}}
***#BGG,,,oDQ{{  J...#FKK000	1 	1 	1s   BB44B=c                    t        t              5 } t        j                  | d      j	                  d      }|j                  d      }|j                  j                  dk(  sJ t        t        j                  |j                  |j                         d      }t        |      D ]  \  }}|t        |j                        dz
  k  r/|j                  |j                  |dz      k  j                         sJ |dkD  sV|j                  |j                  |   k\  j                         rJ  t         j                  |       j	                  d      }t#        ||       d d d        y # 1 sw Y   y xY w)	N   r   r;   r   r   rt   r,   r   )r   r   rg   r@   	set_indexro   indexr:   r   rh   re   __dask_keys__	enumeraterR   	divisionsrp   r=   r   )r   r   rq   rj   iblockr{   s          r(   test_read_csv_indexr    s.   	(	 $rKKbK)33H=V,||  H,,,)LL!&&!//"3v
 "&) 	=HAu3q{{#a''akk!a%&88==???1uq{{1~5::<<<		= ;;r?,,X6&(#$ $ $s   C)E!:*E!%3E!!E*c                     t        t              5 } t        j                  | t	        d            }|}t
        j                  | t	        d            }t        ||       d d d        y # 1 sw Y   y xY w)Nr   r   )r   r   rg   r@   ranger=   r   )r   r   rq   r{   s       r(   test_read_csv_skiprows_ranger	  "  s\    	(	 $rKKU1XK.;;rE!H;5&(#	$ $ $s   AA((A1c                    t        t              5 } t        j                  | dddg      }|dg   }t        j                  | ddg      }|dg   }|j                         j                  |j                  k(  j                         sJ |j                         j                  |j                  k(  j                         sJ 	 d d d        y # 1 sw Y   y xY w)Nr   HighLow)r   rZ   rZ   )r   
timeseriesrg   r@   r=   ro   rl   rp   )r   rF   	df_selectr{   expected_selects        r(   test_usecolsr  *  s    	*	 L[[rFE?[CxL	;;rFE?;;"F8,

##x6;;===!!#**o.D.DDIIKKKL L Ls   B.CCc                 0   t        t              5 } t        j                  | d      }t        j                  | d      }|j                  |j                  k(  sJ t        j                  | d      }|j                  dk(  sJ 	 d d d        y # 1 sw Y   y xY w)N30Br   3064MiBr,   )r   r  rg   r@   r   r   arW   cs       r(   test_string_blocksizer  4  s    	*	 "KKeK,KKdK+}}---KKgK.}}!!!" " "s   A2BBc                    t        d      } t        |       5 }t        j                  |dd      }d|j                  v sJ |j
                  j                         j                         dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nzm
    name, amount
    Alice,100
    Bob,-200
    Charlie,300
    Dennis,400
    Edith,-500
    Frank,600
    Tr   )skipinitialspacer   r;   iX  )r)   r   rg   r@   rQ   r;   maxro   r   r   rF   s      r(   test_skipinitialspacer  >  sx    	
D 
$ 02[[db[A2::%%%yy}}&&(C///	0 0 0s   AA77B c                     t        d      } t        |       5 }t        j                  |d      }|j                  j                         j                  t        k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nzp
    name,amount
    Alice,100.5
    Bob,-200.5
    Charlie,300
    Dennis,400
    Edith,-500
    Frank,600
    r   r   )r)   r   rg   r@   r;   ro   r^   r]   r  s      r(   test_consistent_dtypesr   R  sd    	
D 
$ 22[[r[*yy  "((E1112 2 2s   AA&&A/c                 B   t        d      } t        d      }t               }t        | |d      5  t        j	                  dd      }|j
                  j                  |k(  sJ |j
                  j                         j                  |k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nz@
    name,amount
    Alice,100
    Bob,-200
    Charlie,300
    z8
    name,amount
    1,400
    2,-500
    Frank,600
    z	foo.1.csvz	foo.2.csv	foo.*.csv   r   )r)   r   r   rg   r@   r:   r^   ro   )text1text2string_dtyperF   s       r(   test_consistent_dtypes_2r(  d  s    	E 	E $%L	U;	< 7[[[3ww}},,,ww &&,6667 7 7s   ABBc                    t        d      } t        d      }t        | |d      5  t        j                  dddid      }|j                  j
                  dk(  sJ t        |j                        rJ |j                         }|j                  j
                  dk(  sJ t        |j                  j                  j                        g d	k(  sJ 	 d d d        y # 1 sw Y   y xY w)
NzJ
    fruit,count
    apple,10
    apple,25
    pear,100
    orange,15
    zO
    fruit,count
    apple,200
    banana,300
    orange,400
    banana,10
    r"  r#  fruitr   r$  )r^   r   )applebananaorangepear)r)   r   rg   r@   r*  r^   r   ro   r?   cat
categories)r%  r&  rF   r   s       r(   test_categorical_dtypesr1  }  s    	E 	E 
U;	< Y[[Wj,AR[Pxx~~+++'111jjlyy*,,,ciimm../3XXXXY Y Ys   B(CC c                    t        d      } t        d      }t        j                  j                  j	                  g dd      }t        | |d      5  t        j                  dddd	
      }|j                  j                  j                  du sJ |j                  j                  j                  du sJ t        j                  t        j                  g d|j                        t        j                  g d|j                        d	g d      }t        ||       t        j                  d|dd	
      }|j                  j                  j                  du sJ |j                  j                  j                  du sJ t!        j"                  |j                  j                  j                  |j                         |j                  j                  j$                  du sJ t        ||       t        j                  j                  j	                  g dd      }t        j                  d|dd	
      }|d   j                  j'                         |d<   |j                  j                  j                  du sJ |j                  j                  j                  du sJ |j                  j                  j$                  du sJ t        ||       t        j                  dt        j                  j                  j	                  d      
      }|j                  j                  j                  du sJ t        j                  dd
      }|j                  j                  j                  du sJ 	 d d d        y # 1 sw Y   y xY w)Nz%
    A,B
    a,a
    b,b
    a,a
    z%
    A,B
    a,a
    b,b
    c,c
    )r  rW   r  F)orderedr"  r#  r   )ABr^   )r  rW   r  r  rW   r  )r0  )r   r,   rt   r   r,   rt   r   Tr4  )r)   r=   apitypesCategoricalDtyper   rg   r@   r4  r/  knownr5  rh   Categoricalr0  r   r   assert_index_equalr3  
as_ordered)r%  r&  r^   rq   r{   s        r(   test_categorical_knownr?    s   	E 	E FFLL))/5)IE	U;	< *+[jz0RSxx||!!U***xx||!!U***<<^^2u?O?O $  ^^2u?O?O $ 	 %   

 	&(# [e*0MNxx||!!T)))xx||!!U***
fhhll55u7G7GHxx||##u,,,&(# --ot-L[e*0MN ))446xx||!!T)))xx||!!U***xx||##t+++&(# rvv||<<U<K  
 xx||!!U***[
;xx||!!U***U*+ *+ *+s   LM66M?compressioninfergzipc                &   t               5 }t        j                  t        j                  j                  |d      d      }|j                  t        j                                |j                          t        j                  t        j                  j                  |d      d      }|j                  t        j                                |j                          t        j                  t              5  t        j                  t        j                  j                  |d      |       }d d d        t        j!                               t        t        j#                  d            dz
  dz  k(  sJ 	 d d d        y # 1 sw Y   SxY w# 1 sw Y   y xY w)	Nza.csv.gzwbzb.csv.gzz*.csv.gzr@  r!   r,   rt   )r   rB  openr-   r.   r"   writer   encodecloser   r   r   rg   r@   rR   ro   r&   )r@  tdirr   rF   s       r(   test_compression_multiple_filesrK    s    
 HTIIbggll44d;	!"		IIbggll44d;	!"		\\+& 	VRWW\\$
;UB	V 2::< S)=%>%Ba$GGGGH H	V 	VH Hs%   C(F36E;)AF;F	 FFc                     t        d      5 } t        j                  | d      }t        |j	                               dk(  sJ t        |j                        ddgk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nza,br   r   r  rW   )r   rg   r@   rR   ro   rP   rQ   r   rF   s     r(   test_empty_csv_filerN    sf    	% .B[[A[&2::< A%%%BJJC:---. . .s   AA((A1c                     t        t        d      5 } t        j                  | d      }t	        |j
                        g dk(  sJ 	 d d d        y # 1 sw Y   y xY w)NrW   r   Fsampler9   )r   rA   rg   r@   rP   rQ   rM  s     r(   test_read_csv_no_samplerR    sM    	93	' <2[[E[*BJJ#;;;;< < <s   4AAc                     t        t        d      5  t        j                  dd      } t        j                  dd      }| j                  |j                  k7  sJ 	 d d d        y # 1 sw Y   y xY w)NrW   r   r   Tr   F)r   rA   rg   r@   _namer  rW   s     r(   "test_read_csv_sensitive_to_enforcerV    s[    	93	' "KKK6KKK7ww!''!!!" " "s   A
A&&A/r   
   fmtc                   | r | t         vrt        j                  d| z         t               }dddddj	                  | d      }| rt        t         |    t              nt        }|j                         D ci c]  \  }}||z   | }}}t        |d	      5  | r?|r=t        j                  t              5  t        j                  d
|z   |      }d d d        nt        j                  d
|z   |      }t        j                  d      j                  d      |j                  d      d       d d d        y c c}}w # 1 sw Y   VxY w# 1 sw Y   y xY w)Nz%compress function not provided for %sz.gzz.bz2z.zipz.xz)rB  bz2zipxz rW   r   r   r   r   r   Tru   Frw   )r   r   r   rG   getr   rA   r   r   r   r   rg   r@   r   ro   rz   )	rX  r   r{   suffixfiles2rE   vrenamed_filesrF   s	            r(   test_read_csv_compressionrc     s?    s(";cAB|HF6GKKCQSTF14VHSM9-)F/5||~>tq!QZ]>M>	=s	+ 
 9k* P[[6!9Y[OP P _v5KBJJJ(44$4?  d +	

 
 ?
P P	
 
s+   6D9E4D?A!E?E	EEc                     t        t        d   t              } t        | d      5  t	        j
                  t              5 }t        j                  dd      }|j                  dk(  sJ 	 d d d        t              dk(  sJ t        |d   j                        }d|v sJ d	|v sJ t        j                  d
      5 }t        j                  ddd       }d d d        rJ t	        j                  t               5  t	        j
                  t              5  t        j                  dd      }d d d        d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   }xY w# 1 sw Y   2xY w# 1 sw Y   6xY w# 1 sw Y   y xY w)NrB  rW   r   r   rE  rN   r,   r   zblocksize=NoneT)record)r@  r   foo)r   r   rA   r   r   r   r   rg   r@   r   rR   r$   messagewarningscatch_warningsr   NotImplementedError)r`  wrF   msgre  s        r(   test_warn_non_seekable_filesrm    sc   HV$i0F	6	$ E\\+& 	'!_&AB>>Q&&&	' 1v{{!A$,,}}3&&&$$D1 	RV_&DQB	Rz]]./ 	Ek* E[[e[DE	EE E	' 	'	R 	R
E E	E 	EE Esk   E4)E)AE4;E%E49E(E+E(3E4E		E4E	E4E%!E((E1	-E44E=c                 .   d} t        |       5 }t        j                  |dd      }|j                  j	                         j                         dk(  sJ |j                  j	                         j                         dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nz!a,b
1,2
2,3
3,4
4,5
5,6
6,7r   z
)r   r         )r   rg   r@   rW   rS   ro   r  r  s      r(   test_windows_line_terminatorrq  /  s}    :D	$ =2[[q[@ttxxz!!#'<<<<ttxxz!!#'<<<<= = =s   A3BBrb   )r,   rt   rN   c                    d}t        d|i      5  t        j                  d| d      }t        j                  d|       }t	        ||d       d d d        y # 1 sw Y   y xY w)Nzid0,name0,x0,y0
id,name,x,y
1034,Victor,-0.25,0.84
998,Xavier,-0.48,-0.13
999,Zelda,0.00,0.47
980,Alice,0.67,-0.98
989,Zelda,-0.04,0.03
ztest_header_int.csv@   )rb   r   r   Fr   )r   rg   r@   r=   r   )rb   r   rF   r{   s       r(   test_header_intrt  7  se    	! 	 
)40	1 3[[.v[L;;4V;D"hE23 3 3s   >AA c                    t        dddd      5  t        j                  dd       } t        j	                  ddgd	d
gd      }t        | j                         j                  d      |       d d d        y # 1 sw Y   y xY w)Nz1,2r]  z3,4)z
.tmp.1.csvz
.tmp.2.csvz
.tmp.3.csvz
.tmp.*.csvr   r,   rN   rt      )r   r,   Tru   )r   rg   r@   r=   rh   r   ro   rz   )rF   r{   s     r(   test_header_Nonerw  H  su    	%rO	P A[[d[3<<QF1v 67"**,***5x@A A As   AA77B c                     t        t        dd      t              sJ t        dd      dk(  sJ t        dd      dk(  sJ y )Ni     rN   d   i  rt      )rf   r   rC    r*   r(   test_auto_blocksizer}  O  sC    nT2.444$"c)))$"c)))r*   c                    t        j                  d      } G d d      fd}| j                  |d|       t               sJ y)zp
    psutil returns a total memory of `None` on some systems
    see https://github.com/dask/dask/pull/7601
    psutilc                      e Zd ZdZy)*test__infer_block_size.<locals>.MockOutputN)__name__
__module____qualname__totalr|  r*   r(   
MockOutputr  \  s    r*   r  c                      S r   r|  )r  s   r(   mock_virtual_memoryz3test__infer_block_size.<locals>.mock_virtual_memory_  s	    r*   virtual_memoryN)r   r   setattrr   )monkeypatchr  r  r  s      @r(   test__infer_block_sizer  U  sF    
   *F   02EFr*   c                 `    t        dd      } | t        d      k(  sJ t        | t              sJ y )Nl    J)rN   g    A)r   rC   rf   r   s    r(   test_auto_blocksize_max64mbr  f  s0    }a0ID	!!!i%%%r*   c                   t        j                  d      }|j                         j                  }|j	                         }t        j                  t              }| j                  t        j                  j                  j                  d|       t        ||      }t        t        d      5  t         j#                  d       |j$                  sJ |j&                  d   d   |k(  sJ 	 d d d        y # 1 sw Y   y xY w)	Nr  )wrapsr   rW   r   r2   r,   r   )r   r   r  r  	cpu_countr   Mockr   r  re   	dataframeiocsvr   r   rA   rg   r@   called	call_args)r  r  total_memoryr  mock_read_bytesexpected_block_sizes         r(   test_auto_blocksize_csvr  l  s      *F((*00L  "Iiij1O))--|_M(yA	93	' P
$%%%%%((+K8<OOOOP P Ps   ,;C11C:c                     ddd} t        |       5  t        j                  dd       t        j                  dd      }|j                  d	k(  j	                         sJ 	 d d d        y # 1 sw Y   y xY w)
NzAa,b
0,'abcdefghijklmnopqrstuvwxyz'
1,'abcdefghijklmnopqrstuvwxyz'z.a,b
111111,-11111
222222,-22222
333333,-33333
).overflow1.csv.overflow2.csvr  4   rP  r  #   i8)r   rg   r@   r   rp   )rV   rF   s     r(   test_head_partial_line_fixr  z  sr     RN	E 
5	 )
$R0
 [[)"[5		T!&&((() ) )s   AA))A2c                     d} 	 t         j                  |        J # t        $ r}| t        |      v sJ Y d }~y d }~ww xY w)Nz.not.a.real.file.csv)rg   r@   OSErrorr$   )r   es     r(    test_read_csv_raises_on_no_filesr    s<    	B
Bu SV||s    	=8=c                 "   t        t              5 } t        j                  |       }t        j                  |       }|j                  |j                  k(  sJ t        |j                  j                         t              t        |j                  j                         t              k(  sJ t        |j                  t              sJ t        j                  | ddg      }|j                  |j                  k7  sJ 	 d d d        y # 1 sw Y   y xY w)Nkeyr,   r   )r   	na_values)
r   r   rg   r@   rT  r?   re   keysr$   rf   r  s       r(   $test_read_csv_has_deterministic_namer    s    	(	 "rKKOKKOww!''!!!affkkm-31OOOO!''3'''KKQ1#K6ww!''!!!" " "s   C+DDc                 Z   t        t        t        d      5  t        j                  d      } t        j                  d      }t	        | j
                  j                         t              t	        |j
                  j                         t              k(  sJ 	 d d d        y # 1 sw Y   y xY w)N)z
_foo.1.csvz
_foo.2.csvz
_foo.*.csvr  )r   r   rg   r@   r?   re   r  r$   rU  s     r(   -test_multiple_read_csv_has_deterministic_namer    st    	((C	D PKK%KK%affkkm-31OOOO	P P Ps   B B!!B*c                     t        t              5 } t        j                  | d      }t        j                  | d      }|j                  |j                  k7  sJ 	 d d d        y # 1 sw Y   y xY w)N10kBr   20kB)r   r   rg   r@   rT  )r   r  rW   s      r(   4test_read_csv_has_different_names_based_on_blocksizer    s[    	(	 "rKKfK-KKfK-ww!''!!!" " "s   A
A$$A-c                     t        d      5 } t        j                  | d       }t        |j                        ddgk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nzalice,1
bob,2r   r   r,   )r   rg   r@   rP   rQ   rM  s     r(   test_csv_with_integer_namesr    sO    	"	# *r[[D[)BJJAq6)))* * *s   4A

Ac                 p   d} t        d      D ]  }| dz  } 	 | dz  } d}t        |       5 }t        rt        j	                  |dg      }nt        j	                  |      }d}t        j                  t              5 }t        j	                  |d	dg
      j                  d       d d d        j                  ||z         sJ t        j                  t              5 }t        j	                  |d	      j                  d       d d d        |j                  |      sJ d}t        j                  t              5 }t        j	                  |d	ddi      j                  d       d d d        t        |j                        |k(  sJ t        r@t        j	                  |d	t        t        t        ddg      }t        ||       	 d d d        y t        j                  t              5 }t        j	                  |d	dgddi      j                  d       d d d        t        |j                        ||z   k(  sJ d}t        j                  t              5 }t        j	                  |d	dgt        t        t        d      j                  d       d d d        t        |j                        |k(  sJ t        j	                  |d	t        t        t        d      }t        ||       d d d        y # 1 sw Y   JxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nz*numbers,names,more_numbers,integers,dates
  z1,,2,3,2017-10-31 00:00:00
z"1.5,bar,2.5,3,4998-01-01 00:00:00
a\  

-------------------------------------------------------------

The following columns also failed to properly parse as dates:

- dates

This is usually due to an invalid value in that column. To
diagnose and fix it's recommended to drop these columns from the
`parse_dates` keyword, and manually convert them to dates later
using `dd.to_datetime`.datesparse_datesaU  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

+--------------+---------+----------+
| Column       | Found   | Expected |
+--------------+---------+----------+
| more_numbers | float64 | int64    |
| names        | object  | float64  |
| numbers      | float64 | int64    |
+--------------+---------+----------+

- names
  ValueError(.*)

Usually this is due to dask's dtype inference failing, and
*may* be fixed by specifying dtypes manually by adding:

dtype={'more_numbers': 'float64',
       'names': 'object',
       'numbers': 'float64'}

to the call to `read_csv`/`read_table`.2   )rQ  r  r   r   rP  aa  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

+--------------+---------+----------+
| Column       | Found   | Expected |
+--------------+---------+----------+
| more_numbers | float64 | int64    |
| numbers      | float64 | int64    |
+--------------+---------+----------+

Usually this is due to dask's dtype inference failing, and
*may* be fixed by specifying dtypes manually by adding:

dtype={'more_numbers': 'float64',
       'numbers': 'float64'}

to the call to `read_csv`/`read_table`.

Alternatively, provide `assume_missing=True` to interpret
all unspecified integer columns as floats.r   O)rQ  r^   )more_numbersr   numbers)rQ  r^   r  )rQ  r  r^   aQ  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

The following columns failed to properly parse as dates:

- dates

This is usually due to an invalid value in that column. To
diagnose and fix it's recommended to drop these columns from the
`parse_dates` keyword, and manually convert them to dates later
using `dd.to_datetime`.)r  r   r   r=   r@   r   r   r   rg   ro   r   r$   valuer]   objectr   )r   _date_msgr   r   rl  r  r   s           r(   test_late_dtypesr    s   8D4[ /../11D	"  
$ l2++bwi+8C++b/C6 	0 ]]:& 	X!KK2G9K=EEPVEW	XwwsX~&&&]]:& 	A!KK2K&...@	Awws||9 	, ]]:& 	W!KK2gs^K<DDvDV	W177|s"""++',v%P$I	  C c3Ql lT ]]:& 	(!KK2G9WcN  gg'	( 177|sX~---	& 	 ]]:& 	(!KK$I',v%P	  
 gg'	( 177|s""" kk#(6eL  

 	#sYl l:	X 	X	A 	A4	W 	W	( 	(&	( 	({l ls   AL,:*K,$7L,(K96L,9+L$AL,L,%-LA L,;L AL,,K6	1L,9L	>L,L	L,L	L, L)	%L,,L5c                    d} t        d      D ]  }| dz  } 	 | dz  } t        |       5 }t        j                  |      }t        j                  |dd      }t        ||j                  dt        i             t        j                  |dddd	i
      }t        ||       t        j                  |ddd 
      }t        ||j                  dt        i             d d d        d} t        d      D ]  }| dz  } 	 | dz  } t        |       5 }t        j                  |      }t        j                  |dd	d      }|j                  j                  d	k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nz$numbers,names,more_numbers,integers
r  z
1,foo,2,3
z1.5,bar,2.5,3
r  T)rQ  assume_missingintegersint64)rQ  r  r^   znumbers,integers
z1,2
z1.5,2
r   )rQ  r^   r  )
r  r   r=   r@   rg   r   rB   r]   r  r^   )r   r  r   r   r   rF   s         r(   test_assume_missingr  :  st   2D4[ D	$ 82kk"o kk"Rk=#szz:u"567 kkr$z76K  
 	#s kk"RDkI#szz:u"5678"  D4[ ID	$ +2kk"o [[Bgd[Kzz7***+ +-8 8,+ +s   B0EA
E'E$'E0c                 F   t        t              5 } 	 t        j                  | dd       J # t        $ r}dt        |      v sJ Y d }~nd }~ww xY wt        j                  | d      }t        j                  | dd      }t        ||d       d d d        y # 1 sw Y   y xY w)Nr   r:   )r   	index_colFr   )r  r   )r   r   rg   r@   r   r$   r=   r   )r   r  rF   r   s       r(   test_index_colr  ]  s    	(	 	.r	)KKbFK;5 	)#a&(((	) [[u[-kk"ek<"cu-	. 	. 	.s,   B,	AA	B	AA BB c                 d   t        t              5 } t        j                  | ddddgdg      }t        j                  | dddgdgd      j                  d      }t        ||       t        j                  | dddgdg      j                  d      }t        ||       d d d        y # 1 sw Y   y xY w)Nr   rv  Dater  rb   rZ   r  i rb   rZ   r  r   )rb   rZ   r  r   r  r=   r@   rg   r   r   r   rF   r   s      r(   0test_read_csv_with_datetime_index_partitions_oner  j  s    	*	 [[!A1vF8  
 kkq1a&vh(  

)F
 	 	"c kk"QAVHkMWW
 	"c  s   BB&&B/c                     t        t              5 } t        j                  | ddddgdg      }t        j                  | dddgdgd      j                  d      }t        ||       d d d        y # 1 sw Y   y xY w)Nr   rv  r  r  i  r  r  r  s      r(   .test_read_csv_with_datetime_index_partitions_nr  |  s    	*	 [[!A1vF8  
 kkq1a&vh#  

)F
 	 	"c  s   AA//A8encoding)z	utf-8-sigzutf-16z	utf-16-lez	utf-16-bec                   t         j                  t        dd            }|dz  }|dz  }|dz  }t         j                  ||||d      }t	        d      5 }|j                  || d	       t         j                  || 
      }t        j                  || d      }|j                         }t        t        |j                              |_
        t        ||       d d d        y # 1 sw Y   y xY w)Nr   rz  r   gffffff
@gZӼ?r  rW   r  d.csvF)r  r   )r  r  )r  r   )r=   Seriesr  rh   r   to_csvr@   rg   ro   rR   r   r   )	r  arbrcrdrtest_dfr   r  r  s	            r(   test_encoding_gh601r    s    	5C=	!B	aB	cB	fBll"2B?@G	 BrHE:KKXK.KKXK>IIKAGG%!Q  s   BC  C)c                 H   dj                  dd      } t        |       5 }t        j                  |d      }t	        |t
        j                  |d             t        j                  |d      }t	        |t
        j                  |d             d d d        y # 1 sw Y   y xY w)Nza b c-d
1 2 3
4 5 6 r1   r   )	delimiter)r   r   rg   r@   r   r=   r  s      r(   test_read_csv_header_issue_823r    s    &..sD9D	$ 72[[[&"bkk"$k/0[[t[,"bkk"k567 7 7s   A1BB!c                     t        t              5 } t        j                  | d       }t	        |t
        j                  | d              d d d        y # 1 sw Y   y xY w)Nr  )r   r   rg   r@   r   r=   rM  s     r(   test_none_usecolsr    sJ    	(	 5r[[T[*"bkk"dk345 5 5s   9AAc                 $   t        d      } t        j                         }t        rt	        j
                  t        d      }n!t        rt	        j                  t        d      }|5  t        |       5 }t        j                  |ddgg      }t        j                  |ddgg      }|j                  |j                  k(  j                         sJ t!        |      t!        |      k(  sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)NaX  
    ID,date,time
    10,2003-11-04,180036
    11,2003-11-05,125640
    12,2003-11-01,2519
    13,2003-10-22,142559
    14,2003-10-24,163113
    15,2003-10-20,170133
    16,2003-11-11,160448
    17,2003-11-03,171759
    18,2003-11-07,190928
    19,2003-10-21,84623
    20,2003-10-25,192207
    21,2003-11-13,180156
    22,2003-11-15,131037
    zlist indicesr   nesteddatetimer  )r)   
contextlibnullcontextr   r   r   	TypeErrorr   r   FutureWarningr   rg   r@   r=   rQ   rp   rR   )	pdmc_textctxr   r   rF   s        r(   test_parse_dates_multi_columnr    s    	I& 
 
 
"CmmI^<	ll=9	 'i  	'B++b/?.@+ACRvv.>-?@BJJ#++-22444r7c#h&&&	'' '	' 	'' 's%   %D1A7C:)D:D	?DDc                 @   t        d      } t        |       5 }t        j                  |dd      }t        j                  |dd      }|j
                  |j
                  k(  j                         sJ t        |      t        |      k(  sJ 	 d d d        y # 1 sw Y   y xY w)NzB
    name###amount
    alice###100
    bob###200
    charlie###300z###r   )r   r   )r)   r   rg   r@   r=   rQ   rp   rR   )sep_textr   r   rF   s       r(   test_read_csv_sepr    s    	H 
(	 #rkk"%k9[[x[8

ckk)..0002w#c("""# # #s   A3BBc            	         ddz  dz   } t        | d      5 }t        j                  |d dddd	gd
      j                  d       d d d        y # 1 sw Y   y xY w)Ns   0,my
1,data
r  s	   2,foobarrD  r   r0   r!   r  rW   r   )rb   r   r   r   r   r   r   )r   rg   r@   ro   datar   s     r(   test_read_csv_slash_rr    sg    t#m3D	$T	" $b
* 	 	
 'F'
#$ $ $s   .AAc                     d} t        | d      5 }t        t        j                  |t              t
        j                  |t                     d d d        y # 1 sw Y   y xY w)Ns   a,b
1,2
3,4
5,6rD  r   r6  )r   r   r=   r@   r]   rg   r  s     r(   test_read_csv_singleton_dtyper    sP     D	$T	" Nb"++b+.Be0LMN N Ns   ?AA!c                     t        j                  d       t        d      } t        |       5 }t	        t
        j                  |d      t        j                  |d             d d d        y # 1 sw Y   y xY w)Nr   z
    a,b
    1,2
    )r   )r   r   r)   r   r   r=   r@   rg   )r  r   s     r(   test_read_csv_arrow_enginer    sh    
	"	H 
(	 Xr"++b+3R[[I[5VWX X Xs   7A,,A5c                    t         j                         } t        |       d   }| |   j                  dd      | |<   t	        | d      5  t
        j                  dd dg d	      }t        j                  d
      }|j                  |j                  k(  j                         sJ t        ||       d d d        y # 1 sw Y   y xY w)N   names   NamerW   r   r   r,   r9   )rb   r   r   r2   )rA   copyr?   r   r   rg   r@   r=   rQ   rp   r   )rV   rE   r   rF   s       r(   test_robust_column_mismatchr    s    NNEubAQx1E!H	5s	# kkD1<T  
 [[)*

ckk)..000#s  s   A&B77C c                    t         j                         } t        |       d   }| |   j                  dd      | |<   t	        | d      5  t
        j                  d      }|j                  g dk(  j                         sJ |j                         j                  g dk(  j                         sJ 	 d d d        y # 1 sw Y   y xY w)	Nr  r  s   addressrW   r   r   r9   )r:   r;   r<   address)
rA   r   r?   r   r   rg   r@   rQ   rp   ro   )rV   rE   r   s      r(   "test_different_columns_are_allowedr    s    NNEubAQx4E!H	5s	# Tkk/* 77<<>>>%%)LLQQSSST T Ts   A&B88Cc            	        d} t        |       5 }d}t        j                  t              5  t        j                  ||       d d d        t        t        j                  ||d       t        j                  |d              d d d        d}|| z   } t        |       5 }dt        |      z   }t        j                  t              5  t        j                  ||d       d d d        t        t        j                  ||d d	      t        j                  |d d
             d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   cxY w# 1 sw Y   y xY w)NzAAAAAA,BBBBB,CCCCC,DDDDD,EEEEE
1,2,3,4,5
6,7,8,9,10
11,12,13,14,15r   rP  )rQ  rb   r   z# skip
# these
# lines
rN   )rQ  r   )rQ  rb   r   )rb   r   )	r   r   r   r   rg   r@   r   r=   rR   )r   r   rQ  skiptexts       r(   !test_error_if_sample_is_too_smallr    s+   QD	$ 	
2]]:& 	+KK6K*	+ 	KK6$K7RPT9U	
	
 ,Hd?D	$ 

2c(m#]]:& 	7KK6AK6	7 	KK6$KCKK4!K4	


 

	+ 	+	
 	
"	7 	7

 

sG   D2D&?D2(E
D>AE
&D/	+D22D;>E	E

Ec                     d} ddg}t        |       5 }t        j                  ||d      }t        j                  ||      }t	        ||d       d d d        y # 1 sw Y   y xY w)	Nz?Alice,100
Bob,-200
Charlie,300
Dennis,400
Edith,-500
Frank,600
r:   r;      )r   r   )r   Fr   )r   rg   r@   r=   r   )r   r   r   r   rF   s        r(   test_read_csv_names_not_noner
  7  sh    	 	 XE	$ .2kk"ERk8[[5[)"cu-. . .s   >AA"c                    t         j                  g dg dd      } dD ]  }t        j                  | |      }t	               5 |j                  d       t        j                  t        j                  j                  d            j                         j                  d	      }t        ||        d d d        t	               5 |j                  dd
      }t        j                  |ddi}|t        fdt        |      D              k(  sJ t        j                  t        j                  j                  d            j                         j                  d	      }t        ||        d d d        t	               5 t        j                  j                  d      }|j                  |d      }|t        |      D cg c]&  }t        j                  j                  d| d      ( c}k(  sJ t        j                  |      j                         j                  d	      }t        ||        d d d         y # 1 sw Y   xY w# 1 sw Y   xY wc c}w # 1 sw Y   IxY w)Nr  r,   rt   rN   rv  xyr,   rt   Fr7  *Tru   )r   ro   r   r   c              3  d   K   | ]'  }t         j                  j                  | d        ) yw)z.partN)r-   r.   r"   )r   rK   dns     r(   r   ztest_to_csv.<locals>.<genexpr>Z  s+      "23RA3e-"s   -0
data_*.csvdata_r  )r=   rh   rg   from_pandasr   r  r@   r-   r.   r"   ro   rz   r   re   tupler  )	rF   r   r  rq   rpathsr   rK   r  s	           @r(   test_to_csvr  L  s   	0|D	EB "NN2{+X 	"HHRuH%[[b#!67??AMMSWMXFfb!	"
 X 	"5%8ALL!6v6EE "7<[7I"     [[b#!67??AMMSWMXFfb!	" X 	"b,/BHHRuH-E;@;M67R541    [[_,,.:::EFfb!	" 	"#"	" 	"
	" 	"	" 	"s:   A1IB)IAI+I
/AII	I	II)	c                    t         j                  g dg dd      } t        j                  | d      }t	               5 }t        j                  t              5  t        j                  j                  |d      }|j                  |       d d d        d d d        t         j                  g dg dd      }t        j                  |d      }t	               5 }t        j                  j                  |d	      }|j                  |d
       t        j                  |      j                         j                  d      }t        ||       d d d        t        j                  | d      }t	               5 }|j                  |d
       t        j                  j                  |d	      }|j                  |dd
       t        j                  |      j                         j                  d      }t        ||        d d d        t        j                  |d      }t	               5 }|j                  |d
       t        j                  j                  |d	      }|j                  |dd
       t        j                  |      j                         j                  d      }t        ||       d d d        y # 1 sw Y   (xY w# 1 sw Y   -xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr  r  r  rt   zdata_*_*.csv)r  rW   r  r  r  r   ghr  jrE   lmrK   op)r,   rt   rN   rv  r   rO   r      	   rW     r   r      ry  r	  r	  r  Fr7  Tru   rk  )r   r   )r=   rh   rg   r  r   r   r   r   r-   r.   r"   r  r@   ro   rz   r   )rF   r  r  r   df16rq   s         r(   &test_to_csv_multiple_files_cornercasesr(  j  sL   	0|D	EB
r1A	 R]]:& 	b.1BHHRL	
 <<$ I'	
D. 	tR A	  RWW\\"l+	5!R((*66D6A&$	  	r1A	 R	5!WW\\"l+	#U+R((*66D6A&" 	tR A	  R	5!WW\\"l+	#U+R((*66D6A&$   [	 	 :       sJ   J<2J/	J<A3K	0BKBK"/J9	4J<<K	KK"K+c                    t         j                  g dg dd      } dD ]=  }t        j                  | |      }t	               5 }t
        j                  j                  |d      }|j                  |dd       t        j                  |      j                         j                  d	      }t        ||        d d d        t	               5 }t
        j                  j                  |d      }|j                  |ddd
      }t        j                  |d       t        j                  |      j                         j                  d	      }t        ||        d d d        @ y # 1 sw Y   xY w# 1 sw Y   XxY w)Nr  r  r  r  test.csvFT)r   single_fileru   )r   ro   r+  r   r   )r=   rh   rg   r  r   r-   r.   r"   r  r@   ro   rz   r   re   )rF   r   r  r  r   rq   r  s          r(   test_to_single_csvr,    s.   	0|D	EB "NN2{+X 	"b*-BHHRu$H7[[_,,.:::EFfb!		" X 	"b*-B5%TJALLf-[[_,,.:::EFfb!	" 	""	" 	"	" 	"s   A4E!	BE-!E*	-E7	c                 r   t         j                  g dg dd      } t        j                  | d      }t	               5 }t
        j                  j                  |d      }t        j                  t        d      5  |j                  |d d	d
       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr  r  r  r,   r*  z9name_function is not supported under the single file moder   c                    | S r   r|  )r  s    r(   <lambda>z7test_to_single_csv_with_name_function.<locals>.<lambda>  s     r*   FT)name_functionr   r+  r=   rh   rg   r  r   r-   r.   r"   r   r   r   r  rF   r  r  r   s       r(   %test_to_single_csv_with_name_functionr3    s    	0|D	EB
r1A	 SRWW\\"j)]]M
 	S HHR{%THR		SS S	S 	SS Ss#   <B-9B!B-!B*	&B--B6c                 p   t         j                  g dg dd      } t        j                  | d      }t	               5 }t
        j                  j                  |d      }t        j                  t        d      5  |j                  |ddd	
       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr  r  r  r,   r*  zDheader_first_partition_only cannot be False in the single file mode.r   FT)r   header_first_partition_onlyr+  r1  r2  s       r(   3test_to_single_csv_with_header_first_partition_onlyr6    s    	0|D	EB
r1A	 RWW\\"j)]]X
 	 HH%UPT  		 	 	 s#   <B,9B B, B)	%B,,B5c                    t         j                  g dg dd      } t        j                  | d      }t	               5 }t
        j                  j                  |d      }|j                  |ddd	
       t        j                  t
        j                  j                  |d            j                         }d d d        t        | d       y # 1 sw Y   xY w)Nr  r  r  rt   r   r*  Fr  Tr   r   r+  r  r   )r=   rh   rg   r  r   r-   r.   r"   r  r@   ro   r   )df0rF   	directorycsv_pathrq   s        r(   /test_to_csv_with_single_file_and_exclusive_moder=    s    
,,1E
FC		+B	 EY77<<	:6
		(%ct	DRWW\\)S9:BBDE fcu-	E Es   A7CCc                    t         j                  g dg dd      } t        j                  | d      }t	               5 }t
        j                  j                  t        |      d      }|j                  |ddd	
       t
        j                  j                  |      sJ t        j                  t              5  |j                  |ddd	
       d d d        |j                  |ddd	
       d d d        y # 1 sw Y   'xY w# 1 sw Y   y xY w)Nr  r  r  rt   r8  r*  Fr  Tr9  rk  )r=   rh   rg   r  r   r-   r.   r"   r$   r  existsr   r   FileExistsError)r:  rF   r;  r<  s       r(   2test_to_csv_single_file_exlusive_mode_no_overwriterA    s    
,,1E
FC		+B	 EY77<<I
;
		(%ct	Dww~~h''']]?+ 	IIIhe#4IH	I 			(%ct	DE E
	I 	IE Es$   A9C?7C3C?3C<	8C??Dc                    t         j                  g dg dd      } dD ]  }t        j                  | |      }t	               5 }t
        j                  j                  |d      }|j                  |ddd	       t         j                  |d
      j                  d      }t        ||        d d d         y # 1 sw Y   xY w)Nr  r  r  r  ztest.csv.gzFrB  T)r   r@  r+  rE  ru   )r=   rh   rg   r  r   r-   r.   r"   r  r@   rz   r   )rF   r   r  r  r   rq   s         r(   test_to_single_csv_gziprC    s    	0|D	EB "NN2{+X 	"b-0BHHRu&dHK[[[8DD$DOFfb!		" 	""	" 	"s   A)B66B?	z#to_csv does not support compressionr   c                 @   t         j                  g dg ddg d      } dD ]m  }t        j                  | |      }t	        d      5 }|j                  |d	       t         j                  |d
d      }t        j                  ||        d d d        o y # 1 sw Y   zxY w)Nr  r  r  g      ?g       @g      @g      @r7  r  r  rB  rE  r   )r  r@  )	r=   rh   rg   r  r   r  r@   r   assert_frame_equal)rF   r   r  r   rq   s        r(   test_to_csv_gziprG    s    	"6>R 
 
B  .NN2{+U^ 	.rHHRVH,[[qf[EF!!&"-	. 	..	. 	.s   ABB	z2023.9.1z)https://github.com/dask/dask/issues/10515c                    t         j                  g dg ddg d      } t        j                  | d      }t	               5 }t
        j                  j                  t        |      d      }|j                  |       dt        j                  |      v sJ t        j                  |      sJ t        j                  t
        j                  j                  |d	            j                         }d d d        j                  j                  | j                  j                  k(  j                         sJ y # 1 sw Y   GxY w)
Nr  r  r  rE  r7  rt   r8  createmer  )r=   rh   rg   r  r   r-   r.   r"   r$   r  listdirr@   ro   r  rl   rp   )r:  rF   dirdir0rq   s        r(   test_to_csv_nodirrM    s     ,,"6>R  C 
	+B	 @Sww||CHj1
		$RZZ_,,,zz$RWW\\$45==?@ HHOOsuu||+00222@ @s   B,D44D=c                    t         j                  g dg ddg d      } t        j                  | d      }t	               5 }t        |      }|j                  |       t        j                  |      sJ t        j                  t        j                  j                  |d            j                         }d d d        j                  j                  | j                  j                  k(  j                         sJ y # 1 sw Y   GxY w)	Nr  r  r  rE  r7  rt   r8  r  )r=   rh   rg   r  r   r$   r  r-   rJ  r@   r.   r"   ro   r  rl   rp   r:  rF   rK  rq   s       r(   test_to_csv_simplerP    s    
,,"6>R  C 
	+B	 ?S#h
		#zz#RWW\\#s34<<>	?
 HHOOsuu||+00222? ?s   A5C==Dc                 B   t         j                  ddgddgd      } t         j                  ddgdd	gd      }t        j                  |d
      }t	               5 }t
        j                  j                  |d      }| j                  |d       |j                  |dddd       t        j                  t
        j                  j                  |d            j                         }d d d        t         j                  | |g      }t        |d      sJ y # 1 sw Y   1xY w)Nr  rW   r,   rt   r  r  r  rN   rv  r8  r*  Fr7  T)r   rb   r   r+  r  r   )r=   rh   rg   r  r   r-   r.   r"   r  r@   ro   r>   r   )r:  df1rF   rK  r<  rq   r{   s          r(   ,test_to_csv_with_single_file_and_append_moderS  "  s   
,,Cj1v&C ,,Cj1v&C 
	+B	 ?S77<<Z0

 	 	
 			 	 	
 RWW\\#s34<<>? yy#s$HVX5999? ?s   BDDc                    t         j                  g dg d      } t        j                  | d      }t	               5 }t        |      }|j                  |d       t        j                  |      sJ t        j                  t        j                  j                  |d      d d	g
      j                         }d d d        j                  | k(  j                         sJ y # 1 sw Y   )xY w)Nr  rE  r7  rt   r8  Fr   r  r  )rb   r   )r=   r  rg   r  r   r$   r  r-   rJ  r@   r.   r"   ro   r  rp   rO  s       r(   test_to_csv_seriesrU  =  s    
))(0D)
EC		+B	 YS#h
		#e	$zz#RWW\\#s3DNVVX	Y
 HHO  """Y Ys   A;C  C)c                    ddl m dgfd} t        j                  g dg dd      }t        j                  |d	      }t               5 }|j                  |dd
| i       d   sJ t        j                  t        j                  j                  |d            }t        ||d       d d d        y # 1 sw Y   y xY w)Nr   r^  Fc                     dd<    | i |S )NTr   r|  )argsrJ   flagmp_gets     r(   my_getz$test_to_csv_with_get.<locals>.my_getM  s    Qt&v&&r*   r  r  r  rt   r8  r   r   compute_kwargsr  r   )dask.multiprocessingr^  r=   rh   rg   r  r   r  r@   r-   r.   r"   r   )r\  rF   r   r  rq   rZ  r[  s        @@r(   test_to_csv_with_getr`  H  s    27D' 
0|D	EB
...
+C	 1R

2UK3H
IAwwRWW\\"c23&"%0	1 1 1s   AB77C c                 D   ddl m t        j                  g dg dd      } t        j                  | d      }fd}t               5 }t        j                  t              5  |j                  |d	|
       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   rW  r  r  r  rt   r8  c                      | i |S r   r|  rY  rJ   r[  s     r(   r\  z:test_to_csv_warns_using_scheduler_argument.<locals>.my_geta      t&v&&r*   F)r   r   )r_  r^  r=   rh   rg   r  r   r   r   r  r  rF   r   r\  r  r[  s       @r(   *test_to_csv_warns_using_scheduler_argumentrf  [  s    2	0|D	EB
...
+C' 
 :R\\-( 	:JJr&J9	:: :	: 	:: :s$   
B$B
9B
B	BBc            	        ddl m t        j                  g dg dd      } t        j                  | d      }fd}t               5 }t        j                  t              xr t        j                  t              5  |j                  |d	|d
|i       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   rW  r  r  r  rt   r8  c                      | i |S r   r|  rc  s     r(   r\  z@test_to_csv_errors_using_multiple_scheduler_args.<locals>.my_geto  rd  r*   Fr   )r   r   r^  )r_  r^  r=   rh   rg   r  r   r   r   r   r   r  r  re  s       @r(   0test_to_csv_errors_using_multiple_scheduler_argsri  i  s    2	0|D	EB
...
+C' 
 R]]:&F6<<+F 	JJ%6;PVBW  	 	 	 s$   
5B4?B(B4(B1	-B44B=c                     ddl m fd} t        j                  g dg dd      }t        j                  |d      }t               5 }|j                  |d	| d
d       d d d        y # 1 sw Y   y xY w)Nr   rW  c                 (    |d   dk(  sJ  | i |S )Ntest_kwargs_passedfoobarr|  rc  s     r(   r\  zBtest_to_csv_keeps_all_non_scheduler_compute_kwargs.<locals>.my_get|  s'    *+x777t&v&&r*   r  r  r  rt   r8  Frm  )r   rl  r]  )r_  r^  r=   rh   rg   r  r   r  )r\  rF   r   r  r[  s       @r(   2test_to_csv_keeps_all_non_scheduler_compute_kwargsrn  y  sn    2' 
0|D	EB
...
+C	 
R

)/xP 	 	

 
 
s   
A++A4c                 @   t         j                  dt        d      i      } t        j	                  | d      }|j                  d      }|d   j                  d      sJ |d   j                  d	      sJ t        j                  d       t        j                  d	       y )
Nr4  rW  rt   r8  foo*.csvr   foo0.csvr,   zfoo1.csv)	r=   rh   r  rg   r  r  endswithr-   remove)rF   r   r  s      r(   test_to_csv_pathsrt    s    	sE"I&	'B
...
+CJJz"E8Z(((8Z(((IIjIIjr*   zheader, expected)Fr]  )Tx,y
c                \   t         j                  g g d      }t        j                  |d      }t	               5 }|j                  t        j                  j                  |d      d|        t        j                  j                  t        j                  j                  |d            rJ t        j                  j                  |d      }t        |      5 }|j                         }||k(  sJ 	 d d d        t        j                  |       d d d        y # 1 sw Y   'xY w# 1 sw Y   y xY w)	Nr  r,   r8  z	fooe*.csvFr   rb   z	fooe1.csvz	fooe0.csv)r=   rh   rg   r  r   r  r-   r.   r"   r?  rF  readliners  )rb   r{   dfeddfer  r   fplines           r(   "test_to_csv_header_empty_dataframer}    s    
,,Rb)
*C>>#1>-D	 RBGGLL[1vN77>>"'',,r;"?@@@77<<K0(^ 	$r;;=D8###	$ 			( 	$ 	$	 s$   BD"D0D"D	D""D+z?header,header_first_partition_only,expected_first,expected_next)FFa,1
d,4
)TFru  ru  )FTr~  r  )TTru  r  aabbFzaa,bb
Tr  c                   d}t         j                  g dg dd      }t        j                  ||      }t	               5 }|j                  t        j                  j                  |d      d| |       t        j                  j                  |d	      }t        |      5 }	|	j                         }
|
|k(  sJ 	 d d d        t        j                  |       t        j                  j                  |d
      }t        |      5 }	|	j                         }
|
|k(  sJ 	 d d d        t        j                  |       d d d        y # 1 sw Y   xY w# 1 sw Y   3xY w# 1 sw Y   y xY w)Nrt   )r  rW   r  r  r  r   )r,   rt   rN   rv  r   rO   r  r8  z	fooa*.csvF)r   rb   r5  z	fooa0.csvz	fooa1.csv)r=   rh   rg   r  r   r  r-   r.   r"   rF  rx  rs  )rb   r5  expected_firstexpected_nextpartition_countrF   r   r  r   r{  r|  s              r(   test_to_csv_headerr    s3    O	:AST	UB
...
9C	 R 	

GGLL[)(C	 	 	
 77<<K0(^ 	*r;;=D>)))	* 			(77<<K0(^ 	)r;;=D=(((	) 			(' 	* 	*	) 	)! s>    AED?8AE EE?E	EE	EE c                    t         j                  ddgi      } t        j                  | d      }ddh}t	               5 }|j                  t        j                  j                  |d      dd	       t        j                  j                  |d
      }t        |d      5 }|j                         }d d d        d d d        |v sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nr  r   r,   r8  s   0
s   0
rp  F)rb   r   rq  rb)r=   rh   rg   r  r   r  r-   r.   r"   rF  read)rF   r   r{   r  r   r   raws          r(   test_to_csv_line_endingr    s    	sQCj	!B
...
+C&!H 
 R

277<<J/U
K77<<J/(D! 	Q&&(C	
 (??	 	 s$   ACC.CC	
CCblock_listsrt   rN   )rv  r   rO   c                    t        t        |             }t        |      t        t        t        |                   k(  sJ y r   )rP   r   rR   r   )r  masks     r(   test_block_maskr    s4     
;'(Dt9D!567777r*   c            
     L   t               5 } t        g d      D ]U  \  }}t        t        j                  j                  | t        |      dz         d      5 }|j                  |       d d d        W t        j                  t        j                  j                  | d      ddt        ig d      j                         }t        j                  d	d
gddgddgddgd      }|d   j                  d      |d<   t        ||d       d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N)z0, 1, 2r]  z6, 7, 8r  rk  *.csvTr.   )r4  r5  C)r   r   r   r   rO   r,   r   rt   r#  z0.csvz2.csv)r4  r5  r  r.   r   Fr   )r   r  rF  r-   r.   r"   r$   rG  rg   r@   r/   ro   r=   rh   rB   r   )rJ  rE   r   filerq   rF   s         r(   &test_reading_empty_csv_files_with_pathr    s   	 1T#$>? 	$JAwbggll4Q&93? $4

7#$ $	$ GGLLw' $/!	  

 ') 	 \\VVV '*	
 Z&&z26
&"%0'1 1$ $1 1s%   A
DD'BDDDD#c                L   t         j                  j                  t        |       d      }t        j                  dddg      }|j                  |d       t        j                  |      }|j                  d      }t        ||j                  d      j                                y )Nr*  rW  rv  )rf  barFr7  rf  )r-   r.   r"   r$   r=   rh   r  rg   r@   groupbyr   	get_groupro   )r   r.   rR  ddf1ddfss        r(   test_read_csv_groupby_get_groupr    sz     77<<FZ0D
,,1-.
/CJJt5J!;;tD<<Dc4>>"%--/0r*   c                   t         j                  j                  t        |       d      }t	        d      }t	        t        t        |                  }t        j                  t        ||      D ci c]  \  }}||
 c}}g      }|j                  |       t	        d      }t        j                  |      |   j                  d      }t        ||   |       y c c}}w )Nr*  abcdefghijklmnopqrstuvwxyzhczzkylaaper,   )r-   r.   r"   r$   rP   r  rR   r=   rh   r[  r  rg   r@   rk   r   )r   r.   rQ   rl   r  ra  rR  df2s           r(   test_csv_getitem_column_orderr    s     77<<FZ0D/0G%G%&F
,,#gv*>?$!QA?@
ACJJt =!G
++d
G
$
)
)!
,Cc'lC  @s   0C
c                   t         j                  j                  t        |       d      }d}t        j                  t        |            }t        |d      5 }|j                  |       d d d        t        j                  t        d      5  t        j                  |d       d d d        t        j                  |dd	      }t        ||       y # 1 sw Y   ixY w# 1 sw Y   :xY w)
Nr*  s   a,b
1,"hi
"
2,"oi
"
rD  zEOF encounteredr   r   rP  r,   )rQ  sample_rows)r-   r.   r"   r$   r=   r@   r   rF  rG  r   r   r   rg   r   )r   r.   r  r{   r   rF   s         r(   test_csv_parse_failr  "  s    77<<FZ0D'D{{74=)H	dD	 Q		z):	; %
D$%	T"!	4Bb( % %s   CCCC%c                   dd l }ddlm} t        j                  j                  t        |       d      }t        j                  j                  t        |       d      }t        |d      5 }t        d      D ]O  }|j                  |j                  dd      d	d
|j                  dd      d	d
|j                  dd      d	d       Q 	 d d d         |||       t        |d      5 }t        d      D ]O  }|j                  |j                  dd      d	d
|j                  dd      d	d
|j                  dd      d	d       Q 	 d d d        t        j                  |d dt        d       }t        j                  |d dt        d       }|j                  j                         |j                  j                         k7  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)Nr   )copyfilezold.csvnew_csvrk  rW  r,   i ʚ;09z, r!   r  rN   r0   )rb   r  r^   r   )randomshutilr  r-   r.   r"   r$   rF  r  rG  	randrangerg   r@   re   r  )	r   r  r  old_csv_pathnew_csv_pathr   r  new_dfold_dfs	            r(   6test_csv_name_should_be_different_even_if_head_is_samer  /  s   77<<FY7L77<<FY7L 
lC	  Ar 	AGG##Au-b1F4D4DQ4Nr3RRTU[UeUefginUoprTssuv	 \<( 
lC	  Aq 	AGG##Au-b1F4D4DQ4Nr3RRTU[UeUefginUoprTssuv	 [[TSt  F [[TSt  F ;;!1!1!3333-  s   )AG %AG G	Gc                   t        dd      D cg c]  }| c}t        dd      D cg c]  }| c}d}t        j                  |      }t        |       dz   }t        d      D ]  }|j	                  | d| d	d
        t
        j                  |dz   d      }t        |j                  t        j                  |j                  gdz               y c c}w c c}w )Nr   rz  r   )col1col2)r  /rO   file_r  Fr7  r  Tr   )
r  r=   rh   r$   r  rg   r@   r   r  r>   )r   r  r  rF   	temp_pathr   s         r(   $test_select_with_include_path_columnr  Q  s     #1c]+!+sC5QAa5QRA	1	BFc!I1X ;
		YKuQCt,E	:; ++i')t+
DCchh		277)a-01 ,5Qs
   	C	C	use_namesc                ^   t        d      }|r	g d}ddg}nd x}}t        j                  j                  t	        |       d      }t
        j                  |d       j                  |dd       t
        j                  |d	||
      }t        j                  |d	||d      }t        ||d       y )Nz    city1,1992-09-13,10
    city2,1992-09-13,14
    city3,1992-09-13,98
    city4,1992-09-13,13
    city5,1992-09-13,45
    city6,1992-09-13,64
    )cityr  salesr  r  z	input.csvr   Frw  r   )rb   r   rZ   <   )rb   r   rZ   r   r   )
r   r-   r.   r"   r$   r=   r@   r  rg   r   )r   r  r  r   rZ   r.   rF   r   s           r(   test_names_with_header_0r  `  s     		C )7#77<<F[1DKKDK!((U5(I	T!5'	BB
++  C b#5)r*   )
__future__r   r  rB  r-   rh  r  r   r   unittestr   r   r   r=   rg   fsspecfsspec.compressionr   packaging.versionr	   tlzr
   r   re   	dask.baser   dask.bytes.corer   dask.bytes.utilsr   	dask.corer   dask.dataframe._compatr   r   r   dask.dataframe.io.csvr   r   r   r   r   dask.dataframe.utilsr   r   r   r   
dask.utilsr   r   r   r   compression_fmtsr)   r/   r%   r   r   tsv_text	tsv_text2r  rA   r   	tsv_files	fwf_filesrG   rL   r   r   csv_units_rowtsv_units_rowmarkparametrizer@   
read_tableread_fwfcsv_and_tablerX   r[   r_   rc   rm   rr   r   r   r   r   r   
csv_blocks
tsv_blocksr   r   r   r   r   r   r   r   r   r   r   r   r   r  r	  r  r  r  r   r(  r1  r?  slowrK  rN  rR  rV  rc  r   rm  rq  rt  rw  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r(  r,  r3  r6  r=  rA  rC  xfailrG  skipif__version__rM  rP  rS  rU  r`  rf  ri  rn  rt  r}  r  r  r  rP   r  r  r  r  r  r  r  r  )rX  rE   ra  r  s   0000r(   <module>r     s
   "   	     V"V)*  $ % %  . & %  C C   < ; $))CC)TF2 <" 
EG 	" C& 
EG 
" 
EG  Q)Q	 6?__5FG6AqQ		$&&G		"
 0	"	" $ 
 #%%dE2 ''	i 		"	i  $ $ . . & & $ $ B B$ 
0 
0  6 kk2;;	*R]]BMM9,UV6	6 kk2;;	*R]]BMM9,UV
6	
6 kk2;;	*R]]BMM9,UV6	6 !	bkk9m<	y-@	6	6 _-_-
  12 12
 r{{J/"--1LMKK r{{J/"--1LM,, 	bkk8S1	x6	y&9006 	bkk8Q/	x!R9CC 	bkk8Q/	x!R9
F
F kk2;;	*R]]BMM9,UV	4		4 kk2;;	*R]]BMM9,UV	 r{{I.	0JK
-
- r{{I.	0JK
-
- r{{I.	0JKAA r{{I.	0JK	1	1 r{{I.	0JK
1
1"$&$L"0(2$72Y<<+~ &(9:H ; H .<" tRj1 01
 2 2
. E E*= 9-3 .3 A*"&P)$	"P"*AH +F
.$	 %VW X"75"'J#"
$N
X
	T
:.*"<6 r"&	S.E	" ?@
. A
. F7:#666  3	3
3:6#1&: 
$ +k?-KL M E(''&
ui3
tY0

:& 
Q!i 
	R!b1#!&r+AeAh+	88
1.
1! 
4D2 tUm4$* 5$*o9 *D Ht3 	,s   5	e:&e? f