
    bi,                       d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ  eed      ZdddZddddZd Zd Z d Z!d Z"d Z#d Z$d Z%ejL                  jO                  dddg      d         Z(d! Z)d" Z*d# Z+ejL                  jY                  ejZ                  d$k(  d%&      d'        Z.d( Z/d) Z0eD  cg c]  } | df c} eD  cg c]  } | d*f c} z   Z1ejL                  jO                  d+e1      d,        Z2d- Z3ejL                  jO                  d.d/d0g      d1        Z4ejL                  jO                  d2d3d4g      ejL                  jO                  d5 e5e            d6               Z6d7 Z7d8 Z8ejL                  jr                  d9        Z:ejL                  jO                  d:de;fd;ejv                  fg      d<        Z<d= Z=d> Z>d? Z?yc c} w c c} w )@    )annotationsN)partial)sleep)compr)
open_files)LocalFileSystem)concatvalmap)compute)
read_bytes)compress)	filetextssync)	schedulers   {"amount": 100, "name": "Alice"}
{"amount": 200, "name": "Bob"}
{"amount": 300, "name": "Charlie"}
{"amount": 400, "name": "Dennis"}
s   {"amount": 500, "name": "Alice"}
{"amount": 600, "name": "Bob"}
{"amount": 700, "name": "Charlie"}
{"amount": 800, "name": "Dennis"}
).test.accounts.1.jsonz.test.accounts.2.jsons   a,b
1,2
s   a,b
3,4
s   a,b
5,6
)z.test.fakedata.1.csvz.test.fakedata.2.csvzsubdir/.test.fakedata.2.csvc                    t        j                  t        j                  j	                  |             j                         S N)pathlibPathospathabspathas_uri)r   s    V/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/bytes/tests/test_local.pyto_urir   /   s'    <<-.5577    c                 z    t        j                  t              5  t        h d       d d d        y # 1 sw Y   y xY w)N>   allowed.csvsets/are.csvshould/not/be.csvunordered/so/they.csv)pytestraises	TypeErrorr    r   r   test_unordered_urlpath_errorsr&   3   s0    	y	! 
	

 
 
s   1:c                 \   t        t        d      5  t        d      \  } }t        | t              sJ | d d t        t        t              d      d d k(  sJ | j                  d      sJ t        |t        t        f      sJ t        |d   t        t        f      sJ t        |d   d   d      sJ t        t        t        |            t        t              k\  sJ t        t        |       }t        |      t        t        j!                               k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nbmode.test.accounts.*   r      
dask)r   filesr   
isinstancebytessortedendswithlisttuplehasattrsummaplenr   r	   setvalues)sampler;   resultss      r   test_read_bytesr>   @   s   	5s	# 3#$67&%(((bqzU6%=#34Ra8888u%%%&4-000&)dE]333vay|V,,,3sF#$E
2226&>*7|s5<<>22223 3 3s   DD""D+c                 *   t        t        d      5  t        ddd      \  } }| j                  d      sJ t        ddd      \  } }| j                  d      sJ t        ddd      \  } }| j                  d      sJ 	 d d d        y # 1 sw Y   y xY w)	Nr(   r)   r+   P   r-   )r<   	delimiterr      )r   r/   r   r3   r<   r;   s     r    test_read_bytes_sample_delimiterrD   P   s    	5s	# &#$6rUSu%%%#$;BRWXu%%%#$;AQVWu%%%& & &s   A-B		Bc                     t        t        d      5  t        dd      \  } }t        |       dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr(   r)   r+   z40 Br<   (   )r   r/   r   r9   rC   s     r   test_parse_sample_bytesrH   Z   sD    	5s	# !#$6vF6{b   ! ! !s	   !=Ac                 ~    t        t        d      5  t        dd      \  } }| du sJ 	 d d d        y # 1 sw Y   y xY w)Nr(   r)   r   FrF   )r   r/   r   )r<   _s     r   test_read_bytes_no_samplerK   `   s>    	5s	# 6uE	  s   3<c                     t        t        d      5  t        dd       \  } }t        t	        t
        |            t        t              k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr(   r)   r+   	blocksize)r   r/   r   r7   r8   r9   rC   s     r   test_read_bytes_blocksize_nonerO   f   sP    	5s	# 3#$6$G3sF#$E
2223 3 3s   <AA!rN   g      @z5 Bc                j   t        t        d      5  t        d|       \  }}t        t	        |       }dj                  |      j                  d      }dj                  t        j                               j                  d      }t        |      t        |      k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr(   r)   .test.account*rM   r   r-   )	r   r/   r   r   r	   joinsplitr;   r:   )rN   r<   valsr=   ourlines	testliness         r   test_read_bytes_blocksize_typesrW   l   s    	5s	# /!"2iH6$<(88G$**51HHU\\^,2259	8}I.../ / /s   BB))B2c                     t        t        d      5  t        j                  t              5  t        dd       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr(   r)   rQ   g      @rM   )r   r/   r"   r#   r$   r   r%   r   r   $test_read_bytes_blocksize_float_errsrY   v   sR    	5s	# 8]]9% 	8'37	88 8	8 	88 8s!   AAAA	AA c                    t        t        d      5  t        dd      \  } } }|D ch c]$  }t        j                  j                  |      d   & c}t        j                         k(  sJ 	 d d d        y c c}w # 1 sw Y   y xY w)Nr(   r)   r+   T)include_path   )r   r/   r   r   r   rS   keys)rJ   pathsr   s      r   test_read_bytes_include_pathr_   |   sr    	5s	# J !3$G1e3894d#A&9UZZ\IIIJ J9J Js   A:)A5A:5A::Bc                     t        t        d      5  t        d      dz   } t        | d       \  }}t	        t        t        |            t        t              k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr(   r)   z.test.accounts.*rM   )r   r/   r   r   r7   r8   r9   urlr<   r;   s      r   test_with_urlsrd      s_    	5s	# 3&'#-#C483sF#$E
222	3 3 3s   A
A&&A/win32z!pathlib and moto clash on windows)reasonc                    t        t        d      5  t        j                  d      } t	        | d       \  }}t        t        t        |            t        t              k(  sJ 	 d d d        t        j                  t              5  t        j                  d      } t	        | d        d d d        y # 1 sw Y   NxY w# 1 sw Y   y xY w)Nr(   r)   z./.test.accounts.*rM   zfile://.test.accounts.*)r   r/   r   r   r   r7   r8   r9   r"   r#   OSErrorrb   s      r   test_with_pathsri      s    	5s	# 3ll/0#C483sF#$E
2223 
w	 (ll453$'( (	3 3( (s   AB1#B=1B:=Cc                    t        t        d      5  dD ]  } t        d|       \  }}t        t	        t
        |            t        j                         D cg c]  }t        t        |      | z  d       c}k(  sJ t        t        |       }t        d |D              t        d t        j                         D              k(  sJ d	j                  |      j                  d
      }d	j                  t        j                               j                  d
      }t        |      t        |      k(  rJ  	 d d d        y c c}w # 1 sw Y   y xY w)Nr(   r)   )r,      -   i  rQ   rM   r\   c              3  2   K   | ]  }t        |        y wr   r9   .0rs     r   	<genexpr>z(test_read_bytes_block.<locals>.<genexpr>   s     /!s1v/   c              3  2   K   | ]  }t        |        y wr   rn   rp   vs     r   rr   z(test_read_bytes_block.<locals>.<genexpr>   s     6V!s1v6Vrs   r   r-   )r   r/   r   r4   r8   r9   r;   maxr   r	   r7   rR   rS   r:   )bsr<   rT   rv   r=   rU   rV   s          r   test_read_bytes_blockry      s   	5s	# 3# 	3B%&6"ELFDC'05,+,SVr\A&,    vd|,G/w//36Vu||~6V3VVVVxx(..u5H066u=Ix=C	N222	33 3,3 3s%   AED?
6B:E2E?EEc                    t        t        d      5  dD ]  } t        d| d      \  }}t        d| d      \  }}t        |      D cg c]  }|j                   c}t        |      D cg c]  }|j                   c}k7  sJ t        t        |       }|D cg c]  }|s|	 }}t        d |D              sJ d	j                  |      j                  d      }	d	j                  d
 t        t              D              j                  d      }
|	|
k(  sJ d}t        d| |      \  }}t        t        |       }|D cg c]  }|s|	 }}t        d |D              t        |      dz
  k(  sJ d	j                  |      }d	j                  d t        t              D              }||k(  rJ  	 d d d        y c c}w c c}w c c}w c c}w # 1 sw Y   y xY w)Nr(   r)   )r,   rk   rl   z1.5 kBz.test.accounts*r-   )rN   rA   s   fooc              3  >   K   | ]  }|j                  d         yw)r-   Nr3   ro   s     r   rr   z,test_read_bytes_delimited.<locals>.<genexpr>   s     6Qqzz%(6   r   c              3  .   K   | ]  }t         |     y wr   r/   )rp   ks     r   rr   z,test_read_bytes_delimited.<locals>.<genexpr>   s      Aaq A      }c              3  >   K   | ]  }|j                  d         yw)r   Nr|   ro   s     r   rr   z,test_read_bytes_delimited.<locals>.<genexpr>   s     5Aqzz$'5r}   rB   c              3  .   K   | ]  }t         |     y wr   r   ru   s     r   rr   z,test_read_bytes_delimited.<locals>.<genexpr>   s     <E!H<r   )r   r/   r   r	   keyr   allrR   rS   r2   r7   r9   )rx   rJ   r;   values2ar(   r=   rq   resrU   rV   dourstests                 r   test_read_bytes_delimitedr      s   	5s	#  ' 	 B"#4eTIAv#$5vVJAw#)&>2aAEE2fWo6Vquu6VVVVvf~.G%+1+C+6#6666xx}**51H A6%= AAGGNIy((( A"#4aPIAvvf~.G%+1+C+555SAEEE88C=D88<fUm<<D4<<+	     36V , ,#   sU   6GF9G*F>
=GG!G%BG>GG
A G,G9GG
   zfmt,blocksizec           	        | t         vrt        j                  d       t        t         |    t              }t        |d      5  | r=|r;t        j                  t              5  t        d|d|        d d d        	 d d d        y t        d|d|       \  }}|d d t        t        t              d      d d k(  sJ |j                  d      sJ t        t        |       }d	j                  |      d	j                  t        t              D cg c]  }t        |    c}      k(  sJ 	 d d d        y # 1 sw Y   xY wc c}w # 1 sw Y   y xY w)
N!compression function not providedr(   r)   z.test.accounts.*.jsonr-   )rN   rA   compressionr,   r   r   )r   r"   skipr
   r/   r   r#   
ValueErrorr   r2   r3   r   r	   rR   )fmtrN   files2r<   r;   r=   r   s          r   test_compressionr      s:   
(78HSM5)F	6	$ P9z* +'# #	 P P $#	
 bqzU6%=#34Ra8888u%%%6&>*xx CHHu-N1eAh-N$OOOO)P P $ .O)P Ps7   D> D-0	D>BD>D9D>-D6	2D>>Ec                 P   t        t        d      5  t        d      } t        |       t        t              k(  sJ t	        | t        t                    D ]/  \  }}|5 }|j                         }|t        |   k(  sJ 	 d d d        1 	 d d d        y # 1 sw Y   ExY w# 1 sw Y   y xY wNr(   r)   r+   )r   r/   r   r9   zipr2   read)myfiles	lazy_file	data_filefxs        r   test_open_filesr      s    	5s	# -/07|s5z)))$'$? 	- Iy -aFFHE),,,,- -	-- -- -	- -s$   A
BB<
BBBB%encodingzutf-8asciic                   t        t        d      5  t        dd|       }t        |      t        t              k(  sJ g }|D ],  }|5 }|j	                  |j                                d d d        . t        |      t        t              D cg c]  }t        |   j                  |        c}k(  sJ 	 d d d        y # 1 sw Y   xY wc c}w # 1 sw Y   y xY w)Nr(   r)   r+   rt)r*   r   )	r   r/   r   r9   appendr   r4   r2   decode)r   r   datafiler   r   s         r   test_open_files_text_moder      s    	5s	# P/dXN7|s5z))) 	&D &AFFH%& &	& DzOAeAhooh7OOOOP P
& &OP Ps/   5C B<'%CC+C<CCCr*   r   rbr   c                
   |t         vrt        j                  d       t        t         |   t              }t        |d      5  t        d| |      }g }|D ],  }|5 }|j                  |j                                d d d        . t        t              D cg c]  }t        |    }}| dk(  r|D 	cg c]  }	|	j                          }}	t        |      |k(  sJ 	 d d d        y # 1 sw Y   xY wc c}w c c}	w # 1 sw Y   y xY w)Nr   r(   r)   r+   )r*   r   r   )r   r"   r   r
   r/   r   r   r   r   r2   r   r4   )
r*   r   r   r   r   r   r   r   solr(   s
             r   test_open_files_compressionr      s     (78HSM5)F	6	$ 	!/dL 	&D &AFFH%& &	& "(/AuQx//4<'*+!188:+C+DzS   	! 	!& &/+	! 	!s<   C9 C#:C9C/%C90C4C9#C,(C99Dc                     t        t        d      5  t        t        fD ]1  } t	        j
                  t              5   | dd      \  }}d d d        3 	 d d d        y # 1 sw Y   GxY w# 1 sw Y   y xY w)Nr(   r)   r+   z	not-found)r   )r   r/   r   r   r"   r#   r   )funcr<   r;   s      r   test_bad_compressionr     s}    	5s	# S, 	SDz* S!%&8k!RS S	SS SS SS Ss"   )A)A	
A)A&"A))A2c                     d} t        j                  t        t        f|       5  t	        |        d d d        y # 1 sw Y   y xY w)Nz
not-a-file)match)r"   r#   FileNotFoundErrorrh   r   )fns    r   test_not_foundr     s7    	B	)732	> 2  s	   9Ac                    t        t        d      5  t        d      \  } }t        d      \  } }t        t	        |            }t        t	        |            }|D cg c]  }|j
                   c}|D cg c]  }|j
                   c}k(  sJ t        d       t        D ](  }t        |d      5 }|j                  d       d d d        * t        d      \  } }t        t	        |            }|D cg c]  }|j
                   c}|D cg c]  }|j
                   c}k7  sJ 	 d d d        y c c}w c c}w # 1 sw Y   xY wc c}w c c}w # 1 sw Y   y xY w)Nr(   r)   r+   r\   ab   x)	r   r/   r   r4   r	   _keyr   openwrite)	rJ   r   r(   aabbr   r   cccs	            r   
test_namesr     s)   	5s	# >,-1,-1OO"#$B$1(=R(====a 	Bb$ 1 	 ,-1O"#$B$1(=R(====> > %(= 
 %(=> >sZ   A	ED-.E4D2'E.D7 0E0EE	EE-
E7E <EEcompression_openergzipc                   |\  }}t        |       } t        | dd|      }t        |      dk(  sJ |D ch c]  }|j                   c}dhk(  sJ |D ]  }|5 }|j	                  d       d d d          t        t        j                  |             }|ddgk(  sJ  |t        j                  j                  | |d         d      5 }|j                         }d d d        dk(  sJ y c c}w # 1 sw Y   xY w# 1 sw Y   "xY w)	NrB   wb)numr*   r   s   000z0.partz1.partr   r   )strr   r9   r*   r   r2   r   listdirr   rR   r   )tmpdirr   r   openerr/   r   filr   s           r   test_open_files_writer   (  s   ,K[Fv14[IEu:??!"qAFF"tf,,,  	AGGFO	 	 2::f%&EXx((((	VU1X.	5 FFH;; #	 	
 s   C%C*C6*C3	6C?c           	     *   t        |       } t        t        d      5  t        d      }t	        j
                  t	        j                  |            }t        ||      D ]  \  }}|j                  |j                  k(  sJ t        |j                        t        |j                        k(  sJ |5 }|5 }|j                         |j                         k(  sJ 	 d d d        d d d         	 d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY wr   )r   r   r/   r   cloudpickleloadsdumpsr   r   typefsr   )r   r   myfiles2r   f2f_openf2_opens          r   test_pickability_of_lazy_filesr   :  s    [F	5s	# 7/0$$[%6%6w%?@(+ 	7EAr66RWW$$$:bee,,, 7fb 7G{{}6667 7 7	7	7 77 7 7 77 7s<   BD	-C=0$C1	C=
D	1C:6C==DD		Dc                   t        | dz        }t        j                  |d      5 }|j                  d       d d d        t	        |dd      }|d   5 }t        d	 |D              sJ 	 d d d        y # 1 sw Y   ;xY w# 1 sw Y   y xY w)
Nzmyfile.txt.gzr   r)   s   hello
worldr   r   )r   r*   r   c              3  <   K   | ]  }t        |t                y wr   )r0   r   )rp   lines     r   rr   z'test_py2_local_bytes.<locals>.<genexpr>P  s     7T:dC(7s   )r   r   r   r   r   r   )r   r   r   r/   s       r   test_py2_local_bytesr   H  s    	Vo%	&B	2D	! !Q	 ! rvD9E	q 8Q7Q77778 8! !
8 8s   A2A>2A;>Bc                   t        |       } t        j                         }t        j                  |        t	        dd      5 }|j                  d       d d d        t               j                  d      }t        |      dk(  sJ d|d   v sJ d|d   v sJ t               }t        j                  |       |j	                  |d   d      5 }|j                         }d d d        dk(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w)	Ntmpwhira   r\   /r   rq   )
r   r   getcwdchdirr   r   r   globr9   r   )r   herer   outr   r   s         r   test_abs_pathsr   S  s    [F99;DHHV	eS	 Q	


 
 
%Cs8q==#a&==CF??		BHHTN	Q	 ffh$;;  s   C+
C7+C47D )@
__future__r   r   r   r   sys	functoolsr   timer   r   r"   fsspec.compressionr   fsspec.corer   fsspec.implementations.localr   tlzr	   r
   r.   r   dask.bytes.corer   dask.bytes.utilsr   
dask.utilsr   r/   	csv_filesr   r&   r>   rD   rH   rK   rO   markparametrizerW   rY   r_   rd   skipifplatformri   ry   r   fmt_bsr   r   r   r4   r   r   r   slowr   r   r   r   r   r   )r   s   0r   <module>r      sB   "  	  
     $ " 8   & %  
'V
,	/	/	" +*$1	8

3 &!3 sEl3/ 4/8J3 CLLG+4WX( Y(3  4 "'	'#3+	'*FC9*F	F &1P 2P6- gw%78P 9P $.U,! - /! S > >& -tvtyy>Q/RS T"78g 
(*Fs   .G??H