
    bi0K              	         d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlZ ej                  d      Z ej                  d      Z ej                  dd	      Z ej                  d
        ej                  d      Zd dlmZ d dlmZmZ d dlmZ d dlmZmZ 	 d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z%  ee!d      Z!dZ&dddZ'e	d        Z(ejR                  d        Z*dZ+ ejR                  d      d        Z,ejR                  d        Z-e	e&e'fd        Z. ejR                         d!        Z/d" Z0d# Z1d$ Z2d% Z3d& Z4d' Z5d( Z6ejn                  jq                  d)g d*      d+        Z9ejn                  jq                  d)g d*      d,        Z:ejn                  jq                  d-eD  cg c]  } | df c} eD  cg c]  } | d.f c} z         d/        Z;ejn                  jq                  d0d1d2g      d3        Z<d4 Z=d5 Z> ejR                  d6g7      d8        Z?ejn                  j                  d9      ejn                  jq                  d:d;d<g      d=               ZAd> ZBd? ZCy# e$ r dZY w xY wc c} w c c} w )@    )annotationsN)contextmanager)partials3fsboto3motoz1.3.14)
minversionflaskrequests)compr)get_fs_token_paths
open_files)S3FileSystem)concatvalmap)compute)
read_bytes)compresssync)	schedulertests   {"amount": 100, "name": "Alice"}
{"amount": 200, "name": "Bob"}
{"amount": 300, "name": "Charlie"}
{"amount": 400, "name": "Dennis"}
s   {"amount": 500, "name": "Alice"}
{"amount": 600, "name": "Bob"}
{"amount": 700, "name": "Charlie"}
{"amount": 800, "name": "Dennis"}
)ztest/accounts.1.jsonztest/accounts.2.jsonc               #  F  K   t        t        j                        } 	 d t        j                  j                          t        j                  j	                  |        y# t        j                  j                          t        j                  j	                  |        w xY ww)z
    Get a context manager to safely set environment variables
    All changes will be undone on close, hence environment variables set
    within this contextmanager will neither persist nor change global state.
    N)dictosenvironclearupdate)saved_environs    S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/bytes/tests/test_s3.py!ensure_safe_environment_variablesr    6   se      $M)





-( 	




-(s   B!A >B!?BB!c                     t        ddi      S )Nendpoint_urlhttp://127.0.0.1:5555/)client_kwargs)r        r   s3sor'   E   s    ~/GHIIr&   r#   module)scopec               #     K   t               5  dt        j                  d<   dt        j                  d<   dt        j                  d<   dt        j                  d<   t        j                  t        j                  d      t        j                  	      } d
}	 	 t        j                  t              }|j                  rn(	 |dz  }t        j                  d       |dkD  sJ d       Od  | j                          	 | j!                  d       d d d        y # t        $ r Y bw xY w# t        j"                  $ rI | j%                          t&        j(                  dk(  r#t        j*                  d| j,                   d       Y sw xY w# 1 sw Y   y xY ww)N
foobar_keyAWS_ACCESS_KEY_IDfoobar_secretAWS_SECRET_ACCESS_KEY AWS_SHARED_CREDENTIALS_FILEAWS_CONFIG_FILEzmoto_server s3 -p 5555)stdout   g?r   z!Timed out waiting for moto server   )timeoutwin32zTASKKILL /F /PID z /T)r    r   r   
subprocessPopenshlexsplitDEVNULLr   getendpoint_uriok	Exceptiontimesleep	terminatewaitTimeoutExpiredkillsysplatformcallpid)procr5   rs      r   s3_baserL   M   sg    	*	, "C*6

&'.=

*+46

01(*

$% KK01*:L:L
 LL.44  sNGJJsOQ;C CC;  	 		CIIaI ;"C "C&   (( 	CIIK||w&"3DHH:S AB		C="C "Cs`   FBF%D;=F9D#	F	D FD  F#AE?<F>E??FFFc              #  R   K   t               5 }| d d d        y # 1 sw Y   y xY wwN)
s3_context)rL   fss     r   s3rQ   t   s&     	   s   '	'$'c              #    K   t         j                  dt              }|j                  | d       |j	                         D ]  \  }}|j                  | ||        t        j                  dddi	      }t        j                  j                          |j                          	 | |j                  | d
       y # |j                  | d
       w xY ww)NrQ   r"   zpublic-read-write)BucketACLrT   KeyBodyTr"   r#   )anonr$   )	recursive)r   clientr=   create_bucketitems
put_objectr   r   clear_instance_cacheinvalidate_cacherm)bucketfilesr[   fdatarP   s         r   rO   rO   z   s     \\$\\:F
,?@;;= ;4QT:;			.2J!K 
 
B 	**,&
f%f%s   B"C%B= )C=CCc              #    K   t        j                  d      }t        j                  d      }i dddddddddd	d	d
d
dddddddddddddddddddddddddddddddd d!d"d#dd$ddddddd%d&d&d&d&d&dd'd(d)d*d+d,dd-d.d/d0d1d2dd3ddddddd4d5d6d7d8d9dd:d;dddddd<ddddddd=d>d?d@d@d@ddAd@d@d@d@d@ddBdBdBdBdBddCdDdEdFdGddH}|j                  |      }|j                  |j	                  dI      j                  dJ            }t        j                         }t        j                  |      }|j                  |dKL       dM}t        j                  dNdOP      }	|	j                  t        ||Q       dR}t        ddS      D ]:  }
|j                  dT       |	j                  t        |j!                  |
      |Q       < dU yUw)Vz
    Fixture with sample yellowtrip CSVs loaded into S3.

    Provides the following CSVs:

    * s3://test/nyc-taxi/2015/yellow_tripdata_2015-01.csv
    * s3://test/nyc-taxi/2014/yellow_tripdata_2015-mm.csv
      for mm from 01 - 12.
    numpypandasVendorID      )r   rk   rj   r4      tpep_pickup_datetimez2015-01-15 19:05:39z2015-01-10 20:33:38z2015-01-10 20:33:39tpep_dropoff_datetimez2015-01-15 19:23:42z2015-01-10 20:53:28z2015-01-10 20:43:41z2015-01-10 20:35:31z2015-01-10 20:52:58passenger_counttrip_distancegq=
ףp?gffffff
@g?g      ?g      @pickup_longitudeg    Rg    Rg   `}Rg   Rg   '~Rpickup_latitudeg   `D@g    \D@g   fD@g   `^[D@g   @aD@
RateCodeIDstore_and_fwd_flagNdropoff_longitudeg   b~Rg   Rg   |Rg  FRg   DRdropoff_latitudeg   @`D@g   *aD@g   `iD@g   (\D@g   @_D@payment_typefare_amountg      (@g      -@g      #@g      @g      .@extrag      ?mta_tax
tip_amountg      
@g       @g        tolls_amountg333333?g1@g1@g%@g333333@gL0@)improvement_surchargetotal_amount   i'  Findexz)nyc-taxi/2015/yellow_tripdata_2015-01.csvrQ   r#   rS   rV   z.nyc-taxi/2014/yellow_tripdata_2014-{:0>2d}.csv   r   N)pytestimportorskip	DataFrametakearangerepeatioBytesIOTextIOWrapperto_csvr   r[   r^   test_bucket_namerangeseekformat)rQ   nppdre   sampledffilesfilekeyr[   is              r   s3_with_yellow_tripdatar      s     
		W	%B			X	&B8aA!28$$$$$!
8 	 $$$$$"
8  	qQ1q9!8" 	TccccB#8$ 	!!!!
%82 	  !
38@ 	!aA!4A8B 	#####FC8D 	!!!!!
E8R 	    !
S8` 	A!aA6a8b 	4DSSTBc8d 	SSSSS9e8f 	sssss;g8h 	$3333?i8j 	CCCCC@k8l &)SSSS!I!dtEo8Dr \\$F	RYYq\((/	0B::<DT"EIIe5I!
5C\\$-E\FF
-3TB
:C1b\ Q		!!1szz!}4PQ 
s   GGc                    t        dd      } | j                  dk(  sJ | j                  dk(  sJ t        dd      } | j                  dk(  sJ | j                  dk(  sJ t        j                  t
              5  t        dd       d d d        t        j                  t
              5  t        dd       d d d        y # 1 sw Y   9xY w# 1 sw Y   y xY w)Nr   secret)r   r   )usernamepassword)r   r   )r   r   )DaskS3FileSystemr   r   r   raisesKeyError)rQ   s    r   test_get_s3r      s    	eH	5B66U??99   	58	<B66U??99   	x	  4UU34	x	  767 74 47 7s   8B>'C
>C
Cc                   t         D cg c]  }dt        z   dz   |z    }}t        |fddi|}t        |t         j	                               D ]!  \  }}|5 }|j                  |       d d d        # t        dt        z   dz   fi |\  }}t        t        |       }	t        t         j	                               t        |	      k(  sJ y c c}w # 1 sw Y   xY w)Ns3://z/more/modewbz/more/test/accounts.*)
rc   r   r   zipvalueswriter   r   r   set)
rQ   r'   rd   pathsfilsfilre   r   r   resultss
             r   test_open_files_writer      s    @EF1W''(2Q6FEFe/$/$/Du||~. 	T 	AGGDM	 	  ""%<<@DNFF vf~&Gu||~#g,... G	 	s   CCC	c                $   t        dt        z   dz   fi |\  }}t        |t              sJ |d d t        t        t              d      d d k(  sJ |j                  d      sJ t        |t        t        f      sJ t        |d   t        t        f      sJ t        |d   d   d      sJ t        t        t        |            t        t              k\  sJ t        t        |       }t        |      t        t        j!                               k(  sJ y )Nr   /test/accounts.*r   r      
dask)r   r   
isinstancebytesrc   sortedendswithlisttuplehasattrsummaplenr   r   r   r   )rQ   r'   r   r   r   s        r   test_read_bytesr      s    *: :=O OXSWXNFFfe$$$"1:ve}Q/0!4444??5!!!ftUm,,,fQi$///6!9Q<(((s3 CJ...vf~&Gw<3u||~....r&   c                   t        dt        z   dz   fddd|\  }}|j                  d      sJ t        dt        z   dz   fddd|\  }}|j                  d      sJ t        dt        z   dz   fddd|\  }}|j                  d      sJ y )Nr   r   P   r   )r   	delimiterz/test/accounts.1.jsonrj   )r   r   r   )rQ   r'   r   r   s       r    test_read_bytes_sample_delimiterr     s    ""%77 	NFF ??5!!!""%<< 	NFF ??5!!!""%<< 	NFF ??5!!!r&   c                    t        j                  t              5  t        dt        z   dz   fi | d d d        y # 1 sw Y   y xY w)Nr   z/non-existing/*)r   r   IOErrorr   r   )rQ   r'   s     r   !test_read_bytes_non_existing_globr   &  s@    	w	 K7--0AAJTJK K Ks	   :Ac                    t        dt        z   dz   fdd i|\  }}t        t        t        |            t	        t
              k(  sJ y )Nr   r   	blocksize)r   r   r   r   r   rc   )rQ   r'   _r   s       r   test_read_bytes_blocksize_noner   +  sO    ""%77CGKOIAv s3 CJ...r&   c                    t        dt         dfd dd|\  }}t        |      dk(  sJ t        dt         dfd dd|\  }}t        |      dk(  sJ y )Nr   z*/nyc-taxi/2015/yellow_tripdata_2015-01.csvT)r   rY   rk   z/nyc-taxi/2014/*.csv   )r   r   r   )r   r'   r   Ls       r   'test_read_bytes_blocksize_on_large_datar   2  s    
 !!KL 	DAq q6Q;;
 !!56 	DAq q6R<<r&   r   )r      -   i  c           	     V   t        dt        z   dz   fd|i|\  }}t        t        t        |            t
        j                         D cg c]  }t        t	        |      |z  d       c}k(  sJ t        t        |       }t        d |D              t        d t
        j                         D              k(  sJ dj                  |      j                  d      }dj                  t
        j                               j                  d      }t        |      t        |      k(  sJ y c c}w )	Nr   z/test/account*r   rk   c              3  2   K   | ]  }t        |        y wrN   r   .0rK   s     r   	<genexpr>z(test_read_bytes_block.<locals>.<genexpr>N  s     '!s1v'   c              3  2   K   | ]  }t        |        y wrN   r   r   vs     r   r   z(test_read_bytes_block.<locals>.<genexpr>N  s     .N!s1v.Nr   r&   r   )r   r   r   r   r   rc   r   maxr   r   r   joinr:   r   )	rQ   r   r'   r   valsr   r   ourlines	testliness	            r   test_read_bytes_blockr   D  s   ""%55AJNRGAt C/4||~$*+SVy 1%$    vd|$G'w''3.Nu||~.N+NNNNxx &&u-H(..u5Ix=C	N***$s   
D&c                   t        dt        z   dz   f|dd|\  }}t        dt        z   dz   f|dd|\  }}t        |      D cg c]  }|j                   c}t        |      D cg c]  }|j                   c}k7  sJ t	        t        |       }|D 	cg c]  }	|	s|		 }
}	t        d |
D              sJ dj                  |
      j                  d      }dj                  d t        t              D              j                  d      }||k(  sJ d	}t        dt        z   dz   f||d|\  }}t	        t        |       }|D 	cg c]  }	|	s|		 }
}	t        d
 |
D              t        |
      dz
  k(  sJ dj                  |
      }dj                  d t        t              D              }||k(  sJ y c c}w c c}w c c}	w c c}	w )Nr   z/test/accounts*r   )r   r   s   fooc              3  >   K   | ]  }|j                  d         yw)r   Nr   r   s     r   r   z,test_read_bytes_delimited.<locals>.<genexpr>g  s     .Qqzz% .   r&   c              3  .   K   | ]  }t         |     y wrN   rc   )r   ks     r   r   z,test_read_bytes_delimited.<locals>.<genexpr>i  s     9aq9      }c              3  >   K   | ]  }|j                  d         yw)r   Nr   r   s     r   r   z,test_read_bytes_delimited.<locals>.<genexpr>w  s     -Aqzz$-r   rj   c              3  .   K   | ]  }t         |     y wrN   r   r   s     r   r   z,test_read_bytes_delimited.<locals>.<genexpr>y  s     4E!H4r   )r   r   r   r   r   allr   r:   r   rc   r   r   )rQ   r   r'   r   r   values2abr   rK   resr   r   doursr   s                   r   test_read_bytes_delimitedr   U  s   ""%66 	IAv ""%66 	JAw "&>*aAEE*fWo.Nquu.NNNNvf~&G
#1
#C
#.#....xx}""5)H96%=99??FIy    	A""%66 	IAv vf~&G
#1
#C
#---SA===88C=D884fUm44D4<</ +.N $ $s$   F6(F;G G GGzfmt,blocksize
   c           	        |t         vrt        j                  d       | j                  j	                          t        dt        t         |   t                    5  |r=|r;t        j                  t              5  t        	 d||d| d d d        	 d d d        y t        	 d||d|\  }}|j                  t        t        t              d      d d       sJ |j                  d      sJ t        t        |       }dj!                  |      dj!                  t        t              D cg c]  }t        |    c}      k(  sJ 	 d d d        y # 1 sw Y   xY wc c}w # 1 sw Y   y xY w)	Nz!compression function not providedr   )compressionr   r   r   r   r&   zs3://compress/test/accounts.*)r   r   skip_cacher   rO   r   rc   r   
ValueErrorr   
startswithr   r   r   r   r   )rQ   fmtr   r'   r   r   r   r   s           r   test_compressionr   }  s]   
 (78IIOO	Jx}e <	= P9z* 3 #' 	 P P $+

 	
   ve}Q'7!8"!=>>>u%%%6&>*xx CHHu-N1eAh-N$OOOO)P P $ .O)P Ps7   E7E	EBE(E8EE	EE'r   rtrbc                L   t        dt        z   dz   fd|i|}t        |      t        t              k(  sJ t	        |t        t                    D ]G  \  }}|5 }|j                         }t        |   }|dk(  r||k(  sJ |j                         sJ d d d        I y # 1 sw Y   TxY w)Nr   r   r   r   )r   r   r   rc   r   r   readdecode)	rQ   r   r'   myfiles	lazy_filepathrd   re   sols	            r   test_open_filesr    s    ""%77>BFJG w<3u:%%%wu6 A	4 	A!668D+C"&$,43;@@CJJL@@	A 	AA	A 	As   8BB#	c                    | dz  S )Nrj   r%   )xs    r   <lambda>r	    s
    1q5 r&   c                b   t        dt              5  t        dddi|\  }}t        dddi|\  }}t        |      D cg c]  }|j                   c}t        |      D cg c]  }|j                   c}k(  sJ 	 d d d        t        dt        t        t                    5  t        dddi|\  }}d d d        t              D cg c]  }|j                   c}t              D cg c]  }|j                   c}k7  sJ y c c}w c c}w # 1 sw Y   xY w# 1 sw Y   ixY wc c}w c c}w )Nr   rY   Tr   )rO   rc   r   r   _keyr   double)	rQ   r'   r   r   r   aabbcccs	            r   !test_modification_time_read_bytesr    s   	J	& NMMM1MMM1"(),B,6!90MR0MMMM	N 
Jvu 5	6 NMMM1N %Qi(BGG(vay,IRWW,IIII -0M	N NN N ),Is@   .DDD!D
4D"DD'*D,
DDD$pyarrow)paramsc                X    t        j                  | j                         | j                  S rN   )r   r   param)requests    r   enginer    s    
&==r&   zignore:Dask annotationsmetadata_fileTFc           
     &   t        j                  d      }t        j                  d      }t        j                  d      }dt        z  }|j                  |j	                  d|j
                        |j	                  d|j                        |j	                  d|j                        |j                  j                  g dd      j                  d	      d
|j                  |j	                  d      d            }|j                  |d      }	|	j                  ||||       | j                  |      D 
cg c]  }
|
j                  d      d    }}
|rd|v sJ d|v sJ d|v sJ |j!                  |dd||      }t#        |j$                        dkD  sJ |j&                  j)                  ||       t*        rrt        j,                  t.              5  |j!                  |||ddddi      j1                          d d d        |j!                  |||ddddi      j1                          t3        ||      d    dd!fd"
}t        j,                  t4              5  |j!                  ||||d#d$      j1                          d d d        |j!                  |||d%|i      }|j&                  j)                  ||       |j!                  |||d&d'i      }|j&                  j)                  ||       y c c}
w # 1 sw Y   xY w# 1 sw Y   xY w)(Ndask.dataframerh   rg   s3://%s/test.parquet  dtypehelloyoupeoplesizeOi32i64rd   bhellofoo)namer     	chunksize)r  storage_optionswrite_metadata_file/_common_metadata	_metadatapart.0.parquetT)r   calculate_divisionsr  r/  rk   precache_optionsparquet)methodr  )r  r/  open_file_optionsi@  )r9  	max_block)r/  r   )checkc                0    | sJ  j                   |i |S rN   )open)r<  argskwargsrP   s      r   _openztest_parquet.<locals>._open  s     urww'''r&   F)open_file_funcr<  rB  
cache_typer   )r   r   r   r   r   int32int64float64randomchoiceastypeIndexfrom_pandas
to_parquetlsr:   read_parquetr   	divisionsutils	assert_eqfsspec_parquetr   r   r   r   AssertionError)rQ   r  r'   r  ddr   r   urlre   r   rd   rc   df2rA  df3df4rP   s                   @r   test_parquetrY    s    
		-	.B			X	&B			W	%B
 #3
3C<<99T9299T924rzz2ii&&'A&MTT		
 hhryyUh3  
D 
	,BMMFDm   (*uuSz2!QWWS\"2E2!U***e###u$$$
//5d6SW  C s}}!!!HHtS!  ]]:& 	OO $&9(N#	   gi	 	 "yu$M	 	 	
 ') 
C	6q	9B (
 
~	& 
 16G	 	 	

 ') //+U3	  C HHtS! //'/	  C HHtS!O 3$	 	: s   :K6)K; 'L;LLc           	     P   t        j                  d      }t        j                  d      }t        j                  d      }dt        z  }|j                  |j	                  d|j
                        |j	                  d|j                        |j	                  d|j                        |j                  j                  g dd      j                  d	      d
      }|j                  |d      }|j                  |||dd       |j                  |||ddd       | j                  |      D 	cg c]  }	|	j                  d      d    }
}	d|
v sJ d|
v sJ d|
v sJ |j                  |d||      }|j                   j#                  |j%                  ||g      |d       y c c}	w )Nr  rh   rg   zs3://%s/test.parquet.appendr  r  r  r#  r%  r&  r,  r-  FT)r  r/  write_indexr0  )r  r/  r[  appendignore_divisionsr1  r2  r3  r4  r5  )r   r  r/  )check_index)r   r   r   r   r   rD  rE  rF  rG  rH  rI  rK  rL  rM  r:   rN  rP  rQ  r   )rQ   r  r'   rT  r   r   rU  re   r   rd   rc   rV  s               r   test_parquet_appendr_  "  s   			-	.B			X	&B			W	%B
'*:
:C<<99T9299T924rzz2ii&&'A&MTT		
	D 
	,BMM    MM   (*uuSz2!QWWS\"2E2&&&%u$$$
//	  C HH
		4,   3s   /F#c                   t        j                  d      }t        j                  d      }t        j                  d      }dt        z  }|j                  d|j	                  g d      i      }|j                  |d      }|j                  ||d	|d
       |j                  ||dd	i|       | j                         j                  d	u sJ |j                  ||dd
i|       | j                         j                  d
u sJ |j                  ||ddi|       | j                         j                  dk(  sJ | j                         j                  |dz         5 }	|	j                  dk(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr  rh   rg   r  r'  )r   r   rj   r   r,  r-  FT)r  r[  r/  r0  default_fill_cache)r  r/  default_block_sizei   z
/_metadata)r   r   r   r   arrayrK  rL  rN  currentra  rb  r>  r   )
rQ   r'   r  rT  r   r   rU  re   r   rd   s
             r   test_parquet_wstorageparsre  W  s   			-	.B			X	&B			W	%B
 #3
3C<< 678D		,BMM    OO-u==  
 ::<**e333OO-t<t<  
 ::<**d222OO-u==  
 ::<**e333			3-	. $!{{e###$ $ $s   E77F )D
__future__r   r   r   r9   r7   rF   r@   
contextlibr   	functoolsr   r   r   r   r   r   r   fsspec.compressionr   fsspec.corer   r   r   r   tlzr   r   fsspec.parquetr8  rR  ImportErrorr   r   dask.bytes.corer   dask.bytes.utilsr   r   rc   r    fixturer'   r=   rL   rQ   rO   r   r   r   r   r   r   r   r   markparametrizer   r   r   r  r  r  r  filterwarningsrY  r_  re  )r   s   0r   <module>rt     s   " 	 	   
  %  v6"G$v6h7   G 6z* $ 6 1 +  & %
'V
,  	/	/	  ) ) J J ( h#C  #CL  
 &e & &  T
 T
n7
//"0K
/$ &78+ 9+  &78$ 9$N !"Sc4["5%ACsBi%AAP	P6 $.	A /	A 

J 	{# $ 564-8]" 9 7]"@2j%$y  NH #%As   (I) 1I7
I<)I43I4