bib:ddlmZddlZddlZddlmZddlmZddlZ ddl Z ddl Z ddl mZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZmZe j<j>Z d Z!e j<jEeed k\d dZ#dZ$dZ%dZ&dZ'e j<jEeed k\d e j<jPdZ)dZ*e j<jEeed k\d e j<jWdgde j<jWdgddZ,dZ-dZ.e j<jWde j^gdgdd gd!"ej`fe jbgdd#$ejdfgd%Z3d&Z4e j<jEeed k\d d'Z5d(Z6d)Z7d*Z8d+Z9d,Z:d-Z;d.Zd1Z?y)2) annotationsN) lru_cache)sleep)Version) PY_VERSION)get_deps)tm) assert_eq)tmpdirtmpfilectjdtjgdgddgd}t j |d}t d5}|j|d tj|d }tj||dddddt d5}|jj|d tj|d }tj|j|dddddt j |d }t d5}|j|d tj|d }tj||dddddt d5}|j|d d }|jtj|d }tj||dddddy#1swYFxYw#1swYxYw#1swYxYw#1swYyxYw) Ntablesabcdxy?@@@indexrh5/datarF)compute)pytest importorskippd DataFramedd from_pandasr to_hdfread_hdfr assert_frame_equalrassert_series_equalr%)dfrfnoutrs [/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/dataframe/io/tests/test_hdf.py test_to_hdfr5s ! "6>R B r1A *" Wkk"g& b#a&)* -"  2wkk"g& rttSV,- r1A *" Wkk"g& b#a&)* *" HHR%H 0 kk"g& b#a&) **#** -- ** **s4AG,AG*+AG6AHG'*G36G?H z3.11z@segfaults due to https://github.com/PyTables/PyTables/issues/977)reasonctjdtjgdgddgd}t j |d}tjgdgd dgd }t j |d }t d 5}|j|d t j|d }t||dddt d 5}|j|d t j|d }t||dddt d 5}|j|ddt j|d}t||tj|d}tj||jddtj|d}tj||jdddddt d 5}tj|5}|j|d dddt j|d }t||dddy#1swYvxYw#1swY;xYw#1swYxYw#1swYZxYw#1swYyxYw)Nrrrrrr!rrrrrefghijklmnoprrrr rrrr g@g@g@g @g"@g$@g&@g(@g*@g,@g.@g0@rQr#/data*/data_*cd|dzzSNrrr=s r4z,test_to_hdf_multiple_nodes..|sq1u  name_functionz/data_az/data_aa)r&r'r(r)r*r+r r,r-r r r.ilocHDFStore)r0rdf16rr1r2hdfs r4test_to_hdf_multiple_nodesra9s  ! "6>R B r1A <<$I' * -( DR tR A " Xkk"h'"c " Xkk"h'$ 0" Y.EFkk"i("ckk"i( c2772A;/kk"j) c27712;/0 " [[_ $ HHS( # $kk"h'$ /  00 $ $sI 5H65IB*II(0I*I(6II II% !I((I1ctjdtjgdgddgd}t j |d}tjgdgd dgd }t j |d }t 5}tjj|d }|j|d t j|d }t||dddt 5}tjj|d }|j|d t j|d }t||dddt 5}tjj|d}tjj|d}|j|d |j|d t j||gd }ttj||g|dddt 5}tjj|d }|j|d dt j|d }t||tjtjj|dd }tj||j ddtjtjj|dd }tj||j dddddt#d5}tj$|5} |j| ddddt j|d}t||dddy#1swYxYw#1swYFxYw#1swYxYw#1swYxYw#1swYgxYw#1swYyxYw)Nrrrrrr!rr8rErRrQ data_*.h5r$z data_1.h5z data_2.h5cd|dzzSrVrWrXs r4rYz,test_to_hdf_multiple_files..scQUmrZr[z data_a.h5z data_aa.h5r#rS)r&r'r(r)r*r+r ospathjoinr,r-r concatr r.r]r r^) r0rr_rdnr1r2fn1fn2r`s r4test_to_hdf_multiple_filesrls ! "6>R B r1A <<$I' * -( DR tR A R WW\\"k * Wkk"g&"c  R WW\\"k * Wkk"g&$  .Rggll2{+ggll2{+ g gkk3*g."))T2J'- .  0R WW\\"k * W,CDkk"g&"ckk"'',,r;7A c2772A;/kk"'',,r<8'B c27712;/ 0 " [[_ $ HHS( # $kk"h'"c I.. 0 0 $ $sX AM?3AN BN DN&#N>9N2 *N>?N  NN#&N/2N; 7N>>Octjdtjgdgddgd}t j |d}t d5}|j|d |j|d d t j|d }tt j||g|dddt j |d}t d5}|j|d |j|d d t j|d }t||dddt j |d}t d5}|j|d |j|d d t j|d }tt j||g|dddt j |d}t d5}|j|d |j|d d t j|d }t||dddt j |d}t d5}|j|d|j|d |j|d d dt j|d }tt j||g|dddy#1swYxYw#1swYxYw#1swY1xYw#1swYxYw#1swYyxYw)Nrrrrrr!rr#z/data2rSrmodewrz/data1Froappend) r&r'r(r)r*r+r r,r-r rh)r0rr1r2s r4 test_to_hdf_modes_multiple_nodesrssx ! "6>R B r1A ," X XC(kk"h'"))RH%s+ , r1A " X XC(kk"h'"c  r1A ," X XC(kk"h'"))RH%s+ , r1A " X XC(kk"h'"c  r1A ," X X XC6kk"h'"))RH%s+ ,,A,,,,,,sAAKA KAKA K* A1K6K KK'*K36K?ctjdtjgdgddgd}t j |d}t 5}tjj|d}|jtjj|d d |j|d d t j|d }tt j||g|dddt j |d}t 5}tjj|d}|jtjj|d d |j|d d t j|d }tt j||g|dddt j |d}t 5}tjj|d}|jtjj|dd |j|d d t j|d }t||dddt j |d}t 5}tjj|d}|jtjj|dd |j|d d dt j|d }tt j||g|dddy#1swYCxYw#1swYxYw#1swYxYw#1swYyxYw)Nrrrrrr!rdata*data2r$rrnrSrdata1rpFrq)r&r'r(r)r*r+r rerfrgr,r-r rh)r0rrir1r2s r4 test_to_hdf_modes_multiple_filesrxs ! "6>R B r1A ,R WW\\"g & b'*G4 W3'kk"h'"))RH%s+ , r1A ,R WW\\"g & b'*G4 W3'kk"g&"))RH%s+ , r1A R WW\\"g & b'*G4 W3'kk"g&"c  r1A ,R WW\\"g & b'*G4 W3u5kk"g&"))RH%s+ ,,7,,,,,,s4BL3BMBM  BM3L=M  MM"ct|\}tdfdtfdjDS)N)maxsizecH|sydtfd|Dz}|S)Nrc3.K|] }|ywNrW.0dep_keymax_depth_by_depss r4 z>dependency_depth..max_depth_by_deps..MsH7%g.H)max)keyrdepsrs r4rz+dependency_depth..max_depth_by_depsHs+Cy Hd3iHH HrZc3.K|] }|ywr}rWr~s r4rz#dependency_depth..PsEg )Er)rrrkeys)dsk_rrs @@r4dependency_depthrEs@smGD!t EE EErZc|tjdtjgdgddgd}t j |d}d}t 5}|j|d | }|j|d t j|d }t||dddt5}tjj|d }|j|d | }|j|d t j|d }t||dddy#1swYxYw#1swYyxYw)Nrr8rErrRr!rQcf|jddkrtdd|jdz z|S)NrrKg?)r]rrXs r4 delayed_nopz,test_to_hdf_lock_delays..delayed_nops/ 66!9r> #affQi( )rZr)axismetarSrur$)r&r'r(r)r*r+r applyr,r-r r rerfrg)r_rrr1r2ris r4test_to_hdf_lock_delaysrSs  ! <<$I' * -( DR tR A b GGKaaG 0 Xkk"h'$  R WW\\"g & GGKaaG 0 Wkk"g&$ sA D&4A)D2&D/2D;ctjdtjgdgddgd}t j |d}t 5}tjt5tjj|d}|j|d ddddddt5}tj|5}tjt5|j|d dddddddddy#1swYwxYw#1swY{xYw#1swY2xYw#1swY6xYw#1swYyxYw) Nrrrrrr!rrcrTz /data_*_*)r&r'r(r)r*r+r raises ValueErrorrerfrgr,r r^)r0rrir1r`s r4test_to_hdf_exceptionsrs ! "6>R B r1A $R ]]: & $b+.B HHR # $$ +b [[_ +z* +k* + +++  $ $$$ + + + +++s`D%03D#D%=E D=-D1D=E D" D%%D.1D: 6D==E E  E scheduler)syncthreads processes npartitions)rrrKctjdtjgdgddgd}t j ||}t d5}|j|d | tj|d }t||dddt5}tjj|d }|j|d | t j|d }t||dddt d5}|j|d | t j|d }t||dddy#1swYxYw#1swY`xYw#1swYyxYw) Nrr8rErrRr!rr#r$rrcrS)r&r'r(r)r*r+r r,r-r r rerfrg)rrr0rr1r2ris r4test_to_hdf_schedulersrsC ! $I' * -( BR r{3A " W 2kk"g&"c R WW\\"k * W 2kk"g&"c  " X3kk"h'"c s%7E !AE 7E# EE #E,ctjdtjdddgi}t j |d}t d5}|j|dd d tj|d}tj||dddt d5}|j|dd d tj|d}tj||dddy#1swY_xYw#1swYyxYw) NrAraaaarrr#foo4tabler)format min_itemsizet) r&r'r(r)r*r+r r,r-r r.)r0ddfr1df2s r4test_to_hdf_kwargsrs ! sS&M* +B .. +C '" 2vgA >kk"f% b#&' '" 2vc :kk"f% b#&'' '' ''sAC1&AC=1C:=Dctjdtjgdgddgd}t j |d}t d5}tjtd 5|j|d t ddddddt5}tjj|d }|j|t dddy#1swY\xYw#1swY`xYw#1swYyxYw)Nrr8rErrRr!rQr#z2To preserve order between partitions name_functionmatchrSr[z data_*.csv)r&r'r(r)r*r+r warns UserWarningr,strr rerfrgto_csv)r_rr1ris r4test_to_fmt_warnsrs ! <<$I' * -( DR tR A 6" \\ S  6 HHRH 5 66 (R WW\\"l + 3'((  6 666 ((s0C33C' C3&8C?'C0 ,C33C<?Dz data, comparerrrrr!r)namec tjdtd5}|j|d t j |dddJ#t $r}d t|vsJYd}~nd}~wwxYw dddn #1swYnxYwtd5}|j|dd t j |ddd}|jdk(sJ||j||t j |ddd d djtj |dd d dddn #1swYnxYwtd5}|j}|j|dd t j |dddd}|jdk(sJ||j|dddy#1swYyxYw)Nrr#r$)rdatarr3 chunksizerozformat='table'rrrrr)rstartstoprorrT)r sorted_indexro) r&r'r r,r*r- TypeErrorrrr%r( sort_index)rcomparer1r9r sorted_datas r4 test_read_hdfrAs ! ." BG $ . KKFac : 5 .#s1v- -- .- ...   " BGG 4 KKGqs ;}}!!! T" KKGq L T T V KKG11 5      *"oo' 277; KKGqt# N}}!!! [) ***sHBA A3A.)B.A33BB BD==EA'GG c`tjdtjgdgddgd}t d5}|j |dd tj |d 5tj|dd d ddddddy#1swYxYw#1swYyxYw)zXTest that we can read from a file that's already opened elsewhere in read-only mode.rrrrrr!r#r$rrr3rnrrN) r&r'r(r)r r,r^r*r-r0r1s r4test_read_hdf_multiply_openrms ! "6>R B <" "'' 2 [[# & < KKGqs ; <<< < <<R B  3" "'' 2 ]]:-= > . KKG" - .]]:-C D / KKG2 . /]]:-? @ 3 KKGr 2 3 3 3 . . / / 3 3 3 3sT0D71D #D7-D#D7)D+D7D D7D( $D7+D4 0D77Ec Jtjdtjgdgddgd}t 5}|j t jj|ddd |j t jj|d d d |j t jj|d dd tjjd 5tjt jj|ddd}|jdk(sJtj |j#|tjt jj|ddddd}tjt jj|dddd}tj |j#|tjt jj|d dd}|jdk(sJtj |j#tj$|gdztjt jj|ddd}|jdk(sJtj |j#tj$|gdztjt jj|ddd}|jdk(sJtj |j#tj$|gdzddddddy#1swYxYw#1swYyxYw)Nrrrrrr!one.h5z /foo/datarrtwo.h5z /bar/datarrz/*/datarrrr)rrrrrz*.h5rG)r&r'r(r)r r,rerfrgdaskconfigsetr*r-rr r.r%rh)r0tdirresexpecteds r4test_hdf_globbingrs ! "6>R B FT "'',,tX.K P "'',,tX.K P "'',,tX.K P [[__v_ . F++bggll4:IQRSC??a' ''  ! !#++- 4++ T8,i1ATUC{{ T8,kH  ! !#++- :++bggll4:IQRSC??e+ ++  ! !#++-B4!81D E++bggll48+QRSC??e+ ++  ! !#++-B4!81D E++bggll48)qQC??i/ //  ! !#++-B4!81D E/ F FF  F F FFs&B7N7JN <N N NN"ctjdtjgdgddgd}t 5}|j ddj tjj|dd d |j ddj tjj|d d d tjjd 5tjj|dtjj|d g}tj|d }tj |j#|ddddddy#1swYxYw#1swYyxYw)Nrrrrrr!rr dataframerrrrr)r&r'r(r)r r]r,rerfrgrrrr*r-r r.r%)r0r input_filesrs r4test_hdf_file_listrs+ ! "6>R B  5T   GGLLx (k'     GGLLx (k'  [[__v_ . 577<<h7dH9UVK++k;7C  ! !#++- 4 5 5 5 5 5 5 5s&BE7A;E+E7+E4 0E77FcDtjdtjgdgddgd}t d5}t j |}|j|dd tj|d}t||dddy#1swYyxYw) Nrrrrrr!r#rrr) r&r'r(r)r pathlibPathr,r*r-r )r0r1rfrs r4test_read_hdf_pattern_pathlikers ! "6>R B "||B $K 8kk$ ,#r s A BBcntjdtjgdgddgd}t j |d}t d 5}tj|}|j|d tj|d }t||dddy#1swYyxYw) Nrrrrrr!rrr#r$) r&r'r(r)r*r+r rrr,r-r )r0rr1rfrs r4test_to_hdf_path_pathliker s ! "6>R B .. +C "||B 4!kk$(#s s A B++B4ctjdtd5}d}tjt j j|d}tj|d5}|jd|dddtj|dd }t||k(sJ dddy#1swY;xYw#1swYyxYw) Nrr#(rrprnz/xrr) r&r'r r(r)nprandomrandnr^rrr*r-len)r1Nr0storers r4test_read_hdf_doesnt_segfaultrs ! "  \\"))//!Q/ 0 [[# & #% LLr " #kk"da03x1}} # #s$A C.B;0C;C CCc"tjdtjgdgddgd}t j |d}|j d d d d gk(sJtjd tjd y) Nrrrrrr!rrz foo*.hdf5rz foo0.hdf5z foo1.hdf5) r&r'r(r)r*r+r,reremove)r0rs r4test_hdf_filenamesr%sq ! "6>R B .. +C ::k5 )k;-G GG GIIkIIkrZctjt5tjdddddtjt5tjddgddddtjt 5tjgddddy#1swY|xYw#1swYNxYw#1swYyxYw)Nnonexistant_store_X34HJKz/tmpnonexistant_store_UY56YH)r&rIOErrorr*r-rrWrZr4test_hdf_path_exceptionsr0s w 8 .78 w V /1KLfUV z "  B  88VV  s#B,B8 C,B58CC c tjd ddl G fdd j}G fdd j}G fdd j}t d 5} j |d 5}|j d d }|j|d|d}|j}d|d<|j|j|d|d}|j}d|d<|j|j|d|d}|j}d|d<|jdddtjtjjdd}|j|ddd t!j"|dt!j"|dt!j"|dt!j"|ddddy#1swYxYw#1swYyxYw) Nrrc0eZdZWjZy)'test_hdf_nonpandas_keys..Table1N)__name__ __module__ __qualname__ Float32Colvalue1rsr4Table1rE""$rZrc0eZdZWjZy)'test_hdf_nonpandas_keys..Table2N)rrrrvalue2rsr4Table2rHrrZrc0eZdZWjZy)'test_hdf_nonpandas_keys..Table3N)rrrrvalue3rsr4Table3rKrrZrr#rrn/grouptable1zTable 1rrtable2zTable 2rtable3zTable 3rrKrz/barr)rrroz /group/table1z /group/table2z /group/table3)r&r'r IsDescriptionr open_file create_group create_tablerowrrr(r)rrrr,r*r-) rrrrfh5filerrr barrs @r4test_hdf_nonpandas_keysr>s  *F%%%%%%%%%%%% "$   d  - ''W5E##E8VYGA%%CCM JJL##E8VYGA%%CCM JJL##E8VYGA%%CCM JJL  ll299??2q12 4VG# > D/* D/* D/* D&!/""  ""s&+G.?B2G"1B(G."G+ 'G..G7c0tjdddlm}t j ggdg}|j |dz ddd tjt| 5tj|dz ddddy#1swYyxYw) Nrr)dont_use_fixed_error_message)rBr!zdata.h5fixedr0rp)rrror) r&r'dask.dataframe.io.hdfrr(r)r,rrr*r-)tmp_pathrr0s r4test_hdf_empty_dataframerhsx !B BR( 3BIIh"73IG y(D E0 Hy($/000s )B  B)@ __future__rrer functoolsrtimernumpyrpandasr(r&packaging.versionrrdask.dataframerr*dask._compatibilityr dask.corerdask.dataframe._compatr dask.dataframe.utilsr dask.utilsr r markskip_with_pyarrow_strings pytestmarkr5skipifrarlrsrxrslowrr parametrizerrrr)r.Seriesr/rrrrrrrrrrrrrrWrZr4r+sO"  % *%*&[[ 2 2 *>'&/! MM M`Zz-,`(,V F'&/! MB  BJ+('&/! M&FG 3>4H  >B ' 6(r BLL*>*   ! !    <'&/! M2 2j3$"FJ5(     '"T0rZ