bik<ddlmZddlZddlZddlZddlZddlZddlmZejdejdZ ejdZ ejdZ ddl ZddlZddlmZej&j)ej*d k7d ej&j-d ej&j.gZej2dg dZedj7Zej:j=de_ej?Zej2ddZ ej&jCdgddZ"dZ#ej&jCdgddZ$dZ%dZ&y)) annotationsN) timeseriespandaspysparkpyarrowzdask.dataframe) assert_eqlinuxzAUnnecessary, and hard to get spark working on non-linux platforms)reasonz/pyspark doesn't yet have support for pandas 2.0)paramscXtj|j|jS)N)pytest importorskipparam)requests W/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/tests/test_spark_compat.pyenginer#s   & ==1h)freqUTCmodule)scopec#Ktjtj}tjj j jdjdjddj}||jtjtjur%tjtj|yyw)Nlocalz Dask Testingzspark.sql.session.timeZoner)signal getsignalSIGINTrsql SparkSessionbuildermasterappNameconfig getOrCreatestop threadingcurrent_thread main_thread)prevsparks r spark_sessionr+4s   FMM *D    ((//8  ,e 4   K JJL!Y%:%:%<< fmmT*=sC&C( npartitions) czt|}|jt}|j|jj |dt j||}|j|jjjd}|j|k(sJt|tdy)N overwritemoderr timestampF check_index)strcreateDataFramepdf repartitionwriteparquetdd read_parquetassignr6dt tz_localizer,r)r+r,tmpdirrsdfddfs r$test_roundtrip_parquet_spark_to_daskrGJs [F  ' ' ,COOK &&..vK.H //&/ 0C **s}}//;;EB* CC ??k )) ) c3E*rct|}|jt}|jj |ddt j ||}|j|jjjd}|jjd}|j|jjd  }t|tjdd y) Nr1name)r3 partitionByr4rr5r-)axisr9)rIFr7)r9r:r;r=r>r?r@rAr6rBrCcompute sort_indexrIastyper)r+rDrrErFs r)test_roundtrip_hive_parquet_spark_to_daskrO[s [F  ' ' ,CIIf;FC //&/ 0C **s}}//;;EB* CC ++- " " " *C **#((//%0* 1C c3>>q>)u=rcTt|}tjt|}|j ||d|j j |}|j}|j|jjjd}t||dy)N)r,F)r write_indexrr5r7) r9r? from_pandasr; to_parquetreadr>toPandasrAr6rBrCr)r+r,rDrrFrEs r$test_roundtrip_parquet_dask_to_sparkrVqs [F ..+. 6CNN6&eN<    $ $V ,C ,,.C **s}}//;;EB* CC c3E*rct|}d}d}tjt|tj j |ddg|dzzddg|dzzd }|j d d d d d }t|jDcgc]+}tjjj|-c}sJ|j|}|j|jj|dt j#|dd}t|jDcgc]+}tjjj|-c}sJ|jt%||dycc}wcc}w)Nr.)sizeTFalicebob)abcdInt64Float64booleanstringr1r2rnumpy_nullabler dtype_backendr7)r9pd DataFramerangenprandomrNalldtypesapitypesis_extension_array_dtyper:r<r=r>r?r@r)r+rDr,rYr;dtyperErFs r5test_roundtrip_parquet_spark_to_dask_extension_dtypesrss^ [FK D ,,t!!t!,$!),5!TQY/   C **   C #**U 55e<U VV V  ' ' ,COOK &&..vK.H //&BR/ SC CF::N% . .u 5N  zz c3E*V Os 0F+0Fc ,t|}d}d}tjdtjdtjdtjdtjdtjdg}tjt ||d }|j |}|jd |d jtjjjd d}|j|jj|d t j#|dd}|j$j&j(t*j-d dk(sJ|j$j/j&j(t*j-d dk(sJ|j1dtj2t*j-d dd }t5||dy)Nz8093.234z8094.234z8095.234z8096.234z8097.234z8098.234)r]r^r^r1r2rrfzint64[pyarrow]Fr7)r9decimalDecimalrhrirjr: withColumncastrrrp DecimalTyper<r=r>r?r@r^rr pyarrow_dtypepa decimal128rLrN ArrowDtyper) r+rDr,rY decimal_datar;rErFexpecteds rtest_read_decimal_dtype_pyarrowrs [FK D  # # # # # # L ,,t  C  ' ' ,C ..c#hmmGKK,=,=,I,I!Q,OP QCOOK &&..vK.H //&)/ LC 55;; $ $ a(; ;; ; 55==? . ."--12E EE Ezz!r}}Q23 Hc8/r)' __future__rrxrsysr&r dask.datasetsrrrr~r?numpyrkrrhdask.dataframe.utilsrmarkskipifplatformskipskip_with_pyarrow_strings pytestmarkfixturerrLr;indexrC reset_indexr+ parametrizerGrOrVrsrrrrs"  $H &  i (V#V)** KK R KK@ KK))      d##% II ! !% (  ooh+ +* 3 +4 + >, 3 +4 +%+P#0r