
    bi                   >   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZmZ d dlmZ d dlmZ d dlZd dlZd dlmZ d dlmZmZmZ d d	lmZ d d
lm Z! d dlm"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z= d dl>m?Z?m@Z@mAZAmBZB d dlCmDZDmEZE d dlFmGZGmHZHmIZI d dlJmKZKmLZL d dlMmNZNmOZOmPZPmQZQmRZR d dlSmTZTmUZU d dlVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^ ej                  Z_ G d dej                        Z  G d de"e       Za G d de       Zb G d  d!e       Zc G d" d#ec      Zdd$ Ze G d% d&ed      Zf G d' d(e       Zg G d) d*ed      Zh G d+ d,e       Zid- Zjd. Zkd/ Zl G d0 d1ec      Zm G d2 d3ec      Zn G d4 d5ec      Zo G d6 d7ec      Zp G d8 d9ec      Zq G d: d;ec      Zr G d< d=ec      Zs G d> d?ec      Zt G d@ dAec      Zu G dB dCec      Zv G dD dEec      Zw G dF dGew      Zx G dH dIex      Zy G dJ dKew      Zz G dL dMew      Z{ G dN dOew      Z| G dP dQew      Z} G dR dSew      Z~ G dT dUew      Z G dV dWew      Z G dX dYew      Z G dZ d[ew      Z G d\ d]ew      Z G d^ d_ew      Z G d` dae      Z G db dcew      Z G dd deew      Z G df dgew      Z G dh diew      Z G dj dkew      Z G dl dmew      Z G dn doew      Z G dp dqew      Zdr Z G ds dtec      Z G du dvew      Z G dw dxew      Z G dy dzew      Z G d{ d|ew      Z G d} d~e      Z G d de      Zd]dZ G d dew      Zd Z G d dec      Z G d dew      Z G d dew      Z G d dew      Z G d dew      Z G d dew      Z G d dec      Z G d de      Z G d dew      Zd Z G d dew      Z G d dew      Z G d dew      Z G d dec      Z G d dew      Z G d dew      Zd^dZ G d dew      Z G d dew      Z G d de       Z G d de       Z G d dew      Z G d dew      Z G d de      Z G d dew      Z G d de      Z G d dew      Z G d de       Z G d deec      Z G d de      Z G d de       Z G d deec      Z G dĄ de      Z G dƄ dew      Z G dȄ de      Z G dʄ de      Z G d̄ de      Z G d΄ de      Z G dЄ de      Z G d҄ de      Z G dԄ de      Z G dք de      Z G d؄ de      Z G dڄ de«      Z G d܄ deë      Z G dބ de«      Z G d deë      Z G d de«      Z G d deë      Z G d de«      Z G d deë      Z G d de«      Z G d deë      Z G d de«      Z G d deë      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d  de      Z G d de      Z G d de      Z G d dew      Z G d d	eګ      Z G d
 deګ      Z G d deګ      Z G d de       Z G d de       Z G d de       Z G d de       Zd Zd Zd Zd Z G d deh      Z G d dec      Z G d deh      Z G d  d!e      Z G d" d#eh      Z G d$ d%ec      Z G d& d'e       Zd( Z G d) d*e      Z G d+ d,e      Z G d- d.e      Z G d/ d0e      Z G d1 d2e      Z G d3 d4e      Z G d5 d6e      Z G d7 d8e      Z G d9 d:e      Z G d; d<e      Z G d= d>e      Z G d? d@e      Z G dA dBe      Z G dC dDe      Z G dE dFec      Zej                   G dG dH             Z	 d^	 	 	 	 	 	 	 	 	 d_dIZdJ Z d^dKZd`dMZdN ZefdOZdP ZdQ Z	 d`	 dadRZdLdSdTZdU Z	dbdVZ
ddWdXZdY Zd dZlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d d[lm Z m!Z!m"Z"m#Z# d d\l$m%Z% y(c      )annotationsN)defaultdict)Callable
CollectionMapping)partialAny)PerformanceWarning)merge_sorted	partitionunique)_expr)Expr)FinalizeCompute)AliasDataNodeTaskTaskRefexecute_graph)Array)collections_to_expr)flatten)methods)to_pyarrow_string)
_concat_get_divisions_map_partitions_renameapply_and_enforcehas_parallel_typeis_dataframe_likeis_index_likeis_series_like	safe_headtotal_mem_usage)_BackendData_calc_maybe_new_divisions_convert_to_list_tokenize_partial)	make_metameta_nonempty)CombinedOutput_head_timedeltaoverlap_chunk)drop_overlapget_overlap)clear_known_categoriesdrop_by_shallow_copy	is_scalarraise_on_meta_errorvalid_divisions)Key
no_default)Mfuncnameget_meta_libraryhas_keywordis_arraylikepartial_by_orderpseudorandomrandom_state_datac                      e Zd ZdZdZdZd Zej                  d        Z	d Z
d Zd Zed        Zed	        Zed
        ZdddZd Z fdZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d  Z%d! Z&d" Z'd# Z(d$ Z)d% Z*d& Z+d' Z,d( Z-d) Z.d* Z/d+ Z0d, Z1d- Z2d. Z3d/ Z4d0 Z5d1 Z6d2 Z7d3 Z8ded4Z9ded5Z:dfd6Z;dfd7Z<ded8Z=ded9Z>dgd:Z?dgd;Z@dhd<ZAdhd=ZBdgd>ZCded?ZDdid@ZEdjdAZFdjdBZGdjdCZHdjdDZIdE ZJdF ZKdkdHZLdI ZMdldJZNdK ZOdL ZPdmdMZQeRj                  fdNZTeRj                  fdOZUdGdPdQZVdGeWdfdRZXdndSZYeWeWeWdfdTZZdodUZ[dndVZ\dgdWZ]ej                  dX        Z^dY Z_edZ        Z`ed[        Zaedpd\       Zbej                  dqd]       Zced^        Zded_        Zeed`        ZfdndaZgdb Zhdc Zi xZjS )rr   zPrimary class for all Expressions

    This mostly includes Dask protocols and various Pandas-like method
    definitions to make us look more like a DataFrame.
    Fc                8    | j                   xr t        | ||      S N)_filter_passthroughis_filter_pushdown_availableselfparent
dependentss      Y/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/dataframe/dask_expr/_expr.py_filter_passthrough_availablez"Expr._filter_passthrough_availableQ   s$    '' 
,H&*-
 	
    c                R    | j                   }	 |j                  S # t        $ r Y yw xY wNr   )_metandimAttributeErrorrF   metas     rI   rO   z	Expr.ndimV   s,    zz	99 		s    	&&c                j    t        | j                        D cg c]  }| j                  |f c}S c c}w rB   )rangenpartitions_name)rF   is     rI   __dask_keys__zExpr.__dask_keys__^   s)    ).t/?/?)@AAQAAAs   0c                    t        | fi |S rB   )optimizerF   kwargss     rI   rZ   zExpr.optimizea   s    '''rK   c                ,    t        | j                        S rB   )hashrV   rF   s    rI   __hash__zExpr.__hash__d   s    DJJrK   c                    t        |       S rB   )Indexr_   s    rI   indexz
Expr.indexg   s    T{rK   c                    t        |       S rB   )Sizer_   s    rI   sizez	Expr.sizek       DzrK   c                    t        |       S rB   )NBytesr_   s    rI   nbyteszExpr.nbyteso   s    d|rK   r   c                H   t        t        |             dz   }g }t        | j                        D ]  \  }}t	        |t
              r$|s|j                  |j                  d             :t	        |t              r|j                  }t	        |t        j                  j                  j                        rd}n7t        |      rd}n)t        |      rd}nt!        |      rd}nt#        |      rd}| j%                  |||      } |g|z   }|D cg c]
  }d|z  |z    }}|S c c}w )	N:   z<pandas>z<dataframe>z<index>z<series>z<array> )r9   type	enumerateoperands
isinstancer   extend_tree_repr_linesr&   _datapdcorebasePandasObjectr!   r"   r#   r<    _tree_repr_argument_construction)rF   indent	recursiveheaderlinesrW   oplines           rI   rt   zExpr._tree_repr_liness   s
   $t*%+t}}- 	NEAr"d#LL!4!4Q!78b,/B b"'',,";";<#B&r*&B"2&"B#B'#B!"%"B>>q"fM'	N* 5 167v$77 8s   Dc                :   g }t        | j                  | j                        D ]u  \  }}t        |t              sJt        |t
        j                  t
        j                  f      rA|| j                  j                  |      k7  s`|j                  | d|       w |S )N=)zip_parametersrq   rr   r   rv   Series	DataFrame	_defaultsgetappend)rF   
to_includeparamoperands       rI   _operands_for_reprzExpr._operands_for_repr   s    
!$"2"2DMMB 	:NE7'4(wBLL(ABt~~11%88!!UG1WK"89	: rK   c                    	 t         |   |      S # t        $ r6 t        | j                        r|| j                  j
                  v r| |   cY S  w xY wrB   )super__getattr__rP   r!   rN   columns)rF   key	__class__s     rI   r   zExpr.__getattr__   sQ    	7&s++ 	 ,

8J8J1JCy 		s    ;AAc                R    t        |t              rt        | |      S t        | |      S rB   )rr   r   Filter
ProjectionrF   others     rI   __getitem__zExpr.__getitem__   s&    eT"$&&dE**rK   c                H    t        d| j                  j                   d      )NzThe truth value of a z& is ambiguous. Use a.any() or a.all().)
ValueErrorr   __name__r_   s    rI   __bool__zExpr.__bool__   s+    #DNN$;$;#< =& &
 	
rK   c                    t        | |      S rB   Addr   s     rI   __add__zExpr.__add__       4rK   c                    t        ||       S rB   r   r   s     rI   __radd__zExpr.__radd__       5$rK   c                    t        | |      S rB   Subr   s     rI   __sub__zExpr.__sub__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rsub__zExpr.__rsub__   r   rK   c                    t        | |      S rB   Mulr   s     rI   __mul__zExpr.__mul__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rmul__zExpr.__rmul__   r   rK   c                    t        | |      S rB   PowrF   powers     rI   __pow__zExpr.__pow__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rpow__zExpr.__rpow__   r   rK   c                    t        | |      S rB   Divr   s     rI   __truediv__zExpr.__truediv__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rtruediv__zExpr.__rtruediv__   r   rK   c                    t        | |      S rB   LTr   s     rI   __lt__zExpr.__lt__       $rK   c                    t        ||       S rB   r   r   s     rI   __rlt__zExpr.__rlt__       %rK   c                    t        | |      S rB   GTr   s     rI   __gt__zExpr.__gt__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rgt__zExpr.__rgt__   r   rK   c                    t        | |      S rB   LEr   s     rI   __le__zExpr.__le__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rle__zExpr.__rle__   r   rK   c                    t        | |      S rB   GEr   s     rI   __ge__zExpr.__ge__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rge__zExpr.__rge__   r   rK   c                    t        | |      S rB   )EQr   s     rI   __eq__zExpr.__eq__   r   rK   c                    t        | |      S rB   )NEr   s     rI   __ne__zExpr.__ne__   r   rK   c                    t        | |      S rB   Andr   s     rI   __and__zExpr.__and__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rand__zExpr.__rand__   r   rK   c                    t        | |      S rB   Orr   s     rI   __or__zExpr.__or__   r   rK   c                    t        ||       S rB   r   r   s     rI   __ror__zExpr.__ror__   r   rK   c                    t        | |      S rB   XOrr   s     rI   __xor__zExpr.__xor__   r   rK   c                    t        ||       S rB   r   r   s     rI   __rxor__zExpr.__rxor__   r   rK   c                    t        |       S rB   )Invertr_   s    rI   
__invert__zExpr.__invert__   s    d|rK   c                    t        |       S rB   )Negr_   s    rI   __neg__zExpr.__neg__       4yrK   c                    t        |       S rB   )Posr_   s    rI   __pos__zExpr.__pos__  r   rK   c                    t        | |      S rB   Modr   s     rI   __mod__zExpr.__mod__  r   rK   c                    t        ||       S rB   r   r   s     rI   __rmod__zExpr.__rmod__	  r   rK   c                    t        | |      S rB   FloorDivr   s     rI   __floordiv__zExpr.__floordiv__  s    e$$rK   c                    t        ||       S rB   r  r   s     rI   __rfloordiv__zExpr.__rfloordiv__  s    t$$rK   c                    | |z  }| |z  }||fS rB    rF   r   res1res2s       rI   
__divmod__zExpr.__divmod__  s    u}e|TzrK   c                    || z  }|| z  }||fS rB   r	  r
  s       rI   __rdivmod__zExpr.__rdivmod__  s    }t|TzrK   c                     t        | ||||      S rB   )SumrF   skipnanumeric_onlysplit_everyaxiss        rI   sumzExpr.sum      4{DAArK   c                     t        | ||||      S rB   )Prodr  s        rI   prodz	Expr.prod  s    D&,TBBrK   c                n    |dk(  rt        | ||||      S |dk(  rt        | |||      S t        d| d      )Nr      zaxis=z% not supported. Please specify 0 or 1)Var
VarColumnsr   rF   r  r  ddofr  r  s         rI   varzExpr.var"  sI    19tVT<EEQYdFD,??uTF*OPQQrK   c           	     @    t        | j                  |||||            S )N)r  )Sqrtr"  r   s         rI   stdzExpr.std*  s     DHHT64;HWXXrK   c                "    t        | ||||      S )N)r  r  r  r  )Meanr  s        rI   meanz	Expr.mean-  s    %#
 	
rK   c                     t        | ||||      S rB   )Maxr  s        rI   maxzExpr.max6  r  rK   c                    t        | ||      S N)r  r  r	   rF   r  r  s      rI   anyzExpr.any9      4K@@rK   c                    t        | ||      S r-  )Allr.  s      rI   allzExpr.all<  r0  rK   c                     t        | |||      S N)r  r  r  )IdxMinrF   r  r  r  s       rI   idxminzExpr.idxmin?      l
 	
rK   c                     t        | |||      S r5  )IdxMaxr7  s       rI   idxmaxzExpr.idxmaxD  r9  rK   c                    t        | ||      S )N)dropnar  )Mode)rF   r>  r  s      rI   modez	Expr.modeI  s    D[AArK   c                "    t        | ||||      S )N)r  r  )Minr  s        rI   minzExpr.minL  s    4;TRRrK   c                    t        | ||      S rB   )Count)rF   r  r  s      rI   countz
Expr.countO  s    T<55rK   c                "    ddl m}  || |      S )Nr   )CumSumr  )$dask.dataframe.dask_expr._cumulativerH  )rF   r  rH  s      rI   cumsumzExpr.cumsumR      ?d6**rK   c                "    ddl m}  || |      S )Nr   )CumProdrI  )rJ  rN  )rF   r  rN  s      rI   cumprodzExpr.cumprodW  s    @tF++rK   c                "    ddl m}  || |      S )Nr   )CumMaxrI  )rJ  rQ  )rF   r  rQ  s      rI   cummaxzExpr.cummax\  rL  rK   c                "    ddl m}  || |      S )Nr   )CumMinrI  )rJ  rT  )rF   r  rT  s      rI   cumminzExpr.cummina  rL  rK   c                    t        |       S rB   )Absr_   s    rI   abszExpr.absf  r   rK   c                    t        | |      S rB   )AsType)rF   dtypess     rI   astypezExpr.astypei  s    dF##rK   Nc                     t        | |||      S )Nlowerupperr  )Clip)rF   r_  r`  r  s       rI   clipz	Expr.clipl  s    DU>>rK   c                L    t        | |      rt        | |      S t        | |      S )N)r   )are_co_alignedCombineFirstCombineFirstAlignr   s     rI   combine_firstzExpr.combine_firsto  s&    $&E22$T511rK   c                    t        | ||      S Nfreqhow)ToTimestamp)rF   rk  rl  s      rI   to_timestampzExpr.to_timestampu  s    4d44rK   c                    t        |       S rB   IsNar_   s    rI   isnaz	Expr.isnax  s    DzrK   c                    t        |       S rB   rp  r_   s    rI   isnullzExpr.isnull{  rg   rK   c                    t        | |      S )N)decimals)Round)rF   rv  s     rI   roundz
Expr.round  s    TH--rK   c           
         t        | g||fD cg c]  }t        |t              s| c} st        | ||      S t	        | ||      S c c}w N)condr   )rd  rr   r   
WhereAlignWhererF   r{  r   cs       rI   wherez
Expr.where  sL    dW$%VA*QPTBUa%VWdU;;TE22 &W
   A

A

c           
         t        | g||fD cg c]  }t        |t              s| c} st        | ||      S t	        | ||      S c c}w rz  )rd  rr   r   	MaskAlignMaskr~  s       rI   maskz	Expr.mask  sL    dW$%VA*QPTBUa%VWTE::Dt511 &Wr  )rR   c                    t        | ||||      S rB   )Apply)rF   functionrR   argsr\   s        rI   applyz
Expr.apply  s    T8T488rK   c                     t        | |||      S )N
to_replacevalueregex)Replace)rF   r  r  r  s       rI   replacezExpr.replace  s    t
%uMMrK   c                n    t        |t              rt        | |      st        | |      S t	        | |      S )N)r  )rr   r   rd  FillnaAlignFillna)rF   r  s     rI   fillnazExpr.fillna  s/    eT">$+Ft511d%((rK   c                "    t        | ||||      S )Nmapperrc   r   r  )
RenameAxis)rF   r  rc   r   r  s        rI   rename_axiszExpr.rename_axis  s     $vUGRVWWrK   c                    ddl m} t        | |      st        | ||||      }nt	        | ||||      } |t        |d             |t        |d            fS )Nr   )new_collection)r  
fill_value)positionr  )$dask.dataframe.dask_expr._collectionr  rd  AlignAlignPartitions_AlignAlignGetitem)rF   r   joinr  r  r  aligneds          rI   alignz
Expr.align  s^    GdE**4dJOGT5$TjQGl7Q?@.1-C
 
 	
rK   c                    t        | d|      S )N   )br  )NuniqueApprox)rF   r  s     rI   nunique_approxzExpr.nunique_approx  s    TR[AArK   c                    t        | ||      S rB   )MemoryUsagePerPartition)rF   rc   deeps      rI   memory_usage_per_partitionzExpr.memory_usage_per_partition  s    &tUD99rK   c                4    t        | j                               S rB   )tuple
_divisionsr_   s    rI   	divisionszExpr.divisions  s    T__&''rK   c                    t               rB   NotImplementedErrorr_   s    rI   r  zExpr._divisions      !##rK   c                X    t        | j                        dkD  xr | j                  d   duS )z#Whether divisions are already knownr   Nlenr  r_   s    rI   known_divisionszExpr.known_divisions  s+     4>>"Q&H4>>!+<D+HHrK   c                p    d| j                   v r| j                  d      S t        | j                        dz
  S )NrU   r  )r   r   r  r  r_   s    rI   rU   zExpr.npartitions  s3    D,,,<<..t~~&**rK   c                    	 t        | j                  j                        S # t        $ r# | j                  dk(  r| j
                  gcY S g cY S t        $ r  w xY wNr  )listrN   r   rP   rO   name	Exceptionr_   s    rI   r   zExpr.columns  sR    	

**++ 	yyA~		{"I 		s   ! %AA
Ac                    t               S )a2  Preserves the columns defining the partition mapping from shuffles.

        This property specifies if a column or a set of columns have a unique
        partition mapping that was defined by a shuffle operation. The mapping
        is created by hashing the values and the separating them onto partitions.
        It is important that this property is only propagated if the values
        in those columns did not change in this expression. The property is
        only populated if the mapping was created by the ``partitioning_index``
        function.

        Simply knowing that every value is in only one partition is not a
        satisfying condition, because we also use this property on merge
        operations, where we need these values to be in matching partitions.

        This is also the reason why set_index or sort_values can't set the
        property, they fulfill a weaker condition than what this property enforces.

        Normally, this set contains one tuple of either one or multiple columns.
        It can contain 2, when the operation shuffles multiple columns of the
        result, i.e. a merge operation and the left and right join columns.


        Returns
        -------
            A set of column groups that have a unique partition mapping as
            defined by a shuffle.
        )setr_   s    rI   -unique_partition_mapping_columns_from_shufflez2Expr.unique_partition_mapping_columns_from_shuffle  s    : urK   c                    | j                   S rB   r   r_   s    rI   _projection_columnszExpr._projection_columns  s    ||rK   c                .    | j                   j                  S rB   )rN   r  r_   s    rI   r  z	Expr.name  s    zzrK   c                .    | j                   j                  S rB   )rN   r[  r_   s    rI   r[  zExpr.dtypes  s    zz   rK   c                    |&|j                   j                  | | j                        }t        | j                  |      r, t	        |       | j                  |   g| j
                  dd   S y r  )	predicate
substituteframerd  ro   rq   )rF   rG   r  s      rI   _filter_simplificationzExpr._filter_simplification  sa    ((33D$**EI$**i04:djj3HdmmAB6GHH 1rK   c                    t        |       S rB   )optimize_blockwise_fusionr_   s    rI   fusez	Expr.fuse  s    (..rK   c                    t        |       S rB   )FinalizeComputeDFr_   s    rI   finalize_computezExpr.finalize_compute  s     &&rK   r   T)TFFr   )r   Tr  FF)TF)TFF)FFT)NNN)Nstartr   rB   )outerNNreturnr  )r  r  )kr   
__module____qualname____doc___is_length_preservingrC   rJ   	functoolscached_propertyrO   rX   rZ   r`   propertyrc   rf   rj   rt   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r"  r%  r(  r+  r/  r3  r8  r<  r@  rC  rF  rK  rO  rR  rU  rX  r\  rb  rg  rn  rr  rt  rx  npnanr  r  r  r7   r  r  r  r  r  r  r  r  r  rU   r   r  r  r  r[  r  r  r  __classcell__r   s   @rI   r   r   G   s    "

  B(       :+
                %%

BCRY
BAA



BS6+
,
+
+
$?25. !# 3
  "vv 2
 +/ 9 "&Zu N)  z:AX


B: ( ($ I I + +    <     ! !I/'rK   r   c                      e Zd ZdgZd Zy)r  r  c                4    ddl m}  || j                  d      S Nr   Repartitionr  )%dask.dataframe.dask_expr._repartitionr  r  )rF   r  s     rI   _simplify_downz FinalizeComputeDF._simplify_down  s    E4::q))rK   N)r   r  r  r   r  r	  rK   rI   r  r    s    )K*rK   r  c                  H    e Zd ZdZdgZd Zej                  d        ZddZ	y)Literalz.Represent a literal (known) value as an `Expr`r  c                     yNNNr	  r_   s    rI   r  zLiteral._divisions      rK   c                ,    t        | j                        S rB   )r*   r  r_   s    rI   rN   zLiteral._meta  s    $$rK   c                <    |dk(  sJ t        || j                        S rM   )r   r  rF   r  rc   s      rI   _taskzLiteral._task  s    zzdjj))rK   Nr  r6   rc   intr  r   )
r   r  r  r  r   r  r  r  rN   r  r	  rK   rI   r  r    s0    8)K % %*rK   r  c                     e Zd ZU dZdZded<   g Zded<   dZdZe	j                  d        Ze	j                  dd	       Ze	j                  dd
       ZddZd Ze	j                  d        Zd ZddZd Ze	j                  d        Zy)	BlockwiseaY  Super-class for block-wise operations

    This is fairly generic, and includes definitions for `_meta`, `divisions`,
    `_layer` that are often (but not always) correct.  Mostly this helps us
    avoid duplication in the future.

    Note that `Fused` expressions rely on every `Blockwise`
    expression defining a proper `_task` method.
    NzCallable | None	operationz	list[str]_keyword_onlyFc                    | j                   D cg c]   }t        |t              r|j                  n|" }} | j                  |i | j
                  S c c}w rB   )_argsrr   r   rN   r  _kwargsrF   r   r  s      rI   rN   zBlockwise._meta/  sK    CG::NRJr40b8NNt~~t4t||44 Os   %Ac                    | j                   rO| j                  D ci c]9  }|| j                   v r)| j                  |      t        ur|| j                  |      ; c}S i S c c}w rB   )r  r   r   r7   )rF   ps     rI   r  zBlockwise._kwargs4  sd     ))***t||Aj/P 4<<?" 
 	s   >Ac                    | j                   r]| j                  D cg c]"  }|| j                   vs| j                  |      $ c}| j                  t	        | j                        d  z   }|S | j                  S c c}w rB   )r  r   r   rq   r  )rF   r  r  s      rI   r  zBlockwise._args>  sp    )-)9)9$%QdFXFX=XQc$"2"23567D K}}	s
   A6A6c                V    |j                   dk(  xr |j                  | j                  k  S r  )rU   rO   rF   deps     rI   _broadcast_depzBlockwise._broadcast_depG  s%     !#<499(<<rK   c                    | j                         }|D ]2  }| j                  |      r|j                  |d   j                  k(  r2J  |d   j                  S rM   )dependenciesr  r  )rF   r	  args      rI   r  zBlockwise._divisionsL  s_    
 ((* 	BC&&s+}}Q(A(AAAA	B A(((rK   c                    | j                   rt        | j                         }n"t        t        |             j                         }|dz   | j                  z   S N-)r  r9   ro   r_  deterministic_tokenrF   heads     rI   rV   zBlockwise._nameW  sC    >>DNN+DDJ'--/DczD4444rK   c                    t        |t              r?| j                  |      rt        |j                  df      S t        |j                  |f      S |S )z Return a Blockwise-task argumentr   )rr   r   r  r   rV   )rF   r
  rW   s      rI   _blockwise_argzBlockwise._blockwise_arg_  sG    c4 ""3'		1~..		1~.. JrK   c                    | j                   D cg c]  }| j                  ||       }}| j                  r$t        || j                  g|i | j                  S t        || j                  g| S c c}w )zProduce the task for a specific partition

        Parameters
        ----------
        index:
            Partition index for this task.

        Returns
        -------
        task: tuple
        )r  r  r  r   r  rF   r  rc   r   r  s        rI   r  zBlockwise._taskk  sj     :>D2##B.DD<<dnnDtDt||DDdnn4t44	 Es   A1c                X    | j                   rt        |t              rt        | ||      S y y rB   )_projection_passthroughrr   r   plain_column_projectionrE   s      rI   _simplify_upzBlockwise._simplify_up}  s,    ''Jvz,J*4DD -K'rK   c                Z    | j                   r| j                  j                  S t               S rB   )#_preserves_partitioning_informationr  r  r  r_   s    rI   r  z7Blockwise.unique_partition_mapping_columns_from_shuffle  s"    33::KKKurK   r  dictr  r  r   r  )r   r  r  r  r  __annotations__r  r  r  r  r  rN   r  r  r  r  rV   r  r  r  r  r	  rK   rI   r  r    s     "&I%!M9!#*/'5 5    =
	) 5 5
5$E  rK   r  c                     e Zd ZU g dZdddddddZded<   ej                  d        Zd	 Z	ej                  d
        Z
ddZej                  d        Zej                  d        Zd Zej                  d        ZddZed        Zd Zy)MapPartitions)r  funcrR   enforce_metadatatransform_divisionsclear_divisionsalign_dataframesparent_metarequired_columnstokenr\   nargsNTr   )r\   r%  r&  r'  r(  r)  r  r   c                B    d| j                   v r| j                  d      S y )Nr(  )r   r   r_   s    rI   r(  zMapPartitions.token  s"    d&&&<<((rK   c                4    dt        | j                         dS )NzMapPartitions()r9   r!  r_   s    rI   __str__zMapPartitions.__str__  s     34A66rK   c                    | j                   | j                   }n#t        | j                        j                         }|dz   | j                  z   S r  )r(  r9   r!  r_  r  r  s     rI   rV   zMapPartitions._name  sA    ::!::DDII&,,.DczD4444rK   c                     |j                   dk(  S r  rU   r  s     rI   r  zMapPartitions._broadcast_dep      !##rK   c                    | j                   g| j                  t        | j                        t        | j                        | j                  z    z   S rB   )r  rq   r  r   r)  r_   s    rI   r  zMapPartitions.args  sD    

|dmm  !C(8(8$9DJJ$F
 
 	
rK   c           
        | j                  d      }t        | j                  | j                  D cg c]$  }t        |t              rt        |t
              s|& c}| j                  | j                  || j                        S c c}w NrR   )	r   _get_meta_map_partitionsr  rr   r   _DelayedExprr!  r\   r&  )rF   rR   es      rI   rN   zMapPartitions._meta  st    ||F#'II a&z!\/J 
 IIKK
 	
s   )B
c                0   | j                   D cg c]  }t        |t              s| }}| j                  rt	        d |D              }d|dz   z  S t        | j                  | j                  || j                  | j                   | j                        S c c}w )Nc              3  4   K   | ]  }|j                     y wrB   r1  .0dfs     rI   	<genexpr>z+MapPartitions._divisions.<locals>.<genexpr>  s      >B >   rB   r  )
r  rr   r   r$  r+  r   r%  r#  r!  r\   )rF   r
  dfsmax_partitionss       rI   r  zMapPartitions._divisions  s    "iiAs:c4+@sAA  ># >>Nnq011 -!!$$IIIIKK
 	
 Bs
   BBc                .    t        | j                  d      S )Npartition_info)r;   r!  r_   s    rI   _has_partition_infoz!MapPartitions._has_partition_info  s    499&677rK   c                   | j                   D cg c]  }| j                  ||       }}t        | j                  | j                  ni       }| j                  r|| j
                  |   d|d<   | j                  r<|j                  | j                  | j                  d       t        |t        g|i |S t        || j                  g|i |S c c}w )N)numberdivisionrC  )_funcrN   )r  r  r  r\   rD  r  r"  updater!  rN   r   r   rF   r  rc   r   r  r\   s         rI   r  zMapPartitions._task  s    9=C2##B.CCT[[%<dkk"E## NN51(F#$
   MM!YY!ZZ /A$A&AA		  	 # Ds   Cc                     | |i ||   S rB   r	  )mapped_funcpost_projectionr  r\   s       rI   projected_operationz!MapPartitions.projected_operation  s     D+F+O<<rK   c           
        t        |t              r9| j                  +t        | j                        t        | j                  j
                        z
  x}rt        d|       t        | ||| j                        }| j                  j
                  D cg c]	  }||v s| }}|| j                  j
                  k(  ry  t        |       t        |       | j                  |   t        | j                  | j                  |j
                        | j                  |j
                     g| j                  dd   g|j                  dd   S y y c c}w )Nz1Some elements of `required_columns` are missing: additional_columns   r  )rr   r   r'  r  r  r   KeyErrordetermine_column_projectionro   r   rN  r!  rR   rq   )rF   rG   rH   missingr   cols         rI   r  zMapPartitions._simplify_up   s=   fj)d.C.C.Od334s4::;M;M7NNNwNGyQ  2fjT=R=RG '+jj&8&8KsC7NsKGK$**,,,4<T
JJw'D44diiPIIfnn- ]]12&	 $  /P) Ls   	EEr  r  )r   r  r  r   r   r  r  r  r(  r.  rV   r  r  rN   r  rD  r  staticmethodrN  r  r	  rK   rI   r   r     s    K   It   
7 5 5$ 
 

 
 

$ 8 82 = =
rK   r   c           	     (   |D cg c]  }t        |t        t        f      s| }}t        |       dk\  rt	        d |D              sd}t        |      |D cg c]c  }|j                  dk(  r|j                  nEt        |t              r!t        j                  d|j                        nt        |j                        e }}t        |      D cg c]   \  }}t        |t        t        f      r||f" }	}}t        |||	dS c c}w c c}w c c}}w )Nrm   c              3  4   K   | ]  }t        |d         yw)rU   N)hasattrr<  ds     rI   r>  z"_get_meta_ufunc.<locals>.<genexpr>  s      JqM!: Jr?  z>elemwise with 2 or more DataFrames and Scalar is not supportedr   r	  dtype)r  r   )rr   r   r   r  r3  r  rO   rN   r  emptyr^  r+   rp   r=   )
r@  r  r!  r
  dasksmsgr\  partsrW   r   s
             rI   _get_meta_ufuncrc    s    CSJsT5M$BSCEC
3x1}S JE JJN!#&&   vv{ GG a' 177+"177+E   )QjtUm6TCE  UT??1 Ds   DDA(D	D/Dc                      e Zd Zg dZdZd Zej                  d        Zej                  d        Z	ej                  d        Z
d Zy)	UFuncElemwise)r  r!  rR   r#  r\   Fc                4    dt        | j                         dS NzUFunc(r,  r-  r_   s    rI   r.  zUFuncElemwise.__str__A      +,A..rK   c                F    | j                   t        | j                        d  S rB   rq   r  r   r_   s    rI   r  zUFuncElemwise.argsD      }}S!1!12455rK   c                    | j                   D cg c]%  }t        |t              s|j                  dkD  s$|' c}S c c}w rM   )r  rr   r   rO   rF   r=  s     rI   _dfszUFuncElemwise._dfsH  s-    !YYOr*R*>277Q;OOOs   <<<c                    | j                  d      t        ur| j                  d      }t        |      S t        | j                  | j                  | j
                        }t        |      S r5  )r   r7   rc  rn  r  r!  r*   rQ   s     rI   rN   zUFuncElemwise._metaL  sS    <<z1<<'D  #499diiCDrK   c                `   | j                   rt        | j                  d   t              rt	        | j                        dk(  r	  | j
                  | j                  D cg c]4  }|| j                  d   u rt        j                  |j                        n|6 c}i | j                  }t        |t        j                        rt        j                  |      }t        |      s d g| j                  d   j                  dz   z  }|S | j                  d   j                  S c c}w # t        $ r Y )w xY wNr   r  )r#  rr   rn  rb   r  r!  r  rv   r  r\   r   tolistr5   rU   r  )rF   r
  r  s      rI   r  zUFuncElemwise._divisionsT  s   $$499Q</DII!#!%DII $(99 47$))A,3F/CO
 kk	 i2 'y 9I 'y1!%$))A,*B*BQ*F GI  yy|%%%  s%   D! 9DA D! D! !	D-,D-N)r   r  r  r   r"  r.  r  r  r  rn  rN   r  r	  rK   rI   re  re  7  sk    K / 6 6 P P  &rK   re  c                  X    e Zd Zg dZddddddddZej                  d        Zd Zd Z	y)	MapOverlapAlignr  r!  beforeafterrR   r"  r#  r$  r%  r(  r\   NTFrR   enfore_metadatar#  r\   r$  r%  r(  c                   | j                  d      }| j                  j                  g| j                  t	        | j
                        d  D cg c]   }t        |t              r|j                  n|" c}z   }t        || j                         d   g| j                  | j                  || j                  j                  dd             S c c}w NrR   r   r&  r   r  rN   rq   r  r   rr   r   r6  r	  r!  r\   poprF   rR   r
  r  s       rI   rN   zMapOverlapAlign._meta      ||F#

  !}}S)9)9%:%<=%
 $C.CIIC7%
 
 ( #$IIKKKKOOM40
 	
	%
   %Cc                x    | j                   g| j                  t        | j                        d  z   }t	        |ddiS )Nallow_shuffleF)r  rq   r  r   calc_divisions_for_alignrF   r  s     rI   r  zMapOverlapAlign._divisions  s9    

|dmmC0@0@,A,CDD'CUCCrK   c                   | j                   g| j                  t        | j                        d  z   }t	        |d| j                         i}t        |d   | j                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  | j                   g|dd   S )Nr  r   r  )r  rq   r  r   maybe_align_partitionsr  
MapOverlapr!  rv  rw  rN   r"  r#  r$  r%  r(  r\   r  s     rI   _lowerzMapOverlapAlign._lower  s    

|dmmC0@0@,A,CDD%tIt7HIGIIKKJJJJ!!$$  !!JJKK
 !"X
 	
rK   )
r   r  r  r   r   r  r  rN   r  r  r	  rK   rI   rt  rt  n  sL    K # !I 
 
D
rK   rt  c                      e Zd ZU g dZddddddddZded<   ej                  dd       Ze	d	        Z
ej                  d
        Zej                  d        Zej                  d        Zd Zy)r  ru  NTFrx  r  r   c                &    | j                   }|i }|S rB   )r\   r[   s     rI   r  zMapOverlap._kwargs  s    >FrK   c                    | j                   g| j                  | j                  | j                  gz   | j                  t        | j                        d  z   S rB   )r  r!  rv  rw  rq   r  r   r_   s    rI   r  zMapOverlap.args  sL     ZZLyy$++tzz23mmC 0 01345	
rK   c                   | j                  d      }| j                  j                  g| j                  t	        | j
                        d  D cg c]   }t        |t              r|j                  n|" c}z   }t        || j                         d   g| j                  | j                  || j                  j                  dd             S c c}w r{  r|  r~  s       rI   rN   zMapOverlap._meta  r  r  c                r    | j                  d      }t        |t              rt        j                  |      S |S )Nrv  r   rr   strrv   to_timedelta)rF   rv  s     rI   rv  zMapOverlap.before  s.    h'fc"??6**rK   c                r    | j                  d      }t        |t              rt        j                  |      S |S )Nrw  r  )rF   rw  s     rI   rw  zMapOverlap.after  s.    W%eS!??5))rK   c                `   t        | j                  | j                  | j                        }t	        |t
        | j                  | j                  | j                  | j                  | j                  d d | j                  | j                  t        | j                  dd        g| j                  dd   S r  )CreateOverlappingPartitionsr  rv  rw  r   _overlap_chunkrN   r"  r#  r$  r%  r(  r  r  r  )rF   
overlappeds     rI   r  zMapOverlap._lower  s    0T[[$**U
JJ!!$$  !!JJLL		!"
 YYqr]
 	
rK   r  )r   r  r  r   r   r  r  r  r  r  r  rN   rv  rw  r  r	  rK   rI   r  r    s    K # !It    
 
 
 
    
rK   r  c                  F    e Zd Zg dZej
                  d        Zd ZddZy)r  )r  rv  rw  c                .    | j                   j                  S rB   r  rN   r_   s    rI   rN   z!CreateOverlappingPartitions._meta	      zzrK   c                .    | j                   j                  S rB   r  r  r_   s    rI   r  z&CreateOverlappingPartitions._divisions  s    zz###rK   c                	   i g g }}}d| j                   j                  z   }| j                  r|j                  d        t	        | j                  t
        j                        rv| j                  }t        | j                   j                  dz
        D ]C  }t        j                  | j                   j                  |f|f|||f<   |j                  ||f       E n"t	        | j                  t        j                        rt        j                  | j                   j                        }|j!                         j"                  dd }| j                  |kD  j%                         r|d   }	t        | j                   j                  dz
        D ]  }||dz      }
t'        |
| j                  z
  |	      }||   |}}||kD  r|dkD  r|||   z
  }|dz
  }||kD  r|dkD  rt(        | j                   j                  |dz   ft        ||dz         D cg c]  }| j                   j                  |f c}| j                  f|||f<   |j                  ||f        nt        | j                   j                  dz
        D ]^  }t(        | j                   j                  |dz   f| j                   j                  |fg| j                  f|||f<   |j                  ||f       ` n)|j+                  d g| j                   j                  z         d| j                   j                  z   }| j,                  r.t	        | j,                  t
        j                        rs| j,                  }t        d| j                   j                        D ]C  }t        j.                  | j                   j                  |f|f|||f<   |j                  ||f       E nd| j,                  z  }t        d| j                   j                        D ]S  }t0        | j                   j                  |dz
  f| j                   j                  |f|f|||f<   |j                  ||f       U |j                  d        n)|j+                  d g| j                   j                  z         t3        t5        ||            D ]M  \  }\  }}t6        || j                   j                  |f|| j                  | j,                  f|| j                  |f<   O |S c c}w )Nzoverlap-prependr  r   zoverlap-appendrm   )r  rV   rv  r   rr   numbersIntegralrT   rU   r8   taildatetime	timedeltarv   r   r  diffilocr/  r+  _tail_timedeltars   rw  r  r-   rp   r   _combined_parts)rF   dskprevsnextsname_prependrv  rW   divsdeltaspt_zpt_ilbfirstjkname_appendrw  prevnexts                      rI   _layerz"CreateOverlappingPartitions._layer  sW   BEU(4::+;+;;;;LL$++w'7'78tzz559: 4A./fftzz7G7G6KV-TCq)*LL,!234 DKK););<yy!5!56))!B/ KK&(--/7D"4::#9#9A#=> 8  $AE{ !!3T:#'7Aq#bjQU$)F1I$5E !AA $bjQU
 ,!ZZ--q1u5<A!QUOLqdjj..2L KK	2\1-. lA%67'8* #4::#9#9A#=> 8+!ZZ--q1u5"jj..23 KK	2\1-. lA%678 LL$$**"8"889&)9)99::$**g&6&67

q$**"8"89 3A-.VVdjj6F6F5JE,RCa()LL+q!123 DJJq$**"8"89 3A'))1q51))1-	-Ca() LL+q!123 LL LL$$**"8"889(UE):; 	OA|d!!1%

$CQ 	 
e Ms   S
Nr  	r   r  r  r   r  r  rN   r  r  r	  rK   rI   r  r    s*    .K   $XrK   r  c           	         t        j                  |D cg c]/  }||j                  | j                  j                         |z
  kD     1 c}      }|S c c}w rB   )r   concatrc   rC  )currentprev_rv  r  selecteds        rI   r  r  l  sK    ~~GLMtdjjGMM--/&89	:MH O 	Ns   4Ac                &    t        |||| g|i |S rB   )r.   )r=  r!  rv  rw  r  r\   s         rI   r  r  s  s    vubB4B6BBrK   c                   d}| t        |t        j                        r| j                  d   |k7  ret	        |      | }t        || g|      } t        |      t        |       k(  r3t        |      dkD  r%t        |t        j                        st	        |      |vt        |t        j                        r|j                  d   |k7  rJt	        |      |}t        |||      }t        |      t        |      k(  rt        |      dkD  rt	        |      | ||fD cg c]  }||	 }	}t        j                  |	      }
t        |
| t        |       dkD  rt        |       nd |t        |      dkD  rt        |      f      S d f      S c c}w )NzqPartition size is less than overlapping window size. Try using ``df.repartition`` to increase the partition size.r   )rr   r  r  shaper  r  r  r  r  r-   r   r  r,   )	prev_partcurrent_part	next_partrv  rw  ra  prev_part_inputnext_part_inputr  rb  combineds              rI   r  r  w  s   	*  fg../q!V+)#..'O'yk6JIO$I6(1,"68+=+=>)#..eW--.q!U*)#..'O'iGI?#s9~5#o:NQR:R)#.."L)<N1QNEN~~e$H'3I8JC	NPT'3I8JC	N	
  QU	
  Os   FFc                  J    e Zd Zg dZddddZg dZej                  ZdZ	d Z
y)r  r  r   r  r  r  r  N)r  r  r  Tc                6    | j                   j                         S rB   )r  r  r_   s    rI   r  z_Align._divisions  s    zz$$&&rK   )r   r  r  r   r   r  r8   r  r  r  r  r	  rK   rI   r  r    s,    BK dCI2MI*.''rK   r  c                  Z    e Zd ZddgZej
                  ZdZej                  d        Z
d Zy)r  r  r  Tc                H    | j                   j                  | j                     S rB   )r  rN   r  r_   s    rI   rN   zAlignGetitem._meta  s    zz..rK   c                .    | j                   j                  S rB   r  r_   s    rI   r  zAlignGetitem._divisions      zz###rK   N)r   r  r  r   operatorgetitemr  r  r  r  rN   r  r	  rK   rI   r  r    s:    J'K  I*.'/ /$rK   r  c                  .    e Zd ZddgZddiZedd       Zy)ScalarToSeriesr  rc   r   c                2    t        j                  | |g      S )Nrc   )rv   r   )r  rc   s     rI   r  zScalarToSeries.operation  s    yyug..rK   Nr  )r   r  r  r   r   rW  r  r	  rK   rI   r  r    s'    G$K!I/ /rK   r  c                  .    e Zd ZdgZej
                  ZdZy)DropnaSeriesr  TN)r   r  r  r   r8   r>  r  r  r	  rK   rI   r  r    s    )KI*.'rK   r  c                  J    e Zd Zg dZededZg dZej                  Z	dZ
d Zy)DropnaFrame)r  rl  subsetthreshN)rl  r  r  Tc                   t        |t              r| j                  t        | ||| j                        }| j                  j
                  D cg c]	  }||v s| }}|| j                  j
                  k(  ry  t        |       t        |       | j                  |   g| j                  dd   g|j                  dd   S y y c c}w )NrP  r  )rr   r   r  rT  r  r   ro   rq   rF   rG   rH   r   rV  s        rI   r  zDropnaFrame._simplify_up  s    fj)dkk.E1fjT[[G '+jj&8&8KsC7NsKGK$**,,,4<T
4::g.Cqr1BC$  /F) Ls   	CC)r   r  r  r   r7   r   r  r8   r>  r  r  r  r	  rK   rI   r  r    s,    6K"djII/MI*.'rK   r  c                  V    e Zd ZddgZej
                  Zej                  d        Z	d Z
y)re  r  r   c                    t        | j                  t        | j                  j                        t        | j
                  j                                    S rB   )r*   r  r+   r  rN   r   r_   s    rI   rN   zCombineFirst._meta  s?    NNdjj../djj../
 	
rK   c                   t        |t              rt        | ||      }| j                  j                  D cg c]	  }||v s| }}| j
                  j                  D cg c]	  }||v s| }}| j                  j                  |k(  r| j
                  j                  |k(  ry  t        |       t        |       | j                  |   | j
                  |         g|j                  dd   S y c c}w c c}w r  rr   r   rT  r  r   r   ro   rq   rF   rG   rH   r   rV  frame_columnsother_columnss          rI   r  zCombineFirst._simplify_up  s    fj)1$
KG,0JJ,>,>QS#.SQMQ,0JJ,>,>QS#.SQMQ

""m3JJ&&-74<T
4::m4djj6OP$  *QQ   	C+ C+	C0(C0N)r   r  r  r   r8   rg  r  r  r  rN   r  r	  rK   rI   re  re    s2    G$KI
 
rK   re  c                  d    e Zd Zg dZ eej                        Zej                  d        Z
ddZy)Sample)r  
state_datafracr  c                    | j                   d   j                  g| j                   d   d   gz   | j                   dd  z   } | j                  | S )Nr   r  rm   )rq   rN   r  r  s     rI   rN   zSample._meta  sR    a &&'4==+;A+>*??$--PQPRBSSt~~t$$rK   c                    | j                  | j                  |      g| j                  |   | j                  | j	                  d      gz   }t        || j                  g| S )Nr  )r  r  r  r  r   r   r  )rF   r  rc   r  s       rI   r  zSample._task	  s\    ##DJJ67OOE"IILL#;
 

 D$..0400rK   Nr  )r   r  r  r   rW  r   sampler  r  r  rN   r  r	  rK   rI   r  r     s2    <KW^^,I% %1rK   r  c                  l    e Zd ZU g dZdi iZded<   dgZej                  Z	e
j                  dd       Zy)Queryr  r   expr_kwargsr  zdict[str, Any]r   c                    i | j                   S rB   r  r_   s    rI   r  zQuery._kwargs      #$""##rK   Nr  )r   r  r  r   r   r  r  r8   queryr  r  r  r  r	  rK   rI   r  r    s>    3K!. 3I~3"OMI$ $rK   r  c                  4    e Zd Zg dZdddZed        Zd Zy)r  )r  rc   r  TF)rc   r  c                     t        | d         r| d   j                  t        | i |g      S | d   j                  t        | i |g      S rM   )r#   _constructorr%   _constructor_slicedr  r\   s     rI   r  z!MemoryUsagePerPartition.operation!  sR    $q'"7''$)I&)I(JKKAw**OT,LV,L+MNNrK   c                :    d| j                   j                  dz   z  S NrB   r  r  rU   r_   s    rI   r  z"MemoryUsagePerPartition._divisions'      $**001455rK   N)r   r  r  r   r   rW  r  r  r	  rK   rI   r  r    s*    ,K.IO O
6rK   r  c                  .    e Zd ZdgZej
                  ZdZy)DropDuplicatesBlockwiser  TN)r   r  r  r   r8   drop_duplicatesr  r  r	  rK   rI   r  r  +  s    )K!!I*.'rK   r  c                  &     e Zd ZdZdZ fdZ xZS )Elemwisez
    This doesn't really do anything, but we anticipate that future
    optimizations, like `len` will care about which operations preserve length
    Tc                   t        |t              r| j                  ||      rd }| j                  j                  dk(  ra| j                  dk(  rR| j                  j
                  j                  }t        | |      }|j                  j                  || j                        }| j                  ||      S t        | 1  ||      S Nr  rm   )rr   r   rJ   r  rO   rN   r  r   r  r  r  r   r  )rF   rG   rH   r  r  subsr   s         rI   r  zElemwise._simplify_up9  s    ff%$*L*LJ+
 Izz!#		Qzz'',,!$-",,77djjI	..vyAAw#FJ77rK   )r   r  r  r  r  r  r  r  s   @rI   r  r  1  s    
 !8 8rK   r  c                  N    e Zd ZddgZej
                  d        Zed        Zd Z	y)RenameFramer  r   c                   t               }| j                  d      }| j                  j                  D ]  }t	        |t
              rW| j                  j                  t        |         j                  |      }|j                  t        |j                               j| j                  j                  |g   }|j                  |j                  d           |S )Nr   r  r   )r  r   r  r  rr   r  rN   r  renameaddr   )rF   resultr   elemr  s        rI   r  z9RenameFrame.unique_partition_mapping_columns_from_shuffleJ  s    ,,y)JJLL 	.D$&))$t*5<<W<M

501 ))4&1

6>>!,-	. rK   c                &    | j                  |      S )Nr  )r  r=  r   s     rI   r  zRenameFrame.operationX  s    yyy))rK   c                   t        |t              r4t        | j                  d      t              r| j                  d      j	                         D ci c]  \  }}||
 }}}t        | ||      }t        |      }t        | j                  j                        }|D cg c]  }||v r||   |v r||   n| }}| j                  j                  D cg c]	  }||v s| }}|| j                  j                  k(  ry  t        |       t        |       | j                  |   g| j                  dd   g|j                  dd   S y y c c}}w c c}w c c}w )Nr   r  )rr   r   r   r   itemsrT  r(   r  r  r   ro   rq   )	rF   rG   rH   r   valreverse_mappingr   r  rV  s	            rI   r  zRenameFrame._simplify_up\  s`   fj)jLL#W/
 9=Y8O8U8U8WXHCsCxXOX1$
KG&w/G

 2 23M #  o-/#2F-2W $C(G  '+jj&8&8KsC7NsKGK$**,,,4<T
4::g.Cqr1BC$ )/
) Y
 Ls   EE	EEN)
r   r  r  r   r  r  r  rW  r  r  r	  rK   rI   r  r  G  s=    I&K  * *rK   r  c                       e Zd ZdZed        Zy)ColumnsSetterTc                    t        ||       S rB   )r   r  s     rI   r  zColumnsSetter.operationz  s    w##rK   N)r   r  r  r  rW  r  r	  rK   rI   r  r  w  s    *.'$ $rK   r  c                  .    e Zd ZdgZdZdZdZed        Zy)	_DeepCopyr  Tc                &    | j                  d      S )NTr  )copy)r=  s    rI   r  z_DeepCopy.operation  s    wwDw!!rK   N)	r   r  r  r   r  rC   r  rW  r  r	  rK   rI   r  r    s-    )K"*.'" "rK   r  c                       e Zd ZddgZdZdZdZy)	ToBackendr  optionsTN)r   r  r  r   r  rC   r  r	  rK   rI   r  r    s    I&K"*.'rK   r  c                  ^    e Zd Zg dZddiZdZdZej                  d        Z	e
d        Zd Zy)	RenameSeries)r  rc   sorted_indexr  FTc                    | j                   D cg c])  }t        |t              rt        |j                        n|+ }}t         | j                  |i | j                        S c c}w rB   )r  rr   r   r+   rN   r*   r  r  r   s      rI   rN   zRenameSeries._meta  sa     OSjj
HJz"d';M"((#C
 
 >>??
s   .A$c                `    t        |       r| j                  |      S | j                  |      S )Nr  r  )r#   r  )r=  rc   r  s      rI   r  zRenameSeries.operation  s,    "9959))yyey$$rK   c                   | j                  d      }t        |      r&t        |t              s| j                  j
                  S | j                  r| j                  j                  rvt        j                  d| j                  j
                        }|j                  |      j                  }|j                  st        d      t        |j                               S d| j                  j                   dz   z  S )Nrc   r  r  znThe renamer creates an Index with non-monotonic divisions. This is not allowed. Please set sorted_index=False.rB   )r   r3   rr   r   r  r  r  r  rv   r   r  rc   is_monotonic_increasingr   r  rr  rU   )rF   rc   oldnew_divisionss       rI   r  zRenameSeries._divisions  s    W%UJuh$?::'''4::#=#=))ATZZ%9%9:CJJu-33M 88 J  --/00djj44q899rK   N)r   r  r  r   r   rC   r  r  r  rN   rW  r  r  r	  rK   rI   r  r    sP    4K'I*.'@ @ % %
:rK   r  c                  8    e Zd ZdZddgZddiZej                  Zy)r  Tr  r  N)	r   r  r  r  r   r   r8   r  r  r	  rK   rI   r  r    s$    "G$K$IIrK   r  c                  D    e Zd ZdZg dZdeddZddgZej                  Z
y)r  T)r  r  r  r  NFr  r  r  )r   r  r  r  r   r7   r   r  r8   r  r  r	  rK   rI   r  r    s,    ";K#j5IIg&M		IrK   r  c                  \    e Zd ZdZddgZej                  Zej                  d        Z
ddZy)IsinTr  valuesc                    t        t        | j                  j                        j	                  t        | j                  j                        j
                  dg               S rM   )r*   r+   r  rN   isinr  r_   s    rI   rN   z
Isin._meta  sK    $****+00djj../44aS9
 	
rK   c                     |j                   dk(  S r  r1  r  s     rI   r  zIsin._broadcast_dep  s    !##rK   Nr  )r   r  r  r  r   r8   r+  r  r  r  rN   r  r	  rK   rI   r(  r(    s8    "H%KI
 
$rK   r(  c                  H    e Zd ZdZg dZddddZdgZej                  Z	d Z
y)ra  T)r  r_  r`  r  Nr^  r  c                >    t        |t              rt        | ||      S y rB   rr   r   r  rE   s      rI   r  zClip._simplify_up       fj)*4DD *rK   )r   r  r  r  r   r   r  r8   rb  r  r  r	  rK   rI   ra  ra    s/    "5Kt<IHMIErK   ra  c                  .    e Zd ZdZdZdgZ ee      ZdZ	y)ArrowStringConversionTr  N)
r   r  r  r  rC   r   rW  r   r  r  r	  rK   rI   r2  r2    s&    ")K./I*.'rK   r2  c                  4    e Zd Zg dZddiZej                  Zy)Between)r  leftright	inclusiver7  bothN)r   r  r  r   r   r8   betweenr  r	  rK   rI   r4  r4    s    9Kf%I		IrK   r4  c                  H    e Zd ZdZg dZdddZej                  ZdZ	dZ
d Zy)rm  T)r  rk  rl  Nr  rj  c                    t        t        j                  | j                  j                        j                  | j                  | j                              S ri  )r  rv   rb   r  r  rn  rk  rl  r_   s    rI   r  zToTimestamp._divisions  s=    HHTZZ))*77TYYDHH7U
 	
rK   )r   r  r  r  r   r   r8   rn  r  rC   r  r  r	  rK   rI   rm  rm    s1    "*Kg.II*.'
rK   rm  c                  d    e Zd ZU g dZddiZded<   ej                  Ze	j                  d        Zy)CombineSeriesr  r   r!  r  r  Nr  r   c                    t        t        | j                  j                        j	                  t        | j
                  j                        | j                              S )N)r!  )r*   r+   r  rN   combiner   r!  r_   s    rI   rN   zCombineSeries._meta  sI    $****+33djj../dii 4 
 	
rK   )r   r  r  r   r   r  r8   r@  r  r  r  rN   r	  rK   rI   r=  r=    s7    :K#T*It*		I
 
rK   r=  c                  6    e Zd Zej                  dgz   ZdddZy)CombineFrame	overwriteNT)r  rC  )r   r  r  r=  r   r   r	  rK   rI   rB  rB    s    ++{m;K#$7IrK   rB  c                       e Zd Zg dZddddZdgZ eej                        Z	e
j                   fd       Ze
j                   fd       Z xZS )	ToNumeric)r  errorsdowncastrR   raiseN)rF  rG  rR   rR   c                @    t         |   }|j                  dd        |S r5  )r   r  r}  )rF   r\   r   s     rI   r  zToNumeric._kwargs  s    

64 rK   c                \    | j                  d      | j                  d      S t        |   S r5  )r   r   rN   rF   r   s    rI   rN   zToNumeric._meta  s+    <<+<<''w}rK   )r   r  r  r   r   r  rW  rv   
to_numericr  r  r  r  rN   r  r  s   @rI   rE  rE    s[    9K"dCIHMR]]+I 
  rK   rE  c                  |    e Zd Zg dZddiZddgZej                  d        Ze	d        Z
ej                  d        Zy)
ToDatetime)r  r\   rR   r\   NrR   c                $    | j                  d      S r5  r   r_   s    rI   rN   zToDatetime._meta(  s    ||F##rK   c                 >     t        | d         j                  | i |S rM   )r:   to_datetimer  s     rI   r  zToDatetime.operation,  s$    4Q(44dEfEErK   c                0    | j                  d      x}i S |S )Nr\   rP  r[   s     rI   r  zToDatetime._kwargs0  s     ll8,,F5IrK   )r   r  r  r   r   r  r  r  rN   rW  r  r  r	  rK   rI   rN  rN  #  sc    -K4 Iv&M$ $ F F  rK   rN  c                  B    e Zd Zg dZdddZ eej                        Zy)ToTimedelta)r  unitrF  NrH  )rV  rF  )	r   r  r  r   r   rW  rv   r  r  r	  rK   rI   rU  rU  7  s    -K1IR__-IrK   rU  c                  j     e Zd ZdZddgZej                  ZdZe	j                   fd       Zd Z xZS )rZ  z7A good example of writing a trivial blockwise operationr  r[  Tc                   d }t         |   }| j                  d      }t        |d      r:|j	                         D cg c]  \  }} ||      s| }}}t        ||      }|S  ||      rt        |      }|S c c}}w )Nc                    t        t        j                  j                  j	                  |       t        j
                        xr t        | dd       d u S )N
categories)rr   rv   apitypespandas_dtypeCategoricalDtypegetattrr]  s    rI   _cat_dtype_without_categoriesz3AsType._meta.<locals>._cat_dtype_without_categoriesF  s@    266<<44U;R=P=PQ ?E<6$>rK   r[  r  )cols)r   rN   r   rZ  r  r1   )rF   r`  rR   r[  r  vset_unknownr   s          rI   rN   zAsType._metaD  s    	 w}h'67#$llna0Ma0PK  *$[AD  +62)$/Ds   A;A;c                   t        |t              r#| j                  ||      r| j                  |      S t        |t              r#| j                  d      }t        | ||      }t        |t              rS|j                         D ci c]  \  }}||v s|| }}}|s) t        |      | j                  g|j                  dd   S t        |t              r(| j                  j                  D cg c]	  }||v s| }}| j                  j                  |k(  ry  t        |       | j                  |   |      }t        |t              s|S  t        |      |g|j                  dd   S y c c}}w c c}w )Nr[  r  )rr   r   rJ   r  r   r   rT  r  r  ro   r  rq   r  r   )	rF   rG   rH   r[  r   r   r  rV  r  s	            rI   r  zAsType._simplify_upX  sN   ff%$*L*LJ+
 ..v66fj)\\(+F1$
KG&$'39<<>TxsCSG^#s(TT'4<

IV__QR5HII'4(*.***<*<O3w3OOzz!!W,T$Z

7 3V<Fgt,4<=)<== * U Ps   E)E).	E/8E/)r   r  r  r  r   r8   r\  r  rC   r  r  rN   r  r  r  s   @rI   rZ  rZ  =  s<    AH%KI &>rK   rZ  c                  .    e Zd ZdZdgZej                  Zy)rq  Tr  N)r   r  r  r  r   r8   rr  r  r	  rK   rI   rq  rq  n  s    ")KIrK   rq  c                  L    e Zd ZdZg dZdej                  iZej                  Z
y)r  Tr  r{  r   r   N)r   r  r  r  r   r  r  r   r8   r  r  r	  rK   rI   r  r  t  s$    ",K"&&!IIrK   r  c                  0    e Zd ZdZddgZej                  Zy)rw  Tr  rv  N)r   r  r  r  r   r8   rx  r  r	  rK   rI   rw  rw  {  s    "J'KIrK   rw  c                  L    e Zd ZdZg dZdej                  iZej                  Z
y)r}  Trg  r   N)r   r  r  r  r   r  r  r   r8   r  r  r	  rK   rI   r}  r}    s$    ",K"&&!IIrK   r}  c                   t        |       s| S t        |       r| }n	 | j                  j                  d      }|j                         }|j                         }dx}}	t        j                  |      s||k\  }t        j                  |      s|r||k  n||k  }	|r|	st        d| d| d| d| d| d      | S # t        $ r | j                  }Y w xY w)Nr   Tz1`enforce_runtime_divisions` failed for partition z. Expected a range of [z, z),  but the real range was [z].)
r  r"   rc   get_level_valuesrP   rC  r+  rv   rr  RuntimeError)
r=  rW   division_mindivision_maxlastrc   real_minreal_max	valid_min	valid_maxs
             rI   _check_divisionsrt    s    r7	R	HH--a0E yy{Hyy{H ! I	77< ,	77< 26X-X=T	)?s C%%1N"\N C((0zH:RA
 	

 I'  	HHE	s   B8 8CCc                  R    e Zd ZdgZ ee      ZdZej                  d        Z
ddZy)EnforceRuntimeDivisionsr  Tc                .    | j                   j                  S rB   r  r_   s    rI   rN   zEnforceRuntimeDivisions._meta  r  rK   c                    | j                   D cg c]  }| j                  ||       }}||| j                  |   | j                  |dz      || j                  dz
  k(  gz   }t	        || j
                  g| S c c}w r  )r  r  r  rU   r   r  r  s        rI   r  zEnforceRuntimeDivisions._task  s    9=D2##B.DDNN5!NN519%d&&*+	
 
 D$..0400 Es   A7Nr  )r   r  r  r   rW  rt  r  r  r  r  rN   r  r	  rK   rI   rv  rv    s7    )K-.I*.'   1rK   rv  c                  .    e Zd ZdZdgZej                  Zy)rW  Tr  N)r   r  r  r  r   r8   rX  r  r	  rK   rI   rW  rW    s    ")KIrK   rW  c                  N    e Zd ZdZdZg dZeeeddZg dZe	j                  ZdZy)r  T)r  r  rc   r   r  r   r  N)r   r  r  r  rC   r   r7   r   r  r8   r  r  r  r	  rK   rI   r  r    s<    "AK	I ;MI*.'rK   r  c                  .    e Zd ZdgZej
                  ZdZy)NotNullr  TN)r   r  r  r   r8   notnullr  r  r	  rK   rI   r|  r|    s    )K		I"rK   r|  c                  b    e Zd ZddgZdeiZdgZej                  Z	dZ
ej                  d        Zy)ToFramer  r  Tc                b   t               }t        t        | j                  j                  | j                              | j                  j
                  D ]W  }t        |t              r$|j                  t        fd|D                     7|j                  j                  ||             Y |S )Nc              3  B   K   | ]  }j                  ||        y wrB   )r   )r<  rb  name_mappings     rI   r>  zHToFrame.unique_partition_mapping_columns_from_shuffle.<locals>.<genexpr>  s      FA!1!1!Q!7 Fs   )
r  r  r   r  r   r  rr   r  r  r   )rF   r  r	  r  s      @rI   r  z5ToFrame.unique_partition_mapping_columns_from_shuffle  s    C

 2 2DLLABJJLL 	9D$&

5 F FFG

<++D$78		9
 rK   N)r   r  r  r   r7   r   r  r8   to_framer  rC   r  r  r  r	  rK   rI   r  r    sD    F#K$IHM

I rK   r  c                  B    e Zd Zg dZeddZddgZej                  Z	dZ
y)ToFrameIndex)r  rc   r  Tr  rc   r  rc   N)r   r  r  r   r7   r   r  r8   r  r  rC   r	  rK   rI   r  r    s*    ,K#d3IW%M

IrK   r  c                  2    e Zd ZeddZej                  ZdZy)ToSeriesIndexNr  T)	r   r  r  r7   r   r8   	to_seriesr  r  r	  rK   rI   r  r    s    #d3II*.'rK   r  Fc                V   t        |      }|rVt        |t        j                  j                        st        j                  j	                  |      }| j                  d|      } t        t        |       ||      }| j                  dk(  r| j                         } | j                  |      S )Ng      ?)r  random_stater  )_split)r  rr   r  randomRandomStater  r>   r  rO   r  assign)r=  r  r  shufflerc   s        rI   pd_splitr    s    QA,		(=(=>9900>LYYClY;R!\2E	ww!|[[]99E9""rK   c                  ~    e Zd Zg dZddgZ ee      Zej                  dd       Z
ej                  d        Zd	dZy)
Split)r  r  r  r  r  r  c                    d| j                   iS )Nr  )r  r_   s    rI   r  zSplit._kwargs  s    4<<((rK   c                V    t        | j                  j                  | j                        S rB   )r?   r  rU   r  r_   s    rI   r?   zSplit.random_state_data  s     !7!79J9JKKrK   c                    | j                   D cg c]  }| j                  ||       }}| j                  j                         }| j                  |   |d<   t        || j                  g|i |S c c}w )Nr  )r  r  r  r  r?   r   r  rJ  s         rI   r  zSplit._task  sn    9=D2##B.DD""$!%!7!7!>~D$..:4:6:: Es   A0Nr  r  )r   r  r  r   r  rW  r  r  r  r  r  r?   r  r	  rK   rI   r  r    sT    >K#Y/MX&I) ) L L;rK   r  c                j    | | d   |k(     j                  d      } |dk(  r| | j                  d      S | S )Nr  r  r  r   )dropr   )r=  rW   rO   s      rI   _random_split_taker    sB    	BxLA		#	#H	#	5Bqy"**Q-  IrK   c                  $    e Zd Zg dZ ee      Zy)	SplitTake)r  rW   rO   N)r   r  r  r   rW  r  r  r	  rK   rI   r  r    s    (K/0IrK   r  c                  f    e Zd ZdZg dZdi dZej                  Ze	j                  d        ZddZy)	r  z<A good example of writing a less-trivial blockwise operation)r  r  r  rR   r\   r	  r  c                b    t        | j                  d      | j                  j                        S )NrR   r&  )r*   r   r  rN   r_   s    rI   rN   zApply._meta)  s"    f-4::;K;KLLrK   c                    t        |t        j                  t        | j                  j
                  |f      | j                  g| j                  i | j                  S rB   )	r   r8   r  r   r  rV   r  r  r\   r  s      rI   r  zApply._task-  sS    GGTZZ%%u-.MM	

 YY
 kk
 	
rK   Nr  )r   r  r  r  r   r   r8   r  r  r  r  rN   r  r	  rK   rI   r  r  "  s<    FAKr*IIM M
rK   r  c                       e Zd ZdZg dZddddZddgZej                  Z	e
j                  d        Ze
j                  d
d       Z fd	Z xZS )MapT)r  r
  	na_actionrR   is_monotonicN)r  rR   r  rR   r  c           	        | j                  d      c| j                  D cg c])  }t        |t              rt	        |j
                        n|+ }}t         | j                  |i | j                        S t        | j                  d      | j                  j
                  t        | j                  j
                  dd             S c c}w )NrR   rc   )r&  rc   )r   r  rr   r   r+   rN   r*   r  r  r  r_  r   s      rI   rN   z	Map._meta?  s    <<' ** ,6b$+?bhh'RGD  ^T^^TBT\\BCCLL 

(($****GT:
 	
s   .Cc                    i S rB   r	  r_   s    rI   r  zMap._kwargsM      	rK   c                Z   | j                   s"dt        | j                  j                        z  S t	        | j                  j
                        rWt        t        j                  | j                  j                        j                  | j                  | j                              S t        | j                  t              r_| j                  j                  | j                  j                  k(  r| j                  j                  S d| j                  j                  dz   z  S t        | A         S )NrB   )r  r  )r  r  r  r  r"   rN   r  rv   r   mapr
  r  rr   r   rU   r   r  rK  s    rI   r  zMap._divisionsQ  s       S!5!56664::++,		$**../33DHH3W  $'xx!!TZZ%9%99zz+++ $**"8"81"<==w!##rK   r  )r   r  r  r  r   r   r  r8   r  r  r  r  rN   r  r  r  r  s   @rI   r  r  8  si    "GK"D$GI^,MI
 
  $ $rK   r  c                  v     e Zd Zg dZddddZg dZej                  ZdZ	e
j                  d fd       Z xZS )r  )r  r  r!  r  Tr  F)r  r!  r  c                     ddit         |   S )Nr  r  )r   r  rK  s    rI   r  zVarColumns._kwargsj  s    -UW_--rK   r  )r   r  r  r   r   r  r8   r"  r  r  r  r  r  r  r  s   @rI   r  r  c  s@    =KEBI6MI . .rK   r  c                  6    e Zd Zg dZdddZej                  Zy)NUniqueColumns)r  r  r>  r  T)r  r>  N)r   r  r  r   r   r8   nuniquer  r	  rK   rI   r  r  o  s    -Kd+I		IrK   r  c                  *    e Zd ZdgZej
                  Zy)r$  r  N)r   r  r  r   r  sqrtr  r	  rK   rI   r$  r$  u  s    )KIrK   r$  c                  *    e Zd ZdgZej
                  Zy)ExplodeSeriesr  N)r   r  r  r   r8   exploder  r	  rK   rI   r  r  z  s    )K		IrK   r  c                      e Zd ZddgZd Zy)ExplodeFramer  columnc                V    t        |t              rt        | ||| j                  g      S y rB   )rr   r   r  r  rE   s      rI   r  zExplodeFrame._simplify_up  s(    fj)*4dkk]SS *rK   N)r   r  r  r   r  r	  rK   rI   r  r    s    H%KTrK   r  c                  6    e Zd Zg dZddiZ ee      ZdZd Z	y)Drop)r  r   rF  rF  rH  Tc                    | j                  d      }t        |      r|g}| j                  j                  D cg c]	  }||vs| }}t	        | j                  |      S c c}w Nr   )r   r3   r  r   r   )rF   col_oprV  r   s       rI   r  zDrop._simplify_down  sZ    i(VXF"&**"4"4J368I3JJ$**g.. Ks   	AAN)
r   r  r  r   r   rW  r2   r  r  r  r	  rK   rI   r  r    s'    0K7#I12I*.'/rK   r  c                L   t        t        d|            }| j                  d      } t        j                         5  t        j
                  ddt               |j                         D ]"  \  }}t        |t              r ||       }|| |<   $ 	 d d d        | S # 1 sw Y   | S xY w)Nrm   Fr  ignorez DataFrame is highly fragmented *)messagecategory)
r  r   r  warningscatch_warningsfilterwarningsr   r  rr   r   )r=  pairsr  r  s       rI   r  r    s    1e$%E	e	B		 	 	" 	6'	

  	ID##x("gBtH		 I	 Is   ABB#c                       e Zd ZdZdgZ ee      Zej                  d        Z
ej                  d        Zej                  d        Zej                  d        Z fdZd Zd	 Zd
 Zd Zd Z xZS )AssignzColumn Assignmentr  c                    t        | j                        }| j                  j                  D ch c]2  }t	        |t
              s||vst        |      j                  |      s|4 c}S c c}w rB   )r  keysr  r  rr   r  intersection)rF   r  rV  s      rI   r  z4Assign.unique_partition_mapping_columns_from_shuffle  s^    499~ zzOO
c5)4s8((.	 
 	
 
s   7A(c                &    | j                   dd d   S r  rq   r_   s    rI   r  zAssign.keys      }}QTT""rK   c                &    | j                   dd d   S Nrm   r  r_   s    rI   valszAssign.vals  r  rK   c                    | j                   D cg c]   }t        |t              r|j                  n|" }}t	         | j
                  |i | j                        S c c}w rB   )r  rr   r   rN   r*   r  r  r   s      rI   rN   zAssign._meta  sP    CG::NRJr40b8NN>>?? Os   %Ac                    |dk(  rt         |   |||      S |dz  dk(  r"|dk(  rdnd}|| dt        |      dd  dz  }|S |t        |       z  }|S )	Nr   rm   r   ,rn   r  r   )r   rz   repr)rF   rW   r   r}   sepr   s        rI   rz   z'Assign._tree_repr_argument_construction  sv    67;Ar6JJq5A:Q"CCQtBx"~.a00F  b
#FrK   c                    | j                   S rB   r  r_   s    rI   _node_label_argszAssign._node_label_args  s    }}rK   c                   t        | j                        t        |j                        z  rt        | j                        }t        |j                  |j                        D cg c]  \  }}||vs||g }}}|j                  gt        t        |            z   | j                  dd  z   S |j                  | j                  dd  z   S c c}}w r  )r  r  r   r  r  r  r   rq   )rF   r   r  r  rb  rq   s         rI   _remove_common_columnszAssign._remove_common_columns  s    tyy>C

O+tyy>D+.uzz5::+FX41a!SW-AXHXKK=4(9#::T]]12=NNN>>DMM!"$555 Ys   $C1Cc                   t        | j                  t              r=| j                  | j                        ry t        | j	                  | j                         S t        | j                  t
              rt        | j                  j                  t              r| j                  | j                  j                        ry | j                  j                  d   j                         }|j                  | j                  D cg c]	  }||vs| c}       t        t        | j	                  | j                  j                         |      S y y c c}w r  )
rr   r  r  $_check_for_previously_created_columnr  r   rq   r  rs   r  )rF   new_columnsr  s      rI   r  zAssign._simplify_down  s    djj&)88D466tzzBCC

J/JJJf5
 889I9IJ**--a0557K499Ma8LMN33DJJ4D4DEF 5
/  Ns   <	EEc                    g }| j                   D ].  }t        |t              s|j                  |j                         0 t        t        |      t        |j                        z        S rB   )r  rr   r   rs   r   boolr  r  )rF   childinput_columnsrb  s       rI   r  z+Assign._check_for_previously_created_column  sU     	0A!T"$$QYY/	0 C&UZZ899rK   c                X   t        |t              rt        | ||      }t        |      }t	        |      t	        | j
                        z
  }|t	        | j                  j                        k(  ry t	        | j
                        t	        |      z
  }t        |      t        | j
                        k(  r) t        |      | j                  g|j                  dd   S t        |      dkD  rCg }t        | j
                  | j                        D ]  \  }}||v s|j                  ||g        n| j                  dd  }| j                  j                  D 	cg c]	  }	|	|v s|	 }}	 t        |       t        |       | j                  t        |         g| g|j                  dd   S y c c}	w )Nr  r   )rr   r   rT  r(   r  r  r  r   r  ro   rq   r   r  rs   sorted)
rF   rG   rH   r   ra  r  new_argsr  rb  rV  s
             rI   r  zAssign._simplify_up  sr   fj)1$
KG&w/Gw<#dii.0Ds4::--..tyy>CL0D4yC		N*#tF|DJJE1DEETQ		4995 0DAqG| A/0  ==,&*jj&8&8HsC4KsHGH4<T
4::fWo6BB$ + *( Is   	F'F')r   r  r  r  r   rW  r  r  r  r  r  r  r  rN   rz   r  r  r  r  r  r  r  s   @rI   r  r    s    )KV$I
 
 # # # # @ @6":rK   r  c                  `    e Zd Zg dZdi iZdgZej                  Ze	j                  dd       Zy)Evalr  r  c                    i | j                   S rB   r  r_   s    rI   r  zEval._kwargs  r  rK   Nr  )r   r  r  r   r   r  r8   evalr  r  r  r  r	  rK   rI   r  r    s9    3K#I"OMI$ $rK   r  c                  j    e Zd ZdgZej
                  d        Zej
                  d        Zed        Z	y)CaseWhenr  c                    | j                   dd  }t        dt        |      d      D cg c]  }||   ||dz      f c}S c c}w Nr  r   rm   )rq   rT   r  )rF   r  rW   s      rI   caselistzCaseWhen.caselist  sD    MM!"*/3q61*=>Q1qQx >>>s   ?c           	        | j                   dd  }t        dt        |      d      D cg c]k  }t        ||   t              rt        ||   j                        n||   t        ||dz      t              rt        ||dz      j                        n||dz      fm }}t        t        | j                  j                        j                  |            S c c}w r  )
rq   rT   r  rr   r   r+   rN   r*   r  	case_when)rF   r  rW   r  s       rI   rN   zCaseWhen._meta  s    MM!" 1c!fa(

  .8!d-Cadjj)1 "!AE(D1 "!AE(..11q5

 

 tzz'7'78BB8LMM

s   A0Cc                    t        dt        |      d      D cg c]  }||   ||dz      f }}| j                  t        |            S c c}w )Nr   rm   r  )rT   r  r  r  )serr  rW   s      rI   r  zCaseWhen.operation/  sM    <A!S]TU<VWqXa[(1q5/2WW}}T(^,, Xs   A	N)
r   r  r  r   r  r  r  rN   rW  r  r	  rK   rI   r  r    sQ    )K? ? N N - -rK   r  c                  >    e Zd ZdZdZddgZej                  ZdZ	d Z
y)r   Tr  r  c                   t        | j                  t              rqt        | j                        }|j                  | j                  j                  k7  r9 t        |       t        |       | j                  |      g|j                  dd   S t        |t        t        f      rt        | j                  t        t        f      sk| j                  j                  | |      sOt        | ||d      r@| j                  | j                  |j                  j                  | | j                        z     S t        |t              rd| j                  j                  | |      r;t        | j                  t        t        f      sy t        | j                  | |d      ry t        | ||      S t        |t              r#| j                  j                   | j                     S y )Nr  F)allow_reduction)rr   r  r   rewrite_filtersrV   ro   r  rq   FilterAlignr   rJ   rD   r  r   r  rb   rc   )rF   rG   rH   r  s       rI   r  zFilter._simplify_up<  s|   dnnb)$T^^4F||t~~333#tF|DJtzz625;__QR5H  f{F34ZJJf->
 ::;;D*M 0&*e  ::)9)9)D)DT4::)VV  fj)zz77jI "$**{F.CD1JJj% *4DDfe$::##DNN33 %rK   N)r   r  r  r  rC   r   r  r  r  r  r  r	  rK   rI   r   r   5  s.    "K(K  I*.'*4rK   r   c                       e Zd ZdZddgZej                  Zej                  d        Z
ed        Zej                   fd       Zd Zd Z fd	Zd
 Z xZS )r   Column Selectionr  r   c                   | j                  d      }t        |t              rt        |      n|h}| j                  j
                  D ch c]2  }||v s*t        |t              rt        |      j                  |      r|4 c}S c c}w r  )r   rr   r  r  r  r  r  issubset)rF   r  r   r  s       rI   r  z8Projection.unique_partition_mapping_columns_from_shuffleo  sq    i(!+FD!9#f+x ZZMM
G|z!U3A8P 
 	
 
s   7Bc                    | j                  d      }t        |t              r|S t        |t        j                        rt        |      S |gS r  )r   rr   r  rv   rb   )rF   ra  s     rI   r   zProjection.columnsy  s>    ||I&dD!Kbhh':6MrK   c                Z   t        | j                  j                        r
t        |   S t	        | j                  d      t        t        f      sGt        | j                  d      d      s,t        | j                  j                        j                  d   S | j                  j                  S )Nr   r^  r   )r!   r  rN   r   rr   r   r  slicerZ  r+   r  rK  s    rI   rN   zProjection._meta  s~    TZZ--.7= $,,y1D%=A'LL#WK
 !!1!1277::zzrK   c                <    | j                   | j                  d      gS r  )r  r   r_   s    rI   r  zProjection._node_label_args  s    

DLL344rK   c                    t        | j                        }d|v rd|z   dz   }| dt        | j                  d             dS )Nrn   (r,  [r   ])r  r  r  r   )rF   rx   s     rI   r.  zProjection.__str__  sF    4::$;:#Dqdll9567q99rK   c                @    | j                   dk(  ryt        | 	         S )Nr   r  )rO   r   r  rK  s    rI   r  zProjection._divisions  s    99>w!##rK   c                .   t        | j                  j                        t        | j                        k(  rC| j                  j                  | j                  j                  j                  k(  r| j                  S t        | j                  t              r| j                  j                  d      | j                  d      }t        t              sy t        |t              rt        fd|D              sJ |v sJ | j                  j                  |   S y )Nr   c              3  &   K   | ]  }|v  
 y wrB   r	  )r<  bbas     rI   r>  z,Projection._simplify_down.<locals>.<genexpr>  s     /r27/   )
r  r  r   rN   rO   rr   r   r   r  r3  )rF   r  r  s     @rI   r  zProjection._simplify_down  s    

""#s4<<'88

4::#3#3#8#88 ::djj*-

""9-AY'Aa&At$/Q////Avv::##A&& .rK   )r   r  r  r  r   r  r  r  r  r  r  r  r   rN   r  r.  r  r  r  r  s   @rI   r   r   i  su    I&K  I
 
   	  	 5:$
'rK   r   c                  z    e Zd ZdZdgZeZej                  d        Z	e
d        ZddZej                  d        Zy)	rb   r  r  c                    | j                   j                  }t        |      st        |      r | j                   j                  j                  S |S rB   )r  rN   r#   r!   rc   rQ   s     rI   rN   zIndex._meta  s<    zz$#4T#:::##)))rK   c                    g S rB   r	  r_   s    rI   r  zIndex._projection_columns  r  rK   c                d    t        |t        t        | j                  j                  |f      d      S )Nrc   )r   r_  r   r  rV   r  s      rI   r  zIndex._task  s/    TZZ%%u-.	
 	
rK   c                    | j                   j                  j                  j                  }|| j                   j                  v r|hS |f| j                   j                  v r|fhS t               S rB   )r  rN   rc   r  r  r  )rF   r  s     rI   r  z3Index.unique_partition_mapping_columns_from_shuffle  s[    zz%%**4::KKK6MW

PPPG95LrK   Nr  )r   r  r  r  r   r_  r  r  r  rN   r  r  r  r  r	  rK   rI   rb   rb     s\    )KI   
  rK   rb   c                    | S rB   r	  )r=  r  s     rI   _return_inputr    s    IrK   c                  ,    e Zd ZdgZ ee      ZdZd Zy)ClearDivisionsr  Tc                :    d| j                   j                  dz   z  S r  r  r_   s    rI   r  zClearDivisions._divisions  r  rK   N	r   r  r  r   rW  r  r  r  r  r	  rK   rI   r  r    s    )K]+I*.'6rK   r  c                  .    e Zd ZddgZ ee      ZdZd Zy)SetDivisionsr  r  Tc                $    | j                  d      S )Nr  rP  r_   s    rI   r  zSetDivisions._divisions  s    ||K((rK   Nr  r	  rK   rI   r  r    s!    K(K]+I*.')rK   r  c                  D    e Zd Zg dZej
                  d        Zd Zd Zy)ResolveOverlappingDivisions)r  minsmaxeslensc                .    | j                   j                  S rB   r  r_   s    rI   rN   z!ResolveOverlappingDivisions._meta  r  rK   c                    t        | j                        D cg c]  \  }}|dk7  s| }}}t        |      dk(  ryt        | j                        | j
                  d   fz   S c c}}w )Nr   r  r  )rp   r  r  r  r  r  )rF   rW   lengthnon_emptiess       rI   r  z&ResolveOverlappingDivisions._divisions  s]    *3DII*>NYQ&A+qNN{q TYY4::b>"333	 Os
   A#A#c                   t        | j                        D cg c]  \  }}|dk7  s| }}}t        |      dk(  r&| j                  df| j                  j                  dfiS t        |      D ci c]*  \  }}| j                  |f| j                  j                  |f, }}}t        |j                               }t        dt        | j                              D cg c]'  }| j                  |   | j                  |dz
     k\  s&|) }}| j                  }g }	|D ]  }|	j                  t        ||dz
     ||   f       t        || j                  |dz
  f   ||   f|| j                  |dz
  f<   ||   ||dz      k(  r|dz   |v rj|	j                  ||          t        j                  |	f|| j                  |f<   g }	 |S c c}}w c c}}w c c}w rq  )rp   r  r  rV   r  r  r)  rT   r  r  r  r   r0   r/   r   r  )
rF   rW   r  r  divr  ddf_keysoverlapr  framess
             rI   r  z"ResolveOverlappingDivisions._layer  s   *3DII*>NYQ&A+qNN{q ZZOdjj&6&6%:;; $K0
3 ZZOdjj..44
 
 

% QDII/
499Q<4::aRSeCT3TA
 
 NN	 	A MM;Q1FG
 TZZQ'(!(CQU#$ |yQ//AEW4DMM(1+&$+NNF#;CQ F/	0 
W O

s   F?F?0/G'GGNr  r	  rK   rI   r  r    s)    4K   4,rK   r  c                  L    e Zd ZdZdgZej                  d        Zd Zd Z	d Z
y)Lengthsz$Returns a tuple of partition lengthsr  c                    t               S rB   )r  r_   s    rI   rN   zLengths._meta1	  s	    wrK   c                     yr  r	  r_   s    rI   r  zLengths._divisions5	  r  rK   c                    t        | j                  t              r1t        | j                  j	                         d       }t        |      S y )Nc                    | j                   S rB   r1  exprs    rI   <lambda>z(Lengths._simplify_down.<locals>.<lambda>:	  s    DDTDT rK   r   )rr   r  r  r+  r	  r$  )rF   r  s     rI   r  zLengths._simplify_down8	  s9    djj(+

//17TUE5>! ,rK   c                    d| j                   z   }t        | j                  j                        D ci c]#  }||ft        | j                  j                   |ff% }}t
        t        |j                               f|| j                   df<   |S c c}w )Nzpart-r   )rV   rT   r  rU   r  r  r  r  )rF   r  rW   r  s       rI   r  zLengths._layer=	  s    # 4::112
 1Idjj..233
 
 !&tCHHJ'78TZZO

s   (BN)r   r  r  r  r   r  r  rN   r  r  r  r	  rK   rI   r$  r$  ,	  s3    .)K "
rK   r$  c                  |    e Zd ZdZg dZdedZddgZej                  Z
dZdZej                  dd       Zd	 Zd
 Zy)
ResetIndexz(Reset the index of a Series or DataFrame)r  r  r  F)r  r  r  r  Tc                    d| j                   i}| j                  d      t        ur"|j                  d| j                  d      i       |S )Nr  r  )r  r   r7   rI  r[   s     rI   r  zResetIndex._kwargsQ	  s@    $))$<<z1MM64<<#789rK   c                :    d| j                   j                  dz   z  S r  r  r_   s    rI   r  zResetIndex._divisionsX	  r  rK   c                  	 t        |t              r2| j                  ||      r|| j                     D cg c]/  } |       &t         |       t              s |       j                  1 }}d }t        t        |t                    j                  t        | j                  j                              s| j                  d      xs* | j                  j                  j                  j                  }|t        u r-| j                  j                  j                  j                  d}nT| j                  j                  j                  j                  *| j                  j                  j                  j                  }t        | |      }|j                   j#                  |t%        | j                              }n|| j                  j&                  dk(  rc| j                  d      sR| j                  j                  j                  }t        | |      }|j                   j#                  || j                        }| j)                  ||      S t        |t              r2| j                  j&                  dk(  r| j*                  st        |j                  d      t              ry |j                  d      		| j                  d| j                  j                  j                  j                  fv ry t-        	fd|| j                     D              r' t/        |       | j                  d| j                        S y t1        | ||      }|Ht        |j                        t        |j                  j                        k7  r|j3                  ddi      }|S y c c}w )Nr  rc   r  r  r   c              3  ~   K   | ]4  }t         |       t              xr  |       j                  d       k(   6 yw)r   N)rr   r   r   )r<  r\  rV  s     rI   r>  z*ResetIndex._simplify_up.<locals>.<genexpr>	  s:       qsJ/QACKK	4Jc4QQs   :=T)rr   r   rJ   rV   r   r  r   r  r  r  r   rN   rc   r  r7   r   r  r  rb   rO   r  r  r3  ro   r  substitute_parameters)
rF   rG   rH   r  parentsr  r  r  r  rV  s
            @rI   r  zResetIndex._simplify_up[	  s   ff%$*L*LJ+

 $DJJ/3?:ac6+B G 
 Iww-.77DJJ<N<N8OP||F+Jtzz/?/?/E/E/J/J:%$***:*:*@*@*E*E*M"DZZ%%++00<::++1166D!$-",,77eDJJ>OP	A%dll6.Bzz'',,!$-",,77djjI	..vyAAfj)zz!#DIIfnnY7> nnY/499gtzz/?/?/E/E/J/JKK '

3  &4:djj$		BB,T6:FF!c&..&9SAUAU=V&V55vtnEM% */s   4N	Nr  )r   r  r  r  r   r7   r   r  r8   reset_indexr  rC   r  r  r  r  r  r  r	  rK   rI   r/  r/  G	  sU    2+K
3IV$MI*.' 6-rK   r/  c                  :    e Zd ZddgZej
                  ZdZdZd Z	y)AddPrefixSeriesr  prefixTc                T     t         fd j                  j                  D              S )Nc              3  N   K   | ]  }j                   t        |      z     y wrB   )r9  r  r<  rG  rF   s     rI   r>  z-AddPrefixSeries._divisions.<locals>.<genexpr>	  s     VXT[[3x=0V   "%r  r  r  r_   s   `rI   r  zAddPrefixSeries._divisions	      VAUAUVVVrK   N)
r   r  r  r   r8   
add_prefixr  rC   r  r  r	  rK   rI   r8  r8  	  s'    H%KI*.'WrK   r8  c                  6    e Zd ZddgZej
                  ZdZd Zy)AddSuffixSeriesr  suffixTc                T     t         fd j                  j                  D              S )Nc              3  N   K   | ]  }t        |      j                  z     y wrB   )r  rC  r<  s     rI   r>  z-AddSuffixSeries._divisions.<locals>.<genexpr>	  s     VXS]T[[0Vr=  r>  r_   s   `rI   r  zAddSuffixSeries._divisions	  r?  rK   N)	r   r  r  r   r8   
add_suffixr  r  r  r	  rK   rI   rB  rB  	  s!    H%KI*.'WrK   rB  c                  \    e Zd ZddgZej
                  Zej                  d        Z	d Z
d Zy)	AddPrefixr  r9  c                      j                   j                  D ch c]6  }t        |t              s j                   | nt         fd|D              8 c}S c c}w )Nc              3  <   K   | ]  }j                   |z     y wrB   )r9  r<  trF   s     rI   r>  zJAddPrefix.unique_partition_mapping_columns_from_shuffle.<locals>.<genexpr>	  s     6q4;;?6   )r  r  rr   r  r9  rF   r  s   ` rI   r  z7AddPrefix.unique_partition_mapping_columns_from_shuffle	  s]     ZZMM
  "!U+ ;;-s#6A667
 	
 
   ;Ac                \    t        | j                        }|D cg c]  }||d  	 c}S c c}w rB   )r  r9  )rF   r   
len_prefixrV  s       rI   _convert_columnszAddPrefix._convert_columns	  s+    %
,34SJK 444s   )c                   t        |t              rt        | ||      }| j                  t	        |            }t        |      t        | j                  j                        k(  ry | j                  j                  D cg c]	  }||v s| }} t        |       t        |       | j                  |   | j                  d         |j                  d            S y c c}w )Nr  r   )rr   r   rT  rR  r(   r  r  r   ro   rq   r   r  s        rI   r  zAddPrefix._simplify_up	  s    fj)1$
KG++,<W,EFG7|s4::#5#566&*jj&8&8KsC7NsKGK4<T
4::g.a0@Ay)  * Ls   <	CCN)r   r  r  r   r8   r@  r  r  r  r  rR  r  r	  rK   rI   rH  rH  	  s7    H%KI
 
5rK   rH  c                  V    e Zd ZddgZej
                  Zej                  d        Z	d Z
y)	AddSuffixr  rC  c                      j                   j                  D ch c]6  }t        |t              s|  j                   nt         fd|D              8 c}S c c}w )Nc              3  <   K   | ]  }|j                   z     y wrB   )rC  rK  s     rI   r>  zJAddSuffix.unique_partition_mapping_columns_from_shuffle.<locals>.<genexpr>	  s     6q1t{{?6rM  )r  r  rr   r  rC  rN  s   ` rI   r  z7AddSuffix.unique_partition_mapping_columns_from_shuffle	  s]     ZZMM
  "!U+ #dkk]#6A667
 	
 
rO  c                ^    t        | j                        }|D cg c]  }|d |  
 c}S c c}w rB   )r  rC  )rF   r   
len_suffixrV  s       rI   rR  zAddSuffix._convert_columns	  s-    %
-45cLj[!555s   *N)r   r  r  r   r8   rF  r  r  r  r  rR  r	  rK   rI   rU  rU  	  s2    H%KI
 
6rK   rU  c                  B    e Zd ZddgZ eej                        ZdZd Z	y)AssignIndexr  r  Tc                .    | j                   j                  S rB   )r  r  r_   s    rI   r  zAssignIndex._divisions	  r  rK   N)
r   r  r  r   rW  r   assign_indexr  r  r  r	  rK   rI   r[  r[  	  s'    G$KW112I*.'$rK   r[  c                      e Zd ZdZg dZdddZej                  d        Zej                  d        Z	d Z
dd	Zd
 Zd Zd Zed        Zy)Headz.Take the first `n` rows of the first partition)r  nrU      r  )r`  rU   c                .    | j                   j                  S rB   r  r_   s    rI   rN   z
Head._meta	  r  rK   c                     yr  r	  r_   s    rI   rU   zHead.npartitions	      rK   c                   | j                  d      dk  r2| j                  j                  d   | j                  j                  d   fS | j                  j                  d   | j                  j                  | j                  d         fS )NrU   r  r   )r   r  r  r_   s    rI   r  zHead._divisions	  ss    <<&",::''*DJJ,@,@,DDDJJ  #JJ  m!<=
 	
rK   c                    t               rB   r  r  s      rI   r  z
Head._task	  r  rK   c           
     (   t        | j                  t              r| j                  j                  D cg c]J  }t        |t              r6t        |t
              s&t        || j                  | j                  d            n|L }} t        | j                        | S t        | j                  t              rXt        | j                  j                  t        | j                  | j                  j                        | j                  d            S y c c}w )NrU   )rr   r  r  rq   r   r7  r_  r`  r   ro   rC  rF   r   rq   s      rI   r  zHead._simplify_down	  s    djj(+ **--  ""d+Jr<4P TVVT\\-%@AH  $4

#X..djj$'

  #dffdjjll";T\\-=X  (s   ADc                L    ddl m} t        ||      r|j                  dk(  r| S y y r  dask.dataframe.dask_exprr  rr   new_partitionsrF   rG   rH   r  s       rI   r  zHead._simplify_up
  *    8fk*v/D/D/IK 0J*rK   c                   t        | t              sF| j                  d      }| j                  d      | j                  j                  kD  r%t        d| j                  j                   d|       | j                  }t        | j                        r,t        t        | j                  |      | j                  d      S |dk(  r| j                  j                  dk7  rdnd}t        t        | j                  |      | j                  ||      }|dk7  rFdd	lm} || j                  j                  k7  xr |d
k7  }t         ||d      | j                  d|      }|S y )NrU   zonly z partitions, head received F)safer  Tr   r  r  )rl  )rr   BlockwiseHeadr   r  rU   r   _partitionsr"   rN   BlockwiseHeadIndex
Partitionsr`  rk  r  )rF   rU   
partitionsrp  r  r  s         rI   r  zHead._lower
  s4   $.,,}5K||M*TZZ-C-CC DJJ2233N{m\  ))JTZZ()tzz:6U  '!+

0F0F!0K4QVD!4::z2DFFKE a@"djj&<&<<RPRAR%a8$&&!T L1 /rK   c                    t        | t              r| j                  j                  }n(t	        t        | j                  j                              }| j                  d      dkD  r|d | j                  d       }|S )NrU   r  )rr   PartitionsFilteredr  rr  r  rT   rU   r   )rF   ru  s     rI   rr  zHead._partitions&
  sb    d.///JeDJJ$:$:;<J<<&+#$Adll=&ABJrK   Nr  )r   r  r  r  r   r   r  r  rN   rU   r  r  r  r  r  r  rr  r	  rK   rI   r_  r_  	  ss    8/K*I     
$ 6  rK   r_  c                  Z    e Zd ZdZg dZdZd Zd Zej                  d        Z
d Zd
dZy	)rq  zTake the first `n` rows of every partition

    Typically used after `Partitions(..., [0])` to take
    the first `n` rows of an entire collection.
    )r  r`  rU   rp  Tc                     y rB   r	  r_   s    rI   r  zBlockwiseHead._simplify_down;
      rK   c                     y rB   r	  rE   s      rI   r  zBlockwiseHead._simplify_up>
  rz  rK   c                :    t        | j                               dz
  S r  )r  r  r_   s    rI   rU   zBlockwiseHead.npartitionsA
  s    4??$%))rK   c                `    | j                   j                  d t        | j                        dz    S r  )r  r  r  rr  r_   s    rI   r  zBlockwiseHead._divisionsE
  s*    zz##$?c$*:*:&;a&?@@rK   c                    | j                   rt        }nt        j                  }t	        ||t        | j                  j                  |f      | j                        S rB   )	rp  r$   r8   r  r   r   r  rV   r`  )rF   r  rc   r   s       rI   r  zBlockwiseHead._taskH
  s@    99BBD"gtzz'7'7&?@$&&IIrK   Nr  )r   r  r  r  r   r  r  r  r  r  rU   r  r  r	  rK   rI   rq  rq  1
  sD     8K*.' * *AJrK   rq  c                      e Zd ZddZy)rs  c           	         t        |t        j                  t        | j                  j
                  |f      t        d| j                              S rM   r   r  r  r   r  rV   r  r`  r  s      rI   r  zBlockwiseHeadIndex._taskQ
  s@    (""GTZZ-=-=u,E$FaQUQWQWHX
 	
rK   Nr  r   r  r  r  r	  rK   rI   rs  rs  P
  s    
rK   rs  c                  d    e Zd ZdZddgZddiZej                  d        Zd Z	ddZ
d Zd	 Zd
 Zy)Tailz,Take the last `n` rows of the last partitionr  r`  ra  c                .    | j                   j                  S rB   r  r_   s    rI   rN   z
Tail._meta]
  r  rK   c                4    | j                   j                  dd  S )Nr  r_   s    rI   r  zTail._divisionsa
  s    zz##BC((rK   c                    t               rB   r  r  s      rI   r  z
Tail._taskd
  r  rK   c                   t        | j                  t              ra| j                  j                  D cg c]*  }t        |t              rt        || j                        n|, }} t        | j                        | S t        | j                  t
              rHt        | j                  j                  t        | j                  | j                  j                              S y c c}w rB   )	rr   r  r  rq   r   r  r`  ro   rC  rh  s      rI   r  zTail._simplify_downg
  s    djj(+ **-- %/r4$8R b@H  $4

#X..djj$'

((#dffdjjll*CDD (s   /Cc                L    ddl m} t        ||      r|j                  dk(  r| S y y r  rj  rm  s       rI   r  zTail._simplify_upq
  rn  rK   c                V   t        | t              st        | j                        rBt	        t        | j                  | j                  j                  dz
  g      | j                        S t        t        | j                  | j                  j                  dz
  g      | j                        S y r  )	rr   BlockwiseTailr"   rN   BlockwiseTailIndexrt  r  rU   r`  r_   s    rI   r  zTail._lowerw
  s    $.TZZ()tzzDJJ,B,BQ,F+GH$&&  !4::

(>(>(B'CDdff  /rK   Nr  )r   r  r  r  r   r   r  r  rN   r  r  r  r  r  r	  rK   rI   r  r  W
  sI    6C.KaI   )$E	rK   r  c                  "    e Zd ZdZdZd ZddZy)r  zTake the last `n` rows of every partition

    Typically used after `Partitions(..., [-1])` to take
    the last `n` rows of an entire collection.
    Tc                .    | j                   j                  S rB   r  r_   s    rI   r  zBlockwiseTail._divisions
  r  rK   c                    t        |t        j                  t        | j                  j
                  |f      | j                        S rB   )r   r8   r  r   r  rV   r`  r  s      rI   r  zBlockwiseTail._task
  s/    D!&&'4::+;+;U*C"DdffMMrK   Nr  )r   r  r  r  r  r  r  r	  rK   rI   r  r  
  s     +/'$NrK   r  c                      e Zd ZddZy)r  c           	         t        |t        j                  t        | j                  j
                  |f      t        | j                   d             S rB   r  r  s      rI   r  zBlockwiseTailIndex._task
  sB    TZZ%%u-.466'4 	
 	
rK   Nr  r  r	  rK   rI   r  r  
  s    
rK   r  c                  \     e Zd ZddgZej
                  d        Zd Zd Zd Z	 fdZ
 xZS )Binopr5  r6  c                    | j                         }d|D ch c]  }|j                   c}v xr& t        |D ch c]  }|j                   c}      dk(  S c c}w c c}w r  )r	  rU   r  rO   )rF   depsr  s      rI   _broadcastablezBinop._broadcastable
  sS      "T2c#//22 5.#SXX./14	
2.s
   AAc                R    | j                    d| j                   d| j                   S Nrn   )r5  _operator_reprr6  r_   s    rI   r.  zBinop.__str__
  s(    ))Ad112!DJJ<@@rK   c                   t        |t              r>d}t        | ||      }t        |      }| j                  D cg c]	  }||v s| }}t        | j
                  t              rD| j
                  j                  dkD  r+| j
                  j                  |k7  r| j
                  |   }d}n| j
                  }t        | j                  t              rD| j                  j                  dkD  r+| j                  j                  |k7  r| j                  |   }d}n| j                  }|sy  t        |       t        |       ||      g|j                  dd   S y c c}w )NFr  T)rr   r   rT  r(   r   r5  r   rO   r6  ro   rq   )rF   rG   rH   changedr   rV  r5  r6  s           rI   r  zBinop._simplify_up
  s   fj)G1$
KG&w/G&*llEscWnsEGE499d+IINNQ&II%%0yy)yy4::t,JJOOa'JJ&&'1

7+

4<
T
4 7N&//!":MNN5 * Fs   	EEc                2    | j                   | j                  gS rB   )r5  r6  r_   s    rI   r  zBinop._node_label_args
  s    		4::&&rK   c                   t        | j                        rt        | j                  t              r)t        j                  | j                  j                        n| j                  }t        | j                  t              r)t        j                  | j                  j                        n| j                  }t        | j                  ||            S | j                  rut        | j                               dk(  rY| j                  j                  | j                  j                  k  r| j                  j                  S | j                  j                  S t        | A         S r  )r"   rN   rr   r5  r   rv   r   r  r6  r  r  r  r  r	  rO   r   r  )rF   left_divisionsright_divisionsr   s      rI   r  zBinop._divisions
  s    $ dii. 		$))--.YY  djj$/ 		$**../ZZ  HII  S):):)<%=%Byy~~

/zz+++yy***7%''rK   )r   r  r  r   r  r  r  r.  r  r  r  r  r  s   @rI   r  r  
  s@    7#K
 
AO:'( (rK   r  c                  (    e Zd Zej                  ZdZy)r   +N)r   r  r  r  r  r  r  r	  rK   rI   r   r   
      INrK   r   c                  H    e Zd Zg dZddddZg dZed        Zed        Z	y)MethodOperator)r  r5  r6  r  levelr  r   N)r  r  r  c                    | j                   S rB   r   r_   s    rI   r  zMethodOperator._operator_repr
  s    yyrK   c                (     t        ||       |fi |S rB   r_  )r  r5  r6  r\   s       rI   r  zMethodOperator.operation
  s    "wtT"53F33rK   )
r   r  r  r   r   r  r  r  rW  r  r	  rK   rI   r  r  
  s>    JK"TFI3M  4 4rK   r  c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  subr  r  r	  rK   rI   r   r   
  r  rK   r   c                  .    e Zd Zej                  ZdZd Zy)r   *c                T   t        | j                  t              rt        | j                  t        j
                        rit        | j                  j                  t        j
                        r:| j                  | j                  j                  z  | j                  j                  z  S y y y rB   )rr   r6  r   r5  r  Numberr_   s    rI   r  zMul._simplify_down
  sm    tzz3'499gnn54::??GNN;II

/4::3C3CCC < 6 (rK   N)r   r  r  r  mulr  r  r  r	  rK   rI   r   r   
  s    INDrK   r   c                  (    e Zd Zej                  ZdZy)r   z**N)r   r  r  r  powr  r  r	  rK   rI   r   r     s    INrK   r   c                  (    e Zd Zej                  ZdZy)r   /N)r   r  r  r  truedivr  r  r	  rK   rI   r   r     s      INrK   r   c                  (    e Zd Zej                  ZdZy)r   <N)r   r  r  r  ltr  r  r	  rK   rI   r   r         INrK   r   c                      e Zd Zg dZdddZy)BinOpSeries)r5  r6  r  r  N)r  r  r   r  r  r   r   r	  rK   rI   r  r    s    :K#d3IrK   r  c                      e Zd Zg dZddiZy)
BinOpFrame)r5  r6  r  r  r  Nr  r	  rK   rI   r  r    s    +KIrK   r  c                  (    e Zd Zej                  ZdZy)LTSeriesr  Nr   r  r  r8   r  r  r  r	  rK   rI   r  r         INrK   r  c                  (    e Zd Zej                  ZdZy)LTFramer  Nr  r	  rK   rI   r  r  %  r  rK   r  c                  (    e Zd Zej                  ZdZy)LESeries<=Nr   r  r  r8   ler  r  r	  rK   rI   r  r  *      INrK   r  c                  (    e Zd Zej                  ZdZy)LEFramer  Nr  r	  rK   rI   r  r  /  r  rK   r  c                  (    e Zd Zej                  ZdZy)GTSeries>Nr   r  r  r8   gtr  r  r	  rK   rI   r  r  4  r  rK   r  c                  (    e Zd Zej                  ZdZy)GTFramer  Nr  r	  rK   rI   r  r  9  r  rK   r  c                  (    e Zd Zej                  ZdZy)GESeries>=Nr   r  r  r8   ger  r  r	  rK   rI   r  r  >  r  rK   r  c                  (    e Zd Zej                  ZdZy)GEFramer  Nr  r	  rK   rI   r  r  C  r  rK   r  c                  (    e Zd Zej                  ZdZy)NESeries!=Nr   r  r  r8   ner  r  r	  rK   rI   r  r  H  r  rK   r  c                  (    e Zd Zej                  ZdZy)NEFramer  Nr  r	  rK   rI   r  r  M  r  rK   r  c                  (    e Zd Zej                  ZdZy)EQSeries==Nr   r  r  r8   eqr  r  r	  rK   rI   r  r  R  r  rK   r  c                  (    e Zd Zej                  ZdZy)EQFramer  Nr  r	  rK   rI   r  r  W  r  rK   r  c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  r  r  r  r	  rK   rI   r   r   \      INrK   r   c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  r  r  r  r	  rK   rI   r   r   a  r  rK   r   c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  r  r  r  r	  rK   rI   r   r   f  r  rK   r   c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  r  r  r  r	  rK   rI   r   r   k  r  rK   r   c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  r  r  r  r	  rK   rI   r   r   p  r  rK   r   c                  (    e Zd Zej                  ZdZy)r   &N)r   r  r  r  and_r  r  r	  rK   rI   r   r   u  s    INrK   r   c                  (    e Zd Zej                  ZdZy)r   |N)r   r  r  r  or_r  r  r	  rK   rI   r   r   z  r  rK   r   c                  (    e Zd Zej                  ZdZy)r   ^N)r   r  r  r  xorr  r  r	  rK   rI   r   r     r  rK   r   c                  (    e Zd Zej                  ZdZy)r   %N)r   r  r  r  modr  r  r	  rK   rI   r   r     r  rK   r   c                  (    e Zd Zej                  ZdZy)r  z//N)r   r  r  r  floordivr  r  r	  rK   rI   r  r    s    !!INrK   r  c                  6     e Zd ZdgZd Zd Zd Z fdZ xZS )Unaryopr  c                8    | j                    d| j                   S r  )r  r  r_   s    rI   r.  zUnaryop.__str__  s    %%&a

|44rK   c                    t        |t              rDt        | j                  t              rt	        | ||      S | j                  } t        |       |      S y rB   )rr   r   r  r   r  ro   )rF   rG   rH   r  s       rI   r  zUnaryop._simplify_up  sH    fj)$**d+.tVZHH

4:e$$ *rK   c                    | j                   gS rB   )r  r_   s    rI   r  zUnaryop._node_label_args  s    

|rK   c                    t        | j                        rd| j                  j                  dz   z  S t        |          S r  )r"   rN   r  rU   r   r  rK  s    rI   r  zUnaryop._divisions  s7    $djj44q8997%''rK   )	r   r  r  r   r.  r  r  r  r  r  s   @rI   r  r    s#    )K5%( (rK   r  c                  (    e Zd Zej                  ZdZy)r   ~N)r   r  r  r  invr  r  r	  rK   rI   r   r     r  rK   r   c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  negr  r  r	  rK   rI   r   r     r  rK   r   c                  (    e Zd Zej                  ZdZy)r   r  N)r   r  r  r  posr  r  r	  rK   rI   r   r     r  rK   r   c                  V    e Zd ZdZddgZej                  d        Zd Zd
dZ	d Z
d Zy	)rt  zSelect one or more partitionsr  ru  c                .    | j                   j                  S rB   r  r_   s    rI   rN   zPartitions._meta  r  rK   c                    g }| j                   D ]*  }|j                  | j                  j                  |          , |j                  | j                  j                  dz             t	        |      S r  )ru  r   r  r  r  )rF   r  parts      rI   r  zPartitions._divisions  sc    	OO 	9DTZZ11$78	9--dQh78YrK   c                `    t        || j                  j                  | j                  |   f      S rB   )r   r  rV   ru  r  s      rI   r  zPartitions._task  s'    TDJJ,,dooe.DEFFrK   c                   ddl m} ddlm} t	        | j
                  t              rt	        | j
                  t        t        ||f      s|| j
                  j                  D cg c]E  }t	        |t              r1| j
                  j                  |      st        || j                        n|G }} t        | j
                        | S t	        | j
                  t              rp| j
                  j                   r1| j                  D cg c]  }| j
                  j                   |    }}n| j                  }| j
                  j#                  d|i      S y c c}w c c}w )Nr   )SetIndexBlockwise)ResampleAggregationrr  )rk  r  dask.dataframe.tseries.resampler  rr   r  r  BlockwiseIOFusedrq   r   r  rt  ru  ro   rw  rr  r4  )rF   r  r  r   rq   r  ru  s          rI   r  zPartitions._simplify_down  s   >Gdjj),ZJJe->@ST6
 **--  #2t,TZZ5N5Nr5R r4??3H  $4

#X..

$67zz%%AEQAdjj44Q7Q
Q!__
 ::33]J4OPP 8 Rs   !A
E Ec                2    | j                   | j                  gS rB   )r  ru  r_   s    rI   r  zPartitions._node_label_args  s    

DOO,,rK   Nr  )r   r  r  r  r   r  r  rN   r  r  r  r  r	  rK   rI   rt  rt    s=    'L)K    GQ2-rK   rt  c                       e Zd ZdZedd       Zed	d       Zej                   fd       Z	e fd       Z
d
dZd
dZ xZS )rw  a  Mixin class for partition filtering

    A ``PartitionsFiltered`` subclass must define a ``_partitions`` parameter. When
    ``_partitions`` is defined, the following expressions must produce the same output
    for :cls:`PartitionsFiltered`:

    - ``cls(expr: Expr, ..., _partitions)``
    - ``Partitions(cls(expr: Expr, ...), _partitions)``

    In order to leverage the default ``Expr._layer`` method, subclasses should define
    ``_filtered_task`` instead of ``_task``.
    c                (    | j                  d      duS )z,Whether output partitions have been filteredrr  NrP  r_   s    rI   	_filteredzPartitionsFiltered._filtered  s     ||M*$66rK   c                f    | j                   r| j                  d      S t        | j                        S )zSelected partition indicesrr  )r  r   rT   rU   r_   s    rI   rr  zPartitionsFiltered._partitions  s+     >><<..))**rK   c                    t         |   }| j                  s|S g }| j                  D ]  }|j	                  ||           |j	                  |dz             t        |      S r  )r   r  r  rr  r   r  )rF   full_divisionsr$  r  r   s       rI   r  zPartitionsFiltered.divisions   sm     *~~!! $$ 	7D  !56	7^D1H56]##rK   c                Z    | j                   rt        | j                        S t        |   S rB   )r  r  rr  r   rU   rK  s    rI   rU   zPartitionsFiltered.npartitions  s&    >>t''((w""rK   c                @    | j                  || j                  |         S rB   )_filtered_taskrr  r  s      rI   r  zPartitionsFiltered._task  s     ""4)9)9%)@AArK   c                    t               rB   r  r  s      rI   r  z!PartitionsFiltered._filtered_task  r  rK   )r  r  )r  zlist | tuple | ranger  )r   r  r  r  r  r  rr  r  r  r  rU   r  r  r  r  s   @rI   rw  rw    si     7 7 + + $ $ # #
B$rK   rw  c                  F    e Zd ZdgZd Zed        ZddZd Zed        Z	y)	r7  objc                ^    t        |       j                   dt        | j                         dS )Nr  r,  )ro   r   r  r!  r_   s    rI   r.  z_DelayedExpr.__str__  s(    t*%%&aDHHa88rK   c                .    | j                   j                  S rB   )r!  r   r_   s    rI   rV   z_DelayedExpr._name!  s    xx||rK   c                t   | j                   j                  | j                   j                  | j                   j                        j	                         j                         }|| j                   j                     || j                   j                  df<   |j                  | j                   j                         |S rM   )r!  __dask_optimize__daskr   to_dictr  r}  )rF   dcs     rI   r  z_DelayedExpr._layer%  sw    XX''txx||DLLNSSU "488<< 0DHHLL!
txx||	rK   c                     yr  r	  r_   s    rI   r  z_DelayedExpr._divisions+  r  rK   c                     yrM   r	  r_   s    rI   rO   z_DelayedExpr.ndim.  rd  rK   Nr  )
r   r  r  r   r.  r  rV   r  r  rO   r	  rK   rI   r7  r7    s@    'K9    rK   r7  c                  T    e Zd Zd Zej
                  d        ZddZd Ze	d        Z
y)DelayedsExprc                d    t        |       j                   dt        | j                  d          dS )Nr  r   r,  )ro   r   r  rq   r_   s    rI   r.  zDelayedsExpr.__str__4  s/    t*%%&aDMM!,<(='>a@@rK   c                     d| j                   z   S )Nzdelayed-container-)r  r_   s    rI   rV   zDelayedsExpr._name7  s    #d&>&>>>rK   c                p   ddl m} t        | j                  d   t              rct        | j                        D cg c]'  \  }}t        | j                  |f|j                        ) }}}|D ci c]  }|j                  | }}|S t        | j                  d   |      rt        | j                        j                         }|j                         }|j                         }t        |      D ]/  \  }}	t        | j                  |f|	d         || j                  |f<   1 |S t        d      c c}}w c c}w )Nr   )Delayedz#Expected a Delayed or Future object)dask.delayedr0  rr   rq   r   rp   r   rV   r   r   rZ   rX   __dask_graph__	TypeError)
rF   r0  ixfuttasksrL  r  r*  r  
actual_keys
             rI   r  zDelayedsExpr._layer;  s    (dmmA&0@I$--@X5<Rtzz2&0E  &++155!8+C+ 
 a('2&t}}5>>@D%%'D%%'C
 #,D/ OJ(-tzz2.>
1(NTZZ$%O
 
 ABB ,s   ,D-/D3c                8    dt        | j                        dz   z  S r  )r  rq   r_   s    rI   r  zDelayedsExpr._divisionsR  s    #dmm,q011rK   c                     yrM   r	  r_   s    rI   rO   zDelayedsExpr.ndimU  rd  rK   Nr  )r   r  r  r.  r  r  rV   r  r  r  rO   r	  rK   rI   r,  r,  3  s@    A ? ?.2  rK   r,  c                    d j                   dk(  xr4 j                  dk(  xr# j                  xr t        fd| D              xs j                   dk(  S )zP
    This Series is broadcastable against another dataframe in the sequence
    c                    	 | j                   t        |j                        t        |j                        fk(  S # t        t
        f$ r Y yw xY wNF)r  rC  r   r+  r3  r   )sr=  s     rI   comparez!is_broadcastable.<locals>.compare_  sB    	;;3rzz?C

O"DDD:& 		s   69 A
Ar  c              3  P   K   | ]  }|j                   d k(  s |        yw)rm   NrO   )r<  r=  r>  r=  s     rI   r>  z#is_broadcastable.<locals>.<genexpr>i  s!     =212=s   &&r   )rO   rU   r  r/  )r@  r=  r>  s    `@rI   is_broadcastablerA  Z  s`    
 	
! 	>MMQ	>	> ===	 66Q;rK   c                 r   ddl m} ddlm} t	               }| D cg c]  }|j
                  dkD  s| }}g }|r|j                         }|j                  |v r!|j                  |j                         t        |t              r|j                  |       nm|j
                  dk(  rnt        |t        t        f      rt        |t        ||f      r"|j                         }|j!                  |       n|j                  |       |r|D ch c]  }t#        |g d       }	}t%        |	      dk  S c c}w c c}w )z7Do inputs come from the same parents, modulo blockwise?r   )CumulativeAggregations)	Reduction)r   _series_dataset_info_cacher  )rJ  rC  $dask.dataframe.dask_expr._reductionsrD  r  rO   r}  rV   r  rr   IOr   r7  r(  r  r	  rs   r)   r  )
exprsrC  rD  seenr8  stack	ancestorsr	  itemunique_ancestorss
             rI   rd  rd  n  s    L>5D,1!Q,E,I
IIK77d?aQVVq[ L$/0I'=yIJ>>+LLL&Q' 0   	$ MN   A%%; -.s   D/D/
D4c                ^    t        | t              xr t        | t        t        t        f       S rB   )rr   r  
FromPandas	FromArrayFromDelayedr)  s    rI   is_valid_blockwise_oprS    s.    dI& zz9k20 , rK   c                b    d }	 | j                   } ||       \  } }|s| j                   |k(  r	 | S ,)z.Traverse the expression graph and apply fusionc                ^   t               }| g}t        t               }i }i |rB|j                         }|j                  |v r"|j	                  |j                         t        |      rKt               ||j                  <   |j                  |vr&t               ||j                  <   ||j                  <   |j                         D ]  }|j                  |       t        |      s |j                  |v r(||j                     j	                  |j                         ||j                     j	                  |j                         ||j                  <   ||j                  <    |rB|j                         D cg c]+  \  }}|t               k(  st        fd|D              r|   - }	}}|	r|	j                         }
t               }|
g}g }|r3|j                         }|j                  |v r"|j	                  |j                         |j                  |       ||j                     D ]  }|   }|D ch c]  }|j                   }}|D ch c]  }|j                   }}|j                  |
j                  k(  s|j                  |      r'||j                     |z
  |z
  s|j                  |       ||j                     s|j                  |	D cg c]  }|j                   c}vs|	j                  |        |r3t        |      dkD  rug }|D cg c]  }|j                   }}|D ]1  }||j                         D cg c]  }|j                  |vr| c}z  }3 | j                  |d   t        |g|       }||	 fS |	r| dfS c c}}w c c}w c c}w c c}w c c}w c c}w )Nc              3  <   K   | ]  }t        |            y wrB   )rS  )r<  r   expr_mappings     rI   r>  zBoptimize_blockwise_fusion.<locals>._fusion_pass.<locals>.<genexpr>  s     Qe,\%-@AAQrM  r  r   T)r  r   r}  rV   r  rS  r	  r   r  r3  rU   r  r  r  r  )r*  rJ  rK  rH   r	  r  r   r  rb  rootsrootgroupdep_namer  r=  stack_namesggroup_namesr
group_depsr   local_names_retrW  s                          @rI   _fusion_passz/optimize_blockwise_fusion.<locals>._fusion_pass  sh   u %
99;DzzT!HHTZZ $T*+.5TZZ(::Z/-0UJtzz*/3L,,,. 4W%(1zz\1$TZZ044W]]Cw}}-11$**=29L//3L,4 6 #((*
1CEzQqQQ O
 
 99;D5DFEEyy{::%$T" ,TZZ 8 *H&x0C49":q177":K":49":q177":K":4+;+;;t?R?RSV?W)#))4{B[P S)%cii0SYY).G$%G 6
 S)+* @ 5zA~
8=>uu{{>>" E','9'9';##"==;  # J uQxu1Jz1JKY&_ d Tzq
, #;":G ?#s$   :0N8NNN N%N*)rV   )r*  rc  original_namedones       rI   r  r    s@    Yv 

!$'
d4::.K rK   c                      e Zd ZddgZddiZej                  ZdZdZ	dZ
dZd Zej                  d        Zd Zej                  d	        Zed
        Zed        Zy)Diffr  periodsr  TFc                .    | j                   j                  S rB   r  r_   s    rI   r  zDiff._divisions  r  rK   c                    t         t        | j                  j                        j                  di | j
                        S Nr	  )r*   r+   r  rN   r  r\   r_   s    rI   rN   z
Diff._meta  s1    =tzz'7'78==LLMMrK   c                >    t        |t              rt        | ||      S y rB   r/  rE   s      rI   r  zDiff._simplify_up  r0  rK   c                .    t        | j                        S )Nrh  )r  rh  r_   s    rI   r\   zDiff.kwargs  s    DLL))rK   c                <    | j                   dkD  r| j                   S dS rM   rn  r_   s    rI   rv  zDiff.before      #||a/t||6Q6rK   c                >    | j                   dkD  rdS | j                    S rM   rn  r_   s    rI   rw  z
Diff.after"      LL1$q74<<-7rK   N)r   r  r  r   r   r8   r  r!  r"  r#  r$  r%  r  r  r  rN   r  r\   r  rv  rw  r	  rK   rI   rg  rg    s    I&KAI66DO$ N NE * * 7 7 8 8rK   rg  c                  h    e Zd Zg dZ eej                        ZdZe	j                  d        ZddZy)FillnaCheck)r  method
skip_checkTc                .    | j                   j                  S rB   r  r_   s    rI   rN   zFillnaCheck._meta,  r  rK   c                    | j                   D cg c]  }| j                  ||       }}|| j                  | j                        k7  |d<   t	        || j
                  g| S c c}w )Nr  )r  r  rv  r  r   r  r  s        rI   r  zFillnaCheck._task0  s]    9=D2##B.DDDOODJJ77RD$..0400 Es   A"Nr  )r   r  r  r   rW  r   fillna_checkr  r  r  r  rN   r  r	  rK   rI   rt  rt  '  s:    3KW112I"   1rK   rt  c                      e Zd ZddgZddiZej                  ZdZdZ	dZ
dZd Zej                  d        Zd Zej                  d	        Zed
        Zed        Zy)FFillr  limitNTFc                .    | j                   j                  S rB   r  r_   s    rI   r  zFFill._divisions?  r  rK   c                .    | j                   j                  S rB   r  r_   s    rI   rN   zFFill._metaB  r  rK   c                >    t        |t              rt        | ||      S y rB   r/  rE   s      rI   r  zFFill._simplify_upF  r0  rK   c                .    t        | j                        S )Nr|  )r  r|  r_   s    rI   r\   zFFill.kwargsJ  s    $**%%rK   c                6    | j                   dS | j                   S r  r  r_   s    rI   rv  zFFill.beforeN  s    JJ&q6DJJ6rK   c                     yrM   r	  r_   s    rI   rw  zFFill.afterR  rd  rK   )r   r  r  r   r   r8   ffillr!  r"  r#  r$  r%  r  r  r  rN   r  r\   r  rv  rw  r	  rK   rI   r{  r{  6  s    G$K$I77DO$    E & & 7 7  rK   r{  c                  T     e Zd Zej                  Ze fd       Ze fd       Z xZ	S )BFillc                    t         |   S rB   )r   rw  rK  s    rI   rv  zBFill.beforeZ  s     w}rK   c                    t         |   S rB   )r   rv  rK  s    rI   rw  zBFill.after`  s     w~rK   )
r   r  r  r8   bfillr!  r  rv  rw  r  r  s   @rI   r  r  W  s2    77D 
  rK   r  c                      e Zd Zg dZdddZej                  ZdZdZ	dZ
ej                  d        Zd Zej                  d	        Zej                  d
        Zd Zed        Zed        Zy)Shiftr  rh  rk  r  Nrh  rk  TFc                H    | j                         d   | j                  dS dS )Nr   TF)r  rk  r_   s    rI   r$  zShift.clear_divisionsp  s*     (+3tyy7LtWRWWrK   c                    | j                   | j                  j                  S t        | j                  | j                  | j                         }|d| j                  j
                  dz   z  }|S r  )rk  r  r  r'   rh  rU   )rF   r  s     rI   r  zShift._divisionsv  s[    99::'''-djj$,,		R	4::#9#9A#=>IrK   c                    t         t        | j                  j                        j                  di | j
                        S rk  )r*   r+   r  rN   shiftr\   r_   s    rI   rN   zShift._meta~  s1    >tzz'7'78>>MMNNrK   c                D    t        | j                  | j                        S )Nr  )r  rh  rk  r_   s    rI   r\   zShift.kwargs  s    DLLtyy99rK   c                >    t        |t              rt        | ||      S y rB   r/  rE   s      rI   r  zShift._simplify_up  r0  rK   c                <    | j                   dkD  r| j                   S dS rM   rn  r_   s    rI   rv  zShift.before  rp  rK   c                >    | j                   dkD  rdS | j                    S rM   rn  r_   s    rI   rw  zShift.after  rr  rK   )r   r  r  r   r   r8   r  r!  r"  r#  r%  r  r  r$  r  rN   r\   r  r  rv  rw  r	  rK   rI   r  r  g  s    .Kt,I77DX X
 O O : :E 7 7 8 8rK   r  c                  h    e Zd Zg dZdddZdgZej                  Zd Z	e
j                  dd       Zy)	
ShiftIndexr  r  Nr  rk  c                    | j                   }|| j                  j                   }t        | j                  | j                  |      }|d| j                  j
                  dz   z  }|S r  )rk  rN   r'   r  rh  rU   )rF   rk  r  s      rI   r  zShiftIndex._divisions  sX    yy<::??D-djj$,,M	4::#9#9A#=>IrK   c                :    | j                   d| j                   iS i S )Nrk  )rk  r_   s    rI   r  zShiftIndex._kwargs  s    &*ii&;		"CCrK   r  )r   r  r  r   r   r  r8   r  r  r  r  r  r  r	  rK   rI   r  r    sB    .Kt,IHMI D DrK   r  c                  z    e Zd ZU dZdZded<   d Zd Zej                  d        Z
d Zej                  d	        Zy)
MaybeAlignPartitionsFNzAnyType | None	_expr_clsc                :   | j                   D ch c]  }|j                   c}dhk(  rNg }| j                   D ]&  }|j                  t        |j                               ( 	 t        |      t        |      fS t        | j                    S c c}w # t        $ r Y yw xY w)Nr  r  )	r  rU   rs   r  r  rC  r+  r3  r  )rF   r=  r  s      rI   r  zMaybeAlignPartitions._divisions  s    %)YY/rBNN/A36Dii 0D./0"4y#d)++ (33 0  "!"s   B	!B 	BBc                X    t        |t              r| j                  rt        | ||      S y y rB   )rr   r   r  r  rE   s      rI   r  z!MaybeAlignPartitions._simplify_up  s,    fj)d.J.J*4DD /K)rK   c                    | j                   D cg c]  }t        |t              s| }}|D cg c]  }t        ||      r| c}S c c}w c c}w rB   )rq   rr   r   rA  )rF   r   r@  s      rI   r  zMaybeAlignPartitions.args  sF     MMBbZD-ArBB Br(8b(ABB CBs   AAAAc           	        | j                   t              dk(  sEt        fdD              s1t        | j                        dk(  r2t	        t        d             dk(  r | j                  | j                   S | j                  d   t	        d D              }D ch c]"  }|j                  j                  j                  $ }}t        |      st        d| d      dd	lm} | j                  D cg c]   }t        |t               r ||d |d
      n|" }} | j                  | S t#        | j                  d| j                  i} | j                  | S c c}w c c}w )Nr  c              3  r   K   | ].  }d    j                   |j                   k(  xr |j                   0 ywr   N)r  r  r<  r=  r@  s     rI   r>  z.MaybeAlignPartitions._lower.<locals>.<genexpr>  s7      LNA  BLL0GR5G5GGs   47rm   c                ,    t        | j                        S rB   r  xs    rI   r+  z-MaybeAlignPartitions._lower.<locals>.<lambda>      #akk"2 rK   r   c              3  4   K   | ]  }|j                     y wrB   r1  r;  s     rI   r>  z.MaybeAlignPartitions._lower.<locals>.<genexpr>       ;bnn;r?  zDataFrames are not aligned. We need to shuffle to align partitions with each other. This is not possible because the indexes of the DataFrames have differing dtypes=zX. Please ensure that all Indexes have the same dtype or align manually for this to work.)RearrangeByColumnT)index_shuffler  )r  r  r3  r  r+  r  r  rq   rN   rc   r^  _are_dtypes_shuffle_compatibler3  !dask.dataframe.dask_expr._shuffler  rr   r   r  )rF   rU   r=  r[  r  r  r@  s         @rI   r  zMaybeAlignPartitions._lower  sd    iiHM RU  4>>"a'C2C89Q>!4>>4==11^^A&;s;;K589rbhhnn**9F91&988>x @  L --  ""d+ &b$4PD  "4>>4((%t}}OOt~~t$$/ :s   'E6%Ec                H     | j                   | j                   j                  S rB   )r  rq   rN   r_   s    rI   rN   zMaybeAlignPartitions._meta  s    t~~t}}-333rK   )r   r  r  r  r  r  r  r  r  r  r  r  rN   r	  rK   rI   r  r    sY    # $I~$
4E C C'%R 4 4rK   r  c                D    t        |       dk(  ryt        d | D              S )Nr  Tc              3  n   K   | ]-  }t         j                  j                  j                  |       / y wrB   )rv   r[  r\  is_numeric_dtyper[  s     rI   r>  z1_are_dtypes_shuffle_compatible.<locals>.<genexpr>  s#     @Arvv||,,Q/@s   35)r  r3  )r[  s    rI   r  r    s"    
6{a@@@@rK   c                      e Zd ZddgZeZd Zy)rf  r  r   c                   t        |t              rt        | ||      }| j                  j                  D cg c]	  }||v s| }}| j
                  j                  D cg c]	  }||v s| }}| j                  j                  |k(  r| j
                  j                  |k(  ry  t        |       t        |       | j                  |   | j
                  |         g|j                  dd   S y c c}w c c}w r  r  r  s          rI   r  zCombineFirstAlign._simplify_up  s    fj)1$
KG,0JJ,>,>QS#.SQMQ,0JJ,>,>QS#.SQMQ

""m3JJ&&-74<T
4::m4djj6OP$  *QQr  N)r   r  r  r   re  r  r  r	  rK   rI   rf  rf    s    G$KIrK   rf  c                      e Zd ZdZddgZeZy)r  Tr  r  N)r   r  r  r  r   r  r  r	  rK   rI   r  r  
  s    "G$KIrK   r  c                      e Zd Zg dZeZy)r  r  N)r   r  r  r   r  r  r	  rK   rI   r  r    s    BKIrK   r  c                      e Zd Zg dZeZy)CombineSeriesAlignr>  Nr   r  r  r   r=  r  r	  rK   rI   r  r    s    :KIrK   r  c                      e Zd Zg dZeZy)CombineFrameAlign)r  r   r!  r  rC  Nr  r	  rK   rI   r  r    s    GKIrK   r  c                  &    e Zd ZdZdZddgZeZd Zy)r  Tr  r  c                0    t         j                  | ||      S rB   )r   r  rE   s      rI   r  zFilterAlign._simplify_up%  s    ""4<<rK   N)	r   r  r  r  rC   r   r   r  r  r	  rK   rI   r  r    s"    "K(KI=rK   r  c                      e Zd Zg dZeZd Zy)AssignAlign)r  r  r  c                   t        |t              r(t        | ||      }t        |t              s|g}t	        |      | j
                  hz
  }|t	        | j                  j                        k(  ry | j
                  ht	        |      z
  }t        |      dk(  r) t        |      | j                  g|j                  dd   S | j                  dd  }| j                  j                  D cg c]	  }||v s| }} t        |       t        |       | j                  t        |         g| g|j                  dd   S y c c}w r  )rr   r   rT  r  r  r  r  r   r  ro   rq   r  )rF   rG   rH   r   ra  r  r  rV  s           rI   r  zAssignAlign._simplify_up-  s!   fj)1$
KGgt,")w<4;;-/Ds4::--..KK=3w</D4yA~#tF|DJJE1DEE==,&*jj&8&8HsC4KsHGH4<T
4::fWo6BB$ # *  Is   %	D;/D;N)r   r  r  r   r  r  r  r	  rK   rI   r  r  )  s    .KIrK   r  c                  $    e Zd ZU g dZeZded<   y)r  rg  AnyTyper  N)r   r  r  r   r  r  r  r	  rK   rI   r  r  F  s    ,KIwrK   r  c                      e Zd ZeZy)r|  N)r   r  r  r}  r  r	  rK   rI   r|  r|  K  s    IrK   r|  c                      e Zd Zg dZdZeZy)MapAlign)r  r   r   r  rR   FN)r   r  r  r   r  r  r  r	  rK   rI   r  r  O  s    ?K#IrK   r  c                  ,    e Zd Zej                  dgz   Zy)MapIndexAlignr  N)r   r  r  r  r   r	  rK   rI   r  r  U  s    ''>*::KrK   r  c                  R    e Zd Zg dZdZej                  d        Zd Ze	d        Z
y)OpAlignPartitions)r  r   r   Tc                     t        | j                  j                  | j                        | j                  j                        S rB   )r_  r  rN   r   r   r_   s    rI   rN   zOpAlignPartitions._meta]  s.    1wtzz''1$**2B2BCCrK   c                "   | j                   t              dk(  sEt        fdD              s1t        | j                        dk(  rYt	        t        d             dk(  r@ | j                  | j                  | j                  | j                  g| j                  dd   S ddlm}  || j                  | j                  d	      } || j                  | j                  d	      } | j                  || j                  |g| j                  dd   S )
Nr  c              3  V   K   | ]   }d    j                   |j                   k(   " ywr  r  r  s     rI   r>  z+OpAlignPartitions._lower.<locals>.<genexpr>g  s$     B3q6##r||3B   &)rm   c                ,    t        | j                        S rB   r  r  s    rI   r+  z*OpAlignPartitions._lower.<locals>.<lambda>i  r  rK   rR  r   )RepartitionDivisionsTr$  force)r  r  r3  r  r+  r  _opr  r   r   rq   r  r  )rF   r  r  r   r@  s       @rI   r  zOpAlignPartitions._lowera  s     iiHMBcBB4>>"a'C2C89Q>488DJJPdmmAB>OPPN$JJdnnD
 %JJdnnD
 txxtwwBab0ABBrK   c                &     t        | |      |      S rB   r  r  r   r   r  r\   s        rI   r  zOpAlignPartitions._opw  s    !wub!%((rK   N)r   r  r  r   r  r  r  rN   r  rW  r  r	  rK   rI   r  r  Y  sA    *K"D DC, ) )rK   r  c                  $    e Zd Zg dZed        Zy)MethodOperatorAlign)r  r   r   r  r  r  c                $    t        || |g|i |S rB   )r  r  s        rI   r  zMethodOperatorAlign._op  s    b%@@@@rK   N)r   r  r  r   rW  r  r	  rK   rI   r  r  |  s    IKA ArK   r  c                      e Zd Zg dZdZd Zej                  d        Zej                  d        Z	ej                  d        Z
d Zy)	
UFuncAlign)r  r!  rR   r\   Fc                4    dt        | j                         dS rg  r-  r_   s    rI   r.  zUFuncAlign.__str__  rh  rK   c                F    | j                   t        | j                        d  S rB   rj  r_   s    rI   r  zUFuncAlign.args  rk  rK   c                b    | j                   D cg c]  }t        |t              s| c}S c c}w rB   )r  rr   r   rm  s     rI   rn  zUFuncAlign._dfs  s#    !YY?r*R*>???s   ,,c                    | j                  d      t        ur| j                  d      S t        | j                  | j                  | j
                        S r5  )r   r7   rc  rn  r  r!  r_   s    rI   rN   zUFuncAlign._meta  s=    <<z1<<''tyy$))TYY??rK   c                   t        | j                  d| j                         i}|D cg c]%  }t        |t              s|j
                  dkD  s$|' }}t        |d   | j                  | j                  d| j                  g| S c c}w )Nr  r   F)
r  r  r  rr   r   rO   re  r!  rN   r\   )rF   r  r  r@  s       rI   r  zUFuncAlign._lower  sq    %tyyNDOO<MNEQ*Q"5!&&1*qEESVTYY

E4;;VQUVV Fs   BBBN)r   r  r  r   r"  r.  r  r  r  rn  rN   r  r	  rK   rI   r  r    sj    5K/ 6 6 @ @ @ @
WrK   r  c                      e Zd ZdZdgZej                  d        ZddZd Z	ej                  d        Z
d ZddZdd	Zed
        Zy)r  a  Fused ``Blockwise`` expression

    A ``Fused`` corresponds to the fusion of multiple
    ``Blockwise`` expressions into a single ``Expr`` object.
    Before graph-materialization time, the behavior of this
    object should be identical to that of the first element
    of ``Fused.exprs`` (i.e. the top-most expression in
    the fused group).

    Parameters
    ----------
    exprs : List[Expr]
        Group of original ``Expr`` objects being fused together.
    *dependencies:
        List of external ``Expr`` dependencies. External-``Expr``
        dependencies correspond to any ``Expr`` operand that is
        not already included in ``exprs``. Note that these
        dependencies should be defined in the order of the ``Expr``
        objects that require them (in ``exprs``). These
        dependencies do not include literal operands, because those
        arguments should already be captured in the fused subgraphs.
    rI  c                4    | j                   d   j                  S rM   )rI  rN   r_   s    rI   rN   zFused._meta  s    zz!}"""rK   c                   d| j                   dd   d}|s|gS t               }g }| j                  d   dfg}| j                  D cg c]  }|j                    }}| j                         D 	ci c]  }	|	j                   |	 }
}	|r|j	                         \  }}|j                   |v r$|j                  |j                          |j                  |d      d   }|j                  |j                  dd	d
             |j                         D ]t  }	|	j                   |v r|j                  |	|dz   f       (|	j                   |
v s7|
j	                  |	j                          |j                  |	j                  |dz                v |r|
j                         D ]"  }	|j                  |	j                  d             $ |g|z   }|D cg c]
  }d|z  |z    }}|S c c}w c c}	w c c}w )NzFused(z):r   rm   F)r|   rn   r  r  )rV   r  rI  r	  r}  r  rt   r   r  rs   r)  )rF   r{   r|   r}   rJ  r~   rK  r   fused_groupr  r	  r*  _indentr   s                 rI   rt   zFused._tree_repr_lines  s   $**RS/*"-8Ou**Q-#$04

;uu{{;;262C2C2EF3		3FF!IIKMD'zzT!HHTZZ ((E(B1EDLLc323((* D99+LL#w{!34YY,. $$SYY/LL!5!5gk!BCD    &&( 	2CLL--a01	2 5 167v$771 <F* 8s   G,GGc                   t        | j                  t        j                        }|D cg c]   }|j                  j                  d      d   " }}t        |      dkD  r|d   dz   |d   z   S dj                  |      S c c}w )Nr,  r  r      z-fused-r  )r  rI  r8   _depthrV   splitr  r  )rF   rI  r*  namess       rI   r.  zFused.__str__  ss    tzzqxx06;<d!!#&q)<<u:>8i'%)3388E?"	 =s   %A>c                6    t        |        d| j                   S r  )r  r  r_   s    rI   rV   zFused._name  s    d)Ad66788rK   c                <    | j                   d   j                         S rM   )rI  r  r_   s    rI   r  zFused._divisions  s    zz!}''))rK   c                     |j                   dk(  S r  r1  r  s     rI   r  zFused._broadcast_dep  r2  rK   c                   g }| j                   D ]g  }| j                  |      r|j                  df}n|j                  |f}|j                  ||d         }|j                  |k(  sJ |j                  |       i t        j                  |d|iS )Nr   r  r   )rI  r  rV   r  r   r   r   r  )rF   r  rc   internal_tasksr   subnamerL  s          rI   r  zFused._task  s    ZZ 	%E""5) ;;* ;;.GWQZ0A55G###!!!$	% yy.3d33rK   c                @    t        |      }t        | ||g      }||   S )N)cacher  )r  r   )r  r	  outkeyr  ress        rI   _execute_internal_graphzFused._execute_internal_graph  s&    \"N%vhG6{rK   Nr  r  r  )r   r  r  r  r   r  r  rN   rt   r.  rV   r  r  r  rW  r  r	  rK   rI   r  r    sp    . )K# # D# 9 9*$4  rK   r  c                      e Zd Zd Zy)MinTypec                     y)NTr	  r   s     rI   r   zMinType.__le__	  s    rK   N)r   r  r  r   r	  rK   rI   r  r    s    rK   r  c                   t        |t              rg }n|j                  j                         }|| j                     D cg c]  }t         |       x}t
              s| }}t               }|D ]G  }	|	j                  |v r|j                  |	j                         |j                  |	j                         I | |j                  t        |t                     t        |      }
	 t        |
      }t#        |      dk(  r&|j$                  dk(  rt'        d |D              r|d   S |S c c}w # t        $ r8 t        t        j                  t        |
                  j!                         }Y {w xY w)N)	containerr  c              3  :   K   | ]  }|j                   d k(    yw)r  Nr@  )r<  r  s     rI   r>  z.determine_column_projection.<locals>.<genexpr>/  s     -!-s   r   )rr   rb   r   r  rV   r   r  r  rs   r  r   r  r  r3  _sort_mixedrv   rr  r  rO   r3  )r*  rG   rH   rQ  column_unionr  instr5  rJ  r  flattened_columnss              rI   rT  rT    sI    &% ~~**,'

3Wz13;4QU7VtWGW5D 377d?A112	3 %G$6$GH L)O/0
 	LQKK1-W--A5 X"  O"288D1B,C#DELLNOs   D'D'&D, ,>E-,E-c           	     `   t        j                  | D cg c]  }t        |t               c}t              }t        j                  | D cg c]  }t        |t
               c}t              }t        j                  | D cg c]  }t        j                  |       c}t              }| | z  | z  }t        j                  | |         }t        j                  | |         }t        j                  | |         }|j                         d   j                  |      }	|j                         d   j                  |      }
|j                         d   j                  |      }|j                         d   }t        j                  ||	|
|g      }| j                  |      S c c}w c c}w c c}w )z3order ints before strings before nulls in 1d arraysr]  r   )r  arrayrr   r  r  r  rv   rr  argsortnonzerotakeconcatenate)r)  r  str_pos	tuple_posnull_posnum_posstr_argsorttuple_argsortnum_argsortstr_locs
tuple_locsnum_locs	null_locslocss                 rI   r  r  5  sO   hhF;q
1c*;4HG?1*Q.?tLIxxV44DAHh("iZ/G**VG_-KJJvi01M**VG_-K #((5H""$Q',,];J #((5H  "1%I>>8Xz9EFD;;t <?4s   F!F&F+c                   t        | |||      }t        |t              r)| j                  j                  D cg c]	  }||v s| }}n|| j                  j                  vrg }|| j                  j                  k(  s|s| j
                  dk  ry  t        |       | j                  |   g| j                  dd   }||j                  d      k(  r|S  t        |      ||j                  d            S c c}w )NrP  rm   r  r   )	rT  rr   r  r  r   rO   ro   rq   r   )r*  rG   rH   rQ  r  rV  r  s          rI   r  r  G  s    .fj5GL ,%'+zz'9'9QSL=PQQ	TZZ//	/tzz)))$))a- 	T$Z

<0E4==3DEFv~~i004<y 9:: Rs   	C1C1Tc                t   || j                      D cg c]  } |        |        }}|D ch c]  }t        |t              s|j                   ! }}t        |      dk7  ryt        |      dk(  ry|D ch c]  }t        |t              r|j                   ! }}t	        | ||||      S c c}w c c}w c c}w )Nr  FT)rV   rr   r   r   _check_dependents_are_predicates)	r*  rG   rH   r  r  r5  r8  filtersotherss	            rI   rD   rD   [  s    &tzz2FqacoqsFGF 'A1:a+@qwwAGA
7|q
7|q  'D!jF.CaggDFD+ffj/  GA Es!   B+	B+B0B06B5B5c                j    t        | g       }t        |      dk(  r| S t        |d   |dd       }|| S |S )zRewriting a filter to decompose OR clauses. If a predicate part is part of
    all OR clauses, we can move it to the front so that we can push it down.
    r  r   N)_get_predicate_componentsr  _replace_common_or_components)r  or_componentsr  s      rI   r  r  k  sK     .i<M
=Q*=+;]12=NOF~MrK   c                j   t        | |      s|j                  |        |S t        | j                  |      rt        | j                  ||      }n|j                  | j                         t        | j                  |      rt        | j                  ||      }|S |j                  | j                         |S rB   )rr   r   r5  r  r6  )r  
componentstype_s      rI   r  r  x  s    i')$)..%(.y~~z5Q
)..))//5).y
ER
  	)//*rK   c                f    t        t        | D cg c]  }|j                   c}|             S c c}w rB   )r  r   rV   )r  r8  s     rI   _convert_mappingr    s&    j1QWW1:>??1s   .c                |   t        | g t              }t        |      }|D cg c]  }t        |g t               }}t        t	        t        |            }g }|j                         D ](  t        fd|D              s|j                         * t        |      dk(  ry ||d      }|dd  D ]
  }|||   z  } g }	|g|z   D ]S  }
|
D cg c]	  }||vs| }}t        |      dk(  r|c S |
|d      }|dd  D ]
  ||
   z  } |	j                  |       U |	d   }|	dd  D ]  |z  }	 ||z  S c c}w c c}w )N)r  c              3  &   K   | ]  }|v  
 y wrB   r	  )r<  compr  s     rI   r>  z0_replace_common_or_components.<locals>.<genexpr>  s     4TqDy4r  r   r  )	r  r   r  r  r  r  r3  r   r  )r*  r  and_componentmappingr  and_componentsreplacementsouter_componentr_  result_componentsr!  keep_componentsresult_componentor_components       `         rI   r  r    s   -dBcBM}-G=J89!!Rs3N  #.?@NL\\^ #4^44"# <Al1o.O!" 7)GAJ67
 	N* 3&*Da|.C1DD1$"" 23 $ 	:A/$q'9	:  !123 %Q'Lqr" (#a'(\))E, Es   D4	D9D9c                   |j                   h}|j                         }t               }t               }|r|j                         }	| j                   |	j                   k(  r-|	j                   |v r<|j	                  |	j                          t        |	t              rh|j                  ||	j                      D 
ch c]  }
 |
        |
       j                    c}
       |st        |	t        t        t        f      ry|j	                  |	j                          |j                  |	j                                |r|j                  |      xr |j                  |      S c c}
w r<  )rV   r	  r  r}  r  rr   r7  rI  ApplyConcatApply
TreeReduceShuffleReducers   r  )r*  other_namesrG   rH   r  allowed_expressionsrK  rJ  all_dependentsr8  r  s              rI   r  r    s    "<<.!E5DUN
IIK:: 77d?a& *177 3G1qsQSYYG	
 !.
MJK(Q^^%&+ . ""#67 K<P<P=  Hs   6EE)r  c           
        |D cg c]%  }t        |t              s|j                  dkD  s$|' c}t        d D              sdt	        d D              dz   z  S t        fdD              rd   j
                  S t        t        t        D cg c]  }|j
                   c}             }t        |      dk(  r
|d   |d   f}|S c c}w c c}w )Nr   c              3  4   K   | ]  }|j                     y wrB   )r  r;  s     rI   r>  z+calc_divisions_for_align.<locals>.<genexpr>  s     0br!!0r?  rB   c              3  4   K   | ]  }|j                     y wrB   r1  r;  s     rI   r>  z+calc_divisions_for_align.<locals>.<genexpr>  r  r?  r  c              3  V   K   | ]   }d    j                   |j                   k(   " ywr  r  r  s     rI   r>  z+calc_divisions_for_align.<locals>.<genexpr>  s$     
:3q6r||+
:r  )
rr   r   rO   r3  r+  r  r  r   r   r  )r  rI  r=  r  r@  s       @rI   r  r    s    
F"B!5"''A+2
FC0C00#;s;;a?@@

:c
::1vVL#*FB2<<*FGHII
9~q\9Q<0	 G
 +Gs   CCCCc                    ddl m} |D cg c].  }t        |t              r|j                  dkD  r ||| d      n|0 c}S c c}w )Nr   r  Tr  )r  r  rr   r   rO   )r  rI  r  r=  s       rI   r  r    sP    A   "d#! )4@  s   3Ac                   t        | t              r#rt        | j                        S | j                  S t        | t              r| D cg c]  }t        |       c}S t        | t              rt        fd| D              S t        | t              ri }| D ]  }t        | |         ||<    |S t        | d      rt        | j                        S | S c c}w )zO
    Extract internal cache data (``_meta``) from dd.DataFrame / dd.Series
    c              3  6   K   | ]  }t        |        y wrB   )_extract_meta)r<  _xnonemptys     rI   r>  z _extract_meta.<locals>.<genexpr>  s     =R]2x0=s   r*  )
rr   r   r+   rN   r  r9  r  r  rZ  r*  )r  r;  r:  r  r  s    `   rI   r9  r9    s     !T)1}QWW%>qww>	At	678b(+88	Au	=1===	At	 	3A"1Q42CF	3
	F	QVVX.. 9s   	Cudfc          	         t        t        |       |      5   | t        |d      i t        |d      cddd       S # 1 sw Y   yxY w)z
    Apply a function using args / kwargs. If arguments contain dd.DataFrame /
    dd.Series, using internal cache (``_meta``) for calculation
    r<  TN)r4   r9   r9  )r!  r=  r  r\   s       rI   emulater?  
  sF    
 
Xd^	5 O]4.N-2MNO O Os	   =Ac                   |rt        t        |d         dd      nd}||r|d   j                  }|t        u rH| D cg c])  }t	        |t
              rt        |j                        n|+ }}t        |g|ddi|}d}	nt        |||      }d}	t        |      s\t        |      r|j                  sEt        d | D              s3|	st        j                  d	t               t        t        |g      |
      }t        ||      }|S c c}w )zP
    Helper to generate metadata for map_partitions and map_overlap output.
    r   rc   Nr=  T)rc   r&  Fc              3  ^   K   | ]%  }t        |t              xr |j                  d k(   ' ywr  )rr   r   rO   )r<  r
  s     rI   r>  z+_get_meta_map_partitions.<locals>.<genexpr>$  s.      V47
3/#((a-/Vs   +-zMeta is not valid, `map_partitions` and `map_overlap` expects output to be a pandas object. Try passing a pandas object as meta or a dict or tuple representing the (name, dtype) of the columns. In the future the meta you passed will not work.r  r  )r_  r*   rN   r7   rr   r   r+   r?  r    r<   r  r3  r  warnFutureWarningr   )
r  r@  r!  r\   rR   r&  
meta_indexr
  r  meta_is_emulateds
             rI   r6  r6    s    ?B3q6*GT:tJs!fllz TXXCC)>]399%CGXXt4a4T4V4Z[I d#|D'9djjRU V;?V S  MMa 	 $
; T{3DK1 Ys   .D)r2  r
   r,  rE  r;  r6  r*  r'  rB  r?  ri   r  r  r.  re   r  r-  r  )rH  r  rQ  rP  )rR  r<  rB   )
r*  r   rG   r   rH   z,dict[str, Collection[weakref.ref[BaseExpr]]]rQ  zlist | Noner  objectr  )rG   r   )F(&  
__future__r   r  r  r  r  r  weakrefcollectionsr   collections.abcr   r   r   r   typingr
   r  numpyr  pandasrv   pandas.errorsr   tlzr   r   r   r&  r   rw   
dask._exprr   BaseExprr   dask._task_specr   r   r   r   r   
dask.arrayr   	dask.baser   	dask.corer   dask.dataframer   dask.dataframe._pyarrowr   dask.dataframe.corer   r   r   r   r    r!   r"   r#   r$   r%   dask.dataframe.dask_expr._utilr&   r'   r(   r)   dask.dataframe.dispatchr*   r+   dask.dataframe.rollingr,   r-   r.   dask.dataframe.shuffler/   r0   dask.dataframe.utilsr1   r2   r3   r4   r5   dask.typingr6   r7   
dask.utilsr8   r9   r:   r;   r<   r=   r>   r?   rZ   SingletonExprr  r  r  r   rc  re  rt  r  r  r  r  r  r  r  r  r  r  re  r  r  r  r  r  r  r  r  r  r  r  r  r(  ra  r2  r4  rm  r=  rB  rE  rN  rU  rZ  rq  r  rw  r}  rt  rv  rW  r  r|  r  r  r  r  r  r  r  r  r  r  r  r$  r  r  r  r  r  r  r  r   r   rb   r  r  r  r  r$  r/  r8  rB  rH  rU  r[  r_  rq  rs  r  r  r  r  r   r  r   r   r   r   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r   r   r   r   r   r   r  r  r   r   r   rt  rw  r7  r,  rA  rd  rS  r  rg  rt  r{  r  r  r  r  r  rf  r  r  r  r  r  r  r  r|  r  r  r  r  r  r  total_orderingr  rT  r  r  rD   r  r  r  r  r  r  r  r9  r?  r6  rG  r2  r,  rE  r;  r6  r*  r'  rB  r?  ri   r  r  r.  re   r  r-  r  dask.dataframe.dask_expr.iorH  r  rQ  rP  $dask.dataframe.dask_expr.io._delayedrR  r	  rK   rI   <module>rd     s		   "       # 9 9  !   , / /  ' & I I  )  " 5    = Q Q <  (	 	 	 =={'4 {'|* **d *"f fRPI Pf@84&M 4&n<
d <
~V
 V
rc$ cLC(V	'Y 	'
$9 
$/Y //9 /) 09 <1Y 1$$I $6i 6/i /8y 8,-( -`$K $" "/ /!:8 !:HX h $8 $"	E8 	E/H /h 
( 

H 
8= 8
 & (.( ..>X .>b8 8 H H <1i 1(( / /#h #h &7 /L /	#;H ;(1	 1

H 
,($( ($V	. 	.X 8 
I 
T= T/8 / dX dN$8 $-x -<14Y 14hH' H'V"H "J6X 6)8 ):$ :zd 6A AHWh WWo W B6	 6($( $R4 RjJD) J>
 
)4 )XND) N 
 
B(H B(J% 
4U 4% 

D% 
D% 
% 
 
4% 4
 
{ 
j 
{ 
j 
{ 
j 
{ 
j 
{ 
j 
{ 
j 
 
 
 
 
 
% 
 
% 
% 
u 
(h (0W 
' 
' 
-- --`3$ 3$l4 0$4 $N(%&VdN8: 8B1) 1J BE  )8J )8XD D(D44 D4NA, ,& / 
- 
, 
=& =& :$ 
 # ;H ; ),  )FA+ AW% W6dI dP    '+	%
%% =% $	%
 %P$;( 
 <> @%*R BF&#&R 48 	
* # O"J         ( O N N < <rK   