
    bi                       U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZ d dlZd dlZd dlmZmZ d dlmZ d dlmZ d d	lmZmZmZ erd d
lmZmZ d dlmZ ed   Zde d<   d Z! G d d      Z" G d de"      Z#d*dZ$d+d,dZ%d-dZ& G d de"      Z' G d de"      Z( G d de(      Z) G d de"      Z* G d d e"      Z+ G d! d"      Z, G d# d$e"      Z-d% Z. G d& d'e(      Z/ G d( d)e"      Z0y).    )annotationsN)defaultdict)	Generator)TYPE_CHECKINGLiteral)Taskconvert_legacy_graph)_tokenize_deterministic)Key)ensure_dictfuncnameimport_required)Any	TypeAliasHighLevelGraph)logicalsimplified-logicaltuned-logicalphysicalsimplified-physicalfusedr   OptimizerStagec                ~    ddl m} t        | t              r| S t	        | d      rt        | |      s| j
                  S | S )Nr   Delayedexpr)dask.delayedr   
isinstanceExprhasattrr   )or   s     E/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/_expr.py_unpack_collectionsr$   $   s5    $!Tq&*Q"8vv    c                  :   e Zd ZU g Zded<   i Zded<   dZded<   ded	<   d
ed<   dddZd Zd Z	d Z
d Zd Zd Zd ZdAdZd ZdBdCdZ	 dD	 	 	 	 	 dEdZd Zd Zd Zd Zed        Zd ZdFd ZdGd!Zd" Zd# ZdHd$ZdId%ZdJd&Z dKd'Z!dLdMd(Z"dNd)Z#dNd*Z$d+ Z%d, Z&dOd-Z'dNd.Z(d/ Z)e*jV                  dPd0       Z,e-d1        Z.e*jV                  dPd2       Z/e-d3        Z0e1dQd4       Z2d5 Z3d6 Z4e-d7        Z5dNd8Z6d9 Z7dRd:Z8d; Z9	 	 	 	 dSd<Z:dTd=Z;dUd>Z<dVd?Z=d@ Z>y)Wr    z	list[str]_parameterszdict[str, Any]	_defaultsTbool_pickle_functools_cachelistoperands
str | None_determ_tokenNr.   c                  t        |      }| j                  t        |      d  D ]#  }	 |j                  |j	                  |             % |rJ |       t        j                  |       }||_	        |D cg c]  }t        |       c}|_        |j                   |S # t
        $ r! |j                  | j                  |          Y w xY wc c}w N)r+   r'   lenappendpopKeyErrorr(   object__new__r.   r$   r,   _name)clsr.   argskwargsr,   	parameterinstr"   s           r#   r7   zExpr.__new__:   s    :X9 	:I:

9 56	:
 !6!z~~c"*9ABA,Q/B 	

  :i 89: Cs    B3C'CCc                     y r1    selfs    r#   
_tune_downzExpr._tune_downK       r%   c                     y r1   r?   )rA   parents     r#   _tune_upzExpr._tune_upN   rC   r%   c                    | S r1   r?   r@   s    r#   finalize_computezExpr.finalize_computeQ       r%   c           	         t        | j                  | j                        D cg c]  \  }}| dt        |        c}}S c c}}w )N=)zipr'   r,   repr)rA   paramops      r#   _operands_for_reprzExpr._operands_for_reprT   sC    36t7G7G3W
&/eRugQtBxj!
 	
 
s   Ac                v    dj                  | j                               }t        |       j                   d| dS )N, ())joinrP   type__name__)rA   ss     r#   __str__zExpr.__str__Y   s7    IId--/0t*%%&as!,,r%   c                    t        |       S r1   )strr@   s    r#   __repr__zExpr.__repr__]   s    4yr%   c                >   	 | j                   |   }| j                  |   }t        |      t        |      k7  r&|r|d| dt        |       z  }|S |t        |      z  }|S # t        t        f$ r. |t	        | j                         k  r| j                   |   nd}d}Y {w xY w)N z--no-default-- rK   )r'   r(   
IndexErrorr5   r2   rM   )rA   irO   headerrN   defaults         r#    _tree_repr_argument_constructionz%Expr._tree_repr_argument_construction`   s    	'$$Q'EnnU+G
 8tG}$AeWAd2hZ00  $r(" H% 	'+,s43C3C/D+DD$$Q'"E&G	's   A :BBc                $    d|z  t        |       z   S )Nr_   )rM   rA   indent	recursives      r#   _tree_repr_lineszExpr._tree_repr_lineso   s    V|d4j((r%   c                \    t         j                  j                  | j                               S r1   )oslineseprU   ri   r@   s    r#   	tree_reprzExpr.tree_reprr   s    zzt44677r%   c                ^    ddl m} ddlm} t	        | |      st        d       || ||      S )Nr   )r    )analyzez:analyze is only supported for dask.dataframe.Expr objects.)filenameformat)dask.dataframe.dask_expr._exprr    $dask.dataframe.dask_expr.diagnosticsro   r   	TypeError)rA   rp   rq   DFExprro   s        r#   ro   zExpr.analyzeu   s3    A@$'L  thv>>r%   c                "    ddl m}  || ||      S )Nr   )explain)rs   rw   )rA   stagerq   rw   s       r#   rw   zExpr.explain   s     	AtUF++r%   c                D    | j                         D ]  }t        |        y r1   )ri   print)rA   lines     r#   pprintzExpr.pprint   s!    ))+ 	D$K	r%   c                ,    t        | j                        S r1   )hashr8   r@   s    r#   __hash__zExpr.__hash__   s    DJJr%   c                z    | j                   s$t        t        |       g| j                   | _         | j                   S r1   )r.   r
   rV   r,   r@   s    r#   __dask_tokenize__zExpr.__dask_tokenize__   s3    !!
 "9d!Tdmm!TD!!!r%   c                j    t        | j                        D cg c]  }| j                  |f c}S c c}w )zThe keys for this expression

        This is used to determine the keys of the output collection
        when this expression is computed.

        Returns
        -------
        keys: list
            The keys for this expression
        )rangenpartitionsr8   rA   ra   s     r#   __dask_keys__zExpr.__dask_keys__   s+     */t/?/?)@AAQAAAs   0c                 r    | ^}}}} ||d|i}|j                         D ]  \  }}||j                  |<    |S )Nr.   )items__dict__)r:   typr,   tokencacher=   kvs           r#   _reconstructzExpr._reconstruct   sK    '+$huH2E2KKM 	!DAq DMM!	!r%   c                   t         j                  j                  dd      rt        dt	        |        d      i }t	        |       j
                  rdt	        |       j                  j                         D ]>  \  }}t        |t        j                        s!|| j                  v s0t        | |      ||<   @ t        j                  t	        |       g| j                  | j                  |fS )Nzdask-expr-no-serializeFzSerializing a z object)daskconfiggetRuntimeErrorrV   r*   r   r   r   	functoolscached_propertygetattrr    r   r,   deterministic_token)rA   r   r   r   s       r#   
__reduce__zExpr.__reduce__   s    ;;??3U;T
|7CDD:--T
++113 01a!:!:;T]]@R&tQ/E!H0   J#
]]#
 $$#
 	#
 
 	
r%   c                4   |i }| j                         syg }| j                         D ]d  }|j                  |v r|j                  ||j                            0|j                  |j                  |      dz          |d   ||j                  <   f t	        |      S )zYDepth of the expression tree

        Returns
        -------
        depth: int
           )dependenciesr8   r3   _depthmax)rA   r   resultr   s       r#   r   zExpr._depth   s     =E  "F))+ 3::&MM%

"34MM$++e"4q"89(.r
E$**%3 v;r%   c                   |dv rt         j                  | ||       y 	 t        |       j                  }t         j	                  | d      }|||j                  |      <   y # t        $ r$ t        t        |       j                   d|       w xY w)N)r,   r.   r,   z object has no attribute )	r6   __setattr__rV   r'   __getattribute__index
ValueErrorAttributeErrorrW   )rA   namevalueparamsr,   s        r#   r   zExpr.__setattr__   s    00tT51	$Z++F..tZ@H+0HV\\$'( 	 :&&''@G 	s   ?A -Bc                d    | j                   t        |       j                  j                  |         S r1   )r,   rV   r'   r   )rA   keys     r#   operandzExpr.operand   s(     }}T$Z3399#>??r%   c                b    | j                   D cg c]  }t        |t              s| c}S c c}w r1   )r,   r   r    )rA   r   s     r#   r   zExpr.dependencies   s#    '+}}RG
7D8QRRRs   ,,c                2    t        dt        |        d      )a  The task for the i'th partition

        Parameters
        ----------
        index:
            The index of the partition of this dataframe

        Examples
        --------
        >>> class Add(Expr):
        ...     def _task(self, i):
        ...         return Task(
        ...            self.__dask_keys__()[i],
        ...            operator.add,
        ...            TaskRef((self.left._name, i)),
        ...            TaskRef((self.right._name, i))
        ...        )

        Returns
        -------
        task:
            The Dask task to compute this partition

        See Also
        --------
        Expr._layer
        zcExpressions should define either _layer (full dictionary) or _task (single task).  This expression z defines neither)NotImplementedErrorrV   )rA   r   r   s      r#   _taskz
Expr._task   s(    8 "004T
|;KM
 	
r%   c                    t        | j                        D ci c]-  }| j                  |f| j                  | j                  |f|      / c}S c c}w )a@  The graph layer added by this expression.

        Simple expressions that apply one task per partition can choose to only
        implement `Expr._task` instead.

        Examples
        --------
        >>> class Add(Expr):
        ...     def _layer(self):
        ...         return {
        ...            name: Task(
        ...                name,
        ...                operator.add,
        ...                TaskRef((self.left._name, i)),
        ...                TaskRef((self.right._name, i))
        ...            )
        ...            for i, name in enumerate(self.__dask_keys__())
        ...         }

        Returns
        -------
        layer: dict
            The Dask task graph added by this expression

        See Also
        --------
        Expr._task
        Expr.__dask_graph__
        )r   r   r8   r   r   s     r#   _layerzExpr._layer  sP    B 4++,
 ZZOTZZQ;;
 	
 
s   2Ac                   | j                   |v r|| j                      S | }d| d}d| d}	 d} t        ||             }||}t        |t              s|S |j                   |j                   k7  r|}F|j	                         D ]P  } t        ||      |      }||}t        |t              s|c S ||us2|j                   |j                   k7  sL|}d} n |rg }	d}
|j
                  D ]c  }t        |t              r>|j                  ||      }|||j                   <   |j                   |j                   k7  rd}
n|}|	j                  |       e |
r t        |      |	 }4	 |S )a  Rewrite an expression

        This leverages the ``._{kind}_down`` and ``._{kind}_up``
        methods defined on each class

        Returns
        -------
        expr:
            output expression
        changed:
            whether or not any change occurred
        __down_upTFkind	rewritten)	r8   r   r   r    r   r,   rewriter3   rV   )rA   r   r   r   	down_nameup_name	_continueoutchildnew_operandschangedr   news                r#   r   zExpr.rewrite+  s    ::"TZZ((vUO	dV3-I +'$	*,C{c4(
yyDJJ& **, 	-geW-d3;C!#t,Jd?syyDJJ'>D $I	  LG== )gt,!//ty/IC/2Igmm,yyGMM1"&!C##C() !tDz<0r%   c                   | j                   |v r|| j                      S | }	 |j                         }||}t        |t              s|S |j                   |j                   k7  r|}|j	                         D ]N  }|j                  ||      }||}t        |t              s|c S ||us2|j                   |j                   k7  sL|} n g }d}|j                  D ]  }t        |t              ro||j                      j                  t        j                  |             |j                  ||      }	|	||j                   <   |	j                   |j                   k7  rd}n|}	|j                  |	        |r t        |      | }	 |S )a  Simplify an expression

        This leverages the ``._simplify_down`` and ``._simplify_up``
        methods defined on each class

        Parameters
        ----------

        dependents: defaultdict[list]
            The dependents for every node.
        simplified: dict
            Cache of simplified expressions for these dependents.

        Returns
        -------
        expr:
            output expression
        TF
dependents
simplified)r8   _simplify_downr   r    r   _simplify_upr,   r3   weakrefrefsimplify_oncerV   )
rA   r   r   r   r   r   r   r   r   r   s
             r#   r   zExpr.simplify_onceo  s~   ( ::#djj))%%'C{c4(
yyDJJ& **, 	((z:;C!#t,Jd?syyDJJ'>D	 LG== )gt,w}}-44W[[5FG!//#-* 0 C 14Jw}}-yyGMM1"&!C##C() !tDz<0r%   c                &    |rdnd}t        | |      S )Nr   r   optimize_until)rA   fuserx   s      r#   optimizezExpr.optimize  s    +/5JdE**r%   c                    | S r1   r?   r@   s    r#   r   z	Expr.fuse  rI   r%   c                   | }t               }	 t        |      }|j                  |i       }|j                  |j                  k(  r	 |S |j                  |v rt	        d|d|d      |j                  |j                         |}x)Nr   zOptimizer does not converge. z simplified to zf which was already seen. Please report this issue on the dask issue tracker with a minimal reproducer.)setcollect_dependentsr   r8   r   add)rA   r   seenr   r   s        r#   simplifyzExpr.simplify  s    u+D1J$$
r$JCyyDJJ&  yyD "3D8?3' Rd d  HHSYYD r%   c                     y r1   r?   r@   s    r#   r   zExpr._simplify_down      r%   c                     y r1   r?   )rA   rE   r   s      r#   r   zExpr._simplify_up  r   r%   c                   	 || j                      S # t        $ r Y nw xY w| }|j                         }||}t        |t              s|S g }d}|j
                  D ]R  }t        |t              r-|j                  |      }|j                   |j                   k7  rd}n|}|j                  |       T |r t        |      | }|j                  | j                   |      S )NFT)
r8   r5   _lowerr   r    r,   
lower_oncer3   rV   
setdefault)rA   loweredr   r   r   r   r   r   s           r#   r   zExpr.lower_once  s    	4::&& 		  kkm;C#t$J || 	%G'4(((199-"G$	% $s)\*C !!$**c22s    	c                l    | }i }	 |j                  |      }|j                  |j                  k(  r	 |S |}0)al  Lower an expression completely

        This calls the ``lower_once`` method in a loop
        until nothing changes. This function does not
        apply any other optimizations (like ``simplify``).

        Returns
        -------
        expr:
            output expression

        See Also
        --------
        Expr.lower_once
        Expr._lower
        )r   r8   )rA   r   r   r   s       r#   lower_completelyzExpr.lower_completely  sC    $ //'*CyyDJJ& D	 r%   c                     y r1   r?   r@   s    r#   r   zExpr._lower  r   r%   c                F    t        t        |             j                         S r1   )r   rV   lowerr@   s    r#   	_funcnamezExpr._funcname  s    T
#))++r%   c                \    | j                   s| j                         | _         | j                   S r1   )r.   r   r@   s    r#   r   zExpr.deterministic_token  s*    !! "&!7!7!9D!!!r%   c                :    | j                   dz   | j                  z   S )N-)r   r   r@   s    r#   r8   z
Expr._name   s    ~~#d&>&>>>r%   c                    t               r1   )r   r@   s    r#   _metaz
Expr._meta$  s    !##r%   c                    t               S r1   )_AnnotationsTombstone)r9   s    r#   _annotations_tombstonezExpr._annotations_tombstone(  s    $&&r%   c                    i S r1   r?   r@   s    r#   __dask_annotations__zExpr.__dask_annotations__,  s    	r%   c                R   | g}t               }g }|r|j                         }|j                  |v r!|j                  |j                         |j	                  |j                                |j                         D ]  }|j	                  |        |rt        j                  |      S )a/  Traverse expression tree, collect layers

        Subclasses generally do not want to override this method unless custom
        logic is required to treat (e.g. ignore) specific operands during graph
        generation.

        See also
        --------
        Expr._layer
        Expr._task
        )	r   r4   r8   r   r3   r   r   toolzmerge)rA   stackr   layersr   r   s         r#   __dask_graph__zExpr.__dask_graph__/  s     u99;DzzT!HHTZZ MM$++-(,,. &W%&  {{6""r%   c                "    | j                         S r1   )r   r@   s    r#   r   z	Expr.daskK  s    ""$$r%   c                :    | j                  ||t                     S )a  Substitute a specific term within the expression

        Note that replacing non-`Expr` terms may produce
        unexpected results, and is not recommended.
        Substituting boolean values is not allowed.

        Parameters
        ----------
        old:
            Old term to find and replace.
        new:
            New term to replace instances of `old` with.

        Examples
        --------
        >>> (df + 10).substitute(10, 20)  # doctest: +SKIP
        df + 20
        )_seen)_substituter   )rA   oldr   s      r#   
substitutezExpr.substituteO  s    & S66r%   c           	        | j                   |v r| S t        |t              rd}| j                   |j                   k(  r|S d}t        |t              rt	        d      g }d}| j
                  D ]:  }t        |t              r@|j                  |||      }|j                   |j                   k7  rd}|j                  |       Tdt        |       j                  v r~t        |t              rnt        d |D              r\g }|D ]C  }	|j                  |	j                  |||             |d   j                   |	j                   k7  sBd}E |j                  |       |r?t        |t              s/t        |t        |            r||k(  r|j                  |       d}*|j                  |       = |r t        |       | S |j                  | j                          | S )NFTz)Arguments to `substitute` cannot be bool.Fusedc              3  <   K   | ]  }t        |t                y wr1   )r   r    .0rO   s     r#   	<genexpr>z#Expr._substitute.<locals>.<genexpr>|  s     ?
2t,?   r   )r8   r   r    r)   rt   r,   r   r3   rV   rW   r+   allr   )
rA   r   r   r   substitute_literal	new_exprsupdater   valrO   s
             r#   r   zExpr._substituted  s   ::Kc4 !&zzSYY&
!%#t$ KLL	}} 	*G'4())#sE:==CII-!F  %4:...w-?w?? ! &BJJr~~c3>?2w}}0!%&   %""7D1wS	2sN  %  )=	*@ 4:y))IIdjj!r%   c                ,   |s| S d}g }t        | j                        D ]c  \  }}|t        | j                        k  r5| j                  |   |v r$|j	                  || j                  |             d}S|j	                  |       e |r t        |       | S | S )zSubstitute specific `Expr` parameters

        Parameters
        ----------
        substitutions:
            Mapping of parameter keys to new values. Keys that
            are not found in ``self._parameters`` will be ignored.
        FT)	enumerater,   r2   r'   r3   rV   )rA   substitutionsr   r   ra   r   s         r#   substitute_parameterszExpr.substitute_parameters  s     K#DMM2 	-JAw3t''((T-=-=a-@M-Q##M$2B2B12E$FG##G,	- 4:|,,r%   c                "    | j                         S )z4Operands to include in the node label by `visualize`r   r@   s    r#   _node_label_argszExpr._node_label_args  s      ""r%   c           	        ddl m}m} t        dd      }|xs i }|xs i }|xs i }||d<   d|d<   d|d	<   |j	                  |       |j                  |||
      }	| g}
t               }i }|
r|
j                         }|j                  |v r!|j                  |j                         t        |j                               ||<   |j                         D ]  }|
j                  |        |
ri }|D ]  } ||      }i }|j                         D cg c]1  }t        |t              rt        t!        |            n
t#        |      3 }}t        t!        |            }|r|r| ddj%                  |       dn|} |||      }|j'                  dt#        |             |j'                  dd        |	j(                  |fi |  |j+                         D ].  \  }} ||      }|D ]  } ||      }|	j-                  ||        0 |	S c c}w )Nr   )labelr   graphvizaJ  Drawing dask graphs with the graphviz visualization engine requires the `graphviz` python library and the `graphviz` system library.

Please either conda or pip install as follows:

  conda install python-graphviz     # either conda install
  python -m pip install graphviz    # or pip install and follow installation instructionsrankdirboxshape	helveticafontname)
graph_attr	node_attr	edge_attrrS   rR   rT   )r   r  fontsize20)dask.dotr  r   r   r  Digraphr   r4   r8   r   r   r3   r  r   r    r   rV   r[   rU   r   noder   edge)rA   r  r  r  r  r;   r  r   r  gr   r   r   r   depr   	expr_nameattrsdeps_label
node_labeldep_names                         r#   _to_graphvizzExpr._to_graphviz  s:    	)"h
  %2
O	O	 '
9"	' +	*&!!  
 u99;DzzT!HHTZZ !$T%6%6%8!9L((* "S!"    	'DT
IE
  002 (2#t'<c##c(JD  d4j)F;?F81TYYt_$5Q7VvU3JWc*o6Z.AFF9&&!	'$ ',,. 	,JD$T
I ,9x+,	, 's   6Hc                J    ddl m}  | j                  di |} ||||       |S )a  
        Visualize the expression graph.
        Requires ``graphviz`` to be installed.

        Parameters
        ----------
        filename : str or None, optional
            The name of the file to write to disk. If the provided `filename`
            doesn't include an extension, '.png' will be used by default.
            If `filename` is None, no file will be written, and the graph is
            rendered in the Jupyter notebook only.
        format : {'png', 'pdf', 'dot', 'svg', 'jpeg', 'jpg'}, optional
            Format in which to write output file. Default is 'svg'.
        **kwargs
           Additional keyword arguments to forward to ``to_graphviz``.
        r   )graphviz_to_filer?   )r  r*  r(  )rA   rp   rq   r;   r*  r   s         r#   	visualizezExpr.visualize   s-    " 	.D''Hf-r%   c              #     K   | g}t               }|rh|j                         }|j                  |v r!|j                  |j                         |j	                         D ]  }|j                  |        | |rgyyw)zIterate through all expressions in the tree

        Returns
        -------
        nodes
            Generator of Expr instances in the graph.
            Ordering is a depth-first search of the expression tree
        N)r   r4   r8   r   r   r3   )rA   r   r   r  r!  s        r#   walkz	Expr.walk  ss      u99;DzzT!HHTZZ ((* "S!" J s   A6A;9A;c                    t        t              rt        d D              st        t              sJ d       fd| j                         D        S )aZ  Search the expression graph for a specific operation type

        Parameters
        ----------
        operation
            The operation type to search for.

        Returns
        -------
        nodes
            Generator of `operation` instances. Ordering corresponds
            to a depth-first search of the expression graph.
        c              3  <   K   | ]  }t        |t                y wr1   )
issubclassr    )r   es     r#   r  z'Expr.find_operations.<locals>.<genexpr>=  s     ;AJq$';r  z#`operation` must be`Expr` subclass)c              3  <   K   | ]  }t        |      s|  y wr1   )r   )r   r   	operations     r#   r  z'Expr.find_operations.<locals>.<genexpr>@  s     L
40KLs   )r   tupler  r0  r    r-  )rA   r3  s    `r#   find_operationszExpr.find_operations-  sO     y%(;;;)T*	1 1		1+ MLLr%   c                *   	 t         j                  | |      S # t        $ rq}|j                  d      rt	        d|  d      t        |       j                  }||v r%|j                  |      }| j                  |   cY d }~S t        | d      d }~ww xY w)Nr   z Failed to generate metadata for z=. This operation may not be supported by the current backend.zO

This often means that you are attempting to use an unsupported API function..)	r6   r   r   
startswithr   rV   r'   r   r,   )rA   r   errr'   idxs        r#   __getattr__zExpr.__getattr__B  s    	**455 	~~g& #6tf =R R  t*00Kk!!'',}}S)) % ! " #	s!    	BAB9B?BBr   T)NN)rp   r-   rq   r-   returnNone)r   N)rx   r   rq   r-   r<  r=  r1   )r   r[   r   r   r<  r=  )r   r   r   intr<  r   r<  dict)r   r[   )r   r   r   r@  )F)r   r)   r<  r    )r<  r    )r   r@  )r<  r[   )r<  r   )r
  r@  r<  r    )BTNNN)zdask-expr.svgN)r<  Generator[Expr])r3  ztype | tuple[type]r<  rB  )?rW   
__module____qualname__r'   __annotations__r(   r*   r7   rB   rF   rH   rP   rY   r\   rd   ri   rm   ro   rw   r|   r   r   r   staticmethodr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   propertyr   r8   r   classmethodr   r   r   r   r   r   r  r  r(  r+  r-  r5  r:  r?   r%   r#   r    r    0   s   K "I~"$(T(N*. "

-)8? EI,#,7A,	, "B  
 *@
S
B"
HBHDL+
" 3D6 , , " " ? ? $ $ ' '#8 % %7*3j0# IV.,M*r%   r    c                  4     e Zd ZU dZded<   dd fd
Z xZS )SingletonExpra  A singleton Expr class

    This is used to treat the subclassed expression as a singleton. Singletons
    are deduplicated by expr._name which is typically based on the dask.tokenize
    output.

    This is a crucial performance optimization for expressions that walk through
    an optimizer and are recreated repeatedly but isn't safe for objects that
    cannot be reliably or quickly tokenized.
    z/weakref.WeakValueDictionary[str, SingletonExpr]
_instancesNr/   c               &   t        | d      st        j                         | _        t	        |   | g|d|i|}|j                  }|| j                  v r,| j                  t        j                  k(  r| j                  |   S || j                  |<   |S )NrK  r.   )	r!   r   WeakValueDictionaryrK  superr7   r8   __init__r6   )r9   r.   r:   r;   r=   r8   	__class__s         r#   r7   zSingletonExpr.__new__k  s    sL)$88:CNwsQTQQ&Q

CNN"s||v'F>>%(( $ur%   )rW   rC  rD  __doc__rE  r7   __classcell__rP  s   @r#   rJ  rJ  ]  s    	 @?*. 	 	r%   rJ  c                j   t        t              }| g}t               }|r|j                         }|j                  |v r!|j                  |j                         |j                         D ]D  }|j                  |       ||j                     j                  t        j                  |             F |r|S r1   )
r   r+   r   r4   r8   r   r   r3   r   r   )r   r   r   r   r  r!  s         r#   r   r   w  s    T"JFE5D
yy{::$$& 	<CLLsyy!((T):;	<  r%   c                &    |rdnd}t        | |      S )a  High level query optimization

    This leverages three optimization passes:

    1.  Class based simplification using the ``_simplify`` function and methods
    2.  Blockwise fusion

    Parameters
    ----------
    expr:
        Input expression to optimize
    fuse:
        whether or not to turn on blockwise fusion

    See Also
    --------
    simplify
    optimize_blockwise_fusion
    r   r   r   )r   r   rx   s      r#   r   r     s    ( (,G1FE$&&r%   c                   | }|dk(  r|S |j                         } |dk(  r| S | j                  di       } |dk(  r| S | j                         } |dk(  r| S | j                         } |dk(  r| S | j                         } |dk(  r| S t	        d	|d
      )Nr   r   tuner   r   r   r   r   zStage z not supported.)r   r   r   r   r   )r   rx   r   s      r#   r   r     s    F	 ??D$$ <<Vr<2D   "D
 ==?D%% 99;D
veYo6
77r%   c                  $    e Zd ZdZdgZd ZddZy)LLGExprzLow Level Graph Expressiondskc                6    t        | j                  d            S NrZ  )r+   r   r@   s    r#   r   zLLGExpr.__dask_keys__  s    DLL'((r%   c                6    t        | j                  d            S r\  )r   r   r@   s    r#   r   zLLGExpr._layer  s    4<<.//r%   Nr?  )rW   rC  rD  rQ  r'   r   r   r?   r%   r#   rY  rY    s    $'K)0r%   rY  c                      e Zd Zg dZdddddZed        Zedd       Zd Z	ddZ
d Zej                  dd	       Zed
        ZddZy)HLGExpr)rZ  low_level_optimizeroutput_keyspostcompute_cached_optimizedN)r`  ra  rb  rc  c                $    | j                  d      S r\  r   r@   s    r#   hlgzHLGExpr.hlg  s    ||E""r%   c                   ddl m} t        | d      r| j                  j	                         }n| j                         }t        ||      s' |j                  t        t        |             |d      }|r6t        | d      s*t        j                  dt        |        dt               d }n|r| j                  nd }t        ||| j!                         | j#                         	      S )
Nr   r   r   r?   r  __dask_optimize__zCollection z does not define a `__dask_optimize__` method. In the future this will raise. If no optimization is desired, please set this to `None`.rZ  r`  ra  rb  )dask.highlevelgraphr   r!   r   copyr   r   from_collectionsr[   idwarningswarnrV   PendingDeprecationWarningrh  r_  r   __dask_postcompute__)
collectionoptimize_graphr   rZ  r`  s        r#   from_collectionzHLGExpr.from_collection  s    6:v&//&&(C++-C #~.1.11BzN#SrC '*6I"JMMd:./ 0L L *	 #' 1?
,,D    3"002"779	
 	
r%   c                \    t        | | j                  | j                  | j                        S )N)r`  ra  rb  )HLGFinalizeComputer`  ra  rb  r@   s    r#   rH   zHLGExpr.finalize_compute  s-    ! $ 8 8((((	
 	
r%   c                d   | j                   }t        t              }|j                  j	                         D ]i  }|j
                  s|j
                  }|j                         D ];  \  }}||   j                  |D ci c]  }|t        |      r ||      n| c}       = k t        |      S c c}w r1   )	_optimized_dskr   r@  r   valuesr   r   r  callable)rA   rZ  annotations_by_typelayerannot
annot_typer   r   s           r#   r   zHLGExpr.__dask_annotations__  s     !!CNtCTZZ&&( 	E  ))). %J'
3::NST%U1XeCT	 '(( Us   :B-c                    | j                  d      x}|S | j                  }|j                         }t        |      }|j	                         D ]  }||z  }	 t        |      | _        | j                  S )Nra  )r   rf  get_all_dependenciesr   ry  r+   ra  )rA   keysrZ  r   leafsr  s         r#   r   zHLGExpr.__dask_keys__  su    LL//D<Khh//1L!&&( 	CSLE	;r%   c                    ddl m} | j                  }| j                         }| j                  }| j                  x}	 |||      } |j
                  |      S Nr   r   )rj  r   r`  r   rf  r   )rA   r   	optimizerr  rZ  s        r#   rx  zHLGExpr._optimized_dsk&  sW    6,,	!!#hh111I>C&C#~##C((r%   c                x    | j                   s#t        j                         j                  | _         | j                   S r1   )r.   uuiduuid4hexr@   s    r#   r   zHLGExpr.deterministic_token1  s+    !!!%!1!1D!!!r%   c                0    | j                   }t        |      S r1   )rx  r   )rA   rZ  s     r#   r   zHLGExpr._layer7  s    !!3r%   Tr<  zdict[str, dict[Key, object]]r<  r   r?  )rW   rC  rD  r'   r(   rG  rf  rF  rt  rH   r   r   r   r   rx  r   r   r?   r%   r#   r_  r_    s    K  $!	I # #  
  
D
)
  ) ) " "
 r%   r_  c                      e Zd Zy)_HLGExprGroupNrW   rC  rD  r?   r%   r#   r  r  <  s     	r%   r  c                  l    e Zd Zd Zd ZddZd Zd Zej                  dd       Z
d ZeZddZdd	Zy
)_HLGExprSequencec                     | j                   |   S r1   r,   rA   others     r#   __getitem__z_HLGExprSequence.__getitem__D      }}U##r%   c                R    d| j                  d      d| j                  d      gS )Nzname=r   zdsk=rZ  re  r@   s    r#   rP   z#_HLGExprSequence._operands_for_reprG  s4    DLL(+,4<<&)*
 	
r%   c                "    | j                         S r1   )rP   rf   s      r#   ri   z!_HLGExprSequence._tree_repr_linesM  s    &&((r%   c                h    t        | j                  D cg c]  }|j                          c} S c c}w r1   )r  r,   rH   )rA   rO   s     r#   rH   z!_HLGExprSequence.finalize_computeP  s(    $--!PB""5"5"7!PQQ!Ps   /c           
     0   t        | j                        dk(  ry ddlm} t	        j
                  d | j                        }g }d}|j                         D ]  \  }}t        |      dkD  r|D cg c]  }|j                   }}d} |j                  | }	t        |	||D 
cg c]  }
|
j                          c}
|D cg c]  }|j                   c}      }|j                  |       |j                  |d           |sy t        | S c c}w c c}
w c c}w )Nr   r   r   c                >    t        | t              r| j                  S d S r1   )r   r_  r`  )xs    r#   <lambda>z-_HLGExprSequence._tune_down.<locals>.<lambda>Y  s    z!W/Ea++ 4 r%   FTri  )r2   r,   rj  r   r   groupbyr   rf  r   r  r   rb  r3   r  )rA   r   groupsexprsr   r  groupr   graphsrZ  r   r   	hlg_groups                r#   rB   z_HLGExprSequence._tune_downS  s   t}}"6OMM
  & 	'Iu5zA~/45t$((55*n**F3)(1<A Bq!2 B8= >1 >		 Y'U1X&	' '' 6 !C >s   -D	D;Dc                    ddl m} g }| j                  D ]F  }|j                         }|j                  }|j
                  x}	 |||      }|j                  |       H  |j                  | S r  )rj  r   r,   r   rf  r`  r3   r   )rA   r   r  hlgexprr  rZ  r  s          r#   rx  z_HLGExprSequence._optimized_dskq  st    6 }} 	G((*D++C$888	ET*MM#	 $~##V,,r%   c                ,    t        | j                        S r1   )r   rx  r@   s    r#   r   z_HLGExprSequence.__dask_graph__  s     4..//r%   c                  	 | j                   }t        t              }|j                  j	                         D ]  }|j
                  s|j
                  }|j                         D ]Z  \  }	t        	fd|D              }||   j                  |D ci c]  \  }}t        |t              s|| c}}       ||   rX||= \  t        |      S c c}}w )Nc              3  L   K   | ]  }|t              r |      nf  y wr1   )rz  )r   r   r   s     r#   r  z8_HLGExprSequence.__dask_annotations__.<locals>.<genexpr>  s'      "JK%U1XeE"s   !$)rx  r   r@  r   ry  r   r   r+   r  r   r   )
rA   rZ  r{  r|  r}  r~  annotsr   r   r   s
            @r#   r   z%_HLGExprSequence.__dask_annotations__  s     !!CNtCTZZ&&( 	<E  ))). <%J! "OT" F (
3:: )/ $1#-a1F#G qD /z:/
;<	<  '((s   Cc                    g }| j                   D ]Q  }t        |t              r |j                  |j	                                3|j                  |j	                                S |S r1   )r,   r   r  extendr   r3   rA   all_keysrO   s      r#   r   z_HLGExprSequence.__dask_keys__  sU    -- 	4B"m, 0 0 23 0 0 23		4
 r%   Nr;  r  r  r<  r+   )rW   rC  rD  r  rP   ri   rH   rB   r   r   rx  r   r   r   r   r?   r%   r#   r  r  B  sL    $
)R(< - -0
 F)0r%   r  c                  N    e Zd ZdZd ZddZddZd ZeZd Z	d Z
d Zd	 Zd
 Zy)_ExprSequencezA sequence of expressions

    This is used to be able to optimize multiple collections combined, e.g. when
    being computed simultaneously with ``dask.compute((Expr1, Expr2))``.
    c                     | j                   |   S r1   r  r  s     r#   r  z_ExprSequence.__getitem__  r  r%   c                N    t        j                  d | j                  D              S )Nc              3  <   K   | ]  }|j                           y wr1   r   r   s     r#   r  z'_ExprSequence._layer.<locals>.<genexpr>  s     ?2299;?r  )r   r   r,   r@   s    r#   r   z_ExprSequence._layer  s    {{????r%   c                |    g }| j                   D ]*  }|j                  t        |j                                      , |S r1   )r,   r3   r+   r   r  s      r#   r   z_ExprSequence.__dask_keys__  s9    -- 	6BOOD!1!1!345	6r%   c                `    ddj                  t        t        | j                              z   dz   S )NzExprSequence(rR   rT   )rU   maprM   r,   r@   s    r#   r\   z_ExprSequence.__repr__  s&    3tT]]+C!DDsJJr%   c                4    t        d | j                  D         S )Nc              3  <   K   | ]  }|j                           y wr1   )rH   r   s     r#   r  z1_ExprSequence.finalize_compute.<locals>.<genexpr>  s     <b!!#<r  )r  r,   r@   s    r#   rH   z_ExprSequence.finalize_compute  s    <dmm<
 	
r%   c                    i }| j                   D ]I  }|j                         j                         D ]&  \  }}|j                  |i       j	                  |       ( K |S r1   )r,   r   r   r   r  )rA   r{  rO   r   r   s        r#   r   z"_ExprSequence.__dask_annotations__  sg     -- 	@B//1779 @1#..q"5<<Q?@	@ #"r%   c                ,    t        | j                        S r1   )r2   r,   r@   s    r#   __len__z_ExprSequence.__len__  s    4==!!r%   c                ,    t        | j                        S r1   )iterr,   r@   s    r#   __iter__z_ExprSequence.__iter__  s    DMM""r%   c                .   ddl m} d}g }| j                  D ]  }t        |t        t
        f      r|j                  |       +t        |t              rA|j                  t	         |j                  t        t        |            |d                   ||sd}|j                         }|j                  t	         |j                  |j                  |j                         d                    |rt        j                  dt                |sy t#        | S )	Nr   r   Fr?   r  )rZ  TzComputing mixed collections that are backed by HighlevelGraphs/dicts and Expressions. This forces Expressions to be materialized. It is recommended to use only one type and separate the dask.compute calls if necessary.)rj  r   r,   r   r_  rv  r3   r@  rl  r[   rm  r   r8   r   rn  ro  UserWarningr  )rA   r   issue_warninghlgsrO   opts         r#   r   z_ExprSequence._simplify_down  s    6-- 	B"w(:;<BB%;N;;2K"  $kkm;N;;IIs'9'9';"	* MM.
  &&r%   Nr?  r  )rW   rC  rD  rQ  r  r   r   r\   rY   rH   r   r  r  r   r?   r%   r#   r  r    s=    $@K G

#"#%'r%   r  c                      e Zd Zy)r   Nr  r?   r%   r#   r   r     s    r%   r   c                      e Zd ZdgZd Zy)FinalizeComputer   c                6    | j                   j                         S r1   )r   rH   r@   s    r#   r   zFinalizeCompute._simplify_down  s    yy))++r%   N)rW   rC  rD  r'   r   r?   r%   r#   r  r    s    (K,r%   r  c                    ddl m}m} t        | t              sJ g }| D ]D  }t        |t              r|j                  t        |             .|j                   ||             F  || S )Nr   )ListTaskRef)dask._task_specr  r  r   r+   r3   _convert_dask_keys)r  r  r  new_keysr   s        r#   r  r    sa    -dD!!!H *c4 OO.s34OOGCL)	*
 ?r%   c                  J     e Zd Zd Ze fd       Zd Zed        Zd Z xZ	S )rv  c                    | j                   s| j                  S ddlm} | j                  j                   |j	                  | j                        k(  r| j                  S | S )Nr   r   )rb  rZ  r   r   rq  )rA   r   s     r#   r   z!HLGFinalizeCompute._simplify_down  sH    88O( 887#?#?#II88Or%   c                    dt         |    S )Nz	finalize-)rN  r8   )rA   rP  s    r#   r8   zHLGFinalizeCompute._name   s    57=/**r%   c                "    | j                         S r1   r  r@   s    r#   r   z!HLGFinalizeCompute.__dask_graph__$  s     {{}r%   c                   | j                  d      }|j                  j                  j                         }|j                  j                  j                         }|j                         }t        |j                  t              r|j                  }n|j                  g}|D cg c]&  \  }}t        | j                  |t        |      g| ( }}}ddlm}	m}
 t        |      }|j!                         D ]  }||z  }	 |D ]2  } |
|j"                  |i      ||j"                  <   |||j"                  <   4  |	||      S c c}}w )NrZ  r   )r   MaterializedLayerr  )r   rZ  r   rk  r   r   r   rb  r+   r   r8   r  rj  r   r  r   ry  r   )rA   r   r   r$  r  postcomputesfunc
extra_argstasksr   r  r  r  ts                 r#   rf  zHLGFinalizeCompute.hlg.  s'   ||E"%%'xx$$))+!!#d&&-++L ,,-L %1
 j T#5d#;IjI
 
 	JD	;;= 	CSLE	 	 A-quuaj9F155MDK	  f488
s   #+D?c                    | j                   gS r1   )r8   r@   s    r#   r   z HLGFinalizeCompute.__dask_keys__F  s    

|r%   )
rW   rC  rD  r   rG  r8   r   rf  r   rR  rS  s   @r#   rv  rv    s:    	 + + 9 9.r%   rv  c                  r    e Zd ZdZdgZeeeegZ	d Z
ed        Zej                  d        Zd Zd Zd ZeZy	)
ProhibitReusez
    An expression that guarantees that all keys are suffixes with a unique id.
    This can be used to break a common subexpression apart.
    r   c                T    | j                  | j                  j                               S r1   )_modify_keysr   r   r@   s    r#   r   zProhibitReuse.__dask_keys__S  s       !8!8!:;;r%   c                    | S r1   r?   )objs    r#   	_identityzProhibitReuse._identityV  s    
r%   c                >    t        j                         j                  S r1   )r  r  r  r@   s    r#   _suffixzProhibitReuse._suffixZ  s    zz|r%   c                $   t        |t              r|D cg c]  }| j                  |       c}S t        |t              r| j                  |d         f|dd  z   S t        |t        t
        f      rt        |      }| d| j                   S c c}w )Nr   r   r   )r   r+   r  r4  r>  floatr[   r  )rA   r   kks      r#   r  zProhibitReuse._modify_keys^  s    a456bD%%b)665!%%ad+-!"55C<(AAAdll^$$ 7s   Bc                n    t        | j                  t        | j                              s| j                  S y r1   )r   r   r4  _ALLOWED_TYPESr@   s    r#   r   zProhibitReuse._simplify_downg  s2     II$%%&
 99	
r%   c                   	 ddl m} t	        | j
                  j                               }|D ci c]  }|| j                  |       }}i }|j                         D ]a  \  }}||   }t        ||      rt        j                  dt               |c S t        |t        j                  |j!                  |            ||<   c |j#                  |       |S # t        $ r t        d       }Y w xY wc c}w )Nr   )P2PBarrierTaskzCannot block reusing for graphs including a P2PBarrierTask. This may cause unexpected results. This typically happens when converting a dask DataFrame to delayed objects.)distributed.shuffle._corer  ModuleNotFoundErrorrV   r	   r   r   r  r   r   rn  ro  r  r   r  r  r   r  )rA   r  rZ  old_keysubsdsk2new_keyr  s           r#   r   zProhibitReuse.__dask_graph__p  s    	(@ #499#;#;#=>CFG**733GG $

 	GWGA!^,4   
 ''T"DM	" 	C1 # 	(!$ZN	( Hs   C C-C*)C*N)rW   rC  rD  rQ  r'   r_  rY  rv  r  r  r   rF  r  r   r   r  r  r   r   r   r?   r%   r#   r  r  J  si    
 (Kw(:<LMN<      %: Fr%   r  )r<  r   r  )r   r    r   r)   r<  r    )r   r    rx   r   r<  r    )1
__future__r   r   rk   r  rn  r   collectionsr   collections.abcr   typingr   r   r   r   r  r   r	   dask.tokenizer
   dask.typingr   
dask.utilsr   r   r   r   r   rj  r   r   rE  r$   r    rJ  r   r   r   rY  r_  r  r  r  r   r  r  rv  r  r?   r%   r#   <module>r     s    "  	    # % )   6 1  = =%2#	 	j jZD 4 '28B	0d 	0l d l ^	G 	et ePO'D O'd !  ,d ,
4 4nCD Cr%   