
    bi9                       d dl mZ d dlZd dlZd dlmZ d dlmZ d dlZ	d dl
mZ d dlmZmZ d dlmZ d dlmZmZmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ erd dlZ G d d      Z  G d de      Z! G d de!      Z" G d de!      Z# G d de!      Z$ G d de!      Z% ejL                  e#      d        Z' ejL                  e$      d        Z( ejL                  e%      d        Z) G d de      Z*d"dZ+d Z, G d  d!e      Z-y)#    )annotationsN)product)TYPE_CHECKING)map)TaskTaskRef)tokenize)	BlockwiseBlockwiseDepBlockwiseDepDictblockwise_token)flatten)Layer)normalize_tokencached_cumsumc                      e Zd ZdZd Zd Zy)CallableLazyImportzFunction Wrapper for Lazy Importing.

    This Class should only be used when materializing a graph
    on a distributed scheduler.
    c                    || _         y N)function_path)selfr   s     F/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/layers.py__init__zCallableLazyImport.__init__$   s
    *    c                >    ddl m}   || j                        |i |S )Nr   )import_term)distributed.utilsr   r   )r   argskwargsr   s       r   __call__zCallableLazyImport.__call__'   s"    1.{4--.???r   N)__name__
__module____qualname____doc__r   r!    r   r   r   r      s    +@r   r   c                  D    e Zd ZU dZded<   ded<   dZded<   dd	Zdd
Zy)ArrayBlockwiseDepzg
    Blockwise dep for array-likes, which only needs chunking
    information to compute its data.
    tuple[tuple[int, ...], ...]chunkstuple[int, ...]	numblocksFboolproduces_tasksc                N    || _         t        d |D              | _        d| _        y )Nc              3  2   K   | ]  }t        |        y wr   )len).0chunks     r   	<genexpr>z-ArrayBlockwiseDep.__init__.<locals>.<genexpr>@   s     >es5z>s   F)r*   tupler,   r.   )r   r*   s     r   r   zArrayBlockwiseDep.__init__>   s#    >v>>#r   c                    t        d      )Nz%Subclasses must implement __getitem__)NotImplementedErrorr   idxs     r   __getitem__zArrayBlockwiseDep.__getitem__C   s    !"IJJr   Nr*   r)   r9   r+   )r"   r#   r$   r%   __annotations__r.   r   r:   r&   r   r   r(   r(   4   s*    
 (' ND $
Kr   r(   c                      e Zd ZdZddZy)ArrayChunkShapeDepz(Produce chunk shapes given a chunk indexc                N    t        d t        || j                        D              S )Nc              3  ,   K   | ]  \  }}||     y wr   r&   )r2   ir3   s      r   r4   z1ArrayChunkShapeDep.__getitem__.<locals>.<genexpr>K   s     D(!UU1XDs   )r5   zipr*   r8   s     r   r:   zArrayChunkShapeDep.__getitem__J   s    Dc#t{{.CDDDr   Nr<   )r"   r#   r$   r%   r:   r&   r   r   r?   r?   G   s    2Er   r?   c                  8     e Zd ZU dZded<   d fdZddZ xZS )ArraySliceDepz>Produce slice(s) into the full-sized array given a chunk indexr)   startsc                R    t         |   |       t        d |D              | _        y )Nc              3  6   K   | ]  }t        |d         yw)T)initial_zeroNr   )r2   cs     r   r4   z)ArraySliceDep.__init__.<locals>.<genexpr>U   s     PAM!$??P   )superr   r5   rF   )r   r*   	__class__s     r   r   zArraySliceDep.__init__S   s"     PPPr   c                r    t        d t        || j                        D              }t        d |D              S )Nc              3  <   K   | ]  \  }}||   ||d z      f  yw)   Nr&   )r2   rB   starts      r   r4   z,ArraySliceDep.__getitem__.<locals>.<genexpr>X   s%     SEU1XuQU|,S   c              3  6   K   | ]  }t        g |d    y wr   slice)r2   ss     r   r4   z,ArraySliceDep.__getitem__.<locals>.<genexpr>Y   s     2U_A_t_2rK   )r5   rC   rF   )r   r9   locs      r   r:   zArraySliceDep.__getitem__W   s.    SSdkk=RSS2c222r   r;   r9   r5   )r"   r#   r$   r%   r=   r   r:   __classcell__rM   s   @r   rE   rE   N   s    H''Q3r   rE   c                      e Zd ZddZy)ArrayBlockIdDepc                <    t        |t              st        d      |S )Nz&ArrayBlockIdDep requires a tuple index)
isinstancer5   r7   r8   s     r   r:   zArrayBlockIdDep.__getitem__]   s    #u%%&NOO
r   NrX   )r"   r#   r$   r:   r&   r   r   r\   r\   \   s    r   r\   c                  (     e Zd Zd fdZddZ xZS )ArrayValuesDepc                2    t         |   |       || _        y r   )rL   r   values)r   r*   rb   rM   s      r   r   zArrayValuesDep.__init__e   s     r   c                     | j                   |   S r   )rb   r8   s     r   r:   zArrayValuesDep.__getitem__i   s    {{3r   )r*   r)   rb   znp.ndarray | dictrX   )r"   r#   r$   r   r:   rY   rZ   s   @r   r`   r`   d   s     r   r`   c                    d| j                   fS )NrE   r*   deps    r   normalize_array_slice_deprh   m   s    CJJ&&r   c                    d| j                   fS )Nr\   re   rf   s    r   normalize_array_block_id_deprj   r   s    cjj((r   c                4    d| j                   | j                  fS )Nr`   )r*   rb   rf   s    r   normalize_array_values_deprl   w   s    SZZ33r   c                  d     e Zd ZdZ fdZd Zed        Zd Zd Z	d Z
d Zd	 Zd
 ZddZ xZS )ArrayOverlapLayera\  Simple HighLevelGraph array overlap layer.

    Lazily computed High-level graph layer for a array overlap operations.

    Parameters
    ----------
    name : str
        Name of new output overlap array.
    array : Dask array
    axes: Mapping
        Axes dictionary indicating overlap in each dimension,
        e.g. ``{'0': 1, '1': 1}``
    c                v    t         |           || _        || _        || _        || _        || _        d | _        y r   )rL   r   nameaxesr*   r,   token_cached_keys)r   rp   rq   r*   r,   rr   rM   s         r   r   zArrayOverlapLayer.__init__   s;     			"
 r   c                "    d| j                    dS )NzArrayOverlapLayer<name=''rp   r   s    r   __repr__zArrayOverlapLayer.__repr__   s    )$))A66r   c                x    t        | d      r| j                  S | j                         }|| _        | j                  S )z$Materialize full dict representation_cached_dict)hasattrrz   _construct_graph)r   dsks     r   _dictzArrayOverlapLayer._dict   s<     4($$$'')C #D   r   c                     | j                   |   S r   )r~   )r   keys     r   r:   zArrayOverlapLayer.__getitem__   s    zz#r   c                ,    t        | j                        S r   )iterr~   rw   s    r   __iter__zArrayOverlapLayer.__iter__   s    DJJr   c                ,    t        | j                        S r   )r1   r~   rw   s    r   __len__zArrayOverlapLayer.__len__   s    4::r   c                    t        | d      S )Nrz   )r{   rw   s    r   is_materializedz!ArrayOverlapLayer.is_materialized   s    t^,,r   c                "    | j                         S r   )keysrw   s    r   get_output_keysz!ArrayOverlapLayer.get_output_keys   s    yy{r   c                    | j                   | j                   S | j                  | j                  | j                  cfd        x| _         }|S )Nc                     sfgS t        |       }|dz   t              k(  r%t        |         D cg c]  }f| z   |fz    }}|S t        |         D cg c]  } | |fz     }}|S c c}w c c}w )NrP   )r1   range)r   indrB   resultr*   r   rp   r,   s       r   r   z*ArrayOverlapLayer._dask_keys.<locals>.keys   s    y d)CQw#i.(9>y~9NOA4'D.A4/OO M 9>in8MN1$.NNM PNs   A-A2)rs   rp   r*   r,   )r   r   r*   r   rp   r,   s     @@@@r   
_dask_keyszArrayOverlapLayer._dask_keys   sR    ($$$"&))T[[$..fi	 &*V+Fr   c           	        | j                   }| j                  }| j                  }| j                         }d| j                  z   }d| j                  z   }|rt        d      }nddlm} t        t        t        |            }	t        j                  t        |	|      }
t        j                  |t         t        |
      t        d       t        t               t        j"                  t              }i }i }|D ]O  }t%        |f|z   |      }|du r|f|z   |k7  r
|||f|z   <   +|f|z   ||f|z   <   |g |
d	|z   |
      ||f|z   <   Q t        j&                  ||      }|S )z/Construct graph for a simple overlap operation.zgetitem-zoverlap-z"dask.array.core.concatenate_shapedr   )concatenate_shaped)dimsrq   c                    | d   S )Nr   r&   )as    r   <lambda>z4ArrayOverlapLayer._construct_graph.<locals>.<lambda>   s
    !A$ r   Fr   rv   )rq   r*   rp   r   rr   r   dask.array.corer   listr   r1   	functoolspartial_expand_keys_around_centertoolzpiper   concatfractional_slicemerge)r   deserializingrq   r*   rp   	dask_keysgetitem_nameoverlap_namer   r   expand_key2interior_keysinterior_slicesoverlap_blocksk
frac_slicer}   s                    r   r|   z"ArrayOverlapLayer._construct_graph   sj   yyyyOO%	!DJJ.!DJJ. "44"
 ;CV$%''&T

 

LLL
  	A)4'A+t<JU"w{j(7A! 348<w{! 34&7!'A+LA723	 kk/>:
r   )F)r"   r#   r$   r%   r   rx   propertyr~   r:   r   r   r   r   r   r|   rY   rZ   s   @r   rn   rn   |   sI    ! 7 ! ! -&3r   rn   c                &   d fd}g }t        | dd       D ]Z  \  }} j                  |d            }d}	|dkD  r|d   dk7  r|	dz  }	||   dz
  k  r|d   dk7  r|	dz  }	|j                  |	       \ d t        | dd       D cg c]:  \  }} j                  |d            r |||j                  |d            n|g< }
}}||gg|
z   }
t        t	        |
       }t        fdt        |      D              }||fS c c}}w )a  Get all neighboring keys around center

    Parameters
    ----------
    k: Key
        The key around which to generate new keys
    dims: Sequence[int]
        The number of chunks in each dimension
    name: Option[str]
        The name to include in the output keys, or none to include no name
    axes: Dict[int, int]
        The axes active in the expansion.  We don't expand on non-active axes

    Examples
    --------
    >>> _expand_keys_around_center(('x', 2, 3), dims=[5, 5], name='y', axes={0: 1, 1: 1})  # noqa: E501 # doctest: +NORMALIZE_WHITESPACE
    ([('y', 1.1, 2.1), ('y', 1.1, 3), ('y', 1.1, 3.9), ('y',   2, 2.1), ('y',   2, 3), ('y',   2, 3.9), ('y', 2.9, 2.1), ('y', 2.9, 3), ('y', 2.9, 3.9)], (3, 3))

    >>> _expand_keys_around_center(('x', 0, 4), dims=[5, 5], name='y', axes={0: 1, 1: 1})  # noqa: E501 # doctest: +NORMALIZE_WHITESPACE
    ([('y',   0, 3.1), ('y',   0,   4), ('y', 0.9, 3.1), ('y', 0.9,   4)], (2, 2))
    c                .    t        | t              s| | f} | S r   )r^   r5   depths    r   convert_depthz1_expand_keys_around_center.<locals>.convert_depth  s    %'ENEr   c                     |      }g }|dz
  dkD  r|d   dk7  r|j                  |dz
         |j                  |       |dz   |    dz
  k  r|d   dk7  r|j                  |dz          |S )Ng?r   rP   )append)rB   r   r   rvr   r   s       r   indsz(_expand_keys_around_center.<locals>.inds  su    e$9q=U1X]IIcCi 
		#9tAw{"uQx1}IIcCi 	r   rP   Nr   c                P    t        | t              rt        d | D              S | dk7  S )Nc              3  &   K   | ]	  }|d k7    yw)r   Nr&   )r2   xs     r   r4   zC_expand_keys_around_center.<locals>._valid_depth.<locals>.<genexpr>2  s     -!qAv-s   r   )r^   r5   anyr   s    r   _valid_depthz0_expand_keys_around_center.<locals>._valid_depth0  s'    eU#-u---A:r   c              3  \   K   | ]#  \  }} j                  |d             r|nd % yw)r   rP   N)get)r2   rB   dr   rq   s      r   r4   z-_expand_keys_around_center.<locals>.<genexpr>=  s0      59Q\$((1a.)q0s   ),)	enumerater   r   r   r   r5   )r   r   rp   rq   r   shaperB   r   r   numr   seqshape2r   r   s    ` `         @@r   r   r      sB   .
 EAabE" 3dhhq!n-7uQx1}1HCa1qQ1HCS  !"&As )5TXXa^(DQTXXa^$3%OD  x$
w~
C =Fu=M F ;s   ?Dc                $   | d   ft        d | dd D              z   }g }t        t        | dd |dd             D ]  \  }\  }}|j                  |d      }t	        |t               r|d   }|d   }	n|}|}	||k(  r|j                  t        ddd             \||k  r|	r|j                  t        d|	             ||kD  r|r|j                  t        | d              y t        |      }t        d |D              r| S t        j                  ||fS )a  

    >>> fractional_slice(('x', 5.1), {0: 2})
    (<built-in function getitem>, ('x', 5), (slice(-2, None, None),))

    >>> fractional_slice(('x', 3, 5.1), {0: 2, 1: 3})
    (<built-in function getitem>, ('x', 3, 5), (slice(None, None, None), slice(-3, None, None)))

    >>> fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3})
    (<built-in function getitem>, ('x', 3, 5), (slice(0, 2, None), slice(-3, None, None)))
    r   c              3  D   K   | ]  }t        t        |              y wr   )intround)r2   rB   s     r   r4   z#fractional_slice.<locals>.<genexpr>O  s      A1U1X As    rP   NFc              3  <   K   | ]  }|t        d d d       k(    y wr   rT   )r2   r   s     r   r4   z#fractional_slice.<locals>.<genexpr>e  s     
;c3%dD))
;rR   )
r5   r   rC   r   r^   r   rU   alloperatorgetitem)
taskrq   roundedindexrB   trr   
left_depthright_depths
             r   r   r   C  s    Awj5 AQR AAAGEs48WQR[9: 	6AqAeU#qJ(KJK6LLtT401U{LLq+./UzLL
{D12!" %LE

;U
;;  '511r   c                  H     e Zd ZdZ	 	 	 	 d fd	Zed        Zd Zd Z xZ	S )DataFrameIOLayera  DataFrame-based Blockwise Layer with IO

    Parameters
    ----------
    name : str
        Name to use for the constructed layer.
    columns : str, list or None
        Field name(s) to read in as columns in the output.
    inputs : list or BlockwiseDep
        List of arguments to be passed to ``io_func`` so
        that the materialized task to produce partition ``i``
        will be: ``(<io_func>, inputs[i])``.  Note that each
        element of ``inputs`` is typically a tuple of arguments.
    io_func : callable
        A callable function that takes in a single tuple
        of arguments, and outputs a DataFrame partition.
        Column projection will be supported for functions
        that satisfy the ``DataFrameIOFunction`` protocol.
    label : str (optional)
        String to use as a prefix in the place-holder collection
        name. If nothing is specified (default), "subset-" will
        be used.
    produces_tasks : bool (optional)
        Whether one or more elements of `inputs` is expected to
        contain a nested task. This argument in only used for
        serialization purposes, and will be deprecated in the
        future. Default is False.
    creation_info: dict (optional)
        Dictionary containing the callable function ('func'),
        positional arguments ('args'), and key-word arguments
        ('kwargs') used to produce the dask collection with
        this underlying ``DataFrameIOLayer``.
    annotations: dict (optional)
        Layer annotations to pass through to Blockwise.
    c	           	        || _         || _        || _        || _        || _        || _        || _        || _        t        |t              s@t        t        | j                        D 	
ci c]	  \  }	}
|	f|
 c}
}	| j
                        }n|}t        | j                   |t        t        d                  }t        | A  | j                   d||dfgi |       y c c}
}	w )N)r.   r   rB   )outputoutput_indicesr   indicesr,   r   )rp   _columnsinputsio_funclabelr.   r   creation_infor^   r   r   r   r   r   r   rL   r   )r   rp   columnsr   r   r   r.   r   r   rB   inp
io_arg_mapr   rM   s                r   r   zDataFrameIOLayer.__init__  s     	
,&*&,/))24;;)?@vq#!s@#22J
  J DIIw0B(CD99 #&'# 	 	
 As   'C
c                    | j                   S )z(Current column projection for this layer)r   rw   s    r   r   zDataFrameIOLayer.columns  s     }}r   c           	        ddl m} t        |      }| j                  $t	        | j                        j                  |      rt        | j                  |      r| j                  j                  |      }n| j                  }t        | j                  xs ddz   t        | j                  |      z   || j                  || j                  | j                  | j                        }|S | S )zProduce a column projection for this IO layer.
        Given a list of required output columns, this method
        returns the projected layer.
        r   )DataFrameIOFunctionsubset-)r   r.   r   )dask.dataframe.io.utilsr   r   r   set
issupersetr^   r   project_columnsr   r   r	   rp   r   r.   r   )r   r   r   r   layers        r   r   z DataFrameIOLayer.project_columns  s    
 	@w-<<3t||#4#?#?#H $,,(;<,,66w?,,$'x3.$))W1MMjj#22 ,,E L Kr   c                v    dj                  | j                  t        | j                        | j                        S )Nz3DataFrameIOLayer<name='{}', n_parts={}, columns={}>)formatrp   r1   r   r   rw   s    r   rx   zDataFrameIOLayer.__repr__  s-    DKKIIs4;;'
 	
r   )NFNN)
r"   r#   r$   r%   r   r   r   r   rx   rY   rZ   s   @r   r   r   r  s;    "T &
P  >
r   r   )NN).
__future__r   r   r   	itertoolsr   typingr   tlzr   tlz.curriedr   dask._task_specr   r   	dask.baser	   dask.blockwiser
   r   r   r   	dask.corer   dask.highlevelgraphr   dask.tokenizer   
dask.utilsr   numpynpr   r(   r?   rE   r\   r`   registerrh   rj   rl   rn   r   r   r   r&   r   r   <module>r      s   "        )  U U  % ) $@ @.K K&E* E3% 3'  &   -(' )' /*) +) .)4 *4A AH@F%2^t
y t
r   