
    bi:                    t   U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlm Z  d dl!m"Z"  G d de#      Z$d.dZ% e
jL                         Z'i a(de)d<    ed      Z*de)d<   dd	 	 	 	 	 	 	 d/dZ+d0dZ,e-e.e/e0 e1d      e2e3 e1e4      ejj                  ejl                  ejn                  ej                  ejp                  ejr                  fZ: e"       Z;e;jy                  e:e       e;jy                  ejz                  e>f      d        Z?e;jy                  e      d        Z@e;jy                  eA      d        ZBd1dZCe;jy                  eDeEf      d        ZFe;jy                  e      d        ZGe;jy                  e      d        ZHe;jy                  eef      d        ZIe;jy                  ej                  ej                  f      d        ZLe;jy                  ej                        d         ZNe;jy                  eO      d!        ZP eA       ZQd2d"ZRd3d#ZSd$ ZTe;j                  d%      d&        ZVe;j                  d'      d(        ZWe;j                  d)      d*        ZXe;j                  d+      d,        ZYd4d-ZZy)5    )annotationsN)OrderedDict)Iterable)
ContextVar)partial)curryidentity)Compose)config)literalhash_buffer_hex)Dispatchc                      e Zd Zy)TokenizationErrorN)__name__
__module____qualname__     H/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/tokenize.pyr   r      s    r   r   c                     t        |       }|r$|t        t        |j                                     f}t        j                  t        |      j                         d      j                         S )NF)usedforsecurity)_normalize_seq_funcsorteditemshashlibmd5strencode	hexdigest)argskwargstokens      r   	_tokenizer%   !   sS    '-E*6&,,.+ABB ;;s5z((*EBLLNNr   zdict[int, tuple[int, object]]_SEEN_ENSURE_DETERMINISTICzContextVar[bool | None])ensure_deterministicc                H   t         5  t        i c}ad}	 t        j                          	 t        |i ||rt        j                  |       |acddd       S # t        $ r t        j                  |       }Y Nw xY w# |rt        j                  |       |aw xY w# 1 sw Y   yxY w)a  Deterministic token

    >>> tokenize([1, 2, '3'])  # doctest: +SKIP
    '06961e8de572e73c2e74b51348177918'

    >>> tokenize('Hello') == tokenize('Hello')
    True

    Parameters
    ----------
    args, kwargs:
        objects to tokenize
    ensure_deterministic: bool, optional
        If True, raise TokenizationError if the objects cannot be deterministically
        tokenized, e.g. two identical objects will return different tokens.
        Defaults to the `tokenize.ensure-deterministic` configuration parameter.
    N)tokenize_lockr&   r'   getLookupErrorsetr%   reset)r(   r"   r#   seen_beforer$   s        r   tokenizer0   /   s    * 
  "BU	D!%%'	 d-f-%++E2E   
  	D)--.BCE	D
 %++E2E   s>   BA
A:BA74B6A77B:BBB!c                    	 t         j                         }|s|!t        j                  d      rt	        |       y y # t        $ r d }Y 3w xY w)Nztokenize.ensure-deterministic)r'   r+   r,   r   r   )msgvals     r   _maybe_raise_nondeterministicr4   S   sS    #'') ckfjj)HI$$ Jk  s   < A
	A
c                   t         5  t        |       t        v r dt        t        |          d   fcd d d        S t        t              | ft        t        |       <   	 dt	        t        | j                         d             ft        j                  t        |       d        cd d d        S # t        j                  t        |       d        w xY w# 1 sw Y   y xY w)N__seenr   dictc                    t        | d         S Nr   r   )kvs    r   <lambda>z normalize_dict.<locals>.<lambda>{   s    RU r   key)r*   idr&   lenr   r   r   popds    r   normalize_dictrD   s   s    	 	#a5E>U2a5\!_,	# 	# 5z1}be	#.qwwy&;<  IIbeT"	# 	# IIbeT"	# 	#s(   (C!C&B*C*!CCCc                \    t        t        |       t        | j                               f      S N)r   typelistr   rB   s    r   normalize_ordered_dictrI      s     Qaggi9::r   c                :    dt        t        | t                    fS )Nr-   r=   )r   r   r   ss    r   normalize_setrM      s    
 %fQC&8999r   c                Z   d }t         5  t        |       t        v r dt        t        |          d   fcd d d        S t        t              | ft        t        |       <   	 t	        t        ||             t        t        |       = cd d d        S # t        t        |       = w xY w# 1 sw Y   y xY w)Nc                <    t        | t              r| S t        |       S rF   )
isinstance_IDENTITY_DISPATCHnormalize_token)items    r   _inner_normalize_tokenz3_normalize_seq_func.<locals>._inner_normalize_token   s    d./Kt$$r   r6   r   )r*   r?   r&   r@   tuplemap)seqrT   s     r   r   r      s    % 
 c7eU2c7^A..  USbg	3S9:bg  bg s(   (B!!B!B2B!BB!!B*c                B    t        |       j                  t        |       fS rF   )rG   r   r   )rW   s    r   normalize_seqrY      s    923777r   c                &    dt         |              fS )Nr   )rR   )lits    r   normalize_literalr\      s    oce,,,r   c                H    t        | j                  f| j                  z         S rF   )r   firstfuncsfuncs    r   normalize_composerb      s    

}tzz9::r   c                Z    t        | j                  | j                  | j                  f      S rF   )r   ra   r"   keywordsr`   s    r   normalize_partialre      s     		499dmmDEEr   c                D    t        | j                        | j                  fS rF   )rR   __self__r   )meths    r   normalize_bound_methodri      s    4==)4==88r   c                v    t        | dd       }| t        j                  |      st        |       S t	        |       S )Nrg   )getattrinspectismoduleri   normalize_object)ra   selfs     r   $normalize_builtin_function_or_methodrp      s;     4T*D 0 0 6%d++%%r   c                   t        | dd       }|t        | t              s |       S t        |       t        u rt	        |       S t        | t              rt        j                  |        t        j                  |       rt        | t              st        |       S 	 t        |       S # t        $ r0 t        d| d       t        j                         j                  cY S w xY w)N__dask_tokenize__zObject zr cannot be deterministically hashed. This likely indicates that the object cannot be serialized deterministically.)rk   rP   rG   object_normalize_pure_objectcopyreg
_slotnamesdataclassesis_dataclass_normalize_dataclass_normalize_pickle	Exceptionr4   uuiduuid4hex)omethods     r   rn   rn      s    Q+T2F*Q"5xAw&%a((!T1":a+>#A&&  ##  %aU P P	
 zz| s   
B$ $6CCc                \    t        d       t        j                  |        dt        |       fS )Nzobject() cannot be deterministically hashed. See https://docs.dask.org/en/latest/custom-collections.html#implementing-deterministic-hashing for more information.rs   )r4   _seen_objectsaddr?   )r   s    r   rt   rt      s-    !	  aRU?r   c                   g }d }d }t        d      D ]  }|j                          	 t        j                  | d|j                        }d|v r2|j                          t        j                  | d|j                        }t        j                  ||       t        |      }|r	|r||k(  r n|} t        d       |(t        d       t        t        j                               }||D cg c]  }t        |       c}fS # t        $ rj |j                          	 t        j                  | d|j                        }t        j                  ||       t        |      }n# t        $ r Y Y  w xY wY w xY wc c}w )N      )protocolbuffer_callbacks   __main__)buffersz$Failed to tokenize deterministically)rangeclearpickledumpsappendcloudpickleloadsr   r{   r4   intr|   r}   )r   r   pikpik2_outbufs          r   rz   rz      sN   )+GCD1X N	,,q1gnnMCc!!''Aw~~VLLg."3'D 4C4K)N, 	&&LM
{%&LM$**,9#%999!  	MMO!''Aw~~VS'2&s+  	  :s=   A:C?)E5?E2AEE2	E,'E2+E,,E21E2c                >   t        j                  |       D cg c].  }|j                  t        t	        | |j                  d             f0 }}| j
                  }|j                  D cg c]  }|t	        ||      f }}t        t        |             ||fS c c}w c c}w rF   )	rw   fieldsnamerR   rk   __dataclass_params__	__slots__rn   rG   )objfieldr   paramsattrs        r   ry   ry   
  s     !'', 
_WS%**d%CDEF  %%F8>8H8HItWVT*+IFIDI&66
 Js   3B'Bpandasc                    dd l ddlm t        j	                  j
                        d        } t        j	                  j                        fd       }t        j	                  j                        d        }t        j	                  j                        d        }t        j	                  j                  j                        t        j	                  j                  j                        t        j	                  j                  j                        d                      }t        j	                  j                  j                        d        }t        j	                  j                        d	        }t        j	                  j                        d
        }t        j	                  j                  j                         d        }t        j	                  j"                  j$                  j&                        d        }t        j	                  j"                  j(                  j*                        d        }t        j	                  j"                  j$                  j,                        d        }	t        j	                  t/        j0                              fd       }
t        j	                  j2                  j4                        d        }y )Nr   )PANDAS_GE_210c                    t        |       | j                  | j                  | j                  | j                  | j
                  fS rF   )rG   startstopstepdtyper   )xs    r   normalize_range_indexz.register_pandas.<locals>.normalize_range_index  s-    Aw!&&@@r   c                "   | j                   }t        |j                  j                        rBdd l}r+|j                  |j                  g      j                         }n|j                  |      }t        |       | j                  t        |      fS r9   )arrayrP   arraysArrowExtensionArraypyarrowchunked_array	_pa_arraycombine_chunksrG   r   rR   )indvaluespar   pds      r   normalize_indexz(register_pandas.<locals>.normalize_index   sr    fbii;;<  ))6+;+;*<=LLN&)Cy#((OF$;;;r   c                    | j                   }| j                  g| j                  D cg c]  }t        |       c}z   |D cg c]  }t        |       c}z   S c c}w c c}w rF   )codesr   levelsrR   )r   r   r   s      r   r   z(register_pandas.<locals>.normalize_index3  sS    		XXJ+.::6aq!67+01aq!12	
61s   AAc                V    t        | j                        t        | j                        gS rF   )rR   r   r   )cats    r   normalize_categoricalz.register_pandas.<locals>.normalize_categorical<  s    		*OCII,FGGr   c                V    t        | j                        t        | j                        gS rF   )rR   asi8r   arrs    r   normalize_period_arrayz/register_pandas.<locals>.normalize_period_array@  s!      )?399+EFFr   c                ~    t        | j                        t        | j                        t        | j                        gS rF   )rR   leftrightclosedr   s    r   normalize_interval_arrayz1register_pandas.<locals>.normalize_interval_arrayF  s2     CHH%CII&CJJ'
 	
r   c                    | j                   | j                  t        | j                        t        | j                        gS rF   )r   r   rR   _valuesindexrK   s    r   normalize_seriesz)register_pandas.<locals>.normalize_seriesN  s4     FFGGAII&AGG$	
 	
r   c                    | j                   }t        |j                        | j                  | j                  gz   }t        t        t        |            S rF   )_mgrrH   r   columnsr   rV   rR   )dfmgrdatas      r   normalize_dataframez,register_pandas.<locals>.normalize_dataframeW  s=    ggCJJ2::rxx"88C.//r   c                    	 t        |       t        | j                        fS # t        $ r# t        |       t        | j                        fcY S w xY wrF   )rG   rR   r   AttributeError_datar   s    r   normalize_extension_arrayz2register_pandas.<locals>.normalize_extension_array]  sF    	;Is}}=>> 	;Isyy9::	;s   " )AAc                >    dd l }t        |j                  |             S r9   )numpyrR   asarray)r   nps     r   r   z2register_pandas.<locals>.normalize_extension_arrayd  s    rzz#//r   c                V    t        | j                        t        | j                        gS rF   )rR   
categoriesorderedr   s    r   normalize_categorical_dtypez4register_pandas.<locals>.normalize_categorical_dtypek  s!     0 01?5==3QRRr   c                ,    t        | j                        S rF   )rR   r   r   s    r   normalize_period_dtypez/register_pandas.<locals>.normalize_period_dtypeo  s    uzz**r   c                    j                   S rF   )NA)nar   s    r   normalize_naz%register_pandas.<locals>.normalize_nas  s    uur   c                    | j                   S rF   )freqstr)offsets    r   normalize_offsetz)register_pandas.<locals>.normalize_offsetw  s    ~~r   )r   dask._pandas_compatr   rR   register
RangeIndexIndex
MultiIndexCategoricalr   PeriodArrayDatetimeArrayTimedeltaArrayIntervalArraySeries	DataFramer   api
extensionsExtensionArraytypesCategoricalDtypeExtensionDtyperG   r   offsets
BaseOffset)r   r   r   r   r   r   r   r   r   r   r   r   r   r   s               @@r   register_pandasr     su    2bmm,A -A bhh'< (<$ bmm,
 -
 bnn-H .H bii334bii556bii667G 8 7 5G bii556
 7
 bii(
 )
 bll+0 ,0
 bii;;<; =; bff//>>?0 @0 bffll;;<S =S bff//>>?+ @+ d255k* + bjj334 5r   numbac                 ~    dd l } t        j                  | j                  j                  j
                        d        }y )Nr   c                T    t        | j                         | j                         f      S rF   )rR   _reduce_class_reduce_states)r   s    r   normalize_numba_ufuncz-register_numba.<locals>.normalize_numba_ufunc  s$     1 1 3S5G5G5IJKKr   )r   rR   r   core	serializeReduceMixin)r   r  s     r   register_numbar  |  s5    ejj22>>?L @Lr   r   c                 ~   dd l } t        j                  | j                        d        }t        j                  | j                        d        }t        j                  | j
                        d        }t        j                  | j                        d        }t        j                  | j                        d        }y )Nr   c                0    t        j                  | d      S )N   )r   )r   r   dts    r   normalize_datatypez,register_pyarrow.<locals>.normalize_datatype  s    ||B++r   c                X    dt        | j                        t        | j                        fS )Nzpa.Table)rR   schemar   r  s    r   normalize_tablez)register_pyarrow.<locals>.normalize_table  s)     BII&BJJ'
 	
r   c                X    dt        | j                        t        | j                        fS )Nzpa.ChunkedArray)rR   rG   chunksr   s    r   normalize_chunked_arrayz1register_pyarrow.<locals>.normalize_chunked_array  s)     CHH%CJJ'
 	
r   c                    | j                         }| j                         }t        |      r|d   | j                  dk(  rd |d<   dt        | j                        t        |      fS )Nr   zpa.Array)r   r@   
null_countrR   rG   )r   r   s     r   normalize_arrayz)register_pyarrow.<locals>.normalize_array  sa    ++- ++-w<GAJ2s~~7JGAJ CHH%G$
 	
r   c                    dt        |       fS )Nz	pa.Bufferr   )r   s    r   normalize_bufferz*register_pyarrow.<locals>.normalize_buffer  s    _S122r   )r   rR   r   DataTypeTableChunkedArrayArrayBuffer)r   r  r  r  r  r  s         r   register_pyarrowr    s    bkk*, +, bhh'
 (
 boo.
 /
 bhh'
 (
$ bii(3 )3r   r   c                 :   dd l t        j                  j                        d        } t        j                  j                        fd       }t        j                  j
                        d        }t        j                  j                        d        }y )Nr   c                   | j                   s| j                         | j                  fS | j                  j                  r8	 	 t	        dj                  | j                        j                  dd            }n+	 t	        | j                  d      j                  d            }|| j                  | j                   fS # t        $ r' t	        dj                  | j                              }Y tw xY w# t        t        f$ r t        |       cY S w xY w# t        t        t        f$ r; t	        | j!                         j                  d      j                  d            }Y w xY w)	N-zutf-8surrogatepass)encodingerrors   -K)orderi1)shaperS   r   	hasobjectr   joinflatr    UnicodeDecodeError	TypeErrorrn   ravelviewBufferErrorr   
ValueErrorcopy)r   r   s     r   r  z'register_numpy.<locals>.normalize_array  s*   wwFFHagg&&77+	>*(//%,_ 0 DM&qwwSw'9'>'>t'DE aggqww'' * >*499QVV+<=D> 12 +'**+
  < M&qvvx~~C~'@'E'Ed'KLMs=   5B; 8*D ;-C+(C. *C++C. .D
DAEEc                8    t        j                  |             S rF   )r   ascontiguousarray)mmr   s    r   normalize_mmapz&register_numpy.<locals>.normalize_mmap  s    r33B788r   c                    	 t        |       S # t        $ r0 t        d| d       t        j                         j
                  cY S w xY w)NzCannot tokenize numpy ufunc z. Please use functions of the dask.array.ufunc module instead. See also https://docs.dask.org/en/latest/array-numpy-compatibility.html)rz   r{   r4   r|   r}   r~   r`   s    r   normalize_ufuncz'register_numpy.<locals>.normalize_ufunc  sP    	$$T** 	$).th 7Q Q
 ::<###	$s   
 6AAc                    | j                   S rF   r:   r   s    r   normalize_dtypez'register_numpy.<locals>.normalize_dtype  s    yyr   )r   rR   r   ndarraymemmapufuncr   )r  r7  r9  r;  r   s       @r   register_numpyr?    s    bjj)( *(0 bii(9 )9 bhh'	$ (	$ bhh' (r   c                     t        | ddi|S )Nr(   T)r0   )r"   r#   s     r   _tokenize_deterministicrA    s    T????r   )r"   rs   r#   rs   returnr   )r"   rs   r(   zbool | Noner#   rs   rB  r   )r2   r   rB  None)rW   zIterable[object]rB  ztuple[object, ...])r   rs   rB  ztuple[str, int])r   rs   rB  rU   )rB  r   )[
__future__r   ru   rw   datetimedecimalr   rl   pathlibr   	threadingr   r|   collectionsr   collections.abcr   contextvarsr   	functoolsr   r   tlzr   r	   tlz.functoolzr
   daskr   	dask.corer   dask.hashingr   
dask.utilsr   RuntimeErrorr   r%   RLockr*   r&   __annotations__r'   r0   r4   r   floatr   bytesrG   slicecomplexEllipsisDecimaldatetime	timedeltaPurePathrQ   rR   r   MappingProxyTyper7   rD   rI   r-   rM   r   rU   rH   rY   r\   rb   re   
MethodTypeMethodWrapperTyperi   BuiltinFunctionTyperp   rs   rn   r   rt   rz   ry   register_lazyr   r  r  r?  rA  r   r   r   <module>re     s;   "            # $ "    !   ( 	 	O  	!')$ )1;<S1T . T 8<! ! )4! GM! ! H% 		J	NOOMMMM   *    511489
# :
# +&; '; #: :" 5$-(8 )8 '"- #- '"; #; 7E*+F ,F 5++U-D-DEF9 G9 %334& 5& &!  " 0 :B7 x(c )cL w'L (L y),3 *,3^ w'. (.b@r   