
    bi                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZmZmZ d dlmZ d dlZd dlZd dlmZ d dlmZmZmZ d dlZd dlmZ d d	l m!Z!m"Z" d d
l#m$Z$m%Z%m&Z& d dl'm(Z( d dl)m*Z*  e*d      Z+ e*d      Z, e*d      Z- e*d      Z. e*d      Z/ e*d      Z0 e*d      Z1 ejd                  d      d        Z3d Z4d Z5d Z6ejn                  jq                  d      d        Z9ejn                  jq                  d      d        Z:ejn                  jq                  d      d        Z;ejn                  jq                  d      d        Z<ejn                  jq                  d      d        Z=ejn                  jq                  d      d         Z>ejn                  jq                  d      d!        Z?ejn                  jq                  d      d"        Z@ejn                  jq                  d      d#        ZAejn                  jq                  d      d$        ZBejn                  jq                  d      d%        ZCejn                  jq                  d      d&        ZDejn                  jq                  d      d'        ZEejn                  jq                  d      d(        ZFd) ZGd* ZHd+ ZId, ZJd- ZKd.aLd/ ZM G d0 d1      ZNeOePj                  d2j                  eNj                   eN       j                  eNj                   eN       j                  eNj                  gZUejn                  j                  d3 eM             d4        ZWejn                  j                  d3eU      d5        ZXd6 ZYejn                  j                  d78      ejn                  j                  d9d:dg      d;               Z[d< Z\dd=Z]dd>Z^d? Z_d@ Z`dA Zaejn                  jq                  dB      dC        Zbejn                  jq                  dB      dD        Zcejn                  jq                  dB      dE        Zdejn                  jq                  dB      dF        Zeejn                  jq                  dB      dG        Zfejn                  jq                  dH      dI        Zgejn                  jq                  dB      dJ        Zhejn                  jq                  dB      dK        Ziejn                  jq                  dB      dL        ZjdM ZkdN ZldO ZmdP ZndQ ZodR ZpdS ZqdT Zr G dU dV      Zs G dW dXes      ZtdY ZudZ Zvd[ Zwd\ Zxd] Zyd^ Zzd_ Z{d` Z|da Z}db Z~ejn                  j                  dceeeeg      dd        Zej                    G de df             Zej                    G dg dh             Zej                    G di dj             Z G dk dl      Zdm Zejn                  jq                  e edn      k\  do8      dp        Zejn                  j                  dqg dr      ds        Zejn                  jq                  d      dt        Zejn                  jq                  d      du        Zejn                  j                  dvd.ddwdxdydzdzg d{i dePeg      d|        Zd} Zejn                  jq                  d      ejn                  j                  d~      d               Zejn                  jq                  d      ejn                  j                  dd      d               Zd Zejn                  j                  dqg d      d        Zd Zd Zd Zd Zd Zd Zejn                  jq                  d      d        Zejn                  j                  dd ej2                  dejn                  jq                  d            g      d        Zd Zejn                  jq                  d      d        Zejn                  jq                  d      d        Zejn                  jq                  d      d        Zejn                  jq                  d      d        Zejn                  jq                  d      d        Ze1s G d d      Z e       Z1e1jE                  d      d        Ze1jE                  dd      d        Ze1jK                  d      d        Ze1jK                  dd      d        Ze1jQ                  dgd      d        ZeeeeegZejn                  jq                  d      ejn                  j                  d3e      d               Zejn                  jq                  d      d        Zejn                  jq                  d      d        Zejn                  jq                  d      d        Zejn                  jq                  dB      d        Z G d d      Zd Zd Zd Zd Zy)    )annotationsN)ThreadPoolExecutor)EnumFlagIntEnumIntFlag)Union)Version)composecurrypartial)
PY_VERSION)flattenliteral)TokenizationErrornormalize_tokentokenize)tmpfile)import_or_nonez
dask.arrayzdask.dataframenumpyzscipy.sparsepyarrowpandasnumbaT)autousec               #  ,  K   ddl m} m} |rJ t        j                  t
              5  | j                          ddd       d |rJ t        j                  t
              5  | j                          ddd       y# 1 sw Y   DxY w# 1 sw Y   yxY ww)zBTest that tokenize() and normalize_token() properly clean up stater   _ENSURE_DETERMINISTIC_SEENN)dask.tokenizer   r   pytestraisesLookupErrorgetr   s     S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/tests/test_tokenize.pycheck_clean_stater%   &   s|      ;9	{	# $!!#$	9	{	# $!!#$ $	$ $$ $s3   &BA<)B"B3	B<BBBBc                 0   t         j                  j                  ddi      5  t        | i |}t        | i |}||k(  s	J | |f       t	        j
                  t	        j                  | |f            \  }}t	        j
                  t	        j                  | |f            \  }}t	        j
                  t	        j                  ||f            \  }}t        |i |}||k(  s	J | |f       t        |i |}	||	k(  s	J | |f       	 d d d        |S # 1 sw Y   S xY w)Ntokenize.ensure-deterministicT)daskconfigsetr   cloudpickleloadsdumps)
argskwargsbeforeafterargs2kwargs2args3kwargs3tok2tok3s
             r$   check_tokenizer8   4   s   	94@	A ,4*6* $)&).v. %**;+<+<dF^+LMw$**;+<+<dF^+LMw$**;+<+<eW=M+NOw*'*v~-f~-~*'*t|+dF^+|/,6 M7,6 Ms   CDDc                 z   t        d       t        d      t        d      k(  sJ  G fdd              } t        j                  t              5  t        |        d d d         G d d      } |       }t        j                  t              5  t        |       d d d        y # 1 sw Y   HxY w# 1 sw Y   y xY w)N{   c                  $    e Zd Zd Z fdZd Zy)test_check_tokenize.<locals>.Ac                6    t        j                          | _        y N)randomtokselfs    r$   __init__z'test_check_tokenize.<locals>.A.__init__[   s    }}DH    c                    dfS N rG   )rB   As    r$   
__reduce__z)test_check_tokenize.<locals>.A.__reduce__^   s    b5LrD   c                    | j                   S r>   )r@   rA   s    r$   __dask_tokenize__z0test_check_tokenize.<locals>.A.__dask_tokenize__a   s    88OrD   N)__name__
__module____qualname__rC   rI   rK   )rH   s   r$   rH   r<   Z   s    	'		rD   rH   c                      e Zd Zd Zy)test_check_tokenize.<locals>.Bc                *    t        j                          S r>   )r?   rA   s    r$   rK   z0test_check_tokenize.<locals>.B.__dask_tokenize__j   s    ==?"rD   NrL   rM   rN   rK   rG   rD   r$   BrP   i   s    	#rD   rS   )r8   r   r    r!   AssertionError)arS   brH   s      @r$   test_check_tokenizerW   S   s    3C=N3////  	
A	~	& q# # 	
A	~	& q   s   B%B1%B.1B:c                 >    d} t        t        |       t              sJ y )N         )
isinstancer   strrU   s    r$   test_tokenizer`   r   s    Ahqk3'''rD   znot npc                 .   t        t        j                  d            t        d      k7  sJ t        t        j                  d            t        t        j                  d            k7  sJ t        t        j                  d            t        t        j	                  d            k7  sJ t        t        j                  d            t        d      k7  sJ t        t        j                  d            t        t        j                  d            k7  sJ y )NrZ   1)r8   npint64int32uint32float64rG   rD   r$   test_tokenize_scalarrh   w   s    "((1+&.*;;;;"((1+&.!*EEEE"((1+&.1*FFFF"((1+&.*===="((1+&.A*GGGGrD   c                     t        t        j                  j                  d      j	                  d            t        t        j                  j                  d      j	                  d            k(  sJ y )N    )r8   rc   r?   RandomStaterandom_samplerG   rD   r$   .test_tokenize_numpy_array_consistent_on_valuesrn      s[    
		d#11$7			--d3AA$G	HI I IrD   c                 t    t        t        j                  j                  d      j                  d             y )N   i2dtype)r8   rc   r?   astyperG   rD   r$   /test_tokenize_numpy_array_supports_uneven_sizesru      s'    299##A&--D-9:rD   c                     t         j                  j                  d      } t        | d d d         t        | d d d         k7  sJ y )N   r[   r\   )rc   r?   r8   )arrs    r$   'test_tokenize_discontiguous_numpy_arrayry      s>    
))

1
C#cc(#~c#A#h'????rD   c                 F    t        t        j                  dgd             y )Nz2000-01-01T12:00:00zM8[ns]rr   r8   rc   arrayrG   rD   r$   test_tokenize_numpy_datetimer}      s    2882388DErD   c                 (   t        t        j                  dd            t        t        j                  dd            k(  sJ t        t        j                  dgddg      d         t        t        j                  dgddg      d         k(  sJ y )N      ?f8rr   rZ   r[   )rU   i4)rV   i8r   r{   rG   rD   r$   test_tokenize_numpy_scalarr      s    "((3d(34
D!9    
&+{!;<Q?	&+{1KLQO	PQ Q QrD   c                     t         j                  dd i      5  t        t         j                  d            t        t         j                  d            k7  sJ 	 d d d        y # 1 sw Y   y xY w)Nallc                     yNfoorG   xs    r$   <lambda>z7test_tokenize_numpy_scalar_string_rep.<locals>.<lambda>   s    rD   )	formatterrZ   r[   )rc   printoptionsr8   r|   rG   rD   r$   %test_tokenize_numpy_scalar_string_repr      sX     
E?#;	< Jbhhqk*nRXXa[.IIIIJ J Js   A A$$A-c                    t         j                  g dt              } t        |       t        |       k(  sJ t        t         j                  g dt                    t        t         j                  g dt                    k(  sJ t        t         j                  g dt                    t        t         j                  g dt                    k(  sJ  G d d      }t         j                  dd  |       gt              }t        |       y )N)rU   aaaaarr   )rU   Nr   ))rZ   rU   )rZ   N)rZ   r   c                      e Zd Zy)Ctest_tokenize_numpy_array_on_object_dtype.<locals>.NeedsCloudPickleNrL   rM   rN   rG   rD   r$   NeedsCloudPickler          rD   r   rU   )rc   r|   objectr8   )rU   r   r   s      r$   )test_tokenize_numpy_array_on_object_dtyper      s    
#62A!q 1111"((#5V(DE
#62J    
2&A	!B&Q	RS S S  	#t-/0?A1rD   c                     t         j                  g       } | j                  sJ t        |       t        |       k(  sJ t         j                  g t         j	                               }t        |       t        |      k7  sJ y )Nrr   )rc   r|   stridesr8   rd   rx   arr2s     r$   test_empty_numpy_arrayr      sc    
((2,C;;;#."555588Bbhhj8)D#."6666rD   c                   t        | j                  d            }t        |d      5 }|j                  d       d d d        t        |d      5 }t        j                  |t        j                  ddd      }t        j                  |t        j                  ddd      }t        j                  |t        j                  ddd      }t        |      t        |      k(  sJ t        |      t        |      k(  sJ t        |      t        |      k7  sJ t        |d	d
       t        |d	d
       k(  sJ t        |d	d
       t        |d	d
       k(  sJ t        |d	d       t        |d	d       k(  sJ t        |d	d       t        |d	d       k7  sJ t        |d	d       t        |d	d       k7  sJ |d	d
 }|d	d
 }t        |      t        |      k7  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N	demo_datawbs
   ashekwichtrbrr      )rs   modeoffsetshaperZ   r[   r\   )r^   joinopenwriterc   memmapuint8r8   )tmpdirfnfmmap1mmap2mmap3sub1sub2s           r$   !test_tokenize_numpy_memmap_offsetr      s    
V[[%	&B	b$ 1	 
b$ <1		!288#aq	I		!288#aq	I		!288#aq	Ie$u(====e$u(====e$u(====eAbk*nU1R[.IIIIeAbk*nU1R[.IIIIeAaj)^E!AJ-GGGGeAaj)^E!AJ-GGGGeAaj)^E!AJ-GGGGQr{Qr{d#~d';;;;< < < <s   GFG&G#&G/c                 :   t        d      5 } t        j                  d      }t        j                  | |       t	        t        j                  | d            }d d d        t        d      5 } t        j                  d      }t        j                  | |       t	        t        j                  | d            }d d d        t	              t	              k(  sJ k(  sJ t        d      5 } t        j                  j                  d      }t        j                  | |       t        j                  | d      }t        j                  | d      }t	        |dd d f         }t	        |dd d f         }	t	        |dd	d d f         }
t	        |d d df         }t        ||	|
|h      d
k(  sJ t	        |      t	        |      k(  sJ t	        |dd d f         t	        |dd d f         k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   OxY w# 1 sw Y   y xY w)N.npyr   r   	mmap_mode)
   r   )sizer   rZ   r\      )	r   rc   arangesaver8   loadr?   normallen)r   x1yx2zr   mmmm2rU   rV   cds               r$   test_tokenize_numpy_memmapr      s   	 7BYYq\
B27727567
 
 7BYYq\
B27727567
 "!33336M6	 EBII(+
AWWR3W'ggbCg(2ad8$2ad8$2ac1f:&2ad8$Aq!Q< A%%%b!^C%8888bAh'>#ad)+DDDDE E7 7
7 7E Es&   AG7+AH*DH7HHHc                 r   t        d      5 } t        d      5 }t        j                  d      }t        j                  | |       t        j                  ||       t        j	                  | d      }||z   }t        |      t        |      k(  sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   r   r   )r   rc   r   r   r   r8   )fn1fn2r   rU   rV   s        r$   &test_tokenize_numpy_memmap_no_filenamer      s     
 6C 6CIIaL
Q
QGGC3G'Ea N1$55556 6 6 6 6 6s#   B-A7B!B-!B*	&B--B6c                    t        t        j                        t        t        j                        k7  sJ t        j                  } t        j                  }t	        | t        j
                        sJ t	        |t        j
                        sJ t        |       t        |      k7  sJ t        j
                  j                  d dd      }t        j
                  j                  d dd      }t        j
                  j                  d dd      }t        |      t        |      k7  sJ t        |      t        |      k7  sJ y )Nc                    | dz   S NrZ   rG   r   s    r$   r   z+test_tokenize_numpy_ufunc.<locals>.<lambda>  s
    A rD   rZ   c                    | dz   S r   rG   r   s    r$   r   z+test_tokenize_numpy_ufunc.<locals>.<lambda>  
    Q rD   c                    | dz   S Nr[   rG   r   s    r$   r   z+test_tokenize_numpy_ufunc.<locals>.<lambda>  r   rD   )r8   rc   sincosr]   ufuncda
frompyfunc)np_ufunc	np_ufunc2incinc2inc3s        r$   test_tokenize_numpy_ufuncr     s    "&&!^BFF%;;;;vvHIh)))i***(#~i'@@@@ ((

oq!
4C8815D8815D#."6666#."6666rD   c                    t         j                  d dd      } t        j                  j	                  ddi      5  t        |       t        |       k7  sJ t        |       t        |       k7  sJ 	 d d d        t        j                  j	                  ddi      5  t        j                  t        d      5  t        |        d d d        d d d        t        | d      t        | d      k7  sJ t        j                  t        d	      5  t        | d       d d d        y # 1 sw Y   xY w# 1 sw Y   lxY w# 1 sw Y   pxY w# 1 sw Y   y xY w)
Nc                    | dz   S r   rG   r   s    r$   r   z;test_normalize_numpy_ufunc_unserializable.<locals>.<lambda>  s
    !a% rD   rZ   r'   FTz,Cannot tokenize.*dask\.array\.ufunc.*insteadmatchensure_deterministiczCannot tokenize)
rc   r   r(   r)   r*   r   r   r    r!   r   )r   s    r$   )test_normalize_numpy_ufunc_unserializabler     s)    --A
.C	95A	B <}---s#s';;;;	< 
94@	A ]]%T
 	 SM	 Ce4%9    
(0A	B 1401 1!< <	 	 1 1s;   3DD63D*?D6ED'*D3	/D66D?Ec                     G d d      }  |        }t         j                  j                  ddi      5  t        |      t        |      k7  sJ t	        |      t	        |      k7  sJ 	 d d d        t         j                  j                  ddi      5  t        j                  t        d      5  t        |       d d d        d d d        t        |d      t        |d      k7  sJ t        j                  t        d      5  t        |d       d d d        y # 1 sw Y   xY w# 1 sw Y   lxY w# 1 sw Y   pxY w# 1 sw Y   y xY w)	Nc                      e Zd Zd Zy)/test_normalize_object_unserializable.<locals>.Cc                    J r>   rG   rA   s    r$   rI   z:test_normalize_object_unserializable.<locals>.C.__reduce__3      5rD   NrL   rM   rN   rI   rG   rD   r$   Cr   2      	rD   r   r'   FTz"cannot be deterministically hashedr   r   )r(   r)   r*   r   r   r    r!   r   )r   r   s     r$   $test_normalize_object_unserializabler   1  s(     	
A	95A	B 8{hqk)))q!_Q%7777	8 
94@	A ]]%I
 	 QK	 AE2h	7    
(0T	U /./ /!8 8	 	 / /s;   3DD/,D#8D/ D;D #D,	(D//D8;Ec                     t        t        t        t              } t        t        t        t              }t        t        t        d      }t	        |       t	        |      k(  sJ t	        |       t	        |      k7  sJ y )N)r   r   )r   f3f2f1r8   )r   ghs      r$   1test_tokenize_partial_func_args_kwargs_consistentr   L  s\    B"AB"AB!A!q 1111!q 1111rD   c            
         dddt        ddd      t        j                  d      t        j                  ddd	      t        j                  d
      fD ]  } t        |       | u rJ  y )NrZ   g?rb   r[   r\   z1.1        z
/this/that)slicedecimalDecimaldatetimedatepathlibPurePathr   )is    r$   test_normalize_baser  T  sf    	aAdAr"& 	' q!Q&&&	'rD   c            
        t         j                  j                  ddi      5  t               } t	        |       t	        |       k(  sJ t	        t                     t	        t                     k7  sJ t        t        d      D ch c]  }t	        t                      c}      dk(  sJ t        |       t        |       k(  sJ 	 d d d        t         j                  j                  ddi      5  t        j                  t        d      5  t	                d d d        t        j                  t        d      5  t                d d d        d d d        t	         d      t	        | d      k(  sJ t        j                  t        d      5  t	        | d       d d d        y c c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   }xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr'   Fd   Tdeterministicr   r   )r(   r)   r*   r   r   r   ranger   r    r!   r   )o_s     r$   test_tokenize_objectr  a  s   	95A	B 8H{hqk)))!Xfh%7777
 c
;1HVX&;<CCC q!_Q%77778  
94@	A ]],OD 	QK	]],OD 	A	 AE2h	7    
(	@ /./ /! <8 8"	 		 	 / /sf   AF%F 
%F%*G	F1#G	5F=G		G F%%F.1F:	6G	=G	G		GGc                      G d d      } t         |        d      t         |        d      k7  sJ t        j                  t              5  t         |               ddd       y# 1 sw Y   yxY w)zdTest that the ensure_deterministic override is not lost if tokenize() is
    called recursively
    c                      e Zd Zd Zy)/nested_tokenize_ensure_deterministic.<locals>.Cc                (    t        t                     S r>   r   r   rA   s    r$   rK   zAnested_tokenize_ensure_deterministic.<locals>.C.__dask_tokenize__      FH%%rD   NrR   rG   rD   r$   r   r        	&rD   r   Fr   N)r   r    r!   r   r   s    r$   $nested_tokenize_ensure_deterministicr    sd    
& & ACe4	%9    
(	)   s   A%%A.rZ   c                    d d d d g} | d d \  d }d }| ||gz  } dfd	}| j                  |       d
 }| j                  |       fd}| j                  |       fd}| j                  |       d }| j                  |       d }| j                  |       fd}| j                  |       fd}| j                  |       | S )Nc                    | S r>   rG   r   s    r$   r   z"_local_functions.<locals>.<lambda>      ! rD   c                    | dz   S r   rG   r   s    r$   r   z"_local_functions.<locals>.<lambda>  
    !a% rD   c                    | S r>   rG   r   s    r$   r   z"_local_functions.<locals>.<lambda>  r  rD   c                    | dz   S r   rG   r  s    r$   r   z"_local_functions.<locals>.<lambda>  r  rD   r[   c                    | S r>   rG   r   s    r$   funcz_local_functions.<locals>.func      rD   c                    | S r>   rG   r   s    r$   r   z_local_functions.<locals>.f2  r   rD   rZ   c                     dz          S r   rG   )rU   local_scopes   r$   r  z_local_functions.<locals>.func  s    q~rD   c                      t         dz  a t         S r   )_GLOBALrG   rD   r$   r  z_local_functions.<locals>.func  s    1rD   c                     ||       S r>   rG   r   r   s     r$   r  z_local_functions.<locals>.func      trD   c                     ||       S r>   rG   r'  s     r$   r  z_local_functions.<locals>.func  r(  rD   c                    d } ||       S )Nc                    | dz   S r   rG   r   s    r$   r   z0_local_functions.<locals>.func.<locals>.<lambda>  
    a!e rD   rG   r'  s     r$   r  z_local_functions.<locals>.func      trD   c                    d } ||       S )Nc                    | dz   S )Nr\   rG   r   s    r$   r   z0_local_functions.<locals>.func.<locals>.<lambda>  r,  rD   rG   r'  s     r$   r  z_local_functions.<locals>.func  r-  rD   c                    } ||       S r>   rG   )r   r   rU   s     r$   r  z_local_functions.<locals>.func      trD   c                    } ||       S r>   rG   )r   r   rV   s     r$   r  z_local_functions.<locals>.func  r1  rD   )append)	all_funcsr  r   rU   rV   r#  s      @@@r$   _local_functionsr5    s    	I Ra=DAq $IK
 T
 T
   T  T T T T TrD   c                  "    e Zd Zd Zed        Zy)WithClassMethodc                     y r>   rG   rA   s    r$   r   zWithClassMethod.f      rD   c                     y r>   rG   clss    r$   r   zWithClassMethod.g      rD   N)rL   rM   rN   r   classmethodr   rG   rD   r$   r7  r7    s      rD   r7  r   r  c                    t        |        y r>   r8   r  s    r$   test_tokenize_local_functionsrB    
    4rD   c                    t        |        y r>   r@  rA  s    r$   test_tokenize_special_callablesrE    rC  rD   c                     t               t        z   } | D cg c]  }t        |       }}t        t	        |            t        |      k(  sJ y c c}w r>   )r5  _special_callablesr8   r   r*   )r4  r  tokenss      r$   $test_tokenize_functions_unique_tokenrI    sH     "%77I/89tnT"9F9s6{s6{*** :s   Az3https://github.com/cloudpipe/cloudpickle/issues/453)reasoninstanceFc                V      fd}t         |             t         |             k(  sJ y )Nc                 .     G d d      } r |        S | S )Nc                      e Zd Zy)Itest_tokenize_local_classes_from_different_contexts.<locals>.f.<locals>.CNr   rG   rD   r$   r   rO    s    rD   r   rG   )r   rK  s    r$   r   z>test_tokenize_local_classes_from_different_contexts.<locals>.f  s    	 	 qs%A%rD   r@  )rK  r   s   ` r$   3test_tokenize_local_classes_from_different_contextsrP    s&    & !#."5555rD   c                 P    d } t         |              t         |              k(  sJ y )Nc                     d } | S )Nc                      y)Nr:   rG   rG   rD   r$   r   zKtest_tokenize_local_functions_from_different_contexts.<locals>.f.<locals>.g      rD   rG   )r   s    r$   r   z@test_tokenize_local_functions_from_different_contexts.<locals>.f  s    	 rD   r@  )r   s    r$   5test_tokenize_local_functions_from_different_contextsrU    s%     !#."5555rD   c                     y r>   rG   rU   rV   r   s      r$   r   r         rD   c                     y r>   rG   rW  s      r$   r   r     rX  rD   c                     y r>   rG   r_   s    r$   r   r   !  rX  rD   c                 F    t        t              t        t              k7  sJ y r>   )r8   r   r   rG   rD   r$   test_tokenize_callabler\  %  s    "!3333rD   c                 .   t        t        t        d            t        t        t        d            k7  sJ t        t        t        d            t        t        t        d            k7  sJ t        t	        t        t
                    t        t	        t        t                    k7  sJ t        t        t                    t        t        t                    k7  sJ t        t        t        d            t        t        t        d            k7  sJ y )Nr[   )rV   r\   rZ   )r8   r   r   r   r   r   r   rG   rD   r$   !test_tokenize_composite_functionsr^  )  s    '"*+~gbA>N/OOOO'"*+~gbA>N/OOOO'"b/*nWR_.MMMM%)$uRy(AAAA%a.)^E"N-KKKKrD   znot pdc                    t         j                  g dg ddg d      } t         j                  g dg ddg d      }t        |       t        |      k(  sJ d|j                  _        t        |       t        |      k7  sJ t         j                  g dg dd      } t         j                  g dg dd      }| j
                  j                  d      | d<   t         j                  g d	      | _        t        |       t        |      k7  sJ | j
                  j                  d      |d<   t         j                  g d	      |_        t        |       t        |      k(  sJ y )
NrY   )4asdNr   r   indexr   )rU   rV   rU   categoryr   )r   r   r   )	pd	DataFramer8   rd  namer   rt   IndexcolumnsrU   rV   s     r$   test_tokenize_pandasrl  1  s   
9+=>iPA
9+=>iPA!q 1111AGGL!q 1111
9?;<A
9?;<ASSZZ
#AcF )AI!q 1111SSZZ
#AcF)AI!q 1111rD   c                     t         j                  t         j                  dddg      t         j                  dddg      g      } t        |        y )NrZ   rU   rV   rc  r[   r   r   )rf  concatSeriesr8   )ss    r$   %test_tokenize_pandas_fragmented_indexrq  H  s?    
		299QsCj91299QsCj93QRSA1rD   c                     t         j                  g dt         j                  g dd      dg d      } t         j                  ddgd      | _        t        |        y )	NrY   )r`  u   asdNr   rr   rb  rc  u   xu   y)rf  rg  ro  ri  rj  r8   dfs    r$   $test_tokenize_pandas_invalid_unicoderu  N  sX     
38D	
  
 
B 9i0ABJ2rD   c                     t         j                  dj                         g ddddj                         d gig d      } t        |        y )N   örY   rc  )rf  rg  encoder8   rs  s    r$   (test_tokenize_pandas_mixed_unicode_bytesry  \  sA    			4$t)DE 
 
B 2rD   c                 l     G d d      } t         j                  ddd  |        gi      }t        |       y )Nc                      e Zd Zy):test_tokenize_pandas_cloudpickle.<locals>.NeedsCloudPickleNr   rG   rD   r$   r   r|  g  s    rD   r   r   r   )rf  rg  r8   )r   rt  s     r$    test_tokenize_pandas_cloudpickler}  e  s4      
sUD*:*<=>	?B2rD   znot ddc                 n   t         j                  g dd      t         j                  dgd      t         j                  g dd      t         j                  t         j                  d      gd      t         j                  t         j                  dd	
      gd      t         j                  ddgt         j                  j                  j                  g dd            g} | j                  t         j                  g dd      t         j                  g dd      g       | D ]  }t        |        y )N)rZ   r   NInt64rr   2000z	Period[D])rZ   r   r   zSparse[int]zdatetime64[ns]CET)tzzdatetime64[ns, CET]rU   rV   rW  F)ordered)rU   rV   Nstring)TFNboolean)rf  r|   	TimestampapitypesCategoricalDtypeextendr8   )arraysrx   s     r$   $test_tokenize_pandas_extension_arrayr  o  s     	W-
&-
-0
",,v&'/?@
",,v%,019NO
#J&&,,///O 	 	

F MMHH%XH6HH(	H:	
  srD   c                 6    t        t        j                         y r>   )r8   rf  NArG   rD   r$   test_tokenize_nar    s    255rD   c            	        t         j                  j                  d      t         j                  j                  d      t         j                  j	                  d      t         j                  j                  d      t         j                  d      t         j                  d      t         j                  d      fD ]  } t        |         y )NrZ   r[   )yearsrp   )monthsr   days)rf  offsetsSecond
MonthBeginDayBQuarterEnd
DateOffsetr8   )r   s    r$   test_tokenize_offsetr    s     	

!


a 


q


q!
A
Q
2 	 	v	rD   c                     t         j                  ddg      } t        |        t         j                  j	                  ddgddgg      } t        |        y )NrU   rV   r   rZ   )rf  ri  r8   
MultiIndexfrom_product)idxs    r$   test_tokenize_pandas_indexr    sE    
((C:
C3
--
$
$sCj1a&%9
:C3rD   c                    t        dd       t        d      t        dd      k7  sJ t        dd      t        dd      k7  sJ t        dd      t        dd      k7  sJ t        dd      t        dddi      k7  sJ y )	Nr   rZ   r   r[   r  bar)r   r   r@  rG   rD   r$   test_tokenize_kwargsr    s~    1!qA 6666!q!^A%;;;;!q!^A%;;;;!'>!eU^+LLLLrD   c                 b     G d d      } t         | d            t         | d            k7  sJ y )Nc                      e Zd Zd Zd Zy)$test_tokenize_same_repr.<locals>.Fooc                    || _         y r>   r   rB   r   s     r$   rC   z-test_tokenize_same_repr.<locals>.Foo.__init__  	    DFrD   c                     y)Nza foorG   rA   s    r$   __repr__z-test_tokenize_same_repr.<locals>.Foo.__repr__  s    rD   N)rL   rM   rN   rC   r  rG   rD   r$   Foor    s    		rD   r  rZ   r[   r@  r  s    r$   test_tokenize_same_reprr    s.      #a&!^CF%;;;;rD   c                 b     G d d      } t         | d            t         | d            k7  sJ y )Nc                      e Zd ZdZd Zy)"test_tokenize_slotted.<locals>.Foor   c                    || _         y r>   r   r  s     r$   rC   z+test_tokenize_slotted.<locals>.Foo.__init__  r  rD   NrL   rM   rN   	__slots__rC   rG   rD   r$   r  r    s    		rD   r  rZ   r[   r@  r  s    r$   test_tokenize_slottedr    s.      #a&!^CF%;;;;rD   c                      G d d      } t         | d            t         | d            k7  sJ t         |               y )Nc                      e Zd ZdZddZy)+test_tokenize_slotted_no_value.<locals>.Foorb  Nc                *    ||| _         ||| _        y y r>   rb  rB   r   r   s      r$   rC   z4test_tokenize_slotted_no_value.<locals>.Foo.__init__  s    }} rD   )NNr  rG   rD   r$   r  r    s    		rD   r  rZ   r   r  r@  r  s    r$   test_tokenize_slotted_no_valuer    s8      #(#~cAh'????35rD   c            	     2    G d d      }  G d d|       } |dd      j                   ddik(  sJ t         |dd            t         |dd            t         |dd             t         |dd            g}t        t        |            t        |      k(  sJ y )	Nc                      e Zd ZdZy))test_tokenize_slots_and_dict.<locals>.Foor   N)rL   rM   rN   r  rG   rD   r$   r  r    s    	rD   r  c                      e Zd Zd Zy))test_tokenize_slots_and_dict.<locals>.Barc                &    || _         ||| _        y y r>   rb  r  s      r$   rC   z2test_tokenize_slots_and_dict.<locals>.Bar.__init__  s    DF} rD   NrL   rM   rN   rC   rG   rD   r$   Barr    s    	rD   r  rZ   r[   r   r\   )__dict__r8   r   r*   )r  r  rH  s      r$   test_tokenize_slots_and_dictr    s     c  q!9#q))) 	s1ay!s1ay!s1d|$s1ay!	F s6{s6{***rD   c                 N    G d d      }  | d       | d      }}t        |      t        |      k7  sJ t        |j                        t        |j                        k7  sJ t        |      }t        j                  | d        t        |      }||k7  sJ t        j                  | = y )Nc                      e Zd Zd Zd Zd Zy)!test_tokenize_method.<locals>.Fooc                    || _         y r>   r   r  s     r$   rC   z*test_tokenize_method.<locals>.Foo.__init__  r  rD   c                    | j                   S r>   r   rA   s    r$   rK   z3test_tokenize_method.<locals>.Foo.__dask_tokenize__      66MrD   c                     y)NzHello worldrG   rA   s    r$   helloz'test_tokenize_method.<locals>.Foo.hello  s     rD   N)rL   rM   rN   rC   rK   r  rG   rD   r$   r  r    s    			!rD   r  rZ   r[   c                     | j                   dz   S r   r   rA   s    r$   r   z&test_tokenize_method.<locals>.<lambda>  s    tvvz rD   )r8   r  r   register_lookup)r  rU   rV   r0   r1   s        r$   test_tokenize_methodr    s    ! ! q63q6qA!q 1111!''"nQWW&==== AFS"9:1EU??$rD   c                      G d d      }  G d d|       } | d       | d       |d      }}}t        |      t        |      k7  sJ t        |      t        |      k7  sJ y )Nc                      e Zd Zd Zd Zy)'test_tokenize_callable_class.<locals>.Cc                    || _         y r>   r   r  s     r$   rC   z0test_tokenize_callable_class.<locals>.C.__init__  r  rD   c                    | j                   S r>   r   rA   s    r$   __call__z0test_tokenize_callable_class.<locals>.C.__call__  r  rD   N)rL   rM   rN   rC   r  rG   rD   r$   r   r    s    		rD   r   c                      e Zd Zy)'test_tokenize_callable_class.<locals>.DNr   rG   rD   r$   Dr    r   rD   r  rZ   r[   r@  )r   r  rU   rV   r   s        r$   test_tokenize_callable_classr    sc     A  dAaD!A$!qA!q 1111!q 1111rD   c                      G d d      } t         | dd            t         | dd            k(  sJ t         | dd            t         | dd            k7  sJ y)z(Always use ___dask_tokenize__ if presentc                      e Zd Zd Zd Zd Zy)<test_tokenize_callable_class_with_tokenize_method.<locals>.Cc                     || _         || _        y r>   rb  r  s      r$   rC   zEtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__init__  s    DFDFrD   c                    | j                   S r>   r   rA   s    r$   rK   zNtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__dask_tokenize__  r  rD   c                     y r>   rG   rA   s    r$   r  zEtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__call__  s    rD   N)rL   rM   rN   rC   rK   r  rG   rD   r$   r   r    s    			rD   r   rZ   r[   r\   Nr@  r  s    r$   1test_tokenize_callable_class_with_tokenize_methodr    sV    	 	 !Aq'"nQq!W&====!Aq'"nQq!W&====rD   c                  >    e Zd Zd Zd Zd Zed        Zed        Z	y)HasStaticMethodsc                    || _         y r>   r   r  s     r$   rC   zHasStaticMethods.__init__  s	    rD   c                B    t        t        |             | j                  fS r>   )r   typer   rA   s    r$   rK   z"HasStaticMethods.__dask_tokenize__"  s    tDz*DFF22rD   c                     y r>   rG   rA   s    r$   normal_methodzHasStaticMethods.normal_method%  r9  rD   c                      y r>   rG   rG   rD   r$   static_methodzHasStaticMethods.static_method(  r=  rD   c                     y r>   rG   r;  s    r$   class_methodzHasStaticMethods.class_method,  r=  rD   N)
rL   rM   rN   rC   rK   r  staticmethodr  r>  r  rG   rD   r$   r  r    s9    3    rD   r  c                      e Zd Zy)HasStaticMethods2Nr   rG   rD   r$   r  r  1  s    rD   r  c                 b   t        d      t        d      t        d      }}} t        | j                        t        |j                        k7  sJ t        | j                        t        |j                        k7  sJ t        | j                        t        |j                        k(  sJ t        | j                        t        |j                        k(  sJ t        | j
                        t        |j
                        k(  sJ t        | j
                        t        |j
                        k7  sJ y )NrZ   r[   )r  r  r8   r  r  r  rW  s      r$   test_staticmethodsr  5  s    q!#3A#68I!8L!qA!//*nQ__.MMMM!//*nQ__.MMMM!//*nQ__.MMMM!//*nQ__.MMMM!..)^ANN-KKKK!..)^ANN-KKKKrD   c                     t        dg      t        dg      k7  sJ t        dg      t        d      k7  sJ t        dg      t        dg      k(  sJ d} | | | dgg}t        |      sJ y )NrZ   r[   rZ   r   )r[   r\   )r8   r   rb  s     r$   test_tokenize_sequencesr  A  sy    1#.!"55551#."66661#.!"5555 	A	
A6{A1rD   c                     g d}  G d d      } ||        ||       }}t        | ||        |j                  sJ t        |      t        |      k(  sJ y)zuTest that calling tokenize() recursively doesn't alter the output due to
    memoization of already-seen objects
    rY   c                      e Zd Zd Zd Zy)$test_nested_tokenize_seen.<locals>.Cc                     || _         d | _        y r>   )r   r@   r  s     r$   rC   z-test_nested_tokenize_seen.<locals>.C.__init__S  s    DFDHrD   c                f    | j                   st        | j                        | _         | j                   S r>   )r@   r   r   rA   s    r$   rK   z6test_nested_tokenize_seen.<locals>.C.__dask_tokenize__W  s"    88#DFF+88OrD   N)rL   rM   rN   rC   rK   rG   rD   r$   r   r  R  s    		rD   r   N)r8   r@   )r
  r   c1c2s       r$   test_nested_tokenize_seenr  L  sU     	A  qT1Q4B1b!66M6"!3333rD   c                     t        ddd      dk(  sJ t        ddd      t        ddd      k(  sJ t        ddd      t        ddd      k7  sJ t        ddd      t        ddd      k7  sJ y )NrZ   r   )r   rZ    ba2498a4c5583cc3ec540865185cdafdrZ   r   )r   r[   r[   r@  rG   rD   r$   test_tokenize_dictr  b  s    c*+/QQQQc*+~#A>N/OOOOc*+~A#>N/OOOOc*+~A#>N/OOOOrD   c                 >    t        h d      t        h d      k(  sJ y )N>   r   r   rZ   r[   r@  rG   rD   r$   test_tokenize_setr  j  s$    /0N5   rD   c                     ddl m}   | ddg      } | ddg      } | ddg      }t        |      t        |      k(  sJ t        |      t        |      k7  sJ y )Nr   )OrderedDict)rU   rZ   )rV   r[   )collectionsr  r8   )r  rU   rV   r   s       r$   test_tokenize_ordered_dictr  p  sb    'Xx()AXx()AXx()A!q 1111!q 1111rD   c                 H     G d d      } t         |         |        d       y )Nc                      e Zd Zd Zd Zy)7test_tokenize_dict_doesnt_call_str_on_values.<locals>.Cc                     y)Nr   rG   rA   s    r$   rK   zItest_tokenize_dict_doesnt_call_str_on_values.<locals>.C.__dask_tokenize__}  rT  rD   c                    J r>   rG   rA   s    r$   r  z@test_tokenize_dict_doesnt_call_str_on_values.<locals>.C.__repr__  r   rD   N)rL   rM   rN   rK   r  rG   rD   r$   r   r
  |  s    		rD   r   )rZ   2r@  r  s    r$   ,test_tokenize_dict_doesnt_call_str_on_valuesr  {  s      qs%&rD   c                 N    d} | | d}| | d}t        |      t        |      k(  sJ y )NrY   r   )r[   rZ   r@  )vd1d2s      r$   (test_tokenize_sorts_dict_before_seen_mapr    s1    A1B1B"!3333rD   c                     d} t        d      D ch c]  }|| f }}t        t        d            D ch c]  }|| f }}t        |      t        |      k(  sJ y c c}w c c}w )NrY   r  )r	  reversedr8   )r  r  s1s2s       r$   'test_tokenize_sorts_set_before_seen_mapr    sb    A*	%Q1a&	%B	%"5:.	/Q1a&	/B	/"!3333 
&	/s
   AAc                    t        t        j                  d            t        t        j                  d            k(  sJ t        t        j                  d            t        t        j                  d            k7  sJ y )NrZ   r  r[   )r8   r   	timedeltarG   rD   r$   test_tokenize_timedeltar    sr    (,,!45":    (,,!45":   rD   	enum_typec                     G d d|       }t        |j                        t        |j                        k(  sJ t        |j                        t        |j                        k7  sJ y )Nc                      e Zd ZdZdZy)!test_tokenize_enum.<locals>.ColorrZ   r[   N)rL   rM   rN   REDBLUErG   rD   r$   Colorr    s    rD   r"  )r8   r   r!  )r  r"  s     r$   test_tokenize_enumr#    sP    	  %))$uyy(AAAA%))$uzz(BBBBrD   c                      e Zd ZU ded<   y)
ADataClassintrU   NrL   rM   rN   __annotations__rG   rD   r$   r%  r%    s    
FrD   r%  c                      e Zd ZU ded<   y)
BDataClassfloatrU   Nr'  rG   rD   r$   r*  r*    s    HrD   r*  c                  >    e Zd ZU  ej                  d      Zded<   y)NoValueDataClassF)initr&  rU   N)rL   rM   rN   dataclassesfieldrU   r(  rG   rD   r$   r-  r-    s    [E*As*rD   r-  c                      e Zd ZddZy)GlobalClassc                    || _         y r>   )val)rB   r4  s     r$   rC   zGlobalClass.__init__  s	    rD   N)returnNoner  rG   rD   r$   r2  r2    s    rD   r2  c                 j    G d d      }  G fdd       G d d      } G d dt               }t        t        d	             t        t        d	            t        t        d
            k7  sJ t         |               t         |d	             t         |             t                      k7  sJ y )Nc                      e Zd ZdZy)%test_local_objects.<locals>.LocalTyper  N)rL   rM   rN   r   rG   rD   r$   	LocalTyper9    s    rD   r:  c                      e Zd Z fdZy)*test_local_objects.<locals>.LocalReduciblec                    dfS rF   rG   )rB   LocalReducibles    r$   rI   z5test_local_objects.<locals>.LocalReducible.__reduce__  s    !2%%rD   Nr   )r>  s   r$   r>  r<    s    	&rD   r>  c                      e Zd Zd Zy)-test_local_objects.<locals>.LocalDaskTokenizec                     yr   rG   rA   s    r$   rK   z?test_local_objects.<locals>.LocalDaskTokenize.__dask_tokenize__      rD   NrR   rG   rD   r$   LocalDaskTokenizer@    r   rD   rC  c                      e Zd Zy)&test_local_objects.<locals>.LocalChildNr   rG   rD   r$   
LocalChildrE    r   rD   rF  rZ   r[   )r2  r8   )r:  rC  rF  r>  s      @r$   test_local_objectsrG    s     & & [  ;q>"+a.)^KN-KKKK9;:a=!+-..AQ2RRRRrD   z3.13z)https://github.com/dask/dask/issues/11457c                    t        d      } t        d      }t        |        t        |       t        |      k7  sJ t        d      }t        t               t        t              k7  sJ t        |       t        |      k7  sJ  G d dt               }t        j                  |      sJ t        t               t        |      k7  sJ t         |d            t        |       k7  sJ t        j
                  ddt        t        t        f   fg      }t        |       t         |d            k7  sJ t               }t        |       y )NrZ   r[   c                      e Zd Zy)%test_tokenize_dataclass.<locals>.SubANr   rG   rD   r$   SubArJ    r   rD   rK  r%  rU   )
r%  r8   r*  r/  is_dataclassmake_dataclassr	   r&  r^   r-  )a1a2b1rK  ADataClassRedefinedDifferentlynvs         r$   test_tokenize_dataclassrS    s!    
AB	AB2"!3333 
AB*%
)CCCC"!3333z  ##D)))*%)====$q'"nR&8888 &1%?%?U38_-.&" "0Nq0Q!RRRR 
	B2rD   other))rZ   r   r[   )r      r[   )r   r   rZ   c                X    t        t        ddd            t        t        |        k7  sJ y )Nr   r   r[   )r8   r	  )rT  s    r$   test_tokenize_rangerW    s(     %2q/*nUE].KKKKrD   c                     t         j                  d      } t         j                  d      }d|d<   t        | g      t        |g      k7  sJ y )Ni  r   rk   )rc   r   r8   rb  s     r$   test_tokenize_numpy_arrayrY    sA    
		$A
		$AAdG1#.!"5555rD   c                 j    t         j                  ddt         j                  gd      } t        |        y )Nr   u   JoséOrr   )rc   r|   nanr8   r_   s    r$   $test_tokenize_object_array_with_nansr]    s'    
%BFF+37A1rD   r   rU      ar   y              ?rG   c                    t        |        y r>   r@  r   s    r$   test_tokenize_base_typesr`    s     1rD   c                 b    t        t        ddg            t        t        ddg            k7  sJ y )Nr   rZ   r[   )r8   r   rG   rD   r$   test_tokenize_literalrb    s,    '3(+,wQx?P0QQQQrD   z+ignore:the matrix:PendingDeprecationWarningc                    t         j                  j                  d      } t         j                  | j	                  d            }|j                         }t        |      t        |      k(  sJ d|d d t        |      t        |      k7  sJ y )Nrj   r  rZ   r   )rc   r?   rl   asmatrixrandcopyr8   )rngrU   rV   s      r$   test_tokenize_numpy_matrixrh    st     ))


%C
CHHSM"A	A!q 1111AcrF!q 1111rD   znot spcls_name)dokc                Z   t         j                  j                  d      }t        j	                  dd|      j                  |       }|j                         }t        |      t        |      k(  sJ t        |d      rd|j                  d d n| dk(  rd|d<   nt        t        |       t        |      t        |      k7  sJ |j                         j                  d	      }t         j                  d      |j                  d d |j                  |       }t        |      t        |      k7  sJ y )
Nrj   r   r  )random_statedatarZ   rj  )r\   r\   coo)rc   r?   rl   spre  asformatrf  r8   hasattrrm  
ValueErrorr   row)ri  rg  rU   rV   s       r$    test_tokenize_dense_sparse_arrayrt  #  s    ))


%C
Cc*33H=A	A!q 1111 q&s	U	$1!q 1111 	
% A2AEE#2J	

8A!q 1111rD   c                 x   ddg} | | d<   ddg}||d<   t        |       t        |      k7  sJ g g g}|d   j                  |d          |d   j                  |d          g g g}|d   j                  |d          |d   j                  |d          t        |      t        |      k7  sJ i }||d<   t        |       y )NrZ   r[   r   r\   )r8   r3  )rU   rV   r   r   es        r$    test_tokenize_circular_recursionrw  >  s    	
AAAaD 
AAAaD!q 1111 
RAaDKK!aDKK!	RAaDKK!aDKK!!q 1111 	AAaD1rD   ))i  r   r   )r   rp   r   )r   r      c                    t        j                  ddd      }t        j                  |  }t        |      t        |      k7  sJ y )Nr   r   r   )r   r  r8   )rT  rU   rV   s      r$   test_tokenize_datetime_daterz  X  s<     	dAr"AuA!q 1111rD   c            
        t        t        j                  ddddt        j                  j                               t        t        j                  dddd             t        t        j                  ddd             t        t        j                  dd             t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  dddd            k7  sJ y )NrZ   r[   r\   r   r   )r8   r   timetimezoneutcrG   rD   r$   test_tokenize_datetime_timer  f  s8   8==Aq!X->->-B-BCD8==Aq!,-8==Aq)*8==A&'aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq1	23 3 3rD   c                 h   g d} ddddt         j                  j                  g}t        t	        |      dz         D ]%  }| |d | z   }t        t        j                   |        ' t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddd             k7  sJ y )	NrY   r   r   r   rp   rZ   r[   r\   )r   r}  r~  r	  r   r8   )requiredoptionalr  r.   s       r$   test_tokenize_datetime_datetimer    s   H1aH--112H3x=1$% 1(2A,&x(($/01
 !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAtD	EF F FrD   c                     d} t        j                  t        j                  dt	        j
                  |       g      }|j                          y )Na  
    def inc(x):
        return x + 1

    inc2 = inc
    def sum(x, y):
        return x + y

    from dask.base import tokenize
    assert tokenize(inc) != tokenize(sum)
    # That this is an alias shouldn't matter
    assert tokenize(inc) == tokenize(inc2)

    def inc(x):
        return x + 1

    assert tokenize(inc2) != tokenize(inc)

    def inc(y):
        return y + 1

    assert tokenize(inc2) != tokenize(inc)

    def inc(x):
        y = x
        return y + 1

    assert tokenize(inc2) != tokenize(inc)

    # Test that redefining a function changes the token
    def func(x):
        return x + 1

    result = tokenize(func)

    def func(x):
        return x + 2

    result2 = tokenize(func)
    assert result != result2
    z-c)
subprocessrunsys
executabletextwrapdedentcheck_returncode)scriptprocs     r$   test_tokenize_functions_mainr    s:    (FR >>3>>41HIJDrD   c                     t        j                  ddt        t        j                  d      fgdd i      }  | d       | d	      }}t	        |      t	        |      k7  sJ y )
NrH   paramF)reprrK   c                    | j                   S r>   )r  rA   s    r$   r   z7test_tokenize_dataclass_field_no_repr.<locals>.<lambda>  s
    TZZ rD   )	namespacerZ   r[   )r/  rM  r+  r0  r8   )rH   rN  rO  s      r$   %test_tokenize_dataclass_field_no_reprr    sa    ""
5+++7	89&(?@	A qT1Q4B"!3333rD   c                 n    t        t        j                        t        t        j                        k7  sJ y)ziTop-level functions in the operator module have a __self__ attribute, which is
    the module itself
    N)r8   operatoraddmulrG   rD   r$   test_tokenize_operatorr    s$     (,,'>(,,+GGGGrD   c                 8   t        j                  d      } t        j                  d      }t        j                  d      }t        |       t        |      k(  sJ t        |       t        |      k7  sJ | j                          t        |       t        |      k7  sJ y Nr:   i  r?   Randomr8   rW  s      r$   test_tokenize_random_stater    s}    cAcAcA!q 1111!q 1111HHJ!q 1111rD   c                 t   t         j                  j                  d      } t         j                  j                  d      }t         j                  j                  d      }t        |       t        |      k(  sJ t        |       t        |      k7  sJ | j                          t        |       t        |      k7  sJ y r  rc   r?   rl   r8   rW  s      r$    test_tokenize_random_state_numpyr    s    
		c"A
		c"A
		c"A!q 1111!q 1111HHJ!q 1111rD   moduler?   z	np.random)marksc                n   t        |       } | j                  d       | j                  }t        j                  t        j
                  |            }t        |      t        |      k(  sJ  |        t        j                  t        j
                  |            }t        |      t        |      k(  sJ t        |      t        |      k7  sJ | j                  d       t        j                  t        j
                  |            }t        |      t        |      k(  sJ t        |      t        |      k7  sJ y)zrandom.random() and other methods of the global random state do not compare as
    equal to themselves after a pickle roundtripr[   r:   N)evalseedr?   pickler,   r-   r8   )r  rU   rV   r   r   s        r$   test_tokenize_random_functionsr    s     &\F
KKNAV\\!_%A!q 1111 CV\\!_%A!q 1111!q 1111
KKV\\!_%A!q 1111!q 1111rD   c                     t        j                  d      j                   } t        j                  d      j                   }t        |       t        |      k7  sJ y r  r  rk  s     r$   )test_tokenize_random_functions_with_stater    sB    c!!Ac!!A!q 1111rD   c                     t         j                  j                  d      j                  } t         j                  j                  d      j                  }t        |       t        |      k7  sJ y r  r  rk  s     r$   /test_tokenize_random_functions_with_state_numpyr  "  sN    
		c"))A
		c"))A!q 1111rD   znot pac                     t         j                         } t         j                         }t        |       t        |      k7  sJ y r>   )pard   rg   r8   rk  s     r$   &test_tokenize_pyarrow_datatypes_simpler  )  s0    

A


A!q 1111rD   c                 &   t         j                  t         j                         t         j                         d      } t         j                  t         j	                         t         j                         d      }t        |       t        |      k7  sJ y )Nrb  )r  structre   r  rg   int16r8   rk  s     r$   'test_tokenize_pyarrow_datatypes_complexr  0  sX    
		
56A
		

288:67A!q 1111rD   c                    t         j                  g dg dd      } t         j                  g dg dd      }t         j                  g dg dd      }t        |       t        |      k(  sJ t        |       t        |      k7  sJ y )NrY   rW  rb  )rU   rV   r   )r  tabler8   rW  s      r$   test_pyarrow_tabler  7  sk    
y78A
y78A
y78A!q 1111!q 1111rD   c                 j     G d d      } t         | ddg            t         | ddg            k7  sJ y )Nc                      e Zd Zd Zy)3test_tokenize_opaque_object_with_buffers.<locals>.Cc                8    t         j                  |      | _        y r>   )rc   r|   r   r  s     r$   rC   z<test_tokenize_opaque_object_with_buffers.<locals>.C.__init__D  s    XXa[DFrD   Nr  rG   rD   r$   r   r  C  s    	!rD   r   rZ   r[   r\   r@  r  s    r$   (test_tokenize_opaque_object_with_buffersr  @  s6    ! ! !QF)$q!Qy(AAAArD   c                  $    e Zd Zd Zd ZexZxZZy)
NumbaDummyc                     y)NFrG   rA   s    r$   __bool__zNumbaDummy.__bool__M  rB  rD   c                    d }|S )Nc                    | S r>   rG   rA  s    r$   wrapperz,NumbaDummy._dummy_decorator.<locals>.wrapperQ  s    rD   rG   )rB   r.   r/   r  s       r$   _dummy_decoratorzNumbaDummy._dummy_decoratorP  s     NrD   N)rL   rM   rN   r  r  jit	vectorizeguvectorizerG   rD   r$   r  r  L  s    		 )988i+rD   r  nopythonc                    | |z   S r>   rG   rb  s     r$   	numba_jitr  [      q5LrD   
f8(f8, f8)c                    | |z   S r>   rG   rb  s     r$   numba_jit_with_signaturer  `  r  rD   c                    | |z   S r>   rG   rb  s     r$   numba_vectorizer  e  r  rD   c                    | |z   S r>   rG   rb  s     r$   numba_vectorize_with_signaturer  j  r  rD   f8,f8,f8[:]	(),()->()c                    | |z   |d<   y Nr   rG   r   r   outs      r$   numba_guvectorizer  o  s    UCFrD   z	not numbac                     | dd      dk(  sJ t        |        t        D ]4  } t        |       }t        |t              sJ t        |d   t              r4J  y )NrZ   r[   r\   )r8   all_numba_funcsr   r]   tupler  rH  s     r$   test_tokenize_numbar  }  s]     1:??4 , & &%(((&)U+++,rD   c                     t         D  cg c]  } t        |        }} t        |      t        t        |            k(  sJ y c c} w r>   )r  r8   r   r*   r  s     r$    test_tokenize_numba_unique_tokenr    s;    />?tnT"?F?v;#c&k**** @s   A c                    t         j                  d      d        } t         j                  dd      d        }t         j                  d      d        }t         j                  dd      d        }t         j                  dgd	      d
        }| ||||g}|D cg c]  }t	        |       }}t        |      t        t        |            k(  sJ y c c}w )NTr  c                    | |z   S r>   rG   rb  s     r$   	local_jitz#test_numba_local.<locals>.local_jit      1urD   r  c                    | |z   S r>   rG   rb  s     r$   local_jit_with_signaturez2test_numba_local.<locals>.local_jit_with_signature  r  rD   c                    | |z   S r>   rG   rb  s     r$   local_vectorizez)test_numba_local.<locals>.local_vectorize  r  rD   c                    | |z   S r>   rG   rb  s     r$   local_vectorize_with_signaturez8test_numba_local.<locals>.local_vectorize_with_signature  r  rD   r  r  c                    | |z   |d<   y r  rG   r  s      r$   local_guvectorizez+test_numba_local.<locals>.local_guvectorize  s    QArD   )r   r  r  r  r8   r   r*   )r  r  r  r  r  r4  r  rH  s           r$   test_numba_localr    s    
YYY  YY|dY+ , __d_# $ __\D_1 2 4 5 	 &I 099tnT"9F9v;#c&k**** :s   Cc                     t         j                  g dt         j                        } t         j                  g dt         j                        }t	        | j
                        t	        |j
                        k7  sJ y )NrY   rr   )rc   r|   rd   re   r8   rs   r   s     r$   test_tokenize_np_dtyper    sN    
((9BHH(
-C88IRXX8.D#))$tzz(BBBBrD   c                     t         j                  ddgd      } t        |        t        |       }t	        d t        |      D              sJ y )NrU   rV   zstring[pyarrow]rr   c              3  8   K   | ]  }t        |      d k(    yw)r  N)r^   ).0r@   s     r$   	<genexpr>z5test_tokenize_pandas_arrow_strings.<locals>.<genexpr>  s     ?s3x8#?s   )rf  ro  r8   r   anyr   )serrH  s     r$   "test_tokenize_pandas_arrow_stringsr    sC    
))S#J&7)
8C3S!F?wv????rD   c                      e Zd ZdZd Zy)APickleabler   c                D    t         xj                  dz  c_        t         dfS )NrZ   rG   )r  counterrA   s    r$   rI   zAPickleable.__reduce__  s    q BrD   N)rL   rM   rN   r  rI   rG   rD   r$   r  r    s    GrD   r  c                 X    t               } t        |        t         j                  dk  sJ y r   )r  r   r  r_   s    r$   test_normalize_pickler    s&    AQK !###rD   c                      G d d      } t        j                  t              5  t         |        d       d d d        y # 1 sw Y   y xY w)Nc                      e Zd Zd Zy)Btest_tokenize_recursive_respects_ensure_deterministic.<locals>.Fooc                (    t        t                     S r>   r  rA   s    r$   rK   zTtest_tokenize_recursive_respects_ensure_deterministic.<locals>.Foo.__dask_tokenize__  r  rD   NrR   rG   rD   r$   r  r    r  rD   r  Tr   )r    r!   RuntimeErrorr   r  s    r$   5test_tokenize_recursive_respects_ensure_deterministicr    s;    & & 
|	$ 3T23 3 3s   A  A	c                 :   g } t        d      D ]  }|| g}  i }t        d      D ]  }||i} t        |        t               5 }t        d      D cg c]  }|j                  t        |        }}t	        |D ch c]  }|j                          c}      dk(  sJ t        d      D cg c]  }|j                  t        |       }}t	        |D ch c]  }|j                          c}      dk(  sJ 	 d d d        y c c}w c c}w c c}w c c}w # 1 sw Y   y xY w)Nr  2   rk   rZ   )r	  r   r   submitr   result)nested_listixnested_dictpoolr  futuresr   s          r$   )test_tokenize_nested_sequence_thread_safer    s   KCj (;'( KBi (;'( K 		 7FKDkR4;;<RR01AHHJ01Q666FKDkR4;;<RR01AHHJ01Q666	7 7R0R0	7 7sB   DC=-D8D
D(DDD
'D=DDc                   	
 t        j                  d      } t        j                  d      }|j                  dd      }|j                  dd      }|j                  dd      }t        |      t        |      k(  sJ t        |      t        |      k7  sJ |j                  ddd      }|j                  ddd      }t        |      t        |      k7  sJ |j                  dd	      }t	        | j                  d            }t        |d   t              sJ t        |d         

d
k\  sJ 	
fd	 	t	        |             y )Nr   r   r   r      r[   )stepr\   i@B     c                   t        | t        t        f      r| D ]
  } |        y t        | t              r)| j	                         D ]  \  }} |        |        y t        | t
              rt        |       k(  rt        d      y y )Nzfound a hash)r]   listr  dictitemsr^   r   rT   )rH  tokenkeyvalueassert_no_hasheshex_hash_lens       r$   r  z3test_tokenize_range_index.<locals>.assert_no_hashes  s    ftUm, ( '(%$lln (
U % '( $6{l*$^44 + %rD   )	r    importorskip
RangeIndexr8   r   zerosr]   r^   r   )rc   rf  rU   rV   r   r   rv  largearray_tokenr  r  s            @@r$   test_tokenize_range_indexr    s+   			W	%B			X	&B
aA
aA
aA!q 1111!q 1111
a!$A
a!$A!q 1111MM!W%E!"((2,/Kk!nc*** {1~&L2
5 _U+,rD   r  r   )
__future__r   r/  r   r   r  r  r  r?   r  r  r  concurrent.futuresr   enumr   r   r   r   typingr	   r+   r    packaging.versionr
   tlzr   r   r   r(   dask._compatibilityr   	dask.corer   r   r   r   r   r   
dask.utilsr   dask.utils_testr   r   ddrc   ro  r  rf  r   fixturer%   r8   rW   r`   markskipifrh   rn   ru   ry   r}   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r%  r5  r7  getattrr^   r   __str__r   r   rG  parametrizerB  rE  rI  xfailrP  rU  r   r   r   r\  r^  rl  rq  ru  ry  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  	dataclassr%  r*  r-  r2  rG  rS  rW  rY  r]  r&  r`  rb  filterwarningsrh  rt  rw  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rG   rD   r$   <module>r4     s
   "         
  1 - -    % ' '  * & F F  *L!$%GN#IHw 
$ 
$>>(
 HH H HI I H; ; H@ @
 HF F HQ Q HJ J H " H7 7 H< <2 HE E8 H	6 	6 H7 7" H1 1./62
'/>  HV  HH	JJ	  !1!34 5 !34 5+ OPeT]36 4 Q66			4L H2 2, H 
 H
 
 H  H  H 0 H  H
 
 H M<<+*%02 >$ &	( 		L4,P2'44 tWgt&DEC FC       + + + 
S. '&/!*U  : LL H6 6 H 
 !T3c2tRRsC	HR HIJ2 K 2 HX.2 / 224 2238'FT+\	4H2 H2 2 |v||Kv{{/A/A(/KLM2	2,2 H2 2 H2 2 H2 2 H2 2 HB B 
9 
9 LE D  <$' ( $   - . M?K0 1
 " K 1, 2 !, K + !+
 K + !+@ HC C H@ @ $37"%-rD   