bi~ddlmZddlZddlZddlmZddlmZmZddl m Z ddl m Z m Z mZmZmZmZmZdZdZdd Zd Zdd Z dd Zd ZdZddZGddeZej:ZddeeeeefdZy)) annotationsN)Enum)configutils) GraphNode)flattenget_dependencies ishashableistask reverse_dictsubstoposortcTt|ttfs|g}t}t}i}ttt |}|rWg}|D]K}t ||d}||||<|||<|D])} | |vs|j | |j| +M|}|rW||fS)a<Return new dask with only the tasks required to calculate keys. In other words, remove unnecessary tasks from dask. ``keys`` may be a single key or list of keys. Examples -------- >>> def inc(x): ... return x + 1 >>> def add(x, y): ... return x + y >>> d = {'x': 1, 'y': (inc, 'x'), 'out': (add, 'x', 10)} >>> dsk, dependencies = cull(d, 'out') >>> dsk # doctest: +ELLIPSIS {'out': (, 'x', 10), 'x': 1} >>> dependencies # doctest: +ELLIPSIS {'out': ['x'], 'x': []} Returns ------- dsk: culled dask graph dependencies: Dict mapping {key: [deps]}. Useful side effect to accelerate other optimizations, notably fuse. Tas_list) isinstancelistsetdictrr addappend) dskkeysseen dependenciesoutworknew_workkdependencies_kds L/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/optimization.pycullr#s6 dT3K (v 5D6L C GDM" #D  'A-c1dCNVCF,LO# 'D=HHQKOOA& '  '   ct|d}|turM|dddDcgc]}tj|}}|j |ddj |S|t urt|ddkDrpt|ddtrZ|dddDcgc]}tj|}}|j |dddj |f|dddzSycc}wcc}w)zCreate new keys for fused tasksrN-) typestrr key_splitrjointuplelenr)rtypxnamess r"!default_fused_linear_keys_renamerr2Gs tAw-C cz-1%1R%[9#99 T!Wxx #d1g,*z$q'!*c/J-1%1R%[9#99 T!WQZ !DGABK//::s C.C3c |7t|ts't|ts|g}tt|}||Dcic]}|t ||d}}i}t}|D]k}||}t |dkD} |D]Q} || |vr|j | | |vr|| =|j | 4| r|j | H| |vsM||| <Smg} ttt|j} |r|j\} }| |g} ||vr*|j|}| |=| j|||vr*| j| | vr*| j| } || =| j| | | vr*| j| |r|jDcic]\}}|t|}}}|durt}n |durd}n|}i}t}t}d}| D]} ||| }|duxr ||vxr||v}| j} || }| rm| j}||j!|j| ||j#| t%||| |}|j | |} | rm|j | |r*||<||| <|| ||<|h|| <|j | ||| <|jD]\}}||vs |||<|rn|jD]I\}}||zD]<}||}|j#||j |t%||||||<>K|||z D]}||=||= ||fScc}wcc}}w)aReturn new dask graph with linear sequence of tasks fused together. If specified, the keys in ``keys`` keyword argument are *not* fused. Supply ``dependencies`` from output of ``cull`` if available to avoid recomputing dependencies. **This function is mostly superseded by ``fuse``** Parameters ---------- dsk: dict keys: list dependencies: dict, optional {key: [list-of-keys]}. Must be a list to provide count of each key This optional input often comes from ``cull`` rename_keys: bool or func, optional Whether to rename fused keys with ``default_fused_linear_keys_renamer`` or not. Renaming fused keys can keep the graph more understandable and comprehensive, but it comes at the cost of additional processing. If False, then the top-most key will be used. For advanced usage, a func is also accepted, ``new_key = rename_keys(fused_key_list)``. Examples -------- >>> def inc(x): ... return x + 1 >>> def add(x, y): ... return x + y >>> d = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')} >>> dsk, dependencies = fuse(d) >>> dsk # doctest: +SKIP {'a-b-c': (inc, (inc, 1)), 'c': 'a-b-c'} >>> dsk, dependencies = fuse(d, rename_keys=False) >>> dsk # doctest: +ELLIPSIS {'c': (, (, 1))} >>> dsk, dependencies = fuse(d, keys=['b'], rename_keys=False) >>> dsk # doctest: +ELLIPSIS {'b': (, 1), 'c': (, 'b')} Returns ------- dsk: output graph with keys fused dependencies: dict mapping dependencies after fusion. Useful side effect to accelerate other downstream optimizations. NTrr(F)rrrrr r.rrmapreverseditemspopitempoprreverser2updateremover )rrr rename_keysr child2parent unfusibleparentdepshas_many_childrenchildchains parent2childchainv key_renamerrvfusedaliases is_renamednew_keyvalkeyold_keys r" fuse_linearrPVs `  4 5$%6D74=!KNOa+CDAAO OLI -F#IM -EETM e$,& ' e$" e$i'&, U# - -FHl&8&8&:;?$!QAs1vI?L?d7   !  B EEeGJ  "!%(Gt#Ps(:Pwb?P  %jYY[F  ' ' (8(8(? @  ' ' .s6{E3/C IIe E   % BwKBuI$0$7L !#*)L  KK BuI/0IIKS e BsG%++- :IC'> :W+ G$!r#w93  : :  ~ &sG % & | qPH@s M+)M0c| tSt|tr|St|ttfs|g}t|SN)rrr)r0s r" _flat_setrSs;yu As  D#; ' C q6Mr$cr]tttjtr-j Dcic]\}}|t |c}}t|}Dcic]}|t|c}|r)|jfdj Dt|Dcic] }|vs||c}}i}|D]5}|} ||zD]} | |vr|| } n| } t| | | } | ||<7|j} j D],\}} || vs ||zD]} t| | || } | | |<.| Scc}}wcc}wcc}w)aReturn new dask with the given keys inlined with their values. Inlines all constants if ``inline_constants`` keyword is True. Note that the constant keys will remain in the graph, to remove them follow ``inline`` with ``cull``. Examples -------- >>> def inc(x): ... return x + 1 >>> def add(x, y): ... return x + y >>> d = {'x': 1, 'y': (inc, 'x'), 'z': (add, 'x', 'y')} >>> inline(d) # doctest: +ELLIPSIS {'x': 1, 'y': (, 1), 'z': (, 1, 'y')} >>> inline(d, keys='y') # doctest: +ELLIPSIS {'x': 1, 'y': (, 1), 'z': (, 1, (, 1))} >>> inline(d, keys='y', inline_constants=False) # doctest: +ELLIPSIS {'x': 1, 'y': (, 'x'), 'z': (, 'x', (, 'x'))} c3fK|](\}}t|r|vs|st|s|*ywrR)r r ).0rrFrrs r" zinline..s7 11 !s(LOFSTI  s.1)r) rnextitervaluesrr6rrSr r:rr copy)rrinline_constantsrrrF replaceorderkeysubsrNrMdepreplacedsk2items` ` r"inlinercs2 4\-@-@-B(C#DdK.:.@.@.BCda3q6 C T?D=@A+C33A       -qAHCF-LLG#h,s++ *Cg~!#,c(sC)C  *   <<>DIIKS d?|C00 53gdm4 5DI  KKD B .sE&3E,> E1E1cD sStt|Dcic]}|t|}}t| fd}jDcgc]\}}|||s|}}}|rt ||||D]}|=Scc}wcc}}w)aInline cheap functions into larger operations Examples -------- >>> inc = lambda x: x + 1 >>> add = lambda x, y: x + y >>> double = lambda x: x * 2 >>> dsk = {'out': (add, 'i', 'd'), # doctest: +SKIP ... 'i': (inc, 'x'), ... 'd': (double, 'y'), ... 'x': 1, 'y': 1} >>> inline_functions(dsk, [], [inc]) # doctest: +SKIP {'out': (add, (inc, 'x'), 'd'), 'd': (double, 'y'), 'x': 1, 'y': 1} Protect output keys. In the example below ``i`` is not inlined because it is marked as an output key. >>> inline_functions(dsk, ['i', 'out'], [inc, double]) # doctest: +SKIP {'out': (add, 'i', (double, 'y')), 'i': (inc, 'x'), 'x': 1, 'y': 1} ct|tsHt|r=|vr9|r4 t|j rt fd|Dsyyy#t $rYywxYw)Nc3DK|]}t|tywrRrr)rVr!rs r"rWz6inline_functions..inlinable..as#K67Js1vy1K TF)rrr functions_ofissubsetany TypeError)rNtask dependentsrfast_functionsoutputs r" inlinablez#inline_functions..inlinableYsw4+t 6!3 %..~>sK;Ec?KH   s1A A'&A')r\r)rr r r6rc) rrpror\rrrqrFrrns ``` @r"inline_functionsrr3s6  [F(N=@A+C33A Al+J ))+ 9$!Q1aA 9D 9  (8|  AA  J7B& :sB!B3BcPt|dr|j}t|dr|S)Nfunc)hasattrrt)rts r"unwrap_partialrvts& $ yy $  Kr$ct}|g}tth}|rjg}|D]^}t||vst |r2|j t |d|j|ddN|j|`|}|rj|S)a1Set of functions contained within nested task Examples -------- >>> inc = lambda x: x + 1 >>> add = lambda x, y: x + y >>> mul = lambda x, y: x * y >>> task = (add, (mul, 1, 2), (inc, 3)) # doctest: +SKIP >>> functions_of(task) # doctest: +SKIP set([add, mul, inc]) rr(N)rrr-r)r rrvextend)rmfuncsrsequence_typesrs r"ririzs EE 6DE]N  *DDz^+$<IInT!W56OODH-OOD)  *  Lr$ct|}t|}t|}rdzfd}|tur}t j |}|Dchc]}t j |}}|j |t|}|j|dj|} || S|turt|dkDrt|dtrt j |}|Dchc]}t j |}}|j |t|}|j|ddj|} || f|ddzSyyycc}wcc}w)zCreate new keys for ``fuse`` tasks. The optional parameter `max_fused_key_length` is used to limit the maximum string length for each renamed key. If this parameter is set to `None`, there is no limit. c\r(t|kDrt|ddd}|dd|}|S)Nr0r')r.hash)key_name name_hashmax_fused_key_lengths r"_enforce_max_key_limitz:default_fused_keys_renamer.._enforce_max_key_limitsF CM4H$H>!,bq1I"#8$89:!I;GHr$r'rr(N) r5rXr)r*rr+discardsortedrr,r-r.r) rrit first_keyr/r first_namerr1concatenated_names ` r"default_fused_keys_renamerrsL $BRI y/C!  cz__Y/ -/0#00 j!u  YHHUO%&788 #i.1,IaL#1N__Y/ -/0#00 j!u  Yq\"HHUO&'89;imKK2O, 11s E'6E,ceZdZdZddZy)Defaultrcy)Nz )selfs r"__repr__zDefault.__repr__sr$N)returnr*)__name__ __module__ __qualname__tokenrrr$r"rrs Er$rc tjddur|fS|7t|ts't|ts|g}tt |}|t urtjd}|t usJ|t urtjd}|t usJ|t urtjd}|t usJ||dz}|t urtjd}|t usJ|d|tj|d zzz}|r|s|fS|t urtjd }|t usJ|d urt}n|durd}nt|s td |}|du}|D cic]} | t| d } } nA|jD cic]&\} } | t| tr| n t| d (} } } i} | jD]9\} } | D]!} | | vr| g| | <| | j| #t| | | <;t}| jD]\} } t| d k(s| |xsdvs| vs#t| t r7t#| t$us$t| t&j(t*fsot-fd| Dr|j/| |s$t1d| j3Dr| fSj5}i}g}g}| j6}|j.}|j6}|j8}|j6}|j}|j6}|j}|j:}|j6}|r|}||||vr | |d}||vr ||||| |z |d}||k7rK|| |z}|r|||}|d}|| |z}|r||||||r|gndd d d d| ||z fn|| |} | |z }!| |!z }t|}"|"d k(rG|\}#}$}%}&}'}(})}*t|*}+|)|+d z cxkDrdk\rnn|+d z })|!|*z}!t|!|+k(},|,s|)d z })|(|)z|&z |kr|,s|&|krt|t st=||#|$}-| j9|#| ||#z} ||#=||#|r|%j||%||<||#d|r,|,r|||-|%|&|'|(|)|!fn|||-|%|&d z|'|(d z|)|!fng|-||<nj|$||#<||#|r9|)t?|d z kDrt?|d z })|||||r|gndd |'d |)|!fnn g}%d }&d}'d}.d}(d})t}*d}/||" d}0||" d=|0D]J\}1}1}1}2}3}4}5}6|2d k(r|.d z }.n|2|&kDr|2}&|'|3z }'|(|4z }(|)|5z })t|6|/kDr t|6}/|*|6z}*Lt|*}+|)tA|"d z tCd|+|/z z })|)|+d z cxkDrdk\rnn|+d z })|!|*z}!t|!|+k(},|,s|)d z })|(|)z|&z |kr|.|kr|'|kr|&|kr|,s|&|krt|t st-fd|Ds|}-t}7|0D]M}8|8d}9t=|-|9|8d }-||9=|7||9z}7||9|s1||9d|%j;|8dO| |z} | |7z} |r|%j||%||<|r|||-|%|&d z|'|(d z|)|!fne|-||<ni|0D]}8|8d ||8d<||8d|r?|'|kDr|}'|)t?|d z kDrt?|d z })|||||r|gndd |'d |)|!fnn | |d}|r|rC|jD]0\}:};||;}<|<|<|vs||:||<<|<||:<| |:| |<<|.OsD! 3q695Drhc3JK|]}tt|dk7yw)r(N)r.r)rVrFs r"rWzfuse..SsG1c#a&kQ.Gs!#rr&c3DK|]}t|tywrRrg)rV child_keyrs r"rWzfuse..%s$($-!+3y>9 E(rh)"rgetrrrr_defaultmathlogrcallablerlr r6rr.rr)r-numbersNumberr*rkrallrZr[r8r;rxr intminmax)=rrr ave_width max_width max_heightmax_depth_new_edgesr<rGrr@rFrdepsvals reduciblerH fused_trees info_stackchildren_stackdeps_pop reducible_add reducible_popreducible_removefused_trees_popinfo_stack_appendinfo_stack_popchildren_stack_appendchildren_stack_extendchildren_stack_popr?rBchildren deps_parentedges num_childrenr child_task child_keysheightwidth num_nodesfudgechildren_edgesnum_children_edges no_new_edgesrMnum_single_nodes max_num_edges children_info_ cur_height cur_width cur_num_nodes cur_fudge cur_edges children_deps child_info cur_childroot_key fused_keysaliass=` r"fusers/ Dzz,-6L    4 5$%6D74=!HJJ<= (((XZZ >? )))h&$jj)PQ"(222"'#oHJJ<= ((()dhhy1}&=== JL  hjj!@A (***d0   k "CDD! T)KCFGa#CD99GG%**, 1 Jq$'q-=c1d-S S  E::<4 #A~3aa"  # d)Q I;;= 4 IN$*"%Ss1vy1c!f&*SVgnnc=R*SDtDD MM!   #G GGDy BKJNxxHMMMMMM ''!ooO"))^^N*11*11'++ f !6]1%F ! f%i$v,67"2&E$tE{2)(3"F*2.E(4;6H  #$"5 #.DU i/  #$"6l #i/&."8} 1$'( !""!&),^)<&1A5:: 2Q 6^+E#&u:1C#CL' "U*f4 A&2E)E *3v; B #3v; :F#**95#x ':: yM(3&&--f52M"L=>2'4 "!%!!%?,1,'&0%/F*!]2 *y>M9,/ NM&)3'4**-^)<&S$q(#a1Cm1S*TE1A5:: 2Q 6^+E#&u:1C#CL' "U*f4 A, 9!Y."j0&2E)E *3v; B$'(19(%"&k(+ *7AJ(21 I"&sIz!}"EC "9 )Xi-@@M,Y7* / 4 @ * 1 1*Q- @A$x/ #}4 &&--f52rs" %0f Nb?FLP>B >"LNd ==  nr$