biDddlmZddlZddlZddlZddlZddlmZddlm Z m Z ddl m Z m Z ddlmZmZddlmZddlZddlmZdd lmZmZmZddlmZdd lmZdd lmZm Z m!Z!m"Z"m#Z#dd l$m%Z%m&Z&dd l'm(Z(m)Z)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;mZ>ejr ddlAmBZBmCZCnddlDmBZBmCZCd\dZEdZFdZGe=ed]dZHe=ed]dZIe"ejeje=ed^dZJd_dZLe"ejeje=ed^dZMd_dZOe=ed^dZPe=ed^d ZQe=ed]d!ZRe=ed]d"ZSe=ed_d#d$d%ZTe=ed_d#d$d&ZUe=ed^d'ZVd(ZWe=ed^d)ZXd*ZYejeFd+dfd,ZZejeFd+ddfd-Z[d`d.Z\e=ed]d/Z]e=ed]d0Z^d1ejeFd+ddfd2Z_d3Z`d1dd+ejddfd4Zad1dd+ejddfd5Zb dad6Zce=edad7Zde=e dad8Zed9Zfd:Zge=edad;Zhe=e dad<Zidbd=Zjd>Zkd\d?Zldcd@ZmdcdAZn d^dBZodCZpdDZqe=ed^dEZre=ed^dFZse=ed^dGZte=ed^dHZudIZvdJZwdddKZx dedLZydMZzdNZ{e=edfdOZ|e=edfdPZ}dgdQZ~dgdRZe=edhdSZe=edidTZe=edidUZe=e djdddVdWZdXZ dkdYZe=e djdddVdZZd[Zy#e?$rdZ>YwxYw)l) annotationsN)Iterable)partialreduce)productrepeat)IntegralNumber)mul)Version) accumulatedroppluck)chunk)Array _concatenate2 handle_out implementsunknown_chunk_message)arangediagonal) divide_lookupnannumel_lookup numel_lookup) NUMPY_GE_200) array_safe asarray_safe is_arraylikemeta_from_array validate_axis)oneszeros)tokenize)HighLevelGraph)applydeepmap derived_from) _tree_reduce reductionc d}tjttj|||}||||S)Nc.t|dtdS)N__array_priority__z-inf)getattrfloat)xs P/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/dask/array/reductions.pyzdivide..5sGA3U6]C)keydtype)rdispatchtypebuiltinsmax)abr5r3fs r0divider=4s8 CCtHLLA3$?@AA Q r2c t|fi|SN)rr/kwargss r0numelrB:s  $V $$r2c t|fi|Sr?)rr@s r0nannumelrD>s 1 ' ''r2Fc |>ttjd|jj dt }t |tjtj|||||}|S)Nr4r5axiskeepdimsr5 split_everyout)r-npr"r5sumobjectr)r)r:rHr5rIrJrKresults r0rMrMBs^ }!''2668'6J      F Mr2c ||}n>ttjd|jj dt }t |tjtj|||||SNrFr4r5rG)r-rLr!r5prodrNr)rr:rHr5rIrJrKdts r0rSrSSs`   RWWT1668'6 J       r2c jt|ttjt|||j|| SN)combinerHrIr5rJrK)r) chunk_minrminr5r:rHrIrJrKs r0rZrZe4    gg   r2c|jdk(r#tg||j|jSt j |||S)z-Version of np.min which ignores size 0 arraysrndminr5rHrI)sizerndimr5rLrZr/rHrIs r0rYrYu9vv{"aqvvQWW==vvadX66r2c jt|ttjt|||j|| SrW)r) chunk_maxrr9r5r[s r0r9r9}r\r2c|jdk(r#tg||j|jSt j |||S)z-Version of np.max which ignores size 0 arraysrr^r`)rarrbr5rLr9rcs r0rfrfrdr2c `t|tjtj||d||SNboolrG)r)ranyr[s r0rkrk/       r2c `t|tjtj||d||Sri)r)rallr[s r0rnrnrlr2c ||}nCttjtjd|j dt }t|tjtj|||||SrQ) r-rnansumrLr!r5rNr)rMrTs r0rprps`   U\\"''$agg">?& Q       r2c ||}nCttjtjd|j dt }t|tjtj|||||SrQ) r-rrprLr!r5rNr)nanprodrSrTs r0rrrrs`   U\\"''$agg">?& Q       r2 sequential)methodc ~ttjtjd|||||t j  S)aiDask added an additional keyword-only argument ``method``. method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumsum. Default is 'sequential'. * 'sequential' performs the cumsum of each prior block before the current block. * 'blelloch' is a work-efficient parallel cumsum. It exposes parallelism by first taking the sum of each block and combines the sums via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. rrKrtpreop) cumreductionr nancumsumoperatoraddrLrpr/rHr5rKrts r0ryrys8       ii  r2c ~ttjtjd|||||t j  S)atDask added an additional keyword-only argument ``method``. method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumprod. Default is 'sequential'. * 'sequential' performs the cumprod of each prior block before the current block. * 'blelloch' is a work-efficient parallel cumprod. It exposes parallelism by first taking the product of each block and combines the products via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. rFrv)rxr nancumprodrzr rLrrr|s r0r~r~s:       jj  r2c tj|jrtdt|jdk(r tdt |t t |||j||S)N Arrays chunk sizes are unknown. rzAzero-size array to reduction operation fmin which has no identityrG)rLisnanra ValueErrorrr) _nanmin_skipr5r[s r0nanminr m xx;rrf8c @|r|S||fd|i|}||fd|i|}||dS)Nr5ntotal)r/rMrBr5computing_metarArrs r0 mean_chunkrKs? a'u''A  ) )& )EU ##r2c t|ts|g}|s td|n|}t||jdd|i|}|r|Std|} t| |jdd|i|} || dS)Nc |dSNrrpairs r0r1zmean_combine..c d3ir2axesrHc |dSNrrrs r0r1zmean_combine..i $w-r2rr) isinstancelistr&rrM) pairsrMrBr5rHrrAnsrtotalsrs r0 mean_combinerWs eT "7E' /5B( bt$((=d=f=A / 7F 0M&t , 0 0 Ed Ef EEU ##r2c H|s td|n|}t||}tj|f||d|}|r|Std|}t||jd||d|}tjdd5t |||cdddS#1swYyxYw) Nc |dSrrrs r0r1zmean_agg..prr2rrHr5c |dSrrrs r0r1zmean_agg..wrr2rr=invalidr4r)r&rrLrMerrstater=) rr5rHrrArrrrs r0mean_aggros7E' /5Bbt$A q3t53F3A / 7F 0M&t , 0 0 Rd% R6 RE Hh 7-eQe,---s BB!c ||}n]|jtk(rt}nCttjtj d|jdt}t |tt||||t|d S)NrRshaper5r5FrHrIr5rJrXrK concatenate) r5rNr-rLmeanr"r)rrrrTs r0rr~so   F   RWWRXXD@A7F S      r2cL||}nCttjtjd|jdt }t |tttjtt|||||dtttjt S)NrRrr5)rMrBFrHrIr5rJrKrrX)r-rLrr!r5rNr)rrrrprDrrrTs r0nanmeanrsx   RWWRWW4qww?@'6 R    H=   %,,hG  r2c |r|S||fi|}|jtj}|r ||fi|} n ||fd|i|} tjdd5| |z } ddd| z } tj|j tj rtj| } td|dzD cgc]} || | zfd|i|} } tj| d}| ||dS#1swYxYwcc} w) Nr5rrrrFrHrrM) astyperLint64r issubdtyper5complexfloatingabsrangestack)AorderrMrBr5rimplicit_complex_dtyperArrudixsrs r0 moment_chunkrs a6A AA  A-U-f- Hh 7 AI AA }}QWWb001 FF1I49!UQY4G Hq#ad *% *6 * HB H "A ++ IsC6D6C?c V|d|dz fjdd|i|||||zzfd|i|z}td|dz D]h}tj|tj|tj||z zz } || ||d||z dz f||zzfd|i|zz }j|S)N.rrHrFr)rMrmath factorial) Msr inner_termrrMrHrArkcoeffs r0_moment_helperrs3 >3D3F3c Z 7%)7-37 A1eai Vu%):T^^ETUI=V)VW USCQ./*a-?UdUfU UUV Hr2c t|ts|g}d|d<d|d<|s td|n|}t||}|jdd|i|} |r| Sttd||} ttd||} | jdd|i|} t j d d 5t j| jt jr/t| | } t jt| || z }nt| | | } t| || | z }dddtd |d zDcgc]}t| |||||}}t j|d}| | |dS#1swYQxYwcc}w)Nr5TrIc |dSrrrs r0r1z moment_combine..rr2rrHc |dSrrrs r0r1z moment_combine.. W r2c |dSNrrrs r0r1z moment_combine.. DIr2rrr4rrFrrrr)rrr&rrMrLrrr5rr=rrrr)rrddofr5rMrHrrArrrrrmurorrs r0moment_combiners eT "F7OF:7E' /5B r %B#D#F#A 7#=uED QF w5u=D IB FJJ +D +F +E Hh 7> ==b&8&8 9q!Bvr 2R 78Jq.B%82=J >q%!)$   r2z1c4@ B  "A ++>> s/A=E<F<Fc t|ts|g}||d<|j}d|d<d|d<|s td|n|} t | |} | j d d|i|} |r| St td||} t td||} t | j d d|i|| } tjd d 5tj| jtjr#tjt | | | z }nt | | | | z }dddt| | ||||}| j d d|i||z }t|tr|d krCtj}n2|tj j"urtj||d k<t ||| S#1swYxYw)Nr5TrIc |dSrrrs r0r1zmoment_agg..rr2rrHc |dSrrrs r0r1zmoment_agg..rr2c |dSrrrs r0r1zmoment_agg..rr2rrr4rr)rrcopyr&rrMr=rLrrr5rrrr nanmamasked)rrrr5rMrHrrA keepdim_kwrrrrrrr denominators r0 moment_aggrs eT "F7OJ!JzJw7E' /5B r %B'D'J'A 7#=uED QF w5u=D IB   33 3Q 7B Hh 7> ==r'9'9 :vr 2R 78J%82=J > r2z5#tVDA!%%,T,V,t3K+v& ?&&K BEELL (')vv K!O$ ![ ..#>>s A#GGc|t|tr|dkr td|dkrq|j|}|dk(r-t |j |j d|jSt|j |j d|jS||} nCttjtjd|j d t} |duxrtj|} t|t!t"|| t!t$|| ||| ||d t!t&| S)a Calculate the nth centralized moment. Parameters ---------- a : Array Data over which to compute moment order : int Order of the moment that is returned, must be >= 2. axis : int, optional Axis along which the central moment is computed. The default is to compute the moment of the flattened array. dtype : data-type, optional Type to use in computing the moment. For arrays of integer type the default is float64; for arrays of float types it is the same as the array type. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original array. ddof : int, optional "Delta Degrees of Freedom": the divisor used in the calculation is N - ddof, where N represents the number of elements. By default ddof is zero. Returns ------- moment : Array References ---------- .. [1] Pebay, Philippe (2008), "Formulas for Robust, One-Pass Parallel Computation of Covariances and Arbitrary-Order Statistical Moments", Technical Report SAND2008-6212, Sandia National Laboratories. rzOrder must be an integer >= 0rrr)chunksr5metaNrRrr5)rr)rrF)rr)rr rrMr!rr_metar"r-rLvarr5rN iscomplexobjr)rrrr) r:rrHr5rIrrJrKreducedrUrs r0momentr0sL eX &%!)899 qy%%T%" A: gnnDw}}  MM'..7==     RVVBGG$agg>?& Q"d]Arq/A   >T   %d3  e4  r2c 8||}nCttjtjd|jdt }|duxrtj |}t|tt|tt|||||td|d S) NrRrr5)r)rrF)rHrIr5rJrXnamerKr) r-rLrr!r5rNrr)rrrr r:rHr5rIrrJrKrUrs r0rr|s   RVVBGG$agg>?& Q"d]Arq/A   5KL &     r2c ||}nCttjtjd|jdt }|duxrtj |}t|tttjt|tttj|||||tttj|d S) NrRrr5)rMrBr)rMr)rMFr)r-rLrr!r5rNrr)rrrrprDrrrs r0nanvarrs   RVVBGG$agg>?& Q"d]Arq/A    #9    5 BII6  r2ct|tjjr@|js4|j j rtjjStj|Sr?) rrLr masked_arrayrmaskrnrsqrtr:s r0_sqrtrsD!RUU''(QVVZZ\uu|| 771:r2c\t|dr|jt|St|S)aA version of sqrt that properly handles scalar masked arrays. To mimic ``np.ma`` reductions, we need to convert scalar masked arrays that have an active mask to the ``np.ma.masked`` singleton. This is properly handled automatically for reduction code, but not for ufuncs. We implement a simple version here, since calling `np.ma.sqrt` everywhere is significantly more expensive. _elemwise)hasattrr rrs r0 safe_sqrtr s(q+{{5!$$ 8Or2c tt|||||||}|r ||jk7r|j|}|SN)rHr5rIrrJrK)r rr5rr:rHr5rIrrJrKrOs r0stdrsL  #  F &,,&u% Mr2c tt|||||||}|r ||jk7r|j|}|Sr )r rr5rrs r0nanstdrsN #  F &,,&u% Mr2c t|trV|dj|djk(sJt||djk(s|djdk(rdn|d}n.t||jk(s|jdk(rdn|d}|d}|d}|5||||}|j |}|j |}||fS|||}t t jttt|j}|j||t|}||}||}|r,t j||}t j||}||fS)z3Merge intermediate results from ``arg_*`` functionsvalsargrFNrr`r)rdictrblenravelrrLogridtuplemapslicerinsert expand_dims)datarHargfuncrIrr local_argsindss r0 _arg_combiner"sj$F|  DK$4$44444yDL---f1B1Ba1G a 4yDII-atT!W .>#?@AB D*%T{Dz$i >>$-D..d+C 9r2ctt||jk(s|jdk(rdn|d}|||d}|||d}|jdkDrs|l|\}} tj|j d|j } t dt|| D} tj| | |ddn||z }t|tjjrmd|jvr tjj|} ntjj|} tjj|| } tj ||j d|j"fd|j"fg } || d<|| d<| S#t$$r t'} Y!wxYw) NrFrTr`c3,K|] \}}||zywr?r).0rrs r0 zarg_chunk..sCAa!eCsrZrrr)rrbrL unravel_indexrrrzipravel_multi_indexrrr__name__minimum_fill_valuemaximum_fill_valuefilled empty_liker5 TypeErrorr)funcrr/rH offset_infoarg_axisrroffset total_shapeind total_ind fill_valuerOs r0 arg_chunkr8 sx4yAFF*affkttAwH 4 0D !(T 2Cvvz  "- FK""399;q>177;CC#fc2BCCI)))[ACF ; C$**+ G$$ $11$7J11$7Juu||D*-  FDJJ+?%AS*T F6NF5M M ss dGh ? BBr2c t||||\}}tjtj|r t d|S)Nr:zAll NaN slice encountered)r"rLrkrr)rrrHrIrArrs r0 nanarg_aggr?Bs;T48DIC vvbhhtn455 Jr2c |!tt|jd}nHttr)t |jf|jdk(}nt ddD]N} |j| } t| dkDs!tj| jsEtddt||||} |j} tt!t#t|j$} tt!d|jD}|r t'|t)|j*}nt-d |}tfd t/|jD} t'| |Dcic]\}}| f|z|| f|z|f}}}tj0t3dgt5| }d}t7|r|}|j8}t;j<| ||g }t?|| | || }tA||||||}tC||Scc}}w)a7Generic function for argreduction. Parameters ---------- x : Array chunk : callable Partialed ``arg_chunk``. combine : callable Partialed ``arg_combine``. agg : callable Partialed ``arg_agg``. axis : int, optional split_every : int or dict, optional NTrFz(axis must be either `None` or int, got ''zArg-reductions do not work with arrays that have unknown chunksizes. At some point in your computation this array lost chunking information. A possible solution is with x.compute_chunk_sizes()z arg-reduce-c3XK|]"}ttj|ddd$yw)Nrr)r rzr{)r%bds r0r&z arg_reduction..ss#UbZ b"gqAUs(*rc3LK|]\}}|vrdt|zn|yw)rRN)r)r%rcrHs r0r&z arg_reduction..ys(W!QAI4#a&=14Ws!$r dependencies)r5r)rIr5rJrX)"rrrbrr r r/rrrLrrkrr#rrrr numblocksr(rrr enumerateargminrrrr5r$from_collectionsrr(r)r/rrXaggrHrIrJrKraxrroldkeysoffsetsr1roffdskr5rgraphtmprOs ` r0 arg_reductionrUIs#" |U166]# D( #T166*w! B4&JKK " v;?rxx/335,  $5';GH ID &&C UAKK01 2D7UAHHUVWG '6!''?3 DGW- W9QXXCVW WFD+.  Q ! ecVaZs33 C  IIlA3_Q-?@ AE DE   + +D#QC HE tV5t d }d }||krst|d z ||D]N} g}t-| | || |z || D])\}}}||z|| fz}|||f||<|j/|+||| <P|}|d z}|d z }||krst1j2d d t5j6t5j8|d zz}|d z}|dkDrvt||zd z ||D]N} g}t-| | || |z || D])\}}}||z|| fz}|||f||<|j/|+||| <P|}|d z}|d z }|dkDrv|r||dD]!}t:||j(f|z||f|||z<#t-t=d ||D]:\}}t-||D]&\}}t>||||j(f|z||f|||z<(<t+|d kr|g}n|| g}tAjB||| }tE|||jF| j } tI|| Scc} } wcc} } } wcc}wcc}}w) aGeneric function to perform parallel cumulative scan (a.k.a prefix scan) The Blelloch prefix scan is work-efficient and exposes parallelism. A parallel cumsum works by first taking the sum of each block, then do a binary tree merge followed by a fan-out (i.e., the Brent-Kung pattern). We then take the cumsum of each block and add the sum of the previous blocks. When performing a cumsum across N chunks, this method has 2 * lg(N) levels of dependencies. In contrast, the sequential method has N levels of dependencies. Floating point operations should be more accurate with this method compared to sequential. Parameters ---------- func : callable Cumulative function (e.g. ``np.cumsum``) preop : callable Function to get the final value of a cumulative function (e.g., ``np.sum``) binop : callable Associative function (e.g. ``add``) x : dask array axis : int dtype : dtype Returns ------- dask array rrrr4r5-T)rHrIr5rrFrF)%flattenrechunk npartitionsr-rLr!r5rNrr r rbr*r# map_blocksrrHrrrIrrr(appendr8r9rceillog2rlrrjr$rKrrr)!r0rwrhr/rHr5rKrbase_keybatchesrjnbindices last_index full_indicesrindex prefix_valsrRn_valslevelstridestride2new_valsleft_val right_valr3indexesvaldepsrSrOs! r0prefixscan_blellochr sH: | IIK  q}}  5 }RWWT9:GVL dH %% % qvv &Dmm_AhtT5%EJK LDwHll5td%lHGq{{4() ++  ?H?UVeaqDy%)qc1V  +Wj<LSS4>W\\Oe+>SKS C  F E {7Q;8 *25AJ AJ 7Q3).E8Y#U*eQZ7C %x;CHOOC( ) "* A *F qLG QJE",,q!tyy6Q;1G'H"HIAqj7V+a/A *25AJ AJ 7Q3).E8Y#U*eQZ7C %x;CHOOC( ) "* A *G qLF QJEqj!!_ E! E! %C5 ! !a!6 D MGT!'40  s'VVI%)Hu$%   <1s7|  + +D#D IE 5$'-- 8F c6 ""QW+?Ss0"'N N& N N' N" N'N"N'c f|dk(r| tdt|||||S|dk7rtd|;jdkDr*j j j d |6t|tjd j d t}ttsJtjj|| } |j dt#||||} j$} t'ddd} | fzt'ddfz| fjz dz zz} t)t+t-j$Dcgc]\}}|k7r t/|nd gc}}}t1}|D]p}t3fdt-|D}t4tj6j8|| jfd|if|| df|z<| j:f|z|| f|z<rt/d| D]}|}t)t+t-j$Dcgc]\}}|k7r t/|n|gc}}}t=||D]R\}}| df|z}|| df|zt>j@| j:f|z| ff||<||| j:f|zf|| f|z<TtCjD| || g}tG|| jH| jj8}tK||Scc}}wcc}}w)aGeneric function for cumulative reduction Parameters ---------- func: callable Cumulative function like np.cumsum or np.cumprod binop: callable Associated binary operator like ``np.cumsum->add`` or ``np.cumprod->mul`` ident: Number Associated identity like ``np.cumsum->0`` or ``np.cumprod->1`` x: dask Array axis: int dtype: dtype method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumsum. Default is 'sequential'. * 'sequential' performs the scan of each prior block before the current block. * 'blelloch' is a work-efficient parallel scan. It exposes parallelism by first calling ``preop`` on each block and combines the values via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. preop: callable, optional Function used by 'blelloch' method, like ``np.cumsum->np.sum`` or ``np.cumprod->np.prod`` Returns ------- dask array See also -------- cumsum cumprod blellochNz>cumreduction with "blelloch" method required `preop=` argument)rKrszMInvalid method for cumreduction. Expected "sequential" or "blelloch". Got: rFrnrror4r5rrprc3VK|] \}}|k7rj||nd"ywrFNrn)r%riirHr/s r0r&zcumreduction..s,Veadahhqk"o9Vs&)rextrarF)r)&r/rrrbrqrrrsr-rLr!r5rNrr r rtr*r#rHrrrrIrrrr% full_likerrr(rzgetitemr$rKrrr)r0rhidentr/rHr5rKrtrwmrrfullslcrr{r|rRr5r last_indicesrrN this_slicerSrOs `` r0rxrxsIZ =P #4q$3OO < [\b[e f   | 66A: ##1==#9A }RWWT9:GVL dH %% % qvv &D TE 2Amm_AhtT5%EJK LD DA tT "D 'D.E"dO- -166D=1;L0M MCy?UVeaqDy%)qc1VWG &C-VyQT~VV  LL WWeQWW % e  & T7Oc !"  ffY_TGcM-1a[F  AJ1;;AWXvr2rTz%)s2X   L'2 FHC3.Jw#%!!AFF9s?C8C O #(affY_!EC#  FF  + +D#QC HE 5$!'' @F c6 ""?W"Ys 3L' L-ct|tjjs$t|tjjr}tjj |tjj |z}tjj|tjj |S||zSN)rrrLrrgetdata getmaskarrayr:r;valuess r0 _cumsum_merger!RUU''(Jq"%%:L:L,Mq!BEEMM!$44uu!!&ruu/A/A!/D!EE q5Lr2ct|tjjs$t|tjjr}tjj |tjj |z}tjj|tjj |S||zSrrrs r0_cumprod_mergerrr2c jttjtd|||||tj S)acDask added an additional keyword-only argument ``method``. method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumsum. Default is 'sequential'. * 'sequential' performs the cumsum of each prior block before the current block. * 'blelloch' is a work-efficient parallel cumsum. It exposes parallelism by first taking the sum of each block and combines the sums via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. rrv)rxrLcumsumrrMr|s r0rrs4      ff  r2c jttjtd|||||tj S)anDask added an additional keyword-only argument ``method``. method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumprod. Default is 'sequential'. * 'sequential' performs the cumprod of each prior block before the current block. * 'blelloch' is a work-efficient parallel cumprod. It exposes parallelism by first taking the product of each block and combines the products via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. rFrv)rxrLcumprodrrSr|s r0rrs4      gg  r2c t||j}ttj|}ttj |}t |||||d|j|t| S)aExtract the k largest elements from a on the given axis, and return them sorted from largest to smallest. If k is negative, extract the -k smallest elements instead, and return them sorted from smallest to largest. This performs best when ``k`` is much smaller than the chunk size. All results will be returned in a single chunk along the given axis. Parameters ---------- x: Array Data being sorted k: int axis: int, optional split_every: int >=2, optional See :func:`reduce`. This parameter becomes very important when k is on the same order of magnitude of the chunk size or more, as it prevents getting the whole or a significant portion of the input array in memory all at once, with a negative impact on network transfer too when running on distributed. Returns ------- Selection of x with size abs(k) along the given axis. Examples -------- >>> import dask.array as da >>> x = np.array([5, 1, 3, 6]) >>> d = da.from_array(x, chunks=2) >>> d.topk(2).compute() array([6, 5]) >>> d.topk(-2).compute() array([1, 3]) rT)rrX aggregaterHrIr5rJ output_size) r rbrrtopktopk_aggregater)r5r)r:rrHrJ chunk_combiners r0rr/skH qvv &D EJJ!,M,,2I   ggF  r2ct|jt|j|jft j }|tfdt|jD}|jtj|t}ttj|}ttj|}t!t"rd}n t%}|j&j)t j j+d|j|z dzz} t-||||dt j |dt/|| S) aExtract the indices of the k largest elements from a on the given axis, and return them sorted from largest to smallest. If k is negative, extract the indices of the -k smallest elements instead, and return them sorted from smallest to largest. This performs best when ``k`` is much smaller than the chunk size. All results will be returned in a single chunk along the given axis. Parameters ---------- x: Array Data being sorted k: int axis: int, optional split_every: int >=2, optional See :func:`topk`. The performance considerations for topk also apply here. Returns ------- Selection of np.intp indices of x with size abs(k) along the given axis. Examples -------- >>> import dask.array as da >>> x = np.array([5, 1, 3, 6]) >>> d = da.from_array(x, chunks=2) >>> d.argtopk(2).compute() array([3, 0]) >>> d.argtopk(-2).compute() array([1, 2]) )rr5c3^K|]$}|k(r tdntj&ywr?)rrLnewaxis)r%rrHs r0r&zargtopk..s$T19E$K"**<Ts*-r4rrFroTF) rrXrrHrIr5rJrrr)r rbrrrrLintprrrtrargtopk_preprocessrNrargtopkargtopk_aggregaterr rrrreshaper)r) r:rrHrJidx a_plus_idxrrnaxisrs ` r0rrjsB qvv &D '8 HC eTeAFFmTT UCe666JJ EMMQ/M//15I$D  77>>"'' " * *4166E>A3E+F GD  ggF   r2cBt||||jd|S)N)r3axis1axis2rr4)rrM)r:r3rrr5s r0tracers# AfE ? C CBe C TTr2c $| tdt|ts|f}|Dcgc]}|dkr|jzn|}}t j fd|Dr9j tjDcic] }|||vrdnd c}jtj|||s|nd|r.tjDcgc] \}}||vrdn|c}}nd}t||}|Scc}wcc}wcc}}w) z This works by automatically chunking the reduced axes to a single chunk if necessary and then calling ``numpy.median`` function across the remaining dimensions NzcThe da.median function only works along an axis. The full algorithm is difficult to do in parallelrc3BK|]}j|dkDywrrHr%rMr:s r0r&zmedian..7BAKKOa'7rautorF)rHrI drop_axisr)NotImplementedErrorrrrbr8rkrrrrtrLmedianrIrr)r:rHrIrKrMrErOs` r0rrs  |! @  dH %w48 9b26BKr ) 9D 9||7$77 II%-PBrt27P Q \\ &$D2;1881D EA"*Q! # E FV $F M' :Q FsDDD c | tdt|ts|f}|Dcgc]}|dkr|jzn|}}t j fd|Dr9j tjDcic] }|||vrdnd c}tSttjjdk\r-jjdvr|stj}i}ntj}d |i}j |f||s|nd|r.t#j$Dcgc] \}}||vrd n|c}}ndd |}t'||}|Scc}wcc}wcc}}w) z This works by automatically chunking the reduced axes to a single chunk and then calling ``numpy.nanmedian`` function across the remaining dimensions NzzThe da.nanmedian function only works along an axis or a subset of axes. The full algorithm is difficult to do in parallelrc3BK|]}j|dkDywrrrs r0r&znanmedian..rrrr)rruifrIrF)rHrr)rrrrbr8rkrrrnumbaggr __version__releaser5kind nanmedianrLrtrIrr) r:rHrIrKrMr0rArErOs ` r0rrso  |! @  dH %w48 9b26BKr ) 9D 9||7$77 II%-PBrt27P Q  G'' ( 0 0I = GGLLE !  ||h' Q\\   &$D2;1881D EA"*Q! # E   FV $F M? :Q& FsE"E'7E,)weights interpolationc|XtjdjDr tdt t t j}t|ts|f}|D cgc]} | dkr| jzn| }} tjfd|Dr9jt jD cic] } | | |vrdnd c} trd|i} ni} jtjf||||||s|ndt|trdndt!|||d | } t#|| } | Scc} wcc} w) z This works by automatically chunking the reduced axes to a single chunk if necessary and then calling ``numpy.quantile`` function across the remaining dimensions Nc3&K|] }|dkD ywrrr%n_blockss r0r&zquantile..A1 AzeThe da.quantile function only works along an axis. The full algorithm is difficult to do in parallelrc3BK|]}j|dkDywrrrs r0r&zquantile..-rrrrr)qrtrrHrIrnew_axisr)r8rkrHrrrrrrrrbrrrrtrLquantile_get_quantile_chunksr) r:rrHrKoverwrite_inputrtrIrrrMrArOs ` r0rr sM" | <<AQ[[A A%D  s177|,-D dH %w48 9b26BKr ) 9D 9||7$77 II%-PBrt27P QW% Q\\   # &$D H-4#Aq$9   FV $F M3 :Qs 3EEc t|jdkrdn%ttt |jdd}||jdz}t t jt j|jd|g}tdt|jdz D]}t jt j|j|||j|z}|jt t j|||z||j|z}|S)NrrFrr) rrrr rrrLrrrrutile)r:shapesoriginal_shapesindexersrindexers r0_span_indexersrFs agg,!#QT!'']1R5H)IFqwwqz)Obii !''!* 5v>?@H 1c!''lQ& '))BIIaggaj16QWWQZ3GHbggg&/HIJK1771: Or2c ||hjddhrVt|dk7sH|dt|jdz k7s*t|jdk(s|jddkDrtj||f||||d|Stj |d}t |}t|d} d} t|tsd } |g}g} d t|jddz|rd nd z} t|D]4} tj| |jddz ztj|jdjdz | z}tj|j!t"tj$|j!t"}}|t|t|fz}|t|t|fz}||z }tj&|d k(d |}||z }| j)||z||zzj| 7| r| dj+dStj,| dS)NlinearrFrri)rHrtrrIrFTrRrgg?)issubsetrrrL nanquantilesortrrrrrr!rrMrfloorrintrvrXrusqueezer)r:rrHrtrrIrA sorted_arrr nr_quantiles is_scalar quantilesreshape_shapessingle_qr lower_value higher_valuelowerhigher factor_higher factor_lowers r0_custom_quantilerWsZ] # , ,h-= > t9> 7c!''lQ& & qww<1  772; ~~  '    ,$Ja Hx{#LI a " CIE*"2"23B"788HDRTUNG  GGL !QWWR[1_ 5hhz"&&B&/77; <  %'HHQK$6$6s$;RWWQZ=N=Ns=S\ 5?eK.@-BBCE(Ou\/B.DDEK #!5sMJ #a'  CVm #el&: : C C^ T ! (|###++~~ia00r2c |XtjdjDr tdt t t j}t|ts|f}|D cgc]} | dkr| jzn| }} tjfd|Dr9jt jD cic] } | | |vrdnd c} t\ttjjdk\r6j j"d vr||d k(r|stj$} d |i} n&t&} ||||d } t(r| j+d |ij,| f||s|ndt|trdndt/|||d| } t1|| } | Scc} wcc} w)z This works by automatically chunking the reduced axes to a single chunk and then calling ``numpy.nanquantile`` function across the remaining dimensions Nc3&K|] }|dkD ywrrrs r0r&znanquantile..rrzhThe da.nanquantile function only works along an axis. The full algorithm is difficult to do in parallelrc3BK|]}j|dkDywrrrs r0r&znanquantile..rrrr)rrrrr)rrtrrIr)rHrrr)r8rkrHrrrrrrrrbrrrr rrr5rrrrupdatertrr) r:rrHrKrrtrIrrrMr0rArOs ` r0rrs" | <<AQ[[A A%D  s177|,-D dH %w48 9b26BKr ) 9D 9||7$77 II%-PBrt27P Q  G'' ( 0 0I = GGLLE ! O h ""q!*    MM9g. / Q\\  &$D H-4#Aq$9   FV $F MK :Qs 3F9F>ct|tr t|gng}|r1|t|jDcgc] \}}||vrdn|c}}zS|t|jDcgc] \}}||vs |c}}zScc}}wcc}}w)NrF)rrrrIr)r:rrHrIquantile_chunkrMrEs r0rrs!+Ax!8c!fXbN09!((0C! ',r1tA "!    !((0C Vur1rQU~ VVV ! !WsB+ B8Br?)NNFNN)NFNN)NN)rNF)NNFrNN)F)NF)NNN)NNNrsN)NNNrs)rN)rrrFN)NFN)NNFrF)NrNF) __future__rr8rrzrcollections.abcr functoolsrr itertoolsrrnumbersr r r numpyrLpackaging.versionr tlzr rr dask.arrayrdardask.array.corerrrrrdask.array.creationrrdask.array.dispatchrrrdask.array.numpy_compatrdask.array.utilsrrrrr dask.array.wrapr!r" dask.baser#dask.highlevelgraphr$ dask.utilsr%r&r'r ImportError_array_expr_enableddask.array._array_exprr(r)dask.array._reductions_genericr=rBrDrMrSrZaminrYr9amaxrfrkrnrprrryr~rrrrrrrrrrrrrrrrrr rrr"r8r;r=r?rUr[r^rarJr]rWrjrlrrxrrrrrrrrrrrrrrrr2r0rs^" $%%$%''1LL0(.332>>F %(b   b" BFFBGG b  7 BFFBGG b  7b  b  b"b"b|2b 2b&  b&   99Ee $     $0 -b*b*    ,<    ),\   1/jTXIXb.bLP: b"bLP&BB  CNRJ#ZQRb  b  b  b  76+&u#z    h#Vb2b28vEPbUUb!!Hb--`b    5 55p(    J1Zb    A AAHW}5Gs.N::OO