
    ukiǒ                       U d dl mZ 	 d dlmZmZ d dlZd dlmZmZ d dl	Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlm	Z d dlm Z  d dl!m"Z" d dl#m$Z% d dl&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z2 d dl3m4Z4 i Z5de6d<   i Z7de6d<   ejp                  ejr                  ejt                  ejv                  ejx                  gZ=ej|                  ej~                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  gZNej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  gZ]de6d<   d Z^eZ_eZ` G d d      Za G d d e      Zbd! Zcd" Zd	 	 	 	 	 	 ded#Ze	 	 	 	 	 	 dfd$Zf	 	 	 	 	 	 dfd%Zg G d& d'ej                        Zi G d( d)ej                        Zkej                  d*        Zmdgd+Zndhd,Zo	 	 	 	 	 	 	 	 	 	 did-Zpd. Zqd/ Zrdjd1Zsd2 Ztd3 ZueND ]  Zv euevd04      e5ev<    euevd04      e7ev<    e=D ]  Zv euevd54      e5ev<    euevd54      e7ev<    d6 Zwej                  e4j                  ej                  d7 ej                  e4j                  ej                  e4j                  ej                  d8 ej                   e4j                  ej                  e4j                  ej                  e4j
                  ej                  e4j                  i	Zej                         D ]  \  ZZ ewee      e5e<    ej                  e4j                  ej                  e4j                  ej                  d9 iZej                         D ]  \  ZZ ewee      e7e<    d: Zee5ej"                  <   ee7ej"                  <   d; Zee5ej&                  <   d< Zee5ej*                  <   d= Zee5ej.                  <   d> Zee5ej2                  <   d? Zee5ej6                  <   d@ Zee5ej:                  <   dA Zee5ej>                  <   	 	 dkdBZdC Zee5ejD                  <   dD Zee5ejH                  <   dE ZdF Zee5ejN                  <   dG Zee5ejR                  <   dH Zee5e4jV                  <   ee7e4jV                  <   dI Zee5ejZ                  j\                  <   ee7ejZ                  j\                  <   dJ ZdK ZdL Z	 	 dldMZdN ZdO Zi dPedQedRedS esejj                        dT esejl                        dU esejn                        dV esejp                        dW es eejp                              dX esejr                        dY es eejr                              dZ esejt                        d[ es eejt                              d\ esejv                        d] es eejv                              d^ esejx                        d_ es eejx                              d`da  es eejz                              ee esej|                         esej~                         esej                         esej                         esej                         esej                        db	ZeĐj                         D ]  \  ZZ eeeeƫ        	 	 dldcZ esejp                         es eejp                              eddZeĐj                         D ]  \  ZZ eeeeƫ        y)m    )annotations)CallableSequenceN)Any
NamedTuple)lax)api_util)config)core)lift_jvp)linear_util)pjit)sharding_impls)bcoo_multiply_densebcoo_multiply_sparseBCOO)BCSR)flatten_fun_nokwargs)pytree)partial_eval)tree_flattentree_maptree_unflatten)safe_mapsafe_zip
split_list)_check_tree_and_avals)indexing)sparsezdict[core.Primitive, Callable]sparse_rules_bcoosparse_rules_bcsrzlist[core.Primitive]_densifying_primitivesc                N    | t         v rt        d|  d      t        d|  d      )Nsparse rule for z is not implemented because it would result in dense output. If this is your intent, use sparse.todense() to convert your arguments to dense matrices.z is not implemented.)r"   NotImplementedError)	primitives    \/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/experimental/sparse/transform.py_raise_unimplemented_primitiver(   |   sE    ((
 0 <^ ^ _ _ 	.yk9MNOO    c                  `    e Zd ZU dZded<   ddZddZddZddZddZ	d	 Z
dd
d
d
ddddZy
)SparsifyEnva  Environment for sparse jaxpr evaluation.

  The environment is essentially a collection of buffers and/or tracers
  that may be shared between one or more SparsifyValue objects, which
  represent sparse or dense arrays via indices into the list of buffers.
  zlist[Array]_buffersc                $    t        |      | _        y N)listr,   )selfbufss     r'   __init__zSparsifyEnv.__init__   s    JDMr)   c                    | j                   j                  t        j                  |             t	        | j                         dz
  S )N   )r,   appendjnpasarraylen)r0   arrs     r'   _pushzSparsifyEnv._push   s1    MMS)*t}}!!r)   c                b    |j                   t        d      | j                  |j                      S )z4Get the data buffer associated with a SparsifyValue.z8Internal: requested data from spvalue with data_ref=None)data_refRuntimeErrorr,   r0   spvalues     r'   datazSparsifyEnv.data   s/    STT==))**r)   c                b    |j                   t        d      | j                  |j                      S )z7Get the indices buffer associated with a SparsifyValue.z>Internal: requested indices from spvalue with indices_ref=None)indices_refr=   r,   r>   s     r'   indiceszSparsifyEnv.indices   s/    "YZZ==,,--r)   c                b    |j                   t        d      | j                  |j                      S )z;Get the BCSR indptr buffer associated with a SparsifyValue.z=Internal: requested indices from spvalue with indptr_ref=None)
indptr_refr=   r,   r>   s     r'   indptrzSparsifyEnv.indptr   s/    !XYY==++,,r)   c                ^    t        t        j                  |      | j                  |            S )z2Add a new dense array to the sparsify environment.)SparsifyValuenpshaper:   )r0   r@   s     r'   densezSparsifyEnv.dense   s     $D)9::r)   NF)r<   rB   rE   indices_sortedunique_indicesc               \   ||J | j                  |      }n||t        | j                        k  sJ ||J | j                  |      }n||t        | j                        k  sJ ||J | j                  |      }n||t        | j                        k  sJ t        ||||||	      S )z3Add a new sparse array to the sparsify environment.rL   rM   )r:   r8   r,   rH   )
r0   rJ   r@   rC   rF   r<   rB   rE   rL   rM   s
             r'   r   zSparsifyEnv.sparse   s     D!h!hT]]1C&CCC   JJw'k$s4==7I)III::f%j		#dmm,,,,+z(6~W Wr)   ) )r9   Arrayreturnint)r?   rH   rR   rQ   )NNN)__name__
__module____qualname____doc____annotations__r2   r:   r@   rC   rF   rK   r   rP   r)   r'   r+   r+      sE     "+.-;W4D!%Wr)   r+   c                      e Zd ZU ded<   ded<   dZded<   dZded<   dZd	ed
<   dZd	ed<   ed        Z	d Z
d Zd Zd Zy)rH   ztuple[int, ...]rJ   z
int | Noner<   NrB   rE   Fzbool | NonerL   rM   c                ,    t        | j                        S r.   )r8   rJ   r0   s    r'   ndimzSparsifyValue.ndim   s    tzz?r)   c                    | j                   d uS r.   rB   r[   s    r'   	is_sparsezSparsifyValue.is_sparse   s    4''r)   c                    | j                   d u S r.   r^   r[   s    r'   is_densezSparsifyValue.is_dense   s    t##r)   c                B    | j                         xr | j                  d u S r.   r_   rE   r[   s    r'   is_bcoozSparsifyValue.is_bcoo   s    >>74 77r)   c                B    | j                         xr | j                  d uS r.   rc   r[   s    r'   is_bcsrzSparsifyValue.is_bcsr   s    >>;t ;;r)   )rT   rU   rV   rX   rB   rE   rL   rM   propertyr\   r_   ra   rd   rf   rP   r)   r'   rH   rH      s\    	 +z *j %.+% %.+% ($8<r)   rH   c                .    t        | t        t        f      S r.   )
isinstancer   r   args    r'   <lambda>rl      s    ZdD\: r)   c                "    t        | t              S r.   )ri   rH   rj   s    r'   rl   rl      s    *S-8 r)   c                2      fd}t        ||t              S )zHConvert a pytree of (sparse) arrays to an equivalent pytree of spvalues.c                   t        | t              rHj                  | j                  | j                  | j
                  | j                  | j                        S t        | t              rSj                  | j                  | j                  | j
                  | j                  | j                  | j                        S j                  |       S )NrO   )ri   r   r   rJ   r@   rC   rL   rM   r   rF   rK   )rk   spenvs    r'   array_to_spvaluez,arrays_to_spvalues.<locals>.array_to_spvalue   s    #t\\#))SXXs{{),););),););  = = 
C	\\#))SXXs{{CJJ),););),););  = = [[r)   is_leaf)r   _is_sparse_obj)rp   argsrq   s   `  r'   arrays_to_spvaluesrv      s    

 
"D.	AAr)   c                2      fd}t        ||t              S )zHConvert a pytree of spvalues to an equivalent pytree of (sparse) arrays.c                   | j                         rMt        j                  |       j                  |       f| j                  | j
                  | j                        S | j                         r]t        j                  |       j                  |       j                  |       f| j                  | j
                  | j                        S j                  |       S )N)rJ   rL   rM   )
rd   r   r@   rC   rJ   rL   rM   rf   r   rF   )r?   rp   s    r'   spvalue_to_arrayz,spvalues_to_arrays.<locals>.spvalue_to_array   s    5::g&g(>?g6L6L!(!7!79 9 
	5::g&g(>W@UVg6L6L!(!7!79 9 ZZ  r)   rr   r   _is_spvalue)rp   spvaluesry   s   `  r'   spvalues_to_arraysr}      s    

! 
"Hk	BBr)   c                2      fd}t        ||t              S )zHConvert a pytree of spvalues to an equivalent pytree of abstract values.c                    j                  |       }t        j                  | j                  |j                  |j
                  j                        S r.   )r@   r   ShapedArrayrJ   dtypeaval	weak_type)r?   r@   rp   s     r'   spvalue_to_avalz*spvalues_to_avals.<locals>.spvalue_to_aval  s8    ::gDGMM4::tyy7J7JKKr)   rr   rz   )rp   r|   r   s   `  r'   spvalues_to_avalsr     s    
L 
/8[	AAr)   c                  :    e Zd ZddZed        Zed        Zd Zy)SparseTracerc                    || _         || _        y r.   )_spvalue_trace)r0   tracer?   s      r'   r2   zSparseTracer.__init__  s    DMDKr)   c                p    t        | j                  d      st        d      | j                  j                  S )Nrp   z,Internal: trace does not have spenv defined.)hasattrr   r=   rp   r[   s    r'   rp   zSparseTracer.spenv  s,    4;;(GHH;;r)   c                J    t        | j                  | j                  g      d   S )Nr   )r   rp   r   r[   s    r'   r   zSparseTracer.aval$  s    TZZ$--9!<<r)   c                    | S r.   rP   r[   s    r'   
full_lowerzSparseTracer.full_lower(  s    Kr)   N)r   z
core.Trace)rT   rU   rV   r2   rg   rp   r   r   rP   r)   r'   r   r     s4      
 = =r)   r   c                  <     e Zd ZdZ fdZd Zd ZddZd Z xZ	S )SparseTrace)parent_tracetagrp   c                L    t         |           || _        || _        || _        y r.   )superr2   r   r   rp   )r0   r   r   rp   	__class__s       r'   r2   zSparseTrace.__init__/  s$    	G$DDHDJr)   c                   t        |t              r$| j                  |j                  j                  u r|S t	        j
                  | j                        5  t        | j                  |g      \  }d d d        t        |       S # 1 sw Y   xY wNr?   )	ri   r   r   r   r   set_current_tracer   rv   rp   )r0   valr?   s      r'   to_sparse_tracerzSparseTrace.to_sparse_tracer5  sn    #|$SZZ^^)Cj!!$"3"34 9%djj3%89$009 9s   BBc                v    |D cg c]  } j                  |       }}|D cg c]  }|j                   }}t        d |D              r_|t        vrt	        |       t        j                   j                        5  t        |    j                  gd |D        i |}d d d        nT|j                   j                  t         fd|D              |      }t         j                  |j                  r|n|g      }t         fdD              }|j                  r|S |d   S c c}w c c}w # 1 sw Y   :xY w)Nc              3  <   K   | ]  }|j                           y wr.   )r_   ).0r?   s     r'   	<genexpr>z0SparseTrace.process_primitive.<locals>.<genexpr>@  s     
777
7   c              3  4   K   | ]  }|j                     y wr.   )r   )r   ts     r'   r   z0SparseTrace.process_primitive.<locals>.<genexpr>D  s     A^QR!**A^s   c              3  T   K   | ]  }j                   j                  |       ! y wr.   )rp   r@   r   r?   r0   s     r'   r   z0SparseTrace.process_primitive.<locals>.<genexpr>F  s      CuahDJJOOT[D\Cus   %(c              3  8   K   | ]  }t        |         yw)r   N)r   r   s     r'   r   z0SparseTrace.process_primitive.<locals>.<genexpr>H  s     XT7;;X   r   )r   r   anyr    r(   r   r   r   rp   bind_with_tracetuplerv   multiple_results)	r0   r&   tracersparamsr   r|   out_spvaluesout_bufsout_tracerss	   `        r'   process_primitivezSparseTrace.process_primitive=  s%   189At$$Q'9G9$+,q

,H,

7h
77	+	+&y1!!$"3"34 j(3DJJiA^V]A^ibhij j **4+<+<eCultCu>uw}~h'

	@Z@ZHai`jklX<XXK#44;H+a.H :,j js   D%D*=$D//D8c                    J r.   )
r   rp   r,   sparsify_subtracemainr   r%   dictbindr   )r0   call_primitivefr   r   r|   in_bufsfunr   bufs_outr?   s              r'   process_callzSparseTrace.process_callK  s    5r)   c               |    ~~~t        j                  |       5   |j                  | cd d d        S # 1 sw Y   y xY wr.   )r   r   call_wrapped)r0   r&   r   jvpr   symbolic_zeross         r'   process_custom_jvp_callz#SparseTrace.process_custom_jvp_callV  s<    3				% (Sw'( ( (s   2;)r   zlu.WrappedFun)
rT   rU   rV   	__slots__r2   r   r   r   r   __classcell__)r   s   @r'   r   r   +  s#    .)1I	O(r)   r   c           	        t        j                         5 }t        |||      }t        j                  |      5  |D cg c]  }t	        ||       }	} | |	 }
|
D cg c]  }|j                  |       }}|j                  }d d d        d d d        |j                  D cg c]  }|j                   c}       S c c}w c c}w # 1 sw Y   FxY w# 1 sw Y   JxY wc c}w r   )	r   take_current_tracer   r   r   r   r,   storer   )r   r   r   rp   r|   r1   parentr   r?   
in_tracersoutsout
out_tracesbufferss                 r'   r   r   \  s      FU+E				& HPQWL8QjQ
^d;?@CE**3/@j@g	 ++z2s||23	. R@   3sE   #CCB=CC4CC!C=
CC	CCc                    t        j                         }t               }t        ||      }|j                  }t        | |||      \  }} |j                  | }t        |      }t        | |             S r.   )r   TraceTagr+   rv   r,   r   r   r}   )	wrapped_funru   r   rp   r|   r   r   r   r   s	            r'   sparsify_funr   h  si    #
-%t,(NN''S%J#|Sw'(
h
%	E<>	22r)   c                B     t        j                          fd       }|S )z+Implementation of sparsify() using tracers.c                     t        | t              \  }}t        t        j                  t        j                  d| i             |      \  }}t        ||      }t         |       |      S )Nrr   sparsify
debug_info)	r   rt   r   lu	wrap_initr	   r   r   r   )ru   	args_flatin_treer   out_treer   r   s         r'   _wrappedz'_sparsify_with_tracer.<locals>._wrappedt  sf    %dNCIw0
S ( 3 3JT2 N	PK {I
.C(*c**r)   )	functoolswraps)r   r   s   ` r'   _sparsify_with_tracerr   r  s%    ??3+ + 
/r)   c                   i dfd}dfd}d	fd}t        || j                  |       t        || j                  |       | j                  D ]b  }|j                  }t        ||j                        }	t        d |	D              r0|t        vrt        |       t        |   g|	i |j                  }
nt        d |	D              r0|t        vrt        |       t        |   g|	i |j                  }
n |j                  fd|	D        i |j                  }|j                  r|n|g}g }
t        ||j                        D ]Q  \  }}t        |t        j                         r|
j#                  d        2|
j#                  j%                  |             S t        ||j                  |
       e t        || j                        S )
Nc                    t        | t        j                        rj                  | j                        S t        | t        j
                        sJ |    S r.   )ri   r   LiteralrK   r   Var)varenvrp   s    r'   readzeval_sparse.<locals>.read  sA    #t||$[[!!TXX&&&Xor)   c                d    t        | t        j                        ry j                  |      | <   y r.   )ri   r   DropVarrK   )r   ar   rp   s     r'   write_bufferz!eval_sparse.<locals>.write_buffer  s%    #t||${{1~CHr)   c                N    t        | t        j                        ry |J || <   y r.   )ri   r   r   )r   r   r   s     r'   writezeval_sparse.<locals>.write  s'    #t||$==CHr)   c              3  <   K   | ]  }|j                           y wr.   )rf   r   r   s     r'   r   zeval_sparse.<locals>.<genexpr>  s     
+S3;;=
+r   c              3  <   K   | ]  }|j                           y wr.   )rd   r   s     r'   r   zeval_sparse.<locals>.<genexpr>  s     -sS[[]-r   c              3  @   K   | ]  }j                  |        y wr.   )r@   )r   r   rp   s     r'   r   zeval_sparse.<locals>.<genexpr>  s     ?UZZ_?s   )r   z	core.AtomrR   rH   )r   core.Varr   rQ   rR   None)r   r   r   rH   rR   r   )r   	constvarsinvarseqnsr&   r   r!   r(   r   r    r   r   r   outvarsri   r   r   r5   rK   )jaxprconstsr|   rp   r   r   r   eqnpriminvalsr   r   bufoutvarr   s      `          @r'   eval_sparser     s    )+#
 <&1
5%,,)ZZ &c==DdCJJ'F

+F
++	&	&&t,d#EAFAcjjAc	-f-	-	&	&&t,d#EAFAcjjAc??N3::Nh!22
hc!(CKK8 '+#vfdll+
**T

**U[[%
&	'
 UCKK%)&, 
$	&&r)   c                $     	 	 	 	 	 	 	 	 d fd}|S )Nc                   t        |t              \  }}t        | |      }t        t	        j
                  |t        j                  d|j                  dgt        |      z        i             |      \  }}t        j                  ||      \  }}	}
t        ||
||       }t        |	      t        |      k7  rt        d      | |       fS )Nrr   r   Tr   zkInternal: eval_sparse does not return expected number of arguments. Got {result} for avals {out_avals_flat})r   r{   r   r   r   r   r	   r   	unflattenr8   petrace_to_jaxpr_dynamicr   	Exception)rp   r|   r   spvalues_flatr   in_avals_flatr   r   r   out_avals_flatr   resultr   s               r'   wrappedzsparsify_raw.<locals>.wrapped  s     *(KHM7%e];M0
v**:q+2+<+<dVc-FX=X+Y+-/	0
 	K %'$=$=k=$Y!E>6u=F
>c&k) @ A A8:r)   )rp   r+   r|   rH   r   r   rR   z0tuple[Sequence[SparsifyValue], pytree.PyTreeDef]rP   )r   r	  s   ` r'   sparsify_rawr
    s/    %2>A7& 
.r)   c                X    t        |       t        j                  |       fd       }|S )z5Implementation of sparsify() using jaxpr interpreter.c                 ~    t               }t        ||       } |g|i |\  }}t        ||      }t        ||      S r.   )r+   rv   r}   r   )ru   r   rp   r|   spvalues_outr   r   f_raws          r'   r	  z+_sparsify_with_interpreter.<locals>.wrapped  sH    ME!%.H"5>8>v>L(
UL
1C(C((r)   )r
  r   r   )r   r	  r  s     @r'   _sparsify_with_interpreterr    s.    
q/%??1) ) 
.r)   Fc                2    |rt        |       S t        |       S )a  Experimental sparsification transform.

  Examples:

    Decorate JAX functions to make them compatible with :class:`jax.experimental.sparse.BCOO`
    matrices:

    >>> from jax.experimental import sparse

    >>> @sparse.sparsify
    ... def f(M, v):
    ...   return 2 * M.T @ v

    >>> M = sparse.BCOO.fromdense(jnp.arange(12).reshape(3, 4))

    >>> v = jnp.array([3, 4, 2])

    >>> f(M, v)
    Array([ 64,  82, 100, 118], dtype=int32)
  )r   r  )r   
use_tracers     r'   r   r     s    *  ##%a((r)   c                    |j                         s|j                  r|S t        | |      }|j                  |j                  d      }t        | |      S )z;Return an spvalue representation with deduplicated indices.F)nseremove_zeros)ra   rM   r}   sum_duplicatesr  rv   )rp   r?   r9   s      r'   _ensure_unique_indicesr    sM    711N5'*#swwU;#	E3	''r)   c                      fd}|S )Nc                x   t        |      dk(  sJ |d   }st        | |      }| j                  |      } j                  |fi |}|d   j	                         rL| j                  |j                  ||j                  |j                  |j                  |j                        }|fS | j                  |      }|fS )Nr4   r   )rB   rE   rL   rM   )r8   r  r@   r   r_   r   rJ   rB   rE   rL   rM   rK   )	rp   r|   kwargsr?   r   buf_outout_spvaluelinearr   s	          r'   funcz'_zero_preserving_unary_op.<locals>.func  s    x=AqkG 'ug6g
**W
Cdii&v&G{LL-4-@-@,3,>,>070F0F070F0F	 ! Hk > KK$k>r)   rP   )r   r  r  s   `` r'   _zero_preserving_unary_opr    s    $ 
+r)   )r  Tc                      fd}|S )Nc                f     t        | |      i |}t        | j                  r|      S |g      S r.   )r}   rv   r   )rp   r|   kwdsr  r   	sparse_ops       r'   _sparse_rulez+_standard_sparse_rule.<locals>._sparse_rule  s9    *5(;DtDFet/D/DVSS6(SSr)   rP   )r   r"  r#  s   `` r'   _standard_sparse_ruler$    s    T 
r)   c                 .    t        j                  | fi |S r.   )r   bcoo_concatenater   ks     r'   rl   rl   %      V%<%<Q%D!%D r)   c                 <    t        j                  | d   | dd  fi |S )Nr   r4   )r   bcoo_dynamic_slicer'  s     r'   rl   rl   (  s#    v'@'@1qu'RPQ'R r)   c                 .    t        j                  | fi |S r.   )r   bcsr_concatenater'  s     r'   rl   rl   5  r)  r)   c                   |dk  r t        dt        j                   d| d       t        t        j                  d      | g|d|iS )Nr   r$   z with non-positive exponent z is not implemented because it would result in dense output. If this is your intent, use sparse.todense() to convert your argument to a dense array.Fy)r%   r   integer_pow_pr  )rp   r/  r|   s      r'   _integer_pow_sparser1  ;  sd    !V
 01B1B0CC_`a_b ch h i i 
=	"3#4#4e	<U	SX	SQR	SSr)   c          	        t        |      }t        | |      }|d   j                  t        j                  |d   |      }t        fd|D              }|d   j
                  j                  dz
  }|d   j
                  j                  d   }|d | t        t        |            k(  }|||z   d  t        t        ||z   t                          k(  }	||||z    t        t        |||z               k(  }
i }|r|	r|d   j                  |d<   n|j                  |d<   |r|
r|d   j                  |d<   n|j
                  |d	<   |j                  |d
<   |j                  |d<    | j                  |fi |}|fS )Nr   )permutationc              3  (   K   | ]	  }|     y wr.   rP   )r   irJ   s     r'   r   z$_transpose_sparse.<locals>.<genexpr>J  s     2E!H2      r<   r@   rB   rC   rL   rM   )r   r}   rJ   r   bcoo_transposerC   r\   ranger8   r<   r@   rB   rL   rM   )rp   r3  r|   ru   mat_transposed	out_shapen_batchn_sparsebatch_dims_unchangeddense_dims_unchangedsparse_dims_unchangedr!  r?   rJ   s                @r'   _transpose_sparserB  E  s   k"+	E8	,$
q'--%((akJ.2k22)GOO  1$'!W__""2&(%hw/5w3HH%g&8&9:eE'T\J\^abg^hDi>jj&ww/ABeERY[bem[mLnFoo 
$2{++D!&&DL 3"1+11D$,,DO)88$)88$ELL+d+'
r)   c                6   |\  }}t        j                  |j                  |j                        }|j                         rR|j                         rA|j                  |j                  k7  rt	        d      |j
                  |j
                  k(  rt        j                  | j                  |      | j                  |            }t        j                  j                  r6|j                  |j                  k(  sJ |j                  |j                  k(  sJ | j                  |j                  ||j
                  |j                  |j                        }|fS | j                  |      j                  | j                  |      j                  k7  s7| j                  |      j                  | j                  |      j                  k7  rt	        d      t        j                   | j                  |      | j                  |      g| j                  |      j                  dz
        }t        j                   | j                  |      | j                  |      g| j                  |      j                  dz
        }| j                  |j                  ||      }|fS |j                         r||}}|j                         r|j#                         sJ |j                  |k7  rt	        d      t%        | ||f      \  }}	|j'                         |	z   }
| j)                  |
      }|fS )Nz5Addition between sparse matrices of different shapes.rB   rL   rM   zGAddition between sparse matrices with different batch/dense dimensions.r7  )	dimensiona  Addition between a sparse array X and a dense array Y is not implemented when the output shape is larger than Y.shape. This is to prevent silent densification of a large sparse array. If this is your intent, you can explicitly cast the sparse array to a dense matrix.)r   broadcast_shapesrJ   r_   r%   rB   addr@   r
   enable_checksvaluerL   rM   r   rC   r\   concatenatera   r}   todenserK   )rp   r|   XYr<  out_datar  out_indices
X_promoted
Y_promotedr   s              r'   _add_sparserR  f  s   	$!Q""177AGG4)[[]q{{}ww!'' WXX}}%A

16h				#	#1#3#33331#3#3333LL(010@0@010@0@ ! Bk. ) 
q			%--"2"7"7	75::a=;M;MQVQ[Q[\]Q^QcQc;c ijjOOU]]1%5u}}Q7G$HTYTaTabcTdTiTilmTmnk%**Q-A!?5==YZK[K`K`cdKdehLL(K@k  	{{}a;;=QZZ\))ww)	#$ $
 01v>J





+C++c"K
r)   c                    |\  }}|j                         r8|j                         r(t        | |gt        t        j                     | |       S t        d      )Nz+Subtraction between sparse and dense array.)r_   rR  r    r   neg_pr%   )rp   r|   rL  rM  s       r'   _sub_sparserU    sK    	$!Q[[]q{{}uaI"3CII">ua"HII
K
LLr)   c                   |\  }}|j                         r@|j                         r/|j                  |j                  k(  r|j                  rt        j                  j
                  r6|j                  |j                  k(  sJ |j                  |j                  k(  sJ t        j                  | j                  |      | j                  |            }| j                  |j                  ||j                  |j                  d      }|fS t        | |      \  }}t        ||      }| j                  |j                  |j                  |j                        }|fS |j                         r||}}t        | |      }t        || j                  |            }| j                  |j                  ||j                  |j                  |j                        }|fS )NTrD  )r_   rB   rM   r
   rH  rI  rL   r   mulr@   r   rJ   r}   r   rC   r   )	rp   r|   rL  rM  rN  r  rP  rQ  mats	            r'   _mul_sparserY    s   	$!Q[[]q{{}}}%!*:*:				#	#1#3#33331#3#3333A

16hLL(010@0@04 ! 6k    2%Bj* Z8cLLCHHckkBk  	{{}a#E1-J":uzz!}=H,,qwwamm./.>.>./.>.>  @K r)   c                   |\  }}|j                         rt        d      t        | |      }t        |d| j	                  |      z        }| j                  |j                  ||j                  |j                  |j                        }|fS )NzDivision by a sparse array is not implemented because it would result in dense output. If this is your intent, use sparse.todense() to convert your arguments to a dense array.g      ?rD  )
r_   r%   r}   r   r@   r   rJ   rB   rL   rM   )rp   r|   rL  rM  rP  rN  r  s          r'   _div_sparser[    s    	$!Q[[]
EF F "%+* R%**Q--?@(QWWhAMM-.-=-=-.-=-=  ?+ r)   c                  |\  }t        | |      }t        j                  ||      }|j                  }|dk(  r,| j	                  |j
                  j                               }|fS | j                  ||j
                  |j                        }|fS )N)axesrP   )r}   r   bcoo_reduce_sumrJ   rK   r@   sumrC   )	rp   r]  out_shardingr|   rL  rP  rX  r<  r  s	            r'   _reduce_sum_sparsera    s}    "!!%+*z5#ii)"_++chhlln-K  ,,y#((CKK@K
r)   c          
     t    t        | |      \  }}	t        j                  ||	||||||      }
t        | |
f      S )N)dimension_numbersslice_sizesrM   indices_are_sortedmode
fill_value)r}   r   bcoo_gatherrv   )rp   rc  rd  rM   re  rf  rg  ru   operandstart_indicesr  s              r'   _gather_sparse_rulerk    sI    -eT:'=g}HY*5n1C#'J@& 
EF9	--r)   c                ~    d  fd}t         |      }t        |      \  }|D cg c]  }t        j                  |       }}t	        j
                  t        j                  |j                  j                  j                               |      \  }}	}
t	        j                  ||
      }J |fS c c}w )Nc                     t        |       }t        	|      }t        j                  j                  |	      }t        	|      }t        |      \  }|S r.   )r   rv   r   r   r   r}   r   )
r   ru   r|   r  r   out_flatr   r   r   rp   s
         r'   r	  z _sparsify_jaxpr.<locals>.wrapped  sU     '9-D!%.HellHeDF
UF
+C%c*HhOr)   r   )r}   r   r   get_avalr  r  r   r   r   r   with_unknown_namesClosedJaxpr)rp   r   r|   r	  ru   r   rk   
avals_flatsp_jaxpr_r   r   r   s   ``         @@r'   _sparsify_jaxprru    s     '+( 
E8	,$#D))W.78sc"8*811ll7u{{'='='P'P'RSU_a(Av^^Hf-(				8	 9s   B:c                  t        |||g      \  }}}t        | |g|| \  }	}
t        | |g|| \  }}t        t        | |            \  }}
t        t        | |            \  }}
t        t        | |            \  }}
t	        j
                  j                  g |||t        |      |	t        |      |d}t        | t        ||            S )N)cond_nconsts
cond_jaxprbody_nconsts
body_jaxpr)
r   ru  r   r}   r   while_pr   r8   rv   r   )rp   rx  rw  rz  ry  r|   cond_const_spvaluesbody_const_spvaluesinit_val_spvaluescond_sp_jaxprrt  body_sp_jaxprr   cond_constsbody_consts	init_valsrn  s                    r'   _while_sparser    s    @J|\*A,=*,= %UJa9LaO`a-+E:h@ShVgh- 25:M NO.+q 25:M NO.+q08IJK,)Q[[ W{ W[ W9 W+.{+;+.{+;W( 
E>(H#E	FFr)   c                  t        |      rt        d      t        | |g| \  }}t        t	        | |            \  }}t        d |D              }|t        d t        t        |      t        |      z
        D              z   }|t        d t        t        |j                        t        |      z
        D              z   }|t        d t        t        |      t        |      z
        D              z   }|t        d t        t        |j                        t        |      z
        D              z   }t        j                  j                  ||||||||||	|
|d}t        | t        ||            S )Nz#sparse xla_call with donated_invarsc              3      K   | ]  }d   yw)FNrP   )r   rk   s     r'   r   z_pjit_sparse.<locals>.<genexpr>  s     434   c              3  <   K   | ]  }t         j                    y wr.   r   UNSPECIFIEDr   rt  s     r'   r   z_pjit_sparse.<locals>.<genexpr>  s      &
   &r   c              3  <   K   | ]  }t         j                    y wr.   r  r  s     r'   r   z_pjit_sparse.<locals>.<genexpr>  s      (
   (r   c              3      K   | ]  }d   y wr.   rP   r  s     r'   r   z_pjit_sparse.<locals>.<genexpr>  s      "d"r  c              3      K   | ]  }d   y wr.   rP   r  s     r'   r   z_pjit_sparse.<locals>.<genexpr>"  s      $d$r  )r   in_shardingsout_shardings
in_layoutsout_layoutsdonated_invarsctx_meshnamekeep_unusedinlinecompiler_options_kvs)r   r%   ru  r   r}   r   r:  r8   	out_avalsr   jit_pr   rv   r   )rp   r   r  r  r  r  r  r  r  r  r  r  r|   sp_call_jaxprr   r   rt  rn  s                     r'   _pjit_sparser    s{    	
C
DD+E5D8D-0AB,)Q4)44.  &S^c,&778& ! ,  % (S001C4FFG( # - E "#i.3z?:;"  * e $#m556[9IIJ$  + ZZ__!#/1( 
E>(H#E	FFr)   c              #  v   K   t        | |      D ]$  \  }}|j                         r||gn|gE d {    & y 7 wr.   )r   r_   )r|   r   r?   params       r'   _duplicate_for_sparse_spvaluesr  8  sA      62 Bngu!(!2!2!4u~5'AABAs   -979c          	        t        |||g      \  }}}|rt        d      t        | |g||| \  }	}
t        t	        | |            \  }}
t        t	        | |            \  }}t        t	        | |            \  }}t        |j                  d      ||g      \  }}}g t        ||      t        ||      t        ||      }t        j                  j                  g ||||	|t        |      t        |      d|}t        ||d t        |             }t        ||t        |      d        }t        | ||z         S )Nz#sparse rule for scan with x values.r  )r   r  
num_consts	num_carry)r   r%   ru  r   r}   popr  r   scan_pr   r8   r   rv   )rp   r   r  r  r|   r   const_spvaluescarry_spvaluesxs_spvaluesrs  rt  r   carry
carry_treexsxs_treeconst_linearcarry_linear	xs_linear	sp_linearr   	carry_outxs_outs                          r'   _scan_sparser  <  sz   0:z9%1'-..+ C
DDu]~]]Q\]+(A-e^DE)&!"#5e^#LM%/{CD+"g +5
JJx:y1+3',i=#NLA=#NLA= $K;=)
 	

 	P 	P% 	P" 	PHY#&v;#e*	PHN	P#Z[c%j)9:)'3s5z{#34&	E9v#5	66r)   c          	     R    t         fd|D         \  }}t        d|d   |d   j                  d|d   |d   j                         t        t	         |g            \  }}t        j                  j                  |dt        |      i|}	t        |d   |	      }
t         |
      S )Nc              3  8   K   | ]  }t        |g   y wr.   )ru  )r   r   operandsrp   s     r'   r   z_cond_sparse.<locals>.<genexpr>Z  s%       7$) !0u Hx H  7r   zsparsified true_fun outputr   zsparsified false_fun outputr4   branches)zipr   r  r   r}   r   cond_pr   r   r   rv   )rp   predr  r  r   sp_branchestreedefsru   rt  rn  r   s   `  `       r'   _cond_sparser  Y  s      7-5 7 8+x4 [^%=%=5 [^%=%=? +ED3D83DEF'$ZZ__dJU;-?J6J(x{H-#	E3	''r)   c               \    ~t        | |      j                         }| j                  |      fS r.   )r}   rK  rK   )rp   r?   treer   s       r'   _todense_sparse_ruler  g  s,    
5'*224#
++c
	r)   c                    |j                  d      |j                  d      }|j                  d      }t         g| \  }} fd}t        ||      }t         |      }	t	        j
                  j                  j                  t        j                  |j                  j                        |g|	i |}
t         |
      S )N
call_jaxprjvp_jaxpr_funr  c                 x    t        |       }t        j                  j                  |      }t	        |      S r.   )rv   r   r   r   r}   )arrssparrsr   r  rp   s      r'   r   z$_custom_jvp_sparse_rule.<locals>.funt  s8    t,F
j&&
(9(965
ICeS))r)   r   )r  ru  r   r}   jaxcustom_derivativescustom_jvp_call_pr   r   r   r   r   rv   )rp   r|   r   r  r  r  r   r   r   r   outvalsr  s   `          @r'   _custom_jvp_sparse_ruler  o  s    !'L!9*!'O!<-JJ|,*+E:II-* 	]+#eX.&""4499ll3:#3#3#>#>?	' 
E7	++r)   c                0     t        fd      |       S )zSum array along axis.c                (     | j                   i S r.   )r_  xru   r  s    r'   rl   z_sum.<locals>.<lambda>  s    EAEE4262 r)   r   r0   ru   r  s    ``r'   _sumr    s    	32	3D	99r)   c                0     t        fd      |       S )z;Returns an array containing the same data with a new shape.c                (     | j                   i S r.   )reshaper  s    r'   rl   z_reshape.<locals>.<lambda>  s    IAIIt6v6 r)   r  r  s    ``r'   _reshaper    s    	76	7	==r)   c                0     t        fd      |       S )z-Copy the array and cast to a specified dtype.c                (     | j                   i S r.   )astyper  s    r'   rl   z_astype.<locals>.<lambda>  s    HAHHd5f5 r)   r  r  s    ``r'   _astyper    s    	65	6t	<<r)   c                     t        t        j                  t        j                  |||||            |       }t        |t              s-t        j                  |      dk(  rt        j                  |      }|S )Nidxre  rf  rM   rg  r   )
r   r   partialjnp_indexingrewriting_takeri   r   rI   size	fromdenser9   r  re  rM   rf  rg  r  s          r'   _bcoo_rewriting_taker    sl    F8I%%S=O	nE FFIK& 
FD	!bggfo&:^^F#F	-r)   c                X     t         fdt         j                  d         D              S )Nc              3  (   K   | ]	  }|     y wr.   rP   )r   r5  r9   s     r'   r   z_sparse_iter.<locals>.<genexpr>  s     2c!f2r6  r   )iterr:  rJ   )r9   s   `r'   _sparse_iterr    s!    	2eCIIaL12	22r)   c                      fdS )Nc                     ||       S r.   rP   )r   br   s     r'   rl   z<lambda>.<locals>.<lambda>  s    AaG r)   rP   )r   s   `r'   rl   rl     s    + r)   r  r  r_  __abs____neg____pos__
__matmul____rmatmul____mul____rmul____truediv____rtruediv____add____radd____sub____rsub____pow__c                ,     t        fd      |       S )Nc                0    t        j                  |       S r.   )r6   powerr  r/  s    r'   rl   z<lambda>.<locals>.<lambda>  s    SYYq!_ r)   r  r  s    `r'   rl   rl     s    =(#<=a@ r)   )	__rpow____getitem____iter____gt____ge____lt____le____eq____ne__c                v     t        t        j                  t        j                  |||||            |       }|S )Nr  )r   r   r  r  r  r  s          r'   _bcsr_rewriting_taker    sE    F8I%%S=O	nE FFIK& 
-r)   )r  r  r  )rp   r+   ru   r   rR   r   )rp   r+   r|   r   rR   r   )ru   zlist[ArrayOrSparse])r   r   )
r   z
core.Jaxprr   zSequence[Array]r|   Sequence[SparsifyValue]rp   r+   rR   r  )F)rp   r+   r   zcore.ClosedJaxpr)FFNN)
__future__r   collections.abcr   r   r   typingr   r   numpyrI   r  r   jax._srcr	   r
   r   jax._src.custom_derivativesr   r   r   r   r   jax.experimental.sparse.bcoor   r   r   jax.experimental.sparse.bcsrr   	jax.numpyr6   jax._src.api_utilr   jax._src.libr   jax._src.interpretersr   r  jax.tree_utilr   r   r   jax._src.utilr   r   r   jax._src.lax.control_flowr   jax._src.numpyr   r  jax.experimentalr   r    rX   r!   conj_pcopy_pimag_prT  real_p(_zero_preserving_linear_unary_primitivesabs_pasin_pasinh_patan_patanh_pbessel_i1e_pexpm1_plog1p_psign_psin_psinh_psqrt_psquare_ptan_ptanh_pconvert_element_type_p!_zero_preserving_unary_primitivesacos_pacosh_pbessel_i0e_pcos_pcosh_peq_pexp_pge_pgt_ple_plt_plog_pne_pxor_pr"   r(   rQ   ArrayOrSparser+   rH   rt   r{   rv   r}   r   Tracerr   Tracer   transformation_with_aux2r   r   r   r   r
  r  r   r  r  _primr$  broadcast_in_dim_pbcoo_broadcast_in_dimconcatenate_pconv_general_dilated_pbcoo_conv_general_dilateddot_general_pbcoo_dot_generaldynamic_slice_p	reshape_pbcoo_reshaperev_pbcoo_revslice_p
bcoo_slice	squeeze_pbcoo_squeeze_BCOO_STANDARD_PRIMITIVESitemsr   	bcoo_implbcsr_dot_generalbcsr_broadcast_in_dim_BCSR_STANDARD_PRIMITIVES	bcsr_implr1  r0  rB  transpose_prR  add_prU  sub_prY  mul_pr[  div_pra  reduce_sum_prk  gather_pru  r  r{  r  r  r  r  r  r  r  r  	todense_pr  r  r  r  r  r  r  r  
_swap_argsabsnegativepositivematmulmultiplydividerG  subtractr  greatergreater_equalless
less_equalequal	not_equal_bcoo_methodsmethodimplsetattrr  rP   r)   r'   <module>rt     s}   # D /  "  
     0 &  # X X -  2  4 @ @ 8 8 ; 3 #57 2 757 2 7 ******))**, ( ))**++**++++++**))****,,))**!% !( **++))**(())(((((((())(())1 - "P 	=W =W@<J <2 ;8BB
B 
B&CCC 
C&BBB 
B4;; $/($** /(b 	 	3 4'4'4' &4' 	4'
 4'l.
)<(* / LE6uUKE6uUKEL 6 KE6uTJE6uTJEK &66Df>>V,,R--$$))V__++v  --$$
  1668 COD)1$	BDC V,,&66D  1668 COD)1$	BDCT (; ### $': ### $> &7 #// ""H  + #)) M  + #)) 4  + #))   + #)) 	 '9 #"" #. #6 #,, +<G  "/ #++ (GT !- $** B76 !- #** 
( !- #** 
 '; &"" #&: &"" #,  ?V #((:: ;>U #((:: ;:>= MR15	3 ,

GX  Xcgg	
 Xcll# Xcll# $ *SZZ01 Xcll# hz#,,/0 #**% (:cjj12 Xcgg hz#''*+ Xcll#  hz#,,/0!" @#$ z#)),-%S[[!S&&'SXXS^^$SYYS]]#5: "'') LFD	$ MR15 $*SZZ01% "'') LFD	$r)   