
    uki             	         d Z ddlmZ ddlmZ ddlZddlmZ ddlZddlZddl	m
Z
mZmZ ddlZddlZddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$mZ% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl*m.Z. ddl"m/Z/ ddl"m0Z0 ddl"m1Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@mAZA ddl&mBZB ej                  ej                  ej                  ej                  gZGej                  gZIddZJddZKdddZL G d d e      ZM G d! d"e      ZNdd#ZOdd$ZP e-j                  d%      ZRdd&ZS	 	 dd'ZTeRj                  d(        ZVeRj                  d)        ZXd* ZYd+ ZZd, Z[ e/j                  eReYd       eZe/j                  eR<   e[e0j                  eR<    e#j                  eR e#j                  eVd-.              e-j                  d/      Zad0ea_b        d1Zcddde%j                  d2	 	 	 	 	 dd3Zddde%j                  d4	 	 	 dd5Zeeaj                  d6        Zfeaj                  d7        Zgd8 Zhd9 Zid: Zjehe/j                  ea<   eie/j                  ea<   eje0j                  ea<    e#j                  ea e#j                  efd0.              e-j                  d;      Zldd<dd=Zmd0d<dd>Znelj                  d?        Zoelj                  d@        ZpdA ZqdB ZrdC Zs e/j                  eldeq       ere/j                  el<   ese0j                  el<    e#j                  el e#j                  eod-.              e-j                  dD      Ztd0et_b        ddEZu	 	 	 	 	 	 ddFZvdG Zwetj                  ddH       Zxetj                  ddI       ZyddJZzddKZ{ddLZ|eze/j                  et<   e{e/j                  et<   e|e0j                  et<    e#j                  et e#j                  exd0.              e-j                  dM      Z}ddddN	 	 	 	 	 	 	 ddOZ~	 	 	 	 	 	 	 	 ddPZ	 	 	 	 	 	 	 	 ddQZ	 	 ddRZe}j                  	 	 ddS       Z e#j                  ed-.      ZdT ZdU Z e#j                  ed-.      Z	 	 ddVZ	 	 ddWZ	 	 ddXZ	 	 ddYZ e/j                  e}ede       ee/j                  e}<   ee0j                  e}<    e#j                  e}e        e.j                  e}        e#j                  e}edZ[        e#j                  e}ed\[        e-j                  d]      Zdd^Zd_ Zd` Zda Zej                  db        Zej                  dc        Zdd Zde Zdf Zdg Z e/j                  eeed       ee/j                  e<   ee0j                  e<    e#j                  e e#j                  ed-.              e-j                  dh      Zd0e_b        	 	 	 	 	 	 	 	 	 	 ddiZdj Zej                  ddk       Zej                  ddl       ZddmZdn Zee0j                  e<   ee/j                  e<    e#j                  e e#j                  ed0.              e-j                  do      Zd0e_b        ddpZej                  dq        Zdr Zej                  ds        Zdt Zdu Z e#j                  ed0.      Zee/j                  e<   ee0j                  e<    e#j                  ee        e-j                  dv      Zd0e_b        dddwZddxZej                  dy        Zdz Zd-d-d-d{d|Zd-d-d-d{d}Zd~ Zej                  d        Zd Zd Z e#j                  ed0.      Zee/j                  e<   ee0j                  e<    e#j                  ee       dddd	 	 	 ddZdd	 ddZ	 	 	 	 ddZddZddd	 	 	 ddZd ZddZdd	 	 	 ddZddZd ZddZddZddZ	 	 	 	 	 	 ddZd ZddZddZd-d-ddd	 	 	 	 	 	 	 	 	 	 	 ddZddddddddd	 ddZd ZddZej                   G d de             Zd Zd Z e0j                  eeeeed       y)zHBCOO (Bached coordinate format) matrix object and associated primitives.    )annotations)SequenceN)partial)Any
NamedTupleProtocol)lax)	tree_utilvmap)	JAXSparse)
nfold_vmap_count_stored_elements_dot_general_validated_shapeCuSparseEfficiencyWarningSparseEfficiencyErrorSparseEfficiencyWarningShape
SparseInfo)
coo_spmv_p
coo_spmm_p)mlir)safe_zipunzip2
split_list)api_util)config)core)dispatch)ad)batching)partial_eval)_constranges_like	remaining_dot_general_batch_dim_numsDotDimensionNumbers)GatherDimensionNumbersGatherScatterMode)_unique)Array	ArrayLike	DTypeLike)canonicalize_axisc                l   | \  }}|\  }}|j                   dz
  t        |d u       z   t        fd|D              st        d|dd      d ||f||ffD        \  }}	|t	        d t        ||f|      D              }t        |g|j                  |j                  |j                        }
||	|
fS )	N   c              3  J   K   | ]  }|d u xs d|cxk  xr k  nc   y wNr    ).0bn_batchs     W/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/experimental/sparse/bcoo.py	<genexpr>z,_bcoo_batch_dims_to_front.<locals>.<genexpr>A   s'     ?qQ$Y*!q*7**?    #zEbatch_dims must be None or satisfy 0 < dim < n_batch. Got batch_dims=z for n_batch=.c              3     K   | ]7  \  }}|t        j                  |dg      nt        j                  ||d       9 y wr2   )r	   expand_dimsjnpmoveaxis)r4   argbdims      r7   r8   z,_bcoo_batch_dims_to_front.<locals>.<genexpr>D   sA      #E
#t $(<coocA3S\\#tQ5OO#Es   =?c              3  F   K   | ]  \  }}|	|j                   |     y wNshaper4   r?   dims      r7   r8   z,_bcoo_batch_dims_to_front.<locals>.<genexpr>H   s"     hSX[XgSYYs^h   
!!indices_sortedunique_indices)
ndimboolallNotImplementedErrormaxzipr   rD   rI   rJ   )batched_args
batch_dimsspinfo
batch_sizedataindices	data_bdimindices_bdimbatched_databatched_indicesbatched_spinfor6   s              @r7   _bcoo_batch_dims_to_frontr\   =   s    -$&)\LL1tLD$899'	?J?	?
 1%/MwjC D D#Ey)G\+BC#E, hCw4ThhJz9FLL9-3-B-B-3-B-BD. 
	66    BCOOc                   t        j                  |      }|dk\  sJ | j                  |k(  r| S || j                  k  rO| j                  g d t	        | j
                        D        t        |         }| j                  dd|ddf   }nt        j                  | j                  g | j                  j                  d| j
                   || j                  j                  | j
                  dz   d       }|j                  g d t	        | j
                        D        t        | j                           j                  | j                        }t        j                  | j                  g | j                  j                  dd || j                  j                  d	         }|j                  dd| j                  ddf   j                  | j                        }|j                  d| j                  dddf   j                  t        j                  | j                  | j
                  | j
                  | j                  z    |j                  
            }t!        ||f| j                  | j"                  | j$                        S )zsReturn a copy of `mat` with the specified nse.
  Note that if nse < mat.nse, this will potentially discard data.
  r   c              3  2   K   | ]  }t        d         y wrB   slicer4   is     r7   r8   z _bcoo_set_nse.<locals>.<genexpr>[   s     ?quT{?   .N   rC   c              3  2   K   | ]  }t        d         y wrB   ra   rc   s     r7   r8   z _bcoo_set_nse.<locals>.<genexpr>_   s     >aeDk>re   dtyperD   rI   rJ   )operatorindexnserU   ranger6   rb   rV   r=   
zeros_likerD   atsetarrayn_sparserk   r^   rI   rJ   matro   rU   rV   s       r7   _bcoo_set_nserx   R   s    	s#	/WW^JCGG^88L?E#++,>?LsLMDkk#ttQ,'G>>#((*qCHHNN<CKK,H*q#*qPSPXPXP^P^_b_j_jmn_n_oPp*qrD77O>5+=>OcggOPTTUXU]U]^DnnS[[0e#++2C2CCR2H0e#0es{{O`O`acOd0efGjjhswwh)*..s{{;Gjjcggh)*..syy3;;s{{]`]i]iOi9j?F}}0N OG	tWoSYY // //
1 1r]   c                n  	 | j                   | j                  | j                  }}}t        |||      }|dk(  j	                  t        t        |j                  dz   |j                                    }t        d t        |j                  d |j                         D              }|j	                  |d      }t        j                  ||j                  |j                  |j                  z    |j                        		fd} t        ||j                        ||      }t        t!        ||f|      |	      S )
Nr   rf   c              3  2   K   | ]  \  }}|d k(  s|  ywrf   Nr3   r4   rd   ss      r7   r8   z'bcoo_eliminate_zeros.<locals>.<genexpr>o   s     \AUVZ[U[1\s   Tkeepdimsrj   c                N    t        j                  |d d d f   d d d f   |       S rB   )r=   where)rd   m
fill_values     r7   <lambda>z&bcoo_eliminate_zeros.<locals>.<lambda>r   s%    399Qq$wZD!G)<a@ r]   rC   ro   )rU   rV   rD   _validate_bcoorM   tuplerp   r6   rK   	enumerater=   rt   ru   rk   r   bcoo_sum_duplicatesr^   )
rw   ro   rU   rV   rD   propsmaskdims_to_contractfr   s
            @r7   bcoo_eliminate_zerosr   k   s    3;;		$
w
.%
!)uU]]Q%6		BC	D$\7==%--3P)Q\\	"T	2$yyu}}U]]U^^-KLT[TaTab*@!(Jq%--($7'	T4/?S	IIr]   c                  6    e Zd ZU ded<   ded<   ded<   ded<   y)BCOOPropertiesintr6   ru   n_densero   N)__name__
__module____qualname____annotations__r3   r]   r7   r   r   w   s    ,-,
(r]   r   c                  0    e Zd Zedd       Zedd       Zy)Bufferc                     y rB   r3   selfs    r7   rD   zBuffer.shape~   s    r]   c                     y rB   r3   r   s    r7   rk   zBuffer.dtype   s    r]   N)returnr   )r   r   )r   r   r   propertyrD   rk   r3   r]   r7   r   r   }   s      r]   r   c           
     J   t        ||      }|\  }}}}t        |      }t        d t        | j                  d | |d |       D              rt        d| j                  d|      | j                  |d  |f|||z   d  z   k7  r!t        d| j                  d|d|d|      |S )Nc              3  .   K   | ]  \  }}|d |fv  ywr{   r3   r4   s1s2s      r7   r8   z!_validate_bcoo.<locals>.<genexpr>   s     Yvr2Ar7	Y   z4data batch dimensions not compatible for data.shape=, shape=zInvalid data.shape=	 for nse=
, n_batch=
, n_dense=)_validate_bcoo_indicesr   anyr   rD   
ValueError)rU   rV   rD   r   r6   ru   r   ro   s           r7   r   r      s    
 %
0%$)!'8Wc
,%Y$**Xg2Fhw)XYY
LYPUxX
YY	ZZcVeGh,>,?&@@@
+

}J#{';gZP
QQ	,r]   c           
        t        j                  | j                  t         j                        sJ t	        |      }| j
                  dd  \  }}t        | j
                        dz
  }t        |      |z
  |z
  }|dk\  sJ t        d t        | j
                  d | |d |       D              rt        d| j
                  d|      | j
                  |d  ||fk7  r!t        d| j
                  d|d	|d
|      t        ||||      S )Nrh   r0   r   c              3  .   K   | ]  \  }}|d |fv  ywr{   r3   r   s      r7   r8   z)_validate_bcoo_indices.<locals>.<genexpr>   s     \vr2Ar7	\r   z:indices batch dimensions not compatible for indices.shape=r   zInvalid =indices.shape=r   r   r   )r6   ru   r   ro   )r=   
issubdtyperk   integerr   rD   lenr   r   r   r   )rV   rD   ro   ru   r6   r   s         r7   r   r      s	   	s{{	33	3
,%--$-#x"'J 8+'	A\'--2I5QYRY?)[\\
RGMMCSS\V[U]^
__]]78h/
/ 0
cV;gZ{'T
UU	(GQT	UUr]   bcoo_todensec                Z    t        | j                  | j                  | j                        S )zConvert batched sparse matrix to a dense matrix.

  Args:
    mat: BCOO matrix.

  Returns:
    mat_dense: dense version of ``mat``.
  rS   )_bcoo_todenserU   rV   _info)rw   s    r7   r   r      s     
sxxSYY	??r]   c               ~    t         j                  t        j                  |       t        j                  |      |      S )a  Convert batched sparse matrix to a dense matrix.

  Args:
    data : array of shape ``batch_dims + (nse,) + block_dims``.
    indices : array of shape ``batch_dims + (n_sparse, nse)``
    spinfo : SparseInfo. In particular, this includes the shape
      of the matrix, which is equal to ``batch_dims + sparse_dims + block_dims``
      where ``len(sparse_dims) == n_sparse``

  Returns:
    mat : array with specified shape and dtype matching ``data``
  r   )bcoo_todense_pbindr=   asarrayrU   rV   rS   s      r7   r   r      s.     
		S[[.G0DV		TTr]   c          	        |j                   }t        | |      \  }}}}t        d t        |d | j                   d |       D              }t        t	        j
                  |ddd      t        fdt        |      D              }t        d |d | D              }	t	        j
                  g |	t	        j                  d      dddt              d d }
|s'| j                  |t        |
      | j                  	      } t        j                  || j                        j                  |
|z      j                  |       S )
Nc              3     K   | ];  \  }}|d k(  rt        j                  |t              nt        j                  |       = ywr{   npzerosr   aranger4   r}   i_ss      r7   r8   z%_bcoo_todense_impl.<locals>.<genexpr>   s<      RC *-RXXa%ryy|C R   AAijTindexingsparsec              3  D   K   | ]  }t        d       |fz        y wrB   ra   r4   rd   gridrV   s     r7   r8   z%_bcoo_todense_impl.<locals>.<genexpr>   s$     O!WTU4[!$445O    c              3  F   K   | ]  }t        j                  |        y wrB   r   r   r4   r}   s     r7   r8   z%_bcoo_todense_impl.<locals>.<genexpr>   s     =ryy|=   !rf   ri   r   rk   )rD   r   r   rP   r   meshgridrp   r   sumrL   rk   r=   r   rr   add)rU   rV   rS   rD   r6   ru   _
ind_slices
sparse_indbatch_slices	batch_indr   s    `         @r7   _bcoo_todense_implr      s   
,,%*4%@'8Q R#&uXghw8O#PR R*	r{{JdC	D$OuXOO*=U8G_==,		Ml	MBIIaL	M4	M$Dk#2)	88Gd9oTZZ8HD	5$**	%	(	(Z)?	@	D	DT	JJr]   c               t    |j                   }t        | ||       t        j                  || j                        S rB   )rD   r   r   ShapedArrayrk   )rU   rV   rS   rD   s       r7   _bcoo_todense_abstract_evalr      s.    
,,%w&			%	,,r]   c                   t        | ||      S )Nr   )r   )data_dotrU   rV   rS   s       r7   _bcoo_todense_jvpr      s    	x	88r]   c                  |j                   }t        j                  |      sJ t        j                  |      rt        d      | j                   |k(  sJ | j                  |j
                  j                  k(  sJ t        ||       |fS )N/Cannot transpose with respect to sparse indices)rD   r    is_undefined_primalr   rk   aval_bcoo_extract)ctrU   rV   rS   rD   s        r7   _bcoo_todense_transposer      sw    
,,%				%%	%G$
F
GG	U			TYY__	$$	$	w	#W	,,r]   c               D    t        | ||      \  }}}t        |||      dfS )Nr   r   )r\   r   )rQ   rR   rS   rU   rV   s        r7   _bcoo_todense_batching_ruler      s+    3L*fU$	tWV	4a	77r]   F)multiple_resultsbcoo_fromdenseTz
The error arose for the nse argument of bcoo_fromdense. In order for
BCOO.fromdense() to be used in traced/compiled code, you must pass a concrete
value to the nse (number of stored elements) argument.
ro   r6   r   index_dtypec          	         t        j                  |       } |}|t        | ||      }t        j                  t
        j                  |t              }t        t        | ||||      | j                  dd      S )a  Create BCOO-format sparse matrix from a dense matrix.

  Args:
    mat : array to be converted to BCOO.
    nse : number of specified elements in each batch
    n_batch : number of batch dimensions (default: 0)
    n_dense : number of block_dimensions (default: 0)
    index_dtype : dtype of sparse indices (default: int32)

  Returns:
    mat_bcoo: BCOO representation of the matrix.
  r   Trl   )r=   r   r   r   concrete_or_errorrm   rn   _TRACED_NSE_ERRORr^   _bcoo_fromdenserD   )rw   ro   r6   r   r   nse_arrnse_ints          r7   r   r      so     	C# #'_$S'7;G""8>>7<MN'	ocw*57IId4
I Ir]   r6   r   r   c                   t        j                  |       } t        j                  t        j
                  |t              }t        j                  | ||||      S )aA  Create BCOO-format sparse matrix from a dense matrix.

  Args:
    mat : array to be converted to BCOO, with ``ndim = n_batch + n_sparse + n_dense``.
    nse : number of specified elements in each batch
    n_batch : number of batch dimensions (default: 0)
    n_dense : number of block_dimensions (default: 0)
    index_dtype : dtype of sparse indices (default: int32)

  Returns:
    data : array of shape ``mat.shape[:n_batch] + (nse,) + mat.shape[mat.ndim - n_dense:]``
      and dtype ``mat.dtype``
    indices : array of shape ``mat.shape[:n_batch] + (n_sparse, nse)``
  r   )	r=   r   r   r   rm   rn   r   bcoo_fromdense_pr   )rw   ro   r6   r   r   s        r7   r   r     sN      	C#x~~s4EF#			sWg+6 
 
8 8r]   c                  t        j                  |       } | j                  |z
  |z
  | dk7  }|dkD  r,|j                  t	        |      D cg c]  }|dz    
 c}      }t        t        |d      fd       } ||      }|s)t        j                  |j                  d | dfz   |      }n.t        j                  t        j                  ||      d|dz         }t        ||       }	|j                  t        t	        ||j                                    d   }
t        j                  |
j                   d|z  fz   |      |
k  }||dz   t#        d       fz  |dz  z      }t        j$                  ||	d      }	|	|fS c c}w )	Nr   rf   F)Nbroadcastedc                h    | j                   r%t        j                  | | j                  d        S y)Nsizer   r3   )rK   r=   nonzerorD   )aru   ro   s    r7   _nonzeroz&_bcoo_fromdense_impl.<locals>._nonzero*  s+    vv[[(1CDDr]   .Nrf   rB   )r=   r   rK   r   rp   r   r   r   rD   r>   rt   r   r   listr	   broadcasted_iotark   rb   r   )rw   ro   r6   r   r   r   rd   r   rV   rU   true_nsetrue_nonzerosru   s    `          @r7   _bcoo_fromdense_implr  #  sb   C#XX')(
($q[88uW~6!q1uX67D
:e4 5 TN'	ii

8G,Qx7EGll399Wk:Aw{KG	w	$$XXd5$))456yA(&&x~~tg~7NPWX[cc-1t >7AR RS-	=$	*$	w% 7s   E:c                  | j                   |z
  |z
  }| j                  d | |fz   | j                  ||z   d  z   }| j                  d | ||fz   }t        j                  || j                        t        j                  ||      fS rB   )rK   rD   r   r   rk   )rw   ro   r6   r   r   ru   
data_shapeindex_shapes           r7   _bcoo_fromdense_abstract_evalr	  =  s    XX')(yy'"cV+cii(8J8K.LL*		(7#sHo5+			*cii	0$2B2B;P[2\	\\r]   c                  | \  }|\  }t        |||||      }|\  }	}
t        |      t        j                  u r t        j                  j	                  |	      }nt        |
|      }|t        j                  j	                  |
      f}||fS )Nr   )r   typer    Zerofrom_primal_valuer   )primalstangentsro   r6   r   r   MMdotprimals_outrU   rV   r   tangents_outs                r7   _bcoo_fromdense_jvpr  D  s    "!
%$sGWZef+-$	$Z277ww((.HWd+HBGG55g>?,	l	""r]   c                  | \  }}|j                   |z
  |z
  }|j                  |j                  d | |fz   |j                  ||z   d  z   k(  sJ |j                  |j                  d | ||fz   k(  sJ |j                  |k(  sJ t        |t        j
                        rt        d      t	        j                  |      sJ t        ||t        |j                  j                              S )Nr   r   )rK   rD   rk   
isinstancer    r  r   r   r   r   r   )	r   r  ro   r6   r   r   rU   rV   ru   s	            r7   _bcoo_fromdense_transposer  T  s    -$VVg'(	qwwx(C61AGGGh<N<O4PP	PP	P	!''(7+xo=	==	=	+	%%	%!
F
GG				""	"	tWZ-E	FFr]   c               |    | \  }|\  }d|cxk  r|k  sn t        d|d|      t        |||dz   ||      ||ffS )Nr   z'Expected 0 < bdim <= n_batch; got bdim=r   rf   r   )r   r   )rQ   rR   ro   r6   r   r   r  r@   s           r7   _bcoo_fromdense_batching_ruler  _  sZ    "!
%$
t
w

?$7*M
NN	Wq['Wb	cfjlpeq	qqr]   bcoo_extractassume_uniquec                  t        | t              st        dt        |             t	        j
                  |      }|j                  | j                  k7  r%t        d| j                  d|j                        || j                  }t        | j                  ||      }t        || j                  ffi | j                  j                         S )a   Extract values from a dense array according to the sparse array's indices.

  Args:
    sparr : BCOO array whose indices will be used for the output.
    arr : ArrayLike with shape equal to self.shape
    assume_unique : bool, defaults to sparr.unique_indices
      If True, extract values for every index, even if index contains duplicates.
      If False, duplicate indices will have their values summed and returned in
      the position of the first index.

  Returns:
    extracted : a BCOO array with the same sparsity pattern as self.
  zGFirst argument to bcoo_extract should be a BCOO array. Got type(sparr)=zshape mismatch: sparr.shape=z	 a.shape=r  )r  r^   	TypeErrorr  r=   r   rD   r   rJ   r   rV   r   _asdict)sparrarrr  r   rU   s        r7   r  r  r  s     
E4	 
^RVW\R]Q_`
aa	kk#!WW
4~ZqwwjA
BB((M	u}}a}	E$	tU]]#	=u{{':':'<	==r]   c               2    t         j                  | ||      S )a  Extract BCOO data values from a dense array at given BCOO indices.

  Args:
    indices: An ndarray; see BCOO indices.
    arr: A dense array.
    assume_unique: bool, default=True
      If True, then indices will be assumed unique and a value will be extracted
      from arr for each index. Otherwise, extra work will be done to de-duplicate
      indices to zero-out duplicate extracted values.

  Returns:
    An ndarray; see BCOO data.
  r  )bcoo_extract_pr   )rV   r!  r  s      r7   r   r     s     
		Wc		GGr]   c          	         t        j                  |      }t         |j                        }|s3t	         |j                  d      \   }|}t         |j                        }t        d t        |j                  d |j                    j                  d |j                         D              }t        t        j                  |ddd      t         fdt        |j                        D              }t        d |j                  d |j                   D              }t        j                  g |t        j                  d      dddt              d d	 }	||	z   s|d    }
n#|j                  |	|z      j                  d
d      }
|j                  dk(  r|j                  dk7  r|rTt!        j"                  |
t%        |
j                  |j                  |j                        t        |
j&                              }
nt%        |
j                  |j                  j                        }|j                  t)        d       fz  t)        d      fz   }t        j*                  |
|      j                  |   j-                  |
      }
|s]j                  g|
j                  |j                  dz   d  fd}t        |j                        D ]  }t/        |      }  ||
      }
|
S )NT)rD   return_indexc              3     K   | ];  \  }}|d k(  rt        j                  |t              nt        j                  |       = ywr{   r   r   s      r7   r8   z%_bcoo_extract_impl.<locals>.<genexpr>  s<      bC *-RXXa%ryy|C br   r   r   c              3  D   K   | ]  }t        d       |fz        y wrB   ra   r   s     r7   r8   z%_bcoo_extract_impl.<locals>.<genexpr>  s$     U!WTU4[!$445Ur   c              3  F   K   | ]  }t        j                  |        y wrB   r   r   s     r7   r8   z%_bcoo_extract_impl.<locals>.<genexpr>  s     Gryy|Gr   rf   ri   fillr   moder   rC   c                j    t        j                  |       j                  |   j                  |       S NrC   )r=   rq   rr   r   )rrd   unbatched_out_shapes     r7   r   z_bcoo_extract_impl.<locals>.f  s+    ^^A%89<<Q?CCAFFr]   )r=   r   r   rD   _unique_indicesr   rP   r6   r   r   rp   ru   r   rr   getro   r	   broadcast_in_dim_tuple_replacerK   rb   rq   rs   r   )rV   r!  r  r   sort_indoriginal_propsr   r   r   r   result	out_shapeindr   r   r   r/  s   `              @@r7   _bcoo_extract_implr9    sa   C#
 #))
4%	'syytTGXN"7CII6E b#&syy%--'@'--P^QVQ^Q^B_#`b b*	r{{JdC	D$UuU^^?TUU*GSYY~-FGG,		Ml	MBIIaL	M4	M$Dk#2)	i	YFVVI
*+//V/JF
^^qUYY!^##v||U]]EIIFfkkHZ\f !u}}n>P>PQiMMU4[N*eAh[8c~~fI699#>BB6Jf	)--QU]]Q=N=O0PQG5==! 
q'avx F	-r]   c                   t        |      }t        | |j                        \  }}}}|j                  d | |fz   |j                  |j                  |z
  d  z   }t	        j
                  ||j                        S rB   )rL   r   rD   rK   r   r   rk   )rV   r!  r  r   r6   r   ro   r7  s           r7   _bcoo_extract_abstract_evalr;    sm    
=!3GSYYG'1gsii!SF*SYYsxx'7I7J-KK)			)SYY	//r]   c               T    | j                   |j                   k(  sJ t        || |      S )Nr  )rD   r   )arr_dotrV   r!  r  s       r7   _bcoo_extract_jvpr>    s'    	#))	##	#	w}	EEr]   c               .   |st        d      t        j                  |      sJ t        j                  |      rt        d      | j                  |j
                  j                  k(  sJ |t        | |t        |j
                  j                              fS )Nz2transpose of bcoo_extract with assume_unique=Falser   r   )	rN   r    r   r   rk   r   r   r   rD   )r   rV   r!  r  s       r7   _bcoo_extract_transposer@    sz    	
R
SS				$$	$G$
F
GG	SXX^^	##	#	-GJsxx~~4NO	OOr]   c                  | \  }}t        d |D              sJ |d   |d   }t        j                  ||f      }nr|d   R|d   }t        |j                        }|j                  ||j                  |          t        j                  |||f      }n|d   |d   k7  rt        d      |d   }|j                  dz
  }||k\  rt        d|d|      t        |||      |fS )	Nc              3  $   K   | ]  }|d u 
 y wrB   r3   )r4   r5   s     r7   r8   z._bcoo_extract_batching_rule.<locals>.<genexpr>  s     /qQd]/s   r   rf   z+bcoo_extract with unequal batch dimensions.r0   zbatch_dims=z' out of range for indices with n_batch=r  )r   r	   r<   r  rD   insertr2  rN   rK   r   r   )rQ   rR   r  rV   r!  r@   result_shaper6   s           r7   _bcoo_extract_batching_rulerE    s    ,'3	/J/	//	/]a=Doogw/G!}a=D		?LgmmD12


sL4'
:C!}
1% MNNa=DLL1'	W_

}$LG:N
OO	w=	A4	GGr]   bcoo_transposec                    t         j                   j                  | j                        }t	         fd|D              }t        || j                        S )u-  Transpose a BCOO-format array.

  Args:
    mat: A BCOO-format array.
    permutation:  A tuple or list or ndarray which contains a permutation of
      [0,1,..,N-1] where N is the number of axes of ``mat`` in the order of
      batch, sparse, and dense dimensions. The i’th axis of the returned array
      corresponds to the axis numbered permutation[i] of ``mat``. Transpose
      permutation currently does not support permuting batch axes with non-batch
      axes nor permuting dense axes with non-dense axes.

  Returns:
    A BCOO-format array.
  permutationrS   c              3  <   K   | ]  }j                   |     y wrB   rC   r4   prw   s     r7   r8   z!bcoo_transpose.<locals>.<genexpr>  s     6QCIIaL6   )rD   rJ   )_bcoo_transposerU   rV   r   r   r^   rJ   )rw   rI  buffersr7  s   `   r7   rF  rF    sI     CHHckk{SVS\S\]'6+66)	gYs7I7I	JJr]   c                   t        |      }|t        t        t        |j                                    k(  r| |fS t        j                  | |||      S NrH  )r   rp   r   rD   bcoo_transpose_pr   )rU   rV   rI  rS   s       r7   rN  rN    sR    k"+E%FLL 1233=  wK(. ! 0 0r]   c                   t        |t        t        t        j                  f      st        dt        |       d      t        t        |            t        t        t        |                  k7  rt        d| d| d      t        | ||      \  }}}}|d | }||||z    D 	cg c]  }	|	|z
  	 }
}	|||z   d  D 	cg c]
  }	|	|z
  |z
   }}	|r;t        t        |            t        t        |            k7  rt        d| d|d      |r;t        t        |            t        t        |            k7  rt        d| d|d      ||
|fS c c}	w c c}	w )	Nz8transpose permutation must be a tuple/list/ndarray, got r:   zQtranspose permutation isn't a permutation of operand dimensions, got permutation z for shape zUtranspose permutation cannot permute batch axes with non-batch axes; got permutation z, with n_batch=zUtranspose permutation cannot permute dense axes with non-dense axes; got permutation z, with n_dense=)r  r   r  r   ndarrayr  r  sortedrp   r   r   rN   )rU   rV   rI  rD   r6   ru   r   r   
batch_permrL  sparse_perm
dense_perms               r7   _validate_permutationrY    s   	K%rzz!:	;
NtT_O`Naabc
dd
6+5s5z):#;;
 ''2m;ugQH I I"0w"F'8Wa8G$*&1'7X;M&NOWO+O0;Gh<N<O0PQ1Hw&Q*Qvj)*eE'N.CC
 11<=MWJaQ R Rvj)*eE'N.CC
 11<=MWJaQ R R	[*	,, PQs   #E;Ec                   t        | |||j                        \  }}}t        |       |d|f   j                  g |dz    } | j                  g |fd|D         } | |fS )N.rf   c              3  .   K   | ]  }|z   d z     ywr{   r3   r4   dr6   s     r7   r8   z'_bcoo_transpose_impl.<locals>.<genexpr>+  s     /TAGa/T   )rY  rD   r   	transpose)rU   rV   rI  rS   rV  rW  rX  r6   s          @r7   _bcoo_transpose_implr`  &  s    (=dG[Z`ZfZf(g%*k:
O'/GC$%//RRWRgPQkR'		U	UW	U/T/T	U$	wr]   c                 	 t        | |||j                        \  }}}t        |      	t        j                  |j                        g |		dz      }t        j                  | j                        g |		fd|D           }t        j                  || j                        t        j                  ||j                        fS )Nrf   c              3  .   K   | ]  }|z   d z     ywr{   r3   r\  s     r7   r8   z0_bcoo_transpose_abstract_eval.<locals>.<genexpr>3  s     <aQRQ[1_<ar^  )rY  rD   r   r   rt   r   r   rk   )
rU   rV   rI  rS   rV  r   rX  indices_shaper  r6   s
            @r7   _bcoo_transpose_abstract_evalrd  .  s    3D';PVP\P\]*a
O'((7==)*MJ*M*M1*MN-xx

#$bj$b'$b<aV`<a$bc*			*djj	143C3CMSZS`S`3a	aar]   c                   | \  }}|\  }}t        ||||      }t        ||||      \  }	}||	t        j                  j                  |      ffS rQ  )rN  r    r  r  )
r  r  rI  rS   rU   rV   r   r   r  data_dot_outs
             r7   _bcoo_transpose_jvprg  6  sX    -$+(Ag;vV+#Hg;W]^/,	|RWW%>%>w%GH	HHr]   c               H   | \  }}t        |t        j                        sJ t        j                  |      rt	        d      |j
                  |j                  j
                  k(  sJ t        t        fd|D                    }t        t        t        t        j                  |                  }t        j                  t!        |j"                  dz
        D 	cg c]  }	d c}	t        |j$                  dd        z   t              }
t'        ||
||      \  }}||fS c c}	w )Nr   c              3  <   K   | ]  }j                   |     y wrB   rC   )r4   rL  rS   s     r7   r8   z,_bcoo_transpose_transpose.<locals>.<genexpr>C  s     D1v||ADrM  r0   rf   rh   rj   rH  )r  r    r  r   r   rk   r   r   r   r  mapr   r   argsortr=   r   rp   rK   rD   rN  )r   rU   rV   rI  rS   data_ct
indices_ct	ct_spinforev_permutationrd   dummy_indices
data_transr   s       `        r7   _bcoo_transpose_transposerr  =  s    ':	J	((	(G$
F
GG	$))//	))	)DDDE)S"**["9:;/))gllQ.>(?@1Q@4VXVYHZC[[cfg-!'=o^gh-*a	Z	 As   	Dc                  t        | ||      \  }}}dgd |D        }t        ||||      \  }}|D cg c]  }|d nd
 }}t        ||f|      D 	cg c]   \  }	}|t        j                  |	dg      n|	" }
}	}|
|fS c c}w c c}}	w )Nr   c              3  &   K   | ]	  }|d z     ywr{   r3   )r4   rL  s     r7   r8   z-_bcoo_transpose_batch_rule.<locals>.<genexpr>L  s     9a!e9   rH  )r\   rN  rP   r	   squeeze)rQ   rR   rI  rS   rU   rV   batched_permutationr@   batch_dims_outr?   args_outs              r7   _bcoo_transpose_batch_rulerz  J  s    3L*fU$:9[9:!$=PY_`-$<FGDDLDa/G.G"D'?NCE#t (,|ckk#s#< E( E	>	!! HEs   B%Bbcoo_dot_general)	precisionpreferred_element_typeout_shardingc          
     n   ~~t        | t              rt        |t              r}t        | j                  |j                  |      }t	        | j
                  | j                  |j
                  |j                  | j                  |j                  ||      }t        ||      S t        | t              r/t        | j
                  | j                  |||| j                        S t        |t              r/t        | |j
                  |j                  |||j                        S t        j                  | |||      S )a<  A general contraction operation.

  Args:
    lhs: An ndarray or BCOO-format sparse array.
    rhs: An ndarray or BCOO-format sparse array..
    dimension_numbers: a tuple of tuples of the form
      `((lhs_contracting_dims, rhs_contracting_dims),
      (lhs_batch_dims, rhs_batch_dims))`.
    precision: unused
    preferred_element_type: unused

  Returns:
    An ndarray or BCOO-format sparse array containing the result. If both inputs
    are sparse, the result will be sparse, of type BCOO. If either input is dense,
    the result will be dense, of type ndarray.
  
lhs_spinfo
rhs_spinfodimension_numbersr}  rC   r  r}  r  )r  r}  r  r  r}  )r  r^   r   rD   _bcoo_spdot_generalrU   rV   r   _bcoo_dot_general_bcoo_rdot_generalr	   dot_general)lhsrhsr  r|  r}  r~  rD   bufss           r7   r{  r{  `  s    , Tz#t4(CII):<Esxxchh*-))		1B6LND E""#tSXXs{{CK\4J(+		3 3 #tc388S[[L]5K),4 4 ??37H2HJ Jr]   c                  |\  \  }}\  }}	t        j                  |      t        j                  |      f}
t        j                  |      t        j                  |	      f}|t        j                  |      }t        j                  t        j                  |       t        j                  |      t        j                  |      |
|f||      S Nr  )r   _ensure_index_tupler   rk   bcoo_dot_general_pr   r=   r   )lhs_datalhs_indicesr  r  r}  r  lhs_contractrhs_contract	lhs_batch	rhs_batchcdimsbdimss               r7   r  r    s     :K6< 6I''5''57%''	2''	24%'XX&<=		 	 X!6K8PRUR]R]^aRb495>8N,6 
! 
8 8r]   c                  t        d |D              }t        ||| |||      }d |D        \  }}	t        |j                        |z
  }
g t	        |	      t	        |
|j
                        t	        |	|
      }t        j                  ||      S )Nc              3  ,   K   | ]  }|d d d     y wNri   r3   r4   r]  s     r7   r8   z%_bcoo_rdot_general.<locals>.<genexpr>  s     9]a!DbD'9]s   )r  r  r}  c              3  8   K   | ]  }t        |d            ywr   Nr   r  s     r7   r8   z%_bcoo_rdot_general.<locals>.<genexpr>  s     >qQqT>s   )r   r  r   rD   rp   rK   r	   r_  )r  rhs_datarhs_indicesr  r}  r  dimension_numbers_reversedr6  
n_contractr6   n_swaprI  s               r7   r  r    s     5:9]K\9]4]X{CJ/I4JL& ?,=>*gz :-&W%.W5#=Wgv@VW+	v{	++r]   c          
        t        j                  |       } t        j                  |      }t        j                  |      }t        | j                  |j                  |j                  ||      }|j                  d   |j
                  dz
  |\  \  }}\  }	}
t        t        ||      D cg c]  \  }}|k  s||f c}}      \  }t        t        ||      D cg c]  \  }}|k\  s||f c}}      \  }|	srng |	t        t              |	      }| j                  g |t        | j
                              } |j                  g |t        |j
                              }rGt        fdD              t        j                  g t        t                          }|d|f   }g |
||t        t        |j
                        |
|      }|j                  |      }fd}t        |t              z
        }t        j                   |t        t        |
      t              z
              }t        j"                  |j                  |j$                        } ||| ||      S c c}}w c c}}w )Nr  ri   r0   c              3  (   K   | ]	  }|z
    y wrB   r3   r\  s     r7   r8   z)_bcoo_dot_general_impl.<locals>.<genexpr>  s     Eaa'kE   .c                   t        fdt              D              }|d t               }|t              d  }|rOj                  dkD  r@t	        j
                  d j                  d d D        ddid j                  dz
   }g ||}t        t        t        
      t              z               }t        j                  ||j                  |   j                  dd	      g g f||ff
      }	|r| j                  |   j                  |	      S |	j                  t        t        |	j                  | j                  z
              | j                        S )Nc              3  ,   K   | ]  }d |f     ywr   r3   )r4   rd   r  s     r7   r8   z9_bcoo_dot_general_impl.<locals>.result.<locals>.<genexpr>  s     =CF#=   r0   c              3  F   K   | ]  }t        j                  |        y wrB   )r=   r   )r4   ns     r7   r8   z9_bcoo_dot_general_impl.<locals>.result.<locals>.<genexpr>  s     :aCJJqM:r   ri   r   r   r)  r   r*  )r}  rj   )r   rp   r   rK   r=   r   rD   r  rL   r	   r  rr   r1  r   r   rk   )	out_arrayr  r  r  idx	idx_rightidx_out	idx_batchrR   prodlhs_contracting_blhs_contracting_sru   r}  s     `       r7   r6  z&_bcoo_dot_general_impl.<locals>.result  sL   
=U8_=
=C+S*+,I#'()*G[%%),,:;#4#4Sb#9:.+**Q.0i +I*	*ieC 12T:K5LLMNJ??8SVVI%6%:%:ST%:%UHz:&>?2HJD \\'"&&t,,XXeE$))inn"<=>iooXVVr]   )r=   r   _bcoo_dot_general_abstract_evalr   rD   rK   r   r   r%   rp   r_  r   rt   r   r   r	   r<   r   rk   )r  r  r  r  r}  r  out_avallhs_contractingrhs_contractingr  r  lr.  rhs_contracting_brhs_contracting_srV  rW  rhs_permr6  r  r  r  r6   ru   s       `               @@@@r7   _bcoo_dot_general_implr    s   [["(K(+C#,X]]K<L<Lchh?PDZ8BD( r"(q '?P<$?O&<y))/A1Rq!Q[QF1R *S&&)/A1Sq!Q'\QF1S *T&& #k9kywDUVkYjkJ!!"OJ"Ow1N"OPH''(X*(XuWkFVFV7W(XYK E3DEE))`/`)E(OM^2_`aKc;./KGy G, G/@ Gsxx)_EG(h#W" fg,=(>>?&U3y>7SAR=S3STU#ii7)		8[#	66U1R1Ss   1J
?J
$J
2J
c               :  
 t        j                  t        j                  d      j	                  t        j
                  |j                  | j                        t        j
                  |j                  |j                        ||      }|\  \  }}\  }	}t        | ||j                        \  
}}|	rt        |	      
k\  rt        d|	d
      t        
fd|D              rt        d      t        j                  |j                  |j                        S )Nr  static_argnameszqbcoo_dot_general batch dimensions must be among the batch dimensions in the sparse representation.
got lhs_batch=r   c              3  .   K   | ]  }|z   k\    y wrB   r3   )r4   r]  r6   ru   s     r7   r8   z2_bcoo_dot_general_abstract_eval.<locals>.<genexpr>  s     :Qg 	 :r^  z4bcoo_dot_general: contracting over dense dimensions.)jaxjitr	   r  
eval_shapeShapeDtypeStructrD   rk   r   rO   rN   r   r   r   )r  r  r  r  r}  r  r  r  r   r  r6   ru   s             @@r7   r  r    s     WWS__6efqq


z//
@


syy#))
4-!7	 r 9( *;&?AA*8[*BRBRS'8Q3y>W,
L7*&' '
 	:/::
T
UU			(..(..	99r]   c                >   | j                   t        vr)t        j                  d| j                   dt               y|j                   t
        vr)t        j                  d|j                   dt               y|j                  st        j                  dt               yy)NzJbcoo_dot_general cusparse/hipsparse lowering not available for data.dtype=z). Falling back to default implementation.TzMbcoo_dot_general cusparse/hipsparse lowering not available for indices.dtype=zbcoo_dot_general GPU lowering requires matrices with sorted indices. To sort the rows in your matrix, use e.g. mat = mat.sort_indices(). Falling back to the default implementation.F)rk   CUSPARSE_DATA_DTYPESwarningswarnr   CUSPARSE_INDEX_DTYPESrI   r   s      r7   _bcoo_dot_general_fallbackr    s    	ZZ++MM %%NP+- }}11MM ( ((QS+-   MM $ &?@ r]   c                  t         j                  j                  st        | |||||      S |\  \  }}\  }}	t	        | ||j
                        \  }
}}}	|j                  dk(  rt        nt        }t        | |||||      }|j                  t        vrt        | |||||      S | j                  |j                        } |j                  |j                        }t        |      dk(  rt        |      dk(  r|j                  dv r|
||fdk(  rt        | ||      st        j                   |j
                  d   |j                        |j#                         }}d}dg|j
                  }t%        ||||      \  }}}|j'                  | |||d   dk(  r|j(                  n|||      }|d   S t        |      dk(  rt        |      dk(  r|j                  dv r}|
||fdk(  rut        | ||      sh|d d df   |d d df   }}|d   dk(  }|j
                  }t%        ||||      \  }}}|j'                  | |||d   dk(  r|j(                  n|||      }|d d	 S t        | |||||
      S )Nr  rf   r   )rf   r0   )r   rf   r   F)r_  rD   )r   r0   r   ri   )r  r  r}  )r   bcoo_cusparse_loweringvaluer  r   rD   rK   r   r   r  rk   r  astyper   r  r=   r   ravel!_coo_correct_out_of_bound_indicesr   T)r  r  r  r  r}  r  r  r  r  r   r6   ru   r   coo_matmul_pr  rowcolr_  rD   outs                       r7   _bcoo_dot_general_gpu_implr    s    
	&	&	,	,!(K+5 
 2C.<A"0Z--#/'8Wa"xx1}*,,k3'1	( ^^//!(K+5 
 __X^^,(

8>>"# 
,1Y1!4V9KHg
&)
3(;
Kyy**1-{/@/@A;CTCTCVCI"!!"E7S%SOCe


Hc3%1!_%9CEEs&/u  >C q6MLQ3y>Q#6388v;Mh(I5*8[*M1a4 +ad"3CaA%IE7S%SOCe


Hc3%1!_%9CEEs&/u  >C s8O!(K+
57 7r]   c               $    t        | |||||      S r  r  )lhs_data_dotr  r  r  r  r}  r  s          r7   _bcoo_dot_general_jvp_lhsr  L  s     	<cM^2HU_
a ar]   c               $    t        ||| |||      S r  r  )rhs_dotr  r  r  r  r}  r  s          r7   _bcoo_dot_general_jvp_rhsr  Q  s     	8['M^2HU_
a ar]   c                   t        j                  |      rJ |\  \  }}\  }	}
t        |j                        }t        j                  |      r|j                  j
                  n|j
                  }t        t        |      ||	      }t        t        |      ||
      }t        t        t        |	||            \  }}}t        j                  |      r||f||
ff}t        t        j                  |t        j                  |                  }t        |	      |z   |z   }t        t        t        t        j                  |                  }t        j                   |j
                  dz
  dz  |j                  d   fz         }t#        |j                  d d       |j                  d   dz  z    	 t%        |||        d}|rZt)        |||t+                     \  }}t#         fd|D              }t-        | |||	      }t)        |||t+        |            \  }}n:t/        j0                  | ||	      }t/        j2                  ||      }t5        ||      }|||fS ||f|	|ff}t        t        j                  |t        j                  |                  }t        t        j                  t        |
      |z   |z               }t7        ||| |||
      }||t/        j2                  ||      fS # t&        $ r d}Y :w xY w)Nr0   r   rh   ri   TFrH  c              3  (   K   | ]	  }|     y wrB   r3   )r4   rd   placeholder_shapes     r7   r8   z._bcoo_dot_general_transpose.<locals>.<genexpr>v  s     Ga.q1Gr  r  r  r}  r  )r    r   r   rD   r   rK   r%   rp   rj  r  r$   r   takerk  r   r=   emptyr   rY  rN   rN  r   bcoo_dot_general_sampledr	   r  r_  r   r  )!r   r  r  r  r  r}  r  r  r  r  r  lhs_ndimrhs_ndimlhs_keptrhs_kept	ans_batchans_lhsans_rhsdimslhs_contract_sorted_by_rhsrI  out_axesplaceholder_dataindices_can_be_untransposedr   lhs_indices_Tresult_T_shaperesult_Tr6  out_dense_T	out_denserhs_contract_sorted_by_lhsr  s!                                   @r7   _bcoo_dot_general_transposer  V  s   ##K00	09J6< 6I!!"( 44S9SXX]]sxx(uXi@(uXi@( #D+i8*T U)WgH%")8!4y)6L MD!%bgglBJJ|<T&U!Vy/H,/IIKCRZZ456H yy+"2"2Q"6$!>+BSBSTVBWAY!YZk//458I8I"8MPT8TT),k;HYZ %)!
 #()9;T_0:;L0MOaG;GGn)"c=TXYh!(Mx)3N)CEifa OOBtDk--X6i[)4f;##w)Y!78D!%bgglBJJ|<T&U!VBJJtI1KKhVWXHxbZ6L157F [#--"AAA;  *$)!*s   *K& &K54K5c                   | \  }}}|\  }}}t        | d d |d d ||d n|j                  |         \  }}	}
t        t        |j                        |j                  fd|f|      \  }}t        ||	||
||      }||fS )Nr0   rT   r   r  )r\   rD   r&   r   rK   r  )rQ   rR   r  r}  r  r   r  rhs_bdimnew_lhs_datanew_lhs_indicesnew_lhs_spinfonew_dimension_numbersresult_batch_dimbatched_outs                 r7   _bcoo_dot_general_batch_ruler    s    )!Q.!Q2K!j!nj'tSYYx-@3B/, -H
:chh'!X8I-K))!,Q_9O4IK+ 
&	&&r]   cuda)platformrocmr  c                   |\  \  }}\  }}t        j                  |      t        j                  |      f}t        j                  |      t        j                  |      f}	t        j                  | ||||	f      S )ah  A contraction operation with output computed at given sparse indices.

  Args:
    lhs: An ndarray.
    rhs: An ndarray.
    indices: BCOO indices.
    dimension_numbers: a tuple of tuples of the form
      `((lhs_contracting_dims, rhs_contracting_dims),
      (lhs_batch_dims, rhs_batch_dims))`.

  Returns:
    BCOO data, an ndarray containing the result.
  r  )r   r  bcoo_dot_general_sampled_pr   )
ABrV   r  r  r  r  r  r  r  s
             r7   r  r    s     :K6< 6I''5''57%''	2''	24%	#	(	(Aw<A5> 
) 
K Kr]   c          	     H    t        |t        j                  | |||            S )Nr  r|  )r   r	   r  )r	  r
  rV   r  r|  s        r7   _bcoo_dot_general_sampled_slowr    s     	w1HYen o	ppr]   c                  ~|\  \  }}\  }}|s|s|s|rJ | j                   |j                   cxk(  rdk(  sJ  J |j                   dz
  }	|j                  d   }
|j                  d   }|	|
z   dk(  sJ |	dk(  rO| j                  |d d df      j                  dd      |j                  |d d df      j                  dd      z  S |	dk(  r-| d d d f   |j                  |d      j                  dd      z  S |	dk(  rA| d d d d f   |d d d d f   z  }t	        j
                  |t        |       t        |      |fd	      S t        d
      )Nrf   r0   ri   rh   r   r)  r*  .r   r   rf   r0   too many batch dimensions.)rK   rD   rr   r1  r	   r2  r   r   r	  r
  rV   r  r|  r  r  r  r  r6   ru   ro   r  s                r7    _bcoo_dot_general_sampled_simpler    sq    9J6< 6Ili9E	E	
166	Q			LL1']]2(b#	8	q	  	 \DDA##A#>dd71a4=!%%6a%@A B!|QW:WV_-11v!1LLL!|
AtTM
QtQ}-
-Cc!fc!fc%:IFF
1
22r]   c                  |\  \  }}\  }}|s|rJ t        |      t        |      cxk(  rdk(  sJ  J | j                  |j                  cxk(  rdk(  sJ  J |j                  dz
  }	|j                  d   }
|j                  d   }|	|
z   dk(  sJ |	dk(  r|d   dk(  rdgndg}|d   dk(  rdgndg}| j                  t	        t        d       t        d       f|d   |d d df            j                  dd      } |j                  t	        t        d       t        d       f|d   |d d df            j                  dd      }t        j                  | |||f||ff|      S |	dk(  r|d   dk(  rdgndg}|d   dk(  rdgndg}|j                  t	        t        d       t        d       f|d   |d	            j                  dd      }|d   dk(  rdg}t        j                  | |||f||ff|      S |	dk(  rYt        j                  | |||f||ff|      }t        j                  t        j                  |d
      g |j                  |d      S t        d      )Nrf   r0   ri   rh   r   r)  r*  r  r  )r0   r  r  )r   rK   rD   rr   r3  rb   r1  r	   r  r2  r<   r   r  s                r7   !_bcoo_dot_general_sampled_simple2r    s    :K6< 6I9%	%	\	c,/	41	44	44	4	
166	Q			LL1']]2(b#	8	q	  	 \#A!+!I#A!+!I	^U4[%+6	!gaQRdmTUYY_ersYtA	^U4[%+6	!gaQRdmTUYY_ersYtA??1a\<4PS\^gRh3i%.0 0\#A!+!I#A!+!I	^U4[%+6	!gfoVW[[agtu[vAA!Sl??1a\<4PS\^gRh3i%.0 0\
//!QL,3OR[]fQg2h$-/CT :<Mcii<M<MyYY
1
22r]   c               >   t        j                  |       } t        j                  |      }t        j                  |      }|\  \  }}\  }}|j                  dz
  }|j                  d   }	t        j
                  j                  }
|s@|s>|s<|s:| j                  |j                  cxk(  rdk(  rn n|	|z   dk(  rt        | ||||
      S t        |      dk(  r<|s:| j                  |j                  cxk(  rdk(  rn n|	|z   dk(  rt        | ||||
      S t        | ||||
      S )Nr0   ri   rf   r  )r=   r   rK   rD   r	   	PrecisionHIGHESTr  r   r  r  )r	  r
  rV   r  r  r  r  r  r6   ru   r|  s              r7   _bcoo_dot_general_sampled_implr    s   	kk!n!	kk!n!KK '9J6< 6ILL1']]2(mm##) <9	
&&AFF
a
Hw$6!$;+Aq'M^jstt!I!&&AFF2Ga2GHW^L^bcLc,Q7N_ktuu 
(1gIZfo	ppr]   c                  t        j                  dt        j                  | |fi       }t	        j
                  fd| ||      \  }t        j                  dt        ||fi       }t	        j
                  d |||      \  }|S )N&bcoo_dot_general_sampled_abstract_evalc                 0    t        j                  | digS )Nr  )r	   r  )argsr  s    r7   r   z9_bcoo_dot_general_sampled_abstract_eval.<locals>.<lambda>  s    coot6q_p6q5r r]   )
debug_infoc                     t        |  gS rB   )r   )r  s    r7   r   z9_bcoo_dot_general_sampled_abstract_eval.<locals>.<lambda>  s    }d7K6L r]   )r   r  r	   r  peabstract_eval_funr   )r	  r
  rV   r  dbgdense_resultsparse_results      `   r7   '_bcoo_dot_general_sampled_abstract_evalr%    s    DOOaVR	9#&&'rtuwx257-,D)G\+BB	H#''(LgWc368.-	r]   c                  t        |d      r|j                  j                  n|j                  }t        |d      r|j                  j                  n|j                  }t        |||      }t	        j
                  t        j                  || j                              }t        | ||d      \  }} |d d d d}	 t	        j                  t        j                        | ||fi |	\  }}|||fS )Nr   Tr  )r  r|  r}  r~  )hasattrr   rD   r   r    UndefinedPrimalr   r   rk   r@  get_primitive_transposer	   dot_general_p)
r   r	  r
  rV   r  A_shapeB_shape	mat_shaperw   kwdss
             r7   #_bcoo_dot_general_sampled_transposer/    s    #Av.AFFLLAGG'#Av.AFFLLAGG'*7G=NO)
4++Irxx@A#'GSM+'20$(
 $ 
7	#	#C$5$5	6r1a	H4	H$!Q	
Awr]   c                    t        | |||      S Nr  r  )A_dotr	  r
  rV   r  s        r7   _bcoo_dot_general_sampled_jvp_Ar4  )  s    	!%GGX	YYr]   c                    t        || ||      S r1  r2  )B_dotr	  r
  rV   r  s        r7   _bcoo_dot_general_sampled_jvp_Br7  ,  s    	!!UGGX	YYr]   c               4    fd} t        ||d      |  dfS )Nc                "    t        | ||      S r1  )r  )r	  r
  rV   r  s      r7   implz2_bcoo_dot_general_sampled_batch_rule.<locals>.impl0  s    )!QK\]]r]   r   )in_axesr  r   )rQ   rR   r  r:  s     ` r7   $_bcoo_dot_general_sampled_batch_ruler<  /  s$    ^	3dJ	3\	BA	EEr]   bcoo_spdot_generalc          
         |\  \  }}	\  }
}t        j                  |      t        j                  |	      f}t        j                  |
      t        j                  |      f}t        j                  | |||||||f|      S )Nr  )r   r  bcoo_spdot_general_pr   )r  r  r  r  r  r  r  r}  r  r  r  r  r  r  s                 r7   r  r  D  s     :K6< 6I''5''57%''	2''	24%		"	"8[(K.8Z6;U^:P 
# 
R Rr]   c          	        |j                   }	|j                   }
t        | ||	      }t        |||
      }|j                  |j                  cxk(  rdk(  sJ  J |j                  |j                  cxk(  rdk(  sJ  J D cg c]  }|	|   	 c}D cg c]  }|
|   	 c}k(  sJ t	        d      |j
                  k  sJ t	        d      |j
                  k  sJ g fdt        |	      D        fdt        |
      D        }|d d t        j                  t              f   }|d d t        j                  t              f   }|d d t        j                  t        t        |j
                              t              f   }|d d t        j                  t        t        |j
                              t              f   }|d d d f   |d d d f   k(  j                  d      }t        j                  t        j                  D cg c]  }|	|   	 c}|j                        t        |j                  dz
              }t        j                  t        j                  D cg c]  }|
|   	 c}|j                        t        |j                  dz
              }||k  j                  d      }||k  j                  d      }t        j                   ||d d d f   z  |d d d f   z  | d d d f   |d d d f   z  d      j#                         }t        j$                  |j&                  |j&                  |j                   d   |j                   d   z   gt        j(                  ||            }|j*                  d d d d d |j                   d   f   j-                  |d d d f         }|j*                  d d d d |j                   d   d f   j-                  |d d d f         }|j/                  t1        |      |j                   d         }t3        ||t5        |      |	      S c c}w c c}w c c}w c c}w )
Nr   ri   defaultc              3  2   K   | ]  \  }}|vs|  y wrB   r3   )r4   rd   r}   r  s      r7   r8   z0_bcoo_spdot_general_unbatched.<locals>.<genexpr>_       EDAqA_,DaE   c              3  2   K   | ]  \  }}|vs|  y wrB   r3   )r4   rd   r}   r  s      r7   r8   z0_bcoo_spdot_general_unbatched.<locals>.<genexpr>`  rD  rE  rj   rf   rC   rS   ro   )rD   r   r6   r   rO   ru   r   r=   rt   r   r%   rp   rM   r<   rk   rK   r   r  r  ro   result_typerr   rs   reshaper   _bcoo_sum_duplicatesr   )r  r  r  r  r  r  r  r  out_nse	lhs_shape	rhs_shaper  r  r]  r7  lhs_irhs_ilhs_jrhs_joverlaplhs_fill_valuerhs_fill_value	lhs_valid	rhs_validout_dataout_indicess         ``                  r7   _bcoo_spdot_general_unbatchedrY  Q  s   ))xi8#xi8#		(q	((	((	(		(q	((	((	( /	01)A,	0?4[aYq\4[	[[	[	_b	)CLL	88	8	_b	)CLL	88	8GEIi(EGEIi(EG) a?#>>
?%
a?#>>
?%
a9U3<<-@/#RZ]^^
_%
a9U3<<-@/#RZ]^^
_%
 1d7^uT1W~-2226'??II_5y|5U[[I	%**q.. ??II_5y|5U[[I	%**q.. ~%**2.)~%**2.)YYw1d7!33ia6HH4(8D!G+<<aAAF  		377CGGU[[_u{{2-NO #[ IK+q!%5ekk"o%556::5D>J+q!U[[_%556::5q>J+##CM;3D3DR3HI+ 
hJY<W]d	eeG 
14[$ 6 6s   P-P2P7+P<c               h   |j                   }|j                   }	t        | ||      }
t        |||	      }|
j                  |j                  cxk(  rdk(  sJ  J t        | j                  |j                  |j                  |j                  ||||      \  }}|j                   d   }|\  \  }}\  }}g |t        t        |
j                        |      }g |t        t        |j                        |      }| j                  g |t        |
j                  | j                              } |j                  g |t        |j                  |j                              }|j                  g |t        |
j                  |j                              }|j                  g |t        |j                  |j                              }t        j                  t        t        ||
j                  d        t        |	|j                  d        |D cg c]  }||
j                  z
   c}|D cg c]  }||j                  z
   c}|      }t        ||j                  t        |      z
  d      }t        ||
j                  t        |      z
  d      }t        |t        |            } || |||      S c c}w c c}w )Nr   r  ri   )r  r  r  r  rK  )NNr   r   r;  )r   r   NN)rD   r   r   !_bcoo_spdot_general_abstract_evalr   r%   rp   r6   r_  rK   	functoolsr   rY  r   r   r   )r  r  r  r  r  r  r  r}  rL  rM  r  r  	data_avalr   rK  r  r  r  r  lhs_batch_permrhs_batch_permr]  funcs                          r7   _bcoo_spdot_general_implrb    sl    ))xi8#xi8#		(q	((	((	(2MM;##X]]K4D4DjDU13,)Q OOB'?P<$?O&<y) KYJ5+=y!IJ.JYJ5+=y!IJ. U. U5hmm3T UV( U. U5hmm3T UV(%%&^&^s{{KL\L\9]&^_+%%&^&^s{{KL\L\9]&^_+ 
		8Ickkl34Ickkl340?@1q3;;@0?@1q3;;@
$ 
D#++I6@R	S$	D#++I6@R	S$	D#i.	)$	hX{	;; A@s   J*#J/c          	         |j                   |j                   }t        j                  t        j                  d      j                  t        j                   j                        t        j                  |j                        ||      }	t               }
t        |      }|\  \  }}\  |
j                  s|j                  rt        d      rt              |
j                  k\  sr#t              |j                  k\  rt        d      |rHt        |      |
j                  k  s%t        |      |
j                  |
j                  z   k\  rt        d      |rHt        |      |j                  k  s%t        |      |j                  |j                  z   k\  rt        d      |j                  t              kD  r#|
j                  t        |      kD  rt!        d      |
j                  t        |      kD  r|
j"                  nd|j                  t        |      kD  r|j"                  ndz  }|
j                  |j                  z   t              z
  }t        |t%        j&                  |	j                   |d              }t)        j*                  t-         fdt/        |
j                        D              t-        fd	t/        |
j                        D                    }t)        j*                  t-        fd
t/        |j                        D              t-        fdt/        |j                        D                    }g fdD        |||}g fdD        ||||
j                  |j                  z   dt        |      z  z
  }t1        j2                  ||	j                        }t1        j2                  |j                        }t        |||	j                          ||fS )Nr  r  z)bcoo_spdot_general with dense dimensions.z`bcoo_spdot_general: batch_dims must correspond to batch dimensions of the sparse representation.z?bcoo_spdot_general only supports contraction of sparse indices.zXbcoo_spdot_general: cannot have unused batch dims on rhs with unused sparse dims on lhs.rf   c              3  F   K   | ]  }|vsj                   |     y wrB   rC   )r4   rF   r  r  s     r7   r8   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  !     	T#s)?S(..
	T   	!!c              3  F   K   | ]  }|vsj                   |     y wrB   rC   )r4   rF   r  r  s     r7   r8   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  #     	WS#YBV+

C
 	Wrf  c              3  F   K   | ]  }|vsj                   |     y wrB   rC   )r4   rF   r  r  s     r7   r8   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  re  rf  c              3  F   K   | ]  }|vsj                   |     y wrB   rC   )r4   rF   r  r  s     r7   r8   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rh  rf  c              3  (   K   | ]	  }|     y wrB   r3   r4   rF   rL  s     r7   r8   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>       *in*r  c              3  (   K   | ]	  }|     y wrB   r3   rl  s     r7   r8   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rm  r  r0   )rD   r  r  r	   r  r  r  rk   r   r   rN   rO   r6   minru   r   r   ro   mathr  r   broadcast_shapesr   rp   r   r   )r  r  r  r  r  r  r  r}  rM  r  r  r  r  r  rK  out_n_batchlhs_batch_shaperhs_batch_shaper  rc  r^  indices_avalr  rL  r  s   ````                  @@@r7   r\  r\    sm    ))WWS__6efqq	9hnn5	9hnn5)3	 r 5( 	xi8#xi8#?P<$?O&<y)[[CKK
I
JJC	Nckk1yS^WZWbWbEb
  A  B  B#o.<O@TX[XcXcfifrfrXr@r
_
``#o.<O@TX[XcXcfifrfrXr@r
_
``[[3y>!cllS5I&I
o
pp s?33SWWs?33SWW< 
 ckk)C	N:+8>>+,#?@A'''		Ts{{);	TT		WE#++,>	WW/ ''		Ts{{);	TT		WE#++,>	WW/
*	*  	*
E*	*EE E 	E \\CLL(1s?/C+CC	E- z8>>:)!!-1B1BC,L(..9	L	  r]   c          
     H   t        |j                        }t        |j                        }t        d t        | |      D              }t	        | d d |d d ||      \  }	}
}t	        | dd  |dd  ||      \  }}}t        ||fd|      \  }}t        |	|
||||||      }|||ffS )Nc              3  F   K   | ]  \  }}|	|j                   |     y wrB   rC   rE   s      r7   r8   z1_bcoo_spdot_general_batch_rule.<locals>.<genexpr>  s"     chc3SVSb399S>crG   r0   r  r   r   )r  r  r  r}  )r   rD   rO   rP   r\   r&   r  )rQ   rR   r  r  r}  r  r  r  rT   r  r  r  r  r  r  s                  r7   _bcoo_spdot_general_batch_rulery    s    !!"(!!"(c#lJ2Occ*&?!j!njZ'I#(K&?jnjZ'I#(K(CF$5)7%%#Hk8[6G/9j;QS+ 
')9:	::r]   c                   | \  }}}}|\  }}}	}
t        | i |}t        |      t        j                  u sJ t        |
      t        j                  u sJ d}t        |      t        j                  ur|t        ||||fi |d   z  }t        |	      t        j                  ur|t        |||	|fi |d   z  }||t        j                  j	                  |d         gfS )Nr   rf   )r  r  r    r  r  )r  r  r.  r  r  r  r  r  lhs_indices_dotrhs_data_dotrhs_indices_dotr  rf  s                r7   _bcoo_spdot_general_jvpr~    s    18.(K;AI>,#W55+	o	"''	))	)	o	"''	))	),	,rww&'k8[a\`abcddL	,rww&'+|[a\`abcddL	|RWW%>%>{1~%NO	OOr]   bcoo_sort_indicesc                    t        j                  | j                  d| j                  i\  }}t	        ||f| j
                  d| j                        S )zxSort indices of a BCOO array.

  Args:
    mat : BCOO array

  Returns:
    mat_out : BCOO array with sorted indices.
  rS   Trl   )bcoo_sort_indices_pr   _bufsr   r^   rD   rJ   )rw   rU   rV   s      r7   r  r    sI     &**CIIHciiH-$	tWoSYYt //
1 1r]   c                   t        | ||j                        }|j                  dk(  r| |fS t        t        |j
                  d      } ||      \  }}t        d |j
                        } || |      } | |fS )Nr   Fr   c                    | |   S rB   r3   r]  rL  s     r7   r   z)_bcoo_sort_indices_impl.<locals>.<lambda>*  
    AaD r]   )r   rD   ru   r   _bcoo_sort_indices_unbatchedr6   )rU   rV   rS   r   r   permpermutes          r7   _bcoo_sort_indices_implr  #  sq    
w
5%
^^q=-u}}%P!G*-'4(%--8'	t	$	wr]   c                      j                   \  }} fdt        |      D        }t        j                  g |t        j                   j
                  |      |      ^  }t        j                         |fS )Nc              3  0   K   | ]  }d d |f     y wrB   r3   r4   rd   rV   s     r7   r8   z/_bcoo_sort_indices_unbatched.<locals>.<genexpr>1  s     .gadm.   )num_keys)rD   rp   r	   sortiotark   r=   column_stack)rV   ro   r   idx_colsr  s   `    r7   r  r  .  sc    ==&#q.U1X.(88EhE(DEPQR.7D			'	"D	((r]   c          
        t        | ||j                        }|j                  dk(  r| |fS t        j                  g t        t        |j                  d |j                   | j                  d |j                         |j                  | j                  |j                  dz   d  | j                  | j                        }||fS )Nr   rf   	weak_type)r   rD   ru   r   r   rj  rO   r6   ro   rk   r  )rU   rV   rS   r   data_outs        r7    _bcoo_sort_indices_abstract_evalr  5  s    
w
5%
^^q=1c#w}}^emm,djj%--.HI 1
YY1EMMA-./126**X( 
7	r]   c                   t        | ||      \  }}}t        j                  |||      \  }}d}|d   |d   }d}||f|fS )Nr   rx  rf   r   r  )r\   r  r   )rQ   rR   rS   rU   rV   r  indices_outr  s           r7    _bcoo_sort_indices_batching_ruler  ?  sb    3L*fU$-2242P(K( ]a.KH
K	 (	**r]   c                  t        g | |j                   }|j                  dk(  r| |fS | \  }}|\  }}t        t        |j
                        } ||      \  }	}
t        d |j
                        } |||
      }t        j                  j                  |      }t        |      t        j                  u rt        j                  j                  |      n |||
      }||	f||ffS )Nr   c                    | |   S rB   r3   r  s     r7   r   z(_bcoo_sort_indices_jvp.<locals>.<lambda>S  r  r]   )
r   rD   ru   r   r  r6   r    r  r  r  )r  r  rS   r   rU   rV   r   r   r   r  r  r  r  indices_dot_outrf  s                  r7   _bcoo_sort_indices_jvpr  J  s    

0'
06<<
0%
^^qH-$+(A-u}}=!j+t(%--8'T4 (GG--g6/8<X"''8Q**84W^_gimWn,
K	 <"A	AAr]   r   c                    t        | j                  | j                  | j                  |      \  }}t	        ||f| j
                  dd      S )a  Sums duplicate indices within a BCOO array, returning an array with sorted indices.

  Args:
    mat : BCOO array
    nse : integer (optional). The number of specified elements in the output matrix. This must
      be specified for bcoo_sum_duplicates to be compatible with JIT and other JAX transformations.
      If not specified, the optimal nse will be computed based on the contents of the data and
      index arrays. If specified nse is larger than necessary, data and index arrays will be padded
      with standard fill values. If smaller than necessary, data elements will be dropped from the
      output matrix.

  Returns:
    mat_out : BCOO array with sorted indices and no duplicate indices.
  rG  Trl   )rJ  rU   rV   r   r^   rD   rv   s       r7   r   r   i  sC     'sxxSYYTWX-$	tWoSYYt!
# #r]   c                   |%t        j                  t        j                  |d      }t        j                  | |||      S )N$nse argument of bcoo_sum_duplicates.rG  )r   r   rm   rn   bcoo_sum_duplicates_pr   )rU   rV   rS   ro   s       r7   rJ  rJ  |  s:    _

 
 6\
]C		#	#D'&c	#	JJr]   c          
        t        | ||j                        }t        ||j                  dd      \  }}}|!|j                  dk(  rdn|j	                         }t        |||j                        }|j                  dk(  r(| j                  |j                  d| j                        } t        j                  g t        t        |j                  d |j                   | j                  d |j                         || j                  |j                  dz   d  | j                        }d }	t        |	|j                        }	 |	|||       }||fS )	NTrD   return_inversereturn_true_sizer   rf   ro   rD   r   rj   c                B    | j                   |   j                  |d      S Ndrop)r+  rr   r   )d_outr   r]  s      r7   r   z+_bcoo_sum_duplicates_impl.<locals>.<lambda>  s     ? r]   )r   rD   r0  ru   rO   _adjust_indices_nser   r6   rk   r=   r  rj  r   )
rU   rV   rS   ro   r   r  mappingnse_batchedr  r  s
             r7   _bcoo_sum_duplicates_implr    s'   
w
5%&56<<t'M#+w[~~"!(9C#KSM+
^^q88EMMD

8CDYY >S'--"?NU]]A[\ >>"jj):);<>EIZZQ(?'w.'Xw-(	;	r]   c                  t        | |      }||j                  k  r| dd |d d f   } | S t        j                  t	        j
                  ||j                  |j                  |j                  z    | j                        g | j                  d d ||j                  z
  | j                  d   | j                  dz
  f      }t        j                  | |g| j                  dz
        } | S )	N.rj   rh   ri   rf   )operandrD   broadcast_dimensionsr0   	dimension)r   ro   r	   r2  r=   rt   r6   ru   rk   rD   rK   concatenate)rV   ro   rD   r   r)  s        r7   r  r    s    
 %
0%EIIc4C4l#G 
. iiemmEMMENN,JKSZS`S`aEgmmCR E#		/E7==3DE#LL1,.D
 oowo9IJG	.r]   )r  r%  r  c                   t        | |      }t        t        ||j                  d  |||      }t	        ||j                  d      } ||       S )N)rD   r  r%  r  Fr  )r   r   _unique_indices_unbatchedr6   r   )rV   rD   r  r%  r  r   r   s          r7   r0  r0    sN    
 %
0%'uU]]^/D+,/1! EMMu5!	
7r]   c          	        t        | |      }|j                  dk(  rxd}t        j                  | |df      }|f}|rg |t        j                  |d      }|rg |t        j                  |d      }|rg ||}t        |      dk(  r|d   S |S t        j                  t        j                  |d |j                   | j                        d      }	| |	k\  j                  dd	      }
t        j                  |
|	|       } t        | d||||j                  |	
      }|r)|rdnd}g |d | ||   j                         ||dz   d  }|r>|d   }|| |	k(  j                         j                  |j                        z
  }g |d d |}|S )Nr   rf   rC   int32rj   )r   ri   Tr~   )axisr  r%  r  r   r   r0   )r   ru   r=   rq   r   r   r<   rt   rk   r   r   r*   ro   r  r  )rV   rD   r  r%  r  r   ro   r  r  r   out_of_boundsr  s               r7   r  r    s   
 %
0%
^^q
C..a9K.C1c1399S01c1c1399S01ccK3KcX]3q6++syy)?w}}UW[\*j(--b4-@-IImZ9'a\!1		j	Z#!C
8CI
8s3x~~'
8#cAgh-
8C
b'C
J&++-44SYY?
?C
CH
c
C	*r]   c                   | j                   |j                   k(  s J | j                    d|j                           t        |      | j                   dz   k(  s"J t        |       d| j                   dz           | j                   dkD  r<t        t        || j                   d  |      } t	        || j                         | |      S | |d   k\  ||d   k\  z  }|r?t        j                  |d|       } t        j                  ||d   |      }|d   |d   dz   f}n>t        j                  ||d   |       } t        j                  |d|      }|d   dz   |d   f}| ||fS )Nz != rf   )rD   r_  r   )rK   r   r   r  r   r=   r   )r  r  rD   r_  r   r   s         r7   r  r    sO    
SXX	:#((4z::		Usxx!|	#FE
|41~%FF	#XX\1CHHI&)	=A":a"3,,
q/cU1Xo	.$
))D!S
!C
))D%(C
(C1XuQx!|$E
))D%(C
(C
))D!S
!C1X\58$E	c5r]   c          
        |t        d      t        | ||j                        }t        j                  g |j                  d |j
                   ||j                  |j                  |j                        }t        j                  g t        t        |j                  d |j
                   | j                  d |j
                         || j                  |j
                  dz   d  | j                  | j                        }||fS )Nbcoo_sum_duplicates: nse must be specified when using the function within jit, vmap, and other transformations requiring abstract evaluation.)rk   r  rf   r  )r   r   rD   r   r   r6   ru   rk   r  rj  rO   )rU   rV   rS   ro   r   r  r  s          r7   "_bcoo_sum_duplicates_abstract_evalr    s    [
 [ \ \
w
5%  !V7==%--#@!V#!Vu~~!V(/ARART++c#w}}^emm,djj%--.HI ++::emma'()+,0JJ$..R( 
;	r]   c                   t        | ||      \  }}}t        j                  ||||      \  }}|d   t        j                  |dg      }d}	nd}	||f|	fS )NrG  rf   r   r  rx  )r\   r  r   r	   rv  )
rQ   rR   rS   ro   rU   rV   
new_spinfor  r  r  s
             r7   "_bcoo_sum_duplicates_batching_ruler    so    7jRXY$/44T7:[^4_(K ]++kA3/KHH
K	 (	**r]   c          
     h   t        g | |j                   }| \  }}|\  }}t        ||j                  dd      \  }	}
}|t        j                  |      }	 t        j                  t        j                  |d      }t        |	||j                        }	|j                  dk(  rP|j	                  |j                  d|j                        }|j	                  |j                  d|j                        }t        j                  g t!        t"        |j                  d |j                   |j                  d |j                         ||j                  |j                  dz   d  |j                  	      }|}|j$                  rd
 }nd }t'        ||j                        } |||
|      }t(        j*                  j-                  |	      }t/        |      t(        j*                  u rt(        j*                  j-                  |      n	 |||
|      }||	f||ffS # t
        j                  $ r t        d      w xY w)NTr  r  r  r  r   r   rf   rj   c                B    | j                   |   j                  |d      S r  r  xrd   ys      r7   r   z*_bcoo_sum_duplicates_jvp.<locals>.<lambda>  s    add1gkk!&k9 r]   c                    | S rB   r3   r  s      r7   r   z*_bcoo_sum_duplicates_jvp.<locals>.<lambda>  s    a r]   )r   rD   r0  r=   r   r   r   rm   rn   ConcretizationTypeErrorr   r  ru   r6   rk   r  rj  rO   r   r   r    r  r  r  )r  r  rS   ro   r   rU   rV   r   r   r  r  r  r  rf  r  r  s                   r7   _bcoo_sum_duplicates_jvpr    s    

0'
06<<
0%-$+(A&56<<t'M#+w[
''+
C\

 
 6\
]C $KSM+
^^q88EMMD

8CD||EMMD|OHYY >S'--"?NU]]A[\ >>"jj):);<>EIZZQ(, ]]9GGw.'Xw-(GG--k:/8<X"''8Q**84W^_kmtv~W,
K	 <"A	AA) 
	%	% \
 [ \ \\s   %H H1errorr6   r   on_inefficientc          	       
 | j                   nt        j                        | j                  nt        j                        f| j                   | j                  fk(  r| S | j                  z
  z
  }dvrt        d      dk  rt        d       dk  rt        d       |dk  rt        ddd	| j                   d
      fd}| j                  kD  r]t        d | j                   | j                  | j                  z
   D              r& |d| j                   d| j                   d d       | j                   kD  rOt        d | j                  | j                    D              r& |d| j                   d| j                    d d       | j                  | j                  }}| j                  | j                   
| j                  }|k  r|z
  t        t        
dz         fd       }	 |	||      \  }} |j                  g |j                  d
 t        j                  |j                  

dz          |j                  
dz   d  } |j                  g |j                  d
 t        j                  |j                  

dz          |j                  
dz   d  }}
k  r,
z
  t        t              fd       }	 |	||      \  }}
|kD  r1|z
  t        t        
dz         fd       }	 |	||      \  }}}
kD  r0
z
  t        t        
      
fd       }	 |	||      \  }}
t        ||f      S )a  Update the storage layout (i.e. n_batch & n_dense) of a BCOO matrix.

  In many cases this can be done without introducing undue storage overhead. However,
  increasing ``mat.n_batch`` or ``mat.n_dense`` will lead to very inefficient storage,
  with many explicitly-stored zeros, unless the new batch or dense dimensions have size
  0 or 1. In such cases, ``bcoo_update_layout`` will raise a :class:`SparseEfficiencyError`.
  This can be silenced by specifying the ``on_inefficient`` argument.

  Args:
    mat : BCOO array
    n_batch : optional(int) the number of batch dimensions in the output matrix. If None,
      then n_batch = mat.n_batch.
    n_dense : optional(int) the number of dense dimensions in the output matrix. If None,
      then n_dense = mat.n_dense.
    on_inefficient : optional(string), one of ``['error', 'warn', None]``. Specify the
      behavior in case of an inefficient reconfiguration. This is defined as a reconfiguration
      where the size of the resulting representation is much larger than the size of the
      input representation.

  Returns:
    mat_out : BCOO array
      A BCOO array representing the same sparse array as the input, with the specified
      layout. ``mat_out.todense()`` will match ``mat.todense()`` up to appropriate precision.
  N)r  r  NzJon_inefficent={on_inefficient!r}; expected one of ['error', 'warn', None].r   z"n_batch must be non-negative; got z"n_dense must be non-negative; got zsum of n_batch=z and n_dense=z  cannot be larger than mat.ndim=r:   c                |    dk(  r| dz  } t        |       dk(  r!| dz  } t        j                  | t               y y )Nr  z` To disable this error, set the on_inefficient argument of bcoo_update_layout to 'warn' or None.r  zX To disable this warning, set the on_inefficient argument of bcoo_update_layout to None.)category)r   r  r  r   )msgr  s    r7   _maybe_err_or_warnz.bcoo_update_layout.<locals>._maybe_err_or_warnQ  sP     	 9 :c!#&&	6	!	 / 0cmmC"9: 
"r]   c              3  &   K   | ]	  }|d kD    ywr{   r3   r  s     r7   r8   z%bcoo_update_layout.<locals>.<genexpr>]  s     	DA!a%	Dru  zFor matrix of shape z, increasing n_dense from z to z  results in inefficient storage.c              3  &   K   | ]	  }|d kD    ywr{   r3   r  s     r7   r8   z%bcoo_update_layout.<locals>.<genexpr>`  s     "QQ1q5"Qru  z, increasing n_batch from rf   r   c                    | j                   t        j                  | j                  d        g| j                  d   }t	        j
                  fd| j                  d  D        ddi}t	        j                  t	        j                  |j                  d   j                  f      gt        t        j                  |            }||fS )Nc              3  `   K   | ]%  }t        j                  |j                          ' ywrj   Nr=   r   rk   )r4   r}   rd   s     r7   r8   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>n  s#     Pqcjj!''::P   +.r   r   r   )rI  rp  r  rD   r=   r   r  broadcast_tor   rj  r  )r]  rd   new_dmeshesnew_ir  s    `   r7   _updatez#bcoo_update_layout.<locals>._updatek  s    aii		!''"1+.==e||PAGGBQKP +%)+f 0 0U[[^QVV4L M  9!$SYY!7 9 :eE\r]   r0   c                   j                   d   } | j                  t        j                  | j                   d dz          g| j                   dz   d   }t	        j
                  fdg j                   d  |D        ddi} j                  t        j                  j                   d dz          gj                   dz   d   }t	        j                  g d |d d D        |      }||fS )Nrh   rf   c              3  `   K   | ]%  }t        j                  |j                          ' ywr  r  )r4   r]  rd   s     r7   r8   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s#     Xqcjj!''::Xr  r   r   c              3  <   K   | ]  }|j                           y wrB   )r  )r4   r   s     r7   r8   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s     !A!'')!A   ri   )rD   rI  rp  r  r=   r   r  )r]  rd   ro   r  r  r  r  s    `    r7   r  z#bcoo_update_layout.<locals>._update~  s    GGBKcaii		!''&1q5/2EQWWQUV_Ee||XDWaggbqkDWSVDWX +%)+faii		!''&1q5/2EQWWQUV_EeI!AVCR[!AI5IJeE\r]   c                    t        j                  |  d        j                  t        | d           j	                  |       }|d   }||fS r-  )r=   rq   rr   r   rs   )r]  rd   r  r  r  r   rD   s       r7   r  z#bcoo_update_layout.<locals>._update  sU    nnQeWHI&67::5A23=IMMaPe!feE\r]   c                   j                   d   }t        fdt        	      D              t        j                  |      fz   }g 
 |j                   d   	z
  }t        j
                  d d 	d f   |      }g 
 || j                   | j                  z
  d  }t        j                  | |      j                  |   j                  |       }||fS )Nrh   c              3  0   K   | ]  }d d |f     y wrB   r3   )r4   jrd   s     r7   r8   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s     ,a!AqD',r  ri   rC   )
rD   r   rp   r=   r   r  rK   rq   rr   rs   )r]  rd   ro   r  new_i_shaper  new_d_shaper  current_n_batchr  r6   r   rD   s    `      r7   r  z#bcoo_update_layout.<locals>._update  s    GGBKc,58,,

3/AAcKeOG4KcK1772;?KkqABx5eWeOG4WcWAGGAFFWDTDU<VWknnQk255c:>>qAeE\r]   rC   )r6   rm   rn   r   rK   r   r   rD   rU   rV   r   r   rI  rp  r  r^   )rw   r6   r   r  ru   r  new_datanew_indicescurrent_n_denser  r  r  rD   s    ```      @@@r7   bcoo_update_layoutr    s   B #?CKKw0G'"?CKKw0G'wCKK55JXX')(22
a
bbq[
9'C
DDq[
9'C
DD\
'wjgZ 877:xxjC D D; 		DG8CHHs{{,BC	DD-cii[8R++d7)3SU Vs{{s"Q#))CKK2P"QQ-cii[8R++d7)3SU V ((CKKK(
))%KK/KK/'!AZ?Q./ 0 $Hk:Hkx G0@!A G $		(..[\I\*] ^G!)!0C0D!EGH &+%% P{'8'89I/'J P&*ii0A0A/SbefSf0g&hP'2'8'819L9M'NPK O'!AZ7# $ $Hk:HkO/!AZ?Q./ 0 $Hk:HkO/!AZ?+ , $Hk:HkO	x%U	33r]   )shardingc          	     t    t        t        | j                  | j                  | j                  ||      |      S )a  Expand the size and rank of a BCOO array by duplicating the data.

  A BCOO equivalence to jax.lax.broadcast_in_dim.

  Args:
    mat: A BCOO-format array.
    shape: The shape of the target array.
    broadcast_dimensions: The dimension in the shape of the target array which
      each dimension of the operand (``mat``) shape corresponds to.

  Returns:
    A BCOO-format array containing the target array.
  )rS   rD   r  rC   )r^   _bcoo_broadcast_in_dimrU   rV   r   )rw   rD   r  r  s       r7   bcoo_broadcast_in_dimr    s7     
$SXXs{{399+0:NP 
 r]   c                  t        |j                        t        |      k7  rt        d|j                  d|d      t        | ||j                        }t	        ||j
                  |j                  g      \  }}}t        |d      t        |t        |            kD  rt        d      t        |d      t        |t        |            kD  rt        d      t        ||j
                  d t        |            }	|j                  xr% t        |      t        ||j                   d       z
  }
t        |      |	z
  |
z
  t        j                  |j                  |j
                  |j
                  |j                  z          t        j                  ||	|	z          k7  rt        d	      | |}}|	|
f|j
                  |j                  fk7  rt        j                  |g |d|	 |j                  ||	z   d g ||	fd
|D              }t        j                  |g |d|	 |j                  |j                  g ||	|	dz         }|j                  k7  rhg |d|	 |j                  }t!        j"                  |t$              |	z
  }t!        j&                  ||      j(                  d|f   j+                  |      }||fS )z'BCOO equivalent of lax.broadcast_in_dimzspinfo.shape=z and broadcast_dimensions=z must have the same lengthr   rA  z>Cannot mix batch and sparse dimensions during broadcast_in_dimz>Cannot mix sparse and dense dimensions during broadcast_in_dimNz*Adding sparse dimensions with lengths != 1c              3  .   K   | ]  }|d z   z
    ywr{   r3   )r4   r5   new_n_sparses     r7   r8   z)_bcoo_broadcast_in_dim.<locals>.<genexpr>  s     9cST!a%,:N9cr^  )rD   r  rf   rC   .)r   rD   r   r   r   r6   ru   rO   ro  r   rp  r  rN   r	   r2  ro   r=   rt   r   rq   rr   rs   )rU   rV   rS   rD   r  r   rR   sparse_dims
dense_dimsnew_n_batchnew_n_denser  r  r8  r  s                 @r7   r  r    s    	#233
&A,@+BB\]
^^
w
5%(23G%--Y^YgYgIh(i%*k:Q#k3u:"FF
U
VVa 3z3u:#FF
U
VV (8#e*M+Y#e*s3G3X/Y"Y+Uk)K7,	YYv||EMM5==5>>+IJKtyyY^_jkv  zF  lF  ZG  PH  H
J
KKK( ;EMM5==#AA##HTl{#TUYYT{\7Q7R1STdzd;d9cXb9cdfH &&{?l{#?UYY??HzH;HaHJK
 U^^#;eL[!;599;l;E
))K
%
3C>>+U;>>sCxHLL[YK	;	r]   c          
     `	   t        j                  |      }t        d | D              st        d|        t	        j
                  t        j                  d      j                  | D cg c],  }t        j                  |j                  |j                        . c}|      }t        | D ch c]  }|j                   c}      dkD  rt        d      | D ch c]  }|j                   }}t        |      dk7  rt!        |      dk(  rt        d | D              r,| D cg c]   }|j                  dk(  rt#        |d	      n|" } }n=t        d
 | D              r+| D cg c]   }|j                  dk(  rt#        |d	      n|" } }| D ch c]  }|j                   }}t        |      dk7  rt        d      | d   j                  | d   j$                  }}| D cg c]  }|j&                  j                  d|  }}| D cg c]  }|j(                  j                  d|  }}||k  r6|D 	cg c]  }	|	d| |	|dz   d z    }}	|D 	cg c]  }	|	d| |	|dz   d z    }}	t        t+        |            t        t+        |            cxk(  rdk(  st-        d       t-        d      ||k  r| D ch c]  }|j.                   }
}t        |
      dk7  r$t!        |
      }| D cg c]  }t1        ||       } }t        j                  | D cg c]  }|j&                   c}|      }t        j                  | D cg c]  }|j(                   c}|      }n|||z   k  rt3        j4                  dg| dd D cg c]  }|j                  |    c}z   | d   j&                  j                        }t        j                  | D cg c]  }|j(                   c}|      }t        j                  t7        | |      D cg c]2  \  }}|j&                  j8                  d||z
  f   j;                  |      4 c}}|      }nt-        d      t=        ||f|j                        S c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}	w c c}	w c c}w c c}w c c}w c c}w c c}w c c}w c c}}w )aP  Sparse implementation of :func:`jax.lax.concatenate`

  Args:
    operands : Sequence of BCOO arrays to concatenate. The arrays must have equal
      shapes, except in the `dimension` axis. Additionally, the arrays must have
      have equivalent batch, sparse, and dense dimensions.
    dimension : Positive integer specifying the dimension along which to concatenate
      the arrays. The dimension must be among batch or sparse dimensions of the input;
      concatenation along dense dimensions is not supported.

  Returns:
    A BCOO array containing the concatenation of the inputs.
  c              3  <   K   | ]  }t        |t                y wrB   )r  r^   r4   ops     r7   r8   z#bcoo_concatenate.<locals>.<genexpr>  s     5bZD!5r  zIbcoo_concatenate: expected operands to be a sequence of BCOO arrays. Got r  r  rf   zAbcoo_concatenate requires inputs to have matching nse dimensions.c              3  `   K   | ]&  }|j                   d k(  s|j                  d    dk(   ( yw)r   rf   Nr6   rD   r  s     r7   r8   z#bcoo_concatenate.<locals>.<genexpr>  s'     
B"**/288A;!
B   ..r   r6   c              3  `   K   | ]&  }|j                   d k(  s|j                  d   d k(   ( ywrf   r   Nr  r  s     r7   r8   z#bcoo_concatenate.<locals>.<genexpr>   s'     D"BJJ!ORXXa[ADr  zCbcoo_concatenate requires inputs to have matching batch dimensions.Nz6concatenation of arrays with broadcasted batch indicesri   rj   .z%Concatenation along dense dimensions.rC   )rm   rn   rM   r   r  r  r	   r  r  r   r   rD   rk   r   r   r6   rO   r  ru   rV   rU   rs   rN   ro   rx   r   cumsumr   rr   r   r^   )operandsr  r   r  	n_batchesr6   ru   index_batchesdata_batchesr}   nsesro   r  r  offsetsoffsets                   r7   bcoo_concatenater    s;    nnY')	5H5	5
 $:' ( ( WWS__nEPP:B
CB4BHHbhh/
C Q ( 	x	("**	()A-
X
YY$,-brzz-)- 	^qS^q0

Bh
BBYabSUrzzQ$R3BNbhb	DD	DYabSUrzzQ$R3BNbhb&.//I/^q
Z
[[qk))8A;+?+?8'8@A"2::##HW-A-A4<=b"''--)=,=@MN1Qz	]Qy1}~%66NMN?KL!AjyMAi!mn$55LLL
c- 
!S\):%;
@q
@
V
WW A
V
WW%&rBFF&D&
4yA~Ic3;<R-C(<h<//"A"2::"AYWK(;B;yQH7X%%iiXcr]Krrxx	2KK&qk11779G(;B;wOH//5=h5P#R'1r6 $&::==i'6I1I#J#N#Nv#V #R,35K E
FF	x%X^^	<<a D	( .
 cb/ B=NL ' ="A;K;#Rsf   (1Q+QQ$%Q)%Q.Q3 Q8= Q=(RR)RRR2R/R 
:R%47R*
)
dimensionsr  c          	     x     j                   j                  d j                    j                  j                  d j                   cxk7  r& j                  d j                   k7  rt	        d       t         j                   j                   j                  g      \  }}}t         j                   j                   |xs t        t         j                               j                        \  }}}	t        j                  |      }
t        j                  |      }t        j                  |      }|
dk7  r,|
|vr(t        d j                   d| d j                         |dk7  r/|
|z  |vr(t        d j                   d| d j                          |j#                  |
d	
      }|j#                  |
|z  d	
      }t        |t%        |      t%        |      g      \  }}}t'        j(                   j                  g | j*                  |g | j                   fd|	D              }t'        j(                   j                   g | j*                   j                  g | j                   j                  dz         |s*t-        j.                  g | j*                  d      n_|r\t        fd|D              }|D cg c]'  }t%         j                   j                  |z            ) }}t-        j0                  |t        |      d      }t3        j4                  d      5  t-        j6                   j                   j                   j                   j                  z    j8                        k\  j;                  dd      }ddd       t-        j<                  ||      }t-        j>                  |D cg c]  }|d   	 c}d      t-        j@                  t-        j6                  |j8                              tC        |f|      S c c}w # 1 sw Y   xY wc c}w )a  Sparse implementation of :func:`jax.lax.reshape`.

  Args:
    operand: BCOO array to be reshaped.
    new_sizes: sequence of integers specifying the resulting shape. The size
      of the final array must match the size of the input. This must be specified
      such that batch, sparse, and dense dimensions do not mix.
    dimensions: optional sequence of integers specifying the permutation order of
      the input shape. If specified, the length must match ``operand.shape``.
      Additionally, dimensions must only permute among like dimensions of mat:
      batch, sparse, and dense dimensions cannot be permuted.

  Returns:
    out: reshaped array.
  Nz4reshape of arrays with broadcasted batch dimensions.rf   zJbcoo_reshape: new shape cannot mix batch and sparse dimensions; got shape=z new_shape=z with n_batch=zJbcoo_reshape: new shape cannot mix sparse and dense dimensions; got shape=z with n_dense=right)sidec              3  B   K   | ]  }|j                   z   d z     ywr{   r  rK  s     r7   r8   zbcoo_reshape.<locals>.<genexpr>S  s     +TAAOa,?+T   	new_sizesr  r   rC   c              3  ,   K   | ]  }d |f     ywr   r3   r  s     r7   r8   zbcoo_reshape.<locals>.<genexpr>\  s     <1wsAv<r  clip)r  r+  allowrj   ri   Tr~   r   )r  )"rV   rD   r6   rU   rN   r   ru   rY  r   rp   rK   rp  r  r   cumprodr   r   searchsortedr   r	   rI  ro   r=   
empty_likeravel_multi_indexr  numpy_rank_promotionrt   rk   r   unravel_indexr  r   r^   )rw   r  r  r  batch_shapesparse_shapedense_shaperV  rW  rX  rT   sparse_size
cuml_shapei1i2new_batch_shapenew_sparse_shapenew_dense_shaperU   
index_colsrd   flat_indicesoob_indicesnew_index_colsr  rV   s   `                        @r7   bcoo_reshaper/  (  s   $ 
kk%)E`S_TWT_T_I``
T
UU a ,6cii#++s||A\+](+|[(=HHckk:?uSXX)?)L%*k:yy%*		,'+zz)$*1_:5
 ""%))K	{.QTQ\Q\P]_ ` `A*{2*D
 ""%))K	{.QTQ\Q\P]_ ` ` z8"zK7gF"7A)cRTgWZ[]W^M_7`4/#_ 
HHE/E377E_EUUS[[U+T+TU
W$ KKKKD_DcggDs||D::S[[:#++/:<'
 
nnW,Jo,Jsww,J,JKG<<<J=HIC		#++/23ILI((%:MTZ[L		!	!'	* W		#))CKKs||A[*\07!? ?@CBQU@V W &&|5EFNooH#s9~HrRGiiSYY/?w}}%UW^_G	tWoY	// JW W Is   ,P&7A$P+P7+P4c           	        t        j                  t        j                  d      j	                  t        j
                  | j                  | j                        |      }|D cg c]  }|| j                  k  s| }}|D cg c]4  }| j                  |cxk  r| j                  | j                  z   k  s0n n|6 }}|D cg c]!  }|| j                  | j                  z   k\  s |# }}| j                  | j                  }}|rt        j                  ||      }|s|r8t        j                  |||D cg c]  }|dz   | j                  z
   c}z         }|rt        j                  | j                  | j                  | j                  | j                  z    |j                        }	t        j                  |D cg c]  }|| j                  z
   c}      }
|j                  d|
f   j                  d      }|j                  d|
f   j!                  |	|
   dz
        }t        j"                  |dk  |	|      }t%        ||f| j                        S c c}w c c}w c c}w c c}w c c}w )	z,Sparse implementation of :func:`jax.lax.rev`r  r  rf   rj   .ri   r   rC   )r  r  r	   revr  r  rD   rk   r6   ru   rU   rV   r=   rt   rr   mulr   r   r^   )r  r  r   r]  rR   r  r  rU   rV   r"  spdimss              r7   bcoo_revr5  i  s    
ggcgg7BB


w}}gmm
< C !! &=aW__)<=*=&dq'//Q*cSZScScAc*cd+d%Qagoo@P@P.P)PQ*Q,,$ggg*5G:774JT^1_q!a%':J:J2J1_$_`D99W]]7??GOOgN^N^<^_#*==2LYY[AGOO+ABFjjf%))"-Gjjf%)),v*>*BCGii!\7;G	tWoW]]	33' >dQ 2`
 Bs0   #I8I1I4I!I#I3I:I$c                   t         fdD              t         fdD              rt        d j                   d      t         fdD              }t	        j
                  t         j                        D cg c]  }| j                  z   vr| c}t              }t         fdD              }t        j                   j                  ||z         }t        j                   j                  d|f   |      }t        fd	t         j                        D              }t        ||f| j                    j"                  
      S c c}w )a  Sparse implementation of :func:`jax.lax.squeeze`.

  Squeeze any number of size 1 dimensions from an array.

  Args:
    arr: BCOO array to be reshaped.
    dimensions: sequence of integers specifying dimensions to squeeze.

  Returns:
    out: reshaped array.
  c              3  J   K   | ]  }t        |j                          y wrB   )r.   rK   r4   rF   r!  s     r7   r8   zbcoo_squeeze.<locals>.<genexpr>  s     L#&sCHH5Lr9   c              3  B   K   | ]  }j                   |   d k7    ywr{   rC   r8  s     r7   r8   zbcoo_squeeze.<locals>.<genexpr>  s     331	3r  zPcannot select an axis to squeeze out which has size not equal to one, got shape=z and dimensions=c              3  B   K   | ]  }|j                   k  s|  y wrB   r  r4   r]  r!  s     r7   r8   zbcoo_squeeze.<locals>.<genexpr>  s     >1a#++oQ>s   rj   c              3  z   K   | ]2  }|j                   j                  z   k\  r|j                  z
  d z    4 ywr{   )r6   ru   r;  s     r7   r8   zbcoo_squeeze.<locals>.<genexpr>  s;      9aS[[3<<77 %) 9s   8;.c              3  2   K   | ]  \  }}|vs|  y wrB   r3   )r4   rd   r}   r  s      r7   r8   zbcoo_squeeze.<locals>.<genexpr>  s     N$!Q!::MANrE  rl   )r   r   r   rD   r   rt   rp   ru   r6   r   r	   rv  rU   rV   r   r^   rI   rJ   )	r!  r  rR   rd   r  r  r  r  r7  s	   ``       r7   bcoo_squeezer>    s3    LLL*3
33
 ""%)),=*@ A A>
>>*U3<<%8 @_J>  @GJL+ 9: 9 9*[[:
#:;(CKK[(89:F+N)CII"6NN)	x%Y //@R@R
T T@s    E)stridesc                  t        | t              st        dt        |              |D cg c]  }t	        j
                  |       }}|D cg c]  }t	        j
                  |       }}|#|D cg c]  }t	        j
                  |       }}ndg| j                  z  }t        |      t        |      cxk7  rt        |      cxk7  r| j                  k7  rn nt        d| j                         t        |      | j                  k7  r$t        dt        |       d| j                         t        d |D              rt        d|       t        d	 t        ||| j                        D              st        d
|d|d| j                         t        || j                  | j                  g      \  }}}t        || j                  | j                  g      \  }}	}
t        || j                  | j                  g      \  }}}g }g }t!        t#        |||            D ]  \  }\  }}}|j%                  | j&                  j                  |   | j                  |   k7  rt)        d      nt)        |||             |j%                  | j*                  j                  |   | j                  |   k7  rt)        d      nt)        |||              |j%                  t)        d             |j-                  t)        d      t)        d      g       t!        t#        ||
|            D ]%  \  }\  }}}|j%                  t)        |||             ' | j&                  t/        |         }| j*                  t/        |         }t/        d t        |||      D              }t        || j                  | j                  g      \  }}}| j                  rt1        j2                  t1        j4                  ||j6                        t9        | j                  dz               }t1        j2                  t1        j4                  |	|j6                        t9        | j                  dz               }t1        j2                  t1        j4                  ||j6                        t9        | j                  dz               }t1        j                  ||k\  ||k  z  ||z
  |z  dk(  z  dd      }t1        j:                  |||z
  |z   dz
  |z  ||z
  |z   dz
  |z        }t=        j2                  |d   t9        | j                  dz   | j                  dz   | j>                  z               }t1        j:                  ||d      }tA        jB                  |      }| jD                  |kD  rtG        ||tI        |      |      \  }}t        ||f|      S c c}w c c}w c c}w )a  Sparse implementation of :func:`jax.lax.slice`.

  Args:
    mat: BCOO array to be reshaped.
    start_indices: sequence of integers of length `mat.ndim` specifying the starting
      indices of each slice.
    limit_indices: sequence of integers of length `mat.ndim` specifying the ending
      indices of each slice
    strides: (not implemented) sequence of integers of length `mat.ndim` specifying
      the stride for each slice

  Returns:
    out: BCOO array containing the slice.
  6bcoo_slice: input should be BCOO array, got type(mat)=Nrf   z,bcoo_slice: indices must have size mat.ndim=zlen(strides) = z; expected c              3  &   K   | ]	  }|d k    ywr  r3   r   s     r7   r8   zbcoo_slice.<locals>.<genexpr>  s     !Aa!ru  z5strides must be a sequence of positive integers; got c              3  T   K   | ]   \  }}}d |cxk  xr |cxk  xr |k  nc  " ywr  r3   )r4   startendr   s       r7   r8   zbcoo_slice.<locals>.<genexpr>  s2      X!T %&3&$&& Xs   &(z/bcoo_slice: invalid indices. Got start_indices=z, limit_indices=z and shape=c              3  @   K   | ]  \  }}}||z
  |z   d z
  |z    ywr{   r3   )r4   rD  rE  strides       r7   r8   zbcoo_slice.<locals>.<genexpr>  s3      OsF 	5[6A&(Os   rj   r   ri   T)r  r   r  rC   rG  )%r  r^   r  r  rm   rn   rK   r   r   r   rM   r   rD   r   r6   ru   r   rP   appendrU   rb   rV   extendr   r=   r<   rt   rk   rp   r   r	   r   rp  r  ro   rJ  r   )rw   start_indiceslimit_indicesr?  rd   start_batchstart_sparsestart_dense	end_batch
end_sparse	end_densestride_batchstride_sparsestride_densedata_slicesindex_slicesrD  rE  rG  r  r  	new_shaper   new_shape_sparsestartsendsstrides_keep	keep_datanew_nses                                 r7   
bcoo_slicer_    s%     
C	
LTRUYKX
YY.;<8>>!$<-<.;<8>>!$<-<*12Qx~~a 2G2cCHHnG3}-IWII
CCHH:N
OO\SXX
s7|nKzJ
KK!!!
LWIV
WW	 X%-m]CII%VX 
X
G8H I'%'{399+? @ @ ,6mckkSVS_S_E`+a(+|[%/S\\?Z%[")Z.83;;PSP\P\B].^+,|+,!*3{I|+T!U la	%fchhnnQ&7399Q<&GuT{USXZ]_eMfgs{{'8'8';syy|'KdQVW\^aciQjkl U4[!uT{E$K01!*3{I|+T!U 2a	%fuUC012XXeK()(E,/0+ O&}mWMO O) &i#++s||1LM!q\\__SYY|;;L;LMuUXU`U`cdUdOefF??399Z{7H7HI5QTQ\Q\_`Q`KabDsyyk>O>OPRWX[XcXcfgXgRhiH77K6)kD.@A 6)X5:<T+D ))D;#7(#BQ#F8"S!F]X59hFHK VeCKK!OS[[ST_WZWbWbEb.cdIyyHa0Hii()G
ww2+jy&AwPh 
x%Y	77s =<2s   V5V:2V?c                n   t        d |D              }t        j                  t        j                  d      j                  t        j                  | j                  | j                        ||       t        | t              st        dt        |              t        d |D              }t        d |D              sJ t        d |D              sJ t        |      t        |      cxk7  r| j                  k7  rn nt!        d| j                         t        d	 t#        || j                        D              st        d
| d| j                         t%        || j&                  | j(                  g      \  }}}t%        || j&                  | j(                  g      \  }}}g }	g }
g }g }t+        |r|d   nt,        j.                  d      }t1        t#        ||            D ]  \  }\  }}| j2                  j                  |   | j                  |   k7  }| j4                  j                  |   | j                  |   k7  }|	j7                  |r|n|       |
j7                  |rdn|       |j7                  |r|n|       |j7                  |rdn|        |	j7                  |       |
j7                  | j8                         |j;                  ||g       |j;                  | j8                  | j(                  g       |	j;                  |       |
j;                  |       t        j                  | j2                  |	|
      }t        j                  | j4                  ||      }|}| j(                  r>t=        j>                  ||j                        }t=        j>                  ||j                        }t=        j>                  | j                  | j&                  | j&                  | j(                  z    |j                        }t=        j@                  |dk  ||z   |      }t=        jB                  |d||z
        }t=        jD                  |tG        | j&                  dz               }t=        jD                  |tG        | j&                  dz               }t=        jD                  |tG        | j&                  dz               }t=        j                  ||k\  |||z   k  z  dd      }t=        j@                  |||z
  |      }t        jD                  |d   tG        | j&                  dz   | j&                  dz   | jH                  z               }t=        j@                  ||d      }| j8                  tK        jL                  |      kD  r1tK        jL                  |      }tO        ||tQ        |      |      \  }}t        ||f|      S )a  Sparse implementation of :func:`jax.lax.dynamic_slice`.

  Args:
    mat: BCOO array to slice.
    start_indices: a list of scalar indices, one per dimension. These values
      may be dynamic.
    slice_sizes: the size of the slice. Must be a sequence of non-negative
      integers with length equal to `ndim(operand)`. Inside a JIT compiled
      function, only static values are supported (all JAX arrays inside JIT
      must have statically known size).

  Returns:
    out: BCOO array containing the slice.
  c              3  F   K   | ]  }t        j                  |        y wrB   )rm   rn   rc   s     r7   r8   z%bcoo_dynamic_slice.<locals>.<genexpr>  s     =AhnnQ'=r   slice_sizesr  rA  c              3  F   K   | ]  }t        j                  |        y wrB   )r=   r   rc   s     r7   r8   z%bcoo_dynamic_slice.<locals>.<genexpr>  s     >1A>r   c              3  x   K   | ]2  }t        j                  |j                  t        j                         4 y wrB   )r=   r   rk   r   r   rc   s     r7   r8   z%bcoo_dynamic_slice.<locals>.<genexpr>  s#     HQS^^AGGRZZ0Hs   8:c              3  :   K   | ]  }|j                   d k(    yw)r3   NrC   rc   s     r7   r8   z%bcoo_dynamic_slice.<locals>.<genexpr>  s     2qQWW]2s   z4bcoo_dynamic_slice: indices must have size mat.ndim=c              3  B   K   | ]  \  }}d |cxk  xr |k  nc   ywr  r3   )r4   
slice_size	axis_sizes      r7   r8   z%bcoo_dynamic_slice.<locals>.<genexpr>	  s"     c.Cj)Q*)	))cs   zIslice_sizes must be less than or equal to operand shape, got slice_sizes z for operand shape r   rf   rj   ri   Tr~   r  rC   rG  ))r   r  r  r	   dynamic_slicer  r  rD   rk   r  r^   r  r  rM   r   rK   r   rP   r   r6   ru   r#   r   r  r   rU   rV   rH  ro   rI  r=   rt   r   r  r<   rp   r   rp  r  rJ  r   )rw   rJ  rc  rL  rM  rN  
size_batchsize_sparse
size_dense
data_start
data_sizesindices_startindices_sizeszerord   rD  r   data_is_broadcastindices_is_broadcastr  r  rW  rY  sizesr"  r\  r]  r^  s                               r7   bcoo_dynamic_slicerv    s    ===+''#

-=>II


syy#))
4m! J # 
C	
LTRUYKX
YY>>>-	H-H	HH	H	2M2	22	23{+7sxx7
KCHH:V
WW	cs;X[XaXaGbc	c
 ''2m3FsyykS T T ,6mckkSVS_S_E`+a(+|[(2;cll@[(\%*k:**--	Ma rxx	C$#CZ$@A >a%q)SYYq\9;;,,Q/399Q<?/dU;,a$7!55A2=> DCGGd|$./K JsxxZ@(!!#++}mL+)\\YY|;+<+<=FIIk):):;E99SYYs{{CKK#,,4NOWbWhWhiLYYvz6L#8&AFXXfa!56F__VU3;;?%;<FOOE5q#9:E??<s{{Q1GHL77K6)kFUN.JKRZ^_D))D+"6>KVeCKK!OS[[ST_WZWbWbEb.cdIyyHa0H
ww;''		+&g2+jy&AwPh 
x%Y	77r]   c                B    t        fdt        |       D              S )Nc              3  6   K   | ]  \  }}|k(  rn|  y wrB   r3   )r4   rd   tr8  vals      r7   r8   z!_tuple_replace.<locals>.<genexpr>A  s      A$!Qa3hsA%As   )r   r   )tupr8  rz  s    ``r7   r3  r3  @  s    	A)C.A	AAr]   c                   t        | j                  | j                  | j                  |      \  }}}t	        ||f|      S )a  Sum array element over given axes.

  Args:
    mat: A BCOO-format array.
    shape: The shape of the target array.
    axes:  A tuple or list or ndarray which contains axes of ``mat`` over which
      sum is performed.

  Returns:
    A BCOO-format array containing the result.
  )rS   axesrC   )_bcoo_reduce_sumrU   rV   r   r^   )rw   r}  rW  rX  r7  s        r7   bcoo_reduce_sumr  C  s<     &6	hhCIID&:"(K	x%Y	77r]   c          
         |j                   t        fdD              sJ t         |      \  }}t        t	                    t        fdD              } j                  |       rt        j                  t        j                  z    |j                        t        |j                  dz
              }t        j                  ||k  d      } j                  |j                  kD  r=t        j                  |t        t        |j                   j                                    }t        j                  | d       t              D 	cg c]  }	|	z   vs|	 }
}	|
s9t        j                  t!        |j                   dz   d      |j                        }n|dt#        j                  |
      f   }D ch c]
  }|k  s	| }}|D ]  } j                   |   dk(  rg|j                   |   dk(  r	 |   z   not        j$                   t!         j                   ||         t        t         j                                     n#|j                   |   dk(  r j                  |        j                   |   |j                   |   k(  rJ  t        t        t	        t                    |z
              }t         fd|D              }|t'        j(                  |D 	cg c]  }	 j                   |	    c}	      z  }t        j*                   g || j                   dz   d  g ||t         j                               t        j*                  |g |||j                   dz   d  g ||t        |j                              }t        fd	t        t-                    D              } ||fS c c}	w c c}w c c}	w )
Nc              3  P   K   | ]  }d |cxk  xr t              k  nc   ywr  r  )r4   r   rD   s     r7   r8   z#_bcoo_reduce_sum.<locals>.<genexpr>U  s"     /QQ! c%j  /s   #&c              3  @   K   | ]  }|z   k\  s|z
  d z     ywr{   r3   )r4   axr6   ru   s     r7   r8   z#_bcoo_reduce_sum.<locals>.<genexpr>Z  s&     R2w?Q9QR(]Q&Rs   rj   rf   ri   r   .c              3  <   K   | ]  }j                   |     y wrB   rC   )r4   rd   rU   s     r7   r8   z#_bcoo_reduce_sum.<locals>.<genexpr>}  s     @A$**Q-@rM  c              3  2   K   | ]  }|vs|     y wrB   r3   )r4   rd   r}  rD   s     r7   r8   z#_bcoo_reduce_sum.<locals>.<genexpr>  s     I1D=E!HIs   	
)rD   rM   r   rU  rs   r   r   r=   r<   rt   rk   rp   rK   r	   r   r   r3  r   r2  rp  r  rI  r   )rU   rV   rS   r}  r   ro   
dense_axesr   r   rd   
sparse_idxr  
batch_axesnew_batch_dimsr(  r^  r7  r6   ru   rD   s   `  `             @@@r7   r~  r~  S  se   
,,%	/$/	//	/,T7EB'8Q	D		$ RRR*	*	$	iigw12'--HGLL1J 777Z',Dyy499__T5tyy$)))D#EFd99T4#D !?Faa'k.EF*F	iiw}}gk1Ew}}UGc288J//0G "2rR'\2*2  	/bzz"~	r	a	eBi##D.Rr*SUZ[`aeajaj[kUlm	r	a	xx|::b>W]]2....	/ E'N 3j @AB.@@@/$))J?qTZZ]?@@'	TKKK$**Wq[\2JKO~O
OU7DII5NO
Q$ KKQ/Q7QW]]7Q;<5PQU.U:Ugw||8TUW' IeCJ&7II)	w		!!C G 3  @s   O	'O	
OO,O
c                    t        | j                  | j                  |j                  |j                  | j                  |j                        \  }}}t	        ||f|      S )zAn element-wise multiplication of two sparse arrays.

  Args:
    lhs: A BCOO-format array.
    rhs: A BCOO-format array.

  Returns:
    An BCOO-format array containing the result.
  )r  r  rC   )_bcoo_multiply_sparserU   rV   r   r^   )r  r  rW  rX  r7  s        r7   bcoo_multiply_sparser    sO     &;	hhSXXs{{syy&"(K 
x%Y	77r]   c                  |j                   }|j                   }t        | ||      }t        |||      }	t        |      t        |      k7  rt        d| d|       |j                  |	j                  k7  rt        d| d|	       t        |j                  |	j                        }
t        j                  t        ||
d  ||
d        }t        ||
      } || |||      \  }}||t        j                  ||      fS )NzFbcoo_multiply_sparse: arrays must have same number of dimensions, got z, zIbcoo_multiply_sparse: arrays with differing numbers of dense dimensions: )rL  rM  )rD   r   r   r  r   rN   ro  r6   r]  r   _bcoo_multiply_sparse_unbatchedr   r=   rq  )r  r  r  r  r  r  rL  rM  r  r  r6   _mulrU   rV   s                 r7   r  r    s   ))xi8#xi8#^s9~%
 $+R	{4 5 5[[CKK
 336%r#@ A AS[[)'			:%.wx%8%.wx%8
:$ 
D'	"$xhD-$	w,,Y	B	BBr]   c          
        t        | ||      }t        |||      }|j                  dk(  s|j                  dk(  sJ |j                  r5t        t        | |f|      d      j                  \  } }t        | ||      }n@|j                  r4t        t        ||f|      d      j                  \  }}t        |||      }t        j                  t        t        |d |j                   |d |j                               D 	
cg c]  \  }\  }	}
|	dk7  r|
dk7  r| c}
}	}t              }|j                  |j                  z  }t        j                  |d d d |f   |d d d |f   k(  d      }t        j                  |||j                  |j                  f      \  }}| j                  |   j                  dd	      |j                  |   j                  dd	      z  }t        j                   |j                  |   j                  dt#        |d
      	      |j                  |   j                  dt#        |d
      	            }||fS c c}
}	}w )Nr   rC   r  rf   rj   ri   r   r)  r*  rA  )r   r6   r  r^   r  r=   rt   r   r   ru   r   ro   rM   r   rr   r1  maximumrO   )r  r  r  r  rL  rM  r  r  rd   r   r   r  ro   r   i_lhsi_rhsrU   rV   s                     r7   r  r    s    xi8#xi8#
++
q 01	1 	[[.tX{4KS\/]ghiooHk
;	
:C
{{.tX{4KS\/]ghiooHk
;	
:C	8Ims||<TV_`madamamVn3o)p , ,+!Xb"Qw27  ,36
8$
 	#''# 
Qd]+{4D=/II2	N$T#''8JK,%
++e

 
 f
 
;
++e

 
 f
 
;<$KKnnUVIq8QRnnUVIq8QRT' 
w!,s   >Ic                \    t        | j                  | j                  || j                        S )zAn element-wise multiplication between a sparse and a dense array.

  Args:
    lhs: A BCOO-format array.
    rhs: An ndarray.

  Returns:
    An ndarray containing the result.
  r   )_bcoo_multiply_denserU   rV   r   )sp_matvs     r7   bcoo_multiply_denser    s!     
fkk6>>1V\\	RRr]   c                  |j                   }|j                  dk(  rt        j                  | |      S ||j                   k(  r t        j                  | t	        ||            S t        j
                  |j                   |      |k7  rt        d|d|j                         t        j                  |t        t        |      |j                  z
              }t        | ||      t        t        j                        fd       } || ||      S )zNBroadcasted elementwise multiplication between a BCOO array and a dense array.r   zmultiplication between sparse and dense is only implemented for cases where the output shape matches the sparse matrix shape. Got shape=z
, v.shape=r  c                   j                   d   |j                  j                  z
  k(  sJ t        fdt	        j                   d         D              }t        d t        ||j                         D              }| ||   z  S )Nrf   c              3  0   K   | ]  }d d |f     y wrB   r3   r  s     r7   r8   z5_bcoo_multiply_dense.<locals>._mul.<locals>.<genexpr>  s     ?!1?r  c              3  4   K   | ]  \  }}|d k7  r|nd  ywr  r3   r|   s      r7   r8   z5_bcoo_multiply_dense.<locals>._mul.<locals>.<genexpr>  s     Atq!Q!V"As   )rD   rK   r   r   rp   rP   )rU   rV   r  r8  r   s    `  r7   r  z"_bcoo_multiply_dense.<locals>._mul  sn    ==qvv5555
?uW]]1-='>?
?C
As3/@A
AC!C&=r]   )rD   rK   r	   r3  r   rq  rN   r<   rp   r   r   r   r   r6   )rU   rV   r  rS   rD   r  r   s         @r7   r  r    s     ,,%VVq[774
agg774w233!''5)U2
	177*  
ooas5zAFF234!
w
.%
:' (
 
dGQ	r]   )rJ   indices_are_sortedr+  r   c          	         t         j                   j                   j                         |t        j
                  }t	        j                  |      }|t        j
                  k7  rt        d|d      t        ||||||      }	d}
 t        j                  t        j                  |
      j                  t        j                   j                   j                        t        j                  |j                  |j                        fi |	}|j                   |j"                  |j$                  }t'        |d      g j(                  z  }t+         j(                        D cg c]  }d }}t-         j                        t/        |      D ]*  \  }}|d|f   j1                         ||<   ||   |<   d||<   ,  fd} t3        ||f	      |      }t5        |g |j                  dd
 |j                  dd t7        t+        |j(                                    }|j                  rt7        fdt+        t9        |j                              D              }t;        j<                  |j(                  t>              }t;        j@                  |j(                        |t;        jB                  |z         <   tE        |dt9        |             tE        t+        t9        |                  k7  rtG        |d      }tI        |t7        |            }|jK                  |j                        jM                  |j                        S c c}w )zBCOO version of lax.gather.Nzbcoo_gather: mode=z not yet supported.)r  rc  rJ   r  r+  r   r  r   .c                :    t        |       }t        |      S )Nrb  r1  )rv  r>  )rV   slccollapsed_slice_dimsfull_slice_sizesr  s     r7   
slice_funczbcoo_gather.<locals>.slice_func	  s     
Wg;K
LC(<==r]   r[  ri   rf   r  c              3  *   K   | ]
  }|vr|  y wrB   r3   )r4   rF   offset_dimss     r7   r8   zbcoo_gather.<locals>.<genexpr>'	  s       1sK/  1s   rj   r  rI  )'r   rU   rV   rD   r)   PROMISE_IN_BOUNDSfrom_anyrN   dictr  r  r	   gatherr  r  rk   r  r  start_index_mapr#   rK   rp   r  r   r  r   r/  r   r   r   r   r   r   rt   rs   r  rF  rI  r  )r  rJ  r  rc  rJ   r  r+  r   parsed_moder.  r  r  r  full_start_indicesrd   r;  r  r  r6  rR   rI  r  r  r  s   `                    @@@r7   bcoo_gatherr    s    w> 
\..D!**40+%777
 3dW4GH
II	 1{+@RJ
0$
7/LSWWSZZALL


w}}gmm
<


}22M4G4G
H (
 "--+*??%55/ *0q)A(BW\\(Q-27<<-@AtA'A'--(( da)#q&1779q%a.QGAJ> 04
WJ/0BC&<##CR(<6<<+;<U6;;'(*&
 \\ 1eC,?&@ 1 1J((6;;c2K68ii6LKk123
;'J()Ss:1G-HH "&!4fFk0BCF		'	.	.x~~	>>5 Bs   (	M	rf   )lhs_dilationrhs_dilationr  feature_group_countbatch_group_countr|  r}  r~  c       
           t        j                  t        j                  ||||||||	|
|      } t	        j
                  |      t	        j                  | j                  | j                        t	        j                  |j                  |j                              }t        |t        j                        rt        |j                        dk(  sJ |j                  d   j                  }|d   d| j                  dz
  z  k7  rt!        d      |d   d|j                  dz
  z  k7  rt!        d      |d	   d| j                  dz
  z  k7  rt!        d
      |d   |d   cxk7  rdk7  rt!        d       | j                  d d |j                  d d cxk7  rdk7  rt!        d       t#        | d      r| j$                  j                  n#t#        |d      r|j$                  j                  nd}|d   \  }t'        t)        | |      t)        ||      |      S )N)
window_stridespaddingr  r  r  r  r  r|  r}  r~  rf   r   r  r   r0   z#bcoo convolution with lhs_dilation.r  r  z.bcoo convolution with non-unit window_strides.r  r  z,bcoo convolution with non-unit group counts.)rf   rf   z:bcoo convolution with leading dimensions other than (1, 1)rV   r  r  )r  )r]  r   r	   conv_general_dilatedr  
make_jaxprr  rD   rk   r  r   ClosedJaxprr   eqnsparamsrK   rN   r'  rV   _bcoo_conv_1d_convert_to_1d_for_conv)r  r  r  r  r  r  r  r  r  r|  r}  r~  ra  jaxprr  r   s                   r7   bcoo_conv_general_dilatedr  3	  s    
			#WlN_-AR2H
!$ #..
s33CIIsyyI"33CIIsyyIK%	E4++	,UZZA1EE	E::a=&N1 55
C
DDNtsxx!|44
C
DDA!66
N
OO F+@$AFQF
L
MM G 	YYr]ciim-v-
Z
[[ . '.c9&=""+23	+Bckk''  I('	.sK@.sK@&
( (r]   c                n   t        | t        j                  t        j                  f      r9t        j                  | d      }t        j                  |t        |      dfd      }nt        | t              r| j                  dd      } t        j                  | j                  d      }t        j                  | j                  d      }t        j                  t        j                  |d      | j                  d   k  |d      }nt!        dt#        |        d	      t        ||f| j                  dd  
      S )Nr   rf   rf   r   r0   )r6   r   r   ri   z)bcoo_conv_general_dilated: input of type z not recognized.rC   )r  r  r+   r   rT  r	   rv  r  r   r^   update_layoutrU   rV   r=   r   rD   r  r  )rw   r   rU   rV   s       r7   r  r  Z	  s    cii,-;;sF#D"";TABG#t


Aq

1C;;sxx(Dkk#++v.G99S[[$/#))B-?qID
?S	{JZ[
\\	tWoSYYqr]	33r]   c                H   | j                   | j                  cxk(  r&|j                   cxk(  r|j                  cxk(  rdk(  sJ  J | j                  |j                  k(  sJ t        t	        t
        |            }t        |      dk(  sJ | j                  d d d f   |j                  d d d f   z  j                         }|d   |j                  z
  }| j                  d d d f   |d d d f   z   j                         }|dk  }t        j                  |d|      }t        j                  |d|      }t        d| j                  d   |d   z   |d   z   |j                  d   z
  dz         }t        j                  |d      }t        j                  |d      }t!        ||fdd|f      S )Nrf   r0   r   r  )r   rf      rC   )rK   ru   rk   r   rj  r   r   rU   r  rV   r=   r   rO   rD   r	   r<   r^   )r  r  r  r  r  r  r   dimsizes           r7   r  r  h	  sr   	S\\	BSXX	B	B	BB	BB	B	cii		#c7#$'	W		hhq$w#((47"33::<(1:#&QW%tQw7>>@+
/$		$;/+YYtQ)(399Q<'!*,wqz9CIIaLH1LM'__Xv.(Y7+	x%aG_	==r]   c                      e Zd ZU dZded<   ded<   ded<    ed       Z ed       Z ed	       Z ed
       Z	 ed       Z
ded<   ded<    ed       Z ed       Zddd	 	 	 d, fdZd Zd-dZd-dZd-dZedej(                  ddd	 	 	 	 	 d.d       Zedddd	 	 	 	 	 d/d       Zedddddd	 	 	 	 	 	 	 d0d        Zeddddd!	 	 	 	 	 	 	 d1d"       Zddd#d$	 	 	 d2d%Zd3d4d&Zd-d'Zd5d(Zd6d7d)Zd* Zed+        Z xZ S )8r^   ar  Experimental batched COO matrix implemented in JAX

  Args:
    (data, indices) : data and indices in batched COO format.
    shape : shape of sparse array.

  Attributes:
    data : ndarray of shape ``[*batch_dims, nse, *dense_dims]`` containing the
      explicitly stored data within the sparse matrix.
    indices : ndarray of shape ``[*batch_dims, nse, n_sparse]`` containing the
      indices of the explicitly stored data. Duplicate entries will be summed.

  Examples:
    Create a sparse array from a dense array:

    >>> M = jnp.array([[0., 2., 0.], [1., 0., 4.]])
    >>> M_sp = BCOO.fromdense(M)
    >>> M_sp
    BCOO(float32[2, 3], nse=3)

    Examine the internal representation:

    >>> M_sp.data
    Array([2., 1., 4.], dtype=float32)
    >>> M_sp.indices
    Array([[0, 1],
           [1, 0],
           [1, 2]], dtype=int32)

    Create a dense array from a sparse array:

    >>> M_sp.todense()
    Array([[0., 2., 0.],
           [1., 0., 4.]], dtype=float32)

    Create a sparse array from COO data & indices:

    >>> data = jnp.array([1., 3., 5.])
    >>> indices = jnp.array([[0, 0],
    ...                      [1, 1],
    ...                      [2, 2]])
    >>> mat = BCOO((data, indices), shape=(3, 3))
    >>> mat
    BCOO(float32[3, 3], nse=3)
    >>> mat.todense()
    Array([[1., 0., 0.],
           [0., 3., 0.],
           [0., 0., 5.]], dtype=float32)
  r+   rU   rV   r   rD   c                4    | j                   j                  d   S )Nrh   rV   rD   r   s    r7   r   zBCOO.<lambda>	  s    dll004 r]   c                .    | j                   j                  S rB   )rU   rk   r   s    r7   r   zBCOO.<lambda>	  s    		 r]   c                4    | j                   j                  dz
  S )Nr0   )rV   rK   r   s    r7   r   zBCOO.<lambda>	  s    $,,"3"3a"7 r]   c                4    | j                   j                  d   S r  r  r   s    r7   r   zBCOO.<lambda>	  s    4<<#5#5b#9 r]   c                N    | j                   j                  dz
  | j                  z
  S )Nrf   )rU   rK   r6   r   s    r7   r   zBCOO.<lambda>	  s    $))..1"4t||"C r]   rL   rI   rJ   c                X    t        | j                  | j                  | j                        S rB   )r   rD   rI   rJ   r   s    r7   r   zBCOO.<lambda>	  s#    
4::t7J7J+/+>+>!@ r]   c                2    | j                   | j                  fS rB   )rU   rV   r   s    r7   r   zBCOO.<lambda>	  s    DLL 9 r]   FrH   c                   t        t        j                  |      \  | _        | _        || _        || _        t        | !  |t        |             t        | j                  | j                  | j                         y r-  )rj  r=   r   rU   rV   rI   rJ   super__init__r   r   rD   )r   r  rD   rI   rJ   	__class__s        r7   r  zBCOO.__init__	  sZ    !#++t4DIt|(D(D	GTu.499dllDJJ7r]   c                   | j                   j                  }	 | j                  }| j                  }| j                  }| j
                  }t        | j                        }d|}|r|d|z  }|r|d|z  }| d| | | d}t        | j                  t        j                        r%t        | j                        j                   d| d}|S #  | d}Y TxY w)	N, nse=r   r   ()z(<invalid>)[])r  r   ro   r6   r   rk   r  rD   r  rU   r   Tracerr  )	r   namero   r6   r   rk   rD   extrarepr_s	            r7   __repr__zBCOO.__repr__	  s    >>""D/HHcggjje4::e 3&ke	%[z?*%	%[z?*%awugeWA.e$))T[[)dii))*!E7!4eL#k"es   AC Cc                    t        d      )NzBCOO.reshaperN   r   r  kwargss      r7   rI  zBCOO.reshape	  s    
n
--r]   c                    t        d      )NzBCOO.astyper  r  s      r7   r  zBCOO.astype	  s    
m
,,r]   c                    t        d      )NzBCOO.sumr  r   s    r7   r   zBCOO.sum	  s    
j
))r]   Nr   ro   r   r   r6   c               "    t        |||||      S )z7Create a BCOO array from a (dense) :class:`~jax.Array`.r  )r   )clsrw   ro   r   r   r6   s         r7   	fromdensezBCOO.fromdense	  s     	sWgO Or]   )r   r   r6   c               R   |dk7  s|dk7  rt        d      |j                         }t        j                  |j                        }t        j
                  |j                  |j                  f      j                  |xs t        j                        } | ||f|j                  dd      S )z5Create a BCOO array from a :mod:`scipy.sparse` array.r   z+BCOO.fromscipy with nonzero n_dense/n_batchFrl   )rN   tocoor=   r   rU   r  r  r  r  r  rD   )r  rw   r   r   r6   rU   rV   s          r7   from_scipy_sparsezBCOO.from_scipy_sparse	  s     !|w!| MNN
))+C;;sxx D1299 syy"G gcii#% %r]   r  )rk   r   r   r6   ro   c          
     l   t        |      }t        |      |z
  |z
  }|dk  s|dk  s
|dk  s|dk  rt        d|d|d|d|      t        |||g      \  }}	}
t	        j
                  g |||
|      }t	        j                  g |||t	        j                  |	      |      } | ||f|dd      S )z?Create an empty BCOO instance. Public method is sparse.empty().r   Invalid inputs: shape=r   r   r  Trl   )r   r   r   r   r=   r   fullrt   )r  rD   rk   r   r   r6   ro   ru   r!  r"  r#  rU   rV   s                r7   _emptyzBCOO._empty	  s     %LE5zG#g-H!|w{gkS1W0%G:[zC6RSS-7?R-S*K{996{6C6+6>Dhh44c484cii6M{[GgeD"$ $r]   rk   r   r6   r   c                  d|z
  |z
  }|dk  s
|dk  s|dk  rt        d||f d|d|      dkD  rt        ||z
        }	nt        |z   |      }	|	dk  r| j                  ||f||||      S |dkD  s|dkD  r,| j                  t	        j
                  |||      |||	      S |dk(  rt	        j                  |	|      }
t	        j                  |	|      }t        |d      t        |      t	        j                  t        j                  |t        j                  dk\  fd
fd            t        j                  |t        j                  dk  fdfd            g      }nt	        j                  ||      }
t	        j                  ||      }|t        |      z   }dk  rO|
j                  d t               j!                  d      }
|j                  d t               j!                  |      }nYdkD  rT|
j                  |t              z
  d  j!                  d      }
|j                  |t              z
     j!                  |      }|
d d d f   }
|d d d d f   } | |
|f||fdd      S )Nr0   r   r  r   r   r  rf   rj   r   c                      S rB   r3   rr  s   r7   r   zBCOO._eye.<locals>.<lambda>&
      d r]   c                      S rB   r3   ks   r7   r   zBCOO._eye.<locals>.<lambda>&
      A r]   c                      S rB   r3   r  s   r7   r   zBCOO._eye.<locals>.<lambda>'
  r  r]   c                      S rB   r3   r  s   r7   r   zBCOO._eye.<locals>.<lambda>'
  r  r]   Trl   )r   ro  r  r  r=   eyeonesr   r#   r  r	   subcondr   rr   absrs   )r  r   r  r  rk   r   r6   r   ru   	diag_sizerU   r  rV   rr  s      `         @r7   _eyez	BCOO._eye	
  sj    7{W$H!|w{gk/Ax{';gZPQQ1uaQ-ia!eQ-iA~ZZAe '  : : {gk]]3771a%8#*G'2  4 4 !|XXiu-dJJy4cC^d
a.a  SXXa1flI>?SXXa1flI>?"A Bg XXau%d

1K0g&!,,g	
QwwwA##A&**Wc!f%))!,q5wwq3q6z{#''***QQZ(,,Q/!T']d4&ggq!fT"$ $r]   r  r  c                    t        | |||      S )a  Update the storage layout (i.e. n_batch & n_dense) of a BCOO matrix.

    In many cases this can be done without introducing undue storage overhead. However,
    increasing ``mat.n_batch`` or ``mat.n_dense`` will lead to very inefficient storage,
    with many explicitly-stored zeros, unless the new batch or dense dimensions have size
    0 or 1. In such cases, ``update_layout`` will raise a :class:`SparseEfficiencyError`.
    This can be silenced by specifying the ``on_inefficient`` argument.

    Args:
      n_batch : optional(int) the number of batch dimensions in the output matrix. If None,
        then n_batch = mat.n_batch.
      n_dense : optional(int) the number of dense dimensions in the output matrix. If None,
        then n_dense = mat.n_dense.
      on_inefficient : optional(string), one of ``['error', 'warn', None]``. Specify the
        behavior in case of an inefficient reconfiguration. This is defined as a reconfiguration
        where the size of the resulting representation is much larger than the size of the
        input representation.

    Returns:
      mat_out : BCOO array
        A BCOO array representing the same sparse array as the input, with the specified
        layout. ``mat_out.todense()`` will match ``mat.todense()`` up to appropriate precision.
    r  )r  )r   r6   r   r  s       r7   r  zBCOO.update_layout7
  s    2 dGWUcddr]   c                :    |rt        | |      S t        | |      S )aq  Return a copy of the array with duplicate indices summed.

    Additionally, this operation will result in explicit zero entries removed, and
    indices being sorted in lexicographic order.

    Because the size of the resulting representation depends on the values in the
    arrays, this operation is not compatible with JIT or other transforms. To use
    ``sum_duplicates`` in such cases, you may pass a value to `nse` to specify the
    desired size of the output representation.

    Args:
      nse : integer (optional), if specified, gives the number of specified elements in
        the output sparse representation; if it is larger than the number required, data
        will be padded with zeros and indices will be padded with out-of-bounds values.
        If it is smaller than the number required, data will be silently discarded.
      remove_zeros : bool (default=True). If True, remove explicit zeros from the data
        as part of summing duplicates. If False, then explicit zeros at unique indices
        will remain among the specified elements. Note: remove_zeros=True is incompatible
        with autodiff.
    r   )r   r   )r   ro   remove_zeross      r7   sum_duplicateszBCOO.sum_duplicatesR
  s!    * !$C00 3//r]   c                    t        |       S )z0Return a copy of the matrix with indices sorted.)r  r   s    r7   sort_indiceszBCOO.sort_indicesl
  s    T""r]   c                    t        |       S )z$Create a dense version of the array.)r   r   s    r7   todensezBCOO.todensep
  s    r]   c                    t        |t         j                        ddd   n|      }t         |      }t	         fd|D              }| j
                   j
                   j                  z    D cg c]  }| j
                  z
   }}t	        |      t	        t         j                              k(  r j                  }nd}t        |j                  |j                  f|| j                        S c c}w )z,Create a new array containing the transpose.Nri   r  c              3  <   K   | ]  }j                   |     y wrB   rC   )r4   rd   r   s     r7   r8   z!BCOO.transpose.<locals>.<genexpr>x
  s     0aDJJqM0rM  Frl   )r  rp   rK   rF  r   r6   ru   rI   r^   rU   rV   rJ   )r   r}  r  mat_Tshape_TrL  rW  	is_sorteds   `       r7   r_  zBCOO.transposet
  s    T\5+DbD1tLD4T2E0400G t||dmm/KLN t||# NK N[U5#788%%i iU]]+7(9L9LN NNs   3C3c                f    | j                   | j                  f| j                  j                         fS rB   )rU   rV   r   r  r   s    r7   tree_flattenzBCOO.tree_flatten
  s'    IIt||$djj&8&8&:::r]   c                    t         j                  |       }|\  |_        |_        |j	                         h dk7  rt        d|       |j                  j                  di | |S )N>   rD   rI   rJ   z&BCOO.tree_unflatten: invalid aux_data=r3   )object__new__rU   rV   keysr   __dict__update)r  aux_datachildrenobjs       r7   tree_unflattenzBCOO.tree_unflatten
  s\    
..
C$CHck}}GG@xkBCCCLL#(#Jr]   )r  tuple[Array, Array]rD   Sequence[int]rI   rL   rJ   rL   r   r^   )rw   r+   ro   
int | Noner   r-   r   r   r6   r   r   r^   )r   DTypeLike | Noner   r   r6   r   r   r^   )rD   r   rk   r  r   r-   r   r   r6   r   ro   r   r   r^   )r   r   r  r   r  r   rk   r  r   r-   r6   r   r   r   r   r^   )r6   r  r   r  r  strr   r^   )NT)ro   r  r  rL   r   r^   )r   r+   rB   )r}  Sequence[int] | Noner   r^   )!r   r   r   __doc__r   r   ro   rk   r6   ru   r   r   r  r  r  rI  r  r   classmethodr   r  r  r  r  r  r  r  r  r  r_  r  r  __classcell__)r  s   @r7   r^   r^   }	  s   0f 	+.	,45#
/
0%78'9:(CD'
 @ A%
9
:% ',E8#8=A8(.-* 6:UWU]U] OO+.O7;O O CG'(%!$%36%?C% % =A\caA$$(+$69$BF$ $ EI$+Aa+$!+$69+$HK+$TX+$ +$Z 48t*1e$'e6:e604#N ;  r]   c                .   ||S ||j                   k\  rt        d| d|j                    d      t         | |j                  |       | |j                  |      f|j
                  d | |j
                  |dz   d  z   |j                  |j                        S )NzCannot map in_axis= for BCOO array with n_batch=zK. in_axes for batched BCOO operations must correspond to a batch dimension.rf   rl   r6   r   r^   rU   rV   rD   rI   rJ   )contr   rz  r  s       r7   _bcoo_to_eltr'  
  s    	\J	S[[
*4&0Mckk] [a a b b	tCHHd#T#++t%<=IIet$syy';; //@R@R
T Tr]   c                4   ||S ||j                   kD  rt        d| d|j                    d      t         | ||j                  |       | ||j                  |      f|j
                  d | |fz   |j
                  |d  z   |j                  |j                        S )NzBCOO: cannot add out_axis=r$  zC. BCOO batch axes must be a contiguous block of leading dimensions.rl   r%  )r&  ri  eltr  s       r7   _bcoo_from_eltr*  
  s    	\J	CKK
1$7TUXU`U`Ta bY Y Z Z	tIsxx.YT0RSIIet$	|3cii6FF //@R@R
T Tr]   rB   )rw   r^   ro   r   r   r^   )rw   r^   ro   r  r   r^   )rU   r   rV   r   rD   r  r   r   )rV   r   rD   r  r   r   )rw   r^   r   r+   )rU   r+   rV   r+   rS   r   r   r+   )rw   r+   ro   r  r6   r   r   r   r   r-   r   r^   )rw   r+   ro   r   r6   r   r   r   r   r-   r   r  )r   r^   r!  r,   r  zbool | Noner   r^   )rV   r+   r!  r+   r   r+   )rw   r^   rI  r  r   r^   )
rU   r+   rV   r+   rI  r  rS   r   r   r  )rI  r  rS   r   )r  BCOO | Arrayr  r+  r  r'   r|  Noner}  r,  r   r+  )r  r+   r  r+   r  r+   r  r'   r}  r   r  r   r   r+   )r  r+   r  r+   r  r+   r  r'   r}  r   r  r   r   r+   )r  r   )
r	  r+   r
  r+   rV   r+   r  r'   r   r+   )r  r+   r  r+   r  r+   r  r+   r  r   r  r   r  r'   r}  r   r   r  )r  r   r  r   )rw   r^   r   r^   )
rU   r+   rV   r+   rS   r   ro   r  r   r  )
rw   r^   r6   r  r   r  r  z
str | Noner   r^   )rw   r^   rD   r   r  r  r   r^   )rU   r+   rV   r+   rS   r   rD   r   r  r  r   r  )r  zSequence[BCOO]r  r   r   r^   )rw   r^   r  r  r  r  r   r^   )r!  r^   r  r  r   r^   )
rw   r^   rJ  r  rK  r  r?  r  r   r^   )rw   r^   rJ  zSequence[Any]rc  r  r   r^   )rw   r^   r}  r  r   r^   )
rU   r+   rV   r+   rS   r   r}  r  r   tuple[Array, Array, Shape])r  r^   r  r^   r   r^   )r  r+   r  r+   r  r+   r  r+   r  r   r  r   r   r-  )r  r^   r  r+   r   r+   )
rU   r+   rV   r+   r  r+   rS   r   r   r+   )r  r^   rJ  r+   r  r(   rc  r   rJ   rL   r  rL   r+  zstr | GatherScatterMode | Noner   r^   r  )r  r^   r  r^   r  r  r   r^   )r   
__future__r   collections.abcr   r]  r   rp  rm   typingr   r   r   r  numpyr   r  r	   r
   r   jax.experimental.sparse._baser   jax.experimental.sparse.utilr   r   r   r   r   r   r   r   "jax.experimental.sparse._loweringsr   r   jax._src.interpretersr   	jax.numpyr=   jax._src.utilr   r   r   jax._srcr   r   r   r   r    r!   r"   r   jax._src.lax.laxr#   r$   r%   r&   r'   jax._src.lax.slicingr(   r)   jax._src.numpy.setopsr*   jax._src.typingr+   r,   r-   r.   float32float64	complex64
complex128r  r  r  r\   rx   r   r   r   r   r   	Primitiver   r   r   def_implr   def_abstract_evalr   r   r   r   defjvpprimitive_transposesprimitive_batchersregister_lowering	lower_funr   r   r   r   r   r  r	  r  r  r  primitive_jvpsr#  r  r   r9  r;  r>  r@  rE  rR  rF  rN  rY  r`  rd  rg  rr  rz  r  r{  r  r  r  r  "_bcoo_dot_general_default_loweringr  r  _bcoo_dot_general_gpu_loweringr  r  r  r  simple_implr  r  r  r  r  r  r%  r/  r4  r7  r<  r?  r  rY  rb  r\  ry  r~  r  r  r  r  r  r  r  _bcoo_sort_indices_hlor  r   rJ  r  r  r0  r  r  r  r  r  _bcoo_sum_duplicates_hlor  r  r  r  r/  r5  r>  r_  rv  r3  r  r~  r  r  r  r  r  r  r  r  r  register_pytree_node_classr^   r'  r*  register_vmappabler   r3   r]   r7   <module>rQ     sX   O " $     , ,   
    3  
 F &  6 6     $ * 4T T J ) 7 7 + 

BJJbmmL 
 7*12	JZ X V"  /	@UU  K K" !!- "-
9-8 
		.+T 2*A   '.I  N +   ~~t~~(0 1 "4>>"23 $(  !  59"#ciiII2;ILPI. =>a-0YY8!*8;N8*  2 ##] $]# 	Gr ':  " #,E  ( )0M  , -   '4*1 2  / OS >2 @D H  $ $L !!0 "0FPH* 
		.$ 1 2*A   '.I  N +   ~~t~~(0 1 "4>>"23 $(  !K&0!.08B0GZ0-$   ##b $bI " ':  " #,E  ( )0J  , -   '4*1 2 $T^^$67  (,48"&	)J(;)J $)J .2)J ,8	)JV8)<8.18 #-8 278 ,*=,/2,@J,OT,87?I87t %%:HR: &:* &4T^^U&4 "(77r "0"8 aBLa
aBLa
1BDN1Bf'EO' 
		
7?X Y.I  * +2N  . /   )+M N   ' (   6I   6I ,T^^,FG K,q3,3D $$q %q( --	 .	ZZF
 
		
$&E
)416Y  2 3:^  6 7   DNN1EJL &t~~&:; (,  %R$.R<FR[nR03R8KR,f\ #< #<J ''@! (@!D;$P 5S  0 1*A  & '   +^T^^t.5 6 %dnn%89 '+  $1  ) && '	+B  (d4  *@  % &3S  / 0   *,B C ''<= )-  &#&K
     7<!& AF+05:* ((
 )

+BB *4>>6  ,D  ' (5W  1 2   ,.F G
 <@W[4;D4'1D4@DD4P $(,0(&1>&CV&PD=P 59>01>0#'>0B48T: 04K8,K88<K8ZO8dB8 5"n8C&0C>HCMgC,<
S < (-+07;!=?#9=?"=? !%=? %)	=?
 5=? '+=?@ ,0dVZ23qTX59+/	%( 59	%(N4>* %%P9 P &PhTT   D#sL.$ Or]   