
    ukiB                    &   d dl mZ d dl mZmZmZm	Z
 ej                  Zej                  j                  e       d dlZd dlmZmZmZ ej0                   G d dej2                               Z ej6                  e       G d dej8                               Zddd	d
ej<                  fdZ ej6                  e       G d dej8                               Z ddd	d
ej<                  fdZ! ej6                  e       G d dej8                               Z"ddd	d
ej<                  fdZ# ej6                  e       G d dej8                               Z$ddddd
ej<                  fdZ% ej6                  e       G d dej8                               Z&ddd	d
e&fdZ' ej6                  e       G d dej8                               Z(ddd	d
ej<                  fdZ) ej6                  e       G d dej8                               Z*ddd	d
ej<                  fdZ+ ej6                  e       G d d ej8                               Z,ddd	d
ej<                  fd!Z- ej6                  e       G d" d#ej8                               Z.ddddd
ej<                  fd$Z/ ej6                  e       G d% d&ej8                               Z0ddd	d
ej<                  fd'Z1 ej6                  e       G d( d)ej8                               Z2ddd	d
ej<                  fd*Z3 ej6                  e       G d+ d,ej8                               Z4ddd	d
e4fd-Z5 ej6                  e       G d. d/ej8                               Z6ddddd
ej<                  fd0Z7 ej6                  e       G d1 d2ej8                               Z8ddddd
ej<                  fd3Z9 ej6                  e       G d4 d5ej8                               Z:ddddd6d
ej<                  fd7Z; ej6                  e       G d8 d9ej8                               Z<dddddddd:d
e<fd;Z= ej6                  e       G d< d=ej8                               Z>ddddd>d
e>fd?Z? ej6                  e       G d@ dAej8                               Z@ddddd
ej<                  fdBZA ej6                  e       G dC dDej8                               ZBddd	d
ej<                  fdEZC ej6                  e       G dF dGej8                               ZDddd	d
ej<                  fdHZE ej6                  e       G dI dJej8                               ZFddd	d
ej<                  fdKZG ej6                  e       G dL dMej8                               ZHddd	d
ej<                  fdNZI ej6                  e       G dO dPej8                               ZJddd	d
ej<                  fdQZK ej6                  e       G dR dSej8                               ZLddddd
ej<                  fdTZM ej6                  e       G dU dVej8                               ZNddd	d
ej<                  fdWZO ej6                  e       G dX dYej8                               ZPddddZd
ej<                  fd[ZQ ej6                  e       G d\ d]ej8                               ZRddd	d
eRfd^ZS ej6                  e       G d_ d`ej8                               ZTddddad
eTfdbZU ej6                  e       G dc ddej8                               ZVddd	d
ej<                  fdeZW ej6                  e       G df dgej8                               ZXdddddddhd
ej<                  fdiZY ej6                  e       G dj dkej8                               ZZddd	d
ej<                  fdlZ[ ej6                  e       G dm dnej8                               Z\ddd	d
ej<                  fdoZ] ej6                  e       G dp dqej8                               Z^ddd	d
ej<                  fdrZ_ ej6                  e       G ds dtej8                               Z`ddd	d
ej<                  fduZa ej6                  e       G dv dwej8                               Zbddd	d
ej<                  fdxZc ej6                  e       G dy dzej8                               Zdddd	d
edfd{Ze ej6                  e       G d| d}ej8                               Zfddd	d
ej<                  fd~Zg ej6                  e       G d dej8                               Zhddd	d
ej<                  fdZi ej6                  e       G d dej8                               Zjddd	d
ej<                  fdZk ej6                  e       G d dej8                               Zldddddd
ej<                  fdZm ej6                  e       G d dej8                               Znddd	d
ej<                  fdZo ej6                  e       G d dej8                               Zpddd	d
eej<                  ej                  epf   fdZr ej6                  e       G d dej8                               Zsddd	d
ej<                  fdZt ej6                  e       G d dej8                               Zuddddd
ej<                  fdZv ej6                  e       G d dej8                               Zwddd	d
ej<                  fdZx ej6                  e       G d dej8                               Zyddd	d
ej<                  fdZz ej6                  e       G d dej8                               Z{ddd	d
ej<                  fdZ| ej6                  e       G d dej8                               Z}ddddddd
ej<                  fdZ~ ej6                  e       G d dej8                               Zddd	d
ej<                  fdZ ej6                  e       G d dej8                               Zddddd
ej                  fdZ ej6                  e       G d dej8                               Zddddd
ej<                  fdZ ej6                  e       G d dej8                               Zddddd
ej<                  fdZ ej6                  e       G d dej8                               Zddddddd
efdZ ej6                  e       G d dej8                               Zddd	d
efdZ ej6                  e       G d dej8                               Zddd	d
ej<                  fdZ ej6                  e       G d dej8                               Zddd	d
efdZ ej6                  e       G d dej8                               Zdddddd
ej                  fdZ ej6                  e       G d dej8                               Zddd	d
ej<                  fdZ ej6                  e       G dÄ dej8                               Zddd	d
ej<                  fdńZ ej6                  e       G dƄ dej8                               ZdddddȜd
efdɄZ ej6                  e       G dʄ dej8                               Zddd	d
ej<                  fd̄Z ej6                  e       G d̈́ dej8                               Zddd	d
efdτZ ej6                  e       G dЄ dej8                               Zddddd
ej<                  fd҄Z ej6                  e       G dӄ dej8                               Zddd	d
eej<                  ej                  ef   fdՄZ ej6                  e       G dք dej8                               Zddd	d
efd؄Z ej6                  e       G dل dej8                               Zddd	d
efdۄZ ej6                  e       G d܄ dej8                               Zddd	d
ej<                  fdބZ ej6                  e       G d߄ dej8                               Zddd	d
ej<                  fdZ ej6                  e       G d dej8                               Zddd	d
ej<                  fdZ ej6                  e       G d dej8                               Zddddd
ej<                  fdZ ej6                  e       G d dej8                               Zddd	d
eej<                  ej                  ef   fdZ ej6                  e       G d dej8                               Zddddd
ej<                  fdZ ej6                  e       G d dej8                               Zddddd
ej<                  fdZ ej6                  e       G d dej8                               Zdddddd
efdZ ej6                  e       G d dej8                               Zdddddd
efdZ ej6                  e       G d dej8                               Zddddddd
efdZ ej6                  e       G d dej8                               Zddd	d
efdZ ej6                  e       G d  dej8                               Zddd	d
efdZ ej6                  e       G d dej8                               Zddd	d
ej<                  fdZ ej6                  e       G d dej8                               Zddd	d
efdZy(	      )_cext)equally_sized_accessorget_default_loc_contextget_op_results_or_valuessegmented_accessorN)SequenceUnionOptionalc                       e Zd ZdZy)_DialecttpuN)__name__
__module____qualname__DIALECT_NAMESPACE     X/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jaxlib/mosaic/python/_tpu_gen.pyr   r      s    r   r   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej"                  ej                     fd       Z xZS )AllReduceOpztpu.all_reducer   TNlocipc                   g }i }d }	|j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )NI64AttrcontextdimTPU_ReductionKindAttrkind
attributesresultsoperands
successorsregionsr   r   append_ods_get_default_loc_context
isinstance_ods_ir	AttributeAttrBuildercontainsgetsuper__init__OPERATION_NAME_ODS_REGIONS_ODS_OPERAND_SEGMENTS_ODS_RESULT_SEGMENTS)selfoutputinputr   r!   r   r   r%   r#   r'   _ods_contextr$   _ods_successors	__class__s                r   r2   zAllReduceOp.__init__   sG   HJGOOE/4LsG%%&$$Y/  i(lC u
 tW&&'$$%<=  56t\R v GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   returnc                 4    | j                   j                  d   S Nr   	operationr%   r7   s    r   r9   zAllReduceOp.input/       >>""1%%r   c                 4    | j                   j                  d   S Nr   rA   r#   rB   s    r   r   zAllReduceOp.dim3       >>$$U++r   valuec                 P    |t        d      || j                  j                  d<   y N4'None' not allowed as value for mandatory attributesr   
ValueErrorrA   r#   r7   rH   s     r   r   zAllReduceOp.dim7   '    }MNN',DNNe$r   c                 4    | j                   j                  d   S Nr!   rF   rB   s    r   r!   zAllReduceOp.kind=       >>$$V,,r   c                 P    |t        d      || j                  j                  d<   y NrK   r!   rL   rN   s     r   r!   zAllReduceOp.kindA   '    }MNN(-DNNf%r   c                 4    | j                   j                  d   S r?   rA   r$   rB   s    r   r8   zAllReduceOp.outputG       >>!!!$$r   )r   r   r   r3   r4   r2   builtinspropertyr,   Value
VectorTyper9   IntegerAttrr   setterr-   r!   OpResultr8   __classcell__r<   s   @r   r   r      s   #.,6:t l& &W]]7#5#56 & & ,7&& , , ::-w** - -
 -G%% - - ;;.)) . .
 %g&&w'9'9: % %r   r   r   r=   c                8    t        | |||||      j                  S )Nr8   r9   r   r!   r   r   )r   resultrc   s         r   
all_reducere   K   s    	F%StQS	T	[	[[r   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Z xZS )AllocaSemaphoreOpztpu.sem_allocr   Nr   c                    g }i }d }t        |      }g }|j                  |       d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y Nr"   r*   r)   r1   r2   r3   r4   r5   r6   r7   rd   r   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zAllocaSemaphoreOp.__init__T       HJG/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zAllocaSemaphoreOp.result^   rX   r   r   r   r   r3   r4   r2   rY   rZ   r,   r_   
MemRefTyperd   r`   ra   s   @r   rg   rg   N   sJ    ".,$(T l %g&&w'9'9: % %r   rg   c                2    t        | ||      j                  S Nrd   r   r   )rg   rd   rr   s      r   	sem_allocrs   b   s    	&cb	9	@	@@r   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )	AssumeLayoutOpztpu.assume_layoutr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   r)   r*   r1   r2   r3   r4   r5   r6   r7   rd   r9   r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zAssumeLayoutOp.__init__k       HJGOOE/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zAssumeLayoutOp.inputv   rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zAssumeLayoutOp.resultz   rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r9   r_   rd   r`   ra   s   @r   ru   ru   e   sd    &.,+/D 	l &W]] & & %g&& % %r   ru   c                4    t        | |||      j                  S Nrd   r9   r   r   )ru   rd   r~   s       r   assume_layoutr   ~   s    	vU	C	J	JJr   c                   ,    e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                  fd       Z xZS )AssumeMultipleOpaB  
  This operation is a hint to the compiler that the input `value` is guaranteed
  to be a multiple of `multiple`. This can be used to satisfy divisibility checks
  in some compiler passes.
  
  The result is the same as the input `value`.
  
  Example:
  
  ```mlir
  %val = tpu.assume_multiple %arg0, 16 : index
  ```
  ztpu.assume_multipler   Nr$   r   r   c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   ||d   j                  gdz  }d }
t        | )  | j                  | j                  | j                  | j                  ||||
|||       y )NI32Attrr   multipler      r"   r)   r*   r+   r,   r-   r.   r/   r0   typer1   r2   r3   r4   r5   r6   )r7   rH   r   r$   r   r   r%   r#   r'   r:   r;   r<   s              r   r2   zAssumeMultipleOp.__init__   s    HJGOOE/4Lx**+$$Y/ ' i(<H z 8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rH   zAssumeMultipleOp.value   rC   r   c                 4    | j                   j                  d   S )Nr   rF   rB   s    r   r   zAssumeMultipleOp.multiple       >>$$Z00r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r   rL   rN   s     r   r   zAssumeMultipleOp.multiple   '    }MNN,1DNNj)r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zAssumeMultipleOp.result   rX   r   )r   r   r   __doc__r3   r4   r2   rY   rZ   r,   r[   rH   r]   r   r^   r_   rd   r`   ra   s   @r   r   r      s     ).,154D l &W]] & & 1++ 1 1 ??2G// 2 2
 %g&& % %r   r   r   c                6    t        | ||||      j                  S )NrH   r   r$   r   r   )r   rd   r   s        r   assume_multipler      s    	'sWY	Z	a	aar   c                        e Zd ZdZdZdZddd fd
Zej                  de	j                  e	j                     fd       Z xZS )		BarrierOpzc
  Performs barrier synchronization across all SC vector subcores at the
  specified barrier id.
  ztpu.barrierr   Nr   c                    g }i }d }|j                  |       t        |      }g }d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rw   )r7   
barrier_idr   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zBarrierOp.__init__   s    HJGOOJ/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zBarrierOp.barrier_id   rC   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   	IndexTyper   r`   ra   s   @r   r   r      sO    
 !.,(, l &'--(9(9: & &r   r   c                    t        | ||      S )Nr   r   r   )r   r   s      r   barrierr      s    	jcb	99r   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )		BitcastOpztpu.bitcastr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   r7   r8   r9   r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zBitcastOp.__init__   ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zBitcastOp.input   rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zBitcastOp.output   rX   r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r_   r8   r`   ra   s   @r   r   r      sz     .,+/D 	l &W]]7#5#56 & & %g&&w'9'9: % %r   r   c                4    t        | |||      j                  S Nr8   r9   r   r   )r   rd   r   s       r   bitcastr      s    	&32	>	E	EEr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	BitcastVregOpztpu.bitcast_vregr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   r   s              r   r2   zBitcastVregOp.__init__   ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zBitcastVregOp.input  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zBitcastVregOp.output  rX   r   r   ra   s   @r   r   r      sz    %.,+/D 	l &W]]7#5#56 & & %g&&w'9'9: % %r   r   c                4    t        | |||      j                  S r   )r   rd   r   s       r   bitcast_vregr   	      	fEsr	B	I	IIr   c                   ^    e Zd ZdZdZdZddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                   e	j                     fd       Z xZS )BroadcastInSublanesOpa  
  For each sublane `i`, broadcasts the value in lane `lane + i` along the
  entire sublane. For packed type, imagine the data is compressed unpacked
  along sublane dimension, and the sublane count is multiplied by the packing
  factor.
  For example, for i16 with sublane count 8, `i` above is in [0, 8 * 2).
  If `lane + i` is not in [0, lane_count), then the value in sublane `i` is
  not defined (can be anything).
  ztpu.broadcast_in_sublanesr   Nr   c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y )Nr   r   laner"   r(   )r7   r8   sourcer   r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zBroadcastInSublanesOp.__init__  s    HJGOOF/4LtW&&'$$Y/  i(|D v GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zBroadcastInSublanesOp.source+  rC   r   c                 4    | j                   j                  d   S )Nr   rF   rB   s    r   r   zBroadcastInSublanesOp.lane/  rR   r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r   rL   rN   s     r   r   zBroadcastInSublanesOp.lane3  rU   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zBroadcastInSublanesOp.output9  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r]   r   r^   r_   r8   r`   ra   s   @r   r   r     s     /.,264 l &gmmG$6$67 & & -G'' - - ;;.++ . .
 %g&&w'9'9: % %r   r   c                6    t        | ||||      j                  S )Nr8   r   r   r   r   )r   rd   r   s        r   broadcast_in_sublanesr   =  s    	fV$CTV	W	^	^^r   c                   B    e Zd ZdZdZdddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  ej                     fd       Z xZS )ConcatenateOpztpu.concatenater   Nr   c                   g }i }d }|j                  t        |             t        |      }	t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||	      |d<   d }
t        | )  | j                  | j                  | j                  | j                  ||||
|||       y )Nr   r   	dimensionr"   extend_get_op_results_or_valuesr*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   )r7   sourcesr   r$   r   r   r%   r#   r'   r:   r;   r<   s              r   r2   zConcatenateOp.__init__F  s    HJGOO-g67/4Ly'++,$$Y/  ) i(LI { O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S Nr   r   lenrA   r%   r7   _ods_variadic_group_lengths     r   r   zConcatenateOp.sourcesS  >    !$T^^%<%<!=!AA!E>>""1Q)C%CDDr   c                 4    | j                   j                  d   S Nr   rF   rB   s    r   r   zConcatenateOp.dimensionX      >>$$[11r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   r   rL   rN   s     r   r   zConcatenateOp.dimension\  '    }MNN-2DNNk*r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zConcatenateOp.outputb  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   OpOperandListr   r]   r   r^   r_   r\   r8   r`   ra   s   @r   r   r   @  s    $.,48dt l Ew,, E E 2,, 2 2 3W00 3 3
 %g&&w'9'9: % %r   r   c                6    t        | ||||      j                  S )Nr   r   r$   r   r   )r   rd   r   s        r   concatenater   f  s    	w)WRUZ\	]	d	ddr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  dej                  fd	       Z xZS )
CreateMaskOpztpu.create_maskr   Nr   c                >   g }i }d }|j                  t        |             |j                  t        |             t        |      }	g }
|
j                  |       d }t        |   | j                  | j                  | j                  | j                  ||
|||||       y ri   
r   r   r*   r)   r1   r2   r3   r4   r5   r6   r7   r8   lowhighr   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zCreateMaskOp.__init__o  s    HJGOO-c23OO-d34/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S )Nr      _ods_equally_sized_accessorrA   r%   r7   startelements_per_groups      r   r   zCreateMaskOp.low{  E     ;DNN<S<SUVXY[\^_ `E>>""51C)CDDr   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S )Nr   r   r   r   r   s      r   r   zCreateMaskOp.high  r   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zCreateMaskOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r   r   r   r_   r8   r`   ra   s   @r   r   r   i  s    $.,/3 
l E7(( E E EG)) E E %g&& % %r   r   c                6    t        | ||||      j                  S Nr8   r   r   r   r   )r   rd   r   s        r   create_maskr     s    	V4SR	H	O	OOr   c                   h    e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  d	e	j                  fd
       Zej                  de	j                  fd       Zej                  d	e	j                  fd       Zej                  de	j                  fd       Z xZS )CreateSubelementMaskOpa  
  The "half-sublanes", "quarter-sublanes", etc. (unit is determined by
  the type of `output`) of the mask are masked in the range specified by
  `from` and `to`.
  
  - If `from <= to`, the range `[from, to)` is set and the rest is unset.
  - If `to <= from`, the range `[to, from)` is unset and the rest is set.
  
  All lanes are set identically.
  
  Example:
  
  ```mlir
  %msk = tpu.create_subelement_mask 3, 9 : vector<8x128x2xi1>
  ```
  
  This creates a mask `%msk` where, for all `lane`s, `%msk[*][lane][*]` is:
  
  ```
  [[0, 0], [0, 1], [1, 1], [1, 1], [1, 0], [0, 0], [0, 0], [0, 0]]
  ```
  
  It is currently only supported:
  - In TPU v4, for `num_subelems` of 1 and 2.
  - In TPU v5, for `num_subelems` of 1, 2, and 4.
  ztpu.create_subelement_maskr   Nr   c                f   g }i }d }t        |      }	t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||	      |d<   t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y )Nr   r   fromtor"   r*   r+   r,   r-   r.   r/   r0   r)   r1   r2   r3   r4   r5   r6   )r7   r8   from_r   r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zCreateSubelementMaskOp.__init__  s9   HJG/4Lug''($$Y/   i(E v
 r7$$%$$Y/  i(\B t GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S )Nr   rF   rB   s    r   r   zCreateSubelementMaskOp.from_  rR   r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r   rL   rN   s     r   r   zCreateSubelementMaskOp.from_  rU   r   c                 4    | j                   j                  d   S )Nr   rF   rB   s    r   r   zCreateSubelementMaskOp.to  s    >>$$T**r   c                 P    |t        d      || j                  j                  d<   y )NrK   r   rL   rN   s     r   r   zCreateSubelementMaskOp.to  s'    }MNN&+DNNd#r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zCreateSubelementMaskOp.output  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r]   r   r^   r   r_   r8   r`   ra   s   @r   r   r     s    6 0.,/3 l$ -W(( - - 	<<.,, . .
 +'%% + + 99,g)) , ,
 %g&& % %r   r   c                6    t        | ||||      j                  S )Nr8   r   r   r   r   )r   rd   r   s        r   create_subelement_maskr     s    	vUrsr	R	Y	YYr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Z xZS )DelayOpz	tpu.delayr   Nr   c                    g }i }d }|j                  |       t        |      }g }d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rw   )r7   nanosr   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zDelayOp.__init__  s    HJGOOE/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zDelayOp.nanos  rC   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   IntegerTyper   r`   ra   s   @r   r   r     sH    .,#'D l &W]]7#6#67 & &r   r   c                    t        | ||      S )Nr   r   r   )r   r   s      r   delayr    s    	u#"	--r   c                        e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Z xZS )
DeviceIdOpztpu.device_idr   Nr   c                    g }i }d }t        |      }d }t        	| 	  | j                  | j                  | j
                  | j                  |||||||       y ri   r*   r1   r2   r3   r4   r5   r6   )
r7   r$   r   r   r%   r#   r'   r:   r;   r<   s
            r   r2   zDeviceIdOp.__init__  s    HJG/4LO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zDeviceIdOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r_   r   rd   r`   ra   s   @r   r  r    sL    "., $$4 l %g&&w':':; % %r   r  c                 2    t        | ||      j                  S )Nr   )r  rd   r   s      r   	device_idr    s    	G	4	;	;;r   c                       e Zd ZdZdZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  fd
       Zej                   de	j                  fd       Zej                  de	j"                  e	j                     fd       Z xZS )DynamicGatherOpa5  
  Gathers elements from `source` using `indices`.
  
  The specified `dimensions` of `source` are collapsed together and indexed by
  `indices`.
  
  Given a shape `N0 x N1 x ...`,  the `output[i0, i1, ...]` is given by
  `collapsed_source[j0, j1, ..., indices[i0, i1, ...] mod M]` where
  - `collapsed_source` is the result of collapsing `dimensions` of `source`
    into a new trailing dimension of size `M`.
  - `jk` is the subsequence of `in` for `n` not in `dimensions`.
  
  When a single dimension is specified, this is similar to
  `np.take_along_axis`.
  ztpu.dynamic_gatherr   Nr   c                   g }i }d }	|j                  |       |j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y NDenseI32ArrayAttrr   
dimensionsr"   r(   )r7   r   indicesr  r$   r   r   r%   r#   r'   r:   r;   r<   s               r   r2   zDynamicGatherOp.__init__  s    HJGOOFOOG/4Lz7,,-$$%89 !+ 12:|T | O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zDynamicGatherOp.source*  rC   r   c                 4    | j                   j                  d   S Nr   r@   rB   s    r   r  zDynamicGatherOp.indices.  rC   r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zDynamicGatherOp.dimensions2      >>$$\22r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zDynamicGatherOp.dimensions6  '    }MNN.3DNNl+r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zDynamicGatherOp.output<  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r  r  r  r^   r_   r8   r`   ra   s   @r   r
  r
    s     (.,=AtPT l &gmmG$6$67 & & &w}}W%7%78 & & 3'33 3 3 4g77 4 4
 %g&&w'9'9: % %r   r
  c                8    t        | |||||      j                  S )Nr   r  r  r$   r   r   )r
  rd   r  s         r   dynamic_gatherr  @  s     	JX_ehmo	p	w	wwr   c                       e Zd ZdZdZddddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  fd	       Zej                   d
ej                  fd       Zej                  deej                     fd       Zej                   d
eej                     fd       Zej&                  d        Zej                  deej                     fd       Zej                   d
eej                     fd       Zej&                  d        Zej                  dej*                  ej                     fd       Z xZS )DynamicRotateOpztpu.dynamic_rotater   Nstridestride_dimensionr   r   c                z   g }	i }
d }|	j                  |       |	j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |
||	||||       y )NSI32Attrr   r   r   r!  r"   r(   )r7   rd   rH   amountr   r   r!  r   r   r%   r#   r'   r:   r$   r;   r<   s                  r   r2   zDynamicRotateOp.__init__I  s   HJGOOEOOF/4Ly'++,$$Z0  ) j))\J { 67,,-((4 4: 


!
!*
-fl
K &h/ ##W%6%67((4 HX 


!
!*
-.>
U &00B%C GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rH   zDynamicRotateOp.valuea  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r$  zDynamicRotateOp.amounte  rC   r   c                 4    | j                   j                  d   S r   rF   rB   s    r   r   zDynamicRotateOp.dimensioni  r   r   rH   c                 P    |t        d      || j                  j                  d<   y r   rL   rN   s     r   r   zDynamicRotateOp.dimensionm  r   r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr   rF   rB   s    r   r   zDynamicRotateOp.strides  ,    t~~000>>$$X..r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r*  rF   rN   s     r   r   zDynamicRotateOp.stridey  E    ,1dnn)	T^^..	.
..
#
#H
- 
/r   c                 2    | j                   j                  d= y r*  rF   rB   s    r   r   zDynamicRotateOp.stride      !!(+r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr!  rF   rB   s    r   r!  z DynamicRotateOp.stride_dimension  -    !:!::>>$$%788r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r1  rF   rN   s     r   r!  z DynamicRotateOp.stride_dimension  G    6;dnn 23	t~~88	8
..
#
#$6
7 
9r   c                 2    | j                   j                  d= y r1  rF   rB   s    r   r!  z DynamicRotateOp.stride_dimension      !!"45r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zDynamicRotateOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   rH   r   r$  r]   r   r^   	_Optionalr   deleterr!  r_   rd   r`   ra   s   @r   r  r  C  s   '.,AEX\bfko l0 &W]]7#5#56 & & &gmmG$7$78 & & 2,, 2 2 3W00 3 3
 /i 3 34 / /
 
==.)G$7$78 . . 
>>, , 9	'*=*= > 9 9
 8Ig.A.A$B 8 8 6 6 %g&&w'9'9: % %r   r  r  c          
      <    t        | |||||||      j                  S )Nrd   rH   r$  r   r   r!  r   r   )r  rd   r;  s           r   dynamic_rotater<    s=    	eFi`f  zJ  PS  XZ  
[  
b  
b  br   c                   Z    e Zd ZdZg dZdZdddddddd fd
Zej                  de	j                  e	j                     fd       Zej                  dee	j                  e	j                        fd	       Zej                  de	j                  e	j                     fd
       Zej                  de	j                  e	j                     fd       Zej                  dee	j                  e	j"                        fd       Zej                  dee	j                  e	j"                        fd       Zej                  de	j(                  fd       Zej,                  de	j(                  fd       Zej                  de	j.                  fd       Zej,                  de	j.                  fd       Z xZS )EnqueueDMAOpztpu.enqueue_dma)r   r   r   r   r   r   r   Nsource_semaphorer  core_idprioritystrict_orderingr   r   c                   g }i }d }|j                  |       |j                  |       |j                  |       |j                  |       |j                  |       |j                  |       t        |	      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   g }d }t        | %  | j                  | j                  | j                  | j                  ||||||	|
       y )Nr   r   rB  BoolAttrrC  r"   r(   )r7   r   targettarget_semaphorer@  r  rA  rB  rC  r   r   r%   r#   r'   r:   r$   r;   r<   s                    r   r2   zEnqueueDMAOp.__init__  s   HJGOOFOO$%OOFOO$%OOIOOG/4L8W../((3 8@ 


!
!)
,X|
L (
3 "?G$5$56((4 FU 


!
!*
-o|
T %//@$A GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S NoperandSegmentSizesr   _ods_segmented_accessorrA   r%   r#   r7   operand_ranges     r   r   zEnqueueDMAOp.source  ;    +		 	 		"	"#8	91>M r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S NrJ  r   r   rL  rA   r%   r#   r   rM  s     r   r@  zEnqueueDMAOp.source_semaphore  L    +		 	 		"	"#8	91>M  #=1A5=?4?r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S NrJ  r   r   rK  rM  s     r   rF  zEnqueueDMAOp.target  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S NrJ     r   rK  rM  s     r   rG  zEnqueueDMAOp.target_semaphore  rO  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S NrJ     r   rR  rM  s     r   r  zEnqueueDMAOp.device_id  rS  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S )NrJ     r   rR  rM  s     r   rA  zEnqueueDMAOp.core_id  rS  r   c                 4    | j                   j                  d   S )NrB  rF   rB   s    r   rB  zEnqueueDMAOp.priority  r   r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   rB  rL   rN   s     r   rB  zEnqueueDMAOp.priority  r   r   c                 4    | j                   j                  d   S NrC  rF   rB   s    r   rC  zEnqueueDMAOp.strict_ordering      >>$$%677r   c                 P    |t        d      || j                  j                  d<   y NrK   rC  rL   rN   s     r   rC  zEnqueueDMAOp.strict_ordering  (    }MNN38DNN/0r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   ro   r   r8  r@  rF  rG  r   r  rA  r]   rB  r^   rE  rC  r`   ra   s   @r   r>  r>    s   $.(,KO[_imx|  OS  Y]  bf l. gmmG$6$67   @	'--8J8J*K L @ @ gmmG$6$67   g.@.@ A   @7==1D1D#EF @ @ @yw/B/B!CD @ @ 1++ 1 1 ??2G// 2 2
 8w// 8 8 97#3#3 9 9r   r>  r?  c                ,    t        | |||||||||	
      S )N
r   rF  rG  r@  r  rA  rB  rC  r   r   )r>  rg  s
             r   enqueue_dmarh    sE    	VFEUhx  EN  X_  jr  DS  Y\  ac  
d  dr   c                   R    e Zd ZdZdZddddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  fd	       Zej                  dej                  ej                     fd
       Zej                  deej                  ej                         fd       Zej                  dej$                  fd       Zej(                  dej$                  fd       Z xZS )EnqueueIndirectDMAOpztpu.enqueue_indirect_dmar   Noffset_filteraddr   r   c                .   g }	i }
d }|	j                  |       |	j                  |       |	j                  |       |	j                  |       ||	j                  |       t        |      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   g }d }t        | %  | j                  | j                  | j                  | j                  |
||	||||       y NrE  r   rm  r"   r(   )r7   r   rF  offsets	semaphorerl  rm  r   r   r%   r#   r'   r:   r$   r;   r<   s                  r   r2   zEnqueueIndirectDMAOp.__init__  s!   HJGOOFOOFOOGOOI (//-"@/4L
3))*((4 .1 


!
!*
-c<
H #5) GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zEnqueueIndirectDMAOp.source  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   rF  zEnqueueIndirectDMAOp.target  rC   r   c                 4    | j                   j                  d   S Nr   r@   rB   s    r   rp  zEnqueueIndirectDMAOp.offsets  rC   r   c                 4    | j                   j                  d   S )NrX  r@   rB   s    r   rq  zEnqueueIndirectDMAOp.semaphore   rC   r   c                 |    t        | j                  j                        dk  rd S | j                  j                  d   S )Nr]  r[  r   rB   s    r   rl  z"EnqueueIndirectDMAOp.offset_filter$  3    t~~../!34S9P9PQR9SSr   c                 4    | j                   j                  d   S Nrm  rF   rB   s    r   rm  zEnqueueIndirectDMAOp.add(  rG   r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   rm  rL   rN   s     r   rm  zEnqueueIndirectDMAOp.add,  rO   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   r   rF  rp  rq  r8  r   rl  rE  rm  r^   r`   ra   s   @r   rj  rj    sX   -.,JNTX^bgk l$ &gmmG$6$67 & & &gmmG$6$67 & & &w}} & & &w'9'9: & & TYw}}W5H5H'IJ T T ,7## , , ::-w'' - -r   rj  rk  c          
      (    t        | |||||||      S )Nr   rF  rp  rq  rl  rm  r   r   )rj  r~  s           r   enqueue_indirect_dmar  2  s2    	VFGW`p}  DG  MP  UW  
X  Xr   c                        e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	EraseLayoutOpztpu.erase_memref_layoutr   Nr   c                    g }i }d }|j                  |       t        |      }d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rw   )r7   operandr$   r   r   r%   r#   r'   r:   r;   r<   s             r   r2   zEraseLayoutOp.__init__;  s    HJGOOG/4LO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zEraseLayoutOp.operandD  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zEraseLayoutOp.resultH  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   r  r_   rd   r`   ra   s   @r   r  r  5  s|    ,.,)-4D l &w}}W%7%78 & & %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S )Nr  r$   r   r   )r  rd   r  s       r   erase_memref_layoutr  L  s    	wSR	H	O	OOr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )	ExtFOpztpu.extfr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   )r7   outin_r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zExtFOp.__init__U  s    HJGOOC/4LGNN3O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  z
ExtFOp.in_`  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r  z
ExtFOp.outd  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r  r_   r  r`   ra   s   @r   r  r  O  sd    .,&*t 	l &7== & & %7## % %r   r  c                4    t        | |||      j                  S )Nr  r  r   r   )r  rd   r  s       r   extfr  h  s    	CScb	1	8	88r   c                   Z    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  ej                     fd       Z xZS )FPToSIOpz
tpu.fptosir   Nr   c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y NTPU_RoundingModeEnumr   rounding_moder"   r(   )r7   r8   r9   r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zFPToSIOp.__init__q  s    HJGOOE/4L}g//0$$%;< $1 45m\Z  GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zFPToSIOp.input  rC   r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zFPToSIOp.rounding_mode      >>$$_55r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zFPToSIOp.rounding_mode  '    }MNN16DNNo.r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zFPToSIOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r-   r  r^   r_   r8   r`   ra   s   @r   r  r  k  s    .,:>4 l &W]]7#5#56 & & 6W.. 6 6 7!2!2 7 7
 %g&&w'9'9: % %r   r  c                6    t        | ||||      j                  S )Nr8   r9   r  r   r   )r  rd   r  s        r   fptosir    s    	uMsWY	Z	a	aar   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej"                  ej                     fd       Z xZS )GatherOpz
tpu.gatherr   Nr   c                   g }i }d }	|j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )Nr  r   r  r   r   r"   r(   )r7   r8   r   r  r   r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zGatherOp.__init__  sG   HJGOOF/4Lw))*$$%89 % 127LQ y
 y'++,$$Y/  ) i(LI { GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zGatherOp.source  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zGatherOp.indices      >>$$Y//r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zGatherOp.indices  '    }MNN+0DNNi(r   c                 4    | j                   j                  d   S r   rF   rB   s    r   r   zGatherOp.dimension  r   r   c                 P    |t        d      || j                  j                  d<   y r   rL   rN   s     r   r   zGatherOp.dimension  r   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zGatherOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r  r  r^   r]   r   r_   r8   r`   ra   s   @r   r  r    s   .,@D l& &gmmG$6$67 & & 0w00 0 0 >>1744 1 1
 2,, 2 2 3W00 3 3
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr8   r   r  r   r   r   )r  rd   r  s         r   gatherr    s    	9Z]bd	e	l	llr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Z xZS )GetBarrierSemaphoreOpztpu.sem_barrierr   Nr   c                    g }i }d }t        |      }g }|j                  |       d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rj   )r7   rq  r   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zGetBarrierSemaphoreOp.__init__  s    HJG/4LGNN9O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rq  zGetBarrierSemaphoreOp.semaphore  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r_   ro   rq  r`   ra   s   @r   r  r    sJ    $.,'+ l %))'*<*<= % %r   r  c                2    t        | ||      j                  S )Nrq  r   r   )r  rd   r  s      r   sem_barrierr    s    		C	J	JJr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Z xZS )GetInternalScratchOpztpu.internal_scratchr   Nr   c                    g }i }d }t        |      }g }|j                  |       d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rj   rk   s             r   r2   zGetInternalScratchOp.__init__  rl   r   r=   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zGetInternalScratchOp.result  rX   r   rn   ra   s   @r   r  r    J    ).,$(T l %g&&w'9'9: % %r   r  c                2    t        | ||      j                  S rq   )r  rd   rr   s      r   internal_scratchr    s    	V	<	C	CCr   c                       e Zd ZdZdZdddd fd
Zej                  dej                  fd       Z
e
j                  dej                  fd	       Z
ej                  dej                  ej                     fd
       Z xZS )GetIterationBoundOpztpu.iteration_boundr   Nr   c                x   g }i }d }t        |      }t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |d<   d }	t        
| !  | j                  | j                  | j                  | j                  ||||	|||       y )Nr   r   r   r"   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   )r7   r   r$   r   r   r%   r#   r'   r:   r;   r<   s             r   r2   zGetIterationBoundOp.__init__  s    HJG/4LsG%%&$$Y/  i(lC u O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S rE   rF   rB   s    r   r   zGetIterationBoundOp.dim  rG   r   rH   c                 P    |t        d      || j                  j                  d<   y rJ   rL   rN   s     r   r   zGetIterationBoundOp.dim  rO   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zGetIterationBoundOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r]   r   r^   r_   r   rd   r`   ra   s   @r   r  r    s    (.,%)t 
l ,7&& , , ::-w** - -
 %g&&w':':; % %r   r  c                4    t        | |||      j                  S )Nr   r$   r   r   )r  rd   r  s       r   iteration_boundr    s    	g32	F	M	MMr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  d	e	j                  fd
       Zej                  de	j                  e	j                     fd       Z xZS )IotaOpa  
  Creates a vector that with values that start at 0 and increase along a
  dimension resulting from collapsing the given `dimensions` together in
  row-major order.
  
  Example:
  ```
  tpu.iota {dimensions = array<i32: 2, 0>} : vector<4x3x2xi16>
  ```
  This produces a vector with the following values:
  ```
  [[[0, 4], [0, 4], [0, 4]]
   [[1, 5], [1, 5], [1, 5]]
   [[2, 6], [2, 6], [2, 6]]
   [[3, 7], [3, 7], [3, 7]]]
  ```
  ztpu.iotar   Nr   c                   g }i }d }t        |      }t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |d<   g }	|	j                  |       d }
t        | %  | j                  | j                  | j                  | j                  ||	||
|||       y r  r   )r7   r8   r  r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zIotaOp.__init__6  s    HJG/4Lz7,,-$$%89 !+ 12:|T | GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zIotaOp.dimensionsD  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zIotaOp.dimensionsH  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zIotaOp.outputN  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r  r  r^   r_   r\   r8   r`   ra   s   @r   r  r    s    $ .,04 l 3'33 3 3 4g77 4 4
 %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S )Nr8   r  r   r   )r  rd   r  s       r   iotar  R  s    	v*#"	E	L	LLr   c                   0    e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                   de	j                  fd       Zej                  dee	j$                     fd       Zej                   dee	j$                     fd       Zej(                  d        Zej                  de	j*                  e	j,                     fd       Z xZS )LoadOpz
  Similar to `vector::LoadOp` but with `sublane_mask` and `sublane_stride`.
  When `indices` are negative, it means loading from negative offset
  of `base` address.
  ztpu.loadr   Nsublane_strider   r   c                   g }i }	d }
|j                  |       |j                  t        |             t        |      }t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   |dt	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |	||||
||       y NDenseBoolArrayAttrr   sublane_maskr   r  r"   r)   r   r   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   )r7   rd   baser  r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zLoadOp.__init__a  sb   HJGOODOO-g67/4L|W../$$%9: #/ 23L,W ~ !>7#4#45((3 DR 


!
!)
,^\
R $..>#? GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zLoadOp.baseu  rC   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S Nr   r   r   r   s     r   r  zLoadOp.indicesy  r   r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zLoadOp.sublane_mask~      >>$$^44r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zLoadOp.sublane_mask  '    }MNN05DNNn-r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr  rF   rB   s    r   r  zLoadOp.sublane_stride  -    t~~888>>$$%566r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r  rF   rN   s     r   r  zLoadOp.sublane_stride  G    49dnn 01	T^^66	6
..
#
#$4
5 
7r   c                 2    | j                   j                  d= y r  rF   rB   s    r   r  zLoadOp.sublane_stride      !!"23r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zLoadOp.result  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r  r   r  r  r  r^   r8  r]   r  r9  r_   r\   rd   r`   ra   s   @r   r  r  U  sd    .,LPVZ_c l( &GMM & & Ew,, E E 5G66 5 5 6 : : 6 6
 7i(;(;< 7 7
 6)G,?,?"@ 6 6 4 4 %g&&w'9'9: % %r   r  r  c          	      :    t        | ||||||      j                  S )Nrd   r  r  r  r  r   r   )r  rd   r  s          r   loadr    s2    	vD'esy|  BD  
E  
L  
L  Lr   c                   ~    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Z xZS )LogBufferOpztpu.log_bufferr   Nr   c                f   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y )NDenseI64ArrayAttrr   shapeStrAttrtagr"   r(   )r7   r9   r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zLogBufferOp.__init__  s;   HJGOOE/4Lug''($$%89 ! 125,O w
 sG%%&$$Y/  i(lC u GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zLogBufferOp.input  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zLogBufferOp.shape      >>$$W--r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zLogBufferOp.shape  '    }MNN).DNNg&r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zLogBufferOp.tag  rG   r   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zLogBufferOp.tag  rO   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   r9   r  r  r^   
StringAttrr  r`   ra   s   @r   r  r    s    #.,/3 l$ &W]]7#5#56 & & .W.. . . 	<</22 / /
 ,7%% , , ::-w)) - -r   r  c                "    t        | ||||      S )Nr9   r  r  r   r   )r  r  s        r   
log_bufferr    s    	53CB	GGr   c                   f    e Zd ZdZdZdddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Z xZS )LogOpztpu.logr   N	formattedr   r   c                |   g }i }d }|j                  t        |             t        |      }	t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||	      |d<   |dt        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||	      |d<   g }
d }t        | )  | j                  | j                  | j                  | j                  ||
|||||       y )Nr  r   r  rE  r  r"   r   )r7   inputsr  r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zLogOp.__init__  sF   HJGOO-f56/4LsG%%&$$Y/  i(lC u 9g//0((4 :C 


!
!*
-i
N )5 GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   r   s     r   r
  zLogOp.inputs  r   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  z	LogOp.tag  rG   r   rH   c                 P    |t        d      || j                  j                  d<   y r   rL   rN   s     r   r  z	LogOp.tag  rO   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zLogOp.formatted  r   r   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zLogOp.formatted  r   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r   r
  r  r  r^   rE  r  r`   ra   s   @r   r  r    s    .,/3$ l$ Eg++ E E ,7%% , , ::-w)) - -
 2)) 2 2 3W-- 3 3r   r  r  c                "    t        | ||||      S )Nr
  r  r  r   r   )r  r  s        r   logr    s    	f#	KKr   c                        e Zd ZdZdZdZddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Z xZS )

MaskCastOpaL  
  Cast a mask register into a different packing.
  
  If casting to a type with smaller packing, then values being packed together
  must be identical. For example, for 8x128x4xi1 -> 8x128x2xi1,
  input[i, j, 0] == input[i, j, 1] and input[i, j, 2] == input[i, j, 3] must
  hold for all i, j. Otherwise, the result is undefined.
  ztpu.mask_castr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   rx   s              r   r2   zMaskCastOp.__init__  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zMaskCastOp.input!  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zMaskCastOp.result%  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r_   rd   r`   ra   s   @r   r  r    s     #.,+/D 	l &W]]7#5#56 & & %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S r}   )r  rd   r~   s       r   	mask_castr  )  s    	6CB	?	F	FFr   c                       e Zd ZdZdZddddddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd	       Zej                  dej                  fd
       Zej                   dej                  fd       Zej                  dej                  fd       Zej                   dej                  fd       Zej                  deej&                     fd       Zej                   deej&                     fd       Zej*                  d        Zej                  deej&                     fd       Zej                   deej&                     fd       Zej*                  d        Zej                  dej.                  ej                     fd       Z xZS )MatmulOpz
tpu.matmulr   Ntranspose_lhstranspose_rhs	precisiondimension_numbersr   r   c                l   g }i }d }|j                  |       |j                  |       |j                  |       t        |	      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||||||	|
	       y )
NrE  r   r  r  TPU_ContractPrecisionEnumr  TPU_DotDimensionNumbersAttrr   r"   r(   )r7   rd   lhsrhsaccr  r  r  r   r   r   r%   r#   r'   r:   r$   r;   r<   s                    r   r2   zMatmulOp.__init__2  s)   HJGOOCOOCOOC/4L ='"3"34((4 BO 


!
!*
-m\
R #-_"=  ='"3"34((4 BO 


!
!*
-m\
R #-_"= 9g//0(()DE :C 


!
!"=
>yR^
_ )5 $$g&7&78(()FG J[ 


!
!"?
@AR\h
i '11D&E GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r$  zMatmulOp.lhsO  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r%  zMatmulOp.rhsS  rC   r   c                 4    | j                   j                  d   S ru  r@   rB   s    r   r&  zMatmulOp.accW  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zMatmulOp.transpose_lhs[  r  r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zMatmulOp.transpose_lhs_  r  r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zMatmulOp.transpose_rhse  r  r   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zMatmulOp.transpose_rhsi  r  r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr  rF   rB   s    r   r  zMatmulOp.precisiono  ,    $..333>>$$[11r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r/  rF   rN   s     r   r  zMatmulOp.precisionu  E    /4dnn,	11	1
..
#
#K
0 
2r   c                 2    | j                   j                  d= y r/  rF   rB   s    r   r  zMatmulOp.precision|      !!+.r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr   rF   rB   s    r   r   zMatmulOp.dimension_numbers  s-    $..";";;>>$$%899r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r6  rF   rN   s     r   r   zMatmulOp.dimension_numbers  sG    7<dnn 34	 9 9	9
..
#
#$7
8 
:r   c                 2    | j                   j                  d= y r6  rF   rB   s    r   r   zMatmulOp.dimension_numbers  s    !!"56r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zMatmulOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r$  r%  r&  rE  r  r^   r  r8  r-   r  r9  r   r_   rd   r`   ra   s   @r   r  r  ,  so   .,=AQUaey}  DH  MQ l: &7==!3!34 & & &7==!3!34 & & &7==!3!34 & & 6W-- 6 6 7!1!1 7 7
 6W-- 6 6 7!1!1 7 7
 27#4#45 2 2
 1Yw'8'89 1 1 / / :7+<+<!= : :
 9Yw/@/@%A 9 9 7 7 %g&&w'9'9: % %r   r  r  c                @    t        | |||||||||	
      j                  S )N
rd   r$  r%  r&  r  r  r  r   r   r   )r  rd   r;  s
             r   matmulr<    sF    	Scs-gt  AJ  ^o  ux  }  
@  
G  
G  Gr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	MemRefBitcastOpztpu.memref_bitcastr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   rx   s              r   r2   zMemRefBitcastOp.__init__  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zMemRefBitcastOp.input  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zMemRefBitcastOp.result  rX   r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   r9   r_   rd   r`   ra   s   @r   r>  r>    z    '.,+/D 	l &W]]7#5#56 & & %g&&w'9'9: % %r   r>  c                4    t        | |||      j                  S r}   )r>  rd   r~   s       r   memref_bitcastrE        	e	D	K	KKr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	MemRefReshapeOpztpu.memref_reshaper   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   rx   s              r   r2   zMemRefReshapeOp.__init__  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zMemRefReshapeOp.input  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zMemRefReshapeOp.result  rX   r   rB  ra   s   @r   rH  rH    rC  r   rH  c                4    t        | |||      j                  S r}   )rH  rd   r~   s       r   memref_reshaperM    rF  r   c                   b    e Zd ZdZg dZdZddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                  de	j                   e	j                     fd       Z xZS )MemRefSliceOpztpu.memref_slice)r   rP  r   Nr   c                `   g }i }d }	|j                  |       |j                  t        |             |j                  t        |             t        |      }
g }|j                  |       d }t        |   | j
                  | j                  | j                  | j                  |||||	||       y ri   	r)   r   r*   r1   r2   r3   r4   r5   r6   )r7   rd   mem_refbase_idxdynamic_sizesr   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zMemRefSliceOp.__init__  s    HJGOOGOO-h78OO-m<=/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   rS  zMemRefSliceOp.mem_ref  rO  r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S NrJ  r   rK  rM  s     r   rT  zMemRefSliceOp.base_idx  6    +		 	 		"	"#8	91>M r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S NrJ  r   rK  rM  s     r   rU  zMemRefSliceOp.dynamic_sizes  rY  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zMemRefSliceOp.result  rX   r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   ro   rS  r   rT  rU  r_   rd   r`   ra   s   @r   rO  rO    s    %.$,FJt l w}}W%7%78   --   W22   %g&&w'9'9: % %r   rO  c                8    t        | |||||      j                  S )Nrd   rS  rT  rU  r   r   )rO  rd   r^  s         r   memref_slicer_    s     	fgXeknsu	v	}	}}r   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	MemRefSqueezeOpztpu.memref_squeezer   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   rx   s              r   r2   zMemRefSqueezeOp.__init__  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zMemRefSqueezeOp.input  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zMemRefSqueezeOp.result  rX   r   rB  ra   s   @r   ra  ra    rC  r   ra  c                4    t        | |||      j                  S r}   )ra  rd   r~   s       r   memref_squeezerf    rF  r   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Z xZS )PRNGRandomBitsOpztpu.prng_random_bitsr   Nr   c                    g }i }d }t        |      }g }|j                  |       d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rj   )r7   r8   r   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zPRNGRandomBitsOp.__init__#  rl   r   r=   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zPRNGRandomBitsOp.output-  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r_   r\   r8   r`   ra   s   @r   rh  rh    r  r   rh  c                2    t        | ||      j                  S )Nr8   r   r   )rh  rd   rl  s      r   prng_random_bitsrm  1  s    	SR	8	?	??r   c                   l     e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
 xZS )PRNGSeed32Opztpu.prng_set_seed_32r   Nr   c                    g }i }d }|j                  t        |             t        |      }g }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y ri   	r   r   r*   r1   r2   r3   r4   r5   r6   )r7   seedsr   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zPRNGSeed32Op.__init__:  s    HJGOO-e45/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   r   s     r   rr  zPRNGSeed32Op.seedsD  r   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r   rr  r`   ra   s   @r   ro  ro  4  sB    ).,#'D l EW** E Er   ro  c                    t        | ||      S )Nrr  r   r   )ro  ru  s      r   prng_set_seed_32rv  I  s    	Esr	22r   c                   D    e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                  e	j                      fd       Z xZS )PackElementwiseOpa  
  Packs multiple `sources` elementwise into a single vector of a narrower `target_type`.
  
  The number of `sources` must equal the packing factor, which is the ratio of
  the element bitwidth of the `sources` to the element bitwidth of the
  `target_type`. Elements from the `sources` are interleaved and packed into
  each word of the `output`, ordered from lowest to highest bits,
  corresponding to their order in the `sources`.
  ztpu.pack_elementwiser   Nr   c                   g }i }d }|j                  t        |             t        |      }	t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | -  | j                  | j                  | j                  | j                  ||
|||||       y )NTypeAttrr   target_typer"   r   r   r*   r+   r,   r-   r.   r/   r0   r)   r1   r2   r3   r4   r5   r6   )r7   r8   r   r{  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zPackElementwiseOp.__init__\  s    HJGOO-g67/4L{G--.$$Z0 "- j)+|L } GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   r   s     r   r   zPackElementwiseOp.sourcesk  r   r   c                 4    | j                   j                  d   S )Nr{  rF   rB   s    r   r{  zPackElementwiseOp.target_typep      >>$$]33r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r{  rL   rN   s     r   r{  zPackElementwiseOp.target_typet  '    }MNN/4DNNm,r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zPackElementwiseOp.outputz  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r   r   rz  r{  r^   r_   r\   r8   r`   ra   s   @r   rx  rx  L  s     *.,:>4 l Ew,, E E 47++ 4 4 5w// 5 5
 %g&&w'9'9: % %r   rx  c                6    t        | ||||      j                  S )Nr8   r   r{  r   r   )rx  rd   r  s        r   pack_elementwiser  ~  s    	&'{X[`b	c	j	jjr   c                   6    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd	       Z xZS )

PackMaskOpztpu.pack_vmskr   Nr   c                   g }i }d }|j                  |       |j                  |       t        |      }	g }
|
j                  |       d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y ri   rw   r   s               r   r2   zPackMaskOp.__init__  s    HJGOOCOOD/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zPackMaskOp.low  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r   zPackMaskOp.high  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zPackMaskOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r   r_   r8   r`   ra   s   @r   r  r    s    ".,/3 
l &7==!3!34 & & &GMM'"4"45 & & %g&&w'9'9: % %r   r  c                6    t        | ||||      j                  S r   )r  rd   r   s        r   	pack_vmskr    s    	6s32	F	M	MMr   c                       e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej                   ej"                     fd       Z xZS )PackSubelementsOpztpu.pack_subelementsr   Nr   c                   g }i }d }	|j                  t        |             t        |      }
t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |||||	||       y )Nr  r   	positionsTPU_PackFormatEnumpack_formatr"   r|  )r7   r8   r   r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zPackSubelementsOp.__init__  sO   HJGOO-g67/4Ly'++,$$%89  ) 129lS {
 {G--.$$%9: "- 23KV } GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   r   s     r   r   zPackSubelementsOp.sources  r   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zPackSubelementsOp.positions  r   r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zPackSubelementsOp.positions  r   r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zPackSubelementsOp.pack_format  r  r   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zPackSubelementsOp.pack_format  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zPackSubelementsOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r   r   r  r  r^   r-   r  r_   r\   r8   r`   ra   s   @r   r  r    s   ).,EId l& Ew,, E E 222 2 2 3W66 3 3
 47,, 4 4 5w00 5 5
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr8   r   r  r  r   r   )r  rd   r  s         r   pack_subelementsr    s     	&'Y\gmpuw	x		r   c                   ^    e Zd ZdZdZddddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  ej                     fd       Z xZS )ReciprocalOpztpu.reciprocalr   Napproxr$   r   r   c                   g }i }d }|j                  |       t        |      }	|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   ||d   j                  gdz  }d }
t        | )  | j                  | j                  | j                  | j                  ||||
|||       y )NrE  r   r  r   r   r"   r   )r7   r9   r  r$   r   r   r%   r#   r'   r:   r;   r<   s              r   r2   zReciprocalOp.__init__  s   HJGOOE/4L67,,-((4 4: 


!
!*
-fl
K &h/ 8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zReciprocalOp.input  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zReciprocalOp.approx      >>$$X..r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zReciprocalOp.approx  '    }MNN*/DNNh'r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zReciprocalOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   rE  r  r^   r_   r8   r`   ra   s   @r   r  r    s    #.,&*Ddt l &W]]7#5#56 & & /g&& / / 
==0'** 0 0
 %g&&w'9'9: % %r   r  r  c                6    t        | ||||      j                  S )Nr9   r  r$   r   r   )r  rd   r  s        r   
reciprocalr    s    	E&'sr	R	Y	YYr   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej"                  ej                     fd       Z xZS )ReduceIndexOpztpu.reduce_indexr   Nr   c                   g }i }d }	|j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )Nr   r   axisr    r!   r"   r(   )r7   r8   r9   r  r!   r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zReduceIndexOp.__init__
  sG   HJGOOE/4LtW&&'$$Y/  i(|D v
 tW&&'$$%<=  56t\R v GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zReduceIndexOp.input  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zReduceIndexOp.axis!  rR   r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zReduceIndexOp.axis%  rU   r   c                 4    | j                   j                  d   S rQ   rF   rB   s    r   r!   zReduceIndexOp.kind+  rR   r   c                 P    |t        d      || j                  j                  d<   y rT   rL   rN   s     r   r!   zReduceIndexOp.kind/  rU   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zReduceIndexOp.output5  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r]   r  r^   r-   r!   r_   r8   r`   ra   s   @r   r  r    s   %.,7; l& &W]]7#5#56 & & -G'' - - ;;.++ . .
 -G%% - - ;;.)) . .
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr8   r9   r  r!   r   r   )r  rd   r  s         r   reduce_indexr  9  s    	fE4SUW	X	_	__r   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )	RegionOpz
tpu.regionr   TNr   c                    g }i }d }t        |      }g }|j                  |       d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   )r*   r   r1   r2   r3   r4   r5   r6   r7   results_r   r   r%   r#   r'   r:   r$   r;   r<   s             r   r2   zRegionOp.__init__B  s    HJG/4LGNN8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   rA   r$   r   s     r   r  zRegionOp.results_L  >    !$T^^%;%;!<q!@1!D>>!!!A(B$BCCr   c                      | j                   d   S r?   r'   rB   s    r   regionzRegionOp.regionQ      <<?r   )r   r   r   r3   r4   r2   rY   rZ   r,   OpResultListr  Regionr  r`   ra   s   @r   r  r  <  sg    .,&*t l D,, D D gnn  r   r  c                    t        | ||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )Nr  r   r   r   r   )r  r$   r   )r  r   r   opr$   s        r   r  r  U  sC    sr2"bjjGL1$SW9J71:SPRSr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	ReinterpretCastOpztpu.reinterpret_castr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   rx   s              r   r2   zReinterpretCastOp.__init___  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zReinterpretCastOp.inputj  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zReinterpretCastOp.resultn  rX   r   rB  ra   s   @r   r  r  Y  sz    ).,+/D 	l &W]]7#5#56 & & %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S r}   )r  rd   r~   s       r   reinterpret_castr  r  s    	&32	F	M	MMr   c                        e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	
RelayoutOpztpu.relayoutr   Nr   c                    g }i }d }|j                  |       t        |      }||d   j                  gdz  }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y )Nr   r   r"   )	r)   r*   r   r1   r2   r3   r4   r5   r6   )r7   r9   r$   r   r   r%   r#   r'   r:   r;   r<   s             r   r2   zRelayoutOp.__init__{  s    HJGOOE/4L8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zRelayoutOp.input  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zRelayoutOp.output  rX   r   r   ra   s   @r   r  r  u  s|    !.,'+$ l &W]]7#5#56 & & %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S )Nr9   r$   r   r   )r  rd   r  s       r   relayoutr    s    	%cb	A	H	HHr   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej                   ej                     fd       Z xZS )RepeatOpz
tpu.repeatr   Nr   c                   g }i }d }	|j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )Nr   r   r   timesr"   r(   )r7   r8   r   r   r  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zRepeatOp.__init__  sE   HJGOOF/4Ly'++,$$Y/  ) i(LI {
 ug''($$Y/ ! i(E w GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zRepeatOp.source  rC   r   c                 4    | j                   j                  d   S r   rF   rB   s    r   r   zRepeatOp.dimension  r   r   rH   c                 P    |t        d      || j                  j                  d<   y r   rL   rN   s     r   r   zRepeatOp.dimension  r   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zRepeatOp.times  r  r   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zRepeatOp.times  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zRepeatOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r]   r   r^   r  r_   r8   r`   ra   s   @r   r  r    s   .,>Bt l& &gmmG$6$67 & & 2,, 2 2 3W00 3 3
 .W(( . . 	<</,, / /
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr8   r   r   r  r   r   )r  rd   r  s         r   repeatr    s    	)5VY^`	a	h	hhr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )		ReshapeOpztpu.reshaper   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   )r7   rd   r   r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zReshapeOp.__init__  s    HJGOOF/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zReshapeOp.source  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zReshapeOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r_   rd   r`   ra   s   @r   r  r    sz     .,,0T 	l &gmmG$6$67 & & %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S )Nrd   r   r   r   )r  rd   r  s       r   reshaper    s    	&SR	@	G	GGr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  ej                     fd       Z xZS )	RollVectorsOpztpu.roll_vectorsr   Nr   c                
   g }i }d }|j                  t        |             t        |      }g }	|	j                  |       d }
t        |   | j                  | j                  | j                  | j                  ||	||
|||       y ri   r   r   s              r   r2   zRollVectorsOp.__init__  s    HJGOO-e45/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   r   s     r   r9   zRollVectorsOp.input  r   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zRollVectorsOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r   r9   r_   r\   r8   r`   ra   s   @r   r  r    st    %.,+/D 	l EW** E E %g&&w'9'9: % %r   r  c                4    t        | |||      j                  S r   )r  rd   r   s       r   roll_vectorsr    r   r   c                   8    e Zd ZdZdZdZdddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                  fd       Zej                  d
e	j                  fd       Zej                  dee	j                     fd       Zej                  d
ee	j                     fd       Zej&                  d        Zej                  dee	j                     fd       Zej                  d
ee	j                     fd       Zej&                  d        Zej                  de	j*                  e	j                     fd       Z xZS )RotateOpz
  Rotates the given vector by the given amount in the given dimension, i.e.,
  for a 2D vector of shape (m, n), rotating dim 0 by `amount` will shift a row
  at index `i` to index `(i + amount) % m`
  z
tpu.rotater   Nr   r!  r$   r   r   c                $   g }	i }
d }|	j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   ||	d   j                  gdz  }d }t        | )  | j                  | j                  | j                  | j                  |
||	||||	       y )
Nr#  r   r$  r   r   r!  r   r   r"   r   )r7   rH   r$  r   r   r!  r$   r   r   r%   r#   r'   r:   r;   r<   s                 r   r2   zRotateOp.__init__  s   HJGOOE/4Lvw(()$$Z0 # j)&,G x
 y'++,$$Z0  ) j))\J { 67,,-((4 4: 


!
!*
-fl
K &h/ ##W%6%67((4 HX 


!
!*
-.>
U &00B%C 8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rH   zRotateOp.value'  rC   r   c                 4    | j                   j                  d   S )Nr$  rF   rB   s    r   r$  zRotateOp.amount+  r  r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r$  rL   rN   s     r   r$  zRotateOp.amount/  r  r   c                 4    | j                   j                  d   S r   rF   rB   s    r   r   zRotateOp.dimension5  r   r   c                 P    |t        d      || j                  j                  d<   y r   rL   rN   s     r   r   zRotateOp.dimension9  r   r   c                 f    d| j                   j                  vry | j                   j                  d   S r*  rF   rB   s    r   r   zRotateOp.stride?  r+  r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r*  rF   rN   s     r   r   zRotateOp.strideE  r-  r   c                 2    | j                   j                  d= y r*  rF   rB   s    r   r   zRotateOp.strideL  r/  r   c                 f    d| j                   j                  vry | j                   j                  d   S r1  rF   rB   s    r   r!  zRotateOp.stride_dimensionP  r2  r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r1  rF   rN   s     r   r!  zRotateOp.stride_dimensionV  r4  r   c                 2    | j                   j                  d= y r1  rF   rB   s    r   r!  zRotateOp.stride_dimension]  r6  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zRotateOp.resulta  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   rH   r]   r$  r^   r   r8  r   r9  r!  r_   rd   r`   ra   s   @r   r  r    s     .,9=PT^bhlqu l4 &W]]7#5#56 & & /g)) / / 
==0'-- 0 0
 2,, 2 2 3W00 3 3
 /i 3 34 / /
 
==.)G$7$78 . . 
>>, , 9	'*=*= > 9 9
 8Ig.A.A$B 8 8 6 6 %g&&w'9'9: % %r   r  r  c          
      <    t        | |||||||      j                  S )NrH   r$  r   r   r!  r$   r   r   )r  rd   r  s           r   rotater	  e  s=    	f	&cs  ~E  KN  SU  
V  
]  
]  ]r   c                   &    e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Z xZS )SIToFPOpz
tpu.sitofpr   Nr   c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y r  r(   )r7   r8   r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zSIToFPOp.__init__n  s    HJGOOC/4L}g//0$$%;< $1 45m\Z  GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zSIToFPOp.in_}  rC   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zSIToFPOp.rounding_mode  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zSIToFPOp.rounding_mode  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zSIToFPOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r  r-   r  r^   r_   r8   r`   ra   s   @r   r  r  h  s    .,8< l &7== & & 6W.. 6 6 7!2!2 7 7
 %g&& % %r   r  c                6    t        | ||||      j                  S )Nr8   r  r  r   r   )r  rd   r  s        r   sitofpr    s    	S3SU	V	]	]]r   c                       e Zd ZdZdZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Zej                  de	j                  e	j                     fd       Z xZS )ScanCountOpa  
  ScanCountOp calculates the running duplicate occurrence count of the elements
  in the input vector, %values. The output vector, %counts, contains the running
  duplicate occurrence count for the corresponding element in
  the input vector, where the count is performed in ascending order of element
  indices. For example, if the elements of %values at indices 0, 5, and 7 had
  duplicate values, then the elements of %counts at indices 0, 5, and 7 would
  be 1, 2, and 3, respectively.
  
  A mask vector, %in_mask, specifies which of the elements in the input vector
  are eligible for counting. An element in %values that has its mask set to 0
  will always have a count of 1 in %counts, regardless of the position in the
  vector, or whether there were duplicates or not.
  ztpu.scan_countr   Nr   c                    g }i }d }|j                  |       |j                  |       t        |      }	d }
t        |   | j                  | j
                  | j                  | j                  ||||
|||       y ri   rw   )r7   in_maskvaluesr$   r   r   r%   r#   r'   r:   r;   r<   s              r   r2   zScanCountOp.__init__  s    HJGOOGOOF/4LO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zScanCountOp.in_mask  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r  zScanCountOp.values  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   out_maskzScanCountOp.out_mask  rX   r   c                 4    | j                   j                  d   S r  rW   rB   s    r   countszScanCountOp.counts  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r  r  r_   r  r  r`   ra   s   @r   r  r    s     $.,154D l &w}}W%7%78 & & &gmmG$6$67 & & %((););< % % %g&&w'9'9: % %r   r  c                6    t        | ||||      j                  S )Nr  r  r$   r   r   )r  r$   r   s        r   
scan_countr!    s    	WVW#RT	U	]	]]r   c                       e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  deej                  ej                        fd       Zej                  dej                  fd	       Zej                   d
ej                  fd       Zej                  dej"                  ej                     fd       Z xZS )ScanOpztpu.scanr   Nmaskr   r   c                   g }i }d }	|j                  |       ||j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )Nr    r   r!   r"   r(   )r7   r8   r9   r!   r%  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zScanOp.__init__  s   HJGOOE./4LtW&&'$$%<=  56t\R v GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zScanOp.input  rC   r   c                 |    t        | j                  j                        dk  rd S | j                  j                  d   S r  r   rB   s    r   r%  zScanOp.mask  rx  r   c                 4    | j                   j                  d   S rQ   rF   rB   s    r   r!   zScanOp.kind  rR   r   rH   c                 P    |t        d      || j                  j                  d<   y rT   rL   rN   s     r   r!   zScanOp.kind  rU   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zScanOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r8  r%  r-   r!   r^   r_   r8   r`   ra   s   @r   r#  r#    s    .,26DT l  &W]]7#5#56 & & TIgmmG,>,>?@ T T -G%% - - ;;.)) . .
 %g&&w'9'9: % %r   r#  r$  c                8    t        | |||||      j                  S )Nr8   r9   r!   r%  r   r   )r#  rd   r-  s         r   scanr.    s    	vUDcb	Q	X	XXr   c                        e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	SemaphoreReadOpztpu.sem_readr   Nr   c                    g }i }d }|j                  |       t        |      }d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y ri   rw   )r7   rq  r$   r   r   r%   r#   r'   r:   r;   r<   s             r   r2   zSemaphoreReadOp.__init__  s    HJGOOI/4LO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rq  zSemaphoreReadOp.semaphore	  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zSemaphoreReadOp.result	  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   rq  r_   r   rd   r`   ra   s   @r   r0  r0    s|    !.,+/Td l &w'9'9: & & %g&&w':':; % %r   r0  c                4    t        | |||      j                  S )Nrq  r$   r   r   )r0  rd   r5  s       r   sem_readr6  
	  s    	9g32	N	U	UUr   c                   T    e Zd ZdZg dZdZdddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  dee	j                  e	j                        fd
       Zej                  dee	j                  e	j                        fd       Zej                  dee	j$                     fd       Zej(                  dee	j$                     fd       Zej*                  d        Z xZS )SemaphoreSignalOpztpu.sem_signal)r   r   r   r   r   Nr  rA  	core_typer   r   c                   g }i }	d }
|j                  |       |j                  |       |j                  |       |j                  |       t        |      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |	d<   g }d }t        | %  | j                  | j                  | j                  | j                  |	||||
||       y )NTPU_CoreTypeEnumr   r:  r"   r(   )r7   rq  r$  r  rA  r:  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zSemaphoreSignalOp.__init__	  s   HJGOOIOOFOOIOOG/4L9g//0(();< :C 


!
!"4
5i
V )5 GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   rq  zSemaphoreSignalOp.semaphore&	  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rQ  rK  rM  s     r   r$  zSemaphoreSignalOp.amount-	  rO  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rU  rR  rM  s     r   r  zSemaphoreSignalOp.device_id4	  rS  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rW  rR  rM  s     r   rA  zSemaphoreSignalOp.core_id;	  rS  r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr:  rF   rB   s    r   r:  zSemaphoreSignalOp.core_typeB	  r0  r   rH   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y rB  rF   rN   s     r   r:  zSemaphoreSignalOp.core_typeH	  r2  r   c                 2    | j                   j                  d= y rB  rF   rB   s    r   r:  zSemaphoreSignalOp.core_typeO	  r4  r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   ro   rq  r   r$  r8  r  rA  r-   r:  r^   r9  r`   ra   s   @r   r8  r8  	  si   #.$,594SW]afj l" w'9'9:   gmmG$7$78   @7==1D1D#EF @ @ @yw/B/B!CD @ @ 27#4#45 2 2
 1Yw'8'89 1 1 / /r   r8  r9  c          	      &    t        | ||||||      S )Nrq  r$  r  rA  r:  r   r   )r8  rF  s          r   
sem_signalrG  S	  s+    	Yv\cox  B  GI  
J  Jr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )	SemaphoreWaitOpztpu.sem_waitr   Nr   c                    g }i }d }|j                  |       |j                  |       t        |      }g }	d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   )r7   rq  r$  r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zSemaphoreWaitOp.__init__\	  s    HJGOOIOOF/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rq  zSemaphoreWaitOp.semaphoreg	  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r$  zSemaphoreWaitOp.amountk	  rC   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   rq  r   r$  r`   ra   s   @r   rI  rI  V	  sx    !.,/3 	l &w'9'9: & & &gmmG$7$78 & &r   rI  c                     t        | |||      S )Nrq  r$  r   r   )rI  rN  s       r   sem_waitrO  o	  s    	9V	LLr   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  dej                  fd	       Zej                   d
ej                  fd       Zej                  dej"                  fd       Zej                   d
ej"                  fd       Zej                  dej&                  ej(                     fd       Z xZS )ShuffledLoadOpztpu.shuffled_loadr   Nr   c                   g }i }	d }
|j                  |       |j                  t        |             t        |      }t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |	||||
||       y Nr  r   r  r  sublane_offsetsr"   r  )r7   rd   r  r  r  rT  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zShuffledLoadOp.__init__x	  s\   HJGOODOO-g67/4L|W../$$%9: #/ 23L,W ~
  1 12$$%89 &5 12?LY  ! GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zShuffledLoadOp.base	  rC   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r  r   r   s     r   r  zShuffledLoadOp.indices	  r   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zShuffledLoadOp.sublane_mask	  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zShuffledLoadOp.sublane_mask	  r  r   c                 4    | j                   j                  d   S NrT  rF   rB   s    r   rT  zShuffledLoadOp.sublane_offsets	  rb  r   c                 P    |t        d      || j                  j                  d<   y NrK   rT  rL   rN   s     r   rT  zShuffledLoadOp.sublane_offsets	  re  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zShuffledLoadOp.result	  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   r  r   r  r  r  r^   r  rT  r_   r\   rd   r`   ra   s   @r   rQ  rQ  r	  sA   &.,RV[_ l( &GMM'"4"45 & & Ew,, E E 5G66 5 5 6 : : 6 6
 8w88 8 8 97#<#< 9 9
 %g&&w'9'9: % %r   rQ  c          	      :    t        | ||||||      j                  S )Nrd   r  r  r  rT  r   r   )rQ  rd   r_  s          r   shuffled_loadr`  	  s7    	vD'P\n}  DG  LN  
O  
V  
V  Vr   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  fd	       Zej                  dej                   fd
       Zej$                  dej                   fd       Zej                  dej&                  fd       Zej$                  dej&                  fd       Z xZS )ShuffledStoreOpztpu.shuffled_storer   Nr   c                   g }i }	d }
|j                  |       |j                  |       |j                  t        |             t        |      }t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   g }d }t        | -  | j                  | j                  | j                  | j                  |	||||
||       y rS  r  )r7   valueToStorer  r  r  rT  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zShuffledStoreOp.__init__	  s\   HJGOOL!OODOO-g67/4L|W../$$%9: #/ 23L,W ~
  1 12$$%89 &5 12?LY  ! GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rd  zShuffledStoreOp.valueToStore	  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r  zShuffledStoreOp.base	  rC   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S NrX  r   r   r   r   s     r   r  zShuffledStoreOp.indices	  r   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zShuffledStoreOp.sublane_mask	  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zShuffledStoreOp.sublane_mask	  r  r   c                 4    | j                   j                  d   S rZ  rF   rB   s    r   rT  zShuffledStoreOp.sublane_offsets	  rb  r   c                 P    |t        d      || j                  j                  d<   y r\  rL   rN   s     r   rT  zShuffledStoreOp.sublane_offsets	  re  r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   rd  ro   r  r   r  r  r  r^   r  rT  r`   ra   s   @r   rb  rb  	  s?   '.,X\ae l( &GMM'*<*<= & & &GMM'"4"45 & & Ew,, E E 5G66 5 5 6 : : 6 6
 8w88 8 8 97#<#< 9 9r   rb  c          	      &    t        | ||||||      S )N)rd  r  r  r  rT  r   r   )rb  )value_to_storer  r  r  rT  r   r   s          r   shuffled_storero  	  s/    	n4_k  ~M  SV  []  
^  ^r   c                       e Zd ZdZdZdZddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  dee	j                  e	j                        fd
       Zej                  de	j                   fd       Zej$                  de	j                   fd       Zej                  de	j&                  e	j                     fd       Zej                  de	j&                  e	j                     fd       Zej                  de	j&                  e	j                     fd       Z xZS )SortOpz
  tpu.sort performs a stable sort of key/value pairs in ascending or
  descending order based on keys. Masked-out keys and values are placed at the
  end of the output vectors. An output mask indicates which outputs
  correspond to the valid inputs.
  ztpu.sortr   Nr%  
descendingr   r   c                P   g }
i }d }|
j                  |       |
j                  |       ||
j                  |       t        |      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   g }|j                  |       |j                  |       |j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||
||||	       y )NrE  r   rs  r"   r(   )r7   output_masksorted_keyssorted_valueskeysr  r%  rs  r   r   r%   r#   r'   r:   r$   r;   r<   s                   r   r2   zSortOp.__init__	  s/   HJGOODOOF./4L:w001((4 <F 


!
!*
-j,
O  *,7 GNN;NN;NN=!O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rx  zSortOp.keys
  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r  zSortOp.values
  rC   r   c                 |    t        | j                  j                        dk  rd S | j                  j                  d   S )NrX  r   r   rB   s    r   r%  zSortOp.mask
  rx  r   c                 4    | j                   j                  d   S )Nrs  rF   rB   s    r   rs  zSortOp.descending
  r  r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   rs  rL   rN   s     r   rs  zSortOp.descending
  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   ru  zSortOp.output_mask$
  rX   r   c                 4    | j                   j                  d   S r  rW   rB   s    r   rv  zSortOp.sorted_keys(
  rX   r   c                 4    | j                   j                  d   S ru  rW   rB   s    r   rw  zSortOp.sorted_values,
  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   rx  r  r8  r%  rE  rs  r^   r_   ru  rv  rw  r`   ra   s   @r   rq  rq  	  s    .,TXeiosx| l& &GMM'"4"45 & & &gmmG$6$67 & & TIgmmG,>,>?@ T T 3'** 3 3 4g.. 4 4
 %7++G,>,>? % % %7++G,>,>? % % %W--g.@.@A % %r   rq  rr  c                >    t        | ||||||||	      j                  S )N	ru  rv  rw  rx  r  r%  rs  r   r   )rq  r$   r  s	            r   sortr  0
  sF    	K[P]dhqw  C  PZ  `c  hj  
k  
s  
s  sr   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  fd	       Zej                  d
ej                  fd       Zej                  dej                   ej                     fd       Z xZS )StochasticConvertElementwiseOpz"tpu.stochastic_convert_elementwiser   Nr   c                   g }i }d }	|j                  |       |j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )Nrz  r   dst_typer"   r(   )r7   r8   r9   randomr  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   z'StochasticConvertElementwiseOp.__init__9
  s    HJGOOEOOF/4Lx**+$$Z0 ' j)(LI z GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   z$StochasticConvertElementwiseOp.inputI
  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r  z%StochasticConvertElementwiseOp.randomM
  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  z'StochasticConvertElementwiseOp.dst_typeQ
  r   r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  z'StochasticConvertElementwiseOp.dst_typeU
  r   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   z%StochasticConvertElementwiseOp.output[
  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r  rz  r  r^   r_   r8   r`   ra   s   @r   r  r  3
  s    7.,=Ad l  &W]]7#5#56 & & &gmmG$6$67 & & 1(( 1 1 ??2G,, 2 2
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr8   r9   r  r  r   r   )r  rd   r  s         r   stochastic_convert_elementwiser  _
  s     	'vU6\djmrt	u	|	||r   c                   6    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd	       Z xZS )
StochasticConvertOpztpu.stochastic_convertr   Nr   c                   g }i }d }|j                  |       |j                  |       t        |      }	g }
|
j                  |       d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y ri   rw   )r7   r8   r9   r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zStochasticConvertOp.__init__h
  s    HJGOOEOOF/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zStochasticConvertOp.inputt
  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r  zStochasticConvertOp.randomx
  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zStochasticConvertOp.output|
  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r  r_   r8   r`   ra   s   @r   r  r  b
  s    +.,37D 
l &W]]7#5#56 & & &gmmG$6$67 & & %g&&w'9'9: % %r   r  c                6    t        | ||||      j                  S )Nr8   r9   r  r   r   )r  rd   r  s        r   stochastic_convertr  
  s    	F%CTV	W	^	^^r   c                   z    e Zd ZdZg dZdZddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                  dee	j                     fd       Zej                  de	j$                  fd       Zej(                  de	j$                  fd       Zej                  dee	j*                     fd       Zej(                  dee	j*                     fd       Zej.                  d        Z xZS )StoreOpz	tpu.storer   r   rP  r   r   Nr%  r  r   r   c                   g }	i }
d }|	j                  |       |	j                  |       |	j                  t        |             |	j                  |       t        |      }t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   |dt        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   g }d }t        | )  | j                  | j                  | j                  | j                  |
||	||||       y r  r)   r   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   )r7   rd  r  r  r  r%  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                  r   r2   zStoreOp.__init__
  sn   HJGOOL!OODOO-g67OOD/4L|W../$$%9: #/ 23L,W ~ !>7#4#45((3 DR 


!
!)
,^\
R $..>#? GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   rd  zStoreOp.valueToStore
  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rQ  rK  rM  s     r   r  zStoreOp.base
  rO  r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S r[  rK  rM  s     r   r  zStoreOp.indices
  rY  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rW  rR  rM  s     r   r%  zStoreOp.mask
  rS  r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zStoreOp.sublane_mask
  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zStoreOp.sublane_mask
  r  r   c                 f    d| j                   j                  vry | j                   j                  d   S r  rF   rB   s    r   r  zStoreOp.sublane_stride
  r  r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r  rF   rN   s     r   r  zStoreOp.sublane_stride
  r  r   c                 2    | j                   j                  d= y r  rF   rB   s    r   r  zStoreOp.sublane_stride
  r  r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   r\   rd  r  r   r  r8  r%  r  r  r^   r]   r  r9  r`   ra   s   @r   r  r  
  s   .%,HL]agkpt l* GMM'*<*<=   GMM   w,,   @Igmm, @ @ 5G66 5 5 6 : : 6 6
 7i(;(;< 7 7
 6)G,?,?"@ 6 6 4 4r   r  r  c          
      (    t        | |||||||      S )N)rd  r  r  r  r%  r  r   r   )r  )rn  r  r  r  r%  r  r   r   s           r   storer  
  s2    	n4Wcjn  @N  TW  \^  
_  _r   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  dej                  fd	       Zej                   d
ej                  fd       Zej                  dej"                  ej$                     fd       Z xZS )StridedLoadOpztpu.strided_loadr   Nr   c                   g }i }d }	|j                  |       |j                  t        |             t        |      }
t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||
      |d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |||||	||       y Nr  r   stridesr"   r  )r7   rd   r  r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zStridedLoadOp.__init__
  s   HJGOODOO-g67/4Lw))*$$%89 % 127LQ y GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zStridedLoadOp.base
  rC   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r  r   r   s     r   r  zStridedLoadOp.indices
  r   r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zStridedLoadOp.strides
  r  r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zStridedLoadOp.strides
  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zStridedLoadOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   r  r   r  r  r  r^   r_   r\   rd   r`   ra   s   @r   r  r  
  s    %.,<@T l  &GMM'"4"45 & & Ew,, E E 0w00 0 0 >>1744 1 1
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nrd   r  r  r  r   r   )r  rd   r  s         r   strided_loadr    s    	f4'WZ_a	b	i	iir   c                       e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  fd	       Zej                  dej                   fd
       Zej$                  dej                   fd       Z xZS )StridedStoreOpztpu.strided_storer   Nr   c                   g }i }d }	|j                  |       |j                  |       |j                  t        |             t        |      }
t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||
      |d<   g }d }t        | -  | j                  | j                  | j                  | j                  |||||	||       y r  r  )r7   rd  r  r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zStridedStoreOp.__init__  s   HJGOOL!OODOO-g67/4Lw))*$$%89 % 127LQ y GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rd  zStridedStoreOp.valueToStore   rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r  zStridedStoreOp.base$  rC   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rh  r   r   s     r   r  zStridedStoreOp.indices(  r   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zStridedStoreOp.strides-  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zStridedStoreOp.strides1  r  r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   rd  ro   r  r   r  r  r  r^   r`   ra   s   @r   r  r  
  s    &.,BF4 l  &GMM'*<*<= & & &GMM'"4"45 & & Ew,, E E 0w00 0 0 >>1744 1 1r   r  c                $    t        | |||||      S )N)rd  r  r  r  r   r   )r  )rn  r  r  r  r   r   s         r   strided_storer  7  s    	^$Y`finp	qqr   c                       e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  fd	       Zej                  d
ej                  fd       Zej                  dej                   ej                     fd       Z xZS )SublaneShuffleOpztpu.sublane_shuffler   Nr   c                   g }i }d }	|j                  |       |j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   ||d   j                  gdz  }d }t        | )  | j                  | j                  | j                  | j                  |||||	||       y )Nr  r   patternr   r   r"   r   )r7   r$  r%  r  r$   r   r   r%   r#   r'   r:   r;   r<   s               r   r2   zSublaneShuffleOp.__init__@  s   HJGOOCOOC/4Lw))*$$%89 % 127LQ y 8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r$  zSublaneShuffleOp.lhsO  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   r%  zSublaneShuffleOp.rhsS  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zSublaneShuffleOp.patternW  r  r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zSublaneShuffleOp.pattern[  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zSublaneShuffleOp.resulta  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r$  r%  r  r  r^   r_   rd   r`   ra   s   @r   r  r  :  s    (.,37Td l &7==!3!34 & & &7==!3!34 & & 0w00 0 0 >>1744 1 1
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr$  r%  r  r$   r   r   )r  rd   r  s         r   sublane_shuffler  e  s    	csGWRUZ\	]	d	ddr   c                       e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
e
j                  dej                  fd	       Z
ej                  dej                  fd
       Zej                  dej                  fd       Zej                  dej                  fd       Zej                  dej                   fd       Z xZS )TraceOpz	tpu.tracer  Nr   c                f   g }i }d }t        |      }	t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||	      |d<   t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y Nr  r   messager   levelr"   )r*   r+   r,   r-   r.   r/   r0   r   r1   r2   r3   r4   r5   r6   )r7   r  r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zTraceOp.__init__n  s9   HJG/4Lw))*$$Y/ % i(,G y
 ug''($$Y/ ! i(E w GNN8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zTraceOp.message  r  r   rH   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zTraceOp.message  r  r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zTraceOp.level  r  r   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zTraceOp.level  r  r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r  r   s     r   r  zTraceOp.results_  r  r   c                      | j                   d   S r?   r  rB   s    r   r  zTraceOp.region  r  r   )r   r   r   r3   r4   r2   rY   rZ   r,   r  r  r^   r]   r  r  r  r  r  r`   ra   s   @r   r  r  h  s    .,6:t l$ 0w)) 0 0 >>17-- 1 1
 .W(( . . 	<</,, / /
 D,, D D gnn  r   r  c                    t        | ||||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )N)r  r  r  r   r   r   r   )r  r$   r   )r  r  r  r   r   r  r$   s          r   tracer    sK    'CBO"[][e[eQXL1$SW9J71:SPRSr   c                   &    e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
e
j                  dej                  fd	       Z
ej                  dej                  fd
       Zej                  dej                  fd       Z xZS )TraceStartOpztpu.trace_startr   Nr   c                D   g }i }d }t        |      }t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |d<   t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |d<   g }	d }
t        | !  | j                  | j                  | j                  | j                  ||	||
|||       y r  r  )r7   r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zTraceStartOp.__init__  s-   HJG/4Lw))*$$Y/ % i(,G y
 ug''($$Y/ ! i(E w GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zTraceStartOp.message  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zTraceStartOp.message  r  r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zTraceStartOp.level  r  r   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zTraceStartOp.level  r  r   )r   r   r   r3   r4   r2   rY   rZ   r,   r  r  r^   r]   r  r`   ra   s   @r   r  r    s    $.,,0T l" 0w)) 0 0 >>17-- 1 1
 .W(( . . 	<</,, / /r   r  c                     t        | |||      S )Nr  r  r   r   )r  r  s       r   trace_startr    s    	gU	CCr   c                   .     e Zd ZdZdZddd fd
Z xZS )TraceStopOpztpu.trace_stopr   Nr   c                    g }i }d }t        |      }g }d }t        	| 	  | j                  | j                  | j
                  | j                  |||||||       y ri   r  )
r7   r   r   r%   r#   r'   r:   r$   r;   r<   s
            r   r2   zTraceStopOp.__init__  s    HJG/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   )r   r   r   r3   r4   r2   r`   ra   s   @r   r  r    s    #., T l lr   r  c                     t        | |      S )Nr   )r  r   s     r   
trace_stopr    s    		$$r   c                   Z    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  ej                     fd       Z xZS )TransposeOpztpu.transposer   Nr   c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y )Nr  r   permutationr"   r(   )r7   rd   vectorr  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zTransposeOp.__init__  s    HJGOOF/4L{G--.$$%89 "- 12;U } GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zTransposeOp.vector  rC   r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  zTransposeOp.permutation  r  r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  zTransposeOp.permutation  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zTransposeOp.result  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r  r  r  r^   r_   rd   r`   ra   s   @r   r  r    s    ".,9=$ l &gmmG$6$67 & & 4744 4 4 5w88 5 5
 %g&&w'9'9: % %r   r  c                6    t        | ||||      j                  S )Nrd   r  r  r   r   )r  rd   r  s        r   	transposer    s    	F6{PSXZ	[	b	bbr   c                   &    e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Z xZS )TruncFOpz
tpu.truncfr   Nr   c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y r  r(   )r7   r  r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zTruncFOp.__init__  s    HJGOOC/4L}g//0$$%;< $1 45m\Z  GNN3O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r  zTruncFOp.in_   rC   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zTruncFOp.rounding_mode$  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zTruncFOp.rounding_mode(  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r  zTruncFOp.out.  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r  r-   r  r^   r_   r  r`   ra   s   @r   r  r    s    .,59d l &7== & & 6W.. 6 6 7!2!2 7 7
 %7## % %r   r  c                6    t        | ||||      j                  S )Nr  r  r  r   r   )r  rd   r  s        r   truncfr  2  s    	cs-SR	P	W	WWr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                   fd       Zej                  d
e	j                   fd       Zej                  de	j$                  e	j                     fd       Z xZS )UnpackElementwiseOpa1  
  Unpacks a single vector from `source`, which contains multiple `source_type`
  vectors packed elementwise.
  
  The `index` selects which packed value to extract from each word of `source`.
  An `index` of 0 corresponds to the lowest bits. The extracted values are
  cast to the output element type.
  ztpu.unpack_elementwiser   Nr   c                   g }i }d }	|j                  |       t        |      }
t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||
      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||	||       y )Nrz  r   source_typer   indexr"   r(   )r7   r8   r   r
  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zUnpackElementwiseOp.__init__D  sE   HJGOOF/4L{G--.$$Z0 "- j)+|L }
 ug''($$Y/ ! i(E w GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zUnpackElementwiseOp.sourceW  rC   r   c                 4    | j                   j                  d   S )Nr
  rF   rB   s    r   r
  zUnpackElementwiseOp.source_type[  r  r   rH   c                 P    |t        d      || j                  j                  d<   y )NrK   r
  rL   rN   s     r   r
  zUnpackElementwiseOp.source_type_  r  r   c                 4    | j                   j                  d   S Nr  rF   rB   s    r   r  zUnpackElementwiseOp.indexe  r  r   c                 P    |t        d      || j                  j                  d<   y NrK   r  rL   rN   s     r   r  zUnpackElementwiseOp.indexi  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zUnpackElementwiseOp.outputo  rX   r   )r   r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   rz  r
  r^   r]   r  r_   r8   r`   ra   s   @r   r  r  5  s    ,.,@D l& &gmmG$6$67 & & 47++ 4 4 5w// 5 5
 .W(( . . 	<</,, / /
 %g&&w'9'9: % %r   r  c                8    t        | |||||      j                  S )Nr8   r   r
  r  r   r   )r  rd   r  s         r   unpack_elementwiser  s  s     	F6{Z_ehmo	p	w	wwr   c                   T    e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej"                  fd       Zej                  d	ej"                  fd       Zej                  dej&                  ej                     fd       Z xZS )UnpackSubelementsOpztpu.unpack_subelementsr   Nsign_extendedr   r   c                T   g }i }	d }
|j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |	d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |	d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |	d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |	||||
||       y )	Nr   r   r  r  r  rE  r  r"   r(   )r7   r8   r   r  r  r  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zUnpackSubelementsOp.__init__|  s   HJGOOF/4Lug''($$Y/ ! i(E w
 {G--.$$%9: "- 23KV }  ='"3"34((4 BO 


!
!*
-m\
R #-_"= GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r   zUnpackSubelementsOp.source  rC   r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zUnpackSubelementsOp.index  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zUnpackSubelementsOp.index  r  r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zUnpackSubelementsOp.pack_format  r  r   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zUnpackSubelementsOp.pack_format  r  r   c                 4    | j                   j                  d   S )Nr  rF   rB   s    r   r  z!UnpackSubelementsOp.sign_extended  r  r   c                 P    |t        d      || j                  j                  d<   y )NrK   r  rL   rN   s     r   r  z!UnpackSubelementsOp.sign_extended  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zUnpackSubelementsOp.output  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r   r]   r  r^   r-   r  rE  r  r_   r8   r`   ra   s   @r   r  r  v  sf   +.,JNTX]a l. &gmmG$6$67 & & .W(( . . 	<</,, / /
 47,, 4 4 5w00 5 5
 6W-- 6 6 7!1!1 7 7
 %g&&w'9'9: % %r   r  r  c          	      :    t        | ||||||      j                  S )Nr8   r   r  r  r  r   r   )r  rd   r%  s          r   unpack_subelementsr&    s7    	F6T_o|  CF  KM  
N  
U  
U  Ur   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Z xZS )	UnrollVectorsOpztpu.unroll_vectorsr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j
                  | j                  | j                  | j                  ||	||
|||       y ri   )	r)   r*   r   r1   r2   r3   r4   r5   r6   r   s              r   r2   zUnrollVectorsOp.__init__  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zUnrollVectorsOp.input  rC   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r  r   s     r   r8   zUnrollVectorsOp.output  r  r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r\   r9   r  r8   r`   ra   s   @r   r(  r(    sr    '.,+/D 	l &W]]7#5#56 & & Dg** D Dr   r(  c                    t        | |||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )Nr   r   r   )r(  r$   r   )r8   r9   r   r   r  r$   s         r   unroll_vectorsr-    sE    fEsrB"bjjGL1$SW9J71:SPRSr   c                       e Zd ZdZg dZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  dee	j                  e	j                         fd
       Zej                  de	j$                  e	j                      fd       Z xZS )VectorLoadIdxOpztpu.vector_load_idxr   rP  r   r   Nr$  c                N   g }i }d }	|j                  |       |j                  t        |             |j                  |       t        |      }
g }|j                  |       d }t        |   | j
                  | j                  | j                  | j                  |||||	||       y ri   rR  )r7   rH   r  r  r%  r   r   r%   r#   r'   r:   r$   r;   r<   s                r   r2   zVectorLoadIdxOp.__init__  s    HJGOODOO-g67OOD/4LGNN5O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   r  zVectorLoadIdxOp.base  rO  r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S rX  rK  rM  s     r   r  zVectorLoadIdxOp.indices  rY  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rU  rR  rM  s     r   r%  zVectorLoadIdxOp.mask  rS  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rH   zVectorLoadIdxOp.value  rX   r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   ro   r  r   r  r8  r\   r%  r_   rH   r`   ra   s   @r   r/  r/    s    (.#,37Td l GMM'"4"45   w,,   @IgmmG,>,>?@ @ @ %W%%g&8&89 % %r   r/  c                8    t        | |||||      j                  S )NrH   r  r  r%  r   r   )r/  rd   r7  s         r   vector_load_idxr8    s    	u4tQTY[	\	c	ccr   c                        e Zd ZdZg dZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  dee	j                  e	j                         fd
       Zej                  de	j$                  fd       Zej(                  de	j$                  fd       Zej                  de	j*                  e	j                      fd       Z xZS )VectorLoadOpztpu.vector_loadr0  r   Nr$  c                   g }i }	d }
|j                  |       |j                  t        |             |j                  |       t        |      }t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |	d<   g }|j                  |       d }t        | )  | j                  | j                  | j                  | j                  |	||||
||       y r  r  )r7   rd   r  r  r  r%  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zVectorLoadOp.__init__  s   HJGOODOO-g67OOD/4Lw))*$$%89 % 127LQ y GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   r  zVectorLoadOp.base$  rO  r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S rX  rK  rM  s     r   r  zVectorLoadOp.indices+  rY  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rU  rR  rM  s     r   r%  zVectorLoadOp.mask2  rS  r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zVectorLoadOp.strides9  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zVectorLoadOp.strides=  r  r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   rd   zVectorLoadOp.resultC  rX   r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   ro   r  r   r  r8  r\   r%  r  r  r^   r_   rd   r`   ra   s   @r   r:  r:    s.   $.#,=AtPT l" GMM'"4"45   w,,   @IgmmG,>,>?@ @ @ 0w00 0 0 >>1744 1 1
 %g&&w'9'9: % %r   r:  c          	      :    t        | ||||||      j                  S )Nrd   r  r  r  r%  r   r   )r:  rd   rC  s          r   vector_loadrD  G  s"    	V$W[adik	l	s	ssr   c                       e Zd ZdZg dZdZddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  fd
       Zej                  dee	j                  e	j                        fd       Zej                  de	j&                  fd       Zej*                  de	j&                  fd       Z xZS )VectorStoreIdxOpztpu.vector_store_idxr  r   Nr%  rm  r   r   c                   g }i }	d }
|j                  |       |j                  |       |j                  t        |             |j                  |       t        |      }|dt        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |	d<   g }d }t        | )  | j                  | j                  | j                  | j                  |	||||
||       y ro  r  )r7   rd  r  r  r%  rm  r   r   r%   r#   r'   r:   r$   r;   r<   s                 r   r2   zVectorStoreIdxOp.__init__R  s   HJGOOL!OODOO-g67OOD/4L
3))*((4 .1 


!
!*
-c<
H #5) GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   rd  zVectorStoreIdxOp.valueToStorec  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rQ  rK  rM  s     r   r  zVectorStoreIdxOp.basej  rO  r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S r[  rK  rM  s     r   r  zVectorStoreIdxOp.indicesq  rY  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rW  rR  rM  s     r   r%  zVectorStoreIdxOp.maskx  rS  r   c                 4    | j                   j                  d   S rz  rF   rB   s    r   rm  zVectorStoreIdxOp.add  rG   r   rH   c                 P    |t        d      || j                  j                  d<   y r|  rL   rN   s     r   rm  zVectorStoreIdxOp.add  rO   r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   r\   rd  ro   r  r   r  r8  r%  rE  rm  r^   r`   ra   s   @r   rF  rF  J  s.   ).%,:>DdW[ l" GMM'*<*<=   GMM'"4"45   w,,   @IgmmG,>,>?@ @ @ ,7## , , ::-w'' - -r   rF  rG  c          	      &    t        | ||||||      S )N)rd  r  r  r%  rm  r   r   )rF  )rn  r  r  r%  rm  r   r   s          r   vector_store_idxrP    s    	~D'X\beknsu	vvr   c                   ~    e Zd ZdZg dZdZddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  fd
       Zej                  dee	j                  e	j                        fd       Zej                  de	j&                  fd       Zej*                  de	j&                  fd       Zej                  de	j,                  fd       Zej*                  de	j,                  fd       Z xZS )VectorStoreOpztpu.vector_storer  r   NrG  c                   g }	i }
d }|	j                  |       |	j                  |       |	j                  t        |             |	j                  |       t        |      }t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   |dt        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   g }d }t        | )  | j                  | j                  | j                  | j                  |
||	||||       y )Nr  r   r  rE  rm  r"   r  )r7   rd  r  r  r  r%  rm  r   r   r%   r#   r'   r:   r$   r;   r<   s                  r   r2   zVectorStoreOp.__init__  sj   HJGOOL!OODOO-g67OOD/4Lw))*$$%89 % 127LQ y 3))*((4 .1 


!
!*
-c<
H #5) GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   rd  zVectorStoreOp.valueToStore  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rQ  rK  rM  s     r   r  zVectorStoreOp.base  rO  r   c                 v    t        | j                  j                  | j                  j                  d   d      }|S r[  rK  rM  s     r   r  zVectorStoreOp.indices  rY  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rW  rR  rM  s     r   r%  zVectorStoreOp.mask  rS  r   c                 4    | j                   j                  d   S r  rF   rB   s    r   r  zVectorStoreOp.strides  r  r   rH   c                 P    |t        d      || j                  j                  d<   y r  rL   rN   s     r   r  zVectorStoreOp.strides  r  r   c                 4    | j                   j                  d   S rz  rF   rB   s    r   rm  zVectorStoreOp.add  rG   r   c                 P    |t        d      || j                  j                  d<   y r|  rL   rN   s     r   rm  zVectorStoreOp.add  rO   r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   r\   rd  ro   r  r   r  r8  r%  r  r  r^   rE  rm  r`   ra   s   @r   rR  rR    s{   %.%,CGTW[`d l* GMM'*<*<=   GMM'"4"45   w,,   @IgmmG,>,>?@ @ @ 0w00 0 0 >>1744 1 1
 ,7## , , ::-w'' - -r   rR  c          
      (    t        | |||||||      S )N)rd  r  r  r  r%  rm  r   r   )rR  )rn  r  r  r  r%  rm  r   r   s           r   vector_storer]    s*    	NwX_fjpsy|  BD  
E  Er   c                   |    e Zd ZdZg dZdZdddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Zej                  dee	j                  e	j                         fd       Zej                  dee	j                  e	j                         fd       Zej                  de	j&                  fd       Zej*                  de	j&                  fd       Z xZS )
WaitDMA2Opztpu.wait_dma2)r   r   r   r   r   r   Nr  rA  rC  r   r   c                *   g }	i }
d }|	j                  |       |	j                  |       |	j                  |       |	j                  |       |	j                  |       t        |      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   g }d }t        | %  | j                  | j                  | j                  | j                  |
||	||||       y )NrE  r   rC  r"   r(   )r7   rq  srcdstr  rA  rC  r   r   r%   r#   r'   r:   r$   r;   r<   s                  r   r2   zWaitDMA2Op.__init__  s    HJGOOIOOCOOCOOIOOG/4L"?G$5$56((4 FU 


!
!*
-o|
T %//@$A GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rI  rK  rM  s     r   rq  zWaitDMA2Op.semaphore  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rQ  rK  rM  s     r   rb  zWaitDMA2Op.src  rO  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S rU  rK  rM  s     r   rc  zWaitDMA2Op.dst  rO  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rW  rR  rM  s     r   r  zWaitDMA2Op.device_id  rS  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S rZ  rR  rM  s     r   rA  zWaitDMA2Op.core_id  rS  r   c                 4    | j                   j                  d   S ra  rF   rB   s    r   rC  zWaitDMA2Op.strict_ordering  rb  r   rH   c                 P    |t        d      || j                  j                  d<   y rd  rL   rN   s     r   rC  zWaitDMA2Op.strict_ordering  re  r   )r   r   r   r3   r5   r4   r2   rY   rZ   r,   r[   ro   rq  rb  rc  r8  r   r  rA  rE  rC  r^   r`   ra   s   @r   r_  r_    su   ".&,7;T[_einr l$ w'9'9:   7==!3!34   7==!3!34   @7==1D1D#EF @ @ @yw/B/B!CD @ @ 8w// 8 8 97#3#3 9 9r   r_  r`  c          
      (    t        | |||||||      S )Nrq  rb  rc  r  rA  rC  r   r   )r_  rl  s           r   	wait_dma2rm  #  s1    	iScYX_  rA  GJ  OQ  
R  Rr   c                        e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Z xZS )		WaitDMAOpztpu.wait_dmar   Nr   c                    g }i }d }|j                  |       |j                  |       t        |      }g }	d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   )r7   rq  refr   r   r%   r#   r'   r:   r$   r;   r<   s              r   r2   zWaitDMAOp.__init__,  s    HJGOOIOOC/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rq  zWaitDMAOp.semaphore7  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   rq  zWaitDMAOp.ref;  rC   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   rq  rq  r`   ra   s   @r   ro  ro  &  sx    !.,,0T 	l &w'9'9: & & &7==!3!34 & &r   ro  c                     t        | |||      S )Nrq  rq  r   r   )ro  ru  s       r   wait_dmarv  ?  s    	YCSR	@@r   c                   6    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd	       Z xZS )
WaitIndirectDMAOpztpu.wait_indirect_dmar   Nr   c                   g }i }d }|j                  |       |j                  |       |j                  |       t        |      }	g }
d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y ri   rw   )r7   rq  rb  rc  r   r   r%   r#   r'   r:   r$   r;   r<   s               r   r2   zWaitIndirectDMAOp.__init__H  s    HJGOOIOOCOOC/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   rq  zWaitIndirectDMAOp.semaphoreT  rC   r   c                 4    | j                   j                  d   S r  r@   rB   s    r   rb  zWaitIndirectDMAOp.srcX  rC   r   c                 4    | j                   j                  d   S ru  r@   rB   s    r   rc  zWaitIndirectDMAOp.dst\  rC   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   ro   rq  rb  rc  r`   ra   s   @r   rx  rx  B  s    *.,15$ 
l &w'9'9: & & &7==!3!34 & & &7==!3!34 & &r   rx  c                "    t        | ||||      S )Nrq  rb  rc  r   r   )rx  r~  s        r   wait_indirect_dmar  `  s    	YCScb	QQr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )	WeirdOpz	tpu.weirdr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y ri   rw   r   s              r   r2   zWeirdOp.__init__i  ry   r   r=   c                 4    | j                   j                  d   S r?   r@   rB   s    r   r9   zWeirdOp.inputt  rC   r   c                 4    | j                   j                  d   S r?   rW   rB   s    r   r8   zWeirdOp.outputx  rX   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r[   r9   r_   r8   r`   ra   s   @r   r  r  c  sd    .,+/D 	l &W]] & & %g&& % %r   r  c                4    t        | |||      j                  S r   )r  rd   r   s       r   weirdr  |  s    	e	<	C	CCr   c                   l     e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
 xZS )YieldOpz	tpu.yieldr   Nr   c                    g }i }d }|j                  t        |             t        |      }g }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y ri   rq  r  s             r   r2   zYieldOp.__init__  s    HJGOO-h78/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r=   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   r   r   s     r   r  zYieldOp.results_  r   r   )r   r   r   r3   r4   r2   rY   rZ   r,   r   r  r`   ra   s   @r   r  r    sB    .,&*t l E-- E Er   r  c                    t        | ||      S )Nr  )r  r  s      r   yield_r    s    	(	33r   ) jaxlib.mlir.dialects._ods_commonr   	_ods_cextr   r   r   r*   r   r   r   rL  irr,   globals!register_traceback_file_exclusion__file__rY   typingr   	_Sequencer	   _Unionr
   r8  register_dialectDialectr   register_operationOpViewr   r_   re   rg   rs   ru   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r
  r  r  r<  r>  rh  rj  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r<  r>  rE  rH  rM  rO  r_  ra  rf  rh  rm  ro  rv  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r!  r#  r.  r0  r6  r8  rG  rI  rO  rQ  r`  rb  ro  rq  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r&  r(  r-  r/  r8  r:  rD  rF  rP  rR  r]  r_  rm  ro  rv  rx  r  r  r  r  r  r   r   r   <module>r     s   @  ,, 	   3 3H =  P P w   h'2%'.. 2% (2%h 15 \'BRBR \ h'% % (%& "d Aw/?/? A h'%W^^ % (%0 )- K':J:J K h'1%w~~ 1% (1%f 15$4 bGL\L\ b h'& & (&0  $ : : h'% % (%0 #'4 FG4D4D F h'%GNN % (%0 (, J9I9I J h'.%GNN .% (.%` 8< _IYIY _ h'#%GNN #% (#%J 04$ e7K[K[ e h'%7>> % (%> +/4 PG<L<L P h'H%W^^ H% (H%T 6:d ZwGWGW Z h'&gnn & (&&  .' . h'% % (%" 4D <W5E5E < h'7%gnn 7% (7%r <@Td xW^WgWg x h'S%gnn S% (S%j @DVZ`dim bry  sC  sC b h'Z97>> Z9 (Z9x GKVZdhsw  JN  TX  ]a d  fr d h'3-7>> 3- (3-j OSX\bfko X  uI X h'%GNN % (%, -1dt PHXHX P h'%W^^ % (%0 4 9G,<,< 9 h'$%w~~ $% ($%L 15 b'BRBR b h'2%w~~ 2% (2%h 7;t mHXHX m h'%GNN % (%& #'4 KG4D4D K h'%7>> % (%& %)T Dg6F6F D h'%'.. % (%> %)dt N@P@P N h'1%W^^ 1% (1%f %)T Mg6F6F M h'E%W^^ E% (E%N AE$SW L\c\l\l L h'--'.. -- (--^ *.$ H; H h'.3GNN .3 (.3` #'DT Le L h'% % (%B %)T Gg6F6F G h'f%w~~ f% (f%P 48tW[osy}  CG G  LS  L\  L\ G h'%gnn % (%0 *.$ L7;K;K L h'%gnn % (%0 *.$ L7;K;K L h'+%GNN +% (+%Z CG4 ~T[TdTd ~ h'%gnn % (%0 *.$ L7;K;K L h'%w~~ % (%& %)T @g6F6F @ h'E7>> E (E( $(D 3\ 3 h'/% /% (/%b ;?4 kGL\L\ k h'% % (%: )- N':J:J N h'3% 3% (3%j FJd @W^WgWg @ h'#%7>> #% (#%J !%d Z'JZJZ Z h'2%GNN 2% (2%h 48D `WEUEU ` h'w~~  (0 !T TfW5E5EwG[G[]e5e.f T h'% % (%0 ,0D NW=M=M N h'% % (%.  $$ I7;K;K I h'2%w~~ 2% (2%h 59T igFVFV i h'% % (%0 $(D HW5E5E H h'%GNN % (%2 (, J9I9I J h'a%w~~ a% (a%F 04dTX^bgk ]pw  qA  qA ] h'$%w~~ $% ($%L /3t ^@P@P ^ h',%'.. ,% (,%\ ,0Td ^wG[G[ ^ h')%W^^ )% ()%V '+ Y'BRBR Y h'%gnn % (%, $(Td Vw?O?O V h'C/ C/ (C/J 04TTW[`d Jiz J h'&gnn & (&0 (, M M h'8%W^^ 8% (8%t PTX\ Vahaqaq V h'89gnn 89 (89t Y]ae ^jy ^ h'?%W^^ ?% (?%B IMY]cglp su|  vJ  vJ s h')%W^^ )% ()%V LPTX }]d]m]m } h'%'.. % (%: 6:d _wGWGW _ h'Q4gnn Q4 (Q4f @DTX^bgk _pw _ h'*%GNN *% (*%X 9= j'JZJZ j h'*1W^^ *1 (*1X BF$ rSa r h'(%w~~ (% ((%T 37DT egN^N^ e h'2gnn 2 (2h ,0D TVGDTDTV]VjVjlsDs=t T h'(/7>> (/ ((/T (, D D h'l'.. l (l t % % h'$%'.. $% ($%L 374 cGDTDT c h'$%w~~ $% ($%L ,0D XW=M=M X h';%'.. ;% (;%z CG4 xT[TdTd x h'@%'.. @% (@%D MQVZ_c Uhohxhx U h'Dgnn D (D2 *.$ T6'BRBRT[ThThjyBy;z T h'+%gnn +% (+%Z 37DT dgN^N^ d h'9%7>> 9% (9%v 9=$4 tT[TdTd t h'<-w~~ <- (<-| =AdPTY] wbr w h'J-GNN J- (J-X BF4UY^b Egt E h'D9 D9 (D9L 15dTX^bgk Rpz R h'& & (&0 %)T Ai A h'& & (&: 374 RDU R h'%gnn % (%0 !% D'2B2B D h'Egnn E (E( !T 4g 4r   