
    uki                    .   d dl mZ d dl mZmZmZm	Z
 ej                  Zej                  j                  e       d dlZd dlmZmZmZ ej0                   G d dej2                               Z ej6                  e       G d dej8                               Zddddd	d
eej<                  ej>                  ef   fdZ  ej6                  e       G d dej8                               Z!ddddddd
e!fdZ" ej6                  e       G d dej8                               Z#dddd
e#fdZ$ ej6                  e       G d dej8                               Z%dddd
ej<                  fdZ& ej6                  e       G d dej8                               Z'dddd
ej<                  fdZ( ej6                  e       G d dej8                               Z)dddd
e)fdZ* ej6                  e       G d dej8                               Z+dddd
ej<                  fdZ, ej6                  e       G d  d!ej8                               Z-dddd"d
ej<                  fd#Z. ej6                  e       G d$ d%ej8                               Z/dddd
ej<                  fd&Z0 ej6                  e       G d' d(ej8                               Z1dddd
ej<                  fd)Z2 ej6                  e       G d* d+ej8                               Z3dddd
ej<                  fd,Z4 ej6                  e       G d- d.ej8                               Z5dddd/d
ej<                  fd0Z6 ej6                  e       G d1 d2ej8                               Z7dddd
ej<                  fd3Z8 ej6                  e       G d4 d5ej8                               Z9ddddd6d
ej<                  fd7Z: ej6                  e       G d8 d9ej8                               Z;dddd
e;fd:Z< ej6                  e       G d; d<ej8                               Z=dddd
e=fd=Z> ej6                  e       G d> d?ej8                               Z?dddd
e?fd@Z@ ej6                  e       G dA dBej8                               ZAddddddCd
ej<                  fdDZB ej6                  e       G dE dFej8                               ZCdddddddGd
ej<                  fdHZD ej6                  e       G dI dJej8                               ZEdddd
eej<                  ej>                  eEf   fdKZF ej6                  e       G dL dMej8                               ZGdddd/d
ej<                  fdNZH ej6                  e       G dO dPej8                               ZIdddd
ej<                  fdQZJ ej6                  e       G dR dSej8                               ZKddddTd
ej<                  fdUZL ej6                  e       G dV dWej8                               ZMdddddXd
ej<                  fdYZN ej6                  e       G dZ d[ej8                               ZOdddd/d
ej<                  fd\ZP ej6                  e       G d] d^ej8                               ZQdddd/d
ej<                  fd_ZR ej6                  e       G d` daej8                               ZSdddd"d
ej<                  fdbZT ej6                  e       G dc ddej8                               ZUdddd
ej<                  fdeZV ej6                  e       G df dgej8                               ZWdddd
ej<                  fdhZX ej6                  e       G di djej8                               ZYdddddddddddk
d
ej<                  fdlZZ ej6                  e       G dm dnej8                               Z[dddd
ej<                  fdoZ\ ej6                  e       G dp dqej8                               Z]ddddrd
ej<                  fdsZ^ ej6                  e       G dt duej8                               Z_dddd
ej<                  fdvZ` ej6                  e       G dw dxej8                               Zadddd
eej<                  ej>                  eaf   fdyZb ej6                  e       G dz d{ej8                               Zcdddd
ecfd|Zd ej6                  e       G d} d~ej8                               Zedddd/d
ej<                  fdZf ej6                  e       G d dej8                               Zgdddd/d
ej<                  fdZh ej6                  e       G d dej8                               Zidddd/d
ej<                  fdZj ej6                  e       G d dej8                               Zkdddd
ekfdZl ej6                  e       G d dej8                               Zmdddd
ej<                  fdZn ej6                  e       G d dej8                               Zodddd
eej<                  ej>                  eof   fdZp ej6                  e       G d dej8                               Zqdddd
eqfdZr ej6                  e       G d dej8                               Zsdddddd
ej<                  fdZt ej6                  e       G d dej8                               Zudddd
eej<                  ej>                  euf   fdZv ej6                  e       G d dej8                               Zwdddd
ewfdZx ej6                  e       G d dej8                               Zydddd
ej<                  fdZz ej6                  e       G d dej8                               Z{dddd/d
ej>                  fdZ| ej6                  e       G d dej8                               Z}dddddddd
e}fdZ~ ej6                  e       G d dej8                               Zdddd/d
ej<                  fdZ ej6                  e       G d dej8                               Zdddd/d
ej<                  fdZy)    )_cext)equally_sized_accessorget_default_loc_contextget_op_results_or_valuessegmented_accessorN)SequenceUnionOptionalc                       e Zd ZdZy)_DialectttN)__name__
__module____qualname__DIALECT_NAMESPACE     X/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jaxlib/triton/_triton_ops_gen.pyr   r      s    r   r   c                   H    e Zd ZdZdZdZddddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  dee	j                      fd       Zej                  d
ee	j                      fd       Zej$                  d        Zej                  dee	j                      fd       Zej                  d
ee	j                      fd       Zej$                  d        Z xZS )CallOpah  
  The `tt.call` operation represents a direct call to a function that is
  within the same symbol scope as the call. The operands and result types of
  the call must match the specified function type. The callee is encoded as a
  symbol reference attribute named "callee".
  
  Example:
  
  ```mlir
  %2 = tt.call @my_add(%0, %1) : (f32, f32) -> f32
  ```
  ztt.callr   TN	arg_attrs	res_attrslocipc                j   g }i }	d }
|j                  t        |             t        |      }t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |	d<   |dt        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |	d<   |dt        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |	d<   g }|j                  |       d }t        | )  | j                  | j                  | j                  | j                  |	||||
||       y )NFlatSymbolRefAttrcontextcalleeDictArrayAttrr   r   
attributesresultsoperands
successorsregionsr   r   extend_get_op_results_or_values_ods_get_default_loc_context
isinstance_ods_ir	AttributeAttrBuildercontainsgetsuper__init__OPERATION_NAME_ODS_REGIONS_ODS_OPERAND_SEGMENTS_ODS_RESULT_SEGMENTS)selfresultr!   	operands_r   r   r   r   r&   r$   r(   _ods_contextr%   _ods_successors	__class__s                 r   r4   zCallOp.__init__*   s   HJGOO-i89/4Lvw(()$$%89 # 126<P x 9g//0((9 :C 


!
!/
29l
S )5 9g//0((9 :C 


!
!/
29l
S )5 GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   returnc                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S N   r   len	operationr&   r9   _ods_variadic_group_lengths     r   r;   zCallOp.operands_A   >    !$T^^%<%<!=!AA!E>>""1Q)C%CDDr   c                 4    | j                   j                  d   S )Nr!   rE   r$   r9   s    r   r!   zCallOp.calleeF       >>$$X..r   valuec                 P    |t        d      || j                  j                  d<   y )N4'None' not allowed as value for mandatory attributesr!   
ValueErrorrE   r$   r9   rM   s     r   r!   zCallOp.calleeJ   '    }MNN*/DNNh'r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr   rJ   rK   s    r   r   zCallOp.arg_attrsP   ,    $..333>>$$[11r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y rU   rJ   rR   s     r   r   zCallOp.arg_attrsV   E    /4dnn,	11	1
..
#
#K
0 
2r   c                 2    | j                   j                  d= y rU   rJ   rK   s    r   r   zCallOp.arg_attrs]       !!+.r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr   rJ   rK   s    r   r   zCallOp.res_attrsa   rV   r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r\   rJ   rR   s     r   r   zCallOp.res_attrsg   rX   r   c                 2    | j                   j                  d= y r\   rJ   rK   s    r   r   zCallOp.res_attrsn   rZ   r   )r   r   r   __doc__r5   r6   r4   builtinspropertyr.   OpOperandListr;   r   r!   setter	_Optional	ArrayAttrr   deleterr   __classcell__r>   s   @r   r   r      s    .,=ATW[`d l. E.. E E /g// / / 
==0'33 0 0
 27#4#45 2 2
 1Yw'8'89 1 1 / / 27#4#45 2 2
 1Yw'8'89 1 1 / /r   r   r   r?   c          	          t        | ||||||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )N)r:   r!   r;   r   r   r   r   rB   r   )r   r%   rD   )	r:   r!   r;   r   r   r   r   opr%   s	            r   callrk   r   sb    VFi9`iorwyz"  GI  GQ  GQ  }DL1$SW9J71:SPRSr   c                   r    e Zd ZdZdZdZdddddd fd
Zej                  de	j                  fd       Zej                  d	e	j                  fd
       Zej                  de	j                  fd       Zej                  d	e	j                  fd       Zej                  dee	j                     fd       Zej                  d	ee	j                     fd       Zej"                  d        Zej                  dee	j$                     fd       Zej                  d	ee	j$                     fd       Zej"                  d        Zej                  dee	j$                     fd       Zej                  d	ee	j$                     fd       Zej"                  d        Zej                  de	j*                  fd       Z xZS )FuncOpu  
  Operations within the function cannot implicitly capture values defined
  outside of the function, i.e. Functions are `IsolatedFromAbove`. All
  external references must use function arguments or attributes that establish
  a symbolic connection (e.g. symbols referenced by name via a string
  attribute like SymbolRefAttr). An external function declaration (used when
  referring to a function declared in some other module) has no body. While
  the MLIR textual form provides a nice inline syntax for function arguments,
  they are internally represented as “block arguments” to the first block in
  the region.
  
  Only dialect attribute names may be specified in the attribute dictionaries
  for function arguments, results, or the function itself.
  
  Example:
  
  ```mlir
  // External function definitions.
  tt.func @abort()
  tt.func @scribble(i32, i64, memref<? x 128 x f32, #layout_map0>) -> f64
  
  // A function that returns its argument twice:
  tt.func @count(%x: i64) -> (i64, i64)
    attributes {fruit: "banana"} {
    return %x, %x: i64, i64
  }
  
  // A function with an argument attribute
  tt.func @example_fn_arg(%x: i32 {swift.self = unit})
  
  // A function with a result attribute
  tt.func @example_fn_result() -> (f64 {dialectName.attrName = 0 : i64})
  
  // A function with an attribute
  tt.func @example_fn_attr() attributes {dialectName.attrName = false}
  ```
  ztt.funcrB   TNsym_visibilityr   r   r   r   c                   g }i }	d }
t        |      }t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   |dt        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   |dt        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |	d	<   |dt        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |	d
<   g }d }t        | !  | j                  | j                  | j                  | j                  |	||||
||       y )NSymbolNameAttrr   sym_nameanonymous_722function_typeStrAttrrp   r"   r   r   r#   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   )r9   rs   ru   rp   r   r   r   r   r&   r$   r(   r<   r%   r=   r>   s                 r   r4   zFuncOp.__init__   s<   HJG/4Lx**+$$%56 ' ./,O z
 }g//0$$_5 $1 o.}lS  !>7#4#45((3 DR 


!
!)
,^\
R $..>#? 9g//0((9 :C 


!
!/
29l
S )5 9g//0((9 :C 


!
!/
29l
S )5 GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S )Nrs   rJ   rK   s    r   rs   zFuncOp.sym_name       >>$$Z00r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   rs   rP   rR   s     r   rs   zFuncOp.sym_name   '    }MNN,1DNNj)r   c                 4    | j                   j                  d   S )Nru   rJ   rK   s    r   ru   zFuncOp.function_type       >>$$_55r   c                 P    |t        d      || j                  j                  d<   y )NrO   ru   rP   rR   s     r   ru   zFuncOp.function_type   '    }MNN16DNNo.r   c                 f    d| j                   j                  vry | j                   j                  d   S Nrp   rJ   rK   s    r   rp   zFuncOp.sym_visibility   s-    t~~888>>$$%566r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r   rJ   rR   s     r   rp   zFuncOp.sym_visibility   sG    49dnn 01	T^^66	6
..
#
#$4
5 
7r   c                 2    | j                   j                  d= y r   rJ   rK   s    r   rp   zFuncOp.sym_visibility   s    !!"23r   c                 f    d| j                   j                  vry | j                   j                  d   S rU   rJ   rK   s    r   r   zFuncOp.arg_attrs   rV   r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y rU   rJ   rR   s     r   r   zFuncOp.arg_attrs   rX   r   c                 2    | j                   j                  d= y rU   rJ   rK   s    r   r   zFuncOp.arg_attrs   rZ   r   c                 f    d| j                   j                  vry | j                   j                  d   S r\   rJ   rK   s    r   r   zFuncOp.res_attrs   rV   r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r\   rJ   rR   s     r   r   zFuncOp.res_attrs   rX   r   c                 2    | j                   j                  d= y r\   rJ   rK   s    r   r   zFuncOp.res_attrs  rZ   r   c                      | j                   d   S Nr   r(   rK   s    r   bodyzFuncOp.body      <<?r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   
StringAttrrs   rc   TypeAttrru   rd   rp   rf   re   r   r   Regionr   rg   rh   s   @r   rm   rm   v   s?   $L .,@DPT`djnsw l: 1** 1 1 ??2G.. 2 2
 6W-- 6 6 7!1!1 7 7
 7i(:(:; 7 7
 6)G,>,>"? 6 6 4 4 27#4#45 2 2
 1Yw'8'89 1 1 / / 27#4#45 2 2
 1Yw'8'89 1 1 / / GNN  r   rm   ro   c          	      &    t        | ||||||      S )Nrs   ru   rp   r   r   r   r   )rm   r   s          r   funcr   
  s/    	~ir  H  NQ  VX  
Y  Yr   c                   p     e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Z xZS )	ReturnOpam  
  The `tt.return` operation represents a return operation within a function.
  The operation takes variable number of operands and produces no results.
  The operand number and types must match the signature of the function
  that contains the operation.
  
  Example:
  
  ```mlir
  tt.func @foo() : (i32, f8) {
    ...
    tt.return %0, %1 : i32, f8
  }
  ```
  z	tt.returnr   Nr   r   c                    g }i }d }|j                  t        |             t        |      }g }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y Nr#   	r*   r+   r,   r3   r4   r5   r6   r7   r8   )r9   srcsr   r   r&   r$   r(   r<   r%   r=   r>   s             r   r4   zReturnOp.__init__#  s    HJGOO-d34/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r   zReturnOp.srcs-  rH   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   rb   r   rg   rh   s   @r   r   r     sI      .,"&4 l EG)) E Er   r   r   c                    t        | ||      S )Nr   r   r   )r   r   s      r   return_r   2  s    	t	,,r   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  dej                  fd	       Z xZS )
AddPtrOpz	tt.addptrr   Nr   c                   g }i }d }|j                  |       |j                  |       t        |      }	g }
|
j                  |       d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y r   appendr,   r3   r4   r5   r6   r7   r8   )r9   r:   ptroffsetr   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zAddPtrOp.__init__;  s    HJGOOCOOF/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   rE   r&   rK   s    r   r   zAddPtrOp.ptrG      >>""1%%r   c                 4    | j                   j                  d   S NrB   r   rK   s    r   r   zAddPtrOp.offsetK  r   r   c                 4    | j                   j                  d   S r   rE   r%   rK   s    r   r:   zAddPtrOp.resultO      >>!!!$$r   )r   r   r   r5   r6   r4   r`   ra   r.   Valuer   r   OpResultr:   rg   rh   s   @r   r   r   5  s    .,15$ 
l &7== & & &gmm & & %g&& % %r   r   c                6    t        | ||||      j                  S )Nr:   r   r   r   r   )r   r:   r   s        r   addptrr   S  s    	SSR	H	O	OOr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  dej                  fd	       Z xZS )
	AdvanceOpz
tt.advancer   Nr   c                ,   g }i }d }|j                  |       |j                  t        |             t        |      }	g }
|
j                  |       d }t        |   | j                  | j                  | j                  | j                  ||
|||||       y r   
r   r*   r+   r,   r3   r4   r5   r6   r7   r8   )r9   r:   r   offsetsr   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zAdvanceOp.__init__\  s    HJGOOCOO-g67/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zAdvanceOp.ptrh  r   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S N   rB   rC   rF   s     r   r   zAdvanceOp.offsetsl  rH   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zAdvanceOp.resultq  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   r   r   rb   r   r   r:   rg   rh   s   @r   r   r   V  s    .,264 
l &7== & & Ew,, E E %g&& % %r   r   c                6    t        | ||||      j                  S )Nr:   r   r   r   r   )r   r:   r   s        r   advancer   u  s    	&c7	K	R	RRr   c                        e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Z xZS )AssertOpz
  `tt.assert` takes a condition tensor and a message string.
  If the condition is false, the message is printed, and the program is aborted.
  z	tt.assertr   Nr   c                   g }i }d }|j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   g }	d }
t        | %  | j                  | j                  | j                  | j                  ||	||
|||       y )Nrv   r   messager#   r   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   )r9   	conditionr   r   r   r&   r$   r(   r<   r%   r=   r>   s              r   r4   zAssertOp.__init__  s    HJGOOI/4Lw))*$$Y/ % i(,G y GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zAssertOp.condition  r   r   c                 4    | j                   j                  d   S )Nr   rJ   rK   s    r   r   zAssertOp.message      >>$$Y//r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r   rP   rR   s     r   r   zAssertOp.message  '    }MNN+0DNNi(r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r   r   r   rc   rg   rh   s   @r   r   r   x  s    
 .,04 l & & & 0w)) 0 0 >>17-- 1 1r   r   c                     t        | |||      S )Nr   r   r   r   )r   r   s       r   assert_r     s    	IwCB	GGr   c                   "    e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                  de	j                  fd       Zej                   de	j                  fd       Zej                  de	j                  fd       Zej                   de	j                  fd       Zej                  de	j$                  fd       Z xZS )AtomicCASOpz
  compare $cmp with data $old at location $ptr,
  
  if $old == $cmp, store $val to $ptr,
  
  else store $old to $ptr,
  
  return $old
  ztt.atomic_casr   Nr   c                   g }	i }
d }|	j                  |       |	j                  |       |	j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |
||	||||       y )NTT_MemSemanticAttrr   semTT_MemSyncScopeAttrscoper#   r   )r9   r:   r   cmpvalr   r   r   r   r&   r$   r(   r<   r%   r=   r>   s                  r   r4   zAtomicCASOp.__init__  sa   HJGOOCOOCOOC/4LsG%%&$$%9:  23CN u
 ug''($$%:; ! 34ULQ w GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zAtomicCASOp.ptr  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zAtomicCASOp.cmp  r   r   c                 4    | j                   j                  d   S Nr   r   rK   s    r   r   zAtomicCASOp.val  r   r   c                 4    | j                   j                  d   S Nr   rJ   rK   s    r   r   zAtomicCASOp.sem      >>$$U++r   rM   c                 P    |t        d      || j                  j                  d<   y NrO   r   rP   rR   s     r   r   zAtomicCASOp.sem  '    }MNN',DNNe$r   c                 4    | j                   j                  d   S Nr   rJ   rK   s    r   r   zAtomicCASOp.scope      >>$$W--r   c                 P    |t        d      || j                  j                  d<   y NrO   r   rP   rR   s     r   r   zAtomicCASOp.scope  '    }MNN).DNNg&r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zAtomicCASOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r   r   r   r/   r   rc   r   r   r:   rg   rh   s   @r   r   r     sM    #.,?C l* &7== & & &7== & & &7== & & ,7$$ , , ::-w(( - -
 .W&& . . 	<</** / /
 %g&& % %r   r   c          
      <    t        | |||||||      j                  S )Nr:   r   r   r   r   r   r   r   )r   r:   r   s           r   
atomic_casr     s#    	F#3eY\ac	d	k	kkr   c                       e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  dee	j                     fd
       Zej                  de	j                  fd       Zej"                  de	j                  fd       Zej                  de	j                  fd       Zej"                  de	j                  fd       Zej                  de	j                  fd       Zej"                  de	j                  fd       Zej                  de	j(                  fd       Z xZS )AtomicRMWOpzf
  load data at $ptr, do $rmw_op with $val, and store result to $ptr.
  
  return old value at $ptr
  ztt.atomic_rmwr   Nmaskr   r   c                   g }
i }d }|
j                  |       |
j                  |       ||
j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||
||||	       y )	NTT_AtomicRMWAttrr   atomic_rmw_opr   r   r   r   r#   r   )r9   r:   r   r   r   r   r   r   r   r   r&   r$   r(   r<   r%   r=   r>   s                   r   r4   zAtomicRMWOp.__init__  s   HJGOOCOOC./4L}g//0$$%78 $1 01-V 
 sG%%&$$%9:  23CN u
 ug''($$%:; ! 34ULQ w GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zAtomicRMWOp.ptr  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zAtomicRMWOp.val  r   r   c                 |    t        | j                  j                        dk  rd S | j                  j                  d   S N   r   rC   rK   s    r   r   zAtomicRMWOp.mask  3    t~~../!34S9P9PQR9SSr   c                 4    | j                   j                  d   S )Nr   rJ   rK   s    r   r   zAtomicRMWOp.atomic_rmw_op  r}   r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r   rP   rR   s     r   r   zAtomicRMWOp.atomic_rmw_op#  r   r   c                 4    | j                   j                  d   S r   rJ   rK   s    r   r   zAtomicRMWOp.sem)  r   r   c                 P    |t        d      || j                  j                  d<   y r   rP   rR   s     r   r   zAtomicRMWOp.sem-  r   r   c                 4    | j                   j                  d   S r   rJ   rK   s    r   r   zAtomicRMWOp.scope3  r   r   c                 P    |t        d      || j                  j                  d<   y r   rP   rR   s     r   r   zAtomicRMWOp.scope7  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zAtomicRMWOp.result=  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r   r   rd   r   r/   r   rc   r   r   r   r:   rg   rh   s   @r   r   r     s    #.,JNTX]a l2 &7== & & &7== & & TIgmm, T T 6W.. 6 6 7!2!2 7 7
 ,7$$ , , ::-w(( - -
 .W&& . . 	<</** / /
 %g&& % %r   r   r   c                >    t        | ||||||||	      j                  S )N	r:   r   r   r   r   r   r   r   r   )r   r:   r  s	            r   
atomic_rmwr  A  s8    	F-ScWZbgnrx{  AC  
D  
K  
K  Kr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )		BitcastOpz
tt.bitcastr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y r   r   r9   r:   srcr   r   r&   r$   r(   r<   r%   r=   r>   s              r   r4   zBitcastOp.__init__J      HJGOOC/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zBitcastOp.srcU  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zBitcastOp.resultY  r   r   r   r   r   r5   r6   r4   r`   ra   r.   r   r  r   r:   rg   rh   s   @r   r  r  D  sd    .,)-$ 	l &7== & & %g&& % %r   r  c                4    t        | |||      j                  S Nr:   r  r   r   )r  r:   r  s       r   bitcastr  ]  s    	&csr	:	A	AAr   c                        e Zd ZdZdZdZddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Z xZS )
BroadcastOpz
  For a given tensor, broadcast changes one or more dimensions with size 1
  to a new size, e.g. tensor<1x32x1xf32> -> tensor<2x32x4xf32>.  You cannot
  change the size of a non-1 dimension.
  ztt.broadcastr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y r   r   r  s              r   r4   zBroadcastOp.__init__l  r  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zBroadcastOp.srcw  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zBroadcastOp.result{  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   RankedTensorTyper  r   r:   rg   rh   s   @r   r  r  `  s     ".,)-$ 	l &7==!9!9: & & %g&&w'?'?@ % %r   r  c                4    t        | |||      j                  S r  )r  r:   r  s       r   	broadcastr    s    	F	<	C	CCr   c                   6    e Zd ZdZdZddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd       Zej                  dej                  ej                     fd	       Z xZS )
CatOpztt.catr   Nr   c                   g }i }d }|j                  |       |j                  |       t        |      }	g }
|
j                  |       d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y r   r   r9   r:   lhsrhsr   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zCatOp.__init__      HJGOOCOOC/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  z	CatOp.lhs  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  z	CatOp.rhs  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zCatOp.result  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   r   r  r  r  r   r:   rg   rh   s   @r   r  r    s    .,.2t 
l &7==!9!9: & & &7==!9!9: & & %g&&w'?'?@ % %r   r  c                6    t        | ||||      j                  S Nr:   r  r  r   r   )r  r:   r$  s        r   catr%    s    	f#3CB	?	F	FFr   c                       e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                  de	j                  fd       Zej                   de	j                  fd       Zej                  de	j"                  fd       Z xZS )ClampFOpz
  Clamp operation for floating point types.
  
  The operation takes three arguments: x, min, and max. It returns a tensor of the same shape as x with its values clamped to the range [min, max].
  z	tt.clampfr   Nr%   r   r   c                   g }i }	d }
|j                  |       |j                  |       |j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |	d<   ||d   j                  gdz  }d }t        | )  | j                  | j                  | j                  | j                  |	||||
||       y )NTT_PropagateNanAttrr   propagateNanr   rB   r#   )r   r,   r-   r.   r/   r0   r1   r2   typer3   r4   r5   r6   r7   r8   )r9   xminmaxr+  r%   r   r   r&   r$   r(   r<   r=   r>   s                r   r4   zClampFOp.__init__  s   HJGOOAOOCOOC/4L|W../$$%:; #/ 34\<X ~ 8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r-  z
ClampFOp.x  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r.  zClampFOp.min  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r/  zClampFOp.max  r   r   c                 4    | j                   j                  d   S )Nr+  rJ   rK   s    r   r+  zClampFOp.propagateNan  s    >>$$^44r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r+  rP   rR   s     r   r+  zClampFOp.propagateNan  s'    }MNN05DNNn-r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zClampFOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r-  r.  r/  r/   r+  rc   r   r:   rg   rh   s   @r   r'  r'    s    .,;?Td l  & & & &7== & & &7== & & 5G-- 5 5 6 1 1 6 6
 %g&& % %r   r'  r(  c          	      :    t        | ||||||      j                  S )N)r-  r.  r/  r+  r%   r   r   )r'  r:   )r-  r.  r/  propagate_nanr%   r   r   s          r   clampfr8    s!    	A3CmWZ]bd	e	l	llr   c                   x    e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Zej                  de	j                   e	j                     fd       Z xZS )DescriptorGatherOpaD  
  The `tt.descriptor_gather` op will be lowered to NVIDIA TMA
  gather operations on targets that support it.
  
  `desc_ptr` is a pointer to the TMA descriptor allocated in global memory.
  The descriptor block must have 1 row and the indices must be a 1D tensor.
  Accordingly, the result is a 2D tensor multiple rows.
  ztt.descriptor_gatherr   Nr   c                <   g }i }d }	|j                  |       |j                  |       |j                  |       t        |      }
g }|j                  |       d }t        |   | j                  | j
                  | j                  | j                  |||||	||       y r   r   )r9   r:   desc	x_offsetsy_offsetr   r   r&   r$   r(   r<   r%   r=   r>   s                r   r4   zDescriptorGatherOp.__init__  s    HJGOODOOIOOH/4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r<  zDescriptorGatherOp.desc  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r=  zDescriptorGatherOp.x_offsets  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r>  zDescriptorGatherOp.y_offset   r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zDescriptorGatherOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r<  r  r=  IntegerTyper>  r   r:   rg   rh   s   @r   r:  r:    s     *.,?C l &GMM & & &w'?'?@ & & &g&9&9: & & %g&&w'?'?@ % %r   r:  c                8    t        | |||||      j                  S )Nr:   r<  r=  r>  r   r   )r:  r:   rE  s         r   descriptor_gatherrF    s     	6	T\bejl	m	t	ttr   c                       e Zd ZdZdZdZddddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                   de	j                  fd       Zej                  de	j                  fd       Zej                   de	j                  fd       Zej                  de	j$                  e	j&                     fd       Z xZS )DescriptorLoadOpz
  This operation will be lowered to Nvidia TMA load operation on targets supporting it.
  `desc` is a tensor descriptor object.
  The destination tensor type and shape must match the descriptor otherwise the result is undefined.
  ztt.descriptor_loadr   Ncacheevictr   r   c                   g }i }	d }
|j                  |       |j                  t        |             t        |      }|dt	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   |dt	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |	||||
||       y )NTT_CacheModifierAttrr   rJ  TT_EvictionPolicyAttrrK  r#   r   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   )r9   r:   r<  indicesrJ  rK  r   r   r&   r$   r(   r<   r%   r=   r>   s                 r   r4   zDescriptorLoadOp.__init__  sg   HJGOODOO-g67/4L5'++,(()?@ 27 


!
!"8
9%
V %W- 5'++,(()@A 27 


!
!"9
:5,
W %W- GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r<  zDescriptorLoadOp.desc+  r   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S r   rC   rF   s     r   rP  zDescriptorLoadOp.indices/  rH   r   c                 4    | j                   j                  d   S NrJ  rJ   rK   s    r   rJ  zDescriptorLoadOp.cache4  r   r   rM   c                 P    |t        d      || j                  j                  d<   y NrO   rJ  rP   rR   s     r   rJ  zDescriptorLoadOp.cache8  r   r   c                 4    | j                   j                  d   S NrK  rJ   rK   s    r   rK  zDescriptorLoadOp.evict>  r   r   c                 P    |t        d      || j                  j                  d<   y NrO   rK  rP   rR   s     r   rK  zDescriptorLoadOp.evictB  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zDescriptorLoadOp.resultH  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r<  rb   rP  r/   rJ  rc   rK  r   r  r:   rg   rh   s   @r   rH  rH    s=    (.,594TX l( &GMM & & Ew,, E E .W&& . . 	<</** / /
 .W&& . . 	<</** / /
 %g&&w'?'?@ % %r   rH  rI  c          	      :    t        | ||||||      j                  S )Nr:   r<  rP  rJ  rK  r   r   )rH  r:   r]  s          r   descriptor_loadr^  L  s"    	dG5X]cfkm	n	u	uur   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  fd
       Zej                  de	j                   fd       Zej$                  de	j                   fd       Z xZS )DescriptorReduceOp
  This operation will be lowered to Nvidia TMA store operation on targets supporting it.
  `desc` is a tensor descriptor object.
  The shape and types of `src` must match the descriptor otherwise the result is undefined.
  ztt.descriptor_reducer   Nr   c                   g }i }d }	|j                  |       |j                  |       |j                  t        |             t        |      }
t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||
      |d<   g }d }t        | -  | j                  | j                  | j                  | j                  |||||	||       y )NTT_DescriptorReduceKindAttrr   kindr#   rO  )r9   rd  r<  r  rP  r   r   r&   r$   r(   r<   r%   r=   r>   s                r   r4   zDescriptorReduceOp.__init__[  s   HJGOODOOCOO-g67/4LtW&&'$$%BC  ;<T<X v GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r<  zDescriptorReduceOp.desck  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zDescriptorReduceOp.srco  r   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S Nr   rB   r   rC   rF   s     r   rP  zDescriptorReduceOp.indicess  rH   r   c                 4    | j                   j                  d   S )Nrd  rJ   rK   s    r   rd  zDescriptorReduceOp.kindx      >>$$V,,r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   rd  rP   rR   s     r   rd  zDescriptorReduceOp.kind|  '    }MNN(-DNNf%r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r<  r  r  rb   rP  r/   rd  rc   rg   rh   s   @r   r`  r`  O  s     *.,6:t l  &GMM & & &7==!9!9: & & Ew,, E E -G%% - - ;;.)) . .r   r`  c                $    t        | |||||      S )Nrd  r<  r  rP  r   r   )r`  rn  s         r   descriptor_reducero    s    	Dc7PSXZ	[[r   c                   x    e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Zej                  de	j                  e	j                     fd       Z xZS )DescriptorScatterOpaF  
  The `tt.descriptor_scatter` op will be lowered to NVIDIA TMA
  scatter operations on targets that support it.
  
  `desc_ptr` is a pointer to the TMA descriptor allocated in global memory.
  The descriptor block must have 1 row and the indices must be a 1D tensor.
  Accordingly, the result is a 2D tensor multiple rows.
  ztt.descriptor_scatterr   Nr   c                <   g }i }d }	|j                  |       |j                  |       |j                  |       |j                  |       t        |      }
g }d }t        |   | j                  | j
                  | j                  | j                  |||||	||       y r   r   )r9   r<  r=  r>  r  r   r   r&   r$   r(   r<   r%   r=   r>   s                r   r4   zDescriptorScatterOp.__init__  s    HJGOODOOIOOHOOC/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r<  zDescriptorScatterOp.desc  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r=  zDescriptorScatterOp.x_offsets  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r>  zDescriptorScatterOp.y_offset  r   r   c                 4    | j                   j                  d   S )Nr   r   rK   s    r   r  zDescriptorScatterOp.src  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r<  r  r=  rC  r>  r  rg   rh   s   @r   rq  rq    s     +.,<@T l &GMM & & &w'?'?@ & & &g&9&9: & & &7==!9!9: & &r   rq  c                $    t        | |||||      S )Nr<  r=  r>  r  r   r   )rq  rx  s         r   descriptor_scatterry    s    	$)hTW]`eg	hhr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  fd
       Z xZS )DescriptorStoreOpra  ztt.descriptor_storer   Nr   c                ,   g }i }d }|j                  |       |j                  |       |j                  t        |             t        |      }	g }
d }t        |   | j                  | j                  | j                  | j                  ||
|||||       y r   r   )r9   r<  r  rP  r   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zDescriptorStoreOp.__init__  s    HJGOODOOCOO-g67/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r<  zDescriptorStoreOp.desc  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zDescriptorStoreOp.src  r   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rh  rC   rF   s     r   rP  zDescriptorStoreOp.indices  rH   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r<  r  r  rb   rP  rg   rh   s   @r   r{  r{    s     ).,04 
l &GMM & & &7==!9!9: & & Ew,, E Er   r{  c                "    t        | ||||      S )Nr<  r  rP  r   r   )r{  r  s        r   descriptor_storer    s    	#wCB	OOr   c                       e Zd ZdZdZdZdddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Zej                  de	j                  fd       Zej"                  de	j                  fd       Zej                  de	j$                  fd       Zej"                  de	j$                  fd       Zej                  de	j(                  e	j                     fd       Z xZS )DotOpaE  
  $d = matrix_multiply($a, $b) + $c. $inputPrecision describes how to exercise the TC
  when the inputs are f32. It can be one of: tf32, tf32x3, ieee, bf16x3, bf16x6.
  tf32: use TC with tf32 ops.
  tf32x3: implement the 3xTF32 trick. For more info see the pass in F32DotTC.cpp
  bf16x3: implement the 3xBF16 trick. For more info see the pass in F32DotTC.cpp
  bf16x6: implement the 6xBF16 trick. For more info see the pass in F32DotTC.cpp
  ieee: don't use TC, implement dot in software.
  If the GPU does not have Tensor cores or the inputs are not f32, this flag is ignored.
  ztt.dotr   N)inputPrecisionmaxNumImpreciseAccr%   r   r   c                   g }	i }
d }|	j                  |       |	j                  |       |	j                  |       t        |      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   d }t        | %  | j                  | j                  | j                  | j                  |
||	||||       y )NTT_InputPrecisionAttrr   r  I32Attrr  r#   r   )r9   abcr  r  r%   r   r   r&   r$   r(   r<   r=   r>   s                 r   r4   zDotOp.__init__  sc   HJGOOAOOAOOA/4L!>7#4#45(()@A DR 


!
!"9
:>S_
` $..>#? %%w'8'89((3 L^ 


!
!)
,-?
V (22F'G O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zDotOp.a   r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zDotOp.b  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zDotOp.c  r   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotOp.inputPrecision      >>$$%566r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotOp.inputPrecision  (    }MNN27DNN./r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotOp.maxNumImpreciseAcc  s    >>$$%9::r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotOp.maxNumImpreciseAcc  s(    }MNN6;DNN23r   c                 4    | j                   j                  d   S r   r   rK   s    r   dzDotOp.d   r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  r  r  r/   r  rc   IntegerAttrr  r   r  rg   rh   s   @r   r  r    s   	 .,04W[aejn l& &w778 & & &w778 & & &w778 & & 7g// 7 7 8'"3"3 8 8
 ;'"5"5 ; ; <g&9&9 < <
 %!!'":":; % %r   r  )input_precisionmax_num_imprecise_accr%   r   r   c          
      <    t        | |||||||      j                  S )N)r  r  r  r  r  r%   r   r   )r  r:   )r  r  r  r  r  r%   r   r   s           r   dotr  $  s9    	a1_Qfpw  ~A  FH  
I  
P  
P  Pr   c                       e Zd ZdZdZg dZdZddddddd fd
Zej                  de
j                  e
j                     fd	       Zej                  de
j                  e
j                     fd
       Zej                  de
j                  e
j                     fd       Zej                  dee
j                  e
j                        fd       Zej                  dee
j                  e
j                        fd       Zej                  de
j&                  fd       Zej*                  de
j&                  fd       Zej                  de
j&                  fd       Zej*                  de
j&                  fd       Zej                  de
j.                  fd       Zej*                  de
j.                  fd       Zej                  de
j.                  fd       Zej*                  de
j.                  fd       Zej                  de
j.                  fd       Zej*                  de
j.                  fd       Zej                  de
j6                  e
j                     fd       Z xZS )DotScaledOpz
  $d = matrix_multiply(scale($a, $a_scale), scale($b, $b_scale)) + $c.
  Where scale(x, s) is a function that applies the scale per block following microscaling spec.
  ztt.dot_scaled)rB   rB   rB   r   r   r   Na_scaleb_scale
lhs_k_pack
rhs_k_packr   r   c                p   g }i }d }|j                  |       |j                  |       |j                  |       |j                  |       |j                  |	       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |
dt        |
t        j                        st        j
                  j                  d      s|
n%t        j
                  j                  d      |
|      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   g }|j                  |       d }t        | %  | j                  | j                  | j                  | j                  |||||||	       y )
NTT_ScaleDotElemTypeAttrr   a_elem_typeb_elem_typeBoolAttrfastMathr  r  r#   r   )r9   r  r  r  r  r  r  r  r  r  r  r  r   r   r&   r$   r(   r<   r%   r=   r>   s                       r   r4   zDotScaledOp.__init__4  s}   HJGOOAOOAOOAOOGOOG/4L{G--.$$%>? "- 78l[ }
 {G--.$$%>? "- 78l[ }
 x**+$$Z0 ' j)(LI z :w001((4 <F 


!
!*
-j,
O  *,7 :w001((4 <F 


!
!*
-j,
O  *,7 GNN1O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S NoperandSegmentSizesr   _ods_segmented_accessorrE   r&   r$   r9   operand_ranges     r   r  zDotScaledOp.aW  ;    +		 	 		"	"#8	91>M r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S Nr  rB   r   r  r  s     r   r  zDotScaledOp.b^  r  r   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S Nr  r   r   r  r  s     r   r  zDotScaledOp.ce  r  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S )Nr  r   r   r  rE   r&   r$   rD   r  s     r   r  zDotScaledOp.a_scalel  L    +		 	 		"	"#8	91>M  #=1A5=?4?r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S )Nr     r   r  r  s     r   r  zDotScaledOp.b_scales  r  r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotScaledOp.a_elem_typez      >>$$]33r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotScaledOp.a_elem_type~  '    }MNN/4DNNm,r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotScaledOp.b_elem_type  r  r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotScaledOp.b_elem_type  r  r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotScaledOp.fastMath  ry   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotScaledOp.fastMath  r{   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotScaledOp.lhs_k_pack      >>$$\22r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotScaledOp.lhs_k_pack  '    }MNN.3DNNl+r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zDotScaledOp.rhs_k_pack  r  r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zDotScaledOp.rhs_k_pack  r  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zDotScaledOp.d  r   r   )r   r   r   r_   r5   r7   r6   r4   r`   ra   r.   r   r  r  r  r  rd   r  r  r/   r  rc   r  r  r  r  r  r   r  rg   rh   s   @r   r  r  '  s   
 #.&,PT^bos  AE  KO  TX !lF w778   w778   w778   @yw/G/G!HI @ @ @yw/G/G!HI @ @ 47,, 4 4 5w00 5 5
 47,, 4 4 5w00 5 5
 1(( 1 1 ??2G,, 2 2
 3'** 3 3 4g.. 4 4
 3'** 3 3 4g.. 4 4
 %!!'":":; % %r   r  r  c                F    t        | |||||||||	|
||      j                  S )N)r  r  r  r  r  r  r  r  r  r  r  r   r   )r  r:   )r  r  r  r  r  r  	fast_mathr  r  r  r  r   r   s                r   
dot_scaledr    sR    	qAa[kdmw~  IP  ]g  t~  DG  LN  
O  
V  
V  Vr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                  fd       Zej                  d
e	j                  fd       Zej                  de	j                   fd       Zej                  d
e	j                   fd       Zej                  de	j$                  fd       Zej                  d
e	j$                  fd       Zej                  de	j(                  fd       Z xZS )ElementwiseInlineAsmOpz
  Runs an inline asm block to generate one or more tensors.
  
  The asm block is given `packed_element` elements at a time.  Exactly which
  elems it receives is unspecified.
  ztt.elementwise_inline_asmr   Nr   c                *   g }	i }
d }|	j                  t        |             t        |      }t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   g }|j                  |       d }t        | )  | j                  | j                  | j                  | j                  |
||	||||	       y )
Nrv   r   
asm_stringconstraintsr  purer  packed_elementr#   r)   )r9   r:   r  r  r  r  argsr   r   r&   r$   r(   r<   r%   r=   r>   s                  r   r4   zElementwiseInlineAsmOp.__init__  s   HJGOO-d34/4Lz7,,-$$Y/ !+ i(\J |
 {G--.$$Y/ "- i(lK }
 tW&&'$$Z0  j)$E v
 ~w001$$Y/ %3 i(N   GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r  zElementwiseInlineAsmOp.args  rH   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  z!ElementwiseInlineAsmOp.asm_string  r  r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  z!ElementwiseInlineAsmOp.asm_string  r  r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  z"ElementwiseInlineAsmOp.constraints  r  r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  z"ElementwiseInlineAsmOp.constraints  r  r   c                 4    | j                   j                  d   S Nr  rJ   rK   s    r   r  zElementwiseInlineAsmOp.pure  rj  r   c                 P    |t        d      || j                  j                  d<   y NrO   r  rP   rR   s     r   r  zElementwiseInlineAsmOp.pure  rl  r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  z%ElementwiseInlineAsmOp.packed_element  r  r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  z%ElementwiseInlineAsmOp.packed_element  r  r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rD   rE   r%   rF   s     r   r:   zElementwiseInlineAsmOp.result  >    !$T^^%;%;!<q!@1!D>>!!!A(B$BCCr   )r   r   r   r_   r5   r6   r4   r`   ra   r.   rb   r  r   r  rc   r  r  r  r  r  OpResultListr:   rg   rh   s   @r   r  r    s    /.,Y]bf l6 EG)) E E 3',, 3 3 4g00 4 4
 47-- 4 4 5w11 5 5
 -G$$ - - ;;.(( . .
 7g11 7 7 8'"5"5 8 8
 Dg** D Dr   r  c          
          t        | |||||||      }|j                  }	t        |	      dkD  r|	S t        |	      dk(  r|	d   S |S )N)r:   r  r  r  r  r  r   r   rB   r   )r  r%   rD   )
r:   r  r  r  r  r  r   r   rj   r%   s
             r   elementwise_inline_asmr    sz    V
P[bf  xF  MQ  WZ  _a  b"  np  nx  nx  dkL1$SW9J71:SPRSr   c                   \    e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  ej                     fd       Z xZS )ExpandDimsOpztt.expand_dimsr   Nr(  c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   d }
t        | %  | j                  | j                  | j                  | j                  ||||
|||       y Nr  r   axisr#   r   )r9   r  r  r%   r   r   r&   r$   r(   r<   r=   r>   s              r   r4   zExpandDimsOp.__init__  s    HJGOOC/4LtW&&'$$Y/  i(|D v O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zExpandDimsOp.src$  r   r   c                 4    | j                   j                  d   S Nr  rJ   rK   s    r   r  zExpandDimsOp.axis(  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y NrO   r  rP   rR   s     r   r  zExpandDimsOp.axis,  rl  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zExpandDimsOp.result2  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   r   r  r  r  r  rc   r   r:   rg   rh   s   @r   r  r    s    #.,+/Td l &7==!9!9: & & -G'' - - ;;.++ . .
 %g&&w'?'?@ % %r   r  c                6    t        | ||||      j                  S )Nr  r  r%   r   r   )r  r:   r  s        r   expand_dimsr  6  s    	#D'sr	J	Q	QQr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                  fd       Zej                  d
e	j                  fd       Zej                  de	j                  fd       Zej                  d
e	j                  fd       Zej                  de	j"                  fd       Zej                  d
e	j"                  fd       Zej                  de	j&                  fd       Z xZS )ExternElementwiseOpz
  call an external function $symbol implemented in $libpath/$libname with $args
  return $libpath/$libname:$symbol($args...)
  ztt.extern_elementwiser   Nr   c                *   g }	i }
d }|	j                  t        |             t        |      }t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||      |
d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |
||	||||       y )	Nrv   r   libnamelibpathsymbolr  r  r#   )r*   r+   r,   r-   r.   r/   r0   r1   r2   r   r3   r4   r5   r6   r7   r8   )r9   r:   r   r  r  r  r  r   r   r&   r$   r(   r<   r%   r=   r>   s                  r   r4   zExternElementwiseOp.__init__D  s   HJGOO-d34/4Lw))*$$Y/ % i(,G y
 w))*$$Y/ % i(,G y
 vw(()$$Y/ # i(F x
 tW&&'$$Z0  j)$E v GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r   zExternElementwiseOp.srcs_  rH   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zExternElementwiseOp.libnamed  r   r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zExternElementwiseOp.libnameh  r   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zExternElementwiseOp.libpathn  r   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zExternElementwiseOp.libpathr  r   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zExternElementwiseOp.symbolx  rL   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zExternElementwiseOp.symbol|  rS   r   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zExternElementwiseOp.pure  rj  r   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zExternElementwiseOp.pure  rl  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zExternElementwiseOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   rb   r   r   r  rc   r  r  r  r  r   r:   rg   rh   s   @r   r  r  9  s   
 +.,JNSW l6 EG)) E E 0w)) 0 0 >>17-- 1 1
 0w)) 0 0 >>17-- 1 1
 /g(( / / 
==0',, 0 0
 -G$$ - - ;;.(( . .
 %g&& % %r   r  c          
      <    t        | |||||||      j                  S )Nr:   r   r  r  r  r  r   r   )r  r:   r  s           r   extern_elementwiser    s6    	FwPW`fmqwz  @B  
C  
J  
J  Jr   c                   \    e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  dee	j                     fd	       Zej                  d
ee	j                     fd       Zej                   d        Zej                  de	j"                  fd       Z xZS )FpToFpOpzv
  Floating point casting for custom types (F8), and non-default rounding modes.
  
  F8 <-> FP16, BF16, FP32, FP64
  ztt.fp_to_fpr   Nroundingr   r   c                   g }i }d }|j                  |       t        |      }	|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y )NTT_RoundingModeAttrr   r
  r#   r   )r9   r:   r  r
  r   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zFpToFpOp.__init__  s    HJGOOC/4L8W../(()>? 8@ 


!
!"7
8<
X (
3 GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zFpToFpOp.src  r   r   c                 f    d| j                   j                  vry | j                   j                  d   S Nr
  rJ   rK   s    r   r
  zFpToFpOp.rounding  s,    222>>$$Z00r   rM   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y r  rJ   rR   s     r   r
  zFpToFpOp.rounding  sE    .3dnn
+	t~~00	0
..
#
#J
/ 
1r   c                 2    | j                   j                  d= y r  rJ   rK   s    r   r
  zFpToFpOp.rounding  s    !!*-r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zFpToFpOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  rd   r/   r
  rc   rf   r   r:   rg   rh   s   @r   r  r    s     !.,.2 l &7== & & 1	'"3"34 1 1
 ??0Ig&7&78 0 0 . . %g&& % %r   r  r	  c                6    t        | ||||      j                  S )Nr:   r  r
  r   r   )r  r:   r  s        r   fp_to_fpr    s    	S8	L	S	SSr   c                   ,    e Zd ZdZdZdZddddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  fd
       Zej                   de	j                  fd       Zej                  defd       Zej                   d        Zej&                  d        Zej                  de	j(                  e	j                     fd       Z xZS )GatherOpa$  
  Gather elements from the input tensor using the indices tensor along a
  single specified axis. The output tensor has the same shape as the indices
  tensor. The input and indices tensors must have the same number of
  dimension, and each dimension of the indices tensor that is not the gather
  dimension cannot be greater than the corresponding dimension in the input
  tensor.
  
  The `efficient_layout` attribute is set when the compiler has determined an
  optimized layout for the operation, indicating that it should not be
  changed.
  z	tt.gatherr   Nefficient_layoutr%   r   r   c                (   g }i }	d }
|j                  |       |j                  |       t        |      }t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |	d<   t        |      r+t        j                  j                  t        |            |	d<   d }t        | -  | j                  | j                  | j                  | j                  |	||||
||       y )Nr  r   r  r  r#   )r   r,   r-   r.   r/   r0   r1   r2   boolUnitAttrr3   r4   r5   r6   r7   r8   )r9   r  rP  r  r  r%   r   r   r&   r$   r(   r<   r=   r>   s                r   r4   zGatherOp.__init__  s   HJGOOCOOG/4LtW&&'$$Y/  i(|D v @P@P@T@T"3'A)z*<=O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zGatherOp.src  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   rP  zGatherOp.indices  r   r   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zGatherOp.axis  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zGatherOp.axis  rl  r   c                 2    d| j                   j                  v S Nr  rJ   rK   s    r   r  zGatherOp.efficient_layout      !:!:::r   c                     t        |      r6t        j                  j                         | j                  j
                  d<   y d| j                  j
                  v r| j                  j
                  d= y y r"  r  r.   r  r2   rE   r$   rR   s     r   r  zGatherOp.efficient_layout  W    E{6=6F6F6J6J6Ldnn 23	t~~88	8
..
#
#$6
7 
9r   c                 2    | j                   j                  d= y r"  rJ   rK   s    r   r  zGatherOp.efficient_layout
      !!"45r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zGatherOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  rP  r  r  rc   r  r  rf   r   r:   rg   rh   s   @r   r  r    sX    .,=A4UY^b l  &7==!9!9: & & &w}}W%=%=> & & -G'' - - ;;.++ . .
 ; ; ; 8 8 6 6 %g&&w'?'?@ % %r   r  r  c          	      :    t        | ||||||      j                  S )Nr  rP  r  r  r%   r   r   )r  r:   r+  s          r   gatherr,    s,    	c7HXbiorwy	z  
B  
B  Br   c                       e Zd ZdZdZdddd fd
Zej                  dej                  fd       Z
e
j                  dej                  fd	       Z
ej                  dej                  ej                     fd
       Z xZS )GetNumProgramsOpztt.get_num_programsr   Nr(  c                x   g }i }d }t        |      }t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |d<   d }	t        
| !  | j                  | j                  | j                  | j                  ||||	|||       y NTT_ProgramDimr   r  r#   rw   r9   r  r%   r   r   r&   r$   r(   r<   r=   r>   s             r   r4   zGetNumProgramsOp.__init__      HJG/4LtW&&'$$_5  o.t\J v O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zGetNumProgramsOp.axis'  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zGetNumProgramsOp.axis+  rl  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zGetNumProgramsOp.result1  r   r   r   r   r   r5   r6   r4   r`   ra   r.   r/   r  rc   r   rC  r:   rg   rh   s   @r   r.  r.    s    (.,&* 
l -G%% - - ;;.)) . .
 %g&&w':':; % %r   r.  c                4    t        | |||      j                  S Nr  r%   r   r   )r.  r:   r:  s       r   get_num_programsr;  5  s    	tW#"	E	L	LLr   c                       e Zd ZdZdZdddd fd
Zej                  dej                  fd       Z
e
j                  dej                  fd	       Z
ej                  dej                  ej                     fd
       Z xZS )GetProgramIdOpztt.get_program_idr   Nr(  c                x   g }i }d }t        |      }t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||      |d<   d }	t        
| !  | j                  | j                  | j                  | j                  ||||	|||       y r0  rw   r2  s             r   r4   zGetProgramIdOp.__init__>  r3  r   r?   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zGetProgramIdOp.axisJ  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zGetProgramIdOp.axisN  rl  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zGetProgramIdOp.resultT  r   r   r7  rh   s   @r   r=  r=  8  s    &.,&* 
l -G%% - - ;;.)) . .
 %g&&w':':; % %r   r=  c                4    t        | |||      j                  S r9  )r=  r:   r:  s       r   get_program_idrC  X  s    	T7	C	J	JJr   c                   (    e Zd ZdZdZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  dee	j                     fd	       Zej                  de	j                  e	j                     fd
       Z xZS )HistogramOpz
  Return the histogram of the input tensor. The number of bins is equal to
  the dimension of the output tensor. Each bins has a width of 1 and bins
  start at 0.
  ztt.histogramr   Nr   c                   g }i }d }|j                  |       ||j                  |       t        |      }	g }
|
j                  |       d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y r   r   )r9   r:   r  r   r   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zHistogramOp.__init__g  s    HJGOOC./4LGNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zHistogramOp.srcs  r   r   c                 |    t        | j                  j                        dk  rd S | j                  j                  d   S r   rC   rK   s    r   r   zHistogramOp.maskw  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zHistogramOp.result{  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  rd   r   r   r:   rg   rh   s   @r   rE  rE  [  s     ".,*.DT 
l &7==!9!9: & & TIgmm, T T %g&&w'?'?@ % %r   rE  c                6    t        | ||||      j                  S )Nr:   r  r   r   r   )rE  r:   rK  s        r   	histogramrL    s    	F$CB	G	N	NNr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )	
IntToPtrOpztt.int_to_ptrr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y r   r   r  s              r   r4   zIntToPtrOp.__init__  r  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zIntToPtrOp.src  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zIntToPtrOp.result  r   r   r  rh   s   @r   rN  rN    d    ".,)-$ 	l &7== & & %g&& % %r   rN  c                4    t        | |||      j                  S r  )rN  r:   r  s       r   
int_to_ptrrT        	6s	;	B	BBr   c                   :    e Zd ZdZdZdZddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Z xZS )JoinOpz
  For example, if the two input tensors are 4x8xf32, returns a tensor of
  shape 4x8x2xf32.
  
  Because Triton tensors always have a power-of-two number of elements,
  the two input tensors must have the same shape.
  ztt.joinr   Nr   c                   g }i }d }|j                  |       |j                  |       t        |      }	g }
|
j                  |       d }t        |   | j                  | j
                  | j                  | j                  ||
|||||       y r   r   r  s               r   r4   zJoinOp.__init__  r  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  z
JoinOp.lhs  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  z
JoinOp.rhs  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zJoinOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  r  r   r:   rg   rh   s   @r   rW  rW    s     .,.2t 
l &7==!9!9: & & &7==!9!9: & & %g&&w'?'?@ % %r   rW  c                6    t        | ||||      j                  S r#  )rW  r:   r$  s        r   joinr]    s    	v3CSR	@	G	GGr   c                       e Zd ZdZg dZdZddddddddddd
 fd
Zej                  de	j                  fd       Zej                  dee	j                     fd	       Zej                  dee	j                     fd
       Zej                  de	j                  fd       Zej"                  de	j                  fd       Zej                  dee	j$                     fd       Zej"                  dee	j$                     fd       Zej(                  d        Zej                  de	j$                  fd       Zej"                  de	j$                  fd       Zej                  de	j$                  fd       Zej"                  de	j$                  fd       Zej                  de	j.                  fd       Zej"                  de	j.                  fd       Zej                  de	j2                  fd       Z xZS )LoadOpztt.load)rB   r   r   r   N)
r   otherboundaryCheckpaddingrJ  rK  
isVolatiler%   r   r   c       
            g }i }d }|j                  |       |j                  |       |j                  |       t        |
      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |d	<   |dt        |t        j                        st        j
                  j                  d
      s|n%t        j
                  j                  d
      ||      |d<   d }t        | %  | j                  | j                  | j                  | j                  ||	||||
|       y )NDenseI32ArrayAttrr   ra  TT_PaddingOptionAttrrb  rM  rJ  rN  rK  r  rc  r#   r   )r9   r   r   r`  ra  rb  rJ  rK  rc  r%   r   r   r&   r$   r(   r<   r=   r>   s                    r   r4   zLoadOp.__init__  sk   HJGOOCOODOOE/4L ='"3"34(()<= BO 


!
!"5
6}l
[ #-_"= 7G--.(()?@ 6= 


!
!"8
9'<
X 'y1 5'++,(()?@ 27 


!
!"8
9%
V %W- 5'++,(()@A 27 


!
!"9
:5,
W %W- :w001((4 <F 


!
!*
-j,
O  *,7 O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 |    t        | j                  j                  | j                  j                  d   d      }|d   S r  r  r  s     r   r   z
LoadOp.ptr  r  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S r  r  r  s     r   r   zLoadOp.mask  r  r   c                     t        | j                  j                  | j                  j                  d   d      }t	        |      dkD  r|d   S d S r  r  r  s     r   r`  zLoadOp.other  r  r   c                 4    | j                   j                  d   S Nra  rJ   rK   s    r   ra  zLoadOp.boundaryCheck  r}   r   rM   c                 P    |t        d      || j                  j                  d<   y NrO   ra  rP   rR   s     r   ra  zLoadOp.boundaryCheck  r   r   c                 f    d| j                   j                  vry | j                   j                  d   S Nrb  rJ   rK   s    r   rb  zLoadOp.padding  s,    111>>$$Y//r   c                     ||| j                   j                  d<   y d| j                   j                  v r| j                   j                  d= y y ro  rJ   rR   s     r   rb  zLoadOp.padding  sE    -2dnn	*	dnn//	/
..
#
#I
. 
0r   c                 2    | j                   j                  d= y ro  rJ   rK   s    r   rb  zLoadOp.padding  s    !!),r   c                 4    | j                   j                  d   S rT  rJ   rK   s    r   rJ  zLoadOp.cache  r   r   c                 P    |t        d      || j                  j                  d<   y rV  rP   rR   s     r   rJ  zLoadOp.cache"  r   r   c                 4    | j                   j                  d   S rX  rJ   rK   s    r   rK  zLoadOp.evict(  r   r   c                 P    |t        d      || j                  j                  d<   y rZ  rP   rR   s     r   rK  zLoadOp.evict,  r   r   c                 4    | j                   j                  d   S )Nrc  rJ   rK   s    r   rc  zLoadOp.isVolatile2  r  r   c                 P    |t        d      || j                  j                  d<   y )NrO   rc  rP   rR   s     r   rc  zLoadOp.isVolatile6  r  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zLoadOp.result<  r   r   )r   r   r   r5   r7   r6   r4   r`   ra   r.   r   r   rd   r   r`  re  ra  rc   r/   rb  rf   rJ  rK  r  rc  r   r:   rg   rh   s   @r   r_  r_    s   .","&d$PT\`hly}  HL  RV  [_ l> 7==   @Igmm, @ @ @Yw}}- @ @ 6W66 6 6 7!:!: 7 7
 0y!2!23 0 0
 >>/9W%6%67 / / ??- - .W&& . . 	<</** / /
 .W&& . . 	<</** / /
 3'** 3 3 4g.. 4 4
 %g&& % %r   r_  )
r   r`  boundary_checkrb  rJ  rK  is_volatiler%   r   r   c       
         B    t        | |||||||||	|
      j                  S )N)r   r   r`  ra  rb  rJ  rK  rc  r%   r   r   )r_  r:   )r   r   r`  ry  rb  rJ  rK  rz  r%   r   r   s              r   loadr|  @  sJ    	Cd%~W^fksx  FQ  [b  hk  pr  
s  
z  
z  zr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  d	e	j                  fd
       Zej                  de	j                  fd       Zej                  d	e	j                  fd       Zej                  de	j                  e	j                     fd       Z xZS )MakeRangeOpza
  Returns an 1D int32 tensor.
  
  Values span from $start to $end (exclusive), with step = 1
  ztt.make_ranger   Nr   c                f   g }i }d }t        |      }	t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||	      |d<   t        |t        j                        st        j                  j                  d      s|n%t        j                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | %  | j                  | j                  | j                  | j                  ||
|||||       y )Nr  r   startendr#   )r,   r-   r.   r/   r0   r1   r2   r   r3   r4   r5   r6   r7   r8   )r9   r:   r  r  r   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zMakeRangeOp.__init__O  s9   HJG/4Lug''($$Y/ ! i(E w
 sG%%&$$Y/  i(lC u GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zMakeRangeOp.starta  r   r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zMakeRangeOp.starte  r   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zMakeRangeOp.endk  r   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zMakeRangeOp.endo  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zMakeRangeOp.resultu  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r  r  rc   r  r   r  r:   rg   rh   s   @r   r~  r~  C  s     #.,04 l$ .W(( . . 	<</,, / /
 ,7&& , , ::-w** - -
 %g&&w'?'?@ % %r   r~  c                6    t        | ||||      j                  S )Nr:   r  r  r   r   )r~  r:   r  s        r   
make_ranger  y  s    	F%Scb	I	P	PPr   c                       e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                  de	j                  fd       Zej"                  de	j                  fd       Zej                  de	j$                  fd       Z xZS )MakeTensorDescOpz
  `tt.make_tensor_descriptor` takes both meta information of the parent tensor and the block size,
  and returns a descriptor object which can be used to load/store from the tensor in global memory.
  ztt.make_tensor_descriptorr   Nrb  r   r   c                ,   g }i }	d }
|j                  |       |j                  t        |             |j                  t        |             t        |      }|dt	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |	d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |	||||
||       y )Nrf  r   rb  r#   rO  )r9   r:   baseshapestridesrb  r   r   r&   r$   r(   r<   r%   r=   r>   s                 r   r4   zMakeTensorDescOp.__init__  s   HJGOODOO-e45OO-g67/4L7G--.(()?@ 6= 


!
!"8
9'<
X 'y1 GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                  dddd      \  }}| j                  j                  |   S NrB   r   r   _ods_equally_sized_accessorrE   r&   r9   r  elements_per_groups      r   r  zMakeTensorDescOp.base  =     ;DNN<S<SUVXY[\^_ `E>>""5))r   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S r  r  r  s      r   r  zMakeTensorDescOp.shape  E     ;DNN<S<SUVXY[\^_ `E>>""51C)CDDr   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S )NrB   r   r  r  s      r   r  zMakeTensorDescOp.strides  r  r   c                 4    | j                   j                  d   S ro  rJ   rK   s    r   rb  zMakeTensorDescOp.padding  r   r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   rb  rP   rR   s     r   rb  zMakeTensorDescOp.padding  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zMakeTensorDescOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  rb   r  r  r/   rb  rc   r   r:   rg   rh   s   @r   r  r  |  s   
 /.,>BQU l" *GMM * * EW** E E Ew,, E E 0w(( 0 0 >>17,, 1 1
 %g&& % %r   r  r  c          	      :    t        | ||||||      j                  S )Nr:   r  r  r  rb  r   r   )r  r:   r  s          r   make_tensor_descriptorr    s"    	d%Zagjoq	r	y	yyr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Zej                  de	j                  fd       Zej                  de	j                   fd       Zej$                  de	j                   fd       Zej                  de	j&                  fd       Z xZS )MakeTensorPtrOpz
  `tt.make_tensor_ptr` takes both meta information of the parent tensor and the block tensor, then it returns a
  pointer to the block tensor, e.g. returns a type of `tt.ptr<tensor<8x8xf16>>`.
  ztt.make_tensor_ptrr   Nr   c                \   g }	i }
d }|	j                  |       |	j                  t        |             |	j                  t        |             |	j                  t        |             t        |      }t	        |t
        j                        st
        j                  j                  d      s|n%t        j                  j                  d      ||      |
d<   g }|j                  |       d }t        | -  | j                  | j                  | j                  | j                  |
||	||||       y Nre  r   orderr#   rO  )r9   r:   r  r  r  r   r  r   r   r&   r$   r(   r<   r%   r=   r>   s                  r   r4   zMakeTensorPtrOp.__init__  s+   HJGOODOO-e45OO-g67OO-g67/4Lug''($$%89 ! 125,O w GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                  dddd      \  }}| j                  j                  |   S NrB   r   r   r  r  s      r   r  zMakeTensorPtrOp.base  r  r   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S r  r  r  s      r   r  zMakeTensorPtrOp.shape  r  r   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S )NrB   r   r  r  s      r   r  zMakeTensorPtrOp.strides  r  r   c                     t        | j                  j                  dddd      \  }}| j                  j                  |||z    S )NrB   r   r   r  r  s      r   r   zMakeTensorPtrOp.offsets  r  r   c                 4    | j                   j                  d   S Nr  rJ   rK   s    r   r  zMakeTensorPtrOp.order  r   r   rM   c                 P    |t        d      || j                  j                  d<   y NrO   r  rP   rR   s     r   r  zMakeTensorPtrOp.order  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zMakeTensorPtrOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  rb   r  r  r   re  r  rc   r   r:   rg   rh   s   @r   r  r    s6   
 (.,JNSW l$ *GMM * * EW** E E Ew,, E E Ew,, E E .W.. . . 	<</22 / /
 %g&& % %r   r  c          
      <    t        | |||||||      j                  S )Nr:   r  r  r  r   r  r   r   )r  r:   r  s           r   make_tensor_ptrr    s.    	TY`hmsv{}	~  
F  
F  Fr   c                   d    e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  dej                   fd       Z xZS )MapElementwiseOpztt.map_elementwisern   Nr   c                   g }i }d }|j                  t        |             t        |      }	t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | )  | j                  | j                  | j                  | j                  ||
|||||       y )Nr  r   packr#   r)   )r9   r:   r   r  r   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zMapElementwiseOp.__init__       HJGOO-d34/4LtW&&'$$Y/  i(|D v GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r   zMapElementwiseOp.srcs  rH   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zMapElementwiseOp.pack  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zMapElementwiseOp.pack  rl  r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   r  rF   s     r   r:   zMapElementwiseOp.result  r  r   c                      | j                   d   S r   r   rK   s    r   scalarOpzMapElementwiseOp.scalarOp#  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   rb   r   r  r  rc   r  r:   r   r  rg   rh   s   @r   r  r    s    '.,04 l EG)) E E -G'' - - ;;.++ . .
 Dg** D D   r   r  c                    t        | ||||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )N)r:   r   r  r   r   rB   r   )r  r%   rD   )r:   r   r  r   r   rj   r%   s          r   map_elementwiser  '  sJ    vDtL"XZXbXbgL1$SW9J71:SPRSr   c                   l     e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
 xZS )MapElementwiseReturnOpztt.map_elementwise.returnr   Nr   c                    g }i }d }|j                  t        |             t        |      }g }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y r   r   r9   r:   r   r   r&   r$   r(   r<   r%   r=   r>   s             r   r4   zMapElementwiseReturnOp.__init__1      HJGOO-f56/4LGO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r:   zMapElementwiseReturnOp.result;  rH   r   r   r   r   r5   r6   r4   r`   ra   r.   rb   r:   rg   rh   s   @r   r  r  +  sB    ..,$(T l Eg++ E Er   r  c                    t        | ||      S Nr:   r   r   )r  r  s      r   map_elementwise_returnr  @  s    	v32	>>r   c                        e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Z xZS )	MulhiUIOpzD
  Most significant N bits of the 2N-bit product of two integers.
  z
tt.mulhiuir   Nr(  c                   g }i }d }|j                  |       |j                  |       t        |      }	||d   j                  gdz  }d }
t        |   | j
                  | j                  | j                  | j                  ||||
|||       y Nr   rB   r#   	r   r,   r,  r3   r4   r5   r6   r7   r8   r9   r-  yr%   r   r   r&   r$   r(   r<   r=   r>   s              r   r4   zMulhiUIOp.__init__M      HJGOOAOOA/4L8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r-  zMulhiUIOp.xX  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zMulhiUIOp.y\  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zMulhiUIOp.result`  r   r   r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r-  r  r   r:   rg   rh   s   @r   r  r  C  s      .,&* 	l & & & & & & %g&& % %r   r  c                6    t        | ||||      j                  S Nr-  r  r%   r   r   )r  r:   r  s        r   mulhiuir  d  s    	Q!W#"	=	D	DDr   c                        e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  de	j                  fd
       Z xZS )PreciseDivFOpz+
  Precise div for floating point types.
  ztt.precise_divfr   Nr(  c                   g }i }d }|j                  |       |j                  |       t        |      }	||d   j                  gdz  }d }
t        |   | j
                  | j                  | j                  | j                  ||||
|||       y r  r  r  s              r   r4   zPreciseDivFOp.__init__q  r  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r-  zPreciseDivFOp.x|  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zPreciseDivFOp.y  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zPreciseDivFOp.result  r   r   r  rh   s   @r   r  r  g  s     %.,&* 	l & & & & & & %g&& % %r   r  c                6    t        | ||||      j                  S r  )r  r:   r  s        r   precise_divfr    s    	acb	A	H	HHr   c                        e Zd ZdZdZdZdddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Z xZS )
PreciseSqrtOpz,
  Precise sqrt for floating point types.
  ztt.precise_sqrtr   Nr(  c                    g }i }d }|j                  |       t        |      }||d   j                  gdz  }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y r  r  )r9   r-  r%   r   r   r&   r$   r(   r<   r=   r>   s             r   r4   zPreciseSqrtOp.__init__  s    HJGOOA/4L8A;#3#3"4q"8O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r-  zPreciseSqrtOp.x  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zPreciseSqrtOp.result  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r-  r   r:   rg   rh   s   @r   r  r    sm     %.,#'Td l & & & %g&& % %r   r  c                4    t        | |||      j                  S )Nr-  r%   r   r   )r  r:   r  s       r   precise_sqrtr    s    	G	<	C	CCr   c                       e Zd ZdZdZdZddd fd
Zej                  de	j                  fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                  fd       Zej                  d
e	j                  fd       Zej                  de	j"                  fd       Zej                  d
e	j"                  fd       Z xZS )PrintOpz
  `tt.print` takes a literal string prefix and an arbitrary number of scalar or tensor arguments that should be printed.
  format are generated automatically from the arguments.
  ztt.printr   Nr   c                @   g }i }d }	|j                  t        |             t        |      }
t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   g }d }t        | )  | j                  | j                  | j                  | j                  |||||	||       y )	Nrv   r   prefixr  hexre  isSignedr#   r)   )r9   r  r  r  r  r   r   r&   r$   r(   r<   r%   r=   r>   s                r   r4   zPrintOp.__init__  s   HJGOO-d34/4Lvw(()$$Y/ # i(F x
 sG%%&$$Z0  j)#|D u
 x**+$$%89 ' 128\R z GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r  zPrintOp.args  rH   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zPrintOp.prefix  rL   r   rM   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zPrintOp.prefix  rS   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zPrintOp.hex  r   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zPrintOp.hex  r   r   c                 4    | j                   j                  d   S )Nr  rJ   rK   s    r   r  zPrintOp.isSigned  ry   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r  rP   rR   s     r   r  zPrintOp.isSigned  r{   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   rb   r  r   r  rc   r  r  re  r  rg   rh   s   @r   r  r    s-   
 .,9=$ l, EG)) E E /g(( / / 
==0',, 0 0
 ,7## , , ::-w'' - -
 111 1 1 ??2G55 2 2r   r  c                $    t        | |||||      S )N)r  r  r  r  r   r   )r  )r  r  r  	is_signedr   r   s         r   print_r    s    	CdYCTV	WWr   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Z xZS )	
PtrToIntOpztt.ptr_to_intr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y r   r   r  s              r   r4   zPtrToIntOp.__init__  r  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zPtrToIntOp.src	  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zPtrToIntOp.result	  r   r   r  rh   s   @r   r  r    rR  r   r  c                4    t        | |||      j                  S r  )r  r:   r  s       r   
ptr_to_intr  
	  rU  r   c                   d    e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  dej                   fd       Z xZS )ReduceOpz	tt.reducern   Nr   c                   g }i }d }|j                  t        |             t        |      }	t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||	      |d<   g }
|
j                  |       d }t        | )  | j                  | j                  | j                  | j                  ||
|||||       y r  r)   )r9   r:   r   r  r   r   r&   r$   r(   r<   r%   r=   r>   s               r   r4   zReduceOp.__init__	  r  r   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r   zReduceOp.srcs"	  rH   r   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zReduceOp.axis'	  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zReduceOp.axis+	  rl  r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   r  rF   s     r   r:   zReduceOp.result1	  r  r   c                      | j                   d   S r   r   rK   s    r   	combineOpzReduceOp.combineOp6	  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   rb   r   r  r  rc   r  r:   r   r
  rg   rh   s   @r   r  r  	  s    .,04 l EG)) E E -G'' - - ;;.++ . .
 Dg** D D   r   r  c                    t        | ||||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )N)r:   r   r  r   r   rB   r   )r  r%   rD   )r:   r   r  r   r   rj   r%   s          r   reducer  :	  sJ    vDtD"PRPZPZgL1$SW9J71:SPRSr   c                   l     e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
 xZS )ReduceReturnOpztt.reduce.returnr   Nr   c                    g }i }d }|j                  t        |             t        |      }g }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y r   r   r  s             r   r4   zReduceReturnOp.__init__D	  r  r   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r:   zReduceReturnOp.resultN	  rH   r   r  rh   s   @r   r  r  >	  sB    %.,$(T l Eg++ E Er   r  c                    t        | ||      S r  )r  r  s      r   reduce_returnr  S	  s    	v32	66r   c                       e Zd ZdZdZdZddddd fd
Zej                  de	j                  e	j                     fd       Zej                  defd	       Zej                  d
        Zej                   d        Zej                  defd       Zej                  d        Zej                   d        Zej                  de	j$                  e	j                     fd       Z xZS )	ReshapeOpaZ  
  reinterpret a tensor to a different shape.
  
  If allow_reorder is set the compiler is free to change the order of
  elements to generate more efficient code.
  
  If efficient_layout is set, this is a hint that the destination layout should be kept for performance reason.
  The compiler is still free to change it for better performance.
  z
tt.reshaper   Nallow_reorderr  r   r   c                   g }i }d }	|j                  |       t        |      }
t        |      r+t        j                  j                  t        |            |d<   t        |      r+t        j                  j                  t        |            |d<   g }|j                  |       d }t        |   | j                  | j                  | j                  | j                  |||||	||       y )Nr  r  r#   )r   r,   r  r.   r  r2   r3   r4   r5   r6   r7   r8   )r9   r:   r  r  r  r   r   r&   r$   r(   r<   r%   r=   r>   s                r   r4   zReshapeOp.__init__f	  s    HJGOOC/4LM':J:J:N:N"3';)J7@P@P@T@T"3'A)z*<=GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zReshapeOp.srcu	  r   r   c                 2    d| j                   j                  v S Nr  rJ   rK   s    r   r  zReshapeOp.allow_reordery	  s    dnn7777r   c                     t        |      r6t        j                  j                         | j                  j
                  d<   y d| j                  j
                  v r| j                  j
                  d= y y r  r%  rR   s     r   r  zReshapeOp.allow_reorder}	  sU    E{3:3C3C3G3G3Idnn0	DNN55	5
..
#
#O
4 
6r   c                 2    | j                   j                  d= y r  rJ   rK   s    r   r  zReshapeOp.allow_reorder	  s    !!/2r   c                 2    d| j                   j                  v S r"  rJ   rK   s    r   r  zReshapeOp.efficient_layout	  r#  r   c                     t        |      r6t        j                  j                         | j                  j
                  d<   y d| j                  j
                  v r| j                  j
                  d= y y r"  r%  rR   s     r   r  zReshapeOp.efficient_layout	  r&  r   c                 2    | j                   j                  d= y r"  rJ   rK   s    r   r  zReshapeOp.efficient_layout	  r(  r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zReshapeOp.result	  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  r  r  rc   rf   r  r   r:   rg   rh   s   @r   r  r  V	  s1     .,37$TX]a l &7==!9!9: & & 8T 8 8 5 5 3 3 ; ; ; 8 8 6 6 %g&&w'?'?@ % %r   r  r  c                8    t        | |||||      j                  S )Nr:   r  r  r  r   r   )r  r:   r"  s         r   reshaper#  	  s)    	&cYiorwy	z  
B  
B  Br   c                       e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  d	ej                  fd
       Zej                  dej                  fd       Zej                  d	ej                  fd       Zej                  dej                   fd       Zej                  dej$                  fd       Z xZS )ScanOpztt.scanrn   Nr   c                   g }i }d }	|j                  t        |             t        |      }
t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   t        |t        j
                        st        j                  j                  d      s|n%t	        j                  j                  d      ||
      |d<   g }|j                  |       d }t        | )  | j                  | j                  | j                  | j                  |||||	||       y )Nr  r   r  r  reverser#   r)   )r9   r:   r   r  r'  r   r   r&   r$   r(   r<   r%   r=   r>   s                r   r4   zScanOp.__init__	  sK   HJGOO-d34/4LtW&&'$$Y/  i(|D v
 w))*$$Z0 % j)'<H y GNN6O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r   zScanOp.srcs	  rH   r   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zScanOp.axis	  rj  r   rM   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zScanOp.axis	  rl  r   c                 4    | j                   j                  d   S )Nr'  rJ   rK   s    r   r'  zScanOp.reverse	  r   r   c                 P    |t        d      || j                  j                  d<   y )NrO   r'  rP   rR   s     r   r'  zScanOp.reverse	  r   r   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   r  rF   s     r   r:   zScanOp.result	  r  r   c                      | j                   d   S r   r   rK   s    r   r
  zScanOp.combineOp	  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   rb   r   r  r  rc   r  r'  r  r:   r   r
  rg   rh   s   @r   r%  r%  	  s)   .,9=$ l& EG)) E E -G'' - - ;;.++ . .
 0w'' 0 0 >>17++ 1 1
 Dg** D D   r   r%  c                    t        | |||||      }|j                  }t        |      dkD  r|S t        |      dk(  r|d   S |S )N)r:   r   r  r'  r   r   rB   r   )r%  r%   rD   )r:   r   r  r'  r   r   rj   r%   s           r   scanr0  	  sN    V$T7PRS"_a_i_iU\L1$SW9J71:SPRSr   c                   l     e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
 xZS )ScanReturnOpztt.scan.returnr   Nr   c                    g }i }d }|j                  t        |             t        |      }g }d }	t        
|   | j
                  | j                  | j                  | j                  ||||	|||       y r   r   r  s             r   r4   zScanReturnOp.__init__	  r  r   r?   c                     t        | j                  j                        dz
  dz   }| j                  j                  dd|z    S rA   rC   rF   s     r   r:   zScanReturnOp.result	  rH   r   r  rh   s   @r   r2  r2  	  sB    #.,$(T l Eg++ E Er   r2  c                    t        | ||      S r  )r2  r  s      r   scan_returnr6  	  s    	V	44r   c                        e Zd ZdZdZddd fd
Zej                  dej                  fd       Z
ej                  dej                  ej                     fd       Z xZS )	SplatOpztt.splatr   Nr   c                    g }i }d }|j                  |       t        |      }g }	|	j                  |       d }
t        |   | j                  | j
                  | j                  | j                  ||	||
|||       y r   r   r  s              r   r4   zSplatOp.__init__	  r  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zSplatOp.src
  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zSplatOp.result

  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   r   r  r   r  r:   rg   rh   s   @r   r8  r8  	  so    .,)-$ 	l &7== & & %g&&w'?'?@ % %r   r8  c                4    t        | |||      j                  S r  )r8  r:   r  s       r   splatr=  
  s    	CSR	8	?	??r   c                   <    e Zd ZdZdZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  e	j                     fd	       Zej                  de	j                  e	j                     fd
       Z xZS )SplitOpz
  The input must be a tensor whose last dimension has size 2.  Returns two
  tensors, src[..., 0] and src[..., 1].
  
  For example, if the input shape is 4x8x2xf32, returns two tensors of
  shape 4x8xf32.
  ztt.splitr   Nr(  c                    g }i }d }|j                  |       t        |      }d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y r   r   r9   r  r%   r   r   r&   r$   r(   r<   r=   r>   s             r   r4   zSplitOp.__init__
      HJGOOC/4LO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zSplitOp.src(
  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   outLHSzSplitOp.outLHS,
  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   outRHSzSplitOp.outRHS0
  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  r   rE  rG  rg   rh   s   @r   r?  r?  
  s     .,%)t l &7==!9!9: & & %g&&w'?'?@ % % %g&&w'?'?@ % %r   r?  c                4    t        | |||      j                  S Nr  r%   r   r   )r?  r%   rJ  s       r   splitrK  4
  s    	S'sr	:	B	BBr   c                   j    e Zd ZdZdZddddddd fd
Zej                  dej                  fd       Z
ej                  dej                  fd       Zej                  deej                     fd	       Zej                  dej                  fd
       Zej                   dej                  fd       Zej                  dej"                  fd       Zej                   dej"                  fd       Zej                  dej"                  fd       Zej                   dej"                  fd       Z xZS )StoreOpztt.storer   N)r   ra  rJ  rK  r   r   c                   g }	i }
d }|	j                  |       |	j                  |       ||	j                  |       t        |      }|dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   |dt        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||      |
d<   g }d }t        | %  | j                  | j                  | j                  | j                  |
||	||||       y )	Nre  r   ra  rM  rJ  rN  rK  r#   r   )r9   r   rM   r   ra  rJ  rK  r   r   r&   r$   r(   r<   r%   r=   r>   s                  r   r4   zStoreOp.__init__=
  s   HJGOOCOOE./4L ='"3"34(()<= BO 


!
!"5
6}l
[ #-_"= 5'++,(()?@ 27 


!
!"8
9%
V %W- 5'++,(()@A 27 


!
!"9
:5,
W %W- GO	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r   zStoreOp.ptrU
  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   rM   zStoreOp.valueY
  r   r   c                 |    t        | j                  j                        dk  rd S | j                  j                  d   S r   rC   rK   s    r   r   zStoreOp.mask]
  r   r   c                 4    | j                   j                  d   S rk  rJ   rK   s    r   ra  zStoreOp.boundaryChecka
  r}   r   rM   c                 P    |t        d      || j                  j                  d<   y rm  rP   rR   s     r   ra  zStoreOp.boundaryChecke
  r   r   c                 4    | j                   j                  d   S rT  rJ   rK   s    r   rJ  zStoreOp.cachek
  r   r   c                 P    |t        d      || j                  j                  d<   y rV  rP   rR   s     r   rJ  zStoreOp.cacheo
  r   r   c                 4    | j                   j                  d   S rX  rJ   rK   s    r   rK  zStoreOp.evictu
  r   r   c                 P    |t        d      || j                  j                  d<   y rZ  rP   rR   s     r   rK  zStoreOp.evicty
  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   r   r   rM   rd   r   re  ra  rc   r/   rJ  rK  rg   rh   s   @r   rM  rM  7
  s   .,)-TUY_chl l0 &7== & & &W]] & & TIgmm, T T 6W66 6 6 7!:!: 7 7
 .W&& . . 	<</** / /
 .W&& . . 	<</** / /r   rM  )r   ry  rJ  rK  r   r   c          
      (    t        | |||||||      S )N)r   rM   r   ra  rJ  rK  r   r   )rM  )r   rM   r   ry  rJ  rK  r   r   s           r   storerY  
  s    	SDV[chnqvx	yyr   c                   `    e Zd ZdZdZdZdddd fd
Zej                  de	j                  e	j                     fd       Zej                  de	j                  fd	       Zej                  d
e	j                  fd       Zej                  de	j                   e	j                     fd       Z xZS )TransOpa  
  For example, given a tensor x with shape [1,2,4], transpose(x) with
  order=[2,0,1] rearranges the tensor to have shape [4,1,2].
  
  Although this op is called "trans", it implements both tl.trans() and
  tl.permute().  ("permute" might be a better name, but it's called "trans"
  because originally it only supported 2D tensors.)
  
  ## Implementation note on encodings:
  
  In the TritonGPU dialect (and probably others), an encoding is chosen for
  this op's output so it's a nop from the perspective of code generation.
  
  For example, suppose tensor x has an encoding such that GPU thread [i,j,k]
  has a register containing element [i,j,k] of the tensor.  Now we transpose
  x with order [2,1,0], i.e. we reverse the order of its dimensions.  In
  TritonGPU, we will choose a layout for the output of the transpose so that
  GPU thread [i,j,k] has element [k,j,i] of transpose(x).  But this is the
  same element it had before!  All we've done is "rename" the element that
  thread [i,j,k] has.
  
  The "real" transpose -- i.e. moving data between GPU threads -- occurs in
  convertLayout ops that appear before and/or after the operation.
  
  We do this so that you can chain multiple data-movement ops (e.g.
  transpose+reshape+concat) without going to shared memory after each one.
  ztt.transr   Nr(  c                   g }i }d }|j                  |       t        |      }	t        |t        j                        st        j
                  j                  d      s|n%t        j
                  j                  d      ||	      |d<   d }
t        | %  | j                  | j                  | j                  | j                  ||||
|||       y r  r   )r9   r  r  r%   r   r   r&   r$   r(   r<   r=   r>   s              r   r4   zTransOp.__init__
  s    HJGOOC/4Lug''($$%89 ! 125,O w O	GT(($*;*;T=W=WY]YrYr  @J  T[  fn  {J  T[  ad  ik  lr   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zTransOp.src
  r   r   c                 4    | j                   j                  d   S r  rJ   rK   s    r   r  zTransOp.order
  r   r   rM   c                 P    |t        d      || j                  j                  d<   y r  rP   rR   s     r   r  zTransOp.order
  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zTransOp.result
  r   r   )r   r   r   r_   r5   r6   r4   r`   ra   r.   r   r  r  re  r  rc   r   r:   rg   rh   s   @r   r[  r[  
  s    8 .,,0dt l &7==!9!9: & & .W.. . . 	<</22 / /
 %g&&w'?'?@ % %r   r[  c                6    t        | ||||      j                  S )Nr  r  r%   r   r   )r[  r:   rb  s        r   transrc  
  s    	SwCB	G	N	NNr   c                        e Zd ZdZdZdddd fd
Zej                  dej                  ej                     fd       Zej                  dej                  fd       Z xZS )		UnsplatOpz
tt.unsplatr   Nr(  c                    g }i }d }|j                  |       t        |      }d }	t        
|   | j                  | j
                  | j                  | j                  ||||	|||       y r   r   rA  s             r   r4   zUnsplatOp.__init__
  rB  r   r?   c                 4    | j                   j                  d   S r   r   rK   s    r   r  zUnsplatOp.src
  r   r   c                 4    | j                   j                  d   S r   r   rK   s    r   r:   zUnsplatOp.result
  r   r   )r   r   r   r5   r6   r4   r`   ra   r.   r   r  r  r   r:   rg   rh   s   @r   re  re  
  sq    .,%)t l &7==!9!9: & & %g&& % %r   re  c                4    t        | |||      j                  S rI  )re  r:   rJ  s       r   unsplatrj  
  s    	sG	<	C	CCr   ) jaxlib.mlir.dialects._ods_commonr   	_ods_cextr   r  r   r,   r   r+   r   r  irr.   globals!register_traceback_file_exclusion__file__r`   typingr   	_Sequencer	   _Unionr
   rd   register_dialectDialectr   register_operationOpViewr   r   r  rk   rm   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r%  r'  r8  r:  rF  rH  r^  r`  ro  rq  ry  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r,  r.  r;  r=  rC  rE  rL  rN  rT  rW  r]  r_  r|  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r%  r0  r2  r6  r8  r=  r?  rK  rM  rY  r[  rc  re  rj  r   r   r   <module>rx     s  
 @  ,, 	   3 3H =  P P w   h'X/W^^ X/ (X/t 264TX T]cdkdtdtv}  wK  wK  MS  eS  ^T T h'QW^^ Q (Qf 59DTX^bgk Ypv Y h'"Ew~~ "E ("EH $ -8 - h'%w~~ % (%: (, P9I9I P h'% % (%< *.$ S7;K;K S h'$1w~~ $1 ($1L (, H H h'F%'.. F% (F%P :>$ l7K[K[ l h'P%'.. P% (P%d EIdW[ K`g`p`p K h'% % (%0 !% B'2B2B B h'%'.. % (%< #'4 DG4D4D D h'%GNN % (%: "&$ G73C3C G h'3%w~~ 3% (3%j 37DT mgN^N^ m h')% )% ()%V AE uRYRbRb u h'>%w~~ >% (>%@ 59$SW v\c\l\l v h'0. 0. (0.d 8< \I[ \ h')&'.. )& ()&V ?Ct iPc i h'"E "E ("EH 15 PBS P h'E%GNN E% (E%N %)dX\ae Pjqjzjz P h'F%'.. F% (F%P LPY]jn{  FJ  OS V  X_  Xh  Xh V h'WDW^^ WD (WDr `dhl Tqwx  yI  yI  KR  K_  K_  aw  yw  rx T h'"%7>> "% ("%H '+ R'BRBR R h'T%'.. T% (T%l MQUY J^e^n^n J h'1%w~~ 1% (1%f '+ T'BRBR T h'E%w~~ E% (E%N 484TX B]d]m]m B h'%w~~ % (%> '+ M'BRBR M h'%W^^ % (%> %)dt K@P@P K h'!%'.. !% (!%F $(Td Ow?O?O O h'% % (%0 $(D CW5E5E C h'#%W^^ #% (#%J #'4 HG4D4D H h'v%W^^ v% (v%p $tTQU]aos  ~B  HL  QU z  Za  Zj  Zj z h'3%'.. 3% (3%j +/4 QG<L<L Q h'6%w~~ 6% (6%p EIdW[ z`g`p`p z h'<%gnn <% (<%| JNRV F[b[k[k F h'*w~~ * (*X 04 THXHXZaZnZn  qA  IA  BB T h'EW^^ E (E( +/4 ?<R ? h'% % (%@ "t E9I9I E h'%GNN % (%@ #'DT Ig>N>N I h'%GNN % (%6  $$ D7;K;K D h'A2gnn A2 (A2F 15 X' X h'% % (%0 $(D CW5E5E C h'*w~~ * (*X '+t Tw?O?OQXQeQego?o8p T h'EW^^ E (E( "&$ 7> 7 h'B% B% (B%H +/4TX B]d]m]m B h'8W^^ 8 (8t .2d TvgFVFVX_XlXlntFt?u T h'E7>> E (E(  $ 5 5 h'%gnn % (%0 #t @0@0@ @ h' %gnn  % ( %D DT Cg6J6J C h'E/gnn E/ (E/N #4t4UY^b zgn z h'>%gnn >% (>%@ "&4D OW=M=M O h'% % (%, !dt D8H8H Dr   