
    bi]j                        d dl Z d dlmZ d dlmZmZmZ d dlZd dl	Z	d dl
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ  G d dej.                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Ze G d de             Z G d deee      Z y)    N)	dataclass)OptionalTupleUnion   )ConfigMixinregister_to_config)FromOriginalModelMixin)
BaseOutput   )	Attention)
ModelMixinc                   (     e Zd Z fdZ fdZ xZS )SDCascadeLayerNormc                 $    t        |   |i | y N)super__init__)selfargskwargs	__class__s      e/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/models/unets/unet_stable_cascade.pyr   zSDCascadeLayerNorm.__init__    s    $)&)    c                 r    |j                  dddd      }t        | 	  |      }|j                  dddd      S )Nr   r   r      )permuter   forward)r   xr   s     r   r   zSDCascadeLayerNorm.forward#   s:    IIaAq!GOAyyAq!$$r   __name__
__module____qualname__r   r   __classcell__r   s   @r   r   r      s    *% %r   r   c                   (     e Zd Zg f fd	Zd Z xZS )SDCascadeTimestepBlockc           
          t         |           t        j                  ||dz        | _        || _        |D ])  }t        | d| t        j                  ||dz               + y )Nr   mapper_)r   r   nnLinearmappercondssetattr)r   c
c_timestepr-   cnamer   s        r   r   zSDCascadeTimestepBlock.__init__*   s^    ii
AE2
 	KEDGE7+RYYz1q5-IJ	Kr   c                    |j                  t        | j                        dz   d      }| j                  |d         d d d d d d f   j                  dd      \  }}t	        | j                        D ]I  \  }} t        | d|       ||dz            d d d d d d f   j                  dd      \  }}||z   ||z   }}K |d|z   z  |z   S )Nr   dimr   r   r)   )chunklenr-   r,   	enumerategetattr)	r   r   tabir/   acbcs	            r   r   zSDCascadeTimestepBlock.forward2   s    GGC

Oa'QG/{{1Q4 AtT!12888B1djj) 	"DAq1WTWQC=1!AE(;Aq$<LMSSTU[\S]FBr61r6qA	" AE{Qr   r    r%   s   @r   r'   r'   )   s    ,. Kr   r'   c                   (     e Zd Zd fd	ZddZ xZS )SDCascadeResBlockc           
         t         |           t        j                  ||||dz  |      | _        t        |dd      | _        t        j                  t        j                  ||z   |dz        t        j                         t        |dz        t        j                  |      t        j                  |dz  |            | _        y )Nr   )kernel_sizepaddinggroupsFư>elementwise_affineeps   )r   r   r*   Conv2d	depthwiser   norm
Sequentialr+   GELUGlobalResponseNormDropoutchannelwise)r   r/   c_skiprB   dropoutr   s        r   r   zSDCascadeResBlock.__init__<   s    1a[+YZJZcde&qUM	==IIa&j!a%(GGIq1u%JJwIIa!eQ
r   c                     |}| j                  | j                  |            }|t        j                  ||gd      }| j	                  |j                  dddd            j                  dddd      }||z   S )Nr   r3   r   r   r   )rL   rK   torchcatrQ   r   )r   r   x_skipx_ress       r   r   zSDCascadeResBlock.forwardH   sr    IIdnnQ'(		1f+1-AQYYq!Q23;;Aq!QG5yr   )r   r           r   r    r%   s   @r   r@   r@   ;   s    

r   r@   c                   $     e Zd Z fdZd Z xZS )rO   c                     t         |           t        j                  t	        j
                  ddd|            | _        t        j                  t	        j
                  ddd|            | _        y )Nr   )r   r   r*   	ParameterrU   zerosgammabeta)r   r4   r   s     r   r   zGlobalResponseNorm.__init__S   sL    \\%++aAs";<
LLQ1c!:;	r   c                     t        j                  |ddd      }||j                  dd      dz   z  }| j                  ||z  z  | j                  z   |z   S )Nr   )r   r   T)pr4   keepdim)r4   rb   rE   )rU   rL   meanr^   r_   )r   r   agg_normstand_div_norms       r   r   zGlobalResponseNorm.forwardX   sT    ::a1&$?!X]]r4]%H4%OPzzQ/0499<q@@r   r    r%   s   @r   rO   rO   R   s    <
Ar   rO   c                   &     e Zd Zd fd	Zd Z xZS )SDCascadeAttnBlockc                    t         |           || _        t        |dd      | _        t        ||||z  |d      | _        t        j                  t        j                         t        j                  ||            | _        y )NFrE   rF   T)	query_dimheadsdim_headrS   bias)r   r   	self_attnr   rL   r   	attentionr*   rM   SiLUr+   	kv_mapper)r   r/   c_condnheadrn   rS   r   s         r   r   zSDCascadeAttnBlock.__init___   sb    "&qUM	"Qea5jZahlmrwwy"))FA2FGr   c                 $   | j                  |      }| j                  |      }| j                  rK|j                  \  }}}}t	        j
                  |j                  ||d      j                  dd      |gd      }|| j                  ||      z   }|S )Nrc   r   r   r3   )encoder_hidden_states)	rq   rL   rn   shaperU   rV   view	transposero   )r   r   kvnorm_x
batch_sizechannel_s          r   r   zSDCascadeAttnBlock.forwardg   s    ^^B1>>()%JAFKK
GR@JJ1aPRTU[\]BvR@@r   )TrY   r    r%   s   @r   rh   rh   ^   s    Hr   rh   c                   &     e Zd Zd fd	Zd Z xZS )UpDownBlock2dc                 >   t         |           |dvrt        | d      |rt        j                  |dk(  rdnddd      nt        j
                         }t        j                  ||d	
      }t        j                  |dk(  r||g      | _        y ||g      | _        y )N)updown not supportedr   r   g      ?bilinearT)scale_factormodealign_cornersr   rB   )	r   r   
ValueErrorr*   UpsampleIdentityrJ   
ModuleListblocks)r   in_channelsout_channelsr   enabledinterpolationmappingr   s          r   r   zUpDownBlock2d.__init__r   s    ~%v^455  KK$$,QCj`de 	
 ))K1Emm]G$<kSZ\iRjkr   c                 8    | j                   D ]
  } ||      } |S r   )r   )r   r   blocks      r   r   zUpDownBlock2d.forward~   s#    [[ 	EaA	r   )Tr    r%   s   @r   r   r   q   s    
lr   r   c                   0    e Zd ZU dZej
                  ed<   y)StableCascadeUNetOutputNsample)r!   r"   r#   r   rU   Tensor__annotations__ r   r   r   r      s    FELLr   r   c            ,       l    e Zd ZdZe	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddedededededee   dee   d	ee   d
ee   deee      deee      deee      dee   dee   dee   dee   de	e
ee
   f   de	eee   f   dee   deee      f( fd       Zd ZddZddZd Zd Z	 	 	 	 	 	 	 d dZ xZS )!StableCascadeUNetTr   r   timestep_ratio_embedding_dim
patch_sizeconditioning_dimblock_out_channelsnum_attention_headsdown_num_layers_per_blockup_num_layers_per_blockdown_blocks_repeat_mappersup_blocks_repeat_mappersblock_types_per_layerclip_text_in_channelsclip_image_in_channelseffnet_in_channelspixel_mapper_in_channelsrS   rn   timestep_conditioning_typeswitch_levelc                    t         #|           t        |      t        |      k7  rt        dt        |             t        |      t        |	      k7  rt        dt        |             t        |      t        |
      k7  rt        dt        |             t        |      t        |      k7  rt        dt        |             t        |      t        |      k7  rt        dt        |             t	        |t
              r|ft        |      z  }t	        |t              r|ft        |      z  }|yt        j                  t        j                  ||d   dz  d	
      t        j                         t        j                  |d   dz  |d   d	
      t        |d   dd            | _        |yt        j                  t        j                  ||d   dz  d	
      t        j                         t        j                  |d   dz  |d   d	
      t        |d   dd            | _        t        j                  ||z        | _        |t        j                  |      | _        |t        j                  ||z        | _        t        j$                  dd      | _        t        j                  t        j(                  |      t        j                  ||dz  z  |d   d	
      t        |d   dd            | _        dfd	}t        j,                         | _        t        j,                         | _        t        j,                         | _        t5        t        |            D ]  }|dkD  r| j0                  j7                  t        j                  t        ||d	z
     dd      |t9        ||d	z
     ||   d||d	z
           n!t        j                  ||d	z
     ||   dd                   n-| j0                  j7                  t        j:                                t        j,                         }t5        ||         D ]6  }||   D ],  } ||||   ||   ||   ||         }|j7                  |       . 8 | j.                  j7                  |       |
2t        j,                         }t5        |
|   d	z
        D ]/  }|j7                  t        j                  ||   ||   d	
             1 | j2                  j7                  |        t        j,                         | _        t        j,                         | _        t        j,                         | _         tC        t5        t        |                  D ]  }|dkD  r| j>                  j7                  t        j                  t        ||   dd      |t9        ||   ||d	z
     d||d	z
           n!t        jD                  ||   ||d	z
     dd                   n-| j>                  j7                  t        j:                                t        j,                         }t5        |	ddd   |         D ]i  } tG        ||         D ]V  \  }!}|t        |      d	z
  k  r| |!cxk(  rdk(  rn n||   nd}" ||||   ||   |"||   ||         }|j7                  |       X k | j<                  j7                  |       |ht        j,                         }t5        |ddd   |   d	z
        D ]/  }|j7                  t        j                  ||   ||   d	
             1 | j@                  j7                  |        t        j                  t        |d   dd      t        j                  |d   ||dz  z  d	
      t        jH                  |            | _%        d| _&        y)a  

        Parameters:
            in_channels (`int`, defaults to 16):
                Number of channels in the input sample.
            out_channels (`int`, defaults to 16):
                Number of channels in the output sample.
            timestep_ratio_embedding_dim (`int`, defaults to 64):
                Dimension of the projected time embedding.
            patch_size (`int`, defaults to 1):
                Patch size to use for pixel unshuffling layer
            conditioning_dim (`int`, defaults to 2048):
                Dimension of the image and text conditional embedding.
            block_out_channels (Tuple[int], defaults to (2048, 2048)):
                Tuple of output channels for each block.
            num_attention_heads (Tuple[int], defaults to (32, 32)):
                Number of attention heads in each attention block. Set to -1 to if block types in a layer do not have
                attention.
            down_num_layers_per_block (Tuple[int], defaults to [8, 24]):
                Number of layers in each down block.
            up_num_layers_per_block (Tuple[int], defaults to [24, 8]):
                Number of layers in each up block.
            down_blocks_repeat_mappers (Tuple[int], optional, defaults to [1, 1]):
                Number of 1x1 Convolutional layers to repeat in each down block.
            up_blocks_repeat_mappers (Tuple[int], optional, defaults to [1, 1]):
                Number of 1x1 Convolutional layers to repeat in each up block.
            block_types_per_layer (Tuple[Tuple[str]], optional,
                defaults to (
                    ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), ("SDCascadeResBlock",
                    "SDCascadeTimestepBlock", "SDCascadeAttnBlock")
                ): Block types used in each layer of the up/down blocks.
            clip_text_in_channels (`int`, *optional*, defaults to `None`):
                Number of input channels for CLIP based text conditioning.
            clip_text_pooled_in_channels (`int`, *optional*, defaults to 1280):
                Number of input channels for pooled CLIP text embeddings.
            clip_image_in_channels (`int`, *optional*):
                Number of input channels for CLIP based image conditioning.
            clip_seq (`int`, *optional*, defaults to 4):
            effnet_in_channels (`int`, *optional*, defaults to `None`):
                Number of input channels for effnet conditioning.
            pixel_mapper_in_channels (`int`, defaults to `None`):
                Number of input channels for pixel mapper conditioning.
            kernel_size (`int`, *optional*, defaults to 3):
                Kernel size to use in the block convolutional layers.
            dropout (Tuple[float], *optional*, defaults to (0.1, 0.1)):
                Dropout to use per block.
            self_attn (Union[bool, Tuple[bool]]):
                Tuple of booleans that determine whether to use self attention in a block or not.
            timestep_conditioning_type (Tuple[str], defaults to ("sca", "crp")):
                Timestep conditioning type.
            switch_level (Optional[Tuple[bool]], *optional*, defaults to `None`):
                Tuple that indicates whether upsampling or downsampling should be applied in a block
        zaNumber of elements in `down_num_layers_per_block` must match the length of `block_out_channels`: z_Number of elements in `up_num_layers_per_block` must match the length of `block_out_channels`: zbNumber of elements in `down_blocks_repeat_mappers` must match the length of `block_out_channels`: z`Number of elements in `up_blocks_repeat_mappers` must match the length of `block_out_channels`: z]Number of elements in `block_types_per_layer` must match the length of `block_out_channels`: Nr   rI   r   r   FrE   rF   r   c                     | dk(  rt        |||      S | dk(  rt        ||||      S | dk(  rt        |	      S t        d|  d      )	Nr@   )rB   rS   rh   )rn   rS   r'   )r-   zBlock type r   )r@   rh   r'   r   )

block_typer   rs   rR   rS   rn   r   rB   r   r   s
         r   	get_blockz-StableCascadeUNet.__init__.<locals>.get_block   sr    00(f+_fgg33)+7GZcmtuu77-!=E_  !;zl.!IJJr   r   )r   r   )rB   stride)rS   rn   r   rc   )rR   rS   rn   )r   r   T)'r   r   r6   r   
isinstancefloatboolr*   rM   rJ   rN   r   effnet_mapperpixels_mapperr+   clip_txt_pooled_mapperclip_txt_mapperclip_img_mapper	LayerNorm	clip_normPixelUnshuffle	embeddingr   down_blocksdown_downscalersdown_repeat_mappersrangeappendr   r   	up_blocksup_upscalersup_repeat_mappersreversedConvTranspose2dr7   PixelShuffleclfgradient_checkpointing)$r   r   r   r   r   r   r   r   r   r   r   r   r   r   clip_text_pooled_in_channelsr   clip_seqr   r   rB   rS   rn   r   r   r   r<   
down_blockr}   r   r   block_repeat_mappersup_blockjkrR   r   s$      ` `             `  `            r   r   zStableCascadeUNet.__init__   s   n 	!"c*C&DDstw  yK  uL  tM  N  #$,C(DDqru  wI  sJ  rK  L  #$,F(GGtux  zL  vM  uN  O  #$,D(EErsv  xJ  tK  sL  M  #$,A(BBops  uG  qH  pI  J  gu%j3'9#::Gi&"s+='>>I )!#		,.@.Ca.GUVW			,Q/!35G5JXYZ"#5a#8UX\]	"D $/!#		24Fq4IA4M[\]			,Q/!35G5JXYZ"#5a#8UX\]	"D ')ii0LN^aiNi&j# ,#%99-BDT#UD !-#%99-CEUX`E`#aD &65VZ[j)IIkZ]35G5JXYZ1!4TXY

	K 
	K ==? "#%==? s-./  	FA1u%%,,MM*+=a!e+DY^dhi (3 &.q1u57I!7LSYcopqtupucv  YY'9!a%'@BTUVBWefopq	 %%,,R[[];J4Q78 	-"7": -J%"*1-+A. '
"+A,E %%e,-	- ##J/)5')}}$9!<q@A xA(//		:LQ:OQcdeQftu0vwx((//0DEA 	FF MMO!#%$6 789 $	DA1u!!((MM*+=a+@UZ`de (3 &.q13Ea!e3LSWamnorsnsat  //.q13Ea!e3LZ[de	 !!((7}}H24R48;< +%./DQ/G%H 
+MAz67#>P:QTU:U6UZ[_`ZedeZe/2klF%"*1-+A.% '
"+A,E OOE*
++ NN!!(+'3')}}$7"=a@1DE xA(//		:LQ:OQcdeQftu0vwx&&--.BCI$	DN ==1!4TXYII(+\Z]-KYZ[OOJ'
 ',#r   c                 :   t        |t        j                  t        j                  f      rit        j                  j
                  j                  |j                         |j                  *t        j
                  j                  |j                  d       t        j
                  j                  | j                  j                  d       t        | d      r5t        j
                  j                  | j                  j                  d      nd  t        | d      r5t        j
                  j                  | j                  j                  d      nd  t        | d      rpt        j
                  j                  | j                  d   j                  d       t        j
                  j                  | j                  d   j                  d       t        | d      rpt        j
                  j                  | j                   d   j                  d       t        j
                  j                  | j                   d   j                  d       t        j                  j
                  j                  | j"                  d	   j                  d       t        j
                  j                  | j$                  d	   j                  d       | j&                  | j(                  z   D ]  }|D ]  }t        |t*              rc|j,                  d
   j                  xj.                  t1        j2                  d	t5        | j6                  j8                  d         z        z  c_        vt        |t:              st        j
                  j                  |j<                  j                  d         y )Nr   g{Gz?)stdr   r   r   r   r   r   rc   )r   r*   rJ   r+   rU   initxavier_uniform_weightrm   	constant_normal_r   hasattrr   r   r   r   r   r   r   r   r@   rQ   datanpsqrtsumconfigr   r'   r,   )r   mlevel_blockr   s       r   _init_weightszStableCascadeUNet._init_weights  sj   a"))RYY/0HHMM))!((3vv!!!!&&!,
33::EBI$PaBb,,33>hlBI$PaBb,,33>hl4)GGOOD..q188dOCGGOOD..q188dOC4)GGOOD..q188dOCGGOOD..q188dOC%%dnnQ&7&>&>E
$((1+,,a0  ++dnn< 	>K$ >e%67%%b)0055SI[I[\]I^E_A_9``5'=>GG%%ell&9&91=	>	>r   c                 R   ||z  }| j                   j                  dz  }t        j                  |      |dz
  z  }t	        j
                  ||j                        j                         j                  |       j                         }|d d d f   |d d d f   z  }t	        j                  |j                         |j                         gd      }| j                   j                  dz  dk(  r"t        j                  j                  |dd      }|j!                  |j"                        S )	Nr   r   )devicer3   )r   r   constant)r   )dtype)r   r   mathlogrU   aranger   r   mulexprV   sincosr*   
functionalpadtor   )r   timestep_ratiomax_positionsrhalf_dimembs         r   get_timestep_ratio_embeddingz.StableCascadeUNet.get_timestep_ratio_embedding  s    ]*;;;;q@hh}%A6ll8AHH5;;=AA3$GKKM4j3tQw<'iiCGGI.A6;;33a71<--##Cj#ACvvAGGv$$r   c                    t        |j                        dk(  r|j                  d      }| j                  |      j	                  |j                  d      |j                  d      | j                  j                  z  d      }||| j                  |      }t        |j                        dk(  r|j                  d      }| j                  |      j	                  |j                  d      |j                  d      | j                  j                  z  d      }t        j                  |||gd      }n|}| j                  |      S )Nr   r   r   rc   r3   )r6   rv   	unsqueezer   rw   sizer   r   r   r   rU   rV   r   )r   clip_txt_pooledclip_txtclip_imgclip_txt_poolclips         r   get_clip_embeddingsz%StableCascadeUNet.get_clip_embeddings  s   $$%*+55a8M33ODII  #_%9%9!%<t{{?S?S%SUW
 H$8++H5H8>>"a'#--a0++H5::a (--"2T[[5I5I"I2H 99hx@aHD D~~d##r   c                    g }t        | j                  | j                  | j                        }t	        j
                         r| j                  r|D ]  \  }}} ||      }t        t        |      dz         D ]  }	|D ]~  }
t        |
t              r| j                  |
|      }&t        |
t              r| j                  |
||      }Jt        |
t              r| j                  |
||      }n| j                  |
      } |	t        |      k  s ||	   |      } |j                  d|        |S |D ]  \  }}} ||      }t        t        |      dz         D ]x  }	|D ]W  }
t        |
t              r	 |
|      }t        |
t              r
 |
||      }6t        |
t              r
 |
||      }P |
|      }Y |	t        |      k  sn ||	   |      }z |j                  d|        |S )Nr   r   )zipr   r   r   rU   is_grad_enabledr   r   r6   r   r@   _gradient_checkpointing_funcrh   r'   insert)r   r   r_embedr   level_outputsblock_groupr   
downscalerrepmapr<   r   s              r   _down_encodezStableCascadeUNet._down_encode  s   $**D,A,A4C[C[\  "t'B'B2= +.
JqMs6{Q/ )A!+ I%e->? $ A A% KA'/AB $ A A%D QA'/EF $ A A%G TA $ A A% HAI 3v;%F1IaL) $$Q*+>  3> +.
JqMs6{Q/ )A!+ )%e->? %aA'/AB %aA'/EF %a 1A %aA) 3v;%F1IaL) $$Q*+ r   c           	         |d   }t        | j                  | j                  | j                        }t	        j
                         r| j                  rt        |      D ]  \  }\  }}}	t        t        |	      dz         D ]q  }
t        |      D ]D  \  }}t        |t              r|dk(  r
|dkD  r||   nd }||j                  d      |j                  d      k7  s#|j                  d      |j                  d      k7  re|j                  }t        j                  j                  j!                  |j#                         |j$                  dd  dd      }|j'                  |      }| j)                  |||      }t        |t*              r| j)                  |||      }t        |t,              r| j)                  |||      }3| j)                  ||      }G |
t        |	      k  sg |	|
   |      }t  ||      } |S t        |      D ]u  \  }\  }}}	t        t        |	      dz         D ]H  }
t        |      D ]  \  }}t        |t              r|dk(  r
|dkD  r||   nd }||j                  d      |j                  d      k7  s#|j                  d      |j                  d      k7  re|j                  }t        j                  j                  j!                  |j#                         |j$                  dd  dd      }|j'                  |      } |||      }t        |t*              r
 |||      }t        |t,              r |||      } ||      } |
t        |	      k  s> |	|
   |      }K  ||      }x |S )Nr   r   rc   r   T)r   r   )r   r   r   r   rU   r  r   r7   r   r6   r   r@   r   r   r*   r   interpolater   rv   r   r  rh   r'   )r   r  r  r   r   r  r<   r   upscalerr  r   r   r   skip	orig_types                  r   
_up_decodezStableCascadeUNet._up_decode  sR   !$..$*;*;T=S=ST  "t'B'B3<[3I  //Hhs6{Q/ )A$-h$7 L5%e->?78Av!a%=#3TD#/QVVBZ499R=5PTUTZTZ[]T^bfbkbklnboTo,-GG	$)HH$7$7$C$C$%GGItzz"#Z_c %D %" %&DDO $ A A%D QA'/AB $ A A%D QA'/EF $ A A%G TA $ A A% KAL  3v;%F1IaL%)& QK) V + 4=[3I  //Hhs6{Q/ )A$-h$7 )5%e->?78Av!a%=#3TD#/QVVBZ499R=5PTUTZTZ[]T^bfbkbklnboTo,-GG	$)HH$7$7$C$C$%GGItzz"#Z_c %D %" %&DDO %aA'/AB %aA'/EF %a 1A %aA)  3v;%F1IaL%)& QK) * r   c           	      `   |#|j                  |j                  d      ddd      }| j                  |      }| j                  j                  D ]U  }|dk(  r|}n
|dk(  r|	}nd }|xs t        j                  |      }t        j                  || j                  |      gd      }W | j                  |||      }| j                  |      }t        | d	      rD|B|| j                  t        j                  j                  ||j                  d
d  dd            z   }t        | d      rB|t        j                  j                  | j!                  |      |j                  d
d  dd      z   }| j#                  |||      }| j%                  |||      }| j'                  |      }|
s|fS t)        |      S )Nr   r      scacrpr   r3   )r   r   r   r   r  r   T)r   r   r   r   )r   )	new_zerosr   r   r   r   rU   
zeros_likerV   r   r   r   r   r*   r   r  rv   r   r	  r  r   r   )r   r   r   clip_text_pooled	clip_textr   effnetpixelsr  r  return_dicttimestep_ratio_embedr/   condt_condr   r   r  s                     r   r   zStableCascadeUNet.forward  s    >%%fkk!naA>F  $@@P77 	wAEze=U--n=F#(99.BDDeDeflDm-ntu#v 	w ''8HS\go'p NN6"4)f.@D&&))&qwwrs|*dh)i A 4)BMM--""6*J^b .  A ))!-A4HOOM+?F!9&f55r   )   r  @   r      )r!  r!  )    r"  )r     )r#  r  r   r   r$  )r@   r'   rh   r%  Ni   NrI   NNr   )皙?r&  T)r  r  N)i'  )NN)NNNNNNT)r!   r"   r#    _supports_gradient_checkpointingr	   intr   r   strr   r   r   r   r   r   r   r	  r  r   r$   r%   s   @r   r   r      s   '+$ ,. $)5*207.5<
 :@4
 04%)04,026.8.21?.2=w,w, w, '*	w,
 w, w, "#Jw, #3Zw, $):w, "'sw, %-U3Z$8w, #+5:"6w,   %U3Z0!w,(  (})w,, !)-w,0 %SM1w,2 #+3-3w,6 ueEl*+7w,8 uT{*+9w,: %*#J;w,< uT{+=w, w,r>:%$$$L0n -6r   r   )!r   dataclassesr   typingr   r   r   numpyr   rU   torch.nnr*   configuration_utilsr   r	   loadersr
   utilsr   attention_processorr   modeling_utilsr   r   r   Moduler'   r@   rO   rh   r   r   r   r   r   r   <module>r4     s     ! ) )    B -  + '% %RYY $		 .	A 	A &BII &  j    6
K1G 6r   