
    bi)             7          d dl mZmZmZmZmZ d dlZd dlZd dl	m
c mZ d dlm
Z
 ddlmZmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&  ejN                  e(      Z)	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dide*de+de+de+de+de,de-de*de+dee+   dee+   dee+   dee+   de,de,de,de,de*d e*d!e,d"e-d#ee*   d$ee+   d%ee*   d&e-f2d'Z.	 	 	 	 	 	 	 	 	 	 	 	 	 	 djd(e*de+de+de-de*de+d)e-de+dee+   dee+   de,de,d*e,de,de*d e*d!e,d#ee*   d$ee+   d&e-f(d+Z/	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dkd,e*de+de+de+d-e+de+d.e,de-de*d/ee+   de+dee+   dee+   dee+   de,de,de,de,de*d e*d!e,d"e-d#ee*   d$ee+   d0ee*   d&e-d1e
j`                  f6d2Z1 G d3 d4e
j`                        Z2 G d5 d6e
j`                        Z3 G d7 d8e
j`                        Z4 G d9 d:e
j`                        Z5 G d; d<e
j`                        Z6 G d= d>e
j`                        Z7 G d? d@e
j`                        Z8 G dA dBe
j`                        Z9 G dC dDe
j`                        Z: G dE dFe
j`                        Z; G dG dHe
j`                        Z< G dI dJe
j`                        Z= G dK dLe
j`                        Z> G dM dNe
j`                        Z? G dO dPe
j`                        Z@ G dQ dRe
j`                        ZA G dS dTe
j`                        ZB G dU dVe
j`                        ZC G dW dXe
j`                        ZD G dY dZe
j`                        ZE G d[ d\e
j`                        ZF G d] d^e
j`                        ZG G d_ d`e
j`                        ZH G da dbe
j`                        ZI G dc dde
j`                        ZJ G de dfe
j`                        ZK G dg dhe
j`                        ZLy)l    )AnyDictOptionalTupleUnionN)nn   )	deprecatelogging)apply_freeu   )get_activation)	AttentionAttnAddedKVProcessorAttnAddedKVProcessor2_0)AdaGroupNorm)Downsample2DFirDownsample2DFirUpsample2DKDownsample2DKUpsample2DResnetBlock2DResnetBlockCondNorm2D
Upsample2D)DualTransformer2DModel)Transformer2DModeldown_block_type
num_layersin_channelsout_channelstemb_channelsadd_downsample
resnet_epsresnet_act_fntransformer_layers_per_blocknum_attention_headsresnet_groupscross_attention_dimdownsample_paddingdual_cross_attentionuse_linear_projectiononly_cross_attentionupcast_attentionresnet_time_scale_shiftattention_typeresnet_skip_time_actresnet_out_scale_factorcross_attention_normattention_head_dimdownsample_typedropoutc                    |t         j                  d|	 d       |	}| j                  d      r| dd  n| } | dk(  rt        |||||||||
||      S | dk(  rt	        |||||||||
|||      S | d	k(  r$|d
u rd }n|xs d}t        ||||||||
||||      S | dk(  rO|t        d      t        d6i d|d|d|d|d|d|d|d|d|d|
d|d|d|	d|d|d|d|d |d!|S | d"k(  rF|t        d#      t        d6i d|d|d|d|d|d|d|d|d|
d|d$|d |d%|d&|d|d'|S | d(k(  rt        ||||||||||)
      S | d*k(  rt        ||||||||||+
      S | d,k(  rt        ||||||||
||-
      S | d.k(  rt        ||||||||
|||/      S | d0k(  rt        ||||||||1      S | d2k(  rt        |||||||||||sd34      S d
4      S t        |  d5      )7NztIt is recommended to provide `attention_head_dim` when calling `get_down_block`. Defaulting `attention_head_dim` to .UNetRes   DownBlock2D)r   r   r    r!   r5   r"   r#   r$   r'   r)   r.   ResnetDownsampleBlock2D)r   r   r    r!   r5   r"   r#   r$   r'   r.   skip_time_actoutput_scale_factorAttnDownBlock2DFconv)r   r   r    r!   r5   r#   r$   r'   r)   r3   r.   r4   CrossAttnDownBlock2Dz>cross_attention_dim must be specified for CrossAttnDownBlock2Dr   r%   r   r    r!   r5   r"   r#   r$   r'   r)   r(   r&   r*   r+   r,   r-   r.   r/   SimpleCrossAttnDownBlock2DzDcross_attention_dim must be specified for SimpleCrossAttnDownBlock2Dr3   r<   r=   r2   SkipDownBlock2D)
r   r   r    r!   r5   r"   r#   r$   r)   r.   AttnSkipDownBlock2D)
r   r   r    r!   r5   r"   r#   r$   r3   r.   DownEncoderBlock2D)
r   r   r    r5   r"   r#   r$   r'   r)   r.   AttnDownEncoderBlock2D)r   r   r    r5   r"   r#   r$   r'   r)   r3   r.   KDownBlock2D)r   r   r    r!   r5   r"   r#   r$   KCrossAttnDownBlock2DT)r   r   r    r!   r5   r"   r#   r$   r(   r3   add_self_attention does not exist. )loggerwarning
startswithr:   r;   r>   
ValueErrorr@   rA   rB   rC   rD   rE   rF   rG   )r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   s                            `/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/models/unets/unet_2d_blocks.pyget_down_blockrP   +   s6   8 ! C  DW  CX  XY  Z	
 1-<-G-G	-Roab)XgO-'!#%')!''1$;
 	
 
5	5&!#%')!''$;. 7
 	
 
-	-U""O-7O!#%'!''11$;+
 	
 
2	2&]^^# 
!
)E
 $
 &	

 (
 
 *
 "
 (
 (
  2
 !4
 !4
 "6
 #8
  "6!
" .#
$ %<%
& *'
 	
* 
8	8&cdd) 
!
#
 &
 (	

 
 *
 "
 (
 (
 !4
  2
 %<
 /
 !8
 "6
  "6!
 	
$ 
-	-!#%')!'1$;
 	
 
1	1"!#%')!'1$;
 	
 
0	0!!#%)!''1$;
 	
 
4	4%!#%)!''11$;
 	
 
N	*!#%')!'	
 		
 
3	3$!#%')!' 31+9t
 	
 @E
 	
 ((89
::    mid_block_typer=   mid_block_only_cross_attentionc                     | dk(  rt        |||||||||	|||
|||      S | dk(  rt        |||||||	||||||      S | dk(  rt        |||d|||||d
      S | y t        d	|        )
NUNetMidBlock2DCrossAttn)r%   r   r!   r5   r#   r$   r=   r.   r(   r&   r'   r*   r+   r-   r/   UNetMidBlock2DSimpleCrossAttn)r   r!   r5   r#   r$   r=   r(   r3   r'   r.   r<   r,   r2   UNetMidBlock2Dr   F)
r   r!   r5   r   r#   r$   r=   r'   r.   add_attentionzunknown mid_block_type : )rU   rV   rW   rN   )rR   r!   r   r#   r$   r'   r=   r%   r&   r(   r*   r+   rS   r-   r.   r/   r0   r2   r3   r5   s                       rO   get_mid_blockrY      s    , 22&)E#'!' 3$; 3 3'!5"7-)
 	
" 
:	:,#'!' 3 31'$;.!?!5
 	
 
+	+#'!' 3'$;
 	
 
	4^4DEFFrQ   up_block_typeprev_output_channeladd_upsampleresolution_idxupsample_typereturnc                    |t         j                  d| d       |}| j                  d      r| dd  n| } | dk(  rt        ||||||	||||||      S | dk(  rt	        ||||||	||||||||      S | d	k(  rR|t        d
      t        d6i d|d|
d|d|d|d|d|	d|d|d|d|d|d|d|d|d|d|d|d|d|S | dk(  rL|t        d       t        d6i d|d|d|d|d|d|	d|d|d|d|d|d|d!|d|d"|d#|d|d$|S | d%k(  r%|d&u rd }n|xs d'}t        ||||||	|||||||(      S | d)k(  rt        ||||||	|||||*      S | d+k(  rt        ||||||	||||||,      S | d-k(  rt        ||||	|||||||.      S | d/k(  rt        ||||	||||||||0      S | d1k(  rt        |||||	||||2	      S | d3k(  rt        |||||	||||||4      S t        |  d5      )7NzrIt is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to r7   r8   r9   	UpBlock2D)r   r   r    r[   r!   r]   r5   r\   r#   r$   r'   r.   ResnetUpsampleBlock2D)r   r   r    r[   r!   r]   r5   r\   r#   r$   r'   r.   r<   r=   CrossAttnUpBlock2Dz<cross_attention_dim must be specified for CrossAttnUpBlock2Dr   r%   r   r    r[   r!   r]   r5   r\   r#   r$   r'   r(   r&   r*   r+   r,   r-   r.   r/   SimpleCrossAttnUpBlock2DzBcross_attention_dim must be specified for SimpleCrossAttnUpBlock2Dr3   r<   r=   r2   AttnUpBlock2DFr?   )r   r   r    r[   r!   r]   r5   r#   r$   r'   r3   r.   r^   SkipUpBlock2D)r   r   r    r[   r!   r]   r5   r\   r#   r$   r.   AttnSkipUpBlock2D)r   r   r    r[   r!   r]   r5   r\   r#   r$   r3   r.   UpDecoderBlock2D)r   r   r    r]   r5   r\   r#   r$   r'   r.   r!   AttnUpDecoderBlock2D)r   r   r    r]   r5   r\   r#   r$   r'   r3   r.   r!   
KUpBlock2D)	r   r   r    r!   r]   r5   r\   r#   r$   KCrossAttnUpBlock2D)r   r   r    r!   r]   r5   r\   r#   r$   r(   r3   rI   rJ   )rK   rL   rM   ra   rb   rN   rc   rd   re   rf   rg   rh   ri   rj   rk   )rZ   r   r   r    r[   r!   r\   r#   r$   r]   r%   r&   r'   r(   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r^   r5   s                             rO   get_up_blockrl   G  s\   : ! A  BU  AV  VW  X	
 1)6)A)A))LM!"%R_M#!#% 3')%!''$;
 	
 
1	1$!#% 3')%!''$;. 7
 	
  
.	.&[\\! 
!
)E
 $
 &	

 !4
 (
 *
 
 &
 "
 (
 (
 !4
 !4
 "6
  #8!
" "6#
$ .%
& %<'
( *)
 	
, 
4	4&abb' 
!
#
 &
 !4	

 (
 *
 
 &
 "
 (
 (
 !4
  2
 %<
 /
  !8!
" "6#
$ "6%
 	
( 
/	)5  M)3VM!#% 3')!''1$;'
 	
 
/	)!#% 3')%!'$;
 	
 
-	- !#% 3')%!'1$;
 	
 
,	,!#%)%!''$;'
 	
 
0	0#!#%)%!''1$;'
 	
 
,	&!#%')%!'

 
	
 
/	/"!#%')%!' 31
 	
 &67
88rQ   c                   h     e Zd ZdZdededef fdZdej                  dej                  fdZ	 xZ
S )	AutoencoderTinyBlocka*  
    Tiny Autoencoder block used in [`AutoencoderTiny`]. It is a mini residual module consisting of plain conv + ReLU
    blocks.

    Args:
        in_channels (`int`): The number of input channels.
        out_channels (`int`): The number of output channels.
        act_fn (`str`):
            ` The activation function to use. Supported values are `"swish"`, `"mish"`, `"gelu"`, and `"relu"`.

    Returns:
        `torch.Tensor`: A tensor with the same shape as the input tensor, but with the number of channels equal to
        `out_channels`.
    r   r    act_fnc                    t         |           t        |      }t        j                  t        j
                  ||dd      |t        j
                  ||dd      |t        j
                  ||dd            | _        ||k7  rt        j
                  ||dd      nt        j                         | _        t        j                         | _
        y )Nr	      )kernel_sizepaddingF)rr   bias)super__init__r   r   
SequentialConv2dr?   IdentityskipReLUfuse)selfr   r    ro   	__class__s       rO   rv   zAutoencoderTinyBlock.__init__8  s    'MMIIk<QJIIlLaKIIlLaK
	 l* IIk<QUK 		
 GGI	rQ   xr_   c                 f    | j                  | j                  |      | j                  |      z         S N)r|   r?   rz   )r}   r   s     rO   forwardzAutoencoderTinyBlock.forwardI  s&    yy1		!455rQ   )__name__
__module____qualname____doc__intstrrv   torchTensorr   __classcell__r~   s   @rO   rn   rn   (  s=    C s C "6 6%,, 6rQ   rn   c                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 ddededededededed	ed
ee   dedededef fdZ	dde
j                  dee
j                     de
j                  fdZ xZS )rW   a:  
    A 2D UNet mid-block [`UNetMidBlock2D`] with multiple residual blocks and optional attention blocks.

    Args:
        in_channels (`int`): The number of input channels.
        temb_channels (`int`): The number of temporal embedding channels.
        dropout (`float`, *optional*, defaults to 0.0): The dropout rate.
        num_layers (`int`, *optional*, defaults to 1): The number of residual blocks.
        resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks.
        resnet_time_scale_shift (`str`, *optional*, defaults to `default`):
            The type of normalization to apply to the time embeddings. This can help to improve the performance of the
            model on tasks with long-range temporal dependencies.
        resnet_act_fn (`str`, *optional*, defaults to `swish`): The activation function for the resnet blocks.
        resnet_groups (`int`, *optional*, defaults to 32):
            The number of groups to use in the group normalization layers of the resnet blocks.
        attn_groups (`Optional[int]`, *optional*, defaults to None): The number of groups for the attention blocks.
        resnet_pre_norm (`bool`, *optional*, defaults to `True`):
            Whether to use pre-normalization for the resnet blocks.
        add_attention (`bool`, *optional*, defaults to `True`): Whether to add attention blocks.
        attention_head_dim (`int`, *optional*, defaults to 1):
            Dimension of a single attention head. The number of attention heads is determined based on this value and
            the number of input channels.
        output_scale_factor (`float`, *optional*, defaults to 1.0): The output scale factor.

    Returns:
        `torch.Tensor`: The output of the last residual block, which is a tensor of shape `(batch_size, in_channels,
        height, width)`.

    r   r!   r5   r   r#   r.   r$   r'   attn_groupsresnet_pre_normrX   r3   r=   c                    t         |           ||nt        |dz  d      }|| _        |		|dk(  r|nd }	|dk(  rt	        ||||||d||	      g}nt        ||||||||||

      g}g }|t        j                  d| d       |}t        |      D ]  }| j                  r0|j                  t        |||z  ||||	|dk(  r|nd d	d	d	d	
             n|j                  d        |dk(  r$|j                  t	        ||||||d||	             y|j                  t        ||||||||||

              t        j                  |      | _        t        j                  |      | _        d| _        y )N       defaultspatial	r   r    r!   epsgroupsr5   time_embedding_normnon_linearityr=   
r   r    r!   r   r   r5   r   r   r=   pre_normiIt is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: r7   T
headsdim_headrescale_output_factorr   norm_num_groupsspatial_norm_dimresidual_connectionrt   upcast_softmax_from_deprecated_attn_blockF)ru   rv   minrX   r   r   rK   rL   rangeappendr   r   
ModuleList
attentionsresnetsgradient_checkpointing)r}   r   r!   r5   r   r#   r.   r$   r'   r   r   rX   r3   r=   r   r   _r~   s                    rO   rv   zUNetMidBlock2D.__init__l  s     	)6)BK[\L\^`Ha*+Bi+O-UYK #i/% +!,"/"(#(1"/(;
G  +!,"/"(#(?"/(;,G 
%NN{  }H  |I  IJ  K "-z" 0	A!!!!#)-??!3.A&(3:QU^:^dh,0!'+48  !!$'&)3)$/%0&3&, ',5&3,?
 !$/%0&3&, ',C&3,?!0G0	d --
3}}W-&+#rQ   hidden_statestembr_   c                 2    | j                   d   ||      }t        | j                  | j                   dd        D ]Z  \  }}t        j                         r,| j
                  r |
 |||      }| j                  |||      }F|
 |||      } |||      }\ |S )Nr   rq   r   )r   zipr   r   is_grad_enabledr   _gradient_checkpointing_func)r}   r   r   attnresnets        rO   r   zUNetMidBlock2D.forward  s    'Qt<ab1AB 	<LD&$$&4+F+F#$(T$BM $ A A&-Y] ^#$(T$BM &}d ;	< rQ   )        rq   ư>r   swishr   NTTrq         ?r   )r   r   r   r   r   floatr   r   boolrv   r   r   r   r   r   s   @rO   rW   rW   M  s    D  '0$%) $""#%(r,r, r, 	r,
 r, r, "%r, r, r, c]r, r, r,  r, #r,hU\\ %,,9O [`[g[g rQ   rW   c            '       |    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddededee   dededeeee   f   deded	ed
edee   de	dededede	de	de	def& fdZ
	 	 	 	 	 ddej                  deej                     deej                     deej                     deeeef      deej                     dej                  fdZ xZS )rU   r   r!   r    r5   r   r%   r#   r.   r$   r'   resnet_groups_outr   r&   r=   r(   r*   r+   r-   r/   c                 r   t         |           |xs |}|| _        || _        d| _        || _        |
|
nt        |dz  d      }
t        |t              r|g|z  }|xs |
}t        |||||
||||	||      g}g }t        |      D ]u  }|s*|j                  t        |||z  |||   |||||	             n#|j                  t        |||z  |d||
             |j                  t        ||||||||	||
             w t        j                  |      | _        t        j                  |      | _        d	| _        y )
NTr   r   r   r    r!   r   r   
groups_outr5   r   r   r=   r   )r   r   r(   r   r+   r-   r/   rq   r   r   r(   r   r   F)ru   rv   r   r    has_cross_attentionr&   r   
isinstancer   r   r   r   r   r   r   r   r   r   r   )r}   r   r!   r    r5   r   r%   r#   r.   r$   r'   r   r   r&   r=   r(   r*   r+   r-   r/   r   r   ir~   s                          rO   rv   z UNetMidBlock2DCrossAttn.__init__  s   , 	#2{&(#' #6 )6)BK[\L\^`Ha 2C8,H+IJ+V(-> ')+$,$;+$7(
 
z" '	A'!!&+$(;;$0#?#B,?(9.C)9'5
 !!*+$(;;$0#$,?(5	 NN ,!-"/",#(?"/(;,5'	R --
3}}W-&+#rQ   r   r   encoder_hidden_statesattention_maskcross_attention_kwargsencoder_attention_maskr_   c           	         |'|j                  dd       t        j                  d        | j                  d   ||      }t	        | j
                  | j                  dd        D ]d  \  }}t        j                         r1| j                  r% ||||||d      d   }| j                  |||      }K ||||||d      d   } |||      }f |S )NscaleSPassing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.r   rq   Fr   r   r   r   return_dict)
getrK   rL   r   r   r   r   r   r   r   )	r}   r   r   r   r   r   r   r   r   s	            rO   r   zUNetMidBlock2DCrossAttn.forwardV  s     "-%))'48Dtu'Qt<ab1AB 	<LD&$$&4+F+F $!*?+A#1+A %! ! !% A A&-Y] ^ $!*?+A#1+A %! ! !'}d ;)	<, rQ   )Nr   rq   rq   r   r   r   r   NTrq   r      FFFr   NNNNNr   r   r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   s   @rO   rU   rU     s   
 '+?@ '0$+/ $#$%(#'%*&+!&')d,d, d, sm	d,
 d, d, ',CsO&<d, d, "%d, d, d, $C=d, d, !d, #d,  !!d," ##d,$  $%d,& 'd,( )d,R (,8<15;?9=$||$ u||$$  (5	$
 !.$ !)c3h 8$ !) 6$ 
$rQ   rU   c                    N    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededededee   f fdZ	 	 	 	 	 dde	j                  dee	j                     dee	j                     dee	j                     deeeef      dee	j                     de	j                  fdZ xZS )rV   r   r!   r5   r   r#   r.   r$   r'   r   r3   r=   r(   r<   r,   r2   c                 J   t         |           d| _        |
| _        ||nt	        |dz  d      }|| j                  z  | _        t        ||||||||||	|      g}g }t        |      D ]  }t        t        d      r
t               n	t               }|j                  t        ||| j
                  | j                  ||dd|||             |j                  t        ||||||||||	|              t        j                  |      | _        t        j                  |      | _        y )NTr   r   r   r    r!   r   r   r5   r   r   r=   r   r<   scaled_dot_product_attention	query_dimr(   r   r   added_kv_proj_dimr   rt   r   r,   r2   	processor)ru   rv   r   r3   r   	num_headsr   r   hasattrFr   r   r   r   r   r   r   r   )r}   r   r!   r5   r   r#   r.   r$   r'   r   r3   r=   r(   r<   r,   r2   r   r   r   r   r~   s                       rO   rv   z&UNetMidBlock2DSimpleCrossAttn.__init__~  sR   $ 	#' "4)6)BK[\L\^`Ha$(?(?? '(+$$;+$7(+
 
z" "	A-4Q8V-W')]q]s  )(3..!44&9$1#')=)=' NN +!,"/"(#(?"/(;,"/)"	H --
3}}W-rQ   r   r   r   r   r   r   r_   c                 "   ||ni }|j                  dd       t        j                  d       ||d n|}n|} | j                  d   ||      }t	        | j
                  | j                  dd        D ]  \  }}	 ||f||d|} |	||      } |S )Nr   r   r   rq   r   r   )r   rK   rL   r   r   r   )
r}   r   r   r   r   r   r   maskr   r   s
             rO   r   z%UNetMidBlock2DSimpleCrossAttn.forward  s     <R;]!7ce!%%gt4@NNpq!084>TD "D'Qt<ab1AB 
	8LD& &;# )	M #=$7M
	8 rQ   )r   rq   r   r   r   r   Trq   r   r   FFNr   )r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   s   @rO   rV   rV   }  sm   
  '0$ $"#%(#'#%*.2!R.R. R. 	R.
 R. R. "%R. R. R. R.  R. #R. !R. R. #R.  'sm!R.n (,8<15;?9=%||% u||$%  (5	%
 !.% !)c3h 8% !) 6% 
%rQ   rV   c                       e Zd Z	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
ededededef fdZ	 	 	 ddej                  de
ej                     de
e   de
eeef      deej                  eej                  df   f   f
dZ xZS )r>   r   r    r!   r5   r   r#   r.   r$   r'   r   r3   r=   r)   r4   c                    t         |           g }g }|| _        |t        j	                  d| d       |}t        |      D ]V  }|dk(  r|n|}|j                  t        |||||	|||||

             |j                  t        |||z  ||||	dddd
             X t        j                  |      | _        t        j                  |      | _        |dk(  r1t        j                  t        |d||d	      g      | _        d| _        y |d
k(  r7t        j                  t        |||||	|||||
d      g      | _        d| _        y d | _        d| _        y )Nr   r7   r   r   T	r   r   r   r   r   r   rt   r   r   r?   opuse_convr    rs   namer   )r   r    r!   r   r   r5   r   r   r=   r   downF)ru   rv   r4   rK   rL   r   r   r   r   r   r   r   r   r   downsamplersr   )r}   r   r    r!   r5   r   r#   r.   r$   r'   r   r3   r=   r)   r4   r   r   r   r~   s                     rO   rv   zAttnDownBlock2D.__init__  s   " 	
.%NN{  }I  |J  JK  L ".z" 	A)*a+\KNN +!-"/"(#(?"/(;,  &*<</*="$1(,#'04!	> --
3}}W-f$ " $t,Xjqu!D8 ',#+ ( "!$0%1&3&, ',C&3,?!0!!D( ',# !%D&+#rQ   r   r   upsample_sizer   r_   .c                    ||ni }|j                  dd       t        j                  d       d}t        | j                  | j
                        D ]`  \  }}t        j                         r/| j                  r#| j                  |||      } ||fi |}||fz   }I |||      } ||fi |}||fz   }b | j                  9| j                  D ]$  }| j                  dk(  r |||      } ||      }& ||fz  }||fS )Nr   r   rJ   r   r   )r   rK   rL   r   r   r   r   r   r   r   r   r4   )	r}   r   r   r   r   output_statesr   r   downsamplers	            rO   r   zAttnDownBlock2D.forwardX  s-    <R;]!7ce!%%gt4@NNpqdoo> 	ALFD$$&4+F+F $ A A&-Y] ^ $] M6L M -0@ @ &}d ; $] M6L M -0@ @	A (#00 ?''83$/D$IM$/$>M	? m--Mm++rQ   )r   rq   r   r   r   r   Trq   r   rq   r?   )NNN)r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   r   r   s   @rO   r>   r>     s?     '0$ $"#%("#%[,[, [, 	[,
 [, [, [, "%[, [, [, [,  [, #[,  [, [,@ (,'+;? ,|| , u||$ ,  }	 ,
 !)c3h 8 , 
u||U5<<#455	6 ,rQ   r>   c            +           e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!dedededededeeee   f   deded	ed
edededededededededededef* fdZ		 	 	 	 	 	 d"de
j                  dee
j                     dee
j                     dee
j                     deeeef      dee
j                     dee
j                     dee
j                  ee
j                  df   f   fd Z xZS )#r@   r   r    r!   r5   r   r%   r#   r.   r$   r'   r   r&   r(   r=   r)   r"   r*   r+   r,   r-   r/   c                 v   t         |           g }g }d| _        || _        t	        |t
              r|g|z  }t        |      D ]  }|dk(  r|n|}|j                  t        |||||
|||	||
             |s+|j                  t        |||z  |||   ||
||||
             ]|j                  t        |||z  |d||
              t        j                  |      | _        t        j                  |      | _        |r1t        j                  t        |d||d      g      | _        d	| _        y d | _        d	| _        y )
NTr   r   r   r   r(   r   r+   r,   r-   r/   rq   r   r   r   F)ru   rv   r   r&   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r}   r   r    r!   r5   r   r%   r#   r.   r$   r'   r   r&   r(   r=   r)   r"   r*   r+   r,   r-   r/   r   r   r   r~   s                            rO   rv   zCrossAttnDownBlock2D.__init__|  sx   0 	
#' #6 2C8,H+IJ+V(z" )	A)*a+\KNN +!-"/"(#(?"/(;, (!!&+$(;;$0#?#B,?(5.C-A)9'5 !!*+$(;;$0#$,?(5	A)	T --
3}}W- " $t,Xjqu!D ',# !%D&+#rQ   r   r   r   r   r   r   additional_residualsr_   .c           	      .   |'|j                  dd       t        j                  d       d}t        t	        | j
                  | j                              }	t        |	      D ]  \  }
\  }}t        j                         r1| j                  r%| j                  |||      } ||||||d      d   }n |||      } ||||||d      d   }|
t        |	      dz
  k(  r|||z   }||fz   } | j                  | j                  D ]
  } ||      } ||fz   }||fS )Nr   r   rJ   Fr   r   rq   )r   rK   rL   listr   r   r   	enumerater   r   r   r   lenr   )r}   r   r   r   r   r   r   r   r   blocksr   r   r   r   s                 rO   r   zCrossAttnDownBlock2D.forward  s_    "-%))'48Dtuc$,,89!*6!2 	=A~$$&4+F+F $ A A&-Y] ^ $!*?+A#1+A %! ! !'}d ; $!*?+A#1+A %! ! CK!O#(<(H -0D D)],<<M5	=8 (#00 ; +M :; *],<<Mm++rQ   )r   rq   rq   r   r   r   r   Trq   r   r   rq   TFFFFr   NNNNNN)r   r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   r   s   @rO   r@   r@   {  s    ?@ '0$ $#$#'%("##%*&+%*!&'-Y,Y, Y, 	Y,
 Y, Y, ',CsO&<Y, Y, "%Y, Y, Y, Y, !Y, !Y, #Y,   !Y," #Y,$ #%Y,&  $'Y,( #)Y,* +Y,, -Y,| (,8<15;?9=7;4,||4, u||$4,  (5	4,
 !.4, !)c3h 84, !) 64, 'u||44, 
u||U5<<#455	64,rQ   r@   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededef fdZ	 ddej                  de
ej                     deej                  eej                  df   f   fdZ xZS )r:   r   r    r!   r5   r   r#   r.   r$   r'   r   r=   r"   r)   c                 V   t         |           g }t        |      D ]/  }|dk(  r|n|}|j                  t	        |||||	|||||

             1 t        j                  |      | _        |r1t        j                  t        |d||d      g      | _	        d| _
        y d | _	        d| _
        y )Nr   r   Tr   r   F)ru   rv   r   r   r   r   r   r   r   r   r   )r}   r   r    r!   r5   r   r#   r.   r$   r'   r   r=   r"   r)   r   r   r~   s                   rO   rv   zDownBlock2D.__init__  s      	z" 	A)*a+\KNN +!-"/"(#(?"/(;,	" }}W- " $t,Xjqu!D ',# !%D&+#rQ   r   r   r_   .c                 j   t        |      dkD  s|j                  dd       d}t        dd|       d}| j                  D ]E  }t	        j
                         r | j                  r| j                  |||      }n	 |||      }||fz   }G | j                  | j                  D ]
  } ||      } ||fz   }||fS Nr   r   The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.1.0.0rJ   	r   r   r
   r   r   r   r   r   r   	r}   r   r   argskwargsdeprecation_messager   r   r   s	            rO   r   zDownBlock2D.forwardB  s     t9q=FJJw5A #Ugw(;<ll 	=F$$&4+F+F $ A A&-Y] ^ &}d ;)],<<M	= (#00 ; +M :; *],<<Mm++rQ   
r   rq   r   r   r   r   Tr   Trq   r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   s   @rO   r:   r:     s      '0$ $%(#"#1,1, 1, 	1,
 1, 1, 1, "%1, 1, 1, 1, #1, 1,  1,h KO,"\\,19%,,1G,	u||U5<<#455	6,rQ   r:   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
ededef fdZdej                  dej                  fdZ
 xZS )rD   r   r    r5   r   r#   r.   r$   r'   r   r=   r"   r)   c                    t         |           g }t        |      D ]X  }|dk(  r|n|}|dk(  r$|j                  t	        ||d |||d||
	             5|j                  t        ||d ||||||
|	
             Z t        j                  |      | _        |r*t        j                  t        |d||d      g      | _
        y d | _
        y )Nr   r   r   r   Tr   r   )ru   rv   r   r   r   r   r   r   r   r   r   )r}   r   r    r5   r   r#   r.   r$   r'   r   r=   r"   r)   r   r   r~   s                  rO   rv   zDownEncoderBlock2D.__init__]  s     	z" 	A)*a+\K&)3)$/%1&*&, ',5&3,?
 !$/%1&*&, ',C&3,?!0#	@ }}W- " $t,Xjqu!D !%DrQ   r   r_   c                     t        |      dkD  s|j                  dd       d}t        dd|       | j                  D ]  } ||d       } | j                  | j                  D ]
  } ||      } |S Nr   r   r   r   r   )r   r   r
   r   r   )r}   r   r   r  r  r   r   s          rO   r   zDownEncoderBlock2D.forward  s    t9q=FJJw5A #Ugw(;<ll 	=F"=t<M	= (#00 ; +M :; rQ   r  r   r   r   r   r   r   r   rv   r   r   r   r   r   s   @rO   rD   rD   \  s    
  '0$ $%(#"#=%=% =% 	=%
 =% =% "%=% =% =% =% #=% =%  =%~U\\ u|| rQ   rD   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededef fdZdej                  dej                  fdZ
 xZS )rE   r   r    r5   r   r#   r.   r$   r'   r   r3   r=   r"   r)   c                 L   t         |           g }g }|
t        j                  d| d       |}
t	        |      D ]  }|dk(  r|n|}|dk(  r$|j                  t        ||d |||d||	             n$|j                  t        ||d |||||||	
             |j                  t        |||
z  |
|||dddd
              t        j                  |      | _        t        j                  |      | _        |r*t        j                  t        |d||d	
      g      | _        y d | _        y )Nr   r7   r   r   r   r   Tr   r   r   )ru   rv   rK   rL   r   r   r   r   r   r   r   r   r   r   r   )r}   r   r    r5   r   r#   r.   r$   r'   r   r3   r=   r"   r)   r   r   r   r~   s                    rO   rv   zAttnDownEncoderBlock2D.__init__  su     	
%NN{  }I  |J  JK  L ".z" ,	A)*a+\K&)3)$/%1&*&, ',5&3,?
 !$/%1&*&, ',C&3,?!0  &*<</*="$1(,#'04?,	\ --
3}}W- " $t,Xjqu!D !%DrQ   r   r_   c                 "   t        |      dkD  s|j                  dd       d}t        dd|       t        | j                  | j
                        D ]  \  }} ||d       } ||      } | j                  | j                  D ]
  } ||      } |S r  )r   r   r
   r   r   r   r   )r}   r   r   r  r  r   r   r   s           rO   r   zAttnDownEncoderBlock2D.forward  s    t9q=FJJw5A #Ugw(;<doo> 	0LFD"=t<M /M	0 (#00 ; +M :; rQ   )r   rq   r   r   r   r   Trq   r   Trq   r	  r   s   @rO   rE   rE     s    
  '0$ $"#%(#"#T%T% T% 	T%
 T% T% "%T% T% T% T%  T% #T% T%  T%lU\\ u|| rQ   rE   c                   <    e Zd Zddddddd ej                  d      df	ded	ed
edededededededededef fdZ		 	 dde
j                  dee
j                     dee
j                     dee
j                  ee
j                  df   e
j                  f   fdZ xZS )rC   r   rq   r   r   r   T       @r   r    r!   r5   r   r#   r.   r$   r   r3   r=   r"   c                    t         |           t        j                  g       | _        t        j                  g       | _        |
t        j                  d| d       |}
t        |      D ]  }|dk(  r|n|}| j
                  j                  t        ||||t        |dz  d      t        |dz  d      |||||	             | j                  j                  t        |||
z  |
||ddddd
              |rot        ||||t        |dz  d      |||||	ddd	
      | _        t        j                  t        ||      g      | _        t        j                   d|dd      | _        y d | _        d | _        d | _        y )Nr   r7   r   r   r   r   Tr   firr   r    r!   r   r   r5   r   r   r=   r   use_in_shortcutr   kernelr    r	   rq   rq   rr   stride)ru   rv   r   r   r   r   rK   rL   r   r   r   r   r   resnet_downr   r   rx   	skip_conv)r}   r   r    r!   r5   r   r#   r.   r$   r   r3   r=   r"   r   r~   s                 rO   rv   zAttnSkipDownBlock2D.__init__  s    	--+}}R(%NN{  }I  |J  JK  L ".z" 	A)*a+\KLL +!-"/"{a/4"<1#4b9#(?"/(;, OO"" &*<</*="$&(,#'04#	@ ,()+<1,b1$;+$7( $ D !#|Zf/g.h iDYYq,FSYZDN#D $D!DNrQ   r   r   skip_sampler_   .c                    t        |      dkD  s|j                  dd       d}t        dd|       d}t        | j                  | j
                        D ]  \  }}	 |||      } |	|      }||fz  } | j                  E| j                  ||      }| j                  D ]
  }
 |
|      } | j                  |      |z   }||fz  }|||fS r   )	r   r   r
   r   r   r   r   r  r  )r}   r   r   r  r   r  r  r   r   r   r   s              rO   r   zAttnSkipDownBlock2D.forwardc  s     t9q=FJJw5A #Ugw(;<doo> 	.LFD"=$7M /Mm--M	.
 ( ,,]DAM#00 7)+67 !NN;7-GMm--Mm[88rQ   NNr   r   r   npsqrtr   r   r   r   rv   r   r   r   r   r   r   r   s   @rO   rC   rC     s     '0$ $"#%,RWWS\#N"N" N" 	N"
 N" N" N" "%N" N" N"  N" #N" N"f (,.2	9||9 u||$9 ell+	9 
u||U5<<#45u||C	D9rQ   rC   c                   <    e Zd Zdddddd ej                  d      ddf	ded	ed
edededededededededef fdZ		 	 dde
j                  dee
j                     dee
j                     dee
j                  ee
j                  df   e
j                  f   fdZ xZS )rB   r   rq   r   r   r   Tr  r   r    r!   r5   r   r#   r.   r$   r   r=   r"   r)   c                 &   t         |           t        j                  g       | _        t        |      D ]T  }|dk(  r|n|}| j                  j                  t        ||||t        |dz  d      t        |dz  d      ||||
|	             V |rot        ||||t        |dz  d      ||||
|	ddd      | _	        t        j                  t        ||      g      | _        t        j                  d	|d
d
      | _        y d | _	        d | _        d | _        y )Nr   r   r   r   Tr  r  r  r	   r  r  )ru   rv   r   r   r   r   r   r   r   r  r   r   rx   r  )r}   r   r    r!   r5   r   r#   r.   r$   r   r=   r"   r)   r   r~   s                 rO   rv   zSkipDownBlock2D.__init__  s%    	}}R(z" 	A)*a+\KLL +!-"/"{a/4"<1#4b9#(?"/(;,	$ ,()+<1,b1$;+$7( $ D !#|Zf/g.h iDYYq,FSYZDN#D $D!DNrQ   r   r   r  r_   .c                 P   t        |      dkD  s|j                  dd       d}t        dd|       d}| j                  D ]  } |||      }||fz  } | j                  E| j                  ||      }| j                  D ]
  }	 |	|      } | j                  |      |z   }||fz  }|||fS r   )r   r   r
   r   r   r  r  )
r}   r   r   r  r   r  r  r   r   r   s
             rO   r   zSkipDownBlock2D.forward  s     t9q=FJJw5A #Ugw(;<ll 	.F"=$7Mm--M	. ( ,,]DAM#00 7)+67 !NN;7-GMm--Mm[88rQ   r  r  r   s   @rO   rB   rB     s     '0$ $%,RWWS\#"#9"9" 9" 	9"
 9" 9" 9" "%9" 9" 9" #9" 9"  9"| (,.2	9||9 u||$9 ell+	9 
u||U5<<#45u||C	D9rQ   rB   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededef fdZ	 ddej                  de
ej                     deej                  eej                  df   f   fdZ xZS )r;   r   r    r!   r5   r   r#   r.   r$   r'   r   r=   r"   r<   c                 f   t         |           g }t        |      D ]0  }|dk(  r|n|}|j                  t	        |||||	|||||
|             2 t        j                  |      | _        |r8t        j                  t	        |||||	|||||
|d      g      | _        d| _	        y d | _        d| _	        y )Nr   r   Tr   r    r!   r   r   r5   r   r   r=   r   r<   r   F)
ru   rv   r   r   r   r   r   r   r   r   )r}   r   r    r!   r5   r   r#   r.   r$   r'   r   r=   r"   r<   r   r   r~   s                   rO   rv   z ResnetDownsampleBlock2D.__init__  s      	z" 	A)*a+\KNN +!-"/"(#(?"/(;,"/	$ }}W- "!$0%1&3&, ',C&3,?!0&3!!D* ',# !%D&+#rQ   r   r   r_   .c                 l   t        |      dkD  s|j                  dd       d}t        dd|       d}| j                  D ]E  }t	        j
                         r | j                  r| j                  |||      }n	 |||      }||fz   }G | j                   | j                  D ]  } |||      } ||fz   }||fS r   r   r   s	            rO   r   zResnetDownsampleBlock2D.forward  s     t9q=FJJw5A #Ugw(;<ll 	=F$$&4+F+F $ A A&-Y] ^ &}d ;)],<<M	= (#00 A +M4 @A *],<<Mm++rQ   )
r   rq   r   r   r   r   Tr   TFr   r  r   s   @rO   r;   r;     s      '0$ $%(##=,=, =, 	=,
 =, =, =, "%=, =, =, =, #=, =, =,@ KO,"\\,19%,,1G,	u||U5<<#455	6,rQ   r;   c            $           e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededededededee   f" fdZ	 	 	 	 	 dde	j                  dee	j                     dee	j                     dee	j                     deeeef      dee	j                     dee	j                  ee	j                  df   f   fdZ xZS )rA   r   r    r!   r5   r   r#   r.   r$   r'   r   r3   r(   r=   r"   r<   r,   r2   c                    t         |           d| _        g }g }|| _        || j                  z  | _        t        |      D ]  }|dk(  r|n|}|j                  t        |||||	|||||
|             t        t        d      r
t               n	t               }|j                  t        ||| j                  |||	dd|||              t        j                  |      | _        t        j                  |      | _        |r8t        j                  t        |||||	|||||
|d      g      | _        d| _        y d | _        d| _        y )NTr   r   r   r   r%  F)ru   rv   r   r3   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r}   r   r    r!   r5   r   r#   r.   r$   r'   r   r3   r(   r=   r"   r<   r,   r2   r   r   r   r   r~   s                         rO   rv   z#SimpleCrossAttnDownBlock2D.__init__7  sq   ( 	#' 
"4%)@)@@z" $	A)*a+\KNN +!-"/"(#(?"/(;,"/" .5Q8V-W')]q]s  *(4../&9$1#')=)='-$	J --
3}}W- "!$0%1&3&, ',C&3,?!0&3!!D* ',# !%D&+#rQ   r   r   r   r   r   r   r_   .c                    ||ni }|j                  dd       t        j                  d       d}||d n|}n|}t        | j                  | j
                        D ]`  \  }	}
t        j                         r,| j                  r | j                  |	||      } |
|f||d|}n |	||      } |
|f||d|}||fz   }b | j                   | j                  D ]  } |||      } ||fz   }||fS )Nr   r   rJ   r   r   rK   rL   r   r   r   r   r   r   r   r   )r}   r   r   r   r   r   r   r   r   r   r   r   s               rO   r   z"SimpleCrossAttnDownBlock2D.forward  sJ    <R;]!7ce!%%gt4@NNpq!084>TD "Ddoo> 	=LFD$$&4+F+F $ A A&-Y] ^ $!!*?#'! -	! !'}d ; $!!*?#'! -	! *],<<M'	=* (#00 A +M4 @A *],<<Mm++rQ   )r   rq   r   r   r   r   Trq   r   r   TFFNr   )r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   r   s   @rO   rA   rA   6  s     '0$ $"##'%(##%*.2%\,\, \, 	\,
 \, \, \, "%\, \, \, \,  \, !\, #\, \,  !\," ##\,$ 'sm%\,B (,8<15;?9=5,||5, u||$5,  (5	5,
 !.5, !)c3h 85, !) 65, 
u||U5<<#455	65,rQ   rA   c                        e Zd Z	 	 	 	 	 	 ddedededededededed	ef fd
Z	 ddej                  de
ej                     deej                  eej                  df   f   fdZ xZS )rF   r   r    r!   r5   r   r#   r$   resnet_group_sizer"   c
                 ^   t         |           g }
t        |      D ]9  }|dk(  r|n|}||z  }||z  }|
j                  t	        ||||||||dd
             ; t        j                  |
      | _        |	r+t        j                  t               g      | _	        d| _
        y d | _	        d| _
        y )Nr   	ada_groupF
r   r    r5   r!   r   r   r   r   r   conv_shortcut_bias)ru   rv   r   r   r   r   r   r   r   r   r   )r}   r   r    r!   r5   r   r#   r$   r,  r"   r   r   r   r   r~   s                 rO   rv   zKDownBlock2D.__init__  s     	z" 	A)*a+\K $55F%)::JNN% +!-#"/!)""/(3',	( }}W- "}.? @D ',# !%D&+#rQ   r   r   r_   .c                 ^   t        |      dkD  s|j                  dd       d}t        dd|       d}| j                  D ]E  }t	        j
                         r | j                  r| j                  |||      }n	 |||      }||fz  }G | j                  | j                  D ]
  } ||      } ||fS r   r   r   s	            rO   r   zKDownBlock2D.forward  s     t9q=FJJw5A #Ugw(;<ll 	.F$$&4+F+F $ A A&-Y] ^ &}d ;m--M	. (#00 ; +M :; m++rQ   )r   r   h㈵>gelur   Fr   r  r   s   @rO   rF   rF     s      #!#$+,+, +, 	+,
 +, +, +, +, +, +,\ KO,"\\,19%,,1G,	u||U5<<#455	6,rQ   rF   c                   Z    e Zd Z	 	 	 	 	 	 	 	 ddedededededededed	ed
ededef fdZ	 	 	 	 	 ddej                  de
ej                     de
ej                     de
ej                     de
eeef      de
ej                     deej                  eej                  df   f   fdZ xZS )rG   r   r    r!   r(   r5   r   r,  r"   r3   rH   r#   r$   c                    t         |           g }g }d| _        t        |      D ]_  }|dk(  r|n|}||z  }||z  }|j	                  t        ||||||||dd
             |j	                  t        |||	z  |	||d|
d|	             a t        j                  |      | _	        t        j                  |      | _
        |r+t        j                  t               g      | _        d| _        y d | _        d| _        y )NTr   r.  Fr/  
layer_norm)r(   r!   attention_biasrH   r2   
group_size)ru   rv   r   r   r   r   KAttentionBlockr   r   r   r   r   r   r   )r}   r   r    r!   r(   r5   r   r,  r"   r3   rH   r#   r$   r   r   r   r   r   r~   s                     rO   rv   zKCrossAttnDownBlock2D.__init__  s    	
#' z" 	A)*a+\K $55F%)::JNN% +!-#"/!)""/(3',   $66&(;"/#''9)50
'	B }}W---
3 "}.? @D ',# !%D&+#rQ   r   r   r   r   r   r   r_   .c           	         ||ni }|j                  dd       t        j                  d       d}t        | j                  | j
                        D ]v  \  }}	t        j                         r.| j                  r"| j                  |||      } |	||||||      }n |||      } |	||||||      }| j                  |dz  }q||fz  }x | j                  | j                  D ]
  }
 |
|      } ||fS )Nr   r   rJ   r   embr   r   r   r   r*  )r}   r   r   r   r   r   r   r   r   r   r   s              rO   r   zKCrossAttnDownBlock2D.forwardT  s/    <R;]!7ce!%%gt4@NNpqdoo> 	2LFD$$&4+F+F $ A A!!
 !%!*?#1+A+A! !'}d ; $!*?#1+A+A!   ((-!11;	2> (#00 ; +M :; m++rQ   )r   r   r   T@   Fr2  r3  r   )r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   r   r   s   @rO   rG   rG     sD    !##"$#( #>,>, >, 	>,
 !>, >, >, >, >,  >, !>, >, >,F (,8<15;?9=2,||2, u||$2,  (5	2,
 !.2, !)c3h 82, !) 62, 
u||U5<<#455	62,rQ   rG   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededededef fdZ	 	 ddej                  de
ej                  df   deej                     dee   dej                  f
dZ xZS )re   r   r[   r    r!   r]   r5   r   r#   r.   r$   r'   r   r3   r=   r^   c                    t         |           g }g }|| _        |t        j	                  d| d       |}t        |      D ]e  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t        ||z   ||||||	|
||
             |j                  t        |||z  ||||dddd
             g t        j                  |      | _        t        j                  |      | _        |dk(  r(t        j                  t        |d|	      g      | _        n<|d
k(  r0t        j                  t        |||||||	|
||d      g      | _        nd | _        d| _        || _        y )Nr   r7   rq   r   r   Tr   r?   r   r    r   )r   r    r!   r   r   r5   r   r   r=   r   upF)ru   rv   r^   rK   rL   r   r   r   r   r   r   r   r   r   
upsamplersr   r]   )r}   r   r[   r    r!   r]   r5   r   r#   r.   r$   r'   r   r3   r=   r^   r   r   r   res_skip_channelsresnet_in_channelsr~   s                        rO   rv   zAttnUpBlock2D.__init__  s   $ 	
*%NN{  }I  |J  JK  L ".z" 	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;,  &*<</*="$1(,#'04%	B --
3}}W-F" mmZtbn-o,pqDOh& mm!$0%1&3&, ',C&3,?!0DO$ #DO&+#,rQ   r   res_hidden_states_tuple.r   r   r_   c                    t        |      dkD  s|j                  dd       d}t        dd|       t        | j                  | j
                        D ]u  \  }}	|d   }
|d d }t        j                  ||
gd      }t        j                         r(| j                  r| j                  |||      } |	|      }e |||      } |	|      }w | j                  3| j                  D ]$  }| j                  dk(  r |||	      } ||      }& |S )
Nr   r   r   r   rq   dimr   r   )r   r   r
   r   r   r   r   catr   r   r   rB  r^   )r}   r   rE  r   r   r   r  r  r   r   res_hidden_states	upsamplers               rO   r   zAttnUpBlock2D.forward  s    t9q=FJJw5A #Ugw(;<doo> 	4LFD 7 ;&=cr&B#!II}6G&HaPM$$&4+F+F $ A A&-Y] ^ $] 3 &}d ; $] 3	4 ??&!__ =	%%1$-m$$GM$-m$<M	= rQ   )Nr   rq   r   r   r   r   Trq   r   r?   r  )r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   s   @rO   re   re     s0    # '0$ $"#%(#!Z-Z- !Z- 	Z-
 Z- Z- Z- Z- Z- "%Z- Z- Z- Z-  Z- #Z-  !Z-@ (,'+!||! "'u||S'8!9! u||$	!
  }! 
!rQ   re   c            -           e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d#dededededee   dededeeee   f   d	ed
ededede	dededede	de	de	de	de	def, fdZ
	 	 	 	 	 	 d$dej                  deej                  df   deej                     deej                     deeeef      dee   deej                     d eej                     d!ej                  fd"Z xZS )%rc   r   r    r[   r!   r]   r5   r   r%   r#   r.   r$   r'   r   r&   r(   r=   r\   r*   r+   r,   r-   r/   c                    t         |           g }g }d| _        || _        t	        |t
              r|g|z  }t        |      D ]  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t        ||z   |||	|||
|||
             |s+|j                  t        |||z  |||   ||||||
             l|j                  t        |||z  |d||              t        j                  |      | _        t        j                  |      | _        |r(t        j                  t        |d|      g      | _        nd | _        d| _        || _        y )	NTrq   r   r   r   r   r@  F)ru   rv   r   r&   r   r   r   r   r   r   r   r   r   r   r   r   rB  r   r]   )r}   r   r    r[   r!   r]   r5   r   r%   r#   r.   r$   r'   r   r&   r(   r=   r\   r*   r+   r,   r-   r/   r   r   r   rC  rD  r~   s                               rO   rv   zCrossAttnUpBlock2D.__init__	  s   2 	
#' #6 2C8,H+IJ+V(z" +	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;, (!!&+$(;;$0#?#B,?(5.C-A)9'5 !!*+$(;;$0#$,?(5	E+	X --
3}}W- mmZtbn-o,pqDO"DO&+#,rQ   r   rE  .r   r   r   r   r   r   r_   c	           
         |'|j                  dd       t        j                  d       t        | dd       xr+ t        | dd       xr t        | dd       xr t        | dd       }	t	        | j
                  | j                        D ]  \  }
}|d   }|d d }|	rGt        | j                  ||| j                  | j                  | j                  | j                        \  }}t        j                  ||gd	
      }t        j                         r1| j                   r%| j#                  |
||      } ||||||d      d   } |
||      } ||||||d      d   } | j$                  | j$                  D ]  } |||      } |S )Nr   r   s1s2b1b2rG  rP  rQ  rR  rS  rq   rH  Fr   r   )r   rK   rL   getattrr   r   r   r   r]   rP  rQ  rR  rS  r   rJ  r   r   r   rB  )r}   r   rE  r   r   r   r   r   r   is_freeu_enabledr   r   rK  rL  s                 rO   r   zCrossAttnUpBlock2D.forwarde	  s    "-%))'48Dtu D$% *dD)*dD)* dD)	 	  doo> &	LFD 7 ;&=cr&B#  3>''!%wwwwwwww400 "II}6G&HaPM$$&4+F+F $ A A&-Y] ^ $!*?+A#1+A %! ! !'}d ; $!*?+A#1+A %! !?&	P ??&!__ H	 )- GH rQ   )Nr   rq   rq   r   r   r   r   Trq   r   r   TFFFFr   r   r   r   s   @rO   rc   rc   
	  s	    )-?@ '0$ $#$#'%(!%*&+%*!&'/X-X- X- !	X-
 X- !X- X- X- ',CsO&<X- X- "%X- X- X- X- !X-  !!X-" ##X-$ %X-& #'X-(  $)X-* #+X-, -X-. /X-| (,8<;?'+159=B||B "'u||S'8!9B u||$	B
  (5B !)c3h 8B  }B !.B !) 6B 
BrQ   rc   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 ddededededee   dededed	ed
ededededef fdZ	 	 dde	j                  dee	j                  df   dee	j                     dee   de	j                  f
dZ xZS )ra   r   r[   r    r!   r]   r5   r   r#   r.   r$   r'   r   r=   r\   c                 p   t         |           g }t        |      D ]>  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t	        ||z   ||||||	|
||
             @ t        j                  |      | _        |r(t        j                  t        |d|      g      | _	        nd | _	        d| _
        || _        y )Nrq   r   r   Tr@  F)ru   rv   r   r   r   r   r   r   r   rB  r   r]   )r}   r   r[   r    r!   r]   r5   r   r#   r.   r$   r'   r   r=   r\   r   r   rC  rD  r~   s                      rO   rv   zUpBlock2D.__init__	  s    " 	z" 	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;,		& }}W- mmZtbn-o,pqDO"DO&+#,rQ   r   rE  .r   r   r_   c           
         t        |      dkD  s|j                  dd       d}t        dd|       t        | dd       xr+ t        | dd       xr t        | dd       xr t        | dd       }| j                  D ]  }	|d	   }
|d d	 }|rGt        | j                  ||
| j                  | j                  | j                  | j                  
      \  }}
t        j                  ||
gd      }t        j                         r | j                  r| j                  |	||      } |	||      } | j                   | j                   D ]  } |||      } |S )Nr   r   r   r   rP  rQ  rR  rS  rG  rT  rq   rH  )r   r   r
   rU  r   r   r]   rP  rQ  rR  rS  r   rJ  r   r   r   rB  )r}   r   rE  r   r   r   r  r  rV  r   rK  rL  s               rO   r   zUpBlock2D.forward	  sl    t9q=FJJw5A #Ugw(;< D$% *dD)*dD)* dD)	 	 ll 	<F 7 ;&=cr&B#  3>''!%wwwwwwww400 "II}6G&HaPM$$&4+F+F $ A A&-Y] ^ &}d ;-	<0 ??&!__ H	 )- GH rQ   )
Nr   rq   r   r   r   r   Tr   Tr  r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   s   @rO   ra   ra   	  s    )- '0$ $%(!/-/- !/- 	/-
 /- !/- /- /- /- "%/- /- /- /- #/- /-j (,'+0||0 "'u||S'8!90 u||$	0
  }0 
0rQ   ra   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 	 ddededee   dededededed	ed
edededee   f fdZdde	j                  dee	j                     de	j                  fdZ xZS )rh   r   r    r]   r5   r   r#   r.   r$   r'   r   r=   r\   r!   c                    t         |           g }t        |      D ]X  }|dk(  r|n|}|dk(  r$|j                  t	        |||||	|d||	             5|j                  t        |||||	|||||

             Z t        j                  |      | _        |r/t        j                  t        |d|      g      | _
        || _        y d | _
        || _        y )Nr   r   r   r   Tr@  )ru   rv   r   r   r   r   r   r   r   r   rB  r]   )r}   r   r    r]   r5   r   r#   r.   r$   r'   r   r=   r\   r!   r   r   input_channelsr~   s                    rO   rv   zUpDecoderBlock2D.__init__
  s      	z" 	A,-F[N&)3)$2%1&3&, ',5&3,?
 !$2%1&3&, ',C&3,?!0%	B }}W- mmZtbn-o,pqDO - #DO,rQ   r   r   r_   c                     | j                   D ]  } |||      } | j                  | j                  D ]
  } ||      } |S Nr   )r   rB  )r}   r   r   r   rL  s        rO   r   zUpDecoderBlock2D.forwardM
  sR    ll 	=F"=t<M	= ??&!__ 9	 )- 89 rQ   )Nr   rq   r   r   r   r   Tr   TNr   r   r   r   r   r   r   r   r   rv   r   r   r   r   r   s   @rO   rh   rh   
  s    
 )- '0$ $%(!'+;-;- ;- !	;-
 ;- ;- ;- "%;- ;- ;- ;- #;- ;-  };-zU\\ %,,9O [`[g[g rQ   rh   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 ddededee   dededededed	ed
ededededee   f fdZdde	j                  dee	j                     de	j                  fdZ xZS )ri   r   r    r]   r5   r   r#   r.   r$   r'   r   r3   r=   r\   r!   c                    t         |           g }g }|t        j                  d| d       |}t	        |      D ]  }|dk(  r|n|}|dk(  r$|j                  t        |||||	|d||	             n$|j                  t        |||||	|||||

             |j                  t        |||z  ||||dk7  r|	nd |dk(  r|nd dddd              t        j                  |      | _        t        j                  |      | _        |r/t        j                  t        |d|	      g      | _        || _        y d | _        || _        y )
NjIt is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `out_channels`: r7   r   r   r   r   Tr   r@  )ru   rv   rK   rL   r   r   r   r   r   r   r   r   r   r   rB  r]   )r}   r   r    r]   r5   r   r#   r.   r$   r'   r   r3   r=   r\   r!   r   r   r   r]  r~   s                      rO   rv   zAttnUpDecoderBlock2D.__init__Y
  s   " 	
%NN|  ~J  }K  KL  M ".z" /	A,-F[N&)3)$2%1&3&, ',5&3,?
 !$2%1&3&, ',C&3,?!0  &*<</*="5LPY5YM_c6MQZ6Z]`d(,#'04C/	b --
3}}W- mmZtbn-o,pqDO - #DO,rQ   r   r   r_   c                     t        | j                  | j                        D ]  \  }} |||      } |||      } | j                  | j                  D ]
  } ||      } |S r_  )r   r   r   rB  )r}   r   r   r   r   rL  s         rO   r   zAttnUpDecoderBlock2D.forward
  sm    doo> 	;LFD"=t<M T:M	; ??&!__ 9	 )- 89 rQ   )Nr   rq   r   r   r   r   Trq   r   TNr   r`  r   s   @rO   ri   ri   X
  s    
 )- '0$ $"#%(!'+T-T- T- !	T-
 T- T- T- "%T- T- T- T-  T- #T- T-  }T-l	U\\ 	%,,9O 	[`[g[g 	rQ   ri   c                   0    e Zd Zdddddddd ej                  d      df
d	ed
edededee   dededededede	dedede	f fdZ
	 	 ddej                  deej                  df   deej                     deej                  ej                  f   fdZ xZS )rg   Nr   rq   r   r   r   Tr  r   r[   r    r!   r]   r5   r   r#   r.   r$   r   r3   r=   r\   c                    t         |           t        j                  g       | _        t        j                  g       | _        t        |      D ]f  }||dz
  k(  r|n|}|dk(  r|n|}| j
                  j                  t        ||z   |||t        ||dz  z   d      t        |dz  d      ||	|
||             h |t        j                  d| d       |}| j                  j                  t        |||z  |||ddddd	
             t        ||
      | _        |rt        ||||t        |dz  d      t        |dz  d      ||	|
||ddd      | _        t        j                   |dddd      | _        t$        j                  j'                  t        |dz  d      ||d      | _        t        j*                         | _        || _        y d | _        d | _        d | _        d | _        || _        y )Nrq   r   r   r   r   rc  r7   Tr   r  r  r   r    r!   r   r   r   r5   r   r   r=   r   r  rA  r  r	   r	   r	   r  rr   r  rs   
num_groupsnum_channelsr   affine)ru   rv   r   r   r   r   r   r   r   r   rK   rL   r   r   rL  	resnet_uprx   r  r   	GroupNorm	skip_normSiLUactr]   )r}   r   r[   r    r!   r]   r5   r   r#   r.   r$   r   r3   r=   r\   r   rC  rD  r~   s                     rO   rv   zAttnSkipUpBlock2D.__init__
  s,   " 	--+}}R(z" 	A01Z!^0C,89Q!4LLL 25F F!-"/"14E4JJBO"<1#4b9#(?"/(;,		( %NN|  ~J  }K  KL  M "."&88+&9 "$(#,0	
 '{N*()+<1,b1|q0"5$;+$7( $DN   YY|QFSYcijDN"XX//|q0"5LV`im 0 DN wwyDH - "DN!DN!DNDH,rQ   r   rE  .r   r_   c                    t        |      dkD  s|j                  dd       d}t        dd|       | j                  D ].  }|d   }	|d d }t	        j
                  ||	gd      } |||      }0  | j                  d   |      }|| j                  |      }nd}| j                  J| j                  |      }
| j                  |
      }
| j                  |
      }
||
z   }| j                  ||      }||fS Nr   r   r   r   rG  rq   rH  )r   r   r
   r   r   rJ  r   rL  rn  rp  rr  r  r}   r   rE  r   r  r   r  r  r   rK  skip_sample_statess              rO   r   zAttnSkipUpBlock2D.forward  s    t9q=FJJw5A #Ugw(;<ll 	8F 7 ;&=cr&B#!II}6G&HaPM"=$7M	8 +*=9"..5KK>>%!%!>!%*<!=!%0B!C%(::K NN=$?Mk))rQ   r  r   r   r   r  r  r   r   r   r   r   rv   r   r   r   r   r   r   s   @rO   rg   rg   
  s0    )- '0$ $"#%,RWWS\![-[- ![- 	[-
 [- ![- [- [- [- "%[- [- [-  [- #[- [-B (,%*||%* "'u||S'8!9%* u||$	%* 
u||U\\)	*%*rQ   rg   c                   0    e Zd Zddddddd ej                  d      ddf
d	ed
edededee   dededededede	dede	def fdZ
	 	 ddej                  deej                  df   deej                     deej                  ej                  f   fdZ xZS )rf   Nr   rq   r   r   r   Tr  r   r[   r    r!   r]   r5   r   r#   r.   r$   r   r=   r\   upsample_paddingc                    t         |           t        j                  g       | _        t        |      D ]f  }||dz
  k(  r|n|}|dk(  r|n|}| j                  j                  t        ||z   |||t        ||z   dz  d      t        |dz  d      ||	|
||             h t        ||      | _
        |rt        ||||t        |dz  d      t        |dz  d      ||	|
||ddd	      | _        t        j                  |d
ddd      | _        t        j                  j                  t        |dz  d      ||d      | _        t        j"                         | _        || _        y d | _        d | _        d | _        d | _        || _        y )Nrq   r   r   r   r   r  Tr  rg  r	   rh  r  ri  rj  )ru   rv   r   r   r   r   r   r   r   r   rL  rn  rx   r  r   ro  rp  rq  rr  r]   )r}   r   r[   r    r!   r]   r5   r   r#   r.   r$   r   r=   r\   ry  r   rC  rD  r~   s                     rO   rv   zSkipUpBlock2D.__init__B  s   " 	}}R(z" 	A01Z!^0C,89Q!4LLL 25F F!-"/" 25F F1LbQ"<1#4b9#(?"/(;,		( '{N*()+<1,b1|q0"5$;+$7( $DN   YY|QFSYcijDN"XX//|q0"5LV`im 0 DN wwyDH - "DN!DN!DNDH,rQ   r   rE  .r   r_   c                    t        |      dkD  s|j                  dd       d}t        dd|       | j                  D ].  }|d   }	|d d }t	        j
                  ||	gd      } |||      }0 || j                  |      }nd}| j                  J| j                  |      }
| j                  |
      }
| j                  |
      }
||
z   }| j                  ||      }||fS rt  )r   r   r
   r   r   rJ  rL  rn  rp  rr  r  ru  s              rO   r   zSkipUpBlock2D.forward  s     t9q=FJJw5A #Ugw(;<ll 	8F 7 ;&=cr&B#!II}6G&HaPM"=$7M	8 "..5KK>>%!%!>!%*<!=!%0B!C%(::K NN=$?Mk))rQ   r  rw  r   s   @rO   rf   rf   A  s0    )- '0$ $%,RWWS\! !E-E- !E- 	E-
 E- !E- E- E- E- "%E- E- E- #E- E- E-V (,#*||#* "'u||S'8!9#* u||$	#* 
u||U\\)	*#*rQ   rf   c                       e Zd Z	 	 	 	 	 	 	 	 	 	 	 ddededededee   dededed	ed
edededededef fdZ	 	 dde	j                  dee	j                  df   dee	j                     dee   de	j                  f
dZ xZS )rb   r   r[   r    r!   r]   r5   r   r#   r.   r$   r'   r   r=   r\   r<   c                    t         |           g }t        |      D ]?  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t	        ||z   ||||||	|
|||             A t        j                  |      | _        |r1t        j                  t	        |||||||	|
|||d      g      | _        nd | _        d| _	        || _
        y )Nrq   r   r   Tr   r    r!   r   r   r5   r   r   r=   r   r<   rA  F)ru   rv   r   r   r   r   r   r   rB  r   r]   )r}   r   r[   r    r!   r]   r5   r   r#   r.   r$   r'   r   r=   r\   r<   r   r   rC  rD  r~   s                       rO   rv   zResnetUpsampleBlock2D.__init__  s    $ 	z" 	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;,"/		( }}W- mm!$0%1&3&, ',C&3,?!0&3DO& #DO&+#,rQ   r   rE  .r   r   r_   c                    t        |      dkD  s|j                  dd       d}t        dd|       | j                  D ]b  }|d   }	|d d }t	        j
                  ||	gd      }t	        j                         r | j                  r| j                  |||      }Z |||      }d | j                  | j                  D ]  }
 |
||      } |S rt  )
r   r   r
   r   r   rJ  r   r   r   rB  )r}   r   rE  r   r   r   r  r  r   rK  rL  s              rO   r   zResnetUpsampleBlock2D.forward  s     t9q=FJJw5A #Ugw(;<ll 		<F 7 ;&=cr&B#!II}6G&HaPM$$&4+F+F $ A A&-Y] ^ &}d ;		< ??&!__ ?	 )- >? rQ   )Nr   rq   r   r   r   r   Tr   TFr  rZ  r   s   @rO   rb   rb     s4    )- '0$ $%(!#!B-B- !B- 	B-
 B- !B- B- B- B- "%B- B- B- B- #B- B-  !B-P (,'+|| "'u||S'8!9 u||$	
  } 
rQ   rb   c            (           e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d dededededee   dededed	ed
edededededededededee   f& fdZ	 	 	 	 	 	 d!de	j                  dee	j                  df   dee	j                     dee	j                     dee   dee	j                     deeeef      dee	j                     de	j                  fdZ xZS )"rd   r   r    r[   r!   r]   r5   r   r#   r.   r$   r'   r   r3   r(   r=   r\   r<   r,   r2   c                    t         |           g }g }d| _        || _        || j                  z  | _        t        |      D ]  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t        ||z   ||||||	|
|||             t        t        d      r
t               n	t               }|j                  t        ||| j                  | j                  ||dd|||              t        j                  |      | _        t        j                  |      | _        |r1t        j                  t        |||||||	|
|||d      g      | _        nd | _        d| _        || _        y )	NTrq   r   r   r   r   r~  F)ru   rv   r   r3   r   r   r   r   r   r   r   r   r   r   r   r   r   rB  r   r]   )r}   r   r    r[   r!   r]   r5   r   r#   r.   r$   r'   r   r3   r(   r=   r\   r<   r,   r2   r   r   r   rC  rD  r   r~   s                             rO   rv   z!SimpleCrossAttnUpBlock2D.__init__  s   , 	
#' "4%)@)@@z" &	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;,"/" .5Q8V-W')]q]s  *(4..!44&9$1#')=)='1&	N --
3}}W- mm!$0%1&3&, ',C&3,?!0&3DO& #DO&+#,rQ   r   rE  .r   r   r   r   r   r   r_   c	                    ||ni }|j                  dd       t        j                  d       ||d n|}	n|}	t        | j                  | j
                        D ]}  \  }
}|d   }|d d }t        j                  ||gd      }t        j                         r,| j                  r | j                  |
||      } ||f||	d|}i |
||      } ||f||	d|} | j                  | j                  D ]  } |||      } |S )Nr   r   rG  rq   rH  r   )r   rK   rL   r   r   r   r   rJ  r   r   r   rB  )r}   r   rE  r   r   r   r   r   r   r   r   r   rK  rL  s                 rO   r   z SimpleCrossAttnUpBlock2D.forwardv  sN    <R;]!7ce!%%gt4@NNpq!084>TD "Ddoo> 	LFD !8 ;&=cr&B#!II}6G&HaPM$$&4+F+F $ A A&-Y] ^ $!!*?#'! -	! !'}d ; $!!*?#'! -	!%	2 ??&!__ ?	 )- >? rQ   )Nr   rq   r   r   r   r   Trq   r   r   TFFNr   )r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   r   s   @rO   rd   rd     s    )- '0$ $"##'%(!#%*.2)`-`- `- !	`-
 `- !`- `- `- `- "%`- `- `- `-  `- !`-  #!`-" #`-$ %`-& #'`-( 'sm)`-L (,8<'+15;?9=7||7 "'u||S'8!97 u||$	7
  (57  }7 !.7 !)c3h 87 !) 67 
7rQ   rd   c                        e Zd Z	 	 	 	 	 	 ddedededededededed	ee   d
ef fdZ	 	 dde	j                  dee	j                  df   dee	j                     dee   de	j                  f
dZ xZS )rj   r   r    r!   r]   r5   r   r#   r$   r,  r\   c                    t         |           g }d|z  }|}|dz
  }t        |      D ]C  }|dk(  r|n|}||	z  }||	z  }|j                  t	        |||dz
  k(  r|n|||||||dd
             E t        j                  |      | _        |
r$t        j                  t               g      | _	        nd | _	        d| _
        || _        y )Nr   rq   r   r.  F)
r   r    r!   r   r   r   r5   r   r   r0  )ru   rv   r   r   r   r   r   r   r   rB  r   r]   )r}   r   r    r!   r]   r5   r   r#   r$   r,  r\   r   k_in_channelsk_out_channelsr   r   r   r~   s                    rO   rv   zKUpBlock2D.__init__  s     	L($!^
z" 	A+,6-|K $55F%)::JNN% +45a4Gl"/"!)#"/(3',	( }}W- mm[]O<DO"DO&+#,rQ   r   rE  .r   r   r_   c                    t        |      dkD  s|j                  dd       d}t        dd|       |d   }|t        j                  ||gd      }| j
                  D ]?  }t        j                         r | j                  r| j                  |||      }7 |||      }A | j                  | j                  D ]
  }	 |	|      } |S rt  )
r   r   r
   r   rJ  r   r   r   r   rB  )
r}   r   rE  r   r   r   r  r  r   rL  s
             rO   r   zKUpBlock2D.forward  s     t9q=FJJw5A #Ugw(;<"9""=".!II}6M&NTUVMll 	<F$$&4+F+F $ A A&-Y] ^ &}d ;		< ??&!__ 9	 )- 89 rQ   )r      r2  r3  r   Tr  )r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   s   @rO   rj   rj     s      #+-!/-/- /- 	/-
 /- /- /- /- /- $C=/- /-j (,'+|| "'u||S'8!9 u||$	
  } 
rQ   rj   c                   f    e Zd Z	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededef fdZ	 	 	 	 	 	 ddej                  de
ej                  df   deej                     deej                     deeeef      dee   deej                     deej                     dej                  fdZ xZS )rk   r   r    r!   r]   r5   r   r#   r$   r,  r3   r(   r\   r-   c                    t         |           g }g }||cxk(  xr |k(  nc }||k7  }|rdnd}d| _        |
| _        |r|nd|z  }|}|dz
  }t	        |      D ]  }|dk(  r|n|}||	z  }||	z  }|r||dz
  k(  r|}nd }|j                  t        |||||||||dd             |j                  t        ||dz
  k(  r|n|||dz
  k(  r||
z  n||
z  |
||d|d|		              t        j                  |      | _
        t        j                  |      | _        |r$t        j                  t               g      | _        nd | _        d| _        || _        y )
NTFr   rq   r   r.  )r   r    conv_2d_out_channelsr!   r   r   r   r5   r   r   r0  r6  )r(   r!   r7  rH   r2   r-   )ru   rv   r   r3   r   r   r   r9  r   r   r   r   r   rB  r   r]   )r}   r   r    r!   r]   r5   r   r#   r$   r,  r3   r(   r\   r-   r   r   is_first_blockis_middle_blockrH   r  r  r   r   r   r  r~   s                            rO   rv   zKCrossAttnUpBlock2D.__init__  s     	
$EE%5%3T#' "4 )7A<L$!^
z" '	A+,6-|K $55F%)::JAa$7'5$'+$NN% +!-)="/"!)#"/(3', '(JN':NZ!^+ #&88%);;&(;"/#''9)5%53'	R }}W---
3 mm[]O<DO"DO&+#,rQ   r   rE  .r   r   r   r   r   r   r_   c	           	         |d   }|t        j                  ||gd      }t        | j                  | j                        D ]^  \  }	}
t        j
                         r.| j                  r"| j                  |	||      } |
||||||      }H |	||      } |
||||||      }` | j                  | j                  D ]
  } ||      } |S )NrG  rq   rH  r;  )	r   rJ  r   r   r   r   r   r   rB  )r}   r   rE  r   r   r   r   r   r   r   r   rL  s               rO   r   zKCrossAttnUpBlock2D.forwardV  s     #:""=".!II}6M&NTUVMdoo> 	LFD$$&4+F+F $ A A!!
 !%!*?#1+A+A! !'}d ; $!*?#1+A+A!#	4 ??&!__ 9	 )- 89 rQ   )	r   r   r2  r3  r   rq      TFr   )r   r   r   r   r   r   r   rv   r   r   r   r   r   r   r   r   r   s   @rO   rk   rk      sp     #!#"##&!!&S-S- S- 	S-
 S- S- S- S- S- S-  S- !S- S- S-r (,8<;?'+159=-||- "'u||S'8!9- u||$	-
  (5- !)c3h 8-  }- !.- !) 6- 
-rQ   rk   c                       e Zd ZdZ	 	 	 	 	 	 	 	 ddededededee   deded	ed
edee   def fdZ	de
j                  dedede
j                  fdZde
j                  dedede
j                  fdZ	 	 	 	 	 dde
j                  dee
j                     dee
j                     dee
j                     deeeef      dee
j                     de
j                  fdZ xZS )r9  aN  
    A basic Transformer block.

    Parameters:
        dim (`int`): The number of channels in the input and output.
        num_attention_heads (`int`): The number of heads to use for multi-head attention.
        attention_head_dim (`int`): The number of channels in each head.
        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
        cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
        attention_bias (`bool`, *optional*, defaults to `False`):
            Configure if the attention layers should contain a bias parameter.
        upcast_attention (`bool`, *optional*, defaults to `False`):
            Set to `True` to upcast the attention computation to `float32`.
        temb_channels (`int`, *optional*, defaults to 768):
            The number of channels in the token embedding.
        add_self_attention (`bool`, *optional*, defaults to `False`):
            Set to `True` to add self-attention to the block.
        cross_attention_norm (`str`, *optional*, defaults to `None`):
            The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`.
        group_size (`int`, *optional*, defaults to 32):
            The number of groups to separate the channels into for group normalization.
    rI  r&   r3   r5   r(   r7  r-   r!   rH   r2   r8  c           
         t         |           |	| _        |	r6t        ||t	        d||z              | _        t        |||||d d       | _        t        ||t	        d||z              | _        t        ||||||||
      | _	        y )Nrq   )r   r   r   r5   rt   r(   r2   )r   r(   r   r   r5   rt   r-   r2   )
ru   rv   rH   r   maxnorm1r   attn1norm2attn2)r}   rI  r&   r3   r5   r(   r7  r-   r!   rH   r2   r8  r~   s               rO   rv   zKAttentionBlock.__init__  s     	"4 %mS#a
AR:STDJ")+#$(%)DJ "-c!SJ=N6OP
 3%'-!5	

rQ   r   heightweightr_   c                 l    |j                  dddd      j                  |j                  d   ||z  d      S )Nr   r   r	   rq   rG  permutereshapeshaper}   r   r  r  s       rO   _to_3dzKAttentionBlock._to_3d  s;    $$Q1a0889L9LQ9OQWZ`Q`bdeerQ   c                 f    |j                  ddd      j                  |j                  d   d||      S )Nr   r   rq   rG  r  r  s       rO   _to_4dzKAttentionBlock._to_4d  s5    $$Q1-55m6I6I!6LbRXZ`aarQ   r   r<  r   r   r   c                    ||ni }|j                  dd       t        j                  d       | j                  re| j	                  ||      }|j
                  dd  \  }}	| j                  |||	      } | j                  |fd |d|}
| j                  |
||	      }
|
|z   }| j                  ||      }|j
                  dd  \  }}	| j                  |||	      } | j                  |f|||n|d|}
| j                  |
||	      }
|
|z   }|S )Nr   r   r   r   )r   rK   rL   rH   r  r  r  r  r  r  r  )r}   r   r   r<  r   r   r   norm_hidden_statesr  r  attn_outputs              rO   r   zKAttentionBlock.forward  sQ    <R;]!7ce!%%gt4@NNpq ""!%M3!?/55ab9NFF!%-?!P$$**"&*- )	K ++k66BK'-7M "ZZs;+11!"5![[);VVL djj
"7-B-J>Pf
 %	
 kk+vv>#m3rQ   )r   NFFr  FNr   r   )r   r   r   r   r   r   r   r   r   rv   r   r   r  r  r   r   r   r   r   s   @rO   r9  r9    s   8 -1$!& #(.2)
)
 !)
  	)

 )
 &c])
 )
 )
 )
 !)
 'sm)
 )
VfELL f# fs fu|| fbELL b# bs bu|| b 9= '+15;?9=/||/  (5/ ell#/ !./ !)c3h 8/ !) 6/ 
/rQ   r9  )rq   NNNNFFFFr   r   Fr   NNNr   )r   rq   NNFFFFr   r   FNrq   r   )Nrq   NNNFFFFr   r   Fr   NNNr   )Mtypingr   r   r   r   r   numpyr  r   torch.nn.functionalr   
functionalr   utilsr
   r   utils.torch_utilsr   activationsr   attention_processorr   r   r   normalizationr   r   r   r   r   r   r   r   r   r    transformers.dual_transformer_2dr   transformers.transformer_2dr   
get_loggerr   rK   r   r   r   r   rP   rY   Modulerl   rn   rW   rU   rV   r>   r@   r:   rD   rE   rC   rB   r;   rA   rF   rG   re   rc   ra   rh   ri   rg   rf   rb   rd   rj   rk   r9  rJ   rQ   rO   <module>r     sp   5 4      ' , ( Z Z (	 	 	 F < 
		H	% )*)-#')-(,!&"'!&"#,#!&%(*.(,%)3N;N;N; N; 	N;
 N; N; N; N; #&N; "#N; C=N; "#N; !N; N;  N;  !N;" #N;$ !%N;& 'N;( )N;* #+N;, #3--N;. !/N;0 c]1N;2 3N;p "%())-)-!&"'+0"#,#!&*.())HGHGHG HG 	HG
 HG HG HG #&HG "#HG "#HG HG  HG %)HG HG !HG  !HG" #HG$ #3-%HG& !'HG( )HGj %)())-#')-!&"'!&"#,#!&%(*.(,#'5^9^9^9 ^9 	^9
 ^9 ^9 ^9 ^9 ^9 SM^9 #&^9 "#^9 C=^9 "#^9 ^9   !^9" #^9$ %^9& !'^9( )^9* +^9, #-^9. #3-/^90 !1^92 C=3^94 5^96 YY7^9B"6299 "6J_RYY _DKbii K\zBII zz~,bii ~,BP,299 P,fK,")) K,\L L^dRYY dNm9")) m9`W9bii W9tW,bii W,tT, T,nC,299 C,Ls,BII s,l~BII ~B] ]@b		 bJFryy FR`299 `FC*		 C*Lk*BII k*\aBII aHZryy ZzM M`C")) CNxbii xrQ   