
    bi              4          d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZ  e
j@                  e!      Z" G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z	 	 	 	 	 	 	 	 	 	 	 	 	 dPde#de$de$de$de$de%de&de#d e$d!ee$   d"ee$   d#ee$   d$e%d%e%d&e%d'e%d(e#d)e$d*e$d+ee$ee$   f   d,ee$ee$   f   d-e&d.ed/   f.d0Z'	 	 	 	 	 	 	 	 	 	 	 	 	 	 dQd1e#de$de$de$d2e$de$d3e%de&de#d e$d4ee$   d!ee$   d"ee$   d$e%d%e%d&e%d'e%d(e#d)e$d5ee$   d*e$d+ee$ee$   f   d,ee$ee$   f   d-e&d.ed6   f2d7Z( G d8 d9ejR                        Z* G d: d;ejR                        Z+ G d< d=ejR                        Z, G d> d?ejR                        Z- G d@ dAejR                        Z. G dB dCejR                        Z/ G dD dEejR                        Z0 G dF dGejR                        Z1 G dH dIejR                        Z2 G dJ dKejR                        Z3 G dL dMejR                        Z4 G dN dOejR                        Z5y)R    )AnyDictOptionalTupleUnionN)nn   )	deprecatelogging)apply_freeu   )	Attention)Downsample2DResnetBlock2DSpatioTemporalResBlockTemporalConvLayer
Upsample2D)Transformer2DModel)TransformerSpatioTemporalModelTransformerTemporalModel   )CrossAttnDownBlockMotionCrossAttnUpBlockMotionDownBlockMotionUNetMidBlockCrossAttnMotionUpBlockMotionc                        e Zd Z fdZ xZS )r   c                 B    d}t        dd|       t        |   |i | y )NzImporting `DownBlockMotion` from `diffusers.models.unets.unet_3d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_motion_model import DownBlockMotion` instead.r   1.0.0r
   super__init__selfargskwargsdeprecation_message	__class__s       `/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/models/unets/unet_3d_blocks.pyr"   zDownBlockMotion.__init__0   s,     @#W.AB$)&)    __name__
__module____qualname__r"   __classcell__r(   s   @r)   r   r   /       * *r*   r   c                        e Zd Z fdZ xZS )r   c                 B    d}t        dd|       t        |   |i | y )NzImporting `CrossAttnDownBlockMotion` from `diffusers.models.unets.unet_3d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_motion_model import CrossAttnDownBlockMotion` instead.r   r   r    r#   s       r)   r"   z!CrossAttnDownBlockMotion.__init__7   s,     R,g7JK$)&)r*   r+   r0   s   @r)   r   r   6   r1   r*   r   c                        e Zd Z fdZ xZS )r   c                 B    d}t        dd|       t        |   |i | y )NzImporting `UpBlockMotion` from `diffusers.models.unets.unet_3d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_motion_model import UpBlockMotion` instead.r   r   r    r#   s       r)   r"   zUpBlockMotion.__init__>   s+     |/7,?@$)&)r*   r+   r0   s   @r)   r   r   =   r1   r*   r   c                        e Zd Z fdZ xZS )r   c                 B    d}t        dd|       t        |   |i | y )NzImporting `CrossAttnUpBlockMotion` from `diffusers.models.unets.unet_3d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_motion_model import CrossAttnUpBlockMotion` instead.r   r   r    r#   s       r)   r"   zCrossAttnUpBlockMotion.__init__E   s,     N*G5HI$)&)r*   r+   r0   s   @r)   r   r   D   r1   r*   r   c                        e Zd Z fdZ xZS )r   c                 B    d}t        dd|       t        |   |i | y )NzImporting `UNetMidBlockCrossAttnMotion` from `diffusers.models.unets.unet_3d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_motion_model import UNetMidBlockCrossAttnMotion` instead.r   r   r    r#   s       r)   r"   z$UNetMidBlockCrossAttnMotion.__init__L   s,     X/:MN$)&)r*   r+   r0   s   @r)   r   r   K   r1   r*   r   down_block_type
num_layersin_channelsout_channelstemb_channelsadd_downsample
resnet_epsresnet_act_fnnum_attention_headsresnet_groupscross_attention_dimdownsample_paddingdual_cross_attentionuse_linear_projectiononly_cross_attentionupcast_attentionresnet_time_scale_shifttemporal_num_attention_headstemporal_max_seq_lengthtransformer_layers_per_block%temporal_transformer_layers_per_blockdropoutreturn)DownBlock3DCrossAttnDownBlock3DDownBlockSpatioTemporal CrossAttnDownBlockSpatioTemporalc                 d   | dk(  rt        ||||||||	|||      S | dk(  rI|
t        d      t        di d|d|d|d|d	|d
|d|d|	d|d|
d|d|d|d|d|d|d|S | dk(  rt        |||||      S | dk(  r |
t        d      t	        |||||||
|      S t        |  d      )NrQ   )r;   r<   r=   r>   r?   r@   rA   rC   rE   rJ   rO   rR   z>cross_attention_dim must be specified for CrossAttnDownBlock3Dr;   r<   r=   r>   r?   r@   rA   rC   rE   rD   rB   rF   rG   rH   rI   rJ   rO   rS   )r;   r<   r=   r>   r?   rT   zJcross_attention_dim must be specified for CrossAttnDownBlockSpatioTemporal)r<   r=   r>   r;   rM   r?   rD   rB    does not exist. )rQ   
ValueErrorrR   rS   rT   )r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   s                         r)   get_down_blockrY   R   s   : -'!#%')!''1$;
 	
 
2	2&]^^# 
!
#
 &
 (	

 *
 "
 (
 (
  2
 !4
 !4
 "6
 #8
 "6
 .
  %<!
" #
 	
& 
5	5&!#%')
 	
 
>	>&ijj/#%'!)E) 3 3	
 		
 ((89
::r*   up_block_typeprev_output_channeladd_upsampleresolution_idxtemporal_cross_attention_dim)	UpBlock3DCrossAttnUpBlock3DUpBlockSpatioTemporalCrossAttnUpBlockSpatioTemporalc                 t   | dk(  rt        |||||||||||
|      S | dk(  rL|t        d      t        di d|d|d|d|d	|d
|d|d|d|d|d|	d|d|d|d|d|d|
d|S | dk(  rt        ||||||
|      S | dk(  r"|t        d      t	        |||||||||	|

      S t        |  d      )Nr_   )r;   r<   r=   r[   r>   r\   r@   rA   rC   rJ   r]   rO   r`   z<cross_attention_dim must be specified for CrossAttnUpBlock3Dr;   r<   r=   r[   r>   r\   r@   rA   rC   rD   rB   rF   rG   rH   rI   rJ   r]   rO   ra   )r;   r<   r=   r[   r>   r]   r\   rb   zHcross_attention_dim must be specified for CrossAttnUpBlockSpatioTemporal)
r<   r=   r[   r>   r;   rM   r\   rD   rB   r]   rV   rW   )r_   rX   r`   ra   rb   )rZ   r;   r<   r=   r[   r>   r\   r@   rA   rB   r]   rC   rD   rF   rG   rH   rI   rJ   rK   r^   rL   rM   rN   rO   s                           r)   get_up_blockrd      s   > #!#% 3'%!''$;)
 	
 
.	.&[\\! 
!
#
 &
 !4	

 (
 &
 "
 (
 (
 !4
 !4
 "6
 #8
 "6
 .
  %<!
" *#
$ %
 	
( 
1	1$!#% 3')%
 	
 
:	:&ghh-#% 3'!)E% 3 3)
 	
 &67
88r*   c                   .    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededededef fdZ	 	 	 	 	 ddej                  de
ej                     de
ej                     de
ej                     dede
eeef      dej                  fdZ xZS )UNetMidBlock3DCrossAttnr<   r>   rO   r;   r@   rJ   rA   rC   resnet_pre_normrB   output_scale_factorrD   rF   rG   rI   c                    t         |           d| _        |
| _        ||nt	        |dz  d      }t        ||||||||||	
      g}t        ||d|      g}g }g }t        |      D ]  }|j                  t        ||
z  |
|d||||             |j                  t        ||
z  |
|d||	             |j                  t        ||||||||||	
             |j                  t        ||d|              t        j                  |      | _        t        j                  |      | _        t        j                  |      | _        t        j                  |      | _        y )
NT       
r<   r=   r>   epsgroupsrO   time_embedding_normnon_linearityrh   pre_norm皙?rO   norm_num_groupsr   )r<   r;   rD   rt   rG   rI   r<   r;   rD   rt   )r!   r"   has_cross_attentionrB   minr   r   rangeappendr   r   r   
ModuleListresnets
temp_convs
attentionstemp_attentions)r$   r<   r>   rO   r;   r@   rJ   rA   rC   rg   rB   rh   rD   rF   rG   rI   r{   r|   r}   r~   _r(   s                        r)   r"   z UNetMidBlock3DCrossAttn.__init__  s   $ 	#' #6 )6)BK[\L\^`Ha '(+$$;+$7(
  -	

 
z" ,	A"#66' + (;$1*?%5	 ""(#66' + (;$1	 NN +!,"/"(#(?"/(;, !$1	K,	\ }}W---
3--
3!}}_=r*   hidden_statestembencoder_hidden_statesattention_mask
num_framescross_attention_kwargsrP   c                 N    | j                   d   ||      } | j                  d   ||      }t        | j                  | j                  | j                   dd  | j                  dd        D ]8  \  }}}	}
 ||||d      d   } ||||d      d   } |	||      } |
||      }: |S )Nr   r   r   Fr   r   return_dictr   r   r   )r{   r|   zipr}   r~   )r$   r   r   r   r   r   r   attn	temp_attnresnet	temp_convs              r)   forwardzUNetMidBlock3DCrossAttn.forwardx  s     (Qt<**=ZP25OOT114<<3CT__UVUWEX3
 	L.D)VY !&;'=!	
 M &%'=!	
 M #=$7M%m
KM!	L$ r*   )        r   ư>defaultswishrk   Tr         ?   FTFNNNr   N)r,   r-   r.   intfloatstrboolr"   torchTensorr   r   r   r   r/   r0   s   @r)   rf   rf     s`   
  '0$ $#$%(#'%*&*!&!c>c> c> 	c>
 c> c> "%c> c> c> c> !c> #c> !c> #c>  $c>  !c>P (,8<15;?|| u||$  (5	
 !.  !)c3h 8 
r*   rf   c            '       f    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededededededededef& fdZ	 	 	 	 	 ddej                  de
ej                     de
ej                     de
ej                     dedeeef   deej                  eej                  df   f   fdZ xZS ) rR   r<   r=   r>   rO   r;   r@   rJ   rA   rC   rg   rB   rD   rh   rE   r?   rF   rG   rH   rI   c                    t         |           g }g }g }g }d| _        || _        t	        |      D ]  }|dk(  r|n|}|j                  t        |||||	|||||

             |j                  t        ||d|	             |j                  t        ||z  ||d||	|||	             |j                  t        ||z  ||d||	              t        j                  |      | _        t        j                  |      | _        t        j                  |      | _        t        j                  |      | _        |r1t        j                  t!        |d||d	
      g      | _        d| _        y d | _        d| _        y )NTr   rl   rr   rs   r   r<   r;   rD   rt   rG   rH   rI   ru   opuse_convr=   paddingnameF)r!   r"   rv   rB   rx   ry   r   r   r   r   r   rz   r{   r|   r}   r~   r   downsamplersgradient_checkpointing)r$   r<   r=   r>   rO   r;   r@   rJ   rA   rC   rg   rB   rD   rh   rE   r?   rF   rG   rH   rI   r{   r}   r~   r|   ir(   s                            r)   r"   zCrossAttnDownBlock3D.__init__  s   , 	

#' #6 z" .	A)*a+\KNN +!-"/"(#(?"/(;, !  $1	 " $77' , (;$1*?)=%5
 ""( $77' , (;$1	K.	^ }}W---
3--
3!}}_= " $!%%1 2!
!D ',# !%D&+#r*   r   r   r   r   r   r   rP   .c                 R   d}t        | j                  | j                  | j                  | j                        D ]>  \  }}	}
} |||      } |	||      } |
|||d      d   } ||||d      d   }||fz  }@ | j
                  | j
                  D ]
  } ||      } ||fz  }||fS )NrW   r   Fr   r   r   )r   r{   r|   r}   r~   r   )r$   r   r   r   r   r   r   output_statesr   r   r   r   downsamplers                r)   r   zCrossAttnDownBlock3D.forward  s     25LL$//4??D<P<P3
 	..FItY #=$7M%m
KM &;'=!	
 M &%'=!	
 M m--M%	.( (#00 ; +M :; m--Mm++r*   )r   r   r   r   r   rk   Tr   r   r   r   TFFFFr   )r,   r-   r.   r   r   r   r   r"   r   r   r   r   r   r   r   r   r/   r0   s   @r)   rR   rR     s     '0$ $#$#'%("##%*&+%*!&)b,b, b, 	b,
 b, b, b, "%b, b, b, b, !b, !b, #b,  b,  !b," ##b,$  $%b,& #'b,( )b,N (,8<1515&,||&, u||$&,  (5	&,
 !.&, &, !%S#X&, 
u||U5<<#455	6&,r*   rR   c                        e Zd Z	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
edededef fdZ	 	 ddej                  de
ej                     dedeej                  eej                  df   f   fdZ xZS )rQ   r<   r=   r>   rO   r;   r@   rJ   rA   rC   rg   rh   r?   rE   c                    t         |           g }g }t        |      D ]M  }|dk(  r|n|}|j                  t	        |||||	|||||

             |j                  t        ||d|	             O t        j                  |      | _        t        j                  |      | _	        |r1t        j                  t        |d||d      g      | _        d| _        y d | _        d| _        y )	Nr   rl   rr   rs   Tr   r   F)r!   r"   rx   ry   r   r   r   rz   r{   r|   r   r   r   )r$   r<   r=   r>   rO   r;   r@   rJ   rA   rC   rg   rh   r?   rE   r{   r|   r   r(   s                    r)   r"   zDownBlock3D.__init__'  s      	
z" 	A)*a+\KNN +!-"/"(#(?"/(;, !  $1	!	2 }}W---
3 " $!%%1 2!
!D ',# !%D&+#r*   r   r   r   rP   .c                     d}t        | j                  | j                        D ]  \  }} |||      } |||      }||fz  }  | j                  | j                  D ]
  } ||      } ||fz  }||fS )NrW   r   )r   r{   r|   r   )r$   r   r   r   r   r   r   r   s           r)   r   zDownBlock3D.forwardh  s     !$T\\4??!C 	.FI"=$7M%m
KMm--M		. (#00 ; +M :; m--Mm++r*   )
r   r   r   r   r   rk   Tr   Tr   )Nr   )r,   r-   r.   r   r   r   r   r"   r   r   r   r   r   r   r/   r0   s   @r)   rQ   rQ   &  s     '0$ $%(#"#?,?, ?, 	?,
 ?, ?, ?, "%?, ?, ?, ?, #?, ?,  ?,H (,	,||, u||$, 	,
 
u||U5<<#455	6,r*   rQ   c            *       v    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!dedededededededed	ed
ededededededededededee   f( fdZ	 	 	 	 	 	 d"de	j                  dee	j                  df   dee	j                     dee	j                     dee   dee	j                     dedeeef   de	j                  fd Z xZS )#r`   r<   r=   r[   r>   rO   r;   r@   rJ   rA   rC   rg   rB   rD   rh   r\   rF   rG   rH   rI   r]   c                    t         |           g }g }g }g }d| _        || _        t	        |      D ]  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t        ||z   ||||
|||	||
             |j                  t        ||d|
             |j                  t        ||z  ||d||
|||	             |j                  t        ||z  ||d||
              t        j                  |      | _        t        j                  |      | _        t        j                  |      | _        t        j                  |      | _        |r(t        j                  t!        |d|	      g      | _        nd | _        d
| _        || _        y )NTr   r   rl   rr   rs   r   ru   r   r=   F)r!   r"   rv   rB   rx   ry   r   r   r   r   r   rz   r{   r|   r}   r~   r   
upsamplersr   r]   )r$   r<   r=   r[   r>   rO   r;   r@   rJ   rA   rC   rg   rB   rD   rh   r\   rF   rG   rH   rI   r]   r{   r|   r}   r~   r   res_skip_channelsresnet_in_channelsr(   s                               r)   r"   zCrossAttnUpBlock3D.__init__  s   . 	

#' #6 z" 0	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;, !  $1	 " $77' , (;$1*?)=%5
 ""( $77' , (;$1	O0	b }}W---
3--
3!}}_= mmZtbn-o,pqDO"DO&+#,r*   r   res_hidden_states_tuple.r   r   upsample_sizer   r   r   rP   c	           
         t        | dd       xr+ t        | dd       xr t        | dd       xr t        | dd       }	t        | j                  | j                  | j                  | j
                        D ]  \  }
}}}|d   }|d d }|	rGt        | j                  ||| j                  | j                  | j                  | j                        \  }}t        j                  ||gd      } |
||      } |||	      } ||||d
      d   } ||||d
      d   } | j                  | j                  D ]  } |||      } |S )Ns1s2b1b2r   r   r   r   r   dimr   Fr   r   r   )getattrr   r{   r|   r}   r~   r   r]   r   r   r   r   r   catr   )r$   r   r   r   r   r   r   r   r   is_freeu_enabledr   r   r   r   res_hidden_states	upsamplers                   r)   r   zCrossAttnUpBlock3D.forward  s    D$% *dD)*dD)* dD)	 	 36LL$//4??D<P<P3
 "	.FItY !8 ;&=cr&B#  3>''!%wwwwwwww400 "II}6G&HaPM"=$7M%m
KM &;'=!	
 M &%'=!	
 M;"	H ??&!__ H	 )- GH r*   )r   r   r   r   r   rk   Tr   r   r   TFFFFN)NNNNr   N)r,   r-   r.   r   r   r   r   r   r"   r   r   r   r   r   r   r/   r0   s   @r)   r`   r`     s     '0$ $#$#'%(!%*&+%*!&(,+\-\- \- !	\-
 \- \- \- \- "%\- \- \- \- !\- !\- #\-  !\-" ##\-$  $%\-& #'\-( )\-* !+\-D (,8<'+1515;||; "'u||S'8!9; u||$	;
  (5;  }; !.; ; !%S#X; 
;r*   r`   c                       e Zd Z	 	 	 	 	 	 	 	 	 	 ddedededededededed	ed
ededededee   f fdZ	 	 	 dde	j                  dee	j                  df   dee	j                     dee   dede	j                  fdZ xZS )r_   r<   r[   r=   r>   rO   r;   r@   rJ   rA   rC   rg   rh   r\   r]   c                    t         |           g }g }t        |      D ]\  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t	        ||z   ||||
|||	||
             |j                  t        ||d|
             ^ t        j                  |      | _        t        j                  |      | _	        |r(t        j                  t        |d|      g      | _        nd | _        d| _        || _        y )	Nr   r   rl   rr   rs   Tr   F)r!   r"   rx   ry   r   r   r   rz   r{   r|   r   r   r   r]   )r$   r<   r[   r=   r>   rO   r;   r@   rJ   rA   rC   rg   rh   r\   r]   r{   r|   r   r   r   r(   s                       r)   r"   zUpBlock3D.__init__  s   " 	
z" 	A01Z!^0C,89Q!4LNN 25F F!-"/"(#(?"/(;, !  $1	%	6 }}W---
3 mmZtbn-o,pqDO"DO&+#,r*   r   r   .r   r   r   rP   c           
         t        | dd       xr+ t        | dd       xr t        | dd       xr t        | dd       }t        | j                  | j                        D ]  \  }}|d   }	|d d }|rGt	        | j
                  ||	| j                  | j                  | j                  | j                        \  }}	t        j                  ||	gd      } |||      } |||	      } | j                  | j                  D ]  }
 |
||      } |S )
Nr   r   r   r   r   r   r   r   r   )r   r   r{   r|   r   r]   r   r   r   r   r   r   r   )r$   r   r   r   r   r   r   r   r   r   r   s              r)   r   zUpBlock3D.forwardX  s.    D$% *dD)*dD)* dD)	 	 "%T\\4??!C 	LFI 7 ;&=cr&B#  3>''!%wwwwwwww400 "II}6G&HaPM"=$7M%m
KM)	L, ??&!__ H	 )- GH r*   )
r   r   r   r   r   rk   Tr   TN)NNr   )r,   r-   r.   r   r   r   r   r   r"   r   r   r   r   r/   r0   s   @r)   r_   r_     s$     '0$ $%(!(,9-9- !9- 	9-
 9- 9- 9- 9- "%9- 9- 9- 9- #9- 9- !9-~ (,'+(||( "'u||S'8!9( u||$	(
  }( ( 
(r*   r_   c                   t     e Zd Z	 	 	 d
dededededef
 fdZdej                  dej                  fd	Z xZ	S )MidBlockTemporalDecoderr<   r=   attention_head_dimr;   rI   c                 R   t         
|           g }g }t        |      D ]-  }|dk(  r|n|}	|j                  t	        |	|d ddddd             / |j                  t        |||z  |d|ddd	             t        j                  |      | _        t        j                  |      | _	        y )
Nr   r   h㈵>r   learnedTr<   r=   r>   rm   temporal_epsmerge_factormerge_strategyswitch_spatial_to_temporal_mixrk   )	query_dimheadsdim_headrm   rI   rt   biasresidual_connection)
r!   r"   rx   ry   r   r   r   rz   r}   r{   )r$   r<   r=   r   r;   rI   r{   r}   r   input_channelsr(   s             r)   r"   z MidBlockTemporalDecoder.__init__  s     	
z" 	A,-F[NNN& .!-"&!%!$#,37		 	%!%77+!1 "$(		
 --
3}}W-r*   r   image_only_indicatorc                      | j                   d   ||      }t        | j                   dd  | j                        D ]  \  }} ||      } |||      } |S )Nr   r   r   )r{   r   r}   )r$   r   r   r   r   s        r)   r   zMidBlockTemporalDecoder.forward  sk    
 (Q!5
  QR 0$//B 	LFD /M"%9M	 r*   )i   r   F
r,   r-   r.   r   r   r"   r   r   r   r/   r0   s   @r)   r   r     sc    
 #&!&).). ).  	).
 ). ).V|| $llr*   r   c            	            e Zd Z	 	 d
dedededef fdZdej                  dej                  dej                  fd	Z xZ	S )UpBlockTemporalDecoderr<   r=   r;   r\   c                 2   t         |           g }t        |      D ]-  }|dk(  r|n|}|j                  t	        ||d ddddd             / t        j                  |      | _        |r(t        j                  t        |d|      g      | _	        y d | _	        y )	Nr   r   r   r   r   Tr   r   )
r!   r"   rx   ry   r   r   rz   r{   r   r   )	r$   r<   r=   r;   r\   r{   r   r   r(   s	           r)   r"   zUpBlockTemporalDecoder.__init__  s     	z" 	A,-F[NNN& .!-"&!%!$#,37		 }}W- mmZtbn-o,pqDO"DOr*   r   r   rP   c                     | j                   D ]  } |||      } | j                  | j                  D ]
  } ||      } |S )Nr   )r{   r   )r$   r   r   r   r   s        r)   r   zUpBlockTemporalDecoder.forward  sX    
 ll 	F"%9M	 ??&!__ 9	 )- 89 r*   r   Tr   r0   s   @r)   r   r     sc    
 !## # 	#
 #>|| $ll 
	r*   r   c                        e Zd Z	 	 	 	 ddedededeeee   f   dedef fdZ	 	 	 ddej                  d	e	ej                     d
e	ej                     de	ej                     dej                  f
dZ
 xZS )UNetMidBlockSpatioTemporalr<   r>   r;   rM   rB   rD   c           
         t         
|           d| _        || _        t	        |t
              r|g|z  }t        |||d      g}g }t        |      D ]E  }	|j                  t        |||z  |||	   |             |j                  t        |||d             G t        j                  |      | _        t        j                  |      | _        d| _        y )NTr   r<   r=   r>   rm   r<   r;   rD   F)r!   r"   rv   rB   
isinstancer   r   rx   ry   r   r   rz   r}   r{   r   )r$   r<   r>   r;   rM   rB   rD   r{   r}   r   r(   s             r)   r"   z#UNetMidBlockSpatioTemporal.__init__  s     	#' #6  2C8,H+IJ+V( #'(+	
 
z" 	A.'#66 +;A>(; NN& +!,"/		( --
3}}W-&+#r*   r   r   r   r   rP   c                 H    | j                   d   |||      }t        | j                  | j                   dd        D ]c  \  }}t        j                         r0| j
                  r$ ||||d      d   }| j                  ||||      }J ||||d      d   } ||||      }e |S )Nr   r   r   Fr   r   r   )r{   r   r}   r   is_grad_enabledr   _gradient_checkpointing_func)r$   r   r   r   r   r   r   s          r)   r   z"UNetMidBlockSpatioTemporal.forward+  s     (Q!5
  ab1AB 	gLD&$$&4+F+F $!*?)= %	!
 ! !% A A&-Y]_s t $!*?)= %	!
 ! !'}dQe f!	g$ r*   )r   r   r   r   NNN)r,   r-   r.   r   r   r   r"   r   r   r   r   r/   r0   s   @r)   r   r     s    
 ?@#$#'4,4, 4, 	4,
 ',CsO&<4, !4, !4,r (,8<7;|| u||$  (5	
 'u||4 
r*   r   c                        e Zd Z	 	 ddededededef
 fdZ	 	 ddej                  deej                     d	eej                     d
e	ej                  e	ej                  df   f   fdZ
 xZS )rS   r<   r=   r>   r;   r?   c           	      H   t         |           g }t        |      D ])  }|dk(  r|n|}|j                  t	        |||d             + t        j                  |      | _        |r0t        j                  t        |d|d      g      | _	        d| _
        y d | _	        d| _
        y )Nr   r   r   Tr   )r   r=   r   F)r!   r"   rx   ry   r   r   rz   r{   r   r   r   )	r$   r<   r=   r>   r;   r?   r{   r   r(   s	           r)   r"   z DownBlockSpatioTemporal.__init__N  s     	z" 		A)*a+\KNN& +!-"/			 }}W- " $!%%1!		!D ',# !%D&+#r*   r   r   r   rP   .c                    d}| j                   D ]H  }t        j                         r!| j                  r| j	                  ||||      }n ||||      }||fz   }J | j
                  | j
                  D ]
  } ||      } ||fz   }||fS )NrW   r   )r{   r   r   r   r   r   )r$   r   r   r   r   r   r   s          r)   r   zDownBlockSpatioTemporal.forwardv  s     ll 	=F$$&4+F+F $ A A&-Y]_s t &}dQe f)],<<M	= (#00 ; +M :; *],<<Mm++r*   r   )NN)r,   r-   r.   r   r   r"   r   r   r   r   r   r/   r0   s   @r)   rS   rS   M  s     #&,&, &, 	&,
 &, &,V (,7;	,||, u||$, 'u||4	,
 
u||U5<<#455	6,r*   rS   c                       e Zd Z	 	 	 	 	 ddededededeeee   f   dededef fd	Z	 	 	 dd
ej                  de
ej                     de
ej                     de
ej                     deej                  eej                  df   f   f
dZ xZS )rT   r<   r=   r>   r;   rM   rB   rD   r?   c	           
         t         |           g }	g }
d| _        || _        t	        |t
              r|g|z  }t        |      D ]N  }|dk(  r|n|}|	j                  t        |||d             |
j                  t        |||z  |||   |             P t        j                  |
      | _        t        j                  |	      | _        |r1t        j                  t        |d|dd      g      | _        d	| _        y d | _        d	| _        y )
NTr   r   r   r   r   r   r   F)r!   r"   rv   rB   r   r   rx   ry   r   r   r   rz   r}   r{   r   r   r   )r$   r<   r=   r>   r;   rM   rB   rD   r?   r{   r}   r   r(   s               r)   r"   z)CrossAttnDownBlockSpatioTemporal.__init__  s$    	
#' #6 2C8,H+IJ+V(z" 	A)*a+\KNN& +!-"/	 .' $77 ,;A>(;	( --
3}}W- " $!%%1 !!
!D ',# !%D&+#r*   r   r   r   r   rP   .c                    d}t        t        | j                  | j                              }|D ]i  \  }}t	        j
                         r0| j                  r$| j                  ||||      } ||||d      d   }n ||||      } ||||d      d   }||fz   }k | j                  | j                  D ]
  }	 |	|      } ||fz   }||fS )NrW   Fr   r   r   )	listr   r{   r}   r   r   r   r   r   )
r$   r   r   r   r   r   blocksr   r   r   s
             r)   r   z(CrossAttnDownBlockSpatioTemporal.forward  s    c$,,89" 	=LFD$$&4+F+F $ A A&-Y]_s t $!*?)= %	!
 ! !'}dQe f $!*?)= %	!
 ! *],<<M'	=* (#00 ; +M :; *],<<Mm++r*   )r   r   r   r   Tr   )r,   r-   r.   r   r   r   r   r"   r   r   r   r   r/   r0   s   @r)   rT   rT     s     ?@#$#'#:,:, :, 	:,
 :, ',CsO&<:, !:, !:, :,~ (,8<7;%,||%, u||$%,  (5	%,
 'u||4%, 
u||U5<<#455	6%,r*   rT   c                        e Zd Z	 	 	 	 ddededededee   dededef fd	Z	 	 	 dd
ej                  de
ej                  df   deej                     deej                     dee   dej                  fdZ xZS )ra   r<   r[   r=   r>   r]   r;   r@   r\   c	           	      d   t         |           g }	t        |      D ]8  }
|
|dz
  k(  r|n|}|
dk(  r|n|}|	j                  t	        ||z   |||             : t        j                  |	      | _        |r(t        j                  t        |d|      g      | _	        nd | _	        d| _
        || _        y )Nr   r   r   Tr   F)r!   r"   rx   ry   r   r   rz   r{   r   r   r   r]   )r$   r<   r[   r=   r>   r]   r;   r@   r\   r{   r   r   r   r(   s                r)   r"   zUpBlockSpatioTemporal.__init__  s     	z" 	A01Z!^0C,89Q!4LNN& 25F F!-"/"			 }}W- mmZtbn-o,pqDO"DO&+#,r*   r   r   .r   r   r   rP   c                 :   | j                   D ]e  }|d   }|d d }t        j                  ||gd      }t        j                         r!| j                  r| j                  ||||      }[ ||||      }g | j                  | j                  D ]  } |||      } |S )Nr   r   r   r   )r{   r   r   r   r   r   r   )	r$   r   r   r   r   r   r   r   r   s	            r)   r   zUpBlockSpatioTemporal.forward  s     ll 
	gF 7 ;&=cr&B#!II}6G&HaPM$$&4+F+F $ A A&-Y]_s t &}dQe f
	g ??&!__ H	 )- GH r*   )Nr   r   Tr   )r,   r-   r.   r   r   r   r   r"   r   r   r   r   r/   r0   s   @r)   ra   ra     s     )- !#-#- !#- 	#-
 #- !#- #- #- #-R (,7;'+|| "'u||S'8!9 u||$	
 'u||4  } 
r*   ra   c                   :    e Zd Z	 	 	 	 	 	 	 ddededededee   dedeeee   f   ded	ed
edef fdZ		 	 	 	 dde
j                  dee
j                  df   dee
j                     dee
j                     dee
j                     dee   de
j                  fdZ xZS )rb   r<   r=   r[   r>   r]   r;   rM   r@   rB   rD   r\   c           
      .   t         |           g }g }d| _        |	| _        t	        |t
              r|g|z  }t        |      D ]]  }||dz
  k(  r|n|}|dk(  r|n|}|j                  t        ||z   |||             |j                  t        |	||	z  |||   |
             _ t        j                  |      | _        t        j                  |      | _        |r(t        j                  t        |d|      g      | _        nd | _        d| _        || _        y )NTr   r   r   r   r   F)r!   r"   rv   rB   r   r   rx   ry   r   r   r   rz   r}   r{   r   r   r   r]   )r$   r<   r=   r[   r>   r]   r;   rM   r@   rB   rD   r\   r{   r}   r   r   r   r(   s                    r)   r"   z'CrossAttnUpBlockSpatioTemporal.__init__5  s*    	
#' #6 2C8,H+IJ+V(z" 	A01Z!^0C,89Q!4LNN& 25F F!-"/"	 .' $77 ,;A>(;	, --
3}}W- mmZtbn-o,pqDO"DO&+#,r*   r   r   .r   r   r   r   rP   c                    t        | j                  | j                        D ]  \  }}|d   }	|d d }t        j                  ||	gd      }t        j
                         r0| j                  r$| j                  ||||      } ||||d      d   }m ||||      } ||||d      d   } | j                  | j                  D ]  }
 |
||      } |S )Nr   r   r   Fr   r   r   )	r   r{   r}   r   r   r   r   r   r   )r$   r   r   r   r   r   r   r   r   r   r   s              r)   r   z&CrossAttnUpBlockSpatioTemporal.forwardn  s     doo> 	LFD 7 ;&=cr&B#!II}6G&HaPM$$&4+F+F $ A A&-Y]_s t $!*?)= %	!
 ! !'}dQe f $!*?)= %	!
 !#	0 ??&!__ H	 )- GH r*   )Nr   r   r   r   r   T)NNNN)r,   r-   r.   r   r   r   r   r   r   r"   r   r   r   r/   r0   s   @r)   rb   rb   4  s)    )-?@ #$#'!7-7- 7- !	7-
 7- !7- 7- ',CsO&<7- 7- !7- !7- 7-z (,8<7;'+%||% "'u||S'8!9% u||$	%
  (5% 'u||4%  }% 
%r*   rb   )NNNFTFFr      rk   r   r   r   )NNNFTFFr   r   Nrk   r   r   r   )6typingr   r   r   r   r   r   r   utilsr
   r   utils.torch_utilsr   	attentionr   r   r   r   r   r   r   transformers.transformer_2dr   !transformers.transformer_temporalr   r   unet_motion_modelr   r   r   r   r   
get_loggerr,   loggerr   r   r   r   rY   rd   Modulerf   rR   rQ   r`   r_   r   r   r   rS   rT   ra   rb   rW   r*   r)   <module>r     s7   5 4   ' , !  =  
		H	%*o **7 **M **3 **"= *" $()-(,!&"&!&"#,()#%;<DE-Y;Y;Y; Y; 	Y;
 Y; Y; Y; Y; Y; C=Y; "#Y; !Y; Y;  Y; Y;  !Y;" !#Y;$ #&%Y;& !'Y;( #(U3Z"8)Y;* ,1eCj+A+Y;, -Y;. (/Y;N %)#')-!&"&!&"#,()26#%;<DE1a9a9a9 a9 	a9
 a9 a9 a9 a9 a9 a9 SMa9 C=a9 "#a9 a9  a9  !a9" #a9$ !%a9& #&'a9( #+3-)a9* !+a9, #(U3Z"8-a9. ,1eCj+A/a90 1a92 &3a9HCbii CLK,299 K,\V,")) V,rZ Zzd		 dN<bii <~/RYY /dV Vr>,bii >,Bb,ryy b,J>BII >B_RYY _r*   