bi~PddlmZmZmZmZmZddlZddlmZddl Zddl m Z m Z ddl mZddlmZddlmZdd lmZmZdd lmZmZmZmZmZmZdd lmZmZdd l m!Z!dd l"m#Z#ddl$m%Z%m&Z&m'Z'ddl(m)Z)ejTe+Z,GddejZZ.Gdde!e eZ/y))AnyDictOptionalTupleUnionN) ConfigMixinregister_to_config)UNet2DConditionLoadersMixin)logging)get_activation) Attention FeedForward)ADDED_KV_ATTENTION_PROCESSORSCROSS_ATTENTION_PROCESSORSAttentionProcessorAttnAddedKVProcessor AttnProcessorFusedAttnProcessor2_0)TimestepEmbedding Timesteps) ModelMixin)TransformerTemporalModel)UNetMidBlock3DCrossAttnget_down_block get_up_block)UNet3DConditionOutputceZdZ d dedededededeedeffd Zd ejd ejfd Z xZ S) "I2VGenXLTransformerTemporalEncoderdimnum_attention_headsattention_head_dim activation_fnupcast_attention ff_inner_dimdropoutc t|tj|dd|_t ||||d|d|_t|||d|d|_y)NTh㈵>)elementwise_affineepsF) query_dimheadsdim_headr(biasr&out_bias)r(r% final_dropout inner_dimr0) super__init__nn LayerNormnorm1rattn1rff) selfr"r#r$r%r&r'r( __class__s `/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/models/unets/unet_i2vgen_xl.pyr5z+I2VGenXLTransformerTemporalEncoder.__init__1se \\#$DI %'-   '"   hidden_statesreturnc|j|}|j|d}||z}|jdk(r|jd}|j |}||z}|jdk(r|jd}|S)N)encoder_hidden_statesr)r8r9ndimsqueezer:)r;r?norm_hidden_states attn_output ff_outputs r=forwardz*I2VGenXLTransformerTemporalEncoder.forwardOs"ZZ 6jj!34jP #m3    ")11!4MGGM* !M1    ")11!4Mr>)gegluFNg) __name__ __module__ __qualname__intstrboolrr5torchTensorrI __classcell__r<s@r=r!r!0s} %!&&*  !       sm  <|| r>r!c`eZdZdZdZe d(deedededee dfd ee dfd eedfd ed eed ede eeefdee eeefffd Z e de e effdZde ee e efffdZd)deededdfdZdZdZdZdZdZdZ d*dej2de ej2eefd ej2d!ej2d"eej2d#eej2d$eej2d%ee e efd&ede eeej2ffd'ZxZS)+ I2VGenXLUNeta  I2VGenXL UNet. It is a conditional 3D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample-shaped output. This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented for all models (such as downloading or saving). Parameters: sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): Height and width of input/output sample. in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): The tuple of downsample blocks to use. up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): The tuple of upsample blocks to use. block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): The tuple of output channels for each block. layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. If `None`, normalization and activation layers is skipped in post-processing. cross_attention_dim (`int`, *optional*, defaults to 1280): The dimension of the cross attention features. attention_head_dim (`int`, *optional*, defaults to 64): Attention head dim. num_attention_heads (`int`, *optional*): The number of attention heads. FN sample_size in_channels out_channelsdown_block_types.up_block_typesblock_out_channelslayers_per_blocknorm_num_groupscross_attention_dimr$r#c & t|| } t|t|k7rtd|d|dt|t|k7rtd|d|dt | t s)t| t|k7rtd| d|dt j||z|ddd |_td | |dd | |_ t jt jd |d zdd t jt j|d z|d zdd d t jt j|d z|dd d |_ t|d|d z|d|_t jt jd |d zdd t jt j dt j|d z|dzddd t jt j|dz| ddd |_|dd z} t%|ddd|_|d} t)| | d|_t jt j,| | t jt j,| | |z|_t jt j,| | t jt j,| | |_t j2g|_t j2g|_t | t r| ft|z} |d}t9|D]T\}}|}||}|t|d z k(}t;||||| | dd|| | |d d }|j4j=|Vt?|d| ddd | | d|d |_ d|_!tEtG|}tEtG| }|d}t9|D]\}}|t|d z k(}|}||}|tI|d zt|d z }|sd}|xjBd z c_!nd}tK||d z|||| |dd|| ||d|}|j6j=||}t jL|d|d|_'tQd|_)t j|d|dd |_*y)Nz\Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: z. `up_block_types`: .zbMust provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: z. `down_block_types`: zdMust provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: rrr) kernel_sizepadding)r#r$rX num_layersr^rC)rc)stridercr gelu)r"r#r'r$r%) rhTsilu)act_fnr*F) rerXrY temb_channelsadd_downsample resnet_eps resnet_act_fn resnet_groupsr_r#downsample_paddingdual_cross_attention) rXrlrnrooutput_scale_factorr_r#rprr) rerXrYprev_output_channelrl add_upsamplernrorpr_r#rrresolution_idx) num_channels num_groupsr,)+r4r5len ValueError isinstancerNr6Conv2dconv_inrtransformer_in SequentialSiLUimage_latents_proj_inr!image_latents_temporal_encoderAdaptiveAvgPool2dimage_latents_context_embeddingr time_projrtime_embeddingLinearcontext_embedding fps_embedding ModuleList down_blocks up_blocks enumeraterappendr mid_blocknum_upsamplerslistreversedminr GroupNorm conv_norm_outrconv_actconv_out)r;rWrXrYrZr[r\r]r^r_r$r#time_embed_dimtimestep_input_dimoutput_channelidown_block_type input_channelis_final_block down_blockreversed_block_out_channelsreversed_num_attention_heads up_block_typerurvup_blockr<s r=r5zI2VGenXLUNet.__init__~s2 1  C$7 7nooAAUVdUeefg  ! "c*:&; ;tvHuII_`p_qqrs -s3 : GGI IIn&9K&G H"   ]] II(. 92779biiP^`nFo  ==,r* )3 /#6"83?O;P"P ,A."+,<"= 0 A*M/2N#&8"9A"==N'+)+,#11 $-$7$7$:#$%*J    # #J /) 0.1*2.( ! 3 3B 7)!&   '+84F+G&H#'+H5H,I'J$4Q7 ). 9 1 A}#&8"9A"==N"0 8;N7AE3GYCZ]^C^8_`M"# ##q(#$ #+a/)+$7,) $-$7$@$C%* H NN ! !( +"0 ? 1D \\7I!7LYhnst&v.  "4Q"7ST^_` r>r@ci}dtdtjjdttt fffd |j D]\}}||||S)z Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with indexed by its weight name. namemodule processorsct|dr|j||d<|jD]\}}|d||||S)N get_processor .processorra)hasattrrnamed_children)rrrsub_namechildfn_recursive_add_processorss r=rzAI2VGenXLUNet.attn_processors..fn_recursive_add_processorsHsdv/282F2F2H dV:./#)#8#8#: U%+tfAhZ,@%T U r>)rOrQr6Modulerrr)r;rrrrs @r=attn_processorszI2VGenXLUNet.attn_processors=sm  c 588?? X\]`bt]tXu !//1 BLD& 'fj A Br> processorc Tt|jj}t|tr,t||k7rt dt|d|d|ddt dtjjffd |jD]\}}|||y) a4 Sets the attention processor to use to compute attention. Parameters: processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): The instantiated processor class or a dictionary of processor classes that will be set as the processor for **all** `Attention` layers. If `processor` is a dict, the key needs to define the path to the corresponding cross attention processor. This is strongly recommended when setting trainable attention processors. z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.rrct|drEt|ts|j|n#|j|j |d|j D]\}}|d|||y)N set_processorrra)rr|dictrpopr)rrrrrfn_recursive_attn_processors r=rzDI2VGenXLUNet.set_attn_processor..fn_recursive_attn_processorlsxv/!)T2((3(($z7J)KL#)#8#8#: T%+tfAhZ,@%S Tr>N) rzrkeysr|rr{rOrQr6rr)r;rcountrrrs @r=set_attn_processorzI2VGenXLUNet.set_attn_processorWsD((--/0 i &3y>U+BPQTU^Q_P`a005w6QRWQXXkm   Tc T588?? T!//1 ALD& 'fi @ Ar> chunk_sizer"c|dvrtd||xsd}dtjjdtdtffd |j D] }|||y) aX Sets the attention processor to use [feed forward chunking](https://huggingface.co/blog/reformer#2-chunked-feed-forward-layers). Parameters: chunk_size (`int`, *optional*): The chunk size of the feed-forward layers. If not specified, will run feed-forward layer individually over each tensor of dim=`dim`. dim (`int`, *optional*, defaults to `0`): The dimension over which the feed-forward computation should be chunked. Choose between dim=0 (batch) or dim=1 (sequence length). )rrz-Make sure to set `dim` to either 0 or 1, not rrrr"ct|dr|j|||jD] }|||yNset_chunk_feed_forward)rr"rrchildrenrrr"rfn_recursive_feed_forwards r=rzGI2VGenXLUNet.enable_forward_chunking..fn_recursive_feed_forwardEv78---M* B)%SA Br>N)r{rQr6rrNr)r;rr"rrs @r=enable_forward_chunkingz$I2VGenXLUNet.enable_forward_chunkingzsu f LSERS S _1  Behhoo B3 BUX Bmmo ?F %fj# > ?r>cdtjjdtdtffd |j D] }|ddy)Nrrr"ct|dr|j|||jD] }|||yrrrs r=rzHI2VGenXLUNet.disable_forward_chunking..fn_recursive_feed_forwardrr>r)rQr6rrNr)r;rrs @r=disable_forward_chunkingz%I2VGenXLUNet.disable_forward_chunkingsM Behhoo B3 BUX Bmmo 7F %fdA 6 7r>c jtd|jjDr t}nmtd|jjDr t }n8t dt t|jj|j|y)ze Disables custom attention processors and sets the default attention implementation. c3@K|]}|jtvywN)r<r.0procs r= z:I2VGenXLUNet.set_default_attn_processor..si4t~~!>>ic3@K|]}|jtvywr)r<rrs r=rz:I2VGenXLUNet.set_default_attn_processor..sh$#==hrzOCannot call `set_default_attn_processor` when attention processors are of type N) allrvaluesrrr{nextiterr)r;rs r=set_default_attn_processorz'I2VGenXLUNet.set_default_attn_processors i4K_K_KfKfKhi i,.I h$J^J^JeJeJgh h%IabfgklpmAmAmHmHmJhKcLbMN   *r>ct|jD]9\}}t|d|t|d|t|d|t|d|;y)aFEnables the FreeU mechanism from https://huggingface.co/papers/2309.11497. The suffixes after the scaling factors represent the stage blocks where they are being applied. Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. Args: s1 (`float`): Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to mitigate the "oversmoothing effect" in the enhanced denoising process. s2 (`float`): Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to mitigate the "oversmoothing effect" in the enhanced denoising process. b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. s1s2b1b2N)rrsetattr)r;rrrrrupsample_blocks r= enable_freeuzI2VGenXLUNet.enable_freeusQ$"+4>>!: . A~ ND" - ND" - ND" - ND" -  .r>chd}t|jD]3\}}|D])}t||st||dt ||d+5y)zDisables the FreeU mechanism.>rrrrN)rrrgetattrr)r; freeu_keysrrks r= disable_freeuzI2VGenXLUNet.disable_freeusW- !*4>>!: 5 A~ 5>1-D1Q1]NAt4 5 5r>crd|_|jjD]1\}}dt|jj vs(t d|j|_|jD]%}t|ts|jd'|jty)u1 Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value) are fused. For cross-attention modules, key and value projection matrices are fused. This API is 🧪 experimental. NAddedzQ`fuse_qkv_projections()` is not supported for models having added KV projections.T)fuse) original_attn_processorsritemsrOr<rKr{modulesr|rfuse_projectionsrr)r;_attn_processorrs r=fuse_qkv_projectionsz!I2VGenXLUNet.fuse_qkv_projectionss)-%!%!5!5!;!;!= v A~#n66??@@ !tuu v)-(<(<%lln 3F&),''T'2 3  5 78r>cT|j|j|jyy)uDisables the fused QKV projection if enabled. This API is 🧪 experimental. N)rr)r;s r=unfuse_qkv_projectionsz#I2VGenXLUNet.unfuse_qkv_projectionss)  ( ( 4  # #D$A$A B 5r>sampletimestepfps image_latentsimage_embeddingsrB timestep_condcross_attention_kwargs return_dictc &|j\} } } } }d|jz&d}d}t&fd|jddDrtj dd}|}t j |s|jjdk(}|jjd k(}t|tr%|s|rt jnt j}n$|s|rt jnt j}t j|g||j }n6t!|jd k(r|dj#|j}|j%|jd }|j'|}|j#|j( }|j+||}|j%|jd }|j-|j'|j#|j( }||z}|j/| d |jd | z }|j1| d |j2j4}t j6||gd}|ddddddddf}|j9d ddddj;|jd |jdz|jd|jd|jd}|j=|}|j\}}}}|j9d dddj;|||z|}t j6||gd}|j?|}|jAd|j2jB|j2j4}t j6||gd}|j/| d |jd | z }|j9d ddddj;|jd |jdz|jd|jd|jd}|jE|}|dddfj;| | | | |j9d ddddj;| | z|z| | }|jG|}|j;| | || | j9d dddd}t j6||gd}|j9d ddddj;|jd | zdf|jddz}|jI|}|jK|| |dd }|f} |jLD]>}!tO|!dr|!jPr|!|||| |\}}"n|!||| \}}"| |"z } @|jR|jS|||| |}tU|jVD]\}#}$|#t!|jVdz k(}%| t!|$jX d}"| dt!|$jX } |%s|r| djdd}tO|$dr|$jPr|$|||"||| |}|$|||"|| }|j[|}|j]|}|j_|}|dddfj;d| f|jddzj9d dddd}| s|fSta|S)a The [`I2VGenXLUNet`] forward method. Args: sample (`torch.Tensor`): The noisy input tensor with the following shape `(batch, num_frames, channel, height, width`. timestep (`torch.Tensor` or `float` or `int`): The number of timesteps to denoise an input. fps (`torch.Tensor`): Frames per second for the video being generated. Used as a "micro-condition". image_latents (`torch.Tensor`): Image encodings from the VAE. image_embeddings (`torch.Tensor`): Projection embeddings of the conditioning image computed with a vision encoder. encoder_hidden_states (`torch.Tensor`): The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~models.unets.unet_3d_condition.UNet3DConditionOutput`] instead of a plain tuple. Returns: [`~models.unets.unet_3d_condition.UNet3DConditionOutput`] or `tuple`: If `return_dict` is True, an [`~models.unets.unet_3d_condition.UNet3DConditionOutput`] is returned, otherwise a `tuple` is returned where the first element is the sample tensor. r FNc3.K|] }|zdk7yw)rN)rsdefault_overall_up_factors r=rz'I2VGenXLUNet.forward../sMaq,,1Msz9Forward upsample size to force interpolation output size.Tmpsnpu)dtypedevicer)r)r" output_sizer)r"rrCrs) num_framesrrhas_cross_attention)r?tembrBrr)r?rr)rBrr)r?rres_hidden_states_tuplerB upsample_sizerr)r?rrr r)r)1shaperanyloggerinforQ is_tensorrtyper|floatfloat32float64int32int64tensorrztoexpandrrrrrepeat_interleave new_zerosconfigr_catpermutereshaperrviewrXrrr~rrrrrrrresnetsrrrr)'r;rrrrrrBrrr batch_sizechannelsrheightwidthforward_upsample_sizer  timestepsis_mpsis_npurt_embfps_embemb context_embimage_latents_for_context_embdsimage_latents_context_embs _batch_size _channels_height_width image_embdown_block_res_samplesdownsample_block res_samplesrrrrs' @r=rIzI2VGenXLUNet.forwards*L;A,,7 Hj&% %&t':':$:!!& M6<<;LM M KKS T$( ! y)]]''50F]]''50F)U+*0F (.& u{{ i[fmmTI  !Q &!$**6==9I$$V\\!_5 y) tzz*##E=9jj1&$$T^^C%8%;%;$**%;%MNgo##JA399Q'> @_@_` iii 8a@ !33JAS^SdSdefSgjtSt3u %--aAq!<DD    "]%8%8%; ;    "    "    "  22=A $' " WZXvu E WQ1a # WZ&(50*h G  ;;MJ %--j&%U]^ffghjkmnpqstu FM2:1aA.66 Q*8TVX7Y\b\h\hijik\l7lmf%$$ !#9 %  #) $ 0 0 2 ')>?DTDhDh&6"(*5)+A '# '7VRUbl&m# "k 1 " 2 >> %^^&1%'= $F"+4>>!:  A~#dnn"5"99N0#n6L6L2M1M1OPK%;CYCY?Z>Z%[ ""&; 6r : @ @ D ~'<=.BdBd'"(,7*5"/)+A("(,7"/) - >##F+v&v&a(("j)9FLL) NrCrC)CrossAttnDownBlock3Dr6r6 DownBlock3D) UpBlock3DCrossAttnUpBlock3Dr9r9)i@ir:r rhi@N)Nr)NNNNT) rKrLrM__doc__ _supports_gradient_checkpointingr rrNrrOrr5propertyrrrrrrrrrrrrQrRrrrPrrIrSrTs@r=rVrVasv4(-$&*- + /E !)+#'57@D-|ac]|a|a |a  S/ |ac3h|a""#s(O#|a$%|a&"#'|a(!)|a*"#uSz/2+|a,&eCsO&<=-|a|a|c+=&=!>0 AE2Dd3PbKbFc2c,d AF?(3-?S?Y]?< 7 + .2594 C&488<04;? S4 S4 eS01S4\\ S4 || S4 #5<<0 S4 ( 5S4  -S4!)c3h 8S4S4 $eELL&99 :S4r>rV)0typingrrrrrrQtorch.nnr6torch.utils.checkpointconfiguration_utilsr r loadersr utilsr activationsr attentionrrattention_processorrrrrrr embeddingsrrmodeling_utilsr!transformers.transformer_temporalrunet_3d_blocksrrrunet_3d_conditionr get_loggerrKr rr!rVrr>r=rNs54 B2(.6'H 5   H %..bo 4:{,Go 4r>