biYHddlmZddlmZmZmZmZmZmZddl Z ddl m Z ddl m Z mZddlmZddlmZmZmZmZmZdd lmZdd lmZmZdd lmZmZmZdd l m!Z!dd l"m#Z#ddl$m%Z%m&Z&ejNe(Z)eGddeZ*Gdde#e eZ+Gdde#Z,y)) dataclass)AnyDictListOptionalTupleUnionN) ConfigMixinregister_to_config)PeftAdapterMixin)USE_PEFT_BACKEND BaseOutputloggingscale_lora_layersunscale_lora_layers)AttentionProcessor)ControlNetConditioningEmbedding zero_module)*CombinedTimestepGuidanceTextProjEmbeddings"CombinedTimestepTextProjEmbeddings FluxPosEmbed)Transformer2DModelOutput) ModelMixin)FluxSingleTransformerBlockFluxTransformerBlockcVeZdZUeej ed<eej ed<y)FluxControlNetOutputcontrolnet_block_samplescontrolnet_single_block_samplesN)__name__ __module__ __qualname__rtorchTensor__annotations__g/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/models/controlnets/controlnet_flux.pyrr#s #ELL11%*5<<%88r)rceZdZdZedddddddd d gd d d f d edededededededededeededeffd Ze dZ dZ e d+dedededefdZ d,dejdejdejd ed!ejd"ejd#ej"d$ejd%ejd&ejd'eeeefd(ed)eej.effd*ZxZS)-FluxControlNetModelT@&iiF)8r4N patch_size in_channels num_layersnum_single_layersattention_head_dimnum_attention_headsjoint_attention_dimpooled_projection_dimguidance_embedsaxes_dims_ropenum_modeconditioning_embedding_channelsc Ht|||_||z|_t d| |_| rt nt} | |j||_tj||j|_ tjj||j|_ tjt|Dcgc]}t!|j||c}|_tjt|Dcgc]}t%|j||c}|_tjg|_tt+|j"D]N}|j(j-t/tj|j|jPtjg|_tt+|j&D]N}|j0j-t/tj|j|jP| du|_|j2r%tj4| |j|_| It9| d|_tjj||j|_d|_yd|_t/tjj||j|_d|_ycc}wcc}w)Ni')thetaaxes_dim) embedding_dimr<)dimr:r9)r3r3r3r3)r@block_out_channelsF) super__init__ out_channels inner_dimr pos_embedrrtime_text_embednnLinearcontext_embedderr% x_embedder ModuleListrangertransformer_blocksrsingle_transformer_blockscontrolnet_blockslenappendrcontrolnet_single_blocksunion Embeddingcontrolnet_mode_embedderrinput_hint_blockcontrolnet_x_embeddergradient_checkpointing)selfr5r6r7r8r9r:r;r<r=r>r?r@text_time_guidance_clsi_ __class__s r*rHzFluxControlNetModel.__init__,s ',/AA%ENK:I 6Oq  6..@U !# *=t~~ N((//+t~~F"$--z*    %(;'9  # *,01    +(;'9  * &"$r!2s42234 bA  " " ) )+biiPTP^P^6_*` a b)+ b(9%s499:; iA  ) ) 0 0RYYt~~W[WeWe=f1g h iT) ::,.LL4>>,RD ) * 6$C0Odt%D !*/dnn)UD & ',#%)D !)4UXX__[RVR`R`5a)bD &&+#W  s LLci}dtdtjjdttt fffd |j D]\}}||||S)z Returns: `dict` of attention processors: A dictionary containing all attention processors used in the model with indexed by its weight name. namemodule processorsct|dr|j||d<|jD]\}}|d||||S)N get_processor .processor.)hasattrrinamed_children)rerfrgsub_namechildfn_recursive_add_processorss r*rpzHFluxControlNetModel.attn_processors..fn_recursive_add_processorssdv/282F2F2H dV:./#)#8#8#: U%+tfAhZ,@%T U r))strr%rMModulerrrm)r_rgrerfrps @r*attn_processorsz#FluxControlNetModel.attn_processorsysm  c 588?? X\]`bt]tXu !//1 BLD& 'fj A Br)c Tt|jj}t|tr,t||k7rt dt|d|d|ddt dtjjffd |jD]\}}|||y) a4 Sets the attention processor to use to compute attention. Parameters: processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): The instantiated processor class or a dictionary of processor classes that will be set as the processor for **all** `Attention` layers. If `processor` is a dict, the key needs to define the path to the corresponding cross attention processor. This is strongly recommended when setting trainable attention processors. z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.rerfct|drEt|ts|j|n#|j|j |d|j D]\}}|d|||y)N set_processorrjrk)rl isinstancedictrvpoprm)rerf processorrnrofn_recursive_attn_processors r*r{zKFluxControlNetModel.set_attn_processor..fn_recursive_attn_processorsxv/!)T2((3(($z7J)KL#)#8#8#: T%+tfAhZ,@%S Tr)N) rVrskeysrwrx ValueErrorrqr%rMrrrm)r_rzcountrerfr{s @r*set_attn_processorz&FluxControlNetModel.set_attn_processorsD((--/0 i &3y>U+BPQTU^Q_P`a005w6QRWQXXkm   Tc T588?? T!//1 ALD& 'fi @ Ar)c t|j}||d<||d<||d<||d<|j|}|rP|jj |jj |j j |j j |jj |jj |jj |jj |jj |jj d|jj |jj dt|j|_ |S)Nr7r8r9r:F)strict) rxconfig from_configrKload_state_dict state_dictrLrOrPrSrTrr]) cls transformerr7r8r9r:load_weights_from_transformerr controlnets r*from_transformerz$FluxControlNetModel.from_transformersNk(())|&7"#'9#$(;$%__V, (  0 01F1F1Q1Q1S T  & & 6 6{7R7R7]7]7_ `  ' ' 7 7 8T8T8_8_8a b  ! ! 1 1+2H2H2S2S2U V  ) ) 9 9+:X:X:c:c:ens 9 t  0 0 @ @55@@B5 A 0;:;[;[/\J ,r) hidden_statescontrolnet_condcontrolnet_modeconditioning_scaleencoder_hidden_statespooled_projectionstimestepimg_idstxt_idsguidancejoint_attention_kwargs return_dictreturnc | #| j} | jdd} nd} tr t|| n)| '| j ddt j d|j|}|j|j|}|j\}}}}||jjz}||jjz}|j||||jj||jj}|jddddd d }|j|||zd }||j|z}|j|j d z}| | j|j d z} nd} | |j#||n|j#|| |}|j%|}| j&d k(rt j d | d} |j&d k(rt j d|d}|j(rS| t+d|j-|}t/j0||gd}t/j0| dd| gd} t/j0| |fd}|j3|}d}t5|j6D]S\}}t/j8r%|j:r|j=|||||\}}n|||||\}}||fz}Ud}t5|j>D]S\}}t/j8r%|j:r|j=|||||\}}n|||||\}}||fz}Ud}tA||jBD]\}}||}||fz}d}tA||jDD]\} }|| } || fz}|D!cgc]}!|!|z }}!|D!cgc]}!|!|z }}!tG|dk(rdn|}tG|dk(rdn|}tr tI|| | s||fStK||Scc}!wcc}!w)a The [`FluxTransformer2DModel`] forward method. Args: hidden_states (`torch.FloatTensor` of shape `(batch size, channel, height, width)`): Input `hidden_states`. controlnet_cond (`torch.Tensor`): The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`. controlnet_mode (`torch.Tensor`): The mode tensor of shape `(batch_size, 1)`. conditioning_scale (`float`, defaults to `1.0`): The scale factor for ControlNet outputs. encoder_hidden_states (`torch.FloatTensor` of shape `(batch size, sequence_len, embed_dims)`): Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. pooled_projections (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Embeddings projected from the embeddings of input conditions. timestep ( `torch.LongTensor`): Used to indicate denoising step. block_controlnet_hidden_states: (`list` of `torch.Tensor`): A list of tensors that if specified are added to the residuals of transformer blocks. joint_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain tuple. Returns: If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a `tuple` where the first element is the sample tensor. Nscale?z\Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective.rrr-r izrPassing `txt_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch TensorzrPassing `img_ids` 3d torch.Tensor is deprecated.Please remove the batch dimension and pass it as a 2d torch TensorzA`controlnet_mode` cannot be `None` when applying ControlNet-Union)rEr()rrtembimage_rotary_emb)r r!)&copyryrrgetloggerwarningrPr\shaperr5reshapepermuter]todtyperLrOndimrYr}r[r%catrK enumeraterSis_grad_enabledr^_gradient_checkpointing_funcrTziprUrXrVrr)"r_rrrrrrrrrrrr lora_scale batch_sizechannels height_pwwidth_pwheightwidthrcontrolnet_mode_embidsr block_samples index_blockblocksingle_block_samplesr block_samplecontrolnet_blockr!single_block_samplesamples" r*forwardzFluxControlNetModel.forwards^ " -%;%@%@%B "/33GSAJJ  dJ /%16L6P6PQXZ^6_6kr 6  ,"33ODO8G8M8M 5J)X$++"8"88F 6 66E-55Hfdkk.D.DeT[[McMcO.55aAq!QGO-55j&5.RTUO%(B(B?(SS ;;}223d:  {{=#6#67$>HH  += >%%h:LM !% 5 56K L <<1  NNU ajG <<1  NNU ajG ::& !dee"&"?"?"P $)II/BDY.Z`a$b !ii!g 6A>Gii'*2>>#. "+D,C,C"D = K$$&4+F+F7;7X7X!)$ 84%}8="/*?%5 84%} *],<Q 1J+4S/[m5n+o ''E46@"/$)$(DM',%%'9*?##+A + 73 3 6,9)3G0$05J5VGJJ_anFo1 B 4l1<?1-1,7RN8 I ;\8,F8487 R%&BBBI8AO_6H$))T8" 33E4 7A"/$)$(DM',%%'9*?##+A + 73 3 6,9)3G0$05J5VGJJ_anFo1 B 4l1<?1-1,7RN8 I ;\8,F848;" H%&BBBg1 8D1 8sE(*E.)E4E:)NNNNNNNT)r"r#r$__doc__rHr%rrtensorrr&rrrrqrrr rrrrrs@r*rrs //3+/%) $ $!%;? ]C((]Cell+]Cell+ ]C !K ]C %|| ]C"LL]C""]C]C]C,,]C!)c3h 8]C]C #U* +]Cr)r)- dataclassesrtypingrrrrrr r%torch.nnrMconfiguration_utilsr r loadersr utilsrrrrrattention_processorrcontrolnets.controlnetrr embeddingsrrrmodeling_outputsrmodeling_utilsrtransformers.transformer_fluxrr get_loggerr"rrr,rr(r)r*rs":: B'bb4Quu7'\   H % 9:9 9 c *k3Cc L nCznCr)