
    bi0                     2   d dl mZ d dlmZmZmZmZmZ d dlZd dlm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ d	d
lmZ d	dlmZmZ d	dlmZ d	dlmZ d	dlmZm Z  d	dl!m"Z" ddl#m$Z$  ejJ                  e&      Z'e G d de             Z( G d deee      Z)y)    )	dataclass)AnyDictOptionalTupleUnionN)nn   )ConfigMixinregister_to_config)PeftAdapterMixin)USE_PEFT_BACKEND
BaseOutputloggingscale_lora_layersunscale_lora_layers   )AttentionProcessor)
PatchEmbedPixArtAlphaTextProjection)Transformer2DModelOutput)
ModelMixin)AdaLayerNormSingleRMSNorm)SanaTransformerBlock   )zero_modulec                   2    e Zd ZU eej
                     ed<   y)SanaControlNetOutputcontrolnet_block_samplesN)__name__
__module____qualname__r   torchTensor__annotations__     g/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/models/controlnets/controlnet_sana.pyr   r   $   s    #ELL11r(   r   c            &            e Zd ZdZddgZddgZe	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d(dedee   d	ed
ededee   dee   dee   dede	de	de
dedede
de	dee   ddf$ fd       Zedeeef   fd       Zdeeeeef   f   fdZ	 	 	 	 	 d)dej(                  dej(                  dej*                  d ej(                  d!e	d"eej(                     d#eej(                     d$eeeef      d%e
deeej(                  d&f   ef   fd'Z xZS )*SanaControlNetModelTr   r   patch_embednormNin_channelsout_channelsnum_attention_headsattention_head_dim
num_layersnum_cross_attention_headscross_attention_head_dimcross_attention_dimcaption_channels	mlp_ratiodropoutattention_biassample_size
patch_sizenorm_elementwise_affinenorm_epsinterpolation_scalereturnc                    t         |           |xs |}||z  }t        |||||||dnd       | _        t	        |      | _        t        |	|      | _        t        |dd      | _	        t        j                  t        |      D cg c]  }t        |||||||||||
       c}      | _        t        j                  g       | _        t!        t        j"                  ||            | _        t        t'        | j                              D ]>  }t        j"                  ||      }t!        |      }| j                  j)                  |       @ d| _        y c c}w )	Nsincos)heightwidthr;   r.   	embed_dimr>   pos_embed_type)in_featureshidden_sizegh㈵>T)epselementwise_affine)r8   r3   r4   r5   r9   r<   r=   r7   F)super__init__r   r,   r   
time_embedr   caption_projectionr   caption_normr	   
ModuleListranger   transformer_blockscontrolnet_blocksr   Linearinput_blocklenappendgradient_checkpointing)selfr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   	inner_dim_controlnet_block	__class__s                        r)   rK   zSanaControlNetModel.__init__.   sc   * 	#2{'*<<	 &!# 3':'F8D
 -Y7";HXfo"p#I4DQ #%-- z*  %'&#.G-E(;#1,C%'#
( "$r!2&ryyI'FGs42234 	<A!yyI>*+;<""))*:;	<
 ',#7s   Ec                     i }dt         dt        j                  j                  dt        t         t
        f   ffd| j                         D ]  \  }} |||        |S )z
        Returns:
            `dict` of attention processors: A dictionary containing all attention processors used in the model with
            indexed by its weight name.
        namemodule
processorsc                     t        |d      r|j                         ||  d<   |j                         D ]  \  }} |  d| ||        |S )Nget_processor
.processor.)hasattrrb   named_children)r^   r_   r`   sub_namechildfn_recursive_add_processorss        r)   ri   zHSanaControlNetModel.attn_processors.<locals>.fn_recursive_add_processors   sd    v/282F2F2H
dV:./#)#8#8#: U%+tfAhZ,@%TU r(   )strr$   r	   Moduler   r   rf   )rX   r`   r^   r_   ri   s       @r)   attn_processorsz#SanaControlNetModel.attn_processorsx   sm     
	c 	588?? 	X\]`bt]tXu 	 !//1 	BLD&'fjA	B r(   	processorc           	      T   t        | j                  j                               }t        |t              r,t        |      |k7  rt        dt        |       d| d| d      dt        dt        j                  j                  ffd| j                         D ]  \  }} |||        y)	a4  
        Sets the attention processor to use to compute attention.

        Parameters:
            processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
                The instantiated processor class or a dictionary of processor classes that will be set as the processor
                for **all** `Attention` layers.

                If `processor` is a dict, the key needs to define the path to the corresponding cross attention
                processor. This is strongly recommended when setting trainable attention processors.

        z>A dict of processors was passed, but the number of processors z0 does not match the number of attention layers: z. Please make sure to pass z processor classes.r^   r_   c                     t        |d      rEt        |t              s|j                  |       n#|j                  |j	                  |  d             |j                         D ]  \  }} |  d| ||        y )Nset_processorrc   rd   )re   
isinstancedictrp   poprf   )r^   r_   rm   rg   rh   fn_recursive_attn_processors        r)   rt   zKSanaControlNetModel.set_attn_processor.<locals>.fn_recursive_attn_processor   sx    v/!)T2((3(($z7J)KL#)#8#8#: T%+tfAhZ,@%STr(   N)rU   rl   keysrq   rr   
ValueErrorrj   r$   r	   rk   rf   )rX   rm   countr^   r_   rt   s        @r)   set_attn_processorz&SanaControlNetModel.set_attn_processor   s     D((--/0i&3y>U+BPQTU^Q_P` a005w6QRWQXXkm 
	Tc 	T588?? 	T !//1 	ALD&'fi@	Ar(   hidden_statesencoder_hidden_statestimestepcontrolnet_condconditioning_scaleencoder_attention_maskattention_maskattention_kwargsreturn_dict.c
                 T   |#|j                         }|j                  dd      }
nd}
t        rt        | |
       n)|'|j	                  dd       t
        j                  d       |A|j                  dk(  r2d|j                  |j                        z
  dz  }|j                  d      }|A|j                  dk(  r2d|j                  |j                        z
  dz  }|j                  d      }|j                  \  }}}}| j                  j                  }||z  ||z  }}| j                  |      }|| j                  | j                  |j                  |j                                    z   }| j!                  |||j                        \  }}| j#                  |      }|j%                  |d|j                  d         }| j'                  |      }d	}t)        j*                         r<| j,                  r0| j.                  D ]   }| j1                  ||||||||      }||fz   }" n%| j.                  D ]  } ||||||||      }||fz   } d	}t3        || j4                        D ]  \  }} ||      }||fz   } t        rt7        | |
       |D cg c]  }||z  	 }}|	s|fS t9        |
      S c c}w )Nscale      ?zVPassing `scale` via `attention_kwargs` when not using the PEFT backend is ineffective.r   r   g     )
batch_sizehidden_dtyper'   )r    )copyrs   r   r   getloggerwarningndimtodtype	unsqueezeshapeconfigr;   r,   rT   rL   rM   viewrN   r$   is_grad_enabledrW   rQ   _gradient_checkpointing_funcziprR   r   r   )rX   ry   rz   r{   r|   r}   r~   r   r   r   
lora_scaler   num_channelsrB   rC   ppost_patch_heightpost_patch_widthembedded_timestepblock_res_samplesblockcontrolnet_block_res_samplesblock_res_sampler[   samples                            r)   forwardzSanaControlNetModel.forward   s"    '/446)--gs;JJdJ/+0@0D0DWd0S0_l %.*=*=*B
  ."3"3M4G4G"HHHTN+55a8N "-2H2M2MQR2R&'*@*C*CMDWDW*X&X\d%d"%;%E%Ea%H" 3@2E2E/
L&%KK"".4k5A:+((7%(8(89I9I/J\J\]j]p]pJq9r(ss&*oo-:M:M '6 '
## !% 7 78M N 5 : ::r=K^K^_aKb c $ 1 12G H   "t'B'B00 I $ A A!")*%$	! %68H$H!I 00 
I %!")*%$! %68H$H!
I (*$256GI_I_2` 	^../0@A+GK[J]+](	^ j1Rn'o1C(C'o$'o022#=YZZ (ps   J%)    r   F   r         p   i  i 	  g      @g        Fr   r   Fgư>N)r   NNNT)r!   r"   r#    _supports_gradient_checkpointing_no_split_modules _skip_layerwise_casting_patternsr   intr   floatboolrK   propertyr   rj   r   rl   r   rx   r$   r%   
LongTensorr   r   r   r   __classcell__)r\   s   @r)   r+   r+   )   sd   '+$/>(5v'>$ &(#%"$3525-1 $$(--1%G,G, smG, !	G,
  G, G, $,C=G, #+3-G, &c]G, G, G, G, G, G, G,  "&!G," #G,$ &c]%G,& 
'G, G,R c+=&=!>  0 AE2Dd3PbKbFc2c,d  AP %(9=1559 n[||n[  %||n[ ""	n[
 n[ "n[ !) 6n[ !.n[ #4S>2n[ n[ 
uU\\3&')AA	Bn[r(   r+   )*dataclassesr   typingr   r   r   r   r   r$   r	   configuration_utilsr   r   loadersr   utilsr   r   r   r   r   attention_processorr   
embeddingsr   r   modeling_outputsr   modeling_utilsr   normalizationr   r   transformers.sana_transformerr   
controlnetr   
get_loggerr!   r   r   r+   r'   r(   r)   <module>r      s{    " 4 4   B ' b b 4 > 7 ' 7 @ # 
		H	% 2: 2 2y[*k3C y[r(   