
    bif                        d dl Z d dlmZ d dlZddlmZmZmZ  ej                  e	      Z
d ZddZddZd Zd	 Zd
 Zd Zd Zdedee   defdZd Zd Zd Zd Zd Zd ZddZddZd Zy)    N)List   )is_peft_versionloggingstate_dict_all_zeroc                 d    | j                  dd      \  }}t        j                  ||gd      }|S )Nr   r   dim)chunktorchcat)weightshiftscale
new_weights       b/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/loaders/lora_conversion_utils.pyswap_scale_shiftr      s2    <<q<)LE5E5>q1J    c                    t        | j                               }g d}g d}d}d}|D ]0  t        fd|D              rd}t        fd|D              s/d}2 |r1|r/|D ](  t        fd|D              s| j                         * | S d}	|D ]  t        fd|D              sd}	 n |	s| S i }
g d	}t	               t	               t	               }}}|D ]  }d
|v r| j                  |      |
|<   t        |j                  |      d | d         }|d   |v r|j                  |       U|d   |v r|j                  |       n|d   |v r|j                  |       t        d| d       |D ci c]  }|| D cg c]  }d| | |v s| c} }}}|D ci c]  }|| D cg c]  }d| | |v s| c} }}}|D ci c]  }|| D cg c]  }d| | |v s| c} }}}|D ]  }|dz
  |j                  dz   z  }|dz
  |j                  dz   z  }||   D ]  t        j                  |      |         }dvr||   nd}dvrt        |      nd}|j                  j                  |      d |dz
   t        |      ||gz   j                  |      |dz   d  z         }| j                        |
|<     |D ]  }d }|dk(  r|d   dg}n)|dk(  r|d   dg}n|dk(  r|d   dg}nt        d| d      ||   D ]T  |j                  j                  |      d |dz
   |z   j                  |      |d  z         }| j                        |
|<   V  |D ]  }||j                  dz   z  }||j                  dz   z  }||   D ]  t        j                  |      |         }||   }|dk  rt        |      nd}|j                  j                  |      d |dz
   t        |      ||gz   j                  |      |dz   d  z         }| j                        |
|<     | rt        d      |
S c c}w c c}}w c c}w c c}}w c c}w c c}}w )N)input_blocksmiddle_blockoutput_blocks)down_blocks	mid_block	up_blocksFc              3   &   K   | ]  }|v  
 y wN .0pkeys     r   	<genexpr>z5_maybe_map_sgm_blocks_to_diffusers.<locals>.<genexpr>*        .AqCx.   Tc              3   &   K   | ]  }|v  
 y wr   r   r   s     r   r#   z5_maybe_map_sgm_blocks_to_diffusers.<locals>.<genexpr>,   s     4ac4r%   c              3   &   K   | ]  }|v  
 y wr   r   r   s     r   r#   z5_maybe_map_sgm_blocks_to_diffusers.<locals>.<genexpr>3   s     2182r%   c              3   &   K   | ]  }|v  
 y wr   r   r   s     r   r#   z5_maybe_map_sgm_blocks_to_diffusers.<locals>.<genexpr>:   r$   r%   )resnets
attentions
upsamplerstextr      r   z'Checkpoint not supported because layer z not supported.r   r   r   opdownsamplers01zInvalid middle block id .z:At this point all state dict entries have to be converted.)listkeysanypopsetintsplitadd
ValueErrorlayers_per_blockstrjoin)
state_dictunet_config	delimiterblock_slice_posall_keyssgm_patternsnot_sgm_patternscontains_sgm_patternscontains_not_sgm_patternsis_in_sgm_formatnew_state_dictinner_block_mapinput_block_idsmiddle_block_idsoutput_block_idslayerlayer_idr"   r   middle_blocksr   iblock_idlayer_in_block_idinner_block_idinner_block_keyinner_layers_in_blocknew_keykey_parts                    `           r   "_maybe_map_sgm_blocks_to_diffusersrZ       s   JOO%&HDL@ " % -...$(!4#344(,%	- !: 	$C2\22s#	$   ...#
  N=O ;>%'7%O cU?$.NN5$9N5!5;;y12B?CBGHHA%'##H-aE) $$X.aE) $$X. #J5'Q`!abbc  ( 	*\3,yk(0TX[0[3\\L  ) 	*\3,yk(0TX[0[3\\M  ) 	*]3-	{8*0UY\0\3]]M   :E{;;a?@U{'C'Ca'GH? 		:C 9!5o!FGNAESon=VdO>B#oC(9$:SV!nn		)$%:':;x=/3HIJ))I&':'<=>G
 '1nnS&9N7#		:	:  :6'*C0H!V'*C0H!V'*C0H7s!<== # 	:Cnn		)$%:':;hFS\I]^m^nIooG '1nnS&9N7#		::"  :559:!=!=!AB # 		:C 9!5o!FGN-n=O>Lq>PC(9$:VY!nn		)$%:':;x=/3HIJ))I&':'<=>G
 '1nnS&9N7#		:	: UVV ]
 ]
 ^sZ   6
P= P8P8P= 
Q*Q9Q=Q

QQ#Q'Q8P=QQc                 ~   i }i }i }i }t        d | D              }t        d | D              }t        d | D              }	|s|s|	rt        dd      rt        d      t        | j	                               }
|
D ]@  }|j                  d      s|j                  d      d	   }|d
z   }|dz   }|j                  d      rt        |      }| j                  |      ||<   | j                  |      ||j                  dd      <   |rud|v rdnd}| j                  |j                  dd            ||j                  |d      <   n7|j                  d      r%t        ||      }|j                  d      r9| j                  |      ||<   | j                  |      ||j                  dd      <   n8| j                  |      ||<   | j                  |      ||j                  dd      <   |s|	rd|v rdnd}|j                  d      r5| j                  |j                  dd            ||j                  |d      <   nE|j                  d      r4| j                  |j                  dd            ||j                  |d      <   || v s| j                  |      j                         }|j                  t        ||             C t        |       d	kD  r+t        ddj!                  | j	                                      t"        j%                  d       |j'                         D ci c]  \  }}| d| | }}}|j'                         D ci c]  \  }}| d| | }}}t        |      d	kD  r'|j'                         D ci c]  \  }}d| | c}}nd}||j                  |       i ||}||fS c c}}w c c}}w c c}}w )a  
    Converts a non-Diffusers LoRA state dict to a Diffusers compatible state dict.

    Args:
        state_dict (`dict`): The state dict to convert.
        unet_name (`str`, optional): The name of the U-Net module in the Diffusers model. Defaults to "unet".
        text_encoder_name (`str`, optional): The name of the text encoder module in the Diffusers model. Defaults to
            "text_encoder".

    Returns:
        `tuple`: A tuple containing the converted state dict and a dictionary of alphas.
    c              3   0   K   | ]  }d |v xr d|v   yw)
dora_scale
lora_unet_Nr   r    ks     r   r#   z;_convert_non_diffusers_lora_to_diffusers.<locals>.<genexpr>   s"     [1|q0F\Q5FF[   c              3   <   K   | ]  }d |v xr
 d|v xs d|v   yw)r]   lora_te_	lora_te1_Nr   r_   s     r   r#   z;_convert_non_diffusers_lora_to_diffusers.<locals>.<genexpr>   s-     m]^\Q.XJ!O4W{VWGWXms   c              3   0   K   | ]  }d |v xr d|v   yw)r]   	lora_te2_Nr   r_   s     r   r#   z;_convert_non_diffusers_lora_to_diffusers.<locals>.<genexpr>   s"     Yla/DK14DDYra   <z0.9.0zeYou need `peft` 0.9.0 at least to use DoRA-enabled LoRAs. Please upgrade your installation of `peft`.lora_down.weightr3   r   .lora_up.weight.alphar^   .down..up.z_lora.down.z.lora.down.r]   z.lora_magnitude_vector.)rc   rd   rf   rc   rd   .lora_linear_layer.rf   z7The following keys have not been correctly renamed: 

 , z"Non-diffusers checkpoint detected.text_encoder_2.N)r6   r   r<   r4   r5   endswithr:   
startswith_convert_unet_lora_keyr7   replace_convert_text_encoder_lora_keyitemupdate_get_alpha_namelenr?   loggerinfoitems)r@   	unet_nametext_encoder_nameunet_state_dictte_state_dictte2_state_dictnetwork_alphasdora_present_in_unetdora_present_in_tedora_present_in_te2all_lora_keysr"   	lora_namelora_name_uplora_name_alphadiffusers_namedora_scale_key_to_replacedora_scale_key_to_replace_tealphamodule_nameparamsrJ   s                         r   (_convert_non_diffusers_lora_to_diffusersr      s    OMNN [PZ[[mblmmYjYY15H3(w 
 *+M 7[||./ IIcN1%	 !#44#h. -3C8N /9nnS.AON+HRWcHdON228VDE $=Jn=\Mbo)NN3;;/A<#PQ   6 67PRk lm
 !!"HI;CKN ##$=>0:s0Cn-JT..YeJfn44XvFG1;1D~.KU>>ZfKg~55hGH "%8%2n%DMJ_ - ''(AB"s{{3E|'TU "."8"89UWp"qr ))+6"s{{3E|'TU #>#9#9:VXq#rs
 j(NN?388:E!!//>SX"YZo7[t :TUYU^U^_i_n_n_pUqTrstt
KK45 XgWlWlWno@SV)Ak]3V;oOo]j]p]p]rsFYkSY)*!K=96AsMs ~" M[L`L`Lbc5H[&?;-	(&	0c 
 !^,99=9N>)) pscs   N-N3:N9c                 <   | j                  dd      j                  dd      j                  dd      j                  dd      j                  dd	      j                  d
d	      j                  dd      j                  dd      j                  dd      j                  dd      j                  dd      j                  dd      j                  dd      j                  dd      j                  dd      j                  dd      dv rdvrd }t        j                  |dd!"      d#v rj                  d$d%      d&v rj                  d'd(      d)v sd*v rj                  d+d,      d-v rj                  d.d/      dv rj                  dd      d0v rj                  d0d/      dv r5d1v sd2v r&j                  d1d3      j                  d2d4      S d5v r	 S t        fd6d7D              r	 S 	 S )8zB
    Converts a U-Net LoRA key to a Diffusers compatible key.
    r^    _r3   zinput.blocksr   zdown.blockszmiddle.blockr   z	mid.blockzoutput.blocksr   z	up.blocksztransformer.blockstransformer_blocksz	to.q.lora	to_q_loraz	to.k.lora	to_k_loraz	to.v.lora	to_v_lorazto.out.0.lorato_out_lorazproj.inproj_inzproj.outproj_outz
emb.layerstime_emb_projembztime.emb.projz\.\d+(?=\D*$)r.   )countz.in.zin.layers.2conv1z.out.zout.layers.3conv2r0   r+   r/   convskipzskip.connectionconv_shortcutzconv.shortcutattn1attn2zattn1.processorzattn2.processorffc              3   &   K   | ]  }|v  
 y wr   r   )r    r"   r   s     r   r#   z)_convert_unet_lora_key.<locals>.<genexpr>5  s     FsSN"Fr%   )r   r   )rt   resubr6   )r"   patternr   s     @r   rs   rs     su    [[r2::3DN $++NMJN#++M=IN#++NKHN#++KEN#++O[IN#++KEN#++,@BVWN#++KEN#++KEN#++KEN#++O]KN#++IyAN#++J
CN#++L/JN ?.#H"^1E'//wG. '//H'<>+I'//f='//0A?S .('//Q.('//Q ~-n$>(A+33G=NON+33G=NON  ^#  
F.EF	F  	r   c                    |j                  d      r|j                  d      rdnd}nd}| j                  |d      j                  dd      }|j                  dd	      }|j                  d
d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }d|v sd|v r	 |S d|v r|j                  dd      }|S )zI
    Converts a text encoder LoRA key to a Diffusers compatible key.
    rm   rc   rd   rf   r   r   r3   z
text.model
text_modelz	self.attn	self_attnzq.proj.lorar   zk.proj.lorar   zv.proj.lorar   zout.proj.lorar   ztext.projectiontext_projectionmlp.lora.rn   )rr   rt   )r"   r   key_to_replacer   s       r   ru   ru   =  s	    56'0';';J'G[$[[4<<S#FN#++L,GN#++KEN#++M;GN#++M;GN#++M;GN#++O]KN#++,=?PQNn$(9^(K  
.	  (//:OPr   c                     | j                  d      rd}n| j                  d      rd}nd}||j                  d      d   z   dz   }||iS )	z>
    Gets the correct alpha name for the Diffusers model.
    r^   zunet.rm   text_encoder.rp   r   r   rj   )rr   r:   )r   r   r   prefixnew_names        r   rx   rx   Y  sZ     !!,/		#	#$=	> ",,X6q99HDHer   c                    d d*dfd}d }t        d | D              }|rV| j                         D ci c]9  \  }}|j                  d      r#|j                  dd      j                  d	d
      |; } }}| S t        d | D              }|s | j                         D ci c]  \  }}|j                  dd      | } }}| j                         D ci c]  \  }}|j                  dd      | } }}t        d | D              }|rat	        | d      }|rt
        j                  d       nt
        j                  d       | j                         D ci c]  \  }}d|vs|| } }}t        d | D              }	|	rnt	        | d      }
|
rt
        j                  d       nt
        j                  d       | j                         D ci c]  \  }}|j                  d      r|| } }}t        d | D              }|rat	        | d      }|rt
        j                  d       nt
        j                  d       | j                         D ci c]  \  }}d|vs|| } }}t        d | D              }|rft	        | d      }|rt
        j                  d       nt
        j                  d        | j                         D ci c]  \  }}d!|vsd|vs|| } }}d"d#g}t        d$ | D              r|j                  d%       | j                         D ci c]#  \  }}|j                  d&      rt        ||      |% } }}t        d' | D              r=t
        j                  d(       | j                         D ci c]  \  }}d)|vs|| } }}|r ||       S  ||       S c c}}w c c}}w c c}}w c c}}w c c}}w c c}}w c c}}w c c}}w c c}}w )+Nc                    |dz   | vry | j                  |dz         }|j                  d   }t        j                  ||j                  |j
                  d      }| j                  |dz   |      j                         }||z  }|}	d}
|	dz  |
k  r|	dz  }	|
dz  }
|	dz  |
k  r||	z  ||dz   <   | j                  |d	z         |
z  ||d
z   <   y )N.lora_down.weightr   Fdtypedevicerequires_gradrj         ?r   .lora_A.weightri   .lora_B.weight)r7   shaper   tensorr   r   rv   )sds_sdait_sdsds_keyait_keydown_weightrankdefault_alphar   r   
scale_downscale_ups              r   _convert_to_ai_toolkitzE_convert_kohya_flux_lora_to_diffusers.<locals>._convert_to_ai_toolkitj  s    ((6jj+>!>?   #T1B1B;K]K]mrs

7X-}=BBD 
1nx'!OJMH 1nx' .9:-Ew))*-3ZZBS8S-TW_-_w))*r   c           
         |dz   | vry | j                  |dz         }| j                  |dz         }|j                  d   }t        j                  ||j                  |j
                  d      }| j                  |dz   |      }	|	|z  }
|
}d}|dz  |k  r|dz  }|dz  }|dz  |k  r||z  }||z  }t        |      }||j                  d   |z  g|z  }nt        |      |j                  d   k(  sJ d}||z  dk(  r||z  }d	}d}t        t        |            D ]]  }t        t        |            D ]<  }||k(  r	|xr0 t        j                  |||||   z   ||z  |d
z   |z  f   dk(        }> |||   z  }_ |rt        j                  d|        |D cg c]  }|dz   	 }}|D cg c]  }|dz   	 }}|sk|j                  t        j                  ||             |j                  t        |t        j                   ||d            D ci c]  \  }}||
 c}}       y |j                  t        |t        j"                  ||d            D ci c]  \  }}||
 c}}       d}t        t        |            D ]8  }|||||   z   |z  |d
z   |z  f   j%                         |||   <   |||   z  }: y c c}w c c}w c c}}w c c}}w )Nr   ri   r   Fr   rj   r   r   Tr.   zweight is sparse: r   r   r	   )r7   r   r   r   r   r   ry   sumrangeallrz   r{   rw   dictfromkeyszipr:   r   
contiguous)r   r   r   ait_keysdimsr   	up_weightsd_lora_rankr   r   r   r   r   
num_splits	is_sparseait_rankrR   jr`   ait_down_keysait_up_keysvs                         r   _convert_to_ai_toolkit_catzI_convert_kohya_flux_lora_to_diffusers.<locals>._convert_to_ai_toolkit_cat  s7   ((6jj+>!>?JJw)::;	"((+  1 1+:L:L\a
 

7X-}=$ 
1nx'!OJMH 1nx' "J.(	 ]
<OOA&*45
BDt9	 2222 	*$)#z1HIA3t9% s4y) AAv  ) !eii!!a$q'k/1x<1q5HBT3T"TUYZZ/I T!W 0	:; 8@@!--@@5=>q++>>MM$--{CD MMCU[[TX^_=`,abDAq1a4bc MMCu{{;Xbhi?j,klDAq1a4lm A3t9% )21q47{?ALTUXYTY]eSeDe3e)f)q)q)s{1~&T!W A> c ms   J:(J?K
K

c                    i t        d      D ]  } | d| dd| d        | d| dd| dd| dd| d	g        | d| d
d| d        | d| dd| d        | d| dd| d        | d| dd| d        | d| dd| dd| dd| dg        | d| dd| d        | d| dd| d        | d| dd| d        t        d      D ]O  } | d| dd| dd| dd| d	d| d gg d!"        | d| d#d| d$        | d| d%d| d&       Q fd'}t        d( | D              r |d)d*g|        t        d+ | D              r
 |d,g|        t        d- | D              r
 |d.g|        t        d/ | D              r |d0d1g|        t        d2 | D              r |d3d4g|        t        d5 | D              r |d6d7t        fd8g|        t        | j	                               }i }|r@t        d9 |D              st        d:d;j                  |             |D ]  }|j                  d<      s|j                  d=      d>   }| d?}| d@}t        ||      }	|j                  dA      rI| j                  |      }
|
j                  d>   }|
||	<   | j                  |      ||	j                  dBdC      <   || v s| j                  |      j                         }|z  }|}dD}|dEz  |k  r|dEz  }|dEz  }|dEz  |k  r||	xx   |z  cc<   ||	j                  dBdC      xx   |z  cc<    t!        |       d>kD  r&t"        j%                  dF| j	                                 |r&|j'                         D ci c]  \  }}dG| | }}}i |}|S c c}}w )HN   lora_unet_double_blocks__img_attn_projtransformer.transformer_blocks..attn.to_out.0_img_attn_qkv
.attn.to_q
.attn.to_k
.attn.to_v
_img_mlp_0z.ff.net.0.proj
_img_mlp_2z	.ff.net.2_img_mod_linz.norm1.linear_txt_attn_proj.attn.to_add_out_txt_attn_qkv.attn.add_q_proj.attn.add_k_proj.attn.add_v_proj
_txt_mlp_0z.ff_context.net.0.proj
_txt_mlp_2z.ff_context.net.2_txt_mod_linz.norm1_context.linear&   lora_unet_single_blocks__linear1&transformer.single_transformer_blocks.z	.proj_mlp)   r   r   i 0  )r   _linear2	.proj_out_modulation_linz.norm.linearc                     dD ]Z  }|dk(  rdnd}| D ]J  \  }}}|j                  |      }|j                  |      }|j                  |      }	|r ||	      }	|	
|<   L \ y )Nlora_Alora_Br   	lora_downlora_up)lora_key)orig_lora_key)formatr7   )assignmentssourcer  r  
target_fmt
source_fmt	transform
target_key
source_keyvaluer   s             r   assign_remaining_weightszr_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.assign_remaining_weights%  s    0 //78/C9D /5J
I!+!2!2H!2!EJ!+!2!2!2!OJ"JJz2E  )% 0).F:&//r   c              3   $   K   | ]  }d |v  
 yw)guidance_inNr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>0       2a}!2   )z<time_text_embed.guidance_embedder.linear_1.{lora_key}.weightz5lora_unet_guidance_in_in_layer.{orig_lora_key}.weightN)z<time_text_embed.guidance_embedder.linear_2.{lora_key}.weightz6lora_unet_guidance_in_out_layer.{orig_lora_key}.weightNc              3   $   K   | ]  }d |v  
 yw)img_inNr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>A       -x1}-r  )zx_embedder.{lora_key}.weightz'lora_unet_img_in.{orig_lora_key}.weightNc              3   $   K   | ]  }d |v  
 yw)txt_inNr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>I  r  r  )z"context_embedder.{lora_key}.weightz'lora_unet_txt_in.{orig_lora_key}.weightNc              3   $   K   | ]  }d |v  
 yw)time_inNr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>Q  s     .!yA~.r  )z<time_text_embed.timestep_embedder.linear_1.{lora_key}.weightz1lora_unet_time_in_in_layer.{orig_lora_key}.weightN)z<time_text_embed.timestep_embedder.linear_2.{lora_key}.weightz2lora_unet_time_in_out_layer.{orig_lora_key}.weightNc              3   $   K   | ]  }d |v  
 yw)	vector_inNr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>b  s     0A{a0r  )z8time_text_embed.text_embedder.linear_1.{lora_key}.weightz3lora_unet_vector_in_in_layer.{orig_lora_key}.weightN)z8time_text_embed.text_embedder.linear_2.{lora_key}.weightz4lora_unet_vector_in_out_layer.{orig_lora_key}.weightNc              3   $   K   | ]  }d |v  
 yw)final_layerNr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>s  r  r  z!norm_out.linear.{lora_key}.weightz?lora_unet_final_layer_adaLN_modulation_1.{orig_lora_key}.weight)zproj_out.{lora_key}.weightz3lora_unet_final_layer_linear.{orig_lora_key}.weightNc              3   >   K   | ]  }|j                  d         yw))lora_telora_te1Nrr   r_   s     r   r#   zc_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit.<locals>.<genexpr>  s     Uq||$;<U   zIncompatible keys detected: 

 ro   rh   r3   r   ri   rj   rm   rk   rl   r   r   z!Unsupported keys for ai-toolkit: r   )r   r6   r   r4   r5   r   r<   r?   rq   r:   ru   rr   r7   r   rt   rv   ry   rz   warningr|   )r   rR   r  remaining_keysr   r"   r   r   r   r   r   r   r   r   r   r   r   r   rJ   r   r   r   s                      @r   !_convert_sd_scripts_to_ai_toolkitzP_convert_kohya_flux_lora_to_diffusers.<locals>._convert_sd_scripts_to_ai_toolkit  s   r D	A"*1#^<1!NC	 '*1#];5aS
C5aS
C5aS
C		 #*1#Z81!NC	 #*1#Z81!I>	 #*1#\:1!MB	 #*1#^<1!4DE	 '*1#];5aS8HI5aS8HI5aS8HI		 #*1#Z81!4JK	 #*1#Z81!4EF	 #*1#\:1!4IJ	D	L r 	A&*1#X6<QCzJ<QCzJ<QCzJ<QCyI	 / #*1#X689E	 #*1#_=8<H	'	6		/ 2622$
   -f--$e 	 -f--$k 	 .v..$
   000$
   2622$ <Y(
 p 
 fkkm,UnUU #DTYY~E^D_!`aa% X||$67IIcN1-	"+O<%.Kv"6!?Y!O''(AB"(**S/K#.#4#4Q#7L4?M.1NTjjYeNfM."8"86"JK"f,"JJ7<<>E!L0E!&J"H$q.83"a
 A %q.83 ".1Z?1!."8"86"JKxWK5X8 v;?NN>v{{}oNO^k^q^q^stGZ{TZ}[M:FBtMt4F4m4 us   %O c                    i }d }| D ch c]E  }|j                  d      s2|j                  dd      j                  dd      j                  dd      G }}t        d |D              s
J d|       d	}|D ]$  j                  d
      r6t        j	                  d
      d   j	                  d      d         }d| }nVj                  d      r6t        j	                  d      d   j	                  d      d         }d| }nj                  d      rd}j                  d      rd}nj                  d      rd}nӉj                  d      rd}nj                  d      rd}nj                  d      r#t        j	                  d      d         }d| }nwj                  d      r#t        j	                  d      d         }d| }nCj                  d      r#t        j	                  d      d         }d | }nt        d! d"      d#v rud$v r|d%z  }nkd&v r|d'z  }nat        fd(d)D              rj	                  d#      d   }|d*| z  }n0t        fd+d,D              rj	                  d#      d   }|d*| z  } ||| |       ' |rt        j                  d-      }	d.d/d0d1d2}
d3d4d5}|D ]  j                  d      s|	j                        }|s)t        |j                  d6            }d7| }d8v r$|
j                         D ]  \  }}|v s||z  } n) n'd9v r#|j                         D ]  \  }}|v s||z  } n  ||| |        d	}| rt        d: | D              }|r1t        | j                               }|D ]  | j                          t        |       dkD  r&t!        d;t        | j                                d<      |j                         D ci c]  \  }}|j                  d=      rd>| | }}}|j                         D ci c]  \  }}|j                  d=      sd?| | }}}i ||S c c}w c c}}w c c}}w )@Nc                 @   |  d}|j                  |      }|j                  d   }|  d}|j                  |      }|  d}	|j                  |	      }
|
|z  }|}d}|dz  |k  r|dz  }|dz  }|dz  |k  r||z  }||z  }| d}|||<   |||j                  dd	      <   y )
Nr   r   ri   rj   r   r   r   z.lora_A.z.lora_B.)r7   r   rt   )original_keydiffusers_keyr@   rJ   down_keyr   	lora_rankup_weight_keyr   	alpha_keyr   r   r   r   diffusers_down_keys                  r   _convertzi_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>._convert  s    &'89H$..2K#))!,I+nO<M"}5I'./INN9-E I%EJHq.8+a
A q.8+ &
2K!H,I$1?.!A1<N-.QZN-55j*MNr   r^   r   r   ri   rj   c              3   >   K   | ]  }|j                  d         yw))lora_transformer_rd   Nr!  r_   s     r   r#   zj_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>.<genexpr>  s     ]1<< BC]r"  zall_unique_keys=F+lora_transformer_single_transformer_blocks_r-   r   r   single_transformer_blocks.$lora_transformer_transformer_blocks_transformer_blocks.rd   T!lora_transformer_context_embeddercontext_embedder lora_transformer_norm_out_linearnorm_out.linearlora_transformer_proj_outr   lora_transformer_x_embedder
x_embedder:lora_transformer_time_text_embed_guidance_embedder_linear_z)time_text_embed.guidance_embedder.linear_6lora_transformer_time_text_embed_text_embedder_linear_z%time_text_embed.text_embedder.linear_:lora_transformer_time_text_embed_timestep_embedder_linear_z)time_text_embed.timestep_embedder.linear_zHandling for key (z) is not implemented.attn_	_to_out_0r   _to_add_outr   c              3   &   K   | ]  }|v  
 y wr   r   )r    qkvr`   s     r   r#   zj_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>.<genexpr>  s     FcFr%   to_qto_kto_v.attn.c              3   &   K   | ]  }|v  
 y wr   r   )r    add_qkvr`   s     r   r#   zj_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>.<genexpr>  s     `'A`r%   )
add_q_proj
add_k_proj
add_v_projz(lora_te1_text_model_encoder_layers_(\d+)z.self_attn.q_projz.self_attn.k_projz.self_attn.v_projz.self_attn.out_proj)q_projk_projv_projout_projz.mlp.fc1z.mlp.fc2)fc1fc2r.   ztext_model.encoder.layers.attnr   c              3   >   K   | ]  }|j                  d         ywr^   Nr!  r_   s     r   r#   zj_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers.<locals>.<genexpr>  s     $TAQ\\,%?$Tr"  z\Expected an empty state dict at this point but its has these keys which couldn't be parsed: r3   ztext_model.transformer.r   )rr   rt   r   r9   r:   NotImplementedErrorr6   r   compilesearchgroupr|   r4   r5   r7   ry   r<   )r@   rJ   r/  r`   all_unique_keyshas_te_keysrR   r)  	remaininglayer_patternattn_mappingmlp_mappingmatchkey_fragmentsuffixremaining_all_unetr5   r   transformer_state_dictr   s      `                r   (_convert_mixture_state_dict_to_diffuserszW_convert_kohya_flux_lora_to_diffusers.<locals>._convert_mixture_state_dict_to_diffusers  s    	[8  
<</ II)2.667H"MUUV^`bc
 

 ]_]]tbrbqas_tt]  *	CA||IJ MNrRXXY\]^_`a"<QC @DE FGKQQRUVWXYZ"5aS 9k*"AB 2@A 19: *;< ,Z[ \]^`ab"KA3 OVW XYZ\]^"Gs KZ[ \]^`ab"KA3 O),>qcAV*WXX!|!#!%55M"a'!%77MF-EFF ! 0 4I!vi[%99M`5_`` ! 0 4I!vi[%99MQz>BU*	CX JJ'RSM---1	L #-Z@K$ G||K0%,,Q/A'"<QC @Q;0<0B0B0D ",f'1,)V3M!" aZ0;0A0A0C ",f'1,)V3M!"
 M:~F+G. #!$$T$T!T
)*D "q!" z?Qnost~  uD  uD  uF  pG  oH  HI  J 
 /=.B.B.D"
&*aALLYfLgl1#!"
 "
 =K<P<P<RrDAqVWVbVbcpVq=,a/rr:(:M::K
B"
 ss   A
Q8Q Q 0Q&
Q&c              3   >   K   | ]  }|j                  d         yw)rX  Nr!  r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>2  s     Oqall>:Or"  rX  rh   zlora_A.weightzlora_up.weightzlora_B.weightc              3   b   K   | ]'  }|j                  d       xr d|v xs
 d|v xs d|v  ) yw)r1  r  r  r   Nr!  r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr><  sB      gh()b{a/?/a9PQ>/aU\`aUabs   -/diffusion_model.r^   z!text_encoders.clip_l.transformer.rc   c              3   $   K   | ]  }d |v  
 yw)position_embeddingNr   r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>E  s     $S1%9Q%>$Sr  rm  zThe `position_embedding` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zThe state_dict has position_embedding LoRA params and we currently do not support them. Open an issue if you need this supported - https://github.com/huggingface/diffusers/issues/new.c              3   >   K   | ]  }|j                  d         yw) text_encoders.t5xxl.transformer.Nr!  r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>U  s     ]Q%GH]r"  ztext_encoders.t5xxlzThe `t5xxl` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zT5-xxl keys found in the state dict, which are currently unsupported. We will filter them out.Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.ro  c              3   J   K   | ]  }d |v xr |j                  d        yw)diff_br^   rc   Nr!  r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>d  s'     gWXAR!,,7Q*RRgs   !#.diff_bzThe `diff_b` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.z`diff_b` keys found in the state dict which are currently unsupported. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.c              3   0   K   | ]  }d |v xr d|v   yw).norm.diffNr   r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>t  s      NaGqL9W\9Nra   rv  zThe `diff` LoRA params are all zeros which make them ineffective. So, we will purge them out of the current state dict to make loading possible.zNormalization diff keys found in the state dict which are currently unsupported. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.ru  r  r  c              3   $   K   | ]  }d |v  
 yw)r   Nr   r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>  s     0w!|0r  r   rr  c              3   $   K   | ]  }d |v  
 yw)r   Nr   r_   s     r   r#   z8_convert_kohya_flux_lora_to_diffusers.<locals>.<genexpr>  s     :! A%:r  z`text_projection` keys found in the `state_dict` which are unexpected. So, we will filter out those keys. Open an issue if this is a problem - https://github.com/huggingface/diffusers/issues/new.r   r   )	r6   r|   rr   rt   r   rz   r{   append_custom_replace)r@   r%  rh  has_peft_state_dictr`   r   has_mixturehas_position_embeddingzero_status_pe	has_t5xxlzero_status_t5	has_diffbzero_status_diff_bhas_norm_diffzero_status_difflimit_substringsr   r   s                   @@r   %_convert_kohya_flux_lora_to_diffusersr  i  s   `*AFgRB;N OJOO #((*
1||N+ II(/:BBCSUdeghh

 

   lv K
 Q[QaQaQcdAaii 2LA1Dd
d`j`p`p`rsX\XY[\aii CZPRSSs
s!$$S
$S!S!0=QRNe v ,6+;+;+=_41aAU]^A^!Q$_J_]R\]]	0=RSNe
 p ,6+;+;+=v41aQ\\RtEu!Q$vJvg\fgg	!4Z!K!e
 K
 ,6+;+;+=T41aRSAS!Q$TJTN:NN2:wGe
 K
 ,6+;+;+=g41aPQAQV]efVf!Q$gJg'30Z00##G, #((*
1||67 A/0!3

 
 :z::KKG
 ,6+;+;+=\41aARZ[A[!Q$\J\7
CC,Z88K
 es  ` w  U  h
 ]sY   >N$%N*N0N6%N6N<'N<OOOOO(O=O
Oc                 <   i }t        | j                               }d d}|D ]B  }|j                  d      rt        j                  d|      j                  d      }d| }d|v r|dz  }nOd|v r|d	z  }nEd
|v rd|vr || ||d| dd| dd| dg       n"d|v rd|vr || ||d| dd| dd| dg       d|v r|dz  }nd|v r|dz  }n}|j                  d      rjt        j                  d|      j                  d      }d| }d|v r|dz  }n"d|v rd|vr || ||d| dd| dd| dg       d|v r|dz  }nd|v r|dz  }n|}d|vs/| j                  |      ||<   E t        |       dkD  r&t        dt        | j                                d      |S )!Nc           
      &   | j                  |      }| j                  |j                  dd            }t        |      }||j                  d   |z  g|z  }nt	        |      |j                  d   k(  sJ |D cg c]  }|dz   	 }	}|D cg c]  }|dz   	 }
}|j                  t        j                  |	|             |j                  t        |
t        j                  ||d            D ci c]  \  }}||
 c}}       y c c}w c c}w c c}}w )Nz.down.weightz
.up.weightr   r   r   r	   )r7   rt   ry   r   r   rw   r   r   r   r   r:   )r   r   r   r   r   r   r   r   r`   r   r   r   s               r   
handle_qkvz9_convert_xlabs_flux_lora_to_diffusers.<locals>.handle_qkv  s   jj)JJw~|LM	 ]
<OOA&*45
BDt9	 2222 8@@!--@@5=>q++>> 	dmmM;?@ 	KYPTZ[9\(]^1q!t^_ A> _s   8D
D-D
)zdiffusion_model.double_blocksdouble_blockszdouble_blocks\.(\d+)r.   r   zprocessor.proj_lora1r   zprocessor.proj_lora2r   zprocessor.qkv_lora2upr   r   r   zprocessor.qkv_lora1r   r   r   downr   r   )zdiffusion_model.single_blockssingle_blockszsingle_blocks\.(\d+)r   	proj_lorar   qkv_lorarD  r   z2`old_state_dict` should be at this point but has: r3   r   )	r4   r5   rr   r   r[  r\  r7   ry   r<   )old_state_dictrJ   	orig_keysr  old_key	block_numrX   s          r   %_convert_xlabs_flux_lora_to_diffusersr    sn   N^((*+I`*  HBPQ		"97CII!LI7	{CG%0++'72--&'1d'6I""9)DTU9)DTU9)DTU		 ''1d'6I""9)JO9)JO9)JO		  ++++  RS		"97CII!LI>ykJGg%;&w&4w+>""@:V@:V@:V		  ++++ G &4&8&8&AN7#QHBT >QMdSaSfSfShNiMjjklmmr   r"   
substringsreturnc                     ddj                  d |D              z   dz   }t        j                  ||       }|rG|j                         }|dkD  r| |dz
     dk(  r|dz
  }n|}| d | j	                  dd      }| |d  }||z   S | j	                  dd      S )	N(|c              3   F   K   | ]  }t        j                  |        y wr   )r   escape)r    r   s     r   r#   z"_custom_replace.<locals>.<genexpr>  s     BRYYs^Bs   !)r   r.   r3   r   )r?   r   r[  startrt   )r"   r  r   rc  	start_subboundaryleftrights           r   rz  rz    s     CHHBzBBBSHGIIgs#EKKM	q=SQ/36 1}H H9H~%%c3/HIe|{{3$$r   c                 J   i }t        | j                               }d}d}d}d}dD ]  }| j                  d| d      |d| d<   d| d	|v r| j                  d| d	      |d| d	<   | j                  d
| d      |d| d<   d
| d	|v r| j                  d
| d	      |d| d	<   | j                  d| d      |d| d<   d| d	|v r| j                  d| d	      |d| d	<   | j                  d| d      |d| d<   d| d	|v r| j                  d| d	      |d| d	<   t        d | D              }|r| j                  d| d      |d| d<   d| d	|v r| j                  d| d	      |d| d	<   | j                  d| d      |d| d<   d| d	|v r| j                  d| d	      |d| d	<   | j                  d| d      |d| d<   d| d	|v r| j                  d| d	      |d| d	<   | j                  d| d      |d| d<   d| d	|v s| j                  d| d	      |d| d	<    t	        |      D ]+  }	d|	 d}
dD ]  }| j                  d|	 d| d      ||
 d| d<   d|	 d| d	|v r!| j                  d|	 d| d	      ||
 d| d	<   | j                  d|	 d| d      ||
 d| d<   d|	 d| d	|v r!| j                  d|	 d| d	      ||
 d| d	<   |d k(  r| j                  d|	 d!| d      }t        j                  |g      ||
 d"| d<   t        j                  |g      ||
 d#| d<   t        j                  |g      ||
 d$| d<   | j                  d|	 d%| d      }t        j                  |g      ||
 d&| d<   t        j                  |g      ||
 d'| d<   t        j                  |g      ||
 d(| d<   nt        j                  | j                  d|	 d!| d      d)d*+      \  }}}t        j                  |g      ||
 d#| d<   t        j                  |g      ||
 d$| d<   t        j                  |g      ||
 d"| d<   t        j                  | j                  d|	 d%| d      d)d*+      \  }}}t        j                  |g      ||
 d&| d<   t        j                  |g      ||
 d'| d<   t        j                  |g      ||
 d(| d<   d|	 d!| d	|v rt        j                  | j                  d|	 d!| d	      d)d*+      \  }}}t        j                  |g      ||
 d#| d	<   t        j                  |g      ||
 d$| d	<   t        j                  |g      ||
 d"| d	<   d|	 d%| d	|v rt        j                  | j                  d|	 d%| d	      d)d*+      \  }}}t        j                  |g      ||
 d&| d	<   t        j                  |g      ||
 d'| d	<   t        j                  |g      ||
 d(| d	<   | j                  d|	 d,| d      ||
 d-| d<   d|	 d,| d	|v r!| j                  d|	 d,| d	      ||
 d-| d	<   | j                  d|	 d.| d      ||
 d/| d<   d|	 d.| d	|v r!| j                  d|	 d.| d	      ||
 d/| d	<   | j                  d|	 d0| d      ||
 d1| d<   d|	 d0| d	|v r!| j                  d|	 d0| d	      ||
 d1| d	<   | j                  d|	 d2| d      ||
 d3| d<   d|	 d2| d	|v r!| j                  d|	 d2| d	      ||
 d3| d	<   | j                  d|	 d4| d      ||
 d5| d<   d|	 d4| d	|v r!| j                  d|	 d4| d	      ||
 d5| d	<   | j                  d|	 d6| d      ||
 d7| d<   d|	 d6| d	|v s| j                  d|	 d6| d	      ||
 d7| d	<    | j                  d|	 d8      ||
 d9<   | j                  d|	 d:      ||
 d;<   | j                  d|	 d<      ||
 d=<   | j                  d|	 d>      ||
 d?<   . t	        |      D ]  }	d@|	 d}
dD ]X  }| j                  dA|	 dB| d      ||
 dC| d<   dA|	 dB| d	|v r!| j                  dA|	 dB| d	      ||
 dC| d	<   t        ||z        }||||f}|d k(  r6| j                  dA|	 dD| d      }t        j                  |g      ||
 d#| d<   t        j                  |g      ||
 d$| d<   t        j                  |g      ||
 d"| d<   t        j                  |g      ||
 dE| d<   dA|	 dD| d	|v r| j                  dA|	 dD| d	      }t        j                  |g      ||
 d#| d	<   t        j                  |g      ||
 d$| d	<   t        j                  |g      ||
 d"| d	<   t        j                  |g      ||
 dE| d	<   nit        j                  | j                  dA|	 dD| d      |d*+      \  }}}} t        j                  |g      ||
 d#| d<   t        j                  |g      ||
 d$| d<   t        j                  |g      ||
 d"| d<   t        j                  | g      ||
 dE| d<   dA|	 dD| d	|v rt        j                  | j                  dA|	 dD| d	      |d*+      \  }!}"}#}$t        j                  |!g      ||
 d#| d	<   t        j                  |"g      ||
 d$| d	<   t        j                  |#g      ||
 d"| d	<   t        j                  |$g      ||
 dE| d	<   | j                  dA|	 dF| d      ||
 dG| d<   dA|	 dF| d	|v s8| j                  dA|	 dF| d	      ||
 dG| d	<   [ | j                  dA|	 dH      ||
 d9<   | j                  dA|	 dI      ||
 d;<    dD ]  }| j                  dJ| d      |dG| d<   dJ| d	|v r| j                  dJ| d	      |dG| d	<   t        | j                  dK| d            |dL| d<   dK| d	|v sqt        | j                  dK| d	            |dL| d	<    t        |       d*kD  rt        dM| j                         d      t        |j                               D ]  }%|j                  |%      |dN|% <    |S )ONr   r   r         @r   ztime_in.in_layer..weightz+time_text_embed.timestep_embedder.linear_1..biasztime_in.out_layer.z+time_text_embed.timestep_embedder.linear_2.zvector_in.in_layer.z'time_text_embed.text_embedder.linear_1.zvector_in.out_layer.z'time_text_embed.text_embedder.linear_2.c              3   $   K   | ]  }d |v  
 yw)guidanceNr   r_   s     r   r#   z>_convert_bfl_flux_control_lora_to_diffusers.<locals>.<genexpr>H  s     Hq:?Hr  zguidance_in.in_layer.z+time_text_embed.guidance_embedder.linear_1.zguidance_in.out_layer.z+time_text_embed.guidance_embedder.linear_2.ztxt_in.zcontext_embedder.zimg_in.zx_embedder.r5  r3   double_blocks..img_mod.lin.norm1.linear..txt_mod.lin.norm1_context.linear.r   .img_attn.qkv.
attn.to_v.
attn.to_q.
attn.to_k..txt_attn.qkv.attn.add_q_proj.attn.add_k_proj.attn.add_v_proj.   r   r	   .img_mlp.0.ff.net.0.proj..img_mlp.2.	ff.net.2..txt_mlp.0.ff_context.net.0.proj..txt_mlp.2.ff_context.net.2..img_attn.proj.attn.to_out.0..txt_attn.proj.attn.to_add_out.z.img_attn.norm.query_norm.scalezattn.norm_q.weightz.img_attn.norm.key_norm.scalezattn.norm_k.weightz.txt_attn.norm.query_norm.scalezattn.norm_added_q.weightz.txt_attn.norm.key_norm.scalezattn.norm_added_k.weightr3  single_blocks..modulation.lin.norm.linear.	.linear1.	proj_mlp.	.linear2.	proj_out.z.norm.query_norm.scalez.norm.key_norm.scalefinal_layer.linear.zfinal_layer.adaLN_modulation.1.znorm_out.linear.W`original_state_dict` should be empty at this point but has original_state_dict.keys()=rX  )r4   r5   r7   r6   r   r   r   r   r9   r:   r   ry   r<   )&original_state_dictconverted_state_dictoriginal_state_dict_keys
num_layersnum_single_layers	inner_dim	mlp_ratior  has_guidancerR   block_prefixsample_lora_weightcontext_lora_weightsample_qsample_ksample_v	context_q	context_k	context_vsample_q_biassample_k_biassample_v_biascontext_q_biascontext_k_biascontext_v_biasmlp_hidden_dim
split_sizelora_weight	lora_biasqr`   r   r   q_biask_biasv_biasmlp_biasr"   s&                                         r   +_convert_bfl_flux_control_lora_to_diffusersr    s0   #$7$<$<$>?JII( Bu  ##&7z$IJ 	J8*T[\] xj.2JJ#''*;H:U(KL !#NxjX]!^_
  ##&8
'$JK 	J8*T[\]  z/3KK#''*<XJe(LM !#NxjX]!^_
 ]p\s\s!(73]
FxjPWXY !
%04LL^q^u^u%hZu5_ #J8*TY!Z[ ]p\s\s"8*G4]
FxjPWXY "(515MM^q^u^u&xj6_ #J8*TY!Z[
 H4GHH#''*?z(QR !#NxjX_!`a 'xj6:RR'++.CH:U,ST %'RS[R\\a%bc
 $''*@
'(RS !#NxjX_!`a (z7;SS'++.DXJe,TU %'RS[R\\a%bc
 GZF]F]hZw'G
0
'BC XJe$(@@H[H_H_(5)I #4XJe!DE
 AT@W@WZabjakkrXs@t{8*G<=XJe$(@@BUBYBY\cdlcmmrZsBt ;xj!>?EBuJ : B
,QCq1, q	HTgTkTk =
'BU L>xj!PQ  s-z?C[[ViVmVm$QC}XJeDW$~]8*E%RS ]p\s\s =
'B] L>1FxjPW!XY  s-z?C[[^q^u^u$QC}XJeD_$~5J8*TY%Z[
 8#%8%<%<~aSP^_g^hho=p%q"UZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QR&9&=&=qcQ_`h_iip>q&r#[`[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX 05{{'++nQC~hZW^,_`bcij0,(H V[U^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QR27++'++nQC~hZW^,_`bcij3/	9i \a[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WXs.
%@D\\>Ckk'++nQC~hZW\,]^`agh?;}m TYS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPs.
%@D\\AF'++nQC~hZW\,]^`aghB> Z_YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UV ViUlUl ;xj@V L>z!QR  s+hZu=AYYWjWnWn$QC{8*EBX$~^H:U%ST QdPgPg ;xj@Q L>8*G!LM  s+hZu=AYYReRiRi$QC{8*EBS$~Yxj%NO ^q]t]t ;xj@^ L>1GzQX!YZ  s+hZu=AYY_r_v_v$QC{8*EB`$~5KH:UZ%[\ YlXoXo ;xj@Y L>1B8*G!TU  s+hZu=AYYZmZqZq$QC{8*EB[$~5FxjPU%VW
 ViUlUl ?8*GDV L>z!QR  s/(5AE]]WjWnWn$QCxjFX$~^H:U%ST XkWnWn ?8*GDX L>1A(7!ST  s/(5AE]]YlYpYp$QCxjFZ$~5EhZu%UV_q	h EXD[D[QC>?E
~-?@A EXD[D[QC<=E
~-?@A K^JaJaQC>?K
~-EFG K^JaJaQC<=K
~-EFGAB
J $% ?
3A3a8, 4	HSfSjSj #3H:WET L>hZw!OP  s"28*EBF^^UhUlUl$QC'7zGV$~\(5%QR
 !Y!67N#Y	>JJ8#155qcS[R\\c6deUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRTYT]T]_j^kTl$~Yxj%PQ#A3iz?C[[ 3 7 7.9U]T^^c8d eIW\W`W`bkalWm(L>H:U)STW\W`W`bkalWm(L>H:U)STW\W`W`bkalWm(L>H:U)STV[V_V_aj`kVl(L>8*E)RS${{'++nQCy
RY,Z[]gmn 1a V[U^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRTYT]T]_b^cTd$~Yxj%PQ#A3iz?C[[7<{{+//.9XJV[0\]_iop84FFFH X]W`W`bhaiWj(L>H:U)STW\W`W`bhaiWj(L>H:U)STW\W`W`bhaiWj(L>H:U)STV[V_V_ai`jVk(L>8*E)RS QdPgPg 9XJg>Q L>8*G!LM  s)H:U;?WWReRiRi$QCy
%@S$~Yxj%NOe4	n EXD[D[QC56E
~-?@A EXD[D[QC34E
~-?@A{?
B ) >Q>U>U!(73?
y
':; !
%04LL@S@W@W%hZu5A 9XJe!<= FV##&EhZw$WXF
/zAB -XJe<@XXGW#''*I(SX(YZH #3H:U!CD" !#sXkXpXpXrWttuvww(--/0 S5I5M5Mc5R|C512S  r   c                 t   i }t        | j                               }d}d}d}d}t        |      D ]  }d| d}d}	dD ]  }
| j                  |	 d	| d
|
 d      || d|
 d<   d	| d
|
 d|v r#| j                  |	 d	| d
|
 d      || d|
 d<   | j                  |	 d	| d|
 d      || d|
 d<   |
dk(  r| j                  |	 d	| d|
 d      }t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   | j                  |	 d	| d|
 d      }t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   n"t	        j                  | j                  |	 d	| d|
 d      dd      \  }}}t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j                  | j                  |	 d	| d|
 d      dd      \  }}}t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   d	| d|
 d|v rt	        j                  | j                  |	 d	| d|
 d      dd      \  }}}t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   d	| d|
 d|v rt	        j                  | j                  |	 d	| d|
 d      dd      \  }}}t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   | j                  |	 d	| d|
 d      || d|
 d<   |	 d	| d|
 d|v r#| j                  |	 d	| d|
 d      || d|
 d<   | j                  |	 d	| d|
 d      || d|
 d<   |	 d	| d|
 d|v r#| j                  |	 d	| d|
 d      || d|
 d<   | j                  |	 d	| d |
 d      || d!|
 d<   |	 d	| d |
 d|v r#| j                  |	 d	| d |
 d      || d!|
 d<   | j                  |	 d	| d"|
 d      || d#|
 d<   |	 d	| d"|
 d|v r#| j                  |	 d	| d"|
 d      || d#|
 d<   | j                  |	 d	| d$|
 d      || d%|
 d<   |	 d	| d$|
 d|v r#| j                  |	 d	| d$|
 d      || d%|
 d<   | j                  |	 d	| d&|
 d      || d'|
 d<   |	 d	| d&|
 d|v s| j                  |	 d	| d&|
 d      || d'|
 d<     t        |      D ]}  }d(| d}dD ]n  }
| j                  	 d)| d*|
 d      || d+|
 d<   |	 d)| d*|
 d|v r#| j                  |	 d)| d*|
 d      || d+|
 d<   t        ||z        }||||f}|
dk(  r:| j                  |	 d)| d,|
 d      }t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d-|
 d<   |	 d)| d,|
 d|v r| j                  d)| d,|
 d      }t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d-|
 d<   not	        j                  | j                  |	 d)| d,|
 d      |d      \  }}}} t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  |g      || d|
 d<   t	        j
                  | g      || d-|
 d<   |	 d)| d,|
 d|v rt	        j                  | j                  |	 d)| d,|
 d      |d      \  }!}"}#}$t	        j
                  |!g      || d|
 d<   t	        j
                  |"g      || d|
 d<   t	        j
                  |#g      || d|
 d<   t	        j
                  |$g      || d-|
 d<   | j                  |	 d)| d.|
 d      || d/|
 d<   |	 d)| d.|
 d|v sL| j                  |	 d)| d.|
 d      || d/|
 d<   q  dD ]I  }
| j                  	 d0|
 d      |d/|
 d<   |	 d0|
 d|v s,| j                  |	 d0|
 d      |d/|
 d<   K t        |       dkD  rt        d1| j                         d      t        |j                               D ]  }%|j                  |%      |d2|% <    |S )3Nr   r   r   r  r5  r3   zbase_model.model.r   r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r   r	   r  r  r  r  r  r  r  r  r  r  r  r  r3  r  r  r  r  r  r  r  r  r  rX  )r4   r5   r   r7   r   r   r   r9   r:   ry   r<   )&r  r  r  r  r  r  r  rR   r  original_block_prefixr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r`   r   r   r  r  r  r  r"   s&                                         r   &_convert_fal_kontext_lora_to_diffusersr  I  s   #$7$<$<$>?JII : A,QCq1 3, }	HTgTkTk()s-zQXYU L>xj!PQ  s-z?C[[ViVmVm,-^A3mH:UZ[W$~]8*E%RS ]p\s\s()s-zQXY] L>1FxjPW!XY
 8#%8%<%<,-^A3nXJV]^&" V[U^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QRUZU^U^`r_sUt$~Zz%QR&9&=&=,-^A3nXJV]^'# \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX \a[d[d()\$~5EhZw%WX 05{{'++01s.QYPZZab 0,(H V[U^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QRUZU^U^`h_iUj$~Zz%QR27++'++01s.QYPZZab 3/	9i \a[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WX[`[d[dfoep[q$~5EhZw%WXs.
%@D\\>Ckk'++/D.E^TUSVVdemdnns,tu?;}m
 TYS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPSXS\S\^k]lSm$~Zz%OPs.
%@D\\AF'++/D.E^TUSVVdemdnns,tuB>
 Z_YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UVY^YbYbdrcsYt$~5EhZu%UV ViUlUl()s+hZwWV L>z!QR ((qcXJeTXppWjWnWn,-^A3k(SXYX$~^H:U%ST QdPgPg()s+hZwWQ L>8*G!LM ((qcXJeTXppReRiRi,-^A3k(SXYS$~Yxj%NO ^q]t]t()s+hZwW^ L>1GzQX!YZ ((qcXJeTXpp_r_v_v,-^A3k(SXY`$~5KH:UZ%[\ YlXoXo()s+hZwWY L>1B8*G!TU ((qcXJeTXppZmZqZq,-^A3k(SXY[$~5FxjPU%VW
 ViUlUl()s/(SZ[V L>z!QR ((qc
RWX\ttWjWnWn,-^A3ohZW\]X$~^H:U%ST XkWnWn()s/(SZ[X L>1A(7!ST ((qc
RWX\ttYlYpYp,-^A3ohZW\]Z$~5EhZu%UVw}		AH $% =3A3a8, :	HSfSjSj()s:J8*T[\T L>hZw!OP ((qc9I(SXY]uuUhUlUl,-^A3>NxjX]^V$~\(5%QR
 !Y!67N#Y	>JJ8#155,-^A3izQXY V[U^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRUZU^U^`k_lUm$~Zz%QRTYT]T]_j^kTl$~Yxj%PQ+,N1#YxjPUVZrr 3 7 7.9U]T^^c8d eIW\W`W`bkalWm(L>H:U)STW\W`W`bkalWm(L>H:U)STW\W`W`bkalWm(L>H:U)STV[V_V_aj`kVl(L>8*E)RS${{'++/D.E^TUSVV_`h_iip,qr 1a
 V[U^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRUZU^U^`a_bUc$~Zz%QRTYT]T]_b^cTd$~Yxj%PQ+,N1#YxjPUVZrr7<{{+//3H2IXYWZZcdlcmmr0st"84FFFH
 X]W`W`bhaiWj(L>H:U)STW\W`W`bhaiWj(L>H:U)STW\W`W`bhaiWj(L>H:U)STV[V_V_ai`jVk(L>8*E)RS QdPgPg()s)H:WUQ L>8*G!LM ((qc8*ERVnnReRiRi,-^A3izQVWS$~Yxj%NOq:	=~ ) >Q>U>U$%%8
'J?
y
':; $$$7zGKcc@S@W@W())<XJeLA 9XJe!<= !#sXkXpXpXrWttuvww(--/0 S5I5M5Mc5R|C512S  r   c                    t        | j                               D ci c]  }|| j                  |       }}d }d }d }d }d }i dddd	d
dddddddddddddddddddddd d!d"d#d$d%d&d'd(d)d%d*d+dd,d-d.d/d0d1d2d3}|||||d4}	t        |j                               D ]e  }
|
j                  d5      r |j                  |
      ||
t	        d5      d  <   |
j                  d6      sF|j                  |
      ||
t	        d6      d  <   g t        |j                               D ]E  }
|
d d  }|j                         D ]  \  }}|j                  ||      } |j                  |
      ||<   G t        |j                               D ](  }
|	j                         D ]  \  }}||
vr ||
|        * t        |j                               D ]  }
|j                  |
      |d5|
 <    |S c c}w )7Nc                     |j                  |       }|j                  dd      \  }}t        j                  ||gd      }||| j	                  dd      <   y )Nr   r   r	   final_layer.adaLN_modulation.1r9  )r7   r   r   r   rt   )r"   r@   r   r   r   r   s         r   remap_norm_scale_shift_zI_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_norm_scale_shift_*  sQ    $||A1|-uYYu~15
Wa
3;;?ARSTr   c                 :   d }d| v rz|j                  |       }|j                  dd      \  }}}|| || j                  dd            <   || || j                  dd            <   || || j                  dd            <   y |j                  |       | ||       <   y )	Nc                    | j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d	d
      }|j                  dd      }|j                  dd      }|S )Nzindividual_token_refiner.blocksztoken_refiner.refiner_blockszadaLN_modulation.1r9  r  r7  zt_embedder.mlp.0*time_text_embed.timestep_embedder.linear_1zt_embedder.mlp.2*time_text_embed.timestep_embedder.linear_2
c_embedderztime_text_embed.text_embedderr   r   )rt   )r"   rX   s     r   
rename_keyzS_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_in_.<locals>.rename_key1  s    kk"CEcdGoo&:<MNGooh0BCGoo&8:fgGoo&8:fgGool4STGooeT2GNr   self_attn_qkvr  r   r	   	attn.to_q	attn.to_k	attn.to_v)r7   r   rt   )r"   r@   r  r   rF  rG  rH  s          r   remap_txt_in_z?_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_in_0  s    	 c!^^C(F%||A1|5D$PTJz#++o{"KLMPTJz#++o{"KLMPTJz#++o{"KLM*4..*=Jz#'r   c                 Z   |j                  |       }d| v r@||| j                  dd      <   ||| j                  dd      <   ||| j                  dd      <   y |j                  dd      \  }}}||| j                  dd      <   ||| j                  dd      <   ||| j                  dd      <   y )	Nr   img_attn_qkvr  r  r  r  r   r	   r7   rt   r   r"   r@   r   rF  rG  rH  s         r   remap_img_attn_qkv_zE_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_img_attn_qkv_D  s    $s?CIJs{{>;?@CIJs{{>;?@CIJs{{>;?@%||A1|5D$CGJs{{>;?@CGJs{{>;?@CGJs{{>;?@r   c                 Z   |j                  |       }d| v r@||| j                  dd      <   ||| j                  dd      <   ||| j                  dd      <   y |j                  dd      \  }}}||| j                  dd      <   ||| j                  dd      <   ||| j                  dd      <   y )	Nr   txt_attn_qkvzattn.add_q_projzattn.add_k_projzattn.add_v_projr  r   r	   r   r  s         r   remap_txt_attn_qkv_zE_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_txt_attn_qkv_P  s    $s?IOJs{{>3DEFIOJs{{>3DEFIOJs{{>3DEF%||A1|5D$IMJs{{>3DEFIMJs{{>3DEFIMJs{{>3DEFr   c                 $   d}d| v sd| v r|j                  |       }d| v rB| j                  dd      j                  d      }||| d<   ||| d	<   ||| d
<   ||| d<   y ||||j                  d      d|z  z
  f}t	        j
                  ||d      \  }}}}	| j                  dd      j                  d      }||| d<   ||| d<   ||| d<   |	|| d<   y d| v sd| v r|j                  |       }
d| v rB| j                  dd      j                  d      }|
|| d<   |
|| d<   |
|| d<   |
|| d<   y ||||
j                  d      d|z  z
  f}t	        j
                  |
|d      \  }}}}| j                  dd      j                  d      }||| d<   ||| d<   ||| d<   ||| d<   y | j                  dd      }|j                  d d!      }|j                  d"d#      }|j                  d$d%      }|j                  |       ||<   y )&Nr   zlinear1.lora_A.weightzlinear1.lora_B.weightr   r  single_transformer_blocksz.linear1.lora_A.weightz.attn.to_q.lora_A.weightz.attn.to_k.lora_A.weightz.attn.to_v.lora_A.weightz.proj_mlp.lora_A.weightr   r  r	   z.linear1.lora_B.weightz.attn.to_q.lora_B.weightz.attn.to_k.lora_B.weightz.attn.to_v.lora_B.weightz.proj_mlp.lora_B.weightzlinear1.lora_A.biaszlinear1.lora_B.biasz.linear1.lora_A.biasz.attn.to_q.lora_A.biasz.attn.to_k.lora_A.biasz.attn.to_v.lora_A.biasz.proj_mlp.lora_A.biasz.linear1.lora_B.biasz.attn.to_q.lora_B.biasz.attn.to_k.lora_B.biasz.attn.to_v.lora_B.biasz.proj_mlp.lora_B.biaslinear2r   q_normattn.norm_qk_normattn.norm_k)r7   rt   removesuffixsizer   r:   )r"   r@   hidden_sizelinear1_weightrX   r  r  r`   r   r   linear1_biasr  r  r  r  s                  r    remap_single_transformer_blocks_zR_convert_hunyuan_video_lora_to_diffusers.<locals>.remap_single_transformer_blocks_\  s   "c)-D-K'^^C0N3++o7RS``, DR
gY&>?@CQ
gY&>?@CQ
gY&>?@BP
gY&=>?);^EXEXYZE[^_bm^mEmn
${{>:1M1a++o7RS``, DE
gY&>?@CD
gY&>?@CD
gY&>?@BE
gY&=>?"c)-Bc-I%>>#.L3++o7RS``* BN
gY&<=>AM
gY&<=>AM
gY&<=>@L
gY&;<=);\EVEVWXEY\]`k\kEkl
38;;|Z]^3_0++o7RS``* BH
gY&<=>AG
gY&<=>AG
gY&<=>@H
gY&;<= kk/3NOGooi<Gooh>Gooh>G",.."5Jwr   r  r<  ztime_in.mlp.0r  ztime_in.mlp.2r  zguidance_in.mlp.0z*time_text_embed.guidance_embedder.linear_1zguidance_in.mlp.2z*time_text_embed.guidance_embedder.linear_2zvector_in.in_layerz&time_text_embed.text_embedder.linear_1zvector_in.out_layerz&time_text_embed.text_embedder.linear_2r  r   img_attn_q_normr
  img_attn_k_normr  img_attn_projzattn.to_out.0txt_attn_q_normzattn.norm_added_qtxt_attn_k_normzattn.norm_added_ktxt_attn_projzattn.to_add_outzimg_mod.linearznorm1.linear	img_norm1z
norm1.norm	img_norm2norm2r   znorm1_context.linearnorm2_context
ff_contextznorm.linearz	norm.normznorm_out.normr   
net.0.projnet.2r   )img_mlpztxt_mod.linear	txt_norm1	txt_norm2txt_mlpself_attn_projzmodulation.linearpre_normzfinal_layer.norm_finalzfinal_layer.linearrS  rT  input_embedder)r  r  r  r  r  rX  rk  )r4   r5   r7   rr   ry   r|   rt   )r  r`   r  r  r  r  r  r  TRANSFORMER_KEYS_RENAME_DICTTRANSFORMER_SPECIAL_KEYS_REMAPr"   rX   replace_keyr  special_keyhandler_fn_inplaces                   r   (_convert_hunyuan_video_lora_to_diffusersr,  '  s	   CGH[H`H`HbCcdaA266q99ddb>(
H
N26h$,$E$ 	E$ 	I	$
 	I$ 	F$ 	G$ 	-$ 	=$ 	=$ 	$ 	.$ 	.$ 	*$ 	.$  	\!$" 	W#$$ 0!$)*"1(#=$ D  ++9*A&" (--/0 a>>.)?S?W?WX[?\ S%8%:!;<>>,-CWC[C[\_C` S);%<%>!?@	a (--/0 Fa&'C'I'I'K 	?#Kook:>G	?(<(@(@(EW%	F (--/0 :/M/S/S/U 	:+K+#%s$89	:: (--/0 S5I5M5Mc5R|C512S  W es   G0c                      j                         D ci c]  \  }}|t        d      d  | c}} i d } fd}d} | j                         |      }t        |      D ]  } |d|d        d} | j                         |      }	t        |	      D ]  } |d	|d
        d}
 | j                         |
      }t        |      D ]  } |d|d        t               dkD  rt	        d j                               t        j                               D ]  }j                  |      d| <    S c c}}w )Nrk  c                     t               }| D ]D  }t        j                  ||      }|s|j                  t	        |j                  d                   F t        |      S )Nr.   )r8   r   r[  r;   r9   r\  ry   )r5   r   layersr"   rc  s        r   get_num_layerszH_convert_non_diffusers_lumina2_lora_to_diffusers.<locals>.get_num_layers  sP     	0CIIgs+E

3u{{1~./	0 6{r   c           	         	j                  |  d| d      }	j                  |  d| d      }dD ]  }||  d| d| d<    t        g dt        j                  |g dd	            D ]  \  }}||  d| d| d
<    	j                  |  d| d      |  d| d<   	j                  |  d| d      |  d| d<   t	        dd      D ]N  }	j                  |  d| d| d      |  d| d| d<   	j                  |  d| d| d
      |  d| d| d
<   P |rA	j                  |  d| d      |  d| d<   	j                  |  d| d      |  d| d<   y y )Nr3   z.attention.qkv.lora_A.weightz.attention.qkv.lora_B.weightrE  rI  r   )i 	     r2  r   r	   r   z.attention.out.lora_A.weightz.attn.to_out.0.lora_A.weightz.attention.out.lora_B.weightz.attn.to_out.0.lora_B.weightr.      z.feed_forward.wz.feed_forward.linear_z!.adaLN_modulation.1.lora_A.weightz.norm1.linear.lora_A.weightz!.adaLN_modulation.1.lora_B.weightz.norm1.linear.lora_B.weight)r7   r   r   r:   r   )
r   indexconvert_normr  r  attn_keyr   rO   r  r@   s
           r   process_blockzG_convert_non_diffusers_lumina2_lora_to_diffusers.<locals>.process_block  s@   NNfXQug5Q#RS	..F81UG3O!PQ0 	`HV_ F81UG6(>!RS	` #$<ekk'Scij>k l 	]HfV\ F81UG6(>!RS	] Q[P^P^haw:;Q
xq/KLM Q[P^P^haw:;Q
xq/KLM
 1a[ 	Eblbpbp(!E7/%Gc F81UG3H~!^_ cmbpbp(!E7/%Gc F81UG3H~!^_		 S]SaSa(!E7"CDT F81UG3N!OP T^SaSa(!E7"CDT F81UG3N!OP	 r   znoise_refiner\.(\d+)\.noise_refinerT)r5  zcontext_refiner\.(\d+)\.context_refinerFzlayers\.(\d+)\.r/  r   E`state_dict` should be empty at this point but has state_dict.keys()=rX  )r|   ry   r5   r   r<   r4   r7   )r@   r`   r   r0  r7  noise_refiner_patternnum_noise_refiner_layersrR   context_refiner_patternnum_context_refiner_layerscore_transformer_patternnum_core_transformer_layersr"   r  s   `            @r   0_convert_non_diffusers_lumina2_lora_to_diffusersrA    sv   >H>N>N>PQda!C*+-.1QJ D 6-joo.?AVW+, =oqt<= :!/
0ACZ![-. @'?@  2"01BD\"]./ 6h56 :azO`Nbcdd(--/0 S5I5M5Mc5R|C512S  E Rs   D;c           
        $%& i }| j                         D ci c]  \  }}|t        d      d  | c}}%%D ch c]1  }|j                  d      st        |j	                  d      d         3 }}t        |      }t        |      }t        d %D              xr t        d %D              }t        d %D              rdnd	}t        d
 %D              rdnd}	t        d %D              }
%fd}t        %j                               D ]Z  }|j                  d      r.d|v r*%j                  |       t        j                  d| d       d|v sG|
rJ%j                  |       \ t        ||dz         D ]r  }t        g dg d      D ]  \  }}d| d| d}|%v }d| d| d| d}d| d| d}d| d| d|	 d}d| d| d}|r?%j                  |      }%j                  |      } |||      \  }}||z  ||<   ||z  ||<   n0|%v r%j                  |      ||<   |%v r%j                  |      ||<   d| d| d}d| d| d}|%v sŉ%j                  |      ||<    t        g dg d      D ]  \  }}d| d| d}|%v }d| d| d| d}d| d| d}d| d| d|	 d}d| d| d}|%v r%j                  |      }|||<   |%v r%j                  |      }|||<   |r& ||      \  }}||xx   |z  cc<   ||xx   |z  cc<   d| d| d}d| d| d}|%v s%j                  |      ||<    |rt        d d!gd"d#g      D ]  \  }}d| d| d}|%v }d| d| d| d}d| d| d}d| d| d|	 d}d| d| d}|%v r%j                  |      }|||<   |%v r%j                  |      }|||<   |r& ||      \  }}||xx   |z  cc<   ||xx   |z  cc<   d| d| d}d| d| d}|%v s%j                  |      ||<    t        d$d%gd&d'g      D ]  \  }}d| d| d}|%v }d| d| d| d}d| d(| d}d| d| d|	 d}d| d(| d}|%v r%j                  |      }|||<   |%v r%j                  |      }|||<   |r& ||      \  }}||xx   |z  cc<   ||xx   |z  cc<   d| d| d}d| d(| d}|%v s%j                  |      ||<    u %rt        d) %D              rXd*| d}d+}|%v r%j                  |      ||<   d*|	 d}d,}|%v r%j                  |      ||<   d-%v r%j                  d-      |d.<   t        d/ | D              rH%j                  d0| d      |d1<   %j                  d0|	 d      |d2<   d3%v r%j                  d3      |d4<   d5D ]  &t        &fd6%D              sd7D ]  $$d8k(  rdnd9}&d:k(  rd;nd<}t        $&fd=%D              rF%j                  & d$ d| d      || d>| d<   %j                  & d$ d|	 d      || d>| d<   & d$ d%v s{%j                  & d$ d      || d>| d<     d?D ]  \  }}| d| d}d@| d}|%v r%j                  |      ||<   | d|	 d}d@| d}|%v s?%j                  |      ||<   |j!                  d|	 d      dz   } | %v sp|j!                  d      dAz   }!%j                  |       ||!<    t        %      d8kD  rt#        dB %D              }"|"rO%D ch c]  }|j                  dC      s| }#}t#        dD |#D              st$        t        j'                  dE       nt%        dF%j                               t        |j                               D ]  }|j                  |      |dG| <    |S c c}}w c c}w c c}w )HNrk  blocks.r3   r.   c              3   $   K   | ]  }d |v  
 ywk_imgNr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>%       @qgl@r  c              3   $   K   | ]  }d |v  
 ywv_imgNr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>%       HsZ[TUHsr  c              3   $   K   | ]  }d |v  
 yw)r   Nr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>&  s     #OaHM#Or  r   r  c              3   $   K   | ]  }d |v  
 yw)r   Nr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>'  s     !MA(a-!Mr  r   r  c              3   d   K   | ](  }|j                  d       xr |j                  d       * yw)time_projectionr  N)rr   rq   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>(  s0      %FG&'AAJJy,AA%s   .0c                     | j                   d   }j                  |      j                         }||z  }|}d}|dz  |k  r|dz  }|dz  }|dz  |k  r||fS Nr   r   r   r   r7   rv   )r   r-  r   r   r   r   r   r  s          r   get_alpha_scaleszF_convert_non_diffusers_wan_lora_to_diffusers.<locals>.get_alpha_scales,  sx      ##''	2779
1nx'!OJMH 1nx' 8##r   )rv  rs  normz	Removing zG key from the state dict as it is a norm diff key. This is unsupported.rO  r  r`   r   orF  rG  rH  zto_out.0z.self_attn.rj   r  .attn1.r   r   rs  z.lora_B.biasz.cross_attn..attn2.rF  rJ  rM  rN  zffn.0zffn.2r  r  .ffn.c              3   $   K   | ]  }d |v  
 yw)rO  Nr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>  s     C! A%Cr  ztime_projection.1.z*condition_embedder.time_proj.lora_A.weightz*condition_embedder.time_proj.lora_B.weightztime_projection.1.diff_bz(condition_embedder.time_proj.lora_B.biasc              3   $   K   | ]  }d |v  
 yw)z	head.headNr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>  s     4A{a4r  z
head.head.zproj_out.lora_A.weightzproj_out.lora_B.weightzhead.head.diff_bzproj_out.lora_B.bias)text_embeddingtime_embeddingc              3   &   K   | ]  }|v  
 y wr   r   )r    r`   	text_times     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>  s     ?a9>?r%   )r   r   r   r   r]  z condition_embedder.text_embedderz condition_embedder.time_embedderc              3   0   K   | ]  } d  |v   ywr3   Nr   )r    r`   b_nr`  s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>  s!     Ri[#/14Rs   z.linear_))zff.net.0.projzimg_emb.proj.1)zff.net.2zimg_emb.proj.3z"condition_embedder.image_embedder.r  c              3   $   K   | ]  }d |v  
 yw)rv  Nr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>  s     =A7a<=r  rv  c              3   $   K   | ]  }d |v 
 yw)loraNr   r_   s     r   r#   z?_convert_non_diffusers_wan_lora_to_diffusers.<locals>.<genexpr>  s     :1vQ:r  zThe remaining `state_dict` contains `diff` keys which we do not handle yet. If you see performance issues, please file an issue: https://github.com/huggingface/diffusers//issues/newN`state_dict` should be empty at this point but has original_state_dict.keys()=rX  )r|   ry   rr   r9   r:   minmaxr6   r4   r5   rq   r7   rz   debugr   r   r  r   r<   r{   )'r@   r  r`   r   block_numbers	min_block	max_blockis_i2v_loralora_down_keylora_up_keyhas_time_projection_weightrS  r"   rR   rV  cr-  	has_alphaoriginal_key_Aconverted_key_Aoriginal_key_Bconverted_key_Br   r   r   r   r(  converted_keydiffusers_b_nr   img_ours
img_theirsbias_key_theirsbias_keydiff	diff_keysrc  r  r`  s'                                       @@@r   ,_convert_non_diffusers_wan_lora_to_diffusersr    s*   GQGWGWGYZtq!1S!3467:Z3Fba!,,W`JaSa)bMbM"IM"I@,?@@sSHs_rHsEsK ##O;N#O OHU`M!!M9L!MM(S\K!$ %K^% "	$ ',,./ )<<,-&C-  ##C(LL9SE)pqr#,F  ##C()  9i!m, i\,.RS 	\DAq!!Ks&9I!%88I&qcQCqwON 's'!NCO&qcQCqWMN 's'!NCO155nE/33NC	'7Y'O$
H8Cj8P$_58AH8L$_5 "%88<O<S<STb<c(9!%88<O<S<STb<c(9$QC{1#W=L%aSs,?M226I6M6Ml6[$]33	\8 ,.RS 	\DAq!!L6:I!%88I&qcaS-PN 's'!NCO&qcaS+gNN 's'!NCO!44155nE8C$_5!44/33NC	8A$_5'7Y'O$
H$_5C5$_5A5$QC|A3g>L%aSs,?M226I6M6Ml6[$]3/	\2 Wg.|0LM `1%aSQCv>	%)<<	#*1#\!Am_G!T$+A3gaS"G#*1#\!Ak]'!R$+A3gaS"G!%88"5"9"9."IK<G(9!%88 3 7 7 GI<E(9+;K+S(J(9ZG9(9XE9!(<s'B")!GA3l C#66:M:Q:QR^:_(7/`4 '*\7,CD 	\DAq!!AaS/I!%88I&qc1#Q}oWEN 's%s.AO&qc1#Q{m7CN 's%s.AO!44155nE8C$_5!44/33NC	8A$_5'7Y'O$
H$_5C5$_5A5$QCq73L%aSaS=M226I6M6Ml6[$]3/	\ei\X C/BCC/gFLHM226I6M6Ml6[$]3/}GDLHM226I6M6Ml6[$]3)-@@SfSjSj.T$%OP 444=P=T=T]O73> !9: >Q=T=TWabmannuUv=w !9:!%88?R?V?VWi?j$%;<= 	I?+>??! C),AqM %(88 ;? #
 R>QRR/33yk3%qW^4_` -/?xVd-ef 033yk3%qU\4]^ -/?xVd-ef $AcU'26II/33yk3%w4OP -/?xVb-cd	*%
 	^ Hj )\=/AL@
.YM226I6M6Ml6[$]3(\;-w?L@
.YM226I6M6Ml6[$]3".";";a}G<T"UXa"a"&99,99)DwNH5H5L5L_5](2!	^$ !#=)<==$7Oq1::g;NOIO:	::  KKG
 nSfSkSkSmRopqq(--/0 S5I5M5Mc5R|C512S  I [bj Ps   ___0__c           	         i }| j                         D ci c]  \  }}|t        d      d  | c}}t        D ch c](  }|j                  d      d   j                  d      d   * c}      }t        d D              xr t        d D              }fd}t	        |      D ]  }t        g d	g d
      D ]e  \  }}	j                  d| d| d      }
j                  d| d| d      } ||
d| d|       \  }}|
|z  |d| d|	 d<   ||z  |d| d|	 d<   g t        g d	g d
      D ]e  \  }}	j                  d| d| d      }
j                  d| d| d      } ||
d| d|       \  }}|
|z  |d| d|	 d<   ||z  |d| d|	 d<   g |rxt        ddgddg      D ]e  \  }}	j                  d| d| d      }
j                  d| d| d      } ||
d| d|       \  }}|
|z  |d| d|	 d<   ||z  |d| d|	 d<   g t        ddgddg      D ]e  \  }}	j                  d| d| d      }
j                  d| d| d      } ||
d| d|       \  }}|
|z  |d| d|	 d<   ||z  |d| d|	 d<   g  t              dkD  rt        dj                               t        |j                               D ]  }|j                  |      |d| <    |S c c}}w c c}w )Nr^   blocks_r.   r   r   c              3   $   K   | ]  }d |v  
 ywrE  r   r_   s     r   r#   z8_convert_musubi_wan_lora_to_diffusers.<locals>.<genexpr>  rG  r  c              3   $   K   | ]  }d |v  
 ywrI  r   r_   s     r   r#   z8_convert_musubi_wan_lora_to_diffusers.<locals>.<genexpr>  rK  r  c                     | j                   d   }j                  |dz         j                         }||z  }|}d}|dz  |k  r|dz  }|dz  }|dz  |k  r||fS )Nr   rj   r   r   rR  )r   r"   r   r   r   r   r   r  s          r   rS  z?_convert_musubi_wan_lora_to_diffusers.<locals>.get_alpha_scales  s|      ##''h7<<>
1nx'!OJMH 1nx' 8##r   rU  rW  _self_attn_r   ri   rC  rX  r   r   _cross_attn_rY  rF  rJ  rM  rN  ffn_0ffn_2r  r  rZ  rg  rX  )
r|   ry   r:   r6   r   r   r7   r<   r5   r4   )r@   r  r`   r   
num_blocksrn  rS  rR   rV  rr  r   r   r   r   r"   r  s                  @r   %_convert_musubi_wan_lora_to_diffusersr    s6   AKAQAQASTA1S.0114TCVWaaggi(+11#6q9WXJ@,?@@sSHs_rHsEsK	$ : ],.RS 	_DAq-11GA3k!L]2^_K+//'!Ks/0Z[I#3K71#[YZX[A\#] JJUXbJb 71#WQC~!FGJSV^J^ 71#WQC~!FG	_ ,.RS 	_DAq-11GA3l1#M^2_`K+//'!L?0[\I#3K71#\Z[Y\A]#^ JJUXbJb 71#WQC~!FGJSV^J^ 71#WQC~!FG	_ Wg.|0LM c1155s,qcQb6cd/33gaSQC4_`	'7wqcQ]^_]`Ea'b$
HNY\fNf$wqc>%JKNWZbNb$wqc>%JKc '*\7,CD 	]DAq-11GA3asBS2TUK+//'!AaS0PQI#3K71#QqcAR#S JHSV`H` 71#U1#^!DEHQT\H\ 71#U1#^!DE	]5]B !#jObOgOgOiNklmm(--/0 S5I5M5Mc5R|C512S  o UWs
   K1-K7c                    t        fd| D              st        d      | j                         D ci c]  \  }}|j                   d      | }}}|j                         D ci c]  \  }}d| | }}}|S c c}}w c c}}w )Nc              3   @   K   | ]  }|j                          y wr   r!  r    r`   non_diffusers_prefixs     r   r#   zC_convert_non_diffusers_hidream_lora_to_diffusers.<locals>.<genexpr>D  s     Faq||01Fs   z$Invalid LoRA state dict for HiDream.r3   rX  r   r<   r|   removeprefixr@   r  r`   r   r  s    `   r   0_convert_non_diffusers_hidream_lora_to_diffusersr  C  s    F:FF?@@V`VfVfVhidaQRANN.B-C1+EFIii>R>X>X>Z[dal1#.1[[ j[   A>)Bc                    t        fd| D              st        d      | j                         D ci c]  \  }}|j                   d      | }}}|j                         D ci c]  \  }}d| | }}}|S c c}}w c c}}w )Nc              3   F   K   | ]  }|j                   d         ywrb  r!  r  s     r   r#   z@_convert_non_diffusers_ltxv_lora_to_diffusers.<locals>.<genexpr>L  s#     LAq||34A67Ls   !z&Invalid LoRA state dict for LTX-Video.r3   rX  r  r  s    `   r   -_convert_non_diffusers_ltxv_lora_to_diffusersr  K  s    LLLABBV`VfVfVhidaQRANN.B-C1+EFIii>R>X>X>Z[dal1#.1[[ j[r  c                 p    t        d  D              }|rm j                         D ci c]  \  }}|j                  d      | c}} dt        dt        fd} j                         D ci c]  \  }} ||      | c}} i }t	         j                               }d}d} fd}	|D ]  }|j                  |      s|j                  |d	      }
|j                  ||      j                  |d
      }|j                  |d      } j                  |      } j                  |j                  ||            } |	||      \  }}||z  ||
<   ||z  ||<    t               dkD  rt        d j                               |j                         D ci c]  \  }}d| | }}}|S c c}}w c c}}w c c}}w )Nc              3   >   K   | ]  }|j                  d         ywrW  r!  r_   s     r   r#   z@_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.<genexpr>T  s     Gq\2Gr"  r^   r"   r  c           	         d}d| v r| j                  dd      \  }}n| d}}| d}|t        |      d  }d|v r|j                  dd      \  }}d|z   }n|d}}h d}i }	|D ]4  }
|	j                  t        |
      t	                     j                  |
       6 |j                  d      }g }d}t        |	j                         d	      }|t        |      k  rd
}|D ]Y  }||z   t        |      k  st        ||||z          |	|   v s,|j                  dj                  ||||z                 ||z  }d} n |s|j                  ||          |dz  }|t        |      k  rdj                  |      }| d| | }||rd|z   z   S dz   S )Nr   r3   r.   r   r   >   tor`   r  r  r  r   r  outr;   r`   r;   r  r;   r   imgr   r  modtxtr   r  r  r  r;   r  r;   r`   projr;   r  r  r;   r   r  r   T)reverseF)rsplitry   r:   
setdefaultr8   r;   sortedr5   tuplery  r?   )r"   r   basere  r  restheadtail	protectedprot_by_lenngpartsmergedrR   lengths_descmatchedLhead_convertedconverted_bases                      r   convert_keyzB_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.convert_keyX  s   )Fcz"zz#q1f"BfhaLEE
%Dd{!ZZQ/
dTz!2dI( K ?&&s2w6::2>? JJsOEFA!+"2"2"4dCLc%j.% A1uE
*uU1q1u5E/F+VW./XchhuQQ/?&@AQ"& MM%(+FA c%j. !XXf-N &xq(8?N!vcFlFF2FFr   r   ri   c                     | j                   d   }j                  |      j                         }||z  }|}d}|dz  |k  r|dz  }|dz  }|dz  |k  r||fS rQ  rR  )r   r-  r   r   r   r   r   r@   s          r   rS  zG_convert_non_diffusers_qwen_lora_to_diffusers.<locals>.get_alpha_scales  sv      #y)..0
1nx'!OJMH 1nx' 8##r   r   r   rj   r   r:  rX  )r6   r|   r  r>   r4   r5   rq   rt   r7   ry   r<   )r@   has_lora_unetr`   r   r  r  rD   r*  up_keyrS  r.  diffusers_up_keyr-  r   r   r   r   s   `                r   -_convert_non_diffusers_qwen_lora_to_diffusersr  S  s   GJGGMBLBRBRBTU$!Qann\2A5U
<	GS <	GS <	G| 5?4D4D4FGDAqk!na'G
JOO%&H"HF	$  
J::h!"85E!F yy6:BB6K[\		(H5I$..+K"qyy6'BCI#3K#K J7BZ7O !345>5I !12
J :azO`Nbcdd>R>X>X>Z[dal1#.1[[E V@ HB \s   F&,F,F2)r      )unettext_encoder)diffusion_model)r   typingr   r   utilsr   r   r   
get_logger__name__rz   r   rZ   r   rs   ru   rx   r  r  r>   rz  r  r  r,  rA  r  r  r  r  r  r   r   r   <module>r     s    
   A A 
		H	%vri*X5p8 p9lfR% %$s) % %(j Z	[ |l ^D Nf R: z  e r   