biddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z ddl Z ddlZddlmZmZmZmZmZmZmZmZmZmZmZddlmZmZdd lmZm Z dd l!m"Z"ejFe$Z%id e"d e"d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-Z&Gd.d/Z'y)0N)partial)Path)DictListLiteralOptionalUnion) MIN_PEFT_VERSIONUSE_PEFT_BACKENDcheck_peft_versionconvert_unet_state_dict_to_peftdelete_adapter_layersget_adapter_nameis_peft_availableis_peft_versionloggingset_adapter_layers!set_weights_and_activate_adapters)_create_lora_config_maybe_warn_for_unhandled_keys)_fetch_state_dict#_func_optionally_disable_offloading)_maybe_expand_lora_scalesUNet2DConditionModelUNetMotionModelSD3Transformer2DModelc|SN model_clsweightss Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/loaders/peft.pyr&0FluxTransformer2DModelc|Sr r!r"s r%r&r&1r(CogVideoXTransformer3DModelc|Sr r!r"s r%r&r&2gr(ConsisIDTransformer3DModelc|Sr r!r"s r%r&r&3Wr(MochiTransformer3DModelc|Sr r!r"s r%r&r&4s'r(HunyuanVideoTransformer3DModelc|Sr r!r"s r%r&r&5r(LTXVideoTransformer3DModelc|Sr r!r"s r%r&r&6r1r(SanaTransformer2DModelc|Sr r!r"s r%r&r&7r+r(AuraFlowTransformer2DModelc|Sr r!r"s r%r&r&8r1r(Lumina2Transformer2DModelc|Sr r!r"s r%r&r&9Gr(WanTransformer3DModelc|Sr r!r"s r%r&r&:r'r(CogView4Transformer2DModelc|Sr r!r"s r%r&r&;r1r(HiDreamImageTransformer2DModelc|Sr r!r"s r%r&r&<r6r('HunyuanVideoFramepackTransformer3DModelc|Sr r!r"s r%r&r&=s'r(WanVACETransformer3DModelc|Sr r!r"s r%r&r&>r?r(c|Sr r!r"s r%r&r&?s7r(c|Sr r!r"s r%r&r&@r.r()ChromaTransformer2DModelQwenImageTransformer2DModelc zeZdZUdZdZdZeeed<e dZ d"de fdZ d#de d e d e d ee fd Z d$d eee e fdeeeeeeeeedffdZd%de ddfdZdee ee fddfdZd&dZd&dZdee fdZd'dZd$dZdZdZdZdZdZd eee e ffdZ d(de!de"d ddfd!Z#y))PeftAdapterMixina A class containing all functions for loading and using adapters weights that are supported in PEFT library. For more details about adapters and injecting them in a base model, check out the PEFT [documentation](https://huggingface.co/docs/peft/index). Install the latest version of PEFT, and use this mixin to: - Attach new adapters in the model. - Attach multiple adapters and iteratively activate/deactivate them. - Activate/deactivate all adapters from the model. - Get a list of the active adapters. FN_prepare_lora_hotswap_kwargsct|S)N) _pipeline)r)clsrRs r%_optionally_disable_offloadingz/PeftAdapterMixin._optionally_disable_offloadingVs3YGGr(hotswapc 01ddlm}m}ddlm}ddlm}|jdd} |jdd } |jd d} |jd d} |jd d} |jd d}|jdd}|jdd}|jdd}|jdd0|jdd}|jdd}|jdd }|jdd}d }|rtddr tdddd}t|||| | | | | ||||| \}}| | td|r |r td||jDcic]/\}}|j|ds|j|d|1}}}|J|jDcic]/\}}|j|ds|j|d|1}}}t|dkDrÉ0t|d ivr|std!0d"0t|d ivr|rtd#0d$t!t#|j%}d%|vr t'|}i}|jD]/\1}d&1vs |j(d'kDs|j*d'|d(1<1|zt|d'k\rl|j%Dcgc]}|j|ds|}}|jDcic]\}}||vs |j|d|!}}}0 t-|0t/|||||j10)}|j3|\} }!}"i}#td*d+r||#d<|s |j4$td,d-r dd.lm}$m}%m}&n d/}'t?|'|r01fd0}( |r.(|}$|j@0| %||0|1d}*n@|||f0|d4|#|||0fi|#}*|j4&|fd5|i|j4d|_|jHsd6|_$tU|*0| r|jWna|!r|jYnN|"rL|jZj]D]/}.tO|.t^j`jbs(||.1|9|s6|jdjf}/tDjid8|/d9|d:|/d;yyycc}}wcc}}wcc}wcc}}w#tB$r!})tDjGd20d3|)d})~)wwxYw#tB$r})tK|d rf|jMD]8}+tO|+|s|+jP},|,D]}-0|-vs|+jS0:|j@j0tDjGd70d3|)d})~)wwxYw)=z0.13.1>z0.14.0) check_hotswap_configs_compatiblehotswap_adapter_from_state_dict"prepare_model_for_compiled_hotswapzeHotswapping requires PEFT > v0.14. Please upgrade PEFT to a higher version or install it from source.ci}|jD]j\}}|jdsjdr|dtd ddz}n'|jdr|dtd ddz}|||<l|S)Nz lora_A.weightz lora_B.weightz.weightrqz lora_B.biasz.bias)itemsendswithlen)sdnew_sdkvrfkeys r%map_state_dict_for_hotswapzFPeftAdapterMixin.load_lora_adapter..map_state_dict_for_hotswap$sF " &1::o6#,,:W !"3S^O 4<.7P PAZZ 6 !/S\M 2qe5L LA$%q & "Mr()model state_dictrfconfigz Hotswapping z- was unsuccessful with the following error: )rfrrTzLoading zNo LoRA keys associated to z found with the prefix=zG. This is safe to ignore if LoRA state dict didn't originally have any z related params. You can also try specifying `prefix=None` to resolve the warning. Otherwise, open an issue if you think it's unexpected: https://github.com/huggingface/diffusers/issues/new)5peftrWrXpeft.tuners.tuners_utilsrZhooks.group_offloadingr\popr ValueErrorrr} startswith removeprefixrgetattrnextiterkeysrndimshaperrrrTrPpeft.utils.hotswapryrzr{ ImportErrorrr Exceptionloggererror_hf_peft_config_loadedhasattrmodules isinstanceactive_adaptersdelete_adapterrenable_model_cpu_offloadenable_sequential_cpu_offload componentsvaluestorchnnModule __class____name__warning)2selfrnprefixrUkwargsrWrXrZr\r]r^r_r`rarbrcrdrergrRrhrirprorrr first_keyrankval alpha_keys lora_configis_model_cpu_offloadis_sequential_cpu_offloadis_group_offload peft_kwargsryrzr{msgreincompatible_keysmoduleractive_adapter componentmodel_class_namerfrs2 @@r%load_lora_adapterz"PeftAdapterMixin.load_lora_adapter[sR L;WJJ{D1 $4e<**Y-!::&8$? 7D)::j$/JJ{D1 jj5  **%6=zz.$7 $4d;JJ{D1 "JJ':EB::j$/ x!@D $8iP 02W#+-)!%  H  %&.UV V hXY Y  FPFVFVFXwda\]\h\hlrksstiu\v!..F8169wJw#HPHXw1\]\h\hlrksstiu\vANNfXQ<8!;ww z?Q wt]B?? #L>1qrWT="%EE' 6|nEll T*//"345Iy(]A!,,RXQYYZ|B\a] ]@N@T@T@V"8<1Z[_iZiANNfXQ<0!3"" #/5 .!%!2) KQUPsPsQ M ";=MKtX.3D /04<<H"31'&c** "2 !;J!GJ4T5E5El5SU`a 7"&'1)5#. )-%+#T8DQ[_j)B$ T`(pdo(p%88D ; )48<8Y8Y=A92226D/ ++NI!)UXX__=B9MN  j#~~66  NN-.>-??WPVyYX#$%FF '1 }xw<^"N% |L>Aopqor%st6 4/"&,,.H%fn=.4.D.DO2AH#/>#A$*$9$9,$GHH$$((6 x ~5cdecfgh szTT T*T*TT T#*T# U4 T)AU) U2UUU W2+W- W-!A W--W2rfupcast_before_savingsafe_serializationrdcT ddlm}ddlm m}m}| t |}|t|divrtd|d|j|j ||j|rtjnd | } tjj!|rtd |d |r fd } ntj"} tj$|d||r|}n|}t'||j)} | | | t*j-d| y)al Save the LoRA parameters corresponding to the underlying model. Arguments: save_directory (`str` or `os.PathLike`): Directory to save LoRA parameters to. Will be created if it doesn't exist. adapter_name: (`str`, defaults to "default"): The name of the adapter to serialize. Useful when the underlying model has multiple adapters loaded. upcast_before_saving (`bool`, defaults to `False`): Whether to cast the underlying model to `torch.float32` before serialization. safe_serialization (`bool`, *optional*, defaults to `True`): Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. weight_name: (`str`, *optional*, defaults to `None`): Name of the file to serialize the state dict with. r)get_peft_model_state_dictr)LORA_ADAPTER_METADATA_KEYLORA_WEIGHT_NAMELORA_WEIGHT_NAME_SAFENrrrsz not found in the model.)dtype)rfzProvided path (z#) should be a directory, not a filecddi}RjD]$\}}t|tst||<&t j dd|<t jj|||S)Nformatptr T)indent sort_keys)ri) r}rsetlistjsondumps safetensorsr save_file)r$filenamerirvaluerlora_adapter_metadatas r% save_functionz9PeftAdapterMixin.save_lora_adapter..save_functions$d+(4&;&A&A&CE U%eS19=e1#6E;?**EZcdpt:uH67"((227Hx2XXr(T)exist_okzModel weights saved in ) peft.utilsr lora_baserrrrrrrrto_dicttorfloat32ospathisfilesavemakedirsras_posixrinfo)rsave_directoryrfrrrdrrrlora_layers_to_saver save_pathrrs @@r%save_lora_adapterz"PeftAdapterMixin.save_lora_adapter|s, 9aa  +D1L wt]B? ?}\N:RST T $ 0 0 > F F H7 GG+?%--TG JYe  77>>. )~.>>abc c  Y"JJM NT2  !3 . 5>>@ )95 -i[9:r( adapter_namesr$cts tdt|tr|gn|}t|ts|gt |z}t |t |k7r$tdt |dt |d|Dcgc]}||nd }}t |jj}|||}t|||ycc}w)aJ Set the currently active adapters for use in the diffusion network (e.g. unet, transformer, etc.). Args: adapter_names (`List[str]` or `str`): The names of the adapters to use. adapter_weights (`Union[List[float], float]`, *optional*): The adapter(s) weights to use with the UNet. If `None`, the weights are set to `1.0` for all the adapters. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" ) pipeline.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") pipeline.unet.set_adapters(["cinematic", "pixel"], adapter_weights=[0.5, 0.5]) ``` z.PEFT backend is required for `set_adapters()`.zLength of adapter names z- is not equal to the length of their weights rqN?) r rrstrrr_SET_ADAPTER_SCALE_FN_MAPPINGrrr)rrr$wscale_expansion_fns r% set_adapterszPeftAdapterMixin.set_adapterss> MN N+5mS+I} '4(i#m"44G } W -*3}+=*>>klopwlxkyyz{  9@@1 13.@@;4>>;R;RS$T73)$ wG As CreturncLttts tdddlm}m}|jsd|_n||jvrtd|dt||stdt|d d |_ |||||j|y ) a Adds a new adapter to the current model for training. If no adapter name is passed, a default name is assigned to the adapter to follow the convention of the PEFT library. If you are not familiar with adapters and PEFT methods, we invite you to read more about them in the PEFT [documentation](https://huggingface.co/docs/peft). Args: adapter_config (`[~peft.PeftConfig]`): The configuration of the adapter to add; supported adapters are non-prefix tuning and adaption prompt methods. adapter_name (`str`, *optional*, defaults to `"default"`): The name of the adapter to add. If no name is passed, a default name is assigned to the adapter.  min_versionTPEFT is not available. Please install PEFT to use this function: `pip install peft`.r) PeftConfigrWTzAdapter with name z- already exists. Please use a different name.z8adapter_config should be an instance of PeftConfig. Got z instead.N)r r rrrrrWrrrrrtypebase_model_name_or_path set_adapter)radapter_configrfrrWs r% add_adapterzPeftAdapterMixin.add_adapters '78 "tu u<***.D ' T-- -1,?lmn n.*5J4P^K_J``ij  26.lC &r(c dtt|js tdt |t r|g}t |t |jz }t|dkDrAtddj|dt|jjddl m }d}|jD]^\}}t ||st|d r|j!|n,t|d st|d k7r td ||_d }`|s td y)a Sets a specific adapter by forcing the model to only use that adapter and disables the other adapters. If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT [documentation](https://huggingface.co/docs/peft). Args: adapter_name (Union[str, List[str]])): The list of adapters to set or the adapter name in the case of a single adapter. r0No adapter loaded. Please load an adapter first.rz)Following adapter(s) could not be found: z, zV. Make sure you are passing the correct adapter name(s). current loaded adapters are: rYFrrzYou are trying to set multiple adapters and you have a PEFT version that does not support multi-adapter inference. Please upgrade to the latest version of PEFT. `pip install -U peft` or `pip install -U git+https://github.com/huggingface/peft.git`TzhDid not succeeded in setting the adapter. Please make sure you are using a model that supports adapters.N)r r rrrrrrrrjoinrrrrZ named_modulesrrr)rrfmissingrZ_adapters_has_been_set_rs r%rzPeftAdapterMixin.set_adapters: '78**OP P lC ((>Ll#c$*:*:&;; w>=C\ L FLL (< ( .r(c\ts td|j|jy)Nz-PEFT backend is required for `unfuse_lora()`.)r rr _unfuse_lora_applyrs r% unfuse_lorazPeftAdapterMixin.unfuse_loras#LM M 4**+r(cJddlm}t||r|jyy)NrrY)rrZrunmerge)rrrZs r%rz#PeftAdapterMixin._unfuse_lora_applys; fn - NN  .r(cts tdddlm}ddlm}||t |dr|`t |drd|_||y)Nz-PEFT backend is required for `unload_lora()`.r r[)recurse_remove_peft_layersrrr) r rrr\utilsr!rrrr)rr\r!s r% unload_lorazPeftAdapterMixin.unload_lorasJLM MW6"4( 4 ' 41 2*.D '248r(c@ts tdt|dy)a1 Disables the active LoRA layers of the underlying model. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" ) pipeline.unet.disable_lora() ``` )PEFT backend is required for this method.FrNr rrrs r% disable_lorazPeftAdapterMixin.disable_loras& HI I4/r(c@ts tdt|dy)a/ Enables the active LoRA layers of the underlying model. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" ) pipeline.unet.enable_lora() ``` r%TrNr&rs r% enable_lorazPeftAdapterMixin.enable_loras& HI I4.r(cts tdt|tr|g}|D]7}t ||t |ds|j j|d9y)a Delete an adapter's LoRA layers from the underlying model. Args: adapter_names (`Union[List[str], str]`): The names (single string or list of strings) of the adapter to delete. Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 ).to("cuda") pipeline.load_lora_weights( "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_names="cinematic" ) pipeline.unet.delete_adapters("cinematic") ``` r%rrN)r rrrrrrrr)rrrfs r%delete_adaptersz PeftAdapterMixin.delete_adapterss_. HI I mS )*OM) 9L !$ 5t]+  $$\48  9r( target_rankcheck_compiled)rwarnignorect|dir?|dk(r td|dk(rtjdn|dk7rt d|d||d |_y ) aEnables the possibility to hotswap LoRA adapters. Calling this method is only required when hotswapping adapters and if the model is compiled or if the ranks of the loaded adapters differ. Args: target_rank (`int`, *optional*, defaults to `128`): The highest rank among all the adapters that will be loaded. check_compiled (`str`, *optional*, defaults to `"error"`): How to handle the case when the model is already compiled, which should generally be avoided. The options are: - "error" (default): raise an error - "warn": issue a warning - "ignore": do nothing rrrz-B"9J\ckk6=>W6Xk kr(rO)(rrr functoolsrpathlibrtypingrrrrr rrr"r r r rrrrrrrrutils.peft_utilsrrrrrunet_loader_utilsr get_loggerrrrrOr!r(r%rEsS 77     SM8   H %!5!0!?!@ ! "#E ! !"D !A!%&H!!"D!@!!"D! !C!?!!"D!%&H! ./Q!!" !C#!$!C#E'!.v kv kr(