bi-iddlmZmZmZmZddlZddlZddlmZddlm Z ddl m Z ddl m Z mZmZmZerddlmZmZer dd lmZmZmZej0eZd Zd Zed ZGd dZy))DictListOptionalUnionN)validate_hf_hub_args)nn)load_state_dict)_get_model_fileis_accelerate_availableis_transformers_availablelogging)PreTrainedModelPreTrainedTokenizer)AlignDevicesHook CpuOffloadremove_hook_from_modulezlearned_embeds.binzlearned_embeds.safetensorsc |jdd}|jdd}|jdd}|jdd}|jdd}|jdd}|jdd}|jd d} |jd d} d} | d } d } d d d} g} |D]}t|ttjfsd}| r| | P| j dr? t || xst||||||||  }tjj|d}|+t || xst||||||||  }t|}n|}| j| S#t$r}| s|d}Yd}~Vd}~wwxYw)N cache_dirforce_downloadFproxieslocal_files_onlyhf_tokenrevision subfolder weight_nameuse_safetensorsTtext_inversionpytorch) file_type frameworkz .safetensors) weights_namerrrrtokenrr user_agentcpudevice)pop isinstancedicttorchTensorendswithr TEXT_INVERSION_NAME_SAFE safetensors load_file ExceptionTEXT_INVERSION_NAMEr append)pretrained_model_name_or_pathskwargsrrrrrrrrr allow_pickler$ state_dictspretrained_model_name_or_path model_file state_dictes ^/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/loaders/textual_inversion.py"load_textual_inversion_state_dictsr=%s ;-IZZ 0%8NjjD)Gzz"4d;zz*d+Hzz*d+H ;-I**]D1Kjj!2D9OL &JK)G.'%7$ 9MNJ K$7'K,@,@,P&!05%0%L4L"+'5 ')9&!)"+#- "J"-!2!2!E11 F :FF c FeZdZdZdeeeefddfdZdeddfdZdZ e dZ e d Z e dd eeeeeeej feeeej ffd eeeeefdedd edfdZ ddeeeeefdedd edfdZy )TextualInversionLoaderMixinzY Load Textual Inversion tokens and embeddings to the tokenizer and text encoder. prompt tokenizerrct|ts|g}n|}|Dcgc]}|j||}}t|ts|dS|Scc}w)a Processes prompts that include a special token corresponding to a multi-vector textual inversion embedding to be replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual inversion token or if the textual inversion token is a single vector, the input prompt is returned. Parameters: prompt (`str` or list of `str`): The prompt or prompts to guide the image generation. tokenizer (`PreTrainedTokenizer`): The tokenizer responsible for encoding the prompt into input tokens. Returns: `str` or list of `str`: The converted prompt r)r)r_maybe_convert_prompt)selfr@rApromptsps r<maybe_convert_promptz0TextualInversionLoaderMixin.maybe_convert_promptssY&$'hGGELM4--a;MM&$'1:  NsA c|j|}t|}|D]^}||jvs|}d}|d||jvr$|d|d|z }|dz }|d||jvr$|j||}`|S)a  Maybe convert a prompt into a "multi vector"-compatible prompt. If the prompt includes a token that corresponds to a multi-vector textual inversion embedding, this function will process the prompt so that the special token is replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual inversion token or a textual inversion token that is a single vector, the input prompt is simply returned. Parameters: prompt (`str`): The prompt to guide the image generation. tokenizer (`PreTrainedTokenizer`): The tokenizer responsible for encoding the prompt into input tokens. Returns: `str`: The converted prompt _ )tokenizesetadded_tokens_encoderreplace)rDr@rAtokens unique_tokensr# replacementis r<rCz1TextualInversionLoaderMixin._maybe_convert_prompts ##F+F " >**+,0099:!=   >>**+,0099:!=  - . 2s;Y7Z^abh^i7i=cB`>a=bcBCFGMCNBOO=>  $*;aQ]; ; s< !C $5 5Z[aZbcd d 6  bchbiijkl$ ++-- UG#tuzt{|op   e $  ! !) ,C! -F>))rTc lg}g}t||D]\}}|d|jvrY|g}d}|d||jvr/|j|d||dz }|d||jvr/t d|d|dt |j dkDxr|j ddkD} | rG||gtd|j dDcgc] }|d| c}zz }||D cgc]} | c} z }||gz }|t |j dkDr|dgn|gz }||fScc}wcc} w)N_1rIrJzMulti-vector Token zU already in tokenizer vocabulary. Please choose a different token name or remove the rer)rfrlrNr3rWr[shaperange) rP embeddingsrArmrnrpr#multi_vector_tokensrSis_multi_vectorr;s r<_extend_tokens_and_embeddingsz9TextualInversionLoaderMixin._extend_tokens_and_embeddingss  #J 7 ^ Iu|y2244',g#qn (F(FF'..%!~>FAqn (F(FF!)*=)>?TUhTii\]")//2Q6Q9??1;MPQ;QOugq)//Z[J\A](^AE7!A3(^^^ i"81"88ug% C 4H14L9Q<.S\R]]% ^(>)) )_"8s D, , D1Nr8r#r\rc |xs t|dd}|xs t|dd}t|ts|gn|}t|ts|gn|}|d|t|z}|j ||||t |fi|}t|dkDrtt|dk(rft|dt jrIt|d}t|t|k7r$tdt|dt|d|j|||\}} |j|| |\}} |jjjd tfd | Dr td d } d } |j|j j#D]\} } t| t$j&s!t)| d s.tt| d t*} tt| d t,xs?t)| j.dxr't| j.j0dt,} t2j5d| s| st7| | |j8}|j:}|j=t|t|z|jj}t?|| D]O\}}|jA||jC|}||jD|<t2j5d|dQ|jG||| r|jI|y| r|jK|yy)u Load Textual Inversion embeddings into the text encoder of [`StableDiffusionPipeline`] (both 🤗 Diffusers and Automatic1111 formats are supported). Parameters: pretrained_model_name_or_path (`str` or `os.PathLike` or `List[str or os.PathLike]` or `Dict` or `List[Dict]`): Can be either one of the following or a list of them: - A string, the *model id* (for example `sd-concepts-library/low-poly-hd-logos-icons`) of a pretrained model hosted on the Hub. - A path to a *directory* (for example `./my_text_inversion_directory/`) containing the textual inversion weights. - A path to a *file* (for example `./my_text_inversions.pt`) containing textual inversion weights. - A [torch state dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). token (`str` or `List[str]`, *optional*): Override the token to use for the textual inversion weights. If `pretrained_model_name_or_path` is a list, then `token` must also be a list of equal length. text_encoder ([`~transformers.CLIPTextModel`], *optional*): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). If not specified, function will take self.tokenizer. tokenizer ([`~transformers.CLIPTokenizer`], *optional*): A `CLIPTokenizer` to tokenize text. If not specified, function will take self.tokenizer. weight_name (`str`, *optional*): Name of a custom weight file. This should be used when: - The saved textual inversion file is in 🤗 Diffusers format, but was saved under a specific weight name such as `text_inv.bin`. - The saved textual inversion file is in the Automatic1111 format. cache_dir (`Union[str, os.PathLike]`, *optional*): Path to a directory where a downloaded pretrained model configuration is cached if the standard cache is not used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. local_files_only (`bool`, *optional*, defaults to `False`): Whether to only load local model weights and configuration files or not. If set to `True`, the model won't be downloaded from the Hub. hf_token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from `diffusers-cli login` (stored in `~/.huggingface`) is used. revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier allowed by Git. subfolder (`str`, *optional*, defaults to `""`): The subfolder location of a model file within a larger model repository on the Hub or locally. mirror (`str`, *optional*): Mirror source to resolve accessibility issues if you're downloading a model in China. We do not guarantee the timeliness or safety of the source, and you should refer to the mirror site for more information. Example: To load a Textual Inversion embedding vector in 🤗 Diffusers format: ```py from diffusers import StableDiffusionPipeline import torch model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") pipe.load_textual_inversion("sd-concepts-library/cat-toy") prompt = "A backpack" image = pipe(prompt, num_inference_steps=50).images[0] image.save("cat-backpack.png") ``` To load a Textual Inversion embedding vector in Automatic1111 format, make sure to download the vector first (for example from [civitAI](https://civitai.com/models/3036?modelVersionId=9857)) and then load the vector locally: ```py from diffusers import StableDiffusionPipeline import torch model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") pipe.load_textual_inversion("./charturnerv2.pt", token="charturnerv2") prompt = "charturnerv2, multiple views of the same character in the same outfit, a character turnaround of a woman wearing a black jacket and red shirt, best quality, intricate details." image = pipe(prompt, num_inference_steps=50).images[0] image.save("character.png") ``` rANr\rrIz&You have passed a state_dict contains z* embeddings, and list of tokens of length z% Make sure both have the same length.c3BK|]}|jdk7yw)r{N)rt).0embexpected_emb_dims r< zETextualInversionLoaderMixin.load_textual_inversion..sGS399R=0GszLoaded embeddings are of incorrect shape. Expected each textual inversion embedding to be of shape {input_embeddings.shape[-1]}, but are {embeddings.shape[-1]} F_hf_hookhookszAccelerate hooks detected. Since you have called `load_textual_inversion()`, the previous hooks will be first removed. Then the textual inversion parameters will be loaded and the hooks will be applied again.)recursez'Loaded textual inversion embedding for rd)dtyper'r&)&getattrr)listr[r_r=r+r,rWrqryget_input_embeddingsweightrtany hf_device_map componentsrirModulehasattrrrrrrjrkrr'rresize_token_embeddingsrf add_tokensconvert_tokens_to_idsdatatoenable_model_cpu_offloadenable_sequential_cpu_offload)rDr8r#rAr\r5r4rPr7rvis_model_cpu_offloadis_sequential_cpu_offloadrJ componentr'rinput_embeddingsrptoken_idrs @r<rZz2TextualInversionLoaderMixin.load_textual_inversions}RA{D!A #Jwt^T'J ;TB+ +. ' !+5$ 7%U !9 c"@AAF ##I|=[]cd99Wb[ab  v;?s;/14+a.%,,7";q>2 v;#k"22$@[AQ@RR|~ABH~I}JJ?@ "AA&+W`a "?? T]^ (<<>EEKKBO GJG G_  %$)!    % $ 5 5 7 b 9i3y*5/9')Z:XZd/e,&wy*'EGWXZ&y'9'97C Z *9+=+=+C+CA+FHX Y2  o58L3IG`a b $$"" ,,S^c&k-IJ'<<>EE!$FJ 7 L E9   ' 66u=H.7  ! !( + KKA%J K  L %7  ) ) ) 8 &  . .f . ='rTrPc|xs t|dd}|xs t|dd}g}d}|r}t|tr|g}|jj D]4\}}|j s!|j |vs!|j|3|}6t|dk(rjtdg}|jj D]@\}}|j s-|j||j|j ?|}Bt||D]\}} |j|=|j| =!d} |jD]`}||kDs ||| zkDs|j|} | |j|| z<|j|=|| z|j| j <| dz } b|j|j|jj } |jj"} | d|dz}g}t%|dz| j&dD]*}||vs|j| |j)d,t|dkDr0t+j,|d}t+j,||gd}t/j0|j&d| }||j"_|j5|y)au Unload Textual Inversion embeddings from the text encoder of [`StableDiffusionPipeline`] Example: ```py from diffusers import AutoPipelineForText2Image import torch pipeline = AutoPipelineForText2Image.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") # Example 1 pipeline.load_textual_inversion("sd-concepts-library/gta5-artwork") pipeline.load_textual_inversion("sd-concepts-library/moeb-style") # Remove all token embeddings pipeline.unload_textual_inversion() # Example 2 pipeline.load_textual_inversion("sd-concepts-library/moeb-style") pipeline.load_textual_inversion("sd-concepts-library/gta5-artwork") # Remove just one token pipeline.unload_textual_inversion("") # Example 3: unload from SDXL pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") embedding_path = hf_hub_download( repo_id="linoyts/web_y2k", filename="web_y2k_emb.safetensors", repo_type="model" ) # load embeddings to the text encoders state_dict = load_file(embedding_path) # load embeddings of text_encoder 1 (CLIP ViT-L/14) pipeline.load_textual_inversion( state_dict["clip_l"], tokens=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer, ) # load embeddings of text_encoder 2 (CLIP ViT-G/14) pipeline.load_textual_inversion( state_dict["clip_g"], tokens=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2, ) # Unload explicitly from both text encoders and tokenizers pipeline.unload_textual_inversion( tokens=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer ) pipeline.unload_textual_inversion( tokens=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2 ) ``` rANr\rzNo tokens to remove foundrI)dim)rr)stradded_tokens_decoderrispecialcontentr3r[rWrf_added_tokens_decoder_added_tokens_encoder _update_trie_update_total_vocab_sizer embedding_dimrrurt unsqueezer+catr Embeddingrset_input_embeddings)rDrPrAr\ token_idslast_special_token_idadded_token_id added_tokenrtoken_to_removekey_idr#text_embedding_dimtemp_text_embedding_weightstext_embedding_weights to_appendrStext_embeddings_filtereds r<unload_textual_inversionz4TextualInversionLoaderMixin.unload_textual_inversions@A{D!A #Jwt^T'J  $ &#& /8/M/M/S/S/U ;+ "**"**f4!((8,:)  ; 9~" !<==F/8/M/M/S/S/U ;+ "**$$^4MM+"5"56,:)  ;*-Y)? A %Ho//9//@ A !66 H//H?TW]?]4]!77ARW //0E0NO33H=AVY_A_ // >!     **,*>>@NN&2&G&G&I&P&P#!<=X?TWX?X!Y ,q02M2S2STU2VW NA !  !A  )3I%*YY0F /RXY%Z "#%<<0F0L0LQ0OQc#d /E '',))*BCrT)NNN)rY __module__ __qualname____doc__rrrrGrCr_ staticmethodrqryrrr+r,rrZrrTr<r?r?ns`5d3i+@Mb6C',S$s)T#u||BS=TVZ[_`cejeqeq`q[rVs-s't>c49n-.>12 > 01 >>J375948 zDsDI~./zD12zD01 zDrTr?)typingrrrrr/r+huggingface_hub.utilsrrmodels.modeling_utilsr utilsr r r r transformersrraccelerate.hooksrrr get_loggerrYrjr2r.r=r?rrTr<rs|/. 63``AVV   H %*7EEPWDWDrT