bilddlmZmZmZmZmZddlZddlmZm Z ddl m Z ddl m Z mZddlmZddlmZmZmZmZdd lmZd d lmZmZer ddlmcmZd Znd Zej@e!Z"dZ#ddZ$Gddee Z%y))CallableDictListOptionalUnionN)T5EncoderModel T5Tokenizer)StableDiffusionLoraLoaderMixin)Kandinsky3UNetVQModel) DDPMScheduler) deprecateis_torch_xla_availableloggingreplace_example_docstring) randn_tensor)DiffusionPipelineImagePipelineOutputTFa Examples: ```py >>> from diffusers import AutoPipelineForText2Image >>> import torch >>> pipe = AutoPipelineForText2Image.from_pretrained( ... "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ... ) >>> pipe.enable_model_cpu_offload() >>> prompt = "A photograph of the inside of a subway train. There are raccoons sitting on the seats. One of them is reading a newspaper. The window shows the city in the background." >>> generator = torch.Generator(device="cpu").manual_seed(0) >>> image = pipe(prompt, num_inference_steps=25, generator=generator).images[0] ``` cv||dzz}||dzzdk7r|dz }||dzz}||dzzdk7r|dz }||z||zfS)Nrr)heightwidth scale_factor new_height new_widths m/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.pydownscale_height_and_widthr 1sg<?*J a1$a q(I |Q!#Q  $i,&> >>c%eZdZdZgdZdedededede f fd Z d Z e j d)d ee jdee jdee jdee jfdZdZ d*dZedZedZedZe jeed ddd d ddd d d d d dd d d dgfdeeeefdededeeeeefd eed!eed"eed#eee j<ee j<fd ee jdee jdee jdee jd$eed%ed&ee eee!gd fd'eef d(Z"xZ#S)+Kandinsky3Pipelineztext_encoder->unet->movq)latents prompt_embedsnegative_prompt_embedsnegative_attention_maskattention_mask tokenizer text_encoderunet schedulermovqcNt||j|||||y)N)r)r*r+r,r-)super__init__register_modules)selfr)r*r+r,r- __class__s rr0zKandinsky3Pipeline.__init__Es0  lQZae  r!c|rYtj||dk(||dk(<|jdjdz}|ddd|f}|ddd|f}||fS)Nrr)torch zeros_likesummax)r2 embeddingsr( cut_contextmax_seq_lengths rprocess_embedsz!Kandinsky3Pipeline.process_embedsSsy .3.>.>z.\]J]?^._J~* ++//3779A=N#A$67J+A,>?N>))r!TrNr%r&r(r'c |<|:t|t|ur$tdt|dt|d| |j}|t|trd} n-|t|t r t |} n|jd} d} ||j|d| d d } | jj|}| jj|} |j|| }|d}|j|| |\}} || jd z}|j|jj}nd}|j||}|j\}}}|j!d|d}|j#||z|d}| j!|d} |r'|$|dg| z}nEt|tr|g}n1| t |k7r!t%d|dt |d|d| d |}||j|ddd d d }|jj|}|jj|} |j|| }|d}|ddd|jdf}| ddd|jdf} || jd z}n*t'j(|}t'j(| } |rw|jd}|j||}|j|jk7r@|j!d|d}|j#| |z|d}| j!|d} nd}d} ||| | fS)aX Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `List[str]`, *optional*): prompt to be encoded device: (`torch.device`, *optional*): torch device to place the resulting embeddings on num_images_per_prompt (`int`, *optional*, defaults to 1): number of images that should be generated per prompt do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): whether to use classifier free guidance or not negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. attention_mask (`torch.Tensor`, *optional*): Pre-generated attention mask. Must provide if passing `prompt_embeds` directly. negative_attention_mask (`torch.Tensor`, *optional*): Pre-generated negative attention mask. Must provide if passing `negative_prompt_embeds` directly. Nz?`negative_prompt` should be the same type to `prompt`, but got z != .rr max_lengthTpt)paddingrA truncationreturn_tensors)r(r)dtypedevicer5z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)rCrArDreturn_attention_maskrE)type TypeError_execution_device isinstancestrlistlenshaper) input_idstor(r*r= unsqueezerFrepeatview ValueErrorr6r7)r2promptdo_classifier_free_guidancenum_images_per_promptrGnegative_promptr%r& _cut_contextr(r' batch_sizerA text_inputstext_input_idsrFbs_embedseq_len_ uncond_tokens uncond_inputs r encode_promptz Kandinsky3Pipeline.encode_prompt[s T  /"=F|4#88UVZ[jVkUlmV ~Q( >++F  *VS"9J  Jvt$<VJ&,,Q/J  ..$%# )K)2255f=N(77::6BN ---.M*!,M,0,?,? ~_k,l )M>)N,D,DQ,GGM    (%%++EE%((uV(D ,22'1%,,Q0EqI %**86K+KWVXY '../DaH &+A+I&!#z 1 OS1!0 1 s?33 )/)::J3K_J`ax/ |<33 !0 *#~~!("#*.#' .  ".!7!7!:!:6!B*6*E*E*H*H*P')-):):"#:*;*&*@)B&)?C[]EXEXYZE[C[@[)\&*A!E]}GZGZ[\G]E]B]*^')?BYBcBcdeBf)f&*/)9)9-)H&*/*:*:>*J' &,2215G%;%>%>USY%>%Z "%++}/B/BB)?)F)FqJ_ab)c&)?)D)DZRgEgiprt)u&*A*H*HI^`a*b' &* "&* #4nF]]]r!c|t||||}n;|j|k7rtd|jd||j|}||jz}|S)N) generatorrGrFzUnexpected latents shape, got z , expected )rrQrWrSinit_noise_sigma)r2rQrFrGrgr$r,s rprepare_latentsz"Kandinsky3Pipeline.prepare_latentssg ?"5IfTYZG}}% #A'--P[\a[b!cddjj(GI666r!c |0t|tr|dkrtd|dt|d|Lt fd|Ds8tdj d|D cgc]} | j vs| c} ||td|d |d | | td |7t|t s't|tstd t|||td |d|d |A|?|j|jk7r&td|jd|jd| | td|G|E|jdd|jk7r)td|jddd|jd| | td|I|F|jdd|jk7r)td|jddd|jdyyycc} w)Nrz5`callback_steps` has to be a positive integer but is z of type r?c3:K|]}|jvywN_callback_tensor_inputs.0kr2s r z2Kandinsky3Pipeline.check_inputs.. #F 23A-- -F 2`callback_on_step_end_tensor_inputs` has to be in , but found zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` zLPlease provide `negative_attention_mask` along with `negative_prompt_embeds`rz`negative_prompt_embeds` and `negative_attention_mask` must have the same batch_size and token length when passed directly, but got: `negative_prompt_embeds` z != `negative_attention_mask` z:Please provide `attention_mask` along with `prompt_embeds`z`prompt_embeds` and `attention_mask` must have the same batch_size and token length when passed directly, but got: `prompt_embeds` z != `attention_mask` ) rMintrWrJallrnrNrOrQ) r2rXcallback_stepsr[r%r&"callback_on_step_end_tensor_inputsr(r'rqs ` r check_inputszKandinsky3Pipeline.check_inputss  %z.#/NR`deReGGWX(),  . 9#F 7YF C DTEaEaDbbn|^pHvwbckokGkGbGpqpHoIJ   -";08N}o^00 ^ 5w  FC)@TZ\`IaQRVW]R^Q_`a a  &+A+M9/9JK*++]_   $)?)K""&<&B&BB --:-@-@,AB.445Q8 " -2I2Qkl l ! -2I2U%++BQ/3J3P3PP 66L6R6RSUTU6V5WX/556a9  $)?YZ Z  $)C""2A&.*>*>> --:-@-@!-D,EF&,,-Q0?*D $UpHs !G,5G,c|jSrl_guidance_scaler2s rguidance_scalez!Kandinsky3Pipeline.guidance_scaleBs###r!c |jdkDS)Nrr}rs rrYz.Kandinsky3Pipeline.do_classifier_free_guidanceFs##a''r!c|jSrl)_num_timestepsrs r num_timestepsz Kandinsky3Pipeline.num_timestepsJs"""r!g@ipilr$rXnum_inference_stepsrr[rZrrrg output_type return_dictcallback_on_step_endrzc  |jdd}|jdd}| tddd| tddd|Ltfd|Ds8tdjd |Dcgc]}|jvs|c}d }j }j |||| | || | |_|t|trd }n-|t|tr t|}n| jd }j|j|||| | || | \} } } } jr|jI|,||zd k(r$|tKj$dd z}$||$||tLstOjP| dvrtd| | d k(sjRjU|d !d"}%| d#vrX|%d$zd$z}%|%jWd d }%|%jYj[d dd%d j]j_}%| d&k(rja|%}%n|}%jc|s |%fcdddSte|%'cdddScc}w#1swYyxYw)(u6 Function invoked when calling the pipeline for generation. Args: prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. instead. num_inference_steps (`int`, *optional*, defaults to 25): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. timesteps (`List[int]`, *optional*): Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` timesteps are used. Must be in descending order. guidance_scale (`float`, *optional*, defaults to 3.0): Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. height (`int`, *optional*, defaults to self.unet.config.sample_size): The height in pixels of the generated image. width (`int`, *optional*, defaults to self.unet.config.sample_size): The width in pixels of the generated image. eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://huggingface.co/papers/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. attention_mask (`torch.Tensor`, *optional*): Pre-generated attention mask. Must provide if passing `prompt_embeds` directly. negative_attention_mask (`torch.Tensor`, *optional*): Pre-generated negative attention mask. Must provide if passing `negative_prompt_embeds` directly. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generate image. Choose between [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. callback (`Callable`, *optional*): A function that will be called every `callback_steps` steps during inference. The function will be called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`. callback_steps (`int`, *optional*, defaults to 1): The frequency at which the `callback` function will be called. If not specified, the callback will be called at every step. clean_caption (`bool`, *optional*, defaults to `True`): Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to be installed. If the dependencies are not installed, the embeddings will be created from the raw prompt. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under `self.processor` in [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). Examples: Returns: [`~pipelines.ImagePipelineOutput`] or `tuple` callbackNryz1.0.0zhPassing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`znPassing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`c3:K|]}|jvywrlrmros rrrz.Kandinsky3Pipeline.__call__..rsrtrurvTrr)rZrGr[r%r&r\r(r')rGtext_encoder_offload_hook)totalrF)encoder_hidden_statesencoder_attention_maskrg?)rgr$r%r&r(r'order)rBnprlatentzSOnly the output types `pt`, `pil`, `np` and `latent` are supported not output_type=r)force_not_quantizesample)rrg?r r)images)3poprrxrWrnrLr{r~rMrNrOrPrQrerYr6catboolr, set_timesteps timestepsr rirFhasattrroffloadrr progress_bar enumerater+chunkstep prev_samplelocalsupdategetattr XLA_AVAILABLExm mark_stepr-decodeclampcpupermutefloatnumpy numpy_to_pilmaybe_free_model_hooksr)&r2rXrrr[rZrrrgr%r&r(r'rrr$rrzkwargsrryrqr;rGr]rnum_warmup_stepsritlatent_model_input noise_prednoise_pred_uncondnoise_pred_textcallback_kwargscallback_outputsstep_idximages&` r__call__zKandinsky3Pipeline.__call__Ns@::j$/$4d;   z   %  A  . 9#F 7YF C DTEaEaDbbn|^pHvwbckokGkGbGpqpHoIJ  ''      " .  #  .  *VS"9J  Jvt$<VJ&,,Q/JZ^YkYk   , ,"7+'#9$)$;Zl Z V -~?V  + +!II'=}&MNM"YY(?'PQVVXN $$%8$HNN,, 365!D && / /FE B       NN   44 5$:X:X:d  * * 2 2 4y>,?$..BVBV,VV!)n   %8  9H 5\!),- #1AEAaAaUYYy1}%=gn""YY&*7+9 % '   339C9I9I!9L6%"03"6/!IN]nLn!nJ..--' . + (3&(O?9-3Xa[*9';D!Q'X$.229gFG$4$8$8-$XM-=-A-ABZ\r-s*%5%9%9:JN%[N.>.B.BC\^u.v+I**A9I/IqSTuX\XfXfXlXlNlpqNq '')+N0Ba0G#$(K#K 1g6 LLN[- #`"?? ijuivw(* ((T(J8T-/!CK#-E!KK1-E!IIK//1a;AACIIKE%' --e4E  ' ' )xMH 5H 5P'e4QH 5H 5EpHDH 5H 5s&1S>S> F&TCT) TT ) TrNNNNFNN)NNNNNN)$__name__ __module__ __qualname__model_cpu_offload_seqrnr rr rr r0r=r6no_gradrTensorrerir{propertyrrYrrEXAMPLE_DOC_STRINGrrNrrwr Generatorrrrr __classcell__)r3s@rr#r#;s6    %    !    *U]]_%)049=15:>S^  -S^!) 6S^!.S^"*%,,!7S^S^j #+/ $DL$$((##U]]_12)-#% #;?/0 $#MQ049=15:>%* KO9B %|5c49n%|5!|5 |5 "%T#Y"78 |5 (} |5 |5}|5E%//43H"HIJ|5  -|5!) 6|5!.|5"*%,,!7|5c]|5|5"'xc40@$0F'GH#|5$-1I%|53|5r!r#)r)&typingrrrrrr6 transformersrr loadersr modelsr r schedulersrutilsrrrrutils.torch_utilsrpipeline_utilsrrtorch_xla.core.xla_modelcore xla_modelrr get_loggerrloggerrr r#rr!rrsu88 45-' .C))MM   H %(?Q5*,JQ5r!