biU ddlmZmZmZmZddlZddlZddlm Z ddl m Z ddl m Z mZddlmZddlmZmZmZdd lmZd d lmZmZd d lmZer ddlmcmZdZ ndZ ejBe"Z#dZ$ddZ%GddeZ&y))CallableListOptionalUnionN)XLMRobertaTokenizer)VaeImageProcessor)UNet2DConditionModelVQModel) DDIMScheduler)is_torch_xla_availableloggingreplace_example_docstring) randn_tensor)DiffusionPipelineImagePipelineOutput)MultilingualCLIPTFa Examples: ```py >>> from diffusers import KandinskyImg2ImgPipeline, KandinskyPriorPipeline >>> from diffusers.utils import load_image >>> import torch >>> pipe_prior = KandinskyPriorPipeline.from_pretrained( ... "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 ... ) >>> pipe_prior.to("cuda") >>> prompt = "A red cartoon frog, 4k" >>> image_emb, zero_image_emb = pipe_prior(prompt, return_dict=False) >>> pipe = KandinskyImg2ImgPipeline.from_pretrained( ... "kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16 ... ) >>> pipe.to("cuda") >>> init_image = load_image( ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" ... "/kandinsky/frog.png" ... ) >>> image = pipe( ... prompt, ... image=init_image, ... image_embeds=image_emb, ... negative_image_embeds=zero_image_emb, ... height=768, ... width=768, ... num_inference_steps=100, ... strength=0.2, ... ).images >>> image[0].save("red_frog.png") ``` cv||dzz}||dzzdk7r|dz }||dzz}||dzzdk7r|dz }||z||zfS)Nrrr)hw scale_factornew_hnew_ws s/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py get_new_h_wrWsg q E<?a   q E<?a   < !5 55c#eZdZdZdZdedededede f fd Z d Z d Z d#d Z d ejdejdej dejfdZej$ee d$deeeefdeejej2j2eejeej2j2fdejdejdeeeeefdededededededeeej:eej:fdeedeeeeejgd fd ed!ef d"Z xZ!S)%KandinskyImg2ImgPipelinea Pipeline for image-to-image generation using Kandinsky This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) Args: text_encoder ([`MultilingualCLIP`]): Frozen text-encoder. tokenizer ([`XLMRobertaTokenizer`]): Tokenizer of class scheduler ([`DDIMScheduler`]): A scheduler to be used in combination with `unet` to generate image latents. unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the image embedding. movq ([`VQModel`]): MoVQ image encoder and decoder ztext_encoder->unet->movq text_encodermovq tokenizerunet schedulercnt||j|||||t|ddr/dt |j j jdz znd|_t|ddr |j j jnd}t|j|dd|_ y) N)r"r$r%r&r#r#rrbicubic)vae_scale_factorvae_latent_channelsresample reducing_gap) super__init__register_modulesgetattrlenr#configblock_out_channelsmovq_scale_factorlatent_channelsr image_processor)selfr"r#r$r%r&movq_latent_channels __class__s rr0z!KandinskyImg2ImgPipeline.__init__ws  %  DK4QWY]C^A#dii&&99:Q> ?de DK4QWY]C^tyy//??de0!33 4  rctt||z|}t||z d}|jj|d}|||z fS)Nr)minintmaxr& timesteps)r9num_inference_stepsstrengthdevice init_timestept_startr@s r get_timestepsz&KandinskyImg2ImgPipeline.get_timestepssSC 3h >?ATU )M91=NN,,WX6 -777rc|t||||}n;|j|k7rtd|jd||j|}||jz}|j}t||||}|j |||}|S)N) generatorrCdtypezUnexpected latents shape, got z , expected )rshape ValueErrortoinit_noise_sigma add_noise) r9latentslatent_timesteprJrIrCrHr&noises rprepare_latentsz(KandinskyImg2ImgPipeline.prepare_latentss ?"5IfTYZG}}% #A'--P[\a[b!cddjj(GI666 UieT..%ArNc ht|tr t|nd}|j|dddddd}|j}|j|ddj} | j d |j d k\rt j|| sj|jj| dd|jjdz d f} tjd |jjd | |j|}|jj|} |j|| \} } | j|d } | j|d } | j|d } |r|dg|z}nt!|t!|ur$t#dt!|dt!|dt|t$r|g}n1|t|k7r!t'd|dt|d|d|d |}|j|dddddd}|jj|}|jj|}|j|| \}}|j d}|j)d|}|j+||z|}|j d}|j)d|d}|j+||z|d }|j|d }t j,|| g} t j,|| g} t j,|| g} | | | fS)Nr max_lengthMTpt)paddingrT truncationreturn_attention_maskadd_special_tokensreturn_tensorslongest)rWr[z\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: ) input_idsattention_maskrdimz?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.) isinstancelistr3r$r^rJtorchequal batch_decodemodel_max_lengthloggerwarningrLr_r"repeat_interleavetype TypeErrorstrrKrepeatviewcat)r9promptrCnum_images_per_promptdo_classifier_free_guidancenegative_prompt batch_size text_inputstext_input_idsuntruncated_ids removed_text text_mask prompt_embedstext_encoder_hidden_states uncond_tokens uncond_inputuncond_text_input_idsuncond_text_masknegative_prompt_embeds!uncond_text_encoder_hidden_statesseq_lens r_encode_promptz'KandinskyImg2ImgPipeline._encode_prompts%/vt$>66q$..JiJilmJmprJrGr7stL NNNN334Il^M  (**62..11&9 484E4E$Y5F5 1 1&778MST7U %?%Q%QRgmn%Q%o"//0E1/M &&!#z 1 fT/%::UVZ[jVkUlmV ~Q(OS1!0 1 s?33 )/)::J3K_J`ax/ |<33 !0 >>$&*#'#*L%1$:$:$=$=f$E !+::==fE HLHYHY/@PIZI E "$E -2215G%;%B%B1F[%\ "%;%@%@NcAcel%m "7==a@G0Q0X0XYZ\qst0u -0Q0V0V22GR1 - 0AABW]^A_ "II'=}&MNM).4UWq3r)s & #3Y"?@I8)CCroriginal_samplesrQr@returnc,tjdddtj}d|z }tj|d}|j |j |j }|j |j }||d z}|j}t|jt|jkr=|jd }t|jt|jkr=d ||z d z}|j}t|jt|jkr=|jd }t|jt|jkr=||z||zz} | S) Ng-C6?g{Gz?i)rI?rr`)rCrIg?r]r) rflinspacefloat32cumprodrLrCrIflattenr3rJ unsqueeze) r9rrQr@betasalphasalphas_cumprodsqrt_alpha_prodsqrt_one_minus_alpha_prod noisy_sampless rrNz"KandinskyImg2ImgPipeline.add_noisesr vtTGuv15'**2B2I2IQaQgQg*hLL!1!8!89 (3s:)113/''(3/?/E/E+FF-77;O/''(3/?/E/E+FF&' )B%Bs$J!$=$E$E$G!+112S9I9O9O5PP(A(K(KB(O %+112S9I9O9O5PP(*::=VY^=^^ rrsimage image_embedsnegative_image_embedsrvheightwidthrArBguidance_scalertrH output_typecallbackcallback_steps return_dictc  t|trd}n3t|tr t|}nt dt ||j }|| z}| dkD}|j||| ||\}}}t|trtj|d}t|trtj|d}|rZ|j| d}|j| d}tj||gdj|j|}t|ts|g}td|Ds&t d|Dcgc] }t |c}d tj|Dcgc]}|jj|||!c}d}|j|j|}|j j#|d }|j| d}|j$j'|| |j)|| |\}}t+|j$j,j.| zd z }tj0|g|z|j|}|j2j,j4}t7|||j8\}}|j;||||||f|j|| |j$}t=|j?|D]\}}|rtj|gd zn|}||d }|j3||||dd}|ro|jA|jBdd\}} |jEd \}!}"| jEd \}}#|!| |"|!z zz}tj||#gd}tG|j$j,dr"|j$j,jHdvs#|jA|jBdd\}}|j$jK|||| jL}|,||zdk(r$|tO|j$ddz}$||$||tPswtSjT|j jW|dd}|jY| dvrt d| |jj[|| }|s|fSt]|Scc}wcc}w)a Function invoked when calling the pipeline for generation. Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. image (`torch.Tensor`, `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. image_embeds (`torch.Tensor` or `List[torch.Tensor]`): The clip image embeddings for text prompt, that will be used to condition the image generation. negative_image_embeds (`torch.Tensor` or `List[torch.Tensor]`): The clip image embeddings for negative text prompt, will be used to condition the image generation. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. width (`int`, *optional*, defaults to 512): The width in pixels of the generated image. num_inference_steps (`int`, *optional*, defaults to 100): The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference. strength (`float`, *optional*, defaults to 0.3): Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the `strength`. The number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will be maximum and the denoising process will run for the full number of iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. guidance_scale (`float`, *optional*, defaults to 4.0): Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` (`np.array`) or `"pt"` (`torch.Tensor`). callback (`Callable`, *optional*): A function that calls every `callback_steps` steps during inference. The function is called with the following arguments: `callback(step: int, timestep: int, latents: torch.Tensor)`. callback_steps (`int`, *optional*, defaults to 1): The frequency at which the `callback` function is called. If not specified, the callback is called at every step. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. Examples: Returns: [`~pipelines.ImagePipelineOutput`] or `tuple` rz2`prompt` has to be of type `str` or `list` but is rrr`)rIrCc3K|]8}t|tjjtjf:ywN)rdPILImagerfTensor).0is r z4KandinskyImg2ImgPipeline.__call__..s(Qa:a#))//5<AzInput is in incorrect format: z:. Currently, we only support PIL image and pytorch tensorrO)rCr) text_embedsrF)sampletimestepencoder_hidden_statesadded_cond_kwargsr variance_type)learned learned_range)rHorderT)force_not_quantizer)rVnppilzIOnly the output types `pt`, `pil` and `np` are supported not output_type=)images)/rdrorer3rKrm_execution_devicerrfrrrlrLrIallr8 preprocessr#encoder& set_timestepsrFr>r4num_train_timestepstensorr% in_channelsrr6rR enumerate progress_barsplitrJchunkhasattrrstep prev_sampler2 XLA_AVAILABLExm mark_stepdecodemaybe_free_model_hooks postprocessr)%r9rsrrrrvrrrArBrrtrHrrrrrwrCrur}r~_rrOtimesteps_tensorrPnum_channels_latentstlatent_model_inputr noise_pred variance_prednoise_pred_uncondnoise_pred_textvariance_pred_textstep_idxs% r__call__z!KandinskyImg2ImgPipeline.__call__-s#^ fc "J  %VJQRVW]R^Q_`a a''"77 &4s&:#8<7J7J F13NP_8 4 11 lD ) 99\q9L +T 2$)II.C$K ! &'99:OUV9WL$9$K$KLagh$K$i ! 99&;\%JPQRUU#))&VL %&GEQ5QQ051Ia$q'1I0JKEF  V[\QR4//::1eVL\bcd}226B))""5))4++,Aq+I $$%8$H040B0BCVX`bh0i--dnn33GG(RSVWW,,'8:'EM]McMclrs#yy//;;#FE43I3IJ &&   -vu = & , ,   NN d//0@AB' DAq=XG9q=!9^e 0=| \ )&@"3! #  J+,6,<,%%.?UfCf1gg "YY 4F'GQO --?NN))77;WW * 0 0q1Aq 0 I Ann))# * k #N(:a(? CC1g. O' T   T B8L ##% 1 1hithuvw w$$00 D8O"%00s2J]s T >$Tr) Nrdg333333?g@rNrNrT)"__name__ __module__ __qualname____doc__model_cpu_offload_seqrr rr r r0rFrRrrfr IntTensorrNno_gradrEXAMPLE_DOC_STRINGrrorrrrr>float Generatorrboolr __classcell__)r;s@rr!r!as)&7 &  '  #  !  88, dDN,,||??   4U]]_12<@#& #%&MQ%*GK #J1c49n%J1U\\399??D4FSYY__H]]^J1ll J1 %|| J1 "%T#Y"78 J1J1J1!J1J1J1 #J1E%//43H"HIJJ1c]J18S#u||$rs32 13' .C*))MM   H %&R6X10X1r