
    bit+             
          d dl Z d dlmZmZmZmZmZmZmZ d dl	Z
d dlZd dlZd dlmc mZ d dlmZmZmZmZ ddlmZmZ ddlmZmZ ddlmZmZm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z' dd	l(m)Z) dd
l*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z: ddl;m<Z< ddl=m>Z>  e/       rd dl?m@c mAZB dZCndZC e0j                  eE      ZFdZG	 ddej                  deej                     deJfdZK G d de9e:e!e ee      ZLy)    N)AnyCallableDictListOptionalTupleUnion)CLIPImageProcessorCLIPTextModelCLIPTokenizerCLIPVisionModelWithProjection   )MultiPipelineCallbacksPipelineCallback)PipelineImageInputVaeImageProcessor)FromSingleFileMixinIPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin)AutoencoderKLControlNetModelImageProjectionMultiControlNetModelUNet2DConditionModel)adjust_lora_scale_text_encoder)KarrasDiffusionSchedulers)USE_PEFT_BACKEND	deprecateis_torch_xla_availableloggingreplace_example_docstringscale_lora_layersunscale_lora_layers)empty_device_cacheis_compiled_modulerandn_tensor   )DiffusionPipelineStableDiffusionMixin)StableDiffusionPipelineOutput)StableDiffusionSafetyCheckerTFa  
    Examples:
        ```py
        >>> # !pip install transformers accelerate
        >>> from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler
        >>> from diffusers.utils import load_image
        >>> import numpy as np
        >>> import torch

        >>> init_image = load_image(
        ...     "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png"
        ... )
        >>> init_image = init_image.resize((512, 512))

        >>> generator = torch.Generator(device="cpu").manual_seed(1)

        >>> mask_image = load_image(
        ...     "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_mask.png"
        ... )
        >>> mask_image = mask_image.resize((512, 512))


        >>> def make_canny_condition(image):
        ...     image = np.array(image)
        ...     image = cv2.Canny(image, 100, 200)
        ...     image = image[:, :, None]
        ...     image = np.concatenate([image, image, image], axis=2)
        ...     image = Image.fromarray(image)
        ...     return image


        >>> control_image = make_canny_condition(init_image)

        >>> controlnet = ControlNetModel.from_pretrained(
        ...     "lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16
        ... )
        >>> pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
        ...     "stable-diffusion-v1-5/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
        ... )

        >>> pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
        >>> pipe.enable_model_cpu_offload()

        >>> # generate image
        >>> image = pipe(
        ...     "a handsome man with ray-ban sunglasses",
        ...     num_inference_steps=20,
        ...     generator=generator,
        ...     eta=1.0,
        ...     image=init_image,
        ...     mask_image=mask_image,
        ...     control_image=control_image,
        ... ).images[0]
        ```
encoder_output	generatorsample_modec                     t        | d      r |dk(  r| j                  j                  |      S t        | d      r|dk(  r| j                  j                         S t        | d      r| j                  S t        d      )Nlatent_distsampleargmaxlatentsz3Could not access latents of provided encoder_output)hasattrr1   r2   moder4   AttributeError)r-   r.   r/   s      u/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.pyretrieve_latentsr9   s   st     ~}-+2I))00;;		/K84K))..00		+%%%RSS    c            @           e Zd ZdZdZg dZdgZg dZ	 	 dMded	e	d
e
dedeeee   ee   ef   dededededef fdZ	 	 	 	 dNdeej2                     deej2                     dee   fdZ	 	 	 	 	 dOdeej2                     deej2                     dee   dee   fdZdPdZd Zd Z d Z!d Z"d Z#	 	 	 	 	 	 	 	 	 	 dQd Z$d! Z%	 	 dRd#Z&	 	 	 	 	 	 dSd$Z'd% Z(d&ej2                  d'ejR                  fd(Z*e+d)        Z,e+d*        Z-e+d+        Z.e+d,        Z/e+d-        Z0e+d.        Z1 ejd                          e3e4      ddddddddd/d0dd1dddddddd2ddd3d"ddddd4gfd5ee5ee5   f   d&e6d6e6d7e6d8ee   d9ee   d:ee   d;ed<ed=ed>eee5ee5   f      d?ee   d@ed'eeejR                  eejR                     f      d4eej2                     deej2                     deej2                     dAee6   dBeeej2                        dCee5   dDedEee7e5e8f      dFeeee   f   dGedHeeee   f   dIeeee   f   dee   dJeee9eee7gdf   e:e;f      dKee5   f:dL              Z< xZ=S )T(StableDiffusionControlNetInpaintPipelineaN  
    Pipeline for image inpainting using Stable Diffusion with ControlNet guidance.

    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
    implemented for all pipelines (downloading, saving, running on a particular device, etc.).

    The pipeline also inherits the following loading methods:
        - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
        - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
        - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
        - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
        - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters

    <Tip>

    This pipeline can be used with checkpoints that have been specifically fine-tuned for inpainting
    ([stable-diffusion-v1-5/stable-diffusion-inpainting](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-inpainting))
    as well as default text-to-image Stable Diffusion checkpoints
    ([stable-diffusion-v1-5/stable-diffusion-v1-5](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5)).
    Default text-to-image Stable Diffusion checkpoints might be preferable for ControlNets that have been fine-tuned on
    those, such as [lllyasviel/control_v11p_sd15_inpaint](https://huggingface.co/lllyasviel/control_v11p_sd15_inpaint).

    </Tip>

    Args:
        vae ([`AutoencoderKL`]):
            Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.
        text_encoder ([`~transformers.CLIPTextModel`]):
            Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).
        tokenizer ([`~transformers.CLIPTokenizer`]):
            A `CLIPTokenizer` to tokenize text.
        unet ([`UNet2DConditionModel`]):
            A `UNet2DConditionModel` to denoise the encoded image latents.
        controlnet ([`ControlNetModel`] or `List[ControlNetModel]`):
            Provides additional conditioning to the `unet` during the denoising process. If you set multiple
            ControlNets as a list, the outputs from each ControlNet are added together to create one combined
            additional conditioning.
        scheduler ([`SchedulerMixin`]):
            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
        safety_checker ([`StableDiffusionSafetyChecker`]):
            Classification module that estimates whether generated images could be considered offensive or harmful.
            Please refer to the [model card](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) for
            more details about a model's potential harms.
        feature_extractor ([`~transformers.CLIPImageProcessor`]):
            A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.
    z&text_encoder->image_encoder->unet->vae)safety_checkerfeature_extractorimage_encoderr=   )r4   prompt_embedsnegative_prompt_embedscontrol_imagemaskmasked_image_latentsNTvaetext_encoder	tokenizerunet
controlnet	schedulerr>   r?   requires_safety_checkerc                 Z   t         |           |%|
r#t        j                  d| j                   d       ||t        d      t        |t        t        f      rt        |      }| j                  |||||||||		       t        | dd       r/dt        | j                  j                  j                        dz
  z  nd| _        t#        | j                   	      | _        t#        | j                   d
dd      | _        t#        | j                   dd
      | _        | j+                  |
       y )Nz)You have disabled the safety checker for a   by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .zMake sure to define a feature extractor when loading {self.__class__} if you want to use the safety checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead.)	rE   rF   rG   rH   rI   rJ   r=   r>   r?   rE   r(         )vae_scale_factorFT)rO   do_normalizedo_binarizedo_convert_grayscale)rO   do_convert_rgbrP   )rK   )super__init__loggerwarning	__class__
ValueError
isinstancelisttupler   register_modulesgetattrlenrE   configblock_out_channelsrO   r   image_processormask_processorcontrol_image_processorregister_to_config)selfrE   rF   rG   rH   rI   rJ   r=   r>   r?   rK   rX   s              r8   rU   z1StableDiffusionControlNetInpaintPipeline.__init__   sD    	!&=NN;DNN;K Lj j %*;*Cx 
 j4-0-j9J%!)/' 	 
	
 W^^bdikoVpc$((//*L*L&MPQ&Q Rvw0$BWBWX/!22TXos
 (9!224V[(
$ 	8OPr:   r@   rA   
lora_scalec	                     d}
t        dd|
d        | j                  d	||||||||d|	}t        j                  |d   |d   g      }|S )
Nz`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple.z_encode_prompt()1.0.0Fstandard_warn)promptdevicenum_images_per_promptdo_classifier_free_guidancenegative_promptr@   rA   rg   rM   r    )r   encode_prompttorchcat)rf   rl   rm   rn   ro   rp   r@   rA   rg   kwargsdeprecation_messageprompt_embeds_tuples               r8   _encode_promptz7StableDiffusionControlNetInpaintPipeline._encode_prompt   s}     a$g/BRWX0d00 

"7(C+'#9!

 

 		#6q#9;Nq;Q"RSr:   	clip_skipc
                 H
   |Jt        | t              r:|| _        t        st	        | j
                  |       nt        | j
                  |       |t        |t              rd}
n-|t        |t              rt        |      }
n|j                  d   }
|t        | t              r| j                  || j                        }| j                  |d| j                  j                  dd      }|j                  }| j                  |dd	      j                  }|j                  d
   |j                  d
   k\  rt!        j"                  ||      sj| j                  j%                  |dd| j                  j                  dz
  d
f         }t&        j)                  d| j                  j                   d|        t+        | j
                  j,                  d      r<| j
                  j,                  j.                  r|j0                  j3                  |      }nd}|	(| j                  |j3                  |      |      }|d   }nT| j                  |j3                  |      |d      }|d
   |	dz       }| j
                  j4                  j7                  |      }| j
                  | j
                  j8                  }n/| j:                  | j:                  j8                  }n|j8                  }|j3                  ||      }|j                  \  }}}|j=                  d|d      }|j?                  ||z  |d
      }|rm|j|dg|
z  }n|:tA        |      tA        |      ur$tC        dtA        |       dtA        |       d      t        |t              r|g}n1|
t        |      k7  r!tE        d| dt        |       d| d|
 d	      |}t        | t              r| j                  || j                        }|j                  d   }| j                  |d|dd      }t+        | j
                  j,                  d      r<| j
                  j,                  j.                  r|j0                  j3                  |      }nd}| j                  |j                  j3                  |      |      }|d   }|rK|j                  d   }|j3                  ||      }|j=                  d|d      }|j?                  |
|z  |d
      }| j
                  ,t        | t              rt        rtG        | j
                  |       ||fS )a  
        Encodes the prompt into text encoder hidden states.

        Args:
            prompt (`str` or `List[str]`, *optional*):
                prompt to be encoded
            device: (`torch.device`):
                torch device
            num_images_per_prompt (`int`):
                number of images that should be generated per prompt
            do_classifier_free_guidance (`bool`):
                whether to use classifier free guidance or not
            negative_prompt (`str` or `List[str]`, *optional*):
                The prompt or prompts not to guide the image generation. If not defined, one has to pass
                `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
                less than `1`).
            prompt_embeds (`torch.Tensor`, *optional*):
                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
                provided, text embeddings will be generated from `prompt` input argument.
            negative_prompt_embeds (`torch.Tensor`, *optional*):
                Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
                weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
                argument.
            lora_scale (`float`, *optional*):
                A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
            clip_skip (`int`, *optional*):
                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
                the output of the pre-final layer will be used for computing the prompt embeddings.
        NrM   r   
max_lengthTpt)paddingr{   
truncationreturn_tensorslongest)r}   r   z\The following part of your input was truncated because CLIP can only handle sequences up to z	 tokens: use_attention_mask)attention_mask)r   output_hidden_states)dtyperm    z?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)$rZ   r   _lora_scaler   r   rF   r#   strr[   r_   shaper   maybe_convert_promptrG   model_max_length	input_idsrs   equalbatch_decoderV   rW   r5   r`   r   r   to
text_modelfinal_layer_normr   rH   repeatviewtype	TypeErrorrY   r$   )rf   rl   rm   rn   ro   rp   r@   rA   rg   ry   
batch_sizetext_inputstext_input_idsuntruncated_idsremoved_textr   prompt_embeds_dtypebs_embedseq_len_uncond_tokensr{   uncond_inputs                          r8   rr   z6StableDiffusionControlNetInpaintPipeline.encode_prompt  s6   V !j7U&V)D $.t/@/@*M!$"3"3Z@*VS"9JJvt$<VJ&,,Q/J $ ;<2264>>J..$>>::# ) K )22N"nnVYW[n\ffO$$R(N,@,@,DDU[[N  $~~::#At~~'F'F'JR'O$OP  778	,Q
 t((//1EF4K\K\KcKcKvKv!,!;!;!>!>v!F!%  $ 1 1.2C2CF2K\j 1 k -a 0 $ 1 1"%%f-ncg !2 ! !.b 1IM2B C
 !% 1 1 < < M Mm \("&"3"3"9"9YY""&))//"/"5"5%((/B6(R,22'1%,,Q0EqI%**86K+KWVXY '+A+I&!#z 1#VD<Q(QUVZ[jVkUl mV~Q(  OS1!0 1s?33 )/)::J3K_J` ax/
| <33  !0 $ ;< $ 9 9- X&,,Q/J>>$%# * L t((//1EF4K\K\KcKcKvKv!-!<!<!?!?!G!%%)%6%6&&))&1- &7 &" &<A%>"&,2215G%;%>%>EXag%>%h"%;%B%B1F[]^%_"%;%@%@NcAcelnp%q"($ >?DT#D$5$5zB444r:   c                 |   t        | j                  j                               j                  }t	        |t
        j                        s| j                  |d      j                  }|j                  ||      }|r}| j                  |d      j                  d   }|j                  |d      }| j                  t        j                  |      d      j                  d   }|j                  |d      }||fS | j                  |      j                  }|j                  |d      }t        j                  |      }	||	fS )	Nr|   r   rm   r   T)r   r   dim)nextr?   
parametersr   rZ   rs   Tensorr>   pixel_valuesr   hidden_statesrepeat_interleave
zeros_likeimage_embeds)
rf   imagerm   rn   r   r   image_enc_hidden_statesuncond_image_enc_hidden_statesr   uncond_image_embedss
             r8   encode_imagez5StableDiffusionControlNetInpaintPipeline.encode_image  sD   T''2245;;%.**5*FSSEe4&*&8&8UY&8&Z&h&hik&l#&=&O&OPekl&O&m#-1-?-?  'd .@ .mB. * .L-]-]%1 .^ .* +,JJJ--e4AAL'99:OUV9WL"'"2"2<"@!444r:   c                    g }|rg }|t        |t              s|g}t        |      t        | j                  j                  j
                        k7  rBt        dt        |       dt        | j                  j                  j
                         d      t        || j                  j                  j
                        D ]`  \  }}	t        |	t               }
| j                  ||d|
      \  }}|j                  |d d d f          |sIj                  |d d d f          b n?|D ]:  }|r%|j                  d      \  }}j                  |       |j                  |       < g }t        |      D ]|  \  }}t        j                  |g|z  d      }|r7t        j                  |   g|z  d      }t        j                  ||gd      }|j                  |      }|j                  |       ~ |S )	NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got  images and z IP Adapters.rM   r(   r   r   rm   )rZ   r[   r_   rH   encoder_hid_projimage_projection_layersrY   zipr   r   appendchunk	enumeraters   rt   r   )rf   ip_adapter_imageip_adapter_image_embedsrm   rn   ro   r   negative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsis                 r8   prepare_ip_adapter_image_embedszHStableDiffusionControlNetInpaintPipeline.prepare_ip_adapter_image_embeds  sY    &$&!"*.5$4#5 #$DII,F,F,^,^(__ abefvbwax  yE  FI  JN  JS  JS  Jd  Jd  J|  J|  F}  E~  ~K  L  >A $))"<"<"T"T> 
X9')9 +55E*W&W#DHDUDU+VQ8KEA#%A ##$7a$@A.)001MdTUg1VW
X (? 9#.H[HaHabcHdE02E)001MN##$78	9 #%&/&= 	@"A""'))-@,ADY,Y_`"a*/4yy:OPQ:R9SVk9kqr/s,&+ii1MOb0cij&k#"5"8"8"8"G#**+>?	@ '&r:   c                 l   | j                   d }||fS t        j                  |      r| j                  j	                  |d      }n| j                  j                  |      }| j                  |d      j                  |      }| j                  ||j                  j                  |            \  }}||fS )Npil)output_typer|   r   )images
clip_input)	r=   rs   	is_tensorrb   postprocessnumpy_to_pilr>   r   r   )rf   r   rm   r   has_nsfw_conceptfeature_extractor_inputsafety_checker_inputs          r8   run_safety_checkerz;StableDiffusionControlNetInpaintPipeline.run_safety_checker  s    &# &&& u%*.*>*>*J*J5^c*J*d'*.*>*>*K*KE*R'#'#9#9:Qbf#9#g#j#jkq#r &*&9&9)=)J)J)M)Me)T ': '#E# &&&r:   c                 `   d}t        dd|d       d| j                  j                  j                  z  |z  }| j                  j	                  |d      d   }|d	z  d
z   j                  dd      }|j                         j                  dd	dd      j                         j                         }|S )Nz{The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) insteaddecode_latentsri   Frj   rM   )return_dictr   r(         ?r   )
r   rE   r`   scaling_factordecodeclampcpupermutefloatnumpy)rf   r4   rv   r   s       r8   r   z7StableDiffusionControlNetInpaintPipeline.decode_latents*  s     \"G-@PUVdhhoo444w>U;A>S''1-		##Aq!Q/557==?r:   c                 V   dt        t        j                  | j                  j                        j
                  j                               v }i }|r||d<   dt        t        j                  | j                  j                        j
                  j                               v }|r||d<   |S )Netar.   )setinspect	signaturerJ   stepr   keys)rf   r.   r   accepts_etaextra_step_kwargsaccepts_generators         r8   prepare_extra_step_kwargszBStableDiffusionControlNetInpaintPipeline.prepare_extra_step_kwargs6  s     s7#4#4T^^5H5H#I#T#T#Y#Y#[\\'*e$ (3w/@/@ATAT/U/`/`/e/e/g+hh-6k*  r:   c                 N   t        t        ||z        |      }t        ||z
  d      }| j                  j                  || j                  j
                  z  d  }t        | j                  d      r2| j                  j                  || j                  j
                  z         |||z
  fS )Nr   set_begin_index)minintmaxrJ   	timestepsorderr5   r   )rf   num_inference_stepsstrengthrm   init_timestept_startr   s          r8   get_timestepsz6StableDiffusionControlNetInpaintPipeline.get_timestepsH  s    C 3h >?ATU)M91=NN,,Wt~~7K7K-K-MN	4>>#45NN**7T^^5I5I+IJ-777r:         ?        c                 2    ||dz  dk7  s
||dz  dk7  rt        d| d| d      |0t        |t              r|dk  rt        d| dt        |       d      |Lt	         fd|D              s8t        d	 j
                   d
|D cg c]  }| j
                  vs| c}       ||	t        d| d|	 d      ||	t        d      |7t        |t              s't        |t              st        dt        |             ||
t        d| d|
 d      |	A|
?|	j                  |
j                  k7  r&t        d|	j                   d|
j                   d      |t        |t        j                  j                        st        dt        |       d      t        |t        j                  j                        st        dt        |       d      |dk7  rt        d| d      t         j                  t              rRt        |t              rBt        j                  dt         j                  j                          dt        |       d       t#        t$        d      xr8 t         j                  t&        j(                  j*                  j,                        }t         j                  t.              s&|r8t         j                  j0                  t.              r j3                  |||	       nt         j                  t              s&|rt         j                  j0                  t              rt        |t              st5        d      t7        d |D              rt        d      t        |      t         j                  j                         k7  r8t        dt        |       d t         j                  j                          d!      |D ]  } j3                  |||	        nJ t         j                  t.              s&|r?t         j                  j0                  t.              rt        |t8              st5        d"      t         j                  t              s&|rt         j                  j0                  t              rst        |t              rt7        d# |D              rSt        d      t        |t              r8t        |      t         j                  j                         k7  rt        d$      J t        |      t        |      k7  r$t        d%t        |       d&t        |       d'      t         j                  t              rt        |      t         j                  j                         k7  r[t        d(| d)t        |       d*t         j                  j                          d+t         j                  j                          d	      t;        ||      D ]D  \  }}||k\  rt        d,| d-| d      |d.k  rt        d,| d/      |d0kD  s7t        d1| d2       ||t        d3      |Ut        |t              st        d4t        |             |d   j<                  d5vrt        d6|d   j<                   d7      y y c c}w )8NrN   r   z7`height` and `width` have to be divisible by 8 but are z and r   z5`callback_steps` has to be a positive integer but is z	 of type c              3   :   K   | ]  }|j                   v   y wN)_callback_tensor_inputs).0krf   s     r8   	<genexpr>zHStableDiffusionControlNetInpaintPipeline.check_inputs.<locals>.<genexpr>p  s#      F
23A---F
s   z2`callback_on_step_end_tensor_inputs` has to be in z, but found zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z2`prompt` has to be of type `str` or `list` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` zJThe image should be a PIL image when inpainting mask crop, but is of type zOThe mask image should be a PIL image when inpainting mask crop, but is of type r   z@The output type should be PIL when inpainting mask crop, but is z	You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionz5For multiple controlnets: `image` must be type `list`c              3   <   K   | ]  }t        |t                y wr   rZ   r[   r   r   s     r8   r   zHStableDiffusionControlNetInpaintPipeline.check_inputs.<locals>.<genexpr>  s     8QZ4(8   zEA single batch of multiple conditionings are supported at the moment.zbFor multiple controlnets: `image` must have the same length as the number of controlnets, but got r   z ControlNets.zLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.c              3   <   K   | ]  }t        |t                y wr   r  r  s     r8   r   zHStableDiffusionControlNetInpaintPipeline.check_inputs.<locals>.<genexpr>  s     Rqz!T*Rr  zFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: r   z can't be smaller than 0.r   zcontrol guidance end: z can't be larger than 1.0.zProvide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined.z:`ip_adapter_image_embeds` has to be of type `list` but is )r      zF`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is D)rY   rZ   r   r   allr   r   r[   r   PILImagerI   r   rV   rW   r_   netsr5   Frs   _dynamo
eval_frameOptimizedModuler   	_orig_modcheck_imager   anyr   r   ndim)rf   rl   r   
mask_imageheightwidthcallback_stepsr   rp   r@   rA   r   r   controlnet_conditioning_scalecontrol_guidance_startcontrol_guidance_end"callback_on_step_end_tensor_inputspadding_mask_cropr   is_compiledimage_startends   `                      r8   check_inputsz5StableDiffusionControlNetInpaintPipeline.check_inputsS  s   ( &1*/U5F5ST9XY>VW]V^^cdicjjklmm%z.#/NR`deReGGW X(), 
 .9# F
7YF
 C
 DTEaEaDbbn  |^  pHvw  bc  ko  kG  kG  bGpq  pH  oI  J  -";08N}o ^0 0  ^ 5w  FC)@TZ\`IaQRVW]R^Q_`aa&+A+M9/9J K*++]_ 
 $)?)K""&<&B&BB --:-@-@,A B.445Q8  (eSYY__5 `aefkal`mmno  j#))//: Z(),  e# #cdocppq!rss doo';<&$'DOO$8$8 9::[\_`f\g[hST a!?@ 
ZOOU]]55EEF
 t84??44oFUFM:t(<=4??446JKeT* WXX 8%88 !hiiUs4??#7#788 xy|  ~C  zD  yE  EQ  RU  VZ  Ve  Ve  Vj  Vj  Rk  Ql  ly  z    @  ?@ 5 t84??44oF;UC noot(<=4??446JK7>R4QRR$%lmm94@SIfEgkn$$l F !D 
 5%&#.B*CC/4J0K/LLvwz  |P  xQ  wR  R[  \  doo';<)*c$//2F2F.GG 01G0HcRhNiMj  kC  DG  HL  HW  HW  H\  H\  D]  C^  ^K  LO  PT  P_  P_  Pd  Pd  Le  Kf  fg  h  46JK 	[JE3| .ug5ijminnop  s{ #;E7B[!\]]Sy #9#>X!YZZ	[ ',C,O ^  #.5t< PQUVmQnPop  )+00> \]tuv]w]|]|\}}~  ? /O pHs   ZZc                    t        |t        j                  j                        }t        |t        j                        }t        |t
        j                        }t        |t              xr' t        |d   t        j                  j                        }t        |t              xr t        |d   t        j                        }t        |t              xr t        |d   t
        j                        }	|s!|s|s|s|s|	st        dt        |             |rd}
nt        |      }
|t        |t              rd}n/|t        |t              rt        |      }n||j                  d   }|
dk7  r|
k7  rt        d|
 d|       y y )Nr   zimage must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is rM   zdIf image batch size is not 1, image batch size must be same as prompt batch size. image batch size: z, prompt batch size: )rZ   r  r	  rs   r   npndarrayr[   r   r   r_   r   r   rY   )rf   r   rl   r@   image_is_pilimage_is_tensorimage_is_npimage_is_pil_listimage_is_tensor_listimage_is_np_listimage_batch_sizeprompt_batch_sizes               r8   r  z4StableDiffusionControlNetInpaintPipeline.check_image  s   !%9$UELL9 

3&ud3]
58SYY__8])%6]:eAhPUP\P\;]%eT2Wz%(BJJ7W #%($ f  gk  lq  gr  fs  t   "5z*VS"9 !Jvt$< #F& - 3 3A 6q %59J%Jv  xH  wI  I^  _p  ^q  r  &K r:   Fc                 :   | j                   j                  |||||	      j                  t        j                        }|j
                  d   }|dk(  r|}n|}|j                  |d      }|j                  ||      }|
r|st        j                  |gdz        }|S )Nr  r  crops_coordsresize_moder   r   rM   r   r   r(   )rd   
preprocessr   rs   float32r   r   rt   )rf   r   r  r  r   rn   rm   r   r.  r/  ro   
guess_moder*  	repeat_bys                 r8   prepare_control_imagez>StableDiffusionControlNetInpaintPipeline.prepare_control_image+  s     ,,77&LVa 8 

"5=="
! 	 !;;q>q "I .I''	q'9e4&zIIugk*Er:   c                    ||t        |      | j                  z  t        |      | j                  z  f}t        |t              r)t	        |      |k7  rt        dt	        |       d| d      |	|
|st        d      |s|a|s_|	j                  ||      }	|	j                  d   dk(  r|	}n| j                  |	|      }|j                  ||j                  d	   z  ddd      }|Nt        ||||
      }|r|n| j                  j                  ||
      }|r|| j                  j                  z  n|}n*|j                  |      }|| j                  j                  z  }|f}|r||fz  }|r|fz  }|S )Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.zSince strength < 1. initial latents are to be initialised as a combination of Image + Noise.However, either the image or the noise timestep has not been provided.r   rM   r  )r   r.   r   )r.   rm   r   )r   rO   rZ   r[   r_   rY   r   r   _encode_vae_imager   r'   rJ   	add_noiseinit_noise_sigma)rf   r   num_channels_latentsr  r  r   rm   r.   r4   r   timestepis_strength_maxreturn_noisereturn_image_latentsr   image_latentsnoiseoutputss                     r8   prepare_latentsz8StableDiffusionControlNetInpaintPipeline.prepare_latentsN  s   "  K4000J$///	
 i&3y>Z+GA#i.AQ R&<'gi 
 MX-Y 
  GOOHHF%H8E{{1~" % $ 6 6Ui 6 X)00}?R?RST?U1UWXZ[]^_M? )FRWXE.eDNN4L4L]\ack4lGCRg ? ??X_GJJv&Ednn===G*xG''Gr:   c
                 v   t         j                  j                  j                  ||| j                  z  || j                  z  f      }|j                  ||      }|j                  ||      }|j                  d   dk(  r|}
n| j                  ||      }
|j                  d   |k  rX||j                  d   z  dk(  st        d| d|j                  d    d	      |j                  ||j                  d   z  ddd      }|
j                  d   |k  rX||
j                  d   z  dk(  st        d
| d|
j                  d    d      |
j                  ||
j                  d   z  ddd      }
|	rt        j                  |gdz        n|}|	rt        j                  |
gdz        n|
}
|
j                  ||      }
||
fS )N)sizer   rM   r  r.   r   zvThe passed mask and the required batch size don't match. Masks are supposed to be duplicated to a total batch size of z, but zo masks were passed. Make sure the number of masks that you pass is divisible by the total requested batch size.zyThe passed images and the required batch size don't match. Images are supposed to be duplicated to a total batch size of zq images were passed. Make sure the number of images that you pass is divisible by the total requested batch size.r(   )rs   nn
functionalinterpolaterO   r   r   r7  rY   r   rt   )rf   rC   masked_imager   r  r  r   rm   r.   ro   rD   s              r8   prepare_mask_latentsz=StableDiffusionControlNetInpaintPipeline.prepare_mask_latents  s    xx""..$"7"77$BWBW9WX / 
 wwfEw2#fEBa A%#/ #'#9#9,R[#9#\  ::a=:%

1-2 ..8\

1 O^^ 
 ;;zTZZ]:Aq!DD%%a(:5 4 : :1 ==B 11;FCWC]C]^_C`Ba btt 
 $8#>#>zMaMgMghiMj?jlmoprs#t (Cuyy$!$5PEII+,q01Vj 	
  466fE6R)))r:   r   r.   c                    t        |t              rjt        |j                  d         D cg c]1  }t	        | j
                  j                  |||dz          ||         3 }}t        j                  |d      }n&t	        | j
                  j                  |      |      }| j
                  j                  j                  |z  }|S c c}w )Nr   rM   rE  r   )rZ   r[   ranger   r9   rE   encoders   rt   r`   r   )rf   r   r.   r   r?  s        r8   r7  z:StableDiffusionControlNetInpaintPipeline._encode_vae_image  s    i& u{{1~. !q1q51A!BiXYl[M  "IIm;M,TXX__U-CyYM66Fs   6Cc                     | j                   S r   _guidance_scalerf   s    r8   guidance_scalez7StableDiffusionControlNetInpaintPipeline.guidance_scale  s    ###r:   c                     | j                   S r   )
_clip_skiprQ  s    r8   ry   z2StableDiffusionControlNetInpaintPipeline.clip_skip      r:   c                      | j                   dkD  S )NrM   rO  rQ  s    r8   ro   zDStableDiffusionControlNetInpaintPipeline.do_classifier_free_guidance  s    ##a''r:   c                     | j                   S r   )_cross_attention_kwargsrQ  s    r8   cross_attention_kwargsz?StableDiffusionControlNetInpaintPipeline.cross_attention_kwargs  s    +++r:   c                     | j                   S r   )_num_timestepsrQ  s    r8   num_timestepsz6StableDiffusionControlNetInpaintPipeline.num_timesteps  s    """r:   c                     | j                   S r   )
_interruptrQ  s    r8   	interruptz2StableDiffusionControlNetInpaintPipeline.interrupt  rU  r:   2   g      @rM   r   r   r4   rl   r  rB   r  r  r  r   r   rR  rp   rn   r   r   r   r   r   rY  r  r3  r  r  callback_on_step_endr  c                     |j                  dd      }|j                  dd      } |t        ddd       | t        ddd       t        |t        t        f      r|j
                  }t        | j                        r| j                  j                  n| j                  }!t        |t              s t        |t              rt        |      |gz  }nt        |t              s t        |t              rt        |      |gz  }nSt        |t              sCt        |t              s3t        |!t              rt        |!j                        nd}"|"|gz  |"|gz  }}| j                  |||||| |||||||||||       |
| _        || _        || _        d| _        |t        |t$              rd}#n-|t        |t              rt        |      }#n|j&                  d	   }#|B| j(                  j+                  |||      \  }}| j,                  j/                  ||||
      }$d}%nd}$d}%| j0                  }&t        |!t              r)t        |t2              r|gt        |!j                        z  }t        |!t4              r|!j6                  j8                  n"|!j                  d	   j6                  j8                  }'|xs |'}| j:                  | j:                  j=                  dd      nd}(| j?                  ||&|| j@                  ||||(| jB                  	      \  }}| j@                  rtE        jF                  ||g      }||"| jI                  |||&|#|z  | j@                        })t        |!t4              r4| jK                  ||||#|z  ||&|!jL                  |$|%| j@                  |      }nbt        |!t              rPg }*|D ]F  }+| jK                  |+|||#|z  ||&|!jL                  |$|%| j@                  |      }+|*jO                  |+       H |*}nJ |},| j(                  jQ                  ||||$|%      }-|-jS                  tD        jT                        }-| j,                  jQ                  ||||%|$      }.|-|.dk  z  }/|-j&                  \  }0}0}}| jV                  jY                  |	|&       | j[                  |	||&      \  }1}	|1dd j]                  |#|z        }2|dk(  }3t        |1      | _/        | j`                  j6                  jb                  }4| jd                  j6                  jf                  }5|5dk(  }6| ji                  |#|z  |4|||jL                  |&|||-|2|3d|6      }7|6r|7\  }}8}9n|7\  }}8| jk                  |.|/|#|z  |||jL                  |&|| j@                  	      \  }.}:| jm                  ||      };||d)ind}<g }=to        t        |1            D ]w  }>tq        ||      D ?@cg c]8  \  }?}@dt3        |>t        |1      z  |?k  xs |>dz   t        |1      z  @kD        z
  : }A}?}@|=jO                  t        |!t4              rAd	   nA       y t        |1      |	| jV                  jr                  z  z
  }B| ju                  |	      5 }Ctw        |1      D ]  \  }>}D| jx                  r| j@                  rtE        jF                  |gdz        n|}E| jV                  j{                  |ED      }E|r?| j@                  r3|}F| jV                  j{                  |FD      }F|j}                  d      d   }GnE}F|}Gt        |=|>   t              r%tq        ||=|>         D H?cg c]
  \  }H}?|H|?z   }I}H}?n|}Jt        |Jt              rJd	   }JJ|=|>   z  }I| j                  FDG|I|d      \  }K}L|rm| j@                  raKD Mcg c],  }MtE        jF                  tE        j~                  |M      |Mg      . }K}MtE        jF                  tE        j~                  L      |Lg      }L|5dk(  rtE        jF                  E|.|:gd      }E| je                  ED|| j:                  KL|<d       d	   }N| j@                  rNj}                  d      \  }O}P|O|
|P|Oz
  z  z   }N | jV                  j                  ND|fi |;d!did	   }|5dk(  r}9}Q| j@                  r|.j}                  d      \  }R}0n|.}R|>t        |1      dz
  k  r9|1|>dz      }S| jV                  j                  Q|8tE        j                  |Sg            }QdRz
  Qz  |R|z  z   }|li }T|D ]  }Ut               |U   T|U<     || |>DT      }V|Vj                  d"|      }|Vj                  d#|      }|Vj                  d$|      }|Vj                  d%|      }|>t        |1      dz
  k(  s'|>dz   BkD  r]|>dz   | jV                  jr                  z  d	k(  r>Cj                          |,|>| z  d	k(  r$|>t        | jV                  d&d      z  }W ||WD|       t        st        j                           	 ddd       t        | d'      rL| j                  @| jd                  jS                  d(       | j                  jS                  d(       t                |d)k(  sc| j`                  j                  || j`                  j6                  j                  z  d|*      d	   }| j                  ||&|jL                        \  }}Xn|}d}XXdg|j&                  d	   z  }YnXD Zcg c]  }Z|Z  }Y}Z| j(                  j                  ||Y+      }|+|D >cg c]   }>| j(                  j                  ||,|>|$      " }}>| j                          |s|XfS t        |X,      S c c}@}?w c c}?}Hw c c}Mw # 1 sw Y   sxY wc c}Zw c c}>w )-u)  
        The call function to the pipeline for generation.

        Args:
            prompt (`str` or `List[str]`, *optional*):
                The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.
            image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`,
                    `List[PIL.Image.Image]`, or `List[np.ndarray]`):
                `Image`, NumPy array or tensor representing an image batch to be used as the starting point. For both
                NumPy array and PyTorch tensor, the expected value range is between `[0, 1]`. If it's a tensor or a
                list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a NumPy array or
                a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`. It can also accept image
                latents as `image`, but if passing latents directly it is not encoded again.
            mask_image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`,
                    `List[PIL.Image.Image]`, or `List[np.ndarray]`):
                `Image`, NumPy array or tensor representing an image batch to mask `image`. White pixels in the mask
                are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a
                single channel (luminance) before use. If it's a NumPy array or PyTorch tensor, it should contain one
                color channel (L) instead of 3, so the expected shape for PyTorch tensor would be `(B, 1, H, W)`, `(B,
                H, W)`, `(1, H, W)`, `(H, W)`. And for NumPy array, it would be for `(B, H, W, 1)`, `(B, H, W)`, `(H,
                W, 1)`, or `(H, W)`.
            control_image (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`,
                    `List[List[torch.Tensor]]`, or `List[List[PIL.Image.Image]]`):
                The ControlNet input condition to provide guidance to the `unet` for generation. If the type is
                specified as `torch.Tensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be accepted
                as an image. The dimensions of the output image defaults to `image`'s dimensions. If height and/or
                width are passed, `image` is resized accordingly. If multiple ControlNets are specified in `init`,
                images must be passed as a list such that each element of the list can be correctly batched for input
                to a single ControlNet.
            height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
                The height in pixels of the generated image.
            width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
                The width in pixels of the generated image.
            padding_mask_crop (`int`, *optional*, defaults to `None`):
                The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to
                image and mask_image. If `padding_mask_crop` is not `None`, it will first find a rectangular region
                with the same aspect ration of the image and contains all masked area, and then expand that area based
                on `padding_mask_crop`. The image and mask_image will then be cropped based on the expanded area before
                resizing to the original image size for inpainting. This is useful when the masked area is small while
                the image is large and contain information irrelevant for inpainting, such as background.
            strength (`float`, *optional*, defaults to 1.0):
                Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a
                starting point and more noise is added the higher the `strength`. The number of denoising steps depends
                on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising
                process runs for the full number of iterations specified in `num_inference_steps`. A value of 1
                essentially ignores `image`.
            num_inference_steps (`int`, *optional*, defaults to 50):
                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
                expense of slower inference.
            guidance_scale (`float`, *optional*, defaults to 7.5):
                A higher guidance scale value encourages the model to generate images closely linked to the text
                `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.
            negative_prompt (`str` or `List[str]`, *optional*):
                The prompt or prompts to guide what to not include in image generation. If not defined, you need to
                pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).
            num_images_per_prompt (`int`, *optional*, defaults to 1):
                The number of images to generate per prompt.
            eta (`float`, *optional*, defaults to 0.0):
                Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only
                applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.
            generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
                A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
                generation deterministic.
            latents (`torch.Tensor`, *optional*):
                Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
                tensor is generated by sampling using the supplied random `generator`.
            prompt_embeds (`torch.Tensor`, *optional*):
                Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
                provided, text embeddings are generated from the `prompt` input argument.
            negative_prompt_embeds (`torch.Tensor`, *optional*):
                Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
                not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
            ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
            ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*):
                Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
                IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
                contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
                provided, embeddings are computed from the `ip_adapter_image` input argument.
            output_type (`str`, *optional*, defaults to `"pil"`):
                The output format of the generated image. Choose between `PIL.Image` or `np.array`.
            return_dict (`bool`, *optional*, defaults to `True`):
                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
                plain tuple.
            cross_attention_kwargs (`dict`, *optional*):
                A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in
                [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
            controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 0.5):
                The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added
                to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set
                the corresponding scale as a list.
            guess_mode (`bool`, *optional*, defaults to `False`):
                The ControlNet encoder tries to recognize the content of the input image even if you remove all
                prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended.
            control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0):
                The percentage of total steps at which the ControlNet starts applying.
            control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0):
                The percentage of total steps at which the ControlNet stops applying.
            clip_skip (`int`, *optional*):
                Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
                the output of the pre-final layer will be used for computing the prompt embeddings.
            callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
                A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of
                each denoising step during the inference. with the following arguments: `callback_on_step_end(self:
                DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a
                list of all tensors as specified by `callback_on_step_end_tensor_inputs`.
            callback_on_step_end_tensor_inputs (`List`, *optional*):
                The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
                will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
                `._callback_tensor_inputs` attribute of your pipeline class.

        Examples:

        Returns:
            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
                If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,
                otherwise a `tuple` is returned where the first element is a list with the generated images and the
                second element is a list of `bool`s indicating whether the corresponding generated image contains
                "not-safe-for-work" (nsfw) content.
        callbackNr  ri   zjPassing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`zpPassing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`rM   Fr   )padfilldefaultscale)r@   rA   rg   ry   )r   r  r  r   rn   rm   r   r.  r/  ro   r3  r-  r0  )r  r  r/  r.  r   r   )r   r   rm   r   r  T)r   r;  r<  r=  r>  r   )totalr(   )encoder_hidden_statescontrolnet_condconditioning_scaler3  r   	   r   )ri  rY  down_block_additional_residualsmid_block_additional_residualadded_cond_kwargsr   r   r4   r@   rA   rB   r   final_offload_hookr   latent)r   r.   )r   do_denormalize)r   nsfw_content_detected)Spopr   rZ   r   r   tensor_inputsr&   rI   r  r[   r_   r   r
  r   rP  rT  rX  r^  r   r   rb   get_default_height_widthrc   get_crop_region_execution_devicer   r   r`   global_pool_conditionsrY  getrr   ro   ry   rs   rt   r   r5  r   r   r1  r   r2  rJ   set_timestepsr   r   r[  rE   latent_channelsrH   in_channelsrB  rJ  r   rL  r   r   progress_barr   r_  scale_model_inputr   r   r   r8  tensorlocalsupdater^   XLA_AVAILABLExm	mark_stepr5   rp  r%   r   r   r   r   apply_overlaymaybe_free_model_hooksr+   )[rf   rl   r   r  rB   r  r  r  r   r   rR  rp   rn   r   r.   r4   r@   rA   r   r   r   r   rY  r  r3  r  r  ry   ra  r  ru   rc  r  rI   multr   r.  r/  rm   ry  text_encoder_lora_scaler   control_imagescontrol_image_original_image
init_imagerC   rI  r   r   latent_timestepr<  r:  num_channels_unetr>  latents_outputsr@  r?  rD   r   ro  controlnet_keepr   sekeepsnum_warmup_stepsr~  tlatent_model_inputcontrol_model_inputcontrolnet_prompt_embedsc
cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sampled
noise_prednoise_pred_uncondnoise_pred_textinit_latents_proper	init_masknoise_timestepcallback_kwargsr   callback_outputsstep_idxr   rr  has_nsfws[                                                                                              r8   __call__z1StableDiffusionControlNetInpaintPipeline.__call__  s   | ::j$/$4d;|
 %  C *-=?U,VW1E1S1S.2DT__2UT__..[_[j[j
 0$7JG[]a<b%()=%>BXAY%Y"0$7JG]_c<d#&'=#>BVAW#W 2D9*MacgBh+5jBV+W3z']^D.//,-- %9" 	"#)" .#	
(  .#'=$ *VS"9JJvt$<VJ&,,Q/J( 00II%QWY^_MFE..>>z5RX^o>pL KL#K''j"67JGdfk<l-J,KcR\RaRaNb,b) *o6 44#**AA 	
  9#9
 ?C>Y>Y>eD''++GT:ko 	  150B0B!,,'#9.nn 1C 
1
-- ++!II'=}&MNM'+B+N?? '2200L j/2 66#%(==&; &&)',0,L,L% 7 M 
$89N"/ 6!%!;!;(!),AA*?!$**!- +040P0P) "< " %%n56" +M5 ))44&LVa 5 

  ]]]7
""--vUZf . 
 "TCZ0(..1fe 	$$%8$H)-);); 3hv *< *
&	& $BQ-..z<Q/QR"c/!)n  $xx>> II,,880A5.... $+!5 / 
   ,;)GUM,NGU &*%>%>..,,
&
"" !::9cJ
  +/F/R \* 	 s9~& 	cA   68LMAq eAI.2Rq1uI6NQR6RSSE  ""z*o/V58\ab	c y>,?$..BVBV,VV%89 e	#\!), d#1>> BFAaAaUYYy1}%=gn"%)^^%E%EFXZ[%\" $"B"B*1'*...*J*JK^`a*b'/</B/B1/Ea/H,*<'/<,oa0$7478UWfghWi4j!kDAq!a%!kJ!k,I)!"7>0Ea0H-!69K!KJ?C'*B$1'1) % @O @<&(< $"B"B \r-rVWeii9I9I!9La8P.Q-r*-r+099e6F6FG[6\^r5s+t( %)).4FNb3cij)k&!YY&*7+/+F+F4J2F&7 % ' 	 	
 339C9I9I!9L6%!2^YjGj5k!kJ .$..--j!WmHYmglmnop$)*7'77'+zz!}	1$(	3y>A--)21q5)9.2nn.F.F/nEU8V/+  !9}0CCiRYFYYG'3&(O? 9-3Xa[*9';D!Q'X$.229gFG$4$8$8-$XM-=-A-ABZ\r-s*$4$8$8-$XM I**A9I/IqSTuX\XfXfXlXlNlpqNq '')+N0Ba0G#$(K#K 1g6 LLNId#e	#R 4-.43J3J3VIILLOOu% h&HHOOGdhhoo.L.L$LZ_ktOuE '+&=&=eV]M`M`&a#E#E##"Vekk!n4N;KLx(lLNL$$00K`n0o(nstijT))77
NTUWcdtEt 	##%+,,,EQabb_8 "l* .sSe	# e	#t M
 usE   =m(#Cm90m.
?Am91m4 H.m90m9.
n%n.m99n)NT)NNNN)NNNNNr   )
NNNNNr   r   r   NN)FF)NNNTFF)>__name__
__module____qualname____doc__model_cpu_offload_seq_optional_components_exclude_from_cpu_offloadr   r   r   r   r   r	   r   r   r   r   r   r,   r
   r   boolrU   r   rs   r   r   rx   r   rr   r   r   r   r   r   r   r   r  r5  rB  rJ  	Generatorr7  propertyrR  ry   ro   rY  r\  r_  no_gradr"   EXAMPLE_DOC_STRINGr   r   r   r   r   r   r   r  __classcell__)rX   s   @r8   r<   r<      s<   .` ES!1 2& 8<(,5Q5Q $5Q !	5Q
 #5Q /4+@%BXZnno5Q -5Q 55Q .5Q 55Q "&5Q| 049=&*  - !) 6 UON 049=&*#'t5  -t5 !) 6t5 UOt5 C=t5n52+'\'	!$	8( # $&)" +/%pf#` %* X "=@+*\u||   $ $   ( ( , , # #   U]]_12 )-$()-,0 $#+/#% #;?/0MQ*.049=9=@D%* ;?CF <?:=#' 9BAScc49n%Sc "Sc '	Sc
 *Sc Sc }Sc $C=Sc Sc !Sc Sc "%T#Y"78Sc  (}Sc Sc E%//43H"HIJSc  %,,'!Sc"  -#Sc$ !) 6%Sc& ##56'Sc( "*$u||*<!=)Sc* c]+Sc, -Sc. !)c3h 8/Sc0 (-UDK-?'@1Sc2 3Sc4 !&eT%[&8 95Sc6 $E4;$677Sc8 C=9Sc: '(Cd+T124DF\\]
;Sc@ -1IASc 3 Scr:   r<   )Nr2   )Mr   typingr   r   r   r   r   r   r	   r   r"  	PIL.Imager  rs   torch.nn.functionalrF  rG  r  transformersr
   r   r   r   	callbacksr   r   rb   r   r   loadersr   r   r   r   modelsr   r   r   r   r   models.lorar   
schedulersr   utilsr   r   r    r!   r"   r#   r$   utils.torch_utilsr%   r&   r'   pipeline_utilsr)   r*   stable_diffusionr+   stable_diffusion.safety_checkerr,   torch_xla.core.xla_modelcore	xla_modelr  r  
get_loggerr  rV   r  r   r  r   r9   r<   rq   r:   r8   <module>r     s   "  D D D      h h A D w w q q 9 3   V U D < J ))MM			H	%6 v ck
TLL
T-5eoo-F
T\_
Tzc"zcr:   