biXddlZddlmZddlmZmZmZmZddlZddl m Z ddl Z ddl mZddlmZddlmZddlmZddlmZmZmZd d lmZmZmZd d lmZmZm Z m!Z!d d l"m#Z#m$Z$m%Z%d dl&m'Z'd dl(m)Z)d dl*m+Z+e$jXe-Z.dZ/dZ0Gdde'Z1eejddddZ3eejdddZ4de jjfdZ6dZ7y)N)partial)DictListOptionalUnion) FrozenDict) unreplicate)shard)Image)CLIPImageProcessor CLIPTokenizerFlaxCLIPTextModel)FlaxAutoencoderKLFlaxControlNetModelFlaxUNet2DConditionModel)FlaxDDIMSchedulerFlaxDPMSolverMultistepSchedulerFlaxLMSDiscreteSchedulerFlaxPNDMScheduler)PIL_INTERPOLATIONloggingreplace_example_docstring)FlaxDiffusionPipeline)!FlaxStableDiffusionPipelineOutput) FlaxStableDiffusionSafetyCheckerFa Examples: ```py >>> import jax >>> import numpy as np >>> import jax.numpy as jnp >>> from flax.jax_utils import replicate >>> from flax.training.common_utils import shard >>> from diffusers.utils import load_image, make_image_grid >>> from PIL import Image >>> from diffusers import FlaxStableDiffusionControlNetPipeline, FlaxControlNetModel >>> def create_key(seed=0): ... return jax.random.PRNGKey(seed) >>> rng = create_key(0) >>> # get canny image >>> canny_image = load_image( ... "https://huggingface.co/datasets/YiYiXu/test-doc-assets/resolve/main/blog_post_cell_10_output_0.jpeg" ... ) >>> prompts = "best quality, extremely detailed" >>> negative_prompts = "monochrome, lowres, bad anatomy, worst quality, low quality" >>> # load control net and stable diffusion v1-5 >>> controlnet, controlnet_params = FlaxControlNetModel.from_pretrained( ... "lllyasviel/sd-controlnet-canny", from_pt=True, dtype=jnp.float32 ... ) >>> pipe, params = FlaxStableDiffusionControlNetPipeline.from_pretrained( ... "stable-diffusion-v1-5/stable-diffusion-v1-5", ... controlnet=controlnet, ... revision="flax", ... dtype=jnp.float32, ... ) >>> params["controlnet"] = controlnet_params >>> num_samples = jax.device_count() >>> rng = jax.random.split(rng, jax.device_count()) >>> prompt_ids = pipe.prepare_text_inputs([prompts] * num_samples) >>> negative_prompt_ids = pipe.prepare_text_inputs([negative_prompts] * num_samples) >>> processed_image = pipe.prepare_image_inputs([canny_image] * num_samples) >>> p_params = replicate(params) >>> prompt_ids = shard(prompt_ids) >>> negative_prompt_ids = shard(negative_prompt_ids) >>> processed_image = shard(processed_image) >>> output = pipe( ... prompt_ids=prompt_ids, ... image=processed_image, ... params=p_params, ... prng_seed=rng, ... num_inference_steps=50, ... neg_prompt_ids=negative_prompt_ids, ... jit=True, ... ).images >>> output_images = pipe.numpy_to_pil(np.asarray(output.reshape((num_samples,) + output.shape[-3:]))) >>> output_images = make_image_grid(output_images, num_samples // 4, 4) >>> output_images.save("generated_image.png") ``` ceZdZdZej fdededede de de e e eefded ed ej$ffd Zd e eeeffd Zde ej.eej.ffdZdZddZ ddej6dej6de eefdej>de de!de"ej6de"ej6de!fdZ#e$e% d dej6dej6de eefdej>de de e!ej6fdej6dej6de e!ej6fde&de&fdZ'xZ(S)!%FlaxStableDiffusionControlNetPipelinea4 Flax-based pipeline for text-to-image generation using Stable Diffusion with ControlNet Guidance. This model inherits from [`FlaxDiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). Args: vae ([`FlaxAutoencoderKL`]): Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. text_encoder ([`~transformers.FlaxCLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer ([`~transformers.CLIPTokenizer`]): A `CLIPTokenizer` to tokenize text. unet ([`FlaxUNet2DConditionModel`]): A `FlaxUNet2DConditionModel` to denoise the encoded image latents. controlnet ([`FlaxControlNetModel`]: Provides additional conditioning to the `unet` during the denoising process. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`FlaxDDIMScheduler`], [`FlaxLMSDiscreteScheduler`], [`FlaxPNDMScheduler`], or [`FlaxDPMSolverMultistepScheduler`]. safety_checker ([`FlaxStableDiffusionSafetyChecker`]): Classification module that estimates whether generated images could be considered offensive or harmful. Please refer to the [model card](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) for more details about a model's potential harms. feature_extractor ([`~transformers.CLIPImageProcessor`]): A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. vae text_encoder tokenizerunet controlnet schedulersafety_checkerfeature_extractordtypec >t || |_|#tj d|j d|j ||||||||t|ddr5dt|jjjdz z|_ yd|_ y)Nz)You have disabled the safety checker for a by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .)r r!r"r#r$r%r&r'r r) super__init__r(loggerwarning __class__register_modulesgetattrlenr configblock_out_channelsvae_scale_factor) selfr r!r"r#r$r%r&r'r(r0s r/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/pipelines/controlnet/pipeline_flax_controlnet.pyr-z.FlaxStableDiffusionControlNetPipeline.__init__s   ! NN;DNN;KLjj  %!)/  W^^bdikoVpc$((//*L*L&MPQ&Q Rvwpromptct|ttfstdt ||j |d|j j dd}|jS)Nz2`prompt` has to be of type `str` or `list` but is max_lengthTnp)paddingr< truncationreturn_tensors) isinstancestrlist ValueErrortyper"model_max_length input_ids)r7r: text_inputs r8prepare_text_inputsz9FlaxStableDiffusionControlNetPipeline.prepare_text_inputsse&3+.QRVW]R^Q_`a a^^  ~~66 $ ###r9imagec .t|tjtfstdt |t|tjr|g}t j |Dcgc]}t|t jc}}|Scc}w)Nz9image has to be of type `PIL.Image.Image` or list but is ) rAr rCrDrEjnp concatenate preprocessfloat32)r7rJimgprocessed_imagess r8prepare_image_inputsz:FlaxStableDiffusionControlNetPipeline.prepare_image_inputssw%%++t!45XY]^cYdXefg g eU[[ )GE??TY+ZSJsCKK,H+Z[,[s(!Bc*|j||}|SN)r&)r7featuresparamshas_nsfw_conceptss r8_get_has_nsfw_conceptszr<r@r)rbr(zUnexpected latents shape, got z , expected c b|\}}tj|gdz}tj|jtj|}tj ||j d}jj|||}jjdditj|tj|tj  d\}}jjdditj|tj|tj || j} tj| dd \} } | | | z zz} jj|| ||j\}}||fS) NrrZrrVr$F)encoder_hidden_statescontrolnet_condconditioning_scale return_dictr#)r}down_block_additional_residualsmid_block_additional_residual)axis)rLrMarray timestepsint32 broadcast_torbr%scale_model_inputr$applyr#samplesplitstepto_tuple)rargsrsscheduler_state latents_inputttimestepdown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_prediction_textcontextrurrrJrVr7s r8 loop_bodyzBFlaxStableDiffusionControlNetPipeline._generate..loop_body"s'+ $G_ OOWIM:M /33399EdKA''=+>+>q+ABH NN<<_m]^_M;???;P;P6,/0 -( (#))4&- %#@!* -( (#))4&-0F.B )f 8;yyQUV7W 4 4*^?TWh?h-iiJ(,~~':':?JXY[b'c'l'l'n $G_O+ +r9r%)rqrbr*rVr )methodg?r)rbrDr!r"rGrLrMr#r4 in_channelsr6jaxrandomnormalrOr% set_timestepsinit_noise_sigmaDEBUGrangelax fori_loopr scaling_factorrdecoderclip transpose)r7rorJrVrprqrrrsrtruheightwidth prompt_embeds batch_sizer< uncond_inputnegative_prompt_embeds latents_shaperri_rs` `` ` ` @r8 _generatez/FlaxStableDiffusionControlNetPipeline._generates BC(  B;! urzQWX^W__dejdkklmn n))*VN=S)TUVW  %%a( %%b)  !>>z!~ BC(  ne ,!YY'7*:J:J1:M'MNN:##$q(!/4!8 3U ;-0II7T6UXbXhXhijXk6k,l ):##$q(0MaQUg0V-  #- F^^#- F    *"#34M#)C<"6"6"8"?"?"H &,ll2A&6 #K"$**-@"A"I"I+XbJbdjlqst"u 484L4LM`boqt4u 1 !1XXf%F#$"+,<"=GJAw$&JJ/B1/E$Fq G^^KVUANFZZ'F$ ,- -0Vfggr9)F)NN?)2g@NNrTF))__name__ __module__ __qualname____doc__rLrOrrr rrrrrrrrr r(r-rBrrIr rRrXrnndarrayrrrArrayintrrrrEXAMPLE_DOC_STRINGboolr __classcell__)r0s@r8rrrseR;;%x %x(%x! %x ' %x ( %x 02JLk k %x9%x.%xyy%xN $%T#Y*? $  % T%++=N0N*O  !)L*.04/2kKKk{{kdJ&' k 99 k ! kk#++&k!-k(-kZ12$&47#&*CF AhKKAh{{AhdJ&' Ah 99 Ah ! AheS[[01AhAh Ah(-UCKK-?'@AhAhAh3Ahr9r) NrrrrNrrrr)r)in_axesstatic_broadcasted_argnumsc 4|j||||||||| SrT)r) piperorJrVrprqrrrsrtrus r8rrs0" >> %  r9)r)rc&|j||SrT)rX)rrUrVs r8r]r]s  & &x 88r9xcp|jdd\}}|jdd}|j||zg|S)Nr)rbr)rrrrests r8r^r^ s@ggbqkK 7712;D 199[:- 5 55r9c|jd}|j\}}d||fD\}}|j||ftd}t j |j |dz }|djdddd }|S) NRGBc3,K|] }||dzz yw)rxN).0rs r8 zpreprocess..s '1ABJ 'slanczos)resamplego@rrr*r)convertsizeresizerrLrrr)rJr(whs r8rNrNs MM% E ::DAq 'A 'DAq LL!Q*;I*FL GE IIe  # #E *U 2E $K ! !!Q1 -E Lr9)8re functoolsrtypingrrrrr jax.numpynumpyrLr=flax.core.frozen_dictrflax.jax_utilsr flax.training.common_utilsr PILr transformersr r rmodelsrrr schedulersrrrrutilsrrrpipeline_flax_utilsrstable_diffusionr$stable_diffusion.safety_checker_flaxr get_loggerrr.rrrpmaprr]rr^rNrr9r8rs.. ,&,MMVV KJ7@S   H % AHoh,Aohh  HH 0%   2 d39496s{{6r9