bi ddlZddlmZmZmZmZmZmZmZddl Z ddl m cm Z ddlmZmZmZmZddlmZddlmZmZmZmZddlmZmZmZmZmZm Z ddl!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/dd l0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7m8Z8ddl9m:Z:e)r ddl;mdZ?ndZ?e*jeAZBdZCGdde7e8eeee3e5e ZDy)N)AnyCallableDictListOptionalTupleUnion)CLIPImageProcessor CLIPTextModel CLIPTokenizerCLIPVisionModelWithProjection)PipelineImageInput)FromSingleFileMixinIPAdapterMixinStableDiffusionLoraLoaderMixinTextualInversionLoaderMixin) AutoencoderKLControlNetModelImageProjectionMultiControlNetModelUNet2DConditionModelUNetMotionModel)adjust_lora_scale_text_encoder) MotionAdapter)KarrasDiffusionSchedulers)USE_PEFT_BACKENDis_torch_xla_availableloggingscale_lora_layersunscale_lora_layers)is_compiled_module randn_tensor)VideoProcessor) FreeInitMixin)AnimateDiffFreeNoiseMixin)DiffusionPipelineStableDiffusionMixin)AnimateDiffPipelineOutputTFaT Examples: ```py >>> import torch >>> from diffusers import ( ... AnimateDiffControlNetPipeline, ... AutoencoderKL, ... ControlNetModel, ... MotionAdapter, ... LCMScheduler, ... ) >>> from diffusers.utils import export_to_gif, load_video >>> # Additionally, you will need a preprocess videos before they can be used with the ControlNet >>> # HF maintains just the right package for it: `pip install controlnet_aux` >>> from controlnet_aux.processor import ZoeDetector >>> # Download controlnets from https://huggingface.co/lllyasviel/ControlNet-v1-1 to use .from_single_file >>> # Download Diffusers-format controlnets, such as https://huggingface.co/lllyasviel/sd-controlnet-depth, to use .from_pretrained() >>> controlnet = ControlNetModel.from_single_file("control_v11f1p_sd15_depth.pth", torch_dtype=torch.float16) >>> # We use AnimateLCM for this example but one can use the original motion adapters as well (for example, https://huggingface.co/guoyww/animatediff-motion-adapter-v1-5-3) >>> motion_adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM") >>> vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16) >>> pipe: AnimateDiffControlNetPipeline = AnimateDiffControlNetPipeline.from_pretrained( ... "SG161222/Realistic_Vision_V5.1_noVAE", ... motion_adapter=motion_adapter, ... controlnet=controlnet, ... vae=vae, ... ).to(device="cuda", dtype=torch.float16) >>> pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear") >>> pipe.load_lora_weights( ... "wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora" ... ) >>> pipe.set_adapters(["lcm-lora"], [0.8]) >>> depth_detector = ZoeDetector.from_pretrained("lllyasviel/Annotators").to("cuda") >>> video = load_video( ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-vid2vid-input-1.gif" ... ) >>> conditioning_frames = [] >>> with pipe.progress_bar(total=len(video)) as progress_bar: ... for frame in video: ... conditioning_frames.append(depth_detector(frame)) ... progress_bar.update() >>> prompt = "a panda, playing a guitar, sitting in a pink boat, in the ocean, mountains in background, realistic, high quality" >>> negative_prompt = "bad quality, worst quality" >>> video = pipe( ... prompt=prompt, ... negative_prompt=negative_prompt, ... num_frames=len(video), ... num_inference_steps=10, ... guidance_scale=2.0, ... conditioning_frames=conditioning_frames, ... generator=torch.Generator().manual_seed(42), ... ).frames[0] >>> export_to_gif(video, "animatediff_controlnet.gif", fps=8) ``` c8eZdZdZdZddgZgdZ dCdeded e d e e e fd e d e eeeeeefd edeedeeffd Z dDdeej0deej0deedeefdZdEdZdZdFdefdZdZ dGdZ dEdZ! dHdZ"e#d Z$e#d!Z%e#d"Z&e#d#Z'e#d$Z(e#d%Z)ejTddddd&d'dd(ddddddddd)d*dddddddd+gdfd,e e+ee+fd-eed.eed/eed0ed1ed2ee e+ee+fd3eed4ed5ee ejXeejXfd+eej0deej0deej0d6ee-d7ee-d8eee-d9ee+d:e.d;ee/e+e0fde eeefd?e eeefdeed@ee1eee/gdfdAee+def6dBZ2xZ3S)IAnimateDiffControlNetPipelinea Pipeline for text-to-video generation with ControlNet guidance. This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular device, etc.). The pipeline also inherits the following loading methods: - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings - [`~loaders.StableDiffusionLoraLoaderMixin.load_lora_weights`] for loading LoRA weights - [`~loaders.StableDiffusionLoraLoaderMixin.save_lora_weights`] for saving LoRA weights - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters Args: vae ([`AutoencoderKL`]): Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. text_encoder ([`CLIPTextModel`]): Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). tokenizer (`CLIPTokenizer`): A [`~transformers.CLIPTokenizer`] to tokenize text. unet ([`UNet2DConditionModel`]): A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. motion_adapter ([`MotionAdapter`]): A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. scheduler ([`SchedulerMixin`]): A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. ztext_encoder->unet->vaefeature_extractor image_encoder)latents prompt_embedsnegative_prompt_embedsNvae text_encoder tokenizerunetmotion_adapter controlnet schedulerc t |t|trt j ||}t|t tfr t|}|j|||||||||  t|ddr/dt|jjjdz znd|_t!|j|_t!|jdd |_y) N) r3r4r5r6r7r8r9r.r/r3r%r*)vae_scale_factorTF)r<do_convert_rgb do_normalize)super__init__ isinstancerr from_unet2dlisttuplerregister_modulesgetattrlenr3configblock_out_channelsr<r$video_processorcontrol_video_processor) selfr3r4r5r6r7r8r9r.r/ __class__s z/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.pyr@z&AnimateDiffControlNetPipeline.__init__s  d0 1"..t^DD j4- 0-j9J %)!/'  W^^bdikoVpc$((//*L*L&MPQ&Q Rvw-t?T?TU'5!224V[( $r1r2 lora_scale clip_skipc H |Jt|tr:||_tst |j |nt |j ||t|trd} n-|t|tr t|} n|jd} |t|tr|j||j}|j|d|jjdd} | j} |j|dd j} | jd | jd k\rt!j"| | sj|jj%| dd|jjdz d f}t&j)d |jjd |t+|j j,d r<|j j,j.r| j0j3|}nd}| (|j | j3||}|d}nT|j | j3||d}|d | dz }|j j4j7|}|j |j j8}n/|j:|j:j8}n |j8}|j3||}|j\}}}|j=d|d}|j?||z|d }|rm|j|dg| z}n|:tA|tA|ur$tCdtA|dtA|dt|tr|g}n1| t|k7r!tEd|dt|d|d| d |}t|tr|j||j}|jd}|j|d|dd}t+|j j,d r<|j j,j.r|j0j3|}nd}|j |jj3||}|d}|rK|jd}|j3||}|j=d|d}|j?| |z|d }|j ,t|trtrtG|j |||fS)a Encodes the prompt into text encoder hidden states. Args: prompt (`str` or `List[str]`, *optional*): prompt to be encoded device: (`torch.device`): torch device num_images_per_prompt (`int`): number of images that should be generated per prompt do_classifier_free_guidance (`bool`): whether to use classifier free guidance or not negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. lora_scale (`float`, *optional*): A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. Nr*r max_lengthTpt)paddingrS truncationreturn_tensorslongest)rUrWz\The following part of your input was truncated because CLIP can only handle sequences up to z tokens: use_attention_mask)attention_mask)r[output_hidden_states)dtypedevicez?`negative_prompt` should be the same type to `prompt`, but got z != .z`negative_prompt`: z has batch size z, but `prompt`: zT. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.)$rAr _lora_scalerrr4r strrCrGshapermaybe_convert_promptr5model_max_length input_idstorchequal batch_decodeloggerwarninghasattrrHrZr[to text_modelfinal_layer_normr]r6repeatviewtype TypeError ValueErrorr!)rLpromptr^num_images_per_promptdo_classifier_free_guidancenegative_promptr1r2rPrQ batch_size text_inputstext_input_idsuntruncated_ids removed_textr[prompt_embeds_dtypebs_embedseq_len_ uncond_tokensrS uncond_inputs rN encode_promptz+AnimateDiffControlNetPipeline.encode_prompts6V  !j7U&V)D $.t/@/@*M!$"3"3Z@  *VS"9J  Jvt$<VJ&,,Q/J  $ ;<2264>>J..$>>::# )K)22N"nnVYW[n\ffO$$R(N,@,@,DDU[[N $~~::#At~~'F'F'JR'O$OP  778 ,Q t((//1EF4K\K\KcKcKvKv!,!;!;!>!>v!F!% $ 1 1.2C2CF2K\j 1 k -a 0 $ 1 1"%%f-ncg!2! !.b 1IM2B C !% 1 1 < < M Mm \    ("&"3"3"9"9  YY ""&))// "/"5"5 %((/B6(R ,22'1%,,Q0EqI %**86K+KWVXY  '+A+I&!#z 1 #V D>$%# *Lt((//1EF4K\K\KcKcKvKv!-! " &,2215G%;%>%>EXag%>%h "%;%B%B1F[]^%_ "%;%@%@NcAcelnp%q "    ($ >?DT#D$5$5zB444rOc|t|jjj}t |t j s|j|dj}|j||}|r}|j|djd}|j|d}|jt j|djd}|j|d}||fS|j|j}|j|d}t j|} || fS) NrT)rWr^r]T)r\rdim)nextr/ parametersr]rArgTensorr. pixel_valuesrm hidden_statesrepeat_interleave zeros_like image_embeds) rLimager^rvr\r]image_enc_hidden_statesuncond_image_enc_hidden_statesruncond_image_embedss rN encode_imagez*AnimateDiffControlNetPipeline.encode_image~sDT''2245;;%.**5*FSSEe4 &*&8&8UY&8&Z&h&hik&l #&=&O&OPekl&O&m #-1-?-?  'd.@.mB. *.L-]-]%1.^. *+,JJ J--e4AAL'99:OUV9WL"'"2"2<"@ !44 4rOcg}|rg}|t|ts|g}t|t|jjj k7rBt dt|dt|jjj dt||jjj D]`\}} t| t } |j||d| \} } |j| dddf|sIj| dddfbn?|D]:} |r%| jd\} } j| |j| <g}t|D]|\} } tj| g|zd} |r7tj| g|zd} tj| | gd} | j|} |j| ~|S) NzK`ip_adapter_image` must have same length as the number of IP Adapters. Got z images and z IP Adapters.r*r%rrr^)rArCrGr6encoder_hid_projimage_projection_layersrtziprrappendchunk enumeratergcatrm)rLip_adapter_imageip_adapter_image_embedsr^rvrwrnegative_image_embedssingle_ip_adapter_imageimage_proj_layeroutput_hidden_statesingle_image_embedssingle_negative_image_embedsis rNprepare_ip_adapter_image_embedsz=AnimateDiffControlNetPipeline.prepare_ip_adapter_image_embedssY &$& ! " *.5$4#5 #$DII,F,F,^,^(__ abefvbwaxyEFIJNJSJSJdJdJ|J|F}E~~KL>A $))"<"<"T"T> X9')9+55E*W&W#DHDUDU+VQ8KEA#%A##$7a$@A.)001MdTUg1VW X(? 9#.H[HaHabcHdE02E)001MN##$78  9 #%&/ &= @ "A""'))-@,ADY,Y_`"a */4yy:OPQ:R9SVk9kqr/s,&+ii1MOb0cij&k#"5"8"8"8"G  # * *+> ? @'&rOdecode_chunk_sizecNd|jjjz |z}|j\}}}}}|j dddddj ||z|||}g}t d|jd|D]@} || | |z} |jj| j} |j| Btj|}|dddfj ||df|jddzj ddddd}|j}|S)Nr*rr%rrY) r3rHscaling_factorrcpermutereshaperangedecodesamplerrgrfloat) rLr0rrychannels num_framesheightwidthvideor batch_latentss rNdecode_latentsz,AnimateDiffControlNetPipeline.decode_latentss'dhhoo444w>:A--7 Hj&%//!Q1a088j9PRZ\bdijq'--*,=> (A#A,=(=>M HHOOM:AAM LL ' (  % dAg&& J'CekkRSRTo'UV^^_`bcefhiklm  rOcVdttj|jjj j v}i}|r||d<dttj|jjj j v}|r||d<|S)Neta generator)setinspect signaturer9steprkeys)rLrr accepts_etaextra_step_kwargsaccepts_generators rNprepare_extra_step_kwargsz7AnimateDiffControlNetPipeline.prepare_extra_step_kwargss s7#4#4T^^5H5H#I#T#T#Y#Y#[\\  '* e $(3w/@/@ATAT/U/`/`/e/e/g+hh -6 k *  rO?c R |dzdk7s|dzdk7rtd|d|d|Ltfd|Ds8tdjd|D cgc]} | jvs| c} ||td |d |d | | td |2t|tt t fstd t|||td|d|d |A|?|j|jk7r&td|jd|jdtjtrRt|t rBtjdtjjdt|dtt dxr8tjt"j$j&j(}tjt*s&|rstjj,t*rOt t st/dt t |k7rtd|dt tjts&|rtjj,trt t rt dt st/dt t d|k7rtd|dt dt1 fd Dr tdJtjt*s&|r?tjj,t*rt| t2st/dtjts&|rtjj,trst| t rt1d| DrStd t| t r8t| tjjk7r td!Jt| t4t fs| g} t| t4t fs| g} t| t| k7r$td"t| d#t| d$tjtrt| tjjk7r[td%| d&t| d'tjjd(tjjd t7| | D]D\}}||k\rtd)|d*|d|d+krtd)|d,|d-kDs7td.|d/ycc} w)0Nr;rz7`height` and `width` have to be divisible by 8 but are z and r`c3:K|]}|jvywN)_callback_tensor_inputs).0krLs rN z=AnimateDiffControlNetPipeline.check_inputs..s#F 23A-- -F sz2`callback_on_step_end_tensor_inputs` has to be in z , but found zCannot forward both `prompt`: z and `prompt_embeds`: z2. Please make sure to only forward one of the two.zeProvide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.z:`prompt` has to be of type `str`, `list` or `dict` but is z'Cannot forward both `negative_prompt`: z and `negative_prompt_embeds`: zu`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but got: `prompt_embeds` z != `negative_prompt_embeds` z You have z! ControlNets and you have passed z= prompts. The conditionings will be fixed across the prompts.scaled_dot_product_attentionz>For single controlnet, `image` must be of type `list` but got zExcepted image to have length z but got len(video)=zQFor multiple controlnets: `image` must be type list of lists but got type(video)=z$Expected length of image sublist as z but got len(video[0])=c3RK|]}t|tdk7 yw)rN)rG)rimgrs rNrz=AnimateDiffControlNetPipeline.check_inputs..;s">3s8s58},>s$'zDAll conditioning frame batches for multicontrolnet must be same sizezLFor single controlnet: `controlnet_conditioning_scale` must be type `float`.c3<K|]}t|tywr)rArC)rrs rNrz=AnimateDiffControlNetPipeline.check_inputs..NsRqz!T*RszEA single batch of multiple conditionings are supported at the moment.zFor multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have the same length as the number of controlnetsz`control_guidance_start` has z* elements, but `control_guidance_end` has zI elements. Make sure to provide the same number of elements to each list.z`control_guidance_start`: z has z elements but there are z- controlnets available. Make sure to provide zcontrol guidance start: z4 cannot be larger or equal to control guidance end: rz can't be smaller than 0.rzcontrol guidance end: z can't be larger than 1.0.)rtallrrArbrCdictrrrcr8rrjrkrGnetsrlFrg_dynamo eval_frameOptimizedModuler _orig_modrsanyrrDr)rLrurrrrxr1r2"callback_on_step_end_tensor_inputsrcontrolnet_conditioning_scalecontrol_guidance_startcontrol_guidance_endr is_compiledstartends` ` rN check_inputsz*AnimateDiffControlNetPipeline.check_inputssR A:?eai1nVW]V^^cdicjjklm m - 9#F 7YF C DTEaEaDbbn|^pHvwbckokGkGbGpqpHoIJ   -";08N}o^00 ^ 5w   6Ct;L(MYZ^_eZfYghi i  &+A+M9/9JK*++]_   $)?)K""&<&B&BB --:-@-@,AB.445Q8 doo'; <&$'DOO$8$8 9::[\_`f\g[hST a!?@ Z OOU]]55EEF  t 84??44oFeT*"`aefkal`m noo5zZ' #A*MbWZ[`WaVc!dee t(< =4??446JKeT**U1Xt2L"thlmrhsgu vww58} * #G |Sk]`afghai]j\l!mnn>>> !ghh 5 t 84??44oF;UC noo t(< =4??446JK7>R4QRR$%lmm94@SIfEgkn$$lF!D 505$-@&<%= ". >$8#9 % &#.B*C C/4J0K/LLvwz|PxQwRR[\  doo'; <)*c$//2F2F.GG 01G0HcRhNiMjkCDGHLHWHWH\H\D]C^^KLOPTP_P_PdPdLeKffgh46JK [JE3| .ug5ijminnops{ #;E7B[!\]]Sy #9#>X!YZZ [YpHs V$&V$c ||jr|j||||||||| } t|tr)t ||k7rt dt |d|d|||||j z||j zf} | t| |||} n| j|} | |jjz} | S)Nz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)rr^r]) free_noise_enabled_prepare_latents_free_noiserArCrGrtr<r#rmr9init_noise_sigma) rLrynum_channels_latentsrrrr]r^rr0rcs rNprepare_latentsz-AnimateDiffControlNetPipeline.prepare_latentsvs  " "660*feUTZ\egnG i &3y>Z+GA#i.AQR&<'gi    d++ + T** *   ?"5IfTYZGjj(GDNN;;;rOFc |jj|||jtj}|j dddddj dd}|jd} | dk(r|} n|} |j| d}|j|| }|r| stj|gdz}|S) N)rr)r]rr%r*rrrr) rKpreprocess_videormrgfloat32rflattenrcrr) rLrrrrynum_videos_per_promptr^r]rw guess_modevideo_batch_size repeat_bys rN prepare_videoz+AnimateDiffControlNetPipeline.prepare_videos,,==eFZ_=`cc--d  aAq!,44Q: ;;q> q "I.I'' q'9e4 &zIIugk*E rOc|jSr_guidance_scalerLs rNguidance_scalez,AnimateDiffControlNetPipeline.guidance_scales###rOc|jSr) _clip_skiprs rNrQz'AnimateDiffControlNetPipeline.clip_skip rOc |jdkDS)Nr*rrs rNrwz9AnimateDiffControlNetPipeline.do_classifier_free_guidances##a''rOc|jSr)_cross_attention_kwargsrs rNcross_attention_kwargsz4AnimateDiffControlNetPipeline.cross_attention_kwargss+++rOc|jSr)_num_timestepsrs rN num_timestepsz+AnimateDiffControlNetPipeline.num_timestepss"""rOc|jSr) _interruptrs rN interruptz'AnimateDiffControlNetPipeline.interruptrrO2g@r*pilTr0rurrrnum_inference_stepsrrxrrrrrconditioning_frames output_type return_dictrrrrrcallback_on_step_endrct|jr|jjn |j}t|ts t|trt ||gz}nt|ts t|trt ||gz}nSt|tsCt|ts3t|t rt |jnd}||gz||gz}}|xs-|jjj|jz}|xs-|jjj|jz}d}|j||||||| | |||| ||_ ||_||_d|_|t|t"t$frd}n-|t|tr t |}n| j&d}|j(}t|t r)t|t*r|gt |jz}t|t,r|jj.n"|jdjj.} |xs| }||j1ddnd}!|j2r3|j5|||||j6|| | |!|j8 \} } ng|j;||||j6|| | |!|j8 \} } |j6rt=j>| | g} | jA|d } ||"|jC|||||z|j6}"t|t,r5|jE|||||z|z|||jF|j6| }nct|t rQg}#|D]G}$|jE|$||||z|z|||jF|j6| }%|#jI|%I|#}nJ|jJjM|| |jJjN}&|jjjP}'|jS||z|'|||| jF|| | } |jU| | }(||d "ind})g}*tWt |&D]w}+tY||D,-cgc]8\},}-d t+|+t |&z |,kxs|+dzt |&z |-kDz :}.},}-|*jIt|t,r|.dn|.y|jZr |j\nd}/tW|/D]<}0|jZr#|j_| |0||| jF| \} }&t |&|_0t |&||jJjbzz }1|je|j`5}2tg|&D]\}+}3|jhr|j6rt=j>| gdzn| }4|jJjk|4|3}4|r?|j6r3| }5|jJjk|5|3}5| jmdd}6n|4}5| }6t|*|+tr%tY||*|+D7,cgc] \}7},|7|,z }8}7},n|}9t|9tr|9d}9|9|*|+z}8t=jn|5dd}5|5jqd|5j&d|5j&d|5j&df}5|j|5|3|6||8|d\}:};|j|4|3| |jr|)|:|;jt}<|j6r||=||>|=z zz}<|jJjv|<|3| fi|(jx} |Zi}?|D]}@t{|@|?|@<|||+|3|?}A|Aj}d| } |Aj}d| } |Aj}d| } |+t |&dz k(s'|+dz|1kDr/|+dz|jJjbzdk(r|2jtstj ddd?|dk(r| }Bn/|j| |}C|jj|C|}B|j|sBfStBScc}-},wcc},}7w#1swYxYw)u The call function to the pipeline for generation. Args: prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The height in pixels of the generated video. width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): The width in pixels of the generated video. num_frames (`int`, *optional*, defaults to 16): The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds amounts to 2 seconds of video. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. More denoising steps usually lead to a higher quality videos at the expense of slower inference. guidance_scale (`float`, *optional*, defaults to 7.5): A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide what to not include in image generation. If not defined, you need to pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) from the [DDIM](https://huggingface.co/papers/2010.02502) paper. Only applies to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video generation. Can be used to tweak the same generation with different prompts. If not provided, a latents tensor is generated by sampling using the supplied random `generator`. Latents should be of shape `(batch_size, num_channel, num_frames, height, width)`. prompt_embeds (`torch.Tensor`, *optional*): Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, text embeddings are generated from the `prompt` input argument. negative_prompt_embeds (`torch.Tensor`, *optional*): Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. ip_adapter_image (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not provided, embeddings are computed from the `ip_adapter_image` input argument. conditioning_frames (`List[PipelineImageInput]`, *optional*): The ControlNet input condition to provide guidance to the `unet` for generation. If multiple ControlNets are specified, images must be passed as a list such that each element of the list can be correctly batched for input to a single ControlNet. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated video. Choose between `torch.Tensor`, `PIL.Image` or `np.array`. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead of a plain tuple. cross_attention_kwargs (`dict`, *optional*): A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set the corresponding scale as a list. guess_mode (`bool`, *optional*, defaults to `False`): The ControlNet encoder tries to recognize the content of the input image even if you remove all prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): The percentage of total steps at which the ControlNet starts applying. control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): The percentage of total steps at which the ControlNet stops applying. clip_skip (`int`, *optional*): Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that the output of the pre-final layer will be used for computing the prompt embeddings. callback_on_step_end (`Callable`, *optional*): A function that calls at the end of each denoising steps during the inference. The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. callback_on_step_end_tensor_inputs (`List`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the `._callback_tensor_inputs` attribute of your pipeline class. Examples: Returns: [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: If `return_dict` is `True`, [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. r*) rurrrrxrr1r2rrrrFNrscale) rurr^rrwrxr1r2rPrQ)r1r2rPrQ)repeatsr) rrrryrr^r]rwrrrr)totalr%rYrr)encoder_hidden_statescontrolnet_condconditioning_scalerr )rradded_cond_kwargsdown_block_additional_residualsmid_block_additional_residualr0r1r2latent)rr )frames)Hr"r8rrArCrGrrr6rH sample_sizer<rrrrrrbrrc_execution_devicerrglobal_pool_conditionsgetr_encode_prompt_free_noiserwrQrrgrrrrr]rr9 set_timesteps timesteps in_channelsrrrrfree_init_enabled_free_init_num_iters_apply_free_initrorder progress_barrrscale_model_inputr transposerrrr prev_samplelocalspopupdate XLA_AVAILABLExm mark_steprrJpostprocess_videomaybe_free_model_hooksr+)DrLrurrrr rrxrrrr0r1r2rrr r r rrrrrrQr rrr8multryr^rtext_encoder_lora_scalercond_prepared_videosframe_prepared_videor rrrcontrolnet_keeprsekeepsnum_free_init_itersfree_init_iternum_warmup_stepsr&tlatent_model_inputcontrol_model_inputcontrolnet_prompt_embedsc cond_scalecontrolnet_cond_scaledown_block_res_samplesmid_block_res_sample noise_prednoise_pred_uncondnoise_pred_textcallback_kwargsrcallback_outputsr video_tensorsD rN__call__z&AnimateDiffControlNetPipeline.__call__s p3ET__2UT__..[_[j[j 0$7JG[]aBXAY%Y "0$7JG]_cBVAW#W 2D9*MacgBh+5jBV+W3z']^D.//,--%9 " O499++77$:O:OOM))558M8MM ! !+/Q'#9%*G#9!5   .#'=$  *Vc4["AJ  Jvt$<VJ&,,Q/J'' j"6 7JGdfk" V049=&*#'t5  -t5!) 6t5UOt5C=t5n52+'\&!.#+/&)" J[\nrR%*@$$ ((,,##U]]_)-$& $##% #;?/0MQ*.049=9=@DBF%* ;?CF ?#L7$c]%L7&'L7(!)c3h 8)L7*(-UDK-?'@+L7,-L7.!&eT%[&8 9/L70$E4;$671L72C=3L74'xc40@$0F'GH5L76-1I7L789L7L7rOr-)Ertypingrrrrrrr rgtorch.nn.functionalnn functionalr transformersr r r r image_processorrloadersrrrrmodelsrrrrrr models.lorarmodels.unets.unet_motion_modelr schedulersrutilsrrrr r!utils.torch_utilsr"r#rJr$free_init_utilsr&free_noise_utilsr'pipeline_utilsr(r)pipeline_outputr+torch_xla.core.xla_modelcore xla_modelr.r- get_loggerrNrjEXAMPLE_DOC_STRINGr-rOrNrqsDDD hh1ww:;3nnA-+8D6))MM   H %?Df7"f7rO