
    big0                         d dl Z d dlmZmZmZmZmZmZ d dlZddl	m
Z
 ddlmZ ddlmZmZ ddlmZmZ erdd	lmZ  G d
 de      Zy)    N)TYPE_CHECKINGDictListOptionalTupleUnion   )register_to_config)HookRegistry)SmoothedEnergyGuidanceConfig$_apply_smoothed_energy_guidance_hook   )BaseGuidancerescale_noise_cfg)
BlockStatec                       e Zd ZdZg dZe	 	 	 	 	 	 	 	 	 	 	 	 d!dededededed	ed
eee	e
e	   f      deee
e   f   dedededef fd       Zdej                  j                   ddfdZdej                  j                   fdZ	 d"dddeeeeeeeef   f   f      de
d   fdZ	 	 d#dej.                  deej.                     deej.                     dej.                  fdZedefd       Zede	fd       ZdefdZdefd Z xZS )$SmoothedEnergyGuidancea  
    Smoothed Energy Guidance (SEG): https://huggingface.co/papers/2408.00760

    SEG is only supported as an experimental prototype feature for now, so the implementation may be modified in the
    future without warning or guarantee of reproducibility. This implementation assumes:
    - Generated images are square (height == width)
    - The model does not combine different modalities together (e.g., text and image latent streams are not combined
      together such as Flux)

    Args:
        guidance_scale (`float`, defaults to `7.5`):
            The scale parameter for classifier-free guidance. Higher values result in stronger conditioning on the text
            prompt, while lower values allow for more freedom in generation. Higher values may lead to saturation and
            deterioration of image quality.
        seg_guidance_scale (`float`, defaults to `3.0`):
            The scale parameter for smoothed energy guidance. Anatomy and structure coherence may improve with higher
            values, but it may also lead to overexposure and saturation.
        seg_blur_sigma (`float`, defaults to `9999999.0`):
            The amount by which we blur the attention weights. Setting this value greater than 9999.0 results in
            infinite blur, which means uniform queries. Controlling it exponentially is empirically effective.
        seg_blur_threshold_inf (`float`, defaults to `9999.0`):
            The threshold above which the blur is considered infinite.
        seg_guidance_start (`float`, defaults to `0.0`):
            The fraction of the total number of denoising steps after which smoothed energy guidance starts.
        seg_guidance_stop (`float`, defaults to `1.0`):
            The fraction of the total number of denoising steps after which smoothed energy guidance stops.
        seg_guidance_layers (`int` or `List[int]`, *optional*):
            The layer indices to apply smoothed energy guidance to. Can be a single integer or a list of integers. If
            not provided, `seg_guidance_config` must be provided. The recommended values are `[7, 8, 9]` for Stable
            Diffusion 3.5 Medium.
        seg_guidance_config (`SmoothedEnergyGuidanceConfig` or `List[SmoothedEnergyGuidanceConfig]`, *optional*):
            The configuration for the smoothed energy layer guidance. Can be a single `SmoothedEnergyGuidanceConfig` or
            a list of `SmoothedEnergyGuidanceConfig`. If not provided, `seg_guidance_layers` must be provided.
        guidance_rescale (`float`, defaults to `0.0`):
            The rescale factor applied to the noise predictions. This is used to improve image quality and fix
            overexposure. Based on Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
            Flawed](https://huggingface.co/papers/2305.08891).
        use_original_formulation (`bool`, defaults to `False`):
            Whether to use the original formulation of classifier-free guidance as proposed in the paper. By default,
            we use the diffusers-native implementation that has been in the codebase for a long time. See
            [~guiders.classifier_free_guidance.ClassifierFreeGuidance] for more details.
        start (`float`, defaults to `0.01`):
            The fraction of the total number of denoising steps after which guidance starts.
        stop (`float`, defaults to `0.2`):
            The fraction of the total number of denoising steps after which guidance stops.
    	pred_condpred_uncondpred_cond_segNguidance_scaleseg_guidance_scaleseg_blur_sigmaseg_blur_threshold_infseg_guidance_startseg_guidance_stopseg_guidance_layersseg_guidance_configguidance_rescaleuse_original_formulationstartstopc                    t         |   ||       || _        || _        || _        || _        || _        || _        |	| _        |
| _	        d|cxk  rdk  sn t        d| d      ||cxk  rdk  sn t        d| d      ||t        d      ||t        d      |Ut        |t              r|g}t        |t              st        dt        |       d      |D cg c]  }t        |d	
       }}t        |t               rt        j"                  |      }t        |t              r|g}t        |t              st        dt        |       d      t        t%        t'        |      d       t               r"|D cg c]  }t        j"                  |       }}|| _        t+        t-        | j(                              D cg c]  }d| 	 c}| _        y c c}w c c}w c c}w )N              ?zAExpected `seg_guidance_start` to be between 0.0 and 1.0, but got .z@Expected `seg_guidance_stop` to be between 0.0 and 1.0, but got zjEither `seg_guidance_layers` or `seg_guidance_config` must be provided to enable Smoothed Energy Guidance.zKOnly one of `seg_guidance_layers` or `seg_guidance_config` can be provided.zGExpected `seg_guidance_layers` to be an int or a list of ints, but got auto)fqnzwExpected `seg_guidance_config` to be a SmoothedEnergyGuidanceConfig or a list of SmoothedEnergyGuidanceConfig, but got SmoothedEnergyGuidance_)super__init__r   r   r   r   r   r   r    r!   
ValueError
isinstanceintlisttyper   dict	from_dictnextiterr   rangelen_seg_layer_hook_names)selfr   r   r   r   r   r   r   r   r    r!   r"   r#   layerconfigi	__class__s                   e/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/guiders/smoothed_energy_guidance.pyr,   zSmoothedEnergyGuidance.__init__P   s8     	%,"4,&<#"4!2 0(@%)/C/`as`ttuvww"&7>3>_`q_rrstuu&+>+F|  */B/Njkk*-s3':&;#148 ]^bcv^w]xxyz  at"tW\#?6#R"t"t)40">"H"HI\"])+GH#6"7-t4 J  KO  Pc  Kd  Je  ef  g  T"56=tD`s"tV\#?#I#I&#Q"t"t#6 MRSVW[WoWoSpMq%r(?s&C%r"! #u #u &ss   !G>GG"denoiserreturnc                     | j                         r_| j                  rR| j                  dkD  rBt        | j                  | j
                        D ]  \  }}t        ||| j                  |         y y y y )Nr   )name)_is_seg_enabledis_conditional_count_preparedzipr8   r   r   r   )r9   r?   rB   r;   s       r>   prepare_modelsz%SmoothedEnergyGuidance.prepare_models   sm    !d&9&9d>R>RUV>V #D$>$>@X@X Y gf4XvtGZGZaefg ?W&9!    c                     | j                         rW| j                  rJ| j                  dkD  r:t        j                  |      }| j
                  D ]  }|j                  |d        y y y y )Nr   T)recurse)rC   rD   rE   r   check_if_exists_or_initializer8   remove_hook)r9   r?   registry	hook_names       r>   cleanup_modelsz%SmoothedEnergyGuidance.cleanup_models   sh    !d&9&9d>R>RUV>V#AA(KH!77 >	$$Y$=> ?W&9!rH   datar   input_fieldsc                 D   || j                   }| j                  dk(  rdg}dg}n4| j                  dk(  rddg}| j                         rddgnddg}ng d}g d}g }t        | j                        D ]-  }| j	                  ||||   ||         }|j                  |       / |S )	Nr   r   r   r	   r   r   )r   r   r   r   )_input_fieldsnum_conditions_is_cfg_enabledr6   _prepare_batchappend)r9   rP   rQ   tuple_indicesinput_predictionsdata_batchesr<   
data_batchs           r>   prepare_inputsz%SmoothedEnergyGuidance.prepare_inputs   s     --L!#CM!,  A%FM040D0D0Fm,[ZiLj  &M Mt**+ 	,A,,\4qAQSdefSghJ
+	, rH   r   r   r   c                    d }| j                         s| j                         s|}n| j                         s(||z
  }| j                  r|n|}|| j                  |z  z   }nt| j                         s(||z
  }| j                  r|n|}|| j                  |z  z   }n<||z
  }||z
  }| j                  r|n|}|| j                  |z  z   | j                  |z  z   }| j
                  dkD  rt        ||| j
                        }|i fS )Nr%   )rU   rC   r!   r   r   r    r   )r9   r   r   r   predshift	shift_segs          r>   forwardzSmoothedEnergyGuidance.forward   s    ##%d.B.B.DD%%'-E $ = =9=D$11E99D%%'+E $ = =9;D$--55D+E!M1I $ = =9;D$--558O8OR[8[[D  3&$T9d6K6KLDRxrH   c                 B    | j                   dk(  xs | j                   dk(  S )Nr      )rE   )r9   s    r>   rD   z%SmoothedEnergyGuidance.is_conditional   s#    ##q(ED,@,@A,EErH   c                 ^    d}| j                         r|dz  }| j                         r|dz  }|S )Nr   )rU   rC   )r9   rT   s     r>   rT   z%SmoothedEnergyGuidance.num_conditions   s9    !aN!aNrH   c                    | j                   syd}| j                  ^t        | j                  | j                  z        }t        | j                  | j                  z        }|| j
                  cxk  xr |k  nc }d}| j                  r!t        j                  | j                  d      }n t        j                  | j                  d      }|xr | S )NFTr%   r&   )
_enabled_num_inference_stepsr/   _start_stop_stepr!   mathiscloser   )r9   is_within_rangeskip_start_stepskip_stop_stepis_closes        r>   rU   z&SmoothedEnergyGuidance._is_cfg_enabled   s    }}$$0!$++0I0I"IJO d.G.G!GHN-LnLO((||D$7$7=H||D$7$7=H/x</rH   c                 B   | j                   syd}| j                  ^t        | j                  | j                  z        }t        | j                  | j                  z        }|| j
                  cxk  xr |k  nc }t        j                  | j                  d      }|xr | S )NFTr%   )	rf   rg   r/   r   r   rj   rk   rl   r   )r9   rm   rn   ro   is_zeros        r>   rC   z&SmoothedEnergyGuidance._is_seg_enabled   s    }}$$0!$"9"9D<U<U"UVO !7!7$:S:S!STN-

K^KO,,t66<.w;.rH   )g      @gffffff@g   cAg    @r%   r&   NNr%   Fr%   r&   )N)NN)__name__
__module____qualname____doc___input_predictionsr
   floatr   r   r/   r   r   boolr,   torchnnModulerG   rO   r   strr   r\   Tensorra   propertyrD   rT   rU   rC   __classcell__)r=   s   @r>   r   r      s   -^ G !$$' )(.$'#&?Cgk"%).=s=s "=s 	=s
 !&=s "=s !=s &eCcN&;<=s ##?FbAc#cd=s  =s #'=s =s =s =s~guxx g4 g
>uxx > dh 08c5eTWY\T\oI]C^>^9_0`	l	4 /304	<< ell+  -	
 
: F F F   0 0$/ /rH   r   )rk   typingr   r   r   r   r   r   rz   configuration_utilsr
   hooksr   $hooks.smoothed_energy_guidance_utilsr   r   guider_utilsr   r   "modular_pipelines.modular_pipeliner   r    rH   r>   <module>r      s5     D D  4   u 9 ?]/\ ]/rH   