bi2, ddlmZddlmZmZddlZddlmZddlm Z ddl m Z dd l m Z dd lmZmZmZmZeeZd Zd ZeGd dZGddeZGddeZGddeZdej6j8deddfdZdej6j8dededdfdZddej6j8dede ddfdZ!y) ) dataclass)TupleUnionN) get_logger) unwrap_module)"_ALL_TRANSFORMER_BLOCK_IDENTIFIERS)TransformerBlockRegistry) BaseState HookRegistry ModelHook StateManagerfbc_leader_block_hookfbc_block_hookc eZdZUdZdZeed<y)FirstBlockCacheConfiga_ Configuration for [First Block Cache](https://github.com/chengzeyi/ParaAttention/blob/7a266123671b55e7e5a2fe9af3121f07a36afc78/README.md#first-block-cache-our-dynamic-caching). Args: threshold (`float`, defaults to `0.05`): The threshold to determine whether or not a forward pass through all layers of the model is required. A higher threshold usually results in a forward pass through a lower number of layers and faster inference, but might lead to poorer generation quality. A lower threshold may not result in significant generation speedup. The threshold is compared against the absmean difference of the residuals between the current and cached outputs from the first transformer block. If the difference is below the threshold, the forward pass is skipped. g? thresholdN)__name__ __module__ __qualname____doc__rfloat__annotations__\/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/hooks/first_block_cache.pyrr!s Iurrc&eZdZdfd ZdZxZS)FBCSharedBlockStatecZt|d|_d|_d|_d|_yNT)super__init__head_block_outputhead_block_residualtail_block_residualsshould_compute)self __class__s rr#zFBCSharedBlockState.__init__5s. PT15 SW!$(rc d|_d|_yr!)r&r')r(s rresetzFBCSharedBlockState.reset=s$(!"r)returnN)rrrr#r+ __classcell__r)s@rrr4s )#rrceZdZdZdedefdZdZdejjfdZ dZ ejjd ejd efd Zy ) FBCHeadBlockHookT state_managerrc.||_||_d|_yN)r1r _metadata)r(r1rs rr#zFBCHeadBlockHook.__init__Es*"rcdt|}tj|j|_|Sr3rr getr)r4r(moduleunwrapped_modules rinitialize_hookz FBCHeadBlockHook.initialize_hookJ+(01556F6P6PQ rr9c|jjd||}|jj|i|}t |t }|r||jj |z }n||z }|jj}dx} } |j|} | |_ | s|r*|jd||jj z} n|jd|z} |jj-|sJ|jd||jjz} |rPdgt|z} | | |jj <| | |jj<t | } | }|S| } | }|S|rHdgt|z} ||jj | d<||jj| d<n|} | |_||_|S)N hidden_statesrr )r4_get_parameter_from_args_kwargsfn_reforiginal_forward isinstancetuplereturn_hidden_states_indexr1 get_state _should_compute_remaining_blocksr'r&"return_encoder_hidden_states_indexlenr$r%)r(r9argskwargsoriginal_hidden_statesoutputis_output_tuplehidden_states_residual shared_stater>encoder_hidden_statesr' return_outputr$s r new_forwardzFBCHeadBlockHook.new_forwardOs!%!O!OP_aegm!n---t>v>$VU3 %+DNN,U,U%VYo%o "%+.D%D ",0,>,>,H,H,J 044 ->>?UV&4 # 55a86$..BkBk;ll!- A A! Dv M ~~@@L&& 55a86$..BsBs;tt&!%V 4 KX dnnGGHSh dnnOOP %m 4 #F !. "F %)FS[$8!'-dnn.W.W'X!!$'-dnn._._'`!!$$*!->L */EL , rc:|jj|Sr3)r1r+)r(r9s r reset_statezFBCHeadBlockHook.reset_states   " rrNr,c*|jj}|jy|j}||z jj }|jj }||z j }||j kDSr!)r1rEr%absmeanitemr)r(rNrOprev_hidden_states_residualabsmeanprev_hidden_states_absmeandiffs rrFz1FBCHeadBlockHook._should_compute_remaining_blockss))335  + + 3&2&F&F#),GGLLNSSU%@%D%D%F%K%K%M"44::<dnn$$rN)rrr _is_statefulrrr#r;torchnnModulerRrTcompilerdisableTensorboolrFrrrr0r0BsjLlu  1%((//1f ^^%u||%X\%%rr0cdeZdZddedeffd ZdZdejjfdZ xZ S) FBCBlockHookr1is_tailcLt|||_||_d|_yr3)r"r#r1rgr4)r(r1rgr)s rr#zFBCBlockHook.__init__s$ * rcdt|}tj|j|_|Sr3r6r8s rr;zFBCBlockHook.initialize_hookr<rr9c|jjd||}d}|jj|jjd||}|jj }|j r|j j|i|}|jrdx}} t|trS||jj|jdz }||jj|jdz } n||jz }|| f|_ |S||} | Sddg} || |jj<|| |jj<t| } | S)Nr>rPrr )r4r?rGr1rEr'r@rArgrBrCrDr$r&) r(r9rIrJrKoriginal_encoder_hidden_statesrOrLrNencoder_hidden_states_residualrQs rrRzFBCBlockHook.new_forwardsv!%!O!OP_aegm!n)-& >> < < H-1^^-[-['v. *))335  & &1T[[114B6BF||JNN&)Gfe,t~~HHILLjLjklLmm+t~~PPQT`TrTrstTuu3.4l6T6T-T*5KMk4l 1M ) 12M  "4LMG]M$..CC DOmM$..KK L!-0MrF) rrrrrdr#r;r^r_r`rRr-r.s@rrfrfs/lT  !%((//!rrfr9configr,cfttdi}g}|jD]]\}}|tvs$t |t j js3t|D]\}}|j|d||f_|jd\}} |jd\} } tjd|dt| ||j|D]*\}}tjd|dt||,tjd| dt| |d y ) a Applies [First Block Cache](https://github.com/chengzeyi/ParaAttention/blob/4de137c5b96416489f06e43e19f2c14a772e28fd/README.md#first-block-cache-our-dynamic-caching) to a given module. First Block Cache builds on the ideas of [TeaCache](https://huggingface.co/papers/2411.19108). It is much simpler to implement generically for a wide range of models and has been integrated first for experimental purposes. Args: module (`torch.nn.Module`): The pytorch module to apply FBCache to. Typically, this should be a transformer architecture supported in Diffusers, such as `CogVideoXTransformer3DModel`, but external implementations may also work. config (`FirstBlockCacheConfig`): The configuration to use for applying the FBCache method. Example: ```python >>> import torch >>> from diffusers import CogView4Pipeline >>> from diffusers.hooks import apply_first_block_cache, FirstBlockCacheConfig >>> pipe = CogView4Pipeline.from_pretrained("THUDM/CogView4-6B", torch_dtype=torch.bfloat16) >>> pipe.to("cuda") >>> apply_first_block_cache(pipe.transformer, FirstBlockCacheConfig(threshold=0.2)) >>> prompt = "A photo of an astronaut riding a horse on mars" >>> image = pipe(prompt, generator=torch.Generator().manual_seed(42)).images[0] >>> image.save("output.png") ``` r.rzApplying FBCHeadBlockHook to ''zApplying FBCBlockHook to 'z%Applying FBCBlockHook to tail block 'T)rgN)rrnamed_childrenr rBr^r_ ModuleList enumerateappendpoploggerdebug_apply_fbc_head_block_hookr_apply_fbc_block_hook) r9rnr1remaining_blocksname submoduleindexblockhead_block_name head_blocktail_block_name tail_blocks rapply_first_block_cachers?B!!4b"=M!002@i 9 9IW\W_W_WjWjAk %i0 @LE5  # #vQug%6$> ? @@ #3"6"6q"9OZ"2"6"6r":OZ LL1/1B!DEz=&:J:JK'4 e 1$q9:e]34 LL88IKL*mTBrrr1rcrtj|}t||}|j|tyr3)r check_if_exists_or_initializer0 register_hook_FBC_LEADER_BLOCK_HOOK)rr1rregistryhooks rrzrzs/99%@H M9 5D 4!78rrgcrtj|}t||}|j|tyr3)r rrfr_FBC_BLOCK_HOOK)rr1rgrrs rr{r{s.99%@H  w /D 41rrm)" dataclassesrtypingrrr^utilsrutils.torch_utilsr_commonr _helpersr hooksr r rrrrxrrrrr0rfr_r`rrrzrdr{rrrrs" -7.CC H 0"  $ #) #M%yM%`-9-`5CEHHOO5C=R5CW[5Cp9ehhoo9l9_d9im9 222X\2im2r