biyddlZddlZddlmZmZddlmZddlmZddl m Z m Z m Z m Z mZmZddlZddlZddlmZmZdd lmZdd lmZmZerdd lmZmZdd lmZee Z!d Z"dZ#dZ$dZ%Gdde&eZ'eGddZ(GddZ)GddeZ*GddeZ+GddeZ,ejZddddddddfd ej\j^d!ee&ejZfd"ee&ejZfd#ee&e'fd$e e0d%e1d&e1d'e1d(e1d)e e&d*dfd+Z2d ej\j^d,e(d*dfd-Z3d ej\j^d,e(d*dfd.Z4d ej\j^d,e(d*dfd/Z5d ej\j^d0e)d,e(d*dfd1Z6d ej\j^d0e)d,e(d*dfd2Z7d ej\j^d3e e&d*e ej\jpfd4Z9d ej\j^d3e e&d*e ejtfd5Z;d6e&d7e e&ej\j^fd*e&fd8Zd ej\j^d*e1fd;Z?d ej\j^d*ejZfd<Z@d=ZAd ej\j^d*dfd>ZBy)?N)contextmanager nullcontext) dataclass)Enum)DictListOptionalSetTupleUnion) get_loggeris_accelerate_available)_GO_LC_SUPPORTED_PYTORCH_LAYERS) HookRegistry ModelHook)AlignDevicesHook CpuOffload)send_to_devicegroup_offloadinglayer_execution_trackerlazy_prefetch_group_offloading lazy_leafsceZdZdZdZy)GroupOffloadingType block_level leaf_levelN)__name__ __module__ __qualname__ BLOCK_LEVEL LEAF_LEVEL[/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/hooks/group_offloading.pyrr.s KJr%rceZdZUejed<ejed<eed<eed<eed<eed<dZe e ed<dZ e e ed <dZ e eejj ej fed <y) GroupOffloadingConfig onload_deviceoffload_device offload_type non_blocking record_streamlow_cpu_mem_usageNnum_blocks_per_groupoffload_to_disk_pathstream)rr r!torchdevice__annotations__rboolr/r intr0strr1r cudaStreamr$r%r&r(r(3ss<<LL %%*.(3-.*.(3-.?CFHU5::,,ell:; <Cr%r(c|eZdZ ddeej j dejdejdej j deej j deeej jdeeejd e d e ejjejdfd ee d e d e deedeeddfdZdZedZdZddZdZdZdZdZej6j9dZej6j9dZy) ModuleGroupNmodulesr*r)offload_leader onload_leader parametersbuffersr,r1r-r. onload_selfr0group_idreturnc||_||_||_||_||_|xsg|_|xsg|_|xs| du|_| |_| |_ | |_ | |_ | |_ d|_ |j~||ntt||_t#|j }t$j&j)|jd|d|_g}|jD]R}|j-t/|j |j-t/|j T|j-|j |j-|j t/t0j3|}t5|Dcic] \}}|d| c}}|_|j6j9Dcic]\}}|| c}}|_i|_n|j?|_tAtBdrr?r@r,r1r-rAr.r0_is_offloaded_to_diskr7idrB_compute_group_hashospathjoinsafetensors_file_pathextendlistdictfromkeys enumerate tensor_to_keyitems key_to_tensorcpu_param_dict_init_cpu_param_dicthasattrr2getattrrGcurrent_acceleratortyper8_torch_accelerator_module)selfr<r*r)r=r>r?r@r,r1r-r.rAr0rB short_hash all_tensorsmoduleitensorkvs r&__init__zModuleGroup.__init__As" ,*,*$*}" (>F$,> *&!2$8!%*"  $ $ 0(0(:P:PV[[__%6V\VaVaVeVeVgVrVrVtN6 " ur%c#K |jjDcic](\}}||js|jn|*}}}|d}ycc}}w#d}wxYwwrh)rWrU is_pinnedrk)r^rlrc pinned_dicts r&_pinned_memory_tensorsz"ModuleGroup._pinned_memory_tensorssr &*%8%8%>%>%@!E6&2B2B2Dv((*&PK K  Ks-A%A-AAA%AA""A%c|j|j|j|_|jr4|jj |j j yyNr,)tor)r,rir-r]current_stream)r^rc source_tensors r&_transfer_tensor_to_devicez&ModuleGroup._transfer_tensor_to_devicesT#&&t'9'9HYHY&Z    KK % %d&D&D&S&S&U V r%c|jD]v}|jD]'}|r||n |j}|j||)|j D]'}|r||n |j}|j||)x|jD]'}|r||n |j}|j||)|jD]'}|r||n |j}|j||)yrh)r<r?rirxr@)r^ pinned_memory group_modulerlsourcerms r&_process_tensors_from_modulesz)ModuleGroup._process_tensors_from_moduless LL @L%002 ?1>u-EJJ//v> ?'..0 @2?v.V[[//? @  @__ ;E-:]5) F  + +E6 : ;ll +>U]S  T,, V$kknnT-@-@unU  Vr%c^|j|jy|jy)z5Onloads the group of parameters to the onload_device.N)r0rrr^s r&onload_zModuleGroup.onload_s(  $ $ 0  " " $  $ $ &r%c^|jr|jy|jy)z7Offloads the group of parameters to the offload_device.N)r0rrrs r&offload_zModuleGroup.offload_s%  $ $  ! ! #  # # %r%) NNNFNFFTNNrh)rr r!rr2nnModuler3r ParameterTensorr5r r8r9r7r6rfrXrrqrxr}rrrrcompilerdisablerrr$r%r&r;r;@s489=04"?C(-"' .2"&9 ehhoo&9  9 || 9  9  0 9 T%(("4"4569 $u||,-9 9 ejj''t;<9  ~9  9 9 'sm9 3-9 !9 v*W <":6 9\$V( ^^'' ^^&&r%r;ceZdZdZdZdededdfdZdejjdejjfd Z dejjfd Z dejjfd Z y) GroupOffloadingHooka A hook that offloads groups of torch.nn.Module to the CPU for storage and onloads to accelerator device for computation. Each group has one "onload leader" module that is responsible for onloading, and an "offload leader" module that is responsible for offloading. If prefetching is enabled, the onload leader of the previous module group is responsible for onloading the current module group. FgroupconfigrCNc.||_d|_||_yrh)r next_groupr)r^rrs r&rfzGroupOffloadingHook.__init__s 15 r%racl|jj|k(r|jj|Srhrr=r)r^ras r&initialize_hookz#GroupOffloadingHook.initialize_hook ) :: $ $ . JJ   ! r%c|jj||j_|jj|k(r|jjr|jj|jduxr|jj }|r|jj|jj xr|jj duxr| }|r$|jj j t||jj|jj}t||jj|jj}||fSrs) rr>rArrr1rrr)r,)r^raargskwargsshould_onload_next_groupshould_synchronizes r& pre_forwardzGroupOffloadingHook.pre_forward%s$ :: # # +'-DJJ $ :: # #v -zz%% ""$'+d'B'f4??KfKfGf $''')JJ***mtzz/@/@/LmUmQm " !!--/dDJJ$<$<4::KbKbc (@(@tzzOfOfgV|r%cl|jj|k(r|jj|Srhr)r^raoutputs r& post_forwardz GroupOffloadingHook.post_forwardFrr%)rr r!__doc__ _is_statefulr;r(rfr2rrrrrr$r%r&rrsvLk6KPT ehhoo%((// %((//B588??r%rc&eZdZdZdZdZdZdZy)LazyPrefetchGroupOffloadingHooka A hook, used in conjunction with GroupOffloadingHook, that applies lazy prefetching to groups of torch.nn.Module. This hook is used to determine the order in which the layers are executed during the forward pass. Once the layer invocation order is known, assignments of the next_group attribute for prefetching can be made, which allows prefetching groups in the correct order. Fc0g|_t|_yrh)execution_orderset%_layer_execution_tracker_module_namesrs r&rfz(LazyPrefetchGroupOffloadingHook.__init__VsBD58U2r%chfd}|jD]\}}|dk(s t|dstj|}|j t }|Ed|j _t|||}|j|tjj||S)Ncfd}|S)Nctjjstj ddj j fy)NzAdding z to the execution order)r2r is_compilingloggerdebugrappend) current_namecurrent_submoduler^sr&callbackzoLazyPrefetchGroupOffloadingHook.initialize_hook..make_execution_order_update_callback..callback\sC~~224LL7<.8O!PQ$$++\;L,MNr%r$)rrrr^s`` r&$make_execution_order_update_callbackz]LazyPrefetchGroupOffloadingHook.initialize_hook..make_execution_order_update_callback[s O Or%_diffusers_hookF) named_modulesrYrcheck_if_exists_or_initializeget_hook_GROUP_OFFLOADINGrr,LayerExecutionTrackerHook register_hook_LAYER_EXECUTION_TRACKERradd)r^rarname submoduleregistrygroup_offloading_hooklayer_tracker_hooks` r&rz/LazyPrefetchGroupOffloadingHook.initialize_hookZs  &335 EOD)rz4E!F#AA)LH$,$5$56G$H !$0;@%++8%>?cdhjs?t%u"&&'9;ST::>>tD E r%cft|j}|jDchc]\}}| }}}||jk7rNt|j|z }tj j stjd||j}|jD cgc]\}} | j} }} | D cgc]} | jt} } t|D]} | | jtd|jtd| D]}d|j _|dkDr:|jt}| dj |_d|j$_t|dz D]} |j| \}}|j| dz\}}tj j stj)d|d|| | dzj | | _d| | j$_|Scc}}wcc} }wcc} w) NaBIt seems like some layers were not executed during the forward pass. This may lead to problems when applying lazy prefetching with automatic tracing and lead to device-mismatch related errors. Please make sure that all layers are executed during the forward pass. The following layers were not executed: unexecuted_layers=FrecurseTrrz-Applying lazy prefetch group offloading from z to )lenrrrPr2rrrwarningrrrrange remove_hookr_LAZY_PREFETCH_GROUP_OFFLOADINGrr,rrAr)r^rar num_executedr_execution_order_module_namesunexecuted_layersbase_module_registryr registriesrgroup_offloading_hooksrbhook!base_module_group_offloading_hookname1name2s r&rz,LazyPrefetchGroupOffloadingHook.post_forwardvs64//0 <@>..0*)*, &55DHDXDXYLAyi//Y YWa!b8("3"34E"F!b!b|$ OA qM % %&> % N O (()HRW(X + +D&*DJJ # + ! 0D0M0MN_0` -;QRS;T;Z;Z - 8GL - 8 8 D|a'( EA++A.HE1++AE2HE1>>..0 LUGSWX]W^_`3I!a%3P3V3V "1 % 0?D "1 % 0 0 <  E _(R$Z!bs H",H( H.N)rr r!rrrfrrr$r%r&rrLsL;84r%rc eZdZdZdZdZdZy)rz A hook that tracks the order in which the layers are executed during the forward pass by calling back to the LazyPrefetchGroupOffloadingHook to update the execution order. Fc||_yrhexecution_order_update_callback)r^rs r&rfz"LayerExecutionTrackerHook.__init__s /N,r%c*|j||fSrhr)r^rarrs r&rz%LayerExecutionTrackerHook.pre_forwards ,,.V|r%N)rr r!rrrfrr$r%r&rrs LOr%rrjrFrar)r*r+r/r, use_streamr-r.r0rCc t|trtj|n|}t|trtj|n|}t |}d} |rtj j rtj j} nNttdr3tjj rtj} n td|s |r td|tjk(r | tdt|t|||||| |||  } t|| y)a Applies group offloading to the internal layers of a torch.nn.Module. To understand what group offloading is, and where it is beneficial, we need to first provide some context on how other supported offloading methods work. Typically, offloading is done at two levels: - Module-level: In Diffusers, this can be enabled using the `ModelMixin::enable_model_cpu_offload()` method. It works by offloading each component of a pipeline to the CPU for storage, and onloading to the accelerator device when needed for computation. This method is more memory-efficient than keeping all components on the accelerator, but the memory requirements are still quite high. For this method to work, one needs memory equivalent to size of the model in runtime dtype + size of largest intermediate activation tensors to be able to complete the forward pass. - Leaf-level: In Diffusers, this can be enabled using the `ModelMixin::enable_sequential_cpu_offload()` method. It works by offloading the lowest leaf-level parameters of the computation graph to the CPU for storage, and onloading only the leafs to the accelerator device for computation. This uses the lowest amount of accelerator memory, but can be slower due to the excessive number of device synchronizations. Group offloading is a middle ground between the two methods. It works by offloading groups of internal layers, (either `torch.nn.ModuleList` or `torch.nn.Sequential`). This method uses lower memory than module-level offloading. It is also faster than leaf-level/sequential offloading, as the number of device synchronizations is reduced. Another supported feature (for CUDA devices with support for asynchronous data transfer streams) is the ability to overlap data transfer and computation to reduce the overall execution time compared to sequential offloading. This is enabled using layer prefetching with streams, i.e., the layer that is to be executed next starts onloading to the accelerator device while the current layer is being executed - this increases the memory requirements slightly. Note that this implementation also supports leaf-level offloading but can be made much faster when using streams. Args: module (`torch.nn.Module`): The module to which group offloading is applied. onload_device (`torch.device`): The device to which the group of modules are onloaded. offload_device (`torch.device`, defaults to `torch.device("cpu")`): The device to which the group of modules are offloaded. This should typically be the CPU. Default is CPU. offload_type (`str` or `GroupOffloadingType`, defaults to "block_level"): The type of offloading to be applied. Can be one of "block_level" or "leaf_level". Default is "block_level". offload_to_disk_path (`str`, *optional*, defaults to `None`): The path to the directory where parameters will be offloaded. Setting this option can be useful in limited RAM environment settings where a reasonable speed-memory trade-off is desired. num_blocks_per_group (`int`, *optional*): The number of blocks per group when using offload_type="block_level". This is required when using offload_type="block_level". non_blocking (`bool`, defaults to `False`): If True, offloading and onloading is done with non-blocking data transfer. use_stream (`bool`, defaults to `False`): If True, offloading and onloading is done asynchronously using a CUDA stream. This can be useful for overlapping computation and data transfer. record_stream (`bool`, defaults to `False`): When enabled with `use_stream`, it marks the current tensor as having been used by this stream. It is faster at the expense of slightly more memory usage. Refer to the [PyTorch official docs](https://pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html) more details. low_cpu_mem_usage (`bool`, defaults to `False`): If True, the CPU memory usage is minimized by pinning tensors on-the-fly instead of pre-pinning them. This option only matters when using streamed CPU offloading (i.e. `use_stream=True`). This can be useful when the CPU memory is a bottleneck but may counteract the benefits of using streams. Example: ```python >>> from diffusers import CogVideoXTransformer3DModel >>> from diffusers.hooks import apply_group_offloading >>> transformer = CogVideoXTransformer3DModel.from_pretrained( ... "THUDM/CogVideoX-5b", subfolder="transformer", torch_dtype=torch.bfloat16 ... ) >>> apply_group_offloading( ... transformer, ... onload_device=torch.device("cuda"), ... offload_device=torch.device("cpu"), ... offload_type="block_level", ... num_blocks_per_group=2, ... use_stream=True, ... ) ``` NxpuzOUsing streams for data transfer requires a CUDA device, or an Intel XPU device.z7`record_stream` cannot be True when `use_stream=False`.zO`num_blocks_per_group` must be provided when using `offload_type='block_level'.) r)r*r+r/r,r1r-r.r0)rr7r2r3rr8 is_availabler9rYr ValueErrorr";_raise_error_if_accelerate_model_or_sequential_hook_presentr(_apply_group_offloading) rar)r*r+r/r,rr-r.r0r1rs r&apply_group_offloadingrs r4>mS3QELL/WdM5?PS5TU\\.1ZhN&|4L F :: " " $ZZ&&(F UE "uyy'='='?\\^Fno o -RSS*666;O;Wjkk?G "#%!1!#+1 FFF+r%rc|jtjk(r t||y|jtjk(r t ||yJrh)r+rr"#_apply_group_offloading_block_levelr#"_apply_group_offloading_leaf_level)rars r&rr8sE 1===+FF;    3 > > >*66:ur%c|j9|jdk7r*tjd|jdd|_t }g}g}|j D]`\}}t |tjjtjjfs%|j||f|j|jtdt||jD]}||||jz}|d|d|t|zdz } t||j |j"|j$|d|d|j&|j|j(|j*d| } |j| t||t|zD]} |j|d | ct-|D]$\}} | j.D]} t1| | | &t3||} t5||}| Dcgc]\}}| } }}|Dcgc]\}}| }}}|Dcgc]\}}| }}}t||j |j"|j$||| |d dd d|j6j8d  }|jt1||| yt;||| ycc}}wcc}}wcc}}w)z This function applies offloading to groups of torch.nn.ModuleList or torch.nn.Sequential blocks. In comparison to the "leaf_level" offloading, which is more fine-grained, this offloading is done at the top-level blocks. Nrz\Using streams is only supported for num_blocks_per_group=1. Got config.num_blocks_per_group=z. Setting it to 1.rrT r<r*r)r0r=r>r,r1r-r.rArB.rF_unmatched_group) r<r*r)r0r=r>r?r@r,r1r-rArB)r1r/rrrnamed_childrenrr2r ModuleList Sequentialrrrrr;r*r)r0r,r-r.rSr<_apply_group_offloading_hook2_gather_parameters_with_no_group_offloading_parent/_gather_buffers_with_no_group_offloading_parent __class__r!_apply_lazy_group_offloading_hook)rarmodules_with_group_offloadingunmatched_modulesmatched_module_groupsrrrbcurrent_modulesrBrjr{r?r@rrlrmunmatched_moduleunmatched_groups r&rrAs }} V%@%@A%EkvOjOjNll~  '(#%(E!!002Ai)ehh&9&9588;N;N%OP  $ $dI%6 7 ) - -d 3 q#i.&*E*EF AA'A0K0K,KLOq1Q_)=%=%A$BCH'%44$22%+%@%@.r2-a0#00}}$22"(":": ! E " ( ( /1a#o"667 A-11TF!A3-@ A% A A834M5!MM ML (uV L MMDFLijJ=fFcdG(23HAu%3J3'./)!Vv/G/FWW.Aa1A)WW!!,,**#88$$--..>?O}}$V_VL)&/&Q14/Xs K+ K ? Kct}|jD]\}}t|tst |g|j |j |j|||j|j|j|jd| }t||||j|t|j}t||}t!||}i} |D]0\}} t#||} | | vr| | j%| +| g| | <2i} |D]0\}} t#||} | | vr| | j%| +| g| | <2t| j't| j'z}|D]}| j)|g}| j)|g}||}t g|j |j |||j|||j|j|j|jd|}t||||jTt g|j |j |j||ddddd|jdt*}t-|||yy)a This function applies offloading to groups of leaf modules in a torch.nn.Module. This method has minimal memory requirements. However, it can be slower compared to other offloading methods due to the excessive number of device synchronizations. When using devices that support streams to overlap data transfer and computation, this method can reduce memory usage without any performance degradation. Trr)r<r*r)r=r>r0r?r@r,r1r-r.rArBNF)r<r*r)r0r=r>r?r@r,r1r-r.rArB)rrrrr;r*r)r0r,r1r-r.rrrQrr"_find_parent_module_in_module_dictrrget_GROUP_ID_LAZY_LEAFr)rarr rrr module_dictr?r@parent_to_parametersrl parent_nameparent_to_buffersrm parent_names parent_modulers r&rrs%(E!!//10i)%DE K!00 ..!'! * + Nr%rrc|jd}t|dkDr6dj|}||vr|S|jt|dkDr6y)Nrrr)r!rrMr")rrr%rs r&rr8sP JJsOE e*q.hhuo + %  e*q. r%c tsy|jD]M\}}t|dst|jt t fs4td|dt|dy)N_hf_hookzCannot apply group offloading to a module that is already applying an alternative offloading strategy from Accelerate. If you want to apply group offloading, please disable the existing offloading strategy first. Offending module: z ()) rrrYrr*rrrr\)rarrs r&rrBs{ " $!//1iy*-  i((+;Z*H IUUYTZZ\]abk]l\mmnp  r%c|jD]4}t|ds|jjt}|2|cSy)Nr)r<rYrrr)rarrs r&!_get_top_level_group_offload_hookr-PsL^^%- 9/ 0$-$=$=$F$FGX$Y !$0,, - r%c t|}|duSrh)r-ratop_level_group_offload_hooks r&_is_group_offload_enabledr1Ys#DV#L 't 33r%c^t|}||jjStd)Nz8Group offloading is not enabled for the provided module.)r-rr)rr/s r&_get_group_onload_devicer3^s1#DV#L #/+22@@@ O PPr%cptj|jdj}|ddS)Nzutf-8)hashlibsha256encode hexdigest)rB hashed_ids r&rJrJes.xw78BBDI Sb>r%ct|}|ytj|}|jtd|jt d|jt dt||jy)a| Removes the group offloading hook from the module and re-applies it. This is useful when the module has been modified in-place and the group offloading hook references-to-tensors needs to be updated. The in-place modification can happen in a number of ways, for example, fusing QKV or unloading/loading LoRAs on-the-fly. In this implementation, we make an assumption that group offloading has only been applied at the top-level module, and therefore all submodules have the same onload and offload devices. If this assumption is not true, say in the case where user has applied group offloading at multiple levels, this function will not work as expected. There is some performance penalty associated with doing this when non-default streams are used, because we need to retrace the execution order of the layers with `LazyPrefetchGroupOffloadingHook`. NTr) r-rrrrrrrr)rar0rs r&*_maybe_remove_and_reapply_group_offloadingr<ksv$EV#L #+99&AH *D9 14@ 8$GF$@$G$GHr%)Cr6rK contextlibrr dataclassesrenumrtypingrrr r r r safetensors.torchrr2utilsrr_commonrhooksrraccelerate.hooksrraccelerate.utilsrrrrrrrr7rr(r;rrrr3rrr6r5rrrrrrrrrrrrr-r1r3rJr<r$r%r&rGs 2!:: 74*=/ H '4"B"#t   D D  DN&N&b8)8v^i^B  &0