bi\ddlZddlZddlZddlmZddlmZGddeZGddeZ ddejd e d e d e d e d e defdZ y)N)nn)saved_tensors_hooksc BeZdZdZ d dededededdf fd ZxZS) OffloadActivationsa Context manager under which activation tensors created in the forward pass will be offloaded. Enable the memory efficiency technique of activation offloading, where activations bigger than `min_offload_size` bytes will be offloaded to CPU in the forward and brought back in the backward. This is in contrast to maintaining the activation on GPU VRAM throughout the program. This manager contains the option of using one additional CUDA stream to handle the communication between CUDA and CPU, which is intended to overlap with the default computation stream to improve runtime. We designed synchronization with a few heuristics for optimizing the tradeoff between runtime vs memory usage. Args: use_pin_memory (`bool`, *optional*, defaults to `True`): Whether to offloaded Tensor will be placed in pinned memory on the CPU. Pinned memory allows the Tensor to be moved back onto GPU more quickly but is a limited resource. use_streams (`bool`, *optional*, defaults to `True`): Whether to use streams for performance optimization where the communications get overlapped with the computation. Requires a torch build after torch-2.5.0. min_offload_size (`int`, *optional*, defaults to `1024`): Minimum number of bytes a Tensor must be in order to qualify for offloading. If the tensor is too small, we do not want to waste bandwidth and resources moving it to CPU and back. max_fwd_stash_size (`int`, *optional*, defaults to `5`): Maximum size of the forward stash, or the maximum number of consecutive activations to keep alive during the forward pass. This number must be at least 1. Keeping alive more activations will potentially allow more overlap between the communication and compute streams at the cost of increasing memory usage. Keeping alive fewer activations will conserve memory, but may cause poor overlap between the streams, increasing runtime. Raises: ValueError: if `max_fwd_stash_size` is not at least `1`. Example: ```python >>> with OffloadActivations(): ... outputs = model(inputs, labels=labels) >>> loss = outputs.loss >>> loss.backward() ``` use_pin_memory use_streamsmin_offload_sizemax_fwd_stash_sizereturnNc& |_|_i_d_d_d_d_|_d_ttdr(tjjjnd_jdk(rtjj!ntj"j%_jrjdk(rtj(ntj"j)_i_|dkrt/d||_i_i_d_d_ fd  d t:fd  d t<ffd d tj>d t<fd dtj>d t<f fd }dt<d tj>f fd }dt<d tj>f fd }jr|n|}t@ ||y)NrT< acceleratorcudaxpuz/max_fwd_stash_size should be at least 1 but is c}|jkDr'tjd|djdyy)Nz curr_pct=z!% > self.virtual_memory_safe_pct=z% of virtual memory used)virtual_memory_safe_pctwarningswarn)curr_pctget_cpu_ram_pctselfs [/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/trl/models/activation_offloading.py verify_sufficient_virtual_memoryzEOffloadActivations.__init__..verify_sufficient_virtual_memoryosB&(H$666  +M0L0L/NNfgh7r c>tjjSN)psutilvirtual_memorypercentrrrz4OffloadActivations.__init__..get_cpu_ram_pctts((*22 2rcFxjdz c_jS)Nr) tensor_id)rsr get_tensor_idz2OffloadActivations.__init__..get_tensor_idxs NNa N>> !rxcD|j|jzSr) element_sizenelement)r%s rget_num_bytes_tensorz9OffloadActivations.__init__..get_num_bytes_tensor}s>>#ajjl2 2r activationc jr1t jdk7r tdd _d _ |} }|j j dvr| jk\rt|tjjsttjdr%t|tjjs jrt j j#D]P}|| j$z kr; j |\}} j&j)| j |=Pn j*j- j& jr j*n j&} j.dk(r|ntj0j3|5tj4| j6d }|j9|d |df j|<ddd jr+ j*j;}||f j |<|S|df j|<|S#1swYUxYw) Nrz8Backward pass should have cleared tracker of all tensorsFT)rrBufferrcpu) pin_memorydevice non_blocking)is_first_forward_calllentracker ValueErroris_first_backward_callr/typemin_tensor_size_bytes isinstancetorchr Parameterhasattrr,rlist fwd_stashkeysr s0 wait_events1 wait_streamaccelerator_typerstream empty_likercopy_ record_event) r* num_bytesr#id_evrE cpu_tensoreventr)r$rs r pack_tensorz0OffloadActivations.__init__..pack_tensors"))t||$)$%_``.3*.2+-Z8I%I !!&&/9!;!;;":uxx/A/AB$UXXx8Z TYT\T\TcTc=d###4>>#6#6#89"T-D-D!DD$(NN2$6EAr GG..r2 $r 2! "GG''0$($4$4$''#44=V5::CTCTU[C\!&!1!1*I\I\ej!kJ$$Zd$C"/DLL+## GG002E2.unpack_tensor_single_streams**--16D.**8:.3+-1*t||3 #<=MN1O . $h%=%@%@AVAVei%@%j"+=( -.+ +rc( jrtjj _d fd }tj j jjj| jrd _ jr d _d _ jvrtd j\}}|rtjj}tjj}g | jk(r9 j |k7r*| _t# j$j' d j(vr j(d}dn׉ j*dk(r j,n(tj.j1 j,5|j3 j*d}|}ddd j4j7 j,| j$<tjj9|j;j<  fd}|j?| j=|S#1swYxYw) NctjjD]9}j|}jj |j|=;yr)r=bwd_tensor_stashr? bwd_ev_stashrBrA)rJrNrs r!wait_and_del_remaining_referenceszjOffloadActivations.__init__..unpack_tensor_with_streams..wait_and_del_remaining_referencessV"4#8#8#=#=#?@6 $ 1 1" 5**51 11"56rFTzuntracked tensor with id rrr0cr j }tjj|j j  kDr)|j j j =n) jj}| j <t jjD]<} j|\}} jj| j|=>D]9} j|} jj| j|=;|Sr)rZr:_C_storage_Use_Countuntyped_storage_cdata record_streamr@rHr[r=r>r?rArB) outputsinputsunpacked_tensorrNrJrKrLbrought_back_from_cpu prev_node_idsrstorage_refcountrPs rhookzMOffloadActivations.__init__..unpack_tensor_with_streams..hooks,+/*?*?@P*Q 88667V7V7X7_7_`css+99$''B $ 5 56F G$(GG$8$8$:EBGD--.>?#4>>#6#6#89/ $r 22**2. NN2./ ,6 $ 1 1" 5**51 11"56 #Nrr N) r6r:r^_current_graph_task_id curr_graph_idautogradvariableVariable_execution_enginequeue_callbackrRrr2r4r5_current_autograd_nodecurr_autograd_noder=rZr?r>rDrBrrErSr@rCr_r`ra register_hook) rPr\rTrUgraph_idnoderVrirfrgrhrrs ` @@@runpack_tensor_with_streamsz?OffloadActivations.__init__..unpack_tensor_with_streamss<**%*XX%D%D%F"6''00BBQQRst--16D.**8:.3+-1*t||3 #<=MN1O . $h 88::<xx668 " t111d6M6MQU6U.2D+$()>)>)C)C)E$FM(,%#t~~5/3~~>N/OPQ/R,,1)%)$9$9U$B HYHYZ^ZaZaHbF-E-H-HI^I^mq-H-r*3E0F GG''0?WD))*:;(-xx'B'BC[CkCkCmCtCt'u$"#"#H""4( -.+ +CFFs JJ)"rr8r4r#r2r6rRrrr<r:rcurrent_acceleratorr7rDrcurrent_streamrdefault_streamr@StreamrBr>r5r rZr[rlrsfloatintTensorsuper__init__)rrrr r rOrWrw unpack_tensorrr)r$r __class__s` @@@@rrzOffloadActivations.__init__Fs'%5" %)"&*#%)"-')$=DE=.noopSsMr)rr)rrrs rrzNoOpManager.__init__Rs  t$rrj)rrrrrrrs@rrrIs%%rrmodelrrr r warn_if_no_headr cX t||||}d}t |}t|dr |j}t|drt|dr |j}t|drt |j tjrB|j j fd|j j fdd d }nnt|j d rWt |j jtjr(|j jj fd |j jj fd d d }nt|drB|jj fd|jj fdd d }nt|dr|j} t| drB| j j fd| j j fdd d }nt| dr| jj fd| jj fdd d }nt|ds t|drHt|ddxs |j} | j fd| j fdd d }npt|drdt |j tjr@|j j fd|j j fdd d }|s|rt#j$d|j'D]B\} } d| j)vs| j fd | j fd!d D|S)"a( Returns the activation offloading context manager for the model. All but the last output Linear in every step will be offloaded. If activation offloading is enabled, we return the OffloadActivations context manager. If activation offloading is disabled, we return a NoOpManager context manager. Args: model (`nn.Module`): Model to wrap with the activation offloading context manager. use_pin_memory (`bool`, *optional*, defaults to `True`): Whether to offloaded Tensor will be placed in pinned memory on the CPU. Pinned memory allows the Tensor to be moved back onto GPU more quickly but is a limited resource. use_streams (`bool`, *optional*, defaults to `True`): Whether to use streams for performance optimization where the communications get overlapped with the computation. Requires a torch build after torch-2.5.0. min_offload_size (`int`, *optional*, defaults to `1024`): Minimum number of bytes a Tensor must be in order to qualify for offloading. If the tensor is too small, we do not want to waste bandwidth and resources moving it to CPU and back. max_fwd_stash_size (`int`, *optional*, defaults to `5`): Maximum size of the forward stash, or the maximum number of consecutive activations to keep alive during the forward pass. This number must be at least 1. Keeping alive more activations will potentially allow more overlap between the communication and compute streams at the cost of increasing memory usage. Keeping alive fewer activations will conserve memory, but may cause poor overlap between the streams, increasing runtime. warn_if_no_head (`bool`, *optional*, defaults to `True`): Whether to warn if no output head is detected. If set to `False`, no warning will be raised if no output head is detected. Returns: `contextlib.ContextDecorator`: Activation offloading context manager for the model. )rrr r Fmodule base_model peft_configoutputc$jSr __enter__argsnoop_ctxs rz0get_act_offloading_ctx_manager..s8K]K]K_rc$jSr__exit__rs rrz0get_act_offloading_ctx_manager..sxGXGXGZrT) always_calllinearc$jSrrrs rrz0get_act_offloading_ctx_manager..sRZRdRdRfrc$jSrrrs rrz0get_act_offloading_ctx_manager..shN_N_Narlm_headc$jSrrrs rrz0get_act_offloading_ctx_manager..sHZHZH\rc$jSrrrs rrz0get_act_offloading_ctx_manager..sHDUDUDWrdecoderc$jSrrrs rrz0get_act_offloading_ctx_manager..s8CUCUCWrc$jSrrrs rrz0get_act_offloading_ctx_manager..sx?P?P?Rrc$jSrrrs rrz0get_act_offloading_ctx_manager..sHDVDVDXrc$jSrrrs rrz0get_act_offloading_ctx_manager..s@Q@Q@Srfinal_layer_normln_fNc$jSrrrs rrz0get_act_offloading_ctx_manager..8;M;M;Orc$jSrrrs rrz0get_act_offloading_ctx_manager..x7H7H7Jrheadc$jSrrrs rrz0get_act_offloading_ctx_manager..sXEWEWEYrc$jSrrrs rrz0get_act_offloading_ctx_manager..sARARATraQDuring activation offloading, no output head was detected. If your model has an output head, it will be offloaded. This usually greatly slows training, given the large vocabulary size. To change this behavior, set your output head as model.output and make it an nn.Module. You can disable this warning by passing `warn_if_no_head=False`.ligerc$jSrrrs rrz0get_act_offloading_ctx_manager..rrc$jSrrrs rrz0get_act_offloading_ctx_manager..rr)rrr<rrr9rrModuleregister_forward_pre_hookregister_forward_hookrrrgetattrrrrr named_moduleslower)rrrr r ractivations_handling_ctxoutput_head_detectedunwrapped_modelr final_normnamerrs @rget_act_offloading_ctx_managerrYsR 2%)-  !}HO))00 -'/=2Y)44) o,,bii 8  " " < <=_ `  " " 8 89Zhl 8 m#' _++X 6:oF\F\FcFcegenen;o  " " ) ) C CDf g  " " ) ) ? ?@aos ? t#'  ) ,99:\]556Wei5j# ) ,!)) 7H % NN 4 45W X NN 0 01R`d 0 e#' Wi ( OO 5 56X Y OO 1 12Sae 1 f#'  "4 5RX9Y_.@$G_?K_K_ ,,-OP(()JX\(]# & )j9M9Mryy.Y667YZ223Tbf2g# O  / (557^ f djjl "  , ,-O P  ( ()JX\ ( ]^ $#r)TTrrT) rrr:rtorch.autograd.graphrrrrrr}rr!rrrs* 4i5,i5X %% %$   w$ 99w$w$w$ w$  w$  w$w$r