
    bi                        d dl mZ d dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
mZ ddlmZ  e       r&d dlmZ  G d	 d
ej                   j"                  e      ZddZy)    )annotationsN)Optional)is_hqq_available)BaseTunerLayercheck_adapters_to_merge   )OFTLayer)	HQQLinearc                       e Zd Z	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd	d
dZddZd ZddZd fdZ xZ	S )HqqOFTLinearc                    t         |           t        j                  | |       d| _        || _        | j                  ||||||||	|
|
       y )NF)oft_block_sizemodule_dropoutinit_weightscoftepsblock_shareuse_cayley_neumannnum_cayley_neumann_terms)super__init__r	   fan_in_fan_out_active_adapterupdate_layer)self
base_layeradapter_namerr   r   r   r   r   r   r   r   kwargs	__class__s                N/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/oft/hqq.pyr   zHqqOFTLinear.__init__!   sa     GdJ/"'D#/D --)'#5)A      c                   t        | |      }|sy|D ]  }|| j                  j                         vr!| j                         }i t	        j
                  |j                        d|j                  i}|j                         }| j                  |      }t        j                  |dd      }t        j                  ||j                  |j                              }t        j                  |dd      }|j                  |j                        j                  |j                        }|r2t        j                   |      j#                         st%        d| d      t'        d||j(                  |j                        }	|j+                  dd        |	j,                  |fi | |	| _        | j0                  j3                  |        y)a  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            Noffload_metar   r   z1NaNs detected in the merged weights. The adapter z seems to be brokencompute_dtypedevice)r   lora_Akeysget_base_layercopydeepcopyquant_configr$   
dequantizeget_delta_weighttorch	transposemmtodtyper'   isfiniteall
ValueErrorr
   r&   popquantizer   merged_adaptersappend)
r   
safe_mergeadapter_namesactive_adapterlayerr-   outputoft_dataw_datanew_hqq_layers
             r!   mergezHqqOFTLinear.mergeB   s    4D-HM "/ <!)9)9);;++-h$--0B0B"Ch^UZUgUgh))+00@A6(FIIhnn,EFA68>>255hooFennV&<&@&@&B$KNK[[no  !*$EL_L_hmhtht u  6&&&v>>"/$$++N;1<r"   c                   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }|| j                  j                         vrP| j                         }i t        j                  |j                        d|j                  i}|j                         }| j                  |      }t        j                   |dd      }t        j"                  |j%                         |j'                  |j(                              }t        j                   |dd      }|j'                  |j(                        j'                  |j*                        }t-        d||j.                  |j*                        }|j                  dd        |j0                  |fi | || _        t        | j                        dkD  ryy)z_
            This method unmerges all merged adapter layers from the base weights.
            z Already unmerged. Nothing to do.Nr   r$   r   r%   )mergedwarningswarnlenr:   r8   oft_Rr)   r*   r+   r,   r-   r$   r.   r/   r0   r1   r2   tr3   r4   r'   r
   r&   r9   r   )r   r>   r?   r-   r@   rA   rB   rC   s           r!   unmergezHqqOFTLinear.unmergen   sq    ;;@Ad**+a/!%!5!5!9!9!;!)::++-h$--0B0B"Ch^UZUgUgh))+00@A6(**,		(..0IJA68>>255hooF )$EL_L_hmhtht u  6&&&v>>"/' d**+a/r"   c                <    | j                   |   j                         S N)rJ   
get_weight)r   adapters     r!   r/   zHqqOFTLinear.get_delta_weight   s    ::g&1133r"   c                    | j                   |g|i | |j                  dd       }| j                  r3| j                  r| j	                           | j
                  |g|i |}n| j                  r | j
                  |g|i |}n| j                  D ]  }|| j                  j                         vr | j                  |   }t        j                          }|r2|j                  }	| j                  ||j                  j                        } ||      }  | j
                  |g|i |}r|j                  	      }|S )Nr=   )_check_forward_argsr8   disable_adaptersrF   rL   r   active_adaptersrJ   r)   r0   is_autocast_enabledr4   _cast_input_dtypeweightr3   )
r   xargsr   r=   resultr>   rJ   requires_conversionexpected_dtypes
             r!   forwardzHqqOFTLinear.forward   s.   $D$$Q888"JJ=M$$;;LLN(<T<V<(<T<V<&*&:&: 
!N%TZZ__->>  JJ~6E.3.G.G.I*I'*)* 221ell6H6HIaA
! %T__Q888F">2Mr"   c                *    t         |          }d|z   S )Nzoft.)r   __repr__)r   repr    s     r!   r_   zHqqOFTLinear.__repr__   s    '"$CC<r"   )	   r   g        TFgiUMu?FF   )r   torch.nn.Moduler   strr   intr   re   r   floatr   boolr   rg   r   rf   r   rg   r   rg   r   re   returnNone)FN)r<   rg   r=   zOptional[list[str]]rh   ri   )rh   ri   )rX   torch.Tensorrh   rj   )rh   rd   )
__name__
__module____qualname__r   rD   rL   r/   r]   r_   __classcell__)r    s   @r!   r   r      s     "#$'!% %',,-	'	 	 		
  	 "	 	 	 	 	 !%	 '*	 	B*	<X	0:	4	8	  	 r"   r   c                    d }t        | t              r| j                         }n| }t               rt        |t              rt        ||fi |}|S rN   )
isinstancer   r*   r   r
   r   )targetr   r   
new_moduletarget_base_layers        r!   dispatch_hqqrt      sN    J&.)"113"j):IF!"3\LVL
r"   )rq   rc   r   rd   )
__future__r   r+   rG   typingr   r0   peft.import_utilsr   peft.tuners.tuners_utilsr   r   r?   r	   hqq.core.quantizer
   nnModuler   rt    r"   r!   <module>r}      sH    #     . L  +M uxx M `r"   