
    bi(                        d dl mZ d dlZd dlZd dlmZmZ d dlZd dlm	Z	 d dl
mZmZ d dlmZ ddlmZmZ  e	       r&d d	lmZ  G d
 dej(                  j*                  e      ZddZy)    )annotationsN)AnyOptional)is_hqq_available)BaseTunerLayercheck_adapters_to_merge)	transpose   )	LoraLayerLoraVariant)	HQQLinearc                       e Zd Z	 	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
 fdZddZdddZddZd Z	 	 	 	 	 	 	 	 	 	 ddZddZ	d fdZ
 xZS )HqqLoraLinearc
           
         |	r"t        | j                  j                   d      t        |           t        j                  | |       d| _        || _        | j                  ||||||||	       y )Nz0 does not support lora_bias yet, set it to FalseF)
lora_alphalora_dropoutinit_lora_weights
use_rslorause_dora	lora_bias)	
ValueError	__class____name__super__init__r   fan_in_fan_out_active_adapterupdate_layer)self
base_layeradapter_namerr   r   r   r   r   r   kwargsr   s              O/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/lora/hqq.pyr   zHqqLoraLinear.__init__"   s|      DNN$;$;#<<l!mnnGtZ0"'D#/D %)"3%!#  	    c               "    |sy ddl m}  |       S )Nr
   )DoraLinearVariant)variantsr'   )r   r   r#   r'   s       r$   resolve_lora_variantz"HqqLoraLinear.resolve_lora_variantB   s    3$&&r%   c                   t        | |      }|sy|D ]R  }|| j                  j                         vr!| j                         }i t	        j
                  |j                        d|j                  i}|j                         }|| j                  vr| j                  |      }||z   }n | j                  |   j                  | ||      }|r2t        j                  |      j                         st        d| d      t!        d||j"                  |j$                        }	|j'                  dd        |	j(                  |fi | |	| _        | j,                  j/                  |       U y)a  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            Noffload_metaz1NaNs detected in the merged weights. The adapter z seems to be brokencompute_dtypedevice)r   lora_Akeysget_base_layercopydeepcopyquant_configr+   
dequantizelora_variantget_delta_weight
merge_safetorchisfiniteallr   r   r-   r.   popquantizer    merged_adaptersappend)
r   
safe_mergeadapter_namesactive_adapterlayerr4   output	lora_dataw_datanew_hqq_layers
             r$   mergezHqqLoraLinear.mergeJ   sZ    4D-HM "/ <!)9)9);;++-h$--0B0B"Ch^UZUgUgh))+!):):: $ 5 5n EI#i/F!..~>II$P^`fgFennV&<&@&@&B$KNK[[no  !*$EL_L_hmhtht u  6&&&v>>"/$$++N;/<r%   c                H   | j                   st        j                  d       yt        | j                        dkD  rg| j                  j                         }|| j                  j                         vrP| j                         }i t        j                  |j                        d|j                  i}|j                         }|| j                  vrI| j                  |      }|j!                  |j"                        j!                  |j$                        |z
  }n | j                  |   j'                  | ||      }t)        d||j*                  |j$                        }|j                  dd        |j,                  |fi | || _        t        | j                        dkD  rfyy)z_
            This method unmerges all merged adapter layers from the base weights.
            z Already unmerged. Nothing to do.Nr   r+   r,   )mergedwarningswarnlenr>   r<   r/   r0   r1   r2   r3   r4   r+   r5   r6   r7   todtyper.   unmerger   r-   r=   r    )r   rB   rC   r4   rD   rE   rF   rG   s           r$   rP   zHqqLoraLinear.unmergeu   sa    ;;@Ad**+a/!%!5!5!9!9!;!)9)9);;++-h$--0B0B"Ch^UZUgUgh))+!):):: $ 5 5n EI#YYy7::9;K;KLyXF!..~>FFt^]cdF )$EL_L_hmhtht u  6&&&v>>"/% d**+a/r%   c                    t        | j                  |   j                  | j                  |   j                  z  d      | j                  |   z  S )NF)r	   lora_Bweightr/   scaling)r   adapters     r$   r7   zHqqLoraLinear.get_delta_weight   sK    KK(//$++g2F2M2MM ,,w'	(r%   c          	         | j                   |g|i |}t        |      }g }|D ]5  }|j                  t        |      D 	
cg c]  \  }	}
|
|k(  s|	 c}
}	       7 t        |      D ]  \  }}|dk(  r|| j                  j                         vr)| j                  |   }| j                  |   }| j                  |   }| j                  |   }t        j                          }|r2|j                  }| j                  ||j                  j                        }|||      } | | ||                  |z  }|r|j                        }|||   xx   |z  cc<    |S c c}
}	w )N__base__)r    setr?   	enumerater/   r0   rR   r   rT   r9   is_autocast_enabledrO   _cast_input_dtyperS   rN   )r   xrA   argsr#   resultunique_adapterssub_batch_indices_listrU   indexitemirB   r/   rR   dropoutrT   requires_conversionexpected_dtype	sub_batchrD   s                        r$   _mixed_batch_forwardz"HqqLoraLinear._mixed_batch_forward   s|   
 %T__Q888F!-0O%'"* v&--	R_H`.tdhlsdsu.tuv &/%? <!>!Z/!)9)9);;^4^4++N;,,~6*/*C*C*E&E#&%+\\N..q&--2E2EFA 4Q78	wy'9 :;gE&#YY~6F-a01V;1-<0 M5 /us   EEc           	         | j                   |g|i | |j                  dd       }| j                  r4| j                  r| j	                           | j
                  |g|i |}|S | | j                  |g|d|i|}|S | j                  r | j
                  |g|i |}|S  | j
                  |g|i |}| j                  D ]  }|| j                  j                         vr!| j                  |   }| j                  |   }| j                  |   }	| j                  |   }
t        j                          }|r2|j                  }| j!                  ||j"                  j                        }|| j$                  vr| | | |	|                  |
z  z   }n"| j$                  |   j'                  | |||      }|s|j)                        } |S )NrA   )rB   r\   r^   )_check_forward_argsr<   disable_adaptersrJ   rP   r    rh   active_adaptersr/   r0   rR   r   rT   r9   rZ   rO   r[   rS   r6   forwardrN   )r   r\   r]   r#   rA   r^   rB   r/   rR   rd   rT   re   rf   s                r$   rm   zHqqLoraLinear.forward   s   $D$$Q888"JJ=M$$;;LLN(<T<V<D MC *2221ctc=c\bc@ M? (<T<V<< M9 )<T<V<&*&:&: ;N%T[[-=-=-?? ![[8F![[8F"//?G"ll>:G.3.G.G.I*I'*)/ 221fmm6I6IJ%T->->>!'&
1C*Dw*N!N!%!2!2>!B!J!J +9#)	 "K " +!'>!:1;4 Mr%   c                *    t         |          }d|z   S )Nzlora.)r   __repr__)r   repr   s     r$   ro   zHqqLoraLinear.__repr__   s    '"$CS= r%   )r   r
   g        TFFF)r    torch.nn.Moduler!   strr"   intr   rs   r   floatr   boolr   ru   r   ru   r   ru   returnNone)r   ru   rv   zOptional[LoraVariant])FN)r@   ru   rA   zOptional[list[str]]rv   rw   )rv   rw   )
r\   torch.Tensorr]   r   rA   z	list[str]r#   r   rv   rx   )r\   rx   rv   rx   )rv   rr   )r   
__module____qualname__r   r)   rH   rP   r7   rh   rm   ro   __classcell__)r   s   @r$   r   r       s     "%&*$"#	'	 	 		
 	  	  $	 	 	 	 	@	')	<V	08	$	!$	*-$	>G$	SV$	$	L)	V	! 	!r%   r   c                    d }t        | t              r| j                         }n| }t               rt        |t              rt        ||fi |}|S )N)
isinstancer   r1   r   r   r   )targetr!   r#   
new_moduletarget_base_layers        r$   dispatch_hqqr      sN    J&.)"113"j):IF"#4lMfM
r%   )r~   rq   r!   rr   )
__future__r   r2   rK   typingr   r   r9   peft.import_utilsr   peft.tuners.tuners_utilsr   r   peft.utils.otherr	   rC   r   r   hqq.core.quantizer   nnModuler   r    r%   r$   <module>r      sK    #      . L & ) +M! M!`r%   