biA!ddlmZddlZddlZddlmZmZmZddlZddl m Z ddl m Z m Z ddlmZmZGdde ZGd d e j$eZy) ) annotationsN)AnyLiteralOptional)BaseTunerLayercheck_adapters_to_merge)BlockCircularConvolutionget_circulant_fastcXeZdZdZdZddZd dZdZejdZ y) C3ALayer) c3a_kernel) block_sizec F||_i|_tji|_d|_g|_||_|j}t|tjr$|j|jc|_ |_ ytdt|)NFzUnsupported layer type ) base_layerrnn ParameterDictr_disable_adaptersmerged_adapterskwargsget_base_layer isinstanceLinear in_features out_features ValueErrortype)selfrrs P/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/c3a/layer.py__init__zC3ALayer.__init__"s$**2.!&! ((* j")) ,2<2H2H*JaJa /D d/6tJ7G6HIJ JcX||jjvrtd|d|jj}|j }|j|}t |jtjj|}||jdz S)NzAdapter z not found.) rkeysrrweightdtyper totorchfloat32size)radapterbase_layer_weightbase_layer_weight_dtyper delta_weights rget_delta_weightzC3ALayer.get_delta_weight1s $//..0 0xy <= = //188"3"9"9__W- )*-- *FGJJKbc /44R888r!c |dkrtd||j|zdk7rtd|jd||j|zdk7rtd|jd|||j|<|j j }t jtj|j|z|j|z|tj|j|j|<|j|||j||j|j y)NrzH`block_size` should be a positive integer value but the value passed is zPThe block size should be a factor of the input size. However, the input size is z and the block size is zRThe block size should be a factor of the output size. However, the output size is )r&device)rrrrrr%r Parameterr(zerosr)r1rreset_c3a_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r adapter_namer init_weightsr%s r update_layerzC3ALayer.update_layer;s\ ?ghrgstu u   j (A -bcgcscsbtuLMWLXY    z )Q .deievevdwxOPZO[\ )3 %$$&--(* KK!!Z/  J.mm}}  )  % !!, = 22<@ --.r!c|dury||jjvr0|dk(r-tjj |j|y|dvr|j |j }}dtjdt||zz z}tjd|z}tjj|j|| |y|dk(r`|j }dtjdt|z z}tjj|j|| |ytd|y) NTgaussian)xavier_uniformFg?g@g@kaiming_uniformzUnknown init_weights: ) rr$rinitnormal_rrmathsqrtfloatuniform_r)rr8r9fan_infan_outstdas rr4zC3ALayer.reset_c3a_parametersXs 4   4??//1 1z) =>!::"&"2"2D4E4EDIIcE&72B,C&CDDIIcNS(  !>AF!22))$))C%-$788  !>AF #9,!HII 2r!N)rz nn.ModulereturnNone)rI torch.Tensor) __name__ __module__ __qualname__adapter_layer_namesother_param_namesr r/r:r(no_gradr4r!rr r s<)' K9/:U]]_JJr!r cVeZdZ dfd ZdddZd dZd dZd fd ZxZS) C3ALinearc t|tj||fi|||_|j |||y)N)superr r _active_adapterr:)rrr8rr9r __class__s rr zC3ALinear.__init__os? $ 5f5+ , LAr!c6t||}|sy|D]}||jjvs!|j}|r~|jj j }|j|}||z}tj|jstd|d||j_n9|j|}|jj |z|j_|jj|y)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rrr$rr%datacloner/r(isfiniteallrrappend)r safe_merge adapter_namesactive_adapterr orig_weightsr.s rmergezC3ALinear.merge|s0mD  + L >>,7;;=(OP^O__rs.:J%%*#'#8#8#HL-7->->-C-Cl-RJ%%*$$++N;) 5ZZ1$&&'!+r!c|j}|jr3|jr|j|j|g|i|}n|jr|j|g|i|}n|j|g|i|}|j t j}|jD]}||jjvr |j|j t j}tj|||jdz }||j |jz }|j |}|S)Nr#)r&disable_adaptersrerjrr'r(r)r7rr$r applyr*)rxargsrprevious_dtyperesultrars rforwardzC3ALinear.forwards   {{ $T__Q888F [[$T__Q888F$T__Q888FU]]#A"&"6"6 -!)=)=)??!__^<?? N ,221jAAFF2JN!$$v||,,  ->* r!c*t|}d|zS)Nzc3a.)rV__repr__)rreprXs rrtzC3ALinear.__repr__sg |r!)r8strrintr9z?bool | Literal['gaussian', 'kaiming_uniform', 'xavier_uniform']rIrJ)FN)r_boolr`zOptional[list[str]]rIrJ)rIrJ)rnrKrorrrrIrK)rIrv) rLrMrNr rcrjrrrt __classcell__)rXs@rrTrTmsQ B B B V B  B&rsG# )) L?NJ~NJb[ 8[r!