biBddlZddlZddlmZmZmZddlZddlmZddl mcm Z ddl m Z mZGdde ZGddej eZy)N)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_mergec eZdZdZdZdej ddfdZdede d e d e ddf d Z defd Z defd Z defdZdefdZdeddfdZdddZy) MissLayer) miss_block)miss_r miss_dropout miss_mini_r base_layerreturnNc ||_i|_tji|_i|_tj i|_d|_g|_ d|_ ||_ |j}t|tjr$|j|j c|_|_yt#dt%|)NFTzUnsupported layer type )rr nn ModuleDictr r ParameterDictr _disable_adaptersmerged_adapterscast_input_dtype_enabledkwargsget_base_layer isinstanceLinear in_features out_features ValueErrortype)selfrrs Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/miss/layer.py__init__zMissLayer.__init__s$ MM"-**2.!&!(,% ((* j")) ,2<2H2H*JaJa /D d/6tJ7G6HIJ J adapter_namermini_r init_weightsc |dkrtd|||j|<||j|<|dkDrtj|}ntj }||j |<|j}t|tjrCtjtj||jd|j|<n!tdt!|j"|dk(rB|j$|zdk7s|j|zdk7r td |j'||n\|d k(r1|j|zdk7r td |j)|||n&|r|j+||n|j-||j/||j1|j2y ) zInternal function to create miss adapter Args: adapter_name (`str`): Name for the adapter to add. r (`int`): Rank for the added adapter. init_weights (`bool`): Whether to initialize weights. rz?`r` should be a positive integer value but the value passed is )pT requires_gradz0MiSS is not implemented for base layers of type batz=The weight matrix must be fully divisible into [r, r] blocks.minizmini_r is divided along the out_features dimension. For optimal performance and implementation simplicity,it is recommended that out_features be divisible by mini_r.Error: {self.out_features} % mini_r != 0N)rr r rDropoutIdentityr rrr Parametertorchzerosrr TypeErrorr__name__rreset_bat_parametersreset_mini_parametersreset_miss_parametersreset_miss_parameters_random%_move_adapter_to_device_of_base_layer set_adapteractive_adapters) rr#r$r%r r&rmiss_dropout_layerrs r update_layerzMissLayer.update_layer2s 6^_`^abc c$% L!)/& # !#l!; !# *<,'((* j")) ,,.LLQHYHY9Zjn,oDOOL )NtT^O_OhOhNijk k 5 !#q(D,=,=,AQ,F !`aa  % %lA 6 V #  6)Q. ?  & &|Q ?   & &|Q 7  - -l ; 22<@ --.r"ctjtj||jd|j |<yNTr*rr0r1r2rr rr#r$s r r7zMissLayer.reset_miss_parametersjs-(* U[[DDUDU5Vfj(k %r"ctjtj|j|z||d|j |<yr?r@rAs r r5zMissLayer.reset_bat_parametersms7(* U[[ARARVWAWYZ\]5^nr(s %r"cttjtj||d|j|<yr?)rr0r1r2r )rr#r$r%s r r6zMissLayer.reset_mini_parametersps'(* U[[F5K[_(` %r"ctjj|j|t j dy)N)a)rinitkaiming_uniform_r mathsqrt)rr#s r r8z&MissLayer.reset_miss_parameters_randomss*   !>$))A, Or"scalec|dk(ry|jD]4}||jjvr tjd6y)NzGScaling operation for MiSS not supported! Automatically set scale to 1.r;r keyswarningswarnrrKactive_adapters r scale_layerzMissLayer.scale_layervsH A: "22 eNT__%9%9%;; MMc d  er"c|jD]4}||jjvr tjd6y)Nz?Unscaling operation for MiSS not supported! Keeping scale at 1.rNrRs r unscale_layerzMissLayer.unscale_layers>"22 ]NT__%9%9%;; MM[ \  ]r"NrN)r4 __module__ __qualname__adapter_layer_namesother_param_namesrModuler!strintboolr=r7r5r6r8floatrTrVr"r r r s)AK299K4K&6/6/ 6/ 6/  6/ 6/pl#ltta#aPPee4e]r"r c eZdZdZ ddededededeeefddf fd Z dd ed e e eddfd Z dd Z ddedejfdZddedejfdZdejdededejfdZdeffd ZxZS) MissLinearz, MiSS implemented in a dense layer. r#r$r%r r&rNc t|tj||fi|||_|j|||||fi|||_yrW)superr!r _active_adapterr=miss_fn) rrr#r$r%r r&r __class__s r r!zMissLinear.__init__sR 46v6+,6<XQWX# r" safe_merge adapter_namescXt||}|sy|D]}||jjvs!|j}|jj }|r|jj j}|jdk(r|j||}||z }nt|jdk(r3|j||jjj }|}n2|j||jjj }|}tj|jstd|d|j!||j_n|jdk(r_|j||jjj }|jxj |j!|z c_n|jdk(rQ|j||jjj }|j!||j_nP|j||jjj }|j!||j_|j"j%|y)ab Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If `None`, all active adapters will be merged. Defaults to `None`. Nr,r-z1NaNs detected in the merged weights. The adapter z seems to be broken)rr rOrweightdtypedataclonerhget_delta_weightget_delta_weight_missrr1isfiniteallrtorappend)rrjrkrSr orig_dtype orig_weight delta_weights r mergezMissLinear.merges10mD  +" ">"@K||u,'+'<'<^['Y #|3 /'+'A'A.RVRaRaRhRhRmRm'n &2 '+'A'A.RVRaRaRhRhRmRm'n &2  >>+6::<(OP^O__rs.9^^J-GJ%%*||u,'+'<'<^T__McMcMhMh'i "))..,//*2MM./'+'A'A.RVRaRaRhRhRmRm'n 1=1L )).'+'A'A.RVRaRaRhRhRmRm'n 1=1L )).$$++N;E" :D||v-#{{1doo.J.JdN^N^_mNn.no"//?G ! AvvbzA~*()AFF2JN(:a'? EE!a%67..q$**=A#eii0B 0B0B0eAGGCRL0eRSRXRXY[R\`aRa0ecd0ekm&nqu&uuF v>* r"c*t|}d|zS)Nzmiss.)rf__repr__)rrepris r rzMissLinear.__repr__sg }r")rrr(T)FNrX)F)r4rYrZ__doc__r^r_rarr`r!rlistrzrr1Tensorrqrrrrr __classcell__)ris@r rdrds!)-$$  $  $  $D#I&$ $ 4<4rsG '' Ll]l]^~I~r"