biMddlZddlZddlmZddlZddlZddlmZddlm Z m Z e rddl Z ddl m Z ddl mZmZej ddj#d vxr<ej$j'xr ej$j)dd k\Zere rdd lmZed ZndZej2j4ej2j6ej2j8hZej2j<ej2j>ej2j@ej2jBej2jDej2jFhZ$ej2jJej2jLej2jNej2jPej2jRhZ*ej2jVej2jXej2jZej2j\ej2j^ej2j`ej2jbej2jdej2jfh Z4e$e*ze4zZ5e$e*ze4zZ6e$e*zZ7d ejpdejpde9dejpfdZ:dZ;dgfdZdZ?dZ@dZAdZBd,dZCd,dZDd,dZEd,dZFd,dZGd,d ZHd,d!ZId,d"ZJd,d#ZKd,d$ZLd,d%ZMejZNej2j8eMej2jDeCej2jBeDej2j@eEej2j>eFej2j<eGej2jReHej2jPeIej2jNeJej2jLeKej2jJeLi ZOePeOjZRd&ZSd'ZTGd(d)ejjZVGd*d+ejZXy)-N) nullcontext)is_accelerate_availableis_kernels_available)init_empty_weights)add_hook_to_moduleremove_hook_from_moduleDIFFUSERS_GGUF_CUDA_KERNELSfalse)1trueyes) get_kernelz Isotr0py/ggmlxqweight qweight_typereturnc|tvr||jzS|tvrtj|\}}|j d|j d|z|zf}t j||g|}||j|jjz}|jStj|}td|)Nrz$Unsupported GGUF quantization type: ) UNQUANTIZED_TYPEST DEQUANT_TYPESggufGGML_QUANT_SIZESshapeopsggml_dequantizetodtypeGGMLQuantizationTypeNotImplementedError as_tensor)rrr block_size type_sizerweightys Z/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/diffusers/quantizers/gguf/utils.py_fused_mul_mat_ggufr)Os((799}}$ $ 5 5l C Iq!7==#3y#@:#MN$$WlCUC  !''"$$ $ ;;=00> !$H"WXXc$ttj|jj}|j }i}t j|j}|jD]}||jvs||||<|di|}|S)a Creates a new hook based on the old hook. Use it only if you know what you are doing ! This method is a copy of: https://github.com/huggingface/peft/blob/748f7968f3a31ec06a1c2b0328993319ad9a150a/src/peft/utils/other.py#L245 with some changes ) getattr acceleratehooks __class____name____dict__inspect signature__init__keys parameters)old_hook old_hook_cls old_hook_attrfiltered_old_hook_attrold_hook_init_signatureknew_hooks r(_create_accelerate_new_hookr?ms :++X-?-?-H-HIL%%M%// 0E0EF    !9 '22 2(5a(8 "1 %9545H Or*cHd}t|j}|sy|jD]\}}||zdz} t|||| |t |t j s8||| sB||vsGtrtnt} | 5t|j|j|jdu||j|<dddt||j|_|j|j#d|S#1swYHxYw)Nc>|dz}||vxrt||tS)Nr&) isinstance GGUFParameter) state_dictprefix weight_keys r(_should_convert_to_ggufz:_replace_with_gguf_linear.._should_convert_to_ggufs(h& Z']Jz*7M},]]r*.) compute_dtypeF)listchildrennamed_children_replace_with_gguf_linearrCnnLinearrrr GGUFLinear in_features out_featuresbias_modulestype source_clsrequires_grad_) modelrJrErFmodules_to_not_convertrH has_childrennamemodule module_prefixctxs r(rNrN~s^()L ,,.7 f + !&-]Tjk vryy )' MB22(?(A${C '1&&''KKt+"/ (t$ /36lENN4 + NN4 / / 6'7* L  s =DD! c|jD]W\}}t|tr||vr|jj}t |dd}t rtnt}|5tj|j|j|jdu|}dddtjt|j_|||_ t!|dr.|j"}t%|} t'|t)|| |j+|||j,|<t/|j1} | sLt3||Z|S#1swYxYw)NrT)device_hf_hook)rMrCrQr&rar-rrrrOrPrRrSrT Parameterdequantize_gguf_tensorhasattrrbr?r rrrUrKrL#_dequantize_gguf_and_restore_linear) rYrZr\r]rarTr_ new_moduler8r>r[s r(rfrfsA,,.P f fj )d:P.P]]))F6640D(?(A${C YY&&''KKt+!   !# -CFMM-R SJ "& vz*!??6x@'/":x8 MM& !#-ENN4 FOO-.  /8N O=P@ L5  s .:E..E7  c|jtjjtj}|dddf|dddfdzz|dddfdzz|dddfdzzj dS)Nrrr)viewtorchuint8rint32 unsqueeze)rs r( to_uint32rtso u{{u{{+A adGa1gl "Qq!tW] 2Qq!tW] B M Ma PPr*c|jd}t||t|z gz}tj||dS)Nrdim)rrKsumrpsplit)blocksargsn_maxdimss r(split_block_dimsr~s< LLOE :T*+ +D ;;vt ++r*c|jd}|jtj}|j |ddf}tj ||jddzd\}}}tj |dz|dz|dz d zzgd }tj |dz|dz |dz d zzgd }|j |d f|j |d ffS) Nrrrv?rl0rk)rrorprqreshaperycat)scalesn_blocksdmm_dscmins r( get_scale_minrs||AH [[ %F ^^Xq!, -F FFLL$4$9rBIAq# AHsTzqAvo>?R HB ))QXqa1f_=>B GC JJ!} %s{{Ha='A BBr*ct|d\}}|jtjj |}|jtj }||zS)Nrl)r~rorpfloat16rint8)rzr$r%r rrs r(dequantize_blocks_Q8_0rsJ FA &DAq u}}  'A uzzA q5Lr*c|jd}t|ddd\}}}}|jtjj |}|jtjj |}t |}|j|dftjd|jtjjddz }|j|dd|dzftjddg|jtjjddddz } |dzj tj}| dzj|df} | |dzz}||z|zS) Nrrlrr rar rr) rr~rorprrrtrarangerarrtensorrq) rzr$r%r rrrqhqsqls r(dequantize_blocks_Q5_1rs>||AH#FAq!4LAq"b u}}  'A u}}  'A 2B XqM "ell2ahhekk&Z&b&bcdfh&i iB Xr1jAo6 75<< Aqxxu{{< gaAq B q&U[[ !B t)  h^ ,B rQwB Fa<r*c|jd}t|dd\}}}|jtjj |}t |}|j|dtjd|jtjjddz }|j|dd|dztjddg|jtjjddddz }|dzj tj}|dzj|d}||dzzj tjd z }||zS) Nrrlrrrrrrrm)rr~rorprrrtrrrarrrrqr) rzr$r%r rrrrrs r(dequantize_blocks_Q5_0rs%||AH A.IAr2 u}}  'A 2B Ha ELLAHHEKK$X$`$`abdf$g gB Hb!Z1_ 5 Aqxxu{{: gaAq B q&U[[ !B t)  Xr *B a.  UZZ (2 -B r6Mr*c|jd}t|dd\}}}|jtjj |}|jtjj |}|j |dd|dzftjddg|jtjj ddddz }|dzj |d}||z|zS)Nrrlrrrrr) rr~rorprrrrrarq)rzr$r%r rrrrs r(dequantize_blocks_Q4_1rs||AH1-HAq" u}}  'A u}}  'A Xr1jAo6 75<< Aqxxu{{< gaAq B t)  Xr *B Fa<r*c|jd}t|d\}}|jtjj |}|j |dd|dzftjddg|jtjj dz }|dzj |dfj tjd z }||zS) Nrrlrrrrrrrlrrrk) rr~rorprrrrrarqr)rzr$r%r rrrs r(dequantize_blocks_Q4_0r s||AH VQ 'EAr u}}  'A Xr1jAo6 75<< Aqxxu{{< gl B t)  h^ , / / ;a ?B r6Mr*c|jd}t|tdztdztdz\}}}}|jtj j |}|jtjj |}||zj|tdzdf}|j|dddft jddg|jtjjd z }|d zj|dd f}|j|ddd ft jgd |jtjjd z }|dzj|dd f}||dzzj tj d z } | j|tdzdf} || zj|tfS)Nrrlrrmrr@rrrrrrlrrrrrr) rr~QK_Krorprrrrrrarq) rzr$r%r rrrrrqs r(dequantize_blocks_Q6_Kr-s||AH DAItrzB    [[ $ ' ' .F u}}  'A Vh A67A Xr1b) *ellAq6!((Z_ZeZe.f.n.n/ B t)  hB/ 0B Xr1b) *ellF+D%vF fell +006Jv*5 5 &\dE] *006Jdjk_` 1ell8Ss14YZZkGk4<( (Mls9-B;)FN)r,N) r1 __module__ __qualname__rr# staticmethodr classmethodr __classcell__r0s@r(rDrDs6S  r*rDceZdZ d dfd Zdej fdZdej fdZdej fdZxZ S)rQcFt|||||||_||_yr)rr5rJra)rrRrSrTrJrar0s r(r5zGGUFLinear.__init__s' lD&A* r*inputsct3|jjr|jr|j|S|j |Sr)rr&is_cuda forward_cudaforward_native)rrs r(forwardzGGUFLinear.forwards; ?t{{22v~~$$V, ,""6**r*c"t|j}|j|j}|j%|jj|jnd}t j jj|||}|Sr) rdr&rrJrTrprO functionallinear)rrr&rToutputs r(rzGGUFLinear.forward_nativesj' 44--.37993Htyy||D../d$$++FFDA r*c|jj}t|j|j|j|}|j (||j j|jz }|Sr)r&rr)rrJrT)rrrrs r(rzGGUFLinear.forward_cuda!s_[[++ $VYYt/A/A%BDKKQ[\ 99 diill4#5#56 6F r*)FNN)rN) r1rrr5rprrrrrrs@r(rQrQsM    +ell+ U\\5<<r*rQr)Yr3os contextlibrrrptorch.nnrOutilsrrr.raccelerate.hooksrr getenvlowercuda is_availableget_device_capabilitycan_use_cuda_kernelskernelsrrr!F32F16BF16rQ4_0Q4_1Q5_0Q5_1Q8_0Q8_1STANDARD_QUANT_TYPESQ2_KQ3_KQ4_KQ5_KQ6_K KQUANT_TYPESIQ1_MIQ1_SIQ2_XXSIQ2_XSIQ2_SIQ3_XXSIQ3_SIQ4_XSIQ4_NLIMATRIX_QUANT_TYPESrMMVQ_QUANT_TYPESMMQ_QUANT_TYPESrintr)r?rNrfrrrtr~rrrrrrrrrrrrrrrKr6SUPPORTED_GGUF_QUANT_TYPESrrdrcrDrPrQr,r*r(r!s " B-LBII+W5;;=AUU3 !3 ((*1-2 02" _ %C C..22D4M4M4Q4QSWSlSlSqSqr""""""""""""  """"""""""   ####%%$$##%%##$$$$ %|36II ',69LL&55<<%,,cV[VbVb<"HJbdBGI!R  Q , C&&  -8243(.B&(P((""$:""$:""$:""$:""$:""$:""$:""$:""$:""$:""$: ""6";";"=>>**EHH&&*Zr*