bi ddlZddlZddlmZddlmZmZddlZddl Z ddl m Z ddZ dZ dZdeeej fd eee j$ffd ZGd d eZGd deZd eee j$fdeefdZy)N)Mapping)OptionalUnion) safe_opencd}t|jdk(r!|jtj}d}|j j }tjj||d}|2|t|j}|t|jd||<|jdk(r|d}tj||jd|j}|dd|dd|j|S)Nztorch.bfloat16bfloat16.dat)dtypeshaperzw+)r moder )strr viewtorchint16cpunumpyospathjoinlistr ndimnpmemmapflush)weight weight_nameoffload_folderindexr array tensor_file file_arrays S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/accelerate/utils/offload.pyoffload_weightr#s E 6<<,,U[[) JJL   E'',,~+d/CDK  = $E',tEKK7HIk zzQd ;ekkEKKXJ!HJqM Lct|d}|dk(rd}|d}|dk(rd}tj|||d}t|dd k(r|d }t j |}|ddk(r|j tj}|S) Nr )r rrr)r r r r)tuplerrlenrtensorrr) weight_file weight_infor r rs r"load_offloaded_weightr..s +g& 'E {  E  YY{%u3 GF ;w  A% \\& !F7z)U^^, Mr$c|t|dk(rytjj|d}tjj |r,t |d5}t j|}dddni}j|t |dd5}t j||ddddy#1swYJxYw#1swYyxYw)Nr index.jsonzutf-8)encodingw)indent) r*rrrisfileopenjsonloadupdatedump)rroffload_index_filef current_indexs r"save_offload_indexr>Ds }E anlC ww~~() $w 7 )1 IIaLM ) )   # 8.A -1-..  ) ) ..sCC C  Csave_dir state_dictctj|di}|jD]\}}t||||}t ||y)a Offload a state dict in a given folder. Args: save_dir (`str` or `os.PathLike`): The directory in which to offload the state dict. state_dict (`Dict[str, torch.Tensor]`): The dictionary of tensors to offload. T)exist_ok)rN)rmakedirsitemsr#r>)r?r@rname parameters r"offload_state_dictrGUsQKK4( E%++-Giy$FGuh'r$c2eZdZdZdedefdZdZdZdZ y) PrefixedDatasetz Will access keys in a given dataset by adding a prefix. Args: dataset (`Mapping`): Any map with string keys. prefix (`str`): A prefix to add when trying to access any element in the underlying dataset. datasetprefixc ||_||_yNrJrK)selfrJrKs r"__init__zPrefixedDataset.__init__qs  r$c<|j|j|SrMrNrOkeys r" __getitem__zPrefixedDataset.__getitem__us||t{{mC5122r$ct|jDcgc] }|j|js|"c}Scc}wrM)iterrJ startswithrKrRs r"__iter__zPrefixedDataset.__iter__xs.DLLPSCNN4;;4OSPQQPs AAc,t|jSrM)r*rJrOs r"__len__zPrefixedDataset.__len__{s4<<  r$N) __name__ __module__ __qualname____doc__rr rPrTrXr[r&r$r"rIrIhs*3R!r$rIceZdZdZ d deeejfdee ee jfde fdZ defdZd Zd Zy) OffloadedWeightsLoaderaC A collection that loads weights stored in a given state dict or memory-mapped on disk. Args: state_dict (`Dict[str, torch.Tensor]`, *optional*): A dictionary parameter name to tensor. save_folder (`str` or `os.PathLike`, *optional*): The directory in which the weights are stored (by `offload_state_dict` for instance). index (`Dict`, *optional*): A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default to the index saved in `save_folder`. Nr@ save_folderrc|| | td|in||_||_|I|Gttj j |d5}tj|}ddd|in||_ t|jj|_ |jj|jDcgc]}||jvs|c}||_y#1swYxYwcc}w)NzWNeed either a `state_dict`, a `save_folder` or an `index` containing offloaded weights.r0) ValueErrorr@rbr6rrrr7r8rrkeysall_keysextenddevice)rOr@rbrrhr<rSs r"rPzOffloadedWeightsLoader.__init__s  +"5%-vw w * 2" & =[4bggll; => %! !  % =Re T__1134  TZZTc3dmm;ScTU  % %UsC,C8C8,C5rSc||jvr|j|S|j|}|jd|jdn |j}d} t |dd|5}|j |jd|}dddd|vr"|jtt|d}|jtj|k7r|j|}|Stjj|j|d}t||S#1swYxYw#t $rKt |ddd5}|j |jd|}dddn #1swYnxYwYwxYw)Nsafetensors_filerpt) frameworkrhrr r )r@rgetrhr get_tensor TypeErrortogetattrrrrrrbr.)rOrSr-rhr+r<r,s r"rTz"OffloadedWeightsLoader.__getitem__sr $// !??3' 'jjo ??- . :"kk1Ut{{FF O{+=>$W]^Obc\\+//-*MNFO+%75+g2F#GH}} V 446*Mggll4#3#3uD\B $[+>>OO O{+=>$W\]Oab\\+//-*MNFOOO OsBD6,"D*D6*D3/D66F "E;2 F ;F F  F c,t|jSrM)rVrfrZs r"rXzOffloadedWeightsLoader.__iter__sDMM""r$c,t|jSrM)r*rfrZs r"r[zOffloadedWeightsLoader.__len__s4==!!r$)NNNN)r\r]r^r_dictr rTensorrrrPathLikerrPrTrXr[r&r$r"rarasm /39= ell*+eC$456 (?s?4#"r$rasubmodule_namesc i}|D]N}|j|jDcic]!\}}||k(s|j|dzr||#c}}P|Scc}}w)a Extract the sub state-dict corresponding to a list of given submodules. Args: state_dict (`Dict[str, torch.Tensor]`): The state dict to extract from. submodule_names (`List[str]`): The list of submodule names we want to extract. .)r9rDrW)r@rwresult module_namerSparams r"extract_submodules_state_dictr}ssF&    #-"2"2"4 C+% c8I)JU      M  s&A rM)r7rcollections.abcrtypingrrrrr safetensorsrr#r.r>r rvrtrurGrIrarr}r&r$r"rs #" !*,."(sBKK'7!8(d3PUP\P\K\F](&!g!.@"W@"Fd3 3D.EX\]`Xar$