PiZ ddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZddlmZddlZddlmZddlmZedZ d Zd Zd Zd Zd Zd ZdZdZddddeeeegZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*GddeZ+Gd d!Z,dGd#ed$e-d%e.d&efd'Z/ dHd)eee-fd*e.d+e.d&e e-e ffd,Z0 dGd-e e-ej1fd.e ee-ej2j3fd/e.fd0Z4 dId2ee-efd3e-d&ede-ffd4Z5dd5d6d#ee-efd7ee-efd8e e e-d9e6d&df d:Z7 dJd7ed;e e-de e-d>> # Usage in a checkpointer class >>> def load_checkpoint(self, ...): >>> ... >>> if self._model_type == MY_NEW_MODEL: >>> state_dict = my_custom_state_dict_mapping(state_dict) gemmaGEMMAgemma2GEMMA2llama2LLAMA2llama3LLAMA3llama3_2LLAMA3_2 llama3_vision LLAMA3_VISIONmistralMISTRAL phi3_mini PHI3_MINIphi4PHI4rewardREWARDqwen2QWEN2 clip_text CLIP_TEXT t5_encoder T5_ENCODERN)__name__ __module__ __qualname____doc__r"str__annotations__r$r&r(r*r,r.r0r2r4r6r8r:{/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchtune/training/checkpointing/_utils.pyr r Ns<E3FCFCFCHc(M3(((GS Is   D#FCE3 Is   "J"""""rBr cNeZdZdZdedefdZededdfdZdZ d Z d S) FormattedCheckpointFilesa This class gives a more concise way to represent a list of filenames of the format ``file_{i}_of_{n_files}.pth``. Args: filename_format (str): Format string for the filename. Must have exactly two placeholders, e.g. ``file_{}_of_{}.pth``. max_filename (str): Maximum filename in the list. Should be a string representation of an integer, possibly with leading zeroes. filename_format max_filenamecJ||_||_|dSN)rFrG_validate_filename_format)selfrFrGs rC__init__z!FormattedCheckpointFiles.__init__s, /( &&(((((rBdreturncd|vsd|vrtdt|dts&tdt|dd||d|dS)NrFrGzTMust pass 'filename_format' and 'max_filename' keys to generate checkpoint filenamesz+`max_filename` must be a string, but found z instead.)rFrG) ValueError isinstancer?type)clsrMs rC from_dictz"FormattedCheckpointFiles.from_dicts A % %q)@)@f !N+S11 `d1^CT>U>U``` s/0>*    rBcdtj|jD}t |dkrt ddS)Nc.g|]}|d |dSrA).0xs rC zFFormattedCheckpointFiles._validate_filename_format..s0! ! ! t aDrBz[Filename format string must have exactly two placeholders, e.g. 'file_{i}_of_{n_files}.pth')string FormatterparserFlenrP)rKn_format_placeholderss rCrJz2FormattedCheckpointFiles._validate_filename_formatsn! ! %''--d.BCC! ! !  $ % % * *m  + *rBcbtj}fdt|DS)a Builds a list of checkpoint filenames from the filename format and max filename. Returns: List[str]: List of checkpoint filenames. Example: >>> # Example usage >>> f = FormattedCheckpointFiles(filename_format="file_{}_of_{}.safetensors", max_filename="00003") >>> f.build_checkpoint_filenames() >>> ['file_00001_of_00003.safetensors', 'file_00002_of_00003.safetensors', 'file_00003_of_00003.safetensors'] c g|]W}jt|dzt jjXSrW)rFformatr?zfillr`rG)rYirKs rCr[zGFormattedCheckpointFiles.build_checkpoint_filenames..sj     ' 'AE   T%6!7!788!     rB)intrGrange)rK num_filess` rCbuild_checkpoint_filenamesz3FormattedCheckpointFiles.build_checkpoint_filenamessJ)**     9%%     rBN) r;r<r=r>r?rL classmethoddictrTrJrjrArBrCrErE|s))))))  $  #=    [          rBrEF input_dirfilename missing_okrNc|st|dtj||}|s*|std|d|d|S)a Utility to recover and validate the path for a given file within a given directory. Args: input_dir (Path): Directory containing the file filename (str): Name of the file missing_ok (bool): Whether to raise an error if the file is missing. Returns: Path: Path to the file Raises: ValueError: If the file is missing and missing_ok is False. z is not a valid directory.zNo file with name: z found in .)is_dirrPrjoinpathis_file)rmrnro file_paths rCget_pathrvs     CIAAABBB i22I Qi//11QOxOO9OOOPPP rBTcheckpoint_path weights_onlymmapc t|drdnd}|r_i}t|dd5}|D]}||||< dddn #1swxYwY|}n%t jt|d||}n&#t$r}td |d |d}~wwxYw|S) aW Utility to load a checkpoint file onto CPU in a safe manner. Provides separate handling for safetensors files. Args: checkpoint_path (Union[Path, str]): Path to the checkpoint file. weights_only (bool): Whether to load only tensors, primitive types, and dictionaries (passthrough to torch.load). Default: True mmap (bool): Whether to mmap from disk into CPU memory. Default: True Returns: Dict[str, Any]: State dict from the checkpoint file. Raises: ValueError: If the checkpoint file is not found or cannot be loaded. rTFptcpu) frameworkdeviceN) map_locationryrxzUnable to load checkpoint from z. ) r?endswithr keys get_tensortorchload ExceptionrP) rwrxryis_safetensors_fileresultfk state_dictes rCsafe_torch_loadrsT&W((11.AA LDDu   F?d5III 0Q00A ! QF1II0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 JJO$$") J WWWN?NNNOOUVVW s:z)get_largest_iter_folder..Ls adrB)keyr) recompileoslistdirmatchrggroupappendmax)rrlatest_epoch_folder iter_foldersregexfnamer iter_numbers rCget_largest_iter_folderr:sL Jw  EC66 E""  6ekk!nn--K    4 5 5 5G!,NNCCCAF rBd)ignore_suffixesmax_file_size_mb output_dirrrc  |dzdz}tj|D]X\}}}d|D|dd<tj||}tj||} tj| d|D] dr|rt fd|Dr6tj| } tj| } tj| |krtd| d |d |d tj | stj | | ZdS) a Copies files from the input directory to the output directory, preserving the directory structure. This function will skip copying files that already exist in the output directory or have specific suffixes. It will also skip folders and files that start with '.'. E.g. ".cache/" and ".git". Args: input_dir (Union[str, Path]): The path to the input directory containing files to be copied. output_dir (Union[str, Path]): The path to the output directory where files should be copied. ignore_suffixes (Optional[List[str]]): A list of file suffixes to exclude from copying. Defaults to ['.pt', '.bin', '.safetensors'] if not provided. max_file_size_mb (int): The maximum file size in megabytes to copy. Defaults to 100 MB. Returns: None Example: >>> copy_files('path/to/input_dir', 'path/to/output_dir') This will copy all files from 'path/to/input_dir' to 'path/to/output_dir', except those that already exist in the destination or have the specified suffixes. ic<g|]}|d|S)rq) startswith)rYrMs rCr[zcopy_files..rs)<<<!,,s*;*;<1<<zcopy_files..s@''*0 f%%''''''rBzSkipping copying z to z! as it exceeds the size limit of z MiB.) rwalkpathrelpathjoinmakedirsranygetsizeprintexistsshutilcopy2) rmrrr max_file_sizerootdirsfiles relative_pathdest_dirsrc_file dest_filers @rC copy_filesrRs8%t+d2MWY//#2#2dE=>),, 2 Xy111/ 22 FrBrecipe_checkpointshould_load_recipe_statec|sdSd}|r!tj||}n&tj|td}|rtj|st dt |S)a If recipe_checkpoint is None, look for recipe_state.pt in {output_dir}/{RECIPE_STATE_DIRNAME}/recipe_state.pt. This is to make it easier to resume from a previous run, without having to specify the recipe_checkpoint. Args: output_dir (Path): Directory containing the recipe checkpoint. recipe_checkpoint (Optional[str]): Name of the recipe checkpoint file. Defaults to None. should_load_recipe_state (bool): Whether to load the recipe state from the checkpoint. Returns: Optional[Path]: Path to the recipe checkpoint file if should_load_recipe_state is True, otherwise None. Raises: ValueError: If should_load_recipe_state is True and the recipe checkpoint file is missing. Nzrecipe_state.ptzMIf should_load_recipe_state is True, recipe_checkpoint file must be provided.)rrrRECIPE_STATE_DIRNAMErrPr)rrrrecipe_checkpoint_paths rCget_recipe_checkpoint_pathrs$ $t! !#j:K!L!L!# ,.?" "  " 8N)O)O  [    & ' ''rBadapter_checkpointc |sdSd}|r!tj||}nSt||}tj||d}tj|r|}|rt |ndS)a If adapter_checkpoint is None, look for it in {output_dir}/epoch_{latest_epoch}/adapter_model.pt. This is to make it easier to resume from a previous run, without having to specify the adapter_checkpoint. Args: output_dir (Path): Directory containing the adapter checkpoint. adapter_checkpoint (Optional[str]): Name of the adapter checkpoint file. Defaults to None. should_load_recipe_state (bool): Whether to load the recipe state from checkpoint. pattern (str): Regex pattern to match the epoch folder. Defaults to "epoch_(\d+)". Returns: Optional[Path]: Path to the adapter checkpoint file, or None if not applicable. N)rzadapter_model.pt)rrrrrr)rrrradapter_checkpoint_pathlargest_iter_folder!tentative_adapter_checkpoint_paths rCget_adapter_checkpoint_pathrs& $t" H"$',,z;M"N"N6j'RRR,.GLL +-?- - ) 7>>; < < H&G #,C M4' ( ( (MrBcheckpoint_filescheckpoint_dirhas_adapter_checkpointcN ddtttdttdttfd}t |ts.t |}|}|s|}n|s|}n|}|||d}|S)a! Returns Paths to model checkpoint files, handling resuming from checkpoint, file formating and checking if the files exists. If resuming from checkpoint, the checkpoint files are loaded from the output directory. Otherwise, they are loaded from the checkpoint directory. If checkpoint_fiels is a dictionary, it is converted to a list of formatted checkpoint filenames. Args: checkpoint_files (Union[List[str], Dict[str, str]]): List or dictionary of checkpoint file names. If a dictionary with keys ["filename_format", "max_filename"] is provided, it is converted to a list of formatted checkpoint filenames. checkpoint_dir (Union[str, Path]): Directory containing the checkpoint files. output_dir (Union[str, Path]): Directory to use when resuming from a checkpoint. should_load_recipe_state (bool): Whether to resume from a checkpoint. has_adapter_checkpoint (bool): Indicates if there is an adapter checkpoint. Returns: list[Path]: Sorted list of paths to the checkpoint files. Example: >>> checkpoint_files = ["model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"] >>> checkpoint_dir = "/path/to/checkpoints" >>> output_dir = "/path/to/output" >>> should_load_recipe_state = True >>> has_adapter_checkpoint = False >>> paths = get_model_checkpoint_path( ... checkpoint_files, ... checkpoint_dir, ... output_dir, ... should_load_recipe_state, ... has_adapter_checkpoint ... ) >>> print(paths) [PosixPath('/path/to/output/{largest_epoch}/model-00001-of-00002.safetensors'), PosixPath('/path/to/output/{largest_epoch}/model-00002-of-00002.safetensors')] FrrmrNczg}|D](}t|||}||)t|S)zR Validates that the checkpoint files exist and sorts based on ID. )rvrsorted)rrmrocheckpoint_pathsrrws rCvalidate_checkpoint_filesz.validate_checkpoint_filessP(*! 5 5A&y!Z@@O  # #O 4 4 4 4&'''rB)rmroF) r rr?r rrQrErTrj) rrrrrrformatted_checkpoint_filesrmrs rCget_model_checkpoint_pathrs^((S *(D>( d ((((. & - -S%=%G%G & & "6PPRR $ #" $ # # 00 rBckpt_dirout_dirc|}|}||ks ||jvrtd|d|ddS)z Checks that the output directory is not equal to or a subdirectory of the checkpoint directory. This is necessary to avoid making copies of copies when geting config files from ckpt_dir. ziThe output directory cannot be the same as or a subdirectory of the checkpoint directory. Found ckpt_dir=z and out_dir=rqT)resolveparentsrP)rr _ckpt_dir_out_dirs rCcheck_outdir_not_in_ckptdirrHs{  ""I  H9 X-= = = 1 1 1%, 1 1 1   4rBr)TT)r)NF)NFr)=rrrr]enumrpathlibrtypingrrrrr r r warningsr r safetensorsr torchtune.utils._loggingrloggerADAPTER_CONFIGADAPTER_CONFIG_FNAMEADAPTER_MODEL_FNAMESAFETENSOR_INDEX_FNAMETORCH_INDEX_FNAME SHARD_FNAMEr REPO_ID_FNAMESUFFIXES_TO_NOT_COPY ADAPTER_KEY EPOCHS_KEY MAX_STEPS_KEY MODEL_KEYOPT_KEYSEED_KEYTOTAL_EPOCHS_KEY STEPS_KEYRNG_KEYDATALOADER_KEYr rEr?boolrvrTensornn ParameterrrrgrrrlistrrrArBrCrs DDDDDDDDDDDDDDDDDD !!!!!!////// G   "(%720 %#       %   "   +#+#+#+#+#+#+#+#\B B B B B B B B J$6PT((49%(59(HL( #s(^((((\!0404S%,&'04$U30B+B%CD0404040404j+: sDy $' 498,0 B B B S$YB c4i B d3i( B  B  B B B B N(,%*#(#(#(}#(##(d^ #(#(#(#(P)-%*" %N%N%N  %N#%N %N d^ %N%N%N%NP`DItCH~56`#t)$`c4i `# ` ! `  $Z ````F$$rB