&`ik $ ddlZddlmZn#eef$rZededZ[wwxYw ddlZddlZn#eef$rZededZ[wwxYw ddlm Z n #e$rdZ YnwxYwddl Z ddl Z ddl Z ddl Z ddlmZddlmZmZmZmZmZmZmZmZddlmZdd lmZmZmZdd lm Z dd l!m"Z"erdd l#m$Z$e j%e&Z'd Z(GddeZ) d)dZ*dej+j,de-fdZ. d*dej+j,de-de-de/fdZ0 d+de-dej+j,de-deee-ddf dZ1de-ddde-deee-ddf dZ2 d+dej+j,de-deeej+j3ge/fdee-fdZ4dej+j,de-de/fdZ5dej+j,de-de/fd Z6dej+j,de-ddfd!Z7 d+d"ee-e j8fd#eej+j,deej+j,e-ffd$Z9Gd%d&eZ:e"Gd'd(Z;dS),N)LocalFileSystemzdfsspec is a required dependency of Ray Train and Ray Tune. Please install with: `pip install fsspec`zfpyarrow is a required dependency of Ray Train and Ray Tune. Please install with: `pip install pyarrow`) S3FileSystem)Path) TYPE_CHECKINGCallableDictListOptionalTupleTypeUnion) TempFileLock) SyncConfigSyncer_BackgroundSyncer)_get_ray_train_session_dir) DeveloperAPI) Checkpointz.validate_storage_markercjeZdZdZdedeeffd ZedZ dede fdZ d fd Z xZ S) _ExcludingLocalFilesystemaLocalFileSystem wrapper to exclude files according to patterns. Args: root_path: Root path to strip when matching with the exclude pattern. Ex: root_path="/tmp/a/b/c", exclude=["*a*"], will exclude /tmp/a/b/c/_a_.txt but not ALL of /tmp/a/*. exclude: List of patterns that are applied to files returned by ``self.find()``. If a file path matches this pattern, it will be excluded. root_pathexcludec Vtjdi|||_||_dSN)super__init___exclude _root_path)selfrrkwargs __class__s o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/train/_internal/storage.pyrz"_ExcludingLocalFilesystem.__init__As2""6""" #cdS)N_excluding_localrr s r#fsidz_ExcludingLocalFilesystem.fsidFs!!r$pathreturnc,t|}||j}|g}|r||dz|jD] tfd|DrdS!dS)zdReturn True if `path` (relative to `root_path`) matches any of the `self._exclude` patterns./c3BK|]}tj|VdSN)fnmatch).0 candidateexcls r# z<_ExcludingLocalFilesystem._should_exclude..Us/VV 7?9d33VVVVVVr$TF)r relative_toras_posixis_dirappendrany)r r) relative_pathmatch_candidatesr2s @r#_should_excludez)_ExcludingLocalFilesystem._should_excludeJsDzz((99BBDD )? ;;== 9  # #MC$7 8 8 8M  DVVVVEUVVVVV tt ur$NFc tj|f|||d|}|r fd|DSfd|DS)z+Call parent find() and exclude from result.)maxdepthwithdirsdetailcFi|]\}}|||Srr;)r0r)outr s r# z2_ExcludingLocalFilesystem.find.._sCD#++D11cr$c>g|]}||SrrA)r0r)r s r# z2_ExcludingLocalFilesystem.find..es,MMMT$2F2Ft2L2LMDMMMr$)rfinditems)r r)r=r>r?r!pathsr"s` r#rFz_ExcludingLocalFilesystem.findYs  #hv  IO    N!&  NMMMUMMM Mr$)NFF)__name__ __module__ __qualname____doc__rr strrpropertyr(boolr;rF __classcell__r"s@r#rr4s  $$$c$$$$$$ ""X" C D     N N N N N N N N N Nr$rc tr5t|tjjr|dd|ddtjj||f||d|S)N use_threadsF chunk_sizeisource_filesystemdestination_filesystem)r isinstancepyarrowfs setdefault copy_files)source destinationrVrWr!s r#_pyarrow_fs_copy_filesr_hs0 #97:;RSS0 -/// l$4555 :  ,5       r$rZfs_pathct||} |r||dS||dS#t$r%td|d|dYdSwxYw)Nz(Caught exception when deleting path at (, ):) _is_directory delete_dir delete_file Exceptionlogger exception)rZr`r6s r#_delete_fs_pathrjs 2w ' 'FW  $ MM' " " " " " NN7 # # # # # WWWUBUU'UUUVVVVVVWsAA+A10A1T local_pathfilelockc4t|}|}t||r|ddn|jdd |r\t tj |d5t|||ddddS#1swxYwYdSt|||dS#t$r}|stj |d|d}~wwxYw)aWDownloads a directory or file from (fs, fs_path) to a local path. If fs_path points to a directory: - The full directory contents are downloaded directly into `local_path`, rather than to a subdirectory of `local_path`. If fs_path points to a file: - The file is downloaded to `local_path`, which is expected to be a file path. If the download fails, the `local_path` contents are cleaned up before raising, if the directory did not previously exist. NOTE: This method creates `local_path`'s parent directories if they do not already exist. If the download fails, this does NOT clean up all the parent directories that were created. Args: fs: The filesystem to download from. fs_path: The filesystem path (either a directory or a file) to download. local_path: The local path to download to. filelock: Whether to require a file lock before downloading, useful for multiple downloads to the same directory that may be happening in parallel. Raises: FileNotFoundError: if (fs, fs_path) doesn't exist. rZr`T)parentsexist_okz.lock)rVN) ignore_errors)rresolveexistsrdmkdirparentrosr)normpathr_rgshutilrmtree)rZr`rkrl _local_path exists_beforees r#_download_from_fs_pathr}sBz""**,,K&&((MG,,,>$6666   ===   N!1!1*!=!=DDDEE R R&w bQQQQ R R R R R R R R R R R R R R R R R R #7J" M M M M M M  : M*D 9 9 9 9 sB<1C.-C  C. CC.CC.C.. D8DDrr*cz|s%t||t|||dSt||||dS)a]Uploads a local directory or file to (fs, fs_path). NOTE: This will create all necessary parent directories at the destination. Args: local_path: The local path to upload. fs: The filesystem to upload to. fs_path: The filesystem path where the dir/file will be uploaded to. exclude: A list of filename matches to exclude from upload. This includes all files under subdirectories as well. This pattern will match with the relative paths of all files under `local_path`. Ex: ["*.png"] to exclude all .png images. rn)rWNrkrZr`r)_create_directoryr_"_upload_to_uri_with_exclude_fsspecrs r#_upload_to_fs_pathrse*  R1111z72NNNN&"gwr$z pyarrow.fsct||}tj|}tj|}t ||t ||||dS)N)rrrnrU)rrYrZ FSSpecHandler PyFileSystemrr_)rkrZr`rlocal_fshandler source_fss r#rrs~):wOOOHj&&x00G ''00IW----GyQSr$ file_filtercdtjdd}fd||DS)zReturns the list of filenames at (fs, fs_path), similar to os.listdir. If the path doesn't exist, returns an empty list. NcdS)NTr)xs r#z"_list_at_fs_path..sr$TF)allow_not_found recursivecg|]Y}| tj|jddZS)r,)start)rvr)relpathlstrip)r0 file_inforr`s r#rEz$_list_at_fs_path..sg     ;y ! !   --c22'..:M:MNN   r$)rYrZ FileSelector get_file_info)rZr`rselectors `` r#_list_at_fs_pathrsn$n z&&wPU&VVH     ))(33   r$cj||}|jtjjjkS)z%Returns True if (fs, fs_path) exists.)rtyperYrZFileTypeNotFound)rZr`valids r#_exists_at_fs_pathrs,   W % %E :,5 55r$c||}|jtjjjkrt d|d|d|j S)z~Checks if (fs, fs_path) is a directory or a file. Raises: FileNotFoundError: if (fs, fs_path) doesn't exist. zPath not found: (rb))rrrYrZrrFileNotFoundErroris_file)rZr`rs r#rdrd sZ  ))I~,555 DB D D' D D DEEE  r$c ||dS#t$r%td|d|dYdSwxYw)aWCreate directory at (fs, fs_path). Some external filesystems require directories to already exist, or at least the `netloc` to be created (e.g. PyArrows ``mock://`` filesystem). Generally this should be done before and outside of Ray applications. This utility is thus primarily used in testing, e.g. of ``mock://` URIs. z-Caught exception when creating directory at (rbrcN) create_dirrgrhrirns r#rrsw  g     MB M M' M M M       s+AA storage_pathstorage_filesystemctt|}|r||fStjj|S)aReturns the fs and path from a storage path and an optional custom fs. Args: storage_path: A storage path or URI. (ex: s3://bucket/path or /tmp/ray_results) storage_filesystem: A custom filesystem to use. If not provided, this will be auto-resolved by pyarrow. If provided, the storage_path is assumed to be prefix-stripped already, and must be a valid path on the filesystem. )rMrYrZ FileSystemfrom_uri)rrs r#get_fs_and_pathr)s=|$$L0!<// : ) ), 7 77r$c eZdZdZdedffd Z d dededeed ee e ffd Z deded ee e ffd Z ded ee e ffd Z xZS)_FilesystemSyncerz;Syncer between local filesystem and a `storage_filesystem`.rzpyarrow.fs.FileSystemc H||_tjdi|dSr)rrr)r rr!r"s r#rz_FilesystemSyncer.__init__As."4""6"""""r$Nrkurirr*cD|}tt||j||fS)Nr)rdictr)r rkrrr`s r#_sync_up_commandz"_FilesystemSyncer._sync_up_commandEs7   %*      r$cB|}tt|j||fS)N)rZr`rk)r}rr)r rrkr`s r#_sync_down_commandz$_FilesystemSyncer._sync_down_commandUs2 " *%     r$c@|}tt|j|fS)Nrn)rjrr)r rr`s r#_delete_commandz!_FilesystemSyncer._delete_command`s"(? Q Q QQQr$r.)rIrJrKrLr rrMr r rrrrrrPrQs@r#rr>sEE#84K+L###### DH   $' 2:4. x~      c  s  uXt^?T     R3R54+@RRRRRRRRr$rc eZdZdZ d#deeejfdedee dee j j deed e f d Zd Zd Zd ZdefdZd$dZd%deddfdZedefdZdefdZedefdZedefdZedefdZedefdZedefdZedefdZe deee!e"fdefd Z#e d!e fd"Z$dS)&StorageContexta# Shared context that holds the source of truth for all paths and storage utilities, passed along from the driver to workers. This object defines a few types of paths: 1. *_fs_path: A path on the `storage_filesystem`. This is a regular path which has been prefix-stripped by pyarrow.fs.FileSystem.from_uri and can be joined with `Path(...).as_posix()`. 2. *_driver_staging_path: The temporary staging directory on the local filesystem where driver artifacts are saved to before persisting them to storage. 3. trial_working_directory: The local filesystem path that the remote actors' working directories are moved to by default. This is separated from the driver staging path so that driver syncing does not implicitly upload the trial working directory, for trials on the driver node. Example with storage_path="mock:///bucket/path?param=1": >>> import ray >>> from ray.train._internal.storage import StorageContext >>> import os >>> _ = ray.init() >>> storage = StorageContext( ... storage_path="mock://netloc/bucket/path?param=1", ... experiment_dir_name="exp_name", ... ) >>> storage.storage_filesystem # Auto-resolved # doctest: +ELLIPSIS >> storage.experiment_fs_path 'bucket/path/exp_name' >>> storage.experiment_driver_staging_path # doctest: +ELLIPSIS '/tmp/ray/session_.../artifacts/.../exp_name/driver_artifacts' >>> storage.trial_dir_name = "trial_dir" >>> storage.trial_fs_path 'bucket/path/exp_name/trial_dir' >>> storage.trial_driver_staging_path # doctest: +ELLIPSIS '/tmp/ray/session_.../artifacts/.../exp_name/driver_artifacts/trial_dir' >>> storage.trial_working_directory # doctest: +ELLIPSIS '/tmp/ray/session_.../artifacts/.../exp_name/working_dirs/trial_dir' >>> storage.current_checkpoint_index = 1 >>> storage.checkpoint_fs_path 'bucket/path/exp_name/trial_dir/checkpoint_000001' >>> ray.shutdown() Example with storage_path="/tmp/ray_results": >>> from ray.train._internal.storage import StorageContext >>> storage = StorageContext( ... storage_path="/tmp/ray_results", ... experiment_dir_name="exp_name", ... ) >>> storage.storage_fs_path '/tmp/ray_results' >>> storage.experiment_fs_path '/tmp/ray_results/exp_name' >>> storage.storage_filesystem # Auto-resolved # doctest: +ELLIPSIS )r type_namerrrrr's r#__str__zStorageContext.__str__sj %)%<%F  "&"6  '+&>  "&!4    +/*G     r$ct|jt}|j|j|j|5 ddddS#1swxYwYdS)a On the creation of a storage context, create a validation file at the storage path to verify that the storage path can be written to. This validation file is also used to check whether the storage path is accessible by all nodes in the cluster.N)rexperiment_fs_path!_VALIDATE_STORAGE_MARKER_FILENAMEr5rropen_output_streamr valid_files r#rz&StorageContext._create_validation_files   #%F  (**  **4+BCCC  $ 7 7 C C                     s&A55A9<A9ct|jt}t |j|st d|ddS)z;Checks that the validation file exists at the storage path.rnz>Unable to set up cluster storage with the following settings: a Check that all nodes in the cluster have read/write access to the configured storage path. `RunConfig(storage_path)` should be set to a cloud storage URI or a shared filesystem path accessible by all nodes in your cluster ('s3://bucket' or '/mnt/nfs'). A local path on the head node is not accessible by worker nodes. See: https://docs.ray.io/en/latest/train/user-guides/persistent-storage.htmlN)rrrr5rr RuntimeErrorrs r#rz%StorageContext._check_validation_filesw  #%F  (** "T% ({dest_fs}, {destination}))r]r^rdest_fs)r]r^rVrW) filesystemr)z$Checkpoint successfully created at: ) rhdebugformatr)checkpoint_fs_pathrrrrr_r"info)r rpersisted_checkpoints r#persist_current_checkpointz)StorageContext.persist_current_checkpoints&   DDJF! 3$// EKEE    ##%%% **4+BCCC?/(3#'#:      *33.( 4    Q;OQQRRR##r$Fforcec||jjsdSttj|j }|rdS|rZ|j|j|j|j |jdS|j |j|j dS)aPPersists all artifacts within `trial_local_dir` to storage. This method possibly launches a background task to sync the trial dir, depending on the `sync_period` + `sync_artifacts_on_checkpoint` settings of `SyncConfig`. `(local_fs, trial_working_dir) -> (storage_filesystem, trial_fs_path)` Args: force: If True, wait for a previous sync to finish, launch a new one, and wait for that one to finish. By the end of a `force=True` call, the latest version of the trial artifacts will be persisted. N) local_dir remote_dir) rsync_artifactsr8rvscandirtrial_working_directoryrwaitsync_up trial_fs_pathsync_up_if_needed)r ris_emptys r#persist_artifactsz StorageContext.persist_artifacts4s.  F2:d&BCCDDD   F   K      K  64CU     K        K ) )64CU *     r$cZt|j|jS)aThe path on the `storage_filesystem` to the experiment directory. NOTE: This does not have a URI prefix anymore, since it has been stripped by pyarrow.fs.FileSystem.from_uri already. The URI scheme information is kept in `storage_filesystem` instead. )rrrr5r's r#rz!StorageContext.experiment_fs_pathUs&D($*BCCLLNNNr$cttt|j|jS)zsThe Ray Train/Tune session local directory used to stage files before persisting to the storage filesystem.)rrrrr5r's r#_get_session_pathz StorageContext._get_session_path_s1 & ( ($/4;S  (** r$cjt|dS)a The local filesystem path of the experiment directory on the driver node. The driver is the node where `Trainer.fit`/`Tuner.fit` is being called. This path is of the form: `/tmp/ray/session_/artifacts// /driver_artifacts` This should be used as the temporary staging location for files *on the driver* before syncing them to `experiment_fs_path`. For example, the search algorithm should dump its state to this directory. See `trial_driver_staging_path` for writing trial-specific artifacts. The directory is synced to `{storage_path}/{experiment_dir_name}` periodically. See `_ExperimentCheckpointManager.checkpoint` for where that happens. driver_artifacts)rrr5r's r#experiment_driver_staging_pathz-StorageContext.experiment_driver_staging_pathfs-&D**,,.@AAJJLLLr$c|jtdt|j|jS)zThe trial directory path on the `storage_filesystem`. Raises a ValueError if `trial_dir_name` is not set beforehand. NzBShould not access `trial_fs_path` without setting `trial_dir_name`)rrrrr5r's r#rzStorageContext.trial_fs_path{sF   &T D+T-@AAJJLLLr$c|jtdt|j|jS)aJThe local filesystem path of the trial directory on the driver. The driver is the node where `Trainer.fit`/`Tuner.fit` is being called. This path is of the form: `/tmp/ray/session_/artifacts// /driver_artifacts/` This should be used as the temporary location for files on the driver before persisting them to `trial_fs_path`. For example, callbacks (e.g., JsonLoggerCallback) should write trial-specific logfiles within this directory. NzNShould not access `trial_driver_staging_path` without setting `trial_dir_name`)rrrrr5r's r#trial_driver_staging_pathz(StorageContext.trial_driver_staging_pathsH   &3 D79LMMVVXXXr$c|jtdt|d|jS)aThe local filesystem path to trial working directory. This path is of the form: `/tmp/ray/session_/artifacts// /working_dirs/` Ray Train/Tune moves the remote actor's working directory to this path by default, unless disabled by `RAY_CHDIR_TO_TRIAL_DIR` environment variable. Writing files to this directory allows users to persist training artifacts if `SyncConfig(sync_artifacts=True)` is set. NzHCannot access `trial_working_directory` without setting `trial_dir_name` working_dirs)rrrrr5r's r#rz&StorageContext.trial_working_directorysW   &+   " " $ $nd6I  (** r$cZt|j|jS)a%The current checkpoint directory path on the `storage_filesystem`. "Current" refers to the checkpoint that is currently being created/persisted. The user of this class is responsible for setting the `current_checkpoint_index` (e.g., incrementing when needed). )rrcheckpoint_dir_namer5r's r#rz!StorageContext.checkpoint_fs_paths&D&(@AAJJLLLr$c@t|jS)zEThe current checkpoint directory name, based on the checkpoint index.)r_make_checkpoint_dir_namerr's r#rz"StorageContext.checkpoint_dir_names778UVVVr$run_objcddlm}ddlm}||}t t tj ddr|}nd ||}|S)Nr) ExperimentrTUNE_DISABLE_DATED_SUBDIRz{}_{}) ray.tune.experimentrrrget_trainable_namerOintrvenvirongetr)rrrrun_identifierdir_names r#get_experiment_dir_namez&StorageContext.get_experiment_dir_names222222++++++#66w?? BJNN#>BBCC D D B%HH~~nhhjjAAHr$indexcd|dS)z9Get the name of the checkpoint directory, given an index. checkpoint_06dr)r s r#rz(StorageContext._make_checkpoint_dir_names)U((((r$)NNNr)rrr*r)F)%rIrJrKrLr rMrvPathLiker rrYrZrrrrrrrrrrOrrNrrrrrrrr staticmethodrr r rrr$r#rresEEV-1>B(,(*&%&%C,-&%!&%j) &% %WZ%:; &% ! &%#&&%&%&%&%P        +++++ 0$0$0$0$dtBOCOOOXO3MMMMXM( Ms M M MX MY3YYYXY,X,MCMMMXMWSWWWXW sHd/B)C    \ ))))\)))r$r)NN)Tr.)<fsspecfsspec.implementations.localr ImportErrorModuleNotFoundErrorr|rrY pyarrow.fsrr/loggingrvrxpathlibrtypingrrrr r r r r ray.air._internal.filelockrray.train._internal.syncerrrrray.train.constantsrray.util.annotationsrray.train._checkpointr getLoggerrIrhrrr_rZrrMrjrOr}rrFileInforrrdrrrrrrr$r#rs1 MMM<<<<<<<()   , 4      NNN()   , 5     '''''''LLL TTTTTTTTTTTTTTTTTTTT333333LLLLLLLLLL::::::------1000000  8 $ $%?!1N1N1N1N1N1N1N1NjIM0 W - W W W W W  22 2 22 2222r$(  d3i   B  % 03 >FtCy>Q       DH  (GJ$7#8$#>?@ #Y (67:063646666 !gj+ !c !d ! ! ! ! '*/ # $    &;?88R[()8 !678 7: # %&8888*$R$R$R$R$R)$R$R$RNn)n)n)n)n)n)n)n)n)n)s6 % %2A  AA AAA