&`i6PddlZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z ddlmZddlmZmZddlmZejeZdZdZeeGd d ZGd d ZeGd dejZGddeZdS)N) dataclass)AnyCallableDictListOptionalTuple)tabulate) Deprecated DeveloperAPI)Templatei,icTeZdZUeZeed<eZeed<dZ e ed<dZ e ed<de fdZ d S) SyncConfig sync_period sync_timeoutFsync_artifactsTsync_artifacts_on_checkpointreturnctdtddg|j|jgddddd S) z2Generate an HTML representation of the SyncConfig.zscrollableTable.html.j2z Sync periodz Sync timeout)SettingValuehtmlFkeys)tablefmt showindexheadersnone)table max_height)r renderr rrselfs n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/train/_internal/syncer.py _repr_html_zSyncConfig._repr_html_sj12299 -~>".0AB :   N)__name__ __module__ __qualname__DEFAULT_SYNC_PERIODrint__annotations__DEFAULT_SYNC_TIMEOUTrrboolrstrr$r%r#rrst+K***,L#,,, ND   )- $---  S       r%rcjeZdZdefdZedZedZdZd de e de fd Z dS) _BackgroundProcessfncX||_d|_i|_td|_dSNz-inf)_fn_process_resultfloat _start_time)r"r2s r#__init__z_BackgroundProcess.__init__/s+   ==r%cB|jo|jSN)r6is_aliver!s r# is_runningz_BackgroundProcess.is_running5s}9!7!7!9!99r%c|jSr<)r9r!s r# start_timez_BackgroundProcess.start_time9s r%cjrdSi_fd}tj|_dj_jtj_dS)NFc~ ji}n"#t$r}|jd<Yd}~dSd}~wwxYw|jd<dS)N exceptionresult)r5 Exceptionr7)rDeargskwargsr"s r# entrypointz,_BackgroundProcess.start..entrypointCsl !42622   ,- [) &,DL " " "s  0 +0)targetT) r>r7 threadingThreadr6daemonstarttimer9)r"rGrHrIs``` r#rNz_BackgroundProcess.start=s ? 5  , , , , , , ,"( ;;; #  9;;r%Ntimeoutrc |jsdSd}|r.tj|jz }t||z d}|j||jrBd|_t t|jdt|jd|dd|_|j d}|r||j d}i|_ |S) zWaits for the background process to finish running. Waits until the background process has run for at least `timeout` seconds, counting from the time when the process was started.NrrPr&z. did not finish running within the timeout of z seconds.rCrD) r6rOr@maxjoinr= TimeoutErrorgetattrr5r.r7get)r"rPtime_remainingelapsedrCrDs r#waitz_BackgroundProcess.waitQs} 4  7ikkDO3G 7!2A66N >222 = ! ! # #  DM48ZTX??DD18DDD   L$$[11  O!!(++  r%r<) r&r'r(rr:propertyr>r@rNrr8rrZr/r%r#r1r1.s)8)))) ::X:  X '''(HUOsr%r1c leZdZdZeefdedefdZej dde de de e d e fd Zej dde de de e d e fd Zej de d e fd Zd Zdde efdZ dde de de e d e fdZ dde de de e fdZddedefdZdZdZd e fdZdS)SynceraSyncer class for synchronizing data between Ray nodes and remote (cloud) storage. This class handles data transfer for two cases: 1. Synchronizing data such as experiment state snapshots from the driver to cloud storage. 2. Synchronizing data such as trial checkpoints from remote trainables to cloud storage. Synchronizing tasks are usually asynchronous and can be awaited using ``wait()``. The base class implements a ``wait_or_retry()`` API that will retry a failed sync command. The base class also exposes an API to only kick off syncs every ``sync_period`` seconds. Args: sync_period: The minimum time in seconds between sync operations, as used by ``sync_up/down_if_needed``. sync_timeout: The maximum time to wait for a sync process to finish before issuing a new sync operation. Ex: should be used by ``wait`` if launching asynchronous sync tasks. rrcr||_||_td|_td|_dSr4)rrr8last_sync_up_timelast_sync_down_time)r"rrs r#r:zSyncer.__init__s6 '(!&v#(==   r%N local_dir remote_direxcluderct)a(Synchronize local directory to remote directory. This function can spawn an asynchronous process that can be awaited in ``wait()``. Args: local_dir: Local directory to sync from. remote_dir: Remote directory to sync up to. This is an URI (``protocol://remote/path``). exclude: Pattern of files to exclude, e.g. ``["*/checkpoint_*]`` to exclude trial checkpoints. Returns: True if sync process has been spawned, False otherwise. NotImplementedError)r"rarbrcs r#sync_upzSyncer.sync_up ("!r%ct)a*Synchronize remote directory to local directory. This function can spawn an asynchronous process that can be awaited in ``wait()``. Args: remote_dir: Remote directory to sync down from. This is an URI (``protocol://remote/path``). local_dir: Local directory to sync to. exclude: Pattern of files to exclude, e.g. ``["*/checkpoint_*]`` to exclude trial checkpoints. Returns: True if sync process has been spawned, False otherwise. re)r"rbrarcs r# sync_downzSyncer.sync_downrhr%ct)agDelete directory on remote storage. This function can spawn an asynchronous process that can be awaited in ``wait()``. Args: remote_dir: Remote directory to delete. This is an URI (``protocol://remote/path``). Returns: True if sync process has been spawned, False otherwise. re)r"rbs r#deletez Syncer.deletes "!r%cdS)zRetry the last sync up, sync down, or delete command. You should implement this method if you spawn asynchronous syncing processes. Nr/r!s r#retryz Syncer.retrys r%rPcdS)aWait for asynchronous sync command to finish. You should implement this method if you spawn asynchronous syncing processes. This method should timeout after the asynchronous command has run for `sync_timeout` seconds and raise a `TimeoutError`. Nr/)r"rPs r#rZz Syncer.waits  r%ctj}||jz |jkr!||||}||_|SdS)a}Syncs up if time since last sync up is greater than sync_period. Args: local_dir: Local directory to sync from. remote_dir: Remote directory to sync up to. This is an URI (``protocol://remote/path``). exclude: Pattern of files to exclude, e.g. ``["*/checkpoint_*]`` to exclude trial checkpoints. )rarbrcN)rOr_rrg)r"rarbrcnowrDs r#sync_up_if_neededzSyncer.sync_up_if_neededs[ikk ' '4+; ; ;\\# G"F&)D "M < ;r%ctj}||jz |jkr!||||}||_|SdS)aSyncs down if time since last sync down is greater than sync_period. Args: remote_dir: Remote directory to sync down from. This is an URI (``protocol://remote/path``). local_dir: Local directory to sync to. exclude: Pattern of files to exclude, e.g. ``["*/checkpoint_*]`` to exclude trial checkpoints. )rbrarcN)rOr`rrj)r"rbrarcrqrDs r#sync_down_if_neededzSyncer.sync_down_if_neededs[ikk ) )T-= = =^^%G$F(+D $M > =r% max_retries backoff_sc |dksJd}t|dzD]} |n#t$r}||z }|dkrtj}Yd}~nit dt|d|d|dtj || Yd}~d}~wwxYwdStd|d|) Nrz;The latest sync operation failed with the following error: z Retrying z! more time(s) after sleeping for z seconds...zFailed sync even after z; retries. The latest sync failed with the following error: ) rangerZrE traceback format_excloggererrorreprrOsleeprn RuntimeError)r"rwrxlast_error_tracebackirFattempts_remainings r# wait_or_retryzSyncer.wait_or_retry sMQ#{Q''  A     %01_"&**+4+?+A+A(EEEEE 2Aww22 222%222  9%%% # & FF Xk X XAU X X   s5 CC#ACCcVtd|_td|_dSr4)r8r_r`r!s r#resetz Syncer.reset+s$!&v#(==   r%cdSr<r/r!s r#closez Syncer.close/s r%cdSr<r/r!s r#r$zSyncer._repr_html_2sr%r<)rurv)r&r'r(__doc__r)r,r8r:abcabstractmethodr.rrr-rgrjrlrnrZrrrtr*rrrr$r/r%r#r]r]rs64121111111 IM"""*-"8@" """"* IM"""*-"8@" """"* """"""      HUO    JN*-8@ *JN*-8@(   S    >111   Sr%r]c VeZdZdZeefdedeffd ZdZde e e ffdZ dd e d e d eed efd Z dde de d eed e e e ffdZ dd e d e d eed efdZde de d e e e ffdZd e d efdZde d e e e ffdZddeefdZdZdZxZS)_BackgroundSyncerzASyncer using a background process for asynchronous file transfer.rrcxtt|||d|_d|_dS)N)rr)superrr: _sync_process _current_cmd)r"rr __class__s r#r:z_BackgroundSyncer.__init__9sJ &&//#, 0   " r%cz|jo4|jjo(tj|jjz |jkS)zDReturns whether a previous sync is still running within the timeout.)rr>rOr@rr!s r#_should_continue_existing_syncz0_BackgroundSyncer._should_continue_existing_syncDs@   P"- P d0;;d>OO r% sync_commandc|jrT |n>#t$r1tdt jYnwxYw||_|dS)zoWaits for the previous sync process to finish, then launches a new process that runs the given command.z3Last sync command failed with the following error: N) rrZrEr~warningr|r}rrn)r"rs r#_launch_sync_processz&_BackgroundSyncer._launch_sync_processLs        0 +--00  ) s8AANrarbrcrc|r"td|d|dS||||}||dS)Nz1Last sync still in progress, skipping sync up of  to F) local_pathurircT)rr~debug_sync_up_commandr)r"rarbrc sync_up_cmds r#rgz_BackgroundSyncer.sync_up[s  . . 0 0  LLC'0CC6@CC   5++ j',   !!+...tr%rrctr<re)r"rrrcs r#rz"_BackgroundSyncer._sync_up_commandls "!r%c|r"td|d|dS|||}||dS)Nz3Last sync still in progress, skipping sync down of rF)rrT)rr~r_sync_down_commandr)r"rbrarc sync_down_cmds r#rjz_BackgroundSyncer.sync_downqs  . . 0 0  NNE)3EE9BEE   5//J9/UU  !!-000tr%ctr<re)r"rrs r#rz$_BackgroundSyncer._sync_down_command!!r%c|rtd|dS||}||dS)Nz2Last sync still in progress, skipping deletion of F)rT)rr~r_delete_commandr)r"rb delete_cmds r#rlz_BackgroundSyncer.deletesm  . . 0 0  NNQZQQ   5))j)99  !!*---tr%ctr<re)r"rs r#rz!_BackgroundSyncer._delete_commandrr%rPc|jrN |j|p|jn#t$r}|d}~wwxYw d|_dS#d|_wxYwdS)NrR)rrZrrE)r"rPrFs r#rZz_BackgroundSyncer.waits   * *"''0L4;L'MMMM    N &*"""T")))) * *s ",A =8=A Ac|jstd|j\}}t||_|jjdi|dS)Nz"No sync command set, cannot retry.r/)rrr1rrN)r"cmdrHs r#rnz_BackgroundSyncer.retrys[  ECDD D' V/44  **6*****r%cB|j}d|d<|S)Nr)__dict__copy)r"states r# __getstate__z_BackgroundSyncer.__getstate__s$ ""$$!%o r%r<)r&r'r(rr)r,r8r:rr rrrr.rrr-rgrrjrrlrrZrnr __classcell__)rs@r#rr6s+KK12 ! ! ! ! ! ! ! ! !    x~1F     JN*-8@ $DH"""$'"2:4." x~ """" JN   *- 8@      "c"s"uXt^?T""""      "3"54+@"""" * *HUO * * * *+++r%r)rloggingrKrOr| dataclassesrtypingrrrrrr )ray._private.thirdparty.tabulate.tabulater ray.util.annotationsr r ray.widgetsr getLoggerr&r~r)r,rr1ABCr]rr/r%r#rs  !!!!!!================>>>>>>99999999  8 $ $            ,AAAAAAAAH@@@@@SW@@@Fqqqqqqqqqqr%