
    &`iҖ                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZmZ d dlm Z m!Z!m"Z" d d	l#m$Z$m%Z% d d
l&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2m3Z3 d dl4m5Z5 d dl6m7Z7m8Z8 d dl9m:Z: d dl;m<Z<m=Z= d dl>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZF d dlGmHZH d dlImJZJmKZK d dlLmMZMmNZN d dlOmPZP d dlQmRZRmSZS dZTdZUdZV ejW        eX          ZY G d d          ZZeS G d d                      Z[ G d  d!          Z\ G d" d#          Z]d$e^fd%Z_d&e`d'e`d$e`fd(Zad)ee`ef         d*e`fd+Zbd,d-d$ee`ef         fd.Zce	d/             ZdeS G d0 d-                      ZedS )1    N)contextmanager)partial)Number)Path)AnyCallableDictListOptionalSequenceTupleUnion)binary_to_hexhex_to_binary)EXPR_ERROR_FILEEXPR_ERROR_PICKLE_FILETRAINING_ITERATION)RayActorErrorRayTaskError)_CheckpointManager)_FutureTrainingResult_TrainingResult)StorageContext_exists_at_fs_path)RAY_CHDIR_TO_TRIAL_DIR%RAY_TRAIN_COUNT_PREEMPTION_AS_FAILURE)
CheckpointCheckpointConfig)	TuneError)PlacementGroupFactoryresource_dict_to_pg_factory)
NoopLogger)get_trainable_clsvalidate_trainable)DEBUG_METRICSDONENODE_IPPIDSTDERR_FILESTDOUT_FILETRIAL_ID
TRIAL_INFO)_TrainingRunMetadata)date_strflatten_dict)TuneFunctionDecoderTuneFunctionEncoder)log_once)
DeprecatedDeveloperAPI   i  ztrial_metadata.jsonc                        e Zd ZdZddZd ZdS )	_Locationz7Describes the location at which Trial is placed to run.Nc                 "    || _         || _        d S N)hostnamepidselfr:   r;   s      m/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/experiment/trial.py__init__z_Location.__init__F   s         c                     | j         sdS | j        t          j                    k    rd                    | j                   S d                    | j        | j                   S )N zpid={}z{}:{})r;   r:   platformnodeformatr=   s    r>   __str__z_Location.__str__J   sP    x 	;2]hmoo--??48,,,>>$-:::r@   NN)__name__
__module____qualname____doc__r?   rG    r@   r>   r7   r7   C   s=        AA   ; ; ; ; ;r@   r7   c                   8    e Zd ZdZdZdZdZdZed             Z	dS )ExportFormatzDescribes the format to import/export the trial Trainable.

    This may correspond to different file formats based on the
    Trainable implementation.
    
checkpointmodelonnxh5c                 >   t          t          |                     D ]}| |                                                                         | |<   | |         t          j        t          j        t          j        t          j        fvrt          d| |         z             dS )z^Validates formats.

        Raises:
            ValueError: if the format is unknown.
        z"Unsupported import/export format: N)
rangelenstriplowerrO   
CHECKPOINTMODELONNXH5r   )formatsis     r>   validatezExportFormat.validate`   s     s7||$$ 	S 	SA ))++1133GAJqz'"!	"     Dwqz QRRR	S 	Sr@   N)
rI   rJ   rK   rL   rY   rZ   r[   r\   staticmethodr_   rM   r@   r>   rO   rO   S   sR          JED	BS S \S S Sr@   rO   c                       e Zd ZdZddZed             Zed             Zed             Zede	fd	            Z
e
j        d
e	fd            Z
dS )
_TrialInfozSerializable struct for holding information for a Trial.

    Attributes:
        trial_name: String name of the current trial.
        trial_id: trial_id of the trial
        trial_resources: resources used by trial.
    trialTrialc                 v    t          |          | _        |j        | _        |j        | _        |j        | _        d S r9   )str_trial_nametrial_id	_trial_idplacement_group_factory_trial_resourcesexperiment_dir_name_experiment_name)r=   rc   s     r>   r?   z_TrialInfo.__init__{   s5    u:: % = % 9r@   c                     | j         S r9   )rm   rF   s    r>   experiment_namez_TrialInfo.experiment_name       $$r@   c                     | j         S r9   )rg   rF   s    r>   
trial_namez_TrialInfo.trial_name   s    r@   c                     | j         S r9   )ri   rF   s    r>   rh   z_TrialInfo.trial_id   s
    ~r@   returnc                     | j         S r9   rk   rF   s    r>   trial_resourcesz_TrialInfo.trial_resources   rp   r@   new_resourcesc                     || _         d S r9   rv   )r=   rx   s     r>   rw   z_TrialInfo.trial_resources   s     -r@   N)rc   rd   )rI   rJ   rK   rL   r?   propertyro   rr   rh   r    rw   setterrM   r@   r>   rb   rb   r   s         : : : : % % X%     X    X %!6 % % % X% .-B . . . . . .r@   rb   c                       e Zd ZdZd Zd ZdS )_TemporaryTrialStatezTTemporary trial state.

    Values saved here should not be restored on resume.
    c                 d    t                      | _        d | _        d | _        d | _        d| _        d S )Nr   )r7   location	ray_actor	saving_torestoring_fromnum_restore_failuresrF   s    r>   r?   z_TemporaryTrialState.__init__   s1    !:>:>9=)*!!!r@   c                     i S r9   rM   rF   s    r>   __getstate__z!_TemporaryTrialState.__getstate__   s    	r@   N)rI   rJ   rK   rL   r?   r   rM   r@   r>   r}   r}      s<         
+ + +    r@   r}   rt   c                  d    t          t          d          rt          j        dd          S t          S )Npathconf/PC_PATH_MAX)hasattrosr   _DEFAULT_WIN_MAX_PATH_LENGTHrM   r@   r>   _get_max_path_lengthr      s,    r: /{3...''r@   rootrelative_logdirc                 (   t          |                                                               |          }|                                rI|}|dt	          j                    j        d d         z   z  }t                              d| d| d           |S )N_   zCreating a new dirname z because trial dirname 'z' already exists.)	r   
expanduserjoinpathexistsuuiduuid4hexloggerinfo)r   r   	candidaterelative_logdir_olds       r>   _create_unique_logdir_namer      s    T

%%''00AAI 
-3!1"1"!555Eo E E1E E E	
 	
 	
 r@   configlogdirc                    t           j                            dt          j                               t          j        |d           t          t          t           j                            t          d                              rOt          j
        j                                        t          j
        j        j        k    st          j        |           t          | |          S )NTUNE_ORIG_WORKING_DIRT)exist_ok1)r   environ
setdefaultgetcwdmakedirsboolintgetr   ray_privateworker_mode
LOCAL_MODEchdirr"   )r   r   s     r>   _noop_logger_creatorr      s     J129;;???K&&&&C
5s;;<<==  |"((**cl.A.LLLHVff%%%r@   rc   rd   c                 <   |                                   t          t          | j        j                  }t          j        | j                  }t          |           |t          <   | j
        \  }}||t          <   ||t          <   | j        j        sJ ||| j        d}|S )N)r   )r   logger_creatorstorage)init_local_pathr   r   r   trial_working_directorycopydeepcopyr   rb   r,   log_to_filer*   r)   trial_dir_name)rc   r   trial_configstdout_filestderr_filekwargss         r>   _get_trainable_kwargsr      s    	U]%J  N =..L)%00L$0K +L +L='''' (= F Mr@   c              #   H  K   t           j        j                                        t           j        j        j        k    r`t          j                    }	 t          j        | j                   dV  t          j        |           dS # t          j        |           w xY wdV  dS )z}Context manager changing working directory to trial logdir.
    Used in local mode.

    For non-local mode it is no-op.
    N)	r   r   r   r   r   r   r   r   
local_path)rc   old_dirs     r>   _change_working_directoryr      s       |  ""cl&9&DDD)++	HU%&&&EEEHWBHWs   B Bc            #          e Zd ZdZg dZdZdZdZdZdZ	ddddd	ddddddddd
dddde
dee         dee
         dee         dee         de
dee         deee
ef                  dee         deee
                  dee
         deed ge
f                  deed ge
f                  deee
         eee
         ee
         f         f         dededef"dZd  Zd!ee         fd"Zd#eeef         fd$Zed!efd%            Zed&             Zed'             Z d!ee
         fd(Z!e e"d)          d*                         Z#ed+             Z$ed!e
fd,            Z%ed!e
fd-            Z&e e"d.          d!ee
         fd/                        Z'ed!ee
         fd0            Z(ed!ee
         fd1            Z)ed!efd2            Z*ed3             Z+ed4             Z,ed5             Z-ed!ee.         fd6            Z/ed!ee0         fd7            Z1e2d8             Z3dkd9Z4 e"d:          d;             Z5d< Z6d=eeef         fd>Z7d? Z8d@ Z9dA Z:dB Z;dC Z<dDefdEZ=edF             Z>edG             Z?edH             Z@edI             ZAd!eeB         fdJZCd!eeD         fdKZEdLeBfdMZFdLeGfdNZHdLeIfdOZJ	 dldLeeeDeIeGf                  fdPZKdQ ZLdR ZMd!efdSZNdTe.fdUZOdV ZPdW ZQdX ZRdY ZSdZ ZTed[             ZUed\             ZVd] ZWd^ ZXdmd_ZYd` ZZda Z[d!ee
e
f         fdbZ\e2dmdce
ded!d fdd            Z]dee
fdfZ^e2	 dmdgee
e_j`        f         ded!d fdh            Zadi Zbdj ZcdS )nrd   a;  A trial object holds the state for one model training run.

    Trials are themselves managed by the TrialRunner class, which implements
    the event loop for submitting trial runs to a Ray cluster.

    Trials start in the PENDING state, and transition to RUNNING once started.
    On error, it transitions to ERROR, otherwise TERMINATED on success.

    There are resources allocated to each trial. These should be specified
    using ``PlacementGroupFactory``.

    Attributes:
        trainable_name: Name of the trainable object to be executed.
        config: Provided configuration dictionary with evaluated params.
        trial_id: Unique identifier for the trial.
        path: Path where results for this trial are stored. Can be on
            the local node or on cloud storage.
        local_path: Path on the local disk where results are stored.
        remote_path: Path on cloud storage where results are stored,
            or None if not set.
        relative_logdir: Directory of the trial relative to its
            experiment directory.
        evaluated_params: Evaluated parameters by search algorithm,
        experiment_tag: Identifying trial name to show in the console
        status: One of PENDING, RUNNING, PAUSED, TERMINATED, ERROR/
        error_file: Path to the errors that this trial has raised.

    )results	extra_argrj   
_resources _default_placement_group_factoryPENDINGRUNNINGPAUSED
TERMINATEDERRORNrB   r   FT)r   rh   r   evaluated_paramsexperiment_tagrj   stopping_criterioncheckpoint_configexport_formatsrestore_pathtrial_name_creatortrial_dirname_creatorr   max_failuresstub_setup_default_resourcetrainable_namer   rh   r   r   r   rj   r   r   r   r   r   r   r   r   r   r   c                z   || _         | j         st          |           || _        |t                                          n|| _        t                      | _        t                      | _	        t          j
        |          | _        |pi | _        | j        | _        |pi | _        || _        |pi | _        || _        |r$t%          |t&                    st)          |          }|| _        d| _        || _        | j        r2t%          | j        t0                    rt3          | j                  dk    sd| _        || _        d| _        |
| _        t          j        | _        d| _        || _         || _!        d| _"        d| _#        |	ptI                      }	tK          |	          | j	        _&        || _'        d| _(        |r(tS          tU          j+        |          i           | _(        |r ||           | _"        |r? ||           | _#        tX          j-        j.        | j#        v rt_          d| j#                   d| _0        dS )a  Initialize a new trial.

        The args here take the same meaning as the command line flags defined
        in ray.tune.experiment.config_parser.

        Args:
            _setup_default_resource: Whether to set up default resources.
                When initializing trials from checkpoints, this field is set to false,
                so that setting up default resources can be delayed till after
                ``trial.config`` is loaded from checkpoints.
        N   rH   )r   )rP   metricsz(Trial dirname must not contain '/'. Got )1r   r$   r   rd   generate_idrh   r}   temporary_stater-   run_metadatar   r   r   _Trial__unresolved_configr   r   r   r   
isinstancer    r!   r   rj   r   r   rV   r   _default_result_or_futurer   r   statusr   r   r   custom_trial_namecustom_dirnamer   r   checkpoint_managerr   _restore_checkpoint_resultr   r   from_directoryr   pathsep
ValueError_state_json)r=   r   r   rh   r   r   r   rj   r   r   r   r   r   r   r   r   r   r   s                     r>   r?   zTrial.__init__$  s   F 	y 	/~...,/7/?))+++X355022 y))l $(;  !1 6B,"4":'>$" 	:#%:,
 ,
 	 'B'' '# 1H-'+$&  	,d.99	, t'((A--+D( LP&,m#"4%:"!%" .C1A1C1C/A/0
 0
 0
,
 )EI' 	.=%4\BBB/ / /D+  	>%7%7%=%=D"  	"7"7"="=Dw{d111 Tt?RTT    r@   c                    |                                  }|r| j        s| j        pt                      | _        dS |                    | j                  }|r*| j        r#t          d                    ||                    |r$t          |t                    st          |          }|p| j        pt                      | _        dS )zCompute placement group factory if needed.

        Note: this must be called after all the placeholders in
        self.config are resolved.
        NzResources for {} have been automatically set to {} by its `default_resource_request()` method. Please clear the `resources_per_trial` option.)r#   r   r   r!   rj   default_resource_requestr   r   rE   r   r    )r=   trainable_clsdefault_resourcess      r>   create_placement_group_factoryz$Trial.create_placement_group_factory  s    ..00 	D$@ 	 5V9T9V9V ( F)BB4;OO  	!F 	::@&!#4; ;    	OZ4&
 &
 	O !<<M N N  -4- +,, 	$$$r@   rt   c                    | j         rVt          | j         t          j                  r7	 t          j        | j                   | _         n# t
          $ r
 d| _         Y nw xY w| j         ri| j        j        r]|                     t          | j                             t                    | j                             t                                         | j         S )zCalls ray.get on self._default_result_or_future and assigns back.

        Returns None in case of exceptions.
        Will also set the trial location if runner is set.
        N)r   r   r   	ObjectRefr   r   r   r   set_locationr7   r'   r(   rF   s    r>   _get_default_result_or_futurez#Trial._get_default_result_or_future  s     ) 	6j*CM/
 /
 	66149W1X1X..  6 6 615...6) 	d.B.L 	266w??266s;;    --s   A AAplaceholder_resolversc                 p    ddl m} t          j        | j                  | _         || j        |           d S )Nr   )resolve_placeholders)ray.tune.impl.placeholderr   r   r   r   r   )r=   r   r   s      r>   resolve_config_placeholdersz!Trial.resolve_config_placeholders  sF    BBBBBB mD$<==T[*?@@@@@r@   c                     | j         j        }d |D             s|                                  | j        p|}|                    t
          | j                   |S )Nc                 (    h | ]}|t           k    |S rM   )r+   ).0ks     r>   	<setcomp>z$Trial.last_result.<locals>.<setcomp>  s    333aQ(]]]]]r@   )r   last_resultr   r   r   r+   rh   r=   results     r>   r   zTrial.last_result  sa     ".336333 	>..0003=vF(DM222r@   c                     | j         j        S r9   )r   metric_analysisrF   s    r>   r  zTrial.metric_analysis  s     00r@   c                     | j         j        S r9   )r   metric_n_stepsrF   s    r>   r  zTrial.metric_n_steps  s     //r@   c                 (   | j         j        j        r| j         j        j        S | j         j        sd S t	          j        | j         j        j                                                  \  }}t          ||          | j         _        | j         j        j        S r9   )	r   r   r:   r   r   r   get_current_ip_pidremoter7   r<   s      r>   get_ray_actor_ipzTrial.get_ray_actor_ip   s    (1 	:'099#- 	4 *=DDFF
 
# )2(C(@(@%#,55r@   z#Replaced by `local_experiment_path`c                     | j         S r9   )local_experiment_pathrF   s    r>   	local_dirzTrial.local_dir  s     ))r@   c                     | j         j        S r9   )r   rl   rF   s    r>   rl   zTrial.experiment_dir_name  s    |//r@   c                     | j         j        S r9   )r   experiment_fs_pathrF   s    r>   remote_experiment_pathzTrial.remote_experiment_path  s    |..r@   c                     | j         j        S r9   )r   experiment_driver_staging_pathrF   s    r>   r  zTrial.local_experiment_path  s    |::r@   zReplaced by `local_path`c                      t          d          )Nz%Use `local_path` instead of `logdir`.DeprecationWarningrF   s    r>   r   zTrial.logdir  s     !!HIIIr@   c                     | j         j        S r9   )r   trial_driver_staging_pathrF   s    r>   r   zTrial.local_path$  s    |55r@   c                     | j         j        S r9   )r   trial_fs_pathrF   s    r>   r   z
Trial.path(  s    |))r@   c                 4    t          | j        j                  S r9   )r   r   r   rF   s    r>   has_reported_at_least_oncez Trial.has_reported_at_least_once,  s    D%1222r@   c                 $    | j         j        j        S r9   )r   r   r:   rF   s    r>   node_ipzTrial.node_ip0  s    #,55r@   c                 2    | j         j        j        }|j        S r9   )r   r   r   checkpoint_at_endr=   r   s     r>   r   zTrial.checkpoint_at_end4  s    "5G''r@   c                 2    | j         j        j        }|j        S r9   )r   r   r   checkpoint_frequencyr!  s     r>   checkpoint_freqzTrial.checkpoint_freq9  s    "5G**r@   c                 2    | j         j        j        p| j        S r9   )r   r   latest_checkpoint_resultr   rF   s    r>   r&  zTrial.latest_checkpoint_result>  s     
 0I /.	
r@   c                 ,    | j         r| j         j        ndS )z9Returns the most recent checkpoint if one has been saved.N)r&  rP   rF   s    r>   rP   zTrial.checkpointG  s"    
 ,D)44	
r@   c                 \    t          t          j                    j                  d d         S )N   )rf   r   r   r   clss    r>   r   zTrial.generate_idP  s"    4:<<#$$RaR((r@   c                 N   |                                  }|o|                    | j                  }|s| j        nd }| j        j        j        }t          | j        | j        d | j	        | j
        || j        || j        | j        | j        | j        | j        | j        | j                  S )N)r   rh   r   r   rj   r   r   r   r   r   r   r   r   r   )r#   r   r   rj   r   r   r   rd   r   r   r   r   r   r   r   r   r   r   r   )r=   r   clear_resourcesrj   r   s        r>   resetzTrial.resetT  s     ..00' 
M,R,RK-
 -
 1@ID((T 	  !-@R;!2.$;#6/.*#6"&"<(*L
 
 
 	
r@   zReplaced by `init_local_path()`c                      t          d          )Nz/Use `init_local_path` instead of `init_logdir`.r  rF   s    r>   init_logdirzTrial.init_logdiru  s     !!RSSSr@   c                    | j         sJt          t          | j                  |                                           | _         | j         | j        _        | j        sJ t          | j                  }t                      }t          t          |                    |k    r t                              d| d|            |                    dd           |                                  dS )zInit logdir.z=The path to the trial log directory is too long (max length: zD. Consider using `trial_dirname_creator` to shorten the path. Path: T)parentsr   N)r   r   rf   r  _generate_dirnamer   r   r   r   r   rV   r   warningmkdirinvalidate_json_state)r=   logdir_pathmax_path_lengths      r>   r   zTrial.init_local_pathz  s    # 	?#=D.//1G1G1I1I$ $D  +/*>DL'4?++.00s;  O33NN' /' ' %' '   	$666""$$$$$r@   	resourcesc                     | j         t          j        u rt          d          |}t	          |t
                    rt          |          }|| _        |                                  dS )zEXPERIMENTAL: Updates the resource requirements.

        Should only be called when the trial is not running.

        Raises:
            ValueError: if trial status is running.
        z/Cannot update resources while Trial is running.N)	r   rd   r   r   r   dictr!   rj   r6  )r=   r9  rj   s      r>   update_resourceszTrial.update_resources  si     ;%-''NOOO"+i&& 	M&A)&L&L#'>$""$$$$$r@   c                 f    || j         _        |r"|j                            d          | _        d S d S )NT)debug_metrics_only)r   r   get_auto_filled_metricsr	  r   )r=   r   s     r>   set_ray_actorzTrial.set_ray_actor  sK    )2& 	 .7-N-U-U#' .V . .D***	 	r@   c                     || j         _        dS )zSets the location of the trial.N)r   r   )r=   r   s     r>   r   zTrial.set_location  s    (0%%%r@   c                     || _         |t          j        k    r)| j        j        t          j                    | j        _        |                                  dS )zSets the status of the trial.N)r   rd   r   r   
start_timetimer6  )r=   r   s     r>   
set_statuszTrial.set_status  sK    U]"" +3/3y{{!,""$$$$$r@   c                 <    || _         |                                  d S r9   )r   r6  r!  s     r>   
set_configzTrial.set_config  s!    ""$$$$$r@   c                 <    || _         |                                  d S r9   )r   r6  )r=   r   s     r>   set_experiment_tagzTrial.set_experiment_tag  s"    ,""$$$$$r@   new_storagec                    | j         }| j        j        }|j        D ]F}t	          |j        j                            |j        |j        d          |j	                  |_        G|j
        }|rDt	          |j        j                            |j        |j        d          |j	                  |_        || _         |                                  dS )a1  Updates the storage context of the trial.

        If the `storage_path` or `experiment_dir_name` has changed, then this setter
        also updates the paths of all checkpoints tracked by the checkpoint manager.
        This enables restoration from a checkpoint if the user moves the directory.
           )r   
filesystemN)r   r   r   best_checkpoint_resultsr   rP   r   replacer  storage_filesystemr&  r6  )r=   rJ  original_storager   checkpoint_resultr&  s         r>   set_storagezTrial.set_storage  s      <!.A!3!K 	 	+5&16>>$2K4Mq  '9	, , ,(( $6#N # 	2<-8=EE$2K4Mq  '9	3 3 3$/ #""$$$$$r@   c                     | j         j        S r9   )r   num_failuresrF   s    r>   rU  zTrial.num_failures  s     --r@   c                     | j         j        S r9   )r   num_failures_after_restorerF   s    r>   rW  z Trial.num_failures_after_restore  s     ;;r@   c                     | j         r| j        j        sd S t          | j         | j        j                                                  S r9   )r   r   error_filenamer   as_posixrF   s    r>   
error_filezTrial.error_file  sB     	d&7&F 	4DOT%6%EFFOOQQQr@   c                     | j         r| j        j        sd S t          | j         | j        j                                                  S r9   )r   r   pickled_error_filenamer   rZ  rF   s    r>   pickled_error_filezTrial.pickled_error_file  sF     	d&7&N 	4OT.E
 

(**	r@   c                 ^   | j         j        }|dS | j        j        }t	          | j        j        |                                          }t          ||          rS|                    |          5 }t          j
        |                                          cddd           S # 1 swxY w Y   dS )zReturns the pickled error object if it exists in storage.

        This is a pickled version of the latest error that the trial encountered.
        Nfsfs_path)r   r]  r   rP  r   r  rZ  r   open_input_streamcloudpickleloadsreadall)r=   rY  ra  pickled_error_fs_pathfs        r>   get_pickled_errorzTrial.get_pickled_error  s    
 *A!4\, $L&!
 !

(** 	 -BCCC 	6%%&;<< 6"(556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6t   /&B""B&)B&c                 ^   | j         j        }|dS | j        j        }t	          | j        j        |                                          }t          ||          rS|                    |          5 }|	                                
                                cddd           S # 1 swxY w Y   dS )zReturns the error text file trace as a TuneError object
        if it exists in storage.

        This is a text trace of the latest error that the trial encountered,
        which is used in the case that the error is not picklable.
        Nr`  )r   rY  r   rP  r   r  rZ  r   rc  rf  decode)r=   rY  ra  txt_error_fs_pathrh  s        r>   	get_errorzTrial.get_error  s     *9!4\, !;^LLUUWW->??? 	,%%&788 ,Ayy{{))++, , , , , , , , , , , , , , , ,trj  excc                     | j         j        t          t          j                            dd                    k    r| j        xj        dz  c_        d S | j         xj        dz  c_        d S )NTUNE_RESTORE_RETRY_NUMr   rL  )r   r   r   r   r   r   r   rU  )r=   ro  s     r>   _handle_restore_errorzTrial._handle_restore_error  ss     4JNN3Q779
 9
 
 
 **a/**** 55:5555r@   c                     t          t          t          j                            t
          d                              }|j        r|r| j        xj        dz  c_        d S d S )N0rL  )	r   r   r   r   r   r   	preemptedr   rU  )r=   ro  count_preemption_errorss      r>   _handle_ray_actor_errorzTrial._handle_ray_actor_error%  si    "&
DcJJKK#
 #
 } 	0 7 	0**a/****	0 	0r@   c                     |                                 }t          |t                    r|                     |          S | j        xj        dz  c_        d S )NrL  )as_instanceof_causer   r   rw  r   rU  )r=   ro  causes      r>   _handle_ray_task_errorzTrial._handle_ray_task_error-  sX    ''))e]++ 	7//666 	&&!+&&&&r@   c                 z   | j         r|                     |           nkt          |t                    r|                     |           n@t          |t
                    r|                     |           n| j        xj        dz  c_        | j	        rt          | j        _        t          |t
          t          f          rSt          | j        _        t          | j        d          5 }t!          j        ||           d d d            n# 1 swxY w Y   t          | j        d          5 }|                    d                    | j        j        t+                                           |                    t-          |          dz              d d d            n# 1 swxY w Y   | j                                         d S )NrL  wbza+zFailure # {} (occurred at {})

)is_restoringrr  r   r   rw  r   r{  r   rU  r   r   rY  r   r]  openr^  rd  dumpr[  writerE   r.   rf   invalidate_cache)r=   ro  rh  s      r>   handle_errorzTrial.handle_error6  s     	0&&s++++]++ 	0((----\** 	0'',,,,**a/**? 	)/>D,#m<== -;Q!8$1488 -A$S!,,,- - - - - - - - - - - - - - -dot,, )5<<)6

   
 C4((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 	**,,,,,s%   $DD
D
&A%FFFc           
         |                     t                    rdS | j                                        D ]\  }}t	          |t
                    rt          d          ||vrSt          d          rCt          	                    d| dt          |                                           d| d           ||         |k    r dS dS )	z>Whether the given result meets this trial's stopping criteria.TzcStopping criteria is now flattened by default. Use forward slashes to nest values `key1/key2/key3`.#tune_trial_stop_criterion_not_foundzStopping criterion 'z/' not found in result dict! Available keys are z. If 'zF' is never reported, the run will continue until training is finished.F)r   r&   r   itemsr   r;  r   r2   r   r4  listkeys)r=   r  	criterion
stop_values       r>   should_stopzTrial.should_stopR  s    ::d 	4%)%<%B%B%D%D 	 	!Iz)T**  K   &((ABB NN$y $ $.26;;==.A.A$ $IR$ $ $   	"j00tt 1ur@   c                     | j         pi }|                    t                    r	| j        rdS | j        o&|                    t
          d          | j        z  dk    S )z,Whether this trial is due for checkpointing.Tr   )r   r   r&   r   r$  r   r  s     r>   should_checkpointzTrial.should_checkpointi  sb    !'R::d 	 6 	4  N

-q11D4HHAM	
r@   c                     | j         d uS r9   )rP   rF   s    r>   has_checkpointzTrial.has_checkpoints  s    d**r@   rR  c                     | j         j                            |           | j                            |j                   |                                  | j                                          dS )zwHook for handling checkpoints taken by the Trainable.

        Args:
            checkpoint: Checkpoint taken.
        N)r   r   register_checkpointr   _update_checkpoint_indexr   r6  r  )r=   rR  s     r>   on_checkpointzTrial.on_checkpointv  sh     	,@@ARSSS 	--.?.GHHH""$$$**,,,,,r@   c                     | j         sJ | j        j        j        | j        _        | j        j                            d| j                   d| j        _        d| j        _        dS )zHandles restoration completion.r   Nr   )	r  r   r   r   r   r   r   r   r   rF   s    r>   
on_restorezTrial.on_restore  s_        (,(<(K(S%%004;GGG.2+45111r@   c                 B    | j         j        | j        k    p
| j        dk     S )a  Returns whether the trial qualifies for retrying.

        `num_failures` should represent the number of times the trial has
        failed *up to the moment this method is called.* If we've failed
        5 times and `max_failures=5`, then we should recover, since
        we only pass the limit on the 6th failure.

        Note this may return true even when there is no checkpoint, either because
        `self.checkpoint_freq` is `0` or because the trial failed before
        a checkpoint has been made.
        r   )r   rU  r   rF   s    r>   should_recoverzTrial.should_recover  s(     *d.??X4CTWXCX	
r@   c                    | j         r|                    | j                    |                     t          |                    t
                    |                    t                                         || j        _        t          j	                    | j        _
        | j                                        }t          D ]}|                    |d            t          |                                          D ]J\  }}t!          |t"                    r0| j                            |||                    d                     Kd S )N)r   training_iteration)step)r   updater   r7   r   r'   r(   r   r   rD  last_result_timer   r%   popr/   r  r   r   update_metric)r=   r  metric_resultremove_metricmetricvalues         r>   update_last_resultzTrial.update_last_result  s(    	>MM)<M===)FJJw$7$7CIIJJJ(.%-1Y[[*(--//* 	3 	3MmT2222)-88>>@@ 	 	MFE%(( !//E

3G(H(H 0   	 	r@   c                 <    | j         rd S t          | j                  S r9   )r   r#   r   rF   s    r>   r#   zTrial.get_trainable_cls  s"    9 	4 !4555r@   c                 @    | j         t          j        t          j        fv S r9   )r   rd   r   r   rF   s    r>   is_finishedzTrial.is_finished  s    {u{E,<===r@   c                     | j         j        d uS r9   )r   r   rF   s    r>   r  zTrial.is_restoring  s    #2$>>r@   c                     | j         j        d uS r9   )r   r   rF   s    r>   	is_savingzTrial.is_saving  s    #-T99r@   c                 .    |                      d          S NT)include_trial_id_trainable_namerF   s    r>   __repr__zTrial.__repr__      ##T#:::r@   c                 .    |                      d          S r  r  rF   s    r>   rG   zTrial.__str__  r  r@   c                    | j         r| j         S d| j        v rE| j        d         }t          |t                    r|j        }d                    | j        |          }n| j        }|r|d| j        z   z  }|                    dd          S )z|Combines ``env`` with ``trainable_name`` and ``trial_id``.

        Can be overridden with a custom string creator.
        envz{}_{}r   r   )	r   r   r   typerI   rE   r   rh   rO  )r=   r  r  
identifiers       r>   r  zTrial._trainable_name  s    
 ! 	*))DK+e$C#t$$ #l (;SAAJJ,J 	.#--J!!#s+++r@   c                    | j         r| j         }ndt          t          j                            dd                    }t          |            d| j         }|d |         }|dt                       z  }t          j	        dd|          S )NTUNE_MAX_LEN_IDENTIFIER130r   z[/()])
r   r   r   r   r   rf   r   r.   resub)r=   generated_dirnameMAX_LEN_IDENTIFIERs      r>   r3  zTrial._generate_dirname  s     	2 $ 3!$RZ^^4Mu%U%U!V!V#&t99 D Dt/B D D 12E3E2E F!1XZZ!1!11vgs$5666r@   c                     d | _         d S r9   )r   rF   s    r>   r6  zTrial.invalidate_json_state  s    r@   c                     | j         K|                                 }|                    dd            t          j        |dt
                    | _         | j                                        }| j         |fS )Nr   r   )indentr+  )r   r   r  jsondumpsr1   r   get_json_state)r=   stateruntime_metadata_jsons      r>   r  zTrial.get_json_state  sn    #%%''EIInd+++#z%?RSSSD $ 1 @ @ B B!666r@   
json_statec                     t          j        |t                    }t          |d         |d          }|                    |           |S )Nr*  r   F)r   r   )r  re  r0   rd   __setstate__)r+  r  r   r  	new_trials        r>   from_json_statezTrial.from_json_state  sV    
:+>???"#$)
 
 
	 	u%%%r@   r   c                 8    t          j        |          | _        d S r9   )r-   r  r   )r=   r   s     r>   restore_run_metadatazTrial.restore_run_metadata  s    0@NNr@   r   c                     t          |t                    }|                                st          d| d          |                                }|                     ||          S )Nz%Can't restore trial from path: File `z` not found.)r   )r   TRIAL_STATE_FILENAMEr   FileNotFoundError	read_textr  )r+  r   r   metadata_pathr  s        r>   r   zTrial.from_directory  st     T#788##%% 	#SSSS   #,,..
"":D"999r@   c                     | j                                         }| j        D ]9}t          t	          j        |                    |                              ||<   :|                    dd           d|d<   d|d<   |S )zMemento generator for Trial.

        Sets RUNNING trials to PENDING.
        Note this can only occur if the trial holds a PERSISTENT checkpoint.
        r   Nr   r   )__dict__r   _nonjson_fieldsr   rd  r  r   r  )r=   r  keys      r>   r   zTrial.__getstate__  s     ""$$' 	J 	JC&{'83'H'HIIE#JJ		#T***#m-1)*r@   c                    |d         t           j        k    rt           j        |d<   | j        D ]0}||v r*t	          j        t          ||                             ||<   1|                    dd          }| j        	                    |           |pt          | dd          | _        | j        st          | j                   t                      | _        | j        sJ d S )Nr   r   TF)rd   r   r   r  rd  re  r   r  r  r  getattrr   r$   r   r}   r   rj   )r=   r  r  r   s       r>   r  zTrial.__setstate__   s    ?em++#mE(O' 	J 	JCe||(.}U3Z/H/HIIc
 yy&&U###8GD&%88	y 	4t2333355++++++r@   )rt   rd   r9   )F)drI   rJ   rK   rL   r  r   r   r   r   r   rf   r   r	   r   r    floatr   r
   r   r   r   r   r   r?   r   r;  r   r   r   rz   r   r  r  r
  r3   r  rl   r  r  r   r   r   r  r  r   r$  r   r&  r   rP   classmethodr   r.  r0  r   r<  r@  r   rE  rG  rI  rS  rU  rW  r[  r^  	Exceptionri  r   rn  rr  r   rw  r   r{  r  r  r  r  r  r  r  r  r#   r  r  r  r  rG   r  r3  r6  r  r  r  r   PathLiker   r   r  rM   r@   r>   rd   rd      s        :  O GGFJE "&"&,0+/ CG9=8<.2&*AEDHQU(,'z  z  z z  	z 
 3-z  .)z  #4.z  z  "**?!@z  %T#u*%56z  $$45z  !c+z  smz  %Xwin%=>z   ('C(@Az   8C=%x}0L*MMN!z " #z $ %z & "&'z  z  z  z x'
 '
 '
R.x~ . . . .,AeSjAQ A A A A T    X" 1 1 X1 0 0 X06(3- 6 6 6 6 Z566* * 76 X* 0 0 X0 / / / / X/ ;s ; ; ; X; Z*++J J J J ,+ XJ 6HSM 6 6 6 X6 *hsm * * * X* 3D 3 3 3 X3 6 6 X6 ( ( X( + + X+ 
(?*C 
 
 
 X
 
HZ0 
 
 
 X
 ) ) [)
 
 
 
B Z122T T 32T% % %.%%6K0K*L % % % %&  1 1 1% % %% % %% % %%~ % % % %< . . X. < < X< R R XR
   X8I#6    &8I.    &; ; ; ; ;0= 0 0 0 0,, , , , , NR- -E)\="HIJ- - - -8  .
 
 
+ + + + +- - - - -6 6 6
 
 
   $6 6 6
> > > ? ? X? : : X:; ; ;; ; ;, , , ,&	7 	7 	7     7c3h 7 7 7 7   D W    [O O O O O 9>
: 
:bk)*
:26
:	
: 
: 
: [
:  $, , , , ,r@   )fr   r  loggingr   rC   r  rD  r   
contextlibr   	functoolsr   numbersr   pathlibr   typingr   r   r	   r
   r   r   r   r   r   ray.cloudpicklerd  ray._common.utilsr   r   ray.air.constantsr   r   r   ray.exceptionsr   r   &ray.train._internal.checkpoint_managerr   ray.train._internal.sessionr   r   ray.train._internal.storager   r   ray.train.constantsr   r   ray.tuner   r   ray.tune.errorr   #ray.tune.execution.placement_groupsr    r!   ray.tune.loggerr"   ray.tune.registryr#   r$   ray.tune.resultr%   r&   r'   r(   r)   r*   r+   r,   ray.tune.trainable.metadatar-   ray.tune.utilsr.   r/   ray.tune.utils.serializationr0   r1   ray.utilr2   ray.util.annotationsr3   r4   DEBUG_PRINT_INTERVALr   r  	getLoggerrI   r   r7   rO   rb   r}   r   r   rf   r   r   r   r   rd   rM   r@   r>   <module>r     s      				  				   % % % % % %                   N N N N N N N N N N N N N N N N N N N N 



 % % % % % % : : : : : : : :         
 7 6 6 6 6 6 6 6 E E E E E E N N N N N N N N J J J J J J J J        2 1 1 1 1 1 1 1 $ $ $ $ $ $        ' & & & & &
 D C C C C C C C	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 = < < < < < 1 1 1 1 1 1 1 1 Q Q Q Q Q Q Q Q       9 9 9 9 9 9 9 9 " ,  
	8	$	$; ; ; ; ; ; ; ;  S S S S S S S S<!. !. !. !. !. !. !. !.H       ((c ( ( ( (	S 	3 	3 	 	 	 	&c3h & & & & &  T#s(^    0   " y, y, y, y, y, y, y, y, y, y,r@   