&`iLJddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl m Z ddl mZddlmZmZmZmZmZmZmZmZddlZddlmZddlmZdd lmZmZdd l m!Z!m"Z"dd l#m$Z$m%Z%dd l&m'Z'm(Z(dd l)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl0m1Z1m2Z2m3Z3ddl4m5Z5ddl6m7Z7m8Z8ddl9m:Z:ddl;mZ>m?Z?ddl@mAZAmBZBmCZCmDZDmEZEddlFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNddlOmPZPmQZQddlRmSZSmTZTddlUmVZVmWZWddlXmYZYddlZm[Z[m\Z\ddl]m^Z^m_Z_m`Z`ddlambZbddlcmdZddd lemfZfmgZgdd!lhmiZidd"ljmkZkejlemZneiGd#d$ZoGd%d&ZpeiGd'd(Zqd)eTd*erfd+ZsGd,d-ZtdS).N) defaultdictdeque)datetime)partial)Path)AnyCallableDictListOptionalSetTupleUnion)ResourceRequest)TIME_THIS_ITER_S)PlacementGroupResourceManagerResourceManager)RayActorManager TrackedActor) RayActorError RayTaskError)_FutureTrainingResult_TrainingResult)StorageContext)CheckpointConfig)Callback CallbackList) TuneError_AbortTrialExecution_TuneStopTrialError)_ActorClassCache)_ExperimentCheckpointManager"_find_newest_experiment_checkpoint)_InsufficientResourcesManager)PlacementGroupFactory) ExperimentTrial)_change_working_directory_get_trainable_kwargs _Location_noop_logger_creator _TrialInfo) DEBUG_METRICSDEFAULT_METRICDONERESULT_DUPLICATESHOULD_CHECKPOINT STDERR_FILE STDOUT_FILE TRIAL_INFO) FIFOSchedulerTrialScheduler)BasicVariantGeneratorSearchAlgorithm) NoopStopperStopper) ResumeConfig) flatten_dict warn_if_slow) Verbosity _dedup_logs has_verbosity) _ObjectCache)_ResourceUpdater)TuneFunctionDecoderTuneFunctionEncoder) DeveloperAPI)log_oncecneZdZdZdZddddddddddddddddeedeeee fdee d ee d ee d e d eeefd eeedeedeedeede deegefde fdZdZedZedZedZdeededdfdZddZedefdZ edefdZ!edefd Z"d!Z#d"Z$dd#e d$e fd%Z%d&ee&d e fd'Z'd(edee&fd)Z(d e fd*Z)dd+eefd,Z*d-ee+e,ffd.Z-d/Z.d0Z/d1Z0d2Z1d3e&fd4Z2dd6e d7ede fd8Z3defd9Z4d:Z5d;Z6ddZ8d?Z9d3e&d@efdAZ:deeeffdBZ;d3e&fdCZdFe?fdGZ@ddHZAd3e&de fdIZBd3e&fdJZCd3e&fdKZDd3e&de fdLZEddFe?dNefdOZFdFe?fdPZGdFe?dQeHfdRZI dd3e&dSedTeedUeedVeee&e gdfdWeee&eHgdfdXe deeJjKfdYZLdZZMdd3e&d[ed\e fd]ZNdd3e&d\e fd^ZOd_ZPd3e&dQeHfd`ZQd3e&dQeeReSeTffdaZUdd3e&dQeeHfdbZVdcZWd3e&fddZXdd3e&dfe fdgZYd3e&fdhZZd3e&deee[ffdiZ\djZ]dkZ^d3e&dle+ee ffdmZ_dnZ` dd3e&dleedeeafdoZbdpecfdqZdd3e&dpecfdrZeddsZfd3e&de fdtZgd3e&dle fduZhd3e&fdvZid3e&dweeReSeTffdxZjdyZkd3e&fdzZld3e&d{ed|efd}Zmd3e&d~e fdZndZodZpdZqdd3e&dfe fdZrdZsdZtdZudS)TuneControllerzexperiment_state-{}.jsonRAISENF) search_algplaceholder_resolvers schedulerstopper resume_config fail_fastcheckpoint_period callbacksmetrictrial_checkpoint_configstorage reuse_actorsresource_manager_factory _trainer_apirJrKrLrMrNrOrPrQrRrSrTrUrVrWc` | r | }nt}t||_t|_t d|_i|_i|_tt|_ t|_ g|_ t|_t|_t|_t|_t|_t|_t|_i|_t-d|_t1t2jdd|_d|_| |_t?d|_ t|_!i|_"t1t3j#dd |_$t-t3j#d d |_%t-t3j#d d |_&|p tO|_(||_)|p tU|_+tY|pg|_-t]||_/i|_0tc|j(|_2| |_3| |_4d|_5d|_6d|_7||_8ts|j8ttr[|j8;|_8|j8|j<krt{j>dntd|j8dtt1t2jdd|_Ag|_Bt|_Ci|_Di|_Eg|_Fd|_G|p t|_ItjJ|_KtjM|jKNd|_O|t3j#dd}||_P| p t|_R|S|_Td|_U| |V|d|_UdS#t$r}ttjZr't\tu|t]d|j8rt^dYd}~dSd}~wwxYwt_ddS)N)resource_managerinfTUNE_FORCE_TRIAL_CLEANUP_S600 T) may_keep_oneTUNE_RESULT_BUFFER_LENGTHTUNE_RESULT_BUFFER_MIN_TIME_SgTUNE_RESULT_BUFFER_MAX_TIME_SgY@) for_trainrFzfail_fast='raise' detected. Be careful when using this mode as resources (such as Ray processes, file descriptors, and temporary files) may not be cleaned up properly. To use a safer mode, use fail_fast=True.z,fail_fast must be one of {bool, RAISE}. Got .TUNE_PRINT_ALL_TRIAL_ERRORS1z%Y-%m-%d_%H-%M-%STUNE_GLOBAL_CHECKPOINT_Sauto)rNz Failed to restore the run state.zRestarting experiment.zStarting a new experiment.)`rr_actor_managerr! _class_cacherB_resource_updater_actor_to_trial_trial_to_actorrset_resources_to_pending_trials_pending_trials_pending_trials_list_running_trials_paused_trials_stopped_trials_failed_trials_resetting_trials_staged_trials_started_actors_stopping_actorsfloat_earliest_stopping_actorintosenvironget_actor_cleanup_timeout_actor_force_cleanup_timeout _reuse_actorsrA _actor_cache_trials_to_cache_trial_metadatagetenv_buffer_length_buffer_min_time_s_buffer_max_time_sr7 _search_alg_placeholder_resolversr5_scheduler_algr _callbacksr$_insufficient_resources_manager_pending_trial_queue_times_get_max_pending_trials_max_pending_trials_storage_metric _total_time _iteration _has_errored _fail_fast isinstancestrupperrIwarningswarn ValueErrorbool_print_trial_errors_trials _live_trials_cached_trial_decisions_queued_trial_decisions _stop_queue_should_stop_experimentr9_stoppertime _start_timer fromtimestampstrftime _session_str_checkpoint_periodr_trial_checkpoint_config_create_checkpoint_manager_checkpoint_manager_resumedresume Exceptionr@r>V3_TRIAL_DETAILSloggererror exceptioninfodebug)selfrJrKrLrMrNrOrPrQrRrSrTrUrVrWrYes v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/execution/tune_controller.py__init__zTuneController.__init__Hs$ $ ?7799  <>> -?OPPP,.."2$!7!7;=:<    ) ,/5513!+.55*-%%+.55*-%%-0UU*-%%36%% <>/4U||%+. JNN7 ? ?, , #24)*(d;;;-0EE/1""),G"K"KLL"' 2QSV(W(W"X"X"' I5u = =# #  &@)>)@)@&;#':=??&yB77/L"0 0 0 ,+-'#:4;K#L#L   !# dos + + "o3355DO$*,, 8!XdoXXX$(   get_trialsrJ stop_trial pause_trialget_live_trials_set_trial_status_schedule_trial_save>saverrkhas_resources_for_trial)trial_executorrunner_whitelist_attrexecutor_whitelist_attr)TrialRunnerWrapper_FakeRayTrialExecutorrs r_wrappedzTuneController._wrappedsE! 066###%%%    c|jSN)rrs rresumedzTuneController.resumeds }rc|jSr)rrs rrJzTuneController.search_alg s rc|jSr)rrs r scheduler_algzTuneController.scheduler_algs ""r experimentstotal_num_samplesreturncZ|d}|r|jni}||d<|jjdi|dS)a#Obtains any necessary information from experiments. Mainly used to setup callbacks. Args: experiments: List of Experiments to use. total_num_samples: Total number of samples factoring in grid search samplers. rrN) public_specrsetup)rrr experimentspecs rsetup_experimentsz TuneController.setup_experimentssK!^ )3;z%%$5 !%%%%%%%rcF|j|jdS)z0Calls ``on_experiment_end`` method in callbacks.)trialsN)ron_experiment_endrrs rend_experiment_callbacksz'TuneController.end_experiment_callbacks&s# )))>>>>>rc@|j|jSr)CKPT_FILE_TMPLformatrrs rexperiment_state_file_namez)TuneController.experiment_state_file_name*s"))$*;<< ? ? ? ?  @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ $$%8dFW$XXX ##$7TEV#WWWWWsB??CCforcewaitcJ|j|j||dS)N)save_fnrr)rsync_up_experiment_stater)rrrs r checkpointzTuneController.checkpoint_s6  99$E :     rrct|ddD]=}|jtjkr|j}n$|jtjkr|j}n|j}d}|tj j kr:|}d|j _ d|j _ |tjn|tj jkr|}d|_n^|tj jkr7|}|jtjkr|tjnt)d||J||?dS)Nc|jjSr) run_metadatalast_result_time)ts rz9TuneController._requeue_restored_trials..is !."ArT)keyreversezUnknown resume type: )sortedstatusr'ERRORerrored TERMINATEDfinished unfinishedr; ResumeTypeRESUMErerror_filenamepickled_error_filename set_statusPENDINGRESTARTreset restore_pathSKIPr add_trial)rrrNtrial resume_type trial_to_adds r_requeue_restored_trialsz'TuneController._requeue_restored_trialsds_ AA4    ) )E|u{**+3 !111+4 +6 Ll5<<<$ ;? )8CG )@'' 6666 7 ???${{}} ,0 )) 7 <<<$ &%+55!++E,<=== !F!F!FGGG+++ NN< ( ( ( (= ) )rexperiment_statecg}|dD]\}}tj|}||tj|j}|jj|_|jj|_|jj|_| |tj j j s||||djj|j_|S)Nrr)r'from_json_staterestore_run_metadatacopyrTrstorage_filesystemstorage_fs_pathexperiment_dir_name set_storagerayutilclient is_connectedinit_local_pathappend _timestamp)rr*rtrial_json_statetrial_runtime_metadatar& new_storages r_restore_trialszTuneController._restore_trialss8H8V ! ! 4 4)*:;;E  & &'= > > > )EM22K-1]-MK **.-*GK '.2m.OK +   k * * * 8?&3355 (%%''' MM% $*!9#4#?   rctt|jj|jj}|*t d|jjdt jdtdt|j |jj |5}tj |t}dddn #1swxYwY||d||}|j|jj}|j|r|j||j|r|j||||dS) zResumes all checkpointed trials from previous run. Requires user to manually re-register their objects. Also stops all ongoing trials. )fsNz+Tried to resume experiment from directory 'z-', but no experiment state file of the form 'zD' was found. This is expected if you are launching a new experiment.z9Restoring the run from the latest experiment state file: rr)r#rrr/rrHrrrrnameopen_input_streamrloadsreadallrC __setstate__r=rsync_down_experiment_staterrhas_checkpointrestore_from_dirr can_restorer))rrNnewest_state_pathrr*rdriver_staging_dirs rrzTuneController.resumes? M ,1Q     $UM4UU6D6SUUU    .%&&+ . .   ] - ? ?@Q R R PVW#z!))++;NOOO  P P P P P P P P P P P P P P P *=9:::%%&677  ;;===!]I   * *+= > > B   - -.@ A A A ? & &'9 : : A O , ,-? @ @ @ %%fm<<<<K|]}|VdSr is_finished.0r&s r z-TuneController.is_finished..s.FF55$$&&FFFFFFrc3>K|]}|VdSrrVrXs rrZz-TuneController.is_finished..s.@@%%##%%@@@@@@r)lenrallrrrW)r trials_dones rrWzTuneController.is_finisheds ! " "a ' GFFD4EFFFFFA@@4<@@@@@ =t/;;===rcBfd|jD}|r|dndS)Nc*g|]}|jk |Srtrial_id)rYrtids r z,TuneController.get_trial..s%>>>qAJ#,=,=,=,=,=rrr)rrcr&s ` r get_trialzTuneController.get_trials1>>>>DL>>> *uQxxd*rc|jS)zReturns the list of trials managed by this TrialRunner. Note that the caller usually should not mutate trial state directly. rers rrzTuneController.get_trialss |rc|jS)zAReturns the set of trials that are not in Trial.TERMINATED state.)rrs rrzTuneController.get_live_trialss   rr&c |jr||j||j||jt jkr|j |td5|j | |dddn #1swxYwY||td|d|jt j|jt j|jt j|jt j|jt j|ji}||j ||jt jkrA|j||j|j |dSdS)zAdds a new trial to this TrialRunner. Trials may be added at any time. Args: trial: Trial to queue. scheduler.on_trial_addNz Adding trial z with status )rresolve_config_placeholderscreate_placement_group_factoryrr8rr'rraddr=r on_trial_addr_mark_trial_to_checkpointrrr rpRUNNINGrrPAUSEDrsrtrrurqroplacement_group_factory)rr&status_str_maps rr%zTuneController.add_trials  & K  - -d.I J J J ,,... E""" <5+ + +   ! !% ( ( ( 2 3 3 E E   , ,T]]__e D D D E E E E E E E E E E E E E E E &&u--- GUGGGGHHH M4/ M4/ L$-  d2 K,   u|$((/// <5= ( (  % , ,U 3 3 3  -e.K L P PQV W W W W W ) (s.CC C Xblockingtimeoutc|j}|r|stj}|s|stj|z |krwtd|j}tjd|s.|stj|z |kw|r||dSdS)aAdds next trials to queue if possible. Note that the timeout is currently unexposed to the user. Args: blocking: Blocks until either a trial is available or is_finished (timeout or search algorithm finishes). timeout: Seconds before blocking times out. Returns: Boolean indicating if a new trial was created or not. zBlocking for next trial...r`TF)r next_trialrrWrrsleepr%)rrurvr&starts r_update_trial_queuez"TuneController._update_trial_queue+s ++--  E IKKE  "&"2"2"4"4 9=u9Lw9V9V 9:::(3355 1  "&"2"2"4"4 9=u9Lw9V9V   NN5 ! ! !4urch|j}|j|Sr)riget_live_actors_resourcesrk debug_string)rallocated_resourcess r_used_resources_stringz%TuneController._used_resources_stringKs/"1KKMM%223FGGGrc8|jdSr)rkupdate_avail_resourcesrs r on_step_beginzTuneController.on_step_beginPs 5577777rc^|d|ddS)NF force_all)_cleanup_cached_actors_cleanup_stopping_actorsrs r on_step_endzTuneController.on_step_endSs6 ##e#444 %%%66666rrc`|jr|js|jjdkrd}|j|D]_}t d||d| d| |`dS)NrTrzCleaning up cached actor:  tracked_actor) rrWrwrtotal_max_objectsflush_cached_objectsrr set_on_stop set_on_error _remove_actor)rrrs rrz%TuneController._cleanup_cached_actorsWs   ( ( * * ' !3q88 I!.CCD   < .xs10 y .rc|dS)Nrr)items rrz9TuneController._cleanup_stopping_actors..|s ar)rrr`rrvzForcefully killing actor: T)rkillrZ)r monotonicr{rrrryitemsrriis_actor_startednextpopleftrr remove_actorpoprz)rrnowtimes_rs rrz'TuneController._cleanup_stopping_actorsks"n d33t7RRR F 484I4O4O4Q4Q)(       5  5))E!HQK7$:UUU  58A;.1RRR%66U1Xa[6QQS#(((333$}} A}D$999"33-3PP Y I-IIJJJ#00}SW0XXX  ! % %m 4 4 4' 5  5))E!HQK7$:UUU(  9,1!HQKD ) ) ),1%LLD ) ) )rcT|rtdtd5|dddn #1swxYwYtd5|j|j|jdddn #1swxYwY|||j ds8|j j s,|j || |n>#t"$r1}t$dt)||d}~wwxYw|xjdz c_td 5|dddn #1swxYwYtd 5|j|j|jddddS#1swxYwYdS) Nz%Called step when all trials finished?rzcallbacks.on_step_begin) iterationrg?rz'Trial controller checkpointing failed: r`rzcallbacks.on_step_end)rWrr=rrrr_maybe_update_trial_queue_maybe_add_actorsrirnum_live_actorsron_no_available_trialsr_stop_experiment_if_neededr rrwarningrr)rrs rstepzTuneController.stepsE      ECDD D / * * ! !     ! ! ! ! ! ! ! ! ! ! ! ! ! ! !3 4 4   O ) )/$, *                   &&(((    "'''44 &6 4KKOO%% '')))  OO        NNMSVVMM N N NG  1 - ( (                        1 2 2 X X O ' '$/$, ' W W W X X X X X X X X X X X X X X X X X Xs_AAA.'B!!B%(B%<E F ,FF .GGG)'HH!$H!rc 2|j}||kr#td|d|ddStj|jtj|jtj|j tj |j tj |j i}td|d|d||||vs J||f|||vs J||f|||||||tjkr@|j||j|j|n%|j|j|||dS)aSet trial to a specific status. This will keep track of trials with specific statuses in sets. For PENDING and PAUSED trials we also keep a list of trials to be able to retain FIFO ordering. See ``_maybe_add_actors`` for details. Lastly we also keep a mapping from resources to pending/paused trials to be able to efficiently start trials for cached actors. Trial z already has status z. Skipping update.NzSetting status for trial z from z to )rrrr'r rprprrrqrsrrtrruremovermrqr8rorrdiscardr)rr&rcurrent_statusrss rrz TuneController._set_trial_statuss V # # LLW%WWVWWW X X X F M4/ M4/ L$-  d2 K,    Q Q Q^ Q Q Q Q   ~66668O666N62222UFO222~&--e444v""5))) U] " "  % , ,U 3 3 3  -e.K L P PQV W W W W  -e.K L T T         rc|jD]#}||j|j<$|j|jSr)rget_json_staterrbclearrr&s rrz%TuneController._get_trial_checkpointssR* J JE383G3G3I3ID  0 0 ##%%%##rc:|j|dSr)rrmrs rroz(TuneController._mark_trial_to_checkpoints !!%(((((rc|jrdS|jp |jp|j}t |j|jkr:|| sdSd}t |j|jk8dSdS)z!Ask the searcher for more trials.N)ruT)rrWrprrrsr\rr{)rdont_wait_for_trials rrz(TuneController._maybe_update_trial_queues   ' ' ) )  F  OD$8 OD  n   + + ; ,   T47D !- 0,/0Ms,S,SD ) ) )  T Trc*td5j}dddn #1swxYwY|rt d|jrt d|d|jvr|j vrt d| |tj j |j|j|nHt d|jrt d|d|d t&tffd }|j_jjrjD]}j|sj|s+j|}t d ||s!j| ||jvr9j |j|jdSdS) a;Add actors for pending and paused trials. For actors that have not been staged, yet, we request an actor. For actors that have been staged, already, we try to reuse a cached actor. First, we handle the trial that the scheduler chooses to run. Then, we handle all trials that are pending. Lastly, we see if we have cached actors that we can assign to a pending or paused trial. This can be the case when a trial has not been staged, yet, for instance because the number of staging trials was too large. choose_trial_to_runNtrial_to_run_chosenz#Chose trial to run from scheduler: z [dedup]zStaging trial to run: trial_to_run_reusez)Trying to re-use actor for trial to run: candidatescg}|rjjjkrn|d}|jvr7|jvr||V|jvr|ut d|j |j |j||||zS)Nrz%Scheduling actor for enqueued trial: )rinum_pending_actorsrrrprmr8rw_maybe_reuse_cached_actorrrrmr increase_maxrr_schedule_trial_actor)rnew_candidatesr&rs rrz;TuneController._maybe_add_actors.._maybe_add_actorshs N 2&9T=UUU"q))  444D000"))%000D///225999 LULLMMM#''...!..u/LMMM**5111/ 22"J. .rz+Trying to re-use actor for enqueued trial: )r=rrrr?rbrrrwrmrr'r rmrrrrrrr rqnum_cached_objectsrohas_cached_objectr)r trial_to_runrresource start_trials` rrz TuneController._maybe_add_actors9s-$/ 0 0 T T.BB4==??SSL T T T T T T T T T T T T T T T  =0,2GHH  P,PPPD$777 (<<< DlDDEEE&&|U]CCC#'' 555!..|/STTT**<88883\5JKKLL#L###..|<<< /$u+ / / / / / /<%6$5d6O$P$P!   /  =  8B(::8DD"?IMMOO  O+OO55kBB5h?CCKPPPP"$*===+// <<<)66'?#    s-A  AAc||jvrdS|j}|j|sdS|j|}t d|d|||jvrj|j|}|j |t d|d|| |||j|<||j |<|j j |d}| ||||j|jdS)zMaybe reuse a cached actor for a trial. If an actor has been scheduled for the trial already, this will remove the original actor. TFzReusing ACTOR for trial : z"Removing ORIGINAL ACTOR for trial rr)rvrrrrpop_cached_objectrrrmrrlrri$_live_actors_to_ray_actors_resources set_ray_actor_schedule_trial_resetconfigexperiment_tag)rr&resource_request cached_actororiginal_actor ray_actors rrz(TuneController._maybe_reuse_cached_actors] D* * *4 8 223CDD 5(::;KLL  GGGGGHHH D( ( (!155e<',++ :M+8D  '27D  / 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8   :U : :m : :7 : :     sA FF #F cj||jvr;|j||j|jdS|j}d}|jD]}|j}||kr|}n|r;|j||j|jdSt d)z;Unstage trial, or one with the same resources as ``trial``.NzStarted a trial with resources requested by a different trial, but this trial was lost. This is an error in Ray Tune's execution logic. Please raise a GitHub issue at https://github.com/ray-project/ray/issues)rwrr decrease_maxrrr)rr&rcandidate_trial staged_trialstaged_resourcess r_unstage_trial_with_resourcesz,TuneController._unstage_trial_with_resourcess D' ' '   & &u - - -   * *5+H I I I F!8 /  L+C #333".4     & & 7 7 7   * *?+R S S S F 8   rc|jsdS|jr'|js td|ddS|j|}|j|r#|j |s ||j vr"td|d|dS|j |j |s td|ddStd|d||j|}|j||d d S) aCache trial actor for reuse, if needed. We will only cache as many actors as are needed to fulfill any pending resource requests for actors with the same resource requirements. E.g. if we have 6 running trials and 4 additional staged actors, we will only cache up to 4 of the running trial actors when they finish. One exception is the case when we have no cached actors, yet. In that case, we will always cache the actor in this method. Later, in `_cleanup_cached_actors`, we will check again if we need this cached actor. That method will keep the actor if we don't have any staged trials, because we don't know at that point if the next trial might require the same resources. But because there is no staged trial, it is safe to keep the actor around, as it won't occupy resources needed by another trial until it's staged. FzNot caching actor of trial z5 as the search is over and no more trials are staged.z" as it has not been started, yet: zCould not cache actor of trial zC for reuse, as there are no pending trials requiring its resources.zCaching actor of trial z for re-use: NT)rrrWrwrrrmriris_actor_failedrxr cache_objectrrrrlr)rr&rs r_maybe_cache_trial_actorz'TuneController._maybe_cache_trial_actor"s"! 5   ' ' ) ) $2E  LL2e222   5,U3 #44]CC "22=AA D$888 LL#e## ##   5 --  )=    LL+%+++    5 RuRR=RRSSS,0077    /// D!!!trSTARTEDlogc.|j||j|}td|d|d||||jj|d}|||j |j |j || |tj||||s||dSdS)NzActor for trial rrrrr&)rxrmrlrrrrirrron_trial_startrrrr'rpro_schedule_trial_restore_schedule_trial_train)rrrr&rs rrzTuneController._actor_started]s(   ///$]3 FcFFeFF}FFGGG **5111'L    I&&& &&odl% '    uem444 &&u---++E22 .  & &u - - - - - . .rc||jvri|j|}td|d||j||dtd||j|d|j|dS)NzActor STOPPED for trial rzActor STOPPED: ) rlrrrrmrryrxr)rrr&s rrzTuneController._actor_stoppedvs D0 0 0(,,];;E LLLELL]LL M M M  $ $U + + +    % % % 6}66777 !!-666 $$]33333rrcF|j|}td|d|d|||j|jzvrj||t jtd|d||| |||j || d| d|j |d||dS) NzActor FAILED for trial rz . Exception: rzG failed in its creation task. Unstaging to allow it to be re-scheduled.rF)r)rlrrrprsrr'rpr_trial_task_failurericlear_actor_task_futuresrrrr)rrrr&s rrzTuneController._actor_failedsS$]3  &e & &} & &# & &   T)D,?? @ @  " "5%- 8 8 8 LL3333     . .u 5 5 5  $ $Ui $ @ @ @ 44]CCC !!$'''""4((( ((U(CCC M*****r method_nameargsr on_resultrrc j}d} d} |p t}|pi}rdtffd } rdtdtffd } tddt5j |||| | |} |r| cdddS ddddS#1swxYwYdS) a7Schedule an actor task future for a trial. This is a wrapper around ``ActorManager.schedule_actor_task``. This method retrieves the tracked actor for a trial to kick off the task. It also wraps around the callbacks, retrieving the trial object given the tracked actor. Nrc j|ksJtddd|d| g|Ri|dS#t$ru}tddd||t usjjkr|t tj d}~wwxYw)NFuture z RESOLVED for trial rz, Error handling z result for trial ) rlrrrrrrrI traceback format_exc)rr rrr r rr&s r _on_resultz7TuneController._schedule_trial_task.._on_resultsF 4] CCCCC (k//11((u((((%(( @Ie5d555f55555 @@@LL2+*;*;*=*=22%*22./22I~~DJ)F)F' (<(>(>???@s A C&A0CCrc "|jvr-t|tsJt|nj|ksJtddd| |dS#t$ru}tddd||tusj j kr|ttj d}~wwxYw)Nrz FAILED for trial rrz failure for trial ) rlrrtyperrrrrrrIrr)rrrr rrr&s r _on_errorz6TuneController._schedule_trial_task.._on_errors` (<<<%i??PPiPP?P D$8$GGGGG #k//11##U## ## @HUI..... @@@LL2+*;*;*=*=22%*22./22I~~DJ)F)F' (<(>(>???@s B DA0D  Drz SCHEDULED for trial )rr r rr rr) rmtuplerrrrrr(rir) rr&r r rr rrrrrfutures ``` `` r_schedule_trial_taskz#TuneController._schedule_trial_tasks&,U3   uww2  @ @, @ @ @ @ @ @ @ @ @$  @ @ @) @ @ @ @ @ @ @ @ @.  P{0022PPPPQQQ &u - -  (<<+'$"-=F                            s%CCCc|j|j|}|tjurdS|tjus|tjur||j|j<dSdSr)r setdefaultrbr6STOPPAUSE)rr&decision old_decisions r_queue_decisionzTuneController._queue_decisionsl3>>u~xXX  >. . . F ~* * *h.:N.N.N;CD ( 8 8 8/O.Nrr after_savecZ|tjkr||dS|tjkr||| dS|tjkr||dS|tjkrdStd |)zExecutes action based on decision. Args: trial: Trial to act on. decision: Scheduling decision to undertake. should_checkpointzInvalid decision: {}N) r6CONTINUErrrrrNOOPrr)rr&rr!s r_execute_actionzTuneController._execute_actions ~. . .  & &u - - - - - - - -   U*n  E E E E E , , , OOE " " " " " , , , D3::8DDEE Erc|j|jd}|r:td|d|||||dSdS)Nz$Executing final queued decision for rr!)rrrbrrr')rr&r!final_decisions r_maybe_execute_queued_decisionz-TuneController._maybe_execute_queued_decisions599%.$OO  O LLPuPPPP     : N N N N N  O Orcjoj}js |sjr.jfdjDdSdS)zStops all trials.cxg|]6}|jtjtjhv!|7Sr)rr'rrr)rYrrs rrdz=TuneController._stop_experiment_if_needed..&sH   8EK1A#BBB))!,,BBBrN)rrrstop_allrr set_finishedr)rrOs` rrz)TuneController._stop_experiment_if_needed!sO9(9 = ! ! # # y D4P    ) ) + + +            rc|j|jkr||jrtd|||||dS)NzTrial task failed for trial )exc_infor)rrIrrr_process_trial_failurerr&rs rrz"TuneController._trial_task_failure.s` ?dj ( (O' Y CECCi XXX  ' ' ' C C C C CrcFd|_|||jtjkrT|r@||||j|j |j |dS|jtjtj hvr||j |||j|jd||||j |j |j |dSdS)zHandle trial failure. Attempt trial recovery if possible, clean up state otherwise. Args: trial: Failed trial. exception: Exception prior to invoking this method. T)excr)rrN)rrrr'rpshould_recover _try_recoverron_trial_recoverrrr ron_trial_errorron_trial_completerbrr3s rr2z%TuneController._process_trial_failure6s1! 9%%% <5= ( (U-A-A-C-C (   e  3 3 3 O , ,/$,e -     \emU]; ; ;   . .tU ; ; ;   . .u~T . J J J  % %ey % A A A O * */$,e +      < ;rc|jtjkrtd|dStd||jr7td|dt j|j|j <d|j _ d|j _ | ||r tjn tj|t!||jvrtd|dS|j|}|j||||s||rdStd|d||j|}|j||d||dS) Nz2Not requesting trial STOP as it is ERROR already: z#Requesting to STOP actor for trial rzB is currently saving/pausing. Scheduling STOP after save resolved.z-Will not STOP trial actor as it is not live: rzTerminating actor for trial r)rr'rrr is_savingr6rrrbtemporary_state saving_torestoring_fromrrrr*rmrir rorrrlrr)rr&rrs rrz#TuneController._schedule_trial_stopRs <5; & & LLUeUU V V V F B5BBCCC ? O LL""""   >>%-!>>>#33D%@@@ 225>BBBB..#55%e.?!@!@ 22N<8I+J+J3 O - -/$,e .     . .u 5 5 5   % %e , , , , ,      > F F F$*,,!Y'' ++EQ+????++.y/C/E/EFF@????? sE D$E G1B G,,G1ct|||jdkr||dSdS)Nr)_schedule_trial_exportrrrs rrDz,TuneController._schedule_graceful_trial_stopsE ##E*** <7 " "  % %e , , , , , # "rTr$c(||jvr td|ddS|r1tj|j|j<||dS||| |tj dS)Nz Trial PAUSE requested for trial ) but trial is already stopping. Ignoring.r) rmrrr6rrrbrrrr'rqrr&r$s r_schedule_trial_pausez$TuneController._schedule_trial_pauses , , , LL'5'''    F  8;I;OD ( 8  % %E % 2 2 2 2 2  % %e , , ,  " "5%, 7 7 7 7 7rcd}d}||\}}|dkrd}||f}td|d||||||j|jdS)Nrtrainr`train_bufferedzScheduling future r)r&r r r r)_maybe_buffer_trainingrrrr_on_training_resultr)rr&r r  buffer_length buffer_time_ss rrz$TuneController._schedule_trial_trains '+'B'B5'I'I$ } 1  *K!=1D Q+*;*;*=*=QQ%QQRRR !!#.- "     rcFt|jt|j|jjdz}|j}|dkr4|jr-tdrt dd|fS|dkr"|j dkrt||j |fS||fS)Nr]r` trial_executor_buffer_checkpointz[Disabling buffered training as you passed `checkpoint_at_end` to `tune.CheckpointConfig()`.r) maxrrrrinum_actor_tasksrcheckpoint_at_endrFrrcheckpoint_freq)rr&rQrPs rrNz%TuneController._maybe_buffer_trainings  # ')<)LPR)R S S  + 1  !8 :;; Hm# # 1  !6!:!:}e&;<>MMOOUK                ~* * * MMtM $ $ $ $:;; N N 00MMM N N N N N N N N N N N N N N N  29::  //"o<! 0                &zz*;UCC   $ $V , , ,  * *5 1 1 1 ~+ + +  , ,U:J , K K K ?  LLPUPPhPP Q Q Q/33ENCC Hx$#THH@H,U^<4   1 1 1Os69/D44D8;D85!F""F&)F&>)G33G7:G7c0ttjdddkr^t dDdkrB|jt kr|jnd}|jjt kr |jjnd}|j jt kr |j jnd}t|tr|g}|r |vr|}d}n|r |vr|}t|jj }nv|rptfd|DrUtt!fd|}t |dkr|d}t|j j }nd}d}|r(t#d ||dSdSdS) af Check if any of the required metrics was not reported in the last result. If the only items are ``done`` or any of DEBUG_METRICS, this means that no result was ever received and the trial just returned. This is also okay and will not raise an error. This will ignore checking for the DEFAULT_METRIC. #TUNE_DISABLE_STRICT_METRIC_CHECKINGrr`cPh|]#}|tttgzv!|$Sr)rr-r/)rYks r z:TuneController._validate_result_metrics..is1LLLqatM/B/BdV/K&K&K&K&K&KrNztune.TuneConfig()c3 K|]}|vV dSrr)rY search_metricrAs rrZz:TuneController._validate_result_metrics..s9((0= V+((((((rc |vSrr)rurAs rrz9TuneController._validate_result_metrics..s m6.IrzTrial returned a result which did not include the specified metric(s) `{}` that `{}` expects. Make sure your calls to `tune.report()` include the metric, or set the TUNE_DISABLE_STRICT_METRIC_CHECKING environment variable to 1. Result: {})r|r}r~rr\rr.rrRrrrr__name__anyrfilterrr)rrA base_metricscheduler_metricsearch_metrics report_metriclocations ` rrgz'TuneController._validate_result_metrics^s rz~~CQGG H HA M M LLFLLL M MPQ Q Q*.,.*H*H$,,dK&-??#** #*n<< ''  .#.. 2"0!1 {&88 + .! &6f&D&D 0  344= C((((AO(((%% !%IIII&!! }%%**$1!$4M 011: $   < =CF%x==   S N M Q QN  rc||jvr td|ddS|p|j}||d|j|jd}t||j_ |jj S)NzTrial SAVE requested for trial rHrT)r&r r rr) rmrrrCr_on_saving_resultrrr=r>)rr&rArs rrz#TuneController._schedule_trial_saves , , , LL'%'''   4,5,**,- +  +@*G*G' $..rcheckpoint_valuecJtd5|||dddn #1swxYwYtd5|j|j|j|dddn #1swxYwY||ddS)Nprocess_trial_savezcallbacks.on_trial_saverTr))r=_process_trial_saver on_trial_saverrr+)rr&rs rrz TuneController._on_saving_results. . / / > >  $ $U,< = = = > > > > > > > > > > > > > > >3 4 4   O ) )/$,e *                   ++Ed+CCCCCs377 (BBBctd| |jstd|n |j|j|j||jn)#t$rtdwxYw|||j || |n,#t$rt d||YnwxYwd|j _|j|jd}|r|r|||dSdSdS)zProcesses a trial save. Acts on the decision cached during the last `_process_trial` call. Args: trial: Trial being saved. z Trial %s: Processing trial save.zGot empty checkpoint for trial )rrr&r aError encountered during processing of callbacks. Ray Train/Tune recently changed the checkpoint interface that is passed to callbacks. If you implemented your own callback with an `on_checkpoint` handler, please review the checkpoint interface and adjust your code accordingly.z&Trial %s: Error handling checkpoint %sN)rrr r on_checkpointrrrrron_trial_checkpointrorr=r>rrrbr )rr&rrs rrz"TuneController._process_trial_saves  7??? #. 6 FuFFGGGGO11"&/#|##3#> 2 !   NN' ##$4555(<rrr_requeue_trial)rr&r5s rr7zTuneController._try_recover(s| $((>>> ? 3.2E ! + !!%3!777 DeLLL E"""""rc|j||||tj|j|j||j||j |td5|j | |ddddS#1swxYwYdS)zNotification to TrialScheduler and requeue trial. This does not notify the SearchAlgorithm because the function evaluation is still in progress. )rrjN)rr9rr'r rrindexr8rrmr=rnrrs rrzTuneController._requeue_trial<s5 **4777 uU];;; ++E22333 E""" e$$$ 2 3 3 E E   , ,T]]__e D D D E E E E E E E E E E E E E E E E E Es2.C--C14C1c|jrt|jdkrdS||d|jfd|jd}|jj|dS)Nr export_modelT)r&r r r rr)export_formatsr\rrri_actor_task_eventsresolve_future)rr&rs rrFz%TuneController._schedule_trial_exportUs# s5+?'@'@A'E'E F **&&(- +   .==fEEEEEr new_confignew_experiment_tagc||||tj|}t ||t <|j\}}||t<||t<tt|j j }|j |||d|f||j d|j|jdS)N)logdirr")logger_creatorrTr)set_experiment_tag set_configr.deepcopyr,r4 log_to_filer3r2rr+rTtrial_working_directoryrvrmr_on_trial_resetr)rr&rr extra_config stdout_file stderr_filers rrz$TuneController._schedule_trial_resetis   !3444 $$$}Z00 #-e#4#4 Z #(#4 [$/ [!$/ [! )N    ""5))) !!"0 =*- " rsuccesscD|j||s_d}td|d|t |}|||||dS|j|}||ddS)NzQTrainable runner reuse requires reset_config() to be implemented and return True.z!Could not re-use actor for trial rrREUSED)r) rvrrrrrrrmr)rr&rrrrs rrzTuneController._on_trial_resets %%e,,, /  LLLULLdLL M M M,T22I   y ) ) )  % %ey % A A A F,U3  Mx88888rc:|j|dSr)rr8rs rrequest_stop_trialz!TuneController.request_stop_trials &&&&&rcd|_dSNT)rrs rrequest_stop_experimentz&TuneController.request_stop_experiments'+$$$rc|jr7|j}|||j5dSdSr)rrr)rrs r_process_stop_requestsz%TuneController._process_stop_requestssT  $$&&A OOA        rct|j|jd|||dS)aPause a trial and reset the necessary state variables for resuming later. Args: trial: Trial to pause. should_checkpoint: Whether or not an in-memory checkpoint should be created for this paused trial. Defaults to True. Nr#)rrrbrJrIs rrzTuneController.pause_trials? $((>>> ""5B,0"LP"!h7h7h7_-h7 (UCZ(89 h7 N+ h7 '" h7 -h7h7!c?h7DN+h7 h7"**:!;h7.)h7h7#+8B4G+H"Ih7 !h7h7h7h7T   ,X  X ##X#& +&@C& &&&&$????=C===X=sX0000X0   XXX<   D    ")5k")2>")")")")He@,=L,=,=,=,=\  HSM    <t%::; < < < < > > >+++!!!%Xu%X%X%X%XND3QU@HHHHH 888777<<<<<<(-9-9$-9-9-9-9^*X*X*XX.!u.!c.!.!.!.!`$S#X$$$$ )u)))) ' ' '&&&6 T< T T T TddddL#u#####J: 5: : : : x  5     D9e99999v..L.s....2 4L 4 4 4 4+<+I++++D!%!%<@AE$UUUUuo U  U HeS\4%789 U8UI$6$<=>UU #- UUUUn D D DFFUFcFtFFFF$OOEOtOOOO   DD9DDDDL-?@8)8)8%)8HYr?r@ray.tune.utils.object_cacherAray.tune.utils.resource_updaterrBray.tune.utils.serializationrCrDray.util.annotationsrEray.util.debugrF getLoggerrwrrHrrr|rrrrrrs  ********IIIIIIIIIIIIIIIIIIII ######......LLLLLLLLEEEEEEEE66666666NNNNNNNN666666%%%%%%44444444OOOOOOOOOO;;;;;;FEEEEE11111111                    >=======BBBBBBBB11111111------55555555DDDDDDDDDD444444<<<<<<QQQQQQQQ------######  8 $ $lElElElElElElElE^=33333333D(4(4(4(4(4(4(4(4V""C""""J" " " " " " " " " " r