&`ijddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZddlmZddlmZmZddlmZddlmZddlmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&dd l'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.e r ddl/mZ0ddl1m2Z2ej3e4Z5GddZ6de de de7dee7dee dee e ff dZ8de9de de de9fdZ:de de9dee;ede d e de fd!Z?e,Gd"d#e Z@e,Gd$d%e ZAdS)&N)Path) TYPE_CHECKINGCallableDictListOptionalTupleUnion)TRAINING_ITERATION)_FutureTrainingResult_TrainingResult) Checkpoint) TuneError)Trial)DEFAULT_METRIC) FIFOSchedulerTrialScheduler)SearchGenerator)DomainFunction) format_vars)SafeFallbackEncoder) PublicAPI)log_once)TuneControllerc*eZdZdZdefdZdefdZdS)_PBTTrialStatez%Internal PBT state tracked per-trial.trialcd|j|_d|_d|_d|_d|_d|_dSNr)experiment_tagorig_tag last_scorelast_checkpointlast_perturbation_timelast_train_time last_result)selfrs k/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/schedulers/pbt.py__init__z_PBTTrialState.__init__$s>, .2TX+,#$%  returnc|jjdzdd|jDzdzS)N(z, c32K|]\}}|dv |d|VdS))r#r$r&r%=N).0kvs r) z*_PBTTrialState.__repr__..3sV  Aq q   r+)) __class____name__join__dict__itemsr(s r)__repr__z_PBTTrialState.__repr__.sc N # ii   M//11        r+N)r8 __module__ __qualname____doc__rr*strr=r1r+r)rr!sP//e #      r+rconfig mutationsresample_probabilityperturbation_factorscustom_explore_fnr,ci}tj|}|D]>\}}t|trRt ||||||d\} } ||| i||| imt|ttfrtj |ks |||vrtj |||<d||<tj ddg} | ||} | | z} tt| dt|dz } || ||<d| dkrdnd | | krd nd ||<Rt|tt frtj |kr=t|tr|dn |||<d||<n*tj |}|||z||<d |||<t||t$rt%||||<!t'd t)||r||}| Jd||fS)aTReturn a perturbed config and string descriptors of the operations performed on the original config to produce the new config. Args: config: Original hyperparameter configuration. mutations: Specification of mutations to perform as documented in the PopulationBasedTraining scheduler. resample_probability: Probability of allowing resampling of a particular variable. perturbation_factors: Scaling factors to choose between when mutating a continuous hyperparameter. custom_explore_fn: Custom explore function applied after built-in config perturbations. Returns: new_config: New hyperparameter configuration (after random mutations). operations: Map of hyperparams -> strings describing mutation operations performed N)rFresamplerzshift leftrightz (noop)z* z.Unsupported hyperparameter distribution type: z-Custom explore fn failed to return new config)copydeepcopyr; isinstancedict_exploreupdatelisttuplerandomchoiceindexminmaxlenrrsampleint ValueErrortype)rBrCrDrErF operations new_configkey distributionnested_new_config nested_opsshiftold_idxnew_idxperturbation_factors r)rRrRBs4Jv&&J&__..99\ lD ) )8 ,4s #$$"& --- ) z   s$56 7 7 7   sJ/ 0 0 0 0  tUm 4 4-  "666#;l22#)- "="= 3", 3 r1g..&,,VC[99!E/c'1oos>> config = { ... "a": {"b": 2, "c": 0, "d": {"e": 0.1}}, ... "f": {"g": 0.5}, ... } >>> hyperparam_mutations = { ... "a": {"b": [1, 2], "c": [-1, 0]}, ... } >>> _filter_mutated_params_from_config(config, hyperparam_mutations) == { ... "a": {"b": 2, "c": 0} ... } True Args: config: The config dict that we want to filter. hyperparam_mutations: A dict containing a subset of hyperparameters from config, used to filter the config. Returns: mutated_params: A copy of config containing only params specified in hyperparam_mutations )rPrQ"_filter_mutated_params_from_config)rBrsmutated_params param_name nested_paramss r)rurus8N < < 1 1 1  fZ($ / / <>z"$8$DM*7N: & &)/ );N: & & r+cveZdZdZ d9d ed eed eed ededeeee e e e e ffdededeeefdee dededeffd Zd eed eedefdZdddefdZdddededefdZd ed!ededefd"Zdeddd#eed$eefd%Zd&ed'eded(ed)ef d*Zded(edeeeffd+Z d:d-ed.ed/eed0edef d1Zddded(efd2Zdeeeeeffd3Zdddeefd4Zd5Z d6eedeefd7Z!defd8Z"xZ#S);PopulationBasedTrainingaImplements the Population Based Training (PBT) algorithm. https://www.deepmind.com/blog/population-based-training-of-neural-networks PBT trains a group of models (or agents) in parallel. Periodically, poorly performing models clone the state of the top performers, and a random mutation is applied to their hyperparameters in the hopes of outperforming the current top models. Unlike other hyperparameter search algorithms, PBT mutates hyperparameters during training time. This enables very fast hyperparameter discovery and also automatically discovers good annealing schedules. This Tune PBT implementation considers all trials added as part of the PBT population. If the number of trials exceeds the cluster capacity, they will be time-multiplexed as to balance training progress across the population. To run multiple trials, use `tune.TuneConfig(num_samples=)`. In {LOG_DIR}/{MY_EXPERIMENT_NAME}/, all mutations are logged in `pbt_global.txt` and individual policy perturbations are recorded in pbt_policy_{i}.txt. Tune logs: [target trial tag, clone trial tag, target trial iteration, clone trial iteration, old config, new config] on each perturbation step. Args: time_attr: The training result attr to use for comparing time. Note that you can pass in something non-temporal such as `training_iteration` as a measure of progress, the only requirement is that the attribute should increase monotonically. metric: The training result objective value attribute. Stopping procedures will use this attribute. If None but a mode was passed, the `ray.tune.result.DEFAULT_METRIC` will be used per default. mode: One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. perturbation_interval: Models will be considered for perturbation at this interval of `time_attr`. Note that perturbation incurs checkpoint overhead, so you shouldn't set this to be too frequent. burn_in_period: Models will not be considered for perturbation before this interval of `time_attr` has passed. This guarantees that models are trained for at least a certain amount of time or timesteps before being perturbed. hyperparam_mutations: Hyperparams to mutate. The format is as follows: for each key, either a list, function, or a tune search space object (tune.loguniform, tune.uniform, etc.) can be provided. A list specifies an allowed set of categorical values. A function or tune search space object specifies the distribution of a continuous parameter. You must use tune.choice, tune.uniform, tune.loguniform, etc.. Arbitrary tune.sample_from objects are not supported. A key can also hold a dict for nested hyperparameters. You must specify at least one of `hyperparam_mutations` or `custom_explore_fn`. Tune will sample the search space provided by `hyperparam_mutations` for the initial hyperparameter values if the corresponding hyperparameters are not present in a trial's initial `config`. quantile_fraction: Parameters are transferred from the top `quantile_fraction` fraction of trials to the bottom `quantile_fraction` fraction. Needs to be between 0 and 0.5. Setting it to 0 essentially implies doing no exploitation at all. resample_probability: The probability of resampling from the original distribution when applying `hyperparam_mutations`. If not resampled, the value will be perturbed by a factor chosen from `perturbation_factors` if continuous, or changed to an adjacent value if discrete. perturbation_factors: Scaling factors to choose between when mutating a continuous hyperparameter. custom_explore_fn: You can also specify a custom exploration function. This function is invoked as `f(config)` after built-in perturbations from `hyperparam_mutations` are applied, and should return `config` updated as needed. You must specify at least one of `hyperparam_mutations` or `custom_explore_fn`. log_config: Whether to log the ray config of each model to local_dir at each exploit. Allows config schedule to be reconstructed. require_attrs: Whether to require time_attr and metric to appear in result for every iteration. If True, error will be raised if these values are not present in trial result. synch: If False, will use asynchronous implementation of PBT. Trial perturbations occur every perturbation_interval for each trial independently. If True, will use synchronous implementation of PBT. Perturbations will occur only after all trials are synced at the same time_attr every perturbation_interval. Defaults to False. See Appendix A.1 here https://arxiv.org/pdf/1711.09846.pdf. .. code-block:: python import random from ray import tune from ray.tune.schedulers import PopulationBasedTraining pbt = PopulationBasedTraining( time_attr="training_iteration", metric="episode_reward_mean", mode="max", perturbation_interval=10, # every 10 `time_attr` units # (training_iterations in this case) hyperparam_mutations={ # Perturb factor1 by scaling it by 0.8 or 1.2. Resampling # resets it to a value sampled from the lambda function. "factor_1": lambda: random.uniform(0.0, 20.0), # Alternatively, use tune search space primitives. # The search space for factor_1 is equivalent to factor_2. "factor_2": tune.uniform(0.0, 20.0), # Perturb factor3 by changing it to an adjacent value, e.g. # 10 -> 1 or 10 -> 100. Resampling will choose at random. "factor_3": [1, 10, 100, 1000, 10000], # Using tune.choice is NOT equivalent to the above. # factor_4 is treated as a continuous hyperparameter. "factor_4": tune.choice([1, 10, 100, 1000, 10000]), }) tuner = tune.Tuner( trainable, tune_config=tune.TuneConfig( scheduler=pbt, num_samples=8, ), ) tuner.fit() time_total_sNN@?g333333?g?TF time_attrmetricmodeperturbation_intervalburn_in_periodrsquantile_fractionrDrErF log_config require_attrssynchc |pi}|D]c}t|tttt t fstdt|trtdd|s| std|dks|dkr"td ||dkr"td ||r|dvs Jd t ||_||_d|_|jd krd |_n|jd krd |_||_||_||_||_||_||_| |_i|_| |_| |_| |_| |_t;|j|j|_d|_d|_ dS)Nzk`hyperparam_mutation` values must be either a List, Tuple, Dict, a tune search space object, or a callable.zarbitrary tune.sample_from objects are not supported for `hyperparam_mutation` values.You must use other built in primitives liketune.uniform, tune.loguniform, etc.zZYou must specify at least one of `hyperparam_mutations` or `custom_explore_fn` to use PBT.g?rzQYou must set `quantile_fraction` to a value between 0 and0.5. Current value: '{}'zSperturbation_interval must be a positive number greater than 0. Current value: '{}')rYrZz`mode` must be 'min' or 'max'.rZ?rY)!valuesrPrQrTrUrr TypeErrorrr^rrlsuperr*_metric_mode _metric_op _time_attr_perturbation_interval_burn_in_period_hyperparam_mutations_quantile_fraction_resample_probability_perturbation_factors _trial_state_custom_explore_fn _log_config_require_attrs_synchrZ_next_perturbation_sync_num_checkpoints_num_perturbations)r(rrrrrrsrrDrErFrrrvaluer7s r)r*z PopulationBasedTraining.__init__es5$ 49r)0022  EedD%%JKK " %**  : $ ,= 5  s " "&7!&;&;++162C+D+D  !A % %..4f5J.K.K   L>)))+K)))    :  !DOO Z5 "DO#&;#-%9""3%9"%9"9;"3%+ '*  '  ( ( $ !""#r+r,c |jr|rdS|jr|rdS|r||_|r||_|jdkrd|_n|jdkrd|_|j|jr t|_dS)NFrZrrYrT)rrrr)r(rrspecs r)set_search_propertiesz-PopulationBasedTraining.set_search_propertiess < F 5 : $ 5  "!DL  DJ :  !DOO Z5 "DO < DJ )DLtr+tune_controllerrrc|jLt|jtr2td|jj|j|jr|js8td|jj|j|j |j j j }|j r.|j dkr#tdrtjdt#||j|<|jD]m}||jvrbt|dzrt,dt1|j||j||j||j|<ndS)NzFSearch algorithms cannot be used with {} schedulers. Please remove {}.z{} has been instantiated without a valid `metric` ({}) or `mode` ({}) parameter. Either pass these parameters when instantiating the scheduler, or pass them as parameters to `tune.TuneConfig()`pbt_num_to_keepzUsing `CheckpointConfig.num_to_keep <= 2` with PBT can lead to restoration problems when checkpoint are deleted too early for other trials to exploit them. If this happens, increase the value of `num_to_keep`.z-missingzNCannot find {} in config. Using search space provided by hyperparam_mutations.) search_algrPrr^rlr7r8rrr run_metadatacheckpoint_managercheckpoint_config num_to_keeprwarningswarnrrrrkrBloggerdebugrrevaluated_params)r(rrrros r) on_trial_addz$PopulationBasedTraining.on_trial_adds  % 1j  &7 7 1006N+_-G11 | 4? )*0N+T\4:**  ".AS  ) !-22*++3 M$   $2%#8#8% .3355 B BD5<''D:-..LLB U\41KD1QRRR/4|D/A&t, B Br+resultc~j|vr]dj|}jrt|dzt drt |j|vr]dj|}jrt|dzt drt |j|vs j|vr tj S|j}j }|j kr1t d|dj tj S||j z }|jkr1t d|djtj St d d ||||js||_ \} } tj } |D]/} | jt(jt(jfvrtj} n0|| | jt(jkr tjn| St5fd |Dr t d d nSt d\} } |} g}| D]}|| vr|| vr|| t d| d| d|| |z| z} | D]I}t d||j |_ ||| | Jfd|D}t;|}t;jjz|_t djjt(jkr tjn tjS)NzxCannot find time_attr {} in trial result {}. Make sure that this attribute is returned in the results of your Trainable.zgIf this error is expected, you can change this to a warning message by setting PBT(require_attrs=False)zpbt-time_attr-errorzuCannot find metric {} in trial result {}. Make sure that this attribute is returned in the results of your Trainable.zpbt-metric-errorzStill in burn-in period: z < z#Perturbation interval not reached: zUpdating trial state for trial z at time c3ZK|]%}j|jjko|kV&dSN)rr&r)r2tr(rs r)r5z:PopulationBasedTraining.on_trial_result..LsX!!$4t7SSJr+z?Sync: Other trials are not at perturb time, yet. Pausing trial z to wait.z%Sync: All trials are at perturb time.z!Trial statistics Upper quantile: z Lower quantile: z Not in quantile: zPerturbing trial c4g|]}j|jSr1rr&)r2rr(s r) z;PopulationBasedTraining.on_trial_result..qs4###%a(8###r+zNext perturb at time )rrlr RuntimeErrorrrwarningrrCONTINUErrrr%r_save_trial_stater _quantiles get_trialsstatusrPENDINGPAUSEDPAUSE_checkpoint_or_exploitNOOPanyget_live_trialsappendrZr)r(rrrtime_missing_msgmetric_missing_msgtimestatetime_since_perturblower_quantileupper_quantiledecision other_trial all_trialsnot_in_quantilerall_train_timesmax_last_train_times` ` r)on_trial_resultz'PopulationBasedTraining.on_trial_results- ?& ( (-.4VDOV-L-L  " 5"$771225NN#3444 RTT U U U!* *"E$@@  ; ; ; LLH%HH*.*EHH   "* * MuMMtMMNNN udFE:::{F +/E (-1__->-> *NN%.H.99;;   %%-)FFF-3HEG  ' '   +0,%,*F*F>&&H T)88::. U  6%*666  DEEE151B1B.,7799 "$#22A..1N3J3J'..q111 :'5::'5::)8::,o=N #ALL!8Q!8!8999BFD%a(?//?NN####,7799###'*/&:&:#/2043NN'00, ST5QSSTTT<5<//###) r+rrcZ|j||jz}||_||_||_|S)aSaves necessary trial information when result is received. Args: state: The state object for the trial. time: The current timestep of the trial. result: The trial's result dictionary. trial: The trial object. )rrr#r&r')r(rrrrscores r)rz)PopulationBasedTraining._save_trial_states4&"66  $" r+rrcP|j|}||vrtd|d|jtjkr|jjrOt|jjtr0td|d|jj|_ nrtd|d|j d|j |_ n?td|d| ||j |_ |xjd z c_nd |_ ||vrtj|}||usJ|j|}|j }td|d |dt|trV|} | r | j|_ | j |_ |j }n td |d d }|s"td|d|d S||||d Sd S)z6Checkpoint if in upper quantile, exploits if in lower.zTrial z) is in upper quantile. Saving checkpoint.z is still saving.z* is paused. Use last available checkpoint .z Instructing z to save.rrJNz( is in lower quantile. Exploiting trial z6PBT-scheduled checkpoint save resolved to None. Trial z: didn't save any checkpoint before and can't be exploited.z[pbt]: no checkpoint for trial z. Skip exploit for Trial )rrrrrrtemporary_state saving_torPr r$ checkpoint_schedule_trial_saver'rrVrWresolvemetricsinfo_exploit) r(rrrrrtrial_to_clone clone_stater$training_results r)rz.PopulationBasedTraining._checkpoint_or_exploits!%( N " " LLR%RRR S S S|u|++(2 =z)35J88 =LL!B%!B!B!BCCC,1,A,KE))LL:::&+&6:::-2,::N....+N;K)9O LL666$2666    /+@AA +"1"9"9";";" +.=.EK+2A2LK/&1&AOOLL3)333 '+O"  7n77/477 MM/5. A A A A AA # "r+ trial_state new_staterrac|j|j}}|j}|j} tj|jd|zdz} tj|jd| zdz} |||jtd|jtd|j |g} ttj|jdd5} ttj| t| dddn #1swxYwYtj| rt#j| | t| d5} | tj| td zddddS#1swxYwYdS) zLogs transition during exploit/exploit step. For each step, logs: [target trial tag, clone trial tag, target trial iteration, clone trial iteration, old config, new config]. pbt_policy_z.txtrzpbt_global.txtza+)cls)fileN )r"trial_idospathr9local_experiment_path local_dirr'getr rBopenprintjsondumpsrexistsshutilcopyfilewrite)r(rrrrra trial_nametrial_to_clone_namertrial_to_clone_id trial_pathtrial_to_clone_pathpolicyfs r)_log_config_on_stepz+PopulationBasedTraining._log_config_on_stepsD,7+?AS' >*3W\\  ')AF)J  !gll  $m6G&G&&P       ! !"4a 8 8  & * *+=q A A  !    GLL46F G G   G  $*V)<===A F F F F G G G G G G G G G G G G G G G 7>>- . . = O/ < < < *d # # Hq GGDJv+>???$F G G G H H H H H H H H H H H H H H H H H Hs$/+D&&D*-D*52F44F8;F8cZt|j|j|j|j|jS)a Gets new config for trial by exploring trial_to_clone's config. Args: trial: The current trial that decided to exploit trial_to_clone. trial_to_clone: The top-performing trial with a hyperparameter config that the current trial will explore by perturbing. Returns: new_config: New hyperparameter configuration (after random mutations). operations: Map of hyperparams -> strings describing mutation operations performed )rRrBrrrr)r(rrs r)_get_new_configz'PopulationBasedTraining._get_new_config s4  !  &  &  &  #    r+rM old_params new_paramsr`prefixc d}|s|S|D]}||}||vs7Jd|d|d|||}|||dz }t|tr;|dz }||i} ||||| |dzz }||d } | sd } nd | d } ||d | d |dz }|S)aGenerates a summary of hyperparameter changes from a PBT "explore" step. Example: Given the following hyperparam_mutations: hyperparam_mutations = { "a": tune.uniform(0, 1), "b": list(range(5)), "c": { "d": tune.uniform(2, 3), "e": {"f": [-1, 0, 1]}, }, } This is an example summary output of the operations performed on old_params to get new_params: a : 0.5 --- (* 0.8) --> 0.4 b : 2 --- (resample) --> 4 c : d : 2.5 --- (* 1.2) --> 3.0 e : f : 0 --- (shift right) --> 1 The summary shows the old and new hyperparameter values, with the operation used to perturb labeled in between. If the operation for a certain hyperparameter is not provided, then the summary will just contain arrows without a label. (ex: a : 0.5 -----> 0.4) Args: old_params: Old values of hyperparameters that are perturbed to generate the new config new_params: The newly generated hyperparameter config from PBT exploration operations: Map of hyperparams -> string descriptors the operations performed to generate the values in `new_params` prefix: Helper argument to format nested dict hyperparam configs Returns: summary_str: The hyperparameter change summary to print/log. rMz:`old_params` and `new_params` must both contain the key: 'z' old_params.keys() = z new_params.keys() = z : rz )r`r Nz----->z--- (z) --> )rkrPrr_summarize_hyperparam_changes) r(rr r`r  summary_strrwold_valnew_valnested_operationsoparrows r)r z5PopulationBasedTraining._summarize_hyperparam_changessw^   $ @ @J ,G+++;/9;;'1'8'8;;(2'8'8;;,++ !,G f5j555 5K'4(( @t# $.NN:r$B$B!tAA0!G+ B    ^^J55.$EE-B---E'??E??G???? r+c|j|}|j|}|jj}td|d|j|j|j|j|||\}}t|j |j } t||j } d|d|jd} | | | | |pdz } t| |j r||||||t|j||j } |jt$jkr|jst+dn||d|| ||t3j|j} t7| |j |jj_|xj d z c_ |j!|_!|j"|_"d S) zTransfers perturbed state from trial_to_clone -> trial. If specified, also logs the updated hyperparam state. z [zK] [Exploit] Cloning trial {} (score = {:4f}) into trial {} (score = {:4f}) z8] [Explore] Perturbed the hyperparameter config of trialz: zNo hyperparameters mutated.zzTrials should be paused here only if in synchronous mode. If you encounter this error please raise an issue on Ray Github.Fshould_checkpoint)rrrJN)#rr7r8rrrlrr#rrurBrr rrrnr"rrrrr pause_trialset_experiment_tag set_configrNr$r r'rr_latest_checkpoint_resultrr%r&)r(rrrrr class_namerar`rr explore_info_strnew_tagcheckpoint_to_exploits r)rz PopulationBasedTraining._exploitmss'. %n5 ^,   AJ A A AAG'$& BB   "&!5!5e^!L!L J8  !4#=  8 2   #J # #~ # # #    . .z:z R R -,   $%%%     $ $Y~z   '  *d.H   <5< ' '; <   ' ' ' G G G   ))) $$$-1Ii6O,P,P 0):O    -G 1$-6-M *&/&? ###r+cg}jD]\}}td|||r-td||j)|s|||fdt|dkrggfSttj t|j z}|t|dz kr1ttjt|dz }|d||| dfS)zReturns trials in the lower and upper `quantile` of the population. If there is not enough data to compute this, returns empty lists. zTrial {}, state {}zTrial {} is finishedNc(j|jSr)rr#)rr(s r)z4PopulationBasedTraining._quantiles..s$"3A"6"Ar+rbrJr)rr;rrrl is_finishedr#rsortr[r]mathceilrfloor)r(trialsrrnum_trials_in_quantiles` r)rz"PopulationBasedTraining._quantilesst  -3355 % %LE5 LL-44UEBB C C C  "" C 3::5AABBB+E4E4E4G4G+ e$$$ AAAA BBB v;;!  r6M%( #f++(??@@&& "&F a77),TZF a-H-H)I)I&2223V=Sj|jj kr||o| fd|r|dndS)zEnsures all trials get fair share of time (as defined by time_attr). This enables the PBT scheduler to support a greater number of concurrent trials than can fit in the cluster at any given time. c(j|jSrr)rr(s r)r!z=PopulationBasedTraining.choose_trial_to_run..s$*;E*B*Rr+r"rN) rrrrrrrrr&rr$)r(r candidatesrs` r)choose_trial_to_runz+PopulationBasedTraining.choose_trial_to_runs  $//11 - -E|   {-%%e,,,,%e,<233%%e,,,RRRRSSS *4z!}}4r+c"d|_d|_dSr )rrr<s r) reset_statsz#PopulationBasedTraining.reset_statss"# !r+r(cg}|D]D}|j|}|j.|s||jE|Sr)rr#r#r)r(r(scoresrrs r) last_scoresz#PopulationBasedTraining.last_scoressX 0 0E%e,E+E4E4E4G4G+ e./// r+cBd|j|jS)Nz4PopulationBasedTraining: {} checkpoints, {} perturbs)rlrrr<s r) debug_stringz$PopulationBasedTraining.debug_strings%ELL  !4#:   r+) r{NNr|r}Nr~r~rNTTF)NrM)$r8r>r?r@rArfloatrr rQrTrUrrr boolr*rrrrrr]rrrrrr rrr-r/r2r4 __classcell__)r7s@r)rzrzs2yyz( $"'+ # #'&*4>04"!R$R$R$ R$sm R$ % R$  R$# tT5(F:; ; R$!R$$R$$E5L1R$$H-R$R$R$ !R$R$R$R$R$R$hsm+3C= 0/B,</BU/B/B/B/BbB/B8=BGKB BBBBH#+.8<EJ(CBCB*CBU CB U CBCBCBCBJ(H#(H"(H (H  (H  (H(H(H(HT U E eDRVJFW    2&* MMMMTN M  M MMMM^N@)N@N@ N@N@N@N@`WE$u+tE{":;WWWW253C5QV5555.""" $u+$u+ c        r+rzc eZdZdZdefdZdedeeeee efffdZ ddde fd Z ddde d edefd Z defd Zd S)PopulationBasedTrainingReplayaXReplays a Population Based Training run. Population Based Training does not return a single hyperparameter configuration, but rather a schedule of configurations. For instance, PBT might discover that a larger learning rate leads to good results in the first training iterations, but that a smaller learning rate is preferable later. This scheduler enables replaying these parameter schedules from a finished PBT run. This requires that population based training has been run with ``log_config=True``, which is the default setting. The scheduler will only accept and train a single trial. It will start with the initial config of the existing trial and update the config according to the schedule. Args: policy_file: The PBT policy file. Usually this is stored in ``~/ray_results/experiment_name/pbt_policy_xxx.txt`` where ``xxx`` is the trial ID. Example: .. code-block:: python # Replaying a result from ray.tune.examples.pbt_convnet_example from ray import tune from ray.tune.examples.pbt_convnet_example import PytorchTrainable from ray.tune.schedulers import PopulationBasedTrainingReplay replay = PopulationBasedTrainingReplay( "~/ray_results/pbt_test/pbt_policy_XXXXX_00001.txt") tuner = tune.Tuner( PytorchTrainable, run_config=tune.RunConfig( stop={"training_iteration": 100} ), tune_config=tune.TuneConfig( scheduler=replay, ), ) tuner.fit() policy_filec|t|}|s4td|||_||j\}|_dtj |j|_ ||_ |j |_d|_d|_d|_t%|j|_t)|jd|_dS)NzPolicy file not found: {}z replay_{}r)r expanduserrr^rlas_posixr: _load_policy_policyrrbasenamer!rBcurrent_config_trial _current_stepriter _policy_iternext _next_policy)r(r:initial_configs r)r*z&PopulationBasedTrainingReplay.__init__2s ;''2244 !!## Y8?? @T@T@V@VWWXX X&//11(,'8'89I'J'J$ )001A1A$BR1S1STT$ "k "# .. !2D99r+r,c&g}t|d5}|D]p} tj|}n6#tj$r$t d|dwxYw|t|q dddn #1swxYwYg}d}d}t|D].\} } } } } }|r| |krn| }| }|| |f/|tt|fS)Nrtz#Could not read PBT policy file: {}.) r readlinesrloadsJSONDecodeErrorr^rlrrUreversedrT)r(r: raw_policyfprow parsed_rowr last_new_tag last_old_confold_tagrold_stepnew_stepold_confnew_confs r)r>z*PopulationBasedTrainingReplay._load_policyGs +t $ $ 5||~~ 5 5 !%CJJ+   $=DD[QQ  !!% "3"34444 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5  HP I I 0 0 DGWh(H < 7 7"L$M MM8X. / / / /d8F#3#34444s'B&AB&3A33&B&&B*-B*rrrch|jrtd||_|jjr"|jrtdnG|jjr|js|jj|_n"|jjs|jstd|j|jdS)NzMore than one trial added to PBT replay run. This means the same schedule will be trained multiple times. Do you want to set `n_samples=1`?zsTrial was initialized with a config, which was overwritten. Did you start the PBT replay with a `config` parameter?zNo replay policy found and trial initialized without a valid config. Either pass a `config` argument to `tune.Tuner()`or consider not using PBT replay for this run.)rBr^rBr?rrr)r(rrs r)rz*PopulationBasedTrainingReplay.on_trial_addes ; ;   ;  $,  NNJ    [    +,DKK# DL A  t{+++++r+rct|vr tjS|js tjS|t}||_|j\}}||kr tjSt d|||||}| }||j j _ t|j||}||d||||||_|xjdz c_t)|jd|_tjS)NzXPopulation Based Training replay is now at step {}. Configuration will be changed to {}.rFrrJ)r rrrGrCrrrlrrrrrrnr!rrrrArrFrEr) r(rrrstep change_atrarrs r)rz-PopulationBasedTrainingReplay.on_trial_result}sU V + +!* *  +!* *()! $ 1 : )  !* *  3396$ 3K3K   !55eF5KK ..**  -G't':J SS##EU#CCC   ))) $$$( 1$ !2D99""r+cBd|j|jS)Nz3PopulationBasedTraining replay: Step {}, perturb {})rlrCrr<s r)r4z*PopulationBasedTrainingReplay.debug_strings%DKK   7   r+N)r8r>r?r@rAr*r rrr]r>rrrr4r1r+r)r9r9s..`:C::::*55dDsDyAQrss  NNNNNNNNNNNNNNNNNN000000NNNNNNNN$$$$$$%%%%%%******MMMMMMMM++++++33333333999999333333######B777777AAAAAA  8 $ $        B[" ["[" [" , ["  ) ["  4: ["["["["|K3KKK#KKKK- --+0tUHf1T+U----(( ((,( ((((V T  T  T  T  T  mT  T   T  n j j j j j Mj j  j j j r+