&`iAddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZddlZddlZddl Z!ddl"Z"ddl#m$Z$ddl%Z%ddl&m'Z(ddl)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/ddl0m1Z1dd l2m3Z3dd l4m5Z6dd l7m8Z8m9Z9m:Z:m;Z;mZ>dd l?m@Z@ddlAmBZBddlCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLddlMmNZNddlOmPZPddlQmRZRmSZSddlTmUZUmVZVddlWmXZXddlYmZZZddl[m\Z\ddl]m^Z^ddl_m`Z`ddlambZbmcZcddldmeZeddlfmgZgddlhmiZimjZjmkZkmlZlmmZmddlnmoZoddlpmqZqmrZrddlsmtZtmuZudd lvmwZwmxZxmyZydd!lzm{Z{dd"l|m}Z}m~Z~mZmZmZmZmZdd#lmZmZmZmZmZdd$lmZdd%lmZmZdd&lmZdd'lmZdd(lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd)lmZdd*lmZdd+lmZmZmZdd,lmZmZdd-lmZdd.lmZmZdd/lmZdd0lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd1lmZdd2lmZdd3lmZdd4lmZdd5lmZmZdd6lmZmZmZdd7lmZdd8lmZdd9lmZdd:lmZdd;lmZmZddlmZe\ZZZejeZeGd?d@eeZGdAdBZdS)CN) defaultdict)datetime) TYPE_CHECKINGAnyCallable Collection DefaultDictDictListOptionalSetTupleTypeUnion)version)DEPRECATED_VALUE Deprecateddeprecation_warning)TagKeyrecord_extra_usage_tag) ActorHandleAlgorithmConfig)ALGORITHMS_CLASS_TO_NAME)AggregatorActor_get_env_runner_bundles_get_learner_bundles_get_main_process_bundle _get_offline_eval_runner_bundles) make_callback)ObsPreprocessorConnector)ConnectorPipelineV2) COMPONENT_ENV_RUNNER!COMPONENT_ENV_TO_MODULE_CONNECTORCOMPONENT_EVAL_ENV_RUNNERCOMPONENT_LEARNERCOMPONENT_LEARNER_GROUPCOMPONENT_METRICS_LOGGER!COMPONENT_MODULE_TO_ENV_CONNECTORCOMPONENT_RL_MODULEDEFAULT_MODULE_ID)Columns)validate_module_id) MultiRLModuleMultiRLModuleSpec)RLModule RLModuleSpec)INPUT_ENV_SPACES) EnvContext) EnvRunner)EnvRunnerGroup)_gym_env_creator)collect_episodessummarize_episodes)synchronous_parallel_sample)get_dataset_and_shards) DirectMethod DoublyRobustImportanceSamplingOffPolicyEstimatorWeightedImportanceSampling)OfflineEvaluator)Policy PolicySpec)DEFAULT_POLICY_ID SampleBatch) FilterManager deep_update force_list)FaultTolerantActorManager) DeveloperAPIExperimentalAPI OldAPIStackOverrideToImplementCustomLogic5OverrideToImplementCustomLogic_CallToSuperRecommended PublicAPIoverride)CHECKPOINT_VERSION)CHECKPOINT_VERSION_LEARNER_AND_ENV_RUNNERCheckpointableget_checkpoint_infotry_import_msgpack)update_global_seed_if_necessary)ERR_MSG_INVALID_ENV_DESCRIPTOREnvError) try_import_tf) from_config)&AGGREGATOR_ACTOR_RESULTS ALL_MODULESDATASET_NUM_ITERS_EVALUATEDENV_RUNNER_RESULTSENV_RUNNER_SAMPLING_TIMEREPISODE_LEN_MEANEPISODE_RETURN_MEANEVALUATION_ITERATION_TIMEREVALUATION_RESULTSFAULT_TOLERANCE_STATSLEARNER_RESULTSLEARNER_UPDATE_TIMERNUM_AGENT_STEPS_SAMPLED NUM_AGENT_STEPS_SAMPLED_LIFETIME!NUM_AGENT_STEPS_SAMPLED_THIS_ITERNUM_AGENT_STEPS_TRAINED NUM_AGENT_STEPS_TRAINED_LIFETIMENUM_ENV_STEPS_SAMPLED.NUM_ENV_STEPS_SAMPLED_FOR_EVALUATION_THIS_ITERNUM_ENV_STEPS_SAMPLED_LIFETIMENUM_ENV_STEPS_SAMPLED_THIS_ITERNUM_ENV_STEPS_TRAINEDNUM_ENV_STEPS_TRAINED_LIFETIME NUM_EPISODESNUM_EPISODES_LIFETIME%NUM_TRAINING_STEP_CALLS_PER_ITERATIONOFFLINE_EVAL_RUNNER_RESULTS"OFFLINE_EVALUATION_ITERATION_TIMERRESTORE_ENV_RUNNERS_TIMERRESTORE_EVAL_ENV_RUNNERS_TIMER"RESTORE_OFFLINE_EVAL_RUNNERS_TIMERSTEPS_TRAINED_THIS_ITER_COUNTER SYNCH_ENV_CONNECTOR_STATES_TIMER%SYNCH_EVAL_ENV_CONNECTOR_STATES_TIMERSYNCH_WORKER_WEIGHTS_TIMERTIMERSTRAINING_ITERATION_TIMERTRAINING_STEP_TIMER) LEARNER_INFO) MetricsLogger)(DEFAULT_HISTOGRAM_BOUNDARIES_LONG_EVENTS)DEFAULT_HISTOGRAM_BOUNDARIES_SHORT_EVENTSTimerAndPrometheusLogger)MultiAgentReplayBuffer ReplayBuffer) RunnerGroup)NOT_SERIALIZABLEdeserialize_type) space_utils)AgentConnectorDataTypeAgentIDAgentToModuleMappingFnAlgorithmConfigDict EnvCreator EnvInfoDictEnvType EpisodeIDModuleIDPartialAlgorithmConfigDictPolicyID PolicyState ResultDictSampleBatchTypeShouldModuleBeUpdatedFn StateDictTensorStructType TensorType)DEFAULT_STORAGE_PATH) Checkpoint)PlacementGroupFactory) ExportFormat)Logger UnifiedLogger) ENV_CREATOR_global_registryget_trainable_cls) Resources)TRAINING_ITERATION) Trainable)log_once)Counter Histogram)_Timer) LearnerGroup OfflineDatac%eZdZUdZdZeeed<dZee ed<dZ ee ed<dZ ee ed<dZ eded<dZed ed <d Zgd Zd dgZddgZedeededeeeeedefZdZdZeee dddde dde!e"e#fdeddee$e%dee&e'e(ge%fdee!e$e%e&e%ee)ge*ffddf fdZ+e, ddeedee&ge-fffd Z.dZ/e0edefd Z1e0de2e3fd!Z4e5ee6deddfd"Z7e0ededee8e9fd#Z:ee6de;fd$Zj?j@de;fd'ZAdeBe;e3e3ffd(ZCd)ZDd*ZEd+ZFdd,ZGd-ZHe0de de2e3fd.ZIe0d/eJde2e3fd0ZKe0dd1ZLe,eMfd2eNdeeOfd3ZPe,dddd4d4d4d5d2eNd6eQd7eeRd8eeSd9eeTd:e*d;e*dd2eNd8eeSd9eeTd?e*d@e*dAe*dee9fdBZWeXeYfdCe%de9fdDZZe,ddee2e%de[fdEZ\e,dFeRe%e[ffdGZ]eX dddddddd4d4de e dH dCe%dIee8e9dJee9dKee^j_j`dLee^j_j`dee!eeafdMeebdee&e'e(ge%fdee!e$e%e&e%ee)ge*ffd;e*dj?j@de;fd}ZdeBe;e;dzffd~ZdZeddZeddZdZdZdZedZedZdZeXde2e"dSe"deRe"e"ffdZeXdeRfdZeXddZeXeeddddde[dee$e%dee&e'e(ge%fdee!e$e%e&e%ee)ge*ffdeRf dZeXdeadeefdZeXdZeXdZeXedd  dddddeYd dddddd deedQee2edeedeedeedeedCe%de*dee*dee3dee*dee*de!eeBee2eeRe"effffdZeXedd  ddddeYd dddddd dedQee2edeedeedeedCe%de*dee*dee3dee*dee*fdZedd4dZedd4edZedd4edZxZS) Algorithma An RLlib algorithm responsible for training one or more neural network models. You can write your own Algorithm classes by sub-classing from `Algorithm` or any of its built-in subclasses. Override the `training_step` method to implement your own algorithm logic. Find the various built-in `training_step()` methods for different algorithms in their respective [algo name].py files, for example: `ray.rllib.algorithms.dqn.dqn.py` or `ray.rllib.algorithms.impala.impala.py`. The most important API methods an Algorithm exposes are `train()` for running a single training iteration, `evaluate()` for running a single round of evaluation, `save_to_path()` for creating a checkpoint, and `restore_from_path()` for loading a state from an existing checkpoint. Nconfigmetricsenv_runner_groupeval_env_runner_groupr learner_groupr offline_dataF) tf_session_argslocal_tf_session_args env_configmodel optimizercustom_resources_per_env_runnercustom_resources_per_workerevaluation_configexploration_configreplay_buffer_config extra_python_environs_for_worker input_config output_configrroff_policy_estimation_methodspolicies/rllib_checkpoint.jsonalgorithm_state) policy_idspolicy_mapping_fnpolicies_to_train checkpointpath filesystemzpyarrow.fs.FileSystemrrrreturnc |tkrtdddt||}|dtjdkrEt |t r|}tj |fd|i|S|dtjdkrtd |dtjd krtd |dd |d dkr0|.|dthkr tj }ntdt||||} t| S)afCreates a new algorithm instance from a given checkpoint. Args: path: The path (str) to the checkpoint directory to use or a Ray Train Checkpoint instance to restore from. filesystem: PyArrow FileSystem to use to access data at the `path`. If not specified, this is inferred from the URI scheme of `path`. policy_ids: Optional list of PolicyIDs to recover. This allows users to restore an Algorithm with only a subset of the originally present Policies. policy_mapping_fn: An optional (updated) policy mapping function to use from here on. policies_to_train: An optional list of policy IDs to be trained or a callable taking PolicyID and SampleBatchType and returning a bool (trainable or not?). If None, will keep the existing setup in place. Policies, whose IDs are not in the list (or for which the callable returns False) will not be updated. Returns: The instantiated Algorithm. z)Algorithm.from_checkpoint(checkpoint=...)z#Algorithm.from_checkpoint(path=...)Toldnewerrorcheckpoint_versionz2.0r0.1a Cannot restore a v0 checkpoint using `Algorithm.from_checkpoint()`!In this case, do the following: 1) Create a new Algorithm object using your original config. 2) Call the `restore()` method of this algo object passing it your checkpoint dir or AIR Checkpoint object.z1.0z`checkpoint_info['checkpoint_version']` in `Algorithm.from_checkpoint()` must be 1.0 or later! You are using a checkpoint with version v.formatmsgpackNraYou are trying to restore a multi-agent algorithm from a `msgpack` formatted checkpoint, which do NOT store the `policy_mapping_fn` or `policies_to_train` functions! Make sure that when using the `Algorithm.from_checkpoint()` utility, you also pass the args: `policy_mapping_fn` and `policies_to_train` with your call. You might leave `policies_to_train=None` in case you would like to train all policies anyways.)checkpoint_inforrr)rrrSrVersion isinstancer to_directorysuperfrom_checkpoint ValueErrorrCrDEFAULT_POLICY_MAPPING_FNr#_checkpoint_info_to_algorithm_state from_state) clsrrrrrrkwargsrstate __class__s r/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.pyrzAlgorithm.from_checkpoint+sR ) ) ) ?9     .dJ?? / 0GOE4J4J J J$ ++ +((***577*4QQJQ&QQ Q1 2goe6L6L L LA 1 2W_U5K5K K KE+,@AEEE  8 $ 1 1!(#<05F4GGG(7(Q%%%H   ==+!// >  ##E***logger_creatorc @ t|tr|}t|trDd|vrtj|ntj|||d}nt|trd|vr||}n||}n|}t|t|s(|| }n'|| }|j t||_ |/td|dd|dd | |||||j|\|_|_t|jtr |jjn|j}d|_d|_t/d|j |_|t5jd }t;jd dt?|}t|jd|d|} t@j!"tFstAj$tFdtKj&| tF |j'rd|j'vr fd} n fd} | }tQtR|_*tQtV|_,g|_-g|_.d|_/d|_0d|_1d|_2d|_3d|_4d|_5d|_6d|_7d|_8d|_9d|_:d|_;d|_<d|_=d|_>d|_?d|_@d|_Ad|_Bd|_Cd|_Dd|_Ed|_FtjHd||d|dS)aDInitializes an Algorithm instance. Args: config: Algorithm-specific configuration object. logger_creator: Callable that creates a ray.tune.Logger object. If unspecified, a default logger is created. **kwargs: Arguments passed to the Trainable base class. classT) config_dictNzalgo = Algorithm(env='z', ...)z&algo = AlgorithmConfig().environment('z ').build()Fr)rootstats_cls_lookupz%Y-%m-%d_%H-%M-%Sz[/\\]-_exist_ok)prefixdirtypec|d}|d}|d}t||g|S)z0Creates a custom logger with the default prefix. logger_configrlogdir)r_argsr)copypoprY)rcfgrlogdir_rs rdefault_logger_creatorz2Algorithm.__init__..default_logger_creatorsT 16688C''&//C"ggh77G&3seGLLLLrc(t|dS)z1Creates a Unified logger with the default prefix.N)loggers)r)rrs rrz2Algorithm.__init__..default_logger_creator s(FFFFr)rr)Irdictget_default_configrr from_dictmerge_algorithm_configsupdate_from_dictrto_dict get_state algo_classr environmentvalidatefreeze_get_env_id_and_creatorenv_env_id env_creator__name__local_replay_bufferrrrrrtodaystrftimeresubstrosrexistsrmakedirstempfilemkdtemprrr_timersint _counters_episode_history_episodes_to_be_collectedrr_metrics_step_time(_metrics_run_one_training_iteration_time _metrics_run_one_evaluation_time'_metrics_compile_iteration_results_time_metrics_training_step_time_metrics_evaluate_time._metrics_evaluate_sync_env_runner_weights_time,_metrics_evaluate_sync_connector_states_time)_metrics_step_sync_env_runner_states_time_metrics_load_checkpoint_time_metrics_save_checkpoint_time&_metrics_callback_on_train_result_time(_metrics_callback_on_evaluate_start_time&_metrics_callback_on_evaluate_end_time"_metrics_impala_training_step_time;_metrics_impala_training_step_aggregator_preprocessing_time5_metrics_impala_training_step_learner_group_loop_time8_metrics_impala_training_step_sync_env_runner_state_time4_metrics_impala_sample_and_get_connector_states_time+_metrics_impala_training_step_input_batches0_metrics_impala_training_step_zero_input_batches/_metrics_impala_training_step_env_steps_droppedr__init__) selfrrrrdefault_config env_descrtimestrenv_descr_for_dir logdir_prefixrrrs @rr:zAlgorithm.__init__s( fd # # G!4466N.$// Ef$$#.v6666,6$($@$@*FD%%FFfd++E60A0A+66v>>FF+<&:&:;; G'889I9IJJ'2263C3C3E3EFF   $ $T F  ? 9S999LSLLL       s # # #   *.)E)E J* * & d&&0 d%C%C UDL ! !  $( 8< '4(?' ' '   !n&&//0CDDG "y#s9~~ F F #Dzz2RR5FRRRRM7>>"677 A 04@@@@%]@TUUUF# G&2F(F(FMMMMMMGGGGG4N#6** $S)) ")+&=A?C"8<MQ5EI-LP4@D(;?#SW;QU9NR6BF*BF*LP3MQ5KO3HL/  H  B  E  AOS8SW=RV< )       rcZtddtd|_|jd|jjitddtd|_|jd|jjitdd td|_|jd|jjitd d td|_ |j d|jjitd d td|_ |j d|jjitddtd|_ |j d|jjitddtd|_ |j d|jjitddtd|_ |j d|jjitddtd|_|jd|jjitddtd|_|jd|jjitddtd|_|jd|jjitddtd|_|jd|jjitddtd|_|jd|jjitddtd|_|jd|jjidS) Nrllib_algorithm_step_timezTime spent in Algorithm.step())rllib)name description boundariestag_keysrC/rllib_algorithm_run_one_training_iteration_timez5Time spent in Algorithm._run_one_training_iteration()'rllib_algorithm_run_one_evaluation_timez-Time spent in Algorithm._run_one_evaluation().rllib_algorithm_compile_iteration_results_timez4Time spent in Algorithm._compile_iteration_results()"rllib_algorithm_training_step_timez'Time spent in Algorithm.training_step()rllib_algorithm_evaluate_timez"Time spent in Algorithm.evaluate()5rllib_algorithm_evaluate_sync_env_runner_weights_timezPTime spent on syncing weights to the eval EnvRunners in the Algorithm.evaluate()3rllib_algorithm_evaluate_sync_connector_states_timezBTime spent on syncing connector states in the Algorithm.evaluate()0rllib_algorithm_step_sync_env_runner_states_timezGTime spent in sync_env_runner_states code block of the Algorithm.step()$rllib_algorithm_load_checkpoint_timez)Time spent in Algorithm.load_checkpoint()$rllib_algorithm_save_checkpoint_timez)Time spent in Algorithm.save_checkpoint()-rllib_algorithm_callback_on_train_result_timez*Time spent in callback 'on_train_result()'/rllib_algorithm_callback_on_evaluate_start_timez,Time spent in callback 'on_evaluate_start()'-rllib_algorithm_callback_on_evaluate_end_timez*Time spent in callback 'on_evaluate_end()')rrr$set_default_tagsrrr%r&rr'r(r)r*r+r,r-r.r/r0r1r;s r_set_up_metricszAlgorithm._set_up_metricsIsP"+,8? # # #  00'4>;R1STTT8ABO? 9 9 9 5 5FF dn- .   1::G? 1 1 1 - ->> dn- .   8AAN@ 8 8 8 4 4EE dn- .   ,55A? , , , ( (99 dn- .   '00<? ' ' ' # #44gt~?V5WXXX>GHj@ ? ? ? ; ;LL dn- .   =FF\@ = = = 9 9JJ dn- .   :CCa@ : : : 6 6GG dn- .   .77C@ . . . * *;; dn- .   .77C@ . . . * *;; dn- .   7@@D@ 7 7 7 3 3DD dn- .   9BBF@ 9 9 9 5 5FF dn- .   7@@D@ 7 7 7 3 3DD dn- .     rctSNr)rs rrzAlgorithm.get_default_configs   rc4|jS)aReturns a list of remote worker IDs to fetch metrics from. Specific Algorithm implementations can override this method to use a subset of the workers for metrics collection. Returns: List of remote worker IDs to fetch metrics from. )rhealthy_worker_idsrVs r_remote_worker_ids_for_metricsz(Algorithm._remote_worker_ids_for_metricss$77999rc !"t|tst|tsJ}t|ts8t|tsJt|}||j|_|_tjj jj  jjj r)dtjjD_nj_jjdvr(t$djjdjjr1t)jdjjj_t3t4_d_d_d_d_j d}|K|tBurBd|D}tEd#|d |d d d |j_$jj%r*ddl&m'}jj(p|}|j_)nd_)jj*s jj sdtWj,j-.jjjj sd n jj/j0j1_j2_3j34j356j3r7j3jj3\}}tW|d.jj3j0j1j3j sd n j3j/jj8_9jrj:_n%j9rj9:_j;QjJj<j_j=j_d_>j3j$r^j3j?sRt$dtj3d\_>}t$di_Attttd} jj$FD]I\} } | Gd} | | vr.tE| t| | d | | } npt| tr[t$Idd| z| Jdd\} }tjL| } t| |} t| tr_t| trJQ}t| trjjS| d<| |fi| jA| <ntd| d| | d<KjjUrtjjWjjXfi}jr|Yjnj9r-|Yj9:njj*r3|YtjjWjjXfinrjj%rfj[|td|td}|Yt|jW|jXfij\|d !}j]|"_^i}|j_}|j`}|jaFD]\}}|j_r |j_||<|s|rj^b|||#j^ctd$ztzd %t}jrUjfjjghttfd&|jj'n[j9rTj9fj3jghttfd&|jj'j)rj^jkrHj^ld(}d)|Dj)_mj^jnj)_onj^jpgj)_o|j)_q|j)_rj3r|dd*lsmt}j3jus7j^ctd$ztzd %t}|j3jjvdk|t||+_wd_xjjUrjjydkrj\jd !"tj{dd,-t!t!"fd.tjjpdjjyzDjj/_xttj^ld01}ttjxd21}i_|D]\}}| }|D]I\}} | d|dkr |dd|dgz}|j|<nJ|jvr)t d3|d4|dd5|dd6|d t fd7tjjpdDst d8jtd9jjjtjg:;dS)z#Algorithm.setup..sWWWcceeWWWr)WARNERRORzCurrent log_level is zV. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.z ray.rllibinput_evaluationc2i|]}t|d|iS)r)r)r_opes r z#Algorithm.setup..s$LLLCC63-LLLrzconfig.input_evaluation={}zSconfig.evaluation(evaluation_config=config.overrides(off_policy_estimation_methods=z))Tz/Running OPE during training is not recommended.)rrrhelprr)r validate_envdefault_policy_classrlocal_env_runnerr tune_trial_id)rrhrirrrkrj pg_offset)spaceszCreating evaluation dataset ...) num_workerszEvaluation dataset created)iswisdmdrrrzTrying to import from string: rgammaz$Unknown off_policy_estimation type: ze! Must be either a class path or a sub-class of ray.rllib.offline.offline_evaluator::OfflineEvaluator)input_observation_spaceinput_action_spaceF)rminference_onlyrl_module_spec)multi_rl_module_ckpt_dirmodules_to_loadrl_module_ckpt_dirsr componentsrwdefault)renv_steps_sampledrl_module_state env_to_module module_to_envcLtjSrY)rayget_runtime_context get_node_id)rs rz!Algorithm.setup..s#"9";";"G"G"I"Irc6g|]}|Sr)get)r_node_ids rr`z#Algorithm.setup..s-888*1 888r)OfflineEvaluationRunnerGroup)r local_runner module_state module_specrm)num_cpus max_restartscFg|]}jSr)remoter)r_ragg_clsryr;s rr`z#Algorithm.setup..Ls9NN4;??r)'max_remote_requests_in_flight_per_actorc|j|jfSrY)nodedevice)_learners rrz!Algorithm.setup..[sx}ho.Nrfuncc|j|jfSrY)_node_deviceactors rrz!Algorithm.setup..csEK+Grz9No Learner worker found that matches aggregation worker #z 's node (z) and device (z&)! The Learner workers' locations are c3NK|]}|jvV dSrY)_aggregator_actor_to_learnervalues)r_ learner_idxr;s r z"Algorithm.setup..~sKt@GGIIIrzwSome Learner indices are not mapped to from any AggregatorActors! Final AggregatorActor idx -> Learner idx mapping is: on_algorithm_init algorithmmetrics_logger)r)rrrrrrrrrrU framework_strseed _record_usage"enable_env_runner_and_connector_v2rGcallbacks_class callbacks log_levelloggerinfologging getLoggersetLevel(_create_local_replay_buffer_if_necessaryrrsetremote_requests_in_flightrrmenv_to_module_connectormodule_to_env_connectorrrrrr is_offlineray.rllib.offline.offline_dataroffline_data_classr is_onliner5rrhget_default_policy_classcreate_local_env_runnerrtrial_idget_evaluation_config_objectrr r%_should_create_evaluation_env_runnersrnum_env_runnersr get_spaces env_runnerbuild_env_to_module_connectorbuild_module_to_env_connectorevaluation_datasetope_split_batch_by_episoder:reward_estimatorsr=r?r;r<itemsrrlogrsplit importlib import_modulegetattrr issubclassr@ get_policyr>rtrenable_rl_module_and_learnerr2observation_space action_spaceupdater+build_learner_connectorget_multi_rl_module_specbuild_learner_grouprload_state_pathr{rl_module_specsload_module_stater r&r*sync_env_runner_statesrpeekr]rm is_remoteforeach_learnerlocality_hints_workerslearner_handlesrr)_should_create_offline_evaluation_runners1ray.rllib.offline.offline_evaluation_runner_groupr%offline_eval_rl_module_inference_onlynum_offline_eval_runnersoffline_eval_runner_group_aggregator_actor_manager!num_aggregator_actors_per_learnerrrrrHrange num_learners+max_requests_in_flight_per_aggregator_actorlist enumerate foreach_actorr RuntimeErrorallrWr callbacks_on_algorithm_initr)#r;r config_objrcope_dictrrrr ope_typesrD method_config method_typemodobjpolicyrmlearner_connectorrr|rzr{ module_idsub_module_specrlearner_node_idsrlearner_locationsaggregator_locationsagg_idxaggregator_locationrlearner_locationrrys#` @@rsetupzAlgorithm.setups &/22 %f&@AA A AA0022Jj/:: E!&*DEEEEE,..88DD  ' ' / / /!\JN$DK ( (A4;CSTTT 4;''' ; 9 ;WWz$+:U/V/VWWWDNN![88::DN ; $5 5 5 KK (=    ;  K  k * * 3 3DK4I J J J$(#P#P K$ $      &;?'+ FJ$FJ$ ;??+=>>  ',+/;+T+T{,U,,D (,0;+T+T{,U,,D (#'  " @ 6*E 6 KK9 : : :)?&A*** &D #Q KK4 5 5 5@B$-   $(;#L#R#R#T#T 0 0 D-'++F33Ki''##Ik233 ( 4 K-- 0 1>LMMM&--c155S-c22%c3// +t,, -22  **k+=>>?-1[->M'*/:{6/S/S]/S/S&t,, B;BBB%0M& ! ! ; 3O  K1K,#F $!  dk****+  d8CCEEFFFF;(MM- $ = $ 80[+(, (K(K067G0H0K+12B+CA+F)L))% MM- 1 C 1 >0.2[-Q-Q$.R..K"&!@!@*"A""D  #% '2'B $)9O.9.I.O.O.Q.Q U U* ?"2U5D5T' 2' +> "44-E$3(;5#0::,s25HH#; !O$ %<<;&*l&7&7+-KLVW'8''%4"&">"&">=+ *AA1&*l&7&7+-KLVW'8''%4"&">"&">B  2%/V(,'9'I'III(($885E888D%4 9=8J8SD%55:>9K9T8UD%5 1<!-,2!(==d>TUU  -S)&*&8&B&B#4s#:=P#P','C''(')O PlOk1!%!E!J!01D!E +" P P P.*.& ; 3F  K 9A = =![AA{$BNcjG.G"16Q+GHKK . . .D *!%&66NN7!! $(2@@GGA$$ 13D -0D  ,,&9&=&=&?&?#5F  1K!1(++--a04G4JJJ,=abb,A-a0E-)FQ9'B K$"CCC&4#44.A!.D44/244 1444D#()A)FQ#G#G #;8;;    N K 3#|       rcdS)a'Returns a default Policy class to use, given a config. This class will be used by an Algorithm in case the policy class is not provided by the user in any single- or multi-agent PolicySpec. Note: This method is ignored when the RLModule API is enabled. Nrrrs rrz"Algorithm.get_default_policy_classs trc t|j5|jjo|jdz|jjzdk}|jjo|jdz|jjzdk}i}|r1|jjs |jjr|\}}}n;|jj r| \}}n| \}}|r"|jjs| d}|rD| }|r,|t|tn|}|jj r|jjs|jr|jt(t*f5t|j5|j|j|jt2t4fd|j|jdddn #1swxYwYdddn #1swxYwY|||}n{||jj|j|jtA|j|!|jj" }|#||i|| }dddn #1swxYwY|S) aAImplements the main `Algorithm.train()` logic. Takes n attempts to perform a single training step. Thereby catches RayErrors resulting from worker failures. After n attempts, fails gracefully. Override this method in your Algorithm sub-classes if you would like to handle worker failures yourself. Otherwise, override only `training_step()` to implement the core algorithm logic. Returns: The results dict with stats/infos on sampling, training, and - if required - evaluation. rsrNparallel_train_futurer)rrrr) train_results eval_resultscentral_workerworkersr)timeout_seconds)episodes_this_iterstep_ctxiteration_results)$rr$revaluation_interval iterationoffline_evaluation_intervalevaluation_parallel_to_training'offline_evaluation_parallel_to_training6_run_one_training_iteration_and_evaluation_in_parallelr_run_one_training_iteration)_run_one_training_iteration_old_api_stack_run_one_evaluation_run_one_offline_evaluationrbr!_dont_auto_sync_env_runner_statesrrlog_timer}rzr,rrr]rmrr_compile_iteration_results_sync_filters_if_neededrjr7r\$metrics_episode_collection_timeout_s(_compile_iteration_results_old_api_stack) r;evaluate_this_iterevaluate_offline_this_iterrrtrain_iter_ctxoffline_eval_resultsresultsrs rstepzAlgorithm.steps!$&d&= > >p p   /P^a'4;+JJaO   7X^a'4;+RRVWW ' (*L" I ; I;F IOOQQ ! "N;AI484T4T4V4V1M>> FFHH%&" T$+*U T#77d7SS ) 8'+'G'G'I'I$ 8 !34;;,-?@ $8L{=0  E- ..!AB6 J!1HH'+{26,2C2C(:(F%&-. 3D3"3"/3.J.2.JI   *99"/!-: ,,#'#8#I 1;- &6)7799$(K$T&&&" GG'9+&G&G,&GHYp p p p p p p p p p p p p p p ds\E.J9HAG8, H8G< <H?G< H J9H J9H BJ99J=J=c |j|j|jjt d|j|jjt||j |jj dkr| n| |j ttfid}t d|j|jjt||j | t|iS) zEvaluates current policy offline under `evaluation_config` settings. Returns: A ResultDict only containing the offline evaluation results from the current iteration. from_worker_or_learner_grouprwon_evaluate_offline_startrcallbacks_objectscallbacks_functionsrrTrlatest_merged_onlyon_evaluate_offline_endrrevaluation_metrics)r sync_weightsrrrr r#callbacks_on_evaluate_offline_startrrnum_healthy_remote_runners%_evaluate_offline_with_fixed_duration!_evaluate_offline_on_local_runnerrrbrt!callbacks_on_evaluate_offline_end)r;rs revaluate_offlinezAlgorithm.evaluate_offline,s% &33)-);;L 4     '"n $ O$t|DDD       ) Dq H H  6 6 8 8 8 8  2 2 4 4 4|(( !< =#)    %"n $ M#|#/ ,\::rr c  t|j5||j |cdddS|jjr]|jN|jr5|jj j |jd}n|j }n|j }|j Qt|j5|j |ddddn #1swxYwY|jjr|jjr|jt(t*f5t|j5|j |j|j |jt2t4fd|j|jdddn #1swxYwYdddn #1swxYwYnU||jj|j |jn(|jjr|j|dt|j5tAd|j!|jj"tG||j  dddn #1swxYwYdx}}g}|jj$r?|jjr|%\}}}n|j$}n|j '|j r |&|j \}}}}n|j 'dkr |&|j(\}}}}nh|j 'dkrI|jj)d kr|J|*|\}}}}n|+\}}}}ni}|jjrN|jtXid }t[d r|st\/dn1t2|i}||t`<||tb<||d<||j2tf<|jj$stitj}|j67D]F\}} |D]>} | 8| |jj9} ||:| ?G|r=i|d<|7D]#\}} twj<dg| R} | |d|<$t|j=5tAd|j!|jj>tG||j| dddn #1swxYwYdddn #1swxYwY|S)aeEvaluates current policy under `evaluation_config` settings. Args: parallel_train_future: In case, we are training and avaluating in parallel, this arg carries the currently running ThreadPoolExecutor object that runs the training iteration. Use `parallel_train_future.done()` to check, whether the parallel training job has completed and `parallel_train_future.result()` to get its return values. Returns: A ResultDict only containing the evaluation results from the current iteration. NrTr.rr from_workerrrrron_evaluate_startrr1auto)keyrr5no_eval_resultszoNo evaluation results found for this iteration. This can happen if the evaluation worker(s) is/are not healthy.timesteps_this_iter)split_batch_by_episodeoff_policy_estimatorc.tj|dS)Nr)axis)npmean)xs rrz$Algorithm.evaluate..srwqq'9'9'9ron_evaluate_endr7)?rr)_before_evaluater%_run_offline_evaluation_old_api_stackrrrhealthy_env_runner_ids_worker_manager_actorsrrrr*r9rbroadcast_env_runner_statesrr"r}r{r+rrr]rmrrr$rjr0r rcallbacks_on_evaluate_startrcustom_evaluation_function#_evaluate_with_custom_eval_function_evaluate_on_local_env_runnernum_healthy_remote_workerseval_env_runnerevaluation_duration_evaluate_with_auto_duration_evaluate_with_fixed_durationrbrrwarningrhrnr!rlrrrrestimaterappendtree map_structurer1callbacks_on_evaluate_end)r;r  weights_src env_steps agent_stepsbatchesr estimatesrD estimatorbatchestimate_result estimate_list avg_estimates revaluatezAlgorithm.evaluatees:$&d&A B Bv v   ! ! # # #&2AACC v v v v v v v v {= .)5-DDFF6 #'"7"G"O-DDFFqI#KK#'"4KK"o )5-G.;;5@'+<;A-I"!\22#%JK""": $ Q""""!% : Q Q+/+A046:l6G6G,>,J)*12 7H7&7&372N262N!R !" !" !"""""""""""""""""""""""""""""""(00'+'<'M $ :#51 ? %221<#'3 *=  '&*n(, (O$t|LLL                '( 'I G{5. ";AL @@BB $!# $(;#I#I#K#KLL+33 66tGG  G+FFHHAMM 66t7KLL  G+FFHH1LL;2f<<0<<< 99:OPP $!#::<< $!# " {= #|00*'+ 1   -..|NNJ!3LA BM >?@I <=6? 23B ;9 R'-- '+'='C'C'E'E@@OD)!(@@*3*<*<!37;3Y+=++"$..???? @R;=L!78/8/@/@RR+m'+'999( G! G G!G G! V!G% %V(G% )A-V7J VJ V J !JV&8U* V*U. .V1U. 2VVVc|td|d|jj|jjr?|j||j\}}}|r|st d|d|dn|j}|rt|tst d|d|||fS)NEvaluating current state of z using the custom eval function zxCustom eval function must return `Tuple[ResultDict, int, int]` with `int, int` being `env_steps` and `agent_steps`! Got z, rz6Custom eval function must return dict of metrics! Got ) rrrrWrrrrr)r;rrfrgs rrXz-Algorithm._evaluate_with_custom_eval_function2s)  84 8 8{5 8 8    ; 9 D  66tT=WXX   K  V:CVVGRVVV  ;AACCL :lD#A#A 8(4888  Y 33rc d}|jj|jjz}td|d|d|d|jj}|j |gttfdS)Nrhrq for  rrE) roffline_evaluation_duration!dataset_num_iters_per_eval_runnerrrrrrunr aggregaterbrt)r;unitdurationr+s rr=z+Algorithm._evaluate_offline_on_local_runnerMs K 3k; <   P4PPhPPPPPQQQ0=AACC  I#%@A      rc jt|dr|jtd|jjrtd|jj}|jj}|j}dx}}t d|d|d|dg}|jj r`| |d kr|nd|d kr|nd }|td |Dz }|td |Dz }n|d krqt|D]`} | } || z }|| z }|jr|| an^| } || z }|| z }|jr|| |} |jj st'| | |j} n+|j| gt.t0fd} | |||fS)N input_readera;Can't evaluate on a local worker if this local worker does not have an environment! Try one of the following: 1) Set `evaluation_interval` > 0 to force creating a separate evaluation EnvRunnerGroup. 2) Set `create_local_env_runner=True` to force the local (non-eval) EnvRunner to have an environment to evaluate on.zpCannot run on local evaluation worker parallel to training! Try setting `evaluation_parallel_to_training=False`.rrqrsrtr timestepsepisodes) num_timesteps num_episodesc3>K|]}|VdSrYrgr_es rrz:Algorithm._evaluate_on_local_env_runner..}s*AA1q}}AAAAAArc3>K|]}|VdSrYrfrs rrz:Algorithm._evaluate_on_local_env_runner..~s*==qQ[[]]======rkeep_custom_metricsru)hasattrr}rrrevaluation_duration_unitr\rrrrsamplesumrrgrfrra get_metricsr8keep_per_episode_custom_metricsrryrbr]) r;rrzr{eval_cfgrfrg all_batchesrrrkenv_runner_resultss rrYz'Algorithm._evaluate_on_local_env_runner^s :~ . . :3J3R [ 8 C  {3;2)"## K P4PPhPPPPPQQQ ; 9 *!((*.+*=*=hh4)-););XX)H 3AAAAAAA AK ==H===== =II Z  8__ . ."))++u00222 U__... ).&&u---  .%%''E 5,,.. .K ** *I% *""5)))'3355{= &!3""$,$L"""   L " "#$');< #   "& !9k;FFrc "td|dg}g}|j}|jjfd}dx}}|jjr)|jttfd}n|j tj }tj } |jd} |dkr| dks|s| dz } |jjr|jt t"t$fd d di d } t'|jjt+|jj|tj | z z | z|z } |j|| | d d } | D]2\}}}}||jkr||z }||z }||3tjdn|jfdd} | D]q\}}}||jkr||z }||z }|||jr||r|j}|dkr| dk||dkrtd|jjs*t?|||j j!}|tD}nX|j#|t t"f|jt t"tDfd}d}|jjr$|dkrtd|d||||fS)Nrqz; for as long as the parallelly running training step takes.c||o|dk}|}td|D}td|D}||||fS)Nr)r force_resetc3>K|]}|VdSrYrrs rrzUAlgorithm._evaluate_with_auto_duration.._env_runner_remote..*<K|]}|VdSrYrrs rrzUAlgorithm._evaluate_with_auto_duration.._env_runner_remote..*@@!ammoo@@@@@@r)rrr) workernumrounditerrrrfrgrs r_env_runner_remotezBAlgorithm._evaluate_with_auto_duration.._env_runner_remotes}}!{/Iuz%H((**G<<8<<<<. AMMOO^'Tr!env_runner_sample_and_get_metrics)rr]Calling `sample()` on your remote evaluation worker(s) resulted in all workers crashing! Make sure a) your environment is not too unstable, b) you have enough evaluation workers (`config.evaluation(evaluation_num_env_runners=...)`) to cover for occasional losses, and c) you use the `config.fault_tolerance(restart_failed_env_runners=True)` setting.rruaThis evaluation iteration resulted in an empty set of episode summary results! It's possible that the auto-duration time (roughly the mean time it takes for the training step to finish) is not enough to finish even a single episode. Your current mean training iteration time is aEsec. Try setting the min iteration time to a higher value via the `config.reporting(min_time_s_per_iteration=...)` OR you can also set `config.evaluation_force_reset_envs_before_iteration` to False. However, keep in mind that then the evaluation results may contain some episode stats generated with earlier weights versions.)$rrrrZr,evaluation_force_reset_envs_before_iterationrrrr}r~rrMtimerdonerbr]rmmin1evaluation_auto_duration_max_env_steps_per_samplemax1evaluation_auto_duration_min_env_steps_per_sample$foreach_env_runner_async_fetch_readyrasleeprfrgextendrr_r8rrrqry)r;r  all_metricsrnum_healthy_workersrrfrgtrain_mean_timet0_roundthroughput_estimate_numr+env_sag_srrrkrrrrs @@rr]z&Algorithm._evaluate_with_auto_durations  +4 + + +     #8SSUUkN  9 9 9 9 9#$# K ; 9 J"l//12C0OO#l+CDIO Y[[ ! # #2%:%?%?%A%A aKF{=B 2'+l&7&7**6 $=cB'8 ' '2 '3#KQ U- b0@A222 2  ".SS/'+fnUU0T3:00.E4$t~-- &I4'K&&w//// 4    .SSTTTT?T-4 2 2(E7Dt~-- !2!22I5#4#4#6#66K&&w///-2$**5111*EEGG Y ! # #2%:%?%?%A%AZ ! # # NN=   {= &!3*J """ .l;LL L " "');< #    <,,#%7F-L"&  K D !! NNV# VVV   "9k;FFrcR|jj}|jj}d}g}g}d}|jj}t j}d}|j} |dkr|jj|z|z } | dkrn|dz }|jtj || |j dd } t j} | s | |z |krn| r| }| D]l\} \}}||jkr| ||t|tvr*|ttndz }m|jj}|dk|dkrt d |j|t(t*f dS) Nc2|}||fSrY)rx)runnerrrs r_offline_eval_runner_remotezTAlgorithm._evaluate_offline_with_fixed_duration.._offline_eval_runner_remoteOsjjllGD= rrrrs)rrFr)return_obj_refsrajCalling `run()` on your remote offline evaluation runner(s) resulted in all runners crashing! Make sure a) your dataset is not corrupted, b) you have enough offline evaluation runners (`config.evaluation(num_offline_eval_runners=...)`) to cover for occasional losses, and c) you use the `config.fault_tolerance(restart_failed_offline_eval_runners=True)` setting.ru)rroffline_evaluation_timeout_srr;rrrvforeach_runner_async functoolspartialfetch_ready_async_reqsrar\r[rrr_rryrbrt)r;rntime_outrrnum_units_doner t_last_resultrrunits_left_to_dor+time_nowwidmetrs rr<z/Algorithm._evaluate_offline_with_fixed_durationJs*k: ;; ! ! !  "<W  "A%% 7+EV  1$$ aKF  * ? ?&/' @    4KK %tLG y{{H )x-7(BB ) ( $+   [c44>))""3'''2c+6FFF $%@AFFHHH.I M"A%%T ! # # NNF     #%@A      rc|jj|j}|jj}|jj}|jj}fd}g}g}d}|jdx} } tj} d} |j dkr^|jj |z dkrnG| dz } |jj rdgfdtd|dzDz} |j || | |dd}tj}|s || z |krn|r|} |D]y\}}}}||j kr| |z } | |z } |||d kr*t|vr|tndn|jjd kr|n|z }zn.d krdn|j|jzfd t)|jD}|j fd |d }tj}|s || z |krn|r|} |D]q\}}}||j kr| |z } | |z } |||jr||rd kr|t5|z }n|jjd kr| n| }|jdk^dkrt6d|jj s*t;|||jj}|t}nY|j |tBtDf|jtBtDtfdd}d}|dkr5t6d|jj d|jjd|| | |fS)Nc$| dkr ||jnd dkr ||jnd|o|dk}|}td|D}td|D}||||fS)Nr~rr)rrrc3>K|]}|VdSrYrrs rrzVAlgorithm._evaluate_with_fixed_duration.._env_runner_remote..rrc3>K|]}|VdSrYrrs rrzVAlgorithm._evaluate_with_fixed_duration.._env_runner_remote..rr)r worker_indexrr) rrrr _force_resetrrrfrgrzs rrzCAlgorithm._evaluate_with_fixed_duration.._env_runner_remotes}}04 0C0CC+,,:>*:L:Lc&"566RV(7UaZ %H((**G<<8<<<<.sN!!!&)<<1!14G!GHIIJ!!!r)rrrrrrrrfc,g|]\}}|zk|Srr)r_r worker_idrunits_per_healthy_remote_workers rr`z;Algorithm._evaluate_with_fixed_duration..s>,,,$9::=MMM NMMrcV||fSrYrrs rrz9Algorithm._evaluate_with_fixed_duration.. rrr)rremote_worker_idsrrrruTr4zThis evaluation iteration resulted in an empty set of episode summary results! It's possible that your configured duration timesteps are not enough to finish even a single episode. You have configured rta. For 'timesteps', try increasing this value via the `config.evaluation(evaluation_duration=...)` OR change the unit to 'episodes' via `config.evaluation(evaluation_duration_unit='episodes')` OR try increasing the timeout threshold via `config.evaluation(evaluation_sample_timeout_s=...)` OR you can also set `config.evaluation_force_reset_envs_before_iteration` to False. However, keep in mind that in the latter case, the evaluation results may contain some episode stats generated with earlier weights versions.)#rrrevaluation_num_env_runnersrevaluation_sample_timeout_srrZrrr\rrrrarqrcount_steps_byrollout_fragment_lengthnum_envs_per_env_runnerrr[rfrgrrlenrr_r8rrryrbr])r;rrnrrrrrrrfrgrrrr+rrrrrselected_eval_worker_idsrkrrrrrrzrrs @@@@@rr^z'Algorithm._evaluate_with_fixed_durationsB{3)k< kN ;: 9 9 9 9 9   "8SSUU"## K "A%%#{>O 1$$ aKF{=[ v!!!!!#1kAo66!!!.SS/#'%+$2,7    1T   9;;-8m#;h#F#F-$,M.5  *E4dt~-- &I4'K&&s+++":--6BS5H5H\*//111aa&*[%?;%N%NEETX NN "z))A!9670,,,,,(12EEGG)),,,(.SSTTTT*B?T 9;;-8m#;h#F#F-$,M,3 2 2(E7Dt~-- !2!22I5#4#4#6#66K&&w///-2$**5111:%%"c'll2NN  ;5DD" (#*EEGG M"A%%T ! # # NN=   {= &!3*J """ .l;LL L " "');< #    <,,#%7F#'-L "&  1   NN ,;2 , ,;7 , , ,    "9k;FFrcd}|jjr|}|sgS|jdxxt |z cc<|jp|j}||}tj ||jj s5|d D]}| ddn|jj rtj|jn|jdjd|jt*dzt,zd t*}|j|t0<|j|t4<|jt:t<fd |t<<tj |fd }|||d |jj |S)aTry bringing back unhealthy EnvRunners and - if successful - sync with local. Algorithms that use custom EnvRunners may override this method to disable the default, and create custom restoration logics. Note that "restoring" does not include the actual restarting process, but merely what should happen after such a restart of a (previously failed) worker. Args: env_runner_group: The EnvRunnerGroup to restore. This may be the training or the evaluation EnvRunnerGroup. Returns: A list of EnvRunner indices that have been restored during the call of this method. Ntotal_num_restored_workers policy_statesconnector_configsc4tj|jSrY)r/ from_modulemodule)learners rrz/Algorithm.restore_env_runners..s$5$A'.$Q$QrrrTr}rc|jjr|jjr|jjD]*\}}|jvr|j|d+jD]=\}}||jvr/|j || d>| tj dS)NT)raise_err_if_not_foundF)rO)rris_multi_agentr _rl_modulesrrr remove_module add_modulebuild set_staterr)errrmidmod_specmulti_rl_module_spec state_refs r_sync_env_runnerz7Algorithm.restore_env_runners.._sync_env_runners < TI, T*,)>)C)C)E)E)K)K)M)MXX%Iv (<(LLL // RV/WWW%9%I%O%O%Q%QTTMC")++ ,,S(..2B2BU,SSS LL++ , , , , ,rF)rrrjr)!rrprobe_unhealthy_env_runnersr!rrjrr rputrrrrr/rrrrresult_or_errorsrr&r*rr$rr)rrr]rmforeach_env_runnerenv_runner_restore_timeout_s) r;rrestoredfrom_env_runnerr pol_statesrrrs @@rrestore_env_runnerszAlgorithm.restore_env_runnersesU& ;  F'CCEEH I 3444H E444*;Nt  &#--//EI;A "'"8"?"?"A"A>>JNN#6====>+ '8'D#*(($"22QQ"!% !&00,s25HH#1 !E ,6688 1  ,6688 1 59L4E4E#%CD5F55E0 1I - - - - - -$ ++!&# KD ,   r runner_groupc |r|jsgS|}|r|jdxxt|z cc<|d}|dd|d}t j|fd}|||d|jj |d|d|jj d |j |i |S) Nrrr F)rrcV|tjdSrYrrr)rrs r _sync_runnerz._sync_runners% CGI../////r)rrrrset_dataset_iteratoriterator)rrrrr) rprobe_unhealthy_runnersr!rhealthy_runner_idsforeach_runnerrrr%offline_eval_runner_restore_timeout_s_offline_data_iterators)r;rr from_runnerrr rs @rrestore_offline_eval_runnersz&Algorithm.restore_offline_eval_runnerss> <#< I7799 $  N7 8 8 8CMM I 8 8 8'99;;A>K //""-0 E I 0 0 0 0 0  ' '!"*" $ 6 \ (     ' '+"*" $ 6 \"L$H$RS (   rc F|jjstd|jt t f5|jjdkr1t|j |jj |jj dd\}}n0t|j |jj |jj dd\}}dddn #1swxYwY|j |t|jt tf5|j|t"|jt"i}|j |t&dddn #1swxYwY|jt t(f5|j |jt-t/|t2hz d ddddS#1swxYwYdS) adDefault single iteration logic of an algorithm. - Collect on-policy samples (SampleBatches) in parallel using the Algorithm's EnvRunners (@ray.remote). - Concatenate collected SampleBatches into one train batch. - Note that we may have more than one policy in the multi-agent case: Call the different policies' `learn_on_batch` (simple optimizer) OR `load_batch_into_buffer` + `learn_on_loaded_batch` (multi-GPU optimizer) methods to calculate loss and update the model(s). - Return all collected metrics for the iteration. Returns: For the new API stack, returns None. Results are compiled and extracted automatically through a single `self.metrics.reduce()` call at the very end of an iteration (which might contain more than one call to `training_step()`). This way, we make sure that we account for all results generated by each individual `training_step()` call. For the old API stack, returns the results dict from executing the training step. a+The `Algorithm.training_step()` default implementation no longer supports the old API stack! If you would like to continue using these old APIs with this default `training_step`, simply subclass `Algorithm` and override its `training_step` method (copy/paste the code and delete this error message).rgT) worker_setmax_agent_stepssample_timeout_s_uses_new_env_runners_return_metrics)r max_env_stepsrrrNru)rr~)r/rrw)rrNotImplementedErrorrr"r}r^rr9rtotal_train_batch_sizerryr]rerrrmrrdr|r9rrkeysr[)r;rrlearner_resultss r training_stepzAlgorithm.training_steps,{= %7 \ " "F,E#F G G  {)]::/J#4$(K$F%)[%A*.$( 000,,,0K#4"&+"D%)[%A*.$( 000,,               $ 17IJJJ \ " "F,@#A B B I I"077!2 ))*HII8O L " "? " H H H I I I I I I I I I I I I I I I\ " "F,F#G H H    ! . .-1-?c/"6"6"8"899[MIJJ# /                     s9A2CCCA"FFF1AHHHrc|j |jj}n%|jddgdd}t |t r||S|S)aReturns the (single-agent) RLModule with `model_id` (None if ID not found). Args: module_id: ID of the (single-agent) RLModule to return from the MARLModule used by the local EnvRunner. Returns: The RLModule found under the ModuleID key inside the local EnvRunner's MultiRLModule. None if `module_id` doesn't exist. Nc|jSrY)r)rs rrz&Algorithm.get_module..V s29rrsF)rrjr)rrrrrr.r)r;rrs r get_modulezAlgorithm.get_moduleF sx ? &_+FF*==$$#$#!&> F fm , , ::i(( (MrT)config_overridesnew_agent_to_module_mapping_fnnew_should_module_be_updatedadd_to_learnersadd_to_env_runnersadd_to_eval_env_runnersrr#r$r%r&r'r(ctdd} |jjstdtdt |||gst d|r|j||} d|j_ t|jj <||j |i |j  |j | |j  |j|ffd } |r^| !|j| d} n|j| |j|jd|dure|j^| !|j| d} n|j| |j|jd| S)aVAdds a new (single-agent) RLModule to this Algorithm's MARLModule. Note that an Algorithm has up to 3 different components to which to add the new module to: The LearnerGroup (with n Learners), the EnvRunnerGroup (with m EnvRunners plus a local one) and - if applicable - the eval EnvRunnerGroup (with o EnvRunners plus a local one). Args: module_id: ID of the RLModule to add to the MARLModule. IMPORTANT: Must not contain characters that are also not allowed in Unix/Win filesystems, such as: `<>:"/|?*`, or a dot, space or backslash at the end of the ID. module_spec: The SingleAgentRLModuleSpec to use for constructing the new RLModule. config_overrides: The `AlgorithmConfig` overrides that should apply to the new Module, if any. new_agent_to_module_mapping_fn: An optional (updated) AgentID to ModuleID mapping function to use from here on. Note that already ongoing episodes will not change their mapping but will use the old mapping till the end of the episode. new_should_module_be_updated: An optional sequence of ModuleIDs or a callable taking ModuleID and SampleBatchType and returning whether the ModuleID should be updated (trained). If None, will keep the existing setup in place. RLModules, whose IDs are not in the list (or for which the callable returns False) will not be updated. add_to_learners: Whether to add the new RLModule to the LearnerGroup (with its n Learners). add_to_env_runners: Whether to add the new RLModule to the EnvRunnerGroup (with its m EnvRunners plus the local one). add_to_eval_env_runners: Whether to add the new RLModule to the eval EnvRunnerGroup (with its o EnvRunners plus the local one). Returns: The new MultiRLModuleSpec (after the RLModule has been added). TrNzCan't add a new RLModule to a single-agent setup! Make sure that your setup is already initially multi-agent by either defining >1 RLModules in your `rl_module_spec` or assigning a ModuleID other than z to your (only) RLModule.zjAt least one of `add_to_learners`, `add_to_env_runners`, or `add_to_eval_env_runners` must be set to True!)rrr#r%F)%algorithm_config_overrides_per_modulerrxrc|j||j|jt j|jS)N)rrr,r-)rrrr multi_agentr/r) _env_runner _module_specrr$r%s r_addz"Algorithm.add_module.._add s   ) )#L,>,>,@,@ *   .9"..&D/ ,7"..&B/%01CDD Drrr.)r-rrrr+anyrrr _is_frozenrBrr/ rl_modulerrrr9r) r;rrr#r$r%r&r'r(rr2s ` `` rrzAlgorithm.add_module` sb 9D1111 ${) E*EEE O%79PQRR A   #'#5#@#@#'!1-I $A$$ "' *4,, Y'  ' K # #7@BR6S $    * 5 K # #6T # U U U -ABBB ' 3 K # #6R # S S S +6 E E E E E E E E(  #+'+'<'O'OPT'U'UVW'X$$%88>>>  ! . .-1-?# /    #d * *t/I/U#+'+'A'T'T((($$*==dCCC  & 3 3-1-?# 4    $#r)r$r%remove_from_learnersremove_from_env_runnersremove_from_eval_env_runnersr6r7r8c6d}|r|j|}d|j_|jj=|jjd|j|j|||j||j fd}|r^|!|j |d}n|j ||j |jd |d ure|j ^|!|j |d}n|j ||j |jd |S) a^Removes a new (single-agent) RLModule from this Algorithm's MARLModule. Args: module_id: ID of the RLModule to remove from the MARLModule. IMPORTANT: Must not contain characters that are also not allowed in Unix/Win filesystems, such as: `<>:"/|?*`, or a dot, space or backslash at the end of the ID. new_agent_to_module_mapping_fn: An optional (updated) AgentID to ModuleID mapping function to use from here on. Note that already ongoing episodes will not change their mapping but will use the old mapping till the end of the episode. new_should_module_be_updated: An optional sequence of ModuleIDs or a callable taking ModuleID and SampleBatchType and returning whether the ModuleID should be updated (trained). If None, will keep the existing setup in place. RLModules, whose IDs are not in the list (or for which the callable returns False) will not be updated. remove_from_learners: Whether to remove the RLModule from the LearnerGroup (with its n Learners). remove_from_env_runners: Whether to remove the RLModule from the EnvRunnerGroup (with its m EnvRunners plus the local one). remove_from_eval_env_runners: Whether to remove the RLModule from the eval EnvRunnerGroup (with its o EnvRunners plus the local one). Returns: The new MultiRLModuleSpec (after the RLModule has been removed). N)rr%Fr,rxr-c|j|jd|_t j|jS)N)rr,T)rrrr/_needs_initial_resetr/r)r0rr$s r_removez(Algorithm.remove_module.._remove& sd   , ,y , A A A-9"..&D/04K ,$01CDD DrrTr.)rrrr4rr+rr/r5rrrr9r) r;rr$r%r6r7r8rr<s `` rrzAlgorithm.remove_module sN $  #'#5#C#C#-I$D$$ "'  K  + 9==iNNN ) 5 K # #6T # U U U -ABBB ' 3 K # #6R # S S S  E E E E E E" # #+'+'<'O'O((($$%88AAA  ! . .-1-?# /    )D 0 0*6#+'+'A'T'T((($$*==gFFF  & 3 3-1-?# 4    $#r policy_idc6|j|S)zwReturn policy for the specified id, or None. Args: policy_id: ID of the policy to return. )rr)r;r=s rrzAlgorithm.get_policyU s)))444rcz|j|j|S|j|S)zReturn a dict mapping Module/Policy IDs to weights. Args: policies: Optional list of policies to return weights for, or None for all policies. N) module_ids)r get_weightsr)r;rs rrAzAlgorithm.get_weights^ s=   )%11X1FF F**8444rweightsc|j*|ttt|iii|jj|dS)zSet RLModule/Policy weights by Module/Policy ID. Args: weights: Dict mapping ModuleID/PolicyID to weights. N)rrr'r&r*rrj set_weights)r;rBs rrDzAlgorithm.set_weightsk s_   ) NN+)/,.    .::7CCCCCr) rrr policy_staterrr'r(revaluation_workersr& policy_clsrrrrEc |jjrtd| tkrt ddd|tkrt dddt |d | dur$|j||||||||| | | dur+|j$|j||||||||| | | r| |S| r|jr|j j |Sd Sd S) a{ Adds a new policy to this Algorithm. Args: policy_id: ID of the policy to add. IMPORTANT: Must not contain characters that are also not allowed in Unix/Win filesystems, such as: `<>:"/|?*`, or a dot, space or backslash at the end of the ID. policy_cls: The Policy class to use for constructing the new Policy. Note: Only one of `policy_cls` or `policy` must be provided. policy: The Policy instance to add to this algorithm. If not None, the given Policy object will be directly inserted into the Algorithm's local worker and clones of that Policy will be created on all remote workers as well as all evaluation workers. Note: Only one of `policy_cls` or `policy` must be provided. observation_space: The observation space of the policy to add. If None, try to infer this space from the environment. action_space: The action space of the policy to add. If None, try to infer this space from the environment. config: The config object or overrides for the policy to add. policy_state: Optional state dict to apply to the new policy instance, right after its construction. policy_mapping_fn: An optional (updated) policy mapping function to use from here on. Note that already ongoing episodes will not change their mapping but will use the old mapping till the end of the episode. policies_to_train: An optional list of policy IDs to be trained or a callable taking PolicyID and SampleBatchType and returning a bool (trainable or not?). If None, will keep the existing setup in place. Policies, whose IDs are not in the list (or for which the callable returns False) will not be updated. add_to_env_runners: Whether to add the new RLModule to the EnvRunnerGroup (with its m EnvRunners plus the local one). add_to_eval_env_runners: Whether to add the new RLModule to the eval EnvRunnerGroup (with its o EnvRunners plus the local one). module_spec: In the new RLModule API we need to pass in the module_spec for the new module that is supposed to be added. Knowing the policy spec is not sufficient. Returns: The newly added policy (the copy that got added to the local worker). If `workers` was provided, None is returned. z`Algorithm.add_policy()` is not supported on the new API stack w/ EnvRunners! Use `Algorithm.add_module()` instead. Also see `rllib/examples/self_play_league_based_with_open_spiel.py` for an example.z,Algorithm.add_policy(evaluation_workers=...)z1Algorithm.add_policy(add_to_eval_env_runners=...)Trz(Algorithm.add_policy(add_to_learners=..).Hybrid API stack no longer supported by RLlib!rrgrr*)rrrrErrrN) rrrrrr-r add_policyrrr[ policy_map)r;r=rGrrrrrErrr'r(rrFr&s rrKzAlgorithm.add_policy sH ; 9   !1 1 1 BG     . . . >E     9D1111  % %  ! , ,"3))"3"3' -    #d * *t/I/U  & 1 1"3))"3"3' 2     >??9-- - $ >)C >'29= = > > > >r)rrr7r8rFr6c |tkrtddd|}|tkrtdddfd }|r|j|d |r%|j |j|d d Sd Sd S) aHRemoves a policy from this Algorithm. Args: policy_id: ID of the policy to be removed. policy_mapping_fn: An optional (updated) policy mapping function to use from here on. Note that already ongoing episodes will not change their mapping but will use the old mapping till the end of the episode. policies_to_train: An optional list of policy IDs to be trained or a callable taking PolicyID and SampleBatchType and returning a bool (trainable or not?). If None, will keep the existing setup in place. Policies, whose IDs are not in the list (or for which the callable returns False) will not be updated. remove_from_env_runners: Whether to remove the Policy from the EnvRunnerGroup (with its m EnvRunners plus the local one). remove_from_eval_env_runners: Whether to remove the RLModule from the eval EnvRunnerGroup (with its o EnvRunners plus the local one). z/Algorithm.remove_policy(evaluation_workers=...)z9Algorithm.remove_policy(remove_from_eval_env_runners=...)Frz0Algorithm.remove_policy(remove_from_learners=..)rITrJc8|dS)N)r=rr) remove_policy)rrr=rs rfnz#Algorithm.remove_policy..fn1 s3  #"3"3 !     rrjN)rrrrr) r;r=rrr7r8rFr6rPs ``` rrOzAlgorithm.remove_policy sJ !1 1 1 EO     ,> ( #3 3 3 FE             # P  ! 4 4R$ 4 O O O ( UD,F,R  & 9 9"t 9 T T T T T U U,R,Rrrc|d}|td|d}|std||}|||S)aRecovers an Algorithm from a state object. The `state` of an instantiated Algorithm can be retrieved by calling its `get_state` method. It contains all information necessary to create the Algorithm from scratch. No access to the original code (e.g. configs, knowledge of the Algorithm's class, etc..) is needed. Args: state: The state to recover a new Algorithm instance from. Returns: A new Algorithm instance. algorithm_classNzQNo `algorithm_class` key was found in given `state`! Cannot create new Algorithm.rz+No `config` found in given Algorithm state!)r)rr __setstate__)rrSrnew_algos rrzAlgorithm.from_state@ s ,1995F+G+G  "/  8$$ LJKK K"?&111e$$$r export_dironnxcX||||dS)aExports policy model with given policy_id to a local directory. Args: export_dir: Writable local directory. policy_id: Optional policy id to export. onnx: If given, will export model in ONNX format. The value of this parameter set the ONNX OpSet version to use. If None, the output format will be DL framework specific. N)r export_model)r;rVr=rWs rexport_policy_modelzAlgorithm.export_policy_modelb s,  ""// DAAAAArc||}|td|d||dS)aExports Policy checkpoint to a local directory and returns an AIR Checkpoint. Args: export_dir: Writable local directory to store the AIR Checkpoint information into. policy_id: Optional policy ID to export. If not provided, will export "default_policy". If `policy_id` does not exist in this Algorithm, will raise a KeyError. Raises: KeyError: if `policy_id` cannot be found in this Algorithm. NzPolicy with ID z not found in Algorithm!)rKeyErrorexport_checkpoint)r;rVr=rs rexport_policy_checkpointz"Algorithm.export_policy_checkpointt sN$++ >PYPPPQQ Q  ,,,,,rcheckpoint_dirc t|j5|jjr/|||jj ddddSt j|}|}i}d|vr&d|dvr|d di}|jjr t|d<n t|d<|dz }t|d5}tj||dddn #1swxYwYt|dz d 5}tjd t!|dd t!|t#|t&jt&jd |dddn #1swxYwY|D]`\}}t/|d |dz |z }t1j|d ||} | ||a|jjr:t0j|d} |j| ddddS#1swxYwYdS)aoExports checkpoint to a local directory. The structure of an Algorithm checkpoint dir will be as follows:: policies/ pol_1/ policy_state.pkl pol_2/ policy_state.pkl learner/ learner_state.json module_state/ module_1/ ... optimizer_state/ optimizers_module_1/ ... rllib_checkpoint.json algorithm_state.pkl Note: `rllib_checkpoint.json` contains a "version" key (e.g. with value 0.1) helping RLlib to remain backward compatible wrt. restoring from checkpoints from Ray 2.0 onwards. Args: checkpoint_dir: The directory where the checkpoint files will be stored. ) use_msgpackNrrrzalgorithm_state.pklwbrrr cloudpickle)rrr state_filer ray_version ray_commitTr*rr)rEr)rr.rr save_to_path_use_msgpack_checkpointspathlibPath __getstate__rrQrPopenpickledumpjsonrrrr __version__ __commit__rr-rrrr]rjoinr) r;r_rrrdfpidrE policy_dirrlearner_state_dirs rsave_checkpointzAlgorithm.save_checkpoint s:&d&H I I? C? C{7 !!" $ D"? C? C? C? C? C? C? C? C%\.99N%%''EM5  _h%G%G %h 3 3OR H H {7 A.W*++.@*+(*??Jj$'' &1 E1%%% & & & & & & & & & & & & & & &n'>>DD  +.1%8L2M.N.N"/&)*oo&*=+=+=+?+?&@&@'*&)n                   &3%8%8%:%: P P!\"3d3333+j83>  J6666--((,(OOOO{7 C$&GLL$K$K!"//0ABBB? C? C? C? C? C? C? C? C? C? C? C? C? C? C? C? C? C? Csb/I+B I+C=1 I+=D I+D I+A1F I+F I+F B>I++I/2I/c t|j5|jjr||n>t |}t |}||td|j |jj t|ddddS#1swxYwYdS)Non_checkpoint_loaded)rr1) rr-rrrestore_from_pathrSrrrTr rcallbacks_on_checkpoint_loadedr)r;r_rcheckpoint_datas rload_checkpointzAlgorithm.load_checkpoint s %d&H I I  {7 3&&~6666#6n"E"E"+"O"O###!!/222 &"&.$(K$Nd+++                       sBB33B7:B7)not_componentsr~r~c |jjstdi}|jjr|jr|t ||re|jjd|t|t|t|tgzd||t <nz|t||r!|j |t<|t||r!|j|t<|jr|t ||re|jjd|t|t|t|tgzd||t <|t"||rO|jjd|t"||t"|d||t"<|j|t(<|j|t,<|S)Nz_Algorithm.get_state() not supported on the old API stack! Use Algorithm.__getstate__() instead.)r~r~r)rrrrr_check_componentr#r _get_subcomponentsr*rGr$rr)rr[r%r'rrr(training_iterationr)r;r~r~rrs rr zAlgorithm.get_state s{= 8   ;  A A(((*n3L$/2K 3#'#:#:/$$(2 334GXX(( // (0 3 3! 3 3E./((5z>A 4>>@@9((5z>A 4>>@@9   D$9$9 %z>% % 0Nt/C/M 0223F SS)++,?PP   ''  ( 0 0 0 0E+ ,  !8*n U U -IT-?-I.223JJWW#66+^  ..  ..E) *+/,*@*@*B*B&'%)$; ! rct|vr|jr&|j|tn`|j|tt|j|tt |j|j |j|j ttfd|j|j|jrt|vr|jr%|j|t|j|j|j|j ttfd|j|jt$|vrn|j|t$|j|jd|jr!|j|jdt*|vr%|j |t*t,|vr|t,|_dSdS)NrrrATr.)r#rrrr$rr)rrrrrr]rmrr%r[rr'rr9r(r _iterationr;rs rrzAlgorithm.set_stateS sd 5 ( ( ))%0D*EFFFF,66./0QR,66./0QR  ! 8 8{ O"&,"3"3')GHRS#4###:": 9     % *Cu*L*L# L$..u5I/JKKK  & = =- O"&,"3"3')GHRS#4###:": >    #e + +   ( (/F)G H H H  ! . .-1-?# /   ) *77151C#'8 $u , , L " "5)A#B C C C  & &#$67DOOO ' &rrRct|jfg}|jjr)|jr"|t |jfnc|jjrW|jsP|jr!|t|jf|j r!|t|j f|j r!|t|j f|SrY) r'rrrrrar#rr$rr)r[r%)r;r~s rget_checkpointable_componentsz'Algorithm.get_checkpointable_components s%d&8 9  ; T_    %t7    [ " 4? + !!68TU+ !!68TU      -(    rc:|jfifSrY)rr rVs rget_ctor_args_and_kwargsz"Algorithm.get_ctor_args_and_kwargs s%[ " " $ $ &   rctj|g|Ri|tj|}|tz sd|vr0t|dvr!|j|jd|j dkr|j s|tz r|j j|tz g|Ri||tz r|jj|tz g|Ri||j|jd|jt&t(fd|j |jdS|j dkrV|j rQ|j|j|jt&t(fd|j dSdSdS)N componentTr.rrrA)rrrB)rrzrirjr'is_dirrr9rnum_remote_env_runnersrr$rr)rrrrrr]rm)r;rargsrrs rrzzAlgorithm.restore_from_path sf "!$8888888|D!! * * 2 2 4 4  6 ! !&= AT&T&T  ! . .-1-?# /     7 7 9 9A = =do =88@@BB >,><<?CGM88@@BB >,><<?CGM  ! 8 8{ "&,"3"3')GHRS#4###:": 9      " 9 9 ; ;a ? ?DO ?  ! 8 8{"&,"3"3')GHRS#4##!O 9     @ ? ? ?rresultc t|j5td|j|jjt ||j|dddn #1swxYwYtj ||dS)Non_train_result)rrrr1) rr/r rrcallbacks_on_train_resultrrr log_result)r;rs rrzAlgorithm.log_result s&d&Q R R   !"&.$(K$I"#'<!                    T6*****s8AA Act|dr |j|jt|dr |j|jt|dr |j|jt|dr"|j|jdSdSdS)Nrrrr)rrshutdownrclearrstoprrVs rcleanupzAlgorithm.cleanup s 4 ) ) *d.@.L   ' ' ) ) ) 44 5 5 3  * 6  * 0 0 2 2 2 4+ , , )1F1R  ! & & ( ( ( D1 2 2 .*6  & + + - - - - -  . .66rc||}|||}|||jrt |}n!|j|jrdn|js|j ndd}t|}| |rt|}ng}| |rt|}ng}g}|jrt|}|g|z|z|z|z}t||jS)Nr)CPUGPU)bundlesstrategy)rrr rrrrnum_cpus_for_main_process _fake_gpusnum_gpusrrrrrrplacement_strategy) rr eval_config main_processenv_runner_bundleseval_env_runner_bundlesoffline_eval_runner_bundleslearner_bundlesrs rdefault_resource_requestz"Algorithm.default_resource_request s ''))::6BB 99;;   . 3F;;LL7(AA">  L5V<<  4 4[ A A )&=k&J&J # #&( #  8 8 E E -*J;*W*W ' '*, '  . ;26::ON  !% &* *   %.    rcdS)zPre-evaluation callback.NrrVs rrPzAlgorithm._before_evaluateF s  r env_specifiercttr`tjtrtjtfSdvr fd}|fSt jtfSttrЉ}|drtj d}tj |tj dkrtdtjd Gd d |fd fSt#t$jr|t jtfS|fd fSddfStddz)a}Returns env_id and creator callable given original env id from config. Args: env_specifier: An env class, an already tune registered env ID, a known gym env name, or None (if no env is used). config: The AlgorithmConfig object. Returns: Tuple consisting of a) env ID string and b) env creator callable. rc t|}n/#t$r"ttjwxYw|SrY)rYrrWrVr) env_contextenv_objrs renv_creator_from_classpathzEAlgorithm._get_env_id_and_creator..env_creator_from_classpathb s["-m["I"I%&:A-PP#Ns ,A)env_descriptorremote_worker_envsgymz0.22zCannot specify a gym.Env class via `config.env` while setting `config.remote_worker_env=True` AND your gym version is >= 0.22! Try installing an older version of gym or set `config.remote_worker_env=False`.rs)rceZdZdZdZdS)3Algorithm._get_env_id_and_creator.._wrapperc|j|jfSrY)rrrVs r _get_spacesz?Algorithm._get_env_id_and_creator.._wrapper._get_spaces s#5t7HHHrc.ddlm}t||S)Nr) MultiAgentEnv)ray.rllib.env.multi_agent_envrr)r;rs r_is_multi_agentzCAlgorithm._get_env_id_and_creator.._wrapper._is_multi_agent s$OOOOOO)$ >>>rN)r __module__ __qualname__rrrrr_wrapperr s5III?????rrc.|SrY)r)rrs rrz3Algorithm._get_env_id_and_creator.. s8??3+?+?rc|SrYr)rrs rrz3Algorithm._get_env_id_and_creator.. s==+=+=rNcdSrYr)rs rrz3Algorithm._get_env_id_and_creator.. sDrz {} is an invalid env specifier. zlYou can specify a custom env as either a class (e.g., YourEnvCls) or a registered env id (e.g., "your_env").)rrrcontainsrrrrr6rrmetadatarparserrrrrEnvr)rrrenv_id gym_versionrs` @rrz!Algorithm._get_env_id_and_creatorK s  mS ) )H (mDD $&6&:; &V&VVV %%#####%&@@@%i&7$]''' t , ,0 "F*+" >(088?? =--v1F1FFF$4Q''' ? ? ? ? ?} ? ?(' ???????M3733 >y0$#0    ====== "000 0299-HHPP rrrc|r:|jdkr1tj|j||j|j|jdSdSdS)agSynchronizes the filter stats from `workers` to `central_worker`. .. and broadcasts the central_worker's filter stats back to all `workers` (if configured). Args: central_worker: The worker to sync/aggregate all `workers`' filter stats to and from which to (possibly) broadcast the updated filter stats back to `workers`. workers: The EnvRunnerGroup, whose EnvRunners' filter stats should be used for aggregation on `central_worker` and which (possibly) get updated from `central_worker` after the sync. config: The algorithm config instance. This is used to determine, whether syncing from `workers` should happen at all and whether broadcasting back to `workers` (after possible syncing) should happen. NoFilter) update_remoteruse_remote_data_for_updateN)observation_filterrE synchronizefiltersupdate_worker_filter_stats)sync_filters_on_rollout_workers_timeout_suse_worker_filter_stats)r;rrrs rr$z!Algorithm._sync_filters_if_needed sh.  f7:EE  %&$? & P+1+I         EErc d|S)Na You can adjust the resource requests of RLlib Algorithms by calling `AlgorithmConfig.env_runners(num_env_runners=.., num_cpus_per_env_runner=.., num_gpus_per_env_runner=.., ..)` and `AgorithmConfig.learners(num_learners=.., num_gpus_per_learner=..)`. See the `ray.rllib.algorithms.algorithm_config.AlgorithmConfig` classes (each Algorithm has its own subclass of this class) for more info. The config of this Algorithm is: rr s r resource_helpzAlgorithm.resource_help s 917 9 9 rnowtime_this_iter timestampdebug_metrics_onlyc t||||}d|vrtdt|dts:t|dt sJ|d|d<|S)Nrz3`config` key not found in auto-filled results dict!)rget_auto_filled_metricsr\rrrr )r;rrrr auto_filledrs rrz!Algorithm.get_auto_filled_metrics sgg55 ,>   ; & &PQQ Q+h/66 Dk(3_EE E EE$/$9$A$A$C$CK !rconfig1config2_allow_unknown_configsctj|}d|vr.t|dturt ddd||j}t ||||j|j|j S)aMerges a complete Algorithm config dict with a partial override dict. Respects nested structures within the config dicts. The values in the partial override dict take priority. Args: config1: The complete Algorithm's dict to be merged (overridden) with `config2`. config2: The partial override config dict to merge on top of `config1`. _allow_unknown_configs: If True, keys in `config2` that don't exist in `config1` are allowed and will be added to the final config. Returns: The merged full algorithm config dict. rzcallbacks dict interfaceza class extending rllib.callbacks.callbacks.RLlibCallback; see `rllib/examples/metrics/custom_metrics_and_callbacks.py` for an example.Tr*) rdeepcopyrrrrrF_allow_unknown_subkeys%_override_all_subkeys_if_type_changes_override_all_key_list)rrrrs rrz!Algorithm.merge_algorithm_configs s.-(( ' ! !d7;+?&@&@D&H&H *      " )%(%? "   "  &  5  &    rrrcdS)aEnv validator function for this Algorithm class. Override this in child classes to define custom validation behavior. Args: env: The (sub-)environment to validate. This is normally a single sub-environment (e.g. a gym.Env) within a vectorized setup. env_context: The EnvContext to configure the environment. Raises: Exception: in case something is wrong with the given environment. Nr)rrs rrhzAlgorithm.validate_envs " r TrainIterCtxc Tt|j5|jtt f5|jddkr2t st d}t|5}| |sE|jttf5||j}|r"||j|j|dddn #1swxYwY|jtt$f5t|j5|}dddn #1swxYwYd}dddn #1swxYwY|t+d|jt.d d | |Edddn #1swxYwYdddn #1swxYwY|jjrA|jd d ddd}|j|t8|j}dddn #1swxYwY||fS)a@Runs one training iteration (`self.iteration` will be +1 after this). Calls `self.training_step()` repeatedly until the configured minimum time (sec), minimum sample- or minimum training steps have been reached. Returns: The ResultDict from the last call to `training_step()`. Note that even though we only return the last ResultDict, the user still has full control over the history and reduce behavior of individual metrics at the time these metrics are logged with `self.metrics.log_...()`. frameworktf2Falgorrrestored_env_runner_indicesNTa<`Algorithm.training_step()` should NOT return a result dict anymore on the new API stack! Instead, log all results, timers, counters through the `self.metrics` (MetricsLogger) instance of the Algorithm and return None. The logged results are compiled automatically into one single result dict per training iteration.rsr)reducec*|SrY)rrs rrz7Algorithm._run_one_training_iteration..nsu'8'8':':rrr)rrrrignore_ray_errorsru)rr%rr"r}r~rrtfexecuting_eagerlytf1enable_eager_executionr should_stoprvrr(_make_on_env_runners_recreated_callbacksrr(rr log_valuersrrforeach_actor_async_fetch_readyryrZcompile)r; has_run_oncer)rtraining_step_return_valueremote_aggregator_metricscompiled_metricss rrz%Algorithm._run_one_training_iteration(s`&d&S T TI 6I 6&&0H'IJJ5 5 ;??;//588AUAUAWAW8..000$ !t,,,,-88FF)!\22FB=O=O=Q=Q :RRRRRRRRRRRRRRR,0L 0000000000000006A",!Q## ..A#(/K-88FF),,,,,,,,,,,,,,,5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 n{< ,0,J,j,j::!$'$)&+ -k--) &&-0' $|3355 SI 6I 6I 6I 6I 6I 6I 6I 6I 6I 6I 6I 6I 6I 6I 6V //s'JA#H=H?D'  H'D+ +H.D+ /)HF' -F  F' FF' FF'  H'F+ +H.F+ /A H; HH HH H JH" "J%H" &A)JJ"Jc P|j|jttf5||j}|r7t d|j|jj t||j|dddn #1swxYwY|jttf5| }dddn #1swxYwY|j-|jj |d<|jj|d<|jj|d<t |iS)zRuns offline evaluation step via `self.offline_evaluate()` and handling runner failures. Returns: The results dict from the offline evaluation call. N!on_offline_eval_runners_recreated)rrenv_runner_indicesr1 num_healthy_offline_eval_runners8offline_runners_actor_manager_num_outstanding_async_reqs(num_remote_offline_eval_runners_restarts)rrr"r}rxrr rr+callbacks_on_offline_eval_runners_recreatedrrur?r;num_in_flight_async_reqsnum_remote_runner_restartsrb)r;rrs rr z%Algorithm._run_one_offline_evaluations  ) 5&&0R'STT  <<2 !;*.. KS#&*-1-K/7                       (\ " "F,N#O P P 3 30022L 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3  ) 5.I 2  .G J  .I : #L11s$ABBB?C  C$'C$clt|j5|j|jjrY|jttf5| |j}dddn #1swxYwYn?|j d5| |j}dddn #1swxYwY|r"| |j |j||jjrU|jttf5||}dddn #1swxYwYnu|j t5||}dddn #1swxYwY|j t|jt"|jT|j|d<|j|d<|j|d<dddn #1swxYwYt*|iS)a@Runs evaluation step via `self.evaluate()` and handling worker failures. Args: parallel_train_future: In case, we are training and avaluating in parallel, this arg carries the currently running ThreadPoolExecutor object that runs the training iteration. Use `parallel_train_future.done()` to check, whether the parallel training job has completed and `parallel_train_future.result()` to get its return values. Returns: The results dict from the evaluation call. Nrestore_eval_workersrr r(actor_manager_num_outstanding_async_reqsnum_remote_worker_restarts)rr&rrrrr"r}rwrrrrraropush_units_processedr!rlrZrrrb)r;r rrs rrzAlgorithm._run_one_evaluations &d&K L L- L- L)5;AX..!?@XX$(#;#;D .IIKK0W- L- L- L- L- L- L- L- L- L- L- L- L- L- L- L^#L11s:H"A6* H"6A: :H"=A: >H"B6* H"6B: :H"=B: >AH"D:. H":D> >H"D> H"E;/ H";E? ?H"E? BH""H&)H&cNtj5}jjr|fd}n|fd}|}|\}}dddn #1swxYwY|||fS)a8Runs one training iteration and one evaluation step in parallel. First starts the training iteration (via `self._run_one_training_iteration()`) within a ThreadPoolExecutor, then runs the evaluation step in parallel. In auto-duration mode (config.evaluation_duration=auto), makes sure the evaluation step takes roughly the same time as the training iteration. Returns: A tuple containing the training results, the evaluation results, and the `TrainIterCtx` object returned by the training call. c,SrY)rrVsrrzRAlgorithm._run_one_training_iteration_and_evaluation_in_parallel..sD<<>>rc,SrY)rrVsrrzRAlgorithm._run_one_training_iteration_and_evaluation_in_parallel..sDJJLLrr N) concurrentfuturesThreadPoolExecutorrrsubmitrr)r;executorr evaluation_resultsrr)s` rrz@Algorithm._run_one_training_iteration_and_evaluation_in_parallels  2 2 4 4 K{= (0>>>>))%%)1LLLL))%"&!9!9&;":"" -B,H,H,J,J )M>% K K K K K K K K K K K K K K K(0.@@sA+BBBct|jjjdksJ|jjpd}dii}|jD]*\}}||j ||d|<+|S)a6Runs offline evaluation via `OfflineEvaluator.estimate_on_dataset()` API. This method will be used when `evaluation_dataset` is provided. Note: This will only work if the policy is a single agent policy. Returns: The results dict from the offline evaluation call. rsrI) n_parallelism) rrrjrLrrrrestimate_on_datasetr)r; parallelismr*evaluator_nameoffline_evaluators rrQz/Algorithm._run_offline_evaluation_old_api_stacks4(9DEEJJJJ,GL1 6;151G1M1M1O1O   -N-"55')6 !!7 8   $#rrrcJ|jo|j }| o|jdkp|jS)aDetermines whether we need to create evaluation workers. Returns False if we need to run offline evaluation (with ope.estimate_on_dastaset API) or when local worker is to be used for evaluation. Note: We only use estimate_on_dataset API with bandits for now. That is when ope_split_batch_by_episode is False. TODO: In future we will do the same for episodic RL OPE. r)rrrr)rrrun_offline_evaluations rrz/Algorithm._should_create_evaluation_env_runners,sE  5 ;:: *)  2Q 6 /. rc*|jdup |jdkS)z@Determines whether we need to create offline evaluation workers.Nr)rr)rrs rrz3Algorithm._should_create_offline_evaluation_runnersBs$  34 ? 83a7 rc~t|j5|jrtd|}|t itd|t<|rHt|trt|dkr t|vsJ| ||j rY|j |j d|t <d|j i|d<dddn #1swxYwY|S)Na`Algorithm._timers` is no longer supported on the new API stack! Instead, use `Algorithm.metrics.log_time([some key (str) or nested key sequence (tuple)])`, e.g. inside your custom `training_step()` method, do: `with self.metrics.log_time(('timers', 'my_block_to_be_timed')): ...`rrs)rrrr)rr'rrrrr]rmrrrrbrrrZrrcr)r;rrr+s rr#z$Algorithm._compile_iteration_resultsKs %d&R S S* * |  \#0"4"4"6"6G7>kk"B77c0!44 2 3  -|T22L))Q..*l:::;|,,,$ -HHJJ-HHJJ 22-.?-FFHH/*+M* * * * * * * * * * * * * * * XsDD22D69D6c td|j|jt||||jtd|jt||||jdS)Non_env_runners_recreated)rrr is_evaluationr1on_workers_recreated)rr worker_idsr)r2r)r r"callbacks_on_env_runners_recreatedr in_evaluation)r;rrrs rrz2Algorithm._make_on_env_runners_recreated_callbackszs  &"n!'!J!1#>$2    ""n+6$2  rc |jjrKt|jd|jjd|jjd|jjd|jjd St|jS)Nz(env=z; env-runners=z ; learners=z; multi-agent=))rrrrrrrrrVs r__repr__zAlgorithm.__repr__s ; 3 '::&{6:k6Q K4 ${9 ::& &rc,|jr |jjSdS)z>The local EnvRunner instance within the algo's EnvRunnerGroup.N)rrjrVs rrzAlgorithm.env_runners    :(9 9trc,|jr |jjSdS)zIThe local EnvRunner instance within the algo's evaluation EnvRunnerGroup.N)rrjrVs rr[zAlgorithm.eval_env_runners   % ?-> >trcttj|dttjt |d|jj}|tvrd}ttj|dS)zkRecord the framework and algorithm used. Args: config: Algorithm config dict. rr USER_DEFINEDN) rrRLLIB_FRAMEWORKRLLIB_NUM_WORKERSrrrALL_ALGORITHMSRLLIB_ALGORITHM)r;ralgs rrzAlgorithm._record_usagesp v5vk7JKKKv7VDU=V9W9WXXXn% n $ $ Cv5s;;;;;rexport_formatsc tj|i}tj|vrNtj|tj}||||tj<tj|vrNtj|tj}||||tj<tj |vrptj|tj }||ttj dd||tj <|S)N ONNX_OPSET11)rW) rr  CHECKPOINTrrrrr^MODELrZONNXr getenv)r;r/rVexportedrs r _export_modelzAlgorithm._export_models  n---  "n 4 47<< L,CDDD  ) )$ / / /04H\, -   / /7<< L,>??D  $ $T * * *+/H\' (   . .7<< L,=>>D  $ $TBIlD4Q4Q0R0R $ S S S*.H\& 'rc|jjrtdt||jd}t |dr!|jj|d<t |dr|j|j j |d<|j |d<|j 6|j d r|j |d <|j|t<|S) a Returns current state of Algorithm, sufficient to restore it from scratch. Returns: The current state dict of this Algorithm, which can be used to sufficiently restore the algorithm from scratch without any other information. zgAlgorithm.__getstate__() not supported anymore on the new API stack! Use Algorithm.get_state() instead.)rSrrrrNeval_policy_mapping_fncountersstore_buffer_in_checkpointsr)rrrrr rrrjrr[rr!rrrrrs rrkzAlgorithm.__getstate__s ; 9 5  $Dzzk++--   4+ , , Q"3DNNPPE(O D1 2 2 U*6.2.B.TE* +!Nj  # /DKOO )5 5 /,0+C+M+M+O+OE' (%)$; ! rc|jjrtdt|drd|vr|dr|j|dt j|d|j fdd|j r5| dfd}|j ||j y|jj r@d |vr!|j |d nHtd n-d |vr)t!d rtd d|vr |d|_t$|vr|t$|_d Sd S)zSets the algorithm to the provided state. Args: state: The state dict to restore this Algorithm instance to. `state` may have been returned by a call to an Algorithm's `__getstate__()` method. zgAlgorithm.__setstate__() not supported anymore on the new API stack! Use Algorithm.set_state() instead.rrcR|tjSrYr)rremote_state_refs rrz(Algorithm.__setstate__..s!++cg.>&?&?@@rFrQr:c|tj|dSrY)rrrset_policy_mapping_fn)r_eval_policy_mapping_fnr?s r_setup_eval_workerz2Algorithm.__setstate__.._setup_eval_workers=KK(8 9 9:::++,CDDDDDrNrzI`store_buffer_in_checkpoints` is True, but no replay data found in state!-no_store_buffer_in_checkpoints_but_data_foundzL`store_buffer_in_checkpoints` is False, but some replay data found in state!r;)rrrrrrrrrrrrrr<rr_rr!rr)r;rrCrBr?s @@rrTzAlgorithm.__setstate__s ; 9 5  4+ , , RU1B1BuX1B O % %eHo 6 6 6"wuX77   ! 4 4@@@@!& 5   ) R*/))4L*M*M'EEEEEE*==>PQQQ  # /{6 (E11,66u=R7STTTTNN/'%//H?55/+   ":.DN  & &#$67DOOO ' &r)rrrrc f|ddkrtd|ddd}|ddkrtd }t|d d 5}|,|}||d }nt j|}dddn #1swxYwY|dtj dkr|d|drn|d}tn|dfd|d D|d<t|dtr-t|dpt|d|d<|d} t| t"r| |d} n!t&| |d} | j} t| tt,t.frfd| D} n fd| D} | jd&| |d|d|ini| |d<i|d<D]} t2j|dd| d|ddkrdndz} t2j| std | d!| d"t| d 5}||||d| <nt j||d| <dddn #1swxYwY|||d<||d#t:kr||d#<|djr)t2j|dd$|d%<|S)'aConverts a checkpoint info or object to a proper Algorithm state dict. The returned state dict can be used inside self.__setstate__(). Args: checkpoint_info: A checkpoint info dict as returned by `ray.rllib.utils.checkpoints.get_checkpoint_info( [checkpoint dir or AIR Checkpoint])`. policy_ids: Optional list/set of PolicyIDs. If not None, only those policies listed here will be included in the returned state. Note that state items such as filters, the `is_policy_to_train` function, as well as the multi-agent `policy_ids` dict will be adjusted as well, based on this arg. policy_mapping_fn: An optional (updated) policy mapping function to include in the returned state. policies_to_train: An optional list of policy IDs to be trained or a callable taking PolicyID and SampleBatchType and returning a bool (trainable or not?) to include in the returned state. Returns: The state dict usable within the `self.__setstate__()` method. rrzu`checkpoint` arg passed to `Algorithm._checkpoint_info_to_algorithm_state()` must be an Algorithm checkpoint (but is z)!NrrTr*rdrbF)rawrrrrc$i|] \}}|v || Srr)r_rtfilterrs rrfzAAlgorithm._checkpoint_info_to_algorithm_state..s4'''C*$$V$$$rrrSrch|]}|v| Srr)r_rtrs r z@Algorithm._checkpoint_info_to_algorithm_state..s#QQQsj?P?P?P?P?Prc$i|] \}}|v || Srr)r_rtspecrs rrfzAAlgorithm._checkpoint_info_to_algorithm_state..s0   "+#tsjGXGXCGXGXGXr)rrrrr_rz policy_state.msgpckpklzOGiven checkpoint does not seem to be valid! No policy state file found for PID=z. The file not found is: ris_policy_to_trainrrvr)rrrTrlreadunpackbrmloadrrrrrrrrrrrrrrrtupler/rrrrisfilerr)rrrrrrsdatar worker_stater< new_config new_policiesrtpolicy_state_files ` rrz-Algorithm._checkpoint_info_to_algorithm_stateAsH 6 "k 1 1L0?0GLLL     x ( (I 5 5(t444G /,/ 6 6 '!"vvxx%88 A  ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 0 1GOE4J4J J J (##/ (##0!?L(4 ,|:TJ ''''#/ #:#@#@#B#B'''L #% 12C88 A,<+,,,,A&u->'?@@'(##45HHJJN./:: +<>"E(O &.L,dE(:;; QQQQ|QQQ     /;/A/A/C/C     #J " %"3   )4)*;<<    )E(O-/L )! L L$&GLL#$45##28#< #I#IxxuV %%!w~~&788$G47GG2CGGG +T22La*=D\\!__ _5c::=C[^^ _5c: LLLLLLLLLLLLLLL!,4E 01!- 459III5F 12 ? 7 )+ 019**E% & s%(AB77B;>B;?MM M c|dr|ddrdSd|ddvr|jj|dd<tt|dS)aCreate a MultiAgentReplayBuffer instance if necessary. Args: config: Algorithm-specific configuration data. Returns: MultiAgentReplayBuffer instance based on algorithm config. None, if local replay buffer is not needed. rno_local_replay_bufferNEpisodeReplayBufferr"metrics_num_episodes_for_smoothing)rrr^rYr)r;rs rrz2Algorithm._create_local_replay_buffer_if_necessaryszz011 V ) *4 <0F)GHHHrc|jt5|jddkr2tst i}d}t|5}| |s|jd5| |j }|r"| |j|j |dddn #1swxYwY|jt5|}dddn #1swxYwY|r|}| |dddn #1swxYwYdddn #1swxYwY||fS)Nrrrrestore_workersr)rr~rrrrrrrrrrrrr)r;r+training_step_resultsr)rs rrz3Algorithm._run_one_training_iteration_old_api_stacks \2 3 8 8{{++u44R=Q=Q=S=S4**,,,G$( !4((( 8N(445JKK8&78#'#;#;D>L  ! !"4d ; ; ;BN NGL !G,>$?%6$9$9:JB$O$O !#4#8#8"#M#M ():; 2!!!4+@3 D D   Q;;#'#8'#CFX#X ());ABBBB $$%7888 $ 5 [ ; ; = =! '9  K 7' ' "#  ! < < > >  !  ! : : < <  6  ! < < > >  ( $ # ! !   + +A *GAJJ!4466 ; % 6 6>F>NG+l: ;>F>NG+l: ; >1 +.CC D >1 +.CC D*.8O)PG% & &1 ),AA B >1 ),AA B*.8M)NG% &3::K2L./48N #5 01 4<3C/0+/>:Q+R'( **,, T THAu-25:3Da-H-HF<&&q)) *((** T49%:OQR4S4S--a001"...00 " "JAw!HQKK& x(((rz`Algorithm.compute_single_action` should no longer be used. Get the RLModule instance through `Algorithm.get_module([module ID])`, then compute actions through `RLModule.forward_inference({'obs': [obs batch]})`.)rgr) prev_action prev_rewardr input_dictr= full_fetchexploretimestepepisodeunsquash_action clip_action observationrxryrrzr{r|r}rrc | |jj} n| |jj} d}|%|||| J||tj}n | J|||}|t d|d|jt}t|tj ttfsD tj|}n.#t$r!t!dt#|dwxYw|rt%|dks Jd|d}|s||tj|i}n0|tj|tji}nt!d t+d d |}|d ||gd}|jtj}|-||tj<||| | | \}}}n!||||||| | | \}}}| rt3j||j}n| rt3j||j}|s|r|||fS|S)NzcProvide either `input_dict` OR [`observation`, ...] as args to `Algorithm.compute_single_action()`!z PolicyID 'z9' not found in PolicyMap of the Algorithm's local worker!zObservation type z# cannot be converted to np.ndarray.rsz/Only one preprocessor should be in the pipelinerz2Either observation or input_dict must be provided.0)r)rzr|r}r~)obsrrxryrr|r}r~)rnormalize_actions clip_actionsr,OBSrr\agent_connectorsr!rrLndarrayrrTasarray Exceptionrrr is_identityin_evalrresetrVcompute_single_actionrraction_space_structr)r;rrrxryrrzr=r{r|r}r~rrerr_msgrpp _input_dictacdac_oactionextras rrzAlgorithm.compute_single_actions6  ""k;OO  +2K ;   !#''MM"M" %W[1KK**G***++ >-Y---  $%= >+ D%'@AA   j55     #[(9(9###   5r77a<<>## 5 *#*; ".0s 000QRXa[[000r state_in_)rzr|r}rc g|] }| Srr)r_ridxs rr`z-Algorithm.compute_actions..Os)A)A)AQ!C&)A)A)Ar)rrrrrrrj preprocessorsr transformrraget_initial_staterLrrzipr,rrD PREV_ACTIONS PREV_REWARDSINFOSrcompute_actions_from_input_dictrunbatchrrr)"r;rrrxryrr=r{r|r}rrr state_definedr filtered_obsfiltered_stateagent_idobr preprocessedfiltered obs_batchrzrractionsstatesinfossingle_actionsrEaunbatched_statesrs" @rcompute_actionszAlgorithm.compute_actionss.  ##{<    !;3LT) ++')2n (..00 B BLHb*;F#'' 22>%3I>HHLL ! 0v~i0eLLLH    ) ) )}U""%%eHo6666%%f&>&>&@&@AAAAH\** =EEn-..E00%000Ek9-  "3>J{/ 0  "3>J{/ 0  -(,Jw} %e$$ , ,DAq*+J1 ' '!'!G!G! "H" " %,W55,77  FC K/63MNN K+Av/IJJGCLL&|44 B BMC)A)A)A)A&)A)A)A X & &  J ,e3 3NrzAlgorithm.restore_env_runners)rrcdSrYr)r;rrs rr`zAlgorithm.restore_workersVs rzAlgorithm.env_runner_groupc|jSrY)rrVs rrzAlgorithm.workersZs $$rzAlgorithm.eval_env_runner_groupc|jSrY)rrVs rrFzAlgorithm.evaluation_workersbs ))rrY)NNN)rN)NN)NNNF)rr)rrr__doc__rr r__annotations__rrrr5rrrrrrrr]r`rbrmrprrr__progress_metricsMETADATA_FILE_NAMESTATE_FILE_NAME classmethodrOrRrrrrrrrrrrrrrNrr:rWrLrr r r\rMrrrrArrr,r?r r rrorrXr=rYr]r<r^rrrrr+rr0r"r1r rrr/rrrKrCrrrArDrrmSpacerrrKrO staticmethodrrZr^rwr}rr rrrrrzrrrrrrIrPrrrr4r$rrrfloatrrrJr3rhrr rrrQrrr#rr%propertyrr[rr8rkrTrrrrr&rrrrDrrrr`rrF __classcell__)rs@rrrsh   )-FH_ %,,,(,GXm $+++26h~.5557;8N3:::/3M8N+222,0L(=)000#& -) >zJ 55 355 JJ 2JJ5HJJ )+ )+ " 22 022 1'O Xn9=e+ 6:PT #e+e+e+CO$e+45e+ Z12 e+$Hgy-A8-K$LMe+$ 8$(H_$=>DEG  e+" #e+e+e+e+e+[e+N-1 9= r r )r !"f*!56 r r r r r Yr hH H H T$!?!!![$#!$ :S : : :$# :; Xin On n n n ;:n ` $   $v,    [$# XiCjCCCCJ6;6;Y6;pRVJJ' (:(MNJ JJJYJX4U:sC;O5P44446   ">G>G>G@jGjGjGXR R R R hEGEGEGN$eNetCyeee$#eN$,,c,,,$#,\$GGG$#GR/@HXhEWY2 ,0KOJN $#'(,F$F$F$F$"F$ #4. F$ )11G(H F$'//F&GF$F$!F$"&F$ F$F$F$YF$P LPJN%)(,-1i$i$i$i$)11G(H i$ '//F&G i$ # i$"&i$'+i$ & i$i$i$Yi$V/@55H5V555[5 5 5HT(^$< 5 5 5 5Y 5D4$#7DDDYD(.2#' {> 9=37OS.2PT #'(,.2+(-{>{>{>{>T&\*{> {> $CJ$45 {>sz/0{>0JJKL{>{+{>$Hgy-A8-K$LM{>$ 8$(H_$=>DEG  {>"!#{>$"&%{>&l+'{>. & /{>{>{>[{>z0?UFJ (,-1+-?U?U?U?U$HgY-@$AB ?U $ 8$(H_$=>DEG  ?U"&?U'+?U !?U?U?U[?UB$;\[@0" BBBBsm B  BBB[B"0----  ---[-,Xi[Cc[Cd[C[C[C[CzXicd.Xn=AOAE OOOU3 3#789O!sJsO';!<= O  OOOObXn78y78T78787878rXntE#?O:O4P/Q4Xn %tCH~0E*F    Xn11111fXi++++++*Xi....($ Xi3 ?,FFG3 y// 03 3 3 [$#3 j  \ VS'4/0V:IV x}j( )VVV\Vp"     @ Xi  5:M)M#N  SV    [  Xi#'*.#'#(  h !C=  !   , 26 ) ) $) ,) !) )  ) ) ) [) V '  t   _\ "W0U:~3M-NW0W0W0W0r.2.2.2dRV?2?2' (:(MN?2 ?2?2?2?2B"A z:~5 6"A"A"A"AH$$$,   [ *   [ ---^   < ' ' 'X X < < <"3i58 c3h[&+d+++[+Z<8<8<8[<8|6:PT VVVVZ12V$Hgy-A8-K$LM V $ 8$(H_$=>DEG  V VVV\[VpI0I ( )III[I:''['6tt[tlZN 3726h 37'+&*,0/ "&"&*.&*hhh./h-./h ./ h e_ h{#h[)hhh$h3-h"$hd^h  Z 0$sJ2GGH J !hhh [hTZF 37P 3726&*/ "&"&+/'+PPP&P-./P ./ P ./ P{#PPP$P3-P#4.PtnPPP [PdZ34@@@  A@ Z (%%X  %Z -**X  *****rrc8eZdZdefdZdZdZdefdZdZ dS) rrc0||_d|_d|_dSrY)r time_start time_stop)r;rs rr:zTrainIterCtx.__init__ls rcd|_tj|_d|_d|_|jjjr |jj ttfd|_ |jj tttfd|_t#|jj tt$fi|_t#|jj tt*fi|_np|jjt0|_ |jjt2|_|jjt4|_|jjt6|_|jjj|_|S)Nrrr)failuresrrrmrnrrrrrr]rminit_env_steps_sampledrdr[rpinit_env_steps_trainedrrgrinit_agent_steps_sampledrjinit_agent_steps_trainedr!rkrorfri-num_consecutive_env_runner_failures_tolerancefailure_tolerancerVs r __enter__zTrainIterCtx.__enter__qs )++  9  > Y*.)*;*@*@#%CDa+A++D '+/)*;*@*@ +/MN+A++D '-0 !&&')IJTV'&((--D ) -0 !&&$&FGQS'&((--D ) ) +/)*=>S*TD '*.)*=>S*TD ',0I,?@W,XD ),0I,?@W,XD ) I  J  rc6tj|_dSrY)rr)r;rs r__exit__zTrainIterCtx.__exit__srrc |j|jz S)z4Returns the time we spent in the context in seconds.)rrrVs rrlzTrainIterCtx.get_time_taken_secs~//rcZ|dvr:|xjdz c_|j|jkrtd|jddS|jjjrD|jjjdkrt|jj ttfi |j z |_t|jj tt fi |jz |_n!|jj tt&fd|jz |_|jj tt*t,fd|jz |_n|jjjdkrI|jjt2|j z |_|jjt4|jz |_nH|jjt6|jz |_|jjt8|jz |_|jjj}|jjj}|jjj}|rtAj |j!z |kr|r |j|kr|r |j|krd SdS) N)NFrsz9More than `num_consecutive_env_runner_failures_tolerance=z'` consecutive worker failures! Exiting.FrgrrT)"rrrrrrrrrrr]rgrrrmrdrjrrnrmrr[rprr!rfrirkromin_time_s_per_iteration"min_sample_timesteps_per_iteration!min_train_timesteps_per_iterationrr)r;r+min_t min_sample_ts min_train_tss rrzTrainIterCtx.should_stops m # # MMQ MM}t555"-5 9  >4 y.-?? )../1QR$&/!&((  3 4  )..,.NO$&/!&((  3 4 I%**+-KLVW+12 I%**(+7UV !+1 2 y.-??I'(?@34 I'(?@34 I'(=>12 I'(=>12   9 (K y'I  )++75@@"A&*lm&C&C!'D%)\\%A%A45rN) rrrrr:rrrrlrrrrrrks}Y ###J%%%0E0000TTTTTrr(r rrrimportlib.metadatarorrrirrr collectionsrrtypingrrrrr r r r r rrr gymnasiumrnumpyrL pyarrow.fspyarrowrb packagingrrray.cloudpicklercrmray._common.deprecationrrrray._common.usage.usage_librr ray.actorr%ray.rllib.algorithms.algorithm_configrray.rllib.algorithms.registryrr,ray.rllib.algorithms.utilsrrrrrray.rllib.callbacks.utilsr &ray.rllib.connectors.agent.obs_preprocr!*ray.rllib.connectors.connector_pipeline_v2r"ray.rllib.corer#r$r%r&r'r(r)r*r+ray.rllib.core.columnsr,ray.rllib.core.rl_moduler-(ray.rllib.core.rl_module.multi_rl_moduler.r/"ray.rllib.core.rl_module.rl_moduler0r1 ray.rllib.envr2ray.rllib.env.env_contextr3ray.rllib.env.env_runnerr4ray.rllib.env.env_runner_groupr5ray.rllib.env.utilsr6ray.rllib.evaluation.metricsr7r8ray.rllib.execution.rollout_opsr9ray.rllib.offliner:ray.rllib.offline.estimatorsr;r<r=r>r?#ray.rllib.offline.offline_evaluatorr@ray.rllib.policy.policyrArBray.rllib.policy.sample_batchrCrDray.rllib.utilsrErFrGray.rllib.utils.actor_managerrHray.rllib.utils.annotationsrIrJrKrLrMrNrOray.rllib.utils.checkpointsrPrQrRrSrTray.rllib.utils.debugrUray.rllib.utils.errorrVrWray.rllib.utils.frameworkrXray.rllib.utils.from_configrYray.rllib.utils.metricsrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~r$ray.rllib.utils.metrics.learner_infor&ray.rllib.utils.metrics.metrics_loggerr#ray.rllib.utils.metrics.ray_metricsrrrray.rllib.utils.replay_buffersrr$ray.rllib.utils.runners.runner_grouprray.rllib.utils.serializationrrray.rllib.utils.spacesrray.rllib.utils.typingrrrrrrrrrrrrrrrrrrray.train.constantsrray.tuner#ray.tune.execution.placement_groupsrray.tune.experiment.trialrray.tune.loggerrrray.tune.registryrrrray.tune.resourcesrray.tune.resultrray.tune.trainablerray.utilrray.util.metricsrrray.util.timerr$ray.rllib.core.learner.learner_grouprrrrrtfvrrrrrrrrr s      ######                               GFFFFFFF!!!!!!AAAAAATTTTTT433333KKKKKKJJJJJJ                      +*****777777FEEEEEEE******000000......999999000000HGGGGG444444A@@@@@66666666HHHHHHHHBBBBBBBBBBCCCCCCBAAAAAJJJJJJJJ333333333333''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''P>=====@@@@@@ POOOOOOO<<<<<<LLLLLLLL......(544444EEEEEE22222211111111NNNNNNNNNN((((((......((((((////////!!!!!!;AAAAAA::::::} R  8 $ $ TF*TF*TF*TF*TF* TF*TF* TF*nLFFFFFFFFFFr