&`iddlZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z mZmZmZmZddlZddlZddlmZddlmZddlZddlmZmZmZddlmZddl m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@ddlAmBZBmCZCddlDmEZEddlFmGZGmHZHmIZIddlJmKZKmLZLddlMmNZNmOZOddlPmQZQmRZRddlSmTZTddlUmVZVdd lWmXZXdd!lYmZZZm[Z[m\Z\dd"l]m^Z^dd#l_m`Z`dd$lambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZldd%lmmnZndd&lompZpdd'lqmrZrdd(lsmtZtdd)lumvZvdd*lwmxZxer*dd+lymzZzdd,l{m|Z|dd-l}m~Z~dd.lmZdd/lmZdd0l.mZdd1lamZejeZd2ekd3dfd4ZGd5d6etZGd7d8eZGd9d:eeZdS);N)Enum) TYPE_CHECKINGAnyCallable CollectionDictListOptionalTupleTypeUnion)version)Self)DEPRECATED_VALUE Deprecateddeprecation_warning) RLlibCallback) ConnectorV2DEFAULT_MODULE_ID)Columns)DifferentiableLearnerConfig)validate_module_id)DefaultModelConfig)MultiRLModuleSpec) RLModuleSpec)INPUT_ENV_SINGLE_SPACESINPUT_ENV_SPACES) MultiAgentEnv)is_atari)SampleCollector)SimpleListCollector)MODEL_DEFAULTS) InputReader) IOContext)Policy PolicySpecDEFAULT_POLICY_ID) deep_update force_list merge_dicts) OldAPIStack5OverrideToImplementCustomLogic_CallToSuperRecommended) try_import_tftry_import_torch) NotProvided from_config)DEFAULT_STATS_CLS_LOOKUP) StatsBase) Scheduler)NOT_SERIALIZABLEdeserialize_typeserialize_type)check)TORCH_COMPILE_REQUIRED_VERSION) AgentIDAlgorithmConfigDict EnvConfigDictEnvTypeLearningRateOrScheduleModuleIDMultiAgentPolicyConfigDictPartialAlgorithmConfigDictPolicyIDRLModuleSpecTypeSampleBatchType)Logger)get_trainable_cls) TRIAL_INFO)_Config)log_once)PlacementGroup Algorithm)Learner)DifferentiableLearner LearnerGroupTorchMetaLearner)RLModule) EpisodeType module_specreturnc~t|ttfs tdt |ddS)NzLrl_module_spec must be an instance of RLModuleSpec or MultiRLModuleSpec.Got z instead.) isinstancerr ValueErrortype)rVs y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm_config.py_check_rl_module_specr]esU kL2C#D E E  0 $$ 0 0 0     cM'eZdZdZedZedZedede fdZ edZ dd e e fd Zdefd Zdede fd Zdeeeffd Zedeeefdee effdZdeeeffdZdde ede fdZddZeddZ dde eeefde e ge!fdeddfdZ" dde#fdZ$dde#fdZ% dde#fdZ&dddddde ede ee'e(e)j*e)j*ffd e e+d!e d"dd#f d$Z,ddd%de ede ee-e(e)j*e)j*ffdd&fd'Z.d(e'de fd)Z/e0e0d*d+e ed,e ede fd-Z1e0e0e0e0e2e2e2e2e2e2e2e2d. d/e e3d0e ee4e3fd1e ed2e ede f d3Z5e0fe0e0e0e0e0e0e0e0e0e0e0e0e0d4 d5e ed6e ed7e e3d8e eeefd9e eeefd:e ed;e ede ed?e ed@e edAe eeefdBe ede fdCZ6e0e0fdDe edEe ede fdFZ7e0fe0e0e0e0e0e0e0e0e0e0e2dG de eeefdHe e8dIe e)j*dJe e)j*dKe edLe eee4fdMe edNe edOe edPe edQe ede fdRZ9e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e2e2e2e2e2e2e2e2e2e2e2dS-dTe e dUe e3dVe edWe edXe e3dYe eee)j:fdZe e3d[e ee4e3fd\e ed]e ed^e e4d_e e3d`e e egedae;daffdbe e edcgedae;daffdde edee edfe e3dge eeefdhe edie edje ee3efdke edle edme edne edoe edpe edqe ee0e0e0e0e0e0e0e0e0e0e0dy dze e3d{e eee4e3fd|e ee4e3fd}e e3d~e e4de e3de e3de eZkfd?Zldefd@ZmddAZnddBZodCZpdDZqdEZreVdeeeffdFZsddGZtddHZuddIZvddJZwdKZxdLZydMZzdNZ{dOZ|dPZ}dQZ~eVdefdRZeVdefdSZedTZeddUedVedefdWZdXZdYZedddddZde eIde ede ee-e(e)j*e)j*ffd[e e3// SSSSCLSSS--c4-HHC$) S ! !r^N algo_classc||_i|_i|_d|_d|_d|_d|_d|_d|_d|_ ddd didd didd |_ d d d |_ d|_ tj|_t jdkrdnd|_d|_d|_t jdkrdnd|_d|_i|_d|_d|_i|_d|_d|_d|_d|_d|_d|_ d|_!d|_"d|_#d|_$d|_%d|_&d|_'d|_(d|_)d|_*i|_+d|_,d|_-d|_.d|_/d|_0d|_1d|_2d|_3d|_4d|_5d|_6d|_7d|_8d|_9d|_:d|_;d|_<d|_=t||_?d|_@d|_Ad|_Bd|_Cd|_Dd|_Ed|_Fd|_Gd|_Hd|_Id|_Jd|_Kd|_Ld|_Md|_Nd|_Od|_Pd|_Qd |_Rd|_Sd|_Td|_U tjWt|_Yn#t$rYnwxYwd|_[d|_\i|_]i|_^d|__t|_ad|_bd|_cd|_dd|_ed|_fd|_gd|_hd|_id|_jd|_kd|_ld|_md|_nd|_od|_pd|_qt|d!sd|_si|_td|_ud|_v|wdd"d#|_xtti|_{d$|_||j}|_~d|_d|_d|_d%|_d|_d|_d&|_i|_i|_d|_d|_d|_d|_i|_tjtjg|_d|_d|_d|_i|_i|_d|_d|_d|_i|_d'|_d|_i|_d|_d|_d|_d|_i|_tjtjg|_d(|_d|_d|_d)|_i|_d|_i|_d|_d|_d|_d'|_d*|_d+|_d$|_d,|_d|_d|_d|_i|_d|_d|_d|_d|_d-|_d|_d|_d|_d|_d|_d|_d|_d+|_d|_d|_i|_d|_d|_d.|_d/|_d|_d|_d0|_d|_d|_d1|_d|_d|_d|_d$|_d|_d|_d|_d|_t|_d|_d|_d|_d|_d2|_d|_d|_d|_d|_d|_d.|_d|_d|_d$|_d0|_d/|_i|_d|_i|_i|_d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_t|_ t|_ t|_ t|_ t|_ t|_t|_t|_d|_t|_t|_t|_t|_t|_t|_t|_t|_dS)3aInitializes an AlgorithmConfig instance. Args: algo_class: An optional Algorithm class that this config class belongs to. Used (if provided) to build a respective Algorithm instance from this config. PACKrFtorchT allow_growthCPU)intra_op_parallelism_threadsinter_op_parallelism_threads gpu_optionslog_device_placement device_countallow_soft_placement)rrdarwin aot_eagerinductorNonnxrt action_masksyncgN@ training_onlytruncate_episodesdeepmindNoFilterautogGz?gMbP? global_norm exploration_configenable_rl_module_and_learner"enable_env_runner_and_connector_v2 env_stepsdsampler read_parquet i write_parquetepisodesg^@ig$@ig @g>@WARN(r extra_python_environs_for_driver extra_python_environs_for_workerplacement_strategynum_gpus _fake_gpusnum_cpus_for_main_process framework_str eager_tracingeager_max_retracestf_session_argslocal_tf_session_argstorch_compile_learnerTorchCompileWhatToCompile FORWARD_TRAIN%torch_compile_learner_what_to_compilesysplatform$torch_compile_learner_dynamo_backend!torch_compile_learner_dynamo_modetorch_compile_worker#torch_compile_worker_dynamo_backend torch_compile_worker_dynamo_modetorch_ddp_kwargstorch_skip_nan_gradientsenv env_configobservation_space action_space clip_rewardsnormalize_actions clip_actions _is_ataridisable_env_checking render_envaction_mask_keyenv_runner_clsnum_env_runnerscreate_local_env_runnernum_envs_per_env_runnergym_env_vectorize_modenum_cpus_per_env_runnernum_gpus_per_env_runnercustom_resources_per_env_runner'validate_env_runners_after_constructionepisodes_to_numpy%max_requests_in_flight_per_env_runnersample_timeout_screate_env_on_local_worker_env_to_module_connector0add_default_connectors_to_env_to_module_pipeline_module_to_env_connector0add_default_connectors_to_module_to_env_pipelinemerge_env_runner_statesbroadcast_env_runner_statesepisode_lookback_horizonrollout_fragment_length batch_modecompress_observationsremote_worker_envsremote_env_batch_wait_msenable_tf1_exec_eagerlyr"sample_collectorpreprocessor_prefobservation_filterupdate_worker_filter_statsuse_worker_filter_statssampler_perf_stats_ema_coef _is_online num_learnersnum_gpus_per_learnernum_cpus_per_learner!num_aggregator_actors_per_learner+max_requests_in_flight_per_aggregator_actor local_gpu_idx"max_requests_in_flight_per_learnergammalr grad_clip grad_clip_by_train_batch_size_per_learnertrain_batch_size num_epochsminibatch_sizeshuffle_batch_per_epochcopydeepcopyr#modelAttributeError_learner_connector*add_default_connectors_to_learner_pipelinelearner_config_dict optimizer_learner_classrcallbacks_classcallbacks_on_algorithm_init"callbacks_on_env_runners_recreated+callbacks_on_offline_eval_runners_recreatedcallbacks_on_checkpoint_loaded callbacks_on_environment_createdcallbacks_on_episode_createdcallbacks_on_episode_startcallbacks_on_episode_stepcallbacks_on_episode_endcallbacks_on_evaluate_startcallbacks_on_evaluate_end#callbacks_on_evaluate_offline_start!callbacks_on_evaluate_offline_endcallbacks_on_sample_endcallbacks_on_train_resultexplorerx_prior_exploration_configrrr api_stackcount_steps_byr)r'policiespolicy_map_capacityrjpolicy_mapping_fnpolicies_to_trainpolicy_states_are_swappableobservation_fninput_offline_data_classinput_read_methodinput_read_method_kwargsinput_read_schemainput_read_episodesinput_read_sample_batchesinput_read_batch_sizeinput_filesysteminput_filesystem_kwargsrOBSNEXT_OBSinput_compress_columnsinput_spaces_jsonablematerialize_datamaterialize_mapped_datamap_batches_kwargsiter_batches_kwargsignore_final_observationprelearner_classprelearner_buffer_classprelearner_buffer_kwargsprelearner_module_synch_perioddataset_num_iters_per_learner input_configactions_in_input_normalizedpostprocess_inputsshuffle_buffer_sizeoutput output_configoutput_compress_columnsoutput_max_file_sizeoutput_max_rows_per_fileoutput_write_remaining_dataoutput_write_methodoutput_write_method_kwargsoutput_filesystemoutput_filesystem_kwargsoutput_write_episodesoffline_samplingevaluation_intervalevaluation_durationevaluation_duration_unitevaluation_sample_timeout_s1evaluation_auto_duration_min_env_steps_per_sample1evaluation_auto_duration_max_env_steps_per_sampleevaluation_parallel_to_training,evaluation_force_reset_envs_before_iterationevaluation_configoff_policy_estimation_methodsope_split_batch_by_episodeevaluation_num_env_runnerscustom_evaluation_function in_evaluation)sync_filters_on_rollout_workers_timeout_soffline_evaluation_intervalnum_offline_eval_runnersoffline_evaluation_typeoffline_eval_runner_classoffline_loss_for_module_fnoffline_evaluation_duration'offline_evaluation_parallel_to_trainingoffline_evaluation_timeout_s num_cpus_per_offline_eval_runner num_gpus_per_offline_eval_runner(custom_resources_per_offline_eval_runner#restart_failed_offline_eval_runners#ignore_offline_eval_runner_failures$max_num_offline_eval_runner_restarts%offline_eval_runner_restore_timeout_s.max_requests_in_flight_per_offline_eval_runner0validate_offline_eval_runners_after_construction*offline_eval_runner_health_probe_timeout_s%offline_eval_rl_module_inference_only$broadcast_offline_eval_runner_states"offline_eval_batch_size_per_runner!dataset_num_iters_per_eval_runnerkeep_per_episode_custom_metrics$metrics_episode_collection_timeout_s"metrics_num_episodes_for_smoothingmin_time_s_per_iteration!min_train_timesteps_per_iteration"min_sample_timesteps_per_iteration log_gradientsr3stats_cls_lookupexport_native_model_files"checkpoint_trainable_policies_onlylogger_creator logger_config log_level log_sys_usage fake_samplerseedrestart_failed_env_runnersignore_env_runner_failuresmax_num_env_runner_restarts#delay_between_env_runner_restarts_srestart_failed_sub_environments-num_consecutive_env_runner_failures_tolerance!env_runner_health_probe_timeout_senv_runner_restore_timeout_s _model_config_rl_module_spec%algorithm_config_overrides_per_module_per_module_overrides_validate_config_use_msgpack_checkpoints_torch_grad_scaler_class_torch_lr_scheduler_classes%_tf_policy_handles_more_than_one_loss_disable_preprocessor_api_disable_action_flattening)_disable_initialize_loss_from_dummy_batch!_dont_auto_sync_env_runner_statesrmr env_task_fnenable_connectorssimple_optimizermonitorevaluation_num_episodesmetrics_smoothing_episodestimesteps_per_iterationmin_iter_time_scollect_metrics_timeoutmin_time_s_per_reporting!min_train_timesteps_per_reporting"min_sample_timesteps_per_reportinginput_evaluationpolicy_map_cache worker_clssynchronize_filtersenable_async_evaluation custom_async_evaluation_function_enable_rl_module_apiauto_wrap_old_gym_envs always_attach_evaluation_results buffer_sizeprioritized_replaylearning_startsreplay_batch_sizereplay_sequence_length replay_modeprioritized_replay_alphaprioritized_replay_betaprioritized_replay_eps_disable_execution_plan_api)selfrs r\__init__zAlgorithmConfig.__init__sh %13-02-#) )*&%!"$-.,-%*"AJ$( -.,- & & "&+" % 3 2 <833KK 126.$)!<833KK 015- "(-%!%  !%!$)!,# '+$'($'-#'($'($/1,7;4!%562 $*/'(,%@D=(,%@D='6$+/(()%(+$-%*""'()%',$ 3!+",*.''+$+/($%!$*!12.;<8 34/ )-1* ""',$ ~66DJJ    D #':>7#% " -+/(26/;?8.2+04-,0)*.')-&(,%+/()-&37015.'+$)-& t122 ).2D *&(D #-1)26/ )-/3    *+JLL9 #& !%!?!%+0("  "&"&!/(*%!##( ).&%)" $')$'.{G4D&E#%)" %'+$"$#% ).% $'+$(*%.0+-1*+0("'#$  (/ W5E'F$$4!(,%+0(#2 *,'!%(*%%)" %$( #% (2%+0(AD>AE>/4,<@9!%-/**.'*+'*.'#:>6+/(()%,0$)-& +/'+,(7<4,1)01-01-8:53703804815;2>?;@D=:>75:249125/12.05,48125/(,%12.23/" 8*/&27/#!!! +/'*/' ,0( 480/4,=@:15.,2) #572IK"!%(-%(,%+/(5:2).&*/'9>616.  ,!1 0' '7$*:''7$/'7$(8%1A.2B/ 0 0*#3 '7$0@-%5"&6#0@-,"2/!1&*#+(8%'7$&6#(8%1A.2B/+;(((s:J J&%J&ctjt|}|d|dd|vr1t |dsJ|j|d<|dd|vr1t |dsJ|j|d<|dd|vrt|dtrfi}|d D]7\}}t|tr| ||<2|||<8||d<|dd |d <|d d |d <|dd |d<|dd |d<dD]6}| |tkr||d 7|S)zConverts all settings into a legacy config dict for backward compatibility. Returns: A complete AlgorithmConfigDict, usable in backward-compatible Tune/RLlib use cases. rrmlambda_lambdar$inputrr N callbacksrrcreate_env_on_driverrXcustom_eval_functionr framework)rrrrrrrrrrrrrrrrrr_enable_new_api_stack)rrvarsrnrxrr$rYdictrwr' get_stategetr)rconfig policies_dict policy_id policy_specdep_ks r\to_dictzAlgorithmConfig.to_dictstDzz** <    <      4++ + ++#|F8  JJy ! ! ! v  4** * **"kF7O JJx   Jvj/A4$H$H M*0**Z*@*@*F*F*H*H ; ;& ;k:66;/:/D/D/F/FM),,/:M),,!.F: %jj):DAA{)/4PRS)T)T%&)/4PRV)W)W%&$jj$??{  ( (E*zz%  $444 5$''' r^c i}|d|d}|||||D]%\}||d}|tkr)|dvr.|dkrfd d D}|jdi|P|d krHt kr=ttrtd | |dkr| | dr||<|dkrS|r|_ ttrdvrtdd<|/|dkrSttr-drtdd<|jdi|i|dkr|jdi|i|dkrBttrdvrtdd<|jdi|i|dkr't|t#||'|jdi||S)aModifies this AlgorithmConfig via the provided python config dict. Warns if `config_dict` contains deprecated keys. Silently sets even properties of `self` that do NOT exist. This way, this method may be used to configure custom Policies which do not have their own specific AlgorithmConfig classes, e.g. `ray.rllib.examples.policy.random_policy::RandomPolicy`. Args: config_dict: The old-style python config dict (PartialAlgorithmConfigDict) to use for overriding some properties defined in there. Returns: This updated AlgorithmConfig object. rrNrFru)r multiagentc*i|]}|v||Sr).0krs r\ z4AlgorithmConfig.update_from_dict..s4   EzzuQx"zzr^)rrr r!r"r#rr Terror)r r)r evaluation_rr[)rr custom_modelrreplay_buffer_configr)rr)rrrwr{rH multi_agentr6rYstrr7r environment startswithrr env_runnerstrainingsetattr evaluation)rrk eval_callenable_new_api_stackr~rirs @r\roz AlgorithmConfig.update_from_dicts_&  + * OO@ A A   + NN-A3G     &++--@ *@ *JC..sE.JJCj  /// $$       ! **6****)))e7G.G.GeS))@,U$???Eu5555 $$  E 2222 .. *!& #,,,'.3D+eT**Dv$4U6]$C$CE&M  E ::::eT**Tuyy/H/HT,=R,S,SE.) --e ---- ## --e ----...eT**Dv$4U6]$C$CE&M --e ----***(//  % 8888c5))))$$)$$$ r^c|j}t||d<|d|dd|D}d|vrt |dt rfi}|dD]7\}}t |tr|||<2|||<8||d<|S)zReturns a dict state that can be pickled. Returns: A dictionary containing all attributes of the instance. classrrmc0i|]\}}|tk||Sr)rrrvs r\rz-AlgorithmConfig.get_state..Us)III$!Q18H3H3HA3H3H3Hr^r) __dict__rr[rnrwrYrr'r)rstaterrrs r\rzAlgorithmConfig.get_stateJs ""$$dg , ,II%++--III   :eJ.?#F#F M*/))J*?*?*E*E*G*G ; ;& ;k:66;/:/D/D/F/FM),,/:M),, -E*  r^rc^|d}|}|j||S)aReturns an instance constructed from the state. Args: state: A dictionary containing the state of an `AlgorithmConfig`. See `AlgorithmConfig.get_state` for creating a state. The constructed class will be of ``state["class"]``. Returns: An `AlgorithmConfig` instance with attributes from the `state`. r)rupdate)rprctorrs r\ from_statezAlgorithmConfig.from_statehs3W~u%%% r^cT|}||S)a~Returns a mapping from str to JSON'able values representing this config. The resulting values don't have any code in them. Classes (such as `callbacks_class`) are converted to their full classpath, e.g. `ray.rllib.callbacks.callbacks.RLlibCallback`. Actual code such as lambda functions ware written as their source code (str) plus any closure information for properly restoring the code inside the AlgorithmConfig object made from the returned dict data. Dataclass objects get converted to dicts. Returns: A dict mapping from str to JSON'able values. )r_serialize_dict)rrs r\ serializezAlgorithmConfig.serializes%##F+++r^ copy_frozenctj|}|dur|n1|dur-d|_t |jt r d|j_|S)a9Creates a deep copy of this config and (un)freezes if necessary. Args: copy_frozen: Whether the created deep copy is frozen or not. If None, keep the same frozen status that `self` currently has. Returns: A deep copy of `self` that is (un)frozen. TF)rrfreezermrYrTr`)rrcps r\rzAlgorithmConfig.copysd]4  $   IIKKKK E ! !!BM".@@ 827$/ r^c|jrdSd|_t|jtr|jdSdS)zFreezes this config object, such that no attributes can be set anymore. Algorithms should use this method to make sure that their config objects remain read-only after this. NT)rmrYrTr`rrs r\rzAlgorithmConfig.freezesV ?  F d,o > > ,  " ) ) + + + + + , ,r^c|jsdS||||||||| | dS)$Validates all values in this config.N) r_validate_env_runner_settings_validate_callbacks_settings_validate_framework_settings_validate_resources_settings_validate_multi_agent_settings_validate_input_settings_validate_evaluation_settings_validate_offline_settings _validate_new_api_stack_settings#_validate_to_be_deprecated_settingsrs r\validatezAlgorithmConfig.validates $  F **,,, ))+++ ))+++ ))+++ ++--- %%''' **,,, ''))) --/// 0022222r^Trr{use_copyrMc|||_|j ||jd<|||_|j}t |jt rt |j}||s|ntj||jS)aBuilds an Algorithm from this AlgorithmConfig (or a copy thereof). Args: env: Name of the environment to use (e.g. a gym-registered str), a full class path (e.g. "ray.rllib.examples.envs.classes.random_env.RandomEnv"), or an Env class directly. Note that this arg can also be specified via the "env" key in `config`. logger_creator: Callable that creates a ray.tune.Logger object. If unspecified, a default logger is created. use_copy: Whether to deepcopy `self` and pass the copy to the Algorithm (instead of `self`) as config. This is useful in case you would like to recycle the same AlgorithmConfig over and over, e.g. in a test case, in which we loop over different DL-frameworks. Returns: A ray.rllib.algorithms.algorithm.Algorithm object. Nr)rr{) rrTr{rrYrrGrr)rrr{rrs r\ build_algozAlgorithmConfig.build_algos0 ?DH%103&u-  %"0D _ dos + + <*4?;;Jz'@44T]4-@-@.    r^c4ddlm}m}m}m}m}m} m} g} |j |||} ng#t$rZ} d| j dvr?tdrt d|} n| Yd} ~ nd} ~ wwxYwt| tr| g} n?t| t t"frt!| } nt%d| dt'dj}n&|t*|vr|t*d}n|j}|G|jr@t.jfd jdjjD}t'd j}n&|t*|vr|t*d }n|j}|G|jr@t.jfd jdjjD}| ||| }|jr|||||||jr]||t|j tBr |j j"ntG|j$|j%|||j|| ||S)Nr)"AddObservationsFromEpisodesToBatchAddStatesFromEpisodesToBatchAddTimeDimToBatchAndZeroPadAgentToModuleMappingBatchIndividualItemsEnvToModulePipeline NumpyToTensorpositional argumentzenv-to-module-wrong-signatureaYour `config.env_to_module_connector` function seems to have a wrong or outdated signature! It should be: `def myfunc(env, spaces, device): ...`, where any of these arguments are optional and may be None. `env` is the (vectorized) gym env. `spaces` is a dict of structure `{'__env__': ([vectorized env obs. space, vectorized env act. space]),'__env_single__': ([env obs. space, env act. space])}`. `device` is a (torch) device. z`AlgorithmConfig.env_runners(env_to_module_connector=..)` must return a ConnectorV2 object or a list thereof to be added to a connector pipeline! Your function returned .single_observation_spacec\i|](}|jdj|)Srenvs unwrappedget_observation_spacerrgrs r\rzAAlgorithmConfig.build_env_to_module_connector..2B!.DDSIIr^single_action_spacerc\i|](}|jdj|)Srrrget_action_spacers r\rzAAlgorithmConfig.build_env_to_module_connector..?B!.??DDr^input_observation_spaceinput_action_space connectorsrl_module_specsagent_to_module_mapping_fnr)device)&"ray.rllib.connectors.env_to_modulerrrrr r r r TypeErrorargsrJloggerrrYrlisttuplerZgetattrrris_multi_agentgymspacesrrrpossible_agentsrrappendrl_module_specrr!setrr )rrr.r$rrrrr r r custom_connectorsval_e obs_space act_spacepipelines ` r\build_env_to_module_connectorz-AlgorithmConfig.build_env_to_module_connectors                     ( 4 44S&&II   (AF1I55 ?@@  >    88==DDGDDDD &$ ,, %)F!!D4-00 $(JJ!!!JBFJJJ ?%?AVWWII  $;v$E$E67:II.I  !4  "x{4DI ?%:C>@@ A A A OO7799 : : : OO88:: ; ; ;" (( *$*=?PQQ4D/??!$T]!3!3373I     OO00T=PQQQ R R R OOMM888 9 9 9s6 BABBcddlm}m}m}m}m}m}m} m} g} |j | |} ng#t$rZ} d| j dvr?tdrtd| } n| Yd} ~ nd} ~ wwxYwt| t r| g} n?t| t"t$frt#| } nt'd| dt)dj}n&|t,|vr|t,d}n|j}|G|jr@t0jfd jdjjD}t)d j}n&|t,|vr|t,d }n|j}|G|jr@t0jfd jdjjD}|||| }|jr| ||jr| || | | | | ||!||j"|j#|!||S)Nr) GetActionsListifyDataForVectorEnvModuleToAgentUnmappingModuleToEnvPipelineNormalizeAndClipActionsRemoveSingleTsTimeRankFromBatch TensorToNumpyUnBatchToIndividualItemsr zmodule-to-env-wrong-signatureaYour `config.module_to_env_connector` function seems to have a wrong or outdated signature! It should be: `def myfunc(env, spaces): ...`, where any of these arguments are optional and may be None. `env` is the (vectorized) gym env. `spaces` is a dict of structure `{'__env__': ([vectorized env obs. space, vectorized env act. space]),'__env_single__': ([env obs. space, env act. space])}`. z`AlgorithmConfig.env_runners(module_to_env_connector=..)` must return a ConnectorV2 object or a list thereof to be added to a connector pipeline! Your function returned r rc\i|](}|jdj|)Srrrs r\rzAAlgorithmConfig.build_module_to_env_connector..rr^rrc\i|](}|jdj|)Srrrs r\rzAAlgorithmConfig.build_module_to_env_connector..rr^r)rr)$"ray.rllib.connectors.module_to_envr;r<r=r>r?r@rArBrr&r'rJr(rrYrr)r*rZr+rrr,r-r.rrrr/rrprependr0rr)rrr.r;r<r=r>r?r@rArBr3r4r5r6r7r8s ` r\build_module_to_env_connectorz-AlgorithmConfig.build_module_to_env_connectords   ( 4 44S&AA   (AF1I55 ?@@  X    88==DDGDDDD $$ ,, %)F!!D4-00 $(JJ!!!JBFJJJ ?%?AVWWII  $;v$E$E67:II.I  !4  "x{4DI ?%:C> ? ? ?" ;  !7!7!9!9:::   5577 8 8 8   ]]__ - - -   ZZ\\ * * * OO''&*&>@@ A A A OO77TRRR S S S OO88dSSS T T T" (( *$*=?PQQ4D/??!$T]!3!3373I     OO00T=PQQQ R R R OOMMtFSSS T T Tr^)rr.r1placement_groupr.r1rNrKrQcddlm}||||}||||}|S)aBuilds and returns a new LearnerGroup object based on settings in `self`. Args: env: An optional EnvType object (e.g. a gym.Env) useful for extracting space information for the to-be-constructed RLModule inside the LearnerGroup's Learner workers. Note that if RLlib cannot infer any space information either from this `env` arg, from the optional `spaces` arg or from `self`, the LearnerGroup cannot be created. spaces: An optional dict mapping ModuleIDs to (observation-space, action-space)-tuples for the to-be-constructed RLModule inside the LearnerGroup's Learner workers. Note that if RLlib cannot infer any space information either from this `spces` arg, from the optional `env` arg or from `self`, the LearnerGroup cannot be created. rl_module_spec: An optional (single-agent or multi-agent) RLModuleSpec to use for the constructed LearnerGroup. If None, RLlib tries to infer the RLModuleSpec using the other information given and stored in this `AlgorithmConfig` object. Returns: The newly created `LearnerGroup` object. rrPNrr.)rrVrN)$ray.rllib.core.learner.learner_grouprQget_multi_rl_module_specr)rrr.r1rNrQ learner_groups r\build_learner_groupz#AlgorithmConfig.build_learner_group$sh< FEEEEE  !!::s6:RRN% 99;;&+   r^rPrNcd}|||||}|||}||S)a!Builds and returns a new Learner object based on settings in `self`. This Learner object already has its `build()` method called, meaning its RLModule is already constructed. Args: env: An optional EnvType object (e.g. a gym.Env) useful for extracting space information for the to-be-constructed RLModule inside the Learner. Note that if RLlib cannot infer any space information either from this `env` arg, from the optional `spaces` arg or from `self`, the Learner cannot be created. spaces: An optional dict mapping ModuleIDs to (observation-space, action-space)-tuples for the to-be-constructed RLModule inside the Learner. Note that if RLlib cannot infer any space information either from this `spaces` arg, from the optional `env` arg or from `self`, the Learner cannot be created. Returns: The newly created (and already built) Learner object. NrP)rrV)rR learner_classbuild)rrr.r1learners r\ build_learnerzAlgorithmConfig.build_learnerRsU8 ?f0!::s6:RRN$$Dn$MM r^ module_idc||jvrC||jvr:||j||j|<||jvr |j|S|S)aReturns an AlgorithmConfig object, specific to the given module ID. In a multi-agent setup, individual modules might override one or more AlgorithmConfig properties (e.g. `train_batch_size`, `lr`) using the `overrides()` method. In order to retrieve a full AlgorithmConfig instance (with all these overrides already translated and built-in), users can call this method with the respective module ID. Args: module_id: The module ID for which to get the final AlgorithmConfig object. Returns: A new AlgorithmConfig object for the specific module ID. )rrrro)rrZs r\get_config_for_modulez%AlgorithmConfig.get_config_for_modulexsu( T7 7 7TGGG48IIKK4P4P:9E55D &y 1 2 2 2-i8 8Kr^)rrrrcF|tur||_|tur||_|S)aSets the config's python environment settings. Args: extra_python_environs_for_driver: Any extra python env vars to set in the algorithm's process, e.g., {"OMP_NUM_THREADS": "16"}. extra_python_environs_for_worker: The extra python environments need to set for worker processes. Returns: This updated AlgorithmConfig object. )r1rr)rrrs r\python_environmentz"AlgorithmConfig.python_environments/" ,; > >4TD 1 +; > >4TD 1 r^) rrrrnum_cpus_per_workernum_gpus_per_workercustom_resources_per_workernum_learner_workersnum_cpus_per_learner_workernum_gpus_per_learner_workerrnum_cpus_for_local_workerrrrrc |tkrtddd||_|tkrtddd||_|tkrtddd||_|tkrtd d d||_| tkrtd d d| |_| tkrtd dd| |_| tkrtddd| |_| tkrtddd| |_ |tur||_ |tur||_ |tur||_ |tur||_ |S)aSpecifies resources allocated for an Algorithm and its ray actors/workers. Args: num_cpus_for_main_process: Number of CPUs to allocate for the main algorithm process that runs `Algorithm.training_step()`. Note: This is only relevant when running RLlib through Tune. Otherwise, `Algorithm.training_step()` runs in the main program (driver). num_gpus: Number of GPUs to allocate to the algorithm process. Note that not all algorithms can take advantage of GPUs. Support for multi-GPU is currently only available for tf-[PPO/IMPALA/DQN/PG]. This can be fractional (e.g., 0.3 GPUs). _fake_gpus: Set to True for debugging (multi-)?GPU funcitonality on a CPU machine. GPU towers are simulated by graphs located on CPUs in this case. Use `num_gpus` to test for different numbers of fake GPUs. placement_strategy: The strategy for the placement group factory returned by `Algorithm.default_resource_request()`. A PlacementGroup defines, which devices (resources) should always be co-located on the same node. For example, an Algorithm with 2 EnvRunners and 1 Learner (with 1 GPU) requests a placement group with the bundles: [{"cpu": 1}, {"gpu": 1, "cpu": 1}, {"cpu": 1}, {"cpu": 1}], where the first bundle is for the local (main Algorithm) process, the second one for the 1 Learner worker and the last 2 bundles are for the two EnvRunners. These bundles can now be "placed" on the same or different nodes depending on the value of `placement_strategy`: "PACK": Packs bundles into as few nodes as possible. "SPREAD": Places bundles across distinct nodes as even as possible. "STRICT_PACK": Packs bundles into one node. The group is not allowed to span multiple nodes. "STRICT_SPREAD": Packs bundles across distinct nodes. Returns: This updated AlgorithmConfig object. z.AlgorithmConfig.resources(num_cpus_per_worker)z4AlgorithmConfig.env_runners(num_cpus_per_env_runner)Foldnewrz.AlgorithmConfig.resources(num_gpus_per_worker)z4AlgorithmConfig.env_runners(num_gpus_per_env_runner)z6AlgorithmConfig.resources(custom_resources_per_worker)z=     "/D  $(8 8 8 JJ     .GD * $K 7 7-FD * ; & &$DM [ ( ((DO [ 0 0&8D # r^) rrrrrrrrrrrrrrrrrrrrrrrrrrrc |tur|dkrtddd||_|tur||_|tur||_|tur||_|tur||_|tur||_| tur| |_|tur||_ |tur||_ | tur| |_ | tur| |_ | tur| |_ | tur| |_|tur||_|S)aSets the config's DL framework settings. Args: framework: torch: PyTorch; tf2: TensorFlow 2.x (eager execution or traced if eager_tracing=True); tf: TensorFlow (static-graph); eager_tracing: Enable tracing in eager mode. This greatly improves performance (speedup ~2x), but makes it slightly harder to debug since Python code won't be evaluated after the initial eager pass. Only possible if framework=tf2. eager_max_retraces: Maximum number of tf.function re-traces before a runtime error is raised. This is to prevent unnoticed retraces of methods inside the `..._eager_traced` Policy, which could slow down execution by a factor of 4, without the user noticing what the root cause for this slowdown could be. Only necessary for framework=tf2. Set to None to ignore the re-trace count and never throw an error. tf_session_args: Configures TF for single-process operation by default. local_tf_session_args: Override the following tf session args on the local worker torch_compile_learner: If True, forward_train methods on TorchRLModules on the learner are compiled. If not specified, the default is to compile forward train on the learner. torch_compile_learner_what_to_compile: A TorchCompileWhatToCompile mode specifying what to compile on the learner side if torch_compile_learner is True. See TorchCompileWhatToCompile for details and advice on its usage. torch_compile_learner_dynamo_backend: The torch dynamo backend to use on the learner. torch_compile_learner_dynamo_mode: The torch dynamo mode to use on the learner. torch_compile_worker: If True, forward exploration and inference methods on TorchRLModules on the workers are compiled. If not specified, the default is to not compile forward methods on the workers because retracing can be expensive. torch_compile_worker_dynamo_backend: The torch dynamo backend to use on the workers. torch_compile_worker_dynamo_mode: The torch dynamo mode to use on the workers. torch_ddp_kwargs: The kwargs to pass into `torch.nn.parallel.DistributedDataParallel` when using `num_learners > 1`. This is specifically helpful when searching for unused parameters that are not used in the backward pass. This can give hints for errors in custom models where some parameters do not get touched in the backward pass although they should. torch_skip_nan_gradients: If updates with `nan` gradients should be entirely skipped. This skips updates in the optimizer entirely if they contain any `nan` gradient. This can help to avoid biasing moving-average based optimizers - like Adam. This can help in training phases where policy updates can be highly unstable such as during the early stages of training or with highly exploratory policies. In such phases many gradients might turn `nan` and setting them to zero could corrupt the optimizer's internal state. The default is `False` and turns `nan` gradients to zero. If many `nan` gradients are encountered consider (a) monitoring gradients by setting `log_gradients` in `AlgorithmConfig` to `True`, (b) use proper weight initialization (e.g. Xavier, Kaiming) via the `model_config_dict` in `AlgorithmConfig.rl_module` and/or (c) gradient clipping via `grad_clip` in `AlgorithmConfig.training`. Returns: This updated AlgorithmConfig object. tfez AlgorithmConfig.framework('tfe')z AlgorithmConfig.framework('tf2')Trg)r1rrrrrrrrrrrrrrr)rrrrrrrrrrrrrrrs r\rzAlgorithmConfig.framework/sP^ K ' 'E!!#:: "+D   + +!.D  [ 0 0&8D # + - -#2D 3 3)>D & 3 3)>D & /{ B B4  5 -K ? ?5VD 2 0 C C5  6 { 2 2( >4TD 1 ; . .$4D ! #; 6 6,DD ) r^rrc|turf||_|dur|jr|j|_i|_n@|dur<|js5|j|j|_d|_ntd|tur||_|S)a7Sets the config's API stack settings. Args: enable_rl_module_and_learner: Enables the usage of `RLModule` (instead of `ModelV2`) and Learner (instead of the training-related parts of `Policy`). Must be used with `enable_env_runner_and_connector_v2=True`. Together, these two settings activate the "new API stack" of RLlib. enable_env_runner_and_connector_v2: Enables the usage of EnvRunners (SingleAgentEnvRunner and MultiAgentEnvRunner) and ConnectorV2. When setting this to True, `enable_rl_module_and_learner` must be True as well. Together, these two settings activate the "new API stack" of RLlib. Returns: This updated AlgorithmConfig object. TFNzoconfig.enable_rl_module_and_learner was set to False, but no prior exploration config was found to be restored.)r1rrrr(warningr)rrrs r\rzAlgorithmConfig.api_stacks* ({ : :0LD -+t338O3151H.*,''-66t?V61=.2.LD+59D22NNM .[ @ @6XD 3 r^) rrrrrrrrr rrrrrrrrrrr rc | tkrtdd|tur||_|turt |j|d|tur||_|tur||_|tur||_|tur||_ |tur||_ |tur||_ | tur| |_ | tur| |_ | tur| |_|S)a Sets the config's RL-environment settings. Args: env: The environment specifier. This can either be a tune-registered env, via `tune.register_env([name], lambda env_ctx: [env object])`, or a string specifier of an RLlib supported type. In the latter case, RLlib tries to interpret the specifier as either an Farama-Foundation gymnasium env, a PyBullet env, or a fully qualified classpath to an Env class, e.g. "ray.rllib.examples.envs.classes.random_env.RandomEnv". env_config: Arguments dict passed to the env creator as an EnvContext object (which is a dict plus the properties: `num_env_runners`, `worker_index`, `vector_index`, and `remote`). observation_space: The observation space for the Policies of this Algorithm. action_space: The action space for the Policies of this Algorithm. render_env: If True, try to render the environment on the local worker or on worker 1 (if num_env_runners > 0). For vectorized envs, this usually means that only the first sub-environment is rendered. In order for this to work, your env has to implement the `render()` method which either: a) handles window generation and rendering itself (returning True) or b) returns a numpy uint8 image of shape [height x width x 3 (RGB)]. clip_rewards: Whether to clip rewards during Policy's postprocessing. None (default): Clip for Atari only (r=sign(r)). True: r=sign(r): Fixed rewards -1.0, 1.0, or 0.0. False: Never clip. [float value]: Clip at -value and + value. Tuple[value1, value2]: Clip at value1 and value2. normalize_actions: If True, RLlib learns entirely inside a normalized action space (0.0 centered with small stddev; only affecting Box components). RLlib unsquashes actions (and clip, just in case) to the bounds of the env's action space before sending actions back to the env. clip_actions: If True, the RLlib default ModuleToEnv connector clips actions according to the env's bounds (before sending them into the `env.step()` call). disable_env_checking: Disable RLlib's env checks after a gymnasium.Env instance has been constructed in an EnvRunner. Note that the checks include an `env.reset()` and `env.step()` (with a random action), which might tinker with your env's logic and behavior and thus negatively influence sample collection- and/or learning behavior. is_atari: This config can be used to explicitly specify whether the env is an Atari env or not. If not specified, RLlib tries to auto-detect this. action_mask_key: If observation is a dictionary, expect the value by the key `action_mask_key` to contain a valid actions mask (`numpy.int8` array of zeros and ones). Defaults to "action_mask". Returns: This updated AlgorithmConfig object. z+AlgorithmConfig.environment(env_task_fn=..)Trhr)rrr1rr*rrrrrrrrrr) rrrrrrrrrrr rrs r\rzAlgorithmConfig.environmentsD * * * A     k ! !DH [ ( ( T : : : K / /%6D " { * * ,D  [ ( ((DO { * * ,D  K / /%6D " { * * ,D  { 2 2( 0, the driver (local_worker; worker-idx=0) does not need an environment. This is because it doesn't have to sample (done by remote_workers; worker_indices > 0) nor evaluate (done by evaluation workers; see below). env_to_module_connector: A callable taking an Env as input arg and returning an env-to-module ConnectorV2 (might be a pipeline) object. module_to_env_connector: A callable taking an Env and an RLModule as input args and returning a module-to-env ConnectorV2 (might be a pipeline) object. add_default_connectors_to_env_to_module_pipeline: If True (default), RLlib's EnvRunners automatically add the default env-to-module ConnectorV2 pieces to the EnvToModulePipeline. These automatically perform adding observations and states (in case of stateful Module(s)), agent-to-module mapping, batching, and conversion to tensor data. Only if you know exactly what you are doing, you should set this setting to False. Note that this setting is only relevant if the new API stack is used (including the new EnvRunner classes). add_default_connectors_to_module_to_env_pipeline: If True (default), RLlib's EnvRunners automatically add the default module-to-env ConnectorV2 pieces to the ModuleToEnvPipeline. These automatically perform removing the additional time-rank (if applicable, in case of stateful Module(s)), module-to-agent unmapping, un-batching (to lists), and conversion from tensor data to numpy. Only if you know exactly what you are doing, you should set this setting to False. Note that this setting is only relevant if the new API stack is used (including the new EnvRunner classes). episode_lookback_horizon: The amount of data (in timesteps) to keep from the preceeding episode chunk when a new chunk (for the same episode) is generated to continue sampling at a later time. The larger this value, the more an env-to-module connector can look back in time and compile RLModule input data from this information. For example, if your custom env-to-module connector (and your custom RLModule) requires the previous 10 rewards as inputs, you must set this to at least 10. merge_env_runner_states: True, if remote EnvRunner actor states should be merged into central connector pipelines. Use "training_only" (default) for only doing this for the training EnvRunners, NOT for the evaluation EnvRunners. broadcast_env_runner_states: True, if merged EnvRunner states (from the central connector pipelines) should be broadcast back to all remote EnvRunner actors. use_worker_filter_stats: Whether to use the workers in the EnvRunnerGroup to update the central filters (held by the local worker). If False, stats from the workers aren't used and are discarded. update_worker_filter_stats: Whether to push filter updates from the central filters (held by the local worker) to the remote workers' filters. Setting this to True might be useful within the evaluation config in order to disable the usage of evaluation trajectories for synching the central filter (used for training). rollout_fragment_length: Divide episodes into fragments of this many steps each during sampling. Trajectories of this size are collected from EnvRunners and combined into a larger batch of `train_batch_size` for learning. For example, given rollout_fragment_length=100 and train_batch_size=1000: 1. RLlib collects 10 fragments of 100 steps each from rollout workers. 2. These fragments are concatenated and we perform an epoch of SGD. When using multiple envs per worker, the fragment size is multiplied by `num_envs_per_env_runner`. This is since we are collecting steps from multiple envs in parallel. For example, if num_envs_per_env_runner=5, then EnvRunners return experiences in chunks of 5*100 = 500 steps. The dataflow here can vary per algorithm. For example, PPO further divides the train batch into minibatches for multi-epoch SGD. Set `rollout_fragment_length` to "auto" to have RLlib compute an exact value to match the given batch size. batch_mode: How to build individual batches with the EnvRunner(s). Batches coming from distributed EnvRunners are usually concat'd to form the train batch. Note that "steps" below can mean different things (either env- or agent-steps) and depends on the `count_steps_by` setting, adjustable via `AlgorithmConfig.multi_agent(count_steps_by=..)`: 1) "truncate_episodes": Each call to `EnvRunner.sample()` returns a batch of at most `rollout_fragment_length * num_envs_per_env_runner` in size. The batch is exactly `rollout_fragment_length * num_envs` in size if postprocessing does not change batch sizes. Episodes may be truncated in order to meet this size requirement. This mode guarantees evenly sized batches, but increases variance as the future return must now be estimated at truncation boundaries. 2) "complete_episodes": Each call to `EnvRunner.sample()` returns a batch of at least `rollout_fragment_length * num_envs_per_env_runner` in size. Episodes aren't truncated, but multiple episodes may be packed within one batch to meet the (minimum) batch size. Note that when `num_envs_per_env_runner > 1`, episode steps are buffered until the episode completes, and hence batches may contain significant amounts of off-policy data. explore: Default exploration behavior, iff `explore=None` is passed into compute_action(s). Set to False for no exploration behavior (e.g., for evaluation). episodes_to_numpy: Whether to numpy'ize episodes before returning them from an EnvRunner. False by default. If True, EnvRunners call `to_numpy()` on those episode (chunks) to be returned by `EnvRunners.sample()`. exploration_config: A dict specifying the Exploration object's config. remote_worker_envs: If using num_envs_per_env_runner > 1, whether to create those new envs in remote processes instead of in the same worker. This adds overheads, but can make sense if your envs can take much time to step / reset (e.g., for StarCraft). Use this cautiously; overheads are significant. remote_env_batch_wait_ms: Timeout that remote workers are waiting when polling environments. 0 (continue when at least one env is ready) is a reasonable default, but optimal value could be obtained by measuring your environment step / reset and model inference perf. validate_env_runners_after_construction: Whether to validate that each created remote EnvRunner is healthy after its construction process. preprocessor_pref: Whether to use "rllib" or "deepmind" preprocessors by default. Set to None for using no preprocessor. In this case, the model has to handle possibly complex observations from the environment. observation_filter: Element-wise observation filter, either "NoFilter" or "MeanStdFilter". compress_observations: Whether to LZ4 compress individual observations in the SampleBatches collected during rollouts. enable_tf1_exec_eagerly: Explicitly tells the rollout worker to enable TF eager execution. This is useful for example when framework is "torch", but a TF2 policy needs to be restored for evaluation or league-based purposes. sampler_perf_stats_ema_coef: If specified, perf stats are in EMAs. This is the coeff of how much new data points contribute to the averages. Default is None, which uses simple global average instead. The EMA update rule is: updated = (1 - ema_coef) * old + ema_coef * new Returns: This updated AlgorithmConfig object. z2AlgorithmConfig.env_runners(enable_connectors=...)Frpz0AlgorithmConfig.env_runners(num_rollout_workers)z,AlgorithmConfig.env_runners(num_env_runners)Trgz0AlgorithmConfig.env_runners(num_envs_per_worker)z4AlgorithmConfig.env_runners(num_envs_per_env_runner)z@AlgorithmConfig.env_runners(validate_workers_after_construction)zDAlgorithmConfig.env_runners(validate_env_runners_after_construction)rz`num_envs_per_env_runner` (z) must be larger 0!rz3`rollout_fragment_length` must be int >0 or 'auto'!)rcomplete_episodesz`batch_mode` (z7) must be one of [truncate_episodes|complete_episodes]!rz2AlgorithmConfig.env_runners(synchronize_filter=..)z:AlgorithmConfig.env_runners(update_worker_filter_stats=..)z>ignore_worker_failures is deprecated, and will soon be a no-opz7AlgorithmConfig.env_runners(recreate_failed_workers=..)>AlgorithmConfig.fault_tolerance(restart_failed_env_runners=..)z?AlgorithmConfig.env_runners(restart_failed_sub_environments=..)zCAlgorithmConfig.fault_tolerance(restart_failed_sub_environments=..)zIAlgorithmConfig.env_runners(num_consecutive_worker_failures_tolerance=..)zQAlgorithmConfig.fault_tolerance(num_consecutive_env_runner_failures_tolerance=..)z=AlgorithmConfig.env_runners(worker_health_probe_timeout_s=..)zEAlgorithmConfig.fault_tolerance(env_runner_health_probe_timeout_s=..)z8AlgorithmConfig.env_runners(worker_restore_timeout_s=..)z@AlgorithmConfig.fault_tolerance(env_runner_restore_timeout_s=..))*rrr1rrrZrrrrrrrrrrrrrrrrrrrrYintrrrrr*rrrrrrrrrr)/rrrrrrrrrrrrrrqrrrrrrrrrrrrrrrrrrrrrrrsrtrurvrwrrxryrzr{rnew_exploration_configs/ r\rzAlgorithmConfig.env_runners5sT  0 0 0 H     "2 2 2 FB     "2 2 2 FJ     /2B B B V        , ,"0D  + - -#2D "+ 5 5&!++  2I   ,CD ( ! 4 4*@D ' "+ 5 5+BD ( "+ 5 5+BD ( *+ = =3RD 0 ; . .$4D ! 0 C C5  6 ; . .$4D ! "+ 5 5+BD ( %[ 8 8.HD + "+ 5 5,CD ) "+ 5 5,CD ) ;; N N@  A <; N N@  A $; 6 6,DD ) "+ 5 5+BD ( &k 9 9/JD , "+ 5 5+BD ( %[ 8 8.HD + "+ 5 56<< X0!33*f44 !VWWW+BD ( [ ( (!KKK *Z***)DO + % %"DL K / /%6D " [ 0 0&1%t'>?%'9:%&%& && "'==Q&RD # [ 0 0&8D # #; 6 6,DD ) 2+ E E7  8 K / /%6D " [ 0 0&8D # [ 0 0'9D $ 3 3)>D & "+ 5 5+BD ( &k 9 9/JD , !1 1 1 HP     "%5 5 5 T     #&6 6 6 MT     +.> > > UY      58H H H _h     ),< < < S[     $'7 7 7 NV      r^) rrrrrrrrVlearner_connectorrrrrrrrrrrVrrrc |tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_| tur| |_ | tur| |_ | tur|j | |S)agSets LearnerGroup and Learner worker related configurations. Args: num_learners: Number of Learner workers used for updating the RLModule. A value of 0 means training takes place on a local Learner on main process CPUs or 1 GPU (determined by `num_gpus_per_learner`). For multi-gpu training, you have to set `num_learners` to > 1 and set `num_gpus_per_learner` accordingly (e.g., 4 GPUs total and model fits on 1 GPU: `num_learners=4; num_gpus_per_learner=1` OR 4 GPUs total and model requires 2 GPUs: `num_learners=2; num_gpus_per_learner=2`). num_cpus_per_learner: Number of CPUs allocated per Learner worker. If "auto" (default), use 1 if `num_gpus_per_learner=0`, otherwise 0. Only necessary for custom processing pipeline inside each Learner requiring multiple CPU cores. If `num_learners=0`, RLlib creates only one local Learner instance and the number of CPUs on the main process is `max(num_cpus_per_learner, num_cpus_for_main_process)`. num_gpus_per_learner: Number of GPUs allocated per Learner worker. If `num_learners=0`, any value greater than 0 runs the training on a single GPU on the main process, while a value of 0 runs the training on main process CPUs. num_aggregator_actors_per_learner: The number of aggregator actors per Learner (if num_learners=0, one local learner is created). Must be at least 1. Aggregator actors perform the task of a) converting episodes into a train batch and b) move that train batch to the same GPU that the corresponding learner is located on. Good values are 1 or 2, but this strongly depends on your setup and `EnvRunner` throughput. max_requests_in_flight_per_aggregator_actor: How many in-flight requests are allowed per aggregator actor before new requests are dropped? local_gpu_idx: If `num_gpus_per_learner` > 0, and `num_learners` < 2, then RLlib uses this GPU index for training. This is an index into the available CUDA devices. For example if `os.environ["CUDA_VISIBLE_DEVICES"] = "1"` and `local_gpu_idx=0`, RLlib uses the GPU with ID=1 on the node. max_requests_in_flight_per_learner: Max number of in-flight requests to each Learner (actor). You normally do not have to tune this setting (default is 3), however, for asynchronous algorithms, this determines the "queue" size for incoming batches (or lists of episodes) into each Learner worker, thus also determining, how much off-policy'ness would be acceptable. The off-policy'ness is the difference between the numbers of updates a policy has undergone on the Learner vs the EnvRunners. See the `ray.rllib.utils.actor_manager.FaultTolerantActorManager` class for more details. learner_class: The `Learner` class to use for (distributed) updating of the RLModule. learner_connector: A callable taking an env observation space and an env action space as inputs and returning a learner ConnectorV2 or list of ConnectorV2's as part of pipeline object. add_default_connectors_to_learner_pipeline: If True (default), RLlib's Learners automatically add the default Learner ConnectorV2 pieces to the LearnerPipeline. These automatically perform: a) adding observations from episodes to the train batch, if this has not already been done by a user-provided connector piece b) if RLModule is stateful, add a time rank to the train batch, zero-pad the data, and add the correct state inputs, if this has not already been done by a user-provided connector piece. c) add all other information (actions, rewards, terminateds, etc..) to the train batch, if this has not already been done by a user-provided connector piece. Only if you know exactly what you are doing, you should set this setting to False. learner_config_dict: A dict to insert any settings accessible from within the Learner instance. This should only be used in connection with custom Learner subclasses and in case the user doesn't want to write an extra `AlgorithmConfig` subclass just to add a few settings to the base Algo's own config class. Returns: This updated AlgorithmConfig object. ) r1rrrrrrrr rrrr) rrrrrrrrrVrrrs r\learnerszAlgorithmConfig.learnersst { * * ,D  { 2 2(config.training(add_default_connectors_to_learner_pipeline=..)z>config.learners(add_default_connectors_to_learner_pipeline=..)z'config.training(learner_config_dict=..)z'config.learners(learner_config_dict=..)z5config.training(num_aggregator_actors_per_learner=..)z5config.learners(num_aggregator_actors_per_learner=..)z?config.training(max_requests_in_flight_per_aggregator_actor=..)z?config.learners(max_requests_in_flight_per_aggregator_actor=..)z config.training(num_sgd_iter=..)zconfig.training(num_epochs=..)zGAlgorithmConfig.training(max_requests_in_flight_per_sampler_worker=...)zFAlgorithmConfig.env_runners(max_requests_in_flight_per_env_runner=...))r)rnormrz`grad_clip_by` (z4) must be one of: 'value', 'norm', or 'global_norm'!_use_default_native_modelsz9AlgorithmConfig.training(_use_default_native_models=True)z_use_default_native_models is not supported anymore. To get rid of this error, set `config.api_stack(enable_rl_module_and_learner=True)`. Native models will be better supported by the upcoming RLModule API.rhhelpr)r1rr rrrrrrrrrrrrZrrrrrrrrr,r)rrrrrrrrrrrrrrrrrVrrrs r\rzAlgorithmConfig.trainingO sX@  + + 77     #0D  K / / ;;     '8D # 5[ H H TT     ;  ; k 1 1 ==      $ + +,? @ @ @ ,0@ @ @ KK     6WD 2 6:J J J UU     <  < + + + 64     &J 48H H H A=        =      # #DJ [ DG K ' '&DN { * *#CCC (|(((!-D  '{ : :1MD . ; . .$4D ! [ ( ((DO  , ,"0D  "+ 5 5+BD (  # # J  e $ $ $ 68HII#$$$SH  <= K ' '(CCDN r^)on_algorithm_initon_train_resulton_evaluate_starton_evaluate_endon_evaluate_offline_starton_evaluate_offline_endon_env_runners_recreated!on_offline_eval_runners_recreatedon_checkpoint_loadedon_environment_createdon_episode_createdon_episode_starton_episode_stepon_episode_end on_sample_endr rrrrrrrrrrrrrrrc|t}|turBt|}td|Dst d|d||_|tur||_|tur||_|tur||_|tur||_ |tur||_ |tur||_ |tur||_ | tur| |_ | tur| |_| tur| |_| tur| |_| tur| |_|tur||_|tur||_|tur||_|S)a Sets the callbacks configuration. Args: callbacks_class: RLlibCallback class, whose methods are called during various phases of training and RL environment sample collection. TODO (sven): Change the link to new rst callbacks page. See the `RLlibCallback` class and `examples/metrics/custom_metrics_and_callbacks.py` for more information. on_algorithm_init: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_algorithm_init` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_algorithm_init` # noqa for more information. on_evaluate_start: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_evaluate_start` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_evaluate_start` # noqa for more information. on_evaluate_end: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_evaluate_end` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_evaluate_end` # noqa for more information. on_env_runners_recreated: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_env_runners_recreated` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_env_runners_recreated` # noqa for more information. on_checkpoint_loaded: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_checkpoint_loaded` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_checkpoint_loaded` # noqa for more information. on_environment_created: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_environment_created` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_environment_created` # noqa for more information. on_episode_created: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_episode_created` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_episode_created` # noqa for more information. on_episode_start: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_episode_start` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_episode_start` # noqa for more information. on_episode_step: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_episode_step` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_episode_step` # noqa for more information. on_episode_end: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_episode_end` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_episode_end` # noqa for more information. on_sample_end: A callable or a list of callables. If a list, RLlib calls the items in the same sequence. `on_sample_end` methods overridden in `callbacks_class` take precedence and are called first. See :py:meth:`~ray.rllib.callbacks.callbacks.RLlibCallback.on_sample_end` # noqa for more information. Returns: This updated AlgorithmConfig object. Nc34K|]}t|VdSrb)callable)rcs r\ z,AlgorithmConfig.callbacks.. s(55qx{{555555r^zr`config.callbacks_class` must be a callable or list of callables that returns a subclass of DefaultCallbacks, got rt)rr1r+allrZr r rrrrrr r rrrrrrr)rr rrrrrrrrrrrrrrrto_checks r\rzAlgorithmConfig.callbacks sJ  "+O + - -"/22H55H55555  *&*** $3D K / //@D , + - --D + + - -- 1` setting, the amount of episodes/timesteps to run are split amongst these. A special value of "auto" can be used in case `evaluation_parallel_to_training=True`. This is the recommended way when trying to save as much time on evaluation as possible. The Algorithm then runs as many timesteps via the evaluation workers as possible, while not taking longer than the parallely running training step and thus, never wasting any idle time on either training- or evaluation workers. When using this setting (`evaluation_duration="auto"`), it is strongly advised to set `evaluation_interval=1` and `evaluation_force_reset_envs_before_iteration=True` at the same time. evaluation_duration_unit: The unit, with which to count the evaluation duration. Either "episodes" (default) or "timesteps". Note that this setting is ignored if `evaluation_duration="auto"`. evaluation_auto_duration_min_env_steps_per_sample: If `evaluation_duration` is "auto" (in which case `evaluation_duration_unit` is always "timesteps"), at least how many timesteps should be done per remote `sample()` call. evaluation_auto_duration_max_env_steps_per_sample: If `evaluation_duration` is "auto" (in which case `evaluation_duration_unit` is always "timesteps"), at most how many timesteps should be done per remote `sample()` call. evaluation_sample_timeout_s: The timeout (in seconds) for evaluation workers to sample a complete episode in the case your config settings are: `evaluation_duration != auto` and `evaluation_duration_unit=episode`. After this time, the user receives a warning and instructions on how to fix the issue. evaluation_parallel_to_training: Whether to run evaluation in parallel to the `Algorithm.training_step()` call, using threading. Default=False. E.g. for evaluation_interval=1 -> In every call to `Algorithm.train()`, the `Algorithm.training_step()` and `Algorithm.evaluate()` calls run in parallel. Note that this setting - albeit extremely efficient b/c it wastes no extra time for evaluation - causes the evaluation results to lag one iteration behind the rest of the training results. This is important when picking a good checkpoint. For example, if iteration 42 reports a good evaluation `episode_return_mean`, be aware that these results were achieved on the weights trained in iteration 41, so you should probably pick the iteration 41 checkpoint instead. evaluation_force_reset_envs_before_iteration: Whether all environments should be force-reset (even if they are not done yet) right before the evaluation step of the iteration begins. Setting this to True (default) makes sure that the evaluation results aren't polluted with episode statistics that were actually (at least partially) achieved with an earlier set of weights. Note that this setting is only supported on the new API stack w/ EnvRunners and ConnectorV2 (`config.enable_rl_module_and_learner=True` AND `config.enable_env_runner_and_connector_v2=True`). evaluation_config: Typical usage is to pass extra args to evaluation env creator and to disable exploration by computing deterministic actions. IMPORTANT NOTE: Policy gradient algorithms are able to find the optimal policy, even if this is a stochastic one. Setting "explore=False" here results in the evaluation workers not using this optimal policy! off_policy_estimation_methods: Specify how to evaluate the current policy, along with any optional config parameters. This only has an effect when reading offline experiences ("input" is not "sampler"). Available keys: {ope_method_name: {"type": ope_type, ...}} where `ope_method_name` is a user-defined string to save the OPE results under, and `ope_type` can be any subclass of OffPolicyEstimator, e.g. ray.rllib.offline.estimators.is::ImportanceSampling or your own custom subclass, or the full class path to the subclass. You can also add additional config arguments to be passed to the OffPolicyEstimator in the dict, e.g. {"qreg_dr": {"type": DoublyRobust, "q_model_type": "qreg", "k": 5}} ope_split_batch_by_episode: Whether to use SampleBatch.split_by_episode() to split the input batch to episodes before estimating the ope metrics. In case of bandits you should make this False to see improvements in ope evaluation speed. In case of bandits, it is ok to not split by episode, since each record is one timestep already. The default is True. evaluation_num_env_runners: Number of parallel EnvRunners to use for evaluation. Note that this is set to zero by default, which means evaluation is run in the algorithm process (only if `evaluation_interval` is not 0 or None). If you increase this, also increases the Ray resource usage of the algorithm since evaluation workers are created separately from those EnvRunners used to sample data for training. custom_evaluation_function: Customize the evaluation method. This must be a function of signature (algo: Algorithm, eval_workers: EnvRunnerGroup) -> (metrics: dict, env_steps: int, agent_steps: int) (metrics: dict if `enable_env_runner_and_connector_v2=True`), where `env_steps` and `agent_steps` define the number of sampled steps during the evaluation iteration. See the Algorithm.evaluate() method to see the default implementation. The Algorithm guarantees all eval workers have the latest policy state before this function is called. offline_evaluation_interval: Evaluate offline with every `offline_evaluation_interval` training iterations. The offline evaluation stats are reported under the "evaluation/offline_evaluation" metric key. Set to None (or 0) for no offline evaluation. num_offline_eval_runners: Number of OfflineEvaluationRunner actors to create for parallel evaluation. Setting this to 0 forces sampling to be done in the local OfflineEvaluationRunner (main process or the Algorithm's actor when using Tune). offline_evaluation_type: Type of offline evaluation to run. Either `"eval_loss"` for evaluating the validation loss of the policy, `"is"` for importance sampling, or `"pdis"` for per-decision importance sampling. If you want to implement your own offline evaluation method write an `OfflineEvaluationRunner` and use the `AlgorithmConfig.offline_eval_runner_class`. offline_eval_runner_class: An `OfflineEvaluationRunner` class that implements custom offline evaluation logic. offline_loss_for_module_fn: A callable to compute the loss per `RLModule` in offline evaluation. If not provided the training loss function ( `Learner.compute_loss_for_module`) is used. The signature must be ( runner: OfflineEvaluationRunner, module_id: ModuleID, config: AlgorithmConfig, batch: Dict[str, Any], fwd_out: Dict[str, TensorType]). offline_eval_batch_size_per_runner: Evaluation batch size per individual OfflineEvaluationRunner worker. This setting only applies to the new API stack. The number of OfflineEvaluationRunner workers can be set via `config.evaluation(num_offline_eval_runners=...)`. The total effective batch size is then `num_offline_eval_runners` x `offline_eval_batch_size_per_runner`. dataset_num_iters_per_offline_eval_runner: Number of batches to evaluate in each OfflineEvaluationRunner during a single evaluation. If None, each learner runs a complete epoch over its data block (the dataset is partitioned into at least as many blocks as there are runners). The default is `1`. offline_eval_rl_module_inference_only: If `True`, the module spec is used in an inference-only setting (no-loss) and the RLModule can thus be built in its light version (if available). For example, the `inference_only` version of an RLModule might only contain the networks required for computing actions, but misses additional target- or critic networks. Also, if `True`, the module does NOT contain those (sub) RLModules that have their `learner_only` flag set to True. num_cpus_per_offline_eval_runner: Number of CPUs to allocate per OfflineEvaluationRunner. num_gpus_per_offline_eval_runner: Number of GPUs to allocate per OfflineEvaluationRunner. This can be fractional. This is usually needed only if your (custom) loss function itself requires a GPU (i.e., it contains GPU- intensive computations), or model inference is unusually expensive. custom_resources_per_eval_runner: Any custom Ray resources to allocate per OfflineEvaluationRunner. offline_evaluation_timeout_s: The timeout in seconds for calling `run()` on remote OfflineEvaluationRunner workers. Results (episode list) from workers that take longer than this time are discarded. max_requests_in_flight_per_offline_eval_runner: Max number of in-flight requests to each OfflineEvaluationRunner (actor)). See the `ray.rllib.utils.actor_manager.FaultTolerantActorManager` class for more details. Tuning these values is important when running experiments with large evaluation batches, where there is the risk that the object store may fill up, causing spilling of objects to disk. This can cause any asynchronous requests to become very slow, making your experiment run slowly as well. You can inspect the object store during your experiment through a call to `ray memory` on your head node, and by using the Ray dashboard. If you're seeing that the object store is filling up, turn down the number of remote requests in flight or enable compression or increase the object store memory through, for example: `ray.init(object_store_memory=10 * 1024 * 1024 * 1024) # =10 GB`. broadcast_offline_eval_runner_states: True, if merged OfflineEvaluationRunner states (from the central connector pipelines) should be broadcast back to all remote OfflineEvaluationRunner actors. validate_offline_eval_runners_after_construction: Whether to validate that each created remote OfflineEvaluationRunner is healthy after its construction process. restart_failed_offline_eval_runners: Whether - upon an OfflineEvaluationRunner failure - RLlib tries to restart the lost OfflineEvaluationRunner(s) as an identical copy of the failed one(s). You should set this to True when training on SPOT instances that may preempt any time and/or if you need to evaluate always a complete dataset b/c OfflineEvaluationRunner(s) evaluate through streaming split iterators on disjoint batches. The new, recreated OfflineEvaluationRunner(s) only differ from the failed one in their `self.recreated_worker=True` property value and have the same `worker_index` as the original(s). If this setting is True, the value of the `ignore_offline_eval_runner_failures` setting is ignored. ignore_offline_eval_runner_failures: Whether to ignore any OfflineEvalautionRunner failures and continue running with the remaining OfflineEvaluationRunners. This setting is ignored, if `restart_failed_offline_eval_runners=True`. max_num_offline_eval_runner_restarts: The maximum number of times any OfflineEvaluationRunner is allowed to be restarted (if `restart_failed_offline_eval_runners` is True). offline_eval_runner_health_probe_timeout_s: Max amount of time in seconds, we should spend waiting for OfflineEvaluationRunner health probe calls (`OfflineEvaluationRunner.ping.remote()`) to respond. Health pings are very cheap, however, we perform the health check via a blocking `ray.get()`, so the default value should not be too large. offline_eval_runner_restore_timeout_s: Max amount of time we should wait to restore states on recovered OfflineEvaluationRunner actors. Default is 30 mins. Returns: This updated AlgorithmConfig object. z?AlgorithmConfig.evaluation(always_attach_evaluation_results=..)zThis setting is no longer needed, b/c Tune does not error anymore (only warns) when a metrics key can't be found in the results.Trz5AlgorithmConfig.evaluation(evaluation_num_workers=..)z9AlgorithmConfig.evaluation(evaluation_num_env_runners=..)FrgNrrL)*rrrWr1rLrMrNrPrQrOrRrSrTray.rllib.algorithms.algorithmrMr*_allow_unknown_subkeys%_override_all_subkeys_if_type_changes_override_all_key_listrUrXrVr[r\r]r^r_rorprmrcrdrerbrjrnrkrfrgrhrlri)%rrLrMrNrPrQrOrRrSrTrUrVrWrXr[r\r]r^r_rorrmrcrdrerbrjrnrkrfrgrhrlrirrrMs% r\rzAlgorithmConfig.evaluation sP ,/? ? ? U      "%5 5 5 KO     /ED + k 1 1':D $ k 1 1':D $ #; 6 6,DD ) >4TD 1 +; > >4TD 1 3; F F8  9 ({ : :0LD - 9 L L>  ? 0{ B B4  5 <; N N@  A /k A A3  4 /k A A3  4 0{ B B4  5 6[ H H:  ; 1 C C5  6 r^)%r$r%r&r'r(r)r*r+r,r-r0r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKr$r%r&r'r(r)r*r+r,r-r0r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKc%H|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_| tur| |_ | tur| |_ | tur| |_ | tur| |_ | tur| |_ |tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|turt-|t.s t1dt3|dd}&|d#t1|&dd|dM|jr#t1|&ddt1|&dd ||_|tur||_|tur||_|tur||_ |tur||_!|tur||_"|tur||_#|tur||_$|tur||_%|tur||_&| tur| |_'|!tur|!|_(|"tur|"|_)|#tur|#|_*|$tur|$|_+|%tur|%|_,|S) aCSets the config's offline data settings. Args: input_: Specify how to generate experiences: - "sampler": Generate experiences via online (env) simulation (default). - A local directory or file glob expression (e.g., "/tmp/*.json"). - A list of individual file paths/URIs (e.g., ["/tmp/1.json", "s3://bucket/2.json"]). - A dict with string keys and sampling probabilities as values (e.g., {"sampler": 0.4, "/tmp/*.json": 0.4, "s3://bucket/expert.json": 0.2}). - A callable that takes an `IOContext` object as only arg and returns a `ray.rllib.offline.InputReader`. - A string key that indexes a callable with `tune.registry.register_input` offline_data_class: An optional `OfflineData` class that is used to define the offline data pipeline, including the dataset and the sampling methodology. Override the `OfflineData` class and pass your derived class here, if you need some primer transformations specific to your data or your loss. Usually overriding the `OfflinePreLearner` and using the resulting customization via `prelearner_class` suffices for most cases. The default is `None` which uses the base `OfflineData` defined in `ray.rllib.offline.offline_data.OfflineData`. input_read_method: Read method for the `ray.data.Dataset` to read in the offline data from `input_`. The default is `read_parquet` for Parquet files. See https://docs.ray.io/en/latest/data/api/input_output.html for more info about available read methods in `ray.data`. input_read_method_kwargs: Keyword args for `input_read_method`. These are passed by RLlib into the read method without checking. Use these keyword args together with `map_batches_kwargs` and `iter_batches_kwargs` to tune the performance of the data pipeline. It is strongly recommended to rely on Ray Data's automatic read performance tuning. input_read_schema: Table schema for converting offline data to episodes. This schema maps the offline data columns to ray.rllib.core.columns.Columns: `{Columns.OBS: 'o_t', Columns.ACTIONS: 'a_t', ...}`. Columns in the data set that are not mapped via this schema are sorted into episodes' `extra_model_outputs`. If no schema is passed in the default schema used is `ray.rllib.offline.offline_data.SCHEMA`. If your data set contains already the names in this schema, no `input_read_schema` is needed. The same applies if the data is in RLlib's `EpisodeType` or its old `SampleBatch` format. input_read_episodes: Whether offline data is already stored in RLlib's `EpisodeType` format, i.e. `ray.rllib.env.SingleAgentEpisode` (multi -agent is planned but not supported, yet). Reading episodes directly avoids additional transform steps and is usually faster and therefore the recommended format when your application remains fully inside of RLlib's schema. The other format is a columnar format and is agnostic to the RL framework used. Use the latter format, if you are unsure when to use the data or in which RL framework. The default is to read column data, for example, `False`. `input_read_episodes`, and `input_read_sample_batches` can't be `True` at the same time. See also `output_write_episodes` to define the output data format when recording. input_read_sample_batches: Whether offline data is stored in RLlib's old stack `SampleBatch` type. This is usually the case for older data recorded with RLlib in JSON line format. Reading in `SampleBatch` data needs extra transforms and might not concatenate episode chunks contained in different `SampleBatch`es in the data. If possible avoid to read `SampleBatch`es and convert them in a controlled form into RLlib's `EpisodeType` (i.e. `SingleAgentEpisode`). The default is `False`. `input_read_episodes`, and `input_read_sample_batches` can't be `True` at the same time. input_read_batch_size: Batch size to pull from the data set. This could differ from the `train_batch_size_per_learner`, if a dataset holds `EpisodeType` (i.e., `SingleAgentEpisode`) or `SampleBatch`, or any other data type that contains multiple timesteps in a single row of the dataset. In such cases a single batch of size `train_batch_size_per_learner` will potentially pull a multiple of `train_batch_size_per_learner` timesteps from the offline dataset. The default is `None` in which the `train_batch_size_per_learner` is pulled. input_filesystem: A cloud filesystem to handle access to cloud storage when reading experiences. Can be either "gcs" for Google Cloud Storage, "s3" for AWS S3 buckets, "abs" for Azure Blob Storage, or any filesystem supported by PyArrow. In general the file path is sufficient for accessing data from public or local storage systems. See https://arrow.apache.org/docs/python/filesystems.html for details. input_filesystem_kwargs: A dictionary holding the kwargs for the filesystem given by `input_filesystem`. See `gcsfs.GCSFilesystem` for GCS, `pyarrow.fs.S3FileSystem`, for S3, and `ablfs.AzureBlobFilesystem` for ABS filesystem arguments. input_compress_columns: What input columns are compressed with LZ4 in the input data. If data is stored in RLlib's `SingleAgentEpisode` ( `MultiAgentEpisode` not supported, yet). Note the providing `rllib.core.columns.Columns.OBS` also tries to decompress `rllib.core.columns.Columns.NEXT_OBS`. materialize_data: Whether the raw data should be materialized in memory. This boosts performance, but requires enough memory to avoid an OOM, so make sure that your cluster has the resources available. For very large data you might want to switch to streaming mode by setting this to `False` (default). If your algorithm does not need the RLModule in the Learner connector pipeline or all (learner) connectors are stateless you should consider setting `materialize_mapped_data` to `True` instead (and set `materialize_data` to `False`). If your data does not fit into memory and your Learner connector pipeline requires an RLModule or is stateful, set both `materialize_data` and `materialize_mapped_data` to `False`. materialize_mapped_data: Whether the data should be materialized after running it through the Learner connector pipeline (i.e. after running the `OfflinePreLearner`). This improves performance, but should only be used in case the (learner) connector pipeline does not require an RLModule and the (learner) connector pipeline is stateless. For example, MARWIL's Learner connector pipeline requires the RLModule for value function predictions and training batches would become stale after some iterations causing learning degradation or divergence. Also ensure that your cluster has enough memory available to avoid an OOM. If set to `True` (True), make sure that `materialize_data` is set to `False` to avoid materialization of two datasets. If your data does not fit into memory and your Learner connector pipeline requires an RLModule or is stateful, set both `materialize_data` and `materialize_mapped_data` to `False`. map_batches_kwargs: Keyword args for the `map_batches` method. These are passed into the `ray.data.Dataset.map_batches` method when sampling without checking. If no arguments passed in the default arguments `{'concurrency': max(2, num_learners), 'zero_copy_batch': True}` is used. Use these keyword args together with `input_read_method_kwargs` and `iter_batches_kwargs` to tune the performance of the data pipeline. iter_batches_kwargs: Keyword args for the `iter_batches` method. These are passed into the `ray.data.Dataset.iter_batches` method when sampling without checking. If no arguments are passed in, the default argument `{'prefetch_batches': 2}` is used. Use these keyword args together with `input_read_method_kwargs` and `map_batches_kwargs` to tune the performance of the data pipeline. ignore_final_observation: If the final observation in an episode chunk should be ignored. This concerns mainly column-based data and instead of using a user-provided `NEXT_OBS` sets final observations to zero. This should be used with BC only, as in true Offline RL algorithms the final observation is important. prelearner_class: An optional `OfflinePreLearner` class that is used to transform data batches in `ray.data.map_batches` used in the `OfflineData` class to transform data from columns to batches that can be used in the `Learner.update...()` methods. Override the `OfflinePreLearner` class and pass your derived class in here, if you need to make some further transformations specific for your data or loss. The default is None which uses the base `OfflinePreLearner` defined in `ray.rllib.offline.offline_prelearner`. prelearner_buffer_class: An optional `EpisodeReplayBuffer` class that RLlib uses to buffer experiences when data is in `EpisodeType` or RLlib's previous `SampleBatch` type format. In this case, a single data row may contain multiple timesteps and the buffer serves two purposes: (a) to store intermediate data in memory, and (b) to ensure that RLlib samples exactly `train_batch_size_per_learner` experiences per batch. The default is RLlib's `EpisodeReplayBuffer`. prelearner_buffer_kwargs: Optional keyword arguments for initializing the `EpisodeReplayBuffer`. In most cases this value is simply the `capacity` for the default buffer that RLlib uses (`EpisodeReplayBuffer`), but it may differ if the `prelearner_buffer_class` uses a custom buffer. prelearner_module_synch_period: The period (number of batches converted) after which the `RLModule` held by the `PreLearner` should sync weights. The `PreLearner` is used to preprocess batches for the learners. The higher this value, the more off-policy the `PreLearner`'s module is. Values too small force the `PreLearner` to sync more frequently and thus might slow down the data pipeline. The default value chosen by the `OfflinePreLearner` is 10. dataset_num_iters_per_learner: Number of updates to run in each learner during a single training iteration. If None, each learner runs a complete epoch over its data block (the dataset is partitioned into at least as many blocks as there are learners). The default is `None`. This value must be set to `1`, if RLlib uses a single (local) learner. input_config: Arguments that describe the settings for reading the input. If input is "sample", this is the environment configuration, e.g. `env_name` and `env_config`, etc. See `EnvContext` for more info. If the input is "dataset", this contains e.g. `format`, `path`. actions_in_input_normalized: True, if the actions in a given offline "input" are already normalized (between -1.0 and 1.0). This is usually the case when the offline file has been generated by another RLlib algorithm (e.g. PPO or SAC), while "normalize_actions" was set to True. postprocess_inputs: Whether to run postprocess_trajectory() on the trajectory fragments from offline inputs. Note that postprocessing is done using the *current* policy, not the *behavior* policy, which is typically undesirable for on-policy algorithms. shuffle_buffer_size: If positive, input batches are shuffled via a sliding window buffer of this number of batches. Use this if the input data is not in random enough order. Input is delayed until the shuffle buffer is filled. output: Specify where experiences should be saved: - None: don't save any experiences - "logdir" to save to the agent log dir - a path/URI to save to a custom output directory (e.g., "s3://bckt/") - a function that returns a rllib.offline.OutputWriter output_config: Arguments accessible from the IOContext for configuring custom output. output_compress_columns: What sample batch columns to LZ4 compress in the output data. Note that providing `rllib.core.columns.Columns.OBS` also compresses `rllib.core.columns.Columns.NEXT_OBS`. output_max_file_size: Max output file size (in bytes) before rolling over to a new file. output_max_rows_per_file: Max output row numbers before rolling over to a new file. output_write_remaining_data: Determines whether any remaining data in the recording buffers should be stored to disk. It is only applicable if `output_max_rows_per_file` is defined. When sampling data, it is buffered until the threshold specified by `output_max_rows_per_file` is reached. Only complete multiples of `output_max_rows_per_file` are written to disk, while any leftover data remains in the buffers. If a recording session is stopped, residual data may still reside in these buffers. Setting `output_write_remaining_data` to `True` ensures this data is flushed to disk. By default, this attribute is set to `False`. output_write_method: Write method for the `ray.data.Dataset` to write the offline data to `output`. The default is `read_parquet` for Parquet files. See https://docs.ray.io/en/latest/data/api/input_output.html for more info about available read methods in `ray.data`. output_write_method_kwargs: `kwargs` for the `output_write_method`. These are passed into the write method without checking. output_filesystem: A cloud filesystem to handle access to cloud storage when writing experiences. Should be either "gcs" for Google Cloud Storage, "s3" for AWS S3 buckets, or "abs" for Azure Blob Storage. output_filesystem_kwargs: A dictionary holding the kwargs for the filesystem given by `output_filesystem`. See `gcsfs.GCSFilesystem` for GCS, `pyarrow.fs.S3FileSystem`, for S3, and `ablfs.AzureBlobFilesystem` for ABS filesystem arguments. output_write_episodes: If RLlib should record data in its RLlib's `EpisodeType` format (that is, `SingleAgentEpisode` objects). Use this format, if you need RLlib to order data in time and directly group by episodes for example to train stateful modules or if you plan to use recordings exclusively in RLlib. Otherwise RLlib records data in tabular (columnar) format. Default is `True`. offline_sampling: Whether sampling for the Algorithm happens via reading from offline data. If True, EnvRunners don't limit the number of collected batches within the same `sample()` call based on the number of sub-environments within the worker (no sub-environments present). Returns: This updated AlgorithmConfig object. z!input_config must be a dict, got r z@{} should not be set in the input_config. RLlib uses {} instead.num_cpus_per_read_taskNz.config.env_runners(num_cpus_per_env_runner=..) parallelismz0config.evaluation(evaluation_num_env_runners=..)z&config.env_runners(num_env_runners=..))-r1r$r%r&r'r(r)r*r+r,r-r0r2r3r4r5r6r7r8r9r:r;rYrrZr[rformatrYr<r=r>r?r@rArBrCrDrErFrGrHrIrJrK)'rr$r%r&r'r(r)r*r+r,r-r0r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKmsgs' r\ offline_datazAlgorithmConfig.offline_data sV  $ $ DK [ 0 0&8D # K / /%6D " #; 6 6,DD ) K / /%6D " k 1 1':D $ $K 7 7-FD * 3 3)>D & ; . .$4D ! "+ 5 5+BD ( ! 4 4*@D ' ; . .$4D ! "+ 5 5+BD ( [ 0 0&8D # k 1 1':D $ #; 6 6,DD ) ; . .$4D ! "+ 5 5+BD ( #; 6 6,DD ) ) < <2PD / ( ; ;1ND . { * *lD11  M\8J8JMMM UC 899E JJ0H  ..:% $ )N% )+S !-D  &k 9 9/JD , [ 0 0&8D # k 1 1':D $  $ $ DK  + +!.D  "+ 5 5+BD ( { 2 2(D & ; . .$4D ! r^) rrr r!r"r#rrrrrrr rUr!r"r#rc |tur4|D]} t| dt|ttt fr d|D}t|t r|D]\} } t| tsEt| t tfrt| dkrtd| d| _t| j tt fs/| j (td| d t| j d ||_ntd |tkr't!d d |||tur||_|tur+t|t rt'|}||_|tur||_| tkrt!dd| tkrt!ddd|tur|dvrtd|d ||_|turt|t ttfst/|s | Jdt|t ttfr-t|dkrt0d||_|tur||_|S)a Sets the config's multi-agent settings. Validates the new multi-agent settings and translates everything into a unified multi-agent setup format. For example a `policies` list or set of IDs is properly converted into a dict mapping these IDs to PolicySpecs. Args: policies: Map of type MultiAgentPolicyConfigDict from policy ids to either 4-tuples of (policy_cls, obs_space, act_space, config) or PolicySpecs. These tuples or PolicySpecs define the class of the policy, the observation- and action spaces of the policies, and any extra config. policy_map_capacity: Keep this many policies in the "policy_map" (before writing least-recently used ones to disk/S3). policy_mapping_fn: Function mapping agent ids to policy ids. The signature is: `(agent_id, episode, worker, **kwargs) -> PolicyID`. policies_to_train: Determines those policies that should be updated. Options are: - None, for training all policies. - An iterable of PolicyIDs that should be trained. - A callable, taking a PolicyID and a SampleBatch or MultiAgentBatch and returning a bool (indicating whether the given policy is trainable or not, given the particular batch). This allows you to have a policy trained only on certain data (e.g. when playing against a certain opponent). policy_states_are_swappable: Whether all Policy objects in this map can be "swapped out" via a simple `state = A.get_state(); B.set_state(state)`, where `A` and `B` are policy instances in this map. You should set this to True for significantly speeding up the PolicyMap's cache lookup times, iff your policies all share the same neural network architecture and optimizer types. If True, the PolicyMap doesn't have to garbage collect old, least recently used policies, but instead keeps them in memory and simply override their state with the state of the most recently accessed one. For example, in a league-based training setup, you might have 100s of the same policies in your map (playing against each other in various combinations), but all of them share the same state structure (are "swappable"). observation_fn: Optional function that can be used to enhance the local agent observations to include more state. See rllib/evaluation/observation_function.py for more info. count_steps_by: Which metric to use as the "batch size" when building a MultiAgentBatch. The two supported values are: "env_steps": Count each time the env is "stepped" (no matter how many multi-agent actions are passed/how many multi-agent observations have been returned in the previous step). "agent_steps": Count each individual agent step as one step. Returns: This updated AlgorithmConfig object. Trc,i|]}|tSrr')rps r\rz/AlgorithmConfig.multi_agent.. s>>>Az||>>>r^zYPolicy specs must be tuples/lists of (cls or None, obs_space, action_space, config), got z for PolicyID=NzMulti-agent policy config for z3 must be a dict or AlgorithmConfig object, but got rtzX`policies` must be dict mapping PolicyID to PolicySpec OR a set/tuple/list of PolicyIDs!Frp)rz0AlgorithmConfig.multi_agent(policy_map_cache=..)z+AlgorithmConfig.multi_agent(replay_mode=..)zBAlgorithmConfig.training(replay_buffer_config={'replay_mode': ..})rg)r agent_stepszRconfig.multi_agent(count_steps_by=..) must be one of [env_steps|agent_steps], not zERROR: `policies_to_train` must be a [list|set|tuple] or a callable taking PolicyID and SampleBatch and returning True|False (trainable or not?) or None (for always training all policies).rz`config.multi_agent(policies_to_train=..)` is empty! Make sure - if you would like to learn at least one policy - to add its ID to that list.)r1rrYr2r*r)rrwr'lenrZrr`r[rrr rl_modulerr2r r#rrr(rnr!r") rrrr r!r"r#rrrrpidspecs r\rzAlgorithmConfig.multi_agent sR ; & &  4 4"3d33333(S%$677 ?>>X>>>(D)) !)!1!1IC%dJ77)$u >>#d))q..",!A'+!A!A;>!A!A##CQ't{_d4KLL K3(TSTT?CDK?P?PTTT!)  3 14D D D Be 4 4 4 4 NN9     k 1 1':D $ K / /+T22 C$/0A$B$B!%6D "  , ,"0D  / / / F     * * * A<       , ,%AAA F4BFFF#1D  K / /,tS%.@AA -.. %,,-,-+dC-?@@ ())Q..NN6 &7D " &k 9 9/JD , r^)rqrrrsrtrurvrwcustom_stats_cls_lookuprqrrrsrtrurvrwrc|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|S)aSets the config's reporting settings. Args: keep_per_episode_custom_metrics: Store raw custom metrics without calculating max, min, mean metrics_episode_collection_timeout_s: Wait for metric batches for at most this many seconds. Those that have not returned in time are collected in the next train iteration. metrics_num_episodes_for_smoothing: Smooth rollout metrics over this many episodes, if possible. In case rollouts (sample collection) just started, there may be fewer than this many episodes in the buffer and we'll compute metrics over this smaller number of available episodes. In case there are more than this many episodes collected in a single training iteration, use all of these episodes for metrics computation, meaning don't ever cut any "excess" episodes. Set this to 1 to disable smoothing and to always report only the most recently collected episode's return. min_time_s_per_iteration: Minimum time (in sec) to accumulate within a single `Algorithm.train()` call. This value does not affect learning, only the number of times `Algorithm.training_step()` is called by `Algorithm.train()`. If - after one such step attempt, the time taken has not reached `min_time_s_per_iteration`, performs n more `Algorithm.training_step()` calls until the minimum time has been consumed. Set to 0 or None for no minimum time. min_train_timesteps_per_iteration: Minimum training timesteps to accumulate within a single `train()` call. This value does not affect learning, only the number of times `Algorithm.training_step()` is called by `Algorithm.train()`. If - after one such step attempt, the training timestep count has not been reached, performs n more `training_step()` calls until the minimum timesteps have been executed. Set to 0 or None for no minimum timesteps. min_sample_timesteps_per_iteration: Minimum env sampling timesteps to accumulate within a single `train()` call. This value does not affect learning, only the number of times `Algorithm.training_step()` is called by `Algorithm.train()`. If - after one such step attempt, the env sampling timestep count has not been reached, performs n more `training_step()` calls until the minimum timesteps have been executed. Set to 0 or None for no minimum timesteps. log_gradients: Log gradients to results. If this is `True` the global norm of the gradients dictionary for each optimizer is logged to results. The default is `False`. custom_stats_cls_lookup: A dictionary mapping stat names to their corresponding Stats classes. The Stats classes should be subclasses of :py:class:`~ray.rllib.utils.metrics.stats.StatsBase`. The keys of the dictionary are the stat names, and the values are the corresponding Stats classes. This allows you to use your own Stats classes for logging metrics. You can replace existing values to override some behaviour of RLlib. You can add key-value-pairs to the dictionary to add new stats classes that will be available when logging values with the MetricsLogger throughout RLlib. Returns: This updated AlgorithmConfig object. ) r1rqrrrsrtrurvrwrx) rrqrrrsrtrurvrwrs r\ reportingzAlgorithmConfig.reportingJsB ++ = =3RD 0 /{ B B4  5 .[ @ @6XD 3 #; 6 6,DD ) ,K ? ?5VD 2 -[ @ @6XD 3  + +!.D  "+ 5 5$;D ! r^ryrzcF|tur||_|tur||_|S)a,Sets the config's checkpointing settings. Args: export_native_model_files: Whether an individual Policy- or the Algorithm's checkpoints also contain (tf or torch) native model files. These could be used to restore just the NN models from these files w/o requiring RLlib. These files are generated by calling the tf- or torch- built-in saving utility methods on the actual models. checkpoint_trainable_policies_only: Whether to only add Policies to the Algorithm checkpoint (in sub-directory "policies/") that are trainable according to the `is_trainable_policy` callable of the local worker. Returns: This updated AlgorithmConfig object. )r1ryrz)rryrzs r\ checkpointingzAlgorithmConfig.checkpointings/, %K 7 7-FD * -[ @ @6XD 3 r^)r{r|r}r~rrr|r}r~rrc|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|S)aSets the config's debugging settings. Args: logger_creator: Callable that creates a ray.tune.Logger object. If unspecified, a default logger is created. logger_config: Define logger-specific configuration to be used inside Logger Default value None allows overwriting with nested dicts. log_level: Set the ray.rllib.* log level for the agent process and its workers. Should be one of DEBUG, INFO, WARN, or ERROR. The DEBUG level also periodically prints out summaries of relevant internal dataflow (this is also printed out once at startup at the INFO level). log_sys_usage: Log system resource metrics to results. This requires `psutil` to be installed for sys stats, and `gputil` for GPU metrics. fake_sampler: Use fake (infinite speed) sampler. For testing only. seed: This argument, in conjunction with worker_index, sets the random seed of each worker, so that identically configured trials have identical results. This makes experiments reproducible. Returns: This updated AlgorithmConfig object. )r1r{r|r}r~rr)rr{r|r}r~rrs r\ debuggingzAlgorithmConfig.debuggingsy>  , ,"0D   + +!.D  K ' '&DN  + +!.D  { * * ,D  { " "DI r^)rrrrrrrrrecreate_failed_env_runnersrvrwmax_num_worker_restartsdelay_between_worker_restarts_srxryrzrrrrrrrrc| tkrtddd| tkrtddd| tkrtddd| tkrtdd d| tkrtd d d|tkrtd d d|tkrtddd|tkrtddd|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_ |tur||_ |S)a Sets the config's fault tolerance settings. Args: restart_failed_env_runners: Whether - upon an EnvRunner failure - RLlib tries to restart the lost EnvRunner(s) as an identical copy of the failed one(s). You should set this to True when training on SPOT instances that may preempt any time. The new, recreated EnvRunner(s) only differ from the failed one in their `self.recreated_worker=True` property value and have the same `worker_index` as the original(s). If this setting is True, the value of the `ignore_env_runner_failures` setting is ignored. ignore_env_runner_failures: Whether to ignore any EnvRunner failures and continue running with the remaining EnvRunners. This setting is ignored, if `restart_failed_env_runners=True`. max_num_env_runner_restarts: The maximum number of times any EnvRunner is allowed to be restarted (if `restart_failed_env_runners` is True). delay_between_env_runner_restarts_s: The delay (in seconds) between two consecutive EnvRunner restarts (if `restart_failed_env_runners` is True). restart_failed_sub_environments: If True and any sub-environment (within a vectorized env) throws any error during env stepping, the EnvRunner tries to restart the faulty sub-environment. This is done without disturbing the other (still intact) sub-environment and without the EnvRunner crashing. You can raise `ray.rllib.env.env_runner.StepFailedRecreateEnvError` from your environment's `step` method to not log the error. num_consecutive_env_runner_failures_tolerance: The number of consecutive times an EnvRunner failure (also for evaluation) is tolerated before finally crashing the Algorithm. Only useful if either `ignore_env_runner_failures` or `restart_failed_env_runners` is True. Note that for `restart_failed_sub_environments` and sub-environment failures, the EnvRunner itself is NOT affected and won't throw any errors as the flawed sub-environment is silently restarted under the hood. env_runner_health_probe_timeout_s: Max amount of time in seconds, we should spend waiting for EnvRunner health probe calls (`EnvRunner.ping.remote()`) to respond. Health pings are very cheap, however, we perform the health check via a blocking `ray.get()`, so the default value should not be too large. env_runner_restore_timeout_s: Max amount of time we should wait to restore states on recovered EnvRunner actors. Default is 30 mins. Returns: This updated AlgorithmConfig object. zAlgorithmConfig.fault_tolerance(worker_health_probe_timeout_s)zBAlgorithmConfig.fault_tolerance(env_runner_health_probe_timeout_s)z9AlgorithmConfig.fault_tolerance(worker_restore_timeout_s)z=AlgorithmConfig.fault_tolerance(env_runner_restore_timeout_s)) rrr1rrrrrrrr)rrrrrrrrrrrvrwrrrxryrzs r\fault_tolerancezAlgorithmConfig.fault_tolerances$D '*: : : RQ     "%5 5 5 MQ     #&6 6 6 NQ     #&6 6 6 NR     +.> > > V      58H H H &&      ),< < < T5      $'7 7 7 OS     &[ 8 8.HD + %[ 8 8.HD + &k 9 9/JD , .k A A3  4 ++ = =3RD 0 8 K K=  > -K ? ?5VD 2 '{ : :0LD - r^) model_configr1rmodel_config_dictrrrcT|tkrtddd|tkrtddd|}|tur||_|tur||_|turBt |t std|d |j ||S) aSets the config's RLModule settings. Args: model_config: The DefaultModelConfig object (or a config dictionary) passed as `model_config` arg into each RLModule's constructor. This is used for all RLModules, if not otherwise specified through `rl_module_spec`. rl_module_spec: The RLModule spec to use for this config. It can be either a RLModuleSpec or a MultiRLModuleSpec. If the observation_space, action_space, catalog_class, or the model config is not specified it is inferred from the env and other parts of the algorithm config object. algorithm_config_overrides_per_module: Only used if `enable_rl_module_and_learner=True`. A mapping from ModuleIDs to per-module AlgorithmConfig override dicts, which apply certain settings, e.g. the learning rate, from the main AlgorithmConfig only to this particular module (within a MultiRLModule). You can create override dicts by using the `AlgorithmConfig.overrides` utility. For example, to override your learning rate and (PPO) lambda setting just for a single RLModule with your MultiRLModule, do: config.multi_agent(algorithm_config_overrides_per_module={ "module_1": PPOConfig.overrides(lr=0.0002, lambda_=0.75), }) Returns: This updated AlgorithmConfig object. z3AlgorithmConfig.rl_module(_enable_rl_module_api=..)z:AlgorithmConfig.api_stack(enable_rl_module_and_learner=..)Trgz/AlgorithmConfig.rl_module(model_config_dict=..)z*AlgorithmConfig.rl_module(model_config=..)Fzq`algorithm_config_overrides_per_module` must be a dict mapping module IDs to config override dicts! You provided r ) rrr1rrrYrrZrr)rrr1rrrs r\rzAlgorithmConfig.rl_moduleysN !$4 4 4 IP      0 0 0 E@     -L { * *!-D   , ,#1D 0 C CCTJJ  @<@@@  6 = =5    r^)rrrrrrrrrrrrrrrrc|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|S)a Sets the config's experimental settings. Args: _validate_config: Whether to run `validate()` on this config. True by default. If False, ignores any calls to `self.validate()`. _use_msgpack_checkpoints: Create state files in all checkpoints through msgpack rather than pickle. _torch_grad_scaler_class: Class to use for torch loss scaling (and gradient unscaling). The class must implement the following methods to be compatible with a `TorchLearner`. These methods/APIs match exactly those of torch's own `torch.amp.GradScaler` (see here for more details https://pytorch.org/docs/stable/amp.html#gradient-scaling): `scale([loss])` to scale the loss by some factor. `get_scale()` to get the current scale factor value. `step([optimizer])` to unscale the grads (divide by the scale factor) and step the given optimizer. `update()` to update the scaler after an optimizer step (for example to adjust the scale factor). _torch_lr_scheduler_classes: A list of `torch.lr_scheduler.LRScheduler` (see here for more details https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate) classes or a dictionary mapping module IDs to such a list of respective scheduler classes. Multiple scheduler classes can be applied in sequence and are stepped in the same sequence as defined here. Note, most learning rate schedulers need arguments to be configured, that is, you might have to partially initialize the schedulers in the list(s) using `functools.partial`. _tf_policy_handles_more_than_one_loss: Experimental flag. If True, TFPolicy handles more than one loss or optimizer. Set this to True, if you would like to return more than one loss term from your `loss_fn` and an equal number of optimizers from your `optimizer_fn`. _disable_preprocessor_api: Experimental flag. If True, no (observation) preprocessor is created and observations arrive in model as they are returned by the env. _disable_action_flattening: Experimental flag. If True, RLlib doesn't flatten the policy-computed actions into a single tensor (for storage in SampleCollectors/output files/etc..), but leave (possibly nested) actions as-is. Disabling flattening affects: - SampleCollectors: Have to store possibly nested action structs. - Models that have the previous action(s) as part of their input. - Algorithms reading from offline files (incl. action information). Returns: This updated AlgorithmConfig object. ) r1rrrrrrrr) rrrrrrrrrs r\ experimentalzAlgorithmConfig.experimentalsx ; . .$4D ! #; 6 6,DD ) 0 C C5  6 %K 7 7-FD * %[ 8 8.HD + 4K G G9  : $; 6 6,DD ) &k 9 9/JD , r^c|j{t|jturdS t j|j}n#tjj$rYdSwxYwt||_| |jS)z)True if if specified env is an Atari env.NF) rr[rrr-makerErrorr close)rrs r\r zAlgorithmConfig.is_ataris > ! DH~~S((u htx((9?   uu &c]]DN IIKKK~sAAAcNt|jdkp t|jvS)zReturns whether this config specifies a multi-agent setup. Returns: True, if a) >1 policies defined OR b) 1 policy defined, but its ID is NOT DEFAULT_POLICY_ID. r)rrr)rs r\r,zAlgorithmConfig.is_multi_agent+s&4=!!A%O):$-)OOr^c8|jp|S)aiReturns the Learner sub-class to use by this Algorithm. Either a) User sets a specific learner class via calling `.training(learner_class=...)` b) User leaves learner class unset (None) and the AlgorithmConfig itself figures out the actual learner class by calling its own `.get_default_learner_class()` method. )r get_default_learner_classrs r\rVzAlgorithmConfig.learner_class5s"Fd&D&D&F&FFr^c|jt|jtr|jnt j|jzS)aDefines the model configuration used. This method combines the auto configuration `self _model_config_auto_includes` defined by an algorithm with the user-defined configuration in `self._model_config`.This configuration dictionary is used to configure the `RLModule` in the new stack and the `ModelV2` in the old stack. Returns: A dictionary with the model configuration. )_model_config_auto_includesrYrr dataclassesasdictrs r\rzAlgorithmConfig.model_configAsB/$,d33 8D  #D$677  r^c|}t||jt|jt|jtrWt|tr||j|St|t rtddStj |j}|||S|S)Nz1Cannot merge MultiRLModuleSpec with RLModuleSpec!) get_default_rl_module_specr]rrYrrrrZrr)rdefault_rl_module_specmulti_rl_module_specs r\r1zAlgorithmConfig.rl_module_specTs!%!@!@!B!B4555   + !$"6 7 7 7$. == ,4lCC*11$2FGGG11 68IJJ$K (,}T5I'J'J$$++,BCCC++* )r^c@|j|j|jpdzS|jS)Nr)rrrrs r\rz,AlgorithmConfig.train_batch_size_per_learnerps,  - 5(T->-C!D D11r^rc||_dSrb)rrrs r\rz,AlgorithmConfig.train_batch_size_per_learnerws-2***r^c$|j|jpdzS)aReturns the effective total train batch size. New API stack: `train_batch_size_per_learner` * [effective num Learners]. @OldAPIStack: User never touches `train_batch_size_per_learner` or `num_learners`) -> `train_batch_size`. r)rrrs r\total_train_batch_sizez&AlgorithmConfig.total_train_batch_size{s0D4E4JKKr^r worker_indexcR|jdkr|j|j|jpdzz }t ||kr[|jt ||jz|jpdzz }|dz |jz|krt |St |dzSt |S|jS)aAutomatically infers a proper rollout_fragment_length setting if "auto". Uses the simple formula: `rollout_fragment_length` = `total_train_batch_size` / (`num_envs_per_env_runner` * `num_env_runners`) If result is a fraction AND `worker_index` is provided, makes those workers add additional timesteps, such that the overall batch size (across the workers) adds up to exactly the `total_train_batch_size`. Returns: The user-provided `rollout_fragment_length` or a computed one (if user provided value is "auto"), making sure `total_train_batch_size` is reached exactly in each iteration. rr)rrrrr)rrrdiffs r\get_rollout_fragment_lengthz+AlgorithmConfig.get_rollout_fragment_lengths  '6 1 1'+&A,0D0IJ' #*++/FFF2S+66061484H4MA6OO"A%)EE$NN6777677!;;.// // /r^c|jr |jJdS|j}t|tr|d}n-|d}||pid|_d|_|j|_|jdkrd|_ d|_ nId|_ |j d krd n/ttj|j |jpdz |_ |S) aCreates a full AlgorithmConfig object from `self.evaluation_config`. Returns: A fully valid AlgorithmConfig object that can be used for the evaluation EnvRunnerGroup. If `self` is already an evaluation config object, return None. NFrTrr}rrrr)rYrTrYr`rrorWrrNrrrMrmathceil)rrTeval_config_objs r\get_evaluation_config_objectz,AlgorithmConfig.get_evaluation_config_objects-   )1114 2 ' 9 9 F/444GGOO#iiEi::O  , ,->-D" E E E)-%,0) +/*I'  (J 6 6)N  + 3!39:K3LQ3O00ZSW%=%=3:3S5J4400483I0  & .!.45F.G.J++ZSW%=%=.5.0@//++/3.?+  & .*.*;N ' ' !N$D$D$F$FF  '  %,:N )r^F)rr.r policy_dictsingle_agent_rl_module_specrrc||||\}}|}|jp|ttr;p|_t fd|D}nett sJt|trtjtr1pj|_fd|D} n4p||_fd|jzD}  j | j j j }nn5tjtrjntdd |_ j fd |Dj j j }||jzD]} |j| } |r| jr|| /| jt|tr |j| _nst|jtr%|jj} | td | | _n4| |jvr|j| j| _ntd | d | jt|tr |j| _nbt|jtr|jj} | | _n4| |jvr|j| j| _ntd| d | j| jV|| |t,}|&| j |j| _| j |j| _| j |j | _ |j | z| _ |S)atReturns the MultiRLModuleSpec based on the given env/spaces. Args: env: An optional environment instance, from which to infer the different spaces for the individual RLModules. If not provided, tries to infer from `spaces`, otherwise from `self.observation_space` and `self.action_space`. Raises an error, if no information on spaces can be inferred. spaces: Optional dict mapping ModuleIDs to 2-tuples of observation- and action space that should be used for the respective RLModule. These spaces are usually provided by an already instantiated remote EnvRunner (call `EnvRunner.get_spaces()`). If not provided, tries to infer from `env`, otherwise from `self.observation_space` and `self.action_space`. Raises an error, if no information on spaces can be inferred. inference_only: If `True`, the returned module spec is used in an inference-only setting (sampling) and the RLModule can thus be built in its light version (if available). For example, the `inference_only` version of an RLModule might only contain the networks required for computing actions, but misses additional target- or critic networks. Also, if `True`, the returned spec does NOT contain those (sub) RLModuleSpecs that have their `learner_only` flag set to True. Returns: A new MultiRLModuleSpec instance that can be used to build a MultiRLModule. NrPc:i|]}|tjSrrrrrrs r\rz.s6!!!t}%@AA!!!r^)r!c:i|]}|tjSrr)rrsingle_agent_specs r\rz.s3$$$@A4=):;;$$$r^c li|]0}|tjj|1Sr)rrr!r)rrcurrent_rl_module_specr s r\rz. sU $ $ $  4=2BFF !#4 $ $ $r^)multi_rl_module_classr!modules_to_loadload_state_pathrzWe have a MultiRLModuleSpec (z), but no `RLModuleSpec`s to compile the individual RLModules' specs! Use `AlgorithmConfig.get_multi_rl_module_spec(policy_dict=.., rl_module_spec=..)`.c:i|]}|tjSrrrs r\rz.Es6%%%4=)DEE%%%r^zTThe default rl_module spec cannot have an empty module_class under its RLModuleSpec.zModule class for module z cannot be inferred. It is neither provided in the rl_module_spec that is passed in nor in the default module spec used in the algorithm.zCatalog class for module )get_multi_agent_setuprrrYrrrkeysr! __class__r rrrrZrremove_modules module_class catalog_classrrrrr)rrr.rrr_rr module_specsrZrVrrrr r s ` @@r\rRz(AlgorithmConfig.get_multi_rl_module_specsUN  !77C7OONK "&!@!@!B!B"&!5!O9O ,l ; ;o  ,E/E (:H ' 6#4!!!!(--//!!!$$$ 46GHH H HH0,??W 4DlSS(C).>&8F%4$$$$EPEUEUEWEW$$$LL4M7M&8F%4 $ $ $ $ $(*@*PP$&& $ $ $L(>'G'G*@*V$0$:$J$:$J!7!D (H(($$/: ""8"H,WW3B43)C 6CCC>L+:(>'G'G*@*V%%%%!,!1!1!3!3%%%%;$J$:$J!7!D(H ( ($%';'KKI I I/>yIK +": $33I>>>'/4lCC/E/RK,, 6 F UU#9#I#VL$+(C03K 0'/+/+< (( % (E(E(G(GG(($#r^ct|dr|jr|dvrtd|d|dkrd}t||dS)zzAAlgorithmConfig._validate_env_runner_settings..s&P&P&Pdtz&P&P&Pr^z`gym_env_vectorize_mode` (zD) must be a member of `gymnasium.VectorizeMode`! Allowed values are r )r2r)r- VectorizeModerr)rallowed_vectorize_modess r\rz-AlgorithmConfig._validate_env_runner_settingss"% " # #&P&Pc>O&P&P&P P# #   &.E E E   AT-HAA&=AAA      F Er^c |jsy|j[|jT|jM|jF|j?|j8|j1|j*|j#|j |j |j |j |j |ddSdSdS)zValidates callbacks settings.NzConfig settings `config.callbacks(on_....=lambda ..)` aren't supported on the old API stack! Switch to the new API stack through `config.api_stack(enable_env_runner_and_connector_v2=True, enable_rl_module_and_learner=True)`.)rrr rrrrrrrrrr r rrs r\rz,AlgorithmConfig._validate_callbacks_settings s 6 5A3?1=3?1=/;8D4@2>1=0<6B:FCO!!<#  POr^cfd\}}}d}|jdvr |jdkrdS|jdvrt\}}}nt\}}|jdkr|jr|d|P|jdkrEt j|jtkr#|j s|j r|d|j r8dd l m }|j |j|jfvr|d |j |||||||dS) zGValidates framework settings and checks whether framework is installed.NNNN>tftf2rr<zCannot use `framework=tf` with the new API stack! Either switch to tf2 via `config.framework('tf2')` OR disable the new API stack via `config.api_stack(enable_rl_module_and_learner=False)`.z0torch.compile is only supported from torch 2.0.0r)rz`config.torch_compile_learner_what_to_compile` must be one of [TorchCompileWhatToCompile.forward_train, TorchCompileWhatToCompile.complete_update] but is )rr/r0rrrparse __version__r:rr*ray.rllib.core.learner.torch.torch_learnerrrrCOMPLETE_UPDATE(_check_if_correct_nn_framework_installed_resolve_tf_settings)r_tf1_tf_tfv_torchrrs r\rz,AlgorithmConfig._validate_framework_settings=s*c4  ] 2 2t7IW7T7T F  = 0 0+ooOD#tt(**IFA   % %$*K %   J     "g-- f0114RRR+S/3/HS   P Q Q Q  %       9)7)9B!!EBEE 55dCHHH !!$-----r^cdS)z6Checks, whether resources related settings make sense.Nrrs r\rz,AlgorithmConfig._validate_resources_settingsm r^ct|jtttfr,|jD]&}||jvr|d|d%dSdS)z8Checks, whether multi-agent related settings make sense.z?`config.multi_agent(policies_to_train=..)` contains policy ID (z<) that was not defined in `config.multi_agent(policies=..)`!N)rYr!r)r2r*rr)rrs r\rz.AlgorithmConfig._validate_multi_agent_settingsqs d,tS%.@ A A -  dm++%%>&)>>>    r^c>|jdur|d|jdkr*|js#td|jd|jdkr'|jdkr|jr|d|jdkrG|js|dd S|j d krtd d Sd St|jtr |jdkr |d |jd d Sd S)z7Checks, whether evaluation related settings make sense.Tz`enable_async_evaluation` has been deprecated (you should set this to False)! Use `config.evaluation(evaluation_parallel_to_training=True)` instead.rzYou have specified z evaluation workers, but your `evaluation_interval` is 0 or None! Therefore, evaluation doesn't occur automatically with each call to `Algorithm.train()`. Instead, you have to call `Algorithm.evaluate()` manually in order to trigger an evaluation run.z`evaluation_parallel_to_training` can only be done if `evaluation_num_env_runners` > 0! Try setting `config.evaluation_parallel_to_training` to False.rzU`evaluation_duration=auto` not supported for `evaluation_parallel_to_training=False`!rzWhen using `config.evaluation_duration='auto'`, the sampling unit used is always 'timesteps'! You have set `config.evaluation_duration_unit='episodes'`, which is ignored.z`evaluation_duration` (z) must be an int and >0!N) rrrWrLr(rnr\rRrMrNrYrrs r\rz-AlgorithmConfig._validate_evaluation_settings~s  '4 / /        *Q . .t7O . NN"d&E"""     +q 0 0-2243   E     #v - -7 !!?.*<<V=<43S99 '1,,   $*B     -,r^c|jdkr|jr|d|jdkr<|j|jd<|jr|jpd|jd<dS|jpd|jd<dSdS)z2Checks, whether input related settings make sense.rzOff-policy estimation methods can only be used if the input is a dataset. We currently do not support applying off_policy_estimation_method on a sampler input.datasetrrrN)r$rUrrr<rYrWrrs r\rz(AlgorithmConfig._validate_input_settingss ;) # #(J #   -    ;) # #;?:VD 6 7! M 483R3WVW!-000 483G3L1!-000 $ #r^c,|jsi|jtd|j#td|jd|jr|ddSt|jj dr(td|jj d|js|d  tj |j ttd |j Dd tjDn*#t $rtd YnwxYwt#j|jd d|jr|dd}|j d)||dd|j dikr+||dddSdS)zAChecks, whether settings related to the new API stack make sense.NzYou have setup a RLModuleSpec (via calling `config.rl_module(...)`), but have not enabled the new API stack. To enable it, call `config.api_stack(enable_rl_module_and_learner=True)`.zRYou specified a custom Learner class (via `AlgorithmConfig.training(learner_class=z)`, but have the new API stack disabled. You need to enable it via `AlgorithmConfig.api_stack(enable_rl_module_and_learner=True)`.zYou are using the new API stack EnvRunners (SingleAgentEnvRunner or MultiAgentEnvRunner), but have forgotten to switch on the new API stack! Try setting `config.api_stack(enable_rl_module_and_learner=True)`._on_new_api_stackzYou are running aM on the new API stack! This is the new default behavior for this algorithm. If you don't want to use the new API stack, set `config.api_stack(enable_rl_module_and_learner=False,enable_env_runner_and_connector_v2=False)`. For a detailed migration guide, see here: https://docs.ray.io/en/master/rllib/new-api-stack-migration-guide.htmlzSetting `enable_rl_module_and_learner` to True and `enable_env_runner_and_connector_v2` to False ('hybrid API stack') is not longer supported! Set both to True (new API stack) or both to False (old API stack), instead.c&i|]\}}|dk ||Svf_share_layersrrs r\rzDAlgorithmConfig._validate_new_api_stack_settings..s)OOO$!Q=N8N8NA8N8N8Nr^c&i|]\}}|dk ||SrQrrs r\rzDAlgorithmConfig._validate_new_api_stack_settings..s)SSS$!QAAR B B B OO$*"2"2"4"4OOOSS."6"8"8SSS        NN9       $(G'      "    2    + ! :n % 1   (//OO    :+ , 2 2   (//)+@      3 2sA+D>>$E%$E%cR|jr|jdurtdd|jdvr|d|jdur d|jd<|jdur d|jd <|jd rtd d d |j durdS|j st|j dkri|j dkr7t|jdkr|d|j dn|j dur|dd|_ dS|j tkr|j dvr d|_ dS|jrddlmddlmd|jr|j||j}t1|t2rd|Dn d|D}t7fd|Dr d|_ dSd|_ dSd|_ dS|j dur+|j dkr"|d|j ddSdSdS)NFzAlgorithmConfig.render_envzThe `render_env` setting is not supported on the new API stack! In order to log videos to WandB (or other loggers), take a look at this example here: https://github.com/ray-project/ray/blob/master/rllib/examples/envs/env_rendering_and_recording.py)rhr)rllibrNzF`config.preprocessor_pref` must be either 'rllib', 'deepmind' or None!Trrcustom_preprocessorz 1 not supported yet for framework=rtzhCannot use `simple_optimizer` if `num_gpus` > 1! Consider not setting `simple_optimizer` in your config.)r<rr)DynamicTFPolicy) TorchPolicyc\g|])}t|ttfr t|n|*Sr)rYr*r)r')rrs r\r6zGAlgorithmConfig._validate_to_be_deprecated_settings..sF .8udm-L-LV D))RVr^c*g|]}tSrr)rrs r\r6zGAlgorithmConfig._validate_to_be_deprecated_settings..s9991*,,999r^c3bK|])}|jpdupt|jpf V*dSrb) policy_class issubclass)rrr`radefault_policy_clss r\rzFAlgorithmConfig._validate_to_be_deprecated_settings..sr  &<*<E%)?-?(+6r^z<`simple_optimizer=False` not supported for config.framework(z)!)rrrrrrrrrrrrrr[rzrr,"ray.rllib.policy.dynamic_tf_policyr`ray.rllib.policy.torch_policyrarget_default_policy_classrrYrr.any)rr policy_specsr`rargs @@@r\rz3AlgorithmConfig._validate_to_be_deprecated_settingsPs_  2 te7S7S 0t      !)D D D   X     )T 1 16:DJ2 3  *d 2 27;DJ3 4 :>>/ 0 0  R       D ( ( D2> t}q7H7H!U**tDzz/Bk/Q/Q!!7!%!3777&$..!!N%*D ! ! ! "&6 6 6!88(,%%%$ .NNNNNNEEEEEE%)"?X)-)Q)QRV)W)W&= "(D11 :$,OO$5$5 :9999  !-  2-1D))),1D)))(-%%% "e + +!U**!!?(,(:???, +**r^c|jr<|js5|jdks*|js#|j|j|d|jr*|jj dkrt dddl m }ddlm}|jr*t#|j|s|d|jr*t#|j|s|ddd lm}|jr*t#|j|s|d |jr#|js|js|d |jr'|jr |jd kr|d ddlm}t=|}|jr5|jdkr*|j|vr!|d|jd|dddl m!}|j"r,t#|j"|s|ddSdSdS)NrzXIf no evaluation should be run, `action_space` and `observation_space` must be provided.BCz`ignore_final_observation=True` (zeros-out truncation observations), but the algorithm isn't `BC`. It is recommended to use this setting only with `BC`, b/c other RL algorithms rely on truncation-observations due to value function estimates.) OfflineData)OfflinePreLearnerzZUnknown `offline_data_class`. OfflineData class needs to inherit from `OfflineData` class.z]Unknown `prelearner_class`. PreLearner class needs to inherit from `OfflinePreLearner` class.)EpisodeReplayBufferzUnknown `prelearner_buffer_class`. The buffer class for the prelearner needs to inherit from `EpisodeReplayBuffer`. Specifically it needs to store and sample lists of `Single-/MultiAgentEpisode`s.zSetting `input_read_batch_size` is only allowed in case of a dataset that holds either `EpisodeType` or `BatchType` data (i.e. rows that contains multiple timesteps), but neither `input_read_episodes` nor `input_read_sample_batches` is set to `True`.r}zWhen recording episodes only complete episodes should be recorded (i.e. `batch_mode=='complete_episodes'`). Otherwise recorded episodes cannot be read in for training.)OfflinePolicyEvaluationTypes eval_lossz!Unknown offline evaluation type: zx.Available types of offline evaluation are either `'eval_loss' to evaluate the training loss on a validation dataset or r )OfflineEvaluationRunnerzyUnknown `offline_eval_runner_class`. OfflineEvaluationRunner class needs to inherit from `OfflineEvaluationRunner` class.)# is_offline is_onlinerWrLrrrr6rrzr(rnray.rllib.offline.offline_dataro$ray.rllib.offline.offline_prelearnerrpr%rfr74ray.rllib.utils.replay_buffers.episode_replay_bufferrqr8r+r)r*r@rJr2ray.rllib.offline.offline_policy_evaluation_runnerrrr)r]+ray.rllib.offline.offline_evaluation_runnerrtr^)rrorprqrroffline_eval_typesrts r\rz*AlgorithmConfig._validate_offline_settingssD O N 4q88D>>>>JJJJJJ  " :  #[, ,     ,        !#4* *     2            '   (*=1 1     0     %   $ (,(F        K * #666   D         "">??  ( , ;;,4PPP   VD4PVV@RVVV    XWWWWW  ) *  *,C3 3     8         r^c|jS)zDefines if this config is for online RL. Note, a config can be for on- and offline training at the same time. )rrs r\rvzAlgorithmConfig.is_online$s r^ct|joet|jtp9t|jtot|jdto|jdko|jS)z*Defines, if this config is for offline RL.rr)boolr$rYrr)rrs r\ruzAlgorithmConfig.is_offline,su    24;,,Wt{D11UjQQT6U6U  2 y( 21 r^crd|vr|d|d<d|vrt|d|d<t|d|d<t|d|d<t|dtrt|d|d<d|vrRt|ddtr$t|dd|dd<t|ddtr$t|dd|dd<t|d d tr$t|d d |d d <|d }|t|d t t frt|d |d <|d r t|d <|dr t|d<t|d t t frt|d |d <|d r t|d <|dr t|d<|S)Nr rrrrrr[rrrrrr r!) rnr8rYr[rr2r*r)r6)r ma_configs r\rzAlgorithmConfig._serialize_dict<s  & &"(**->"?"?F;  f  ,VG_==F7O,VK-@AA{%3F;M4N%O%O!" fUmT * * :*6%=99F5M !V + + v4599&AA4 H H ,6D-.v666F) *6 2 f1266v>> E E 3A+,V444F' ( 0 fWo)).994 @ @ .<w///F7ON + JJ|,,  )-- 33c5\BB D(,Yz-B(C(C *%}}011 B1A -.}}011 B1A -. fjj,,sEl ; ; :!%fZ&8!9!9F:  ::) * * ;*:F& ' ::) * * ;*:F& ' r^r~rvc&|dkrd}n>|dkrd}n5|dkrd}n,|dkrd}n#|d krd }n|d krd }n|d krd}n|dkrd}|r|dkrtdddn|dkrtdddn|dkrtdddn||dkrtdddnc|dkrtdddnJ|d krtd!d"dn1|d#krtd$d%dn|d&krtd'd(d|S))Nrr rrrrXrrrr$rrnum_cpus_for_driverr num_workersrrrrTrgrz!config.metrics_smoothing_episodesz)config.metrics_num_episodes_for_smoothingrzconfig.min_iter_time_szconfig.min_time_s_per_iterationrzconfig.min_time_s_per_reportingrz)config.min_sample_timesteps_per_reportingz)config.min_sample_timesteps_per_iterationrz(config.min_train_timesteps_per_reportingz(config.min_train_timesteps_per_iterationrzconfig.timesteps_per_iterationzY`config.min_sample_timesteps_per_iteration` OR `config.min_train_timesteps_per_iteration`rzconfig.evaluation_num_episodeszK`config.evaluation_duration` and `config.evaluation_duration_unit=episodes`r)r~rvs r\r{z'AlgorithmConfig._translate_special_keysms9 +  #CC * * *.CC * * *.CC K  !CC G^^CC H__CC ) ) )-CC M ! !#C 2 ///#1> 444#;C )))#09 222#99 <<<#CC ;;;#BB 111#8A  111#8A  r^c|jdvr|s|stddSdS|jdkr|stddSdS)z?Check if tf/torch experiment is running and tf/torch installed.>r<r=zTensorFlow was specified as the framework to use (via `config.framework([tf|tf2])`)! However, no installation was found. You can install TensorFlow via `pip install tensorflow`rzPyTorch was specified as the framework to use (via `config.framework('torch')`)! However, no installation was found. You can install PyTorch via `pip install torch`.N)r ImportError)rrDrErGs r\rBz8AlgorithmConfig._check_if_correct_nn_framework_installeds   . . C !Y     7 * * !G+ *  r^c`|r|jdkru|jdkr|dkrtd|s|td|jd|jdd S|r'|jdkrtdd Sd Sd S) zCheck and resolve tf settings.r=rzxYou configured `framework`=tf2, but your installed pip tf-version is < 2.0! Make sure your TensorFlow version is >= 2.x.zExecuting eagerly (framework='z'), with eager_tracing=z. For production workloads, make sure to set eager_tracing=True in order to match the speed of tf-static-graph (framework='tf'). For debugging purposes, `eager_tracing=False` is the best choice.r<zYour framework setting is 'tf', meaning you are using static-graph mode. Set framework='tf2' to enable eager execution with tf2.x. You may also then want to set eager_tracing=True in order to reach similar execution speed as with static-graph mode.N)rrZexecuting_eagerlyenable_eager_executionr(infor)rrDrFs r\rCz$AlgorithmConfig._resolve_tf_settingss  D&%//!U**taxx ) ))++ .++--- KK<1C<<'+'9<<<      d(D00 KK3       00r^)rrr.default_policy_classrc tj|p|j}t|tt t fr d|D}d}d}t|tjj r/tj |j \}}n|t|dr't|jtjr|j}n6t|dr&t|jtjr|j}t|dr't|jtjr|j}n6t|dr&t|jtjr|j}|I|"| t(dgd}|#| t(ddgd}|D]\}}t|t,st-|x||<}|j| |||_|jnt|d r|jn||||vr||d} n2tt2rd} |j} t td rjrjn} t;| dkrjn| dt?fd | Dr} n| rV| D]S} | | dd |kr?| (| | krtAd | } Tn'||} n"|jr|j} ntAd|d| ||_|jnt|d r|jn||||vr||d} n2tt2rd} |j} t td rjrjn} t;| dkrjn!| dt?fd| Dr} n| rV| D]S} | | dd |kr?| (!| | krtAd!| } Tn'||} n"|jr|j} ntAd|d| ||_t||j"tFsp||j""t||j"tHsJ|d%||j"pi||_"|j&/tO|j&st |j&dfd }n|j&}||fS)an Compiles complete multi-agent config (dict) from the information in `self`. Infers the observation- and action spaces, the policy classes, and the policy's configs. The returned `MultiAgentPolicyConfigDict` is fully unified and strictly maps PolicyIDs to complete PolicySpec objects (with all their fields not-None). Examples: .. testcode:: import gymnasium as gym from ray.rllib.algorithms.ppo import PPOConfig config = ( PPOConfig() .environment("CartPole-v1") .framework("torch") .multi_agent(policies={"pol1", "pol2"}, policies_to_train=["pol1"]) ) policy_dict, is_policy_to_train = config.get_multi_agent_setup( env=gym.make("CartPole-v1")) is_policy_to_train("pol1") is_policy_to_train("pol2") Args: policies: An optional multi-agent `policies` dict, mapping policy IDs to PolicySpec objects. If not provided uses `self.policies` instead. Note that the `policy_class`, `observation_space`, and `action_space` properties in these PolicySpecs may be None and must therefore be inferred here. env: An optional env instance, from which to infer the different spaces for the different policies. If not provided, tries to infer from `spaces`. Otherwise from `self.observation_space` and `self.action_space`. Raises an error, if no information on spaces can be infered. spaces: Optional dict mapping policy IDs to tuples of 1) observation space and 2) action space that should be used for the respective policy. These spaces were usually provided by an already instantiated remote EnvRunner. Note that if the `env` argument is provided, tries to infer spaces from `env` first. default_policy_class: The Policy class to use should a PolicySpec have its policy_class property set to None. Returns: A tuple consisting of 1) a MultiAgentPolicyConfigDict and 2) a `is_policy_to_train(PolicyID, SampleBatchType) -> bool` callable. Raises: ValueError: In case, no spaces can be infered for the policy/ies. ValueError: In case, two agents in the env map to the same PolicyID (according to `self.policy_mapping_fn`), but have different action- or observation spaces according to the infered space information. c,i|]}|tSrr)rrs r\rz9AlgorithmConfig.get_multi_agent_setup...s>>>cZ\\>>>r^Nrrrrrrrr/c3JK|]}|kVdSrb)r)rrg env_unwrapped one_obs_spaces r\rz8AlgorithmConfig.get_multi_agent_setup..xsJ&;;C@@MQr^)rhzTwo agents in your environment map to the same policyID (as per your `policy_mapping_fn`), however, these agents also have different observation spaces!z3`observation_space` not provided in PolicySpec for z and env does not have an observation space OR no spaces received from other workers' env(s) OR no `observation_space` specified in config!c3JK|]}|kVdSrb)r)rrgr one_act_spaces r\rz8AlgorithmConfig.get_multi_agent_setup..sJ&66s;;}Lr^zTwo agents in your environment map to the same policyID (as per your `policy_mapping_fn`), however, these agents also have different action spaces!z.`action_space` not provided in PolicySpec for z and env does not have an action space OR no spaces received from other workers' env(s) OR no `action_space` specified in config!Frc |vSrbr)rbatchpolss r\is_policy_to_trainzAAlgorithmConfig.get_multi_agent_setup..is_policy_to_trains d{"r^rb)(rrrrYr2r)r*rayactor ActorHandler _get_spacesremoterxrr-Spacerrrrrwr'rerrr r/ get_agent_idsrrrrZrrr`rror!r)rrrr.r env_obs_space env_act_spacerrr6 mapping_fnaidsrgr7rrrrrs @@@@r\rz%AlgorithmConfig.get_multi_agent_setups:x=!:T];; hdE 2 3 3 ?>>X>>>H   c390 1 1 1+.73?3I3I3K3K+L+L (M==_s677 6J,ci== 6!$ < 122 6z%sy88 6!$ 5 s122 1z'88 1!$ 7 n-- 1* #)33 1!$ 0   $ & +;dV D DQ G $ & +;dD\ J J1 M !)  5 5 7 7L L  Ck:66 G.8+.FF '/4H4T-A *,418k1J1J S PS %#-- &s AII }==4 $I!%!7J"=2CDD;)9; 55+88:: D 4yyA~~(5(G (5(K(KDQRG(T(T #'U%2 $U#'UUC)z#tDAAASHH%.$9(5(K(KC(P(P'0)1)1+5)H+&+&%& -:,O,OPS,T,T ". -II+ $ 6II$CCCC3< /'/18k1J1J S PS %#-- &s AII }==2 $I!%!7J"=2CDD;)9; 55+88:: D 4yyA~~(5(B (5(F(FtAw(O(O #'P%2 $P#'PPC)z#tDAAASHH%.$9(5(F(Fs(K(Ky(X(X*4)C+&+&%& -:,J,J3,O,O ". -II& $ 1II$>>>> .7 *hsm2ODD }+3zSM($8833(,yyUy'C'C'T'TSM(.B(( $  ! -ht?U6V6V -t-..D # # # # # # #"&!7 +++r^zAlgorithmConfig.build_algo)rirc|j|i|Srb)rrr'ris r\rWzAlgorithmConfig.buildst////r^z*AlgorithmConfig.get_multi_rl_module_spec()cdSrbrrs r\get_marl_module_specz$AlgorithmConfig.get_marl_module_specrIr^zAlgorithmConfig.env_runners(..)cdSrbrrs r\rolloutszAlgorithmConfig.rolloutsrIr^cdSrbrrs r\ explorationzAlgorithmConfig.explorationrIr^r~cdSrbrrs r\rz+AlgorithmConfig.recreate_failed_env_runners r^c*tddddS)Nz+AlgorithmConfig.recreate_failed_env_runners*AlgorithmConfig.restart_failed_env_runnersTrgrrs r\rz+AlgorithmConfig.recreate_failed_env_runners s,=<      r^%AlgorithmConfig._enable_new_api_stackcdSrbrrs r\rz%AlgorithmConfig._enable_new_api_stack  r^c*tddddS)Nrz,AlgorithmConfig.enable_rl_module_and_learnerTrgrrs r\rz%AlgorithmConfig._enable_new_api_stacks,7>      r^z2AlgorithmConfig.enable_env_runner_and_connector_v2cdSrbrrs r\uses_new_env_runnersz$AlgorithmConfig.uses_new_env_runners"rr^AlgorithmConfig.num_env_runnerscdSrbrrs r\rsz#AlgorithmConfig.num_rollout_workers'rr^c*tddddS)Nz#AlgorithmConfig.num_rollout_workersrTrgrrs r\rsz#AlgorithmConfig.num_rollout_workers,s,51      r^&AlgorithmConfig.evaluation_num_workerscdSrbrrs r\rz&AlgorithmConfig.evaluation_num_workers4rr^c*tddddS)Nrz*AlgorithmConfig.evaluation_num_env_runnersTrgrrs r\rz&AlgorithmConfig.evaluation_num_workers9)8<    r^'AlgorithmConfig.num_envs_per_env_runnercdSrbrrs r\rtz#AlgorithmConfig.num_envs_per_workerBrr^c*tddddS)Nz#AlgorithmConfig.num_envs_per_workerrTrgrrs r\rtz#AlgorithmConfig.num_envs_per_workerG)59    r^*AlgorithmConfig.ignore_env_runner_failurescdSrbrrs r\rvz&AlgorithmConfig.ignore_worker_failuresPrr^c*tddddS)Nz&AlgorithmConfig.ignore_worker_failuresrTrgrrs r\rvz&AlgorithmConfig.ignore_worker_failuresUrr^rcdSrbrrs r\rwz'AlgorithmConfig.recreate_failed_workers^rr^c*tddddS)Nz'AlgorithmConfig.recreate_failed_workersrTrgrrs r\rwz'AlgorithmConfig.recreate_failed_workerscs)9<    r^+AlgorithmConfig.max_num_env_runner_restartscdSrbrrs r\rz'AlgorithmConfig.max_num_worker_restartslrr^c*tddddS)Nz'AlgorithmConfig.max_num_worker_restartsrTrgrrs r\rz'AlgorithmConfig.max_num_worker_restartsqs)9=    r^3AlgorithmConfig.delay_between_env_runner_restarts_scdSrbrrs r\rz/AlgorithmConfig.delay_between_worker_restarts_szrr^c*tddddS)Nz/AlgorithmConfig.delay_between_worker_restarts_srTrgrrs r\rz/AlgorithmConfig.delay_between_worker_restarts_ss)AE    r^=AlgorithmConfig.num_consecutive_env_runner_failures_tolerancecdSrbrrs r\rxz9AlgorithmConfig.num_consecutive_worker_failures_tolerances r^c*tddddS)Nz9AlgorithmConfig.num_consecutive_worker_failures_tolerancerTrgrrs r\rxz9AlgorithmConfig.num_consecutive_worker_failures_tolerances)KO    r^1AlgorithmConfig.env_runner_health_probe_timeout_scdSrbrrs r\ryz-AlgorithmConfig.worker_health_probe_timeout_srr^c*tddddS)Nz-AlgorithmConfig.worker_health_probe_timeout_srTrgrrs r\ryz-AlgorithmConfig.worker_health_probe_timeout_ss)?C    r^,AlgorithmConfig.env_runner_restore_timeout_scdSrbrrs r\rzz(AlgorithmConfig.worker_restore_timeout_srr^c*tddddS)Nz(AlgorithmConfig.worker_restore_timeout_srTrgrrs r\rzz(AlgorithmConfig.worker_restore_timeout_ss):>    r^7AlgorithmConfig.validate_env_runners_after_constructioncdSrbrrs r\ruz3AlgorithmConfig.validate_workers_after_constructionrr^c*tddddS)Nz3AlgorithmConfig.validate_workers_after_constructionrTrgrrs r\ruz3AlgorithmConfig.validate_workers_after_constructions)EI    r^'AlgorithmConfig.num_cpus_per_env_runnercdSrbrrs r\r_z#AlgorithmConfig.num_cpus_per_workerrr^c*tddddS)Nz#AlgorithmConfig.num_cpus_per_workerrTrgrrs r\r_z#AlgorithmConfig.num_cpus_per_workerrr^'AlgorithmConfig.num_gpus_per_env_runnercdSrbrrs r\r`z#AlgorithmConfig.num_gpus_per_workerrr^c*tddddS)Nz#AlgorithmConfig.num_gpus_per_workerrTrgrrs r\r`z#AlgorithmConfig.num_gpus_per_workerrr^/AlgorithmConfig.custom_resources_per_env_runnercdSrbrrs r\raz+AlgorithmConfig.custom_resources_per_workerrr^c*tddddS)Nz+AlgorithmConfig.custom_resources_per_workerrTrgrrs r\raz+AlgorithmConfig.custom_resources_per_workers)=A    r^AlgorithmConfig.num_learnerscdSrbrrs r\rbz#AlgorithmConfig.num_learner_workersrr^c*tddddS)Nz#AlgorithmConfig.num_learner_workersrTrgrrs r\rbz#AlgorithmConfig.num_learner_workerss)5.    r^$AlgorithmConfig.num_cpus_per_learnercdSrbrrs r\rcz+AlgorithmConfig.num_cpus_per_learner_workerrr^c*tddddS)Nz+AlgorithmConfig.num_cpus_per_learner_workerrTrgrrs r\rcz+AlgorithmConfig.num_cpus_per_learner_worker)=6    r^$AlgorithmConfig.num_gpus_per_learnercdSrbrrs r\rdz+AlgorithmConfig.num_gpus_per_learner_worker rr^c*tddddS)Nz+AlgorithmConfig.num_gpus_per_learner_workerrTrgrrs r\rdz+AlgorithmConfig.num_gpus_per_learner_workerrr^)AlgorithmConfig.num_cpus_for_local_workercdSrbrrs r\rez)AlgorithmConfig.num_cpus_for_local_workerrr^c*tddddS)Nrz)AlgorithmConfig.num_cpus_for_main_processTrgrrs r\rez)AlgorithmConfig.num_cpus_for_local_workers);;    r^rb)rWN)NNTr;)NNr)T)rz __module__ __qualname____doc__ staticmethodrerj classmethodrrrrrr r[rr<rrBrorrrrr rrrrrr.rr>rrFrrr9rGrMr@r r-rrDrTrCrYr\r1r^rrfloatrjrrr=rr7r r r!rr.rr?rrrrr%r$rrArr;rErr4rrrrrrrpropertyr r,rVrr1rsetterrrrrrrrrrr'rrRrr r"r'rrnrr.rwrrrrrrrrrrrrrvrurr{rBrCr-r&rrrWrrrrrrrsrrtrvrwrrrxryrzrur_r`rarbrcrdre __classcell__rs@r\r`r`nsI)<!!\!!!\! DT[2; ; [; zf<f<8D>f<f<f<f

<tCH~%c 2B[2,4S>,,,,"$& , , , ,$;333;:3(.29= & & eCL) *& !"f*!56&  &  & & & & T  oo  oooobsskssssr II  IIII\"&HL596: ,,,g ,hci.B(CCDE , !!12 , ""23 , ,,,,b"&HL $$$g $hci.B(CCDE $  $$$$L x D    JI/:=H:E5@3>!yyyC=y ~ y %SM y "$sCx.1 y (S#X7y (~y08}y,4C=y/7smy'tny.6c]y+33-y#4S>2y #+4.!y" #yyyyz8C=H))&.tn)-5TN)  ))))Z.9^/:1<,7%05@,7'2/:#.)4$^^^ eCL) *^]+ ^ $CI. ^ sy) ^TN^uT5[12^$D>^tn^'tn^4.^"#^ !^^^^F*5)42=5@1I6A0;=H$/"-,72=5@-8=a@"*$AaB%-TNCaD%TNEaF#4#89GaH%TNIaJ#+5/KaL$C=MaN%SMOaP"*$QaR&.e_Sal maaaaL '2AL EP8C%uuusmu'uS%_'=> u 'uUCZ'89 u ,4C= u6>e_u }u-5SMu Y0u$ !3:#34mT-%889;  u"5=TN#u$&d38n5%u& 'uuuut"-/:%0&1*56A$/(32= +$/*:4D%2B3> EP8C9OOOO + , O E? O sm O#3-O'/smOSMO! O"*$O~OD>O( Y0)O*$ !3:#34mT-%889;  +O65=TN7O8&d38n59O: ;OOOOj  U HSEPGREP    JULWHSFQEPDOCN9UUU! $}%tD,?'@@ A U $E(DN*B$CD U"%$x.(@"ABU$E(DN*B$CDU"%$x.(@"ABU$, (DN* +$ U"* (DN* +" U #+ (DN* +# !U&,4 (DN* +, 'U,'uXtH~-E'FG-U.!)xh/G)H I/U0%U8T(^+C%DE1U2#54>)A#BC3U4"%$x.(@"AB5U6!xh'?!@A7U8 hX&> ?@9U: ;UUUUt.99D2=KVKV7B:EGR 8C5@4?9D5@2=6A8C9DI>I>IFQAL)9/Wggg&c]g&eCHo6 g #+3- g 6A.94?/:*52=6A+62=-8.93>+62=3>8C7B'26A-8-8 +(37B0;2=6A-85@+63>0;*5OtttsHi[+-E$FFGHt%TN t $E#x-$89 t #+4. t$DcN3t&d^t$,D>t (}t#3-t"*$t!)c 3t#4.t"*$t %TN!t"&d^#t$#+4.%t&#4.'t("*$)t*#+4.+t,)1 -t.(0}/t0tn1t2&.d^3t4%TN5t6&c]7t8 9t: ~;t<"*$s)!4=t>'uo?t@#+3-AtB&.d^CtD&c]EtF%-TNGtH$C=ItJ#+4.KtL (~MtN#3-OtP Qttttv -8  6A-8(3.>$*+rrr ,j.BB C r &c] r$ g}-x7 8 r$ *X&(O1Ld1R(SS T r&.d^r!*r! r, -rrrrn;F@K-5TN  @:E(3#.(3'2),,,!"f*!56, ~ , C= ,  ~ ,tn,sm, ,,,,b6A5@5@?J:EGR=H8C$4/ 0 0(82B&6!1'LLL%-TNL%-TN L &.c] L .6e_ L*2$L8@}L,4E?L'/uoL( )LLLLbMX5@ *.DDDuT#s(^5G%GHID!!12 D 08 55 60 D DDDDR,03>3> @K4?5@DOQQQ#4.Q#+4. Q #+4. Q &. $t*d8T$Z#788 9& Q08~Q$,D>Q%-TNQ4Q QQQQf$X0PPPPXP GtI G G GX G  X $**X*62c222X2 "(3#3$333)(3LLLLXL'0'0'0C'0'0'0'0VH # $HHHHT++++Z     ",< " " " " "5i#1E+F " " " ""&CG)- YY g Yc5CI)=#>>?@Y! Y  YYYY|"&HL$7;>Br$r$r$g r$hci.B(CCDE r$  r$d3 ?34r$&.l%;r$ r$r$r$r$h(((((&###:((((($#D#### ,,,, &&&&%%%'''&&& T#s(^   X "####    :........`      >>>@MMM4tttpfffPjjjX4X  D    X  ..\.`HHSH4H3HHH\HT*>:>!%HL7; F,F,F,56F,g  F, hci.B(CCDE F, 'tF|4 F, )8X4OQU4U+VV WF,F,F,[F,PZ0>>>00?>0Z@MMM  NM Z5TBBB  CB Z5TBBB  CB Z L  X  !'  (' Z;4HHH  IHX !  "! ZHPTUUU  VUX Z5TBBB  CBX     Z.sA   {$? @ @      r^z`differentiable_learner_configs` must be a list of `DifferentiableLearnerConfig` instances, but at least one instance is not a `DifferentiableLearnerConfig`.N) rr/ray.rllib.core.learner.torch.torch_meta_learnerrSrfrrrYrr)r[r)rrSrs r\rz&DifferentiableAlgorithmConfig.validatesT  UTTTTT$88::r?r@rArBrCrDrEray.tune.loggerrFray.tune.registryrGray.tune.resultrH ray.tune.tunerIray.utilrJray.util.placement_grouprKrrMray.rllib.core.learnerrN-ray.rllib.core.learner.differentiable_learnerrOrQrQrrSrTrU getLoggerrzr(r]r`rrrrr^r\r9s                           """"""  877777999999,,,,,,******877777LLLLLLFFFFFF;;;;;;CCCCCCCC777777::::::LLLLLLUUUUUU++++++66666622222266666666;;;;;;@@@@@@@@@@FEEEEEEE@@@@@@@@KKKKKK333333999999 -,,,,,FFFFFF                          #"""""//////&&&&&&!!!!!!3333333888888......SSSSSSAAAAAAPPPPPP;;;;;;222222  8 $ $ '7 D    x^ x^ x^ x^ x^ gx^ x^ x^ v}u3u3u3u3u3Ou3u3u3p$$$$$T$$$$$r^