&`i:QFddlmZmZmZmZddlmZddlmZddl m Z ddl m Z m Z ddlmZmZmZmZddlmZddlmZdd lmZdd lmZdd lmZmZdd lmZdd l m!Z!m"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ddl,m-Z-m.Z.m/Z/ddl0m1Z1Gdde Z2Gdde Z3dS))CallableOptionalTypeUnion)Self)deprecation_warning) Algorithm)AlgorithmConfig NotProvided)+AddNextObservationsFromEpisodesToTrainBatch"AddObservationsFromEpisodesToBatchAddOneTsToEpisodesAndTruncateGeneralAdvantageEstimation)Learner) TrainingData) RLModuleSpec)synchronous_parallel_sample)multi_gpu_train_one_steptrain_one_step)Policy) OldAPIStackoverride)LEARNER_RESULTSLEARNER_UPDATE_TIMERNUM_AGENT_STEPS_SAMPLEDNUM_ENV_STEPS_SAMPLEDOFFLINE_SAMPLING_TIMER SAMPLE_TIMERSYNCH_WORKER_WEIGHTS_TIMERTIMERS)EnvType ResultDictRLModuleSpecType)LoggerceZdZdZdfd Zeeeeeeeeddee dee dee dee d ee d ee d e ffd Z eed e fd Z eed eedeffdZeed e ffd Zeed e ffd Zee ddeeeefdeegefd dffd Zee dfd Zeedfd ZefdZxZS) MARWILConfigaDefines a configuration class from which a MARWIL Algorithm can be built. .. testcode:: import gymnasium as gym import numpy as np from pathlib import Path from ray.rllib.algorithms.marwil import MARWILConfig # Get the base path (to ray/rllib) base_path = Path(__file__).parents[2] # Get the path to the data in rllib folder. data_path = base_path / "offline/tests/data/cartpole/cartpole-v1_large" config = MARWILConfig() # Enable the new API stack. config.api_stack( enable_rl_module_and_learner=True, enable_env_runner_and_connector_v2=True, ) # Define the environment for which to learn a policy # from offline data. config.environment( observation_space=gym.spaces.Box( np.array([-4.8, -np.inf, -0.41887903, -np.inf]), np.array([4.8, np.inf, 0.41887903, np.inf]), shape=(4,), dtype=np.float32, ), action_space=gym.spaces.Discrete(2), ) # Set the training parameters. config.training( beta=1.0, lr=1e-5, gamma=0.99, # We must define a train batch size for each # learner (here 1 local learner). train_batch_size_per_learner=2000, ) # Define the data source for offline data. config.offline_data( input_=[data_path.as_posix()], # Run exactly one update per training iteration. dataset_num_iters_per_learner=1, ) # Build an `Algorithm` object from the config and run 1 training # iteration. algo = config.build() algo.train() .. testcode:: import gymnasium as gym import numpy as np from pathlib import Path from ray.rllib.algorithms.marwil import MARWILConfig from ray import tune # Get the base path (to ray/rllib) base_path = Path(__file__).parents[2] # Get the path to the data in rllib folder. data_path = base_path / "offline/tests/data/cartpole/cartpole-v1_large" config = MARWILConfig() # Enable the new API stack. config.api_stack( enable_rl_module_and_learner=True, enable_env_runner_and_connector_v2=True, ) # Print out some default values print(f"beta: {config.beta}") # Update the config object. config.training( lr=tune.grid_search([1e-3, 1e-4]), beta=0.75, # We must define a train batch size for each # learner (here 1 local learner). train_batch_size_per_learner=2000, ) # Set the config's data path. config.offline_data( input_=[data_path.as_posix()], # Set the number of updates to be run per learner # per training step. dataset_num_iters_per_learner=1, ) # Set the config's environment for evalaution. config.environment( observation_space=gym.spaces.Box( np.array([-4.8, -np.inf, -0.41887903, -np.inf]), np.array([4.8, np.inf, 0.41887903, np.inf]), shape=(4,), dtype=np.float32, ), action_space=gym.spaces.Discrete(2), ) # Set up a tuner to run the experiment. tuner = tune.Tuner( "MARWIL", param_space=config, run_config=tune.RunConfig( stop={"training_iteration": 1}, ), ) # Run the experiment. tuner.fit() NcRddi|_t|ptd|_d|_d|_d|_d|_d|_ d|j d <d |_ d |_ d |_ d |_d|_d|_d |_d|_d|_d S)z$Initializes a MARWILConfig instance.typeStochasticSampling) algo_classF?g:0yE>gY@vf_share_layersNsamplerTg-C6?i)exploration_configsuper__init__MARWIL _is_onlinebetabc_logstd_coeff'moving_average_sqd_adv_norm_update_rate!moving_average_sqd_adv_norm_startvf_coeffmodel grad_clipinput_postprocess_inputslrlambda_train_batch_sizematerialize_datamaterialize_mapped_data"_set_off_policy_estimation_methods)selfr* __class__s v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/algorithms/marwil/marwil.pyr1zMARWILConfig.__init__s ( #  J$8&999 "7;416. (- $%  "& $!%',$38///)r4r5r6r7r8r:r4r5r6r7r8r:returnc tjdi||tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|S)aSets the training related configuration. Args: beta: Scaling of advantages in exponential terms. When beta is 0.0, MARWIL is reduced to behavior cloning (imitation learning); see bc.py algorithm in this same directory. bc_logstd_coeff: A coefficient to encourage higher action distribution entropy for exploration. moving_average_sqd_adv_norm_update_rate: The rate for updating the squared moving average advantage norm (c^2). A higher rate leads to faster updates of this moving avergage. moving_average_sqd_adv_norm_start: Starting value for the squared moving average advantage norm (c^2). vf_coeff: Balancing value estimation loss and policy optimization loss. grad_clip: If specified, clip the global norm of gradients by this amount. Returns: This updated AlgorithmConfig object. ) r0trainingr r4r5r6r7r8r:) rCr4r5r6r7r8r:kwargsrDs rErJzMARWILConfig.trainings@ ""6""" { " "DI + - -#2D 2+ E E7  8 -K ? ?5VD 2 ; & &$DM K ' '&DN rFct|jdkrddlm}t|St d|jd)Ntorchr)DefaultPPOTorchRLModule) module_classThe framework ' is not supported. Use 'torch' instead.) framework_str:ray.rllib.algorithms.ppo.torch.default_ppo_torch_rl_modulerNr ValueError)rCrNs rEget_default_rl_module_specz'MARWILConfig.get_default_rl_module_specsm   ( (       -DEEE E'!3''' rFrcX|jdkrddlm}|Std|jd)NrMr)MARWILTorchLearnerrPrQ)rR6ray.rllib.algorithms.marwil.torch.marwil_torch_learnerrWrT)rCrWs rEget_default_learner_classz&MARWILConfig.get_default_learner_classsa   ( (      & %'!3''' rFc Ptjdi|d|vrd|_|S)zuSets the evaluation related configuration. Returns: This updated AlgorithmConfig object. off_policy_estimation_methodsTrI)r0 evaluationrB)rCrKrDs rEr\zMARWILConfig.evaluation"s: $$V$$$ *f 4 46:D 3 rFc tjdi|d|vrOddlm}t |d|s&t d|dd|S)Nprelearner_classr)OfflinePreLearnerz`prelearner_class` z is not a subclass of `OfflinePreLearner`. Any class passed to `prelearner_class` needs to implement the interface given by `OfflinePreLearner`.rI)r0 offline_dataray.rllib.offline.offline_datar_ issubclassgetrT)rCrKr_rDs rEr`zMARWILConfig.offline_data4s &&v&&&  ' ' H H H H H Hfjj);<<>OPP  +&**5G*H*H+++ rFenvlogger_creatorr cx|jstddt||S)NzMARWIL used to have off_policy_estimation_methods is and wis by default. This haschanged to off_policy_estimation_methods: \{\}.If you want to use an off-policy estimator, specify it in.evaluation(off_policy_estimation_methods=...)F)olderror)rBrr0build)rCrdrerDs rErizMARWILConfig.buildHsL 6  A      ww}}S.111rFc<t|||}|t|t t |t|j |j |S)N)input_observation_spaceinput_action_spacedevice)gammar>) r0build_learner_connectorprependr insert_afterr r appendrrnr>)rCrkrlrmpipelinerDs rEroz$MARWILConfig.build_learner_connectorYs7722$;13   688999  . 7 9 9     &TZ N N N   rFcXt|jdks |jdkr|d|jdur |jdkr|d|jdkr%|js |jr|ddSdSdSdS)Nr,r+z"`beta` must be within 0.0 and 1.0!Fz`postprocess_inputs` must be True for MARWIL (to calculate accum., discounted returns)! Try setting `config.offline_data(postprocess_inputs=True)`.rzWhen using a local Learner (`config.num_learners=0`), the number of iterations per learner (`dataset_num_iters_per_learner`) has to be defined! Set this hyperparameter through `config.offline_data(dataset_num_iters_per_learner=...)`.)r0validater4 _value_errorr< num_learnersdataset_num_iters_per_learnerenable_rl_module_and_learnerrCrDs rEruzMARWILConfig.validate}s  9s??di#oo   B C C C  "e + + C   B      " "6 #1 #   7      # " " " " "rFc@tj|jddzS)NF)r4r-)r0_model_auto_keysr4rzs rEr|zMARWILConfig._model_auto_keyss ww'49QV*W*WWWrFN)NNrGN)__name__ __module__ __qualname____doc__r1rr r rfloatrrJr#rUrrstrrYr\r`r!rr$rirorupropertyr| __classcell__)rDs@rEr&r&,snn`686868686868pXo!,+6CN=H$/%0...uo."% . 2:% . ,4E? .5/.E?. ......`Xo ,<    Xo 5i#1E+F    Xo "Xo&Xo.29=22 eCL) *2!"f*!562  222222 Xo  !!!!!!FXo:XXXXXXXXXXrFr&ceZdZeeedefdZeeedede e e fdZ eeddZ edefdZdS) r2rGctSr})r&)clss rEget_default_configzMARWIL.get_default_configs~~rFconfigcb|ddkrddlm}|S|ddkrddlm}|Sddlm}|S)N frameworkrMr)MARWILTorchPolicytf)MARWILTF1Policy)MARWILTF2Policy)/ray.rllib.algorithms.marwil.marwil_torch_policyr,ray.rllib.algorithms.marwil.marwil_tf_policyrr)rrrrrs rEget_default_policy_classzMARWIL.get_default_policy_classs + ' ) )      % $ K D ( (      # " T T T T T T" "rFNc|jjs|S|jt t f5|jjdkp|jjdk}|j |jj |jj|}|rt|}nt|}dddn #1swxYwY|jt tf5|jjd ||jj |jjd|j j}|j|t$ddddS#1swxYwYdS) a-Implements training logic for the new stack Note, this includes so far training with the `OfflineData` class (multi-/single-learner setup) and evaluation on `EnvRunner`s. Note further, evaluation on the dataset itself using estimators is not implemented, yet. r) num_samples num_shardsreturn_iterator)data_iterators)batchN) training_dataminibatch_size num_iters)keyrI)r"enable_env_runner_and_connector_v2_training_step_old_api_stackmetricslog_timer rrwrxr`sampletrain_batch_size_per_learnerrr learner_groupupdateiter_batches_kwargs aggregater)rCrbatch_or_iteratorrlearner_resultss rE training_stepzMARWIL.training_steps){= 74466 6 \ " "F,B#C D D F F (1,B;<A  !% 1 8 8 KD;3!0 !9!!  F ,?  7 7 9 9A = =89  %22!-"4"4"6"677[3                '' 444s599;rs222222222222""""""777777444444NNNNNNNN 322222======;;;;;;+*****========                     #"""""qXqXqXqXqX?qXqXqXh xxxxxYxxxxxrF