&`i !ddlmZmZmZddlmZmZddlmZm Z ddl m Z ddl m Z ddlmZddlmZddlmZdd lmZmZGd d e ZGd d eZdS))OptionalTypeUnion)AlgorithmConfig NotProvided)MARWIL MARWILConfig)"AddObservationsFromEpisodesToBatch)+AddNextObservationsFromEpisodesToTrainBatch)Learner) RLModuleSpec)override)LearningRateOrScheduleRLModuleSpecTypeceZdZdZdfd Zeeeeeeeeeddee dee dee dee d ee d ee d ee d dffd Z eed eedeffdZeed efdZee dfd Zeedfd ZefdZxZS) IQLConfigaDefines a configuration class from which a new IQL Algorithm can be built .. testcode:: :skipif: True from ray.rllib.algorithms.iql import IQLConfig # Run this from the ray directory root. config = IQLConfig().training(actor_lr=0.00001, gamma=0.99) config = config.offline_data( input_="./rllib/offline/tests/data/pendulum/pendulum-v1_enormous") # Build an Algorithm object from the config and run 1 training iteration. algo = config.build() algo.train() .. testcode:: :skipif: True from ray.rllib.algorithms.iql import IQLConfig from ray import tune config = IQLConfig() # Print out some default values. print(config.beta) # Update the config object. config.training( lr=tune.grid_search([0.001, 0.0001]), beta=0.75 ) # Set the config object's data path. # Run this from the ray directory root. config.offline_data( input_="./rllib/offline/tests/data/pendulum/pendulum-v1_enormous" ) # Set the config object's env, used for evaluation. config.environment(env="Pendulum-v1") # Use to_dict() to get the old-style python config dict # when running with tune. tune.Tuner( "IQL", param_space=config.to_dict(), ).fit() Nct|ptd|_d|_d|_d|_d|_d|_d|_ d|_ d|_ dS)N) algo_classg?g?ga2U0*3?Tr?) super__init__IQLbeta expectileactor_lr critic_lrvalue_lrlrtwin_qtarget_network_update_freqtau)selfr __class__s p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/algorithms/iql/iql.pyrzIQLConfig.__init__<sq J$5#666     +,')rrrrrr r!rrrrrr r!returnc tjdi||tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_ |S)aSets the training related configuration. Args: beta: The temperature to scaling advantages in exponential terms. Must be >> 0.0. The higher this parameter the less greedy (exploitative) the policy becomes. It also means that the policy is fitting less to the best actions in the dataset. twin_q: If a twin-Q architecture should be used (advisable). expectile: The expectile to use in expectile regression for the value function. For high expectiles the value function tries to match the upper tail of the Q-value distribution. actor_lr: The learning rate for the actor network. Actor learning rates greater than critic learning rates work well in experiments. critic_lr: The learning rate for the Q-network. Critic learning rates greater than value function learning rates work well in experiments. value_lr: The learning rate for the value function network. target_network_update_freq: The number of timesteps in between the target Q-network is fixed. Note, too high values here could harm convergence. The target network is updated via Polyak-averaging. tau: The update parameter for Polyak-averaging of the target Q-network. The higher this value the faster the weights move towards the actual Q-network. Return: This updated `AlgorithmConfig` object. ) rtrainingrrrrrrr r!) r"rrrrrr r!kwargsr#s r$r)zIQLConfig.trainingYsN ""6"""  $ $ DK K ' '&DN ; & &$DM K ' '&DN ; & &$DM %[ 8 8.HD + k ! !DH r%r cX|jdkrddlm}|Std|jd)Ntorchr)IQLTorchLearnerThe framework z) is not supported. Use `'torch'` instead.) framework_str0ray.rllib.algorithms.iql.torch.iql_torch_learnerr- ValueError)r"r-s r$get_default_learner_classz#IQLConfig.get_default_learner_classsS   ( ( X X X X X X" ")!3))) r%ct|jdkrddlm}t|St d|jd)Nr,r)DefaultIQLTorchRLModule) module_classr.z' is not supported. Use `torch` instead.)r/:ray.rllib.algorithms.iql.torch.default_iql_torch_rl_moduler4r r1)r"r4s r$get_default_rl_module_specz$IQLConfig.get_default_rl_module_specsm   ( (       -DEEE E'!3''' r%ct|||}|d|d|tt |S)N)input_observation_spaceinput_action_spacedeviceAddOneTsToEpisodesAndTruncateGeneralAdvantageEstimation)rbuild_learner_connectorremove insert_afterr r )r"r9r:r;pipeliner#s r$r>z!IQLConfig.build_learner_connectors7722$;13   78884555  . 7 9 9   r%ct|jdkr|dd|jcxkrdksn|ddSdS)NgzFFor meaningful results, `beta` (temperature) parameter must be >> 0.0!rz@For meaningful results, `expectile` parameter must be in (0, 1).)rvalidater _value_errorrr"r#s r$rCzIQLConfig.validates  9     X   T^))))c))))   R     *)r%c>tjd|jizS)Nr)r_model_config_auto_includesrrEs r$rGz%IQLConfig._model_config_auto_includessww2h 5LLLr%N)r&N)__name__ __module__ __qualname____doc__rrr rrboolfloatrintr)rrstrr2rr7r>rCpropertyrG __classcell__)r#s@r$rrs!((T:Xl"-%05@6A5@4?*7777E? 7 12 7 23 7127%-SM7e_7 777777rXl 5i#1E+F    Xl ,<    Xl  2Xl      MMMMXMMMMMr%rcPeZdZdZeeedefdZdS)rzOImplicit Q-learning (derived from MARWIL). Uses MARWIL training step. r&ctSrH)r)clss r$get_default_configzIQL.get_default_configs{{r%N) rIrJrKrL classmethodrrrrVr(r%r$rrsZ  Xf?[r%rN)typingrrr%ray.rllib.algorithms.algorithm_configrr"ray.rllib.algorithms.marwil.marwilrr Cray.rllib.connectors.common.add_observations_from_episodes_to_batchr Oray.rllib.connectors.learner.add_next_observations_from_episodes_to_train_batchr ray.rllib.core.learner.learnerr "ray.rllib.core.rl_module.rl_moduler ray.rllib.utils.annotationsrray.rllib.utils.typingrrrrr(r%r$rasN((((((((((NNNNNNNNCCCCCCCC322222;;;;;;000000KKKKKKKKGMGMGMGMGM GMGMGMT     &     r%