&`idZddlZddlmZddlmZmZmZmZm Z m Z m Z m Z ddl ZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZdd l m!Z!ddl"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@ddlAmBZBddlCmDZDmEZEmFZFmGZGddlHmIZImJZJmKZKmLZLejMeNZOGddeZPdedeeQfdZRGddeZSdS)ap Deep Q-Networks (DQN, Rainbow, Parametric DQN) ============================================== This file defines the distributed Algorithm class for the Deep Q-Networks algorithm. See `dqn_[tf|torch]_policy.py` for the definition of the policies. Detailed documentation: https://docs.ray.io/en/master/rllib-algorithms.html#deep-q-networks-dqn-rainbow-parametric-dqn N) defaultdict)AnyCallableDictListOptionalTupleTypeUnion)Self)DEPRECATED_VALUE) Algorithm)AlgorithmConfig NotProvided) DQNTFPolicy)DQNTorchPolicy)Learner) RLModuleSpec)synchronous_parallel_sample)multi_gpu_train_one_steptrain_one_step)Policy)MultiAgentBatch) deep_update)override) ALL_MODULESENV_RUNNER_RESULTSENV_RUNNER_SAMPLING_TIMERLAST_TARGET_UPDATE_TSLEARNER_RESULTSLEARNER_UPDATE_TIMERNUM_AGENT_STEPS_SAMPLED NUM_AGENT_STEPS_SAMPLED_LIFETIMENUM_ENV_STEPS_SAMPLEDNUM_ENV_STEPS_SAMPLED_LIFETIMENUM_TARGET_UPDATESREPLAY_BUFFER_ADD_DATA_TIMERREPLAY_BUFFER_RESULTSREPLAY_BUFFER_SAMPLE_TIMER REPLAY_BUFFER_UPDATE_PRIOS_TIMER SAMPLE_TIMERSYNCH_WORKER_WEIGHTS_TIMER TD_ERROR_KEYTIMERS)convert_to_numpy)sample_min_n_steps_from_buffer*update_priorities_in_episode_replay_buffer"update_priorities_in_replay_buffervalidate_buffer_config)LearningRateOrSchedule ResultDictRLModuleSpecTypeSampleBatchTypec2eZdZdZd&fd Zeeeeeeeeeeeeeeeeeeeeeeeeeddee dee dee dee e e e efd eed eed ee d ee d eedee deedeedee deedee dee dee dee e ee e ffdeeee eeee geefdeedeedeedee def0fdZeed'fd Zeed(d e de fd!Zeedefd"Zeeedeeefffd# Zeede ed$effd%Z xZ!S)) DQNConfiga/Defines a configuration class from which a DQN Algorithm can be built. .. testcode:: from ray.rllib.algorithms.dqn.dqn import DQNConfig config = ( DQNConfig() .environment("CartPole-v1") .training(replay_buffer_config={ "type": "PrioritizedEpisodeReplayBuffer", "capacity": 60000, "alpha": 0.5, "beta": 0.5, }) .env_runners(num_env_runners=1) ) algo = config.build() algo.train() algo.stop() .. testcode:: from ray.rllib.algorithms.dqn.dqn import DQNConfig from ray import tune config = ( DQNConfig() .environment("CartPole-v1") .training( num_atoms=tune.grid_search([1,]) ) ) tune.Tuner( "DQN", run_config=tune.RunConfig(stop={"training_iteration":1}), param_space=config, ).fit() .. testoutput:: :hide: ... Nc ddddd|_t|ptd|_dd g|_d |_d |_d |_d |_ | tj dd|_ d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_dg|_d|_d|_d|_d|_d|_d|_ d|_!dddd d!|_"d|_#tH|_%tH|_&tH|_'tH|_(d|_)tH|_*tH|_+tH|_,dS)"z!Initializes a DQNConfig instance. EpsilonGreedy?g{Gz?')typeinitial_epsilon final_epsilonepsilon_timesteps) algo_classauto)rr<)r=g?gD@ global_normgMb@? F)explore)evaluation_configNiig:0yE>g$g$@g?ThuberrPrioritizedEpisodeReplayBufferiPg333333?g?)r>capacityalphabeta)-exploration_configsuper__init__DQNrollout_fragment_lengthepsilon grad_clip grad_clip_bylrtrain_batch_size evaluationr overridesmin_time_s_per_iteration"min_sample_timesteps_per_iterationtarget_network_update_freq(num_steps_sampled_before_learning_startsstore_buffer_in_checkpoints adam_epsilontau num_atomsv_minv_maxnoisysigma0duelinghiddensdouble_qn_stepbefore_learn_on_batchtraining_intensitytd_error_loss_fn$categorical_distribution_temperature burn_in_lenreplay_buffer_config lr_scheduler buffer_sizeprioritized_replaylearning_startsreplay_batch_sizereplay_sequence_lengthprioritized_replay_alphaprioritized_replay_betaprioritized_replay_eps)selfrB __class__s p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/algorithms/dqn/dqn.pyrQzDQNConfig.__init__{s$"!!& # #  J$5#666 9?$ !-0 * " /*CE*R*R*RSSS)-%26/ +.'8<5+0(      u   %)""& '4715% % ! ,"2/!1&*#(8%'7$&6###)r]rpr_rqrTr`rUr^rarbrcrdrerfrgrhrirjrkrlrmrnror]rpr_rqrTr`rUr^rarbrcrdrerfrgrhrirjrkrlrmrnroreturnc btjdi||tur||_|tur+t d|jid|iddgdg}|d|_|tur||_|tur||_|tur||_|tur||_ |tur||_ |tur||_ | tur| |_ | tur| |_ | tur| |_| tur| |_| tur| |_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|tur||_|S)uWSets the training related configuration. Args: target_network_update_freq: Update the target network every `target_network_update_freq` sample steps. replay_buffer_config: Replay buffer config. Examples: { "_enable_replay_buffer_api": True, "type": "MultiAgentReplayBuffer", "capacity": 50000, "replay_sequence_length": 1, } - OR - { "_enable_replay_buffer_api": True, "type": "MultiAgentPrioritizedReplayBuffer", "capacity": 50000, "prioritized_replay_alpha": 0.6, "prioritized_replay_beta": 0.4, "prioritized_replay_eps": 1e-6, "replay_sequence_length": 1, } - Where - prioritized_replay_alpha: Alpha parameter controls the degree of prioritization in the buffer. In other words, when a buffer sample has a higher temporal-difference error, with how much more probability should it drawn to use to update the parametrized Q-network. 0.0 corresponds to uniform probability. Setting much above 1.0 may quickly result as the sampling distribution could become heavily “pointy” with low entropy. prioritized_replay_beta: Beta parameter controls the degree of importance sampling which suppresses the influence of gradient updates from samples that have higher probability of being sampled via alpha parameter and the temporal-difference error. prioritized_replay_eps: Epsilon parameter sets the baseline probability for sampling so that when the temporal-difference error of a sample is zero, there is still a chance of drawing the sample. store_buffer_in_checkpoints: Set this to True, if you want the contents of your buffer(s) to be stored in any saved checkpoints as well. Warnings will be created if: - This is True AND restoring from a checkpoint that contains no buffer data. - This is False AND restoring from a checkpoint that does contain buffer data. epsilon: Epsilon exploration schedule. In the format of [[timestep, value], [timestep, value], ...]. A schedule must start from timestep 0. adam_epsilon: Adam optimizer's epsilon hyper parameter. grad_clip: If not None, clip gradients during optimization at this value. num_steps_sampled_before_learning_starts: Number of timesteps to collect from rollout workers before we start sampling from replay buffers for learning. Whether we count this in agent steps or environment steps depends on config.multi_agent(count_steps_by=..). tau: Update the target by au * policy + (1- au) * target_policy. num_atoms: Number of atoms for representing the distribution of return. When this is greater than 1, distributional Q-learning is used. v_min: Minimum value estimation v_max: Maximum value estimation noisy: Whether to use noisy network to aid exploration. This adds parametric noise to the model weights. sigma0: Control the initial parameter noise for noisy nets. dueling: Whether to use dueling DQN. hiddens: Dense-layer setup for each the advantage branch and the value branch double_q: Whether to use double DQN. n_step: N-step target updates. If >1, sars' tuples in trajectories will be postprocessed to become sa[discounted sum of R][s t+n] tuples. An integer will be interpreted as a fixed n-step value. If a tuple of 2 ints is provided here, the n-step value will be drawn for each sample(!) in the train batch from a uniform distribution over the closed interval defined by `[n_step[0], n_step[1]]`. before_learn_on_batch: Callback to run before learning on a multi-agent batch of experiences. training_intensity: The intensity with which to update the model (vs collecting samples from the env). If None, uses "natural" values of: `train_batch_size` / (`rollout_fragment_length` x `num_env_runners` x `num_envs_per_env_runner`). If not None, will make sure that the ratio between timesteps inserted into and sampled from the buffer matches the given values. Example: training_intensity=1000.0 train_batch_size=250 rollout_fragment_length=1 num_env_runners=1 (or 0) num_envs_per_env_runner=1 -> natural value = 250 / 1 = 250.0 -> will make sure that replay+train op will be executed 4x asoften as rollout+insert op (4 * 250 = 1000). See: rllib/algorithms/dqn/dqn.py::calculate_rr_weights for further details. td_error_loss_fn: "huber" or "mse". loss function for calculating TD error when num_atoms is 1. Note that if num_atoms is > 1, this parameter is simply ignored, and softmax cross entropy loss will be used. categorical_distribution_temperature: Set the temperature parameter used by Categorical action distribution. A valid temperature is in the range of [0, 1]. Note that this mostly affects evaluation since TD error uses argmax for return calculation. burn_in_len: The burn-in period for a stateful RLModule. It allows the Learner to utilize the initial `burn_in_len` steps in a replay sequence solely for unrolling the network and establishing a typical starting state. The network is then updated on the remaining steps of the sequence. This process helps mitigate issues stemming from a poor initial state - zero or an outdated recorded state. Consider setting this parameter to a positive integer if your stateful RLModule faces convergence challenges or exhibits signs of catastrophic forgetting. Returns: This updated AlgorithmConfig object. rpF)rPtrainingrr]rrpr_rqrTr`rUr^rarbrcrdrerfrgrhrirjrkrlrmrnro)rzr]rpr_rqrTr`rUr^rarbrcrdrerfrgrhrirjrkrlrmrnrokwargsnew_replay_buffer_configr{s r|rzDQNConfig.trainings` ""6""" %[ 8 8.HD + { 2 2(3')BC')=>'('( (( $)AAW(XD % &k 9 9/JD , k ) )*D  + % %"DL { * * ,D  K ' '&DN 3; F F8  9 k ! !DH K ' '&DN  # #DJ  # #DJ  # #DJ  $ $ DK + % %"DL + % %"DL ; & &$DM  $ $ DK 3 3)>D & [ 0 0&8D # ; . .$4D ! /{ B B4  5 k ) )*D  r}c bt|jr|j|dnc|jst ||jddkr<|jdkr|d|j r|d|j dvr|d|jsI|j d kr>|j |j kr.|d |j d |j d |j d d|j vrFd|j dcxkr |jkr)nn&td|jd|j ddddlm}|jrRt'|jdt*s2t-|jd|s|ddS|jsat'|jdt*rd|jdvst-|jd|r|ddSdSdS)NzW`lr_schedule` is deprecated and must be None! Use the `lr` setting to setup a schedule.r>ParameterNoisecomplete_episodeszParameterNoise Exploration requires `batch_mode` to be 'complete_episodes'. Try setting `config.env_runners(batch_mode='complete_episodes')`.zOParameterNoise Exploration and `noisy` network cannot be used at the same time!)rJmsez,`td_error_loss_fn` must be 'huber' or 'mse'!rCz Your `rollout_fragment_length` (z) is smaller than `n_step` (z:)! Try setting config.env_runners(rollout_fragment_length=z). max_seq_lenrzYour defined `burn_in_len`=z" is larger or equal `max_seq_len`=zC! Either decrease the `burn_in_len` or increase your `max_seq_len`.)EpisodeReplayBufferz[When using the new `EnvRunner API` the replay buffer must be of type `EpisodeReplayBuffer`.EpisodeadWhen using the old API stack the replay buffer must not be of type `EpisodeReplayBuffer`! We suggest you use the following config to run DQN on the old API stack: `config.training(replay_buffer_config={'type': 'MultiAgentPrioritizedReplayBuffer', 'prioritized_replay_alpha': [alpha], 'prioritized_replay_beta': [beta], 'prioritized_replay_eps': [eps], })`.)rPvalidateenable_rl_module_and_learnerrq _value_error in_evaluationr3rO batch_moderermrSrj model_configro ValueError4ray.rllib.utils.replay_buffers.episode_replay_bufferr"enable_env_runner_and_connector_v2 isinstancerpstr issubclass)rzrr{s r|rzDQNConfig.validates   , +!!8 % -&t,,,&v.2BBB?&999%%< :%%2  (8 8 8   L M M M" ,66,t{::   #43O##*.+##;###    T. . .D%m4HHHH8HHHHHHDd.>DD!%!2=!ADDD        3 t8@#FF t8@BUVV    )     8 44V>3rx 78899::r}ceZdZeeedefdZeeedede e e fdZ eededdffd Z eed dZdZdefd ZxZS) rRr~ctSr)r9)clss r|get_default_configzDQN.get_default_configTs{{r}rc6|ddkrtStS)N frameworkr)rr)rrs r|get_default_policy_classzDQN.get_default_policy_classYs + ' ) )! ! r}Nc$t||jjr_|jrZ|j,|jddgdd|_dS|jj |_dSdSdS)Nc4|jSr)module is_stateful)ers r|zDQN.setup..jsry4466r}rHF)remote_worker_idslocal_env_runnerr) rPsetuprrenv_runner_group env_runnerforeach_env_runner_module_is_statefulrr)rzrr{s r|rz DQN.setupcs  f ; 9 Pd>S P&+/+@+S+S66'(c%*,T,, ,((( ,0?+A+M+M+O+O((( P P P Pr}cj|jjs|S|S)a-DQN training iteration function. Each training iteration, we: - Sample (MultiAgentBatch) from workers. - Store new samples in replay buffer. - Sample training batch (MultiAgentBatch) from replay buffer. - Learn on training batch. - Update remote workers' new policy weights. - Update target network every `target_network_update_freq` sample steps. - Return all collected metrics for the iteration. Returns: The results dict from executing the training iteration. )rr_training_step_old_api_stack_training_step_new_api_stack)rzs r| training_stepzDQN.training_stepqs7"{= 74466 600222r}c  t|j\}}t|D]}|jt t f5t|jd|jj dd\}}dddn #1swxYwY|j |t|jt tf5|j |dddn #1swxYwY|jjdkrHt!|jtt$fi}n(|jtt(fd}||jjkrt|D]}|jt t,f5|j |jj|jj|j|jjddzt;|jt=|jdr |jjnd|jj |jj!d d }|j "}|j |gtFdddn #1swxYwY|jt tHf5|j%&|t(|jtt(ft$|jtt$fi }tOtP} |D]w} | )D]`\} } tT| vrR| | +tY| -tTaxd | )D} |j |t\dddn #1swxYwY|jt t^f5ta|j | dddn #1swxYwY|jt tbf5te|d3thhz } |j5|j%| ddddddS#1swxYwYdSdS)NT) worker_setconcatsample_timeout_s_uses_new_env_runners_return_metrics)key agent_steps)defaultrrrorN) num_itemsrjbatch_length_Tlookbackmin_batch_length_TgammarNsample_episodes)episodes timestepscPi|]#\}}|ttj|di$S)r)axis)r-r concatenate).0 module_idss r| z4DQN._training_step_new_api_stack..sB!!!(Iq"L".2K2K2K#L!!!r}) replay_buffer td_errors)from_worker_or_learner_grouppolicies global_varsinference_only)6rrrangemetricslog_timer.rrrr aggregaterr'local_replay_bufferaddcount_steps_bysumpeekr#valuesr%r^r)samplerrjrrgetrhasattrrorrp get_metricsr(r! learner_groupupdaterritemsr-extendr/popr r*r1r,setkeysr sync_weights)rz store_weightr_renv_runner_results current_tsreplay_buffer_resultslearner_resultsrresrmodule_resultsmodules_to_updates r|rz DQN._training_step_new_api_stacks0DT[0Q0Q- -|$$ 7 7A&&0I'JKK  /J#4%)[%A*.$( 000,,                L " "#5;M " N N N&&0L'MNN 7 7(,,X666 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ; % 6 6 !!')IJTV"&((JJ **#%CDa+J M M M233H H \**F4N+OPP#7>>"&+"D#{1!4"k6::=!LLM"%T%=!>!>#4; >>,4;+B+B"k/![=AA&II(,# ?  H*-1,D,P,P,R,R)L**./5J+1:\**F4H+IJJ"Q"Q&*&8&?&?!): $ 1 1%79W$X!"!"= $ 1 1(:(H%&!"!"#'@''O(!,D 1 1I.""9<""5I~+~== )) 4 ; ;$4(6(:(:<(H(H(M(M(O(O%&%&!"!"!""!!,5OO,=,=!!!IL**?*PPPE"Q"Q"Q"Q"Q"Q"Q"Q"Q"Q"Q"Q"Q"Q"QJ\**F4T+UVV>&*&>"+&&0J'KLL  $'(:(?(?(A(A$B$Bk]$R!%22151C. $#' 3                  ] N Mso'BB B C;;C? C? C2KK K ?EQ  Q Q <RR# &R# AT33T7:T7ci}t|j\}}t|D]}|jt5t |jd|jj}dddn #1swxYwY|sicS|jtxx| z cc<|jtxx| z cc<|j |d|jti}|j|jjdkrtnt}||jjkrt|D]}t#|j |jj|jjdk}|jdpd} | ||j|j}|jd durt)||}nt+||}t-|j |j|||jt.} || z |jjkr_|j} |j| fd |jt8xxd z cc<||jt.<|jt:5|j| dddn #1swxYwY|S) zvTraining step for the old API stack. More specifically this training step relies on `RolloutWorker`. T)rrrNtimestepr)count_by_agent_stepsrkc|Srr)bas r|rz2DQN._training_step_old_api_stack..7sUVr}simple_optimizerc2||vo|Sr) update_target)ppid to_updates r|rz2DQN._training_step_old_api_stack..Ns9,B1B1Br}rH)r)rrr_timersr+rrr _countersr"rr$ env_stepsrrrr^r0rrrrr2rr]rget_policies_to_trainforeach_policy_to_trainr&r,r) rz train_resultsrrrnew_sample_batchrcur_ts train_batchpost_fn last_updater$s r|rz DQN._training_step_old_api_stacks  1ET[0Q0Q- -|$$ ; ;Al+  4O#4%)[%A555                $   N2 3 3 37G7S7S7U7U U 3 3 3 N0 1 1 15E5O5O5Q5Q Q 1 1 1  $ ( ()9 : : : : '<= ;->>('*   DKH H H233* P* P<,K6)-)C})T +//*ABBW%gk43H$+VV ;??#566$>>$24$E$EMM$rIs  ######JJJJJJJJJJJJJJJJJJJJ""""""444444444444NNNNNNNN>>>>>>DDDDDD******;;;;;;+*****999999''''''000000*322222   8 $ $jjjjjjjjZ;;T%[;;;;6HHHHH)HHHHHr}