&`i ddlZddlmZmZddlmZmZmZddlm Z m Z ddl m Z ddl mZddlmZmZddlmZdd lmZdd lmZdd lmZdd lmZmZGd de ZdS)N)AnyDict)&LEARNER_RESULTS_CURR_ENTROPY_COEFF_KEYLEARNER_RESULTS_KL_KEY PPOConfig)AddOneTsToEpisodesAndTruncateGeneralAdvantageEstimation)LearnerValueFunctionAPI)5OverrideToImplementCustomLogic_CallToSuperRecommendedoverride)LambdaDefaultDict)NUM_ENV_STEPS_SAMPLED_LIFETIME)convert_to_numpy) Scheduler)ModuleID TensorTypec FeZdZeed fd Zeedeffd Zeeede e e fddffd Z e eedeefdZejded ed eddfd ZxZS) PPOLearnerreturnNcttfd_tfd_jqjjrgjtj tjj jj dSdSdS)Ncvtj|jjjS)N)fixed_value_or_schedule frameworkdevice)rconfigget_config_for_module entropy_coeffr_device module_idselfs x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/algorithms/ppo/ppo_learner.pyz"PPOLearner.build..%s6iK55i@@N.| chj|jSN)_get_tensor_variablerrkl_coeffr!s r$r%z"PPOLearner.build..2s-d77 11)<<Er&)gammalambda_)superbuildr#entropy_coeff_schedulers_per_modulecurr_kl_coeffs_per_module_learner_connectorr*add_default_connectors_to_learner_pipelineprependrappendr r+r,)r# __class__s`r$r.zPPOLearner.builds          0FW    F F &  # / F 0  # + +,I,K,K L L L  # * **++T[5H      0 / / /r&r"c tj|fi|}|j|d|j|d|Sr()r- remove_moduler/popr0)r#r"kwargs marl_specr5s r$r7zPPOLearner.remove_moduleMsY)EGG))>>v>>  044YEEE &**9d;;;r& timestepsc>t||jjD]\}}|j|}|j|| td}|j |tf|d|jrV|tf|j vrFt!|j |tf}||||dS)N)r;r)timestep)window)r"rkl_loss)r-after_gradient_based_updatemodule _rl_modulesitemsrrr/updategetrmetrics log_valuer use_kl_lossrrpeek_update_module_kl_coeff)r#r;r"rBrnew_entropy_coeffr@r5s r$rAz&PPOLearner.after_gradient_based_updateVs8 ++i+@@@!%!8!>!>!@!@   Iv[66yAAF!% H!fimm,JANNfOO  L " "BC! #    "  674<GG*L%%y2H&IJJ,,'!#-'  r&ctgSr(r )clss r$rl_module_required_apisz"PPOLearner.rl_module_required_apisxs !!!r&rr@cdS)aODynamically update the KL loss coefficients of each module. The update is completed using the mean KL divergence between the action distributions current policy and old policy of each module. That action distribution is computed during the most recent update/call to `compute_loss`. Args: module_id: The module whose KL loss coefficient to update. config: The AlgorithmConfig specific to the given `module_id`. kl_loss: The mean KL loss of the module, computed inside `compute_loss_for_module()`. N)r#r"rr@s r$rKz"PPOLearner._update_module_kl_coeffsr&)rN)__name__ __module__ __qualname__rr r.rr7r rstrrrA classmethodlisttyperOabcabstractmethodrfloatrK __classcell__)r5s@r$rrsx Xg------^Xgx; XgS>  ;:@ Xg"T """["                 r&r)rYtypingrrray.rllib.algorithms.ppo.pporrrray.rllib.connectors.learnerrr ray.rllib.core.learner.learnerr 0ray.rllib.core.rl_module.apis.value_function_apir ray.rllib.utils.annotationsr r"ray.rllib.utils.lambda_defaultdictrray.rllib.utils.metricsrray.rllib.utils.numpyr#ray.rllib.utils.schedules.schedulerrray.rllib.utils.typingrrrrQr&r$rhs}  322222MMMMMMA@@@@@32222299999977777777v v v v v v v v v v r&