&`i1ddlZddlZddlmZmZmZmZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZdd lmZmZdd lmZmZdd lmZdd lmZejZ eGd deZ!dS)N)AnyDictListOptional)Dataset) FQETorchModel)OffPolicyEstimator)compute_q_and_v_values)OfflineEvaluator)Policy) SampleBatch convert_ma_batch_to_sample_batch) DeveloperAPIoverride)convert_to_numpy)SampleBatchTypec eZdZdZee ddedededee ffd Z eed e d e e e ffd Zeed e d e e eeffd ZdZeed ed e e e ffdZeedddeded e e e ffdZxZS) DirectMethodaThe Direct Method estimator. Let s_t, a_t, and r_t be the state, action, and reward at timestep t. This method trains a Q-model for the evaluation policy \pi_e on behavior data generated by \pi_b. Currently, RLlib implements this using Fitted-Q Evaluation (FQE). You can also implement your own model and pass it in as `q_model_config = {"type": your_model_class, **your_kwargs}`. This estimator computes the expected return for \pi_e for an episode as: V^{\pi_e}(s_0) = \sum_{a \in A} \pi_e(a | s_0) Q(s_0, a) and returns the mean and standard deviation over episodes. For more information refer to https://arxiv.org/pdf/1911.06854.pdfNpolicygammaepsilon_greedyq_model_configc\t|||t|dr)|jdddks Jd|pi}|dt }|d ||d||_t|jds Jdd S) a Initializes a Direct Method OPE Estimator. Args: policy: Policy to evaluate. gamma: Discount factor of the environment. epsilon_greedy: The probability by which we act acording to a fully random policy during deployment. With 1-epsilon_greedy we act according the target policy. q_model_config: Arguments to specify the Q-model. Must specify a `type` key pointing to the Q-model class. This Q-model is trained in the train() method and is used to compute the state-value estimates for the DirectMethod estimator. It must implement `train` and `estimate_v`. TODO (Rohan138): Unify this with RLModule API. config frameworktorchz,Framework must be torch to use DirectMethod.type)rr estimate_vz'self.model must implement `estimate_v`!N)super__init__hasattrrgetpoprmodel)selfrrrr model_cls __class__s ~/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/offline/estimators/direct_method.pyr"zDirectMethod.__init__&s0 777 68 $ $ > !!+w777BBB=CBB(-2"&&v}== Y       J    5 5 4 5 5  5 5episodereturnci}|d}d}t|jD]}||||j|zzz }||dd}||d<||d<|S)Nrewardsr v_behaviorv_target)rangecountr_compute_v_target)r'r,estimates_per_epsioder/r1tr2s r*estimate_on_single_episodez'DirectMethod.estimate_on_single_episodeRs ")$ w}%% 5 5A '!*tz1}4 4JJ))'"1"+66.8l+,4j)$$r+batchc\i}|d}|}||}||d<||d<|S)Nr/r1r2)r5)r'r9r6r/r1r2s r*estimate_on_single_step_samplesz,DirectMethod.estimate_on_single_step_samplesbsH!# " ))%00.8l+,4j)$$r+cX|j|}t|}|S)N)r&rr)r' init_stepr2s r*r5zDirectMethod._compute_v_targetqs):((33#H--r+ct|}|j|}dtj|iS)zTrains self.model on the given batch. Args: batch: A SampleBatchType to train on Returns: A dict with key "loss" and value as the mean training loss. loss)rr&trainnpmean)r'r9lossess r*r@zDirectMethod.trainvs9177!!%((((r+.) n_parallelismdatasetrDc t||zd}|t|d|jj|jdd}|d}|d}||z }|d|z tj |z }||||dS) aoCalculates the Direct Method estimate on the given dataset. Note: This estimate works for only discrete action spaces for now. Args: dataset: Dataset to compute the estimate on. Each record in dataset should include the following columns: `obs`, `actions`, `action_prob` and `rewards`. The `obs` on each row shoud be a vector of D dimensions. n_parallelism: The number of parallel workers to use. Returns: Dictionary with the following keys: v_target: The estimated value of the target policy. v_behavior: The estimated value of the behavior policy. v_gain: The estimated gain of the target policy over the behavior policy. v_std: The standard deviation of the estimated value of the target. r0pandasF) model_class model_statecompute_q_values) batch_size batch_format fn_kwargsr/v_values)r1r2 v_gain_mean v_gain_ste) maxr4 map_batchesr r&r) get_staterBstdmathsqrt) r'rErDrK updated_dsr1r2rOrPs r*estimate_on_datasetz DirectMethod.estimate_on_datasets.M91== (( "!!#z3#z3355$) )    __Y// ??:..+ NN: & & 3di 6P6P P  % &$    r+)rN)__name__ __module__ __qualname____doc__rr r floatrrr"r strrr8rr;r5rr@r rintrX __classcell__)r)s@r*rrs J JX !! !$)- )5)5)5)5 )5 ! )5)5)5)5)5"!)5VX !! %+ %$sCx. % % %"! %X !! %  % c4;  % % %"! % X !! )? )tCH~ ) ) )"! )X8;. . . . 25. c3h. . .  . . . . . r+r)"loggingrUtypingrrrrnumpyrAray.datar,ray.rllib.offline.estimators.fqe_torch_modelr1ray.rllib.offline.estimators.off_policy_estimatorr *ray.rllib.offline.offline_evaluation_utilsr #ray.rllib.offline.offline_evaluatorr ray.rllib.policyr ray.rllib.policy.sample_batchr rray.rllib.utils.annotationsrrray.rllib.utils.numpyrray.rllib.utils.typingr getLoggerloggerrr r+r*rpsb ,,,,,,,,,,,,FFFFFFPPPPPPMMMMMM@@@@@@######WWWWWWWW>>>>>>>>222222222222    ] ] ] ] ] %] ] ] ] ] r+