&`iN3zddlmZmZmZmZmZmZmZmZddl Z ddl m Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZmZdd lm Z m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'm(Z(m)Z)e \Z*Z+e!\Z,Z+edddddddddddddddddddddde-de-deeeeeeegee)ee)ffdeege'fdeeeegee-e)ffdeeeeeeeefeegefdeeeee-e)fee)eegee-e)ffdeeede)gee-e)ffdeeegee-e)ffdeeee'gdfdeeee j.e j.e'gdfdeeee j.e j.e'gdfdeeee j/j.e j/j.e'gdfdeeee j.e j.e'gdfd eeee j/j.e j/j.e'gdfd!eee)ee)gee)e)ffd"eeeee)e)e)gee)e0ee)ffd#eeee j/j.e j/j.e'gefd$eeee j/j.e j/j.e'geeeeffd%eeeegee(e1ffd&eeedgdfd'eee0d(eeege2fd)eef0d*Z3dS)+)AnyCallableDictListOptionalTupleTypeUnionN) ModelCatalog)ModelV2)TorchDistributionWrapper) TorchModelV2)Policy) SampleBatch) TorchPolicy)NullContextManager add_mixins) OldAPIStackoverride)try_import_jaxtry_import_torch)LEARNER_STATS_KEY)convert_to_numpy)AlgorithmConfigDictModelGradients TensorType)get_default_configstats_fnpostprocess_fnextra_action_out_fnextra_grad_process_fnextra_learn_fetches_fn optimizer_fnvalidate_spaces before_initbefore_loss_init after_init_after_loss_initaction_sampler_fnaction_distribution_fn make_modelmake_model_and_action_distcompute_gradients_fnapply_gradients_fnmixinsget_batch_divisibility_reqname frameworkloss_fnrrrr r!ztorch.optim.Optimizerr"r#r$r%r&r'r(r)r*r+r,r-r.r/r0returnc6 ttt|}G    fdd|}fd}t ||_||_||_|S)aHelper function for creating a new Policy class at runtime. Supports frameworks JAX and PyTorch. Args: name: name of the policy (e.g., "PPOTorchPolicy") framework: Either "jax" or "torch". loss_fn (Optional[Callable[[Policy, ModelV2, Type[TorchDistributionWrapper], SampleBatch], Union[TensorType, List[TensorType]]]]): Callable that returns a loss tensor. get_default_config (Optional[Callable[[None], AlgorithmConfigDict]]): Optional callable that returns the default config to merge with any overrides. If None, uses only(!) the user-provided PartialAlgorithmConfigDict as dict for this Policy. postprocess_fn (Optional[Callable[[Policy, SampleBatch, Optional[Dict[Any, SampleBatch]], Optional[Any]], SampleBatch]]): Optional callable for post-processing experience batches (called after the super's `postprocess_trajectory` method). stats_fn (Optional[Callable[[Policy, SampleBatch], Dict[str, TensorType]]]): Optional callable that returns a dict of values given the policy and training batch. If None, will use `TorchPolicy.extra_grad_info()` instead. The stats dict is used for logging (e.g. in TensorBoard). extra_action_out_fn (Optional[Callable[[Policy, Dict[str, TensorType], List[TensorType], ModelV2, TorchDistributionWrapper]], Dict[str, TensorType]]]): Optional callable that returns a dict of extra values to include in experiences. If None, no extra computations will be performed. extra_grad_process_fn (Optional[Callable[[Policy, "torch.optim.Optimizer", TensorType], Dict[str, TensorType]]]): Optional callable that is called after gradients are computed and returns a processing info dict. If None, will call the `TorchPolicy.extra_grad_process()` method instead. # TODO: (sven) dissolve naming mismatch between "learn" and "compute.." extra_learn_fetches_fn (Optional[Callable[[Policy], Dict[str, TensorType]]]): Optional callable that returns a dict of extra tensors from the policy after loss evaluation. If None, will call the `TorchPolicy.extra_compute_grad_fetches()` method instead. optimizer_fn (Optional[Callable[[Policy, AlgorithmConfigDict], "torch.optim.Optimizer"]]): Optional callable that returns a torch optimizer given the policy and config. If None, will call the `TorchPolicy.optimizer()` method instead (which returns a torch Adam optimizer). validate_spaces (Optional[Callable[[Policy, gym.Space, gym.Space, AlgorithmConfigDict], None]]): Optional callable that takes the Policy, observation_space, action_space, and config to check for correctness. If None, no spaces checking will be done. before_init (Optional[Callable[[Policy, gym.Space, gym.Space, AlgorithmConfigDict], None]]): Optional callable to run at the beginning of `Policy.__init__` that takes the same arguments as the Policy constructor. If None, this step will be skipped. before_loss_init (Optional[Callable[[Policy, gym.spaces.Space, gym.spaces.Space, AlgorithmConfigDict], None]]): Optional callable to run prior to loss init. If None, this step will be skipped. after_init (Optional[Callable[[Policy, gym.Space, gym.Space, AlgorithmConfigDict], None]]): DEPRECATED: Use `before_loss_init` instead. _after_loss_init (Optional[Callable[[Policy, gym.spaces.Space, gym.spaces.Space, AlgorithmConfigDict], None]]): Optional callable to run after the loss init. If None, this step will be skipped. This will be deprecated at some point and renamed into `after_init` to match `build_tf_policy()` behavior. action_sampler_fn (Optional[Callable[[TensorType, List[TensorType]], Tuple[TensorType, TensorType]]]): Optional callable returning a sampled action and its log-likelihood given some (obs and state) inputs. If None, will either use `action_distribution_fn` or compute actions by calling self.model, then sampling from the so parameterized action distribution. action_distribution_fn (Optional[Callable[[Policy, ModelV2, TensorType, TensorType, TensorType], Tuple[TensorType, Type[TorchDistributionWrapper], List[TensorType]]]]): A callable that takes the Policy, Model, the observation batch, an explore-flag, a timestep, and an is_training flag and returns a tuple of a) distribution inputs (parameters), b) a dist-class to generate an action distribution object from, and c) internal-state outputs (empty list if not applicable). If None, will either use `action_sampler_fn` or compute actions by calling self.model, then sampling from the parameterized action distribution. make_model (Optional[Callable[[Policy, gym.spaces.Space, gym.spaces.Space, AlgorithmConfigDict], ModelV2]]): Optional callable that takes the same arguments as Policy.__init__ and returns a model instance. The distribution class will be determined automatically. Note: Only one of `make_model` or `make_model_and_action_dist` should be provided. If both are None, a default Model will be created. make_model_and_action_dist (Optional[Callable[[Policy, gym.spaces.Space, gym.spaces.Space, AlgorithmConfigDict], Tuple[ModelV2, Type[TorchDistributionWrapper]]]]): Optional callable that takes the same arguments as Policy.__init__ and returns a tuple of model instance and torch action distribution class. Note: Only one of `make_model` or `make_model_and_action_dist` should be provided. If both are None, a default Model will be created. compute_gradients_fn (Optional[Callable[ [Policy, SampleBatch], Tuple[ModelGradients, dict]]]): Optional callable that the sampled batch an computes the gradients w.r. to the loss function. If None, will call the `TorchPolicy.compute_gradients()` method instead. apply_gradients_fn (Optional[Callable[[Policy, "torch.optim.Optimizer"], None]]): Optional callable that takes a grads list and applies these to the Model's parameters. If None, will call the `TorchPolicy.apply_gradients()` method instead. mixins (Optional[List[type]]): Optional list of any class mixins for the returned policy class. These mixins will be applied in order and will have higher precedence than the TorchPolicy class. get_batch_divisibility_req (Optional[Callable[[Policy], int]]): Optional callable that returns the divisibility requirement for sample batches. If None, will assume a value of 1. Returns: Type[TorchPolicy]: TorchPolicy child class constructed from the specified args. ceZdZ fdZee d fd Ze fdZe fdZefdZ efdZ e fdZ efd Z efd Z d Zd ZxZS)&build_policy_class..policy_clsc ||_x|_|jd<r||||j r ||||jrE Jd|||||_tj||jd\}}ngr||||\|_}nNtj||jd\}}tj||||jd|_t }t|j|s Jd|_|j |||||j|jdrdn| |dd |j |jj p }|r|||j |j ||d |jdrdn  r ||||d |_dS) Nr2zAEither `make_model` or `make_model_and_action_dist` must be None!model)r2) obs_space action_space num_outputs model_configr2z5ERROR: Generated Model must be a TorchModelV2 object! in_evaluation max_seq_len) observation_spacer;configr9lossaction_distribution_classr)r*r?r0T)auto_remove_unneeded_view_reqsrr)rAr2r9r get_action_dist get_model_v2r isinstance parent_cls__init__view_requirementsupdater@r;!_initialize_loss_from_dummy_batchglobal_timestep)selfr:r;rA dist_class_ logit_dim model_cls_before_loss_initr(r*r)r'r%r&r2r0r3r+r,rHrr$s t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/policy/policy_template.pyrIz/build_policy_class..policy_cls.__init__s DK9B ADNT[5 Lit{KKK H D)\4;GGG 199%:99(ZivNN , < $+g"6)!!! AA, )C)C)\6**& JJ )5(D $+g"6))))% I*6'!- )!%W!5'  %I I G GF G G )DO O $ $"+)j![9FTTw*4"3'="7OM:+E %     " ) )$**F G G G 0 >J   !!$0$2CV  2 2/3!%_!=K8 3      H  y,GGG$%D Nc|5t|||}r||||cdddS|cdddS#1swxYwYdSN)_no_grad_contextsuperpostprocess_trajectory)rN sample_batchother_agent_batchesepisode __class__rs rTrZz=build_policy_class..policy_cls.postprocess_trajectoryLs&&(( $ $$ww== "5w   ")>l,? $ $ $ $ $ $ $ $$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s3A$A$$A(+A(cPr |||S|||S)zCalled after optimizer.zero_grad() and loss.backward() calls. Allows for gradient processing before optimizer.step() is called. E.g. for gradient clipping. )extra_grad_process)rN optimizerrBr!rHs rTr`z9build_policy_class..policy_cls.extra_grad_process_s;% L,,T9dCCC!44T9dKKKrUcr,t|}ttiifi|S|SrW)rdictrextra_compute_grad_fetches)rNfetchesr"rHs rTrdzAbuild_policy_class..policy_cls.extra_compute_grad_fetchesksU% C*+A+A$+G+GHH.3??w???!<.policy_cls.compute_gradientsts5# A++D%888!33D%@@@rUcTr||dS||dSrW)apply_gradients)rN gradientsr.rHs rTrjz6build_policy_class..policy_cls.apply_gradients{s@! <""433333**4;;;;;rUc|5r|||||}n|||||}||cdddS#1swxYwYdSrW)rXextra_action_out_convert_to_numpy)rN input_dict state_batchesr9 action_dist stats_dictr rHs rTrmz7build_policy_class..policy_cls.extra_action_outs&&(( : :&!4!4j- ""JJ",!.policy_cls.optimizers9 8)\$ << '11$77  rUc|5r ||}n|j||}||cdddS#1swxYwYdSrW)rXrHextra_grad_inforn)rN train_batchrrrs rTrvz6build_policy_class..policy_cls.extra_grad_infos&&(( : :T!)$ !.policy_cls._no_grad_contexts)~((}}&%'' 'rUc:|jdkrt|S|Sry)r2r)rNdatas rTrnz8build_policy_class..policy_cls._convert_to_numpys"~(('---KrU)NN)__name__ __module__ __qualname__rIrrrZr`rdrgrjrmrarvrXrn __classcell__)r^r(r*r)r'r.r%r&r-r r!r"r2r0r3r+r,r#rHrrr$s@rT policy_clsr7shS %S %S %S %S %S %S %S %S %S %S %S %S %S %S %S %S %S %j &  BF $ $ $ $ $ $   $$ *   L L L L L   L *   C C C C C   C *   A A A A A   A *   < < < < <   < *   : : : : :   : *           *   : : : :   : ( ( (        rUrc 2tditfi|S)aCreates a Torch|JAXPolicy cls based on settings of another one. Keyword Args: **overrides: The settings (passed into `build_torch_policy`) that should be different from the class that this method is called on. Returns: type: A new Torch|JAXPolicy sub-class. Examples: >> MySpecialDQNPolicyClass = DQNTorchPolicy.with_updates( .. name="MySpecialDQNPolicyClass", .. loss_function=[some_new_loss_function], .. ) )build_policy_classrc) overridesoriginal_kwargss rT with_updatesz(build_policy_class..with_updatess)""GGD$F$FI$F$FGGGrU)localscopyrr staticmethodrr~r)r1r2r3rrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0baserrrrHs `` ````````````````` ` @@rTrr s=dhhmmooOJ j& ) )DrrrrrrrrrrrrrrrrrrrrrrrrrrrTrrrhHHHHH&+<88JJ"J rU)4typingrrrrrrr r gymnasiumgymray.rllib.models.catalogr ray.rllib.models.modelv2r (ray.rllib.models.torch.torch_action_distr $ray.rllib.models.torch.torch_modelv2rray.rllib.policy.policyrray.rllib.policy.sample_batchrray.rllib.policy.torch_policyrray.rllib.utilsrrray.rllib.utils.annotationsrrray.rllib.utils.frameworkrr$ray.rllib.utils.metrics.learner_inforray.rllib.utils.numpyrray.rllib.utils.typingrrrjaxrPrzstrSpacespacestypercintrrrUrTrsa                    111111,,,,,,MMMMMM======******555555555555::::::::========FFFFFFFFBBBBBB222222RRRRRRRRRR   Q    q GKQU   RV       #'DHs___ __ Wd#;S*_@UUVG_N%XvhS*_8M.M%NOO_P&-.0GGHQ_V&#)SY0CDdJKW_\&#)SY0CDdJK]_b SZ%sz'79L Mt S c_l&#)SY0CDdJKm_r SZ%sz'79L Mt S s_| *d:./z:7M1NNO}_B% Wj*j A *dD$44 5 7 C_N SZ%sz'79L Mw V O_X!) SZ%sz'79L M '4 899 : < !Y_d#&+&nd.B(CCDe_j!&12D89k_p T$Z q_r!)6(C-)@ As_t +u___ ___rU