&`i<ddlZddlZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl mZddlZddlmZddlmZddlmZddlmZdd lmZmZmZmZmZdd lmZdd l m!Z!m"Z"dd l#m$Z$m%Z%dd l&m'Z'ddl(m)Z)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMddlNmOZOddlPmQZQmRZRmSZSddlTmUZUmVZVddlWmXZXejYdZZeXdGdde*e2Z[dS)N) defaultdict) Collection DefaultDictListOptionalUnion)DictInfoToList) Deprecated)AlgorithmConfig) RLlibCallback) make_callback)!COMPONENT_ENV_TO_MODULE_CONNECTOR!COMPONENT_MODULE_TO_ENV_CONNECTORCOMPONENT_RL_MODULEDEFAULT_AGENT_IDDEFAULT_MODULE_ID)Columns)RLModule RLModuleSpec)INPUT_ENV_SINGLE_SPACESINPUT_ENV_SPACES) EnvContext)ENV_STEP_FAILURE EnvRunner)SingleAgentEpisode) force_list)override)Checkpointable)ERR_MSG_INVALID_ENV_DESCRIPTOREnvError) get_device)ENV_TO_MODULE_CONNECTOREPISODE_DURATION_SEC_MEANEPISODE_LEN_MAXEPISODE_LEN_MEANEPISODE_LEN_MINEPISODE_RETURN_MAXEPISODE_RETURN_MEANEPISODE_RETURN_MINMODULE_TO_ENV_CONNECTORNUM_AGENT_STEPS_SAMPLED NUM_AGENT_STEPS_SAMPLED_LIFETIMENUM_ENV_STEPS_SAMPLEDNUM_ENV_STEPS_SAMPLED_LIFETIME NUM_EPISODESNUM_EPISODES_LIFETIMENUM_MODULE_STEPS_SAMPLED!NUM_MODULE_STEPS_SAMPLED_LIFETIMERLMODULE_INFERENCE_TIMER SAMPLE_TIMERTIME_BETWEEN_SAMPLINGWEIGHTS_SEQ_NO)unbatch) EpisodeID ResultDict StateDict) ENV_CREATOR_global_registry) PublicAPIz ray.rllibalpha) stabilityceZdZdZeedeffd Zeedddddddeded e d e d e d e e f d Z dddddde ede ed e d e d e d e e f dZeedZeed efdZee d-ddde eeeefde eeeefd efdZeeded dfdZeedZeedZeedZeedZeed.dZeedZeedZdZ d-d Z!d!ed"ed#e e fd$Z"d%Z#d&Z$e%d'd()d*Z&e%d+d()d,Z'xZ(S)/SingleAgentEnvRunnerz9The generic environment runner for the single agent case.configc tjd d|i||d|_|di|_dt |jjD|_t|j|j sdn |jj |_ d|_ d|_|j '|j dks|jjs|jjdkr||j|j |j|j |_d|_d|_||j|j |j|_d |_d t5|jD|_d|_g|_t=t>|_ d|_!d|_"|jj#|_#dS) zInitializes a SingleAgentEnvRunner instance. Args: config: An `AlgorithmConfig` object containing all settings needed to build this `EnvRunner` class. rB tune_trial_idspacesc"g|] }| SrG).0clss y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/env/single_agent_env_runner.py z1SingleAgentEnvRunner.__init__..Ts+0 0 0 CCEE0 0 0 rN)envrEdevice)rMrETcg|]}dSNrGrH_s rJrKz1SingleAgentEnvRunner.__init__..s%> > > D> > > rLrG)$super__init__getrDrErrBcallbacks_class _callbacksr! worker_indexnum_gpus_per_env_runner_devicerMnum_envscreate_env_on_local_workernum_env_runnersmake_envbuild_env_to_module_connector_env_to_module_cached_to_modulemodule make_modulebuild_module_to_env_connector_module_to_env_needs_initial_resetrange _episodes _shared_data_done_episodes_for_metricsrlist_ongoing_episodes_for_metrics_weights_seq_no_time_after_samplingepisodes_to_numpy)selfrBkwargs __class__s rJrTzSingleAgentEnvRunner.__init__Fs 111&111"(**_"="=jj2.. 0 0 ' (CDD0 0 0  " K& OAADK,O   48   % 1$${5%{*a// MMOOO#kGGT\H  "&+/  #kGGH   +/!> >  ..> > > !DF'    *%&%)! "&!>rLNF num_timesteps num_episodesexplorerandom_actions force_resetrtrurvrwrxreturnc z|jt|d||J|j:|jt t j|jz |jt|j d|j t5| |j j }|9|7|j jdkr'|j |j|jz}|"|dksJ|||||}n@|!|dksJ|||| }n||j|| }t'd |j|j jt-||j|  dddn #1swxYwYt j|_|S) aRuns and returns a sample (n timesteps or m episodes) on the env(s). If neither `num_timesteps` nor `num_episodes` are provided and the config `batch_mode` is "truncate_episodes" then `config.get_rollout_fragment_length(self.worker_index) * self.num_envs` timesteps will be sampled. Args: num_timesteps: The number of timesteps to sample during this call. The episodes returned will contain the total timesteps greater than or equal to num_timesteps and less than num_timesteps + num_envs_per_env_runner. Note that only one of `num_timesteps` or `num_episodes` may be provided. num_episodes: The number of episodes to sample during this call. Note that only one of `num_timesteps` or `num_episodes` may be provided. explore: If True, will use the RLModule's `forward_exploration()` method to compute actions. If False, will use the RLModule's `forward_inference()` method. If None (default), will use the `explore` boolean setting from `self.config` passed into this EnvRunner's constructor. You can change this setting in your config via `config.env_runners(explore=True|False)`. random_actions: If True, actions will be sampled randomly (from the action space of the environment). If False (default), actions or action distribution parameters are computed by the RLModule. force_reset: Whether to force-reset all vectorized environments before sampling. Useful if you would like to collect a clean slate of new episodes via this call. Note that when sampling n episodes (`num_episodes != None`), this is fixed to True. Returns: A list of `SingleAgentEpisode` instances, carrying the sampled data. Nz2 doesn't have an env! Can't call `sample()` on it.)keyvalue)r{r|windowtruncate_episodesr)rtrvrwrx)rurvrw on_sample_end) env_runnermetrics_loggersamplescallbacks_objectscallbacks_functionsrq)rM ValueErrorrnmetrics log_valuer5time perf_counterr6rmlog_timer4rBrv batch_modeget_rollout_fragment_lengthrXr[_sampler rWcallbacks_on_sample_enddict)rprtrurvrwrxrs rJsamplezSingleAgentEnvRunner.samplespR 8 KKK "-,2J2JK  $ 0 L " ")'))D,EE #    &    \ " "< 0 04 4 +-% (K*.AAAK;;D.s(L(L(L!(L(L(LrLFTrr})t) rl_modulebatchepisodesrv shared_datarmetrics_prefix_keyrsc(i|]\}}||SrGrG)rHkv env_indexs rJ z0SingleAgentEnvRunner._sample..ns#%Q%Q%Q$!Qa9%Q%Q%QrL observationinfos)ractionrewardr terminated truncatedextra_model_outputs)rrrvrrrron_episode_starton_episode_stepon_episode_endcPg|]"}|jj#S))len_lookback_buffer)cutrBepisode_lookback_horizon)rHepsrps rJrKz0SingleAgentEnvRunner._sample..s<...DK,PQQ...rL)metric num_steps)7rfrgr[rhri _reset_envsrACTIONSrM action_spacerranum_env_steps_sampled_lifetimerBr]rrr3rbforward_explorationforward_inferencerer*popACTIONS_FOR_ENV _try_env_steprrr7setitemsrmr6is_reset add_env_resetadd add_env_stepr`r"_make_on_episode_callbackis_doneroappendto_numpy _new_episoderjextendrvalidaterlid__log_env_steps_metrics_num_env_steps_sampled_increase_sampled_metricslen)rprtrurvrwrxdone_episodes_to_returnrrtsrto_env to_moduleglobal_env_steps_lifetimeactionsactions_for_envresults observationsrewards terminateds truncatedsrcall_on_episode_startextra_model_outputongoing_episodes_to_returnongoing_episodes_continuationsrs` @rJrzSingleAgentEnvRunner._samples'=?  ,,2d6O2(L(LuT]7K7K(L(L(L LHt~.0 0K$+   X{G < < <).D % %~H+K  #(,D %$1$=R-   C,DVDV% OTX%:%A%A%C%C !2  ,,,)-&J; K7<1>49 1;- ../GHH!%!@!@%)B"A"" ../GHHJJ!%!>!>y!I!IJJJJJJJJJJJJJJJ,,"k %# + L(?'A-jj11G$jj)@'JJO((99G***||"/!-##1 $ $EL AL'; E$+L$9$977;K;K'L$'EE !"4=11   %Q%Q%Q%Q&,,..%Q%Q%Q"595I">2~i09Y'55$0$;#I.6*--i8888 !GBY'44$0$;&y1&y1#I.#.y#9",Y"7,>5{&)-)<)<%#"k + L(?'A*=**&#4=11% ;% ;  55522*Ix 22)9h I&.;1HC22()X -L/66x 7J7S7S7U7UVVVV066x 7JKKKl**%%i:::[%2$=R-   C,DVDV` '../FGGG &("  $....>... * ~ ; ;5A:: 237;BB3GGG);.55cllnnEEEE/55c::::swG@@D"c$ii/N"doo&7&77N&$*=*=*?*??&&6sw?  % %0B     '--///|""$$$rL)not_components componentsrc t|ji}|t||r^|jjd|t||t|d||t<|j|t<|t||r!|j |t<|t||r!|j |t<|S)N)rrrG) r.r_check_componentrrb get_state_get_subcomponentsrmr6rr`rre)rprrrqstates rJrzSingleAgentEnvRunner.get_states01TU  !4j. Q Q 9)>)>*223F SS#66'  **  **E% &%)$8E. !  -z>   W8<7J7T7T7V7VE3 4  -z>   W8<7J7T7T7V7VE3 4 rLrc|t|vr%|j|tt|vr%|j|tt |vr|td}|dks |j|kr|t }t|tj rtj|}t|trt|vr |t}|j||dkr||_t|vr|t|_dSdS)Nr)rr` set_staterrerrUr6rm isinstanceray ObjectRefrrrbr.r)rprweights_seq_norl_module_states rJrzSingleAgentEnvRunner.set_state/s9 , 5 5   ) )%0Q*R S S S , 5 5   ) )%0Q*R S S S % ' '#YY~q99N""d&:^&K&K"'(;"<os}==?&)go&>&>O55I)_<<&56G&HO %%o666!!'5$ *U 2 2278V2WD / / / 3 2rLcdd|jifS)NrGrB)rBrs rJget_ctor_args_and_kwargsz-SingleAgentEnvRunner.get_ctor_args_and_kwargsQs  t{ #  rLcXtj|}|i|SrP)r get_metadataupdate)rpmetadatas rJrz!SingleAgentEnvRunner.get_metadataXs4!.t44    rLcTt|jft|jft|jfgSrP)rrbrr`rrers rJget_checkpointable_componentsz2SingleAgentEnvRunner.get_checkpointable_componentsbs.!$+ . .0C D .0C D  rLc8|jrt|dsJdS)aChecks that self.__init__() has been completed properly. Ensures that the instance has a `MultiRLModule` and an environment defined. Raises: AssertionError: If the EnvRunner Actor has NOT been properly initialized. rbN)rMhasattrrs rJassert_healthyz#SingleAgentEnvRunner.assert_healthyjs'x3GD(3333333rLc VjZ jn?#t$r2}td|jdYd}~nd}~wwxYwjj}t|ts(t|j j jj n|jjstdtjjtrltjt"jjrHd}tjt"jjt'j|fdfd i}nIt+jjr$d}t'j|fd fd  i}n jj}t'jjj} t1t'j|fjj|d |_nH#t&jj$r1}t;t=jjj|d}~wwxYwjj _ j jjksJd _!tEdj#jj$tKj&jj'dS)aCreates a vectorized gymnasium env and stores it in `self.env`. Note that users can change the EnvRunner's config (e.g. change `self.config.env_config`) and then call this method to create new environments with the updated configuration. Nz7Tried closing the existing env, but failed with error: r)rX num_workersremotez`config.env` is not provided! You should provide a valid environment to your config through `config.environment([env descriptor e.g. 'CartPole-v1'])`.zrllib-single-agent-env-v0cSrPrG) entry_pointenv_ctxsrJz/SingleAgentEnvRunner.make_env..sKK$8$8rLc$d|izSNr[rG)r[r r s rJr z/SingleAgentEnvRunner.make_env..s KKz84455rL)r vector_entry_pointc8jSrPrBrM)r rpsrJr z/SingleAgentEnvRunner.make_env..sDKOOG$<$<rLcBjd|izSrr)r[r rps rJr z/SingleAgentEnvRunner.make_env..s$DKOOz84455rL)r[vectorization_modeTon_environment_created)rrrM env_contextr)(rMclose ExceptionloggerwarningargsrB env_configrrrXrremote_worker_envsrstrr<containsr;rUgymregistercallable VectorizeModegym_env_vectorize_moder make_vecnum_envs_per_env_runnererrorErrorr rformatr[rfr rW callbacks_on_environment_createdrr unwrapped)rperenv_namevectorize_moder r s` @@rJr^zSingleAgentEnvRunner.make_envwsH 8         #vay##  [+ *j11 ! !. ,{5 GG!G{ 'M   - - '2B2K 3 3  '3H*.{DKOLLK L88888$$$$$    JJ dko & & '2H L<<<<<$$$$$    JJ{H*4;+MNN % ![@'5! DHHy   .5dkoFF   "X. } CCCCC%)!  $"o $ L#|H&#  s,& A"(AA">1G00H5,H00H5c0|j |jjnd} |j||d}||_|j|jdS#t$r d|_YdSwxYw)NT)rMrEinference_only) rMr+rBget_rl_module_specrbuildrbtorZNotImplementedError)rprM module_specs rJrcz SingleAgentEnvRunner.make_modules$(H$8dh  d (, (F(F 1 1$)G))K&++--DK KNN4< ( ( ( ( (#   DKKKK sA'BBBcJ|j|jdSdSrP)rMrrs rJstopzSingleAgentEnvRunner.stops, 8  HNN      rLcXt|jD]}||||j||jr|jndd\}}t|}t|jD]+}|| ||||,d|_ |j r0| |j ||||j tf|_ t|jD]}|d||dS)N)seedoptionsr)rrrvrrrr)rgr[rrlr_try_env_resetrf_seedr7rrarbr`rr"r)rprrrvrrrs rJrz SingleAgentEnvRunner._reset_envsspt}-- 3 3I   i 2 2 2 2 *00222#11#8Bd 2   e |,, t}--  I Y  - -(3I& .    "& ; %)%8%8+!' $;#= &9&&D "t}-- T TI  * *+=y( S S S S T TrLc||n|j}t|jj|jj||<|d||dS)N)rron_episode_created)rhrrMrrr)rprrs rJrz!SingleAgentEnvRunner._new_episode s\'3880"h?5    &&';YQQQQQrLwhichidxrc t||||j|jj|j|}|dkr|j||j|d<t||jt|j d||dS)N)episoderrrMrrrprev_episode_chunks callbacks_r) rrrMr+rbrlrr rWgetattrrB)rpr?r@rrqs rJrz.SingleAgentEnvRunner._make_on_episode_callback(sSM<"k     $ $ $,0,N !-F( )  "o ' 5I%5I5I J J       rLc`|jt|d|jttf|d|jt t f|d|jt|d|jt|dd|jttf|d|jtt f|d|jt|d|S)Nsum)r lifetime_sumT)rwith_throughput) rrr-r+rr1rr/r.r,r2r0)rprnum_episodes_completeds rJrz.SingleAgentEnvRunner._increase_sampled_metrics?sg 4iNNN  $&6 7      %'8 9       "     * !     -/? @ !     .0A B !     ! "!    rLc  tdttj|jj|jjpdz }|jt|||jt|||jt|||jdtf|||jdtf|||jt|d||jt|d||jt |d||jt"|d|dS)Nr})r~agent_episode_return_meanmodule_episode_return_meanmin)rr~max)rOintmathceilrB"metrics_num_episodes_for_smoothingr]rrr%r(r#rrr&r)r$r')rplengthretsecwins rJrz)SingleAgentEnvRunner._log_episode_metricsis   KB{27a9     /DDD 2CDDD 8#cJJJ  (*: ;S      )+< =s3    uSQQQ 13uSQQQ uSQQQ 13uSQQQQQrLz6SingleAgentEnvRunner.get_state(components='rl_module')T)newr'cdSrPrGrprrqs rJ get_weightsz SingleAgentEnvRunner.get_weightss rLz SingleAgentEnvRunner.set_state()cdSrPrGrZs rJ set_weightsz SingleAgentEnvRunner.set_weightss rLrP)ryN))__name__ __module__ __qualname____doc__rrr rTrPboolrrrrrrr9rrrrrr:rrrrrrr^rcr7rrrrrr r[r] __classcell__)rrs@rJrArABsCC XiM?/M?M?M?M?M?M?^Xi" $!uuuu u  u  uu  !uuuut(,&*$!WDWDWD }WDsm WD  WD  WDWD  !WDWDWDWDrXi     Xi%Z%%%%2Xn=AAE U3 3#789!sJsO';!<=   :XnXyXTXXXXBXn   XnXn   Xi 4 4 4Xi` ` ` ` DXi(Xi )T)T)TVRRRR  " .23E.F    .(((TRRR@Z D   Z6dCCC  DC     rLrA)\loggingrQr collectionsrtypingrrrrr gymnasiumr gymnasium.wrappers.vectorr rray._common.deprecationr %ray.rllib.algorithms.algorithm_configr ray.rllib.callbacks.callbacksr ray.rllib.callbacks.utilsr ray.rllib.corerrrrrray.rllib.core.columnsr"ray.rllib.core.rl_module.rl_modulerr ray.rllib.envrrray.rllib.env.env_contextrray.rllib.env.env_runnerrr"ray.rllib.env.single_agent_episoderray.rllib.utilsrray.rllib.utils.annotationsrray.rllib.utils.checkpointsrray.rllib.utils.errorrr ray.rllib.utils.frameworkr!ray.rllib.utils.metricsr"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6"ray.rllib.utils.spaces.space_utilsr7ray.rllib.utils.typingr8r9r:ray.tune.registryr;r<ray.util.annotationsr= getLoggerrrArGrLrJrs ######AAAAAAAAAAAAAA444444 ......AAAAAA777777333333+*****EEEEEEEECCCCCCCC000000@@@@@@@@AAAAAA&&&&&&000000666666JJJJJJJJ000000.766666CCCCCCCCCC;;;;;;;;******  ; ' '  WO  O  O  O  O  9nO  O  O  O  O  rL