&`ixddlZddlZddlmZmZmZmZmZmZm Z m Z ddl Z ddl ZddlmZddlmZddlmZddlmZmZddlmZmZmZmZmZmZddlm Z dd l!m"Z"m#Z#d Z$ej%e&Z'e#d Gd de j(Z)e"de e*efde dfdZ+eGddeZ,eGddZ-dS)N)CallableDictListOptionalSetTupleTypeUnion) Deprecated)BaseEnv) EnvContext) OldAPIStackoverride)AgentID EnvCreatorEnvIDEnvTypeMultiAgentDict MultiEnvDict)log_once) DeveloperAPI PublicAPIstatebeta) stabilityceZdZUdZdZeeeej fe d<dZ eeeej fe d<gZ e ee d<gZe ee d<dZeej e d<dZeej e d<fd Zddd d eed eed eeefffdZded eeeeeeffdZd(dZded ej fdZded ej fdZed efdZed efdZ d)deee efdej dej d dfdZe e!ddd e"efdZ#e d*d!ee$ege%fd"ed#e&d$ed%e&d d&f d'Z'xZ(S)+ MultiAgentEnvzlAn environment that hosts multiple independent agents. Agents are identified by AgentIDs (string). Nobservation_spaces action_spacesagentspossible_agentsobservation_space action_spacectt|dst|_|jst |j|_|js |j|_dSdSN _agent_ids) super__init__hasattrsetr&r listr!copy)self __class__s q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/env/multi_agent_env.pyr(zMultiAgentEnv.__init__8s t\** $!eeDO{ 0t//DK# 6#';#3#3#5#5D  6 6seedoptionsr2r3returncNt||dS)aResets the env and returns observations from ready agents. Args: seed: An optional seed to use for the new episode. Returns: New observations for each ready agent. .. testcode:: :skipif: True from ray.rllib.env.multi_agent_env import MultiAgentEnv class MyMultiAgentEnv(MultiAgentEnv): # Define your env here. env = MyMultiAgentEnv() obs, infos = env.reset(seed=42, options={}) print(obs) .. testoutput:: { "car_0": [2.4, 1.6], "car_1": [3.4, -3.2], "traffic_light_1": [0, 3, 5, 1], } r1N)r'reset)r-r2r3r.s r/r6zMultiAgentEnv.resetEs&B  4 11111r0 action_dictct)aReturns observations from ready agents. The returns are dicts mapping from agent_id strings to values. The number of agents in the env can vary over time. Returns: Tuple containing 1) new observations for each ready agent, 2) reward values for each ready agent. If the episode is just started, the value will be None. 3) Terminated values for each ready agent. The special key "__all__" (required) is used to indicate env termination. 4) Truncated values for each ready agent. 5) Info values for each agent id (may be empty dicts). .. testcode:: :skipif: True env = ... obs, rewards, terminateds, truncateds, infos = env.step(action_dict={ "car_0": 1, "car_1": 0, "traffic_light_1": 2, }) print(rewards) print(terminateds) print(infos) .. testoutput:: { "car_0": 3, "car_1": -1, "traffic_light_1": 0, } { "car_0": False, # car_0 is still running "car_1": True, # car_1 is terminated "__all__": False, # the env is not terminated } { "car_0": {}, # info for car_0 "car_1": {}, # info for car_1 } )NotImplementedError)r-r7s r/stepzMultiAgentEnv.stephs d"!r0cdS)z Tries to render the environment.Nr-s r/renderzMultiAgentEnv.renders  r0agent_idc|j |j|St|jtjjr||jjvr |j|S|jSN)r isinstancer"gymspacesrr-r?s r/get_observation_spacez#MultiAgentEnv.get_observation_spaces^  " .*84 4 t-sz ? ? *D2999)(3 3 ) )r0c|j |j|St|jtjjr||jjvr |j|S|jSrA)rrBr#rCrDrrEs r/get_action_spacezMultiAgentEnv.get_action_spaces^   )%h/ / t(#*/ : : %D-444$X. . $ $r0c*t|jSrA)lenr r=s r/ num_agentszMultiAgentEnv.num_agentss4;r0c*t|jSrA)rJr!r=s r/max_num_agentszMultiAgentEnv.max_num_agentss4'(((r0groups obs_space act_spacec*ddlm}|||||S)a,Convenience method for grouping together agents in this env. An agent group is a list of agent IDs that are mapped to a single logical agent. All agents of the group must act at the same time in the environment. The grouped agent exposes Tuple action and observation spaces that are the concatenated action and obs spaces of the individual agents. The rewards of all the agents in a group are summed. The individual agent rewards are available under the "individual_rewards" key of the group info return. Agent grouping is required to leverage algorithms such as Q-Mix. Args: groups: Mapping from group id to a list of the agent ids of group members. If an agent id is not present in any group value, it will be left ungrouped. The group id becomes a new agent ID in the final environment. obs_space: Optional observation space for the grouped env. Must be a tuple space. If not provided, will infer this to be a Tuple of n individual agents spaces (n=num agents in a group). act_space: Optional action space for the grouped env. Must be a tuple space. If not provided, will infer this to be a Tuple of n individual agents spaces (n=num agents in a group). .. testcode:: :skipif: True from ray.rllib.env.multi_agent_env import MultiAgentEnv class MyMultiAgentEnv(MultiAgentEnv): # define your env here ... env = MyMultiAgentEnv(...) grouped_env = env.with_agent_groups(env, { "group1": ["agent1", "agent2", "agent3"], "group2": ["agent4", "agent5"], }) r)GroupAgentsWrapper)+ray.rllib.env.wrappers.group_agents_wrapperrR)r-rNrOrPrRs r/with_agent_groupszMultiAgentEnv.with_agent_groupss/^ SRRRRR!!$ 9EEEr0zMultiAgentEnv.possible_agentsF)newerrorct|dst|_t|jtst|j|_|jr|jnt|jSr%)r)r*r&rBr r=s r/ get_agent_idszMultiAgentEnv.get_agent_idssct\** $!eeDO$/3// 3!$/22DO"&/Gts4;7G7GGr0rmake_envnum_envs remote_envsremote_env_batch_wait_msrestart_failed_sub_environmentsr c`ddlm}|r|||d||}nt||g||}|S)a Converts an RLlib MultiAgentEnv into a BaseEnv object. The resulting BaseEnv is always vectorized (contains n sub-environments) to support batched forward passes, where n may also be 1. BaseEnv also supports async execution via the `poll` and `send_actions` methods and thus supports external simulators. Args: make_env: A callable taking an int as input (which indicates the number of individual sub-environments within the final vectorized BaseEnv) and returning one individual sub-environment. num_envs: The number of sub-environments to create in the resulting (vectorized) BaseEnv. The already existing `env` will be one of the `num_envs`. remote_envs: Whether each sub-env should be a @ray.remote actor. You can set this behavior in your config via the `remote_worker_envs=True` option. remote_env_batch_wait_ms: The wait time (in ms) to poll remote sub-environments for, if applicable. Only used if `remote_envs` is True. restart_failed_sub_environments: If True and any sub-environment (within a vectorized env) throws any error during env stepping, we will try to restart the faulty sub-environment. This is done without disturbing the other (still intact) sub-environments. Returns: The resulting BaseEnv object. r) RemoteBaseEnvT) multiagentr]r^)rZ existing_envsr[r^)ray.rllib.env.remote_base_envr`MultiAgentEnvWrapper)r-rZr[r\r]r^r`envs r/ to_base_envzMultiAgentEnv.to_base_env srL @?????  -)A0O CC'!#f!0O C r0)r4N)NN)NrYFrF))__name__ __module__ __qualname____doc__rrrrrCSpace__annotations__rr rr!r"r#r(intdictrrr6r:r>rFrHpropertyrKrMstrrTrr rrXrrboolrf __classcell__)r.s@r/rrsY>Bgsy&8!9:AAA8.MultiEnvNconfigctintjdd}t t r!fdt|D_n!fdt|D_t_ t_ fdt|D_ fdt|D_ tt|_j_dS)NrKrYc8g|]}tjSr<)rCmake).0_rss r/ z?make_multi_agent..MultiEnv.__init__..s$OOOqSX&9::OOOr0c&g|] }Sr<r<)r{r|rwrss r/r}z?make_multi_agent..MultiEnv.__init__..s%MMMQ0088MMMr0c6i|]}|j|jSr<envsr"r{ir-s r/ z?make_multi_agent..MultiEnv.__init__..s1'''6749Q<1'''r0c6i|]}|j|jSr<rr#rs r/rz?make_multi_agent..MultiEnv.__init__..s$!S!S!S1!TYq\%>!S!S!Sr0)r'r(r,deepcopypoprBrprangerr* terminateds truncatedsrrr+r r!)r-rwnumr.rss`` r/r(z+make_multi_agent..MultiEnv.__init__~sF GG     ~ v..**\1--C-s33 NOOOOE#JJOOO MMMMM%**MMM "uuD !eeDO'''';@::'''D #"T!S!S!Sc !S!S!SD uSzz**DK#';#3#3#5#5D r0r1r2r3cRt|_t|_ii}}t|jD]e\}}|||\||<||<|j|||st df||fS)Nr1zMultiEnv does not contain obs.) r*rr enumeraterr6rcontainsloggerwarning)r-r2r3obsinfosrres r/r6z(make_multi_agent..MultiEnv.resets"uuD !eeDORC#DI.. E E3#&99$9#H#H Aa.q1::3q6BBENN#CDDD: r0cXiiiiif\}}}}}t|dkrtd|D]~\}}|j||\||<||<||<||<||<||r|j|||r|j|t|j|jzt|jk|d<t|jt|jk|d<|||||fS)Nrz>The environment is expecting an action for at least one agent.__all__)rJ ValueErroritemsrr:raddr) r-r7rrew terminated truncatedinforactions r/r:z'make_multi_agent..MultiEnv.stepsW46BB4F 1Cj)T;1$$ T)..00 + + 6GKyH$v,,EAA 1 y|T!Wa=,$((+++Q<+O''*** %((84?(J$K$Ks PP%Jy !$'t#7#73ty>>#IIi ZD8 8r0cd|jD}|djd|djdkrd}nd}tj||S)Nc6g|]}|Sr<)r>)r{es r/r}z=make_multi_agent..MultiEnv.render..s ;;;AQXXZZ;;;r0rrY)axis)rshapenp concatenate)r- render_images concat_dims r/r>z)make_multi_agent..MultiEnv.renders_ <;;;;MQ%a(=+;+A!+DDD  >-jAAA Ar0rA)rgrhrir r(rrrrmrnr6r:r>rr)r.rss@r/MultiEnvrv}s 6 6: 6 6 6 6 6 6 6< - 15QU   #     !   -  9 9 !  96 -  B B !  B B B B Br0r)r)rsrs` r/make_multi_agentrIsThQBQBQBQBQBQBQB=QBQBQBf Or0ceZdZdZ ddeegefdeddedefdZ e e d e e e e e e e ffd Ze e d e d d fd Ze e  dd d ddeedeedeed ee e e ffdZe e ddeed d fdZe e  dded eeeefeeffdZe e ddeed d fdZee e d ejjfdZee e d ejfdZe e d e e!fdZ"ddeed d fdZ#d S)rdziInternal adapter of MultiAgentEnv to BaseEnv. This also supports vectorization if num_envs > 1. FrZrbrr[r^c||_||_||_||_t |_t |_t|j|jkr\|j|t|jt|j|jk\|jD]}t|tsJ| d|jdj |_ dS)aUWraps MultiAgentEnv(s) into the BaseEnv API. Args: make_env: Factory that produces a new MultiAgentEnv instance taking the vector index as only call argument. Must be defined, if the number of existing envs is less than num_envs. existing_envs: List of already existing multi-agent envs. num_envs: Desired num multiagent envs to have at the end in total. This will include the given (already created) `existing_envs`. restart_failed_sub_environments: If True and any sub-environment (within this vectorized env) throws any error during env stepping, we will try to restart the faulty sub-environment. This is done without disturbing the other (still intact) sub-environments. N)idxr)rZrr[r^r*rrrJappendrBr_init_env_state unwrapped_unwrapped_env)r-rZrbr[r^res r/r(zMultiAgentEnvWrapper.__init__s,! !   /N,55%%$)nnt},, I  T]]3ty>>:: ; ; ;$)nnt},,9 2 2Cc=11 1 11 1 &&&"il4r0r4ciiiiif\}}}}}t|jD].\}}|\||<||<||<||<||</|||||ifSrA)r env_statespoll)r-rrewardsrrrr env_states r/rzMultiAgentEnvWrapper.polls8:2r2r7I4Wk:u%do66 ! !LAy   A A1 aG[*eR??r0r7Nc|D]\}}||jvs ||jvrtd|d|j|} ||\}}}}} nj#t $r]} |jrJt | j d| || }i}ddi}ddi}i} n| Yd} ~ nd} ~ wwxYwt|tt fs Jdt|ts Jd t|ts Jd t|ts Jd t| ts Jd t|trt| t|} | rN| d hkrGtd| |d|vr"td|d|vr"td||dr|j||dr|j||j|||||| dS)NzEnv z. is already done and cannot accept new actionsr)env_idrTFz+Not a multi-agent obs dict or an Exception!zNot a multi-agent reward dict!z#Not a multi-agent terminateds dict!z"Not a multi-agent truncateds dict!zNot a multi-agent info dict! __common__z}Key set for infos must be a subset of obs (plus optionally the '__common__' key for infos concerning all/no agents): {} vs {}zfIn multi-agent environments, '__all__': True|False must be included in the 'terminateds' dict: got {}.zeIn multi-agent environments, '__all__': True|False must be included in the 'truncateds' dict: got {}.)rrrrrr: Exceptionr^r exceptionargs try_restartrBrnr* differenceformatkeysrrobserve) r-r7r agent_dictrerrrrrr info_diffs r/ send_actionsz!MultiAgentEnvWrapper.send_actionss4"-"3"3"5"54 4  FJ)))Vt-F-F Q6QQQ)F#C ?Bxx ?S?S<Wk:uu   7 $$QVAY///$$F$333C G#,d"3K"+U!3JEEGEEEE dI& = =< = =gt,, N N.N N N,k400 W W2W W W0j$// U U1U U U/eT** J J,J J J*#t$$ JJ11#c((;; |n!%DI//00F # #C-33w3OOJC#y)) L7@D c@R@RRDOC(,ty~~I!#t,,KK.KKK,$***$++C000$/))O**3///GCL"IcNN !!r0cDt|tr|g}|.ttt |j}|D]} |j|nF#t$r9}tdr t d|d|Yd}~nd}~wwxYwt d|d| |x|j |_ |j|<t d|ddS)N close_sub_envzBTrying to close old and replaced sub-environment (at vector index=z"), but closing resulted in error: z+Trying to restart sub-environment at index .zSub-environment at index z restarted successfully.)rBrmr+rrJrcloserrrrrZrre)r-rrrs r/rz MultiAgentEnvWrapper.try_restartmsO fc " " XF >%DI//00F V VC  #$$&&&&   O,,NNM!$MMIJMM  NNOOOO P P P8< c8J8J JDOC $ty~ NNTsTTT U U U U V VsA.. B18/B,,B1as_dictcd|rdt|jDSd|jDS)Nc$i|] \}}||jSr<re)r{_idrs r/rz=MultiAgentEnvWrapper.get_sub_environments..s XXX>3 CXXXr0cg|] }|j Sr<r)r{rs r/r}z=MultiAgentEnvWrapper.get_sub_environments..s777e 777r0)rr)r-rs r/get_sub_environmentsz)MultiAgentEnvWrapper.get_sub_environmentssA  YXXYt=W=WXXX X77t7777r0cv|d}t|tsJ|j|SNr)rBrmrr>)r-rs r/ try_renderzMultiAgentEnvWrapper.try_renders; >F&#&&&&&y '')))r0c&|jdjSrrr=s r/r"z&MultiAgentEnvWrapper.observation_spacesy|--r0c&|jdjSrrr=s r/r#z!MultiAgentEnvWrapper.action_spacesy|((r0c@|jdSr)rrXr=s r/rXz"MultiAgentEnvWrapper.get_agent_idssy|))+++r0rc|fdjD_dSt|tsJt j|jj|<dS)zResets all or one particular sub-environment's state (by index). Args: idx: The index to reset at. If None, reset all the sub-environments' states. Nc:g|]}t|jSr<)_MultiAgentEnvStater^)r{rer-s r/r}z8MultiAgentEnvWrapper._init_env_state..s6$C)MNNr0)rrrBrmrr^)r-rs` r/rz$MultiAgentEnvWrapper._init_env_states| ;9DOOO c3'' ' ''#6 # D$$DOC r0FrA)$rgrhrirjrrmrrrqr(rr rrrrrrrnrrr rrprrrorCrDr"rkr#rrrXrr<r0r/rdrdsN16 "5"5C5'>*"5O,"5 "5 *. "5"5"5"5HXg@    @@@@*Xg5 55555nXg#'"#"& """"sm " $ " % l23 4 """"@XgVV(5/VTVVVV(Xg#888 tCL!4=0 18888Xg**%*D****  Xg.3:?...X. Xg)ci)))X)Xg,s7|,,,,8C=Dr0rdc eZdZddedefdZdeeeeeeffdZdeded ed ed ef d Z d d dde e de e deeeffdZ d S)rFrereturn_error_as_obsct|tsJ||_||_d|_i|_i|_ddi|_ddi|_i|_ dS)NFr) rBrrer initializedlast_obs last_rewardslast_terminatedslast_truncateds last_infos)r-rers r/r(z_MultiAgentEnvState.__init__sc#}-----#6   !*E 2 )51r0r4c|js|d|_|j}i}d|jdi}d|jdi}|j}|ds|dst |tr_|j}i|_|j}t |tr d|d<d|d<i|_|j}i|_i|_|j}i|_nz| D]e}||jvr|j|||<|j|=||jvr|j|||<|j|=||jvr|j|||<|j|=fd|jd<d|jd<|||||fS)NTrF) rr6rrrrrBrrr)r- observationsrrrrags r/rz_MultiAgentEnvState.polls $ JJLLL#D }  $"7 "BC !5i!@A   " 1)$ 1, 22 1 'G "D /K, 22 .)- I&(- 9%$&D !-J#%D DMOE DOO#'')) 1 1***"&"3B"7GBK)"-...&*&;B&?KO-b1---%)%9"%=JrN,R0+0i(*/Y'Wk:uDDr0rrrrrc||_|D].\}}||jvr|j|xx|z cc<$||j|</|D]0\}}||jvr|j|p||j|<&||j|<1|D]0\}} ||jvr|j|p| |j|<&| |j|<1||_dSrA)rrrrrr) r-rrrrrrrdts r/rz_MultiAgentEnvState.observes% ]]__ * *EBT&&&!"%%%*%%%%()!"%% &&(( . .EBT***,0,A",E,J%b)),-%b))%%'' - -EBT)))+/+?+C+Hq$R((+,$R((r0Nr1r2r3c> |j||}nJ#t$r=}|jr*t|jd||f}n|Yd}~nd}~wwxYw|\|_|_i|_ ddi|_ ddi|_ |j|jfS)Nr1rrF) rer6rrrrrrrrrr)r-r2r3 obs_and_infosrs r/r6z_MultiAgentEnvState.resets   HNNgNFFMM   '   +++ !1   *7& t!*E 2 )51}do--s A&3A!!A&r)rgrhrirrqr(rrrrrrmrnr6r<r0r/rrs  M     6E    6E6E6E6Ep    $  #       :#"& ...sm.$ . ~~- . ......r0r).r,loggingtypingrrrrrrr r gymnasiumrCnumpyrray._common.deprecationr ray.rllib.env.base_envr ray.rllib.env.env_contextr ray.rllib.utils.annotationsrrray.rllib.utils.typingrrrrrrray.utilrray.util.annotationsrr ENV_STATE getLoggerrgrEnvrrprrdrr<r0r/rsi JJJJJJJJJJJJJJJJJJJJ......******000000========88888888   8 $ $ VhhhhhCGhhhV FsJ/F /FFFFR \\\\\7\\ \~ s.s.s.s.s.s.s. s.s.s.r0