&`izl\ddlZddlZddlZddlmZmZmZmZmZm Z m Z m Z ddl Z ddlZddlZddlmZddlmZmZddlmZddlmZddlmZmZddlmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%erdd l&m'Z'ej(ej(ej)ej)ej*ej*ej+ej+ej,ej,ej-ej-ej.ej.ej/ej/ej0ej0ej1ej1ej2ej2d d ddddiZ3ej4e5Z6e%dGddZ7dS)N) TYPE_CHECKINGAnyDictListOptionalSetTupleUnion)Columns) MultiRLModuleMultiRLModuleSpec)SingleAgentEpisode) flatten_dict)OverrideToImplementCustomLogic5OverrideToImplementCustomLogic_CallToSuperRecommended)unpack_if_needed) ReplayBuffer) EpisodeTypeModuleID) PublicAPI)AlgorithmConfig agent_indexdones unroll_idalpha) stabilityc4eZdZdZedddddddeeejejfdee dee e e fd e e e ff d Zed e e ejfd e e ejffd Zed efdZed e e e ffdZdeed eefdZdd efdZeeedddddfded e e ee ejffde e e fdedeee deedejdejd e e e fd e e ee!ffdZ"eeeddfded e e ee ejffde e e fdedeee d e e ee!ff dZ#dS)OfflinePreLearnera?Class that coordinates data transformation from dataset to learner. This class is an essential part of the new `Offline RL API` of `RLlib`. It is a callable class that is run in `ray.data.Dataset.map_batches` when iterating over batches for training. It's basic function is to convert data in batch from rows to episodes (`SingleAGentEpisode`s for now) and to then run the learner connector pipeline to convert further to trainable batches. These batches are used directly in the `Learner`'s `update` method. The main reason to run these transformations inside of `map_batches` is for better performance. Batches can be pre-fetched in `ray.data` and therefore batch trransformation can be run highly parallelized to the `Learner''s `update`. This class can be overridden to implement custom logic for transforming batches and make them 'Learner'-ready. When deriving from this class the `__call__` method and `_map_to_episodes` can be overridden to induce custom logic for the complete transformation pipeline (`__call__`) or for converting to episodes only ('_map_to_episodes`). Custom `OfflinePreLearner` classes can be passed into `AlgorithmConfig.offline`'s `prelearner_class`. The `OfflineData` class will then use the custom class in its data pipeline. N)spaces module_spec module_stateconfigrrr r!kwargsc 8||_|jj|_|jj|_||_|j||pd\|_|_|j|j|j|_ |jj |_ |j |_ d|_|js |js|jr6|jjp|j}|j|jjz}|di||_dSdS)N)NN)input_observation_spaceinput_action_spacer)r"input_read_episodesinput_read_sample_batchesbuild_module set_stateobservation_space action_spacebuild_learner_connector_learner_connectorpolicies_to_train_policies_to_trainis_multi_agent_is_multi_agentiter_since_last_module_update is_statefulprelearner_buffer_classdefault_prelearner_buffer_class default_prelearner_buffer_kwargsprelearner_buffer_kwargsepisode_buffer)selfr"rr r!r#r7r:s x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/offline/offline_prelearner.py__init__zOfflinePreLearner.__init__QsM(. )-)H /3{/T&&1&7&7&9&9  |,,,5;4Jl1 1#'+"E"E$($:#0#F# #  #'+"?%+%:23*  * |'')) '  387 $ 5+67 % #:"9##*##D     batchreturnc |jrddlddlfd|dD}||}|j||j|jj|j r |jj ddnd|j ddpddd }nG|j rt|j|dt"|jjz|jj d }||}|j||j|jj|j r |jj ddnd|j ddpddd }nM||j|t"|jjzd |jj|j|j d }||j i|id}|D]}|||s||=t3|S)aqPrepares plain data batches for training with `Learner`'s. Args: batch: A dictionary of numpy arrays containing either column data with `self.config.input_read_schema`, `EpisodeType` data, or `BatchType` data. Returns: A `MultiAgentBatch` that can be passed to `Learner.update` methods. rNclg|]0}tj|j1S)) object_hook)r from_stateunpackbdecode).0statemnpmsgpacks r= z.OfflinePreLearner.__call__..sM222#-OOEszOBB222r?item max_seq_lenn_stepT) num_itemsbatch_length_TrOsample_episodesto_numpy)rTschemainput_compress_columnsepisodesF)rUrTrVr-r.) rl_moduler@rW shared_datametrics)r(rK msgpack_numpy_validate_episodesr;addsampler"train_batch_size_per_learnerr+r6 model_configgetr)r_map_sample_batch_to_episoder4SCHEMAinput_read_schemarV_map_to_episodesr-r.r0_should_module_be_updatedr)r<r@rW module_idrJrKs @@r=__call__zOfflinePreLearner.__call__s  #C  NNN ' ' ' '22222#6] 222H..x88H   # #H - - -*11+B<++-- t{7;;M1MMM{x338q!%2  HH +& ">>$ =='+{'I ? ..x88H   # #H - - -*11+B<++-- t{7;;M1MMM{x338q!%2  HH261F1F$ =='+{'I"&"8!.2G222HL''l(   % %I11)UCC %)$E"""r?cddlm}|S)zSets the default replay buffer.r)EpisodeReplayBuffer)4ray.rllib.utils.replay_buffers.episode_replay_bufferrj)r<rjs r=r8z1OfflinePreLearner.default_prelearner_buffer_class s(      #"r?c:|jjdz|jjdS)zSets the default arguments for the replay buffer. Note, the `capacity` might vary with the size of the episodes or sample batches in the offline dataset. )capacity batch_size_B)r"r_)r<s r=r9z2OfflinePreLearner.default_prelearner_buffer_kwargss( @2E KD   r?rWc.td|Dstdt}t}|D]M}|j|vrB|j|jjvr/||j||N|S)aValidate episodes sampled from the dataset. Note, our episode buffers cannot handle either duplicates nor non-ordered fragmentations, i.e. fragments from episodes that do not arrive in timestep order. Args: episodes: A list of `SingleAgentEpisode` instances sampled from a dataset. Returns: A set of `SingleAgentEpisode` instances. Raises: ValueError: If not all episodes are `done`. c3$K|] }|jV dSN)is_done)rHepss r= z7OfflinePreLearner._validate_episodes..6s$3333;333333r?zWhen sampling from episodes (`input_read_episodes=True`) all recorded episodes must be done (i.e. either `terminated=True`) or `truncated=True`).)all ValueErrorsetid_r;episode_id_to_indexr])r<rWunique_episode_idscleaned_episodesrts r=r\z$OfflinePreLearner._validate_episodes"s(33(33333 ( !UU55 * *C111G4#6#JJJ"&&sw/// $$S)))r?c|jsdSt|js|t|jvS|||S)z:Checks which modules in a MultiRLModule should be updated.T)r2callablerx)r<rgmulti_agent_batchs r=rfz+OfflinePreLearner._should_module_be_updatedIsT& I4$122 ID$; < << <**96GHH Hr?Fr3rUrTrVignore_final_observationr-r.c pgg} t|tjD]\} |rMtj|vr|tjnd|vr|dnd} nd} |rZtjvrt | n| } |r)t jdtj| } nWtjvr+t |tj n|tj } ttj |vr+t|tj ntjj| | | gitj|vr|tjnigtjvr+t |tjn|tjg|tjg|tj|vrtjndtj|vr|tjndfd|Dd }|r|| |d | iS) z!Maps a batch of data to episodes.rNc d|zS)Nrr')xs r=z4OfflinePreLearner._map_to_episodes..s !a%r?rFci|]F\}}|vr=|vr'|dv#||vrt|n|gGS)rrtypevaluesrrHkvirVrUs r= z6OfflinePreLearner._map_to_episodes..s ) ) ) !AqVOO ! 8 8 !)I I I $:::-QqT222!"1!J I Ir?r ryagent_id observationsinfosactionsrewards terminated truncatedextra_model_outputslen_lookback_bufferrW) enumerater OBSAGENT_IDrtree map_structurecopydeepcopyNEXT_OBSrEPS_IDstruuiduuid4hexINFOSACTIONSREWARDS TERMINATEDS TRUNCATEDSitemsrTappend)r3r@rUrTrVrr-r.r#rWobsr unpacked_obsunpacked_next_obsepisoders ` ` @r=rez"OfflinePreLearner._map_to_episodesSsB "8!=2fW[&9 :;;c )c )FAs '500&!123A66 "-0E99f]34Q77! P ) {&<<<%S))) , (,(:'|)D)D))%% #;*@@@)vg6F/G)H)KLLL"6'*:#; #6'/#:;A>?$!'"56%??w233$    g01U::$F7+=$>?BB ) ) ) ) ) ) %*KKMM ) ) ))*k666p'$$&&&((((H%%r?c  pgg}t|tjD]P\ |r'd|vr|d dnd}nd}|r4t tr6t  fdt jdD nt tj rJtjvr!t n n tdt dtj |vr} tjvr1t |tj  dn#|tj  dn  dtj|vr]tj|vrI|tj d}|tj d}ntj|vr;tj|vr'|tj d}d}nntj|vr;tj|vr'|tj d}d}nd |vr|d  d}d}nd }d}t#tj|vr1t |tj dnt'jj| tj|vr|tj nigt/ ztjvr+t |tj n|tj |tj || fd |Dd } |r| || Rd |iS)z6Maps an old stack `SampleBatch` to new stack episodes.rrNc$g|] }|df S).r')rHrrs r=rLzBOfflinePreLearner._map_sample_batch_to_episode..s!DDD13q#v;DDDr?zUnknown observation type: z. When mapping from old recorded `SampleBatches` batched observations should be either of type `np.array` or - if the column is compressed - of `str` type.FdoneTci|]E\}}|vr<|vr&|dv#||vrt|n|FSrrrs r=rzBOfflinePreLearner._map_sample_batch_to_episode..Zs{ ) ) )!AqVOO ! 8 8 !)I I I 666,AaD111qT !J I Ir?rrW)rr r isinstancerrrangeshapenpndarraytolist TypeErrorrrrrrrrrrrrlenrrrrT) r3r@rUrTrVrWrrrrrrs ` ` @@r=rbz.OfflinePreLearner._map_sample_batch_to_episodes"8!=2fW[&9 :;;I )I )FAs m,55&/03A66  { ) c3''+3//CEDDDcil0C0CDDDCC RZ00 "'+.2HHH)666 ZZ\\C $LT#YYLLL'*+u44JJ!'+.2HHH)vg6F/G)H)KB)OPPP"6'*:#;!&"9:1=)' ) ) ) ) ) )%*KKMM ) ) ))*S***\'$$&&&((((H%%r?rr)$__name__ __module__ __qualname____doc__rrr gymSpacer rrrrr>rrrrhpropertyrr8r9rrrr\boolrf staticmethodrcr listrrerbr'r?r=rr5s_4; 9=376: 888"8sy#)345 8 /0 8 tHcM23 8sCx.888;:8t$}#d3 ?3}#S"*_8M}#}#}#$#}#~####X#  $sCx.    X  % /0%  % % % % NIIdIIII$"(6:38'+"&v&v&v&CtRZ/001v&S#Xv& v& !)c 3 v& #+4. v&9v&iv&sCx.v& c4 $$ %v&v&v&\$#v&p$"(6: _&_&_&CtRZ/001_&S#X_& _& !)c 3 _& c4 $$ % _&_&_&\$#_&_&_&r?r)8rloggingrtypingrrrrrrr r gymnasiumrnumpyrrray.rllib.core.columnsr (ray.rllib.core.rl_module.multi_rl_moduler r "ray.rllib.env.single_agent_episoderray.rllib.utilsrray.rllib.utils.annotationsrrray.rllib.utils.compressionr,ray.rllib.utils.replay_buffers.replay_bufferrray.rllib.utils.typingrrray.util.annotationsr%ray.rllib.algorithms.algorithm_configrrr MODULE_IDrrrrrrrTrc getLoggerrloggerrr'r?r=rsM  NNNNNNNNNNNNNNNNNNNN ******UUUUUUUUAAAAAA((((((988888EEEEEE88888888******FEEEEEE NGN g& w( K OW_ OW_ M7= g& , * Iwy= W $  8 $ $ Wx&x&x&x&x&x&x&x&x&x&r?