&`iP?ddlZddlmZmZddlmZddlmZmZm Z ddl m Z ddl m Z ddlmZmZe Gdd Ze Gd d eZGd d eZe GddZe GddZe GddZe GddZdS)N)ListOptional) DataIterator)MultiAgentBatch SampleBatchconcat_samples)unflatten_dict) DeveloperAPI) DeviceType EpisodeTypec <eZdZdZdddddededed ed ed d f d Zd S)MiniBatchIteratorBasez+The base class for all minibatch iterators.Tr num_epochsshuffle_batch_per_epochnum_total_minibatchesbatchrrminibatch_sizerreturnNcdS)aInitializes a MiniBatchIteratorBase instance. Args: batch: The input multi-agent batch. num_epochs: The number of complete passes over the entire train batch. Each pass might be further split into n minibatches (if `minibatch_size` provided). The train batch is generated from the given `episodes` through the Learner connector pipeline. minibatch_size: The size of minibatches to use to further split the train batch into per epoch. The train batch is generated from the given `episodes` through the Learner connector pipeline. num_total_minibatches: The total number of minibatches to loop through (over all `num_epochs` epochs). It's only required to set this to != 0 in multi-agent + multi-GPU situations, in which the MultiAgentEpisodes themselves are roughly sharded equally, however, they might contain SingleAgentEpisodes with very lopsided length distributions. Thus, without this fixed, pre-computed value, one Learner might go through a different number of minibatche passes than others causing a deadlock. N)selfrrrrrs s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/utils/minibatch_utils.py__init__zMiniBatchIteratorBase.__init__s 8 )__name__ __module__ __qualname____doc__rintboolrrrrrr s}55 (,%&      "&     #        rrc LeZdZdZdddddededed ed ed d f fd ZdZxZ S)MiniBatchCyclicIteratorawThis implements a simple multi-agent minibatch iterator. This iterator will split the input multi-agent batch into minibatches where the size of batch for each module_id (aka policy_id) is equal to minibatch_size. If the input batch is smaller than minibatch_size, then the iterator will cycle through the batch until it has covered `num_epochs` epochs. rTrrrrrrrrNcFt||||||_||_||_||_d|jD|_d|jD|_ d|_ ||_ dS)z/Initializes a MiniBatchCyclicIterator instance.)rrrci|]}|dSrr.0mids r z4MiniBatchCyclicIterator.__init__..OsEEE#sAEEErci|]}|dSr'rr(s rr+z4MiniBatchCyclicIterator.__init__..Qs#R#R#RsC#R#R#RrrN) superr_batch_minibatch_size _num_epochs_shuffle_batch_per_epochpolicy_batcheskeys_start_num_covered_epochs_minibatch_count_num_total_minibatches)rrrrrr __class__s rrz MiniBatchCyclicIterator.__init__8s  !)$;     -%(?%FE)=)B)B)D)DEEE #R#Re6J6O6O6Q6Q#R#R#R !&;###rc#&K|jdkr/t|j|jks|jdkrD|j|jkr5i}|jjD]\}}t|dkrtd|d|j |}|j }g}|j r]|tj Jdd}t#|||j t|z z}nd}||z||kr||d}||||} | dks Jd|| z}d}|j|xxdz cc<|jr|||z||k||z} | |kr|||| t+|||<| |j |<t-|t|j}|V|xjdz c_|jdkr0t|j|jk|jdkr|j|jk1dSdSdSdS) NrzThe batch for module_id zt is empty! This will create an infinite loop because we need to cover the same number of samples for each module_id.z}MiniBatchCyclicIterator requires SampleBatch.SEQ_LENSto be present in the batch for slicing a batch in the batch dimension B.c@t|tjSN)lenrSEQ_LENSbs rget_lenz1MiniBatchCyclicIterator.__iter__..get_lens"1[%9#:;;;rc t|Sr;r<r>s rr@z1MiniBatchCyclicIterator.__iter__..get_lens "1vv rzLength of a sample must be > 0!r)r7minr5valuesr0r6r.r2itemsr< ValueErrorr4r/_slice_seq_lens_in_Bgetrr=r!appendr1shufflerr) r minibatch module_id module_batchsn_stepssamples_to_concatr@sample len_samplees r__iter__z MiniBatchCyclicIterator.__iter__Vs< +q0007799::T=MMM+a//)D,GGGI+/;+E+K+K+M+MF +F +' <|$$))$I9III K *.$&!  4&'++K,@AAMM'NMM <<<" --/#l2C2CCEGG&&&'kWW\%:%:::)!""-F%,,V444!(J%>>>+L>>>z)GA,Y7771<7774/$,,...'kWW\%:%:::Kq55%,,\!A#->???(66G'H'H )$)* I&& ( 3t{3C3CDDIOOO  ! !Q & ! !q+q0007799::T=MMM+a//)D,GGGGG0/GG0/r) rrrr rr!r"rrT __classcell__r8s@rr$r$.s(,%&<<<< <  < "& < #< <<<<<<<]']']']']']']'rr$c*eZdZdeffd ZdZxZS)MiniBatchDummyIteratorrc Jtj|fi|||_dSr;)r-rr.)rrkwargsr8s rrzMiniBatchDummyIterator.__init__s-))&))) rc#K|jVdSr;)r.)rs rrTzMiniBatchDummyIterator.__iter__skr)rrrrrrTrUrVs@rrXrXsSorrXc >eZdZdedededeefdZdefdZ dS) MiniBatchRayDataIteratoriteratordevicer num_itersc ||_d|D|_|jjd||d|j|_t |j|_||_dS)Nc&i|]\}}|dk ||S) return_stater)r)kvs rr+z5MiniBatchRayDataIterator.__init__..s(OOOA1;N;N1;N;N;Nr) batch_sizer_r) _iteratorrE_kwargsiter_torch_batches_batched_iterableiter_epoch_iterator _num_iters)rr^r_rr`rZs rrz!MiniBatchRayDataIterator.__init__s"OOOOO "C!B" %" " l" "  $D$:;;#rrc #Kd}|j ||jkr|jD]}|dz }t|}td|Dt d|D}|V|jr ||jkrn#t|j|_|jsdS|j||jkdSdS)Nrrc4i|]\}}|t|Sr)r)r)rL module_datas rr+z5MiniBatchRayDataIterator.__iter__..s62I{";{#;#;rc 3K|]?}ttt|V@dSr;)r<nextrkrD)r)rps r z4MiniBatchRayDataIterator.__iter__..sZ""'Dk&8&8&:&:!;!;<<==""""""r) env_steps) rmrlr rrEsumrDrkrj)r iterationrs rrTz!MiniBatchRayDataIterator.__iter__s! o%T_)D)D-  Q &u--'6;kkmm"""+0<<>>"""    ?yDO'C'CE(,D,B'C'C$E9o%T_)D)D)D)D)D)DrN) rrrrr r!rrrrTrrrr]r]so$$ $  $ C= $$$$0/rr]c(eZdZdZdedefdZdZdS)ShardBatchIteratorzIterator for sharding batch into num_shards batches. Args: batch: The input multi-agent batch. num_shards: The number of shards to split the batch into. Yields: A MultiAgentBatch of size len(batch) / num_shards. r num_shardsc"||_||_dSr;)r. _num_shards)rrrys rrzShardBatchIterator.__init__s %rc#Kt|jD]}i}|jjD]z\}}t jt||jz }||z}t||zt|}|t|t|||<{t|t|}|VdSr;) ranger{r.r2rEmathceilr<rCr!r) ri batch_to_sendpid sub_batchrfstartend new_batchs rrTzShardBatchIterator.__iter__ st'((  A M"&+"<"B"B"D"D F FY!Ys9~~8H'HII "Q%*,c)nn==%.s5zzCHH/D%E c""' s:GGIOOOO  rN)rrrr rr!rrTrrrrxrxsO&o&3&&&&rrxcZeZdZdZ d deededeefdZdeefdZ dS) ShardEpisodesIteratorzMIterator for sharding a list of Episodes into `num_shards` lists of Episodes.Nepisodesrylen_lookback_buffercFt|td|_||_||_t d|D|_dt|jD|_|j}t|jD]}|||z z}||j|<||z}dS)aInitializes a ShardEpisodesIterator instance. Args: episodes: The input list of Episodes. num_shards: The number of shards to split the episodes into. len_lookback_buffer: An optional length of a lookback buffer to enforce on the returned shards. When spitting an episode, the second piece might need a lookback buffer (into the first piece) depending on the user's settings. T)keyreversec34K|]}t|VdSr;rB)r)rSs rrsz1ShardEpisodesIterator.__init__..2s( : :AQ : : : : : :rcg|]}dSr'rr)_s r z2ShardEpisodesIterator.__init__..3sCCCaCCCrN) sortedr< _episodesr{_len_lookback_bufferru _total_lengthr}_target_lengths)rrryrremaining_lengthrNlen_s rrzShardEpisodesIterator.__init__s  c4@@@%$7! : : : : :::CC51A+B+BCCC-t'(( % %A# Q7D&*D  #  $   % %rrc#Kdt|jD}dt|jD}d}|t|jkrh|j|}|t |}||t|z|j|kr>|||||xxt|z cc<|dz }n|j|||z }|dkrw|d||t|d|j }}|||||xxt|z cc<||j|<n(|dksJ||||dz }|t|jkh|D]} | VdS)a2Runs one iteration through this sharder. Yields: A sub-list of Episodes of size roughly `len(episodes) / num_shards`. The yielded sublists might have slightly different total sums of episode lengths, in order to not have to drop even a single timestep. cg|]}gSrrrs rrz2ShardEpisodesIterator.__iter__..Bs8881B888rcg|]}dSr'rrs rrz2ShardEpisodesIterator.__iter__..Cs6661666rrrN)r) r}r{r<rindexrCrrIslicer) rsublistslengths episode_indexepisode min_indexr slice_partremaining_partsublists rrTzShardEpisodesIterator.__iter__:s98d&6 7 788866eD$455666 c$.1111n]3G c'll33Iy!CLL0D4H4SSS#**7333 """c'll2"""" $(#7 #BWYEW#W #a'' 1!1 12 !"2D99040I& !/JY'..z:::I&&&#j//9&&&4BDN=11+q0000Y'..w777!Q&M=c$.1111@   GMMMM  rr;) rrrr rr r!rrrTrrrrrs}WW .2 %%{#%%&c] %%%%6-${+------rrc$eZdZdZdefdZdZdS)ShardObjectRefIteratora,Iterator for sharding a list of ray ObjectRefs into num_shards sub-lists. Args: object_refs: The input list of ray ObjectRefs. num_shards: The number of shards to split the references into. Yields: A sub-list of ray ObjectRefs with lengths as equal as possible. ryc"||_||_dSr;) _object_refsr{)r object_refsrys rrzShardObjectRefIterator.__init__vs'%rc#Kt|j}||jz}||jz}d}t|jD]%}||z||krdndz}|j||V|}&dS)Nrr)r<rr{r})rn sublist_sizeremaining_elementsrrrs rrTzShardObjectRefIterator.__iter__zs ! " "D,, !11t'((  A,&q3E/E/E!!1MC#E#I. . . .EE   rN)rrrr r!rrTrrrrrjsH&&&&&     rr)r~typingrrray.datarray.rllib.policy.sample_batchrrrray.rllib.utilsr ray.rllib.utils.annotationsr ray.rllib.utils.typingr r rr$rXr]rxrrrrrrs !!!!!!!!!!!!!!VVVVVVVVVV******444444::::::::        DD'D'D'D'D'3D'D'D'N277777777t@KKKKKKKK\r