
    &`ih                     :   d Z ddlZddlZddlmZmZ ddlZddlZddl	m
Z
 ddlmZ ddlmZ ddlmZmZ ddlmZmZmZ dd	lmZ  e            \  ZZZ e            \  ZZ ej        e          Ze	 	 	 	 	 	 d+de
de de!de deee"                  dee         de!de"fd            Z#edd
ddedede"de!fd            Z$eddddd
dd
d
ddd
dde"de!fd             Z%e	 	 	 	 d,d!edeee                   d"e d#e d$e!d%ed&         fd'            Z&ed(e d)e de"fd*            Z'dS )-a6  RNN utils for RLlib.

The main trick here is that we add the time dimension at the last moment.
The non-LSTM layers of the model see their inputs as one flat batch. Before
the LSTM cell, we reshape the input to add the expected time dimension. During
postprocessing, we dynamically pad the experience batches so that this
reshaping is possible.

Note that this padding strategy only works out if we assume zero inputs don't
meaningfully affect the loss function. This happens to be true for all the
current algorithms: https://github.com/ray-project/ray/issues/2992
    N)ListOptional)SampleBatch)OldAPIStack)	summarize)try_import_tftry_import_torch)SampleBatchType
TensorTypeViewRequirementsDict)log_onceF   zerobatchmax_seq_lenshufflebatch_divisibility_reqfeature_keysview_requirements_enable_new_api_stackpaddingc                 h     j         rdS d _         |dk    rJt           t          j                           |z  dk    o"t	           t          j                           dk    }nd}d}	|rd v sd v r                     t          j                  }
t          j	         d                   rPt          fdD                       s
J d	            t          d                   t          |
          k    rd}	d}nd}n|std
 v sd v rl                     t          j                  8t           d
                   t           t          j                           k    rd}	|r|d
         j        d}n)d}n&|s|}d}| _        n|r                                  dS g }|pg }                                 D ]p\  }}|                    d          r|                    |           0|s>|s|                    d          s'n	 	 |t          j        fvr|                    |           qt#           fd|D              fd|D                                  t          j                                       t          j                                       t          j                                       t          j                  |||	|d||          \  }}}
t)          |          D ])\  }}t          j         |         ||                    |<   *t)          |          D ]\  }}||          |<   t-          j        |
           t          j        <   |rt	          |
           _        t1          d          rAt2                              d                    t9          |||
|d                               dS dS )a  Applies padding to `batch` so it's choppable into same-size sequences.

    Shuffles `batch` (if desired), makes sure divisibility requirement is met,
    then pads the batch ([B, ...]) into same-size chunks ([B, ...]) w/o
    adding a time dimension (yet).
    Padding depends on episodes found in batch and `max_seq_len`.

    Args:
        batch: The SampleBatch object. All values in here have
            the shape [B, ...].
        max_seq_len: The max. sequence length to use for chopping.
        shuffle: Whether to shuffle batch sequences. Shuffle may
            be done in-place. This only makes sense if you're further
            applying minibatch SGD after getting the outputs.
        batch_divisibility_req: The int by which the batch dimension
            must be dividable.
        feature_keys: An optional list of keys to apply sequence-chopping
            to. If None, use all keys in batch that are not
            "state_in/out_"-type keys.
        view_requirements: An optional Policy ViewRequirements dict to
            be able to infer whether e.g. dynamic max'ing should be
            applied over the seq_lens.
        _enable_new_api_stack: This is a temporary flag to enable the new RLModule API.
            After a complete rollout of the new API, this flag will be removed.
        padding: Padding type to use. Either "zero" or "last". Zero padding
            will pad with zeros, last padding will pad with the last value.
    NTr   r   Fstate_in	state_outc              3   d   K   | ]*}t          |          t          d                    k    V  +dS )r   N)len).0r   	state_inss     s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/policy/rnn_sequencing.py	<genexpr>z6pad_batch_to_sequences_of_same_size.<locals>.<genexpr>f   sK        7?HYq\!2!22         z9All state_in tensors should have the same batch_dim size.
state_in_0state_out_0c                      g | ]
}|         S  r%   r   kr   s     r   
<listcomp>z7pad_batch_to_sequences_of_same_size.<locals>.<listcomp>   s    999aq999r!   c                      g | ]
}|         S r%   r%   r&   s     r   r(   z7pad_batch_to_sequences_of_same_size.<locals>.<listcomp>   s    444AuQx444r!   )feature_columnsstate_columnsepisode_ids
unroll_idsagent_indicesseq_lensr   dynamic_maxstates_already_reduced_to_initr   handle_nested_datar   pad_infos_with_empty_dictsrnn_ma_feed_dictz'Padded input for RNN/Attn.Nets/MA:

{}
)featuresinitial_statesr/   r   )zero_paddedr   r   CUR_OBSmaxAGENT_INDEXgetSEQ_LENStreeflattenall
shift_fromr   r   items
startswithappendchop_into_sequencesEPS_ID	UNROLL_ID	enumerateunflatten_asnparrayr   loggerinfoformatr   )r   r   r   r   r   r   r   r   meets_divisibility_reqsr1   r/   r0   
state_keysfeature_keys_r'   vfeature_sequencesr6   ir   s   `                  @r   #pad_batch_to_sequences_of_same_sizerT   "   s   N  E!!k)*++.DDI 9E+1233q8 	  #'%*"  2*"5"59M9M 99[122
 Lz!233	 	     CL     K KJK K 
 9Q<  CMM1115. KKKK" %!7!7 99[)**63,<
 <
{+,--<. <. .2*  	 !2<!@!K!SKK  KK$ ,'  	MMOOO J &BM $ $1<<
## 	$a    	$6K	$\\+..	$ 	$QU	$ +.///  ###2E9999=9994444444IIk01199[233ii 788;/00'E#83 3 3/~x -(( E E1$U1X/@/CDDa*%% % %1!!$a"$(8"4"4E+
 *MM"## 
8??$5*8$,'2	  	 		
 	
 	
 	
 	

 
r!   tf)	framework
time_majorpadded_inputsr/   rV   rW   c                6   |dv r|du s
J d            t                               |           } t                               |           d         }t                               |          d         }||z  }t                               t                               |d          t                               |d          t                               |           dd         gd          }t                               | |          S |dk    r|t                              |           } | j        d         }|j        d         }||z  }||f| j        dd         z   }|                     |          }	|r|		                    dd          }	|	S |d	k    sJ d

                    |                      t          j        |           } | j        d         }|j        d         }||z  }||f| j        dd         z   }|                     |          }	|r|		                    dd          }	|	S )a  Adds a time dimension to padded inputs.

    Args:
        padded_inputs: a padded batch of sequences. That is,
            for seq_lens=[1, 2, 2], then inputs=[A, *, B, B, C, C], where
            A, B, C are sequence elements and * denotes padding.
        seq_lens: A 1D tensor of sequence lengths, denoting the non-padded length
            in timesteps of each rollout in the batch.
        framework: The framework string ("tf2", "tf", "torch").
        time_major: Whether data should be returned in time-major (TxB)
            format or not (BxT).

    Returns:
        TensorType: Reshaped tensor of shape [B, T, ...] or [T, B, ...].
    )tf2rU   Fz$time-major not supported yet for tf!r   axisr   NtorchrI   zUnknown framework: {})rU   convert_to_tensorshapeconcatexpand_dimsreshaper]   	as_tensorview	transposerM   rI   asarray)
rX   r/   rV   rW   padded_batch_sizenew_batch_size	time_size	new_shapebatch_major_shapepadded_outputss
             r   add_time_dimensionrm      s.   6 M!!U"""$J""",,];;HH]33A6(++A.%7	II~A66yq11''+
   
 
	 zz-333	g		66)/2 "*%7	+Y7-:Mabb:QQ&++,=>> 	<+55a;;ND   "9"@"@"K"K   
=11)/2 "*%7	+Y7-:Mabb:QQ&../@AA 	<+55a;;Nr!   T)r,   r-   r.   r0   r   r/   r1   r2   _extra_paddingr   r3   r3   c                 P	   |t          |          dk    rd}g }d}t          j        t          j        ||          t          j        |t          j                  dz            }|D ].}|||k    s||k    r|                    |           d}|dz  }|}/|r|                    |           t          j        |t          j                  }|rt          |          |z   }t          |          |z  }g }| D ]}t          |t                    rt          j        |          }|                    g            t          j        |          D ]u}t          |t          j                  st          j        |          }|j        t          k    r7|r5t          |d         t                    rd t!          |          D             }nh|j        t          k    s|j        j        t          j        u rdg|z  }n9t          j        |ft          j        |          dd         z   |j                  }d}d}|D ]V}t!          |          D ]}||         |||z   <   |dz  }|dk    r$t!          ||          D ]}||dz
           |||z   <   ||z  }W|t          |          k    s
J |            |d                             |           w|	r|}ng }|D ]}t          |t                    rt          j        |          }g }t          j        |          D ]}t          |t          j                  st          j        |          }g }d}|D ]"}|                    ||                    ||z  }#|                    t          j        |                     |                    t          j        ||                     |rt          j                            t          |                    } t1          t          j        |                    D ]b\  }}|j        }!t          j        |t          |          df|j        dd         z             }||          }t          j        ||!          }|||<   ct1          |          D ]\  }}||          }|||<   ||          }|
sd	 |D             }|||fS )
a   Truncate and pad experiences into fixed-length sequences.

    Args:
        feature_columns: List of arrays containing features.
        state_columns: List of arrays containing LSTM state values.
        max_seq_len: Max length of sequences. Sequences longer than max_seq_len
            will be split into subsequences that span the batch dimension
            and sum to max_seq_len.
        episode_ids (List[EpisodeID]): List of episode ids for each step.
        unroll_ids (List[UnrollID]): List of identifiers for the sample batch.
            This is used to make sure sequences are cut between sample batches.
        agent_indices (List[AgentID]): List of agent ids for each step. Note
            that this has to be combined with episode_ids for uniqueness.
        dynamic_max: Whether to dynamically shrink the max seq len.
            For example, if max len is 20 and the actual max seq len in the
            data is 7, it will be shrunk to 7.
        shuffle: Whether to shuffle the sequence outputs.
        handle_nested_data: If True, assume that the data in
            `feature_columns` could be nested structures (of data).
            If False, assumes that all items in `feature_columns` are
            only np.ndarrays (no nested structured of np.ndarrays).
        _extra_padding: Add extra padding to the end of sequences.
        padding: Padding type to use. Either "zero" or "last". Zero padding
            will pad with zeros, last padding will pad with the last value.
        pad_infos_with_empty_dicts: If True, will zero-pad INFOs with empty
            dicts (instead of None). Used by the new API stack in the meantime,
            however, as soon as the new ConnectorV2 API will be activated (as
            part of the new API stack), we will no longer use this utility function
            anyway.

    Returns:
        f_pad: Padded feature columns. These will be of shape
            [NUM_SEQUENCES * MAX_SEQ_LEN, ...].
        s_init: Initial states for each sequence, of shape
            [NUM_SEQUENCES, ...].
        seq_lens: List of sequence lengths, of shape [NUM_SEQUENCES].

    .. testcode::
        :skipif: True

        from ray.rllib.policy.rnn_sequencing import chop_into_sequences
        f_pad, s_init, seq_lens = chop_into_sequences(
            episode_ids=[1, 1, 5, 5, 5, 5],
            unroll_ids=[4, 4, 4, 4, 4, 4],
            agent_indices=[0, 0, 0, 0, 0, 0],
            feature_columns=[[4, 4, 8, 8, 8, 8],
                             [1, 1, 0, 1, 1, 0]],
            state_columns=[[4, 5, 4, 5, 5, 5]],
            max_seq_len=3)
        print(f_pad)
        print(s_init)
        print(seq_lens)


    .. testoutput::

        [[4, 4, 0, 8, 8, 8, 8, 0, 0],
         [1, 1, 0, 0, 1, 1, 0, 0, 0]]
        [[4, 4, 5]]
        [2, 3, 1]
    Nr   )dtype    r   c                     g | ]}i S r%   r%   )r   _s     r   r(   z'chop_into_sequences.<locals>.<listcomp>  s    333333r!   lastc                     g | ]
}|d          S )r   r%   )r   fs     r   r(   z'chop_into_sequences.<locals>.<listcomp>  s    ===aQqT===r!   )r   rI   addrJ   int64rC   int32r9   
isinstancelistr=   r>   ndarrayrp   objectdictrangetypestr_zerosr_   rH   randompermutationrG   rb   )"r*   r+   r   r,   r-   r.   r0   r   r/   r1   r2   rn   r   r3   prev_idseq_len
unique_idsuidlengthrR   colrw   f_padseq_baserS   len_
seq_offsetr6   state_columninitial_state_flatss_initr   
orig_shapes"                                     r   rD   rD     s   ` 3x==A--VF;..HZrx000B6
 

  	 	C#w7k;Q;Q(((qLGGG 	%OOG$$$8HBH555  5(mmn4]][(F '0 '0c4   	 (3--C  $$$c"" "	0 "	0Aa,,  HQKK
 6!!. "qtT** " 43U6]]333F""aglbg&=&= &RXa[[_!<AGLLLHA  	( 	("'++  J34Q4E(Z/0FAAf$$&+D+&>&> @ @
78Qxh344K'A;;;;;;b!((////E"	0H & W&) 	W 	WL,-- 6!x55!#\,// 	< 	<!!RZ00 $A$  DMM!A$'''IAA"))"(6*:*:;;;;!!$"3LBT"U"UVVVV )i++CMM::dl+<==>> 	% 	%DAqJ
1s8}}b1AGABBK?@@A+A
1j))A#$a  n-- 	" 	"DAq+A !N1K(  >==+<===nh66r!   sample_batchzero_pad_max_seq_lenpre_overlapzero_init_statesreturnr   c                    | |                      t          j                  }nH|                      t          j                  )t          d          rt                              d           |q||z
  }t          d          r/t                              d                    |||                     t          t          |           |          \  }}|g|z  |r|gng z   }|t          |          dk    s
J d            d}g }	|D ]+}
||z
  }|}||
z   |		                    ||f           ||
z  },g }|	D ]\  }dd|}dk     r||d}n\| t          j
                 dk    rnd         }||d	         k    }|d         s&t          t          d
|z
                      z   d}#fd|                                 D             }n!fd|                                 D             }|rd}d                    |          }||v rlt          j        | |         dd                   ||<   |                    d                    |          d           |dz  }d                    |          }||v lnxd}d                    |          }||v r]| d                    |                   dz
           ||<   |d                    |          = |dz  }d                    |          }||v ]|	                    t          |z
  g                     |dk    r|D ]}|                    |d           |S )a  Slices batch along `seq_lens` (each seq-len item produces one batch).

    Args:
        sample_batch: The SampleBatch to timeslice.
        seq_lens (Optional[List[int]]): An optional list of seq_lens to slice
            at. If None, use `sample_batch[SampleBatch.SEQ_LENS]`.
        zero_pad_max_seq_len: If >0, already zero-pad the resulting
            slices up to this length. NOTE: This max-len will include the
            additional timesteps gained via setting pre_overlap (see Example).
        pre_overlap: If >0, will overlap each two consecutive slices by
            this many timesteps (toward the left side). This will cause
            zero-padding at the very beginning of the batch.
        zero_init_states: Whether initial states should always be
            zero'd. If False, will use the state_outs of the batch to
            populate state_in values.

    Returns:
        List[SampleBatch]: The list of (new) SampleBatches.

    Examples:
        assert seq_lens == [5, 5, 2]
        assert sample_batch.count == 12
        # self = 0 1 2 3 4 | 5 6 7 8 9 | 10 11 <- timesteps
        slices = timeslice_along_seq_lens_with_overlap(
            sample_batch=sample_batch.
            zero_pad_max_seq_len=10,
            pre_overlap=3)
        # Z = zero padding (at beginning or end).
        #             |pre (3)|     seq     | max-seq-len (up to 10)
        # slices[0] = | Z Z Z |  0  1 2 3 4 | Z Z
        # slices[1] = | 2 3 4 |  5  6 7 8 9 | Z Z
        # slices[2] = | 7 8 9 | 10 11 Z Z Z | Z Z
        # Note that `zero_pad_max_seq_len=10` includes the 3 pre-overlaps
        #  count (makes sure each slice has exactly length 10).
    N!overriding_sequencing_informationz~Found sequencing information in a batch that will be ignored when slicing. Ignore this warning if you know what you are doing..no_sequence_lengths_available_for_time_slicingzTrying to slice a batch along sequences without sequence lengths being provided in the batch. Batch will be sliced into slices of size {} = {} - {} = zero_pad_max_seq_len - pre_overlap.r   zCCannot timeslice along `seq_lens` when `seq_lens` is empty or None!Tru   g      ?c                     i | ]^\  }}|t           j        k    |t          j        t          j        f|j        d d         z   |j                  |         g          _S )r   N)r_   rp   )r   r<   rI   concatenater   r_   rp   )r   r'   rQ   
data_beginendzero_lengths      r   
<dictcomp>z9timeslice_along_seq_lens_with_overlap.<locals>.<dictcomp>-  s}     	 	 	 Aq,,, 2>~'C17SSS*S.)  -,,r!   c                 L    i | ] \  }}|t           j        k    ||         !S r%   )r   r<   )r   r'   rQ   beginr   s      r   r   z9timeslice_along_seq_lens_with_overlap.<locals>.<dictcomp>8  s?       Aq,,, 1U3Y<,,,r!   zstate_in_{}r   zstate_out_{})r/   )r   exclude_states)r;   r   r<   r   rK   warningrM   divmodr   rC   rE   intsumrA   rI   
zeros_likepopright_zero_pad)r   r/   r   r   r   r   num_seq_lenslast_seq_lenstartslicesr   	pre_beginslice_begin
timesliceszero_init_states_eps_idsis_last_episode_idsdatarS   keytsr   r   r   r   s                        @@@@r   %timeslice_along_seq_lens_with_overlapr     sM   V ##K$899K011=(/C
 C
= NN&   *[8DEE 	NNE FLV!5{F F	   &,C,=,={%K%K"l()L8*2\NN

 	X!2!2!2L "3!22 EF  K'	goy+s3444J#) 6E 6E{C
,199%K$J $";#56

uuPQTW7WXG")WR["8&q) )!#c,?&?"@"@AA"[0
$(!"	 	 	 	 	 	 )..00	 	 	DD    (..00  D  	.A&&q))C++M,s*;AaC*@AAS	..q114888Q#**1-- ++ A&&q))C++()>)>q)A)AB519uCTUS	..q112Q#**1--	 ++ 	+dcEk]CCCDDDD a 	U 	UB*>tTTTTr!   b_dimt_dimc                 X    |dv rAt                                           t                                          fd} fd}n&|dk    r fd} fd}nt          d| d          t          j        t
          j        |          t          j        t
          j        |          fS )	aS  Produces two functions to fold/unfold any Tensors in a struct.

    Args:
        b_dim: The batch dimension to use for folding.
        t_dim: The time dimension to use for folding.
        framework: The framework to use for folding. One of "tf2" or "torch".

    Returns:
        fold: A function that takes a struct of torch.Tensors and reshapes
            them to have a first dimension of `b_dim * t_dim`.
        unfold: A function that takes a struct of torch.Tensors and reshapes
            them to have a first dimension of `b_dim` and a second dimension
            of `t_dim`.
    rZ   c                     | | S t                               |           } t                               |           }|dd          }t                               | t                               z  g|gd                    S )N   r   r[   rU   r^   r_   rb   r`   itemr_   
other_dimsr   r   s      r   fold_mappingz)get_fold_unfold_fns.<locals>.fold_mappingq  sl    |''--DHHTNNEqrrJ::dBII
.KRSI$T$TUUUr!   c                     | | S t                               |           } | j        }|dd          }t                               | t                               gg|gd                    S )Nr   r   r[   r   r   s      r   unfold_mappingz+get_fold_unfold_fns.<locals>.unfold_mappingz  sd    |''--DJEqrrJ::dBIIw.LSTI$U$UVVVr!   r]   c                 Z   | | S t                               |           } t          |                                           }t          |d d                   \  }}f||fk    s"J d                    f||f                      |dd          }|                     z  g|z             S )Nr   zVAll tensors in the struct must have the same batch and time dimensions. Got {} and {}.r]   rc   r|   sizerM   rb   )r   r   current_b_dimcurrent_t_dimr   r   r   s        r   r   z)get_fold_unfold_fns.<locals>.fold_mapping  s    |??4((D		$$D+/RaR>>(M=5>m]%CCCC--3VEN]M$B. . DCC abbJ<<* <===r!   c                 .   | | S t                               |           } t          |                                           }|d         }|dd          }|z  k    s!J d                    |z                        |                     g|z             S )Nr   r   zwThe first dimension of the tensor must be equal to the product of the desired batch and time dimensions. Got {} and {}.r   )r   r   r   r   r   r   s       r   r   z+get_fold_unfold_fns.<locals>.unfold_mapping  s    |??4((D		$$D GMabbJ EEM111HHN!55=I I 211 << ;<<<r!   z
framework z not implemented!)rU   r^   
ValueError	functoolspartialr=   map_structure)r   r   rV   r   r   s   ``   r   get_fold_unfold_fnsr   \  s%     E$$U++$$U++	V 	V 	V 	V 	V 	V	W 	W 	W 	W 	W 	W 	W 
g			> 	> 	> 	> 	> 	>$	= 	= 	= 	= 	= 	= 	=  BiBBBCCCT/>>	@QNA A  r!   )Fr   NNFr   )Nr   r   T)(__doc__r   loggingtypingr   r   numpyrI   r=   ray.rllib.policy.sample_batchr   ray.rllib.utils.annotationsr   ray.rllib.utils.debugr   ray.rllib.utils.frameworkr   r	   ray.rllib.utils.typingr
   r   r   ray.utilr   tf1rU   tfvr]   rs   	getLogger__name__rK   r   boolstrrT   rm   rD   r   r   r%   r!   r   <module>r      sB         ! ! ! ! ! ! ! !      5 5 5 5 5 5 3 3 3 3 3 3 + + + + + + E E E E E E E E T T T T T T T T T T      }Rq		8	$	$  "#(,8<"'[
 [
[
[
 [
  	[

 49%[
   45[
  [
 [
 [
 [
 [
| 
 F F FF F 	F
 F F F FR  #(',x7 x7 x7 x7 !%x7 x7 x7 x7v  %) !!Q Q!QtCy!Q Q 	Q
 Q 
-Q Q Q Qh Ns N3 N3 N N N N N Nr!   