
    &`i1G                        d Z ddlmZmZmZ ddlZddlZddl	Z	ddl
mZmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/  e$            \  Z0Z1e! G d dee1j2                              Z3 G d dee1j2                  Z4dS )a  
[1] - Attention Is All You Need - Vaswani, Jones, Shazeer, Parmar,
      Uszkoreit, Gomez, Kaiser - Google Brain/Research, U Toronto - 2017.
      https://arxiv.org/pdf/1706.03762.pdf
[2] - Stabilizing Transformers for Reinforcement Learning - E. Parisotto
      et al. - DeepMind - 2019. https://arxiv.org/pdf/1910.06764.pdf
[3] - Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context.
      Z. Dai, Z. Yang, et al. - Carnegie Mellon U - 2019.
      https://www.aclweb.org/anthology/P19-1285.pdf
    )DictOptionalUnionN)BoxDiscreteMultiDiscrete)deprecation_warning)ModelV2)SlimFC)GRUGateRelativeMultiHeadAttentionSkipConnection)RecurrentNetwork)TorchModelV2)SampleBatch)ViewRequirement)OldAPIStackoverride)try_import_torch)get_base_struct_from_space)flatten_inputs_to_1d_tensorone_hot)ListModelConfigDict
TensorType)log_oncec                   d    e Zd ZdZdddddddddd	ej        j        d
ej        j        dee         de	de
dedededededededef fdZ ee          dee         dedeee         ffd            Z ee          deej                 fd            Z ee          defd            Z xZS )GTrXLNetaI  A GTrXL net Model described in [2].

    This is still in an experimental phase.
    Can be used as a drop-in replacement for LSTMs in PPO and IMPALA.

    To use this network as a replacement for an RNN, configure your Algorithm
    as follows:

    Examples:
        >> config["model"]["custom_model"] = GTrXLNet
        >> config["model"]["max_seq_len"] = 10
        >> config["model"]["custom_model_config"] = {
        >>     num_transformer_units=1,
        >>     attention_dim=32,
        >>     num_heads=2,
        >>     memory_tau=50,
        >>     etc..
        >> }
       @      2       g       @)num_transformer_unitsattention_dim	num_headsmemory_inferencememory_traininghead_dimposition_wise_mlp_diminit_gru_gate_biasobservation_spaceaction_spacenum_outputsmodel_confignamer$   r%   r&   r'   r(   r)   r*   r+   c                b   t                                          |||||           t          j                            |            || _        || _        || _        |	| _        |
| _        || _	        |d         | _
        |j        d         | _        t          | j        | j                  | _        | j        g| _        g }t!          | j                  D ]}t#          t%          | j        | j        ||dt          j                  t)          | j        |                    }t#          t                              t,          j                            | j                  t          | j        |dt          j                  t          || j        dt          j                            t)          | j        |                    }|                    ||g           t          j        | | _        | j                            |           d	| _        d	| _        d	| _        | j        Ht          | j        | j        t          j        
          | _        t          | j        dd	
          | _        n| j        | _        t!          | j                  D ]}t=          dd| j        f          }t?          d                     |          d                     | j                  | j
        |          | j!        d                     |          <   t?          |d          | j!        d                     |          <   d	S )a  Initializes a GTrXLNet.

        Args:
            num_transformer_units: The number of Transformer repeats to
                use (denoted L in [2]).
            attention_dim: The input and output dimensions of one
                Transformer unit.
            num_heads: The number of attention heads to use in parallel.
                Denoted as `H` in [3].
            memory_inference: The number of timesteps to concat (time
                axis) and feed into the next transformer unit as inference
                input. The first transformer unit will receive this number of
                past observations (plus the current one), instead.
            memory_training: The number of timesteps to concat (time
                axis) and feed into the next transformer unit as training
                input (plus the actual input sequence of len=max_seq_len).
                The first transformer unit will receive this number of
                past observations (plus the input sequence), instead.
            head_dim: The dimension of a single(!) attention head within
                a multi-head attention unit. Denoted as `d` in [3].
            position_wise_mlp_dim: The dimension of the hidden layer
                within the position-wise MLP (after the multi-head attention
                block within one Transformer unit). This is the size of the
                first of the two layers within the PositionwiseFeedforward. The
                second layer always has size=`attention_dim`.
            init_gru_gate_bias: Initial bias values for the GRU gates
                (two GRUs per Transformer unit, one after the MHA, one after
                the position-wise MLP).
        max_seq_lenr   )in_sizeout_sizeT)in_dimout_dimr&   r)   input_layernormoutput_activation)fan_in_layerF)r3   r4   use_biasactivation_fnN)r3   r4   r;   r   g      g      ?)shapezstate_out_{}-{}:-1)shiftbatch_repeat_valuespacestate_in_{})r@   used_for_training)"super__init__nnModuler$   r%   r&   r'   r(   r)   r2   r<   obs_dimr   linear_layerlayersranger   r   ReLUr   
Sequentialtorch	LayerNormextendattention_layerslogits
values_out
_value_outr.   r   r   formatview_requirements)selfr,   r-   r.   r/   r0   r$   r%   r&   r'   r(   r)   r*   r+   rP   i	MHA_layerE_layerr@   	__class__s                      x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/rllib/models/torch/attention_net.pyrD   zGTrXLNet.__init__>   sO   \ 	|[,	
 	
 	
 		4   %:"*" 0. '6(.q1"4<$BTUUU()t122 #	: #	:A&*- .'%$(&(g   %T%79KLL
 
 
I %H&&t'9:: $ 2!6!&&(g	    5!%!3!&&(g	    %T%79KLL!  G( ##Y$89999 !#/? @+,,, ' *) g  DK %*Qd  DOO  $1D t122 	 	Ac$*<)>???E>M%%a((ood&;<<#'#3? ? ?D"=#7#7#:#:; @Ou@ @ @D">#8#8#;#;<<	 	    stateseq_lensreturnc           	          |J |t           j                 }t          |          }|j        d         |z  }t                              |d|gt          |j        dd                    z             }|}g }t          t           j                            D ]X}	|	dz  dk    r"  j        |	         |||	dz                     }-  j        |	         |          }|	                    |           Y|d d         } j
        7 
                    |          }
                     |           _         j        }n	|}
 j        }t                              |
d|g           fd|D             fS )Nr   r   r!   )memoryc                 T    g | ]$}t                               |d j        g          %S )ra   )rM   reshaper%   ).0mrV   s     r[   
<listcomp>z$GTrXLNet.forward.<locals>.<listcomp>   s<     3
 3
 3
;<EMM!b$"45663
 3
 3
r\   )r   OBSlenr<   rM   rd   listrJ   rI   appendrQ   rR   rS   r.   r%   )rV   
input_dictr]   r^   observationsBTall_outmemory_outsrW   outr6   s   `           r[   forwardzGTrXLNet.forward   s    ### "+/2MMq!Q&}}2q'D);ABB)?$@$@@
 
 s4;''(( 		, 		,A1uzz($+a.qAvGGG
 )$+a.11""7++++ "#2#&;"++g&&C"oog66DO&GGC(G}}S2w-00 3
 3
 3
 3
@K3
 3
 3
 
 	
r\   c                     g S )N rV   s    r[   get_initial_statezGTrXLNet.get_initial_state   s    	r\   c                 f    | j         
J d            t                              | j         dg          S )Nz3Must call forward first AND must have value branch!ra   )rS   rM   rd   rv   s    r[   value_functionzGTrXLNet.value_function   s5     O''@ (''}}T_rd333r\   )__name__
__module____qualname____doc__gymspacesSpacer   intr   strfloatrD   r   r
   r   r   rs   r   npndarrayrw   ry   __classcell__rZ   s   @r[   r   r   (   s        8 &' "!%'$'M M M:+M j&M c]	M
 &M M  #M M M M M M  #M "M M M M M M^ Xg)
!%j!1)
=G)

d:&	')
 )
 )
 )
X X4
#3      Xg4
 4 4 4 4 4 4 4 4r\   r   c                   T    e Zd ZdZdej        j        dej        j        dedede	f
 fdZ
 ee          dee	ef         d	ee         d
edeee         ffd            Z ee          deeej                 ee         f         fd            Z ee          defd            Z xZS )AttentionWrapperzGGTrXL wrapper serving as interface for ModelV2s that set use_attention.	obs_spacer-   r.   r/   r0   c                 n   t          d          rt          d           t          j                            |            t                                          ||d ||           |d         | _        |d         | _        t          | j	                  | _
        d| _        t          j        | j
                  D ]}t          |t                    r| xj        |j        z  c_        -t          |t"                    r(| xj        t%          j        |j                  z  c_        j|j        5| xj        t-          t%          j        |j                            z  c_        | xj        t-          t1          |                    z  c_        | j        r| xj        | j        | j        z  z  c_        | j        r| xj        | j        z  c_        |}|d         | _        | j        Nt6          j                            t=          d          t=          d	          | j        ft$          j        
          }n|}tA          ||d |d|d         | j        |d         |d         |d         |d         |d         |d                   | _!        || _        tE          | j        | j        d tF          j        j$        j%                  | _&        tE          | j        dd tF          j        j$        j%                  | _'        | j!        j(        | _(        | j)        | j(        d         _*        | j        rKtW          tX          j-        | j	        d.                    | j                            | j(        tX          j/        <   | j        rGtW          tX          j0        d.                    | j                            | j(        tX          j1        <   d S d S )N!deprecate_attention_wrapper_torchz5ray.rllib.models.torch.attention_net.AttentionWrapper)oldattention_use_n_prev_actionsattention_use_n_prev_rewardsr   r%   z-infinf)r<   dtypegtrxlattention_num_transformer_unitsattention_num_headsattention_head_dimattention_memory_inferenceattention_memory_trainingattention_position_wise_mlp_dimattention_init_gru_gate_bias)r$   r%   r&   r)   r'   r(   r*   r+   )r3   r4   r;   initializerr   obsr=   )r@   r>   )r>   )2r   r	   rE   rF   rD   rC   use_n_prev_actionsuse_n_prev_rewardsr   r-   action_space_struct
action_dimtreeflatten
isinstancer   nr   r   sumnvecr<   r   prodri   r.   r%   r~   r   r   r   float32r   r   r   rM   initxavier_uniform__logits_branch_value_branchrU   r   r@   r   r   ACTIONSrT   PREV_ACTIONSREWARDSPREV_REWARDS)
rV   r   r-   r.   r/   r0   r@   cfgin_spacerZ   s
            r[   rD   zAttentionWrapper.__init__	  s    788 	K    		4   L$dKKK"./M"N"./M"N#=d>O#P#P \$":;; 	3 	3E%** 357*E=11 326%*#5#55(3rwu{';';#<#<<3s5zz??2 " 	J 7$/ II" 	8 77 1'z~~fuU||D4D3Fbj &  HH !H
 "%&G"H,/0-. !=> ;<"%&G"H"#AB
 
 

" ' %&%5	
 
 
 $&5	
 
 
 "&!=.2nu%+ " 	?N#'ood&=>>@ @ @D";#;<
 " 	?N#8??4;R+S+S@ @ @D";#;<<<	 	r\   rl   r]   r^   r_   c                    |J |                      |g d           \  }}g }| j        r|t          j                 }| j        d         rVt          || j        d          }t                              ||j	        d         dg          }|
                    |           ndt          | j        t                    r]t          | j                  D ]G}	|
                    t          |d d |	f                                         | j                             Hnt          | j        t"                    rt          d| j        | j        j	        d                   D ]\}	|
                    t          |d d |	|	| j        j	        d         z   f                                         | j                             ]nO|
                    t                              |                                d| j        | j        z  g                     | j        rW|
                    t                              |t          j                                                 d| j        g                     |r t                              |g|z   d          }|x|d	<   |d
<   |                     |||          \  | _        }
|                     | j                  }||
fS )N_disable_action_flatteningT)spaces_struct	time_axisr   ra   )r@   r   )dimobs_flatr   )_wrapped_forwardr   r   r   r/   r   r   rM   rd   r<   rk   r   r-   r   rJ   r   r   r   r   r   r   catr   	_featuresr   )rV   rl   r]   r^   wrapped_out_prev_a_rprev_n_actionsflatrW   rq   	model_outs               r[   rs   zAttentionWrapper.forwardn  s    ###..z2tDDQ  " ,	'(@AN  !=> '2""&":"   }}TDJqM2+>??%%%%
 d/:: "4#:;;   # .qqq!t 4 : : < <DDU        1=AA "42D4E4KA4N  
 
 !# .$%AAq1t/@/Fq/I+I'I$I!""'%''&*&7	     
 OO*0022!84?!JK    " 	OO{78>>@@01     	E))[MH$<!)DDK 6A@
:E!2&*jjUH&M&M#''77	+%%r\   c                 N      fdt           j        j                  D             S )Nc                     g | ]I}t                               j        j        d                     |                   j        j                  JS )rA   )rM   zerosr   rU   rT   r@   r<   )re   rW   rV   s     r[   rg   z6AttentionWrapper.get_initial_state.<locals>.<listcomp>  sZ     
 
 
  KK
,]-A-A!-D-DEKQ 
 
 
r\   )rJ   r   r$   rv   s   `r[   rw   z"AttentionWrapper.get_initial_state  s=    
 
 
 
 4:;<<	
 
 
 	
r\   c                     | j         
J d            t                              |                     | j                   dg          S )NzMust call forward() first!ra   )r   rM   rd   r   rv   s    r[   ry   zAttentionWrapper.value_function  s?    ~))+G)))}}T//??"FFFr\   )rz   r{   r|   r}   r~   r   r   r   r   r   rD   r   r   r   r   r   rs   r
   r   r   r   rw   ry   r   r   s   @r[   r   r     s       QQc:#c j&c 	c
 &c c c c c c cJ XM&j)M& JM& 	M&
 d:&	'M& M& M&  M&^ Xg
5bj)94
;K)K#L 
 
 
 
 XgG
 G G G G G G G Gr\   r   )5r}   typingr   r   r   	gymnasiumr~   numpyr   r   gymnasium.spacesr   r   r   ray._common.deprecationr	   ray.rllib.models.modelv2r
   ray.rllib.models.torch.miscr   ray.rllib.models.torch.modulesr   r   r   $ray.rllib.models.torch.recurrent_netr   $ray.rllib.models.torch.torch_modelv2r   ray.rllib.policy.sample_batchr   !ray.rllib.policy.view_requirementr   ray.rllib.utils.annotationsr   r   ray.rllib.utils.frameworkr   "ray.rllib.utils.spaces.space_utilsr   ray.rllib.utils.torch_utilsr   r   ray.rllib.utils.typingr   r   r   ray.utilr   rM   rE   rF   r   r   ru   r\   r[   <module>r      sG  	 	 ) ( ( ( ( ( ( ( ( (          9 9 9 9 9 9 9 9 9 9 7 7 7 7 7 7 , , , , , , . . . . . .         
 B A A A A A = = = = = = 5 5 5 5 5 5 = = = = = = = = = = = = = = 6 6 6 6 6 6 I I I I I I L L L L L L L L D D D D D D D D D D      	r Z4 Z4 Z4 Z4 Z4 Z4 Z4 Z4zDG DG DG DG DG|RY DG DG DG DG DGr\   