
     `iJa                        d dl mZmZmZ d dlZd dlmZ ddlmZ ddl	m
Z
mZ ddlmZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-  ej.        e/          Z0 G d de          Z1 G d de*          Z2 G d de'          Z3 G d de+          Z4	 	 	 d3dej5        dej6        dej6        dej6        deej6                 de7d ee7         d!ee7         d"e8ej6        ej6        f         fd#Z9 G d$ d%e#          Z: G d& d'e          Z; G d( d)e)          Z< G d* d+e(          Z= G d, d-e$          Z> G d. d/e%          Z? G d0 d1e&          Z@g d2ZAdS )4    )CallableOptionalUnionN   )ACT2FN)CacheDynamicCache)PretrainedConfiglayer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)deprecate_kwarg   )GemmaAttentionGemmaForCausalLMGemmaForSequenceClassificationGemmaForTokenClassificationGemmaMLP
GemmaModelGemmaPreTrainedModelGemmaRMSNormGemmaRotaryEmbeddingapply_rotary_pos_emb	repeat_kvc                        e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d# fd"	Z xZ	S )$Gemma2Configa  
    This is the configuration class to store the configuration of a [`Gemma2Model`]. It is used to instantiate an Gemma2
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the Gemma2-7B.
    e.g. [google/gemma2-7b](https://huggingface.co/google/gemma2-7b)
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Gemma2 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`Gemma2Model`]
        hidden_size (`int`, *optional*, defaults to 2304):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 9216):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 26):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*, defaults to 4):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        head_dim (`int`, *optional*, defaults to 256):
            The attention head dimension.
        hidden_activation (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
            The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"`
            if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        eos_token_id (`int`, *optional*, defaults to 1):
            End of stream token id.
        bos_token_id (`int`, *optional*, defaults to 2):
            Beginning of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        query_pre_attn_scalar (`float`, *optional*, defaults to 256):
            scaling factor used on the attention scores
        sliding_window (`int`, *optional*, defaults to 4096):
            in Gemma2, every other layer uses sliding window attention. This is the size of the sliding window.
        layer_types (`list`, *optional*):
            Attention pattern for each layer.
        final_logit_softcapping (`float`, *optional*, defaults to 30.0):
            scaling factor when applying tanh softcapping on the logits.
        attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
            scaling factor when applying tanh softcapping on the attention scores.

    ```python
    >>> from transformers import Gemma2Model, Gemma2Config
    >>> # Initializing a Gemma2 gemma2-7b style configuration
    >>> configuration = Gemma2Config()
    >>> # Initializing a model from the gemma2-7b style configuration
    >>> model = Gemma2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```gemma2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm   	   $              gelu_pytorch_tanh    {Gz?ư>Tr      r        @F           N      >@      I@c                     t                      j        d||||d| || _        |	| _        || _        || _        || _        || _        || _        || _	        |
| _
        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        | j        #d t-          | j                  D             | _        t/          | j        | j                   d S )N)pad_token_idbos_token_ideos_token_idtie_word_embeddingsc                 @    g | ]}t          |d z   dz            rdndS )r;   r   sliding_attentionfull_attention)bool).0is     }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/gemma2/modular_gemma2.py
<listcomp>z)Gemma2Config.__init__.<locals>.<listcomp>   sA          STtQUaK'8'8N##>N          )super__init__
vocab_sizemax_position_embeddingshidden_sizeintermediate_sizenum_hidden_layersnum_attention_headshead_dimnum_key_value_headsinitializer_rangerms_norm_eps	use_cache
rope_thetaattention_biasattention_dropouthidden_activationquery_pre_attn_scalarsliding_windowfinal_logit_softcappingattn_logit_softcappinglayer_typesranger   )selfrR   rT   rU   rV   rW   rY   rX   r`   rS   rZ   r[   r\   rB   rD   rC   rE   r]   r^   r_   ra   rb   re   rc   rd   kwargs	__class__s                             rL   rQ   zGemma2Config.__init__   s2   8 	 	
%%% 3		
 	

 	
 	
 	
 %'>$&!2!2#6  #6 !2("$,!2!2%:",'>$&<#&#   X]^b^tXuXu     D 	d.0FGGGGGrN   )r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   Tr   r;   r   Tr<   Fr=   r6   r>   Nr?   r@   )
__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planrQ   __classcell__ri   s   @rL   r$   r$   2   s       J JX J#4"5%.%.%.%."+ )"+  &(9:#%568IJ!"_$56  - $ ! $#3<H <H <H <H <H <H <H <H <H <HrN   r$   c                       e Zd ZdS )Gemma2RMSNormNrj   rk   rl   rO   rN   rL   ru   ru              DrN   ru   c                        e Zd Z fdZ xZS )	Gemma2MLPc                 x    t                                          |           t          |j                 | _        d S N)rP   rQ   r   r`   act_fnrg   configri   s     rL   rQ   zGemma2MLP.__init__   s.       V56rN   )rj   rk   rl   rQ   rr   rs   s   @rL   ry   ry      s8        7 7 7 7 7 7 7 7 7rN   ry   c                       e Zd ZdS )Gemma2RotaryEmbeddingNrv   rO   rN   rL   r   r      rw   rN   r   r=   modulequerykeyvaluer,   dropoutscalingsoftcapreturnc                    |
| j         dz  }t          || j                  }	t          || j                  }
t          j        ||	                    dd                    |z  }|||z  }t          j        |          }||z  }|$|d d d d d d d |	j        d         f         }||z   }t          j	        
                    |dt          j                                      |j                  }t          j	                            ||| j                  }t          j        ||
          }|                    dd                                          }||fS )	N      r   r   )dimdtype)ptrainingr;   )rX   r"   num_key_value_groupstorchmatmul	transposetanhshapenn
functionalsoftmaxfloat32tor   r   r   
contiguous)r   r   r   r   r,   r   r   r   rh   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                 rL   eager_attention_forwardr      sR    /4'3 ;<<JUF$?@@L<z';';Aq'A'ABBWLL#g-z,//#g-!$QQQ111.D
0@0D.D%DE#k1 =((2U](SSVVW\WbccL=((6?([[L,|\::K''1--88::K$$rN   c                   @    e Zd Zdedef fdZ eddd          	 	 dd	ej        d
e	ej        ej        f         de
ej                 de
e         de
ej                 dee         de	ej        e
ej                 e
e	ej                          f         fd            Z xZS )Gemma2Attentionr~   	layer_idxc                     t                                          ||           | j        j        | _        | j        j        | _        d| _        |j        dz  | _        |j        |         dk    r|j	        nd | _	        d S )NTr   rG   )
rP   rQ   r~   rd   r_   	is_causalra   r   re   rb   rg   r~   r   ri   s      rL   rQ   zGemma2Attention.__init__  sw    +++&*k&H#!%!>3T97=7I)7TXk7k7kf33qurN   past_key_valuer&   4.58new_nameversionNr+   position_embeddingsr,   cache_positionrh   r   c                 \   |j         d d         }g |d| j        R }|                     |                              |                              dd          }	|                     |                              |                              dd          }
|                     |                              |                              dd          }|\  }}t          |	|
||          \  }	}
|&|||d}|                    |
|| j	        |          \  }
}t          }| j        j        dk    rt          | j        j                 } || |	|
||f| j        r| j        nd| j        | j        | j        d|\  }} |j        g |dR                                  }|                     |          }||fS )Nr   r;   r   )sincosr   eagerr=   )r   r   rb   r   )r   rX   q_projviewr   k_projv_projr!   updater   r   r~   _attn_implementationr   r   r_   r   rb   rd   reshaper   o_proj)rg   r+   r   r,   r&   r   rh   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsattention_interfacer   r   s                     rL   forwardzGemma2Attention.forward	  s    $)#2#.88b8$-88{{=1166|DDNNqRSTT[[//44\BBLLQPQRR
{{=1166|DDNNqRSTT&S#7jRUWZ#[#[ j&#&snUUL'6'='=j,X\Xfht'u'u$J(?;+w66"9$+:Z"[$7$7%
 /3mDD**L./%
 %
 %
 %
!\ *k);;;;;;FFHHkk+..L((rN   )NN)rj   rk   rl   r$   intrQ   r   r   Tensortupler   r   
LongTensorr   r   r   rr   rs   s   @rL   r   r      s#       v| v v v v v v v _%0A6RRR ,059+) +)|+) #5<#=>+) !.	+)
 "%+) !!12+) -.+) 
u|Xel3XeEL>Q5RR	S+) +) +) SR+) +) +) +) +)rN   r   c                   h    e Zd Zdedef fdZ eddd          	 	 	 	 	 	 dd
ej        de	ej        ej        f         de
ej                 de
ej                 de
e         de
e         de
e         de
ej                 de	ej        e
e	ej        ej        f                  f         fd            Z xZS )Gemma2DecoderLayerr~   r   c                    t                                                       |j        | _        || _        |j        |         | _        t          ||          | _        t          |          | _	        t          |j        |j                  | _        t          |j        |j                  | _        t          |j        |j                  | _        t          |j        |j                  | _        d S )N)r~   r   )eps)rP   rQ   rT   r~   re   attention_typer   	self_attnry   mlpru   r[   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr   s      rL   rQ   zGemma2DecoderLayer.__init__9  s    !-$0;()LLLV$$,V-?VEXYYY(5f6HfNa(b(b(b%)6v7IvOb)c)c)c&*78JPVPc*d*d*d'''rN   r   r&   r   r   NFr+   r   r,   position_idsoutput_attentionsr\   r   r   c	                 <   |}
|                      |          } | j        d||||||||d|	\  }}|                     |          }|
|z   }|}
|                     |          }|                     |          }|                     |          }|
|z   }|f}|r||fz  }|S )N)r+   r   r,   r   r&   r   r\   r   rO   )r   r   r   r   r   r   )rg   r+   r   r,   r   r&   r   r\   r   rh   residualself_attn_weightsoutputss                rL   r   zGemma2DecoderLayer.forwardF  s     !,,];; ,:4> 
,
' 3)%+/)
,
 
,
 
,
 
,
(( 55mDD =0 66}EE//77FF =0 " 	,)++GrN   )NNNFFN)rj   rk   rl   r$   r   rQ   r   r   r   r   r   r   r   rI   FloatTensorr   rr   rs   s   @rL   r   r   8  sN       e| e e e e e e e _%0A6RRR
 2637+/,1$)59* *|* #5<#=>* !.	*
 u/0* "%* $D>* D>* !!12* 
u (51BEDU1U+V"WW	X* * * SR* * * * *rN   r   c                       e Zd ZdS )Gemma2PreTrainedModelNrv   rO   rN   rL   r   r   t  rw   rN   r   c                       e Zd Zdef fdZ	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 dee	         deej
                 d	ee         d
ee         dee         deej                 dee         defdZ xZS )Gemma2Modelr~   c                     t                                                     t          j        fdt	          j                  D                       | _        d S )Nc                 0    g | ]}t          |          S rO   )r   )rJ   r   r~   s     rL   rM   z(Gemma2Model.__init__.<locals>.<listcomp>|  s$    dddy	22dddrN   )rP   rQ   r   
ModuleListrf   rV   r.   r}   s    `rL   rQ   zGemma2Model.__init__y  sU       mddddE&JbDcDcddd
 
rN   Nr)   r,   r   r&   r*   r\   r   output_hidden_statesr   rh   r   c
                 8   ||n| j         j        }||n| j         j        }||n| j         j        }|d u |d uz  rt	          d          | j        r%| j        r|rt                              d           d}|| 	                    |          }|r|| j        st          | j                   }|	B||                                nd}t          j        |||j        d         z   |j                  }	||	                    d          }t#          |x}t$                    s'| j         |||	||d}t'          di |t)          di |d	}|}|                     ||          }t          j        | j         j        d
z  |j                  }||z  }|rdnd }|rdnd }| j        d | j         j                 D ]<}|r||fz  } ||f|||j                 |||||	d|
}|d         }|r||d         fz  }=|                     |          }|r||fz  }t;          ||||          S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)r~   r   r;   )device)r~   input_embedsr,   r   r&   r   )rH   rG   g      ?)r   rO   )r   r,   r   r&   r   r\   r   )last_hidden_stater&   r+   
attentions)r~   r   r   r\   
ValueErrorgradient_checkpointingr   loggerwarning_oncer-   r	   get_seq_lengthr   aranger   r   	unsqueeze
isinstancedictr   r   
rotary_embtensorrT   r   r.   rV   r   r/   r   )rg   r)   r,   r   r&   r*   r\   r   r   r   rh   past_seen_tokenscausal_mask_mappingmask_kwargsr+   r   
normalizerall_hidden_statesall_self_attnsdecoder_layerlayer_outputss                        rL   r   zGemma2Model.forward  s    2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	-t";< 	[YZZZ& 	4= 	Y 	j   I  --i88M 	?00*$+>>>O!CRC^==???de"\ "2]5H5K"KTaTh  N )33A66L ?-FF 	 + -"0"0#2 , K #5"C"C{"C"C%F%U%U%U%U# # & #oom\JJ
 \$+"93">mFYZZZ
%
2 #7@BBD0:d![)H4;+H)HI 	6 	6M# 6!m%55!)M
$72=3OP) /"3#-
 
 
 
M *!,M  6=#3"55		-00 	2-!11&+++%	
 
 
 	
rN   )	NNNNNNNNN)rj   rk   rl   r$   rQ   r   r   r   r   r   r   rI   r   r   r   r   rr   rs   s   @rL   r   r   x  s7       
| 
 
 
 
 
 
 151537+/59$(,0/359k
 k
E,-k
 !.k
 u/0	k

 "%k
   12k
 D>k
 $D>k
 'tnk
 !!12k
 +,k
 
!k
 k
 k
 k
 k
 k
 k
 k
rN   r   c                   <    e Zd Z fdZ	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 dee         deej	                 d	eej                 d
ee
         dee
         dee
         deej                 deeej        f         defdZ xZS )Gemma2ForCausalLMc                     t                                          |           t          |          | _        |                                  d S r{   )rP   rQ   r   model	post_initr}   s     rL   rQ   zGemma2ForCausalLM.__init__  s@        ((
rN   Nr   r)   r,   r   r&   r*   labelsr\   r   r   r   logits_to_keepr   c                    ||n| j         j        }|	|	n| j         j        }	 | j        d||||||||	|
d	|}|j        }t          |t                    rt          | d          n|}|                     |dd|ddf                   }| j         j	        2|| j         j	        z  }t          j        |          }|| j         j	        z  }d}| | j        ||| j        fi |}t          |||j        |j        |j                  S )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma2ForCausalLM

        >>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```N)	r)   r,   r   r&   r*   r\   r   r   r   )losslogitsr&   r+   r   rO   )r~   r   r   r   r   r   r   slicelm_headrc   r   r   loss_functionrR   r   r&   r+   r   )rg   r)   r,   r   r&   r*   r   r\   r   r   r   r   rh   r   r+   slice_indicesr  r   s                     rL   r   zGemma2ForCausalLM.forward  sb   B 2C1N--TXT_Tq$8$D  $+Jj 	 ,64: ,
)%+'/!5),
 ,
 ,
 ,
  18B>SV8W8Wk~ot444]kmAAA}aaa,?@AA;.:dkAAFZ''FdkAAF%4%ffdoPPPPD%#3!/)
 
 
 	
rN   )NNNNNNNNNNr   )rj   rk   rl   rQ   r   r   r   r   r   r   rI   r   r   r   r   rr   rs   s   @rL   r   r     sJ            151537+/59-1$(,0/35934F
 F
E,-F
 !.F
 u/0	F

 "%F
   12F
 )*F
 D>F
 $D>F
 'tnF
 !!12F
 c5</0F
 
 F
 F
 F
 F
 F
 F
 F
 F
rN   r   c                       e Zd ZdS )Gemma2ForSequenceClassificationNrv   rO   rN   rL   r  r  <  rw   rN   r  c                       e Zd ZdS )Gemma2ForTokenClassificationNrv   rO   rN   rL   r	  r	  @  rw   rN   r	  )r$   r   r   r   r  r	  )r=   NN)Btypingr   r   r   r   torch.nnr   activationsr   cache_utilsr   r	   configuration_utilsr
   r   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   utils.deprecationr   gemma.modeling_gemmar   r   r   r   r   r   r   r   r    r!   r"   
get_loggerrj   r   r$   ru   ry   r   Moduler   floatr   r   r   r   r   r   r   r  r	  __all__rO   rN   rL   <module>r     si    - , , , , , , , , ,        ! ! ! ! ! ! . . . . . . . . J J J J J J J J R R R R R R R R B B B B B B 9 9 9 9 9 9 O O O O O O O O 5 5 5 5 5 5 & & & & & & 0 0 0 0 0 0 0 0 0 0 0 0 0 0                          
	H	%	%ZH ZH ZH ZH ZH# ZH ZH ZHz	 	 	 	 	L 	 	 	7 7 7 7 7 7 7 7	 	 	 	 	0 	 	 	 ## %  %I %< % 
 % <	 %
 U\* %  % e_ % e_ % 5<%& %  %  %  %F5) 5) 5) 5) 5)n 5) 5) 5)p9 9 9 9 93 9 9 9x	 	 	 	 	0 	 	 	r
 r
 r
 r
 r
* r
 r
 r
jL
 L
 L
 L
 L
( L
 L
 L
^	 	 	 	 	&D 	 	 		 	 	 	 	#> 	 	 	  rN   