
     `i                     *   d dl mZmZmZmZ d dlZd dlmc mZ	 d dlmZ ddl
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(  e#            rd dl)m*Z* ddl+m,Z,  e$j-        e.          Z/ G d ded          Z0 G d dej1                  Z2 G d dej1                  Z3 G d dej1                  Z4 G d dej1                  Z5 G d  d!ej1                  Z6d" Z7dBd#Z8d$ej9        d%e:d&ej9        fd'Z;	 dCd)ej1        d*ej9        d+ej9        d,ej9        d-eej9                 d.e<d/e<fd0Z= G d1 d2ej1                  Z> G d3 d4e          Z?e" G d5 d6e                      Z@ G d7 d8ej1                  ZAe" G d9 d:e@                      ZB	 	 	 dDd<eej9        eCej9                 df         d=ee:         d-eej9                 d&eej9        e:f         fd>ZD G d? d@e@e          ZEg dAZFdS )E    )CallableOptional	TypedDictUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)GradientCheckpointingLayer)BaseModelOutputWithPastMoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringis_torch_flex_attn_availablelogging)deprecate_kwarg   )GraniteMoeSharedConfig)	BlockMask)make_flex_block_causal_maskc                   d    e Zd ZU dZej        ed<   ej        ed<   eed<   eed<   ej        ed<   dS )GraniteFlashAttentionKwargsa  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    Attributes:
        cu_seq_lens_q (`torch.LongTensor`)
            Gets cumulative sequence length for query state.
        cu_seq_lens_k (`torch.LongTensor`)
            Gets cumulative sequence length for key state.
        max_length_q (`int`):
            Maximum sequence length for query state.
        max_length_k (`int`):
            Maximum sequence length for key state.
        seq_idx (`torch.IntTensor):
            Index of each packed sequence.
    cu_seq_lens_qcu_seq_lens_kmax_length_qmax_length_kseq_idxN)	__name__
__module____qualname____doc__torch
LongTensor__annotations__int	IntTensor     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/granitemoeshared/modeling_granitemoeshared.pyr    r    3   sb          " ########_r0   r    F)totalc                   L     e Zd ZdZdef fdZdej        dej        fdZ xZ	S )GraniteMoeSharedMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    configc                 D   t                                                       |j        | _        |j        | _        t
          |j                 | _        t          j	        | j        | j        dz  d          | _
        t          j	        | j        | j        d          | _        d S )N   Fbias)super__init__hidden_size
input_sizeshared_intermediate_sizer	   
hidden_act
activationr   Linearinput_linearoutput_linearselfr5   	__class__s     r1   r;   zGraniteMoeSharedMLP.__init__U   s     ,!: !23Idot7G!7KRWXXXYt'7uUUUr0   hidden_statesreturnc                     |                      |          }|                    dd          }|                     |d                   |d         z  }|                     |          }|S )Nr7   dimr   r   )rB   chunkr@   rC   )rE   rG   chunked_hidden_statess      r1   forwardzGraniteMoeSharedMLP.forward^   sj    ))-88 - 3 3A2 3 > >(=a(@AADYZ[D\\**=99r0   )
r&   r'   r(   r)   r   r;   r*   TensorrO   __classcell__rF   s   @r1   r4   r4   L   s|         V5 V V V V V VU\ el        r0   r4   c                   ,     e Zd Zd fd	Zd Zd Z xZS )GraniteMoeSharedRMSNormư>c                     t                                                       t          j        t	          j        |                    | _        || _        dS )zF
        GraniteMoeSharedRMSNorm is equivalent to T5LayerNorm
        N)r:   r;   r   	Parameterr*   onesweightvariance_epsilon)rE   r<   epsrF   s      r1   r;   z GraniteMoeSharedRMSNorm.__init__g   sD     	l5:k#:#:;; #r0   c                    |j         }|                    t          j                  }|                    d                              dd          }|t          j        || j        z             z  }| j        |                    |          z  S )Nr7   rJ   T)keepdim)	dtypetor*   float32powmeanrsqrtrZ   rY   )rE   rG   input_dtypevariances       r1   rO   zGraniteMoeSharedRMSNorm.forwardo   s|    #)%((77 $$Q'',,R,>>%Ht?T4T(U(UU{]--k::::r0   c                 H    t          | j        j                   d| j         S )Nz, eps=)tuplerY   shaperZ   )rE   s    r1   
extra_reprz"GraniteMoeSharedRMSNorm.extra_reprv   s&    )**II$2GIIIr0   )rU   )r&   r'   r(   r;   rO   ri   rQ   rR   s   @r1   rT   rT   f   sb        $ $ $ $ $ $; ; ;J J J J J J Jr0   rT   c                   6     e Zd Zdedededdf fdZd Z xZS )GraniteMoeSharedParallelExpertsnum_expertsr=   output_sizerH   Nc                     t                                                       t          j        t	          j        |||                    | _        || _        || _        || _	        dS )a  
        Initialize the GraniteMoeSharedParallelExperts module.
        The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with
        many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and
        [ScatterMoE](https://github.com/shawntan/scattermoe), as well as the
        [MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py)
        used in vllm.

        Args:
            num_experts (int):
                Number of experts.
            input_size (int):
                Size of the input.
            output_size (int):
                Size of the output.
        N)
r:   r;   r   rW   r*   emptyrY   rl   r=   rm   )rE   rl   r=   rm   rF   s       r1   r;   z(GraniteMoeSharedParallelExperts.__init__{   sW    " 	l5;{K#T#TUU&$&r0   c                    |                     |d          }g }t          | j                  D ];}|                    t	          j        ||         | j        |                              <t          j        |d          }|S )a  
        Forward pass of the GraniteMoeSharedParallelExperts module.

        Args:
            inputs (Tensor):
                Input tensor.
            expert_size:
                Expert size information.

        Returns:
            Tensor: Output tensor.
        r   rK   )	splitrangerl   appendFlinearrY   r*   cat)rE   inputsexpert_size
input_listoutput_listiresultss          r1   rO   z'GraniteMoeSharedParallelExperts.forward   s     \\+1\55
t'(( 	H 	HAqx
1t{1~FFGGGG)KQ///r0   r&   r'   r(   r-   r;   rO   rQ   rR   s   @r1   rk   rk   z   sh        'C 'S 's 't ' ' ' ' ' '.      r0   rk   c                   2     e Zd Zdededef fdZd Z xZS )GraniteMoeSharedTopKGatingr=   rl   top_kc                     t                                                       || _        || _        || _        t          j        ||d          | _        dS )a  
        Initialize the top-k gating mechanism.
        Args:
            input_size (`int`):
                Size of the input.
            num_experts (`int`):
                Number of experts.
            top_k (`int`):
                Number of top experts to select.
        Fr8   N)r:   r;   rl   r=   r   r   rA   layer)rE   r=   rl   r   rF   s       r1   r;   z#GraniteMoeSharedTopKGating.__init__   sM     	&$
Yz;UCCC


r0   c                    |                      |                                          }|                    | j        d          \  }}t	          j        |d                              |          }t	          j        |                    d          | j	        g|j
        |j                  }|                    d|d          }|                                                    d          }|                                }|                                }	|	                    d          \  }
}|                    | j        d          }|                                }||         }|||||fS )Nr   rK   r   r^   devicetrunc)rounding_mode)r   floattopkr   r*   softmaxtype_aszerossizerl   r^   r   scatterlongsumtolistflattensortdiv)rE   rG   logitstop_k_logitstop_k_indicestop_k_gatesr   gatesrx   top_k_experts_index_sorted_expertsbatch_indexbatch_gatess                 r1   rO   z"GraniteMoeSharedTopKGating.forward   sS   M**0022&,kk$*!k&D&D#mmLa888@@OO a  $"23;;LU`Ug
 
 
 a22jjll&&q)) "((** &--//"/"4"4Q"7"7*..tz.QQ "))++!"67#[+{FRRr0   r}   rR   s   @r1   r   r      sq        D3 DS D D D D D D D&S S S S S S Sr0   r   c                   .     e Zd ZdZdef fdZd Z xZS )GraniteMoeSharedMoEz
    A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts.

    Args:
        config:
            Configuration object with model hyperparameters.
    r5   c                    t                                                       |j        | _        |j        | _        t
          |j                 | _        t          |j	        | j        | j        dz            | _
        t          |j	        | j        | j                  | _        t          | j        |j	        |j                  | _        d S )Nr7   )r=   rl   r   )r:   r;   r<   r=   intermediate_sizer	   r?   r@   rk   num_local_expertsrB   rC   r   num_experts_per_tokrouterrD   s     r1   r;   zGraniteMoeSharedMoE.__init__   s     ,!3 !23;$dot7G!7K
 
 =$d&6
 
 10,
 
 
r0   c                 T   |                                 \  }}}|                    d|          }|                     |          \  }}}}}	||         }
|                     |
|          }|                    dd          }|                     |d                   |d         z  }|                     ||          }||dddf         z  }t          j        ||z  | j	        f|j
        |j                  }|                    d||          }|                    ||| j	                  }||	fS )a  
        Forward pass of the mixture of experts layer.

        Args:
            layer_input (Tensor):
                Input tensor.

        Returns:
            Tensor:
                Output tensor.
            Tensor:
                Router logits.
        rJ   r7   rK   r   r   Nr   )r   reshaper   rB   rM   r@   rC   r*   r   r=   r^   r   	index_addview)rE   layer_inputbszlengthemb_sizer   r   r   rx   router_logitsexpert_inputsrG   rN   expert_outputsr   layer_outputs                   r1   rO   zGraniteMoeSharedMoE.forward   s5    !, 0 0 2 2VX!))"h77BF++kBZBZ?;[-#K0))-EE - 3 3A2 3 > >(=a(@AADYZ[D\\++M;GG'+aaag*>>S6\4?;>CW`n`uvvvq+~FF#((fdoFF]**r0   )r&   r'   r(   r)   r   r;   rO   rQ   rR   s   @r1   r   r      s^         
5 
 
 
 
 
 
&+ + + + + + +r0   r   c                     | dd| j         d         dz  f         }| d| j         d         dz  df         }t          j        | |fd          S )z*Rotates half the hidden dims of the input..NrJ   r7   rK   )rh   r*   rv   )xx1x2s      r1   rotate_halfr     s]    	
3"!'"+"""	#B	
3q """	#B9rc2YB''''r0   c                     |                     |          }|                     |          }| |z  t          |           |z  z   }||z  t          |          |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkcossinposition_idsunsqueeze_dimq_embedk_embeds           r1   apply_rotary_pos_embr     sc    ( --
&
&C
--
&
&C3w;q>>C/0G3w;q>>C/0GGr0   rG   n_reprH   c                     | j         \  }}}}|dk    r| S | dddddddddf                             |||||          } |                     |||z  ||          S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rh   expandr   )rG   r   batchnum_key_value_headsslenhead_dims         r1   	repeat_kvr   5  s    
 2?1D.Ehzz!!!!QQQaaa"23::5BUW\^bdlmmM  (;e(CT8TTTr0           modulequerykeyvalueattention_maskscalingdropoutc                 R   t          || j                  }t          || j                  }	t          j        ||                    dd                    |z  }
|$|d d d d d d d |j        d         f         }|
|z   }
t          j                            |
dt          j	                  
                    |j                  }
t          j                            |
|| j                  }
t          j        |
|	          }|                    dd                                          }||
fS )Nr7   r   rJ   )rL   r^   )ptrainingr   )r   num_key_value_groupsr*   matmul	transposerh   r   
functionalr   r`   r_   r^   r   r   
contiguous)r   r   r   r   r   r   r   kwargs
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r1   eager_attention_forwardr   A  s    3 ;<<JUF$?@@L<z';';Aq'A'ABBWLL!$QQQ111.D
0@0D.D%DE#k1 =((2U](SSVVW\WbccL=((6?([[L,|\::K''1--88::K$$r0   c                   t    e Zd ZdZddedee         f fdZ eddd	          	 	 	 	 	 	 dde	j
        dee	j
                 dee	j                 dee         dedee	j                 deee	j
        e	j
        f                  dee	j
        ee	j
                 eee	j
                          f         fd            Z xZS )GraniteMoeSharedAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr5   	layer_idxc                    t                                                       || _        || _        |(t                              d| j        j         d           |j        | _        |j	        | _	        |j
        | _        | j	        | j        z  | _        |j        | _        | j        | j        z  | _        d| _        |j        | _        | j        | j        z  | j	        k    r t%          d| j	         d| j         d          t'          j        | j	        | j        | j        z  |j                  | _        t'          j        | j	        | j        | j        z  |j                  | _        t'          j        | j	        | j        | j        z  |j                  | _        t'          j        | j	        | j	        |j                  | _        d S )NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r8   )r:   r;   r5   r   loggerwarning_oncerF   r&   attention_dropoutr<   num_attention_heads	num_headsr   r   r   	is_causalattention_multiplierr   
ValueErrorr   rA   attention_biasq_projk_projv_projo_projrE   r5   r   rF   s      r1   r;   z"GraniteMoeSharedAttention.__init__a  s   ",!8 , , ,   "(!9!-3(DN:#)#= $(Nd6N$N!2MDN*t/???8RVRb 8 8%)^8 8 8  
 i 0$.4=2PW]Wlmmmi 0$2JT]2Zagavwwwi 0$2JT]2Zagavwwwi 0$2BI^___r0   past_key_valuepast_key_values4.58new_nameversionFrG   r   r   	use_cachecache_positionposition_embeddingsrH   c                    |                                 \  }	}
}|                     |          }|                     |          }|                     |          }|                    |	|
| j        | j                                      dd          }|                    |	|
| j        | j                                      dd          }|                    |	|
| j        | j                                      dd          }||nd\  }}|t          ||||          \  }}|&|||d}|
                    ||| j        |          \  }}t          }| j        j        dk    rt          | j        j                 } || ||||f| j        sdn| j        | j        d|\  }}|                    |	|
d          }|                     |          }||fS )	Nr   r7   )NN)r   r   r   eagerr   )r   r   rJ   )r   r   r   r   r   r   r   r   r   r   updater   r   r5   _attn_implementationr   r   r   r   r   )rE   rG   r   r   r   r   r   r   r   r   q_lenr   query_statesr   r   r   r   cache_kwargsattention_interfacer   r   s                        r1   rO   z!GraniteMoeSharedAttention.forward  s    &**,,UA{{=11[[//
{{=11#((eT^T]SS]]^_abcc__S%1I4=YYccdeghii
#((eT5Mt}]]gghiklmm*=*I&&|S*';L*VY[^'_'_$L*&#&snUUL'6'='=j,X\Xfht'u'u$J(?;+w66"9$+:Z"[$7$7	%
  $}HCC$2HL	%
 	%
 	%
 	%
!\ "&&sE266kk+..L((r0   N)NNNFNN)r&   r'   r(   r)   r   r   r-   r;   r   r*   rP   r+   r
   boolrg   rO   rQ   rR   s   @r1   r   r   ^  sR       GG` `5 `(3- ` ` ` ` ` `@ _%0A6RRR 2637+/59KO0) 0)|0) !.0) u/0	0)
 "%0) 0) !!120) &eEL%,,F&GH0) 
u|Xel3XeEL>Q5RR	S0) 0) 0) SR0) 0) 0) 0) 0)r0   r   c                       e Zd Zdedef fdZ eddd          	 	 	 	 	 	 	 	 dd
ej        de	ej                 de	ej
                 de	e         de	e         de	e         de	ej
                 de	e         de	eej        ej        f                  dee         deej        e	eej        ej        f                  f         fd            Z xZS )GraniteMoeSharedDecoderLayerr5   r   c                    t                                                       |j        | _        t          ||          | _        |j        dk    rt          |          | _        t          |j        |j	                  | _
        t          |j        |j	                  | _        |j        | _        |j        dk    rd nt          |          | _        d S )N)r5   r   r   r[   )r:   r;   r<   r   	self_attnr   r   block_sparse_moerT   rms_norm_epsinput_layernormpost_attention_layernormresidual_multiplierr>   r4   
shared_mlpr   s      r1   r;   z%GraniteMoeSharedDecoderLayer.__init__  s    !-2&IVVV#a''$7$?$?D!6v7IvObccc(?@RX^Xk(l(l(l%#)#= "("AQ"F"F$$L_`fLgLgr0   r   r   r   r   NFrG   r   r   output_attentionsr   r   output_router_logitsr   r   rH   c
                 d   |}|                      |          } | j        d||||||||	d|
\  }}||| j        z  z   }|}|                     |          }|                     |          \  }}| j        |}n||                     |          z   }~||| j        z  z   }|f}|r||fz  }|r||fz  }|S )a1  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            output_router_logits (`bool`, *optional*):
                Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
                should not be returned during inference.
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs. Can be used to provide `GraniteFlashAttentionKwargs` for
                padding-free training and/or improve torch.compile performance.
        )rG   r   r   r   r  r   r   r   Nr/   )r  r  r  r  r  r  )rE   rG   r   r   r   r  r   r   r  r   r   residualself_attn_weightsmoe_hidden_statesr   outputss                   r1   rO   z$GraniteMoeSharedDecoderLayer.forward  s   N !,,];; ,:4> 
,
')%+/) 3
,
 
,
 
,
 
,
(( !=43K#KK !55mDD+/+@+@+O+O(=?"-MM-0N0NNM =43K#KK " 	,)++G 	(''Gr0   )NNNFFNFN)r&   r'   r(   r   r-   r;   r   r*   rP   r   r+   r
   r  rg   r   r    FloatTensorrO   rQ   rR   s   @r1   r  r    s       h5 h# h h h h h h _%0A6RRR 2637+/,1$)59/4KOO O|O !.O u/0	O
 "%O $D>O D>O !!12O 'tnO &eEL%,,F&GHO 45O 
u (51BEDU1U+V"WW	XO O O SRO O O O Or0   r  c                   J     e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZ fdZ xZS )GraniteMoeSharedPreTrainedModelr5   modelTr  r   Fc                     t                                          |           t          |t                    r-|j        j                            d| j        j                   d S d S )Nr   )rb   std)	r:   _init_weights
isinstancerk   rY   datanormal_r5   initializer_range)rE   r   rF   s     r1   r  z-GraniteMoeSharedPreTrainedModel._init_weights"  sd    f%%%f=>> 	TM&&CT[5R&SSSSS	T 	Tr0   )r&   r'   r(   r   r,   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraphr  rQ   rR   s   @r1   r  r    s~         """"&*#78#4"5N"T T T T T T T T Tr0   r  c                   |     e Zd ZU ej        ed<   ddef fdZ ej                    e	d                         Z
 xZS )GraniteMoeSharedRotaryEmbeddinginv_freqNr5   c                    t                                                       t          |d          rSt          |j        t
                    r9|j                            d|j                            d                    | _        nd| _        |j        | _	        |j        | _
        || _        t          | j                 | _        |                     | j        |          \  }| _        |                     d|d           | j        | _        d S )Nrope_scaling	rope_typetypedefaultr-  F)
persistent)r:   r;   hasattrr   r/  dictgetr0  max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr5   r   rope_init_fnattention_scalingregister_bufferr-  original_inv_freq)rE   r5   r   r-  rF   s       r1   r;   z(GraniteMoeSharedRotaryEmbedding.__init__+  s    6>** 	'z&:Mt/T/T 	'#044[&BUBYBYZ`BaBabbDNN&DN"("@$*$B!/?+/+<+<T[&+Q+Q($(ZeDDD!%r0   c                 X   | j         d d d d f                                                             |j        d         dd                              |j                  }|d d d d d f                                         }t          |j        j        t                    r|j        j        dk    r|j        j        nd}t          j
        |d          5  |                                |                                z                      dd          }t          j        ||fd	          }|                                | j        z  }|                                | j        z  }	d d d            n# 1 swxY w Y   |                    |j        
          |	                    |j        
          fS )Nr   rJ   r   mpscpuF)device_typeenabledr7   rK   )r^   )r-  r   r   rh   r_   r   r   r1  strr*   autocastr   rv   r   r;  r   r^   )
rE   r   r   inv_freq_expandedposition_ids_expandedrA  freqsembr   r   s
             r1   rO   z'GraniteMoeSharedRotaryEmbedding.forward<  s    !M$4-8>>@@GGHZ[\H]_acdeehhijiqrr ,QQQaaaZ 8 > > @ @'1!(-'E'Ek!(-[`J`J`ahmmfk^UCCC 	5 	5&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))d44C''))d44C		5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 vvAGv$$cff17f&;&;;;s   BE++E/2E/r  )r&   r'   r(   r*   rP   r,   r   r;   no_gradr   rO   rQ   rR   s   @r1   r,  r,  (  s         l/ /5 / / / / / /" U]__< <  _< < < < <r0   r,  c                       e Zd Zdef fdZe	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej	                 deej                 dee
eeej                 f                  deej                 d	ee         d
ee         dee         dee         dee         deej                 de
eef         fd            Z	 dde
ej	        df         dej	        dej	        ded
ef
dZedej	        dededej        dej	        defd            Z xZS )GraniteMoeSharedModelr5   c                    t                                                     j        | _        j        | _        t          j        j        j        | j                  | _        t          j	        fdt          j                  D                       | _        t          j        j                  | _        d| _        j        | _        j        | _        j        | _        | j        | j        z  | _        j        | _        j        | _        j        | _        | j        dk    rt1                    nd | _        |                                  d S )Nc                 0    g | ]}t          |          S r/   )r  ).0r   r5   s     r1   
<listcomp>z2GraniteMoeSharedModel.__init__.<locals>.<listcomp>U  s$    nnn)&)<<nnnr0   r
  Frope)r:   r;   pad_token_idpadding_idx
vocab_sizer   	Embeddingr<   embed_tokens
ModuleListrr   num_hidden_layerslayersrT   r  normgradient_checkpointingembedding_multiplierr   r   r   r7  
rope_thetaposition_embedding_typer,  
rotary_emb	post_initrD   s    `r1   r;   zGraniteMoeSharedModel.__init__N  s8      !. +L):F<NPTP`aamnnnneTZTlNmNmnnn
 
 ,F,>FDWXXX	&+#$*$?!!-3(DN:'-'E$ +'-'E$EIEaekEkEk9&AAAqu 	r0   N	input_idsr   r   r   inputs_embedsr   r  output_hidden_statesr  return_dictr   rH   c                     ||n| j         j        }||n| j         j        }||n| j         j        }|
|
n| j         j        }
|d u |d uz  rt          d          | j        r%| j        r|rt          	                    d           d}|| 
                    |          }|| j        z  }|r|t          | j                   }|B||                                nd}t          j        |||j        d         z   |j                  }||                    d          }|                     |||||          }|}d }| j        |                     ||          }|rdnd }|rdnd }|	rdnd }| j        D ]B}|r||fz  } |||||||||	|		  	        }|d         }|r||d         fz  }|	r||d
         fz  }C|                     |          }|r||fz  }|
st/          d ||||fD                       S t1          |||||          S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)r5   r   r   r   r/   )r   r   r   r  r   r   r  r   rJ   c              3      K   | ]}||V  	d S r  r/   )rN  vs     r1   	<genexpr>z0GraniteMoeSharedModel.forward.<locals>.<genexpr>  s1        bcbobobobobo r0   )last_hidden_stater   rG   
attentionsr   )r5   r  rb  r   use_return_dictr   rZ  r   r   r   rU  r[  r   get_seq_lengthr*   arangerh   r   r   _update_causal_maskr^  rX  rY  rg   r   )rE   r`  r   r   r   ra  r   r  rb  r  rc  r   r   past_seen_tokensr   rG   r   all_hidden_statesall_self_attnsall_router_logitsdecoder_layerlayer_outputss                         r1   rO   zGraniteMoeSharedModel.forwardg  s     2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]-t";< 	[YZZZ& 	4= 	Y 	j   I  --i88M%(AA 	?0*$+>>>O!CRC^==???de"\ "2]5H5K"KTaTh  N )33A66L..M>?L]
 

 &"?&"&//-"N"N #7@BBD0:d"6@BBD![ 	: 	:M# 6!m%55!)M*) /"3#-%9$7
 
 
M *!,M  6=#3"55# :!mB&7%99!		-00   	2-!11 	  )?<M~^      &+++%+
 
 
 	
r0   Fr   input_tensorc           	      $   | j         j        dk    r||dk                                    r|S d S | j         j        dk    r+t          |t          j                  rt          |          }|S ||                                nd}||j        nd}| j         j        dk    r#|s!|st          j
        |||| j                  rd S |j        }|j        d         }	|r|                                }
n/t          |t          j                  r|j        d	         n||	z   dz   }
|                     ||	|
|||j        d         
          }| j         j        dk    r@|>|j        j        dv r0|s.t	          j        |          j        }t          j        ||          }|S )Nflash_attention_2r   flex_attentionr   Fsdpa)ra  past_key_values_lengthis_trainingr   rJ   )sequence_lengthtarget_lengthr^   r   
batch_size)cudaxpunpu)r5   r   anyr   r*   rP   r   rl  is_compileabler   _ignore_causal_mask_sdpar   r^   rh   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr   r1  finfomin_unmask_unattended)rE   r   ru  r   r   r  ro  using_compilable_cacher^   r|  r}  r   	min_dtypes                r1   rn  z)GraniteMoeSharedModel._update_causal_mask  s    ;+/BBB)~/D.I.I.K.K)%%4;+/???.%,77 M!<^!L!L!!
 @O?Z?99;;;`aCRC^!?!?di ;+v55>T5]n5%>*'7 M	    t"&,Q/! 	+??AAMM nel;;<$R((%7!;  PP+')#)!, Q 
 
 K,66*%*.DDD% E E**.I0CKQZ[[Kr0   r|  r}  r^   r~  c                    | |                                  dk    r| }nMt          j        |          j        }t          j        ||f|||j                  }|dk    rt          j        |d          }|t          j        ||j                  |                    dd          k    z  }|ddddddf         	                    |ddd          }| |
                                }| j        d         }	|ddddddd|	f         | ddddddf                             |j                  z   }
|
dk    }
|ddddddd|	f                             |
|          |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuer^   r   r   )diagonalre  rJ   r   )rL   r*   r  r  fullr   triurm  r   r   clonerh   r_   masked_fill)r   r|  r}  r^   r   r~  r   r   r  mask_lengthpadding_masks              r1   r  zKGraniteMoeSharedModel._prepare_4d_causal_attention_mask_with_cache_position  s   < %.*<*<*>*>!*C*C(KKE**.I* -0Ye\j\q  K !###jqAAA5<n>STTTWeWmWmnprsWtWtttK%dD!!!QQQ&67>>z1bRTUUK))//11,226*111aaaL[L+@ANSTSTSTVZ\`bcbcbcScDdDgDg&E E    ,q05@AAAqqq,;,AV5W5c5c )6 6AAAqqq!!!\k\12 r0   )NNNNNNNNNNN)F)r&   r'   r(   r   r;   r   r   r*   r+   rP   r   r
   listr  r  rg   r   rO   rn  staticmethodr-   r^   r  rQ   rR   s   @r1   rK  rK  L  s9       5      2  151537KO59$(,0/3/3&*59h
 h
E,-h
 !.h
 u/0	h

 "%tE4E/F(F"GHh
   12h
 D>h
 $D>h
 'tnh
 'tnh
 d^h
 !!12h
 
u--	.h
 h
 h
 ^h
` #(B BelK78B lB 	B
 B  B B B BH 444 4 {	4
 4 4 4 4 \4 4 4 4 4r0   rK  r7   gate_logitsrl   c                    | t          | t                    sdS t          | t                    r/| d         j        t          j        fd| D             d          }t          j        j                            |d          }t          j        ||d          \  }}t          j        j        	                    ||          }|@t          j
        |                                d          }	t          j
        |d          }
nD|j        \  }}|j        d         ||z  z  }|dddddddf                             |||||f                              d||                                        }t          j        |                                |z  d          t          j        |d          z  }	|ddddddf                             ||||j        d         f                              d|j        d                                                 }t          j        ||z  d          t          j        |d          z  }
|j        j        |j        j        nd}|j        d         t%          |          z  }t          j        |	dd|||j        d         z   f         |
                    d          z            }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                 :    g | ]}|                               S r/   )r_   )rN  
layer_gatecompute_devices     r1   rO  z,load_balancing_loss_func.<locals>.<listcomp>p  s&    -j-j-jPZjmmN.K.K-j-j-jr0   rK   rJ   r   )r   rg   r   r*   rv   r   r   r   r   one_hotrb   r   rh   r   r   r_   r   indexr-   r   )r  rl   r   r   concatenated_gate_logitsrouting_weightsr   selected_expertsexpert_masktokens_per_expertrouter_prob_per_expertr~  r|  rW  expert_attention_mask router_per_expert_attention_maskdevice_indexrankoverall_lossr  s                      @r1   load_balancing_loss_funcr  N  s   : *[%"@"@q+u%% s$Q.#(9-j-j-j-j^i-j-j-jpq#r#r#r h)112JPR1SSO*_eDDDA(%--.>LLK!J{'8'8':':BBB "'O!C!C!C&4&:#
O4:1=*B^_ 4AAAtT12V&
OUKXYYWR,,R	 	 "Ik&7&7&9&9<Q&QWXYYY\a\e!q]
 ]
 ]
 
 4AAAt+,V&
O_EZ[\E]^__WR.q122R	 	) "'?=]+]cd!e!e!ehmhq,!i
 i
 i
 "
 4C3I3O3[?)//abL #c,&7&77D9!!!TD?+@+C$CCCDG]GgGghiGjGjj L +%%r0   c                        e Zd ZdgZdef fdZe	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej	                 deej
                 deej	                 d	eeeeej                 f                  d
eej                 deej	                 dee         dee         dee         dee         dee         deej	                 deeej
        f         deeef         fd            Z xZS )GraniteMoeSharedForCausalLMzlm_head.weightr5   c                 F   t                                          |           t          |          | _        |j        | _        t          j        |j        |j        d          | _        |j	        | _	        |j
        | _        |j        | _        |                                  d S )NFr8   )r:   r;   rK  r  rS  r   rA   r<   lm_headrouter_aux_loss_coefr   rl   r   r_  rD   s     r1   r;   z$GraniteMoeSharedForCausalLM.__init__  s       *622
 +y!3V5FUSSS$*$?!!3#)#=  	r0   Nr   r`  r   r   r   ra  labelsr   r  rb  r  rc  r   logits_to_keeprH   c                     ||n| j         j        }|
|
n| j         j        }
|	|	n| j         j        }	||n| j         j        } | j        d||||||||	|
||d|}|d         }t          |t                    rt          | d          n|}| 	                    |dd|ddf                   }|| j         j
        z  }d}|/|                                } | j        ||fd| j         j        i|}d}|
rRt          |r|j        n|d         | j        | j        |          }|%|| j        |                    |j                  z  z  }|s |f|dd         z   }|
r|f|z   }||f|z   n|S t+          ||||j        |j        |j        |j                  S )	ax  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeSharedForCausalLM

        >>> model = GraniteMoeSharedForCausalLM.from_pretrained("ibm/PowerMoE-3b")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r`  r   r   r   ra  r   r  rb  r  rc  r   r   rS  rJ   r   )lossaux_lossr   r   rG   rj  r   r/   )r5   r  r  rb  rk  r  r   r-   slicer  logits_scalingr   loss_functionrS  r  r   rl   r   r  r_   r   r   r   rG   rj  )rE   r`  r   r   r   ra  r  r   r  rb  r  rc  r   r  r   r  rG   slice_indicesr   r  r  outputs                         r1   rO   z#GraniteMoeSharedForCausalLM.forward  sO   P 2C1N--TXT_Tq$8$D  $+Jj 	 %9$D  $+Jj 	 &1%<kk$+B] $* 
)%+'/!5!5#)
 
 
 
   
8B>SV8W8Wk~ot444]kmAAA}aaa,?@AA$+44\\^^F%4%   ;1 	 D  	M/)4E%%'"+ (	 H !1HKK4L4LLL 	DY,F# ."v-'+'7D7V##VC(#3!/)!/
 
 
 	
r0   )NNNNNNNNNNNNr   )r&   r'   r(   _tied_weights_keysr   r;   r   r   r*   r+   rP   r   r
   r  r  r  r-   rg   r   rO   rQ   rR   s   @r1   r  r    s       *+5        151537KO59-1$(,0/3/3&*5934k
 k
E,-k
 !.k
 u/0	k

 "%tE4E/F(F"GHk
   12k
 )*k
 D>k
 $D>k
 'tnk
 'tnk
 d^k
 !!12k
 c5</0k
  
u//	0!k
 k
 k
 ^k
 k
 k
 k
 k
r0   r  )r  rK  r  )Nr   )r   )Nr7   N)Gtypingr   r   r   r   r*   torch.nn.functionalr   r   rt   activationsr	   cache_utilsr
   r   
generationr   modeling_attn_mask_utilsr   modeling_layersr   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.deprecationr   configuration_granitemoesharedr   !torch.nn.attention.flex_attentionr   integrations.flex_attentionr   
get_loggerr&   r   r    Moduler4   rT   rk   r   r   r   r   rP   r-   r   r   r   r   r  r  r,  rK  rg   r  r  __all__r/   r0   r1   <module>r     so  , 8 7 7 7 7 7 7 7 7 7 7 7                 ! ! ! ! ! ! . . . . . . . . ) ) ) ) ) ) > > > > > > 9 9 9 9 9 9 j j j j j j j j j j K K K K K K K K F F F F F F F F & & & & & & J J J J J J J J J J 0 0 0 0 0 0 B B B B B B  !! K;;;;;;JJJJJJ 
	H	%	%    )5    2    ")   4J J J J Jbi J J J(* * * * *bi * * *Z-S -S -S -S -S -S -S -S`9+ 9+ 9+ 9+ 9+") 9+ 9+ 9+x( ( (   6	UU\ 	U# 	U%, 	U 	U 	U 	U& % %I%<% 
% <	%
 U\*% % % % % %:T) T) T) T) T)	 T) T) T)n^ ^ ^ ^ ^#= ^ ^ ^B T T T T To T T T"!< !< !< !< !<bi !< !< !<H ~ ~ ~ ~ ~; ~ ~ ~F "&
-1	S& S&u|U5<%8$>?S&#S& U\*	S&
 5<S& S& S& S&l|
 |
 |
 |
 |
"A? |
 |
 |
~ f
e
er0   