
     `i;                        d dl mZmZmZ d dlZd dlmc mZ d dlmZ ddl	m
Z
 ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZmZ ddlmZm Z m!Z! ddl"m#Z# ddl$m%Z%  e             rd dl&m'Z' ddl(m)Z)  e!j*        e+          Z,	 	 	 d;deej-        e.ej-                 df         dee/         deej-                 deej-        e/f         fdZ0 G d dej1                  Z2 G d dej1                  Z3d Z4d<dZ5 G d d ej1                  Z6 G d! d"ej1                  Z7 G d# d$ej1                  Z8d%ej-        d&e/dej-        fd'Z9 G d( d)ej1                  Z:	 d=d+ej1        d,ej-        d-ej-        d.ej-        deej-                 d/e;d0e;fd1Z< G d2 d3e          Z=e G d4 d5e                      Z>e G d6 d7e>                      Z? G d8 d9e>e          Z@g d:ZAdS )>    )CallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)GradientCheckpointingLayer)BaseModelOutputWithPastMoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)auto_docstringis_torch_flex_attn_availablelogging)deprecate_kwarg   )GraniteMoeConfig)	BlockMask)make_flex_block_causal_mask   gate_logitsnum_expertsattention_maskreturnc                    | t          | t                    sdS t          | t                    r/| d         j        t          j        fd| D             d          }t          j        j                            |d          }t          j        ||d          \  }}t          j        j        	                    ||          }|@t          j
        |                                d          }	t          j
        |d          }
nD|j        \  }}|j        d         ||z  z  }|dddddddf                             |||||f                              d||                                        }t          j        |                                |z  d          t          j        |d          z  }	|ddddddf                             ||||j        d         f                              d|j        d                                                 }t          j        ||z  d          t          j        |d          z  }
|j        j        |j        j        nd}|j        d         t%          |          z  }t          j        |	dd|||j        d         z   f         |
                    d          z            }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                 :    g | ]}|                               S  )to).0
layer_gatecompute_devices     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/models/granitemoe/modeling_granitemoe.py
<listcomp>z,load_balancing_loss_func.<locals>.<listcomp>O   s&    -j-j-jPZjmmN.K.K-j-j-j    dimr   )
isinstancetupledevicetorchcatr   
functionalsoftmaxtopkone_hotmeanfloatshapeexpandreshaper%   sumindexint	unsqueeze)r   r   top_kr    concatenated_gate_logitsrouting_weights_selected_expertsexpert_masktokens_per_expertrouter_prob_per_expert
batch_sizesequence_lengthnum_hidden_layersexpert_attention_mask router_per_expert_attention_maskdevice_indexrankoverall_lossr(   s                      @r)   load_balancing_loss_funcrQ   -   s   : *[%"@"@q+u%% s$Q.#(9-j-j-j-j^i-j-j-jpq#r#r#r h)112JPR1SSO*_eDDDA(%--.>LLK!J{'8'8':':BBB "'O!C!C!C&4&:#
O4:1=*B^_ 4AAAtT12V&
OUKXYYWR,,R	 	 "Ik&7&7&9&9<Q&QWXYYY\a\e!q]
 ]
 ]
 
 4AAAt+,V&
O_EZ[\E]^__WR.q122R	 	) "'?=]+]cd!e!e!ehmhq,!i
 i
 i
 "
 4C3I3O3[?)//abL #c,&7&77D9!!!TD?+@+C$CCCDG]GgGghiGjGjj L +%%r+   c                   ,     e Zd Zd fd	Zd Zd Z xZS )GraniteMoeRMSNormư>c                     t                                                       t          j        t	          j        |                    | _        || _        dS )z@
        GraniteMoeRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parameterr2   onesweightvariance_epsilon)selfhidden_sizeeps	__class__s      r)   rW   zGraniteMoeRMSNorm.__init__   sD     	l5:k#:#:;; #r+   c                    |j         }|                    t          j                  }|                    d                              dd          }|t          j        || j        z             z  }| j        |                    |          z  S )Nr   r.   T)keepdim)	dtyper%   r2   float32powr8   rsqrtr[   rZ   )r\   hidden_statesinput_dtypevariances       r)   forwardzGraniteMoeRMSNorm.forward   s|    #)%((77 $$Q'',,R,>>%Ht?T4T(U(UU{]--k::::r+   c                 H    t          | j        j                   d| j         S )Nz, eps=)r0   rZ   r:   r[   )r\   s    r)   
extra_reprzGraniteMoeRMSNorm.extra_repr   s&    )**II$2GIIIr+   )rT   )__name__
__module____qualname__rW   ri   rk   __classcell__r_   s   @r)   rS   rS      sb        $ $ $ $ $ $; ; ;J J J J J J Jr+   rS   c                   |     e Zd ZU ej        ed<   ddef fdZ ej                    e	d                         Z
 xZS )GraniteMoeRotaryEmbeddinginv_freqNconfigc                    t                                                       t          |d          rSt          |j        t
                    r9|j                            d|j                            d                    | _        nd| _        |j        | _	        |j        | _
        || _        t          | j                 | _        |                     | j        |          \  }| _        |                     d|d           | j        | _        d S )Nrope_scaling	rope_typetypedefaultrs   F)
persistent)rV   rW   hasattrr/   rv   dictgetrw   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrt   r   rope_init_fnattention_scalingregister_bufferrs   original_inv_freq)r\   rt   r1   rs   r_   s       r)   rW   z"GraniteMoeRotaryEmbedding.__init__   s    6>** 	'z&:Mt/T/T 	'#044[&BUBYBYZ`BaBabbDNN&DN"("@$*$B!/?+/+<+<T[&+Q+Q($(ZeDDD!%r+   c                 X   | j         d d d d f                                                             |j        d         dd                              |j                  }|d d d d d f                                         }t          |j        j        t                    r|j        j        dk    r|j        j        nd}t          j
        |d          5  |                                |                                z                      dd          }t          j        ||fd	          }|                                | j        z  }|                                | j        z  }	d d d            n# 1 swxY w Y   |                    |j        
          |	                    |j        
          fS )Nr   r.   r   mpscpuF)device_typeenabledr   r,   )rb   )rs   r9   r;   r:   r%   r1   r/   rx   strr2   autocast	transposer3   cosr   sinrb   )
r\   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r)   ri   z!GraniteMoeRotaryEmbedding.forward   s    !M$4-8>>@@GGHZ[\H]_acdeehhijiqrr ,QQQaaaZ 8 > > @ @'1!(-'E'Ek!(-[`J`J`ahmmfk^UCCC 	5 	5&,,..1F1L1L1N1NNYYZ[]^__E)UEN333C''))d44C''))d44C		5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 vvAGv$$cff17f&;&;;;s   BE++E/2E/N)rl   rm   rn   r2   Tensor__annotations__r   rW   no_gradr   ri   ro   rp   s   @r)   rr   rr      s         l/ // / / / / / /" U]__< <  _< < < < <r+   rr   c                     | dd| j         d         dz  f         }| d| j         d         dz  df         }t          j        | |fd          S )z*Rotates half the hidden dims of the input..Nr.   r   r,   )r:   r2   r3   )r   x1x2s      r)   rotate_halfr      s]    	
3"!'"+"""	#B	
3q """	#B9rc2YB''''r+   c                     |                     |          }|                     |          }| |z  t          |           |z  z   }||z  t          |          |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r@   r   )qkr   r   r   unsqueeze_dimq_embedk_embeds           r)   apply_rotary_pos_embr      sc    ( --
&
&C
--
&
&C3w;q>>C/0G3w;q>>C/0GGr+   c                   6     e Zd Zdedededdf fdZd Z xZS )GraniteMoeParallelExpertsr   
input_sizeoutput_sizer!   Nc                     t                                                       t          j        t	          j        |||                    | _        || _        || _        || _	        dS )a  
        Initialize the GraniteMoeParallelExperts module.
        The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with
        many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and
        [ScatterMoE](https://github.com/shawntan/scattermoe), as well as the
        [MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py)
        used in vllm.

        Args:
            num_experts (int):
                Number of experts.
            input_size (int):
                Size of the input.
            output_size (int):
                Size of the output.
        N)
rV   rW   r   rX   r2   emptyrZ   r   r   r   )r\   r   r   r   r_   s       r)   rW   z"GraniteMoeParallelExperts.__init__   sW    " 	l5;{K#T#TUU&$&r+   c                    |                     |d          }g }t          | j                  D ];}|                    t	          j        ||         | j        |                              <t          j        |d          }|S )a  
        Forward pass of the GraniteMoeParallelExperts module.

        Args:
            inputs (Tensor):
                Input tensor.
            expert_size:
                Expert size information.

        Returns:
            Tensor: Output tensor.
        r   r,   )	splitranger   appendFlinearrZ   r2   r3   )r\   inputsexpert_size
input_listoutput_listiresultss          r)   ri   z!GraniteMoeParallelExperts.forward   s     \\+1\55
t'(( 	H 	HAqx
1t{1~FFGGGG)KQ///r+   rl   rm   rn   r?   rW   ri   ro   rp   s   @r)   r   r      sh        'C 'S 's 't ' ' ' ' ' '.      r+   r   c                   2     e Zd Zdededef fdZd Z xZS )GraniteMoeTopKGatingr   r   rA   c                     t                                                       || _        || _        || _        t          j        ||d          | _        dS )a  
        Initialize the top-k gating mechanism.
        Args:
            input_size (`int`):
                Size of the input.
            num_experts (`int`):
                Number of experts.
            top_k (`int`):
                Number of top experts to select.
        FbiasN)rV   rW   r   r   rA   r   Linearlayer)r\   r   r   rA   r_   s       r)   rW   zGraniteMoeTopKGating.__init__  sM     	&$
Yz;UCCC


r+   c                    |                      |                                          }|                    | j        d          \  }}t	          j        |d                              |          }t	          j        |                    d          | j	        g|j
        |j                  }|                    d|d          }|                                                    d          }|                                }|                                }	|	                    d          \  }
}|                    | j        d          }|                                }||         }|||||fS )Nr   r,   r   rb   r1   trunc)rounding_mode)r   r9   r6   rA   r2   r5   type_aszerossizer   rb   r1   scatterlongr=   tolistflattensortdiv)r\   rf   logitstop_k_logitstop_k_indicestop_k_gatesr   gatesr   top_k_expertsrD   index_sorted_expertsbatch_indexbatch_gatess                 r)   ri   zGraniteMoeTopKGating.forward$  sS   M**0022&,kk$*!k&D&D#mmLa888@@OO a  $"23;;LU`Ug
 
 
 a22jjll&&q)) "((** &--//"/"4"4Q"7"7*..tz.QQ "))++!"67#[+{FRRr+   r   rp   s   @r)   r   r     sq        D3 DS D D D D D D D&S S S S S S Sr+   r   c                   .     e Zd ZdZdef fdZd Z xZS )GraniteMoeMoEz
    A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts.

    Args:
        config:
            Configuration object with model hyperparameters.
    rt   c                    t                                                       |j        | _        |j        | _        t
          |j                 | _        t          |j	        | j        | j        dz            | _
        t          |j	        | j        | j                  | _        t          | j        |j	        |j                  | _        d S )Nr   )r   r   rA   )rV   rW   r]   r   intermediate_sizer   
hidden_act
activationr   num_local_expertsinput_linearoutput_linearr   num_experts_per_tokrouterr\   rt   r_   s     r)   rW   zGraniteMoeMoE.__init__I  s     ,!3 !235f6NPTP_aeaqtuauvv6v7OQUQacgcrss*0,
 
 
r+   c                 T   |                                 \  }}}|                    d|          }|                     |          \  }}}}}	||         }
|                     |
|          }|                    dd          }|                     |d                   |d         z  }|                     ||          }||dddf         z  }t          j        ||z  | j	        f|j
        |j                  }|                    d||          }|                    ||| j	                  }||	fS )a  
        Forward pass of the mixture of experts layer.

        Args:
            layer_input (Tensor):
                Input tensor.

        Returns:
            Tensor:
                Output tensor.
            Tensor:
                Router logits.
        r.   r   r,   r   r   Nr   )r   r<   r   r   chunkr   r   r2   r   r   rb   r1   	index_addview)r\   layer_inputbszlengthemb_sizerD   r   r   r   router_logitsexpert_inputsrf   chunked_hidden_statesexpert_outputsr   layer_outputs                   r)   ri   zGraniteMoeMoE.forwardX  s5    !, 0 0 2 2VX!))"h77BF++kBZBZ?;[-#K0))-EE - 3 3A2 3 > >(=a(@AADYZ[D\\++M;GG'+aaag*>>S6\4?;>CW`n`uvvvq+~FF#((fdoFF]**r+   )rl   rm   rn   __doc__r   rW   ri   ro   rp   s   @r)   r   r   @  s^         
/ 
 
 
 
 
 
+ + + + + + +r+   r   rf   n_repc                     | j         \  }}}}|dk    r| S | dddddddddf                             |||||          } |                     |||z  ||          S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r:   r;   r<   )rf   r   batchnum_key_value_headsslenhead_dims         r)   	repeat_kvr   y  s    
 2?1D.Ehzz!!!!QQQaaa"23::5BUW\^bdlmmM  (;e(CT8TTTr+   c                   t    e Zd ZdZddedee         f fdZ eddd	          	 	 	 	 	 	 dde	j
        dee	j
                 dee	j                 dee         dedee	j                 deee	j
        e	j
        f                  dee	j
        ee	j
                 eee	j
                          f         fd            Z xZS )GraniteMoeAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNrt   	layer_idxc                    t                                                       || _        || _        |(t                              d| j        j         d           |j        | _        |j	        | _	        |j
        | _        | j	        | j        z  | _        |j        | _        | j        | j        z  | _        d| _        |j        | _        | j        | j        z  | j	        k    r t%          d| j	         d| j         d          t'          j        | j	        | j        | j        z  |j                  | _        t'          j        | j	        | j        | j        z  |j                  | _        t'          j        | j	        | j        | j        z  |j                  | _        t'          j        | j	        | j	        |j                  | _        d S )NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r   )rV   rW   rt   r   loggerwarning_oncer_   rl   attention_dropoutr]   num_attention_heads	num_headsr   r   num_key_value_groups	is_causalattention_multiplierscaling
ValueErrorr   r   attention_biasq_projk_projv_projo_projr\   rt   r   r_   s      r)   rW   zGraniteMoeAttention.__init__  s   ",!8 , , ,   "(!9!-3(DN:#)#= $(Nd6N$N!2MDN*t/???8RVRb 8 8%)^8 8 8  
 i 0$.4=2PW]Wlmmmi 0$2JT]2Zagavwwwi 0$2JT]2Zagavwwwi 0$2BI^___r+   past_key_valuepast_key_values4.58new_nameversionFrf   r    r   	use_cachecache_positionposition_embeddingsr!   c                    |                                 \  }	}
}|                     |          }|                     |          }|                     |          }|                    |	|
| j        | j                                      dd          }|                    |	|
| j        | j                                      dd          }|                    |	|
| j        | j                                      dd          }||nd\  }}|t          ||||          \  }}|&|||d}|
                    ||| j        |          \  }}t          }| j        j        dk    rt          | j        j                 } || ||||f| j        sdn| j        | j        d|\  }}|                    |	|
d          }|                     |          }||fS )	Nr   r   )NN)r   r   r  eager        )dropoutr  r.   )r   r  r  r  r   r   r   r   r   r   updater   eager_attention_forwardrt   _attn_implementationr   trainingr   r  r  )r\   rf   r    r   r
  r  r  r  kwargsr   q_lenrD   query_states
key_statesvalue_statesr   r   cache_kwargsattention_interfaceattn_outputattn_weightss                        r)   ri   zGraniteMoeAttention.forward  s    &**,,UA{{=11[[//
{{=11#((eT^T]SS]]^_abcc__S%1I4=YYccdeghii
#((eT5Mt}]]gghiklmm*=*I&&|S*';L*VY[^'_'_$L*&#&snUUL'6'='=j,X\Xfht'u'u$J(?;+w66"9$+:Z"[$7$7	%
  $}HCC$2HL	%
 	%
 	%
 	%
!\ "&&sE266kk+..L((r+   r   )NNNFNN)rl   rm   rn   r   r   r   r?   rW   r   r2   r   
LongTensorr	   boolr0   ri   ro   rp   s   @r)   r   r     sR       GG` `/ `HSM ` ` ` ` ` `@ _%0A6RRR 2637+/59KO0) 0)|0) !.0) u/0	0)
 "%0) 0) !!120) &eEL%,,F&GH0) 
u|Xel3XeEL>Q5RR	S0) 0) 0) SR0) 0) 0) 0) 0)r+   r   r  modulequerykeyvaluer  r  c                 R   t          || j                  }t          || j                  }	t          j        ||                    dd                    |z  }
|$|d d d d d d d |j        d         f         }|
|z   }
t          j                            |
dt          j	                  
                    |j                  }
t          j                            |
|| j                  }
t          j        |
|	          }|                    dd                                          }||
fS )Nr   r   r.   )r-   rb   )pr  r   )r   r   r2   matmulr   r:   r   r4   r5   rc   r%   rb   r  r  
contiguous)r%  r&  r'  r(  r    r  r  r  r  r  r"  causal_maskr!  s                r)   r  r    s    3 ;<<JUF$?@@L<z';';Aq'A'ABBWLL!$QQQ111.D
0@0D.D%DE#k1 =((2U](SSVVW\WbccL=((6?([[L,|\::K''1--88::K$$r+   c                       e Zd Zdedef fdZ eddd          	 	 	 	 	 	 	 	 dd
ej        de	ej                 de	ej
                 de	e         de	e         de	e         de	ej
                 de	e         de	eej        ej        f                  deej        e	eej        ej        f                  f         fd            Z xZS )GraniteMoeDecoderLayerrt   r   c                 b   t                                                       |j        | _        t          ||          | _        |j        dk    rt          |          | _        t          |j        |j	                  | _
        t          |j        |j	                  | _        |j        | _        d S )N)rt   r   r   r^   )rV   rW   r]   r   	self_attnr   r   block_sparse_moerS   rms_norm_epsinput_layernormpost_attention_layernormresidual_multiplierr  s      r)   rW   zGraniteMoeDecoderLayer.__init__  s    !-,FiPPP#a''$1&$9$9D!01CI\]]](9&:LRXRe(f(f(f%#)#=   r+   r	  r
  r  r  NFrf   r    r   output_attentionsr  r  output_router_logitsr  r!   c
                    |}|                      |          } | j        d||||||||	d|
\  }}||| j        z  z   }|}|                     |          }|                     |          \  }}||| j        z  z   }|f}|r||fz  }|r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            output_router_logits (`bool`, *optional*):
                Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
                should not be returned during inference.
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )rf   r    r   r
  r9  r  r  r  r$   )r6  r3  r8  r7  r4  )r\   rf   r    r   r
  r9  r  r  r:  r  r  residualself_attn_weightsr   outputss                  r)   ri   zGraniteMoeDecoderLayer.forward  s    N !,,];; ,:4> 
,
')%+/) 3
,
 
,
 
,
 
,
(( !=43K#KK !55mDD'+'<'<]'K'K$} =43K#KK " 	,)++G 	(''Gr+   )NNNFFNFN)rl   rm   rn   r   r?   rW   r   r2   r   r   r#  r	   r$  r0   FloatTensorri   ro   rp   s   @r)   r0  r0    ss       
>/ 
>C 
> 
> 
> 
> 
> 
> _%0A6RRR 2637+/,1$)59/4KOH H|H !.H u/0	H
 "%H $D>H D>H !!12H 'tnH &eEL%,,F&GHH 
u (51BEDU1U+V"WW	XH H H SRH H H H Hr+   r0  c                   J     e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZ fdZ xZS )GraniteMoePreTrainedModelrt   modelTr0  r
  Fc                     t                                          |           t          |t                    r-|j        j                            d| j        j                   d S d S )Nr  )r8   std)	rV   _init_weightsr/   r   rZ   datanormal_rt   initializer_range)r\   r%  r_   s     r)   rE  z'GraniteMoePreTrainedModel._init_weights^  sd    f%%%f788 	TM&&CT[5R&SSSSS	T 	Tr+   )rl   rm   rn   r   r   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraphrE  ro   rp   s   @r)   rA  rA  R  s~         &*#12#4"5N"T T T T T T T T Tr+   rA  c                       e Zd Zdef fdZe	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej	                 deej                 dee
eeej                 f                  deej                 d	ee         d
ee         dee         dee         dee         deej                 de
eef         fd            Z	 dde
ej	        df         dej	        dej	        ded
ef
dZedej	        dededej        dej	        defd            Z xZS )GraniteMoeModelrt   c                    t                                                     j        | _        j        | _        t          j        j        j        | j                  | _        t          j	        fdt          j                  D                       | _        t          j        j                  | _        d| _        j        | _        j        | _        j        | _        | j        | j        z  | _        j        | _        j        | _        j        | _        | j        dk    rt1                    nd | _        |                                  d S )Nc                 0    g | ]}t          |          S r$   )r0  )r&   r   rt   s     r)   r*   z,GraniteMoeModel.__init__.<locals>.<listcomp>m  s$    hhh9#FI66hhhr+   r2  Frope)rV   rW   pad_token_idpadding_idx
vocab_sizer   	Embeddingr]   embed_tokens
ModuleListr   rK   layersrS   r5  normgradient_checkpointingembedding_multiplierr   r   r   r~   
rope_thetaposition_embedding_typerr   
rotary_emb	post_initr   s    `r)   rW   zGraniteMoeModel.__init__f  s7      !. +L):F<NPTP`aamhhhhfNfHgHghhh
 
 &f&8f>QRRR	&+#$*$?!!-3(DN:'-'E$ +'-'E$?C?[_e?e?e3F;;;ko 	r+   N	input_idsr    r   r
  inputs_embedsr  r9  output_hidden_statesr:  return_dictr  r!   c                     ||n| j         j        }||n| j         j        }||n| j         j        }|
|
n| j         j        }
|d u |d uz  rt          d          | j        r%| j        r|rt          	                    d           d}|| 
                    |          }|| j        z  }|r|t          | j                   }|B||                                nd}t          j        |||j        d         z   |j                  }||                    d          }|                     |||||          }|}d }| j        |                     ||          }|rdnd }|rdnd }|	rdnd }| j        D ]B}|r||fz  } |||||||||	|		  	        }|d         }|r||d         fz  }|	r||d
         fz  }C|                     |          }|r||fz  }|
st/          d ||||fD                       S t1          |||||          S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)rt   r   r   r1   r$   )r    r   r
  r9  r  r  r:  r  r.   c              3      K   | ]}||V  	d S r   r$   )r&   vs     r)   	<genexpr>z*GraniteMoeModel.forward.<locals>.<genexpr>  s1        bcbobobobobo r+   )last_hidden_stater
  rf   
attentionsr   )rt   r9  re  r  use_return_dictr  r]  r  r   r   rY  r^  r
   get_seq_lengthr2   aranger:   r1   r@   _update_causal_maskra  r[  r\  r0   r   )r\   rc  r    r   r
  rd  r  r9  re  r:  rf  r  r  past_seen_tokensr.  rf   r  all_hidden_statesall_self_attnsall_router_logitsdecoder_layerlayer_outputss                         r)   ri   zGraniteMoeModel.forward  s     2C1N--TXT_Tq$8$D  $+Jj 	 "+!6IIDK<Q	%0%<kk$+B]-t";< 	[YZZZ& 	4= 	Y 	j   I  --i88M%(AA 	?0*$+>>>O!CRC^==???de"\ "2]5H5K"KTaTh  N )33A66L..M>?L]
 

 &"?&"&//-"N"N #7@BBD0:d"6@BBD![ 	: 	:M# 6!m%55!)M*) /"3#-%9$7
 
 
M *!,M  6=#3"55# :!mB&7%99!		-00   	2-!11 	  )?<M~^      &+++%+
 
 
 	
r+   Fr   input_tensorc           	      $   | j         j        dk    r||dk                                    r|S d S | j         j        dk    r+t          |t          j                  rt          |          }|S ||                                nd}||j        nd}| j         j        dk    r#|s!|st          j
        |||| j                  rd S |j        }|j        d         }	|r|                                }
n/t          |t          j                  r|j        d	         n||	z   dz   }
|                     ||	|
|||j        d         
          }| j         j        dk    r@|>|j        j        dv r0|s.t	          j        |          j        }t          j        ||          }|S )Nflash_attention_2r  flex_attentionr   Fsdpa)rd  past_key_values_lengthis_trainingr   r.   )rJ   target_lengthrb   r  rI   )cudaxpunpu)rt   r  anyr/   r2   r   r   ro  is_compileabler   _ignore_causal_mask_sdpar  rb   r:   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr1   rx   finfomin_unmask_unattended)r\   r    rx  r  r
  r9  rr  using_compilable_cacherb   rJ   r  r.  	min_dtypes                r)   rq  z#GraniteMoeModel._update_causal_mask  s    ;+/BBB)~/D.I.I.K.K)%%4;+/???.%,77 M!<^!L!L!!
 @O?Z?99;;;`aCRC^!?!?di ;+v55>T5]n5%>*'7 M	    t"&,Q/! 	+??AAMM nel;;<$R((%7!;  PP+')#)!, Q 
 
 K,66*%*.DDD% E E**.I0CKQZ[[Kr+   rJ   r  rb   rI   c                    | |                                  dk    r| }nMt          j        |          j        }t          j        ||f|||j                  }|dk    rt          j        |d          }|t          j        ||j                  |                    dd          k    z  }|ddddddf         	                    |ddd          }| |
                                }| j        d         }	|ddddddd|	f         | ddddddf                             |j                  z   }
|
dk    }
|ddddddd|	f                             |
|          |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuerb   r1   r   )diagonalrh  r.   r   )r-   r2   r  r  fullr1   triurp  r<   r;   cloner:   r%   masked_fill)r    rJ   r  rb   r  rI   r  r.  r  mask_lengthpadding_masks              r)   r  zEGraniteMoeModel._prepare_4d_causal_attention_mask_with_cache_position/  s   > %.*<*<*>*>!*C*C(KKE**.I* -0Ye\j\q  K !###jqAAA5<n>STTTWeWmWmnprsWtWtttK%dD!!!QQQ&67>>z1bRTUUK))//11,226*111aaaL[L+@ANSTSTSTVZ\`bcbcbcScDdDgDg&E E    ,q05@AAAqqq,;,AV5W5c5c )6 6AAAqqq!!!\k\12 r+   )NNNNNNNNNNN)F)rl   rm   rn   r   rW   r   r   r2   r#  r   r   r	   listr?  r$  r0   r   ri   rq  staticmethodr?   rb   r  ro   rp   s   @r)   rQ  rQ  d  s9       /      2  151537KO59$(,0/3/3&*59h
 h
E,-h
 !.h
 u/0	h

 "%tE4E/F(F"GHh
   12h
 D>h
 $D>h
 'tnh
 'tnh
 d^h
 !!12h
 
u--	.h
 h
 h
 ^h
b #(B BelK78B lB 	B
 B  B B B BH 444 4 {	4
 4 4 4 4 \4 4 4 4 4r+   rQ  c                        e Zd ZdgZdef fdZe	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej	                 deej
                 deej	                 d	eeeeej                 f                  d
eej                 deej	                 dee         dee         dee         dee         dee         deej	                 deeej
        f         deeef         fd            Z xZS )GraniteMoeForCausalLMzlm_head.weightrt   c                 F   t                                          |           t          |          | _        |j        | _        t          j        |j        |j        d          | _        |j	        | _	        |j
        | _        |j        | _        |                                  d S )NFr   )rV   rW   rQ  rB  rW  r   r   r]   lm_headrouter_aux_loss_coefr   r   r   rb  r   s     r)   rW   zGraniteMoeForCausalLM.__init__k  s       $V,,
 +y!3V5FUSSS$*$?!!3#)#=  	r+   Nr   rc  r    r   r
  rd  labelsr  r9  re  r:  rf  r  logits_to_keepr!   c                     ||n| j         j        }|
|
n| j         j        }
|	|	n| j         j        }	||n| j         j        } | j        d||||||||	|
||d|}|d         }t          |t                    rt          | d          n|}| 	                    |dd|ddf                   }|| j         j
        z  }d}|/|                                } | j        ||fd| j         j        i|}d}|
rRt          |r|j        n|d         | j        | j        |          }|%|| j        |                    |j                  z  z  }|s |f|dd         z   }|
r|f|z   }||f|z   n|S t+          ||||j        |j        |j        |j                  S )	al  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeForCausalLM

        >>> model = GraniteMoeForCausalLM.from_pretrained("ibm/PowerMoE-3b")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rc  r    r   r
  rd  r  r9  re  r:  rf  r  r   rW  r.   r   )lossaux_lossr   r
  rf   rm  r   r$   )rt   r9  r:  re  rn  rB  r/   r?   slicer  logits_scalingr9   loss_functionrW  rQ   r   r   r   r  r%   r1   r   r
  rf   rm  )r\   rc  r    r   r
  rd  r  r  r9  re  r:  rf  r  r  r  r>  rf   slice_indicesr   r  r  outputs                         r)   ri   zGraniteMoeForCausalLM.forwardx  sO   P 2C1N--TXT_Tq$8$D  $+Jj 	 %9$D  $+Jj 	 &1%<kk$+B] $* 
)%+'/!5!5#)
 
 
 
   
8B>SV8W8Wk~ot444]kmAAA}aaa,?@AA$+44\\^^F%4%   ;1 	 D  	M/)4E%%'"+ (	 H !1HKK4L4LLL 	DY,F# ."v-'+'7D7V##VC(#3!/)!/
 
 
 	
r+   )NNNNNNNNNNNNr   )rl   rm   rn   _tied_weights_keysr   rW   r   r   r2   r#  r   r   r	   r  r?  r$  r?   r0   r   ri   ro   rp   s   @r)   r  r  h  s       *+/        151537KO59-1$(,0/3/3&*5934k
 k
E,-k
 !.k
 u/0	k

 "%tE4E/F(F"GHk
   12k
 )*k
 D>k
 $D>k
 'tnk
 'tnk
 d^k
 !!12k
 c5</0k
  
u//	0!k
 k
 k
 ^k
 k
 k
 k
 k
r+   r  )r  rQ  rA  )Nr   N)Nr   )r  )Btypingr   r   r   r2   torch.nn.functionalr   r4   r   activationsr   cache_utilsr	   r
   
generationr   modeling_attn_mask_utilsr   modeling_layersr   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   utilsr   r   r   utils.deprecationr   configuration_granitemoer   !torch.nn.attention.flex_attentionr   integrations.flex_attentionr   
get_loggerrl   r   r   r0   r?   rQ   ModulerS   rr   r   r   r   r   r   r   r   r9   r  r0  rA  rQ  r  __all__r$   r+   r)   <module>r     s    - , , , , , , , , ,                 ! ! ! ! ! ! . . . . . . . . ) ) ) ) ) ) > > > > > > 9 9 9 9 9 9 j j j j j j j j j j K K K K K K K K F F F F F F F F J J J J J J J J J J 0 0 0 0 0 0 6 6 6 6 6 6  !! K;;;;;;JJJJJJ 
	H	%	% "&
-1	S& S&u|U5<%8$>?S&#S& U\*	S&
 5<S& S& S& S&nJ J J J J	 J J J*!< !< !< !< !<	 !< !< !<J( ( (   8* * * * *	 * * *\-S -S -S -S -S29 -S -S -S`5+ 5+ 5+ 5+ 5+BI 5+ 5+ 5+r	UU\ 	U# 	U%, 	U 	U 	U 	UT) T) T) T) T)") T) T) T)| % %I%<% 
% <	%
 U\*% % % % % %6V V V V V7 V V Vr T T T T T T T T" @ @ @ @ @/ @ @ @F|
 |
 |
 |
 |
5 |
 |
 |
~ T
S
Sr+   