
    .`i?                     l   d dl mZ d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d dlmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;  G d de	j<                  Z= G d de	j<                  Z> G d de	j<                  Z?e G d  d!e	j<                              Z@ G d" d#e	j<        e2e1          ZA e4d$%           G d& d'eA                      ZBdS )(    )Iterable)partial)islice)AnyN)nn)PretrainedConfig)	Attention)support_torch_compile)CacheConfig
VllmConfig)get_pp_groupget_tensor_model_parallel_rank$get_tensor_model_parallel_world_sizesplit_tensor_along_last_dim tensor_model_parallel_all_gather)
SiluAndMul)RMSNorm)MergedColumnParallelLinearQKVParallelLinearRowParallelLinear)LogitsProcessor)pooler_for_token_classify)QuantizationConfig)get_rope)ParallelLMHeadVocabParallelEmbedding)default_weight_loader)IntermediateTensors   )SupportsLoRA
SupportsPP)default_pooling_type)StageMissingLayeris_pp_missing_parameter'make_empty_intermediate_tensors_factorymake_layersmaybe_prefixno_init_weightsc                   J     e Zd Z	 	 ddededededz  deddf fd	Zd
 Z xZS )InternLM2MLPN hidden_sizeintermediate_size
hidden_actquant_configprefixreturnc                    t                                                       t          ||gdz  d|| d          | _        t	          ||d|| d          | _        |dk    rt          d| d          t                      | _        d S )	N   Fz.gate_up_projbiasr/   r0   z.w2siluzUnsupported activation: z!. Only silu is supported for now.)	super__init__r   gate_up_projr   w2
ValueErrorr   act_fn)selfr,   r-   r.   r/   r0   	__class__s         x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/internlm2.pyr8   zInternLM2MLP.__init__6   s     	6!#%+++
 
 
 $%>>>
 
 
 X:XXX   !ll    c                     |                      |          \  }}|                     |          }|                     |          \  }}|S N)r9   r<   r:   )r=   xgate_up_s       r?   forwardzInternLM2MLP.forwardS   sB    &&q))
KK  wwqzz1r@   )Nr+   )	__name__
__module____qualname__intstrr   r8   rF   __classcell__r>   s   @r?   r*   r*   5   s         37# ## # 	#
 )4/# # 
# # # # # #:      r@   r*   c                        e Zd Z	 	 	 	 	 ddedededeeef         dz  ded	edz  d
edz  deddf fdZ	de
j        fdZde
j        de
j        de
j        fdZ xZS )InternLM2AttentionN    r+   r,   	num_headsnum_kv_headsrope_parametersmax_position_embeddingscache_configr/   r0   r1   c	           
         t                                                       || _        t                      | _        t                      | _        || _        | j        | j        z  dk    sJ | j        | j        z  | _        || _	        | j	        | j        k    r| j	        | j        z  dk    sJ n| j        | j	        z  dk    sJ t          d| j	        | j        z            | _        || j        z  | _        | j        | j        z  | _        | j        | j        z  | _        t          | j        | j        z            | _        | j        dz  | _        || _        t'          || j        | j        | j	        d|| d          | _        t+          | j        | j        z  |d|| d          | _        t/          | j        ||          | _        t3          | j        | j        | j        | j        ||| d	
          | _        d S )Nr   r   g      Fz.wqkvr4   z.wo)max_positionrS   z.attn)rR   rU   r/   r0   )r7   r8   r,   r   tp_sizer   tp_ranktotal_num_headsrQ   total_num_kv_headsmaxrR   head_dimq_sizekv_sizerJ   key_value_groupsscalingrT   r   wqkvr   wor   
rotary_embr	   attn)
r=   r,   rQ   rR   rS   rT   rU   r/   r0   r>   s
            r?   r8   zInternLM2Attention.__init__[   s    	&;==577(#dl2a7777-="."dl22 *T\9Q>>>>> <$"99Q>>>>4#:dl#JKK#t';;nt}4(4=8 #DNT5F$F G G}d*'>$%M #%###
 
 
	 $ 4=0%>>>
 
 
 #M0+
 
 

 NML*%%###
 
 
			r@   qkvc                 n   |j         d         }| j        dk    r{| j        | j        | j        g| j        z  }t	          |          }t          j        ||d          }|d d d         |dd d         z   |dd d         z   }t          j        |d          }|                    || j	        | j
        dz   | j                  }t          j        || j
        ddgd          \  }}}|                    || j        | j        z            }|                    || j        | j        z            }|                    || j        | j        z            }| j        dk    r]t          t          | j                  } ||          | j                 } ||          | j                 } ||          | j                 }|||fS )	Nr   r   )dim   r3   )num_partitions)shaperX   r^   r_   r   torchsplitcatviewr[   r`   r]   reshaper   r   rY   )r=   rf   seq_lenqkv_mapqkvsplitters           r?   	split_qkvzInternLM2Attention.split_qkv   s   )A,<!{DL$,?$,NG2377C+c7333Cccc(SAY&QTT2C)CR(((ChhT,d.Ca.G
 
 +cD$91a#@bIII1aIIgt{T\9::IIgt|dl:;;IIgt|dl:;;<!:4<XXXHDL)ADL)ADL)A!Qwr@   	positionshidden_statesc                     |                      |          \  }}|                     |          \  }}}|                     |||          \  }}|                     |||          }|                     |          \  }	}|	S rB   )rb   ry   rd   re   rc   )
r=   rz   r{   rf   rE   ru   rv   rw   attn_outputoutputs
             r?   rF   zInternLM2Attention.forward   sx    
 =))Q..%%1ay!Q//1ii1a((GGK((	r@   )NrP   NNr+   )rG   rH   rI   rJ   dictrK   r   r   r   r8   rn   Tensorry   rF   rL   rM   s   @r?   rO   rO   Z   s&        26'++/26A
 A
A
 A
 	A

 c3h$.A
 "%A
 "D(A
 )4/A
 A
 
A
 A
 A
 A
 A
 A
FU\    0
<
 |
 
	
 
 
 
 
 
 
 
r@   rO   c                        e Zd Z	 	 	 ddededz  dedz  deddf
 fdZd	ej	        d
ej	        dej	        dz  de
ej	        ej	        f         fdZ xZS )InternLMDecoderLayerNr+   configrU   r/   r0   r1   c                    t                                                       |j        | _        t          |dd          }t	          | j        |j        |j        |j        |||| d          | _        t          | j        |j
        |j        || d          | _        t          |j        |j                  | _        t          |j        |j                  | _        d S )NrT   rP   z
.attention)r,   rQ   rR   rS   rT   rU   r/   r0   z.feed_forward)r,   r-   r.   r/   r0   eps)r7   r8   r,   getattrrO   num_attention_headsnum_key_value_headsrS   	attentionr*   r-   r.   feed_forwardr   rms_norm_epsattention_normffn_norm)r=   r   rU   r/   r0   rT   r>   s         r?   r8   zInternLMDecoderLayer.__init__   s     	!-")&2KT"R"R+(03"2$;%%(((	
 	
 	
 )($6(%+++
 
 
 &f&8f>QRRR 28KLLLr@   rz   r{   residualc                     ||}|                      |          }n|                      ||          \  }}|                     ||          }|                     ||          \  }}|                     |          }||fS )N)rz   r{   )r   r   r   r   )r=   rz   r{   r   s       r?   rF   zInternLMDecoderLayer.forward   s     $H //>>MM&*&9&9-&R&R#M8' ' 
 
 #'--x"H"Hx))-88h&&r@   )NNr+   )rG   rH   rI   r   r   r   rK   r8   rn   r   tuplerF   rL   rM   s   @r?   r   r      s         ,026M M M "D(M )4/	M
 M 
M M M M M M<'<' |' ,%	'
 
u|U\)	*' ' ' ' ' ' ' 'r@   r   c                        e Zd Zdeddededee         f fdZdej	        dej	        fd	Z
	 	 ddej	        dej	        ded
z  dej	        d
z  dej	        ez  f
dZ xZS )InternLM2Modelr+   )r0   
layer_typevllm_configr0   r   c                   t                                                       |j        j        |j        |j        | _        j        | _        t          j        j	                  | _
        t          j        fd| d          \  | _        | _        | _        t!          j	        j                  | _        t'          ddgj	                  | _        d S )Nc                 "     |           S )Nr0    )r0   rU   r   r   r/   s    r?   <lambda>z)InternLM2Model.__init__.<locals>.<lambda>  s!    ::l6   r@   z.layersr   r   r{   r   )r7   r8   model_config	hf_configrU   r/   r   
vocab_sizer   r,   tok_embeddingsr&   num_hidden_layersstart_layer	end_layerlayersr   r   normr%   make_empty_intermediate_tensors)r=   r   r0   r   rU   r   r/   r>   s      `@@@r?   r8   zInternLM2Model.__init__   s    	)3"/"/ +4
 
 9D$       %%%9
 9
 9
5$.$+ F.F4GHHH	/Vj)6+=0
 0
,,,r@   	input_idsr1   c                 ,    |                      |          S rB   )r   r=   r   s     r?   embed_input_idszInternLM2Model.embed_input_ids  s    ""9---r@   Nrz   intermediate_tensorsinputs_embedsc                 p   t                      j        r||}n|                     |          }d }n|J |d         }|d         }t          | j        | j        | j                  D ]} ||||          \  }}t                      j        st          ||d          S | 	                    ||          \  }}|S )Nr{   r   )r{   r   )
r   is_first_rankr   r   r   r   r   is_last_rankr   r   )	r=   r   rz   r   r   r{   r   layerrE   s	            r?   rF   zInternLM2Model.forward  s     >>' 		8( - $ 4 4Y ? ?HH'3330AM+J7HDK)94>JJ 	P 	PE&+eI}h&O&O#M88~~* 	&"/XFF    99]H==qr@   NN)rG   rH   rI   r   r   rK   typer8   rn   r   r   r   rF   rL   rM   s   @r?   r   r      s        1E
 
 
  
 	

 -.
 
 
 
 
 
>. .%, . . . . <@-1 < < 2D8	
 |d* 
+	+       r@   r   c                   4    e Zd ZdgddgdZdeddeded	ee         f fd
Zde	j
        de	j
        fdZ	 dde	j
        de	j
        dedz  de	j
        dz  de	j
        f
dZde	j
        de	j
        dz  fdZdeeee	j
        f                  dee         fdZ xZS )InternLM2ForCausalLMrb   w1w3)rb   r9   r+   r0   
model_typer   r0   r   c          	         t                                                       |j        j        }|j        }|| _        || _         ||t          |d                    | _        t          |j	        |j
        |t          |d                    | _        | j        j        r| j        j        j        | j        _        t          |j	                  | _        | j        j        | _        d S )Nmodel)r   r0   r~   )r/   r0   )r7   r8   r   r   r/   r   r'   r   r   r   r,   r~   tie_word_embeddingsr   weightr   logits_processorr   )r=   r   r0   r   r   r/   r>   s         r?   r8   zInternLM2ForCausalLM.__init__>  s     	)3"/(Z#L,I,I
 
 

 %%11	
 
 
 ;* 	B!%!:!ADK /0A B BJ6 	,,,r@   r   r1   c                 6    | j                             |          S rB   )r   r   r   s     r?   r   z$InternLM2ForCausalLM.embed_input_ids\  s    z)))444r@   Nrz   r   r   c                 6    |                      ||||          }|S rB   )r   )r=   r   rz   r   r   r{   s         r?   rF   zInternLM2ForCausalLM.forward_  s)     

y"6
 
 r@   r{   c                 <    |                      | j        |          }|S rB   )r   r~   )r=   r{   logitss      r?   compute_logitsz#InternLM2ForCausalLM.compute_logitsk  s      &&t{MBBr@   weightsc                 2   ddg}t          |                                           }t                      }|D ]\  }}d|v r
|D ]i\  }}}	||vr|                    ||          }|                    d          r||vr;t          ||           rL||         }
|
j        } ||
||	            nU|                    d          r||vrt          ||           r||         }
t          |
dt                    } ||
|           |	                    |           |S )N)r9   r   r   )r9   r   r   zrotary_emb.inv_freqz.biasweight_loader)
r   named_parameterssetreplaceendswithr$   r   r   r   add)r=   r   stacked_params_mappingparams_dictloaded_paramsnameloaded_weight
param_nameweight_nameshard_idparamr   s               r?   load_weightsz!InternLM2ForCausalLM.load_weightsr  ss    &%"

 4002233"%%%#* 	$ 	$D-$,,5K 4 41
Kd**||K<<==)) d+.E.E*466 #D) % 3e]H=== ==)) d+.E.E*466 #D) '@U V Ve]333d####r@   rB   )rG   rH   rI   packed_modules_mappingr   r   rK   r   r8   rn   r   r   r   rF   r   r   r   r   r   rL   rM   s   @r?   r   r   8  sw       t  +9
 
 
  
 	

 (
 
 
 
 
 
<5 5%, 5 5 5 5 .2
 
<
 <
 2D8	

 |d*
 

 
 
 
| 
	   "HU33D-E$F "3s8 " " " " " " " "r@   r   ALL)tok_pooling_typec                        e Zd ZdZdeddededee         f fdZ	 	 dd	e	j
        d
e	j
        dedz  de	j
        dz  de	j
        ez  f
dZ xZS )InternLM2ForRewardModelTr+   r   r   r0   r   c                   t          | d t          t          f          5  t                                          |||           d d d            n# 1 swxY w Y   |j        j        }|j        j        | _        t          |j	        ddd| j        t          |d          d          | _        |j        j        }|J t          |          | _        d S )Nc                 "    t          d|           S )Nr~   )r#   )mods    r?   r   z2InternLM2ForRewardModel.__init__.<locals>.<lambda>  s    )(C88 r@   )targets)r   r0   r   r   Fv_head)r5   input_is_parallelparams_dtyper0   return_bias)r(   r   r   r7   r8   r   r   
head_dtyper   r,   r'   r   pooler_configr   pooler)r=   r   r0   r   r   r   r>   s         r?   r8   z InternLM2ForRewardModel.__init__  s,    88$n5
 
 
 		 		
 GG'%    		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 )3%2='#11
 
 
 $0>(((/>>s   %AAANr   rz   r   r   r1   c                     |                      ||||          }|                    | j                  }|                     |          }|S rB   )r   tor   r   )r=   r   rz   r   r   r{   r   s          r?   rF   zInternLM2ForRewardModel.forward  sN     

y"6
 
 &((99]++r@   r   )rG   rH   rI   is_pooling_modelr   r   rK   r   r8   rn   r   r   rF   rL   rM   s   @r?   r   r     s         +9"? "? "?  "? 	"?
 ("? "? "? "? "? "?P <@-1 < < 2D8	
 |d* 
+	+       r@   r   )Ccollections.abcr   	functoolsr   	itertoolsr   typingr   rn   r   transformersr   vllm.attention.layerr	   vllm.compilation.decoratorsr
   vllm.configr   r   vllm.distributedr   r   r   r   r   %vllm.model_executor.layers.activationr   $vllm.model_executor.layers.layernormr   !vllm.model_executor.layers.linearr   r   r   +vllm.model_executor.layers.logits_processorr   )vllm.model_executor.layers.pooler.tokwiser   'vllm.model_executor.layers.quantizationr   +vllm.model_executor.layers.rotary_embeddingr   3vllm.model_executor.layers.vocab_parallel_embeddingr   r   -vllm.model_executor.model_loader.weight_utilsr   vllm.sequencer   
interfacesr    r!   interfaces_baser"   utilsr#   r$   r%   r&   r'   r(   Moduler*   rO   r   r   r   r   r   r@   r?   <module>r      s   % $ $ $ $ $                          ) ) ) ) ) ) * * * * * * = = = = = = / / / / / / / /              = < < < < < 8 8 8 8 8 8         
 H G G G G G O O O O O O F F F F F F @ @ @ @ @ @        P O O O O O - - - - - - 0 0 0 0 0 0 0 0 1 1 1 1 1 1               " " " " "29 " " "Jf f f f f f f fR3' 3' 3' 3' 3'29 3' 3' 3'l ; ; ; ; ;RY ; ; ;|\ \ \ \ \29j, \ \ \~ u---3 3 3 3 32 3 3 .-3 3 3r@   