
    .`i                         d dl mZ d dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZ d dlmZ d dlmZ  G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z  G d dej                  Z! G d dej        j                  Z"dS )    )IterableN)$get_tensor_model_parallel_world_size)divide)
SiluAndMul)MMEncoderAttention)Conv2dLayer)RMSNorm)MergedColumnParallelLinearQKVParallelLinearRowParallelLinear)QuantizationConfig)default_weight_loader)AIMv2Configc                   P     e Zd Zdededef fdZdej        dej        fdZ	 xZ
S )AIMv2SwiGLUFFNconfigquant_configprefixc                    t                                                       |j        }|j        }|j        }t          ||gdz  ||| d          | _        t          ||||| d          | _        t                      | _
        d S )N   .fc13)biasr   r   z.fc2
input_sizeoutput_sizer   r   r   )super__init__intermediate_sizehidden_sizeuse_biasr
   fc13r   fc2r   act_fn)selfr   r   r   hidden_featuresin_featuresr   	__class__s          t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/aimv2.pyr   zAIMv2SwiGLUFFN.__init__   s     	 2(.!%###
 
 
	 %&#%???
 
 
 !ll    xreturnc                     |                      |          \  }}|                     |          }|                     |          \  }}|S N)r!   r#   r"   )r$   r*   _s      r(   forwardzAIMv2SwiGLUFFN.forward4   s<    yy||1KKNNxx{{1r)   __name__
__module____qualname__r   r   strr   torchTensorr/   __classcell__r'   s   @r(   r   r      s{        #!#1C#MP# # # # # #0 %,        r)   r   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )AIMv2PatchEmbedr   c                     t                                                       t          |j        |j        |j        |j        f|j        |j        f          | _        t          |j        |j                  | _	        d S )N)kernel_sizestrideeps)
r   r   r   num_channelsr   
patch_sizeprojr	   rms_norm_epsnorm)r$   r   r'   s     r(   r   zAIMv2PatchEmbed.__init__<   sw    *F,=>%v'89	
 
 
	 F.F4GHHH			r)   r*   r+   c                     |                      |                              d                              dd          }| j                            |          }|S Nr      )rB   flatten	transposerD   forward_nativer$   r*   s     r(   r/   zAIMv2PatchEmbed.forwardF   sI    IIaLL  ##--a33I$$Q''r)   	r1   r2   r3   r   r   r5   r6   r/   r7   r8   s   @r(   r:   r:   ;   sq        I{ I I I I I I %,        r)   r:   c                   H     e Zd Zdef fdZdej        dej        fdZ xZS )AIMv2ViTPreprocessorr   c                     t                                                       |j        |j        z  dz  }t	          |          | _        t          j        t          j	        d||j
        f                    | _        d S rF   )r   r   
image_sizerA   r:   
patchifiernn	Parameterr5   zerosr   	pos_embed)r$   r   num_patchesr'   s      r(   r   zAIMv2ViTPreprocessor.__init__M   sf    (F,==!C)&11ek1k6CU2V&W&WXXr)   r*   r+   c                     |                      |          }|j        \  }}}| j                            |j                  }||d d d |f         z   }|S r-   )rQ   shaperU   todevice)r$   r*   tokensr.   NrU   s         r(   r/   zAIMv2ViTPreprocessor.forwardT   sW    ##,1aN%%fm44	)AAArrE**r)   rL   r8   s   @r(   rN   rN   L   sq        Y{ Y Y Y Y Y Y %,        r)   rN   c                   P     e Zd Zdededef fdZdej        dej        fdZ	 xZ
S )AIMv2Attentionr   r   r   c           	         t                                                       || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j         d| j         d          | j        dz  | _	        t          | j        | j        | j        |j        || d          | _        t          | j        | j        |j        || d          | _        t!                      | _        t%          | j        | j                  | _        t)          | j        | j        | j	                  | _        d S )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      z.qkv)r   	head_sizetotal_num_headsr   r   r   z.projr   )r   r   r   r   	embed_dimnum_attention_heads	num_headshead_dim
ValueErrorscaler   qkv_biasqkvr   r    rB   r   tp_sizer   num_heads_per_partitionr   attnr$   r   r   r   r'   s       r(   r   zAIMv2Attention.__init__]   sZ    	+3$.8=4>)T^;;'%)^' 'N' ' '  
 ]D(
$m N%???
 
 
 &~%###
 
 
	 <=='-dndl'K'K$&($-
 
			r)   r*   r+   c                     |                      |          \  }}|                    dd          \  }}}|                     |||          }|                     |          \  }}|S )N   )dim)ri   chunkrl   rB   )r$   r*   ri   r.   qkvs          r(   r/   zAIMv2Attention.forward   s]    !Q))A2)&&1aIIaAyy||1r)   r0   r8   s   @r(   r^   r^   \   s|        &
!&
1C&
MP&
 &
 &
 &
 &
 &
P %,        r)   r^   c                   P     e Zd Zdededef fdZdej        dej        fdZ	 xZ
S )
AIMv2Blockr   r   r   c                 0   t                                                       t          ||| d          | _        t	          |j        |j                  | _        t          ||| d          | _	        t	          |j        |j                  | _
        d S )Nz.attn)r   r   r>   z.mlp)r   r   r^   rl   r	   r   rC   norm_1r   mlpnorm_2rm   s       r(   r   zAIMv2Block.__init__   s     	"6F6F6F
 
 
	 f0f6IJJJ!ooo
 
 
 f0f6IJJJr)   r*   r+   c                     ||                      | j                            |                    z   }||                     | j                            |                    z   }|S r-   )rl   ry   rJ   rz   r{   rK   s     r(   r/   zAIMv2Block.forward   sS    		$+44Q7788833A66777r)   r0   r8   s   @r(   rw   rw      s        K!K1CKMPK K K K K K %,        r)   rw   c            	       b     e Zd Zddddedededz  def fdZd	ej	        d
ej	        fdZ
 xZS )AIMv2TransformerN require_post_normr   r   r   r   r   c                   t                                                       t          j        fdt	          j                  D                       | _        |r"t          j        j	                  | _
        d S d | _
        d S )Nc           	      >    g | ]}t           d |           S )z.blocks.)r   )rw   ).0ir   r   r   s     r(   
<listcomp>z-AIMv2Transformer.__init__.<locals>.<listcomp>   sF        6<68N8N18N8NOOO  r)   r>   )r   r   rR   
ModuleListrangenum_hidden_layersblocksr	   r   rC   post_trunk_normr$   r   r   r   r   r'   s    `` `r(   r   zAIMv2Transformer.__init__   s     	m     v788  
 
  	(#*6+=6CV#W#W#WD   #'D   r)   r[   r+   c                 h    | j         D ]} ||          }| j        |                     |          }|S r-   )r   r   )r$   r[   blocks      r(   r/   zAIMv2Transformer.forward   sC    [ 	# 	#EU6]]FF+))&11Fr)   )r1   r2   r3   r   r   boolr4   r   r5   r6   r/   r7   r8   s   @r(   r~   r~      s         *.( ( (( )(
  $;( ( ( ( ( ( (*el u|        r)   r~   c            	            e Zd Zddddedededz  def fdZd	ej	        d
ej	        fdZ
deeeej	        f                  d
ee         fdZ xZS )
AIMv2ModelNr   r   r   r   r   r   c                    t                                                       t          |          | _        t	          |||| d          | _        d S )Nz.trunk)r   r   r   )r   r   rN   preprocessorr~   trunkr   s        r(   r   zAIMv2Model.__init__   sZ     	088%%/$$$	
 
 



r)   pixel_valuesr+   c                 Z    |                      |          }|                     |          }|S r-   )r   r   )r$   r   r*   s      r(   r/   zAIMv2Model.forward   s)    l++JJqMMr)   weightsc                    ddg}t          |                                           }t                      }|D ]\  }}|                    d          r| j        j        '|D ]>\  }}}	||vr|                    ||          }||         }
|
j        } ||
||	            n*||         }
t          |
dt                    } ||
|           |
                    |           |S )N)r   z.fc1r   )r   z.fc3rG   ztrunk.post_trunk_normweight_loader)dictnamed_parametersset
startswithr   r   replacer   getattrr   add)r$   r   stacked_params_mappingparams_dictloaded_paramsnameloaded_weight
param_nameweight_nameshard_idparamr   s               r(   load_weightszAIMv2Model.load_weights   s    ! "

 4002233"%%%#* 	$ 	$D-  788J.65K 4 41
Kd**||K<<#D) % 3e]H===#D) '@U V Ve]333d####r)   )r1   r2   r3   r   r   r   r4   r   r5   r6   r/   r   tupler   r   r7   r8   s   @r(   r   r      s         *.
 
 

 )

  $;
 
 
 
 
 
 
"EL U\    HU33D-E$F 3s8        r)   r   )#collections.abcr   r5   torch.nnrR   vllm.distributedr   vllm.distributed.utilsr   %vllm.model_executor.layers.activationr   9vllm.model_executor.layers.attention.mm_encoder_attentionr   vllm.model_executor.layers.convr   $vllm.model_executor.layers.layernormr	   !vllm.model_executor.layers.linearr
   r   r   'vllm.model_executor.layers.quantizationr   -vllm.model_executor.model_loader.weight_utilsr   $vllm.transformers_utils.configs.ovisr   Moduler   r:   rN   r^   rw   r~   r    r)   r(   <module>r      sM   % $ $ $ $ $        A A A A A A ) ) ) ) ) ) < < < < < < X X X X X X 7 7 7 7 7 7 8 8 8 8 8 8         
 G F F F F F O O O O O O < < < < < <    RY   @    bi   "    29    / / / / /RY / / /d       (    ry   >7 7 7 7 7 7 7 7 7 7r)   