.`i3i ( U d Z ddlmZmZmZmZ ddlmZmZm Z m
Z
ddl Z ddl m Z ddl
mZ ddlmZ ddlmZmZmZmZ dd lmZ dd
lmZ dd lmZm Z dd l m Z m Z m Z dd
l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z:m;Z; e j< d Z= G d de. Z> G d de. Z?e>e?z Z@e
eAd< deeBe jC f fdZD G d d e$ ZE G d d e4 ZF G d d e6 ZG G d! d"e3eG ZH G d# d$e5eG ZI G d% d&e jJ ZK G d' d(e jJ ZL G d) d*e ZM ejN eIeGeH+ G d, d-e2 ZOdS ).zCInference-only MiniCPM-O model compatible with HuggingFace weights. )CallableIterableMappingSequence) AnnotatedAnyLiteral TypeAliasN)nn) BatchFeature)BaseModelOutputWithPast)ACT2FNWhisperAttention
WhisperConfigWhisperEncoder)
VllmConfig)BaseDummyOptions)MULTIMODAL_REGISTRYMultiModalKwargsItems)MultiModalDataDictMultiModalFieldConfig
NestedTensors) AudioItemAudioProcessorItemsDictEmbeddingItems ModalityDataModalityDataItemsMultiModalDataItemsMultiModalDataParser)PromptReplacement PromptUpdatePromptUpdateDetails) TensorSchema TensorShape )_MAX_FRAMES_PER_VIDEO MiniCPMV2_6MiniCPMVDummyInputsBuilder MiniCPMVMultiModalDataParserMiniCPMVMultiModalProcessorMiniCPMVProcessingInfo_minicpmv_field_config)AutoWeightsLoadercast_overflow_tensors maybe_prefixcpuc e Zd ZU dZdZed ed< eej e
ej z e ddddh f ed< eej e
ej z e dd f ed
< d S ) MiniCPMOAudioFeatureInputsz
Dimensions:
- bns: Batch size * number of audios * number of slices
- bn: Batch size * number of audios
- c: Number of channels
- l: Length
- s: Number of slices
audio_featurestypebnscl dynamic_dimsbnsaudio_feature_lensN __name__
__module__ __qualname____doc__r4 r __annotations__r torchTensorlistr$ w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/models/minicpmo.pyr2 r2 O s '7D'"
#666
tEL)) E33%888 : "
tEL)) D# rG r2 c e Zd ZU dZdZed ed< eej e
ej z e ddddh f ed< dS ) MiniCPMOAudioEmbeddingInputsz
Dimensions:
- bn: Batch size * number of audios
- s: Number of slices
- h: Hidden size (must match language model backbone)
Length of each slice may vary, so pass it as a list.
audio_embedsr4 r: r; hr8 Nr= rF rG rH rJ rJ o s| %3D'.
!222
tEL)) D#s#777 9 rG rJ MiniCPMOAudioInputs hf_inputsc t di t | t j d t j d t j d dS )Naudio)r3 r< rK rF )dictr, r batched)rN s rH _minicpmo_field_configrS sb
+
+ ,4W==08AA*27;; rG c e Zd Zdeeej f deeeej f geeef f ddf fd Z xZ
S )MiniCPMOAudioEmbeddingItemsdatafields_factoryreturnNc T t |ddh| d S )NimagerK )modalityrequired_fieldsrW )super__init__)selfrV rW __class__s rH r^ z$MiniCPMOAudioEmbeddingItems.__init__ s? + ,)
rG ) r> r? r@ r strrC rD r r r^
__classcell__r` s @rH rU rU s
c5<'(
!
S%,&
' ( C.. /
1
rG rU c j e Zd Zdeeej f ee z de e
e
f dz f fd Z xZ S ) MiniCPMOMultiModalDataParserrV rX Nc t |t rt |t S t | S )N)rW )
isinstancerQ rU rS r] _parse_audio_data)r_ rV r` s rH rh z.MiniCPMOMultiModalDataParser._parse_audio_data sN dD ! ! . 5
ww((...rG )
r> r? r@ rQ ra rC rD r r r r rh rb rc s @rH re re ss
/3 $% Y(??
/
38 $t +
/
/
/
/
/
/
/
/
/
/rG re c e Zd ZdZdeeedz f f fd Z ddeded edefd
Z defd Z
defd Z defd
Z defdZ
defdZdefdZdedefdZdedeeef defdZ xZS )MiniCPMOProcessingInfo()rX Nc P i t dd iS )NrP )r] get_supported_mm_limits)r_ r` s rH rm z.MiniCPMOProcessingInfo.get_supported_mm_limits s( C%''1133CWdCCCrG Tr%
audio_lens chunk_input chunk_lengthc Z | }| ||| S N)ro rp )get_hf_processorget_audio_placeholder)r_ rn ro rp hf_processors rH rt z,MiniCPMOProcessingInfo.get_audio_placeholder s; ,,.. 11 #% 2
rG c dS )N rF r_ s rH get_default_audio_pool_stepz2MiniCPMOProcessingInfo.get_default_audio_pool_step s qrG c dS )Ni> rF rx s rH get_default_audio_sampling_ratez6MiniCPMOProcessingInfo.get_default_audio_sampling_rate s urG c 4 | j S N)
get_hf_configaudio_chunk_lengthrx s rH get_chunk_lengthz'MiniCPMOProcessingInfo.get_chunk_length s !!##66rG c Z | }d}|dz
dz dz }||z
|z dz S )Nd r% rw )ry )r_ pool_stepfbank_feat_in_chunkcnn_feat_in_chunks rH get_max_audio_tokens_per_chunkz5MiniCPMOProcessingInfo.get_max_audio_tokens_per_chunk sC 4466 ! 01 4 :Q >!I-);a??rG c dS )N rF rx s rH 'get_max_audio_chunks_with_most_featuresz>MiniCPMOProcessingInfo.get_max_audio_chunks_with_most_features s rrG c X | }| |z S r} )r r )r_
num_chunkss rH get_max_audio_tokensz+MiniCPMOProcessingInfo.get_max_audio_tokens s* AACC
2244zAArG r c | }| }t ||z |z dz S )Nr% )r{ r int)r_ r
sampling_ratenum_tokens_per_chunks rH get_audio_len_by_num_chunksz2MiniCPMOProcessingInfo.get_audio_len_by_num_chunks sC <<>>
# B B D D :
-0DDEEIIrG seq_len mm_countsc | dd }| dd }| dd }| |z }| |z }| ||z
|z
}t |t |d z t } t | d S )NrZ r videorP r% )getget_max_image_tokensr get_max_video_framesminmaxr& )
r_ r r
max_images
max_videos
max_audiosmax_image_tokensmax_audio_tokensmax_total_framesmax_frames_per_videos
rH !get_num_frames_with_most_featuresz8MiniCPMOProcessingInfo.get_num_frames_with_most_features s
]]7A..
]]7A..
]]7A..
4466C 4466C 44 & &)9 9
# J 2 2 24I
'+++rG Tr% )r> r? r@
audio_patternr ra r rm boolrt ry r{ r r r r r r rb rc s @rH rj rj s )MDcDj )A D D D D D D !
S 7# 7 7 7 7@ @ @ @ @ Bc B B B BJc Jc J J J J
,, 38$,
, , , , , , , ,rG rj c ~ e Zd Zdeeef def fd Z ddedeeef deeef dz def fd
Z xZ
S ) MiniCPMODummyInputsBuilderr rX c | dd }| j j |z }t | |z S )NrP r )r infor r] get_dummy_text)r_ r
num_audiosaudio_prompt_textsr` s rH r z)MiniCPMODummyInputsBuilder.get_dummy_text sD ]]7A..
!Y 4z Aww%%i003EEErG Nr
mm_optionsc J | dd }| j | j z }|r| d nd }d| ||| i}i t ||| |S )NrP r )lengthr overrides)r r r r{ _get_dummy_audiosr] get_dummy_mm_data) r_ r r r r audio_lenaudio_overrides
audio_mm_datar` s rH r z,MiniCPMODummyInputsBuilder.get_dummy_mm_data s ]]7A..
I = = ? ?i7799
:
6@I*..111T
T++ Z? ,
gg''JGG
rG r} ) r> r? r@ r ra r r r r r rb rc s @rH r r s FS(9 Fc F F F F F F =A
38$
C!1 12T9
rG r c
^ e Zd ZdefdZ ddedededefdZd e ee
f d
e ee
f d e ee
f de ee f fd Z d e ee
f d
e ee
f d e ee
f de ee f f fd
Z
dede ee
f dedee f fd Zdede ee
f de eef fdZ xZS )MiniCPMOMultiModalProcessorrX c P t | j S )N) target_sr)re r r{ rx s rH _get_data_parserz,MiniCPMOMultiModalProcessor._get_data_parser s* +i??AA
rG Tr% rn ro rp c < | j ||| S rr )r rt )r_ rn ro rp s rH get_audio_prompt_textsz2MiniCPMOMultiModalProcessor.get_audio_prompt_texts s, y.. #% /
rG mm_data mm_kwargs
tok_kwargsc | d x}i S | d|i dt t
f }t
|t ri }nt| | j j gt | z dd |D ii |ddi|ddh }d t |d |d D }||d< |S )
NaudiosrP c g | ]}|gS rF rF ).0rP s rH
z>MiniCPMOMultiModalProcessor.process_audios..: s #G#G#GUG#G#G#GrG ro Tr3 r< )promptsr r r out_keysc 0 g | ]\ }}|d d d |f S r} rF )r feat feature_lens rH r z>MiniCPMOMultiModalProcessor.process_audios..B s= ' ' '%D+ QQQ _%' ' 'rG ) r r
parse_mm_data get_itemsrU r rg _base_call_hf_processorr r lenzip)r_ r r r r
parsed_audios audio_inputsunpadded_audio_featuress rH process_audiosz*MiniCPMOMultiModalProcessor.process_audios& s1 kk(++ +F 4I
! ! # #
]GV,
-
-
Yw!<>Q R
S
S m%@ A A E LL 77 01C
4F4FF!#G#G #G#G#GH)r rU r rg r r r summapr get_audio_lengthr" select_textr )r r single_audio_embedsr r r_ s rH get_audio_replacementzNMiniCPMOMultiModalProcessor._get_prompt_updates..get_audio_replacementf s ''57JK F &"=>>
>&,jj&:&:>&J# I A AC!45566 # 3 3H = = &2++I66
rG rP )r[ target replacement)r] _get_prompt_updatesr r r r )r_ r r r base_updatesaudio_placeholderr r` s `` rH r z/MiniCPMOMultiModalProcessor._get_prompt_updatesX s ww22 #9' 3
!I 3 C $
( 1
rG rN c t | S r} )rS )r_ rN r s rH _get_mm_fields_configz1MiniCPMOMultiModalProcessor._get_mm_fields_config s
&i000rG r )r> r? r@ r r r r ra r r objectr r r r r r r! r r r r rb rc s @rH r r s
"6
!
% f%% 3;'% CK( %
m# $ % % % % N
f%
3;'
CK(
m# $
'
%'
!(V 4'
- '
, '
'
'
'
'
'
R1 1 !(V 41
++ , 1 1 1 1 1 1 1 1rG r c L e Zd Zdedef fd Zdej dej fdZ xZS )MultiModalProjectorin_dimout_dimc t t j ||d | _ t j | _ t j ||d | _ d S )NT) in_features out_featuresbias)r] r^ r Linearlinear1ReLUrelulinear2)r_ r r r` s rH r^ z MultiModalProjector.__init__ s\
yV'PTUUU GII yW7QUVVV rG r3 rX c | | | }| | }|S r} )r r r )r_ r3
hidden_statess rH forwardzMultiModalProjector.forward s6 $,,~">">??
]33
rG ) r> r? r@ r r^ rC rD r rb rc s @rH r r sy Ws WS W W W W W W el u| rG r c Z e Zd Zdedef fd Zdej dej dej fdZ xZ S )MiniCPMWhisperEncoderLayerconfig layer_idxc t |j | _ t | j |j |j || | _ t j | j | _
|j | _ t |j
| _ |j | _ t j | j |j | _ t j |j | j | _ t j | j | _ d S )N) embed_dim num_headsdropoutr r )r] r^ d_modelr r encoder_attention_headsattention_dropout self_attnr LayerNormself_attn_layer_normr r activation_function
activation_fnactivation_dropoutr encoder_ffn_dimfc1fc2final_layer_norm)r_ r r r` s rH r^ z#MiniCPMWhisperEncoderLayer.__init__ s
)n 4,
%'L $@$@! ~ #F$> ?"("; 9T^V-CDD9V 3T^DD " T^ < < rG r attention_maskrX c z |}| | }| || \ }}t j || j | j }||z }|}| | }| | | }t j || j | j }|
| }t j || j | j }||z }|j t j
k rt | }|f}|S )N)r r ptraining)r r r
functionalr r r
r r r r dtyperC float16r. )r_ r r residual_outputss rH r z"MiniCPMWhisperEncoderLayer.forward sC
! 11-@@
>>') *
q
-- T\DM .
!=0
--m<<
**488M+B+BCC
-- T 4t} .
//
-- T\DM .
!=0
%- / / 1- @ @M "rG )
r> r? r@ r r r^ rC rD r rb rc s @rH r r s =} = = = = = = =$! |! !
! ! ! ! ! ! ! !rG r c V e Zd Zdef fd Z ddej dej dz defdZ xZ S ) MiniCPMWhisperEncoderr c t t j fdt j D | _ d S )Nc 2 g | ]}t | S ))r )r )r ir s rH r z2MiniCPMWhisperEncoder.__init__.. s6
+6Q???
rG )r] r^ r
ModuleListrangeencoder_layerslayers)r_ r r` s `rH r^ z MiniCPMWhisperEncoder.__init__ sd
m
v 455
rG Ninput_featuresr rX c 0 | | j j j | j j j }t
j | | }t
j | | }| ddd }| j
j }|d |j d d d f }||z }t
j || j | j
}d}t | j D ]N\ }}||fz }d} | j
r!t! j g }
|
| j k rd} | rd } : ||| } | d }O| | }||fz }t) ||
S ) Nr devicer rw r% r rF FT)NN)last_hidden_stater )toconv1weightr r% r r geluconv2permuteembed_positionsshaper r enumerater! rC rand layerdrop
layer_normr
) r_ r" r
inputs_embeds embed_posr encoder_statesidx
encoder_layerto_dropdropout_probability
layer_outputss rH r z MiniCPMWhisperEncoder.forward s (***#)$*2C2J +
**4::n+E+EFF
**4::m+D+DEE
%--aA66
(/ 6
3A 6 6 9: % 1
-- T\DM .
"+DK"8"8 1 1 C +}.> >NG}
#&+jnn#& 77 "G
1 ,
-
!"! !
!.a 0
66
'=*::& +(
rG r} )
r> r? r@ r r^ rC rD r
r rb rc s @rH r r s
}
/33
3
3
t+3
! 3
3
3
3
3
3
3
3
rG r )r dummy_inputsc
e Zd Zg dddgdZededededz fd Zd
d d ed
ef fdZ d
d d ed
efdZ
de e ee
j f dee fdZdedfdededede
j dede
j f dZde
j fdZd edee
j fd Zd ededz fd Zd ede f fd Z d!ede
j ee
j z fd"Z d#e f fd$ Z xZ S )%MiniCPMO)q_projk_projv_proj gate_projup_proj)qkv_proj gate_up_projr[ r rX Nc | d rdS | d rdS | d rdS t d )NrZ z(./)r z()rP rk z0Only image, video or audio modality is supported)
startswith
ValueError)clsr[ r s rH get_placeholder_strz MiniCPMO.get_placeholder_str se w ' ' )(( w ' ' )(( w ' ' )((KLLLrG )prefix vllm_configrK c t || | |d 5 | |t |d | _ d d d d S # 1 swxY w Y d S )N)rL rK rP apm)r] r^ _mark_tower_modelinit_audio_moduler/ rN )r_ rL rK r` s rH r^ zMiniCPMO.__init__* s
[@@@
#
#K
9
9 -- ' VU0K0K . DH s +A33A7:A7c | j j }t | }t |j dz }t j | j j | j j | _ t || j
| _ d| _ |S )N )stride)r r )
r audio_configr r r
r AvgPool1daudio_pool_stepaudio_avg_poolerr r audio_projection_layeraudio_encoder_layer)r_ rL rK rU modelaudio_output_dims rH rP zMiniCPMO.init_audio_module2 s {/ %l33 | ;q @AA " K ' 0K!
!
!
':#T^'
'
'
# $& rG weightsc P t | dg }| | S )Ntts)
skip_prefixes)r- load_weights)r_ r] loaders rH ra zMiniCPMO.load_weights@ s* "4w???""7+++rG rT r size
chunk_sizenum_left_chunksr% num_lookheadc t j |||t j }t j || }||z }|dk rt j | } n t j ||z
d }
|
|z } |dz } t j | |z |z | } t j || d }
| d } | d } |
| k |
| k z }|S )N)r% r )r% r )r r% )r )rC zerosr arange
zeros_likeclamp unsqueeze)r_ rc rd re r% rf ret row_indices
chunk_indices
start_indicesstart_chunk_indicesend_chunk_indices end_indices col_indicess rH subsequent_chunk_maskz MiniCPMO.subsequent_chunk_maskD s k$V5:FFFl4777 #z1
Q ! ,[ 9 9MM #(+mo.MST"U"U"U /* #dk&A A
[
(')+,'-# 'B&D&D5;&D&W&W#&(CCCrG rV c | j j }|d }t |t rt | }|d j d }t
d |D }|d j }|d j }t j
|||f|| } t | D ] \ }
} | j d } | | |
dd | f< n|} |d }
t |
t j r|
d }
t j |
}| j \ }}}|d z
d
z d z }t j d||j |j d || }| d || }||k }| |d d | |d || }| | j j j j | j j j j }|dk rWt/ |d z }| ||d|j }t j |t j | }t7 d
||< | | | j | j }| | }| d d
}| | }| d d
}| ! | \ }} | } t t j } d} tE t |
D ]}
t t j } tE t |
|
D ]/}| # || d | | d d f | d z
} 0| # t j$ | | S )Nr3 r c 3 0 K | ]}|j d V dS )rT N)r. )r items rH z3MiniCPMO.get_audio_hidden_states..w s( < r? r@ packed_modules_mapping classmethodra r rI r r^ rP r r rC rD setra
CPU_DEVICEr% ru
LongTensorr{ r2 rE r r rM r rQ r r r rb rc s @rH r= r=
s
M3 M3 M3: M M M [M BD z 3 KM
C ,HU3 3D-E$F ,3s8 , , , , " )
@De>N D D D D\".\"
el \" \" \" \"|
t #
. f 9(9
U\* *9 9 9 9 %T % % % % % % % % % %rG r= )PrA collections.abcr r r r typingr r r r
rC r transformersr transformers.modeling_outputsr
,transformers.models.whisper.modeling_whisperr r r r vllm.configr vllm.config.multimodalr vllm.multimodalr r vllm.multimodal.inputsr r r vllm.multimodal.parser r r r r r r vllm.multimodal.processingr r! r" vllm.utils.tensor_schemar# r$ minicpmvr& r' r( r) r* r+ r, utilsr- r. r/ r% r r2 rJ rM rB ra rD rS rU re rj r r Moduler r r register_processorr= rF rG rH r s 2 J I I A A A A A A A A A A A A 5 5 5 5 5 5 5 5 5 5 5 5 % % % % % % A A A A A A # " " " " " 3 3 3 3 3 3 F F F F F F F F
? > > > > > > > J I I I I I I I I I
U\%
@ < & !== Y
gc5<.?&@
"4
" / / / / /#? / / / A, A, A, A, A, 3 A, A, A,H
!;T"U r1 r1 r1j
")
4 4 4 4 4 4 4 4n=
=
=
=
=
N =
=
=
@ (' +
p% p% p% p% p%{ p% p%
p% p% p%rG