.`i+ UddlmZmZddlmZddlmZmZmZm Z m Z m Z ddl m Z mZmZddlmZddlmZddlmZmZmZmZmZmZddlZddlZdd lmZm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'd dl(m)Z)d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4d dl5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;mm?Z?er ddl@mAZAd dlBmCZCneDZAeDZCe"eEZFedeGeHeIZJeGeHeIzZKeeLd< edddde$deGdeMdeHeIfd ZNedd!d"de$d#eOeId$fd%eMdeGfd&ZPdd'de$dzd(eKd)eMdeGfd*ZQdd'de$dzd(eKd)eMdeHeIfd+ZRGd,d-eZSe Gd.d/ZTGd0d1ZUeKeTzZVeeLd2< eeIgeVfeVzZWeeLd3< e Gd4d5eeJZXeKeXzZYeeLd6< eeIgeYfeYzZZeeLd7< Gd8d9eGeZ[e Gd:d;eZ\e Gd<d=e\Z]e Gd>d?e\Z^Gd@dAeZ_GdBdCeZ`edDe_e`zEZadFe eadeeGeHeaffdGZbGdHdIeZce dJGdKdLZdGdMdNeZeddOd#eHeIdPeHeIdQeIdeeefdRZfd#eHeIdPeHeIdSeHeIdeHeIfdTZge GdUdVZheOeOeGeIfeOeceIffZiddWdXeJdYdZde$dzd[eId\d]deOe[dzeHeiff d^Zjd_ekeGeIfd`ekeGeIfdeMfdaZldXeJdYdZde$dzdeOeHeJd]ffdbZmdXeHeIdYdZde$dzdeOeHeId]ffdcZndXeGdYdZde$dzdeOeGd]ffddZodXeHeIdYdZde$dzde ehfdeZpdXeHeIdYdZde$dzde eGeHehffdfZqekeGeHeMfZr e eGeHe edfZs e eGeHeIdzfZt edge;EZuGdhdieZvGdjdkeeeuZwGdldmeweuZxdS)n)ABCabstractmethod) defaultdict)Callable Generator ItemsViewIterableMappingSequence) dataclassfieldreplace)Enum) lru_cache) TYPE_CHECKINGGeneric NamedTupleProtocol TypeAliascastN)TypeVar assert_never) init_logger) TokenizerLike)flatten_2d_lists full_groupby)MultiModalHasher) MultiModalDataDictMultiModalEncDecInputsMultiModalFieldConfigMultiModalHashesMultiModalInputsMultiModalKwargsItemMultiModalKwargsItemsMultiModalKwargsOptionalItemsMultiModalUUIDDictPlaceholderRange)DictEmbeddingItemsEmbeddingItemsMultiModalDataItemsMultiModalDataParser)BaseProcessingInfoget_current_request_idtimed_preprocessor_operation)BaseDummyInputsBuilder) BatchFeature)BaseMultiModalProcessorCache_S PromptSeqi)maxsizeTadd_special_tokens tokenizertextr8returnc0|||S)Nr7)encode)r9r:r8s x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/multimodal/processing/processor.py_cached_encoder?Fs   D5G  H HHFskip_special_tokens token_ids.rBcJ|t||S)NrA)decodelist)r9rCrBs r>_cached_decoderGPs$   DOOAT  U UUr@ use_cacheseqrIct|tr|S|td|s||St |t |S)Nz8You cannot decode tokens when `skip_tokenizer_init=True`) isinstancestr ValueErrorrErGtupler9rJrIs r> _seq2textrQZsb #s STTT %$$$ )U3ZZ 0 00r@ct|tr<|td|s||dSt ||dS|S)Nz6You cannot encode text when `skip_tokenizer_init=True`Fr7)rLrMrNr=r?rPs r> _seq2tokensrSlsh #sH  UVV V C##CE#BB BiGGGG Jr@c 6eZdZ ddedzdedededzfdZdS) _GetMatchIndexrr9Nprompt start_idxr;cdSN)selfr9rVrWs r>__call__z_GetMatchIndex.__call__s Sr@r)__name__ __module__ __qualname__rr5intr\rZr@r>rUrU~sa   4'  t r@rUceZdZUdZeed<dS) PromptIndexz#Resolves to an index in the prompt.get_match_indexN)r^r_r`__doc__rU__annotations__rZr@r>rcrcs$--######r@rccfeZdZedefdZededefdZedefdZdS)PromptIndexTargetsr;c$tddS)z Resolves to the start of the prompt (before the first token). This results in a match even if the prompt is empty. rcdSNrrZr9rVrWs r>z*PromptIndexTargets.start..s!r@r]rcrZr@r>startzPromptIndexTargets.startsCCCDDDr@rJc v ddtdzdtdtdtdzffd }t|S) zN Resolves to a location in the prompt after the given prefix. rr9NrVrWr;c|dkrdS}t|trt||d}nt||d}t |}|d||kr|ndS)NrFrH)rLrMrQrSlen)r9rVrWprefix match_idxrJs r>rdz2PromptIndexTargets.prefix..get_match_indexs A~~tF&#&& I"9fFFF%Y%HHHF I &z z 2f < <99$ Fr@r])rr5rarc)rJrds` r>rszPromptIndexTargets.prefixss G G$t+ G G G4Z  G G G G G G(?+++r@c$tddS)z Resolves to the end of the prompt (after the last token). This results in a match even if the prompt is empty. rc t|SrYrrrls r>rmz(PromptIndexTargets.end..s #f++r@r]rnrZr@r>endzPromptIndexTargets.endsMMMNNNr@N) r^r_r` staticmethodrcror5rsrxrZr@r>rhrhsE;EEE\E,I,+,,,\,6OOOO\OOOr@rh UpdateTargetPromptUpdateTargetceZdZUdZeed< dZeedze ge j fdzed< e deddfdZ e ded eddfd Ze ded eddfd Ze ded eeddfdZdS)PromptUpdateDetailszEDetails about the token sequence or text that are part of the update.fullNis_embedrJr;zPromptUpdateDetails[_S]c"t|S)N)r~)r})rJs r>from_seqzPromptUpdateDetails.from_seqs",,,,r@ embed_textcldtdzdtdtjffd }t ||S)Nr9r~r;ct|d}t||}tjtj|tj|S)NFrHrStorchisintensor)r9r~embed_token_idsrCrs r>rz1PromptUpdateDetails.select_text..is_embedsT))Z5QQQO#It44I: Y'' _-- r@r~rrr5rTensorr})rJrrs ` r> select_textzPromptUpdateDetails.select_textsR    4 I %,      #h????r@embed_token_idcldtdzdtdtjffd }t ||S)Nr9r~r;cTt||}tj|kSrY)rSrr)r9r~rCrs r>rz5PromptUpdateDetails.select_token_id..is_embeds'#It44I< **n< select_token_idz#PromptUpdateDetails.select_token_idsR  =  4 =I =%, = = = = = = #h????r@rcldtdzdtdtjffd }t ||S)Nr9r~r;ct||}tjtj|tjSrYr)r9r~rCrs r>rz6PromptUpdateDetails.select_token_ids..is_embeds@#It44I: Y'' _-- r@rr)rJrrs ` r>select_token_idsz$PromptUpdateDetails.select_token_idssR    4 I %,      #h????r@)r^r_r`rer4rfrrrr5rrryrrMrrarrFrrZr@r>r}r}sPOO HHHQUHh ,i8%,FG$NUUU -b-6---\- @  @ @ # @ @ @\ @ @  @ @ # @ @ @\ @ @  @c @ # @ @ @\ @ @ @r@r}PromptUpdateInfoPromptUpdateContentceZdZdZdZdS) UpdateModeinsertrN)r^r_r`INSERTREPLACErZr@r>rr%s FGGGr@rceZdZUdZeed< eed< eede fdZ eede fdZ de defdZde defd Zde dd fd Zd S) PromptUpdatezA Defines how to update a prompt with placeholder tokens. modalitytargetr;ct)z3The placeholder tokens that are part of the update.NotImplementedErrorr[s r>contentzPromptUpdate.content6 "!r@ct)z!Defines how to update the prompt.rrs r>modezPromptUpdate.mode<rr@item_idxcH|j}t|r ||}|SrY)rcallable)r[rrs r>_resolve_targetzPromptUpdate._resolve_targetBs- F   &VH%%F r@c|j}t|r ||}t|tst|}|SrY)rrrLr}r)r[rrs r>_resolve_contentzPromptUpdate._resolve_contentIsS, G   (gh''G'#677 <)227;;Gr@ResolvedPromptUpdatec t|j||j||||S)z Given the index of the processed item within [`modality`][vllm.multimodal.processing.PromptUpdate.modality], output a copy of this object with its lazy attributes resolved. )rrrrr)rrrrr)r[rs r>resolvezPromptUpdate.resolveSsJ $]''11))(33     r@N)r^r_r`rerMrfr{propertyrrrrrrarzrr}rrrZr@r>rr*sMMM4 1 ","""^X""j"""^X" 1D    (>       r@rcpeZdZUdZedZeed< edefdZ ede fdZ dS) PromptInsertiona Defines how to insert placeholder tokens into a prompt. Example: For each image, insert a number of `` feature placeholders equal to the feature size of the vision encoder after the `` token: ```python PromptInsertion( modality="image", target="", insertion="" * image_feature_size, ) ``` Insert these tokens at the start of the prompt: ```python PromptInsertion( modality="image", target=PromptIndexTargets.start(), insertion="" * image_feature_size, ) ``` Insert these tokens after a prefix `Images:`: ```python PromptInsertion( modality="image", target=PromptIndexTargets.prefix("Images:"), insertion="" * image_feature_size, ) ``` Insert these tokens at the end of the prompt: ```python PromptInsertion( modality="image", target=PromptIndexTargets.end(), insertion="" * image_feature_size, ) ``` Frepr insertionr;c|jSrY)rrs r>rzPromptInsertion.contents ~r@ctjSrY)rrrs r>rzPromptInsertion.modes   r@N) r^r_r`rer rrrfrrrrrZr@r>rrbs--^&+U%6%6%6I"666,X!j!!!X!!!r@rcpeZdZUdZedZeed< edefdZ ede fdZ dS) PromptReplacementa Defines how to replace portions of an input prompt with placeholder tokens. Example: For each image, replace one `` input placeholder in the prompt with a number of `` feature placeholders equal to the feature size of the vision encoder: ```python PromptReplacement( modality="image", target="", replacement="" * image_feature_size, ) ``` As above, but further pad the feature placeholders with `` and ``, which are not supposed to be passed to the vision encoder: ```python PromptReplacement( modality="image", target="", replacement=PromptUpdateDetails( full="".join( [ "", "" * image_feature_size, "", ] ), features="" * image_feature_size, ), ) ``` To avoid unnecessary tokenization during prompt replacement, we recommended passing token sequences instead of text: ```python PromptReplacement( modality="image", target=[image_token_id], replacement=PromptUpdateDetails( full=( [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id] ), features=[image_token_id] * image_feature_size, ), ) ``` Fr replacementr;c|jSrY)rrs r>rzPromptReplacement.contents r@ctjSrY)rrrs r>rzPromptReplacement.modes !!r@N) r^r_r`rer rrrfrrrrrZr@r>rrs55n(-u%'8'8'8K$888 ,   X "j"""X"""r@rceZdZUeed<dS)_HasModalityAttrrN)r^r_r`rMrfrZr@r>rrsMMMMMr@rc*eZdZedefdZdS)_HasModalityPropr;cdSrYrZrs r>rz_HasModalityProp.modalitys"sr@N)r^r_r`rrMrrZr@r>rrs+ "#"""X"""r@r_M)boundvaluesc&t|dS)z} Convenience function to apply [`full_groupby`][vllm.utils.collection_utils.full_groupby] based on modality. c|jSrY)r)xs r>rmz'full_groupby_modality..sajr@key)r)rs r>full_groupby_modalityrs $8$8 9 9 99r@c$eZdZUeed<eed<dS)PromptTargetMatchrWend_idxNr^r_r`rarfrZr@r>rr "NNN LLLLLr@r)frozenc .eZdZUdZeed< eed< eed< eed< e dZ e ed< d d d e ed e d zdedeefdZd d d ed e d zdedeefdZd d d e eezd e d zdedeefdZdefdZdefdZd S)rz A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its lazy attributes resolved, apart from those related to tokenization. rrrrFrrrrWrVr9NrWr;c#K|j}t|tr-||||}|t ||VdSt ||}t |||D]}t |j|jVdS)7Yield each instance of `self.target` found in `prompt`.Nr) rrLrcrdrrSiter_token_matchesrWr)r[rVr9rWrrttarget_token_idsmatchs r>rz'ResolvedPromptUpdate.iter_token_matches$s fk * * ..y&)LLI$' 9===== F&y&99'0@IVVV D DE#EOU]CC C C C C D Dr@c#K|j}t|tr-||||}|t ||VdSt ||}t jt j|||D]8}t | | V9dS)rN)pos) rrLrcrdrrQrefinditerescaperorx)r[rVr9rWrrt target_textrs r>iter_text_matchesz&ResolvedPromptUpdate.iter_text_matches:s fk * * ..y&)LLI$' 9===== F 622 [;!7!7YOOO @ @E#EKKMM599;;?? ? ? ? ? @ @r@ct|tr||||S||||S)rr)rLrMrr)r[rVr9rWs r> iter_matchesz!ResolvedPromptUpdate.iter_matchesPsM fc " " R))&)y)QQ Q&&vyI&NNNr@c$t||S)N)rr)r[rs r> with_targetz ResolvedPromptUpdate.with_target]stF++++r@ct|tst|}t||S)N)r)rLr}rr)r[rs r> with_contentz!ResolvedPromptUpdate.with_content`s;'#677 <)227;;GtW----r@)r^r_r`rerMrfrarrzr rr}rFrrrrrrrrrrZr@r>rrs MMM4MMMJ + 1#(5e#4#4#4G 444= DDDS D!4'D  D $ % DDDD6 @@@@!4'@  @ $ % @@@@6 O O OS C O!4' O  O $ % O O O O,,,,,,.$4......r@rc$eZdZUeed<eed<dS) _TokenMatchrWrNrrZr@r>rrgrr@rr match_idsrWc#Kt|}t|}|dkrdS|||z dzkr<||z}||||krt||V|}n|dz }|||z dzk:dSdS)zh Yield each occurrence of `match_ids` in `token_ids`. Note that empty matches are ignored. rNr-)rWr)rrr)rCrrW prompt_len match_lenrs r>rrlsYJIIA~~ j9,q0 0 0i' Yw& '9 4 4 7CCC C C C II NI j9,q0 0 0 0 0 0 0r@new_idscPttt}d}t||D]D}|j}|j}|||||||}E|||dt |S)z} Replace each occurrence of `match_ids` in `token_ids` with `new_ids`. Note that empty matches are ignored. rN)rFrarrWrappendr)rCrrout_seqs prev_end_idxrrWrs r>replace_token_matchesrsDI  HL#Iy99O - ,y"89:::     OOIlmm,--- H % %%r@ceZdZUeed<eed<eed<eeed<ejdzed<e defdZ de fd Z dS) PlaceholderFeaturesInforrrWtokensNrr;c*t|jSrY)rrrrs r>lengthzPlaceholderFeaturesInfo.lengths4;r@cDt|j|j|jS)N)offsetrr)r(rWrrrs r>to_rangez PlaceholderFeaturesInfo.to_ranges* >;]    r@) r^r_r`rMrfrarFrrrrr(rrZr@r>rrsMMMMMMNNN IlT!!!!     X  *      r@r)rrVmm_prompt_updatesMultiModalPromptUpdatesrcurrent_result"MultiModalPromptUpdatesApplyResultcd}ttttftttff}|D]\}}t |D]k\} } ||| t | D]G\} } || f|vrn;| |||D]!} || j}n || jkr| | f||| f<Hlt|d}|tj kritt}d}|D]I}|\}\} }| j| jkr||0|sd}||J|}||fS)Nrc|ddS)Nr-rrZ)items r>rmz_find_matches..s471:r@rFT)dictrOrMraritems enumeraterrsortedrrrF _MatchToApplyrWrr)rVrr9rrr mm_matchesrmodality_updatesr item_updates update_idxupdatermatches_to_applymatches_to_apply_has_non_empty_matchesr_s r> _find_matchesrs#DeCHou->-C'DDEGGJ&7&=&=&?&?""&/0@&A&A   "Hlh'1=&/ &=&=  " Fh':55E#00*1  E |%{,, 7 z#_all_items_found..sE Hh N8,,r@)allr)rrs` r>_all_items_foundrsI "1"7"7"9"9  r@cd|D}ttttz}d|D}d|D}t ||r|g|fSd} t |||||\}} |n| D]\\} } \} } || | | }|jj}|tj kr| j }n'|tj kr| j }nt||||||t|trt!||nt#||| || | <| j }d|D}t ||rn0|||dt%tt&||fS)Nc4i|]\}}|t|SrZrwrmrs r> z"_apply_matches..$NNN5aUNNNr@c<i|]\}}|dgt|zSrYrwrs r>rz"_apply_matches..s8666#+1eD6CJJ 666r@cHi|]\}}|td|D S)c3K|]}|duV dSrYrZrrs r>rz,_apply_matches... s&**q}******r@sumrrress r>rz"_apply_matches.. sB/5q#3**c*** * *r@rT)rrcHi|]\}}|td|D S)c3K|]}|duV dSrYrZr"s r>rz,_apply_matches...6s&..Q1D=......r@r$r&s r>rz"_apply_matches..5sB   391cAs..#.....   r@)rrFrMrarrrr~rrrrrWrrrLrQrSrr4)rVrr9rr out_resultrrrr rrrr matched_updatematched_contentend_idx_to_inserts r>_apply_matchesr.sg ON4E4K4K4M4MNNNNC$s)O$&&H66/@/F/F/H/H666J 9C9I9I9K9KO88$x##L'!.   %% " " "  < 9I ) ) 5 Xx"55*.x8B:NN,49Oz((($)M!!+++$)O!!T""" OOF<0A#AB C C C OOfc**= )_555 O<<    .8Jx  *!=LL  =G=M=M=O=O    NO < <  O'R OOF<==)*** R( # #Z //r@cLt|||\}}t||fS) Apply the updates in `mm_prompt_updates` to `prompt`. Matches are exclusive even when multiple modalities share the same placeholder tokens. In that case, the modality that appears earlier in `mm_prompt_updates` takes priority. )r.r)rVrr9 token_id_seqsresults r>apply_token_matchesr3@s.+63DiPPM6 M * *F 22r@cXt|||\}}d||fS)r0)r.join)rVrr9textsr2s r>apply_text_matchesr8Qs/#6+>6 !!r@c #Kd|D}d|D}t||rdSt|}d}||krd}|D]\}} ||} | ||dkr(| | D]} | j} t || j} t| }||z}|dks||kr?|||| krF| j}|||| j}t|| || |V|}||xxdz cc<d}n|rt||rdSn|s|dz }||kdSdS) a& Yield each set of placeholder tokens found in `prompt`. Matches are exclusive even when multiple modalities share the same placeholder tokens. In that case, the modality that appears earlier in `mm_prompt_updates` takes priority. Note that empty matches are ignored. c4i|]\}}|t|SrZrwrs r>rz&_iter_placeholders..prr@ci|]}|dSr]rZ)rrs r>rz&_iter_placeholders..qsJJJHHaJJJr@NrF)rrrWrrr-T) rrrrgetrrSr~rr)rVrr9ritem_idx_by_modalityrrWfoundrrrr rcontent_tokens_fullcontent_len_full end_idx_fullcontent_is_embeds r>_iter_placeholdersrCbsON4E4K4K4M4MNNNNJJ8IJJJ(<==VJI j *;*A*A*C*C% %  &H&+H5H>--h::::*84   .&1)W\&J&J##&':#;#; (+;; #q((L:,E,E)L015HHH'.'7$'3+;+;Iw|+T+T(1!)!)"+2!1 !-I(222a7222 EE#I& #N4HIIFF     NIW j r@c\t|||}tt|SrY)rCrr)rVrr9its r>find_mm_placeholdersrFs- F$5y A AB %b)) * **r@_Ic.eZdZUeed<eed<eed<dS)MultiModalProcessingInfokwargshashesprompt_updatesN)r^r_r`r&rfr"rrZr@r>rIrIs6 )))) ++++++r@rIceZdZdZdddedddedzddffd Zed Zed Z dd d e de de e e fdedzdef dZdefdZde deddfdZde defdZedede e e fde e effdZedede e e fdedeefdZdeede e efdefdZ dede e e fdedefdZ!d e"ed!ede e e"e#ffd"Z$dede%e e e fe e e fffd#Z&d e de e e fd$e e e fd%e e e fdef d&Z'd'e dede e e fd(e e e fde(f d)Z)d'e dede e e fd(e e e fde%e"eee(ff d*Z*d'e d(e e e fde"efd+Z+d,e"ede"efd-Z,dede e e fd(e e e fdefd.Z-d e e"ezdede e e fd(e e e fd/e(de%e"eee(ff d0Z.dd dede e e fd(e e e fdedzde/f d1Z0ded2ed3e/de%e1effd4Z2d5e3d6ede3fd7Z4ded3e/d8e1d9ed:ede%e5eff d;Z6dd d e e"ezd2ede e e fd(e e e fdedzde%e"ee7e(ff d<Z8dd d e e"ezd2ede e e fd(e e e fdedzde%e"ee7e(ff d=Z9d e"ed!ede%e"ee:ffd>Z;d e d!ede%e e:ffd?ZdCede e efddfdDZ?dEe e e"e#fde e efddfdFZ@dedGe"ed$e5d!edHe(de%e"ee e e"e#fff dIZA dKdd d e e"ezde de e e fd(e e e fdzdedzdef dJZBxZCS)LBaseMultiModalProcessorz Abstract base class to process multi-modal inputs to be used in vLLM. Not to be confused with `transformers.ProcessorMixin`. N)cacheinfo dummy_inputszBaseDummyInputsBuilder[_I]rOr;ct||_||_||_||_|j|_|j |_ dSrY) super__init__rPrQrO_get_data_parser data_parserget_supported_mm_limits_supported_mm_limitsget_allowed_mm_limits_allowed_mm_limits)r[rPrQrO __class__s r>rTz BaseMultiModalProcessor.__init__sy  ( 0022%)I$E$E$G$G!"&)"A"A"C"Cr@c|jSrY)rXrs r>supported_mm_limitsz+BaseMultiModalProcessor.supported_mm_limitss ((r@c|jSrY)rZrs r>allowed_mm_limitsz)BaseMultiModalProcessor.allowed_mm_limitss &&r@mm_uuidsrVmm_datahf_processor_mm_kwargsrac4|||||S)Nr`)apply)r[rVrbrcras r>r\z BaseMultiModalProcessor.__call__szz&'+AHzUUUr@c|jjj}d}|jr#|jjj}t |S)a Construct a parser to preprocess multi-modal data items before passing them to [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data]. You can support additional modalities by creating a subclass of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser] that has additional subparsers. N)expected_hidden_size)rPctx model_configget_multimodal_configenable_mm_embedsget_inputs_embeds_sizer,)r[ mm_configrgs r>rUz(BaseMultiModalProcessor._get_data_parsersYIM.DDFF #  % W#'9=#=#T#T#V#V #9MNNNNr@r num_itemsc|j|d}|j|d}||}t||}||kr#d|d|d}||kr|dz }t |dS)NrzAt most  z"(s) may be provided in one prompt.z4 Set `--limit-mm-per-prompt` to increase this limit.)r]r<r_minrN)r[rrnsupported_limit allowed_limitlimitmsgs r>validate_num_itemsz*BaseMultiModalProcessor.validate_num_items s 266xCC.228Q??  "+OO]33 u  QUQQXQQQCO++MMS// !  r@c|j|}|jjj}|jsI|D]4\}}t|ttfrtd|d5|D](\}}| |t|)|S)a3 Normalize [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict] to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems] before passing them to [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data]. z,You must set `--enable-mm-embeds` to input `z_embeds`)rV parse_mm_datarPrhrirjrkrrLr*r)rNrvrr)r[rbmm_itemsrmrrs r> _to_mm_itemsz$BaseMultiModalProcessor._to_mm_items"s#11'::IM.DDFF ) #+>>#3#3  %en6H%IJJ$/$/// (~~// : :OHe  # #Hc%jj 9 9 9 9r@ hf_inputsct)z?Given the HF-processed data, output the metadata of each field.r)r[r{rcs r>_get_mm_fields_configz-BaseMultiModalProcessor._get_mm_fields_config=s "!r@ry out_mm_kwargsct)aZ Given the original multi-modal items for this modality and HF-processed data, output the updates to perform. The information returned by this method is used to update token inputs which bypass the HF processor. It is also used to update the output of HF processor if the HF process does not apply prompt updates to text inputs. Moreover, this information is critical to determine the token positions in order to construct [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange] for each multi-modal item. r)r[ryrcr~s r>_get_prompt_updatesz+BaseMultiModalProcessor._get_prompt_updatesFs *"!r@rLrc:fdt|DS)Nc xi|]5\}|fdt|dD6S)c.g|]fdDS)c:g|]}|SrZ)r)rr rs r> zYBaseMultiModalProcessor._bind_and_group_updates....ds%@@@f))@@@r@rZ)rrupdatess @r>rzNBaseMultiModalProcessor._bind_and_group_updates...cs?A@@@@@@r@r)ranger<)rrrrs @r>rzCBaseMultiModalProcessor._bind_and_group_updates..bsn   "'  %n&8&81&E&E F F   r@)r)r[rLrs `r>_bind_and_group_updatesz/BaseMultiModalProcessor._bind_and_group_updates]s8     &;>%J%J     r@c^||||}|||}|D]W\}}t |D]B\}} t | dkr*t dt |||CX|S)N)ryrcr~r-aDetected %d prompt updates for `mm_items[%r][%s]`. Multiple prompt updates per item is now deprecated and may be removed in v0.13. Instead, please specify dynamic update targets in the same prompt update definition by passing a function to `PromptUpdate.target`.)rrget_all_countsrrrrlogger warning_once) r[ryrcr~unbound_prompt_updatesrrrLritem_prompt_updatess r>_get_mm_prompt_updatesz.BaseMultiModalProcessor._get_mm_prompt_updatesjs "&!9!9#9'":" "  !88 "  # # % %   ):(?(?(A(A   $Hn1:>1J1J  --*++a//''? N++      ! r@ new_token_idsrcV|j}t|||SrY)rP get_tokenizerrF)r[rrr9s r>_find_mm_placeholdersz-BaseMultiModalProcessor._find_mm_placeholderss* I++-- #M3DiPPPr@cTtttf}tttf}|D]P}||||Q||fSrY)rrMobjectrr get_processor_dataget_passthrough_data)r[ryprocessor_datapassthrough_datars r>_get_hf_mm_dataz'BaseMultiModalProcessor._get_hf_mm_datasc6k*,,V ,..__&& B BE  ! !%":":"<"< = = =  # #E$>$>$@$@ A A A A///r@ mm_kwargs tok_kwargsc t|jjd5|jj|jjdi|t dd|i|t di||cdddS#1swxYwYdS)zc Call the HF processor on the prompt text and associated multi-modal data. hf_processorr:NrZ)r0rPrhcall_hf_processorget_hf_processorr)r[rVrbrrs r>_call_hf_processorz*BaseMultiModalProcessor._call_hf_processors*$)- H H  9=22* *77Y77,,&,G,,//y/J//                  sA A11A58A5 prompt_texttokenization_kwargscZtd|D S)z Return whether the HF processor applies prompt updates. For most HF processors, this should be `True` when multi-modal data items are passed, but `False` when multi-modal embeddings are passed. c3NK|] }t|ttfV!dSrY)rLr*r))rrs r>rzHBaseMultiModalProcessor._hf_processor_applies_updates..sF   u~/AB C C      r@)anyr)r[rryrcrs r>_hf_processor_applies_updatesz5BaseMultiModalProcessor._hf_processor_applies_updatessA  !**       r@c||\}}|||||}|||d\}|||||} ||| fS)z Apply the HF processor on the prompt text and multi-modal data together. In addition, return whether prompt updates have been applied. )rVrbrr input_idsrryrcr)rrr poptolistr) r[rryrcrrrprocessed_data prompt_idsis_update_applieds r>_apply_hf_processor_text_mmz3BaseMultiModalProcessor._apply_hf_processor_text_mms,0+?+?+I+I((00",* 1   .///&**;77>>@@  >>##9 3 ?  >+<<_apply_hf_processor_text_onlyz5BaseMultiModalProcessor._apply_hf_processor_text_onlysA ;;#(,,#% 3 <   Aqr@ prompt_tokensc|S)a Apply the HF processor on the prompt tokens only. Most HF processors accept prompt text but not prompt tokens. If the HF processor adds or removes tokens that are not related to multi-modal data, you should override this method so it is consistent with the output of [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only] on the corresponding text. rZ)r[rs r>_apply_hf_processor_tokens_onlyz7BaseMultiModalProcessor._apply_hf_processor_tokens_onlys r@c|}||j||||\}}}|S)aK Apply the HF processor on the multi-modal data only. Since HF processor requires that text and multi-modal items correspond to each other, we generate dummy text using [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder] to go along with the multi-modal data. r)rrrQget_dummy_text)r[ryrcr mm_countsrmm_processed_datas r>_apply_hf_processor_mm_onlyz3BaseMultiModalProcessor._apply_hf_processor_mm_onlys]++-- "&"B"B)88CC#9 3 #C# #  a! r@enable_hf_prompt_updatect|tr2|r|||||S|||}n||}||||}||dfS)a Apply the HF processor on the prompt text and multi-modal data. In addition, return whether prompt updates have been applied (for most HF processors, this should be `True`). Note: If `enable_hf_prompt_update=False`, we use HF processor to perform prompt updates if available; HF processor requires that the prompt corresponds to multi-modal items. r)ryrcrF)rLrMrrrr)r[rVryrcrrrrs r>_apply_hf_processor_mainz0BaseMultiModalProcessor._apply_hf_processor_main*s( fc " " F& 77 &%+A(; 8;;FDWXXJJ==fEEJ <<#9 3=   ,e33r@c  |jj i}|pi}|D]\ } |vr| }t|tr|g}g}t |D][\} } || } | sr3| | n| } |tj dd i | iF|| \|| < fd|D| <|S)zCreate MM hashes to be returned. Note: When overrides are provided via callers of `apply`, `_hash_mm_items` will be bypassed and the overrides will be used. Nmodel_idcBg|]}tjddi|iS)rrZ)r hash_kwargs)rrrcrrrs r>rz:BaseMultiModalProcessor._hash_mm_items..sf$$$ %0!)#T*1. $$$r@rZ) rPrrrLrMrget_all_items_for_hashrrr)r[ryrcrrarKrmm_uuids_per_modalitycomputedir item_uuidrrs `` @@r>_hash_mm_itemsz&BaseMultiModalProcessor._hash_mm_itemsSs9%#%>r'~~//+ + OHe8##(0(:%3S99D-B,C)')()E)E)G)GHH33GAt 5a 8I")1*.* -6,Ayyt ,8)1#+T"2#9#6 ! 2222#+x  $$$$$$$!&$$$x   r@ mm_data_items mm_hashesc tfd|D}d|D}i}|D]L\}}g} |D]=} ||| } | td|d| d| | >| ||<M|||fS)NcBi|]\}}||SrZ) is_cached)rrrKrOs r>rzDBaseMultiModalProcessor._get_cache_missing_items..s8   2B(FHeoof--   r@cHi|]\}}|dt|D S)cg|] \}}|| SrZrZ)ridxitem_is_cacheds r>rzOBaseMultiModalProcessor._get_cache_missing_items...s2'C%r@)r)rritems_is_cacheds r>rzDBaseMultiModalProcessor._get_cache_missing_items..sR   */ +4_+E+E   r@zCache miss for z at index z but data is not provided.)rrNrrz) r[rOrr mm_is_cachedmm_missing_idxsmm_missing_dataridxsmissing_modality_datardatas ` r>_get_cache_missing_itemsz0BaseMultiModalProcessor._get_cache_missing_itemss#     FOooFWFW     .:-?-?-A-A    -3355 > >NHd$& ! 7 7$X.s3<$5(55c555 *006666(=OH % %T..????r@ cached_update new_item_idxc$t||S)z Override this if other attributes of `ResolvedPromptUpdate` also need to be recomputed after retrieving from the cache. )rr)r[rrs r>_recompute_cached_prompt_updatez7BaseMultiModalProcessor._recompute_cached_prompt_updates}|<<<rmz:BaseMultiModalProcessor._merge_mm_kwargs..sAr@r-c<g|]}|SrZ)r)rr rr[s r>rz.s9"<_merge_mm_kwargsz(BaseMultiModalProcessor._merge_mm_kwargss) &&(( 9 9F# 9 9 --i8888 9*#s(3II>>#C.BT.I)J$JKDQQ +Ch?S6T1U,U V ! ! !* 1 1   Hf.228R@@N%>%B%B8R%P%P "'0'8'8  #)#H-h7  ':8'D$*89I*J'+ABR+S('111Q6111.0DDDDD"'";";D)"L"Lh'..v666%h/66&- ,*-88  !677+++r@c|||||d\}}}tj||||} t |jjd5|||||} dddn #1swxYwY|||| } t| | | } || |fS)NTrVryrcrrhashingr`rJrKrL) rr%from_hf_inputsr}r0rPrhrrrI) r[rVrrcrrarrrrrrmm_infos r>_apply_hf_processorz+BaseMultiModalProcessor._apply_hf_processorsF  ) )"#9 3$( *     *8   & &'8:P Q Q  *$)- C C  ++&#! ,I               !77  "    +,    7$555s"BB B c4|j}||\}}||r||||||St|jjd5|||||} dddn #1swxYwYt|jjd5|||| \} } dddn #1swxYwY||| ||d\} } }tj | | | |}| | ||}t|jjd5| || | || \}}dddn #1swxYwYt|| | }| ||fS) zy Apply the HF processor on the full prompt text, caching the results and reusing cached results. N)rVrrcrrarr` cache_lookup)rOrrFr)rrrrr)rOrrr0rPrhrrrr%rr}rrrI)r[rVrrcrrarOrrrrmm_missing_data_itemsrmm_missing_processed_datarrrrrrs r>_cached_apply_hf_processorz2BaseMultiModalProcessor._cached_apply_hf_processor&s "22=AA  =,=+++'=$7! , *$)- C C  ++&#! ,I               *$)- H H  262O2O+#3P33 /L/                 ) )*#9 3$) *    % 2@ %  & &)+A    %)$?$? ! " % % ! *$)- H H  +/+@+@#)"3*C ,A,, (I(               +,    7$555s6A>>BB#C  CCE88E<?E<cV|j}t|||SrY)rPrr3r[rVrr9s r>_apply_token_matchesz,BaseMultiModalProcessor._apply_token_matches{s* I++-- "6+_apply_text_matchesz+BaseMultiModalProcessor._apply_text_matchess* I++-- !&*;YGGGr@rCc|j}|||\}}td|Ds;|t ||d|\}}t||d}tttttft}| D][\}} t| D]F\} } | Jd|d| d||||| | gG\||t#|} || fS)Nc3HK|]}td|DVdS)c3K|]}|duV dSrYrZ)rr s r>rzJBaseMultiModalProcessor._apply_prompt_updates...s'EE: $&EEEEEEr@N)r)r update_idxss r>rz@BaseMultiModalProcessor._apply_prompt_updates..sM   EEEEE E E      r@FrHz0Failed to apply prompt replacement for mm_items[z][])rPrrrrrrQrSrrMrFr rrrrrr) r[rCrr9r match_resultnew_textmatched_updatesrrrr  placeholderss r>_apply_prompt_updatesz-BaseMultiModalProcessor._apply_prompt_updatess I++-- &*&?&?  ' ' # |  +2244     N&*%=%=)Y%@@@!&& "Hl ( 8uMMMM%c49M0N+O&OPQUVV%1%7%7%9%9   !Hk(1+(>(>  $*!--: (::.6:::.--  )00&x0::FG  11   ! !  l**r@c|D]Z\}}||g}t||kr,td|d|d|d|dt|d [dS)NExpected there to be rpz- items in keyword arguments corresponding to z data items, but only found z! There is likely a problem with your implementation of merged multi-modal processor for this model (usually arising from an inconsistency between `_call_hf_processor` and `_get_mm_fields_config`).rr<rr RuntimeError)r[rrr item_countrs r>_validate_mm_kwargsz+BaseMultiModalProcessor._validate_mm_kwargss %3$8$8$:$:   HjMM(B//E5zzZ''"IJIIII:DIIII=@ZZIII(  r@ mm_updatesc |D]W\}}||g}t||kr)td|d|d|dt|d XdS)Nr z! prompt updates corresponding to rp items, but instead found z prompt updates! This is likely because you forgot to include input placeholder tokens (e.g., ``, `<|image_pad|>`) in the prompt. If the model has a chat template, make sure you have applied it before calling `LLM.generate`.r )r[rrrr rs r>_validate_mm_updatesz,BaseMultiModalProcessor._validate_mm_updatess %3$8$8$:$:   Hj%>>(B77L<  J.."NJNN(2NN5=NN%(%6%6NNN/  r@mm_placeholdersc |D]W\}}||g}t||kr)td|d|d|dt|d XdS)Nr z& prompt placeholders corresponding to rprz prompt placeholders! Make sure the implementation of `_call_hf_processor` and `_get_mm_fields_config` are consistent with each other.r )r[rrrr rs r>_validate_mm_placeholdersz1BaseMultiModalProcessor._validate_mm_placeholderss %3$8$8$:$:   Hj*..x<_maybe_apply_prompt_updatesz3BaseMultiModalProcessor._maybe_apply_prompt_updatess"0022   N;;; !!"3^DDD  L"88!O  * *?N K K K K*.*D*D!++ 'J  * *?N K K K?**r@ct}||jj|||}|i}||||||\}} } t |jjd5|||| j| j | \}} dddn #1swxYwYd| D} td|| j| j | S)a" Process multi-modal inputs to be used in vLLM. The main steps are: 1. Apply HF Processor on prompt text and multi-modal data together, outputting token IDs and processed tensors. 2. Find and update sequences in the token IDs with placeholder tokens. The number of placeholder tokens equals the feature size of the multi-modal data outputted by the multi-modal encoder. 3. Extract information about the placeholder tokens from the processed token IDs. N)rra prompt_update)ryrrrrc.i|]\}}|d|DS)c6g|]}|SrZ)r)rrs r>rz..Gs @@@4t}}@@@r@rZ)rrrs r>rz1BaseMultiModalProcessor.apply..Fs=! ! ! &, @@<@@@! ! ! r@ multimodal)typeprompt_token_idsrrr) r/rPrhcreate_timing_statsrzrr0rrJrLrr#rK) r[rVrbrcrra request_idryrrrrmm_placeholder_rangess r>rezBaseMultiModalProcessor.applysy,,--  ! IM - -j 9 9 9$$W--  &"$   + +   " 3 ,     *$)- I I  *.*J*J!%!.")"8"3 +K++ 'J               ! ! *9*?*?*A*A! ! !   'nn1     s(B55B9<B9rY)Dr^r_r`rerGr3rTrr]r_rMrr rr'r#r\r,rUrarvr+rzrr2r!r}r%r rrrrrrFrrrOrrboolrrrrrrr"rMultiModalIsCachedrrrr&rrIrrrrrrrrrrre __classcell__r[s@r>rNrNs 6: DDDD3D ,d2 D  DDDDDD&))X)''X'/3 VVVV$V!(V 4 V %t+ V VVVVO"6OOOO("""  """"*# 6""!(V 4" ++ , """^""%"!(V 4"- " ,  """^",   .   S)   !      !% !!(V 4 !- ! ! ! ! ! !DQCyQ3Q d233 4 QQQQ 0% 0 wsF{#WS&[%99 : 0 0 0 0 f%  3;' CK( (  & !(V 4  %S&[1       & = =& =!(V 4 = %S&[1 = tCy,, - = = = =D%S&[1 c *Cy c"!%!!(V 4!%S&[1 !  !!!!2'4d3i'4&'4!(V 4 '4 %S&[1 '4"&'4 tCy,, -'4'4'4'4^/3 @@@%@!(V 4@%S&[1 @ %t+ @ @@@@D @+ @+ @$ @ !#66 7 @ @ @ @D =+ = =  = = = =1,+1,$1,) 1, 1 1, $; 1, ,.EE F1,1,1,1,t/3/6/6/6d3i/6+/6!(V 4 /6 %S&[1 /6%t+/6 tCy2D8 9/6/6/6/6p/3S6S6S6d3iS6+S6!(V 4 S6 %S&[1 S6%t+S6 tCy2D8 9S6S6S6S6jIS I3I tCy<< = IIIIHH3H s66 7 HHHH2+92+32+ tCy'#t,C'D"DEE F 2+2+2+2+h0 S)  &+ S)  & d+B&C!CD S)  "+%+I+1 + 3 +  + tCy'#t,C'D"DEE F++++@<@ @ /3@ @ @ d3i@ $@ !(V 4 @ %S&[1D8 @ %t+@  @ @ @ @ @ @ @ @ r@rNc&eZdZedeeezdedeeezfdZdeeezdedeeezfdZ deeezdede fdZ ddd deeezded e ee fd e ee fdzd edzdef fd ZxZS)EncDecMultiModalProcessorrVrbr;ct)z Create input prompt for the encoder. HF processor will be applied on this prompt during profiling and generation. rr[rVrbs r>create_encoder_promptz/EncDecMultiModalProcessor.create_encoder_promptUs "!r@c|S)z$Create input prompt for the decoder.rZr)s r>create_decoder_promptz/EncDecMultiModalProcessor.create_decoder_promptas  r@encoder_inputsc|j}|||}t|tr||d}n|}t dd|di|}||d<|S)NFr7encoder_prompt_token_idsrrZ)rPrr,rLrMr=r )r[rVrbr-r9decoder_prompt_rawdecoder_prompt_ids mm_inputss r>_get_enc_dec_inputsz-EncDecMultiModalProcessor._get_enc_dec_inputsis I++-- !77HH (# . . 4!*!1!1"u"2""  "4 *  %34F%G   ); $%r@Nr`rcrrac|||}t|||||}||||S)a; Process multi-modal inputs to be used in vLLM. The main processing steps are modified to fit encoder-decoder model: 1. Create encoder prompt from input prompt text. 2. Apply the HF processor on encoder prompt. 3. Copy the input prompt text as decoder prompt inputs. r`)rVrbr-)r*rSrer3) r[rVrbrcrraencoder_promptr-r[s r>rezEncDecMultiModalProcessor.applysm 33FGDD   "  '  '')(   r@rY)r^r_r`rrMrFrarr*r,r#r3r rr'r rer$r%s@r>r'r'Tsw "d3i "$ " tCy " " "^ "d3i$ tCy d3i$) 6<@  /3   d3i $ !(V 4  %S&[1D8  %t+            r@r')yabcrr collectionsrcollections.abcrrrr r r dataclassesr r renumr functoolsrtypingrrrrrrregexrrtyping_extensionsrr vllm.loggerrvllm.tokenizersrvllm.utils.collection_utilsrrhasherrinputsrr r!r"r#r$r%r&r'r(parser)r*r+r,contextr.r/r0rQr1%transformers.feature_extraction_utilsr2rOr3rr^rrMrFrar4r5rfr"r?rOrGrQrSrUrcrhrzr{r}rrrrrrrrrrrrrrrrrrrrr.r3r8rCrFr#rrrGrIrNr'rZr@r>rGs3 $##############WWWWWWWWWWWWWWWW1111111111 33333333######))))))FFFFFFFF%%%%%%                          100000*BBBBBB4444444L#) X   WT3S ""T#Y 9&&&3 4 $ IIII I I  #Y IIII 4 !& VVVVS#XV V  VVVV 111t#1 1 1  1111, t#    #Y $X $$$$$$$ $ -O-O-O-O-O-O-O-O`$k1 i111!)# )< = LILLL ?@?@?@?@?@'"+?@?@ ?@D(*==)==="*3%1A*A!BEU!UYUUUd  4 4 4 4 4 34 4  4 n A!A!A!A!A!lA!A! A!H I"I"I"I"I" I"I" I"Xx#####x###  WT),<<===:(2,:9S$r(]3K::::   $U.U.U.U.U.U.U.U.p* CyCy  { :&Cy&Cy&#Y& #Y &&&&6          *eCHou->-C'DDE  4"4"4" 4"04"t#4"  4" 9 4" : d=1 124"4"4"4"ncN#s(^ ?0 ?00?0t#?0 489 9: ?0?0?0?0D3 I303t#3 49: :; 3333"" "0"t#" 34 45 """""B IB0Bt#B%& BBBBJ+ I+0+t#+ S$./ /0 ++++#tDz/* "#tH5I,J'K"KL &-S$sTz2B-B%C" WT+,,,,,,,,z,,, D D D D D c72;D D D NH H H H H 7 ;H H H H H r@