.`i<:ddlmZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlmZddlmZddlmZddlmZmZd d lmZmZmZmZmZmZmZd d lm Z d d l!m"Z"m#Z#m$Z$m%Z%erdd l&m'Z'mZm(Z(ddl)m*Z*ee+Z,e de-dZ.e de$Z/e de$dZ0Gdde e0Z1Gdde e/Z2Gdde e/Z3edGdde e/Z4Gdd Z5d!S)")Mapping) dataclass)Lock) TYPE_CHECKINGGenericLiteralProtocolTypeVarcast)BaseDummyOptions)ObservabilityConfig) init_logger) TokenizerLikecached_tokenizer_from_config)BaseMultiModalProcessorCacheBaseMultiModalReceiverCacheMultiModalProcessorOnlyCacheMultiModalProcessorSenderCacheMultiModalReceiverCacheShmObjectStoreReceiverCacheShmObjectStoreSenderCache)MultiModalInputs)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoInputProcessingContext) ModelConfigr VllmConfig)SupportsMultiModalNr )bound_I_I_coT)r" covariantc"eZdZdZdedefdZdS)ProcessingInfoFactory Constructs a [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor] instance from the context. ctxreturncdSN)selfr)s l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/multimodal/registry.py__call__zProcessingInfoFactory.__call__0s N)__name__ __module__ __qualname____doc__rr$r0r-r1r/r'r')sD  # r1r'c.eZdZdZdedeefdZdS)DummyInputsBuilderFactoryz Constructs a [`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder] instance from the context. infor*cdSr,r-)r.r8s r/r0z"DummyInputsBuilderFactory.__call__=sr1N)r2r3r4r5r#rr0r-r1r/r7r76s9 DRC$:2$>CCCCCCr1r7c NeZdZdZdddedeededzdeefdZdS) MultiModalProcessorFactoryr(Ncacher8 dummy_inputsr=r*cdSr,r-)r.r8r>r=s r/r0z#MultiModalProcessorFactory.__call__Gs '*cr1) r2r3r4r5r#rrrr0r-r1r/r;r;@sv6: ****-R0* ,d2 * ! $ ******r1r;)frozencneZdZUeeed<eeed<eeed<dddede dzfdZ dS) _ProcessorFactoriesr8 processorr>Nr<r)r=c||}||}||||S)Nr<r8r>rC)r.r)r=r8dummy_inputs_builders r/build_processorz#_ProcessorFactories.build_processorVs@ yy~~#0066~~d$8~FFFr1) r2r3r4r'r#__annotations__r;r7rrrGr-r1r/rBrBPs  ####)"----+B//// 6: GGG #G,d2 GGGGGGr1rBceZdZdZdddeeefdzfdZdddefdZ ddddddd e dzd eee fdzd e dzdeee ff d Z ddd ddd e dzd e dzdeee ffdZdeedeedeefdZd)dZ d*ddd ddedzdefdZ d+ddddd ddedzdefdZ d+dddddd ddedzd e dzdeef dZ d+dddddddeee fdzd e dzd e dzdedzdef dZddde fdZd d!ded"fd#Z d d!de dzfd$Z!d d!de"dzfd%Z#d d!de$dzfd&Z%d d!d'e&de$dzfd(Z'dS),MultiModalRegistryzL A registry that dispatches data processing according to the model. model_configrr*NcxjsdSfdjjD}t|dkr|ndS)z Extract multimodal dummy options from model config. Returns None if no configurable options are found, otherwise returns a mapping of modality names to their dummy options. NcNi|]!}j|x|"Sr,)multimodal_configget_dummy_options).0mrKopts r/ z:MultiModalRegistry._extract_mm_options..ssA   #5GGJJJW sWWWr1r)rNlimit_per_promptlen)r.rK mm_optionsrRs ` @r/_extract_mm_optionsz&MultiModalRegistry._extract_mm_optionsfse- 4     !3D   !__q00zzd:r1c|jsdS||d}|}|t fd|Drt ddSdS)z Checks if the model supports multimodal inputs. Returns True if the model is multimodal with any non-zero supported modalities, otherwise returns False, effectively running in text-only mode. FN tokenizerc3JK|]}|dkVdS)rN)get_limit_per_prompt)rPmodality mm_configs r/ z@MultiModalRegistry.supports_multimodal_inputs..sJ    * *8 4 4 9      r1zcAll limits of multimodal modalities supported by the model are set to 0, running in text-only mode.T)is_multimodal_model_create_processing_infoget_supported_mm_limitsget_multimodal_configalllogger info_once)r.rKr8supported_modalitiesr^s @r/supports_multimodal_inputsz-MultiModalRegistry.supports_multimodal_inputs{s/ 5++LD+II#;;== 6688      0         ;   5tr1)r=profiler_limitsobservability_configr=rirjc|jsiS||||}||j}d|D|j|j}| fd|DS|||}d|dDS) z Get the maximum number of tokens per data item from each modality based on underlying model configuration. r<Nc&i|]\}}|dk |dS)rrr-)rPr]limits r/rSzJMultiModalRegistry.get_max_tokens_per_item_by_modality..s+   +He%RS))Ha)))r1)seq_len mm_countscPi|]"\}}|ddk||#S)r)get)rPr] max_tokensros r/rSzJMultiModalRegistry.get_max_tokens_per_item_by_modality..sB(Hj==1--11*111r1)rorCcHi|]\}}|td|D S)c3$K|] }|jV dSr,)get_num_embeds)rPitems r/r_zTMultiModalRegistry.get_max_tokens_per_item_by_modality...s%GG$$-GGGGGGr1)sum)rPr] placeholderss r/rSzJMultiModalRegistry.get_max_tokens_per_item_by_modality..sE   &, cGG,GGGGG   r1mm_placeholders)r`create_processorallowed_mm_limitsitemsr8get_mm_max_tokens_per_item max_model_lenget_dummy_mm_inputs) r.rKr=rirjrCmax_tokens_per_item mm_inputsros @r/#get_max_tokens_per_item_by_modalityz6MultiModalRegistry.get_max_tokens_per_item_by_modalitys1/ I)) .e*    "'9O  />/D/D/F/F   (nGG .H    *,?,E,E,G,G  ,, -    *34E*F*L*L*N*N    r1)r=rjcR|jsiS||||}|jS)z Get the maximum number of multi-modal input instances for each modality that are allowed per prompt for a model class. r<)r`rzr{)r.rKr=rjrCs r/get_mm_limits_per_promptz+MultiModalRegistry.get_mm_limits_per_promptsA/ I)) .e*  **r1rCr8r>c<dtdtffd }|S)a/ Register a multi-modal processor to a model class. The processor is constructed lazily, hence a factory method should be passed. When the model receives multi-modal data, the provided function is invoked to transform the data into a dictionary of model inputs. model_clsr*cd|jvrtd|t|_|S)N_processor_factoryzfModel class %s already has a multi-modal processor registered to %s. It is overwritten by the new one.rE)__dict__rewarningrBr)rr>r8rCr.s r/wrapperz6MultiModalRegistry.register_processor..wrappers]#y'999J ,?)#,,,I (  r1)r!)r.rCr8r>rs```` r/register_processorz%MultiModalRegistry.register_processorsH q Q         "r1r cnddlm}||\}}t|dsJtd|S)Nr)get_model_architecturerr ) vllm.model_executor.model_loaderrhasattrr )r.rKrr_s r/_get_model_clsz!MultiModalRegistry._get_model_clssPKKKKKK--l;; 1y"677777()444r1zObservabilityConfig | NonerZcH|t|}t|||S)N)rj)rr)r.rKrjrZs r/_create_processing_ctxz)MultiModalRegistry._create_processing_ctxs6  4\BBI% ):N    r1rYc||}|j}||||}||Sr,)rrrr8)r.rKrjrZr factoriesr)s r/raz*MultiModalRegistry._create_processing_infosI'' 55 0 )),8LiXX~~c"""r1)rZr=c|jst|jd||}|j}||||}|||S)zT Create a multi-modal processor for a specific model and tokenizer. z is not a multimodal modelr<)r` ValueErrormodelrrrrG)r.rKrjrZr=rrr)s r/rzz#MultiModalRegistry.create_processorsv/ P 2NNNOO O'' 55 0 )),8LiXX((E(:::r1)r=rjrCroc|j}|||||}||j}|j||||}||j|j|j |j }|d} t| } | |kr| dg|| z z|S)z Create dummy data for profiling the memory usage of a model. The model is identified by `model_config`. Nr<)rnrorV)promptmm_datahf_processor_mm_kwargstokenization_kwargsprompt_token_idsr) r~rzr{r>get_dummy_processor_inputsrWapplyrrrrrUextend) r.rKror=rjrCrnprocessor_inputsrr total_lens r/rz&MultiModalRegistry.get_dummy_mm_inputs2s,  --2%.I  !3I$1LL// ==M   OO#*$,#3#J 0 D $  %%78()) w    # #QC7Y+>$? @ @ @r1c|jsdS||}|sdSt|dks Jdtt |}||S)zY Get the maximum length of the encoder input for encoder-decoder models. rrzdEncoder-decoder models are expected to implement the multimodal interface with at most one modality.)is_encoder_decoderrrUnextiter)r.rKrrfirst_modalitys r/get_encdec_max_encoder_lenz-MultiModalRegistry.get_encdec_max_encoder_len\s}. 1==lKK  1:!### O$## d:..//.))r1 vllm_configr)Nprocessor_onlylrushmc|j}||sdS|}|jdkrdS|j}|jdko|jdkp|j}|sdS|}|jS)Nrrr) rKrhrcmm_processor_cache_gbparallel_config_api_process_countdata_parallel_sizedata_parallel_external_lbmm_processor_cache_type)r.rrKr^ris_ipc_supporteds r/_get_cache_typez"MultiModalRegistry._get_cache_typeps#/ ..|<< 4!6688  *a / /4&5*=B  .! 3 98    $## 6688 00r1c||}|dS|dkrt|jS|dkrt|jS|dkrt |St d|)z4Return a `BaseMultiModalProcessorCache`, if enabled.NrrrUnknown cache type: )rrrKrrrr.r cache_types r/processor_cache_from_configz.MultiModalRegistry.processor_cache_from_configs ))+66  4 + + +/ 0HII I 5 1+2JKK K 5 ,[99 9BJBBCC Cr1c\||}|dSt|jS)z4Return a `MultiModalProcessorOnlyCache`, if enabled.N)rrrKrs r/ processor_only_cache_from_configz3MultiModalRegistry.processor_only_cache_from_configs3 ))+66  4+K,DEEEr1c||}|dvrdS|dkrt|jStd|)z>Return a `BaseMultiModalReceiverCache` for the engine process.)NrrNrr)rrrKrrs r/!engine_receiver_cache_from_configz4MultiModalRegistry.engine_receiver_cache_from_configs[ ))+66 8 8 84 5 *;+CDD DBJBBCC Cr1shared_worker_lockc||}|dvrdS|dkrt||Std|)z>Return a `BaseMultiModalReceiverCache` for the worker process.)NrrNrr)rrr)r.rrrs r/!worker_receiver_cache_from_configz4MultiModalRegistry.worker_receiver_cache_from_configs[ ))+66 8 8 84 5 .{B*.   #  ;  !4'      >B # +/ # # ## #; # !4' #  # # # #>B; +/59 ;;;#;;; !4' ; ,d2 ; !!3 4;;;;0/3( 6:;?48(((#(38$t+( ,d2 ( 2D8 (+T1( ((((T*}*****(1!1 5 611114D!D & ,DDDD" F! F & , F F F F D! D %t + D D D D D! D% D %t + D D D D D Dr1rJN)6collections.abcr dataclassesrmultiprocessing.synchronizerrtypingrrrr r r vllm.config.multimodalr vllm.config.observabilityr vllm.loggerrvllm.tokenizersrrr=rrrrrrrinputsr processingrrrr vllm.configrr%vllm.model_executor.models.interfacesr r2retyper!r#r$r'r7r;rBrJr-r1r/rs$#####!!!!!!888888KKKKKKKKKKKKKKKK333333999999######GGGGGGGG%$$$$$IHHHHHHHHHHHHHHHH X   GCt01222 WT+,,,1TBBB     HUO   DDDDD DDD * * * * *" * * *  $ G G G G G'"+ G G G ^D^D^D^D^D^D^D^D^D^Dr1