
    .`i`                     :   d dl mZ d dlmZmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZ d dlmZ d dlmZmZmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1  ee2          Z3 G d d          Z4dS )    )Mapping)Anycast)assert_never)ModelConfigObservabilityConfig)init_logger)MULTIMODAL_REGISTRYMultiModalRegistry)BaseMultiModalProcessorCache)MultiModalDataDictMultiModalEncDecInputsMultiModalInputsMultiModalUUIDDict)BaseMultiModalProcessor)renderer_from_config)TokenizerLike)json_iter_leaves)MultiModalCacheStats   )DecoderOnlyInputsEmbedsInputsEmbedsPromptEncoderDecoderInputsExplicitEncoderDecoderPromptProcessorInputs
PromptTypeSingletonInputsSingletonPrompt
TextPromptTokenInputsTokensPromptembeds_inputstoken_inputs)"is_explicit_encoder_decoder_promptparse_singleton_promptc                   l    e Zd Zdedfdededz  dededz  ddf
 fdZe	de
dz  fd            Zde
fd	Zdedz  fd
Zdedz  fdZdedz  fdZdee         fdZdee         dz  dee         fdZ	 d/deeef         dz  deeef         fdZ	 d/dedeeef         dz  dee         fdZdefdZ	 d/dddeee         z  dedeeef         dz  deeef         dz  dedz  defdZ de!de"fdZ#	 d/dee         deeef         dz  dee         fdZ$	 d/ddde%deeef         dz  dedz  de&ez  fdZ'	 d/ddde(deeef         dz  dedz  de&ez  fd Z)	 d/ddde*deeef         dz  dedz  de+fd!Z,d"e+d#e+dz  de-fd$Z.	 d/de+e/z  d%e+dz  de0e+e+f         fd&Z1	 d/ddde2deeef         dz  dedz  de-fd'Z3d(e4de4fd)Z5	 d/ddde*deeef         dz  dedz  de4fd*Z6	 d/ddde2deeef         dz  dedz  de7fd+Z8	 d/ddde2deeef         dz  dedz  de7fd,Z9de:dz  fd-Z;d0d.Z< xZ=S )1InputPreprocessorNmodel_configobservability_configmm_registrymm_processor_cachereturnc                     t                                                       || _        || _        t	          |          | _        || _        || _        |rt                      nd | _	        d S N)
super__init__r)   r*   r   rendererr+   r,   r   mm_cache_stats)selfr)   r*   r+   r,   	__class__s        j/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/inputs/preprocess.pyr1   zInputPreprocessor.__init__/   sj     	($8!,\::&"48JT2444PT    c                     | j         j        S r/   )r2   	tokenizerr4   s    r6   r9   zInputPreprocessor.tokenizer@   s    }&&r7   c                 4    | j                                         S r/   )r2   get_tokenizerr:   s    r6   r<   zInputPreprocessor.get_tokenizerD   s    }**,,,r7   c                 `    | j         t                              d           d S | j         j        S )Nz@Using None for BOS token id because tokenizer is not initialized)r9   loggerwarning_oncebos_token_idr:   s    r6   get_bos_token_idz"InputPreprocessor.get_bos_token_idG   7    >!R   4~**r7   c                 `    | j         t                              d           d S | j         j        S )Nz@Using None for EOS token id because tokenizer is not initialized)r9   r>   r?   eos_token_idr:   s    r6   get_eos_token_idz"InputPreprocessor.get_eos_token_idP   rB   r7   c                 J   | j         j        st                              d           dS | j         | j         j        t                              d           dS t          | j         j        dd          }|.t                              d           |                                 }|S )z
        Obtain the decoder start token id employed by an encoder/decoder
        model. Returns None for non-encoder/decoder models or if the
        model config is unavailable.
        zSUsing None for decoder start token id because this is not an encoder/decoder model.NzLUsing None for decoder start token id because model config is not available.decoder_start_token_idzaFalling back on <BOS> for decoder start token id because decoder start token id is not available.)r)   is_encoder_decoderr>   r?   	hf_configgetattrrA   )r4   dec_start_token_ids     r6   get_decoder_start_token_idz,InputPreprocessor.get_decoder_start_token_idY   s      3 	8   4$(9(C(K1   4$')A4
 
 %  
 "&!6!6!8!8!!r7   c                 8    |                                  }|J |gS )aU  
        Specifically for encoder/decoder models:
        generate a default decoder prompt for when
        the user specifies only the encoder prompt.

        Encoder/decoder models utilize the decoder
        prompt in different ways; as new models are
        added, it is intended that this function
        will be extended to produce differing
        default decoder prompts, depending on the
        model variety.

        Absent a special case, the default behavior
        of this method is to mirror the behavior of
        the HuggingFace (HF) GenerationMixin for a None
        decoder prompt, which is to employ a logit processor
        setting to force the first decoded token to be <BOS>.
        Here, this behavior is approximated by having the
        "default" decoder prompt be <BOS>.

        However, it is possible that in the future
        other models may have different or more
        complex logic for the default decoder prompt.
        This motivates having a special helper method
        for default decoder prompts.

        Returns:

        * prompt_token_ids
        )rA   )r4   r@   s     r6   #_get_default_enc_dec_decoder_promptz5InputPreprocessor._get_default_enc_dec_decoder_prompt{   s*    @ ,,..'''~r7   decoder_input_idsc                     |                                  }|J ||                                 }t          |          dk    s|d         |k    r|g|z   }|S )a  
        Prepares `decoder_input_ids` for generation with encoder-decoder models.

        Based on:
        https://github.com/huggingface/transformers/blob/4037a2b5b1278736e566aec12e169100275545ea/src/transformers/generation/utils.py
        specifically,
        `GenerationMixin._prepare_decoder_input_ids_for_generation()`.

        Arguments:

        * decoder_input_ids: input token ids to preprocess

        Returns:

        * Processed token list
        Nr   )rL   rN   len)r4   rO   rG   s      r6   )_prepare_decoder_input_ids_for_generationz;InputPreprocessor._prepare_decoder_input_ids_for_generation   sx    * "&!@!@!B!B%111$ !% H H J J !""a'' #'===!7 8;L L  r7   	overridesc                     t          t          t          f                     }| j        j        rd|d<   |r|                    |           |S )NFadd_special_tokens)dictstrr   r)   rH   update)r4   rS   kwargss      r6   _get_tokenization_kwz&InputPreprocessor._get_tokenization_kw   sQ     c3h!!/ 	1 ,1F'( 	%MM)$$$r7   prompttokenization_kwargsc                     |                                  }|                     |          }| j        j        }|r*|                    dd          r|                                } |j        |fi |S )zn
        Apply the model's tokenizer to a text prompt, returning the
        corresponding token IDs.
        do_lower_caseF)r<   rZ   r)   encoder_configgetlowerencode)r4   r[   r\   r9   r_   s        r6   _tokenize_promptz"InputPreprocessor._tokenize_prompt   s}     &&((	"778KLL*9 	$n00%HH 	$\\^^Fy>>*=>>>r7   c                     t          | d          s7| j                            | j        | j        | j        | j                  | _        | j        S )N_mm_processor)r9   cache)hasattrr+   create_processorr)   r*   r9   r,   re   r:   s    r6   _get_mm_processorz#InputPreprocessor._get_mm_processor   sW    t_-- 	!%!1!B!B!).-	 "C " "D !!r7   mm_uuidsmm_datamm_processor_kwargsrk   c                    |                                  }|i }|                    |||||          }|d         }t          d t          |          D                       }	|	st	          d| d          |S )z
        Apply the model's multi-modal processor to a multi-modal prompt,
        returning the corresponding token IDs and metadata.
        N)hf_processor_mm_kwargsr\   rk   	mm_hashesc              3   @   K   | ]}t          |t                    V  d S r/   )
isinstancerW   ).0leafs     r6   	<genexpr>z8InputPreprocessor._process_multimodal.<locals>.<genexpr>  s=       $
 $
&*JtS!!$
 $
 $
 $
 $
 $
r7   z*mm_hashes must contain only strings, got: z_. This is likely due to an incorrect custom implementation of MultiModalProcessor.apply method.)ri   applyallr   
ValueError)
r4   r[   rl   rm   r\   rk   mm_processormm_inputrp   contains_only_stringss
             r6   _process_multimodalz%InputPreprocessor._process_multimodal   s     --//&"$%%#6 3 & 
 
 [)	 !$ $
 $
.>y.I.I$
 $
 $
 !
 !
 % 	4Y 4 4 4   r7   parsed_contentc                 .   | j         j        st          d          |d         }|j        dk    r|                    d          }|j        dk    rt          d          |                                }t          ||                    d          	          S )
Nz?You must set `--enable-prompt-embeds` to input `prompt_embeds`.prompt_embeds   r   )dim   z6prompt_embeds must be of shape (seq_len, hidden_size).
cache_salt)r   r   )r)   enable_prompt_embedsrx   ndimsqueezecpur#   r`   )r4   r}   r   s      r6   _process_embedsz!InputPreprocessor._process_embeds  s      5 	Q   '7 "")11a188M""UVVV
 &))++'N4F4F|4T4T
 
 
 	
r7   inputsc                 z    |rd|vs| j         |S |d         }| j         j        dk    r|| d          S |d |         S )N
truncation
max_lengthleft)r9   truncation_side)r4   r   r\   r   s       r6   _truncate_inputsz"InputPreprocessor._truncate_inputs9  s_     $	#666~%M(6
>)V33:+,,''+:+&&r7   c                "   |                      |d         |          }|                    d          x}r0|                     |||                    d          pi ||          }nt          |          }|                    d          x}r||d<   |S )Nprompt_token_idsmulti_modal_datarm   r\   rk   r   )r   r`   r|   r$   )r4   r}   r\   rk   r   r   r   r   s           r6   _process_tokensz!InputPreprocessor._process_tokensJ  s      00-.0C
 

  .112DEEE 		4--  ""#899?R$7! .  FF ""233F'++L999: 	.#-F< r7   c                (   |d         }|                     d          x}r0|                     |||                     d          pi ||          }n&|                     ||          }t          |          }|                     d          x}r||d<   |S )Nr[   r   rm   r   r\   r   )r`   r|   rc   r$   )	r4   r}   r\   rk   prompt_textr   r   r   r   s	            r6   _process_textzInputPreprocessor._process_textf  s     %X.  .112DEEE 	4-- ""#899?R$7! .  FF  $44$7  5     ""233F'++L999: 	.#-F< r7   c                   t          |          }|d         dk    r|                     |d                   S |d         dk    r|                     |d         |          S |d         dk    r|                     |d         ||          S |d         dk    r,|                     t	          |d         	          ||          S t          |           d
S )z
        Extract the singleton inputs from a prompt.

        Arguments:

        * prompt: single encoder or decoder input prompt

        Returns:

        * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance
        typeembedscontenttokensrj   textr   rW   )r[   N)r&   r   r   r   r    r   )r4   r[   r\   rk   parseds        r6   _prompt_to_llm_inputsz'InputPreprocessor._prompt_to_llm_inputs  s	   $ (//&>X%%''y(9:::&>X%%''y!! (    &>V##%%y!$7! &   
 &>U""%%&"3444$7! &    	Vr7   encoder_inputsdecoder_inputsc                    |d         dk    s|r|d         dk    rt          d          t          t          t          z  |          }t          t          t          z  d z  |          }|U| j        j        j        dk    r|d                                         }n|                     d           }t          |          }n3d|v rt          d          |                     |d                   }||d<   t          ||          S )	Nr   r   =Embedding inputs are not supported for encoder-decoder modelswhisperr   r   zJMulti-modal decoder inputs of encoder-decoder models are not supported yet)encoderdecoder)rx   r   r!   r   r)   rI   
model_typecopyrR   r$   r   )r4   r   r   dec_token_idss       r6   _build_enc_dec_llm_inputsz+InputPreprocessor._build_enc_dec_llm_inputs  s5    6"h.. /v&(22O  
 k,<<nMMk,<<tC^TT! *5BB
 !//A B G G I I $ N Nt T T)-88NN!^33 ;  
 !JJ12 M 2?N-.#""
 
 
 	
r7   decoder_inputs_to_overridec                 r   |d         dk    s|r|d         dk    rt          d          t          t          t          z  t          z  |          }t          t          t          z  dz  |          }|d         dk    rd|vrt          d          t          t          |          }t          |d                   }|p|}t          d|d         |d	         |d
         |d                   }|                    d          x}r||d<   n0|d         dk    rt          g           }|p|}nt          |           ||fS )zx
        For encoder/decoder models only:
        Separate Encoder/Decoder inputs from a MultiModalEncDecInputs
        r   r   r   N
multimodalencoder_prompt_token_idszXYou should register an encoder-decoder multi-modal processor for encoder-decoder models.r   	mm_kwargsrp   mm_placeholders)r   r   r   rp   r   r   token)r   )	rx   r   r!   r   r   RuntimeErrorr$   r`   r   )r4   r   r   r   decoder_prompt_inputsr   r   s          r6   _split_enc_dec_mm_inputsz*InputPreprocessor._split_enc_dec_mm_inputs  s    6Nh&&) '*62h>>O  
 **-CC
 
 &***T1&&
 &
" &>\)))77"  
 0&99F)&1K*LMMN$>$H&!-!!67I!J - - &'8 9  N $ZZ555z :/9|,F^w&&)2>>>N7A6NN   ~--r7   c                   t          |          r}t          t          |          }|                     |d         ||          }|d         x}d}n|                     ||          }| j        j        r|                     ||          \  }}nT|                     t          t          |          ||          }| j        j        r|                     |          \  }}n|}d}|                     ||          S )ab  
        For encoder/decoder models only:
        Process an input prompt into an
        [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
        instance.

        There are two types of input prompts:
        singleton prompts which carry only the
        encoder prompt, and explicit encoder/decoder
        prompts which carry both the encoder and the
        decoder prompts as member variables.

        This function handles the following scenarios:
        * Singleton encoder prompt: extract encoder prompt
          token ids & infer default decoder prompt token ids
        * Explicit encoder/decoder prompt: extract encoder
          and decoder prompt token ids

        Note that for Explicit encoder/decoder prompts,
        each sub-prompt (encoder or decoder prompt) can
        have any possible singleton type; thus this
        method relies on helper functions to obtain
        token ids for the sub-prompts.

        Arguments:

        * prompt: an input prompt

        Returns:

        * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
          instance
        encoder_promptr   decoder_promptNr   )	r%   r   r   r   r)   is_multimodal_modelr   r   r   )	r4   r[   r\   rk   prompt_r   decoder_inputr   r   s	            r6   _process_encoder_decoder_promptz1InputPreprocessor._process_encoder_decoder_prompt  s1   T .f55  	&7@@G!77()$7! 8  N
 "))9!::C!%!%!;!;!7J "< " "
  4 151N1N"N2 2.
 //_f--$7! 0  F
  4 &151N1Nv1V1V.!'!%--nnMMMr7   prompt_inputsc                 H    d|v rt          t          t          z  |          }|S )Nr   )r   r!   r   )r4   r   s     r6   _build_decoder_only_llm_inputsz0InputPreprocessor._build_decoder_only_llm_inputsf  s1     .. .. M r7   c                \    |                      |||          }|                     |          S )a1  
        For decoder-only models:
        Process an input prompt into a
        [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.

        Arguments:

        * prompt: input prompt

        Returns:

        * [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
        r   )r   r   )r4   r[   r\   rk   prompt_compss        r6   _process_decoder_only_promptz.InputPreprocessor._process_decoder_only_promptq  s=    * 11 3 2 
 
 22<@@@r7   c                    | j         j        r|                     |||          S t          |          rt	          d          |                     t          t          |          ||          S )Nrj   z9Cannot pass encoder-decoder prompt to decoder-only modelsr   )r)   rH   r   r%   rx   r   r   r   )r4   r[   r\   rk   s       r6   _preprocesszInputPreprocessor._preprocess  s     / 	 77#! 8    .f55 	K   00&)) 3 1 
 
 	
r7   c                   |                      |||          }| j        rk| j        d| j                            d          }| j        xj        dz  c_        | j        xj        |j        z  c_        | j        xj        |j        z  c_        |S )zPreprocess the input prompt.rj   NT)deltar   )r   r,   r3   
make_statsrequestsqueriestotalhits)r4   r[   r\   rk   resr   s         r6   
preprocesszInputPreprocessor.preprocess  s     v':XNN" 	3t':'F+66T6BBE((A-((''5;6''$$
2$$
r7   c                 B    | j         }|d S t                      | _         |S r/   )r3   r   )r4   r3   s     r6   stat_mm_cachezInputPreprocessor.stat_mm_cache  s)    ,!4244r7   c                 p    | j         | j                                          | j        d| j        _        d S d S )NT)r,   clear_cacher3   resetr:   s    r6   clear_mm_cachez InputPreprocessor.clear_mm_cache  sB    ".#//111*(,D%%% +*r7   r/   )r-   N)>__name__
__module____qualname__r
   r   r   r   r   r1   propertyr   r9   r<   intrA   rE   rL   listrN   rR   rV   rW   r   rZ   rc   r   ri   r   r   objectr   r   r|   r   r   r   r   r"   r!   r   r    r   r   r   r   r   r   r   tupler   r   r   r   r   r   r   r   r   r   r   r   __classcell__)r5   s   @r6   r(   r(   .   s        <@*=BFU U!U 2D8U (	U
 94?U 
U U U U U U" '=4/ ' ' ' X'-} - - - -+#* + + + ++#* + + + + "C$J  "  "  "  "D"T#Y " " " "H#!9t+#! 
c#! #! #! #!N ,0 S>D( 
c3h   ( 6:? ?? "#s(^d2? 
c	? ? ? ?&	"#: 	" 	" 	" 	"  6:& /3& & &d3i& $& %S&[1D8	&
 "#s(^d2& %t+& 
& & & &P
$
 

 
 
 
> OS' '3i'6:38nt6K'	c' ' ' '( 6:
 /3  $ "#s(^d2
 %t+ 
'	'   > 6:
 /3  " "#s(^d2
 %t+ 
'	'   B 6:(
 /3( ( (( "#s(^d2(
 %t+( 
( ( ( (T+
'+
 ($.+
 
	+
 +
 +
 +
` >B;. ;."88;. %4d$:;. 
/	0	;. ;. ;. ;.@ 6:LN
 /3LN LN LNLN "#s(^d2LN
 %t+LN 
LN LN LN LN\	(	 
	 	 	 	 6:A
 /3A A AA "#s(^d2A
 %t+A 
A A A A@ 6:

 /3
 
 

 "#s(^d2

 %t+
 

 
 
 
@ 6:
 /3   "#s(^d2
 %t+ 
   $3d:    - - - - - - - -r7   r(   N)5collections.abcr   typingr   r   typing_extensionsr   vllm.configr   r   vllm.loggerr	   vllm.multimodalr
   r   vllm.multimodal.cacher   vllm.multimodal.inputsr   r   r   r   vllm.multimodal.processingr   vllm.renderersr   vllm.tokenizersr   vllm.utils.jsontreer   vllm.v1.metrics.statsr   datar   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   parser%   r&   r   r>   r(    r7   r6   <module>r      s5   $ # # # # #         * * * * * * 8 8 8 8 8 8 8 8 # # # # # # C C C C C C C C > > > > > >            ? > > > > > / / / / / / ) ) ) ) ) ) 0 0 0 0 0 0 6 6 6 6 6 6                                 N M M M M M M M	X		]
- ]
- ]
- ]
- ]
- ]
- ]
- ]
- ]
- ]
-r7   