
    .`i3                        d dl mZmZ d dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ  ee          Z ej        ej                   ej        d          k    ZdZ G d d          Z G d dee          Z G d de          Z G d de          Zde de!de"e          de#de$e e!f         dz  f
dZ%dS )    )ABCabstractmethodN)version)	Tokenizer)DecodeStream)PreTrainedTokenizerFast)init_logger)TokenizerLike)convert_prompt_ids_to_tokensdetokenize_incrementally)&length_from_prompt_token_ids_or_embeds)EngineCoreRequestz0.21.1zInvalid prefix encounteredc                       e Zd Zd Zedee         fd            Zdee         dede	dz  fdZ
ded	ede	fd
Zededz  dedd fd            ZdS )IncrementalDetokenizerc                     g | _         d S N	token_idsselfs    n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/engine/detokenizer.py__init__zIncrementalDetokenizer.__init__   s    $&    returnc                     | j         S r   r   r   s    r   output_token_idsz'IncrementalDetokenizer.output_token_ids"   s
    ~r   new_token_idsstop_terminatedNc                 :    | j                             |           d S r   )r   extend)r   r   r   s      r   updatezIncrementalDetokenizer.update&   s    m,,,tr   finisheddeltac                     dS N  )r   r"   r#   s      r   get_next_output_textz+IncrementalDetokenizer.get_next_output_text*   s    rr   	tokenizerrequestc                     |j         J |t                      S t          r%t          |t                    rt          ||          S t          ||          S r   )sampling_paramsr   USE_FAST_DETOKENIZER
isinstancer   FastIncrementalDetokenizerSlowIncrementalDetokenizer)clsr)   r*   s      r   from_new_requestz'IncrementalDetokenizer.from_new_request-   sc     &222)+++ 	BJy:Q$R$R 	B-iAAA *)W===r   )__name__
__module____qualname__r   propertylistintr   boolstrr!   r(   classmethodr
   r   r2   r'   r   r   r   r      s        ' ' ' $s)    XDI  t    T $ 3     > 4'> #> 
"	> > > [> > >r   r   c                   ~     e Zd Zdef fdZdee         dededz  fdZ	e
dedefd	            Zd
ededefdZ xZS )BaseIncrementalDetokenizerr*   c                    t                                                       |j        }|J |j        g }n*t	          |j        t
                    r	|j        g}n|j        }|| _        |j        | _        |j        | _        | j        r.| j        s't          d | j        D                       dz
  | _	        nd| _	        d| _
        d| _        d S )Nc              3   4   K   | ]}t          |          V  d S r   )len).0ss     r   	<genexpr>z6BaseIncrementalDetokenizer.__init__.<locals>.<genexpr>V   s(      )D)DQ#a&&)D)D)D)D)D)Dr      r   r&   )superr   r,   stopr.   r:   
min_tokensinclude_stop_str_in_outputmaxstop_buffer_length_last_output_text_offsetoutput_text)r   r*   params	stop_list	__class__s       r   r   z#BaseIncrementalDetokenizer.__init__B   s     (!!!;IIS)) 	$III	 +*0*K' 9 	(T< 	(&))D)D$))D)D)D&D&Dq&HD##&'D#-.% r   r   r   r   Nc                    |sdS |r| j         s|d         }|dd         }nd}t          | j                  }|D ]w}| j                            |           | xj        |                     |          z  c_        | j        r1t          | j                  | j        k    rt          | j                  }x|| j                            |           d}| j        rut          | j                  | j        k    rXt          | j        t          | j                  |z
  | j        | j                   }||\  }}|dk    r| j        d|         | _        |S )z
        Update RequestState for the request_id by:
            1) Detokenize the new token ids incrementally.
            2) Evaluate stop criteria.

        Return matched stop string or None.
        N)rL   new_char_countrF   include_in_output)
rH   r@   rL   r   appenddecode_nextrG   r   rF   check_stop_strings)	r   r   r   skipped_stop_token_idstop_check_offsetnew_token_idstop_stringrF   truncate_tos	            r   r!   z!BaseIncrementalDetokenizer.update^   s     	4 	)4#B 	) %2"$5!)#2#.MM$(!
   011) 	: 	:LN!!,/// 0 0 > >> :3t'<#=#=#P#P$'(8$9$9! ,N!!"7888 9 
	FT233doEE% ,"4#3447HHY"&"A	  D +/(["$$'+'7'ED$r   next_token_idc                     t           r   )NotImplementedError)r   r\   s     r   rU   z&BaseIncrementalDetokenizer.decode_next   s    !!r   r"   r#   c                     |rdn| j         }|s|r| j        d|          n| j        S t          | j                  |z
  }| j        }||k     r|| _        | j        ||         S dS )zVIf delta is True, only new text since the last call to
        this method is returnedr   Nr&   )rJ   rL   r@   rK   )r   r"   r#   buffer_lengthlengthlast_offsets         r   r(   z/BaseIncrementalDetokenizer.get_next_output_text   s    
 &B4+B 	 !( =.11&
 T%&&63,2D)#K$677rr   )r3   r4   r5   r   r   r7   r8   r9   r:   r!   r   rU   r(   __classcell__rO   s   @r   r=   r=   A   s         1      81DI 1 1t 1 1 1 1f " " " " " ^"T $ 3        r   r=   c                   N     e Zd Zdedef fdZdedefdZdededz  fdZ	 xZ
S )	r/   r)   r*   c                    t                                          |           |j        }|J |j        | _        |j        | _        t          | j                  | _        |j        | _        |j	        pg }|}t          |          }|dk    rOt          dt          |dz   d                    D ]-}|| d          }d| j                            |          vr|} n.|D ]}	|                     |	           |j        p|j        | _        | j        sut!          | j        dd           x}
	 Ad | j                                                                        D             x| j        _        }
|
rd| _        |
| _        d S d	| _        d S d S )
Nskip_special_tokens   rD      u   �added_token_idsc                 $    i | ]\  }}||j         S r'   )content)rA   tidtoks      r   
<dictcomp>z7FastIncrementalDetokenizer.__init__.<locals>.<dictcomp>   s6     D D D S D D Dr   FT)rE   r   r,   
request_idrh   r   stream
_tokenizerr)   prompt_token_idsr@   rangemindecode_protected_stepspaces_between_special_tokensgetattrget_added_tokens_decoderitemsrk   last_special)r   r)   r*   r,   rt   prompt_suffix
prompt_lenisuffixrn   rk   rO   s              r   r   z#FastIncrementalDetokenizer.__init__   s   !!!!1***!,#2#F "t7OPPP$-$8 #39r(''
>>1c*q."5566  )1"##. 5 5f = ===$*ME >
 ! 	& 	&C  %%%% / =< 	*
 1 	: $+4>;Ld#S#SSD D$(N$K$K$M$M$S$S$U$UD D D .
  :$)!'6$$$ 6:222!	: 	:r   r\   r   c                     |                      |          }| j        s0| j                            |          }|d u}|r	| j        r|}|| _        |pdS r%   )rx   ry   rk   getr}   )r   r\   tokenspecial_token
is_specials        r   rU   z&FastIncrementalDetokenizer.decode_next   si    $$]331 	+ 044]CCM&d2J &d/ &% *D{r   Nc                    	 | j                             | j        |          }n# t          t          f$ r  t
                              d|           d }Y nt          $ r}t          |          	                    t                    s|t
                              d| j                   t          | j                  | _         | j                             | j        |          }Y d }~nd }~ww xY w|S )Nz Encountered invalid token id: %rzXEncountered invalid prefix detokenization error for request %s, resetting decode stream.rg   )rr   stepr)   OverflowError	TypeErrorlogger	exception	Exceptionr:   
startswithINVALID_PREFIX_ERR_MSGwarningrq   r   rh   )r   r\   r   es       r   rx   z*FastIncrementalDetokenizer._protected_step   s   	DK$$T^]CCEEy) 	 	 	 ?OOOEEE 	D 	D 	Dq66$$%;<< 
 NN<  
 '4;STTTDKK$$T^]CCEEEEEE	D s    # .C)	C)BC$$C))r3   r4   r5   r   r   r   r8   r:   rU   rx   rc   rd   s   @r   r/   r/      s        0:"9 0:DU 0: 0: 0: 0: 0: 0:d     S S4Z        r   r/   c                   `     e Zd Zdedef fdZedee         fd            Z	dede
fdZ xZS )r0   r)   r*   c                    t                                          |           || _        |j        }|J t	          |j        |j                  | _        |j        0t          ||j        |j	                  \  | _
        | _        | _        ndg| j        z  | _
        d| _        d| _        | j                            |j        p
dg| j        z             |j	        | _	        |j        | _        d S )N)r)   
prompt_idsrh   r&   r   )rE   r   r)   r,   r   rt   prompt_embedsr   r   rh   tokensprefix_offsetread_offsetread_offestr   r    ry   )r   r)   r*   rM   rO   s       r   r   z#SlowIncrementalDetokenizer.__init__  s    !!!"(!!!@$g&;
 

 #/,'&7(.(B   >DK+T-=-= $0DK!"D Dg6O1#:OPPP#)#= -3-Q***r   r   c                 F    | j         s| j        n| j        | j         d          S r   )r   r   r   s    r   r   z+SlowIncrementalDetokenizer.output_token_ids"  s+     ?5DNN.!2!23	
r   r\   c           	          t          | j        | j        | j        | j        | j        | j        | j                  \  }}}}| j                            |           || _        || _        |S )N)r)   all_input_idsprev_tokensr   r   rh   ry   )	r   r)   r   r   r   r   rh   ry   r    )r   r\   
new_tokensdecoded_textr   r   s         r   rU   z&SlowIncrementalDetokenizer.decode_next*  sw    ?Wn.,( $ 8*.*L@
 @
 @
<
L- 	:&&&*&r   )r3   r4   r5   r
   r   r   r6   r7   r8   r   r:   rU   rc   rd   s   @r   r0   r0     s        R- R:K R R R R R R> 
$s) 
 
 
 X
         r   r0   rL   rR   rF   rS   r   c                     |r|sdS |D ]Y}t          |          }|                     |d|z
  |z
            }|dk    r4|r||z  }|t          |           k    r|dfc S ||fc S dS )a5  Check if any stop strings are matched and truncate sequence
    output text accordingly.

    Returns tuple (stop_string, offset) if matched or else None.

    Where stop_string is the matched stop string and offset is the
    length to which output_text should be truncated, or -1 for no
    truncation.
    NrD   rQ   )r@   find)rL   rR   rF   rS   stop_strstop_string_len
stop_indexs          r   rV   rV   <  s       t $ $h-- %%hN0B_0TUU
 	$/)JS----|### ####4r   )&abcr   r   
tokenizers	packagingr   r   tokenizers.decodersr   transformersr   vllm.loggerr	   vllm.tokenizersr
   !vllm.tokenizers.detokenizer_utilsr   r   
vllm.utilsr   vllm.v1.enginer   r3   r   parse__version__r-   r   r   r=   r/   r0   r:   r8   r7   r9   tuplerV   r'   r   r   <module>r      s@   $ # # # # # # #                       , , , , , , 0 0 0 0 0 0 # # # # # # ) ) ) ) ) )        > = = = = = , , , , , ,	X		 %w}Z%;<<h@W@WW  6  >  >  >  >  >  >  >  >Fe e e e e!7 e e ePV V V V V!; V V Vr7 7 7 7 7!; 7 7 7t### s)# 	#
 38_t# # # # # #r   