
    .`i                     b    d dl mZ d dlmZmZmZmZ erd dlmZ d dl	m
Z
  G d de          ZdS )    )Path)TYPE_CHECKINGAnyProtocoloverload)BatchEncoding)ChatCompletionMessageParamc                      e Zd Zedddddeez  dededz  dedz  dd f
d	            Zdefd
Z	e
dee         fd            Ze
dee         fd            Ze
defd            Ze
defd            Ze
defd            Ze
defd            Ze
defd            Ze
defd            Ze
defd            ZdefdZdefdZ	 	 	 	 d.deee         z  dedz  dedededz  ddfdZdeeef         fdZdeeef         fdZ	 	 	 d/dededz  dedz  dedee         f
d Z	 d0d!ed"         d#eeeef                  dz  deee         z  fd$Zed%edefd&            Zed%ee         dee         fd'            Zd%eee         z  deee         z  fd(Zd%ee         defd)Z d1d*ee         ez  d+edefd,Z!	 d1d*ee         d+edee         fd-Z"dS )2TokenizerLikeFN)trust_remote_coderevisiondownload_dirpath_or_repo_idr   r   r   returnc                    t           NNotImplementedError)clsr   r   r   r   argskwargss          l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/tokenizers/protocol.pyfrom_pretrainedzTokenizerLike.from_pretrained   s
     "!    c                     t           r   r   selfs    r   num_special_tokens_to_addz'TokenizerLike.num_special_tokens_to_add       !!r   c                     t           r   r   r   s    r   all_special_tokensz TokenizerLike.all_special_tokens       !!r   c                     t           r   r   r   s    r   all_special_idszTokenizerLike.all_special_ids    r"   r   c                     t           r   r   r   s    r   bos_token_idzTokenizerLike.bos_token_id$   r"   r   c                     t           r   r   r   s    r   eos_token_idzTokenizerLike.eos_token_id(   r"   r   c                     t           r   r   r   s    r   pad_token_idzTokenizerLike.pad_token_id,   r"   r   c                     t           r   r   r   s    r   is_fastzTokenizerLike.is_fast0   r"   r   c                     t           r   r   r   s    r   
vocab_sizezTokenizerLike.vocab_size4   r"   r   c                     t           r   r   r   s    r   max_token_idzTokenizerLike.max_token_id8   r"   r   c                     t           r   r   r   s    r   truncation_sidezTokenizerLike.truncation_side<   r"   r   c                 :    t          t          |                     S r   )hashidr   s    r   __hash__zTokenizerLike.__hash__@   s    BtHH~~r   c                     | j         S r   )r.   r   s    r   __len__zTokenizerLike.__len__C   s
    r   Ttext	text_pairadd_special_tokens
truncation
max_lengthr   c                     t           r   r   )r   r9   r:   r;   r<   r=   s         r   __call__zTokenizerLike.__call__F   s
     "!r   c                     t           r   r   r   s    r   	get_vocabzTokenizerLike.get_vocabP   r   r   c                     t           r   r   r   s    r   get_added_vocabzTokenizerLike.get_added_vocabS   r   r   c                     t           r   r   )r   r9   r<   r=   r;   s        r   encodezTokenizerLike.encodeV   s
     "!r   messagesr	   toolsc                     t           r   r   )r   rF   rG   r   s       r   apply_chat_templatez!TokenizerLike.apply_chat_template_   s
     "!r   tokensc                     d S r    r   rJ   s     r   convert_tokens_to_idsz#TokenizerLike.convert_tokens_to_idsg   s    9<r   c                     d S r   rL   rM   s     r   rN   z#TokenizerLike.convert_tokens_to_idsj   s    EHSr   c                     t           r   r   rM   s     r   rN   z#TokenizerLike.convert_tokens_to_idsm   r   r   c                     t           r   r   rM   s     r   convert_tokens_to_stringz&TokenizerLike.convert_tokens_to_stringp   r   r   idsskip_special_tokensc                     t           r   r   r   rS   rT   s      r   decodezTokenizerLike.decodes   r   r   c                     t           r   r   rV   s      r   convert_ids_to_tokensz#TokenizerLike.convert_ids_to_tokensv   s
    
 "!r   )NTFN)NNTr   )F)#__name__
__module____qualname__classmethodstrr   boolr   intr   propertylistr!   r$   r&   r(   r*   r,   r.   r0   r2   r6   r8   r?   dictrA   rC   rE   r   rI   r   rN   rR   rW   rY   rL   r   r   r   r      s       
 #(##'	" 	" 	"t	"  		"
 *	" Dj	" 
	" 	" 	" [	""3 " " " " "DI " " " X" "c " " " X" "c " " " X" "c " " " X" "c " " " X" " " " " X" "C " " " X" "c " " " X" " " " " X"#         !%#' !%" "DIo" :" !	"
 " $J" 
" " " ""4S> " " " ""c3h " " " " #'!%#'" "" 4K" $J	"
 !" 
c" " " " .2" "34" DcN#d*"
 
tCy" " " " <C<C<<< X<HDIH$s)HHH XH"C$s)O "d3i " " " ""tCy "S " " " "" "$s)c/ " "QT " " " " %*" "#Y" "" 
c	" " " " " "r   r   N)pathlibr   typingr   r   r   r   transformersr   vllm.entrypoints.chat_utilsr	   r   rL   r   r   <module>rh      s          9 9 9 9 9 9 9 9 9 9 9 9 G******FFFFFFo" o" o" o" o"H o" o" o" o" o"r   