
    .`i]                     z    d dl mZ d dlmZmZ d dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ  G d	 d
e          ZdS )    )Path)Anyoverload)BatchEncoding)ChatCompletionMessageParam   )encode_messages)CachedHfTokenizer)TokenizerLikec                       e Zd Zedddddeez  dededz  dedz  dd	f
 fd
            Zdeddf fdZ		 d1de
d         de
eeef                  dz  dee
e         z  fdZdefdZede
e         fd            Zede
e         fd            Zedefd            Zedefd            Zedefd            Zedefd            Zedefd            Zedefd            Zedefd            ZdefdZdefdZ	 	 	 	 d2dee
e         z  dedz  d ed!ed"edz  dd#fd$Zdeeef         fd%Zdeeef         fd&Z	 	 	 d3ded!edz  d"edz  d ede
e         f
d'Ze d(edefd)            Z!e d(e
e         de
e         fd*            Z!d(ee
e         z  dee
e         z  fd+Z!d(e
e         defd,Z"d4d-e
e         ez  d.edefd/Z#	 d4d-e
e         d.ede
e         fd0Z$ xZ%S )5DeepseekV32TokenizerFNtrust_remote_coderevisiondownload_dirpath_or_repo_idr   r   r   returnr   c                d     t                      j        |g|R |||d|}t          |          S )Nr   )superfrom_pretrainedr   )	clsr   r   r   r   argskwargs	tokenizer	__class__s	           p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/tokenizers/deepseek_v32.pyr   z$DeepseekV32Tokenizer.from_pretrained   s\     ,EGG+

 
 0%
 
 
 
	 $I...    r   c                     t                                                       || _        t          |dd          | _        | j                                        | _        t          | j                  | _        d S )Nname_or_path )	r   __init__r   getattrr   get_added_vocab_added_vocablen_added_vocab_size)selfr   r   s     r   r!   zDeepseekV32Tokenizer.__init__%   sc    "#I~rBB N::<<!$T%6!7!7r   messagesr   toolsc                                          dd          }                     dd          }|p|}d}|sd}                     d|          }|                                }|6t          |          dk    r#|                    dddi           ||d         d	<   |d
         d         dk    }t	          ||          }	t          |fi |	}
                     dd          rfddD             } | j        |
fddi|S |
S )NthinkingFenable_thinkingchatconversationr   rolesystemr)   user)thinking_modedrop_thinkingtokenizeTc                 *    i | ]}|v ||         S  r7   ).0kr   s     r   
<dictcomp>z<DeepseekV32Tokenizer.apply_chat_template.<locals>.<dictcomp>H   s+          !"a6kk6!9kkkr   )
truncation
max_lengthadd_special_tokens)getcopyr%   insertdictr	   encode)r'   r(   r)   r   r+   r,   r3   r.   r4   encode_config
prompt_strtokenizer_kwargss      `        r   apply_chat_templatez(DeepseekV32Tokenizer.apply_chat_template.   s\    ::j%00 **%6>>." 	#"Mzz.(;;$$&&UaOOA1222#(HQK  !V,6=VVV$X????
::j$'' 	       &B      4; #( #   r   c                 F    t          |                     d                    S )Nr    )r%   rB   r'   s    r   num_special_tokens_to_addz.DeepseekV32Tokenizer.num_special_tokens_to_addS   s    4;;r??###r   c                     | j         j        S N)r   all_special_tokensrH   s    r   rL   z'DeepseekV32Tokenizer.all_special_tokensV   s    ~00r   c                     | j         j        S rK   )r   all_special_idsrH   s    r   rN   z$DeepseekV32Tokenizer.all_special_idsZ       ~--r   c                     | j         j        S rK   )r   bos_token_idrH   s    r   rQ   z!DeepseekV32Tokenizer.bos_token_id^       ~**r   c                     | j         j        S rK   )r   eos_token_idrH   s    r   rT   z!DeepseekV32Tokenizer.eos_token_idb   rR   r   c                     | j         j        S rK   )r   pad_token_idrH   s    r   rV   z!DeepseekV32Tokenizer.pad_token_idf   rR   r   c                     | j         j        S rK   )r   is_fastrH   s    r   rX   zDeepseekV32Tokenizer.is_fastj   s    ~%%r   c                     | j         j        S rK   )r   
vocab_sizerH   s    r   rZ   zDeepseekV32Tokenizer.vocab_sizen   s    ~((r   c                     | j         j        S rK   )r   max_token_idrH   s    r   r\   z!DeepseekV32Tokenizer.max_token_idr   rR   r   c                     | j         j        S rK   )r   truncation_siderH   s    r   r^   z$DeepseekV32Tokenizer.truncation_sidev   rO   r   c                 :    t          t          |                     S rK   )hashidrH   s    r   __hash__zDeepseekV32Tokenizer.__hash__z   s    BtHH~~r   c                      | j         | j        z   S rK   )rZ   r&   rH   s    r   __len__zDeepseekV32Tokenizer.__len__}   s    !777r   Ttext	text_pairr=   r;   r<   r   c                 6    |                      |||||          S )N)rf   r=   r;   r<   )r   )r'   re   rf   r=   r;   r<   s         r   __call__zDeepseekV32Tokenizer.__call__   s.     ~~1!!  
 
 	
r   c                 4    | j                                         S rK   )r   	get_vocabrH   s    r   rj   zDeepseekV32Tokenizer.get_vocab   s    ~'')))r   c                 4    | j                                         S rK   )r$   r?   rH   s    r   r#   z$DeepseekV32Tokenizer.get_added_vocab   s     %%'''r   c                 >    | j                             ||||          S )N)r;   r<   r=   )r   rB   )r'   re   r;   r<   r=   s        r   rB   zDeepseekV32Tokenizer.encode   s/     ~$$!!1	 % 
 
 	
r   tokensc                     d S rK   r7   r'   rm   s     r   convert_tokens_to_idsz*DeepseekV32Tokenizer.convert_tokens_to_ids   s    9<r   c                     d S rK   r7   ro   s     r   rp   z*DeepseekV32Tokenizer.convert_tokens_to_ids   s    EHSr   c                 6    | j                             |          S rK   )r   rp   ro   s     r   rp   z*DeepseekV32Tokenizer.convert_tokens_to_ids   s    ~33F;;;r   c                 6    | j                             |          S rK   )r   convert_tokens_to_stringro   s     r   rt   z-DeepseekV32Tokenizer.convert_tokens_to_string   s    ~66v>>>r   idsskip_special_tokensc                 :    | j                             ||          S N)rv   )r   decoder'   ru   rv   s      r   ry   zDeepseekV32Tokenizer.decode   s    ~$$S>Q$RRRr   c                 :    | j                             ||          S rx   )r   convert_ids_to_tokensrz   s      r   r|   z*DeepseekV32Tokenizer.convert_ids_to_tokens   s)    
 ~33%8 4 
 
 	
r   rK   )NTFN)NNT)F)&__name__
__module____qualname__classmethodstrr   boolr   r   r!   listrA   r   intrF   rI   propertyrL   rN   rQ   rT   rV   rX   rZ   r\   r^   rb   rd   rh   rj   r#   rB   r   rp   rt   ry   r|   __classcell__)r   s   @r   r   r      s       
 #(##'/ / /t/  	/
 */ Dj/ 
/ / / / / [/&8- 8D 8 8 8 8 8 8 .2# #34# DcN#d*#
 
tCy# # # #J$3 $ $ $ $ 1DI 1 1 1 X1 .c . . . X. +c + + + X+ +c + + + X+ +c + + + X+ & & & & X& )C ) ) ) X) +c + + + X+ . . . . X.#    8 8 8 8 8 !%#' !%
 
DIo
 :
 !	

 
 $J
 

 
 
 
 *4S> * * * *(c3h ( ( ( ( #'!%#'
 

 4K
 $J	

 !
 
c
 
 
 
 <C<C<<< X<HDIH$s)HHH XH<C$s)O <d3i < < < <?tCy ?S ? ? ? ?S S$s)c/ S SQT S S S S %*
 
#Y
 "
 
c	
 
 
 
 
 
 
 
r   r   N)pathlibr   typingr   r   transformersr   vllm.entrypoints.chat_utilsr   deepseek_v32_encodingr	   hfr
   protocolr   r   r7   r   r   <module>r      s                          & & & & & & B B B B B B 2 2 2 2 2 2 ! ! ! ! ! ! # # # # # #k
 k
 k
 k
 k
, k
 k
 k
 k
 k
r   