
    .`i                     &   d dl Zd dlmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d d
lmZ ddlmZ e
rd dlm Z m!Z!  ee"          Z#dddddZ$e G d d                      Z% e%d e$&                                D                       Z'dddde(ez  ddde(fdZ) ee)          Z*d-d Z+ ed!ee"          Z,ed#ddd$de(ez  d%e-e,         d&e.d'e(dz  d(e(dz  d)e,fd*Z/ ee/          Z0d.d,Z1dS )/    N)	dataclassfield)	lru_cache)Path)TYPE_CHECKING)TypeVarassert_never)init_logger)check_gguf_fileget_gguf_file_path_from_hfis_ggufis_remote_ggufsplit_remote_gguf)list_filtered_repo_files)resolve_obj_by_qualname   )TokenizerLike)ModelConfig
RunnerType)deepseek_v32DeepseekV32Tokenizer)grok2Grok2Tokenizer)hfCachedHfTokenizer)mistralMistralTokenizer)r   r   r   r   c                       e Zd ZU  ee          Zeeeeef         f         ed<   dedededdfdZ	dede
e         fd	Zdedefd
ZdS )_TokenizerRegistry)default_factory
tokenizerstokenizer_modemodule
class_namereturnNc                 j    || j         v rt                              d|||           ||f| j         |<   d S )NzT%s.%s is already registered for tokenizer_mode=%r. It is overwritten by the new one.)r!   loggerwarningselfr"   r#   r$   s       l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/tokenizers/registry.pyregisterz_TokenizerRegistry.register-   sK    T_,,NN4   ,2:*>'t    c                     || j         vrt          d|d          | j         |         \  }}t                              d| d|           t	          | d|           S )Nz+No tokenizer registered for tokenizer_mode=.zLoading z for tokenizer_mode=)r!   
ValueErrorr'   
debug_oncer   r)   s       r+   load_tokenizer_clsz%_TokenizerRegistry.load_tokenizer_cls;   s{    00PNPPPQQQ!_^<
IZIInIIJJJ&&'?'?:'?'?@@@r-   c                 F    |                      |          } |j        |i |S )N)r2   from_pretrained)r*   r"   argskwargstokenizer_clss        r+   load_tokenizerz!_TokenizerRegistry.load_tokenizerD   s-    //??,},d=f===r-   )__name__
__module____qualname__r   dictr!   strtuple__annotations__r,   typer   r2   r8    r-   r+   r   r   (   s          .3U4-H-H-HJS%S/)*HHHs C S T    A Am9L A A A A>S >m > > > > > >r-   r   c                 *    i | ]\  }\  }}|d | |fS )zvllm.tokenizers.rA   ).0modemod_relnamecls_names       r+   
<dictcomp>rG   J   sA       )D); 	/+//:  r-   generateauto)runner_typer"   tokenizer_namerJ   r   r"   c          	         |                     d          }|                     d          }t          j        rddlm} ddlm} t          |                                           sT || |          5   |t          |           ||t          j        j        g d          }	|	} d d d            n# 1 swxY w Y   t          |           rst          |           r,t          |           j        |d<   t          |           j        } n8t#          |           r)t%          |           \  } }
t'          | |
|	          }||d<   d
|vr-|dk    s|dk    rd|d
<   n|dk    rd|d
<   nt)          |           |dk    r,|                     dd          rt+          d          d}d|d<   |dk    rWt,          j                            d          r8ddg}t3          t          |           ||          }t5          |          dk    rd}|dk    r7dg}t3          t          |           ||          }t5          |          dk    rd}|dk    rd}|| ||fS )Nrevisiondownload_dirr   )snapshot_download)get_lock)z.*.ptz.*.safetensorsz.*.bin)model_id	cache_dirrM   local_files_onlyignore_file_pattern	gguf_file)rM   truncation_siderH   draftleftpoolingrightslowuse_fastFz5Cannot use the fast tokenizer in slow tokenizer mode.r   rI   mistral_commonztekken.jsonztokenizer.model.v*)model_name_or_pathallow_patternsrM   r   ztokenizer.tok.jsonr   )getenvsVLLM_USE_MODELSCOPE modelscope.hub.snapshot_downloadrO   -vllm.model_executor.model_loader.weight_utilsrP   r   existsr=   huggingface_hub	constantsHF_HUB_OFFLINEr   r   nameparentr   r   r   r	   r0   	importlibutil	find_specr   len)rK   rJ   r"   r5   r6   rM   rN   rO   rP   tokenizer_path
quant_typerU   r_   
files_lists                 r+   resolve_tokenizer_argsrr   Q   s*    "::j11H%zz.99L 0 	GFFFFF 	KJJJJJ N##**,, 	0 .,77 	0 	0!2!2 00*%%4%>%M(M(M(M" " " "0	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 ~ ,>** 	,"&~"6"6";F;!.118NNN++ 	,):>)J)J&NJ2!  I
 #,F;&&*$$w(>(>(.F$%%I%%(/F$%%%%%::j%(( 	VTUUU"z IN$<$<=M$N$N')=>-">22)
 
 


 z??Q&N ./-">22)
 
 


 z??Q$N >477s   01B--B14B1configr   c                 X    t          | j        f| j        | j        | j        | j        d|S N)rJ   r"   rM   trust_remote_code)cached_resolve_tokenizer_args	tokenizerrJ   r"   tokenizer_revisionrv   )rs   r6   s     r+   tokenizer_args_from_configrz      sF    (&,* 2    r-   _T)bounddefaultF)r7   rv   rM   rN   r7   rv   rM   rN   r%   c                    t          | g|R |||d|\  }} }}|t          k    rt                              |          }n|} |j        | g|R i |}	|	j        st                              d           |	S )zHGets a tokenizer for the given model name via HuggingFace or ModelScope.)rv   rM   rN   ziUsing a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead.)rw   r   TokenizerRegistryr2   r4   is_fastr'   r(   )
rK   r7   rv   rM   rN   r5   r6   r"   tokenizer_cls_rx   s
             r+   get_tokenizerr      s     4Q4	4 4 ,!4 4 4 40NND& %%*==nMM&..~OOOOOOI 
A	
 	
 	

 r-   model_configc                 j    | j         rd S t          | j        f| j        | j        | j        | j        d|S ru   )skip_tokenizer_initcached_get_tokenizerrx   rJ   r"   ry   rv   )r   r6   s     r+   cached_tokenizer_from_configr      sV    ' t ,#20&8    r-   )rs   r   )r   r   )2importlib.utilrk   dataclassesr   r   	functoolsr   pathlibr   typingr   rf   typing_extensionsr   r	   	vllm.envsra   vllm.loggerr
   "vllm.transformers_utils.gguf_utilsr   r   r   r   r   "vllm.transformers_utils.repo_utilsr   vllm.utils.import_utilsr   protocolr   vllm.config.modelr   r   r9   r'   _VLLM_TOKENIZERSr   itemsr   r=   rr   rw   rz   r{   r@   boolr   r   r   rA   r-   r+   <module>r      s       ( ( ( ( ( ( ( (                             3 3 3 3 3 3 3 3       # # # # # #              H G G G G G ; ; ; ; ; ; # # # # # # :99999999	X		 =(
%.	   > > > > > > > >@ '& -=-C-C-E-E     !+ 	Y8 Y8 Y8$JY8 Y8 	Y8 Y8 Y8 Y8x !*	*@ A A     WT>>> ,##  $J 8 	
 Dj *    D !y//      r-   