
    &`i=
                         d Z ddlZddlmZmZmZ erddlmZmZ eddef         Z	 ej
        e          Zde	de	fd	ZdS )
z'Utility functions for batch processing.    N)TYPE_CHECKINGAnyUnion)PreTrainedTokenizerPreTrainedTokenizerFastr   r   	tokenizerreturnc                 ~   t          | dd          t          | d          r| j        } t          | dd          t          | j                  t          | j                  t          | dd          t          |            G fdd| j                  }d| j        j         |_        || _        | S )a  Get tokenizer with cached properties.
    This will patch the tokenizer object in place.
    By default, transformers will recompute multiple tokenizer properties
    each time they are called, leading to a significant slowdown. This
    function caches these properties for faster access.
    Args:
        tokenizer: The tokenizer object.
    Returns:
        The patched tokenizer object.
    chat_templateNr   all_special_tokens_extendedc                       e Zd Zefd            Zefd            Zefd            Ze fd            ZfdZdS )-get_cached_tokenizer.<locals>.CachedTokenizerc                     S N )selftokenizer_all_special_idss    q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/llm/_internal/batch/utils.pyall_special_idsz=get_cached_tokenizer.<locals>.CachedTokenizer.all_special_ids.   s	    ,,    c                     S r   r   )r   tokenizer_all_special_tokenss    r   all_special_tokensz@get_cached_tokenizer.<locals>.CachedTokenizer.all_special_tokens2   s	    //r   c                     S r   r   )r   %tokenizer_all_special_tokens_extendeds    r   r   zIget_cached_tokenizer.<locals>.CachedTokenizer.all_special_tokens_extended6   s	    88r   c                     S r   r   )r   r   s    r   r   z;get_cached_tokenizer.<locals>.CachedTokenizer.chat_template:   s	      r   c                     S r   r   )r   tokenizer_lens    r   __len__z5get_cached_tokenizer.<locals>.CachedTokenizer.__len__>   s	      r   N)	__name__
__module____qualname__propertyr   r   r   r   r   )r   r   r   r   r   s   r   CachedTokenizerr   -   s        		- 	- 	- 	- 
	- 
	0 	0 	0 	0 
	0 
	9 	9 	9 	9 
	9 
	! 	! 	! 	! 
	!	! 	! 	! 	! 	! 	! 	!r   r$   Cached)	getattrhasattrr   setr   r   len	__class__r    )r   r$   r   r   r   r   r   s     @@@@@r   get_cached_tokenizerr+      s	    I==My+&& F'	  #IEEM #I$= > >#&y'C#D#D 
 -40$- -) 	NNM! ! ! ! ! ! ! ! ! ! !)- ! ! !(  G	(;(DFFO)Ir   )__doc__loggingtypingr   r   r   transformersr   r   AnyTokenizer	getLoggerr    loggerr+   r   r   r   <module>r3      s    - -  , , , , , , , , , , JIIIIIIII*,EsJK		8	$	$7L 7\ 7 7 7 7 7 7r   