
     `i[                     J    d dl mZ d dlmZ d dlmZ d dlmZmZ dede	fdZ
dS )	    )Path)Any)TikTokenConverter)TIKTOKEN_VOCAB_FILETOKENIZER_FILEencoding
output_dirc                 @   t          |          }|                    d           |dz  t          z  }|t          z  }t	          |                                          }t	          |                                          }	 ddlm} ddlm	} t          | t                    r ||           }  || j        |           n# t          $ r t          d          w xY wt          || j        | j                                                  }|                    |           d	S )
a  
    Converts given `tiktoken` encoding to `PretrainedTokenizerFast` and saves the configuration of converted tokenizer
    on disk.

    Args:
        encoding (`str` or `tiktoken.Encoding`):
            Tokenizer from `tiktoken` library. If `encoding` is `str`, the tokenizer will be loaded with
            `tiktoken.get_encoding(encoding)`.
        output_dir (`str`):
            Save path for converted tokenizer configuration file.
    T)exist_oktiktokenr   )get_encoding)dump_tiktoken_bpezY`tiktoken` is required to save a `tiktoken` file. Install it with `pip install tiktoken`.)
vocab_filepatternadditional_special_tokensN)r   mkdirr   r   strabsoluter   r   tiktoken.loadr   
isinstance_mergeable_ranksImportError
ValueErrorr   _pat_str_special_tokens	convertedsave)	r   r	   	save_filetokenizer_filesave_file_absoluteoutput_file_absoluter   r   	tokenizers	            v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/integrations/tiktoken.pyconvert_tiktoken_to_fastr$      sT    j!!Jd###Z'*==I.0NY//1122~668899	v))))))333333h$$ 	.#|H--H(35GHHHH v v vtuuuv "%x/@\d\t  ikk  NN'(((((s    =B> >CN)pathlibr   typingr   #transformers.convert_slow_tokenizerr   $transformers.tokenization_utils_fastr   r   r   r$        r#   <module>r+      s                A A A A A A T T T T T T T T#)s #) #) #) #) #) #) #)r*   