
    Pi                     l    d dl mZmZ dZdZdZdZdZdZd Z	d	 Z
d
 Zd Zd Zd Zd Ze	e
eeeeedZdS )    )data_gym_to_mergeable_bpe_ranksload_tiktoken_bpe<|endoftext|>z<|fim_prefix|>z<|fim_middle|>z<|fim_suffix|>z<|endofprompt|>zO'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\sc                  P    t          dddd          } ddt          | t          did	S )
NzIhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpezLhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json@1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5@196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashgpt2Q  P  nameexplicit_n_vocabpat_strmergeable_ranksspecial_tokens)r   r50k_pat_str	ENDOFTEXTr   s    n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/tiktoken_ext/openai_public.pyr   r      sG    5bhY\	  O !*$e,      c                  L    t          dd          } ddt          | t          didS )NzGhttps://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken@306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930expected_hash	r50k_baser   r   r   r   r   r   r   s    r   r   r   !   A    'QX  O
 !*$e,  r   c                  L    t          dd          } ddt          | t          didS )NGhttps://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken@94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069r   	p50k_basei  r   r   r    r   s    r   r%   r%   /   r!   r   c                  x    t          dd          } t          dt          dt          dt          di}dt
          | |d	S )
Nr#   r$   r   r   r&   ij  ik  	p50k_editr   r   r   r   )r   r   
FIM_PREFIX
FIM_MIDDLE
FIM_SUFFIXr   r   r   s     r   r(   r(   =   sP    'QX  O  
E:ujZ_`N*(	  r   c            
      |    t          dd          } t          dt          dt          dt          dt
          di}d	d
| |dS )NzIhttps://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken@223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7r   i i i i i cl100k_basezu'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\sr)   )r   r   r*   r+   r,   ENDOFPROMPTr-   s     r   r0   r0   K   s^    'SX  O
 	6FFFVN  P*(	  r   c                      t          dd          } t          dt          di}d                    g d          }d|| |d	S )
NzHhttps://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken@446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2dr   ? iR |)zi[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?zi[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?z
\p{N}{1,3}z ?[^\s\p{L}\p{N}]+[\r\n/]*z
\s*[\r\n]+z	\s+(?!\S)z\s+
o200k_baser)   )r   r   r1   join)r   r   r   s      r   r6   r6   _   sm    'RX  O  f=N hh	
 	
 	

 
G *(	  r   c                      t                      } d}| d         }| d         }i | d         ddddd	d
ddddddddddd t          dd          D             z  }||||dS )No200k_harmonyr   r   r   i> r4   i@ iA iB iC iD iE iF iG iH iI iJ iK iL )z<|startoftext|>r   z<|reserved_200000|>z<|reserved_200001|>z
<|return|>z<|constrain|>z<|reserved_200004|>z<|channel|>z	<|start|>z<|end|>z<|message|>z<|reserved_200009|>z<|reserved_200010|>z<|reserved_200011|>z<|call|>c                     i | ]	}d | d|
S )z<|reserved_z|> ).0is     r   
<dictcomp>z!o200k_harmony.<locals>.<dictcomp>   s&    ???A	q			a???r   iM i r)   )r6   range)base_encr   r   r   r   s        r   r9   r9   {   s    ||HDy!G01O
#
$!%%%%%%!  " 	@?vv)>)>???#@N& *(	  r   )r   r   r%   r(   r0   r6   r9   N)tiktoken.loadr   r   r   r*   r+   r,   r1   r   r   r   r%   r(   r0   r6   r9   ENCODING_CONSTRUCTORSr;   r   r   <module>rC      s    L L L L L L L L	


 [ 
           (  8  @ "    r   