
    Pi                    f    d dl mZ d dlZd dlZd dlZddZddZdddZ	 	 	 dd dZd!dZ	dd"dZ
dS )#    )annotationsNblobpathstrreturnbytesc                   d| vr>t          | dd          5 }|                                cd d d            S # 1 swxY w Y   |                     d          r4dd l}|                    |           }|                                 |j        S 	 dd l}n"# t          $ r}t          d          |d }~ww xY w|	                    | d          5 }|                                cd d d            S # 1 swxY w Y   d S )Nz://rbr   	buffering)zhttp://zhttps://Oblobfile is not installed. Please install it by running `pip install blobfile`.)
openread
startswithrequestsgetraise_for_statuscontentblobfileImportErrorBlobFile)r   fr   respr   es         a/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/tiktoken/load.py	read_filer      s   H(DA... 	!6688	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 233 ||H%%|   ]
 
	 
		8T	*	* avvxx                 s6   8<<B 
B1B,,B1
C++C/2C/dataexpected_hashboolc                Z    t          j        |                                           }||k    S N)hashlibsha256	hexdigest)r   r   actual_hashs      r   
check_hashr%      s(    .&&0022K-''    
str | Nonec                   d}dt           j        v rt           j        d         }nYdt           j        v rt           j        d         }n8dd l}t           j                            |                                d          }d}|dk    rt          |           S t          j        | 	                                          
                                }t           j                            ||          }t           j                            |          rxt          |dd	          5 }|                                }d d d            n# 1 swxY w Y   |t          ||          r|S 	 t          j        |           n# t           $ r Y nw xY wt          |           }	|r&t          |	|          st#          d
|  d| d          dd l}
	 t          j        |d           |dz   t)          |
                                          z   dz   }t          |d          5 }|                    |	           d d d            n# 1 swxY w Y   t          j        ||           n# t           $ r |r Y nw xY w|	S )NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r	   r
   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r!   sha1encoder#   existsr   r   r%   removeOSError
ValueErroruuidmakedirsr   uuid4writerename)r   r   user_specified_cache	cache_dirr1   	cache_key
cache_pathr   r   contentsr;   tmp_filenames               r   read_file_cachedrF   $   s	   rz))J34				+	+J34		GLL!4!4!6!68HII	$B"""X__..//99;;Ii33J	w~~j!! 
*da000 	A6688D	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 Jt]$C$C K	Ij!!!! 	 	 	D	 ""H 
Z-@@ 
Ih I I= I I I
 
 	

 KKK	
I----!C'#djjll*;*;;fD,%% 	GGH	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
	,
++++    		 	
 Os[   D<<E E E1 1
E>=E>=AI H/#I /H33I 6H37I I I Fvocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashclobber_one_byte_tokensdict[bytes, int]c                   d t          d          D             }d |D             d}t          d          D ]5}||vr/|                    |           |t          d|z             <   |dz  }6t          |          dk    sJ t	          | |                                          }d |                    d          dd         D             }	dfdd t          |          D             }
~t          |
          }|	D ]$\  }}||
 |           |          z   <   |dz  }%dd l}|	                    t	          ||                    }fd|
                                D             }|                    dd            |                    dd            |r#|D ] }t          |          dk    r||         |
|<   !|
|k    sJ |
S )Nc                z    g | ]8}t          |                                          #t          |          d k    6|9S ) )chrisprintable.0bs     r   
<listcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>b   s;    XXXQQ1C1C1E1EX#a&&TW--q---r&      c                .    i | ]}t          |          |S  )rP   rR   s     r   
<dictcomp>z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>d   s     @@@1SVVQ@@@r&   r      c                P    g | ]#}t          |                                          $S rX   )tuplesplit)rS   	merge_strs     r   rU   z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>o   s*    aaay%	))**aaar&   
valuer   r   r   c                :    t          fd| D                       S )Nc              3  (   K   | ]}|         V  d S r    rX   )rS   rT   data_gym_byte_to_bytes     r   	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>r   s)      ==!*1-======r&   r   )ra   rd   s    r   decode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymq   s&    ====u======r&   c                6    i | ]\  }}t          |g          |S rX   rf   )rS   irT   s      r   rY   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>v   s&    FFF41asQFFFr&   c                .    i | ]\  }} |          |S rX   rX   )rS   kvrg   s      r   rY   z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>   s)    RRRTQ??1--qRRRr&   s   <|endoftext|>s   <|startoftext|>)ra   r   r   r   )rangeappendrP   lenrF   decoder]   	enumeratejsonloadsitemspop)rG   rH   rI   rJ   rK   rank_to_intbytenrT   vocab_bpe_contents
bpe_merges	bpe_ranksfirstsecondrr   encoder_jsonencoder_json_loadedrk   rd   rg   s                    @@r   data_gym_to_mergeable_bpe_ranksr   Z   sI    YX%++XXXO@@@@@	A4[[  O##""1%%%34!#dQh--0FA4'''' *..IIPPRRaa<N<T<TUY<Z<Z[\]_[_<`aaaJ> > > > > >
 GF9_+E+EFFFI 	IA#  vFG	//%((??6+B+BBC	QKKK
 ::./@BSTTUULRRRR\=O=O=Q=QRRR,d333.555 6$ 	6 	6A1vv{{215	!+++++r&   rz   tiktoken_bpe_fileNonec                   	 dd l }n"# t          $ r}t          d          |d }~ww xY w|                    |d          5 }t          |                                 d           D ]T\  }}|                    t          j        |          dz   t          |          	                                z   dz              U	 d d d            d S # 1 swxY w Y   d S )Nr   r   r.   c                    | d         S )NrZ   rX   )xs    r   <lambda>z#dump_tiktoken_bpe.<locals>.<lambda>   s
    1Q4 r&   )key       
)
r   r   r   sortedrt   r>   base64	b64encoder   r6   )rz   r   r   r   r   tokenranks          r   dump_tiktoken_bper      sK      ]
 
	 
		,d	3	3 Qq!)//"3"3HHH 	Q 	QKE4GGF$U++d2SYY5E5E5G5GG%OPPPP	QQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Qs"    
&!&A:CCCc                &   t          | |          }i }|                                D ]i}|s	 |                                \  }}t          |          |t	          j        |          <   B# t          $ r}t          d|d|            |d }~ww xY w|S )NzError parsing line z in )rF   
splitlinesr]   intr   	b64decode	Exceptionr:   )r   r   rD   retliner   r   r   s           r   load_tiktoken_bper      s     1=AAH
C##%% [ [ 		[**,,KE4+.t99C ''(( 	[ 	[ 	[R4RR?PRRSSYZZ	[Js   ;A))
B3B		B)r   r   r   r   )r   r   r   r   r   r   r    )r   r   r   r'   r   r   )NNF)rG   r   rH   r   rI   r'   rJ   r'   rK   r   r   rL   )rz   rL   r   r   r   r   )r   r   r   r'   r   rL   )
__future__r   r   r!   r/   r   r%   rF   r   r   r   rX   r&   r   <module>r      s    " " " " " "   				   .( ( ( (
3 3 3 3 3r "&$($)7 7 7 7 7t	Q 	Q 	Q 	Q      r&   