
    `i4Q                       d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
mZmZmZmZmZmZ 	 d dlmZ n# e$ r dZY nw xY wd dlZddlmZ  ej        e          Z G d d	          Ze G d
 de                      Ze G d dee                      Z G d de          Z G d de          Z G d de          Z G d de          Z dS )    )annotationsN)Path)AnyCallableSequenceMappingIterableProtocolClassVarruntime_checkable)SentencePieceProcessor   )
GGUFWriterc                      e Zd ZU ded<   ded<   ded<   ded<   	 	 	 d%d&dZd'dZd(d)dZd*dZd+dZd,d"Z	d+d#Z
d+d$Zd
S )-SpecialVocab	list[str]mergeszdict[str, bool]add_special_tokendict[str, int]special_token_idsz(str | Sequence[Mapping[str, str]] | Nonechat_templateFNpathstr | os.PathLike[str]load_mergesboolspecial_token_typesIterable[str] | Nonen_vocab
int | Nonec                    i | _         i | _        || _        || _        g | _        d | _        ||| _        nd| _        |                     t          |                     d S )N)boseosunkseppadclsmask)	r   r   r   r   r   r   r   _loadr   )selfr   r   r   r   s        ^/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/gguf/vocab.py__init__zSpecialVocab.__init__   si    
 "$!#&!*':D$$'YD$

4::    returnstrc                p    d                     t          | j                  | j        pd| j        pd          S )NzG<SpecialVocab with {} merges, special tokens {}, add special tokens {}>unset)formatlenr   r   r   r)   s    r*   __repr__zSpecialVocab.__repr__-   s<    X__d4?AWAb[b
 
 	
r,   gwr   quietNonec                   | j         rM|s0t                              dt          | j                    d           |                    | j                    n!| j        rt                              d           | j                                        D ]k\  }}t          |d| dd           }|"t                              d| d| d           >|s t                              d	| d
|             ||           l| j
                                        D ]k\  }}t          |d| dd           }|"t                              d| d| d           >|s t                              d| d|             ||           l| j        @|s"t                              d| j                    |                    | j                   d S d S )NzAdding z
 merge(s).zJAdding merges requested but no merges found, output may be non-functional.add_	_token_idz"No handler for special token type z	 with id  - skippingzSetting special token type z to add_add__tokenzNo handler for add_z_token with value zSetting add_z
_token to zSetting chat_template to )r   loggerinfor2   add_token_mergesr   warningr   itemsgetattrr   r   add_chat_template)r)   r5   r6   typtokid
id_handlervalueadd_handlers           r*   add_to_ggufzSpecialVocab.add_to_gguf2   s)   ; 	i DBc$+&6&6BBBCCC,,,, 	iNNghhh06688 	 	JC7>rCX#CXCXCXZ^7_7_J!dCddRWdddeee LJ#JJ5JJKKKJu06688 	 	JC9@E[PSE[E[E[]a9b9bK"^S^^E^^^___ CA3AA%AABBBK) NL8JLLMMM  !344444 *)r,   r   c                    |                      |           |                     |           | j        r| j        s|                     |           d S d S d S N)_try_load_from_tokenizer_json_try_load_from_config_jsonr   r   _try_load_merges_txt)r)   r   s     r*   r(   zSpecialVocab._loadN   sl    **4000''--- 	,DK 	,%%d+++++	, 	, 	, 	,r,   c                r   |dz  }|                                 sdS t          |dd          5 }t          |d                                          }|                    d          s|                    d           d}nd	}g }|D ]}|d	z  }|                                }|s|                    d d
          }t          |          dk    r&t          	                    |j
         d| d           m|                    |d          d|d	                     	 d d d            n# 1 swxY w Y   || _        dS )Nz
merges.txtFrutf-8encoding #r   r         z: Line z: Entry malformed, ignoring T)is_fileopennextstrip
startswithseeksplitr2   r>   rA   nameappendr   )	r)   r   merges_filefp
first_lineline_numr   linepartss	            r*   rO   z!SpecialVocab._try_load_merges_txtT   s   \)""$$ 	5+sw777 	82b"++--J((-- 


F 	8 	8Azz|| 

4++u::??NNk&6#d#dx#d#d#deeeq66E!H667777	8	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8$ ts   C*D%%D),D)rE   tidr   c           	        t          |t                    sd S |dk     rt          d| d|           | j        || j        k     r|| j        v rd S || j        |<   d S t
                              d| d| d| j         d           d S )Nr   z%invalid value for special token type z: zSpecial token type z, id z out of range, must be under r;   )
isinstanceint
ValueErrorr   r   r>   rA   )r)   rE   ri   s      r*   _set_special_tokenzSpecialVocab._set_special_tokenm   s    #s## 	F77QSQQCQQRRR<3#5#5d,,,*-D"3'FrSrrsrrY]Yerrrsssssr,   c                Z   |dz  }|                                 rt          |d          5 }t          j        |          }d d d            n# 1 swxY w Y   | j        r,|                    di                               d          }t          |t                    r|rt          |d         t                    r|| _	        nt          |d         t                    rt          |d                   dk    rt          |d         d         t                    ret          d |D                       r:t                              d	t          t          d
          dz                        d |D             | _	        nt!          d          |                    di           }ni }|dz  }|                                 sdS t          |d          5 }t          j        |          }d d d            n# 1 swxY w Y   d }	|dz  }
|
                                 rPt          |
d          5 }t          j        |                              d          }	d d d            n# 1 swxY w Y   |                    d|	          }|t          |t          t          f          r|| _        nt                              d|d           | j        D ]}|                    d| d          }t          |t&                    r
|| j        |<   |                    | d          }t          |t                    r|nDt          |t*                    r.|                    d          }t          |t                    s|nt-          fd|D             d           }|                     ||           dS )Ntokenizer.jsonrR   rS   modelr   r   rX   c              3  (   K   | ]}|D ]}d |v V  	dS )rY   N ).0pairss      r*   	<genexpr>z=SpecialVocab._try_load_from_tokenizer_json.<locals>.<genexpr>   s3      IIDDIIqsaxIIIIIIIr,   z'Spaces in merges detected, encoding as rY      c                L    g | ]!}d                      d |D                       "S )rY   c                L    g | ]!}d                      d |D                       "S )rU   c              3  d   K   | ]+}|d k    rt          t          |          dz             n|V  ,dS )rY   rx   N)chrord)rt   cs     r*   rw   zSSpecialVocab._try_load_from_tokenizer_json.<locals>.<listcomp>.<listcomp>.<genexpr>   sU       ,& ,&,- >?#XXCFFSL(9(9(91,& ,& ,& ,& ,& ,&r,   join)rt   parts     r*   
<listcomp>zISpecialVocab._try_load_from_tokenizer_json.<locals>.<listcomp>.<listcomp>   sT     !" !" !" )-	 %'GG ,& ,&15,& ,& ,& %& %&!" !" !"r,   r   )rt   ru   s     r*   r   z>SpecialVocab._try_load_from_tokenizer_json.<locals>.<listcomp>   sV     ' ' ' !%  HH!" !" 15!" !" !"	 	' ' 'r,   zUnknown tokenizer merges formatadded_tokensztokenizer_config.jsonTzchat_template.jsonr   z$Bad type for chat_template field in z - ignoringr9   r=   contentc              3  t   K   | ]2}|                     d           k    |                     d          V  3dS )r   idN)get)rt   atok
tc_contents     r*   rw   z=SpecialVocab._try_load_from_tokenizer_json.<locals>.<genexpr>   sD      ^^DDHHY<O<OS]<]<]$<]<]<]<]^^r,   )rZ   r[   jsonloadr   r   rk   listr.   r   r2   anyr>   rA   r|   r}   rm   r   r   r   r   dictr\   rn   )r)   r   tokenizer_filef	tokenizerr   r   tokenizer_config_filetokenizer_configchat_template_altchat_template_filer   rE   	add_entryentryentry_contentmaybe_token_idr   s                    @r*   rM   z*SpecialVocab._try_load_from_tokenizer_jsony   s    00!!## 	n999 )Q IaLL	) ) ) ) ) ) ) ) ) ) ) ) ) ) ) L"w3377AAfd++ L L!&)S11 L&,#F1It44 LVAY19L9LQ[\bcd\efg\hjmQnQn9L IIFIIIII n"NN+lUXY\]`YaYadgYgUhUh+l+lmmm' ' )/' ' ' ))JKKK$==<<LLL $'> >$,,.. 	4'G<<< 	,#y||	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, !$88%%'' 	F(W=== F$(IaLL$4$4_$E$E!F F F F F F F F F F F F F F F(,,_>OPP J}sDk$J$J !.DNNfBWfffggg+ 	9 	9C(,,-?C-?-?-?@@I)T** 8.7&s+$((C88E%%% "

E4((  %		) 4 4!-55 *

!^^^^L^^^ N ##C8888ts5   AAAG33G7:G7*(II"%I"c                "   |dz  }|                                 sdS t          |d          5 }t          j        |          }d d d            n# 1 swxY w Y   | j        D ].}|                     ||                    | d                     /dS )Nzconfig.jsonFrR   rS   r:   T)rZ   r[   r   r   r   rn   r   )r)   r   config_filer   configrE   s         r*   rN   z'SpecialVocab._try_load_from_config_json   s    ]*""$$ 	5+'222 	"aYq\\F	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"+ 	H 	HC##Cs4E4E4E)F)FGGGGts   AAA)FNN)r   r   r   r   r   r   r   r   r-   r.   )F)r5   r   r6   r   r-   r7   )r   r   r-   r7   )r   r   r-   r   )rE   r.   ri   r   r-   r7   )__name__
__module____qualname____annotations__r+   r4   rJ   r(   rO   rn   rM   rN   rs   r,   r*   r   r      s        &&&&%%%%;;;; AF48"    "
 
 
 

5 5 5 5 58, , , ,   2
t 
t 
t 
tE E E EN     r,   r   c                  $    e Zd ZU ded<   ded<   dS )	BaseVocabzClassVar[str]tokenizer_modelra   N)r   r   r   r   rs   r,   r*   r   r      s*         """"r,   r   c                  H    e Zd ZU ded<   ded<   ded<   ded<   dd
ZddZdS )Vocabrl   
vocab_sizer   added_tokens_dictr   added_tokens_listr   fname_tokenizer	base_pathc                    d S rL   rs   )r)   r   s     r*   r+   zVocab.__init__         r,   r-   -Iterable[tuple[bytes, float, gguf.TokenType]]c                    d S rL   rs   r3   s    r*   
all_tokenszVocab.all_tokens   r   r,   Nr   r   r-   r   )r   r   r   r   r+   r   rs   r,   r*   r   r      sW         OOO%%%%    ,,,,NNNNNNr,   r   c                      e Zd ZdZdZddZdS )NoVocabno_vocabr-   r.   c                    dS )Nz3<NoVocab for a model without integrated vocabulary>rs   r3   s    r*   r4   zNoVocab.__repr__   s    DDr,   Nr   )r   r   r   r   ra   r4   rs   r,   r*   r   r      s8         ODE E E E E Er,   r   c                  >    e Zd ZdZdZddZddZdd	Zdd
ZddZ	dS )BpeVocabgpt2bper   r   c           
         i }|dz  x}                                 rt          |d          5 }t          j        |           _        d d d            n# 1 swxY w Y   	 t          |dz  d          5 }t          j        |          }d d d            n# 1 swxY w Y   n# t
          $ r Y nw xY w|dz  }t          |d          5 }t          j        |          }d d d            n# 1 swxY w Y   |d         }|d         dk    s(|                    d	d
          s|d         d         dk    rt          d          |d          _        |                    d          x} fd|D             }t           j                  }t          t          ||t          |          z                       }	t          |                                          }
|	|
k    r=|t          |
          z   dz
  }t          dt          |
           d| d| d|
           t          |                                d           }| _        d |D              _        | _         j        t           j                  z    _        | _        d S )Nz
vocab.jsonrR   rS   added_tokens.jsonrp   rq   typeBPEbyte_fallbackFdecoder	ByteLevelzCannot find GPT-2 BPE tokenizervocabr   c                L    i | ] }|d          j         v|d          |d         !S )r   r   )r   )rt   itemr)   s     r*   
<dictcomp>z%BpeVocab.__init__.<locals>.<dictcomp>	  sA      F  F  F$(#'	?$*#D#D !%Yd#D#D#Dr,   r   zExpected the z1 added token ID(s) to be sequential in the range z - z; got c                    | d         S Nr   rs   )text_idxs    r*   <lambda>z#BpeVocab.__init__.<locals>.<lambda>  s
    (1+ r,   keyc                    g | ]\  }}|S rs   rs   )rt   textidxs      r*   r   z%BpeVocab.__init__.<locals>.<listcomp>  s    $C$C$CktST$C$C$Cr,   )existsr[   r   r   r   FileNotFoundErrorr   r2   r   rangesortedvaluesrm   rB   r   r   vocab_size_baser   r   )r)   r   r   r   r   tokenizer_jsonr   addedr   expected_ids
actual_idsexpected_end_idrB   s   `            r*   r+   zBpeVocab.__init__   s   ')(<77O??AA  	Fo888 *A!Yq\\
* * * * * * * * * * * * * * *)&99GLLL 0PQ#'9Q<<L0 0 0 0 0 0 0 0 0 0 0 0 0 0 0$    (*::O o888 .A!%1. . . . . . . . . . . . . . . /=W.EO'500O4G4GY^4_4_0!),V4CC'(IJJJ(1DJ'++N;;;H F  F  F  F,1 F  F  F 4:
E*j3|;L;L.LMMNNl113344
:%%(3z??:Q>O SS__ S S *S S/>S SFPS S T T T |))++1M1MNNN$0$C$CU$C$C$C$.$($83t?U;V;V$V$3sY   AAA B! 4B	B! BB! BB! !
B.-B.C((C,/C,r-   r   c              #     K   d | j                                         D             }t          | j                   D ]!\  }}||         dt          j        j        fV  "d S )Nc                    i | ]\  }}||	S rs   rs   rt   encoded_tokr   s      r*   r   z'BpeVocab.bpe_tokens.<locals>.<dictcomp>  s    SSS_["[SSSr,   g        )r   rB   	enumerategguf	TokenTypeNORMAL)r)   reverse_vocabi_s       r*   
bpe_tokenszBpeVocab.bpe_tokens  sn      SS
@P@P@R@RSSSdj)) 	? 	?DAq"C)>>>>>>	? 	?r,   c              #  t   K   | j         D ]-}d}|                    d          |t          j        j        fV  .d S N     @rR   )r   encoder   r   CONTROLr)   r   scores      r*   r   zBpeVocab.added_tokens"  sR      * 	F 	FDE++g&&t~/EEEEEE	F 	Fr,   c              #  r   K   |                                  E d {V  |                                 E d {V  d S rL   )r   r   r3   s    r*   r   zBpeVocab.all_tokens'  sR      ??$$$$$$$$$$$&&&&&&&&&&&r,   r.   c                B    d| j          dt          | j                   dS )Nz<BpeVocab with  base tokens and  added tokens>r   r2   r   r3   s    r*   r4   zBpeVocab.__repr__+  s)    s!5ssDLbHcHcssssr,   Nr   r   r   )
r   r   r   r   ra   r+   r   r   r   r4   rs   r,   r*   r   r      s        OD24 24 24 24h? ? ? ?F F F F
' ' ' 't t t t t tr,   r   c                  >    e Zd ZdZdZddZddZdd	Zdd
ZddZ	dS )SentencePieceVocabllamaspmr   r   c           	        t           t          d          i }|dz  x}                                rR	 t          |dz  d          5 }t	          j        |          }d d d            n# 1 swxY w Y   n=# t          $ r Y n1w xY w|j        dz  x}                                st          d          t                      | _        | j        	                    t          |                     | j                                        fd|                                D             t          t          t                    z                       }t!                                                    }||k    rt%          d| d	|           || _        fd
|D             | _        | _        | j        t          | j                  z   | _        || _        d S )Nzsentencepiece is not installedztokenizer.modelr   rR   rS   zCannot find tokenizer.modelc                (    i | ]\  }}|k    ||S rs   rs   )rt   piecer   r   s      r*   r   z/SentencePieceVocab.__init__.<locals>.<dictcomp>G  s*    ```)%rU_O_O_BO_O_O_r,   zExpected new token IDs z to be sequential; got c                     g | ]
}|         S rs   rs   )rt   r   
new_tokenss     r*   r   z/SentencePieceVocab.__init__.<locals>.<listcomp>P  s    "K"K"Kb:b>"K"K"Kr,   )r   RuntimeErrorr   r[   r   r   r   parentsentencepiece_tokenizerLoadFromFiler.   r   rB   r   r   r2   r   keysrm   r   r   r   r   )	r)   r   r   r   r   expected_new_idsactual_new_idsr   r   s	          @@r*   r+   zSentencePieceVocab.__init__3  s1   !)?@@@')(+<<<ODDFF 		C)&99GLLL 0PQ#'9Q<<L0 0 0 0 0 0 0 0 0 0 0 0 0 0 0$   %.%58I%II/QQSS 	C#$ABBB'='?'?$$11#o2F2FGGG1<<>>
````|7I7I7K7K```
j*s:2N O OPP!*//"3"344~--p7Gpp`nppqqq #/"K"K"K"KN"K"K"K","&"6T=S9T9T"T"1s5   A6 	A*A6 *A..A6 1A.2A6 6
BBr-   r   c              #  >  K   | j         }t          |                                          D ]}|                    |          }|                    d          }|                    |          }t          j        j        }|	                    |          rt          j        j
        }|                    |          rt          j        j        }|                    |          rt          j        j        }|                    |          rt          j        j        }|||fV  d S )NrR   )r   r   r   	IdToPiecer   GetScorer   r   r   	IsUnknownUNKNOWN	IsControlr   IsUnusedUNUSEDIsByteBYTE)r)   r   r   r   r   r   toktypes          r*   sentencepiece_tokensz'SentencePieceVocab.sentencepiece_tokensU  s     0	y++--.. 	' 	'A''**E <<00D$--a00En+G""1%% 1.0""1%% 1.0 !!!$$ 0./"" ..-w&&&&&)	' 	'r,   c              #  t   K   | j         D ]-}d}|                    d          |t          j        j        fV  .d S r   )r   r   r   r   USER_DEFINEDr   s      r*   r   zSentencePieceVocab.added_tokensm  sR      * 	K 	KDE++g&&t~/JJJJJJ	K 	Kr,   c              #  r   K   |                                  E d {V  |                                 E d {V  d S rL   )r  r   r3   s    r*   r   zSentencePieceVocab.all_tokensr  sT      ,,.........$$&&&&&&&&&&&r,   r.   c                B    d| j          dt          | j                   dS )Nz<SentencePieceVocab with r   r   r   r3   s    r*   r4   zSentencePieceVocab.__repr__v  s+    }4+?}}RUVZVlRmRm}}}}r,   Nr   r   r   )
r   r   r   r   ra   r+   r  r   r   r4   rs   r,   r*   r   r   /  s        OD 2  2  2  2D' ' ' '0K K K K
' ' ' '~ ~ ~ ~ ~ ~r,   r   c                  T    e Zd ZdZdZddZddZddZddZddZ	d Z
ddZddZdS )LlamaHfVocabr   hfftr   r   c                    |dz  }t          |d          5 }t          j        |          }d d d            n# 1 swxY w Y   |d         }|d         dk    o,|                    dd          o|                    d	d
           }|rt	          d          |sC|d         dk    s(|                    d	d          r|d         d         dk    rt          d          	 ddlm} n"# t          $ r}t          d          |d }~ww xY w|	                    ||d
           _
         j
        j        sJ g  _        t                       _        t                       _        t#           j
                                                                        d           D ]S\  }	}
|
 j
        j        k    r> j                            |	           |
 j        |	<    j                            |
           T fd j
        j        D              _        t           j
        j                   _         j
        j         _         j        t9           j                  z    _        | _        d S )Nrp   rR   rS   rq   r   r   ignore_mergesFr   Tz'Llama 3 must be converted with BpeVocabr   r   zCannot find Llama BPE tokenizerr   )AutoTokenizerzsTo use LlamaHfVocab, please install the `transformers` package. You can install it with `pip install transformers`.)	cache_dirlocal_files_onlyc                    | d         S r   rs   )xs    r*   r   z'LlamaHfVocab.__init__.<locals>.<lambda>  s
    AaD r,   r   c                P    i | ]"}|j                                         |         #S rs   )r   	get_vocab)rt   tokr)   s     r*   r   z)LlamaHfVocab.__init__.<locals>.<dictcomp>  s>     
 
 
 ))++C0
 
 
r,   )r[   r   r   r   	TypeErrorr   transformersr  ImportErrorfrom_pretrainedr   is_fastr   r   r   setadded_tokens_idsr   get_added_vocabrB   r   rb   addall_special_tokensspecialsall_special_idsspecial_idsr   r2   r   )r)   r   r   r   r   r   	is_llama3r  er  tokidxs   `          r*   r+   zLlamaHfVocab.__init__~  s    #&66/G444 	*!Yq\\N	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* +9*AF#u, ?1D1D_V[1\1\ ?#''>>> 	  	GEFFF 	GF#u,,O4G4GY^4_4_,i(0J>>#$EFFF	2222222 	 	 	F  	 '66! 7 
 

 ~%%%% "$!%!$ "N**,,2244..
 
 
 	2 	2KC 222&--c222.4&s+%))&111
 
 
 
~8
 
 
 t~=>>  $~8#3c$:P6Q6QQ.s%   9= =C# #
D-C==Dr-   r   c              #  T  K   d | j                                                                         D             }t          | j                  D ][}|| j        v r||                             d          }||                     |          |                     ||| j	                  fV  \d S )Nc                    i | ]\  }}||	S rs   rs   r   s      r*   r   z*LlamaHfVocab.hf_tokens.<locals>.<dictcomp>  s+     
 
 
 /RB
 
 
r,   rR   )
r   r  rB   r   r   r%  r   get_token_scoreget_token_typer+  )r)   r   token_id
token_texts       r*   	hf_tokenszLlamaHfVocab.hf_tokens  s      
 
37>3K3K3M3M3S3S3U3U
 
 
 d233 	 	H4000 'x077@@J d228<<d>Q>Q*d&6? ?     	 	r,   r3  rl   r4  bytesr+  set[int]gguf.TokenTypec                    t          j        d|          rt          j        j        S ||v rt          j        j        nt          j        j        S )Ns   <0x[0-9A-Fa-f]{2}>)re	fullmatchr   r   r  r   r   )r)   r3  r4  r+  s       r*   r2  zLlamaHfVocab.get_token_type  sC    <.
;; 	'>&& *2[)@)@t~%%dnF[[r,   floatc                    dS )Nr   rs   )r)   r3  s     r*   r1  zLlamaHfVocab.get_token_score  s	     wr,   c              #    K   | j         D ]}|| j        v rH|                     | j        |         d| j                  }|                     | j        |                   }nt
          j        j        }d}|                    d          ||fV  d S )Nr,   r   rR   )	r   r)  r2  r+  r1  r   r   r  r   )r)   r   r  r   s       r*   r   zLlamaHfVocab.added_tokens  s      * 	7 	7Dt}$$--dmD.A3HXYY,,T]4-@AA.5++g&&w66666	7 	7r,   c                :    d| j         j        v pd| j         j        v S )Nz<0x0A>
)r   r   r3   s    r*   has_newline_tokenzLlamaHfVocab.has_newline_token  s!    4>//O44>;O3OOr,   c              #  r   K   |                                  E d {V  |                                 E d {V  d S rL   )r5  r   r3   s    r*   r   zLlamaHfVocab.all_tokens  sR      >>#########$$&&&&&&&&&&&r,   r.   c                B    d| j          dt          | j                   dS )Nz<LlamaHfVocab with r   r   r   r3   s    r*   r4   zLlamaHfVocab.__repr__  s*    wT%9wwCPTPfLgLgwwwwr,   Nr   r   )r3  rl   r4  r6  r+  r7  r-   r8  )r3  rl   r-   r<  r   )r   r   r   r   ra   r+   r5  r2  r1  r   rA  r   r4   rs   r,   r*   r  r  z  s        OD@/ @/ @/ @/D   $\ \ \ \   
	7 	7 	7 	7P P P' ' ' 'x x x x x xr,   r  )!
__future__r   r:  loggingr   ospathlibr   typingr   r   r   r   r	   r
   r   r   sentencepiecer   r!  r   gguf_writerr   	getLoggerr   r>   r   r   r   r   r   r   r  rs   r,   r*   <module>rL     s   " " " " " " 				   				       d d d d d d d d d d d d d d d d d d d d"4444444 " " "!"  # # # # # #		8	$	$r r r r r r r rj        
 O O O O OIx O O OE E E E Ei E E EHt Ht Ht Ht Htu Ht Ht HtVH~ H~ H~ H~ H~ H~ H~ H~Vxx xx xx xx xx5 xx xx xx xx xxs   9 AA