
     `iv                     r   d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej        e          Zi dddddddddddddddd
dddddddddddd
dddddddddddd
dddddddddddddd d!ddddddddddd"d#d$ddddddddddd
d%dddddddddddddd&d'ddddddddddd
d(d)d*d+d,d-d.ddddddddddd
d/d0dd1dd2d3d4d5d6d7d8d9d:dd2d;d<dd=d>dddddddd?dd@	dAd0dBdCdDd1d2dEdFdddddddGdHdIdddd2ddJdKdLddM	dddddddddNdd
dddddddddddOddPdddddddddddOddPd5d6d7d8d9d:dd2d;d<dd=ddddddddddd
dQZdRdSdTdUdVd)d*d+d,dWdX
dYdd)d*d+d,dZd[Zd\ Z G d] d^          Z G d_ d`e          Z  G da dbe          Z! G dc dde          Z" G de dfe          Z# G dg dhe          Z$ G di dje          Z%i de de!dke!d$e!d%e!d.e"d/e#d'e#d>e#dAe#dFe#d4e$dIe#dle#dme%dne%doe$e e dpZ&dqe'dre(ee)f         fdsZ*dS )tz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers
processors)BPEUnigram   
AddedToken)GemmaConverterGPT2ConverterLlamaConverterQwen2ConverterT5Converter)logging)tqdmgeneral
model_type_model_name_or_path)architecturenamellamamax_position_embeddingsnum_hidden_layersintermediate_sizehidden_sizehead_dim
rope_thetanum_attention_headsnum_key_value_headsrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonr$   mistralqwen2qwen2moenum_expertsnum_experts_per_tok)r%   r&   r'   r(   r)   r*   r+   r,   r-   r$   expert_countexpert_used_countlfm2conv_L_cache)r%   r&   r'   r(   r)   r*   r+   r,   r-   r$   zshortconv.l_cacheqwen3	qwen3_moe)r%   r&   r'   r(   r)   r*   attention.key_lengthr+   r,   r-   r$   r3   r4   falcon	tokenizerbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idphi3bloomn_layern_headlayer_norm_epsilon)r&   r(   r+   r$   attention.layer_norm_epsilont5n_positions
num_layersd_ffd_modeld_kv	num_headsrelative_attention_num_bucketsdecoder_start_token_id)r%   r&   r'   r(   r9   r+   r,   rI   z attention.relative_buckets_countrR   r$   stablelmlayer_norm_eps)	r%   r&   r'   r(   r)   r+   r,   rI   r$   gpt2n_ctxn_embdr'   )r&   r%   r(   r'   r+   rI   
starcoder2norm_epsilon)r&   r%   r(   r'   r+   r,   rI   mambaconv_kernel
state_sizetime_step_rank)	r$   r%   r(   r-   r&   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizenorm_epssliding_window)r%   r&   r'   r(   r)   r*   r9   r+   r,   r-   zattention.sliding_windowr$   )nemotrongemma2gemma3umt5decitokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr@   rA   rB   rC   zggml.add_space_prefixchat_template)rl   rk   r@   rA   rB   rC   )r;   tokenizer_configc                    t          |t                    s|g}t          |          dk    r|d         }d }n |d         dk    rt          d          |\  }}|dv rt	          | d                   } n|dv rt          | d                   } nz|dk    rt          | d                   } n^|dk    rBt          d	t          |                                                     	                                } n|dk    rt          | |          } | S )
N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   ro   r            
      )            B)
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_value	data_typearray_data_types      r/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/transformers/integrations/ggml.pyr   r   =  s   i&&  K	
9~~aL	Q<1oppp%."	?...VAY	g		vay!!	afQi	asDLL))1133::<<	a"6?;;M    c                       e Zd Zd ZdS )GGUFTokenizerSkeletonc                 \   |                                 D ]\  }}t          | ||           t          | d          sBt          | d          rt          | d          st          d          | j        }| j        fdt          |          D             t                              d           g }t                                                     D ]\  }}g }t          dt          |                    D ]6}	|d |	         ||	d          }}
|
|v r||v r|                    |
||f           7t          |fdd	
          }|                    |           t          |d d	
          }d |D             }|| _        nVd | j        D             | _        t          | d          s0d t          t          | j                            D             | _        t          | d          sg | _        t          | d          sd | _        t          | d          r| j        | j        | _        d S d S d S )Nri   rf   rg   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                 (    i | ]\  }}||         S  r   ).0itrg   s      r   
<dictcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>a  s#    @@@daQq	@@@r   z:Merges were not in checkpoint, building merges on the fly.ro   c                 <    | d                  | d                  fS )Nr   ro   r   )xvocabs    r   <lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>k  s    U1Q4[%!+4N r   T)keyreversec                     | d         S )Nr   r   )vals    r   r   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>m  s
    CF r   c                 .    g | ]}|d          |d         fS )r   ro   r   )r   r   s     r   
<listcomp>z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>n  s%    9993s1vs1v&999r   c                 R    g | ]$}t          |                    d                     %S ) )tuplesplit)r   merges     r   r   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>q  s,    LLLu5S!1!122LLLr   c                     g | ]}d S Nr   )r   _s     r   r   z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>s  s    EEEtEEEr   added_tokensr>   unknown_token_id)itemssetattrhasattrr~   rf   rg   	enumerateloggerwarningr   ranger}   appendsortedextendri   r   r>   r   )selfdict_kvrf   ri   r   piece_scorelocalindexpiece_lpiece_rrg   r   s               @@r   __init__zGGUFTokenizerSkeleton.__init__V  s   KKMM 	  	 DAqD!QtX&& 	F4** '$2I2I  r   [F[F@@@@i.?.?@@@ENNWXXXF&*5;;==&9&9 % %"{"1c%jj11 F FE',VeV}eEFFmWG&((W->->gw%DEEEu*N*N*N*NX\]]]e$$$$F(:(:DIIIF99&999F DKKLLLLLDK4** FEEU3t{3C3C-D-DEEEt^,, 	# "Dt^,, 	% $D 4+,, 	61B1J $ 5D	6 	61J1Jr   N)__name__
__module____qualname__r   r   r   r   r   r   U  s#        '6 '6 '6 '6 '6r   r   c                   2    e Zd Zd Zd Zd Zd Zd Zd ZdS )GGUFLlamaConverterc                     t          |          | _        | j        | _        i | _        t	          | j        dd          dk    | _        d S )Nre   r   )r   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dicts     r   r   zGGUFLlamaConverter.__init__  sF    *>::
"&*!#$+DJ8H'$R$RV]$]!!!r   c                 P    t          t          |j        |j                            S r   r|   ziprf   rg   r   r   s     r   r   zGGUFLlamaConverter.vocab      Cel33444r   c                     |j         S r   ri   r   s     r   ri   zGGUFLlamaConverter.merges  
    |r   c           	      (   |                      | j                  }|                     | j                  }d t          |          D             }|j        |j        |j                 nd }t          |dd           |j        |j                 nd }t          |dd           |j        |j                 nd }t          t          |||dd                    }g }	t          | j        d          sv|%|	                    t          |dd                     |%|	                    t          |dd                     |%|	                    t          |dd                     ntt          j        t          j        | j        j                  d	k              d
         }
|
D ]7}|	                    t          | j        j        |         dd                     8t#          |	          d
k    r|                    |	           t#          | j        j                  d
k    r)|                    d | j        j        D                        || j        d<   || j        d<   || j        d<   | j        r*d | j        d<   d| j        d<   d| j        d<   d| j        _        |S )Nc                      i | ]\  }\  }}||S r   r   r   r   word_scores       r   r   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>  #    NNN!2NT6T1NNNr   r<   r=   T)	unk_tokenfuse_unkbyte_fallbackrh   F
normalizedspecialrq   r   c                 2    g | ]}t          |d d           S )Fr   r   )r   added_tokens     r   r   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>  s'    uuukKE5IIIuuur   r   	eos_token	bos_tokenrj   clean_up_tokenization_spaceslegacy)r   r   ri   r   r>   rf   r   r<   r   r	   r   r   r   npwherer   rh   r}   add_special_tokensr   
add_tokensr   r   r   r   )r   r   vocab_scoresri   	bpe_vocabr   r   r   r;   special_tokensspecial_tokens_idxidxs               r   r;   zGGUFLlamaConverter.tokenizer  s   zz$*--TZ((NNi6M6MNNN	8=8J8VEL!344\`	8?~W[8\8\8hEL!344nr	8?~W[8\8\8hEL!344nr	#"  
 
	 tz<00 	j$%%juVZ&[&[&[\\\$%%juVZ&[&[&[\\\$%%juVZ&[&[&[\\\ "$"(4:3H*I*IQ*N!O!OPQ!R) j j%%j1B31GTYcg&h&h&hiiii~!##((888tz&''1,,  uu]a]g]tuuu   /8{+.7{+.7{+$ 	39=D"#56EID"#AB/4D"8,-2D#*r   c                    t          j                    t          j                    t          j        dd          g}| j        r|t          j        ddd          gz  }|r|t          j        dd          gz  }t          j        |          S )N   ▁r   FTrj   trim_offsets	use_regexro   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementrj   sequences       r   decoderzGGUFLlamaConverter.decoder  s    !##MOOUC((
 $ 	i+UQVbfggghhH 	>!<<<==H ***r   c                    |                      | j                  }|                     | j                  }|||_        d}d}t          | j        d          r| j        j        }|                     ||          }|||_        |                     ||          |_        |                                 }|r||_        | j	        r5t          j        ddd          |_        t          j        g           |_        |S )Nr   Trj   Fr   )r;   r   
normalizerr   r   rj   pre_tokenizerr   post_processorr   r   r   r   r   )r   r;   r   r   rj   r   r   s          r   	convertedzGGUFLlamaConverter.converted  s   NN4:..	 __TZ00
!#-I 4*,>?? 	H#6G**;8HII$&3I# LL6FGG	,,.. 	6'5I$ $ 	<&4&>!&Ud' ' 'I#
 $/#7#;#;I r   N)	r   r   r   r   r   ri   r;   r   r  r   r   r   r   r     sr        ^ ^ ^5 5 5  8 8 8t+ + +! ! ! ! !r   r   c                   *     e Zd Zd Zdef fdZ xZS )GGUFQwen2Converterc                 <    t          |          | _        i | _        d S r   r   r   r   r   s     r   r   zGGUFQwen2Converter.__init__       "7"G"G!#r   returnc           	      :   d t          | j        j                  D             }| j        j        }t	                                          ||          }|                    t          ddd          t          ddd          t          ddd          g           |S )Nc                     i | ]\  }}||	S r   r   r   r   r   s      r   r   z0GGUFQwen2Converter.converted.<locals>.<dictcomp>      RRRWQqRRRr   <|endoftext|>FTr   z<|im_start|>z
<|im_end|>)r   r   rf   ri   superr  r   r   r   r   ri   r;   	__class__s       r   r  zGGUFQwen2Converter.converted  s    RR	$2I2P(Q(QRRR(/GG%%eV44	$$?udKKK>eTJJJ<E4HHH	
 	
 	
 r   r   r   r   r   r   r  __classcell__r  s   @r   r  r    sS        $ $ $9          r   r  c                   8    e Zd Zd Zd Zd Zd Zd ZdefdZ	dS )	GGUFPhi3Converterc                 T    t          |          | _        | j        | _        i | _        d S r   r   r   r   r   r   s     r   r   zGGUFPhi3Converter.__init__  s)    *>::
"&*!#r   c                 P    t          t          |j        |j                            S r   r   r   s     r   r   zGGUFPhi3Converter.vocab  r   r   c                     |j         S r   r   r   s     r   ri   zGGUFPhi3Converter.merges  r   r   c                    |                      | j                  }|                     | j                  }d t          |          D             }t	          t          ||                    }|                    t          ddddd          t          ddd          t          dddd	          t          d
ddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          t          dddd	          g           |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |j        |j	        |j                 nd | j
        d<   |S )Nc                      i | ]\  }\  }}||S r   r   r   s       r   r   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>  r   r   </s>TF)rstriplstripr   r   r  r   z<|assistant|>)r  r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   	pad_token)r   r   ri   r   r   r	   r   r   r>   rf   r   r=   r<   r?   )r   r   r   ri   r   r;   s         r   r;   zGGUFPhi3Converter.tokenizer  s&   zz$*--TZ((NNi6M6MNNN	c)V4455	$$6$uX\]]]?udKKK?4ESWXXX-duVZ[[[-duVZ[[[-duVZ[[[-duVZ[[[<PTUUU9TeTRRR-duVZ[[[-duVZ[[[:dudSSS	
 	
 	
$ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ 160B0NEL+,,TX 	{+ r   c                     t          j                    t          j                    t          j        |d          g}|r|t          j        dd          gz  }t          j        |          S )Nr   ro   r   )r   r   r   r   r   r   r   s       r   r   zGGUFPhi3Converter.decoder@  se    !##MOO[#..
  	>!<<<==H ***r   r  c                     |                      | j                  }d}d}t          | j        d          r| j        j        }|                     ||          |_        |S )Nr   Trj   )r;   r   r   r   rj   r   )r   r;   r   rj   s       r   r  zGGUFPhi3Converter.convertedK  s`    NN4:..	4*,>?? 	H#6G LL6FGG	r   N)
r   r   r   r   r   ri   r;   r   r   r  r   r   r   r  r    s{        $ $ $
5 5 5  % % %N	+ 	+ 	+
9 
 
 
 
 
 
r   r  c                   *     e Zd Zd Zdef fdZ xZS )GGUFGPTConverterc                 <    t          |          | _        i | _        d S r   r  r   s     r   r   zGGUFGPTConverter.__init__Y  r  r   r  c                     d t          | j        j                  D             }| j        j        }t	                                          ||          }|S )Nc                     i | ]\  }}||	S r   r   r
  s      r   r   z.GGUFGPTConverter.converted.<locals>.<dictcomp>^  r  r   )r   r   rf   ri   r  r  r  s       r   r  zGGUFGPTConverter.converted]  sN    RR	$2I2P(Q(QRRR(/GG%%eV44	r   r  r  s   @r   r"  r"  X  sS        $ $ $9          r   r"  c                   2    e Zd Zd Zd Zd Zd ZdefdZdS )GGUFT5Converterc                     dg|d<   t          |          | _        d t          | j        j                  D             | _        | j        | _        i | _        d S )N
dummy textri   c                     i | ]\  }}||	S r   r   )r   r   r   s      r   r   z,GGUFT5Converter.__init__.<locals>.<dictcomp>j  s    GGG$!QAGGGr   )r   r   r   rf   token2idr   r   r   s     r   r   zGGUFT5Converter.__init__e  sW    $0>x *>::
GG)DJ4E*F*FGGG"&*!#r   c                 P    t          t          |j        |j                            S r   r   r   s     r   r   zGGUFT5Converter.vocabn  r   r   c                     t          | j        dd          r_g }t          | j        dd          r|t          j        d          gz  }|t          j        dd          gz  }t          j        |          S d S )Nr   Trj   r   )prependr   )patternr   )r   r   r   Prependr   r   )r   r   r   s      r   r   zGGUFT5Converter.normalizerq  s    4*Hd;; 	2Ht.0BDII A[0???@@,S%HHHIIH'111tr   c                 T    t          j        ddgg dd| j        d         fg          S )N$Ar  )r2  r  z$Br  )singlepairr   )r   TemplateProcessingr+  )r   s    r   r   zGGUFT5Converter.post_processorz  s@    ,&>---v./
 
 
 	
r   r  c                    |                      | j                  }t          t          || j        j        d                    }|                     | j                  }|||_        d}d}t          | j        d          r| j        j        }| 	                    ||          }|||_	        | 
                    ||          |_
        |                                 }|r||_        |S )NFunk_idr   r   Trj   )r   r   r   r
   r>   r   r   r   rj   r   r   r   )r   r   r;   r   r   rj   r   r   s           r   r  zGGUFT5Converter.converted  s    zz$*--z.#  
 
	 __TZ00
!#-I 4*,>?? 	H#6G**;8HII$&3I# LL6FGG	,,.. 	6'5I$r   N)	r   r   r   r   r   r   r   r   r  r   r   r   r'  r'  d  sk        $ $ $5 5 5  
 
 
9      r   r'  c                   2    e Zd Zd Zd Zd Zd ZdefdZdS )GGUFGemmaConverterc                 `    dg|d<   t          |          | _        | j        | _        i | _        d S )Nr)  ri   r  r   s     r   r   zGGUFGemmaConverter.__init__  s5    $0>x *>::
"&*!#r   c                 x   t          t          |j        |j                            }g }|D ]\  }}|dk    r|                    d|f           #d|v rOt          |                                          dk    r*dt          |          z  }|                    ||f           v|                    ||f           |S )Nz<0x09>	r   r   r   )r|   r   rf   rg   r   r}   strip)r   r   original_vocabupdated_vocabtokenscoreunderscoress          r   r   zGGUFGemmaConverter.vocab  s    c%,==>>* 	5 	5LE5  $$dE]3333#ekkmm"4"4"9"9#c%jj0$$k5%9::::$$eU^4444r   c                 ,    t          j        dd          S )Nr   r   )r   r   r   s     r   r   zGGUFGemmaConverter.normalizer  s    "3...r   c                     t          j        dd          t          j                    t          j                    g}|r|t          j        dd          gz  }t          j        |          S )Nr   r   ro   r   )r   r   r   r   r   r   r   s       r   r   zGGUFGemmaConverter.decoder  se    UC((!##MOO
  	>!<<<==H ***r   r  c                    |                      | j                  }t          t          || j        j        | j                            }|                     | j                  }|||_        d}d}t          | j        d          r| j        j	        }| 
                    ||          |_
        |                     ||          }|||_        |S )Nr7  r   Trj   )r   r   r   r
   r>   handle_byte_fallbackr   r   r   rj   r   r   )r   r   r;   r   r   rj   r   s          r   r  zGGUFGemmaConverter.converted  s    zz$*--z."7  
 
	 __TZ00
!#-I 4*,>?? 	H#6G LL6FGG	**;8HII$&3I#r   N)	r   r   r   r   r   r   r   r   r  r   r   r   r:  r:    sk        $ $ $  / / /	+ 	+ 	+9      r   r:  	qwen2_moer`   ra   gemma3_textrc   )rd   decilmr   r  c                 j    | }t          |         |          }|                                }||j        fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr  r   )r   r   tokenizer_class_name	converterfast_tokenizers        r   convert_gguf_tokenizerrP    s<     ('(<=nMMI((**N9666r   )+__doc__r   numpyr   
tokenizersr   r   r   r   r   tokenizers.modelsr	   r
    r   convert_slow_tokenizerr   r   r   r   r   utilsr   utils.loggingr   
get_loggerr   r   GGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGr   r   r   r  r  r"  r'  r:  rL  strr   dictrP  r   r   r   <module>r^     s    
           S S S S S S S S S S S S S S * * * * * * * *       o o o o o o o o o o o o o o                   
	H	%	%$% 
 3*2) *& 5#8,:" $ 3*2) *& 5#8,:" %> 3*2) $& 5#8,:" ?V 3*2) $& 5#8,:"%2 Wr 3*2) $& 5#8,:"+ sL 3*2) $& 5#8,:" Md 3*2) $& * 5#8,:"%2 eB 3*2) $& 5#8,:" CZ ++!/!/	 [f 3*2) $& 5#8,:" g~  ) ("(< L 	'#%% & +#8(<,L":" Mf 3*2) $ 5#8(8"
 
g|  !$4 ((< }L *3)2 5#8(6 M^ "3),@*(&.-
 
_v 4*2) $& 5#8,6"  4*2) $& !+ 5#8,:$4" " 4*2) $& !+ 5#8,:$4" " (#%% & +#8(<,L":"  4*2) $& 5#8,:" g   F ''++!/!/!3  )"++!/!/   0  0(6 (6 (6 (6 (6 (6 (6 (6Vv v v v v v v vr       (H H H H H H H HV	 	 	 	 	} 	 	 	< < < < <k < < <~> > > > > > > >B # 	
 #       " 	/       %!" O#$  '   .7 7yRVAW 7 7 7 7 7 7r   