
    Pi8             
      &   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlZd dl Z!d dl"Z"erd d	l"m#Z# d
ej$        vr5e	j%        &                    d e' ee(          j)        dz                       d dl*Z*d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2 d dl3m4Z4 d dl5m6Z6  ej7        d          Z8 G d de          Z9 G d de          Z: edd          Z; G d d          Z< G d de<          Z= G d de<          Z>e<?                    d            G d! d"e=                      Z@e<?                    d#d$           G d% d$e=                      ZAe<?                    d&           G d' d(e=                      ZBe<?                    d)           G d* d+e=                      ZCe<?                    d,d-           G d. d/e=                      ZDe<?                    d0           G d1 d2e=                      ZEe<?                    d3d4           G d5 d6e=                      ZFe<?                    d7           G d8 d9e=                      ZGe<?                    d:           G d; d<e=                      ZHe<?                    d=d>d?           G d@ dAe=                      ZIe<?                    dBdCdDdEdFdGdHdI           G dJ dIe=                      ZJe<?                    dK           G dL dMeJ                      ZKe<?                    dGdN           G dO dPe>                      ZLe<?                    dQdR           G dS dTe>                      ZMe<?                    dU           G dV dWeJ                      ZNe<?                    dU           G dX dYe>                      ZOe<?                    dN           G dZ d[eJ                      ZPe<?                    d\           G d] d^e=                      ZQe<?                    d_           G d` dae=                      ZRe<?                    db           G dc dde=                      ZSe<?                    de           G df dge=                      ZTe<?                    dh           G di dje=                      ZUe<?                    dk           G dl dme=                      ZVe<?                    dn           G do dpe=                      ZWe<?                    dqdrds           G dt dqe=                      ZXe<?                    du           G dv due=                      ZYe<?                    dw           G dx dye=                      ZZe<?                    dz           G d{ d|e=                      Z[e<?                    d}           G d~ de[                      Z\e<?                    dddd           G d de=                      Z]e<?                    ddd           G d de>                      Z^e<?                    d           G d de^                      Z_e<?                    d           G d de>                      Z`e<?                    d           G d de=                      Zae<?                    d           G d de=                      Zbe<?                    d           G d deX                      Zce<?                    d           G d deb                      Zde<?                    d           G d de=                      Zee<?                    d           G d de=                      Zfe<?                    d           G d de=                      Zge<?                    d           G d deg                      Zhe<?                    d           G d de=                      Zie<?                    dd           G d de=                      Zje<?                    d           G d de=                      Zke<?                    d           G d de=                      Zle<?                    d           G d de=                      Zme<?                    dddd           G d de=                      Zne<?                    ddd           G d den                      Zoe<?                    dd           G d den                      Zpe<?                    d           G d den                      Zqe<?                    dddæ           G dĄ den                      Zre<?                    ddǦ           G dȄ den                      Zse<?                    dɦ           G dʄ de=                      Zte<?                    d̦           G d̈́ de=                      Zue<?                    ddЦ           G dф de=                      Zve<?                    dЦ           G dӄ de>                      Zwe<?                    dզ           G dք dev                      Zxe<?                    dئ           G dل de=                      Zye<?                    dۦ           G d܄ de=                      Zze<?                    dަ           G d߄ dez                      Z{e<?                    dd           G d de=                      Z|e<?                    d           G d de|                      Z}e<?                    ddd           G d de=                      Z~e<?                    d           G d de=                      Ze<?                    d           G d de=                      Ze<?                    d           G d de=                      Ze<?                    d           G d de=                      Ze<?                    d          e<?                    d           G d de=                                  Ze<?                    d           G d de=                      Ze<?                    d            G d de=                      Ze<?                    dd           G d den                      Ze<?                    d           G d d	e=                      Ze<?                    d
           G d de=                      Ze<?                    d           G d de=                      Ze<?                    d          e<?                    d          e<?                    d           G d de=                                              Ze<?                    d           G d deb                      Ze<?                    d           G d de=                      Ze<?                    d          e<?                    d          e<?                    d          e<?                    d           G d d e=                                                          Ze<?                    d!           G d" d!e=                      Ze<?                    d#           G d$ d%e=                      Ze<?                    d&d'           G d( d)e=                      Ze<?                    d*           G d+ d,e=                      Ze<?                    d-d.d/           G d0 d.e=                      Ze<?                    d1           G d2 d3e=                      Ze<?                    d4           G d5 d6e=                      Ze<?                    d7           G d8 d9e=                      Ze<?                    d:           G d; d<eJ                      Ze<?                    d=d>           G d? d@e                      Ze<?                    dAdB           G dC dDee                      Ze<?                    dE           G dF dGe=                      Ze<?                    dH          e<?                    dI           G dJ dKe=                                  Ze<?                    dL           G dM dLe=                      Ze<?                    ds           G dN dOe>                      Ze<?                    dL           G dP dQe                      Ze<?                    dH           G dR dSe                      Ze<?                    dT           G dU dVe                      Ze<?                    dW           G dX dYe=                      Ze<?                    dZ           G d[ d\e=                      Ze<?                    d]           G d^ d_eJ                      Ze<?                    d`           G da dbe=                      Ze<?                    dc          e<?                    dd           G de dfe=                                  Ze<?                    dg           G dh die=                      Z G dj dkeJ          Z G dl dmeL          Z G dn doe*j                  Zd~drZddvZddzZdd|Zed}k    r e             dS dS (      )annotationsN)IntEnum)Path)sha256)
TYPE_CHECKINGAnyCallableContextManagerIterableIteratorLiteralSequenceTypeVarcast)chain)
AutoConfig)TensorNO_LOCAL_GGUF   zgguf-py)MistralTokenizerTypeMistralVocab)TokenizerVersion)DATASET_MEANDATASET_STD)
Tekkenizer)SentencePieceTokenizerz
hf-to-ggufc                  &    e Zd ZdZdZdZdZdZdZdS )SentencePieceTokenTypesr                  N)	__name__
__module____qualname__NORMALUNKNOWNCONTROLUSER_DEFINEDUNUSEDBYTE     j/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/bin/convert_hf_to_gguf.pyr   r   -   s,        FGGLFDDDr.   r   c                      e Zd ZdZdZdS )	ModelTyper   r   N)r$   r%   r&   TEXTMMPROJr-   r.   r/   r1   r1   6   s        DFFFr.   r1   AnyModeltype[ModelBase])boundc                  2   e Zd ZU ej        i ej        i iZded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded <   d!ed"<   d#Zded$<   d#d#d#d%d%d&d&d#d#d%d%d'dad.Z	e
dbd3            Zdcddd8Zded:ZdfdgdAZdhdidCZdjdkdGZdH ZdldLZdmdPZdndQZdR ZdS ZdodUZdV ZdW ZedpdX            ZedqdY            Ze
drd\            Ze
d]             Ze
ej        fdsd`            Zd%S )t	ModelBasez+dict[ModelType, dict[str, type[ModelBase]]]_model_classesr   	dir_modelgguf.LlamaFileTypeftype	fname_outboolis_big_endianzgguf.GGUFEndian	endianessuse_temp_filelazy	list[str]
part_namesis_safetensorsdict[str, Any]hparamszset[str] | Nonetensor_nameszgguf.GGUFWritergguf_writer
str | None
model_namePath | Nonemetadata_overridedir_model_cardremote_hf_model_idzgguf.MODEL_ARCH
model_archintblock_countzgguf.TensorNameMap
tensor_mapFis_mistral_formatNr   )r?   rA   eagerrM   rK   split_max_tensorssplit_max_sizedry_runsmall_first_shardrG   rO   rU   rV   rW   rX   rY   dict[str, Any] | Nonec          
     Z    t                     t          u s,t                     t          u st                     t          u r$t	          t                     j        d          | _        | _        | _        | _	        |rt          j        j        nt          j        j         _        | _        | pd u _         _        d _        d fd}| _        n{ j        sdnd}t                               j        |d           _        t/           j                  d	k     _         j        s&t                               j        d
d           _        |%t                               j         j                  n| _        d  _        | _        | _        | _         j        t          j        j        k    rtA                                                     \  }}|j!        tD          j#        k    r:tH          %                    d|j!         d           t          j        j&         _        n9tH          %                    d|j!         d           t          j        j'         _        t          j(        d t          j)         j*                  j         j        |	|
||           _+        d S )Nz$ should not be directly instantiatedTreturnIterator[tuple[str, Tensor]]c               3  ^  K   t                               d            t          j        j                                      } t          d |                                 D                       _        | 	                                D ]#\  }}|t                              |          fV  $d S )Nz(Using remote model with HuggingFace id: c              3     K   | ]}|V  d S Nr-   ).0names     r/   	<genexpr>zAModelBase.__init__.<locals>.get_remote_tensors.<locals>.<genexpr>v   s"      'O'O'O'O'O'O'O'Or.   )loggerinfoggufutilitySafetensorRemoteget_list_tensors_hf_modelsetkeysrH   itemsLazyTorchTensorfrom_remote_tensor)remote_tensorsrb   remote_tensorrO   selfs      r/   get_remote_tensorsz.ModelBase.__init__.<locals>.get_remote_tensorss   s      [GY[[\\\!%!>!X!XYk!l!l$''O'O9L9L9N9N'O'O'O$O$O!+9+?+?+A+A T T'D-!C!CM!R!RSSSSST Tr.   modelconsolidatedz.safetensorsr   pytorch_modelz.binz/choosing --outtype f16 from first tensor type ()z0choosing --outtype bf16 from first tensor type ()patharchr@   rA   rV   rW   rX   rY   r\   r]   ),typer8   	TextModelMmprojModel	TypeErrorr$   r:   r<   r=   r?   rf   
GGUFEndianBIGLITTLEr@   rA   rB   rO   rE   get_tensorsrT   get_model_part_namesrD   lenload_hparamsrG   rH   rM   rK   rN   LlamaFileTypeGUESSEDnextdtypetorchfloat16rd   re   
MOSTLY_F16MOSTLY_BF16
GGUFWriterMODEL_ARCH_NAMESrP   rI   )rq   r:   r<   r=   r?   rA   rU   rM   rK   rV   rW   rX   rY   rG   rO   rr   prefix_first_tensors   `             `    r/   __init__zModelBase.__init__^   s   
 ::""T

i''T

k))tDzz2ZZZ[[["
"*0=Y,,4?CY*IA"4D"@	"4)"&DT T T T T T T  2D$($:NWWF'<<T^VUcddDO"%do"6"6":D& j"+"@"@Q`bh"i"iY`Yhy--dnd>TUUUnu !2$' :+333"4#3#3#5#566OA|!U]22clN`cccddd!/:

d|Oadddeee!/;
  ?4;PQUQ`;amqm{  LP  L^=N_mw~  Sde e er.   rw   r   strr\   c                X    |j         |j        }}| | | }|                    |          S r`   )stemsuffix	with_name)clsrw   r   r   r   new_names         r/   add_prefix_to_filenamez ModelBase.add_prefix_to_filename   s6    y$+f,d,F,,~~h'''r.   rk   Iterable[str]optionalr   c                     t           fd|D             d           }| j        |         S |rd S t          d|           )Nc              3  .   K   | ]}|j         v |V  d S r`   rG   )ra   krq   s     r/   rc   z(ModelBase.find_hparam.<locals>.<genexpr>   s/      99!qDL'8'8A'8'8'8'899r.   could not find any of: )r   rG   KeyError)rq   rk   r   keys   `   r/   find_hparamzModelBase.find_hparam   s]    9999t9994@@?<$$ 	4777888r.   r]   c           	   #  ^  K   t                      }| j        s| j        rdnd}|dz  }| j        |z  }|                                rt                      | _        t                              d| d           t          |dd          5 }t          j
        |          }|                    d	          t          t                    st          d
|          | j                                                                       d d d            n# 1 swxY w Y   n|| _        i n	|| _        i | j        D ]~}t                              d| d           | j        r;ddlm} t)          t*          t,                    || j        |z  dd                    }n?t/          j        t3          j
        t5          | j        |z            ddd                    }|5 }	|                    |	                                           |	                                D ]}
| j        rM| j        r0|	                    |
          }t:                              |          }n?|	                    |
          }n)|	|
         }| j        rt:                               |          }|
|fV  	 d d d            n# 1 swxY w Y   tC          |"                    | j                            dk    rtG          | j        $                    |                    }tG          |$                    | j                            }tG          t          fd|D                                 }tC          |          dk    r(tC          |          dk    rt          d| d|           t          d| d|           d S )Nzmodel.safetensorszpytorch_model.binz.index.jsonz%gguf: loading model weight map from ''rutf-8encoding
weight_mapzCan't load 'weight_map' from zgguf: loading model part 'r   )	safe_openptcpu)	frameworkdeviceT)map_locationmmapweights_onlyc              3  0   K   | ]}|v |         V  d S r`   r-   )ra   nr   s     r/   rc   z(ModelBase.get_tensors.<locals>.<genexpr>   s+      &Y&Yjz!}&Y&Yr.   z#Missing or incomplete model files: z
Missing tensors: zOMismatch between weight map and model parts for tensor names:
Missing tensors: z
Extra tensors: )%rj   rT   rE   r:   is_filerH   rd   re   openjsonloadget
isinstancedict
ValueErrorupdaterk   rD   safetensorsr   r   r
   r   
contextlibnullcontextr   r   rB   	get_slicerm   from_safetensors_slice
get_tensor
from_eagerr   symmetric_differencesorted
difference)rq   tensor_names_from_parts
index_name
index_filefindex	part_namer   ctx
model_partrb   datamissingextramissing_filesr   s                  @r/   r   zModelBase.get_tensors   s     ,/EE% 	040C\,,I\J-'J*4J!!##  $'EE!QJQQQRRR*cG<<< @,0IaLLE!&<!8!8J!)J1M1M)()W)W)WXXX%,,Z__->->???@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ %<!

 7DJ 	% 	%IKKAYAAABBB" L111111>#.		$.9:T`dmr0s0s0stt ,UZDNY<V8W8Wfkrv  FJ  .K  .K  .K  L  L %
'..z/@/@AAA&OO-- % %D* 	D9 ?#-#7#7#=#=D#2#I#I$#O#ODD#-#8#8#>#>DD)$/9 D#2#=#=d#C#CD*$$$$%% % % % % % % % % % % % % % %" &;;D<MNNOORSSST.99:QRRSSG2==d>OPPQQE"3&Y&Y&Y&Yg&Y&Y&Y#Y#YZZM5zzQ3}#5#5#9#9  "?} "? "?5<"? "? @ @ @ ! ";5<"; ";38"; "; < < < TSs&   A?DDD CJ//J3	6J3	.weightr   gguf.MODEL_TENSORbid
int | Noner   c                    |t           j        | j                 vrt          d|d| j                  t           j        |         }d|v r|J |                    |          }||z   S )NzMissing z for MODEL_TENSORS of {bid}r   )rf   MODEL_TENSORSrP   r   TENSOR_NAMESformat)rq   r   r   r   rb   s        r/   format_tensor_namezModelBase.format_tensor_name   st    d(999XXXT_XXYYY%c*d?????;;3;''Df}r.   rb   c                    |t           j        | j                 vrdS t           j        |         }d|v r|dS |                    |          }n|dS |||z   k    S )NFr   r   )rf   r   rP   r   r   )rq   rb   r   r   r   key_names         r/   match_model_tensor_namez!ModelBase.match_model_tensor_name   sm    d(9995)#.h{u3//HHu6)**r.   r   .biastry_suffixesSequence[str]c                f    | j                             ||          }|t          d|          |S )N)r   r   zCan not map tensor )rS   get_namer   )rq   rb   r   r   s       r/   map_tensor_namezModelBase.map_tensor_name   s>    ?++<+PP;4;;<<<r.   c                     t          d          )Nz7set_gguf_parameters() must be implemented in subclassesNotImplementedErrorrq   s    r/   set_gguf_parameterszModelBase.set_gguf_parameters   s    !"[\\\r.   
data_torchr   Iterable[tuple[str, Tensor]]c                4    ~|                      |          |fgS r`   r   rq   r   rb   r   s       r/   modify_tensorszModelBase.modify_tensors  s!    %%d++Z899r.   r   n_dims gguf.GGMLQuantizationType | boolc                    ~~~~dS NFr-   rq   rb   r   r   r   s        r/   tensor_force_quantzModelBase.tensor_force_quant  s    (Cur.   c                    dS )Nr-   r-   r   s    r/   generate_extra_tensorsz ModelBase.generate_extra_tensors  s    rr.   c                    t          d  j        j                                        D                       t	          d          z   }t                                                                                      D ]\  }}|                    d          r|j	        }|j	        t          j        t          j        fvr|                    t          j                  }d |                    d          D ]'}|                                rt!          |           n(                     ||          D ]-\  }|                                }t	          |j                  dk    r|                                }t	          |j                  }                     ||          }|dk    s                    d          rt*          j        j        }|du rSt1           fd	t*          j        j        t*          j        j        t*          j        j        t*          j        j        t*          j        j        t*          j        j        t*          j        j         t*          j        j!        t*          j        j"        t*          j        j#        t*          j        j$        t*          j        j%        t*          j        j&        t*          j        j'        t*          j        j(        t*          j        j)        t*          j        j*        fD                       s                    d
          st*          j        j        }|du rt1           fdt*          j        j+        t*          j        j,        t*          j        j-        t*          j        j.        t*          j        j/        t*          j        j0        fD                       r: j1        t*          j2        j3        t*          j2        j4        fv rt*          j        j5        }tm          |tn                    r$ j1        t*          j2        j8        k    rt*          j        j        }n j1        t*          j2        j9        k    rt*          j        j5        }n̉ j1        t*          j2        j:        k    rt*          j        j;        }n j1        t*          j2        j<        k    rt*          j        j=        }nt j1        t*          j2        j3        k    rt*          j        j>        }nH j1        t*          j2        j4        k    rt*          j        j?        }nt          d j1        jA                   	 t*          jB        C                    ||          }ni# t*          jD        $ rW}	t          F                    d|	d           t*          j        j5        }t*          jB        C                    ||          }Y d }	~	nd }	~	ww xY w|j	        t          jH        k    rt+          jI        |j        |          n|j        }
ddJ                    d t          |
          D                        d}t          L                    d| d dz   d| d|jA         d|             jM        N                    ||           /d S )Nc              3  :   K   | ]\  }}t          |          V  d S r`   )r   )ra   r   ss      r/   rc   z,ModelBase.prepare_tensors.<locals>.<genexpr>  s,      OOda3q66OOOOOOr.   z.weight,)z.attention.masked_biasz.attention.biasz.rotary_emb.inv_freq.r   r   z_norm.weightFc              3  F   K   | ]}                     |          V  d S r`   r   ra   r   r   r   rq   s     r/   rc   z,ModelBase.prepare_tensors.<locals>.<genexpr>7  sI         44XsCHH     r.   r   c              3  F   K   | ]}                     |          V  d S r`   r  r  s     r/   rc   z,ModelBase.prepare_tensors.<locals>.<genexpr>Q  sI       
/ 
/ 003DD
/ 
/ 
/ 
/ 
/ 
/r.   zUnknown file type: z%s, %szfalling back to F16{, c              3  4   K   | ]}t          |          V  d S r`   )r   )ra   r   s     r/   rc   z,ModelBase.prepare_tensors.<locals>.<genexpr>~  s(      *K*Ka3q66*K*K*K*K*K*Kr.   }z%-r   , z --> z
, shape = 	raw_dtype)OmaxrS   mappingvaluesr   r   r   r   endswithr   r   r   float32tosplit	isdecimalrQ   r   numpyshaper   rf   GGMLQuantizationTypeF32anyMODEL_TENSORFFN_GATE_INPPOS_EMBDTOKEN_TYPES
SSM_CONV1DSHORTCONV_CONVTIME_MIX_FIRSTTIME_MIX_W1TIME_MIX_W2TIME_MIX_DECAY_W1TIME_MIX_DECAY_W2TIME_MIX_LERP_FUSEDPOSNET_NORM1POSNET_NORM2V_ENC_EMBD_POSA_ENC_EMBD_POSALTUP_CORRECT_COEFALTUP_PREDICT_COEF
TOKEN_EMBDPER_LAYER_TOKEN_EMBDOUTPUTALTUP_ROUTERLAUREL_LLAUREL_Rr<   r   MOSTLY_TQ1_0MOSTLY_TQ2_0F16r   r>   ALL_F32r   r   BF16MOSTLY_Q8_0Q8_0TQ1_0TQ2_0r   rb   quantsquantize
QuantErrorrd   warningnpuint8quant_shape_from_byte_shapejoinreversedre   rI   
add_tensor)rq   max_name_lenrb   r   	old_dtypepartr   r   
data_qtypeer  	shape_strr   r   s   `           @@r/   prepare_tensorszModelBase.prepare_tensors  s   OOdo.E.L.L.N.NOOOOORUV`RaRaa %d&A&A&C&CTEUEUEWEW X X q	R q	RD*}}bcc "(I u}'EEE']]5=99
 C

3  >>## d))CE *.)<)<Zs)S)S _R _R$* "'')) tz??a''%++--DTZ?C?V?VW[]egjlr?s?s
 Q;;("3"3N"C"C;!%!:!>J &&       !-: -6 -9 -8 -< -< -9 -9 -? -? -A -: -: -< -< -@ -@#$     '. $,,Y77/ '2 "&!:!>J&&3 
/ 
/ 
/ 
/ 
/ 
/ )4)>)0)6)2)2 
/ 
/ 
/ 
, 
,& z*7*7&  
 &*%>%B
 j$// RzT%7%???%)%>%B

t'9'DDD%)%>%B

t'9'EEE%)%>%C

t'9'EEE%)%>%C

t'9'FFF%)%>%D

t'9'FFF%)%>%D

()Ptz)P)PQQQB;//jAADD B B BNN8Q0EFFF!%!:!>J;//jAADDDDDDB
 UYT^bdbjTjTj8ZPPPptpz P*K*K8E??*K*K*K!K!KOOO	 3L333nnnD}}y}}WaWf}}r{}}~~~ ++Hdj+QQQQ_R%q	R q	Rs     UV'AV""V'c                X    | j                             t          j        j                   d S r`   )rI   add_typerf   GGUFTypeMODELr   s    r/   set_typezModelBase.set_type  s$    !!$-"566666r.   
vocab_onlyc                   | j                                         \  }}}}t          j                            | j        | j        | j        |          | _        | j	        r| j	        | j        _
        | j        j
        | j        j
        | j        _
        | j        j        '|dk    r!t          j        ||||          | j        _        |                                  t                              d           | j                            | j                    t                              d           |                                  t                              d           | j                             t          j                   d S )Nr   zSet meta modelzSet model parameterszSet model quantization version)rI   get_total_parameter_countrf   Metadatar   rM   rN   rK   metadatarO   rb   r:   
size_labelrP  rd   re   set_gguf_meta_modelr   add_quantization_versionGGML_QUANT_VERSION)rq   rQ  total_paramsshared_paramsexpert_paramsexpert_counts         r/   prepare_metadatazModelBase.prepare_metadata  sA   CGCSCmCmCoCo@m]L**4+A4CVX\Xgiuvv " 	9!%!8DM =%!%!4DM =#+q0@0@'+|]Taco'p'pDM$$%%%))$*:;;;*+++  """455511$2IJJJJJr.   c                     t          d          )Nz/write_vocab() must be implemented in subclassesr   r   s    r/   write_vocabzModelBase.write_vocab  s    !"STTTr.   c                4   |                                   |                     d           | j                            | j                   | j                                         | j                            d           | j                                         d S )NFrQ  rw   T)progress)rK  r^  rI   write_header_to_filer=   write_kv_data_to_filewrite_tensors_to_filecloser   s    r/   writezModelBase.write  s    ///--4>-BBB..000...===     r.   c                    g }t          j        |           D ]A}|                    |          r*|                    |          r|                    |           B|                                 |S r`   )oslistdir
startswithr  appendsort)r:   r   r   rD   filenames        r/   r   zModelBase.get_model_part_names  sv     "

9-- 	, 	,H""6** ,x/@/@/H/H ,!!(+++r.   c                P   |rCt          | dz  dd          5 }t          j        |          }d d d            n# 1 swxY w Y   |S 	 t          j        | d                                          }n# t          $ r}t                              d|  d|            t                              d	           t          | d
z  dd          5 }t          j        |          }d d d            n# 1 swxY w Y   Y d }~nd }~ww xY wd|v r|d         |d<   d|v r|d         d         |d<   |S )Nzparams.jsonr   r   r   Ftrust_remote_codez!Failed to load model config from : z"Trying to load config.json insteadconfig.json
llm_configtext_configthinker_config)	r   r   r   r   from_pretrainedto_dict	Exceptionrd   r>  )r:   rT   r   configrI  s        r/   r   zModelBase.load_hparams  s    	i-/wGGG &11& & & & & & & & & & & & & & &M	&  /	USSS[[]]FF 	& 	& 	&NNOyOOAOOPPPNN?@@@i-/wGGG &11& & & & & & & & & & & & & & &	&
 6!!$*<$8F=!v%%$*+;$<]$KF=!sJ   9= =(A0 0
C?:AC:	C*C:*C.	.C:1C.	2C::C?namesCallable[[AnyModel], AnyModel]c                      sJ d fd}|S )Nmodelclsr4   r\   c                    | j         t          j        j        k    rt          j        nt          j        }D ]}| j        |         |<   | S r`   )rP   rf   
MODEL_ARCHr3   r1   r2   r9   )r  
model_typerb   r   r}  s      r/   funcz ModelBase.register.<locals>.func  sQ    -5-@DODZ-Z-Z))`i`nJ @ @7?":.t44Or.   )r  r4   r\   r4   r-   )r   r}  r  s   `` r/   registerzModelBase.register  s7    u	 	 	 	 	 	 	
 r.   c                
   | j                                         D ]h\  }}t                              |j         d           t          |                                          D ]}t                              d|             id S )Nz models:z  - )r9   rl   rd   errorrb   r   rk   )r   r  model_classesrb   s       r/   print_registered_modelsz!ModelBase.print_registered_models  s    ),);)A)A)C)C 	, 	,%JLLJO555666}113344 , ,]D]]++++,	, 	,r.   rx   r5   c                n    	 | j         |         |         S # t          $ r t          d|d          d w xY w)NzArchitecture z not supported!)r9   r   r   )r   rx   r  s      r/   from_model_architecturez!ModelBase.from_model_architecture  sW    	Y%j1$77 	Y 	Y 	Y%&Md&M&M&MNNTXX	Ys    4)r:   r   r<   r;   r=   r   r?   r>   rA   r>   rU   r>   rM   rL   rK   rJ   rV   rQ   rW   rQ   rX   r>   rY   r>   rG   rZ   rO   rJ   )rw   r   r   r   r\   r   Frk   r   r   r>   r\   r   ry   )Nr   )r   r   r   r   r   r   r\   r   r   )
rb   r   r   r   r   r   r   r   r\   r>   r   rb   r   r   r   r\   r   r   r   rb   r   r   r   r\   r   
rb   r   r   r   r   r   r   rQ   r\   r   r\   r   rQ  r>   )r:   r   r   r   r   r   r\   rC   )r:   r   rT   r>   )r}  r   r\   r~  )rx   r   r\   r5   ) r$   r%   r&   r1   r2   r3   r9   __annotations__rT   r   classmethodr   r   r   r   r   r   r   r   r   r   rK  rP  r^  r`  ri  staticmethodr   r   r  r  r  r-   r.   r/   r8   r8   >   si        "CN    
 OOOOOOJJJ!!!!    """"""""   """"#####mr',E26QU*+1V[+0SWz~	6e 6e 6e 6e 6e 6ep ( ( ( [(
9 9 9 9 9<< << << <<|    + + + + +    ] ] ]: : : :
      tR tR tRl7 7 7K K K K:U U U! ! !    \    \.    [ , , [, =F^ Y Y Y Y [Y Y Yr.   r8   c                       e Zd ZU ej        Zded<    fdZed             Z	d Z
d! fdZd	 Zd
 Zd"dZd#dZd$dZd%dZd%dZd Zd&dZd Zd Zd Zd'dZd%d Z xZS )(r{   r   hf_archc                ^    t                      j        |i | | j        s t          | j        | j                  | _        nd| _        d| j        v ri | j        | j        d         | _        |                     g d          | _        t          j
        | j        | j                  | _        d S )N rw  )n_layersnum_hidden_layersn_layer
num_layers)superr   rT   get_model_architecturerG   r  r  r   rR   rf   get_tensor_name_maprP   rS   rq   argskwargs	__class__s      r/   r   zTextModel.__init__  s    $)&)))% 	1$,PPDLLDLDL((JdlJdl=.IJDL++,f,f,fgg24?DDTUUr.   c                F    d| j         vrt          d| j                  d S )NrP   z"Missing property 'model_arch' for )__dict__r}   r$   )r   s    r/   __init_subclass__zTextModel.__init_subclass__  s2     s|++QQQRRR ,+r.   c                .    |                                   d S r`   _set_vocab_gpt2r   s    r/   	set_vocabzTextModel.set_vocab      r.   rQ  r>   c           
         t                                          |           | j                                        d         }| j        j                            d          d         }| j                                        r|sVt          j
        | j        j        | j        j        | j        j        | j        j        | j        j        ||dk     rdnd           }nCt          j
        | j        j        | j        j        | j        j        | j        j        d d d          }| j        | d	z  | _        n1| j        j        t          j        | j        j        |          z  | _        t$                              d
           |                                  d S )Nrb  r   r   r   LoRAr  vocab)rV  output_typer  .ggufzSet model tokenizer)r  r^  rI   rS  r<   rb   	partitionr=   is_dirrf   naming_conventionrU  basenamefinetuneversionrV  parentfill_templated_filenamerd   re   r  )rq   rQ  rZ  r  fname_defaultr  s        r/   r^  zTextModel.prepare_metadata	  s     J 777'AACCAF:?44S99!< >  "" 	t N%)%;DM<NPTP]Pfhlhuh~  AE  AN  AV  X\  Xe  Xp  r}  T`  cd  Td  Td  JP  JP  jn  &o  &o  &o%)%;DM<NPTP]Pfhlhuh~  AE  AN  AV  cg  uy  FM  &N  &N  &N "^.E.E.EEDNN "^2T5QRVR`Regr5s5ssDN)***r.   c                   | j                             | j                   |                     g dd          x}7| j                             |           t
                              d|            |                     g dd          x}7| j                             |           t
                              d|            |                     g dd          x}7| j                             |           t
                              d|            |                     g d	d          x}7| j         	                    |           t
                              d
|            |                     ddgd          x}7| j         
                    |           t
                              d|            | j                            d          x}7| j                             |           t
                              d|            |                     ddgd          x}7| j                             |           t
                              d|            |                     g dd          x}7| j                             |           t
                              d|            | j                            d          x}	7| j                             |	           t
                              d|	            | j                            d          x}
7| j                             |
           t
                              d|
            | j                            d          x}4| j                             |           | j                             |           | j                             | j                   t
                              d| j                    d S )N)max_position_embeddingsn_ctxn_positions
max_lengthTr   zgguf: context length = )hidden_sizen_embddimzgguf: embedding length = )intermediate_sizen_inner
hidden_dimzgguf: feed forward length = )num_attention_headsn_headn_headszgguf: head count = num_key_value_heads
n_kv_headszgguf: key-value head count = 
rope_thetazgguf: rope theta = rms_norm_epsnorm_epsgguf: rms norm epsilon = )layer_norm_epslayer_norm_epsilonnorm_epsilonzgguf: layer norm epsilon = num_local_expertszgguf: expert count = num_experts_per_tokzgguf: experts used count = head_dimgguf: file type = )rI   add_block_countrR   r   add_context_lengthrd   re   add_embedding_lengthadd_feed_forward_lengthadd_head_countadd_head_count_kvrG   r   add_rope_freq_baseadd_layer_norm_rms_epsadd_layer_norm_epsadd_expert_countadd_expert_used_countadd_key_lengthadd_value_lengthadd_file_typer<   )rq   r  r  n_ffr  	n_head_kvr  	f_rms_eps
f_norm_eps	n_expertsn_experts_usedr  s               r/   r   zTextModel.set_gguf_parameters%  s   (()9:::%%&g&g&grv%wwwE  E//666KK9%99:::&&'G'G'GRV&WWWFd11&999KK<F<<===$$%S%S%S^b$cccDp44T:::KK=t==>>>&&'S'S'S^b&cccFp++F333KK6f66777))+@,*OZ^)___Il..y999KKC	CCDDD,**<888JE//
;;;KK:j::;;;))>:*FQU)VVVIc33I>>>KK?I??@@@**+c+c+cnr*sssJ  A//
;;;KKBjBBCCC))*=>>>IK--i888KK;	;;<<<"l../DEEENR22>BBBKKFnFFGGG((444HA++H555--h777&&tz22255566666r.   c                .   t          | j        j                  dk    rt          d          |                     d           | j                            | j                   | j                                         | j                                         d S )Nr   z)Splitting the vocabulary is not supportedTrb  rc  )	r   rI   tensorsr   r^  re  r=   rf  rh  r   s    r/   r`  zTextModel.write_vocabS  s    t'((A--HIII...--4>-BBB..000     r.   tokenstr | bytesr\   c                   t          |t          t          f          r|                    d          }n@t          |t                    r)|                                                    d          }n|}|dv }|p)|                    d          o|                    d          }|p)|                    d          o|                    d          }|p)|                    d          o|                    d	          }|S )
Nr   r   )z<pad>z<mask>z<2mass>z[@BOS@]z<||>u   <｜u   ｜>z<unused>)r   bytes	bytearraydecode
memoryviewtobytesrm  r  )rq   r  
token_textseems_specials       r/   does_token_look_specialz!TextModel.does_token_look_special\  s   eeY/00 	w77JJz** 	///AAJJJ # '
 

 &d**?*?*E*E*c*J]J]^bJcJc%h**?*?*G*G*gJL_L_`fLgLg &h**?*?	*J*J*gzObObcfOgOgr.    tuple[list[str], list[int], str]c                    g }g }ddl m} |                    | j                  }| j                            dt          |j                            }t          |j        	                                          |k     sJ | 
                    |          }d |j                                        D             }|                                }|j        }	t          |          D ]}
|
|vr>|                    d|
 d           |                    t           j        j                   E||
         }||v r|	|
         j        sl|}|                    |                    |d                    }||k    r:t,                              t1          |           d	t1          |           d
           |	|
         j        s|                     |          r%|                    t           j        j                   nr|                    d                    d          d          }|                    t           j        j                   n$|                    t           j        j                   |                    |           |||fS )Nr   AutoTokenizer
vocab_sizec                    i | ]\  }}||	S r-   r-   ra   encoded_tokid_s      r/   
<dictcomp>z,TextModel.get_vocab_base.<locals>.<dictcomp>  s    ZZZ.>k3kZZZr.   [PAD]Fadd_special_tokens  is encoded and decoded back to  using AutoTokenizer   ▁r   r
  )transformersr   ry  r:   rG   r   r   r  r  r  get_vocab_base_prerl   get_added_vocabadded_tokens_decoderrangern  rf   	TokenTyper+   
normalizedr  encoderd   re   reprspecialr  r)   replacer*   r'   )rq   tokenstoktypesr   	tokenizerr  tokprereverse_vocabadded_vocabr  ir  previous_tokens                r/   get_vocab_basezTextModel.get_vocab_baset  s    ......!11$.AA	\%%lC	4H4HII
9?))++,,z9999((33ZZ)/BWBWBYBYZZZ//11(=z"" 	% 	%A%%kQkkk*** 56666*1-K'' 02= D). ) 0 01A1A%\a1A1b1b c c)U22"KK4+?+?  )C  )Caefkalal  )C  )C  )C  D  D  D+A.6 E$:V:VW\:]:] E (>???? !&o.D.DW.M.Ms S S (CDDDDOODN$9:::e$$$$x''r.   c                D   d}|                     |          }t          t          |                                                                                     }t                              d|            t                              d|            d }|dk    rd}|dk    rd}|dk    rd}|d	k    rd}|d
k    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd}|dk    rd }|d!k    rd"}|d#k    rd$}|d%k    rd&}|d'k    rd(}|d)k    rd*}|d+k    rd,}|d-k    rd.}|d/k    rd0}|d1k    rd2}|d3k    rd}|d'k    rd4}|d5k    rd6}|d7k    rd8}|d!k    rd9}|d:k    rd;}|d<k    rd=}|d>k    rd?}|d@k    rdA}|dBk    rdC}|dDk    rdE}|dFk    rdG}|dHk    rdI}|dJk    rdK}|dLk    rdM}|dNk    rdO}|dPk    rdQ}|dRk    rdS}|dTk    rdU}|dVk    rdW}|dXk    rdY}|dZk    rd[}|d\k    rd]}|d^k    rd_}|d`k    rda}|dbk    rdc}|ddk    rde}|dfk    rdg}|dhk    rdi}|djk    rdk}|dlk    rdm}|dnk    rdo}|dpk    rdq}|drk    rds}|dtk    rdu}|dvk    rdw}|dxk    rdy}|Jt                              dz           t                              d{           t                              d|           t                              d}           t                              d~           t                              d           t                              d           t                              d           t                              d           t                              d|            t                              d{           t                              dz           t          d          t                              dt          |                      t                              d|            |S )Nu  
 

 


 	 		 	
  
   
    
     
🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lLzchktok: zchkhsh: @b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99bzchatglm-bpe@81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516@a1336059768a55c99a734006ffb02203cd450fed003e9a71886c88acf24fdbc2glm4@9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902@1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35z
minerva-7b@7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664hunyuan@bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6zhunyuan-dense@a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6z	falcon-h1@60476e1243776c4fb1b993dbd7a5f15ac22f83c80afdf425fa5ae01c8d44ef86@3eda48b4c4dc7de733d1a8b3e3b4a85243dbbf704da2ee9d42c6beced8897896@48f8e02c0359c0bbdd82f26909171fac1c18a457bb47573ed1fe3bbb2c1cfd4b@81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890kimi-k2@d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97cqwen2@0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5z	llama-bpe@049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754zdeepseek-llm@347715f544604f9118bb75ed199f68779f423cabb20db6de6f31b908d04d7821zdeepseek-coder@8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652edfalcon@0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19fzbert-bge@9d032fcbd5501f4a38150912590928bfb36091efb5df11b8e2124b0390e3fb1efalcon3@8e62295832751ca1e8f92f2226f403dea30dc5165e448b5bfa05af5340c64ec7zbert-bge-large@b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166mpt@35d91631860c815f952d711435f48d356ebac988362536bed955d43bfa436e34	starcoder@3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454gpt-2@32d85c31273f8019248f2559fed492d929ea28b17e51d81d3bb36fff23ca72b3	stablelm2@6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ffrefact@9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8z	command-r@e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aeaolmo@a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79edbrx@c7699093ba4255a91e702aa38a596aa81669f3525dae06c2953267dde580f448z
jina-v1-enz
jina-v2-en@171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643z
jina-v2-es@27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6z
jina-v2-de@c136ed14d01c2745d4f60a9596ae66800e2b61fa45643e72436041855ad4089dz	smaug-bpe@c7ea5862a53e4272c035c8238367063e2b270d51faa48c0f09e9d5b54746c360z	poro-chat@7967bfa498ade6b757b064f31e964dddbb80f8f9a4d68d4ba7998fcf281c531azjina-v2-code@7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52ceeviking@b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901jais@7b3e7548e4308f52a76e8229e4e6cc831195d0d1df43aed21ac6c93da05fec5f	codeshell@63b97e4253352e6f357cc59ea5b583e3a680eaeaf2632188c2b952de2588485etekken@855059429035d75a914d1eda9f10a876752e281a054a7a3d421ef0533e5b6249smollm@3c30d3ad1d6b64202cd222813e7736c2db6e1bd6d67197090fc1211fbc612ae7bloom@bc01ce58980e1db43859146dc51b1758b3b88729b217a74792e9f8d43e479d21zgpt3-finnish@4e2b24cc4770243d65a2c9ec19770a72f08cffc161adbb73fcbb6b7dd45a0aaeexaone@fcace8b9cac38ce847670c970cd5892031a753a1ef381abd1d9af00f713da085zphi-2@60824e3c0d9401f89943cbb2fff727f0e2d4c545ba4df2d6e4f09a6db0f5b450	chameleon@8b5a93ed704057481f240da0be7e7dca721d7f8f4755263b6807227a2cbeae65zroberta-bpe@ad851be1dba641f2e3711822f816db2c265f788b37c63b4e1aeacb9ee92de8ebgigachat@d4c8f286ea6b520b3d495c4455483cfa2302c0cfcd4be05d781b6a8a0a7cdaf1megrez@877081d19cf6996e2c4ff0e1236341e9b7bde288f5311a56a937f0afbbb3aeb5zdeepseek-v3@b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5zdeepseek-r1-qwen@ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352ezgpt-4o@7dec86086fcc38b66b7bc1575a160ae21cf705be7718b9d5598190d7c12db76fsuperbpe@1994ffd01900cfb37395608534236ecd63f2bd5995d6cb1004dda1af50240f15trillion@96a5f08be6259352137b512d4157e333e21df7edd3fcd152990608735a65b224
bailingmoe@d353350c764d8c3b39c763113960e4fb4919bea5fbf208a0e3b22e8469dc7406llama4@0e9433cbbb161f89e264eb32e8e64bfe69e834973ffca5d41d3948a604a3e2a3pixtral@d5f1dd6f980fec569fb218a81a7658ac45fc56b38c5a0adeb1c232fbe04ef5ecz
seed-coder@b0a6b1c0bd5998ebd9df08611efde34a4ff03faed45ae09c43e6b31ebd4b94cfza.x-4.0@f6791d196f87ce6b56a7d234be618e0d58f8cda3549416635b2bebcd22cd95c4zmidm-2.0@169bf0296a13c4d9b7672313f749eb36501d931022de052aad6e36f2bf34dd51lfm2@2085e1638f6c377a0aa4ead21b27bb4cb941bf800df86ed391011769c1758dfbexaone4@a1e163ecab2e718a4c829d1148b6e86824ec36163bb71941c3dca9cd5ac25756mellum
zV**************************************************************************************z5** WARNING: The BPE pre-tokenizer was not recognized!z2**          There are 2 possible reasons for this:zN**          - the model has not been added to convert_hf_to_gguf_update.py yetz>**          - the pre-tokenization config has changed upstreamz`**          Check your model files and convert_hf_to_gguf_update.py and update them accordingly.z;** ref:     https://github.com/ggml-org/llama.cpp/pull/6920z**z** chkhsh:  zBBPE pre-tokenizer was not recognized - update get_vocab_base_pre()ztokenizer.ggml.pre: )	r  r   r   	hexdigestrd   debugr>  r   r  )rq   r  chktxtchktokchkhshress         r/   r  zTextModel.get_vocab_base_pre  s    d!!&))F**,,--7799((()))((()))
 WWWCWWWCWWWCWWWCWWWCWWWCWWW!CWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWW CWWW"CWWWCWWWCWWWCWWW"CWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWW CWWWCWWWCWWWCWWWCWWWCWWWCWWW CWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWW$CWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWCWWWC;NN4   NNstttNNRSSSNNOPPPNNklllNN[\\\NN}~~~NNXYYYNN4   NN2&22333NNstttNN4   %&jkkk7DII77888((()))
r.   Nonec                :    | j                             d           d S )Nnone)rI   add_tokenizer_modelr   s    r/   _set_vocab_nonezTextModel._set_vocab_none  s    ,,V44444r.   c                p   |                                  \  }}}| j                            d           | j                            |           | j                            |           | j                            |           t          j        | j        d          }|	                    | j                   d S Ngpt2Tload_merges
r!  rI   r  add_tokenizer_preadd_token_listadd_token_typesrf   SpecialVocabr:   add_to_ggufrq   r  r  r  special_vocabs        r/   r  zTextModel._set_vocab_gpt2      #'#6#6#8#8 &,,V444**6222''///((222)$.dKKK!!$"233333r.   c           	        | j         }| j        }g }g }ddlm} |                    |d          }|d         }t          |                                                                          |k     sJ |                     |          }g }	i }
|j	        }|
                                D ]\  }}||
t                              |          <   t          |          dk    r6t                              |||          }t          |          dk    sJ |	                    d	                    t#          t          j        |                               |j        }d
 i |
|
                                D             }t'          |          D ]}||vr>|                    d| d           |                    t(          j        j                   D||         |v r@|                    ||                    |                    t(          j        j                   |                    ||                    |                    t(          j        j                   | j                            d           | j                            |           | j                            |           | j                            |           t)          j        |d          }|	|_        t          |j                   dk    rB|!                    d|j        d                    |!                    d|j        d                    |!                    d|j        d                    |"                    | j                   d S )Nr   r  Trr  r  r   max_rankr   r
  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z-TextModel._set_vocab_qwen.<locals>.<dictcomp>  s    ddd/?{C{dddr.   r  r  r  Fr  bos<|endoftext|>eosunk)#r:   rG   r  r   ry  r  	get_vocabr  r  mergeable_ranksrl   	QwenModeltoken_bytes_to_stringr   bpern  rB  mapspecial_tokensr  rf   r  r+   r)   r'   rI   r  r  r  r  r  mergesspecial_token_ids_set_special_tokenr  )rq   r:   rG   r  r  r   r  r  r  r  r  r  r  rankmergedr  r  r  r  s                      r/   _set_vocab_qwenzTextModel._set_vocab_qwen  sM   N	, ......!11)t1TT	\*
9&&((//1122Z????((33#3*0022 	R 	RKE4<@E)11%8895zzQ]]?ED]IIFv;;!####MM#((3y'F#O#OPPQQQQ  .ddC[eC[{C[CaCaCcCcdddz"" 		7 		7A%%kQkkk*** 56666q![00mA./// 67777mA./// 56666,,V444**6222''///((222))GGG%}.//144,,UI4L_4]^^^,,UI4L_4]^^^((	0H0YZZZ!!$"233333r.   Tc                   |                                  \  }}}| j                            d           | j                            d           | j                            |           | j                            |           | j                            |           t          j        | j	        t          |                    }|                    | j                   d S Nllamadefaultn_vocab)_create_vocab_sentencepiecerI   r  r  r  add_token_scoresr  rf   r  r:   r   r  )rq   r  r  scoresr  r  s         r/   _set_vocab_sentencepiecez"TextModel._set_vocab_sentencepiece  s    #'#C#C#E#E ,,W555**9555''///))&111((222)$.#f++NNN!!$"233333r.   c           
     ~
   ddl m} | j        dz  }|                                st	          d|            |            }|                    t          |                     |                     ddgd          p|                                }d	 t          |          D             }d
g|z  }t          j        g|z  }t          |                                          D ]}||k    r%t                              d| d|dz
               n|                    |          }	|	                    d          }
|                    |          }t          j        }|                    |          rt          j        }ne|                    |          rt          j        }nC|                    |          rt          j        }n!|                    |          rt          j        }|
||<   |||<   |||<   | j        dz  }|                                rt3          |dd          5 }t5          j        |          }|D ]`}||         }||k    r$t                              d| d|dz
              4|                    d          ||<   d||<   t          j        ||<   a	 d d d            n# 1 swxY w Y   | j        dz  }|                                rt3          |dd          5 }t5          j        |          }|                    di           }|                                D ]G\  }}t?          |          }|d         }||k    r$t                              d| d|dz
              G||         t          j        k    r[||         |                    d          k    r<t                              d| d||                              d          d|           |                    d          s| !                    |          rt          j        ||<   n8|"                    d                     d          d          }t          j        ||<   d||<   |                    d          ||<   I	 d d d            n# 1 swxY w Y   |tG          |          k    r|tG          |          z
  }t          $                    d| d| d           t          d|dz             D ]^}|%                    tM          d | dd                     |%                    d           |%                    t          j                   _|||fS )!Nr   SentencePieceProcessortokenizer.modelFile not found: vocab_size_per_layer_inputr  Tr  c                @    g | ]}d | d                     d          S r  r  r   r  ra   r  s     r/   
<listcomp>z9TextModel._create_vocab_sentencepiece.<locals>.<listcomp>  .    VVVq{a{{{11'::VVVr.        zignore tokens from : id is out of range, max=r   r   added_tokens.jsonr   r   ignore token      @tokenizer_config.jsonr  contentreplacing token rt   -> r  r  r
  Padding vocab with  token(s) - [PAD1] through [PADr  r  )'sentencepiecer  r:   r   FileNotFoundErrorLoadFromFiler   r   r  r  r   r+   rd   r>  	IdToPiecer  GetScorer'   	IsUnknownr(   	IsControlr)   IsUnusedIsByter,   r   r   r   r*   r   rl   rQ   r  r  r  r   r  rn  r  )rq   r  tokenizer_pathr  r  r  r  r  token_idpiecetextscoretoktypeadded_tokens_filer   added_tokens_jsonr   tokenizer_config_filetokenizer_config_jsonr  
token_datar  	pad_countr  s                           r/   r  z%TextModel._create_vocab_sentencepiece  sK   888888*;;%%'' 	I#$G~$G$GHHH**,,	s>22333%%('
  &   4 '1133 	
 WVE*DUDUVVV'j:56=>Ki224455 	) 	)H:%%iXiiYcfgYgiijjj''11E<<((D&&x00E-4G""8,, 719$$X.. 719##H-- 718!!(++ 716#F8$F8!(HX N-@@$$&& 	N'w??? 
N1$(IaLL!, N NC05H:--'kx'k'k[ehi[i'k'klll '*zz'':':F8$'.F8$)@)MHX&&N
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N !%1H H ((** 	=+S7CCC =q(,	!%'<'@'@AWY['\'\$,@,F,F,H,H = =(Hj"8}}H!+I!6E:--'kx'k'k[ehi[i'k'klll )-D-KKK!(+u||G/D/DDD"NN+{h+{+{&QYJZJaJabiJjJj+{+{rw+{+{|||!~~i00 RD4P4PQV4W4W R-D-L** %o.D.DW.M.Ms S S-D-Q*'.F8$',||G'<'<F8$$!== = = = = = = = = = = = = = =* F##"S[[0ILLeyeeYbeeefff1i!m,, @ @eK1KKK'BBBCCCg&&& 7 >????vx''s&   !A8J&&J*-J*"FQ77Q;>Q;c                   t          j        | j                  }g }g }g }|                                D ]E\  }}}|                    |           |                    |           |                    |           Ft          |          |j        k    sJ | j                            d           | j        	                    d           | j        
                    |           | j                            |           | j                            |           t          j        | j        t          |                    }|                    | j                   d S r  )rf   LlamaHfVocabr:   
all_tokensrn  r   r  rI   r  r  r  r  r  r  r  )	rq   r  r  r  r  r  r  r  r  s	            r/   _set_vocab_llama_hfzTextModel._set_vocab_llama_hf3  sE   !$.11$)$4$4$6$6 	% 	% D%MM$MM%   OOG$$$$6{{e.....,,W555**9555''///))&111((222)$.#f++NNN!!$"233333r.   c           	     .   | j         dz                                  sJ | j                            dd          }d                    d          g}t
          j        j        g}t          | j         dz  dd          5 }|	                                }|D ],}|
                    d          }t          |          d	k    sJ t          j        d                    |d
d                             t          |d                   }	}t!          |t"                    r|                    d          n|}t!          |t$                    sJ t          |          |	k    sJ t'          |          dd         }
|                    |
                    d                     |                    t
          j        j                   .	 d d d            n# 1 swxY w Y   |t          |          z
  }|dk    sJ t-          t          |          |          D ]R}|                    d| d                    d                     |                    t
          j        j                   S| j                            d           | j                            |           | j                            |           t          j        | j         d          }|j        ~t=          t>                    j         dz  dz  dz  }|                                r?t          |dd          5 }|!                                }d d d            n# 1 swxY w Y   nd}||_        |"                    dd           d|j#        d<   d|j#        d<   |$                    | j                   d S )Nzrwkv_vocab_v20230424.txtr  i   z<s>r   r   r   r
  r    r   r   r   r  r  rwkvFr  models	templateszllama-cpp-rwkv-world.jinjaz
rwkv-worldeoti  r  r  )%r:   r   rG   r   r  rf   r  r)   r   	readlinesr  r   astliteral_evalrB  rQ   r   r   r  r  rn  r'   r  r+   rI   r  r  r  r  chat_templater   __file__r  readr  r  r  )rq   r  r  r  r   lineslinepartsr  	token_lenr  	remainderr  r  template_pathtemplates                   r/   _set_vocab_rwkv_worldzTextModel._set_vocab_rwkv_worldI  s   !;;DDFFFFF\%%lE::
$||G445#~56$.#==sWUUU 	7YZKKMME 	7 	7

35zzQ#&#3CHHU1R4[4I4I#J#JCPUVXPYNNy1;E31G1GRW---U!%/////5zzY...."&u++ad"3
j//88999 56666	7	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 V,	A~~~~s6{{J// 	3 	3AMM++++,,W55666OODN12222,,V444''///((222)$.eLLL&. NN1H<{JMiiM$$&& (-w??? (1 vvxxH( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (*2M'((44412'.12'.!!$"233333s%   <EGGGL<<M M rK    Literal['gpt-neox', 'llama-spm']r  rQ   c                
   t          t          j        d                   dz  d| dz  }t                              dt
          j                            |t          j                               d           t          j	        |d          }|dk    rd	nd
}|
                    t          j        j        j                  sJ | j                            t!          j        d                                       d                     |
                    t          j        j        j                  | j                            r-t!          j        d                                       d          n|           |
                    t          j        j        j                  sJ | j                            fdj        D             d |                    |dk    r`|
                    t          j        j        j                  sJ | j                            fdj        D             d |                    |
                    t          j        j        j                  sJ | j                            fdj        D             d |                    |dk    rX|
                    t          j        j        j                  sJ | j                            fdj        D                        |
                    t          j        j        j                  x=| j                            j        d                                          d                    |
                    t          j        j        j!                  x=| j        "                    j        d                                          d                    |
                    t          j        j        j#                  x=| j        $                    j        d                                          d                    |
                    t          j        j        j%                  x=| j        &                    j        d                                          d                    |
                    t          j        j        j'                  x=| j        (                    j        d                                          d                    |
                    t          j        j        j)                  x?| j        *                    j        d                                          d                    d S d S )Nr   r  zggml-vocab-r  zUsing tokenizer from 'r   r   gpt-neoxr>  r  r  r   c                D    g | ]}t          j        |                   S r-   r  r  ra   r  fields     r/   r  z0TextModel._set_vocab_builtin.<locals>.<listcomp>  s'    (S(S(S1u{1~)>)>(S(S(Sr.   	llama-spmc                Z    g | ]'}j         |                                         d          (S r   r  tolistr  s     r/   r  z0TextModel._set_vocab_builtin.<locals>.<listcomp>  s0    .^.^.^au{1~/D/D/F/Fq/I.^.^.^r.   c                Z    g | ]'}j         |                                         d          (S r  r  r  s     r/   r  z0TextModel._set_vocab_builtin.<locals>.<listcomp>  s0    )Y)Y)Y%+a.*?*?*A*A!*D)Y)Y)Yr.   c                D    g | ]}t          j        |                   S r-   r   r  s     r/   r  z0TextModel._set_vocab_builtin.<locals>.<listcomp>  s'    .Y.Y.YuU[^/D/D.Y.Y.Yr.   )+r   sysrw   rd   r>  rk  relpathgetcwdrf   
GGUFReader	get_fieldKeys	TokenizerrO  rI   r  r  r  r  PREr  LISTr  r   SCORESr  
TOKEN_TYPEr  MERGESadd_token_mergesBOS_IDadd_bos_token_idr  EOS_IDadd_eos_token_idUNK_IDadd_unk_token_idPAD_IDadd_pad_token_idADD_BOSadd_add_bos_tokenADD_EOSadd_add_eos_token)rq   rK   r  r  vocab_readerdefault_prer  s         @r/   _set_vocab_builtinzTextModel._set_vocab_builtinv  s   chqk**X58Wj8W8W8WW_PRPYP[P[0\0\___```~s;;)Z77eeY&&ty':'@AAu,,U5;r?-C-C-J-J7-S-STTT&&ty':'>??**UZ+k5R+A+A+H+H+Q+Q+Q`klll&&ty':'?@@u''(S(S(S(S
(S(S(ST_U_T_(`aaa$$ **49+>+EFFELL5--.^.^.^.^SXS].^.^.^_j`j_j.klll&&ty':'EFFu(()Y)Y)Y)Yej)Y)Y)YZe[eZe)fggg$$ **49+>+EFFELL5--.Y.Y.Y.Yej.Y.Y.YZZZ!++DI,?,FGGGET--ek"o.D.D.F.Fq.IJJJ!++DI,?,FGGGET--ek"o.D.D.F.Fq.IJJJ!++DI,?,FGGGET--ek"o.D.D.F.Fq.IJJJ!++DI,?,FGGGET--ek"o.D.D.F.Fq.IJJJ!++DI,?,GHHHEU..u{2/E/E/G/G/JKKK!++DI,?,GHHHEU..u{2/E/E/G/G/JKKKKK VUr.   c                |   d }| j         dz  }|                                rXt          |d          5 }t          j        |          }d d d            n# 1 swxY w Y   |D ]}|d         dk    r
|d         } n|t          | j         |z  dz  d          5 }t          j        |          }d d d            n# 1 swxY w Y   |d         rt
          j        j        }nC|d	         rt
          j        j        }n)|d
         rt
          j        j	        }nt          d          | j                            |           d S d S )Nzmodules.jsonr   r   rz   z$sentence_transformers.models.Poolingrw   ru  pooling_mode_mean_tokenspooling_mode_cls_tokenpooling_mode_lasttokenz0Only MEAN, CLS, and LAST pooling types supported)r:   r   r   r   r   rf   PoolingTypeMEANCLSLASTr   rI   add_pooling_type)rq   pooling_pathmodule_pathr   modulesmodpoolingpooling_types           r/   _try_set_pooling_typezTextModel._try_set_pooling_type  s   n~5   	kG444 ')A,,' ' ' ' ' ' ' ' ' ' ' ' ' ' '  v;"HHH#&v;LE I
 #dn|3mCgVVV 'Z[)A,,' ' ' ' ' ' ' ' ' ' ' ' ' ' '12 ^#/412 ^#/312 ^#/4)*\]]]--l;;;;; $#s#   AAAB88B<?B<r  )r  r  r\   r>   r\   r  r\   r   r\   r  )T)rK   r  r  rQ   )r$   r%   r&   r1   r2   r  r  r   r  r  r  r^  r   r`  r  r!  r  r  r  r  r  r  r  r  r%  r5  __classcell__r  s   @r/   r{   r{     s        JLLLV V V V V S S [S       8,7 ,7 ,7\! ! !   0*( *( *( *(`c c c cL5 5 5 54 4 4 444 44 44l
4 
4 
4 
4X( X( X(t4 4 4,+4 +4 +4Z+L +L +L +LZ< < < < < < < <r.   r{   c                       e Zd ZU ej        Zej        j        Zde	d<   de	d<   g dZ
dZde	d<   dZde	d	<   d
Zde	d<   d
Zde	d<    fdZddZddZd Zd Zd ZdddZdddZdd dZ xZS )!r|   rF   preprocessor_configglobal_config)r  r  r  r  depthTr>   has_vision_encoderFhas_audio_encoderNrZ   hparams_visionhparams_audioc                    t                      j        |i | | j        t          j        j        k    rt          d          | j        sld| j        vr
i | j        d<   d| j        vr
i | j        d<   i | j        | j        d         }|	                    d|	                    dd                    | _
        n>d | j                                        D             }|	                    dd          | _
        | j
        dk    s
J d	            dd l}|                    | j                  | _        |                                 | _        |                                 | _        | j        | j        t'          d
          | j        p| j        p| j        | _        | j        o| j        }|rdn|                     | j        d          | _        t          j        t          j        j        | j                  | _        | j        sNt7          | j        dz  dd          5 }t;          j        |          | _        d d d            d S # 1 swxY w Y   d S d S )NzGMmprojModel must be subclassed with model_arch = gguf.MODEL_ARCH.MMPROJrw  audio_configr  r  r   c                "    i | ]\  }}|d v	||S ))vision_encoderaudio_encoderr-   ra   r   vs      r/   r  z(MmprojModel.__init__.<locals>.<dictcomp>  s0       ABe9e9e19e9e9er.   r  zn_embd not found in hparamsz1vision_config / audio_config not found in hparams   Tzpreprocessor_config.jsonr   r   r   ) r  r   rP   rf   r  r3   r}   rT   rG   r   n_embd_textrl   copydeepcopyr=  get_vision_configrA  get_audio_configrB  r   r@  r?  r   n_block_keysrR   r  rS   r   r:   r   r   r<  )rq   r  r  rw  rL  have_multiple_encodersr   r  s          r/   r   zMmprojModel.__init__  s   $)&)))?do444efff % 	@DL00.0]+T\11/1^,IT\IT\--HIK*}koohXY>Z>Z[[D !%!3!3!5!5  K  +|Q??D!###%B### 	!]]4<88"4466!2244&4+=+EPQQQ *Pd.@PDL "&!7!SD<S"8g33d>N>NtO`bf>g>g24?3I4K[\\ % 	8dn'AA3QXYYY 8]^+/9Q<<(8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8	8 	8s   H>>IIr\   c                L    | j         sdnd}| j                            |          S )Nvision_configrF  )rT   r=  r   )rq   config_names     r/   rN  zMmprojModel.get_vision_config  s+    -1-CYooIY!%%k222r.   c                6    | j                             d          S )NrD  r=  r   r   s    r/   rO  zMmprojModel.get_audio_config  s    !%%n555r.   c                X    | j                             t          j        j                   d S r`   )rI   rM  rf   rN  r3   r   s    r/   rP  zMmprojModel.set_type   s$    !!$-"677777r.   c                    | j                             | j                   | j        r| j                             d           | j                             | j                   | j                             |                     dg                     | j         	                    |                     dg                     | j         
                    |                     dg                     | j                             |                     dg                     | j                             |                     | j                             | j                             |                     dg                     | j        rt           n| j        d         }| j        rt$          n| j        d         }| j                             |           | j                             |           | j        r| j                             d           | j                             | j                   | j                             |                     dg                     | j                             |                     dg                     | j                             |                     | j                             | j                             |                     dg                     | j        s| j        st;          d	          d S d S )
NT
image_size
patch_sizer  r  r  
image_mean	image_stdz4MmprojModel must have either vision or audio encoder)rI   r  r<   r?  add_clip_has_vision_encoderadd_vision_projection_dimrK  add_vision_image_sizefind_vparamadd_vision_patch_sizeadd_vision_embedding_lengthadd_vision_feed_forward_lengthadd_vision_block_countrP  add_vision_head_countrT   r   r<  r   add_vision_image_meanadd_vision_image_stdr@  add_clip_has_audio_encoderadd_audio_projection_dimadd_audio_embedding_lengthfind_aparamadd_audio_feed_forward_lengthadd_audio_block_countadd_audio_head_countr   )rq   r[  r\  s      r/   r   zMmprojModel.set_gguf_parameters  s   &&tz222" 	=88>>>66t7GHHH 2243C3C\N3S3STTT2243C3C\N3S3STTT889I9I=/9Z9Z[[[;;D<L<LNaMb<c<cddd33D4D4DTEV4W4WXXX2243C3CEZD[3\3\]]] *.)?kTE]^jEkJ'+'=h4C[\gChI22:>>>11)<<<! 	]77===55d6FGGG 778H8H-8Y8YZZZ::4;K;KM`La;b;bccc2243C3CDDU3V3VWWW11$2B2BDYCZ2[2[\\\& 	Ut/E 	USTTT	U 	U 	U 	Ur.   c                     t          d          )Nz*MmprojModel does not support vocab writing)r   r   s    r/   r`  zMmprojModel.write_vocab&  s    EFFFr.   rk   r   r   r   c                L    | j         J |                     | j         ||          S r`   )rA  _find_paramrq   rk   r   s      r/   r`  zMmprojModel.find_vparam)  s,    "... 3T8DDDr.   c                L    | j         J |                     | j         ||          S r`   )rB  rq  rr  s      r/   rk  zMmprojModel.find_aparam-  s,    !--- 2D(CCCr.   objc                |    t          fd|D             d           }||         S |rd S t          d|           )Nc              3  $   K   | ]
}|v |V  d S r`   r-   )ra   r   rt  s     r/   rc   z*MmprojModel._find_param.<locals>.<genexpr>2  s'      00!qCxxAxxxx00r.   r   )r   r   )rq   rt  rk   r   r   s    `   r/   rq  zMmprojModel._find_param1  sY    0000t000$77?s8O 	4777888r.   r\   rZ   r  r  )rt  rF   rk   r   r   r>   r\   r   )r$   r%   r&   r1   r3   r  rf   r  rP   r  rP  r?  r@  rA  rB  r   rN  rO  rP  r   r`  r`  rk  rq  r9  r:  s   @r/   r|   r|     si        !J'J''''!!!!VVVL########## -1N0000+/M////*8 *8 *8 *8 *8X3 3 3 36 6 6 68 8 8!U !U !UFG G GE E E E ED D D D D9 9 9 9 9 9 9 9 9r.   r|   GPTNeoXForCausalLMc                  4    e Zd Zej        j        Zd Zdd
ZdS )GPTNeoXModelc                   | j         d         }| j                            | j         d                    | j                            | j         d                    | j                            |           | j                            | j         d                    | j                            t          | j         d         | j         d         | j         d         z  z                       | j                            | j         d                    | j        	                    | j         
                    dd                     | j                            | j         d	                    d S )
Nr  r  r  r  
rotary_pctr  use_parallel_residualTr  )rG   rI   r  r  r  r  add_rope_dimension_countrQ   r  add_parallel_residualr   r  rq   rR   s     r/   r   z GPTNeoXModel.set_gguf_parameters>  s9   l#67++DL9R,STTT--dl=.IJJJ((55500>Q1RSSS11\*dl=.IT\ZoMp.pqrr	
 	
 	
 	''5J(KLLL..t|/?/?@WY]/^/^___++DL9I,JKKKKKr.   r   r   rb   r   r   r   r\   r   c           
     `   ~| j                             d| j                             d                    }| j                             d| j                             d                    }g }t          j        d|          r|                    |d||z  |f          }t          j        |d d dd d d d f                             d|f          |d d d	d d d d f                             d|f          |d d d
d d d d f                             d|f          fd          }t                              d           nt          j        d|          r|                    |d||z  f          }t          j        |d d dd d f                             |f          |d d d	d d f                             |f          |d d d
d d f                             |f          fd          }t                              d           |	                    | 
                    |          |f           |S )Nr  r  r  n_embedz9gpt_neox\.layers\.\d+\.attention\.query_key_value\.weightr    r   r  r   r   r  %re-format attention.linear_qkv.weightz7gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias#re-format attention.linear_qkv.bias)rG   r   rematchreshaper   catrd   re   rn  r   	rq   r   rb   r   r  r  r  qkv_weightsqkv_biass	            r/   r   zGPTNeoXModel.modify_tensorsL  sZ   !!(DL,<,<=R,S,STT,""=$,2B2B92M2MNN,.8PRVWW 	? %,,faF9JG-TUUK1aaa
+33RMBB1aaa
+33RMBB1aaa
+33RMBB
   J KK?@@@@XPRVWW 
	?!))61g6G*HIIHQQQ111W%--wj99QQQ111W%--wj99QQQ111W%--wj99
   J KK=>>>,,T22J?@@@r.   Nr  )	r$   r%   r&   rf   r  GPTNEOXrP   r   r   r-   r.   r/   rz  rz  :  sD        (JL L L$ $ $ $ $ $r.   rz  BloomForCausalLM
BloomModelc                  4    e Zd Zej        j        Zd Zdd
ZdS )r  c                   | j                             d| j                             d                    }| j                             d| j                             d                    }| j                            | j                             d|                     | j                            |           | j                            d|z             | j                            | j         d                    | j                            |           | j                            |           | j        	                    | j         d                    | j        
                    | j                   d S )	Nr  r  r  r  
seq_lengthr!   r  r  )rG   r   rI   r  r  r  r  r  r  r  r  r<   )rq   r  r  s      r/   r   zBloomModel.set_gguf_parametersw  s6   ,""=$,2B2B92M2MNN!!(DL,<,<=R,S,STT++DL,<,<\7,S,STTT--g66600W===((i)@AAA''///**6222++DL9M,NOOO&&tz22222r.   r   r   rb   r   r   r   r\   r   c           
        ~| j                             d| j                             d                    }| j                             d| j                             d                    }t          j        dd|          }g }t          j        d|          r|                    |d||z  |f          }t          j        |d d d	d d d d f                             d
|f          |d d dd d d d f                             d
|f          |d d dd d d d f                             d
|f          fd	          }t          	                    d           nt          j        d|          r|                    |d||z  f          }t          j        |d d d	d d f                             |f          |d d dd d f                             |f          |d d dd d f                             |f          fd	          }t          	                    d           |
                    |                     |          |f           |S )Nr  r  r  r  ztransformer\.r  z/h\.\d+\.self_attention\.query_key_value\.weightr    r   r  r   r   r  r  z-h\.\d+\.self_attention\.query_key_value\.biasr  )rG   r   r  subr  r  r   r  rd   re   rn  r   r  s	            r/   r   zBloomModel.modify_tensors  sl   !!(DL,<,<=R,S,STT,""=$,2B2B92M2MNNv&D11,.8FMM 	? %,,faF9JG-TUUK1aaa
+33RMBB1aaa
+33RMBB1aaa
+33RMBB
   J KK?@@@@XFMM 
	?!))61g6G*HIIHQQQ111W%--wj99QQQ111W%--wj99QQQ111W%--wj99
   J KK=>>>,,T22J?@@@r.   Nr  )	r$   r%   r&   rf   r  BLOOMrP   r   r   r-   r.   r/   r  r  s  sA        &J
3 
3 
3& & & & & &r.   MPTForCausalLMc                  :    e Zd Zej        j        Zd Zd ZddZ	dS )MPTModelc                J   	 |                                   d S # t          $ r |                                  | j                            d           | j                            d           | j                            d           | j                            d           Y d S w xY w)NFr    r   r   )r  r{  r  rI   r   r  r  r  r   s    r/   r  zMPTModel.set_vocab  s    	1  """"" 	1 	1 	1))+++..u555--a000--a000--a000000	1s    BB"!B"c                ^   | j         d         }| j                            | j         d                    | j                            | j         d                    | j                            |           | j                            d| j         d         z             | j                            | j         d                    | j         d                             d          x}r| j                            |           | j        	                    d           | j         d         d	         +| j        
                    | j         d         d	                    | j         d         d
         r-| j                            | j         d         d                    d S | j                            d           d S )Nr  max_seq_lend_modelr!   r  attn_config
kv_n_headsh㈵>clip_qkvalibialibi_bias_max        )rG   rI   r  r  r  r  r  r   r  r  add_clamp_kqvadd_max_alibi_bias)rq   rR   r  s      r/   r   zMPTModel.set_gguf_parameters  s   l:.++DL,GHHH--dl9.EFFF((55500T\)5L1LMMM''Y(?@@@m488FFF: 	;..z:::++D111<&z2>**4<+Fz+RSSS<&w/ 	5//]0KL\0]^^^^^//44444r.   r   r   rb   r   r   r   r\   r   c                    ~d|v r.|                      |d          }|                    dd          }n|                      |d          }||fgS )Nscales)r   r   z.scalesr   z
act.scalesr   )r   r  rq   r   rb   r   r   s        r/   r   zMPTModel.modify_tensors  sg    t++D?^+__H'',??HH++D?S+TTH:&''r.   Nr  )
r$   r%   r&   rf   r  MPTrP   r  r   r   r-   r.   r/   r  r    sP        $J	1 	1 	15 5 5"	( 	( 	( 	( 	( 	(r.   r  OrionForCausalLMc                  2    e Zd Zej        j        Zd Zd ZdS )
OrionModelc                .    |                                   d S r`   r  r   s    r/   r  zOrionModel.set_vocab      %%'''''r.   c                <   | j         d         }| j         d         }| j                             d|          }d}d| j         v r| j         d         }n=d| j         v r| j         d         }n&d| j         v r| j         d         }nt          d          | j                            | j                   | j                            d	           | j                            |           | j                            | j         d
                    | j        	                    |           | j        
                    | j         d                    | j                            |           | j                            |           | j                            | j         d                    d S )Nr  r  r  r   max_sequence_lengthr  model_max_length(gguf: can not find ctx length parameter.Meta AI original pthr  r  r  )rG   r   r   rI   r  r<   add_tensor_data_layoutr  r  r  r  r  r  r  )rq   rR   
head_counthead_count_kv
ctx_lengths        r/   r   zOrionModel.set_gguf_parameters  s   l#67\"78
(()>
KK
 DL00&;<JJ&$,66&?@JJ4<//&89JJGHHH&&tz222//0FGGG++J777--dl=.IJJJ((55500>Q1RSSS''
333**=999 	++DL,HIIIIIr.   N)	r$   r%   r&   rf   r  ORIONrP   r  r   r-   r.   r/   r  r    sA        &J( ( (J J J J Jr.   r  BaichuanForCausalLMBaiChuanForCausalLMc                  X    e Zd Zej        j        Zd Zd ZddZ	dddZ
	 dddZddZdS )BaichuanModelc                .    |                                   d S r`   r  r   s    r/   r  zBaichuanModel.set_vocab  r  r.   c                   | j         d         }| j         d         }| j                             d|          }d}d| j         v r| j         d         }n=d| j         v r| j         d         }n&d| j         v r| j         d         }nt          d          | j                            d	           | j                            |           | j                            | j         d
                    | j                            |           | j                            | j         d                    | j        	                    | j         d
         | j         d         z             | j        
                    |           | j                            |           | j                            | j         d                    | j                            | j                   | j                             d          pi }|                    d|                    d                    dk    rOd|v rM| j                            t           j        j                   | j                            |d                    d S d S d S Nr  r  r  r   r  r  r  r  r  r  r  r  rope_scaling	rope_typerz   linearfactorrG   r   r   rI   r  r  r  r  r  r~  r  r  r  r  r<   add_rope_scaling_typerf   RopeScalingTypeLINEARadd_rope_scaling_factorrq   rR   r  r  r  r  s         r/   r   z!BaichuanModel.set_gguf_parameters  N   l#67\"78
(()>
KK
 DL00&;<JJ&$,66&?@JJ4<//&89JJGHHH//0FGGG++J777--dl=.IJJJ((55500>Q1RSSS11$,}2MQUQ]^sQt2tuuu''
333**=999//^0LMMM&&tz222|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r   r   rb   r   r   r   r\   r   c                F   | j         d         }| j                             d|          }g }||d| dk    rt                              d|            |                     t
          j        j        |          |                     |d||          f|                     t
          j        j	        |          |                     |d||          f|                     t
          j        j
        |          |                     |d          fg}n|                     |          |fg}|S )	Nr  r  model.layers.z.self_attn.W_pack.weightzUnpacking and permuting layer r   r   r   )rG   r   rd   re   r   rf   r  ATTN_Q_reverse_hf_permute_partATTN_KATTN_V_reverse_hf_partr   )rq   r   rb   r   r  r  r  s          r/   r   zBaichuanModel.modify_tensors!  s*   \"78
(()>
KK,.?t'Ts'T'T'TTTKK>>>???(():)A3GG11*aZXXZ(():)A3GG11*a][[](():)A3GG))*a88:GG ,,T22J?@Gr.   Nweightsr  rQ   	n_kv_headc                    |||k    r||z  } |j         |d|j        d         |z  dz  g|j        dd          R                      dd                               |j                  S Nr   r   r   r  r  swapaxesrq   r  r  r  s       r/   _reverse_hf_permutez!BaichuanModel._reverse_hf_permute6  y     Vy%8%8y F GOFAw}Q'76'AQ'F[WXWYWYIZ[[[Xa^^WW]##	
r.   n_partr  c                v    |j         d         dz  }|                     |||z  ||z  |z   df         ||          S Nr   r    .)r  r  )rq   r  r  r  r  r   s         r/   r  z&BaichuanModel._reverse_hf_permute_part@  sJ     M!!''F
1v:>0I30N(OQWYbcccr.   c                L    |j         d         dz  }|||z  ||z  |z   df         S r  )r  )rq   r  r  r   s       r/   r  zBaichuanModel._reverse_hf_partF  s4    M!!q6z!f*q.0#566r.   r  r`   r  r   r  rQ   r  r   r\   r   )
r  r   r  rQ   r  rQ   r  r   r\   r   )r  r   r  rQ   r\   r   )r$   r%   r&   rf   r  BAICHUANrP   r  r   r   r  r  r  r-   r.   r/   r  r    s        )J( ( (M M M>   *
 
 
 
 
 RVd d d d d7 7 7 7 7 7r.   r  XverseForCausalLMc                  D    e Zd Zej        j        Zd Zd ZddZ	dddZ
dS )XverseModelc                4   | j         dz                                  sJ | j         }| j        }g }g }ddlm} |                    |          }|                    dt          |j                            }t          |
                                                                          }||k    rt          d          d |j                                        D             }	|                                }
t          |          D ]}|	|                             d          }|dk    r+t"          j        j        }d	| d
                    d          }nxt)          j        d|          rt"          j        j        }nQ|	|         |
v r6|j        |         j        rt"          j        j        }n#t"          j        j        }nt"          j        j        }|                    |           |                    |           | j                            d           | j                            d           | j                            |           | j                             |           t#          j!        |t          |                    }|"                    | j                   d S )Ntokenizer.jsonr   r  r  z.Vocabulary size exceeds expected maximum size.c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z)XverseModel.set_vocab.<locals>.<dictcomp>`  s    (j(j(j>Nk3k(j(j(jr.   r       <r  s   <0x[0-9A-Fa-f]{2}>r  r  r  )#r:   r   rG   r  r   ry  r   r   r  r  r  r  r   rl   r  r  r  rf   r  r,   r  	fullmatchr  r  r)   r*   r'   rn  rI   r  r  r  r  r  r  )rq   r:   rG   r  r  r   r  r  max_vocab_indexr  r  r  r  r  r  s                  r/   r  zXverseModel.set_vocabO  sv   !11::<<<<<N	,  ......!11)<<	[[s9?/C/CDD
 i1133::<<==j((MNNN(j(jR[RaRgRgRiRi(j(j(j//11j)) 	% 	%H&x077@@JW$$.-....55g>>

4jAA 0.-x(K771(;C :"n4GG"n9GG./MM*%%%OOG$$$$,,W555**9555''///((222))S[[III!!$"233333r.   c                   | j         d         }| j         d         }| j                             d|          }d}d| j         v r| j         d         }n=d| j         v r| j         d         }n&d| j         v r| j         d         }nt          d          | j                            d	           | j                            |           | j                            | j         d
                    | j                            |           | j                            | j         d                    | j        	                    | j         d
         | j         d         z             | j        
                    |           | j                            |           | j                            | j         d                    | j                            | j                   | j                             d          pi }|                    d|                    d                    dk    rOd|v rM| j                            t           j        j                   | j                            |d                    d S d S d S r  r  r  s         r/   r   zXverseModel.set_gguf_parameters~  r  r.   r   r   rb   r   r   r   r\   r   c                4   ~| j         d         }| j                             d|          }|                    d          r|                     |||          }|                    d          r|                     |||          }|                     |          |fgS Nr  r  q_proj.weightk_proj.weight)rG   r   r  r  r   )rq   r   rb   r   r  r  s         r/   r   zXverseModel.modify_tensors  s    \"78
(()>
KK ==)) 	V11*j*UUJ==)) 	Y11*j-XXJ%%d++Z899r.   Nr  r  rQ   r  c                    |||k    r||z  } |j         |d|j        d         |z  dz  g|j        dd          R                      dd                               |j                  S r  r  r  s       r/   r  zXverseModel._reverse_hf_permute  r  r.   r  r`   r  )r$   r%   r&   rf   r  XVERSErP   r  r   r   r  r-   r.   r/   r  r  K  sm        'J-4 -4 -4^M M M>: : : :
 
 
 
 
 
 
r.   r  FalconForCausalLMRWForCausalLMc                  4    e Zd Zej        j        Zd Zdd
ZdS )FalconModelc                >   | j                             d          }|| j         d         }| j                             d          }|| j         d         }| j                             d          }|| j                             dd          }| j                            d           | j                            d	           | j                            | j         d
                    | j                            d| j         d
         z             | j                            |           | j                            |           | j        	                    |           | j        
                    | j         d                    | j                            | j                   d S )Nr  r  r  r  num_kv_headsr  r      jploskir  r!   r  )rG   r   rI   r  r  r  r  r  r  r  r  r  r<   )rq   rR   r  r  s       r/   r   zFalconModel.set_gguf_parameters  su   l&&':;;,y1K!!"788>\(+FL$$^44	((a88I++D111//	:::--dl=.IJJJ00T\-5P1PQQQ((555''///**9555++DL9M,NOOO&&tz22222r.   r   r   rb   r   r   r   r\   r   c                D   ~d|v r|                      ddg          }|                      ddgd          pd}| j        d	         |z  }|                    |||z  d
z   |||z            }|d d d df                             ||z  ||z            }|d d dgf                             ||z  ||z            }	|d d dgf                             ||z  ||z            }
t	          j        ||	|
f                              |          }|                     |          |fgS )Nquery_key_valuer  r  r  r  Tr  r   r  r   r  )r   rG   viewr  r   r  
reshape_asr   )rq   r   rb   r   r  r  r  qkvqr   rI  s              r/   r   zFalconModel.modify_tensors  sP    $$%%'<h&GHHF((.+)FQU(VV[Z[I|M2f<H//)Vy-@1-DhPX[aPabbCAAAssF##FX$5x&7HIIAAAAtG$$Y%98f;LMMAAAAtG$$Y%98f;LMMAAq!9--88DDJ%%d++Z899r.   Nr  )	r$   r%   r&   rf   r  FALCONrP   r   r   r-   r.   r/   r  r    sA        'J3 3 3.: : : : : :r.   r  GPTBigCodeForCausalLMc                  ,    e Zd Zej        j        Zd ZdS )StarCoderModelc                >   | j         d         }| j                            | j         d                    | j                            | j         d                    | j                            d| j         d         z             | j                            |           | j                            | j         d                    | j                            d           | j                            | j         d                    | j        	                    | j
                   d S )Nr  r  r  r!   r  r   r  )rG   rI   r  r  r  r  r  r  r  r  r<   r  s     r/   r   z"StarCoderModel.set_gguf_parameters  s    l9-++DL,GHHH--dl8.DEEE00T\(5K1KLLL((555''X(>???**1---++DL9M,NOOO&&tz22222r.   N)r$   r%   r&   rf   r  	STARCODERrP   r   r-   r.   r/   r  r    s-        *J
3 
3 
3 
3 
3r.   r  GPTRefactForCausalLMc                  D     e Zd Zej        j        Z fdZd ZddZ	 xZ
S )RefactModelc                J   t                                                       t          j        | j        dg d          }|                    dd           |                    dd           |                    dd	           d |_        |                    | j                   d S )
NFr   r   middler  r  special_token_typesr   r   r   r    r  r   )	r  r  rf   r  r:   r  r  r  rI   )rq   r  r  s     r/   r  zRefactModel.set_vocab  s     )$.e@e@e@eg g g((1555((1555((1555&*#!!$"233333r.   c                   | j         d         }d|z  }t          d|z  dz            }d}|||z   dz
  |z  z  }| j         d         }| j                            | j         d                    | j                            | j         d                    | j                            |           | j                            |           | j                            | j         d	                    | j                            d           | j        	                    | j         d
                    | j        
                    | j                   d S )Nr  r!   r   r       r   r  r  r  r  )rG   rQ   rI   r  r  r  r  r  r  r  r  r<   )rq   r  	inner_dimmultiple_offf_dimrR   s         r/   r   zRefactModel.set_gguf_parameters  s6   \(+

N	Y*++
k!9A!=+ MNl9- 	++DL,GHHH--dl8.DEEE00888((555''X(>???**1---//=Q0RSSS&&tz22222r.   r   r   rb   r   r   r   r\   r   c                   | j         d         }d|z  }t          d|z  dz            }d}|||z   dz
  |z  z  }| j         d         }d}	| j         d         |z  }
g }|h|d| d	k    r|                    |                     t          j        j        |          |d |	|
z           f           |                    |                     t          j        j        |          ||	|
z  d          f           n|d| d
k    r;|                    |                     t          j        j        |          |f           n|d| dk    r|                    |                     t          j        j	        |          |d |         f           |                    |                     t          j        j
        |          ||d          f           t          |          dk    r*|                    |                     |          |f           |S )Nr  r!   r   r    r  r   r  ztransformer.h.z.attn.kv.weightz.attn.q.weightz.mlp.gate_up_proj.weightr   )rG   rQ   rn  r   rf   r  r  r  r  FFN_GATEFFN_UPr   r   )rq   r   rb   r   r  r  r  r  r  r  r  r  s               r/   r   zRefactModel.modify_tensors!  s$   \(+

N	Y*++
k!9A!=+ MNh'	<)V3,.?<<<<<< 7 78I8PRU V VXbcxdmpxdxcxXyz{{{ 7 78I8PRU V VXbclowcwcxcxXyz{{{{=#===== 7 78I8PRU V VXbcddddG#GGGGG 7 78I8RTW X XZdelflelZmnooo 7 78I8PRU V VXbcicjcjXklmmmw<<1NND0066
CDDDr.   r  )r$   r%   r&   rf   r  REFACTrP   r  r   r   r9  r:  s   @r/   r  r    se        'J
4 
4 
4 
4 
43 3 3(       r.   r  StableLmForCausalLMStableLMEpochForCausalLMLlavaStableLMEpochForCausalLMc                  r     e Zd ZU ej        j        Zd Zd ZdZ	de
d<   dZde
d<   ddZdddZ fdZ xZS )StableLMModelc                    | j         dz                                  r|                                  d S |                                  d S )Nr  )r:   r   r  r  r   s    r/   r  zStableLMModel.set_vocabA  sP    N--6688 	#  """""   """""r.   c                    | j         }|d         }| j                            |d                    | j                            |d                    | j                            |           | j                            |d                    |                     ddg          }| j                            t          ||d         |d         z  z                       | j        	                    |d                    | j        
                    |d                    | j                            d	|v r|d	         nd
           | j                            |                     ddg                     | j                            | j                   d S )Nr  r  r  r  partial_rotary_factorrope_pctr  r  r}  Tr  r  )rG   rI   r  r  r  r  r   r~  rQ   r  r  r  r  r  r<   )rq   rG   rR   rotary_factors       r/   r   z!StableLMModel.set_gguf_parametersH  s   ,12++G4M,NOOO--gm.DEEE((555009L1MNNN((*A:)NOO11#mw}G]ahi~aG  7A  3B  3B  	C  	C  	C''0E(FGGG**73H+IJJJ..SjnuSuSuw7N/O/O{  	A  	A  	A++D,<,<>NPZ=[,\,\]]]&&tz22222r.   Nlist[dict[str, Tensor]] | None_q_norms_k_normsr   r   rb   r   r   r   r\   r   c                   | j         d         }| j         d         }|                    d          dk    r|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |k    r#|                     ||| j        |         d          S g S |                    d          dk    r|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |k    r#|                     ||| j        |         d	          S g S |                     |          |fgS )
Nr  r  zq_layernorm.normsr  c                    g | ]}i S r-   r-   ra   r   s     r/   r  z0StableLMModel.modify_tensors.<locals>.<listcomp>c       E E E E E Er.   q_layernormzk_layernorm.normsc                    g | ]}i S r-   r-   r)  s     r/   r  z0StableLMModel.modify_tensors.<locals>.<listcomp>p  r*  r.   k_layernorm)	rG   findr%  r  rR   r   _stack_qk_normr&  r   rq   r   rb   r   r  r  s         r/   r   zStableLMModel.modify_tensors[  sl   34L!67	99())R//???}$ E EU43C-D-D E E E'1DM#t$4=%&&&00**3c8JMZZZ	99())R//???}$ E EU43C-D-D E E E'1DM#t$4=%&&)33**3	4=;M}]]]	%%d++Z899r.   r+  rQ   r  normsdict[str, Tensor]
layer_namec                    g }t          |          D ],}d| d| d| d}|                    ||                    ||= -t          j        |d          }d| d| d}	|                     |	          }
|
|fgS )Nr  z.self_attn.z.norms.r   r   r  )r  rn  r   stackr   )rq   r   r  r1  r3  datasxidenamer   merged_namer   s              r/   r/  zStableLMModel._stack_qk_norm{  s     == 	 	CSCSSJSSsSSSELLu&&&e[A...
IcIIjIII''44:&''r.   c                   t                                                       | j        | j        Z| j        d | j        D             ng | j        d | j        D             ng z   }t	          |          dk    rt          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   rk   ra   dr   s      r/   r  z1StableLMModel.prepare_tensors.<locals>.<listcomp>  -    <<<q16688<<a<<<<r.   c                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z1StableLMModel.prepare_tensors.<locals>.<listcomp>  r?  r.   r   zUnprocessed norms: )r  rK  r%  r&  r   r   )rq   r1  r  s     r/   rK  zStableLMModel.prepare_tensors  s    !!!=$(A AE@Y<<DM<<<<_a@D@Y<<DM<<<<_aE
 5zzA~~ !>u!>!>??? )B(A ~r.   r  )r+  )r   rQ   r  rQ   r1  r2  r3  r   )r$   r%   r&   rf   r  STABLELMrP   r  r   r%  r  r&  r   r/  rK  r9  r:  s   @r/   r  r  =  s         )J# # #3 3 3  04H3333/3H3333: : : :@( ( ( ( (@ @ @ @ @ @ @ @ @r.   r  LLaMAForCausalLMLlamaForCausalLMMistralForCausalLMMixtralForCausalLMVLlama3ForCausalLMLlavaForConditionalGenerationVoxtralForConditionalGeneration
LlamaModelc                       e Zd ZU ej        j        ZdZ fdZd Z	d Z
 fdZedd            ZdZded<   ddZddZ fdZ xZS )rI  Tc                     t                      j        |i | | j        dk    r%| j                            dd          | j        d<   d S d S )NrF  r      )r  r   r  rG   r   r  s      r/   r   zLlamaModel.__init__  s\    $)&)))<///26,2B2BCXZ\2]2]DL./// 0/r.   c                    t          | j                  }t                              d|j         d|j         d           | j                            |j                   g }g }g }|	                                D ]E\  }}}|
                    |           |
                    |           |
                    |           Ft          |          |j        k    s#J dt          |           d|j         d            |j        t          j        k    rF| j                            d           | j                            |                                           t                              d|j         d	|j         d	|j         d	|j         d	           | j                            |j                   | j                            |j                   | j                            |j                   | j                            |j                   | j                            |           | j                            |           | j                            |           | j                            |j                   | j                            d
           | j                            d           t?          t@                    j!        dz  }tD          #                    ||          }	| j        $                    |	           d S )NzConverting tokenizer z	 of size r   ztoken count (z) != vocab size (rv   rY  z+Setting bos, eos, unk and pad token IDs to r  TFzmodels/templates/)%r   r:   rd   re   tokenizer_typer  rI   r  gguf_tokenizer_modelr  rn  r   r   rY  r  r  extract_vocab_merges_from_modelbos_ideos_idunk_idpad_idr  r  r  r  r  r  r  add_vocab_sizer   r"  r   r  r  MistralModelget_community_chat_templateadd_chat_template)
rq   r  r  r  r  r  r  r  template_dirr  s
             r/   _set_vocab_mistralzLlamaModel._set_vocab_mistral  s   T^,,VE$8VV5CSVVV	
 	
 	
 	,,U-GHHH$)$4$4$6$6 	% 	% D%MM$MM%   OOG$$$$6{{e....MCKKMM%:JMMM /.. #7#>>>..x888--5577   	y%,yy%,yyZ_Zfyyjojvyyy	
 	
 	
 	))%,777))%,777))%,777))%,777''///))&111((222''(8999**4000**5111H~~,/BB;;E<PP**844444r.   c                ~   | j         r|                                 S | j        dz  }| j        dz  }|                                r(|                                s|                                  	 |                                  nQ# t
          $ rD 	 |                                  n+# t
          t          f$ r |                                  Y nw xY wY nw xY w| j	        
                    dd          dk    rt          j        | j        dg d          }|                    d	d
           |                    dd           |                    dd           |                    dd           |                    | j                   | j        dz  }|                                rbt!          |dd          5 }t#          j        |          }d|v r | j                            |d                    d d d            n# 1 swxY w Y   | j	        
                    dd          dk    r| j                            d           d S d S )Nztekken.jsonr  r  i }  i}  Fr  r  r   i}  r   i}  r  i	}  r  i
}  r  r   r   r   add_prefix_spacei   )rT   rZ  r:   r   r  r  r  r}   r  rG   r   rf   r  r  r  rI   r   r   r   add_add_space_prefixr   )rq   path_tekken_jsonpath_tokenizer_jsonr  r  r   r  s          r/   r  zLlamaModel.set_vocab  s   ! 	-**,,,>M9"n/??##%% 	&.A.I.I.K.K 	&##%%%	'))++++  	' 	' 	''((****%y1 ' ' '$$&&&&&'	' <L%00E99 -E&K&K&K  M ,,Xu===,,Xu===,,Xu===,,Uu===%%d&6777 $1H H ((** 	e+S7CCC eq(,	!%%)>>>$99:OPb:cddde e e e e e e e e e e e e e e <L%00E99..u55555 :9sH   -B 
CB"!C"%C
C	C

CC29G77G;>G;c                \   t                                                       | j        }| j        s | j                            |d                    |                    d          x}|d         |d         z  }| j                            |           | j                            d          pi }|                    d|                    d                    dk    rOd	|v rM| j                            t          j
        j                   | j                            |d	                    d S d S d S 
Nr  r  r  r  r  r  rz   r  r  )r  r   rG   rT   rI   rU  r   r~  r  rf   r  r  r  rq   rG   rope_dimr  r  s       r/   r   zLlamaModel.set_gguf_parameters  s(   ##%%%,% 	C++GL,ABBBJ///H8}-9N1OOH11(;;;|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r  r   r  rQ   r  r   c                    |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R                      dd                               | j                  S r  r  r  r  r  s      r/   permutezLlamaModel.permute  r     Vy%8%8F7=+;v+E+J_W][\[][]M^___!Q''	)r.   Nr$  _expertsr   rb   r   r   r\   r   c           	         |                      ddg          }|                      ddg          }g d}dv p*dv p&dv p"d	v pd
v pt          fd|D                       }|rg S | j        dk    rdz   nW                    d          r                    dd          n+                    d          r                    dd          | j        rb                    d          rt                              |||          }                    d          rt                              |||          }	                    d          dk    r| j
        d         }|J | j        #d t          | j                  D             | _        || j        |         <   t          | j        |                   |dz  k    rg }	dD ]}
g }t          |          D ]B}d| d| d|
 d}|                    | j        |         |                    | j        |         |= Ct!          j        |d          }d | d!|
 d}|                     |          }|	                    ||f           |	S g S |                               |fgS )"Nr  r  r  r  )vision_encoder.vision_language_adapter.patch_merger.pre_mm_projector_normvision_towervision_modelaudio_towermodel.connectormulti_modal_projectorc              3  B   K   | ]}                     |          V  d S r`   rm  ra   r   rb   s     r/   rc   z,LlamaModel.modify_tensors.<locals>.<genexpr>.  sC         ''     r.   rI  model.zmodel.text_modelztext_model.r  language_model.r  zq_proj.biasr  zk_proj.biasblock_sparse_moe.expertsr  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z-LlamaModel.modify_tensors.<locals>.<listcomp>I  r*  r.   r    w1w2w3r  .block_sparse_moe.experts.r   r   r   r  layers..feed_forward.experts.)r   r  r  rm  r  undo_permuter  rI  rf  r.  rG   rh  r  rR   r   rn  r   r5  r   )rq   r   rb   r   r  r  vision_prefixesis_multimodal_tensorr  r  widr6  r7  r8  r9  r   s     `             r/   r   zLlamaModel.modify_tensors  s0   !!9.C"DEE$$l4I%JKK	
 
 
  .5  % $  !D(  '$.	 
     -     	   	7I\\))d?DD__/00 	7<<r22DD__.// 	7<< 1266D 	O}}=>> L'//
FFKK
}}=>> O'//
FINN
 99/00B66%89I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 . ; ;C*,E$Y// 6 6 a a as a aUX a a aT]3%7%>??? M#.u55!&U!:!:!:J"SC"S"Ss"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c              #    K   |                      dgd          x}r|                    dd                                          dk    r| j                            dd          }| j                            d	          x}| j        d
         | j        d         z  }d|t	          j        d|dt          j                  |z  z  z  }|                    dd          }|                    dd          }|                    dd          }| j                            dd          }||z  }	||z  }
g }|D ]{}dt          j        z  |z  }||
k     r|	                    d           0||	k    r|	                    |           L||z  |z
  ||z
  z  }|	                    dd|z
  |z  |z   z             || 
                    t          j        j                  t	          j        |t          j                  fV  d S d S d S Nr  Tr  r  r  llama3r       @r  r  r        ?r   r   r   r         @low_freq_factorhigh_freq_factor      @ original_max_position_embeddings    r   r   r   lowerrG   r   aranger  mathpirn  r   rf   r  
ROPE_FREQStensorrq   r  baser  freqsr  r  r  old_context_lenlow_freq_wavelenhigh_freq_wavelenrope_factorsfreqwavelensmooths                  r/   r   z!LlamaModel.generate_extra_tensorsf  s<     ++^,<t+LLL< 	R006688HDD|''g>><++J777C@,}5F[9\\CtQQem(T(T(TWZ(Z[\%))(C88"."2"23Dc"J"J#/#3#34F#L#L "&,"2"23UW["\"\#2_#D $36F$F!  "! R RD$'kD0G!222$++A.... #333$++F3333"1G";o"MRbetRt!u$++A!f*1F1O,PQQQQ..t/@/KLLel[goto|N}N}N}~~~~~~7	 	DDr.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z.LlamaModel.prepare_tensors.<locals>.<listcomp>  -    BBBQBBAqBBBBr.   r   Unprocessed experts: r  rK  rh  r   r   rq   expertsr  s     r/   rK  zLlamaModel.prepare_tensors  n    !!!=$BB$-BBBG7||a !B!B!BCCC	 %$  r.   r  r   r  rQ   r  r   r  r  )r$   r%   r&   rf   r  LLAMArP   r  r   rZ  r  r   r  rf  rh  r  r   r   rK  r9  r:  s   @r/   rI  rI    s         &JL^ ^ ^ ^ ^/5 /5 /5b'6 '6 '6RM M M M M  ) ) ) \) 04H3333F: F: F: F:P   <D D D D D D D D Dr.   ArceeForCausalLMc                  6     e Zd Zej        j        Z fdZ xZS )
ArceeModelc                   t                                                       |                                  | j                            d          pi }|                    d|                    d                    dk    rod|v rm| j                            t          j        j	                   | j        
                    |d                    | j                            |d                    d S d S d S Nr  r  rz   yarnr  r  r  r   r5  rG   r   rI   r  rf   r  YARNr  add_rope_scaling_orig_ctx_lenrq   r  r  s     r/   r   zArceeModel.set_gguf_parameters      ##%%%""$$$|''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;klllll MLQiQir.   )	r$   r%   r&   rf   r  ARCEErP   r   r9  r:  s   @r/   r  r    sK        &Jm m m m m m m m mr.   r   Mistral3ForConditionalGenerationc                  <     e Zd ZdZ fdZddZ fdZddZ xZS )LlavaVisionModelr  c                *    t                      j        |i | | j                            d          dk    r>| j                            dd          | j        d<   |                     d          | _        nz| j        rV| j                            dd           | j        d<   | j        d         
J d            |                     dg          | _        nt          d	| j        d                    t                              d
| j                    d S )Nr  ru  r  r  z[IMG_BREAK]r  z!norm_eps not found in params.jsonimage_break_token_idzUnsupported model type: zImage break token id: )r  r   rG   r   get_token_idimg_break_tok_idrT   r=  r`  r   rd   re   r  s      r/   r   zLlavaVisionModel.__init__  s   $)&)))<L))Y66-1\-=-=>NPT-U-UDL)*$($5$5m$D$DD!!# 	V'+'9'='=j$'O'ODL$<
+779\777$($4$46L5M$N$ND!!T\8RTTUUUDT-BDDEEEEEr.   r  r   r\   rQ   c                L   | j         dz  }t          |dd          5 }t          j        |          d         }|                                D ].\  }}|d         |k    rt          |          c cd d d            S /	 d d d            n# 1 swxY w Y   t          d| d          )	Nr  r   r   r   r  r  zToken 'z ' not found in tokenizer config.)r:   r   r   r   rl   rQ   r   )rq   r  r  r   r  r  r  s          r/   r  zLlavaVisionModel.get_token_id  s    $1H H'w??? 	$1#'9Q<<0F#G #7#=#=#?#? $ $Zi(E11s88OO		$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 2$	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$
 J5JJJKKKs   AB:BBBc                J   t                                                       | j        }|                    d          dk    r| j                            t          j        j                   | j        	                    |d                    |d         dk    r| j        
                    d           n?|d         dk    r| j                            d           nt          d|d                    d	| j        v r)| j                            | j        d	                    d S d S d S )
Nr  ru  r  
hidden_actsiluTgeluUnsupported hidden_act: spatial_merge_size)r  r   rG   r   rI   add_clip_projector_typerf   VisionProjectorTypePIXTRAL"add_vision_attention_layernorm_epsadd_vision_use_siluadd_vision_use_gelur   r=  add_vision_spatial_merge_sizerq   rG   r  s     r/   r   z$LlavaVisionModel.set_gguf_parameters  s)   ##%%%,;;|$$	1144T5M5UVVV??HX@YZZZ |$.. 44T::::&&00 44T:::: !SGL<Q!S!STTT $t'999 >>t?QRf?ghhhhh 21 :9r.   r   r   rb   r   r   r   c                   ~| j         s| j        d         n|                     dg          }|}d}t          fd|D                       r                    d          r#| j         st
                              |||          }                    d          r#| j         st
                              |||          }|                               |fgS | j         sdnd}| j        dk    rg|v rct          
                    d	            || j                 }t          j        t          j        j                 |                               |fgS g S )
Nr  )multi_modal_projector.vision_tower.rj  rk  rl  rm  c              3  B   K   | ]}                     |          V  d S r`   rt  ru  s     r/   rc   z2LlavaVisionModel.modify_tensors.<locals>.<genexpr>  s/      DD6tv&&DDDDDDr.   rx  ry  embed_tokens.weightztok_embeddings.weightr   z,Extracting [IMG_BREAK] token embedding from )rT   rG   r`  r  r  rI  rf  r   r  rd   re   rf   r   r  V_TOK_EMBD_IMG_BREAK)	rq   r   rb   r   r  r  valid_prefixes	embed_keyimg_break_embds	     `      r/   r   zLlavaVisionModel.modify_tensors  s   7;7M|DL.//SWScScezd{S|S| 	 	
 DDDD^DDDDD 	>}}=>> LtG] L'//
FFKK
}}=>> OtG] O'//
FINN
))$//<==151Gd))Md	 1$$d):):KKMtMMNNN'(=>N$T%6%KLD))$//@AA	r.   )r  r   r\   rQ   r  )	r$   r%   r&   r  r   r  r   r   r9  r:  s   @r/   r  r    s        
 F F F F FL L L Li i i i i&               r.   r   Idefics3ForConditionalGenerationSmolVLMForConditionalGenerationc                  6     e Zd Z fdZ fdZd ZddZ xZS )SmolVLMModelc                2    t                      j        |i | | j        d         dk    rk| j                            dd          | j        d<   | j                            dd          | j        d<   | j                            dd          | j        d<   d S d S )	Nr  smolvlm_visionr  i  r     r  i   )r  r   rG   r   r  s      r/   r   zSmolVLMModel.__init__  s    $)&)))<%)999 +/,*:*:=$*O*ODL'26,2B2BCXZ\2]2]DL./040@0@ATVZ0[0[DL,--- :9r.   c                   t                                                       | j                            t          j        j                   | j                            | j        	                    dd                     | j        
                    | j        	                    dd                     | j                            d           d S )Nr  r  scale_factorr   T)r  r   rI   r  rf   r  IDEFICS3r  rG   r   !add_vision_projector_scale_factorr=  r  rq   r  s    r/   r   z SmolVLMModel.set_gguf_parameters  s    ##%%%001I1RSSS;;DL<L<LM]_c<d<deee::4;M;Q;QR`bc;d;deee,,T22222r.   c                6    ~~~d|v rt           j        j        S dS )N.embeddings.F)rf   r  r  r   s        r/   r   zSmolVLMModel.tensor_force_quant	  s%    6T!!,00ur.   r   r   rb   r   r   r   r\   r   c                T    ~d|v pd|v pd|v }|r|                      |          |fgS g S )Nrn  ro  rq  r   )rq   r   rb   r   is_vision_tensors        r/   r   zSmolVLMModel.modify_tensors		  sO    )T1h^t5KhO`dhOh 	>))$//<==	r.   r  r$   r%   r&   r   r   r   r   r9  r:  s   @r/   r  r    sy        \ \ \ \ \3 3 3 3 3         r.   r  Llama4ForConditionalGenerationc                  V     e Zd Zej        j        ZdZ fdZd Z	 fdZ
d fdZ xZS )Llama4ModelFc                     t                      j        |i | | j        d         | j        d<   | j        d         | j        d<   d S )Nr  intermediate_size_moeintermediate_size_mlpr  r   rG   r  s      r/   r   zLlama4Model.__init__	  sO    $)&)))04=P0Q,-,0L9P,Q()))r.   c                .    |                                   d S r`   r  r   s    r/   r  zLlama4Model.set_vocab	  r  r.   c                    t                                                       | j                            | j        d                    | j                            | j        d                    d S )Ninterleave_moe_layer_stepr  )r  r   rI   add_interleave_moe_layer_steprG   add_expert_feed_forward_lengthr  s    r/   r   zLlama4Model.set_gguf_parameters!	  s]    ##%%%66t|D_7`aaa77E\8]^^^^^r.   r   r   rb   r   r   r   c                L   |                     d          r|                    dd          }d|v r|                    dd          }|                    dd          }|j        d         dz  }|                    dd                              |d	          \  }}|                     |          |f|                     |          |fgS |                    d
          r|dz  }|                    dd          }d|v sd|v rg S t                                          |||          S )Nrw  r  gate_up_projup_proj.weightgate_proj.weightr  r   r  r  	down_projr   rr  ro  )	rm  r  r  	transposer  r   r  r  r   )
rq   r   rb   r   name_up	name_gatedim_halfgate_proj_weightup_proj_weightr  s
            r/   r   zLlama4Model.modify_tensors&	  sE   ??,-- 	7<< 1266D T!!ll>3CDDG^5GHHI!'+q0H/9/C/CB/K/K/Q/QRZ`b/Q/c/c,n%%i002BC%%g..? 
 ==%% 	6ID#--b"55J"d**n.D.DIww%%j$<<<r.   r   r   rb   r   r   r   )r$   r%   r&   rf   r  LLAMA4rP   r  r   r  r   r   r9  r:  s   @r/   r  r  	  s        'JLR R R R R  _ _ _ _ _
= = = = = = = = = =r.   r  c                  &     e Zd Z fdZdd
Z xZS )Llama4VisionModelc                   t                                                       | j                            t          j        j                   | j                            | j        d                    | j        	                    t          d| j        d         z                       | j        d         dk    sJ | j                            d           d S )Nr  r  pixel_shuffle_ratior  r  T)r  r   rI   r  rf   r  r  r  rG   r  rQ   r  r  s    r/   r   z%Llama4VisionModel.set_gguf_parameters@	  s    ##%%%001I1PQQQ;;DL<TUUU::3sT\RgEh?h;i;ijjj|L)V3333,,T22222r.   r   r   rb   r   r   r   r\   r   c                    ~d|v sd|v rPd|v r	d|vr|dz  }d|v r't           j        t           j        j                 dz   |fgS |                     |          |fgS g S )Nrr  ro  positional_embedding_vlmr   zmulti_modal_projector.linear_1)rf   r   r  V_MMPROJ_FCr   r   s       r/   r   z Llama4VisionModel.modify_tensorsH	  s    "d**n.D.D)T11it6K6K	!/477*4+<+HIIUWabcc))$//<==	r.   r  )r$   r%   r&   r   r   r9  r:  s   @r/   r  r  >	  sL        3 3 3 3 3
 
 
 
 
 
 
 
r.   r  c                  8     e Zd Zej        j        Zd fdZ xZS )	Mistral3Modelr   r   rb   r   r   r   c                    |                     dd          }d|v sd|v rg S t                                          |||          S )Nrw  r  rr  rn  )r  r  r   rq   r   rb   r   r  s       r/   r   zMistral3Model.modify_tensorsY	  sM    ||-r22"d**n.D.DIww%%j$<<<r.   r  )	r$   r%   r&   rf   r  r  rP   r   r9  r:  s   @r/   r  r  U	  sG        &J= = = = = = = = = =r.   r  DeciLMForCausalLMc                       e Zd Zej        j        Zedd            Zedd	            Z	 fd
Z
d Z fdZedd            ZddZddZ fdZ xZS ) 	DeciModelffn_multfloatr  rQ   r\   c                h    t          d| z  |z  dz            }t                              |d          S )Nr   r    r  )rQ   r  _find_multiple)r  r  r  s      r/   _ffn_mult_to_intermediate_sizez(DeciModel._ffn_mult_to_intermediate_sized	  s7      Hv 5 9::''(93???r.   r   r   c                .    | |z  dk    r| S | |z   | |z  z
  S Nr   r-   )r   r   s     r/   r  zDeciModel._find_multiplej	  s(     q5A::H1uAr.   c                     t                      j        |i | d j        v r j        d         } j        t	          |          k    sJ t                       _        t                       _        t                      }t          t	          |                    D ]S}||         d         d         ||         d         d         du r@ j        	                    d            j        	                     j        d                    n j        	                    d            j        	                    d           n_ j        	                     j        d         ||         d         d         z              j        	                     j        d                    ||         d         d	         |	                    d
           ,|	                    ||         d         d	                    U j        t	           j                  k    sJ  j        t	           j                  k    sJ  j        t	          |          k    sJ t           j        t
                    r t           j        d         t                    sJ t           j        t
                    r t           j        d         t                    sJ t          |t
                    rt          |d         t                    sJ  fd|D              _        d S d S )Nblock_configs	attentionn_heads_in_groupreplace_with_linearTr   r  ffnr  r  c                \    g | ](}t                               |j        d                    )S )r  )r  r  rG   )ra   
multiplierrq   s     r/   r  z&DeciModel.__init__.<locals>.<listcomp>	  sA     ) ) ) 88T\R_E`aa) ) )r.   )r  r   rG   rR   r   list_num_kv_heads
_num_headsr  rn  r   rQ   r  	_ffn_dims)rq   r  r  _block_configs_ffn_multipliersilr  s   `     r/   r   zDeciModel.__init__q	  s   $)&)))dl**26,2ON#s>':':::::!%D"ffDO#vv C//00 S S!"%k23EFN%b)+67LMQUUU*11!444..t|<Q/RSSSS*11!444..q1111&--dl;P.QUcdfUghsUt  vH  VI  /I  J  J  JO**4<8M+NOOO!"%e,Z8@$++C0000$++N2,>u,Ej,QRRRR#s4+='>'>>>>>#s4?';';;;;;#s+;'<'<<<<<d0$77bJtGYZ[G\^a<b<bbbbdot44\DOTUDVX[9\9\\\\.55`*EUVWEXZ_:`:````) ) ) )"2) ) )DNNNQ +*r.   c                   | j                             dd          dk    r|                                 \  }}}| j                            d           | j                            |           | j                            |           | j                            |           t          j	        | j
        d          }|                    | j                   d S |                                  d S )Nr  i  r  Tr  )rG   r   r!  rI   r  r  r  r  rf   r  r:   r  r  r  s        r/   r  zDeciModel.set_vocab	  s     <L&11V;;'+':':'<'<$FHf00888..v666++F333,,X666 -dn$OOOM%%d&677777 $$&&&&&r.   c                   d| j         v r| j        t          | j                  k    sJ | j        t          | j                  k    sJ | j        t          | j                  k    sJ | j                             d          x}| j                            |           | j        	                    | j                   | j        
                    | j                   | j                            | j                   | j                            | j                   | j                            | j         d                    | j                            | j         d                    | j                            | j         d                    | j                            | j         d         | j         d         z             | j                            | j         d         | j         d         z             | j                            | j                   nyt)                                                       d| j         v rP| j         d         | _        | j        t          | j                  k    sJ | j        	                    | j                   | j         }| j                            |d                    |                    d	          x}|d         |d         z  }| j                            |           | j                             d
          pi }|                    d|                    d                    dk    rOd|v rM| j                            t2          j        j                   | j                            |d                    d S d S d S )Nr  r  r  r  r  r  num_key_value_heads_per_layerr  r  r  r  rz   r  r  )rG   rR   r   r!  r"  r#  r   rI   r  r  r  r  r  r  r  r  r  r  r  r<   r  r   rU  r~  r  rf   r  r  r  )rq   r  rG   rc  r  r  s        r/   r   zDeciModel.set_gguf_parameters	  s?   dl**#s4+='>'>>>>>#s4?';';;;;;#s4>':':::::"l..|<<<
I 33J???..t/ABBB++DO<<<44T^DDD,,T-=>>>//=V0WXXX11$,}2MNNN33DL4PQQQ++DL,G4<XmKn,nooo--dl=.IT\ZoMp.pqqq**4:6666GG''))).$,>>04=\0]"'3t/A+B+BBBBB 2243EFFF,''(=>>>J///H8}-9N1OOH11(;;;|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r  r   r  r  r   c                    |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R                      dd                               | j                  S r  r  re  s      r/   rf  zDeciModel.permute	  rg  r.   r   rb   r   r   r   c                    | j         d         }|\d| j         v r| j         d         |         }nYd| j         v r| j        |         }| j        |         }n5| j                             d          }n| j                             d          }|                    d          rt
                              |||          }|                    d          rt
                              |||          }|                     |          |fgS )Nr  r)  r  r  rx  ry  )rG   r!  r"  r   r  r  rf  r   r0  s         r/   r   zDeciModel.modify_tensors	  s   34?.$,>> L)HI#N		 DL00 .s3	- L,,-BCC		(()>??I==9:: 	G"**:vvFFJ==9:: 	J"**:vyIIJ%%d++Z899r.   c              #    K   |                      dgd          x}r|                    dd                                          dk    r| j                            dd          }| j                            d	          x}| j        d
         | j        d         z  }d|t	          j        d|dt          j                  |z  z  z  }|                    dd          }|                    dd          }|                    dd          }| j                            dd          }||z  }	||z  }
|	|
k    sJ g }|D ]{}dt          j        z  |z  }||
k     r|	                    d           0||	k    r|	                    |           L||z  |z
  ||z
  z  }|	                    dd|z
  |z  |z   z             || 
                    t          j        j                  t	          j        |t          j                  fV  d S d S d S r  r  r  s                  r/   r   z DeciModel.generate_extra_tensors	  L     ++^,<t+LLL< 	R006688HDD|''g>><++J777C@,}5F[9\\CtQQem(T(T(TWZ(Z[\%))(C88"."2"23Dc"J"J#/#3#34F#L#L "&,"2"23UW["\"\#2_#D $36F$F!'+<<<<<!! R RD$'kD0G!222$++A.... #333$++F3333"1G";o"MRbetRt!u$++A!f*1F1O,PQQQQ..t/@/KLLel[goto|N}N}N}~~~~~~7	 	DDr.   c                H    t                                                       d S r`   r  rK  r  s    r/   rK  zDeciModel.prepare_tensors
      !!!!!r.   )r  r  r  rQ   r\   rQ   )r   rQ   r   rQ   r\   rQ   r  r  r  )r$   r%   r&   rf   r  DECIrP   r  r  r  r   r  r   rf  r   r   rK  r9  r:  s   @r/   r  r  `	  s       %J@ @ @ \@
    \. . . . .`' ' ' !M !M !M !M !MF ) ) ) \): : : :&   <" " " " " " " " "r.   r  BitnetForCausalLMc                  L     e Zd Zej        j        Zd Z fdZddZ	ddZ
 xZS )BitnetModelc                .    |                                   d S r`   r  r   s    r/   r  zBitnetModel.set_vocab
  r  r.   c                    t                                                       | j                            t          j        j                   | j                            d           d S )Nr  )r  r   rI   r  rf   r  r  r  r  s    r/   r   zBitnetModel.set_gguf_parameters
  sS    ##%%%..t/C/JKKK0055555r.   weightr   r\   c                <   |j         }|                                }|                                                                                    d          }d|z  }||z                                                      dd          |z  }|                    |          S )Nr  )minr   r  )r   r  absmeanclamproundrz   )rq   r7  r   scaleiscaleresults         r/   weight_quantzBitnetModel.weight_quant
  s    

!!##))d)33U 6/((**00Q77&@{{5!!!r.   r   rb   r   r   r   r   c           
   #     K                         |          t           fdt          j        j        t          j        j        t          j        j        t          j        j        t          j        j        t          j        j	        t          j        j
        fD                       r                     |          }|fV  d S )Nc              3  F   K   | ]}                     |          V  d S r`   r  r  s     r/   rc   z-BitnetModel.modify_tensors.<locals>.<genexpr>+
  sE        Ct++Hc3??      r.   )r   r  rf   r  r  r  r  ATTN_OUTr  FFN_DOWNr  rA  r  s   `  `@r/   r   zBitnetModel.modify_tensors(
  s      ''--      $$$&$&&L
      
	7 **:66J$$$$$$r.   )r7  r   r\   r   r  )r$   r%   r&   rf   r  BITNETrP   r  r   rA  r   r9  r:  s   @r/   r4  r4  
  sy        'J( ( (6 6 6 6 6
	" 	" 	" 	"% % % % % % % %r.   r4  GrokForCausalLMc                  ^     e Zd ZU ej        j        Zd Z fdZ fdZ	dZ
ded<   ddZ xZS )	GrokModelc                .    |                                   d S r`   r  r   s    r/   r  zGrokModel.set_vocab>
  r  r.   c                :     t                      j        |i | d S r`   )r  r   r  s      r/   r   zGrokModel.__init__A
  s%    $)&)))))r.   c                H    t                                                       d S r`   r  r   r  s    r/   r   zGrokModel.set_gguf_parametersD
      ##%%%%%r.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        |                     d          dk    r| j        d         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t          |          D ]B}d| d| d| d	}	|                    | j        |         |	                    | j        |         |	= Ct          j        |d
          }d| d| d	}
| 	                    |
          }|                    ||f           |S g S | 	                    |          |fgS )N.moe.r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z,GrokModel.modify_tensors.<locals>.<listcomp>Q
  r*  r.   r    )r  linear_1linear_vztransformer.decoder_layer.r   r   r   r  
r.  rG   rh  r  rR   r   rn  r   r5  r   )rq   r   rb   r   r  r  r  r6  r7  r8  r9  r   s               r/   r   zGrokModel.modify_tensorsI
  s   99W##%89I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 > ; ;C*,E$Y// 6 6 YS Y Ys Y YS Y Y YT]3%7%>??? M#.u55!&U!:!:!:J"Us"U"U"U"U"UK#33K@@HNNHj#9::::	%%d++Z899r.   r  )r$   r%   r&   rf   r  GROKrP   r  r   r   rh  r  r   r9  r:  s   @r/   rI  rI  :
  s         %J( ( (* * * * *& & & & & 04H3333#: #: #: #: #: #: #: #:r.   rI  DbrxForCausalLMc                  <    e Zd Zej        j        Zd Zdd
ZddZ	dS )	DbrxModelc                   | j         d         }| j         d         }| j                            | j         d                    | j                            | j         d                    | j                            | j         d                    | j                            |d                    | j                            | j         d                    | j                            |d                    | j                            |d	                    | j        	                    |d
                    | j        
                    |d                    | j                            |d                    | j                            d           | j                            | j                   t                              d| j                    d S )N
ffn_configr  r  r  r  ffn_hidden_sizer  r  r  r  moe_num_experts	moe_top_kr  r  )rG   rI   r  r  r  r  r  r  r  r  r  r  r  r  r<   rd   re   )rq   rZ  r  s      r/   r   zDbrxModel.set_gguf_parameterss
  s   \,/
l=1((j)ABBB++DL,GHHH--dl9.EFFF00<M1NOOO''Y(?@@@**;|+DEEE++K,EFFF&&{:'>???))*5F*GHHH..z+/FGGG++D111&&tz22255566666r.   r   r   rb   r   r   r   r\   r   c                   ~| j         d         d         }| j         d         d         }| j         d         }d dd d}d}|                                D ]c}	|                    |	          dk    rH|                    d	          dk    r/d
}|                    |||          }||	         x}

 |j        |
 } nd|                     |s|n|d	z   d          }||fgS )NrZ  r\  r[  r  )r   r   r   )zffn.experts.mlp.w1zffn.experts.mlp.w2zffn.experts.mlp.v1Fr  r   Tr  r  )rG   rk   r.  r  rf  r   )rq   r   rb   r   n_expertr  r  exp_tensor_namesr  exp_tensor_namepermute_tensorr   s               r/   r   zDbrxModel.modify_tensors
  s   <-.?@|L)*;<i( 372;268 8 /4466 	 	Oyy))R//DIIi4H4HB4N4N'__XtVDD
&6&GGNT!3!3^!DJ ''G(Q	AQ`l'mm:&''r.   r   r   rQ   r   c                    ~~~|dk    S )Nr   r-   r   s        r/   r   zDbrxModel.tensor_force_quant
  s    (Czr.   Nr  r  )
r$   r%   r&   rf   r  DBRXrP   r   r   r   r-   r.   r/   rX  rX  o
  sV        %J7 7 70!( !( !( !(F     r.   rX  MiniCPMForCausalLMc                  L     e Zd Zej        j        Z fdZddZd Z	ddZ
 xZS )MiniCPMModelc                x   t                                                       t          | j        d                   }| j                            |           t                              d|            | j        d         | j        d         dz  z  }| j                            |           t                              d|            | j        d         | j        d         z  }| j        	                    |           t                              d	|            | j        
                    d
          pi }|
                    d|
                    d                    dk    rW| j                            t          j        j                   t                              dt          j        j                    d S d S )N	scale_embz"gguf: (minicpm) embedding_scale = scale_depthr        ?z!gguf: (minicpm) residual_scale = r  dim_model_basezgguf: (minicpm) logit_scale = r  r  rz   longropez$gguf: (minicpm) rope_scaling_type = )r  r   r  rG   rI   add_embedding_scalerd   re   add_residual_scaleadd_logit_scaler   r  rf   r  LONGROPE)rq   embedding_scaleresidual_scalelogit_scaler  r  s        r/   r   z MiniCPMModel.set_gguf_parameters
  s   ##%%%[ 9::,,_===JJJKKKm4t|DW7X\_7__++N;;;HHHIIIl=1DLAQ4RR((555B[BBCCC|''77=2K)9)9&)A)ABBjPP2243G3PQQQKK^t?S?\^^_____ QPr.   r\   r   c              #    K   | j         d         | j         d         z  }|                     dgd          }||                    dd           }|                    dd           }||t          d          t	          |          t	          |          k    st	          |          |dz  k    rt          d	|dz             |                     t          j        j	                  t          j        |t          j        
          fV  |                     t          j        j                  t          j        |t          j        
          fV  d S d S )Nr  r  r  Tlong_factorshort_factorNMissing the required key rope_scaling.long_factor or rope_scaling_short_factorr   2The length of rope long and short factors must be r  )rG   r   r   r   r   r   r   rf   r  ROPE_FACTORS_LONGr   r  r  ROPE_FACTORS_SHORT)rq   	rope_dimsr  long_factorsshort_factorss        r/   r   z#MiniCPMModel.generate_extra_tensors
  s     L/4<@U3VV	''(8$??#'++M4@@L(,,^TBBM#}'<oppp<  C$6$666#l:K:Ky[\}:\:\ !eV_bcVc!e!efff**4+<+NOOQVQ]^jrwr  RA  RA  RA  B  B  B  B**4+<+OPPRWR^_lty  uB  SC  SC  SC  D  D  D  D  D  D $#r.   c                .    |                                   d S r`   r  r   s    r/   r  zMiniCPMModel.set_vocab
  r  r.   r   r   rb   r   r   r   c                F   ~| j         d         }| j                             d          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                     |          |fgS r  rG   r   r  rI  rf  r   r0  s         r/   r   zMiniCPMModel.modify_tensors
  s    34L$$%:;;	 ==/++ 	H#++JGGJ==/++ 	K#++J	JJJ%%d++Z899r.   r  r  )r$   r%   r&   rf   r  MINICPMrP   r   r   r  r   r9  r:  s   @r/   rg  rg  
  s        (J` ` ` ` ` D D D D"( ( (: : : : : : : :r.   rg  MiniCPM3ForCausalLMc                  D    e Zd Zej        j        Zd ZddZd Z	dddZ
dS )MiniCPM3Modelc                z   | j         }| j                            | j                   | j                            |d                    | j                            |d                    | j                            | j                   | j                            |d                    | j        	                    |d                    | j        
                    |d                    | j                            |d                    | j                            |d                    d|v r(|d          | j                            |d                    | j                            |d	                    | j                            |d
         |d         z              | j                            |d                    d S )Nr  r  r  r  r  r  r  q_lora_rankkv_lora_rankqk_nope_head_dimqk_rope_head_dim)rG   rI   r  r<   r  r  r  rR   r  r  r  r  rU  add_q_lora_rankadd_kv_lora_rankr  r~  )rq   rG   s     r/   r   z!MiniCPM3Model.set_gguf_parameters
  s   ,&&tz222++G4M,NOOO--gm.DEEE(()9:::009L1MNNN''0E(FGGG**73H+IJJJ//0GHHH''(=>>>G##(>(J,,W]-CDDD))'.*ABBB''0B(CgN`Fa(abbb11':L2MNNNNNr.   r\   r   c              #    K   |                      dgd          }|%| j        d         }|                    dd           }|                    dd           }||t          d          t	          |          t	          |          k    st	          |          |dz  k    rt          d|dz             |                     t          j        j	                  t          j        |t          j        	          fV  |                     t          j        j                  t          j        |t          j        	          fV  d S d S )
Nr  Tr  rv  rw  rx  r   ry  r  )r   rG   r   r   r   r   r   rf   r  rz  r   r  r  r{  )rq   r  r|  r}  r~  s        r/   r   z$MiniCPM3Model.generate_extra_tensors  s     ''(8$??#%78I'++M4@@L(,,^TBBM#}'<oppp<  C$6$666#l:K:Ky[\}:\:\ !eV_bcVc!e!efff**4+<+NOOQVQ]^jrwr  RA  RA  RA  B  B  B  B**4+<+OPPRWR^_lty  uB  SC  SC  SC  D  D  D  D  D  D $#r.   c                .    |                                   d S r`   r  r   s    r/   r  zMiniCPM3Model.set_vocab  r  r.   Nr  r   r  rQ   r  r   c                    |||k    r||z  } |j         |d|j        d         |z  dz  g|j        dd          R                      dd                               |j                  S r  r  r  s       r/   r  z!MiniCPM3Model._reverse_hf_permute  r  r.   r  r`   r  )r$   r%   r&   rf   r  MINICPM3rP   r   r   r  r  r-   r.   r/   r  r  
  sp        )JO O O$D D D D"( ( (
 
 
 
 
 
 
r.   r  QWenLMHeadModelc                  b    e Zd Zej        j        Zed             Zeddd            Z	d Z
d ZdS )r  c                    ddl m}  |            d                    fd|                     d          D                       S )Nr   bytes_to_unicoder  c                :    g | ]}t          |                   S r-   ordra   charbyte_encoders     r/   r  z3QwenModel.token_bytes_to_string.<locals>.<listcomp>(  $    PPPDSYY/PPPr.   latin-1*transformers.models.gpt2.tokenization_gpt2r  rB  r  br  r  s     @r/   r  zQwenModel.token_bytes_to_string$  V    OOOOOO''))wwPPPPAHHY<O<OPPPQQQr.   Nr  dict[bytes, int]r  r  r  r   r\   list[bytes]c                n   d |D             }	 d }d }t          t          |d d         |dd                              D ]7\  }}|                     |d         |d         z             }||||k     r|}|}8||||k    rn3|J |d |         ||         ||dz            z   gz   ||dz   d          z   }|S )Nc                .    g | ]}t          |g          S r-   r  ra   r  s     r/   r  z!QwenModel.bpe.<locals>.<listcomp>,       +++s+++r.   Tr  r   r   r   	enumeratezipr   	r  r  r  r  min_idxmin_rankr  pairr  s	            r/   r  zQwenModel.bpe*     ++U+++	bGH$SssU122Y%?%?@@ $ $4&**47T!W+<==#)9TH__G#HH$8X=Q=Q&&&(7(OuW~gk8J'J&KKeT[^_T_T`T`NaaE	b r.   c                .    |                                   d S r`   )r  r   s    r/   r  zQwenModel.set_vocab;  r  r.   c                   | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d         | j        d         z             | j                             | j        d                    | j         	                    | j        d                    | j         
                    | j                   d S )Nr  r  r  r  rotary_emb_baser  r  )rI   r  rG   r  r  r  r  r~  r  r  r  r<   r   s    r/   r   zQwenModel.set_gguf_parameters>  s&   ++DL9R,STTT((6I)JKKK--dl=.IJJJ00>Q1RSSS++DL9J,KLLL11$,}2MQUQ]^sQt2tuuu''5J(KLLL//=Q0RSSS&&tz22222r.   r`   r  r  r  r  r  r   r\   r  )r$   r%   r&   rf   r  QWENrP   r  r  r  r  r   r-   r.   r/   r  r     s}        %JR R \R
     \   	3 	3 	3 	3 	3r.   r  
Qwen2ModelQwen2ForCausalLM"Qwen2AudioForConditionalGenerationc                  H     e Zd Zej        j        Zd Z fdZd fdZ	 xZ
S )r  c                z    	 |                                   d S # t          $ r |                                  Y d S w xY wr`   r  r  r  r   s    r/   r  zQwen2Model.set_vocabN  U    	#))+++++  	# 	# 	#  """"""	#    ::c                   t                                                       |                                  | j                            d          pi }|                    d|                    d                    dk    rod|v rm| j                            t          j        j	                   | j        
                    |d                    | j                            |d                    d S d S d S r  r  r  s     r/   r   zQwen2Model.set_gguf_parametersT  r  r.   r   r   rb   r   r   r   r\   r   c              #  ^  K   | j         dk    rd| }d|v r|                    dd          }|                    d          s?|                    d          s*|                    d          s|                    d          rg S t                                          |||          E d {V  d S )	Nr  rv  rw  r  mlprr  ro  rp  )r  r  rm  r  r   r  s       r/   r   zQwen2Model.modify_tensors]  s      <<''"D??D$$<< 1266D??5!! 	T__5L%M%M 	??>22	6:oom6T6T	 I77))*dC@@@@@@@@@@@r.   r  )r$   r%   r&   rf   r  QWEN2rP   r  r   r   r9  r:  s   @r/   r  r  J  s~        &J# # #m m m m m	A 	A 	A 	A 	A 	A 	A 	A 	A 	Ar.   
DreamModelc                  P     e Zd Zej        j        ZddZd Z fdZ	d fdZ
 xZS )r  r\   r  c                V   g }g }ddl m} |                    | j        d          }|                                }| j                            dt          |                    }t          |	                                          |k     sJ | 
                    |          }d |                                D             }|                                }	t          |          D ]D}
|
|vr>|                    d|
 d           |                    t          j        j                   E||
         |	v r|                    ||
                    t%          |d	          re|
|j        v r\|j        |
         j        r%|                    t          j        j                   |                    t          j        j                   |                    t          j        j                   |                    ||
                    |                    t          j        j                   F|||fS )
Nr   r  Trr  r  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z-DreamModel.get_vocab_base.<locals>.<dictcomp>z      UUU.>k3kUUUr.   r  r  r  r  r   ry  r:   r  rG   r   r   r  r  r  rl   r  r  rn  rf   r  r+   hasattrr  r  r)   r*   r'   rq   r  r  r   r  
vocab_dictr  r  r  r  r  s              r/   r!  zDreamModel.get_vocab_basem      ......!11$.TX1YY	((**
\%%lC
OODD
:$$&&''*4444((33UU*BRBRBTBTUUU//11z"" 	7 	7A%%kQkkk*** 56666q![00mA.///9&<== <!yGeBeBe 5a8@ E (>???? (CDDDD OODN$:;;;;mA./// 56666x''r.   c                z    	 |                                   d S # t          $ r |                                  Y d S w xY wr`   r  r   s    r/   r  zDreamModel.set_vocab  r  r  c                   t                                                       |                                  | j                            d           | j                            d          pi }|                    d|                    d                    dk    rmd|v ri| j                            t          j	        j
                   | j                            |d                    | j                            |d                    | j                            d          }|| j                            |           d S d S )	NFr  r  rz   r  r  r  mask_token_id)r  r   r5  rI   add_causal_attentionrG   r   r  rf   r  r  r  r  add_mask_token_id)rq   r  r  r  s      r/   r   zDreamModel.set_gguf_parameters  s+   ##%%%""$$$ 	--e444|''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;klll ((99$..}===== %$r.   r   r   rb   r   r   r   r   c              #  ^   K   t                                          |||          E d {V  d S r`   )r  r   r  s       r/   r   zDreamModel.modify_tensors  s;      77))*dC@@@@@@@@@@@r.   r6  r  )r$   r%   r&   rf   r  DREAMrP   r!  r  r   r   r9  r:  s   @r/   r  r  i  s        &J#( #( #( #(J# # #> > > > >$A A A A A A A A A Ar.   LLaDAModelLMc                  l     e Zd Zej        j        ZdZddZd Z	 fdZ
edd            Zd fdZ xZS )
LLaDAModelTr\   r  c                V   g }g }ddl m} |                    | j        d          }|                                }| j                            dt          |                    }t          |	                                          |k     sJ | 
                    |          }d |                                D             }|                                }	t          |          D ]D}
|
|vr>|                    d|
 d           |                    t          j        j                   E||
         |	v r|                    ||
                    t%          |d	          re|
|j        v r\|j        |
         j        r%|                    t          j        j                   |                    t          j        j                   |                    t          j        j                   |                    ||
                    |                    t          j        j                   F|||fS )
Nr   r  Trr  r  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z-LLaDAModel.get_vocab_base.<locals>.<dictcomp>  r  r.   r  r  r  r  r  s              r/   r!  zLLaDAModel.get_vocab_base  r  r.   c                b    |                                   | j                            d           d S )NT)r  rI   r   r   s    r/   r  zLLaDAModel.set_vocab  s3     	**400000r.   c                p   t                                                       |                                  | j        }| j                            |d                    |                    d          x}U|                    d|                    d                    }|                    d|                    d                    |z  }| j                            |           | j                            dd          }| j                            |           | j                            dd          }| j        	                    |           | j                            d	d
          }| j        
                    |           | j                            d           | j                            d           d S )Nr  r  r  r  r  r  r     mlp_hidden_sizei 0  F)r  r   r5  rG   rI   rU  r   r~  r  r  r  r  add_diffusion_shift_logits)rq   rG   rc  r  context_lengthembedding_lengthfeed_forward_lengthr  s          r/   r   zLLaDAModel.set_gguf_parameters  s   ##%%%""$$$ ,''(=>>>J///H8kk"7Y9O9OPPG{{='++i2H2HIIWTH11(;;; ))*?FF++N;;;  <++It<<--.>??? #l../@%HH001DEEE 	--e444 	33E:::::r.   r  r   r  rQ   r  r   c                    |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R                      dd                               | j                  S r  r  re  s      r/   rf  zLLaDAModel.permute  rg  r.   r   rb   r   r   r   c              #    K   | j                             d| j                             d                    }| j                             d| j                             d                    }| j        rb|                    d          rt                              |||          }|                    d          rt                              |||          }t                                          |||          E d {V  d S )Nr  r  r  r  rx  ry  )rG   r   r  r  r  rf  r  r   )rq   r   rb   r   r  r  r  s         r/   r   zLLaDAModel.modify_tensors  s      !!"79I9I)9T9TUUL$$%:DL<L<L\<Z<Z[[	 	O}}=>> L'//
FFKK
}}=>> O'//
FINN
 77))*dC@@@@@@@@@@@r.   r6  r  r  )r$   r%   r&   rf   r  LLADArP   r  r!  r  r   r  rf  r   r9  r:  s   @r/   r  r    s        &JL#( #( #( #(J1 1 1; ; ; ; ;> ) ) ) \)A A A A A A A A A Ar.   r  Ernie4_5_ForCausalLMc                  D     e Zd Zej        j        Zd Z fdZddZ	 xZ
S )Ernie4_5Modelc                .    |                                   d S r`   r  r   s    r/   r  zErnie4_5Model.set_vocab  r  r.   c                H    t                                                       d S r`   rM  r  s    r/   r   z!Ernie4_5Model.set_gguf_parameters  rN  r.   r   r   rb   r   r   r   r\   r   c                p   | j         d         }| j         d         }| j                             d          x}| j         d         |z  }d|v r|                    dd          }d|v r|                    dd	          }|                    dd
          }|                    dd          }	||z  }
||z  }||z  }|                    |
||gd          \  }}}|                     |          |f|                     |          |f|                     |	          |fgS d|v r|                    dd          }|                    dd          }|j        d         dz  }|                    |d          \  }}|                     |          |f|                     |          |fgS |                     |          |fgS )Nr  r  r  r  zernie.rv  qkv_projzqkv_proj.weightr  r  zv_proj.weightr   r  up_gate_projzup_gate_proj.weightr  r  r   )rG   r   r  r  r   r  )rq   r   rb   r   	num_headsr  r  name_qname_kname_vtotal_q_dimtotal_k_dimtotal_v_dimq_proj_weightk_proj_weightv_proj_weightr  r  r  r  r   s                        r/   r   zErnie4_5Model.modify_tensors  s   L!67	|$9:((444H=|M2i?Ht<<(33D \\"3_EEF\\"3_EEF\\"3_EEF#h.K&1K&1K:D:J:JKYdfqKrxy:J:z:z7M=-%%f--}=%%f--}=%%f--}=  T!!ll#8:JKKG%:<NOOI!'*a/H/9/?/?a/?/P/P,n%%i002BC%%g..?  %%d++Z899r.   r  )r$   r%   r&   rf   r  ERNIE4_5rP   r  r   r   r9  r:  s   @r/   r  r    se        )J( ( (& & & & &": ": ": ": ": ": ": ":r.   r  Ernie4_5_MoeForCausalLMc                  b     e Zd ZU ej        j        ZdZded<    fdZ	 fdZ
ddZ fdZ xZS )Ernie4_5MoeModelNr$  rh  c                     t                      j        |i | d t          | j                  D             | _        d S )Nc                    g | ]}i S r-   r-   r)  s     r/   r  z-Ernie4_5MoeModel.__init__.<locals>.<listcomp>J  s    ======r.   )r  r   r  rR   rh  r  s      r/   r   zErnie4_5MoeModel.__init__H  sC    $)&)))==U4+;%<%<===r.   c                   t                                                       | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            d          x}| j        	                    |           | j                            d          x}w| j        
                    |           |dk    rY| j                            d          x}?| j                            d	          x}%| j                            ||z             d S d S d S d S d S )
Nr\  moe_kmoe_layer_intervalmoe_layer_start_indexmoe_intermediate_sizemoe_num_shared_expertsr   r  r  )r  r   rI   r  rG   r  r  add_leading_dense_block_countr   r  add_expert_shared_count%add_expert_shared_feed_forward_length)rq   r   shared_expert_countshared_expert_intermediate_sizer  r  s        r/   r   z$Ernie4_5MoeModel.set_gguf_parametersL  s   ##%%%))$,7H*IJJJ..t|G/DEEE66t|DX7YZZZ66t|D[7\]]]%)\%5%56M%N%NN![;;<QRRR#'<#3#34L#M#MMZ445HIII"Q&&t|O_O_`sOtOt,t,K  ,B  _c  _k  _o  _o  pE  _F  _F  HF  H[  GS FFGfj}G}~~~~~ [Z&&  ,B  ,B  GS  GSr.   r   r   rb   r   r   r   r\   r   c           	        |                     d          r|                    dd          }t          j        d|          }|rg S t          j        d|          }|rg S t          j        d|          }|rg S t          j        d|          }|rg S |                    d          dk    r| j        d	         }|J | j        #d
 t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t          |          D ]B}	d| d|	 d| d}
|
                    | j        |         |
                    | j        |         |
= Ct          j        |d          }d| d| d}|                     |          }|
                    ||f           |S g S |                     |          |fgS )Ne_score_correction_biase_score_correction.biaszmodel.mtp_block.(\d+)zmodel.mtp_emb_norm.(\d+)zmodel.mtp_hidden_norm.(\d+)zmodel.mtp_linear_proj.(\d+)mlp.expertsr  r\  c                    g | ]}i S r-   r-   r)  s     r/   r  z3Ernie4_5MoeModel.modify_tensors.<locals>.<listcomp>v  r*  r.   r    )	gate_projup_projr  r  .mlp.experts.r   r   r   r  )r  r  r  r  r.  rG   rh  r  rR   r   rn  r   r5  r   )rq   r   rb   r   r  r  r  w_namer6  r7  ename_to_retriever9  r   s                r/   r   zErnie4_5MoeModel.modify_tensorsY  s;   ==233 	V<< 9;TUUD 1488 	I 4d;; 	I7>> 	I7>> 	I 99]##r))%67I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 D ; ;F*,E$Y// B B,cC,c,cc,c,cTZ,c,c,c)T]3%78I%JKKK M#./@AA!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z4Ernie4_5MoeModel.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  z Ernie4_5MoeModel.prepare_tensors  r  r.   r  )r$   r%   r&   rf   r  ERNIE4_5_MOErP   rh  r  r   r   r   rK  r9  r:  s   @r/   r  r  C  s         -J/3H3333> > > > >    5: 5: 5: 5:nD D D D D D D D Dr.   r  Qwen2VLModelQwen2VLForConditionalGeneration"Qwen2_5_VLForConditionalGenerationQwen2_5OmniModelc                  D     e Zd Zej        j        Z fdZd ZddZ	 xZ
S )r  c           	         t                                                       | j        d         d         }|dgt          ddt	          |          z
            z  z  }| j                            |           d S )Nr  mrope_sectionr   r!   )r  r   rG   r  r   rI   add_rope_dimension_sections)rq   r  r  s     r/   r   z Qwen2VLModel.set_gguf_parameters  so    ##%%%^4_E!s1a#m*<*<&<====44]CCCCCr.   c                z    	 |                                   d S # t          $ r |                                  Y d S w xY wr`   r  r   s    r/   r  zQwen2VLModel.set_vocab  r  r  r   r   rb   r   r   r   r\   r   c                6   ~|                     d          r|                    dd          }|                     d          s?|                     d          s*|                     d          s|                     d          rg S |                     |          |fgS )Nthinker.r  visualaudiotalker	token2wav)rm  r  r   r   s       r/   r   zQwen2VLModel.modify_tensors  s    ??:&& 	0<<
B//D??8$$ 	(@(@ 	))	-1__[-I-I	 I%%d++Z899r.   r  )r$   r%   r&   rf   r  QWEN2VLrP   r   r  r   r9  r:  s   @r/   r  r    sl         (JD D D D D# # #: : : : : : : :r.   c                  6     e Zd Z fdZ fdZd ZddZ xZS )Qwen2VLVisionModelc                    t                      j        |i | | j        J | j                            dd          | j        d<   | j                            d          | j        d<   | j                            d          | j        d<   d| j        v rF| j                            d          | j        d	<   | j                            d          | j        d<   d S d S )
NrY  i0  r  r  r>  r  	embed_dimr  r  )r  r   rA  r   r  s      r/   r   zQwen2VLVisionModel.__init__  s    $)&)))"...,0,?,C,CLRU,V,VL)595H5L5L[5Y5Y12373F3J3J73S3S/0$---7;7J7N7N}7]7]D 34151D1H1H1U1UD... .-r.   c                   t                                                       | j        J | j        }| j        d         }|dk    r+| j                            t          j        j                   n.|dk    s|dk    r|dk    r*| j                            t          j        j	                   n)| j                            t          j        j
                   | j                            d           |                    d          }|
J d            |d         d	z   }t          d	t          |                    D ],}||         ||d	z
           z
  |k    rt          d
|           -| j                            |           nt          d| j        d                    | j                            | j                            dd                     d S )Nr  qwen2_vl
qwen2_5_vlqwen2_5_omniTfullatt_block_indexesz0fullatt_block_indexes is required for qwen2_5_vlr   r   zInvalid fullatt_block_indexes: zUnknown QwenVL model type: r  ư>)r  r   rA  r=  rI   r  rf   r  r#  QWEN25OQWEN25VLr  r   r  r   r   add_vision_n_wa_patternr  )rq   rG   r  r,  n_wa_patternr  r  s         r/   r   z&Qwen2VLVisionModel.set_gguf_parameters  s   ##%%%"...%'5
##44T5M5UVVVV<'':+G+G^++ 889Q9YZZZZ 889Q9Z[[[00666$+KK0G$H$H!(446h44403a7L1c"78899 ` `(+.CAE.JJlZZ$%^G\%^%^___ [44\BBBB]4;Ml;[]]^^^;;D<N<R<RSacg<h<hiiiiir.   c                `    ~~~d|v rt           j        j        S d|v rt           j        j        S dS Nz.patch_embd.z.position_embd.Frf   r  r4  r  r   s        r/   r   z%Qwen2VLVisionModel.tensor_force_quant  <    vX%%,00((,00ur.   r   r   rb   r   r   r   r\   r   c                4   ~|                     d          rd|v r|j        dk    r|j        \  }}n|j        d         }|dz  dk    sJ |dz  }|d |         }|||dz           }||dz  d          }	|                     |                    dd                    |f|                     |                    dd                    |f|                     |                    dd	                    |	fgS d
|v r|j        \  }
}}}}~
~~~|dk    s
J d            t
          j        t
          j        j                 dz   |d d d d ddf         ft
          j        t
          j        j                 dz   |d d d d ddf         fgS |                     |          |fgS g S )Nzvisual..qkv.r   r   r    r  r  r   rI  zpatch_embed.proj.weightz;Current implmentation only support temporal_patch_size of 2r   .z	.weight.1r   )	rm  ndimr  r   r  rf   r   r  V_ENC_EMBD_PATCH)rq   r   rb   r   c3r   cwqwkwvc1c2ktkhkws                  r/   r   z!Qwen2VLVisionModel.modify_tensors  s   ??9%% 	B $?a''&,EB#)!,BAv{{{{!G^1q5)A'))$,,uc*B*BCCRH))$,,uc*B*BCCRH))$,,uc*B*BCCRH 
 +d22%/%5"BBBQwww ]www&t'8'IJYVZdefefefhihihiklnqeqZrs&t'8'IJ[XZdefefefhihihiklnqeqZrs 
 --d33Z@AA	r.   r  r  r:  s   @r/   r%  r%    s~        	V 	V 	V 	V 	Vj j j j j6         r.   r%  c                  Z     e Zd ZdZdZ fdZ fdZddZddZdd	Z	d
 Z
d fdZ xZS )Qwen25OmniModelTc                     t                      j        |i | | j        J | j        d         | j        d<   | j        d         | j        d<   | j        d         | j        d<   d S )Nr  r  encoder_ffn_dimr  encoder_attention_headsr  )r  r   rB  r  s      r/   r   zQwen25OmniModel.__init__  sw    $)&)))!---,0,>y,I=)262DEV2W./484FG`4a0111r.   c                
   t                                                       | j        J | j                            | j        d                    | j                            | j                            dd                     d S Nnum_mel_binsr  r  )r  r   rB  rI   add_audio_num_mel_bins!add_audio_attention_layernorm_epsr   r  s    r/   r   z#Qwen25OmniModel.set_gguf_parameters  sy    ##%%%!---//0B>0RSSS::4;M;Q;QRbdh;i;ijjjjjr.   r\   rZ   c                B    | j         d                             d          S )Nrx  rS  rV  r   s    r/   rN  z!Qwen25OmniModel.get_vision_config  s    !"2377HHHr.   c                B    | j         d                             d          S )Nrx  rD  rV  r   s    r/   rO  z Qwen25OmniModel.get_audio_config!  s    !"2377GGGr.   r   c              #  4  K   | j         J d}d}| j         d         }t          j        |          |dz  dz
  z  }t          j        | t          j        |dz                                            z            }t          j        |          d d t          j        f         |t          j        d d f         z  }t          j        t          j	        |          t          j
        |          gd                              t          j                  }d|fV  d S )	N'  i  r  r   r   r  r  z"audio_tower.embed_positions.weight)rB  r?  logr   expr  r  newaxisr  sincosr  r  )rq   max_timescalelengthchannelslog_timescale_incrementinv_timescalesscaled_timepos_embds           r/   r   z&Qwen25OmniModel.generate_extra_tensors$  s     !---%m4"$&"7"78q=1;L"M$;#;el8WX=>Y>Y>_>_>a>a#abbl6**111bj=9N2:WXWXWX=<YY9ei44ei6L6LMSTUUUXX_d_lXmm3X>>>>>>r.   c                >    ~~~d|v rd|v rt           j        j        S dS Nz.convr   Frf   r  r4  r   s        r/   r   z"Qwen25OmniModel.tensor_force_quant0  -    6d??yD00,00ur.   r   r   rb   r   r   r   c                @   |                     d          r|                    dd          }|                     d          r;d|v sd|v r|                    d          }d|v rg S |                     |          |fgS t	                                          |||          S )Nr  r  rp  
conv1.bias
conv2.biasr  audio_bos_eos_token)rm  r  	unsqueezer   r  r   r  s       r/   r   zQwen25OmniModel.modify_tensors6  s    ??:&& 	0<<
B//D??=)) 		>t##|t';';'11"55
$,, 	))$//<==ww%%j$<<<r.   rw  r  r  )r$   r%   r&   r?  r@  r   r   rN  rO  r   r   r   r9  r:  s   @r/   rE  rE    s        b b b b bk k k k kI I I IH H H H
? 
? 
? 
?  = = = = = = = = = =r.   rE  InternVisionModelc                  2     e Zd Z fdZd Zd ZddZ xZS )rg  c                B   | j         J t          | j         d         t                    r| j         d         d         | j         d<   t          | j         d         t                    r| j         d         d         | j         d<   t                                                       | j        }| j                            t          j	        j
                   | j                            |d                    |d         dk    r| j                            d           n?|d         dk    r| j                            d           nt          d	|d                    | j                            d
          }|J | j                            t%          d|z                       d S )NrY  r   rZ  r  r  r  Tr  r  downsample_ratior  )rA  r   r   r  r   rG   rI   r  rf   r  INTERNVLr  r  r  r   r=  r   r  rQ   )rq   rG   rj  r  s      r/   r   z%InternVisionModel.set_gguf_parametersJ  s   "...d),7>> 	U040CL0QRS0TD-d),7>> 	U040CL0QRS0TD-##%%%,001I1RSSS;;GDT<UVVV< F**006666\"f,,006666O8MOOPPP-112DEE+++::3sEU?U;V;VWWWWWr.   c                `    ~~~d|v rt           j        j        S d|v rt           j        j        S dS r3  r4  r   s        r/   r   z$InternVisionModel.tensor_force_quanta  r5  r.   c                0    ddddddd}||v r||         }|S )Nzmlp1.0.biaszmlp1.0.weightzmlp1.1.biaszmlp1.1.weightzmlp1.3.biaszmlp1.3.weight)z+model.multi_modal_projector.layer_norm.biasz-model.multi_modal_projector.layer_norm.weightz)model.multi_modal_projector.linear_1.biasz+model.multi_modal_projector.linear_1.weightz)model.multi_modal_projector.linear_2.biasz+model.multi_modal_projector.linear_2.weightr-   )rq   rb   	names_maps      r/   _mapping_interns1_namez(InternVisionModel._mapping_interns1_namei  s;    ;H=L9F;J9F;J
 
	 9T?Dr.   r   r   rb   r   r   r   r\   r   c                   ~g d}|                                t          fd|D                       r3                    d          rdz   dv sdv sdv r                    d          sdz  d	v r|j        d
k    r|j        \  }}n|j        d         }|dz  dk    sJ |dz  }|d |         }|||d
z           }	||d
z  d          }
|                                         dd                    |f|                                         dd                    |	f|                                         dd                    |
fgS |                               |fgS g S )N)ro  r  model.vision_towermodel.multi_modal_projectorc                :    g | ]}                     |          S r-   rt  ru  s     r/   r  z4InternVisionModel.modify_tensors.<locals>.<listcomp>{  s%    DDDF''DDDr.   ro  r  z.lsz.lambda_position_embeddingr   r7  r   r   r    zattn.qkvzself_attn.q_projzself_attn.k_projzself_attn.v_proj)ro  r  rm  r  r8  r  r   r  )rq   r   rb   r   vision_prefixr:  r   r;  r<  r=  r>  s     `        r/   r   z InternVisionModel.modify_tensorsv  s   ddd**400DDDDmDDDEE 	> ~.. .&-t!3!37Kt7S7S]a]j]jkt]u]u7S	!$?a''&,EB#)!,BAv{{{{!G^1q5)A'))$,,zCU*V*VWWY[\))$,,zCU*V*VWWY[\))$,,zCU*V*VWWY[\ 
 ))$//<==	r.   r  )r$   r%   r&   r   r   ro  r   r9  r:  s   @r/   rg  rg  H  so        X X X X X.           r.   WavTokenizerDecc                  D     e Zd Zej        j        Zdd	Zd
 Z fdZ	 xZ
S )WavTokenizerDecModelr   r   rb   r   r   r   r\   r   c                ^   ~|                     d          s*|                     d          s|                     d          rt                              d|           g S t                              |                     |           d|j                    |                     |          |fgS )Nzcodebook.cluster_sizezcodebook.embed_avgzcodebook.initedz	Skipping r  )r  rd   r  re   r   r  r   s       r/   r   z#WavTokenizerDecModel.modify_tensors  s     566	233	 /00	 LL-T--...It++D11IIz7GIIJJJ%%d++Z899r.   c                .    |                                   d S r`   )r  r   s    r/   r  zWavTokenizerDecModel.set_vocab  r  r.   c                F   t                                                       | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j        	                    | j        d         d                    | j        
                    | j        d         d                    | j                            | j        d	         d                    | j                            | j        d	         d                    | j                            d
           d S )Nr  n_embd_featuresr  group_norm_epsilongroup_norm_groupsposnetr  r  convnextF)r  r   rI   rU  rG   add_features_lengthr  add_group_norm_epsadd_group_norm_groupsadd_posnet_embedding_lengthadd_posnet_block_countadd_convnext_embedding_lengthadd_convnext_block_countr  r  s    r/   r   z(WavTokenizerDecModel.set_gguf_parameters  s]   ##%%%''l1KLLL,,>O1PQQQ00f1EFFF++>R1STTT..>Q1RSSS44T\(5KH5UVVV//T\(5KI5VWWW66t|J7OPX7YZZZ11t|J7OPY7Z[[[--e44444r.   r  )r$   r%   r&   rf   r  WAVTOKENIZER_DECrP   r   r  r   r9  r:  s   @r/   rx  rx    se        1J: : : :  5 5 5 5 5 5 5 5 5r.   rx  Qwen2MoeForCausalLMc                  X     e Zd ZU ej        j        Z fdZdZde	d<   ddZ
 fdZ xZS )Qwen2MoeModelc                t   t                                                       | j                            d          x}| j                            |           | j                            d          x}7| j                            |           t                              d|            | j                            d          x}7| j        	                    |           t                              d|            | j                            d          pi }|                    d|                    d                    d	k    rod
|v rm| j        
                    t          j        j                   | j                            |d
                    | j                            |d                    d S d S d S )Nnum_expertsr   #gguf: expert feed forward length = r  z*gguf: expert shared feed forward length = r  r  rz   r  r  r  )r  r   rG   r   rI   r  r  rd   re   r  r  rf   r  r  r  r  )rq   r  r   r  r  r  s        r/   r   z!Qwen2MoeModel.set_gguf_parameters  s   ##%%%))-888IE--i888%)\%5%56M%N%NN![;;<QRRRKKU>SUUVVV/3|/?/?@a/b/bb+oBBCbcccKKfEdffggg |''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;klllll MLQiQir.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	     j   |                     dd          }|                    d          s?|                    d          s*|                    d          s|                    d          rg S |                    d          dk    r| j        d	         }|J | j        #d
 t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t          |          D ]B}d| d| d| d}	|                    | j        |         |	                    | j        |         |	= Ct          j
        |d          }d| d| d}
|                     |
          }|                    ||f           |S g S |                     |          |fgS )Nrw  r  r  ro  rq  rr  r  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z0Qwen2MoeModel.modify_tensors.<locals>.<listcomp>  r*  r.   r    r  r  r  r  r  r   r   r   r  )r  rm  r.  rG   rh  r  rR   r   rn  r   r5  r   rq   r   rb   r   r  r  r  r6  r7  r8  r9  r   s               r/   r   zQwen2MoeModel.modify_tensors  s   ||-r22??5!! 	T__^%D%D 	XlHmHm 	qu  rA  rA  B_  r`  r` 	I99Y2%%]3I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 D ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z1Qwen2MoeModel.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  zQwen2MoeModel.prepare_tensors  r  r.   r  )r$   r%   r&   rf   r  QWEN2MOErP   r   rh  r  r   rK  r9  r:  s   @r/   r  r    s         )Jm m m m m$ 04H3333&: &: &: &:PD D D D D D D D Dr.   r  Qwen3ForCausalLMc                  &    e Zd Zej        j        ZdS )
Qwen3ModelN)r$   r%   r&   rf   r  QWEN3rP   r-   r.   r/   r  r            &JJJr.   r  Qwen3MoeForCausalLMc                  B     e Zd Zej        j        Z fdZd Zd Z	 xZ
S )Qwen3MoeModelc                     t                      j        |i | t                              | j        d          }|                    dd g          d         | _        d S )NFarchitecturesr   )r  r   r8   r   r:   r   origin_hf_arch)rq   r  r  rG   r  s       r/   r   zQwen3MoeModel.__init__  sY    $)&)))((??%kk/D6BB1Er.   c                    | j         dk    r|                                  d S 	 |                                  d S # t          $ r |                                  Y d S w xY w)N InternS1ForConditionalGeneration)r  _set_vocab_interns1r  r  r  r   s    r/   r  zQwen3MoeModel.set_vocab  s{    "DDD$$&&&F	#))+++++  	# 	# 	#  """"""	#s   9 AAc                   g }g }ddl m} |                    | j        d          }t	          |d|                                          }| j                            dt          |                    }t          |
                                          |k     sJ |                     |          }d |                                D             }|                                }	|j        }
t          |          D ]o}||vr>|                    d| d	           |                    t"          j        j                   E||         }||	v r|
|         j        sl|}|                    |                    |d
                    }||k    r:t.                              t3          |           dt3          |           d           |
|         j        s|                     |          r%|                    t"          j        j                   nI|                    t"          j        j                   n$|                    t"          j        j                   |                    |           q| j                             d           | j        !                    |           | j        "                    |           | j        #                    |           t#          j$        | j        d          }| j        dz  }g }|%                                rQtM          |d          5 }tO          j(        |                              dg           }d d d            n# 1 swxY w Y   | j        dz  }|%                                rtM          |d          5 }tO          j(        |                              di           }
d |
                                D             }|D ]"}||v r|)                    |||                    #	 d d d            n# 1 swxY w Y   |)                    dd           |)                    dd           |*                    | j                   d S )Nr   r  Trr  r  r  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z5Qwen3MoeModel._set_vocab_interns1.<locals>.<dictcomp>+  s    PPP.>k3kPPPr.   r  r  Fr	  r  r  r  r  zspecial_tokens_map.jsonr   r   additional_special_tokensr  c                P    i | ]#\  }}|d          |d         t          |          $S )r  r  )rQ   )ra   r  r   s      r/   r  z5Qwen3MoeModel._set_vocab_interns1.<locals>.<dictcomp>V  sM      !B  !B  !B+%quv  rA  !Bi3u::  !B  !B  !Br.   r  i]P r  i[P )+r  r   ry  r:   getattrr  rG   r   r   r  r  r  rl   r  r  r  rn  rf   r  r+   r  r  r  rd   re   r  r  r  r)   r*   r'   rI   r  r  r  r  r  r   r   r   r   r  r  )rq   r  r  r   r  r  r  r  r  r  r  r  r  r   r  special_tokens_map_filer  r   tokenizer_cfg_filetoken2ids_maps                       r/   r  z!Qwen3MoeModel._set_vocab_interns1  s	    ......!11$.TX1YY		7I,?,?,A,ABB\%%lCJJ??
5<<>>""Z////((33PP%++--PPP//11(=z"" 	% 	%A%%kQkkk*** 56666*1-K'' 02= D). ) 0 01A1A%\a1A1b1b c c)U22"KK4+?+?  )C  )Caefkalal  )C  )C  )C  D  D  D+A.6 E$:V:VW\:]:] E (>???? (CDDDDOODN$9:::e$$$$,,V444**6222''///((222)$.dKKK"&.3L"L$&!"**,, 	^-'BBB ^a,0IaLL,<,<=XZ\,],])^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^!^.GG%%'' 	V(W=== V'+y||'7'78NPR'S'S$ !B  !BQeQkQkQmQm  !B  !B  !B6 V VE--%88e@TUUUVV V V V V V V V V V V V V V V 	((777((777!!$"233333s%   )MMM
A,PP
P)r$   r%   r&   rf   r  QWEN3MOErP   r   r  r  r9  r:  s   @r/   r  r    se        )JF F F F F
	# 	# 	#=4 =4 =4 =4 =4 =4 =4r.   r  GPT2LMHeadModelc                  4    e Zd Zej        j        Zd Zdd
ZdS )	GPT2Modelc                   | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             d| j        d         z             | j                             | j        d                    | j                             | j        d                    | j                             | j	                   d S )Nr  r  r  r!   r  r  
rI   r  rG   r  r  r  r  r  r  r<   r   s    r/   r   zGPT2Model.set_gguf_parametersc  s    ((i)@AAA++DL,ABBB--dl8.DEEE00T\(5K1KLLL''X(>???++DL9M,NOOO&&tz22222r.   r   r   rb   r   r   r   r\   r   c                    ~g }|                     d          r|S |                     d          r|                    dd          }|                     |          }|                    ||f           |S )N)
.attn.biasz.attn.masked_bias).c_attn.weight.c_proj.weight.c_fc.weightr  r   r   )r  r  r   rn  )rq   r   rb   r   r  r   s         r/   r   zGPT2Model.modify_tensorsl  s    ,. ==<== 	N==_`` 	4#--a33J''--*-...r.   Nr  )	r$   r%   r&   rf   r  GPT2rP   r   r   r-   r.   r/   r  r  _  sA        %J3 3 3     r.   r  PhiForCausalLMc                  ,    e Zd Zej        j        Zd ZdS )	Phi2Modelc                N   |                      ddg          }|                      dg          }|                      ddg          }|                      ddg          }| j                            |                      dd	g                     | j                            |           | j                            d
|z             | j                            |           | j                            |           | j                            |           | j                            |                      ddg                     | j        	                    t          ||z            |z             | j                            | j                   | j                            d           d S )Nr  r  r!  r  r  r  r  r  r  r!   r  r  F)r   rI   r  r  r  r  r  r  r  r~  rQ   r  r<   r   )rq   rR   rot_pctr  r  s        r/   r   zPhi2Model.set_gguf_parameters  s   &&(;Y'GHH""$;#<==!!=(";<<!!#8("CDD++D,<,<mMf=g,h,hiii--f55500V<<<((555''///**6222++D,<,<>RTd=e,f,fggg11#g6F2G2G62QRRR&&tz222**511111r.   N)r$   r%   r&   rf   r  PHI2rP   r   r-   r.   r/   r  r    s-        %J2 2 2 2 2r.   r  Phi3ForCausalLMc                  :    e Zd Zej        j        Zd Zd ZddZ	dS )Phi3MiniModelc                $   | j         dz  }|                                rmt          |dd          5 }t          j        |          }|d         }|dk    r |                                 cd d d            S 	 d d d            n# 1 swxY w Y   ddlm} | j         d	z  }|                                st          d
|            |            }|	                    t          |                     | j                            d|                                          }d t          |          D             }	dg|z  }
t          j        g|z  }t          |                                          D ]}|                    |          }|                    d          }|                    |          }t          j        }|                    |          rt          j        }ne|                    |          rt          j        }nC|                    |          rt          j        }n!|                    |          rt          j        }||	|<   ||
|<   |||<   | j         dz  }|                                rt          |dd          5 }t          j        |          }|D ]`}||         }||k    r$t8                              d| d|dz
              4|                    d          |	|<   d|
|<   t          j        ||<   a	 d d d            n# 1 swxY w Y   | j         dz  }|                                rGt          |dd          5 }t          j        |          }|                    di           }|                                D ]\  }}tA          |          }|d                             d          }||         t          j        k    r[|	|         |k    rOt8          !                    d| d|	|         "                    d          d|"                    d                     ||	|<   d|
|<   t          j        ||<   |                    d          rt          j        ||<   	 d d d            n# 1 swxY w Y   | j         dz  }|                                r8t          |dd          5 }t          j        |          }|                    dg           }|D ]}tA          |d                   }|d                             d          }||         t          j        k    r[|	|         |k    rOt8          !                    d| d|	|         "                    d          d|"                    d                     ||	|<   d|
|<   t          j        ||<   |                    d          rt          j        ||<   	 d d d            n# 1 swxY w Y   | j#        $                    d           | j#        %                    d           | j#        &                    |	           | j#        '                    |
           | j#        (                    |           tS          j*        | j         tW          |	                    }|,                    | j#                   d S )Nr  r   r   r   tokenizer_classGPT2Tokenizerr   r  r  Error: Missing r  c                @    g | ]}d | d                     d          S r  r  r  s     r/   r  z+Phi3MiniModel.set_vocab.<locals>.<listcomp>  r  r.   r  r  r  r  r   r  r  r  r  rt  r  r  r  added_tokensidr  r  r  )-r:   r   r   r   r   r  r  r  r   r  r   rG   r   r  r  r   r+   r  r  r  r'   r  r(   r  r)   r  r  r,   rd   r  r*   rl   rQ   r>  r  rI   r  r  r  r  r  rf   r  r   r  )rq   r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  
foken_datar  tokenizer_filetokenizer_jsonr  r  s                              r/   r  zPhi3MiniModel.set_vocab  so    $1H H ((** 	2+S7CCC 2q(,	!%"78I"J"o55//11	2 2 2 2 2 2 2 2 62 2 2 2 2 2 2 2 2 2 2 2 2 2 2 	988888*;;%%'' 	A?~??@@@**,,	s>22333\%%lI4H4H4J4JKK
VVE*DUDUVVV'j:56=>Ki224455 	) 	)H''11E<<((D&&x00E-4G""8,, 719$$X.. 719##H-- 718!!(++ 716#F8$F8!(HX N-@@$$&& 	N'w??? N1$(IaLL!, N NC05H:--%iX%i%iYcfgYg%i%ijjj '*zz'':':F8$'.F8$)@)MHX&&NN N N N N N N N N N N N N N N !%1H H ((** 	M+S7CCC Mq(,	!%'<'@'@AWY['\'\$,@,F,F,H,H 
M 
M(Hj"8}}H&y188AAE)-D-KKK!(+u44"NN  ,Lh  ,L  ,L&QYJZJaJabiJjJj  ,L  ,Lrwr~r~  @G  sH  sH  ,L  ,L  M  M  M',F8$'.F8$)@)MHX&!~~i00 M-D-L*
MM M M M M M M M M M M M M M M *::!!## 	MncG<<< M!%1-11."EE". 
M 
MJ":d#344H&y188AAE)-D-KKK!(+u44"NN  ,Lh  ,L  ,L&QYJZJaJabiJjJj  ,L  ,Lrwr~r~  @G  sH  sH  ,L  ,L  M  M  M',F8$'.F8$)@)MHX&!~~i00 M-D-L*
MM M M M M M M M M M M M M M M 	,,W555**9555''///))&111((222)$.#f++NNN!!$"233333sJ   6BBB?A8LLL DQ**Q.1Q.&DWWWc                   |                      ddg          }|                      ddg          }|                      ddg          }|                      ddg          }|                      d	g          }|                      d
dg          }|                      dg          }| j                            dd          }t          ||z            |z  }	| j                            |           | j                            |           | j                            |           | j                            |                      dg                     | j        	                    |           | j        
                    |           | j                            |           | j                            |           | j                            |	           | j                            |                      dg                     | j                            | j                   | j                            d          }
|
d}
| j                            |
           d S )Nr  r  r  r  r  r  r  r  r  r  r  r  r!  r  r  r  sliding_windowr   )r   rG   r   rQ   rI   r  r  r  r  r  r  r  r  r~  r  r  r<   add_sliding_window)rq   rR   r  r  r  rms_epsmax_pos_embdsorig_max_pos_embdsr  r|  r  s              r/   r   z!Phi3MiniModel.set_gguf_parameters  s;   &&(;Y'GHH!!=(";<<!!#8("CDD$$&;[%IJJ	""N#344((-9R)STT!--/Q.RSS,""#:C@@&())V3	++M:::667IJJJ--f555001A1ACVBW1X1XYYY((555''///**9555//88811)<<<++D,<,<l^,L,LMMM&&tz222))*:;;!N++N;;;;;r.   r\   r   c           
   #    K   |                      ddg          }|                      ddg          }|                      ddg          }|                      dg          }| j                            dd	          }t          ||z            |z  }|                      d
gd          }|d S ||z  }|                    d|                    dd                                                    }	t          |	          dk    rt          d          |	dk    s|	dk    rG|d	k    r>t          j        dt          j	        |          t          j	        |          z  z             nd	}
n<|	dk    r#|d	k    rdt          j	        |          z  d	z   nd	}
nt          d|	 d          | j                            |
           |                    dd           }|                    dd           }||t          d          t          |          t          |          k    st          |          |dz  k    r6t          d|dz   dt          |           dt          |           d          |                     t          j        j                  t%          j        |t$          j                   fV  |                     t          j        j                  t%          j        |t$          j                   fV  d S )!Nr  r  r  r  r  r  r  r!  r  r  Tr  rz   r  r   z*Missing the required key rope_scaling.typesurm  r   r  皙?zThe rope scaling type z is not supported yetrv  rw  rx  r   ry  z. long_factors = z, short_factors = r   r  )r   rG   r   rQ   r  r   r   r  sqrtrR  r   rI   add_rope_scaling_attn_factorsr   r   rf   r  rz  r   r  r  r{  )rq   r  r  r  r  r  r|  r  r>  rope_scaling_typeattn_factorr}  r~  s                r/   r   z$Phi3MiniModel.generate_extra_tensors   sE     !!=(";<<!!#8("CDD((-9R)STT!--/Q.RSS,""#:C@@&())V3	 ''(8$??F 22(,,[,:J:J6SU:V:VWW]]__ !!Q&&GHHH$$(9Z(G(G[`cf[f[f$)A$(CU:V:V(V$VWWWloKK&((9>#/#55#KK%&g?P&g&g&ghhh66{CCC#''t<<$((>>=#8klll|M 2 222c,6G6G9WX=6X6X  mR[^_R_  m  mru  wC  sD  sD  m  m  X[  \i  Xj  Xj  m  m  m  n  n  n&&t'8'JKKU\Zfnsn{M|M|M|}}}}&&t'8'KLLel[hpup}N~N~N~r.   Nr  )
r$   r%   r&   rf   r  PHI3rP   r  r   r   r-   r.   r/   r  r    sZ        %Jf4 f4 f4P< < <:&@ &@ &@ &@ &@ &@r.   r  PhiMoEForCausalLMc                  X     e Zd ZU ej        j        ZdZded<    fdZ	ddZ
 fdZ xZS )PhiMoeModelNr$  rh  c                    t                                                       | j                            | j        d                    | j                            | j        d                    d S )Nr  r  )r  r   rI   r  rG   r  r  s    r/   r   zPhiMoeModel.set_gguf_parametersO  s]    ##%%%..t|<Q/RSSS))$,7J*KLLLLLr.   r   r   rb   r   r   r   r\   r   c           	        |                     d          dk    r| j        d         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t          |          D ]B}d| d| d	| d
}	|                    | j        |         |	                    | j        |         |	= Ct          j        |d          }d| d| d
}
| 	                    |
          }|                    ||f           |S g S | 	                    |          |fgS )Nrz  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z.PhiMoeModel.modify_tensors.<locals>.<listcomp>[  r*  r.   r    r|  r  r  r   r   r   r  rT  r  s               r/   r   zPhiMoeModel.modify_tensorsT  s   99/00B66%89I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 1 ; ;F*,E$Y// 6 6 d d ds d dU[ d d dT]3%7%>??? M#.u55!&U!:!:!:J"`#"`"`QW"`"`"`K#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z/PhiMoeModel.prepare_tensors.<locals>.<listcomp>}  r  r.   r   r  r  r  s     r/   rK  zPhiMoeModel.prepare_tensorsx  r  r.   r  )r$   r%   r&   rf   r  PHIMOErP   rh  r  r   r   rK  r9  r:  s   @r/   r  r  I  s         'J/3H3333M M M M M
": ": ": ":HD D D D D D D D Dr.   r  PlamoForCausalLMc                  F    e Zd Zej        j        Zd Zd Zd Z	d Z
ddZdS )
PlamoModelc                .    |                                   d S r`   r  r   s    r/   r  zPlamoModel.set_vocab  r  r.   c                   | j         }|d         }| j                            d           | j                            |d                    | j                            |d                    | j                            |           | j                            |d                    | j                            d           | j                            |d                    | j        	                    | j
                   d S )Nr  r  r  r  r  r"   r  )rG   rI   r  r  r  r  r  r  r  r  r<   rq   rG   rR   s      r/   r   zPlamoModel.set_gguf_parameters  s    ,12++D111--gm.DEEE009L1MNNN((555''0E(FGGG**1---//0GHHH&&tz22222r.   c                    |                                 dk    sJ |                    dddd          }t          j        |d          }t          j        |d          }|S )N   r     r"   rJ  r  )r   r   r   r    sizer  r   rf  rq   r   s     r/   shuffle_attn_q_weightz PlamoModel.shuffle_attn_q_weight  s^      L0000''1c488
]:|<<
]:|<<
r.   c                    |                                 dk    sJ |                    dddd          }t          j        |d          }t          j        |d          }|S )Nr  r  r  r"   rJ  )r   r   r   r    r  r  s     r/   shuffle_attn_output_weightz%PlamoModel.shuffle_attn_output_weight  s^      L0000''aC88
]:|<<
]:|<<
r.   r   r   rb   r   r   r   r\   r   c                    ~|                      |          }|                    d          r|                     |          }n*|                    d          r|                     |          }||fgS )Nzattn_q.weightzattn_output.weight)r   r  r  r  r  s        r/   r   zPlamoModel.modify_tensors  s    ''-- _-- 	E33J??JJ344 	E88DDJ:&''r.   Nr  )r$   r%   r&   rf   r  PLAMOrP   r  r   r  r  r   r-   r.   r/   r  r    sn        &J( ( (3 3 3    ( ( ( ( ( (r.   r  Plamo2ForCausalLMPLaMo2ForCausalLMc                  :    e Zd Zej        j        Zd Zd ZddZ	dS )Plamo2Modelc                
   | j         dz  }| j         dz  }|                                st          d|           t          |dd          5 }t	          j        |          }d d d            n# 1 swxY w Y   g }g }g }t          |dd          5 }t          |          D ]\  }}	|	                                rt	          j        |	          }
|
d         	                    d          }t          |
d                   }t          |
          d	k    r|
d	         nd
}|                    |           |                    |           |dk    r%|                    t          j        j                   |dk    r%|                    t          j        j                   |dk    r&|                    t          j        j                   (|
d         }|                    d          r;|                    d          r&|                    t          j        j                   |                    t          j        j                   	 d d d            n# 1 swxY w Y   | j        d         }|t          |          k    r|t          |          z
  }t,                              d| d| d           t1          d|dz             D ]c}|                    t3          d| dd                     |                    d           |                    t          j        j                   d| j                            d           | j                            d           | j                            |           | j                            |           | j                             |           d|v rP|d         H|!                    |d         	                    d                    }| j        "                    |           d|v rP|d         H|!                    |d         	                    d                    }| j        #                    |           d|v rP|d         H|!                    |d         	                    d                    }| j        $                    |           d|v rP|d         H|!                    |d         	                    d                    }| j        %                    |           d|v rP|d         H|!                    |d         	                    d                    }| j        &                    |           | j        '                    d           | j        (                    d           d S )Nztokenizer.jsonlr  z"PLaMo 2 tokenizer file not found: r   r   r   r   r   r   r'   r(   r)   r,   z<|plamo:r  r  r  r  r  r  r  plamo2r  	bos_token	eos_token	pad_token	sep_token	unk_tokenr!   F))r:   r   r  r   r   r   r  striploadsr  r  r   rn  rf   r  r(   r)   r,   rm  r  r'   rG   rd   r  r  r  r+   rI   r  r  r  r  r  r   r  r  r  add_sep_token_idr  add_eot_token_idr]  )rq   tokenizer_jsonl_pathtokenizer_config_pathr   tokenizer_configr  r  r  line_numr  r  r  r  token_type_str	token_strr  r  r  r  s                      r/   r  zPlamo2Model.set_vocab  s     $~0AA $1H H#++-- 	a#$_I]$_$_``` 'w??? 	,1#y||	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, &g>>> 	C!"+A,, C C$::<< C!%D!1!1J&qM0099E!*Q-00E69*oo6I6IZ]]xNMM%(((MM%((( &22 (>????'944 (>????'611 (;<<<< %/qM	$//
;; C	@R@RSW@X@X C$OODN,BCCCC$OODN,ABBB1C	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C6 \,/
F##"S[[0ILLeyeeYbeeefff1i!m,, 7 7eK1KKK'BBBCCCg&&& 56666 	,,X666**9555''///))&111((222 ***/?/L/X||$4[$A$H$H$Q$QRRH--h777***/?/L/X||$4[$A$H$H$Q$QRRH--h777***/?/L/X||$4[$A$H$H$Q$QRRH--h777***/?/L/X||$4[$A$H$H$Q$QRRH--h777***/?/L/X||$4[$A$H$H$Q$QRRH--h777 	))!,,,--e44444s%   A..A25A2F6IIIc                Z   | j         }|d         }| j                            | j         d                    |                    dd          }|                    dd          }g }|rrt	          |          D ]b}||dz  k    r
||dz
  k    }n||z  |dz  k    }|r|                    d           9|                    |                    d	d
                     c|r| j                            |           | j                            |                    dd                     | j                            |                    dd                     | j        	                    |           | j        
                    |                    dd                     | j                            |                    dd                     | j                            |                    dd                     | j                            |                    dd                     | j                            |                    dd
                     | j                            |                    dd                     |                    dd          |                    dd          z  }| j                            |           | j                            d           | j                            |                    dd                     | j                            | j                   d S )Nr  r  
mamba_stepr   mamba_enabledTr   r   r  r!   r  r  r  r  r  rL  r  r-  r  rQ  mamba_d_state@   mamba_d_convmamba_num_headshidden_size_per_headrJ  r  i 4  )rG   rI   rU  r   r  rn  r  r  r  r  r  r  r  add_ssm_state_sizeadd_ssm_conv_kerneladd_ssm_time_step_rankadd_ssm_inner_sizeadd_ssm_group_countr  r  r<   )	rq   rG   rR   r
  r  mamba_layersr  is_mambar  s	            r/   r   zPlamo2Model.set_gguf_parameters	  s   ,12''\(BCCC
 [[q11
OT:: 
	O;'' 	O 	O:?33 ![1_ 4HH !JJ!ODH O ''**** ''4I1(M(MNNNN 	=..|<<<++GKK8QSW,X,XYYY--gkk-.N.NOOO((555''4I2(N(NOOO//NE0R0RSSS++GKKe,L,LMMM 	++GKK,L,LMMM,,W[[-K-KLLL//<Mr0R0RSSS#KK(92>>McehAiAii++,=>>>,,Q/// 	00=PRW1X1XYYY&&tz22222r.   r   r   rb   r   r   r   r\   r   c                   ~|                     d          rt          j        |           }n|                     d          r |                    d          d         dz   }n|                     d          r |                    d          d         dz   }n|                     d          r |                    d          d         dz   }nZ|                     d	          r |                    d	          d         d
z   }n%|                     d          r|                    d          d         dz   }n|                     d          r|                    d          d         dz   }n|                     d          r"t          j        |          }|j        dk    sJ n|                     d          r|dz  }nk|                     d          r|dz  }nP|                     d          r|dz  }n5|                     d          r|dz  }n|                     d          r|dz  }|                     |          }||fgS )N.A_log.dt_biasr   .dt_proj.biasz.dt_norm_weightz.dt_norm.weightz.B_norm_weightz.B_norm.weightz.C_norm_weightz.C_norm.weightz	.k_weightz	.k.weightz	.q_weightz	.q.weightz.conv1d.weightr   z.pre_mixer_norm.weightr  z.post_mixer_norm.weightg?z.pre_mlp_norm.weightz.post_mlp_norm.weightgWfѷ?z.norm.weight)r  r   rS  
rpartitionsqueezer8  r   r  s        r/   r   zPlamo2Model.modify_tensors7  sY   =="" 	)J///JJ]]:&& 	??:..q1OCDD]],-- 	??#455a8;LLDD]]+,, 	??#344Q7:JJDD]]+,, 	??#344Q7:JJDD]];'' 	??;//2[@DD]];'' 	??;//2[@DD]]+,, 	z22J?a''''']]344 		#JJ]]455 	'!JJ]]122 	#JJ]]233 	.(JJ]]>** 	#J''--:&''r.   Nr  )
r$   r%   r&   rf   r  PLAMO2rP   r  r   r   r-   r.   r/   r  r    sU        'JQ5 Q5 Q5f,3 ,3 ,3\!( !( !( !( !( !(r.   r  CodeShellForCausalLMc                  8    e Zd Zej        j        Zd ZdZddZ	dS )CodeShellModelc                   | j         d         }| j                            | j         d                    | j                            | j         d                    | j                            d| j         d         z             | j                            |           | j                            | j         d                    | j                            | j         d                    | j                            | j         d                    | j        	                    | j
                   | j                            d           | j                            t          j        j                   | j                            d	           d S )
Nr  r  r  r!   r  num_query_groupsr  r  r  )rG   rI   r  r  r  r  r  r  r  r  r<   r  r  rf   r  r  r  r  s     r/   r   z"CodeShellModel.set_gguf_parameters_  sJ   l9-++DL,GHHH--dl8.DEEE00T\(5K1KLLL((555''X(>???**4<8J+KLLL++DL9M,NOOO&&tz222++G444..t/C/JKKK0055555r.   Fr   r   rb   r   r   r   r\   r   c                   ~|                      t          j        j                  }|                      t          j        j                  }|                     |          }| j        ss||                      t          j        j                  k    rK| j        rCd| j        v r:t          	                    | d| d           | j        
                    d           n||k    rd| _        ||fgS )Nztransformer.wte.weightz not found before z, assuming they are tiedT)r   rf   r  r.  r,  r   _has_tok_embdrH   rd   r  removerq   r   rb   r   output_nametok_embd_namer   s          r/   r   zCodeShellModel.modify_tensorsp  s    --d.?.FGG//0A0LMM''-- ! 	&h$2I2I$J[Jb2c2c&c&c  C%=AR%R%Rfffffggg!(()ABBB&&!%D:&''r.   Nr  )
r$   r%   r&   rf   r  	CODESHELLrP   r   r%  r   r-   r.   r/   r!  r!  [  sH        *J6 6 6 M( ( ( ( ( (r.   r!  InternLM2ForCausalLMc                  :    e Zd Zej        j        Zd Zd ZddZ	dS )InternLM2Modelc                x   ddl m} ddl m} | j        dz  }g }g }g }|                                s1t
                              d|            t          j        d           |	                                }|
                    t          |d                                                     |j        j        } |            }	|	                    t!          |                     | j                            d|	                                          }
t)          |
          D ]n}|	                    |          }|                    d	          }|	                    |          }|d
k    r3t
                              d| d           d                    d	          }t2          j        }|	                    |          rt2          j        }ne|	                    |          rt2          j        }nC|	                    |          rt2          j         }n!|	!                    |          rt2          j"        }|#                    d          rt2          j         }|$                    |           |$                    |           |$                    |           p| j        dz  }|                                rt          |dd	          5 }tK          j&        |          }|D ]^}|$                    |                    d	                     |$                    d           |$                    t2          j'                   _	 d d d            n# 1 swxY w Y   d}d }| j        dz  }|                                rQt          |dd	          5 }tK          j&        |          }|                    di           }|(                                D ]\  }}tS          |          }|d         }||k    r|}|                    d	          }||         t2          j         k    r[||         |k    rOt
                              d| d||         *                    d	          d|*                    d	                     |||<   d||<   t2          j'        ||<   |                    d          rt2          j        ||<   	 d d d            n# 1 swxY w Y   | j        dz  }|                                rBt          |dd	          5 }tK          j&        |          }|                    dg           }|D ]}tS          |d                   }|d         }||k    r|}|                    d	          }||         t2          j         k    r[||         |k    rOt
                              d| d||         *                    d	          d|*                    d	                     |||<   d||<   t2          j'        ||<   |                    d          rt2          j        ||<   	 d d d            n# 1 swxY w Y   | j+        ,                    d           | j+        -                    d           | j+        .                    |           | j+        /                    |           | j+        0                    |           | j+        1                    |           te          j3        | j        ti          |                     }|j5        d!         }|+||j5        d!<   t
                              d"| d#| d$           |6                    | j+                   d S )%Nr   r  sentencepiece_model_pb2r  r  r   rbr  r   r  zInternLM2 convert token 'u   ' to '🐉'!u   🐉z[UNUSEDr  r   r   r  
<|im_end|>r  r  r  r  rt  r  r  r  r  r  r  r  r  r  zReplace eos:z with a special token:z8 in chat mode so that the conversation can end normally.)7r  r  r0  r:   r   rd   r  r
  exit
ModelProtoParseFromStringr   r  normalizer_specadd_dummy_prefixr  r   rG   r   r  r  r  r  r  r>  r   r'   r  r(   r  r)   r  r+   r  r,   rm  rn  r   r   r*   rl   rQ   r  rI   r  r  r  r  r  r]  rf   r  r   r  r  ) rq   r  rs   r  r  r  r  sentencepiece_model
add_prefixr  r  r  r  r  r  r  r  r   r  r   chat_eos_tokenchat_eos_token_idr  r  r  r  r  r  r  r  r  old_eoss                                    r/   r  zInternLM2Model.set_vocab  s   
 	988888BBBBBB*;;   %%'' 	LL;>;;<<<HQKKK#..00++D,F,F,K,K,M,MNNN(8I
**,,	s>22333\%%lI4H4H4J4JKK
j)) 	% 	%H''11E<<((D&&x00Ew M4MMMNNN}}W---4G""8,, 719$$X.. 719##H-- 718!!(++ 716	** 918MM$MM%   OOG$$$$ N-@@$$&& 	J'w??? J1$(IaLL!, J JCMM#**W"5"5666MM'***OO$;$HIIIIJJ J J J J J J J J J J J J J J &  $1H H ((** 	M+S7CCC Mq(,	!%'<'@'@AWY['\'\$,@,F,F,H,H M M(Hj"8}}H&y1E..,4)!LL11E)-D-KKK!(+u44"NN  ,Lh  ,L  ,L&QYJZJaJabiJjJj  ,L  ,Lrwr~r~  @G  sH  sH  ,L  ,L  M  M  M',F8$'.F8$)@)MHX&!~~i00 M-D-L*MM M M M M M M M M M M M M M M$ *::!!## 	MncG<<< M!%1-11."EE". M MJ":d#344H&y1E..,4)!LL11E)-D-KKK!(+u44"NN  ,Lh  ,L  ,L&QYJZJaJabiJjJj  ,L  ,Lrwr~r~  @G  sH  sH  ,L  ,L  M  M  M',F8$'.F8$)@)MHX&!~~i00 M-D-L*MM M M M M M M M M M M M M M M$ 	,,W555**9555''///))&111((222--j999)$.#f++NNN1%8( 6GM+E2NN V' V VIZ V V V W W W 	!!$"233333s9   ?A6MM	MD'R66R:=R:2DXXXc                   | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j         	                    | j        d                    | j         
                    | j                   | j                            d	          pi }|                    d
|                    d                    dk    rOd|v rM| j                             t          j        j                   | j                             |d                    d S d S d S )Nr  r  r  r  r  r  r  r  r  r  rz   r  r  )rI   r  rG   r  r  r  r  r  r  r  r  r<   r   r  rf   r  r  r  )rq   r  s     r/   r   z"InternLM2Model.set_gguf_parameters  s   ++DL9R,STTT((6I)JKKK--dl=.IJJJ00>Q1RSSS++DL,FGGG''5J(KLLL//^0LMMM**4<8M+NOOO&&tz222|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r   r   rb   r   r   r   r\   r   c                   | j         d         }| j         d         }| j         d         }||z  }||z  }||z  }	|                    dd          }|                    d          s|                    d          rg S |[d| d	|v rR|}
|
                    |	|d
z   ||f          }
|
d d d |f         |
d d df         |
d d df         }}}t                              |                    d|j        d         f          ||          }t                              |                    d|j        d         f          ||          }|                    d|j        d         f          }|                     t          j	        j
        |          |f|                     t          j	        j        |          |f|                     t          j	        j        |          |fgS |                     |          |fgS )Nr  r  r  rw  r  r  ro  r  z.attention.wqkvr   r  r  )rG   r  rm  r  rI  rf  r  r   rf   r  r  r  r  r   )rq   r   rb   r   r  r  r  q_per_kvr  
num_groupsr  r  r   rI  s                 r/   r   zInternLM2Model.modify_tensors  s   L!67	|$9:m,,Y&(*
||-r22??5!! 	T__^%D%D 	I?CsCCCtKKC++z8a<6JKKC!!!ZxZ-(#aaae*c!!!R%j!qA ""199b!'"+->#?#?IVVA""199b!'"+->#?#?LYYA		2qwr{+,,A (():)A3GGK(():)A3GGK(():)A3GGK  ))$//<==r.   Nr  )
r$   r%   r&   rf   r  	INTERNLM2rP   r  r   r   r-   r.   r/   r-  r-    sW        *J|4 |4 |4|M M M> > > > > >r.   r-  InternLM3ForCausalLMc                  D     e Zd Zej        j        Zd Z fdZddZ	 xZ
S )InternLM3Modelc                   |                                  \  }}}| j                            d           | j                            d           | j                            |           | j                            |           | j                            |           t          j        | j	        t          |                    }| j	        dz  }|                                rt          |dd          5 }t          j        |          }d|v r | j                            |d                    d	|v r{|d	                                         D ]`\  }}	|	                    d
          rFt%          |          }|	d         }
|                    |
|           |
dk    rd|j        v r
||j        d<   ad d d            n# 1 swxY w Y   |                    | j                   d S )Nr  r  r  r  r   r   r   r\  r  r  r  r2  r  )r  rI   r  r  r  r  r  rf   r  r:   r   r   r   r   r   r]  rl   r   rQ   r  r  r  )rq   r  r  r  r  r  r   r  r  r  r  s              r/   r  zInternLM3Model.set_vocab:  s6   #'#C#C#E#E ,,W555**9555''///))&111((222)$.#f++NNN $1H H ((** 	R+S7CCC Rq(,	!%%)>>>$99:OPb:cddd)-BBB0EF\0]0c0c0e0e R R,*%>>)44 R'*8}}H$.y$9E)<<UHMMM$44-Ba9a9aIQ ? FR R R R R R R R R R R R R R R 	!!$"233333s   3B8F77F;>F;c                N   t                                                       | j        }| j                            |d                    |                    d          x}|d         |d         z  }| j                            |           | j                            d          pi }|                    d|                    d                    dk    rOd	|v rM| j                            t          j	        j
                   | j                            |d	                    d S d S d S ra  )r  r   rG   rI   rU  r   r~  r  rf   r  r  r  rb  s       r/   r   z"InternLM3Model.set_gguf_parametersX  s   ##%%%,''(=>>>J///H8}-9N1OOH11(;;;|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r   r   rb   r   r   r   r\   r   c                   | j         d         }| j                             d          }|                    dd          }|                    d          s|                    d          rg S |                    d          rt
                              |||          }|                    d          rt
                              |||          }|                     |          |fgS )	Nr  r  rw  r  r  ro  rx  ry  )rG   r   r  rm  r  rI  rf  r   r0  s         r/   r   zInternLM3Model.modify_tensorsf  s    34L$$%:;;	||-r22??5!! 	T__^%D%D 	I==9:: 	H#++JGGJ==9:: 	K#++J	JJJ%%d++Z899r.   r  )r$   r%   r&   rf   r  r  rP   r  r   r   r9  r:  s   @r/   rD  rD  6  sj        &J4 4 4<M M M M M: : : : : : : :r.   rD  	BertModelBertForMaskedLMCamembertModelBertForSequenceClassificationc                  ^     e Zd Zej        j        Z fdZ fdZd Z	ddZ
ddZddZ xZS )rH  c                     t                      j        |i | d | _        | j                            d          x}r!t          |          dk    r|d         dk    rd }|| _        d S )Nid2labelr   r   LABEL_0)r  r   r  rG   r   r   cls_out_labels)rq   r  r  rP  r  s       r/   r   zBertModel.__init__x  sx    $)&)))!\--j999> 	&>""a''N1,=,J,J!%,r.   c                F   t                                                       | j                            d           |                                  | j        rJ| j                            d t          | j                                                  D                        d S d S )NFc                    g | ]\  }}|S r-   r-   rH  s      r/   r  z1BertModel.set_gguf_parameters.<locals>.<listcomp>  s    :m:m:mA1:m:m:mr.   )	r  r   rI   r  r5  rP  add_classifier_output_labelsr   rl   r  s    r/   r   zBertModel.set_gguf_parameters  s    ##%%%--e444""$$$ 	o99:m:mPTPcPiPiPkPkIlIl:m:m:mnnnnn	o 	or.   c                X   |                                  \  }}}t          |          | _        | j                            | j                            dd                     d }t          t          ||                    }| j        	                    d           | j        
                    |           | j                            |           | j                            |           t          j        | j        t          |                    }|                    | j                   d S )Ntype_vocab_sizer   c                    |                      d          r|                     d          r| S |                      d          r
| dd          S d| z   S )N[r  z##r   u   ▁)rm  r  )toks    r/   phantomz$BertModel.set_vocab.<locals>.phantom  sY    ~~c"" s||C'8'8 
~~d## 122wc>!r.   bertr  )r!  r   r  rI   add_token_type_countrG   r   r   r  r  r  r  r  rf   r  r:   r  )rq   r  r  r  rY  r  s         r/   r  zBertModel.set_vocab  s   #'#6#6#8#8 &f++
 	--dl.>.>?PRS.T.TUUU	" 	" 	" c'6**++ 	,,V444**6222''///((222 )$.#f++NNN!!$"233333r.   r   r   rb   r   r   r   r\   r   c                   ~|                     d          r
|dd          }|                    d          r|d d         dz   }|                    d          r|d d         dz   }|d	v rg S |                     d
          rg S |                     d          rg S | j        r|dk    rd}|dk    rd}|                     |          |fgS )Nzbert.r"   z.gammair   z.betar   )zembeddings.position_idszpooler.dense.weightzpooler.dense.biaszcls.predictionszcls.seq_relationshipzclassifier.weightzclassifier.out_proj.weightzclassifier.biaszclassifier.out_proj.bias)rm  r  rP  r   r   s       r/   r   zBertModel.modify_tensors  s   ??7## 	8D=="" 	)9y(D==!! 	'9w&D ZZZI??,-- 	I??122 	I 	2***3(((1%%d++Z899r.   r  c                    | j                             d          x}1d|z   | _        d| j         v r| j         dxx         | j        z  cc<   d S d S d | _        d S Npad_token_idr   r  )rG   r   _position_offset)rq   r`  s     r/   _xlmroberta_tokenizer_initz$BertModel._xlmroberta_tokenizer_init  sw     L,,^<<<LI$%$4D!(DL8867774;PP77777 98 %)D!!!r.   c                   dt           j        d<   ddlm} ddlm} | j        dz  }i }i }|                                sD| j        dz  }| j        dz  }|                                st          d	|           dd
lm	} ddl
m} |                    | j                  }	t          |dd          5 }
t          j        |
          }d d d            n# 1 swxY w Y   |                                r>t          |dd          5 }
t          j        |
          }d d d            n# 1 swxY w Y   |	j        }|	j        } ||d         d                   }t%          | j                            dd          |	j                  }n|                                }|                    t          |d                                                     |j        j        dk    sJ |j        j        }|j        j        }|j        j        } |            }	|	                    tA          |                     t%          | j                            dd          |	                                          }d tC          |          D             }dg|z  }tD          j#        g|z  }tI          |	|          rtC          |	                                          D ]}|	%                    |          }|&                    d          }|	'                    |          }tD          j(        }|	)                    |          rtD          j*        }ne|	+                    |          rtD          j,        }nC|	-                    |          rtD          j#        }n!|	.                    |          rtD          j/        }|||<   |||<   |||<   n1|	0                                }|                    d          }|                    ||d                             dd                    }tC          |	j                  D ]}|	1                    |          }|	1                    |          x}|&                    d          }|d         d         |         d         }tD          j(        }||k    rtD          j*        }n8||	j2        v rtD          j,        }n"||3                                v rtD          j4        }|||<   |||<   |||<   tI          |	|          rg d|dd         z   }g d|dd         z   }tD          j,        tD          j,        tD          j,        tD          j*        g|dd         z   }| j5        tl          j7        j8        k    rd|d<   d |d<   tD          j,        |d<   | j9        :                    d!           | j9        ;                    d"           | j9        <                    |           | j9        =                    |           | j9        >                    |           | j9        ?                    |           | j9        @                    | j                            d#d                     | j9        A                    |           |r| j9        B                    |           tm          jC        | j        t          |          $          }|E                    | j9                   d S )%Npython&PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATIONr   r  r/  zsentencepiece.bpe.modelr  r  r  )	b64decoder  r   r   r   
normalizerprecompiled_charsmapr  r1  r   c                @    g | ]}d | d                     d          S r  r  r  s     r/   r  z3BertModel._xlmroberta_set_vocab.<locals>.<listcomp>  r  r.   r  r  rs   rS  r    r  )s   <s>s   <pad>s   </s>s   <unk>r  )r  r  r  r  s   <mask>i r  t5r  rU  r  )Frk  environr  r  r0  r:   r   r  base64rf  r  r   ry  r   r   r   r\  clean_up_tokenization_spacesr  rG   r   r  r4  r5  r  trainer_specr  r6  r7  remove_extra_whitespacesrh  r  r   r  r   r+   r   r  r  r  r'   r  r(   r  r)   r  r  r,   r  _convert_id_to_tokenall_special_idsr  r*   rP   rf   r  NOMIC_BERT_MOErI   r  r  r  r  r  r]  r[  add_remove_extra_whitespacesadd_precompiled_charsmapr  r   r  )rq   r  rs   r  r  r  r  rf  r   r  fpr9  remove_whitespacesrh  r  r8  r  r  r  r  r  r  r  r  r  r  unk_token_idr  s                               r/   _xlmroberta_set_vocabzBertModel._xlmroberta_set_vocab  sZ    @H
;<888888BBBBBB*CC "%%'' #	X!^.>>N$(N5L$L!!))++ M'(K>(K(KLLL((((((222222%55dnEEIncG<<< /!%2/ / / / / / / / / / / / / / / %,,.. :/wGGG :2,0IbMM): : : : : : : : : : : : : : : #3J!*!G#,9^L-IJ`-a#b#b T\--lA>>	@TUUJJ"'"2"2"4"4//^T0J0J0O0O0Q0QRRR&3>!CCCC,<MJ!4!D!]#6#F#[ ..00I""3~#6#6777T\--lA>>	@T@T@V@VWWJVVE*DUDUVVV'j:56=>Ki!788 +	1!)"6"6"8"899 - -!++H55||G,,!**84418&&x00 ;5=GG((22 ;5=GG''11 ;5<GG%%h// ;5:G#'x #(x %,""#-& $3355K-11+>>I&??9nW6M6Q6QRZ\]6^6^__L!)"677 1 1!66x@@&;;HEEEER <<00D*73G<XFqIE5<G<//"9"A!Y%>>>"9"A![%7%7%9%999"9"F
 (,F8$',F8$)0HX&i!788 	C:::VAbD\IF)))F1R4L8F'/'/'/'/	
 2H $/"@@@!*v!$v#:#B ,,T222**9555''///))&111((222--j999--dl.>.>?PRS.T.TUUU556HIII 	L556JKKK)$.#f++NNN!!$"233333s$   1CCCD$$D(+D(r  r8  )r$   r%   r&   rf   r  BERTrP   r   r   r  r   rb  rx  r9  r:  s   @r/   rH  rH  t  s        %J- - - - -o o o o o4 4 48: : : :@) ) ) )~4 ~4 ~4 ~4 ~4 ~4 ~4 ~4r.   DistilBertModelDistilBertForMaskedLM#DistilBertForSequenceClassificationc                  B     e Zd Zej        j        Z fdZd fd
Z xZ	S )rz  c                    | j                             d           t                              d           t	                                                       d S )Ng-q=z gguf: layer norm epsilon = 1e-12)rI   r  rd   re   r  r   r  s    r/   r   z#DistilBertModel.set_gguf_parametersT  sI    ++E2226777##%%%%%r.   r   r   rb   r   r   r   r\   r   c                    |                     d          r
|dd          }|                     d          rg S t                                          |||          S )Nzdistilbert.   vocab_rm  r  r   r  s       r/   r   zDistilBertModel.modify_tensorsY  s[    ??=)) 	9D ??8$$ 	Iww%%j$<<<r.   r  )
r$   r%   r&   rf   r  ry  rP   r   r   r9  r:  s   @r/   rz  rz  P  s`        %J& & & & &
= = = = = = = = = =r.   RobertaModel RobertaForSequenceClassificationc                  L     e Zd Zej        j        Z fdZ fdZd fdZ	 xZ
S )r  c                     t                      j        |i | | j                            d          x}1d|z   | _        d| j        v r| j        dxx         | j        z  cc<   d S d S d | _        d S r_  )r  r   rG   r   ra  )rq   r  r  r`  r  s       r/   r   zRobertaModel.__init__h  s    $)&))) !L,,^<<<LI$%$4D!(DL8867774;PP77777 98 %)D!!!r.   c                   | j         dz  }|                                rI|                                  | j                            | j                            dd                     dS t                                                      S )z)Support BPE tokenizers for roberta modelsr  rU  r   N)	r:   existsr  rI   r[  rG   r   r  r  )rq   bpe_tok_pathr  s     r/   r  zRobertaModel.set_vocabs  s    ~(88   		'  """
 11$,2B2BCTVW2X2XYYYYY 77$$&&&r.   r   r   rb   r   r   r   r\   r   c                    |                     d          r
|dd          }|dk    r| j        || j        d d d f         }t                                          |||          S Nzroberta.r  z%embeddings.position_embeddings.weightrm  ra  r  r   r  s       r/   r   zRobertaModel.modify_tensors  u     ??:&& 	8D :::$0'(=(>(>qqq(@A
ww%%j$<<<r.   r  r$   r%   r&   rf   r  ry  rP   r   r  r   r9  r:  s   @r/   r  r  d  sy        %J	) 	) 	) 	) 	)' ' ' ' '= = = = = = = = = =r.   NomicBertModelc                  ^     e Zd Zej        j        Zd fdZd fdZddZ	 fdZ
ddZ xZS )r  r:   r   r<   r;   r=   r  r   c                   |                     dd           }|t                              |d          }t          |                    d                    | _        | j        rt          j        j        nt          j        j	        | _
         t                      j        |||fd|i| |                                 | _        | j        r|                                  | j        d         | j                            dd          }}|dk    r|dk    rd| j        d<   n,|dk    r|dk    rd| j        d<   nt#          d	| d
|           | j        r| j        d         dk    snnJ | j        d         du sJ | j        d         | j        k    sJ | j        d         | j        k    sJ | j        d         | j        k    sJ | j        d         du sJ | j        d         dk    sJ | j        d         du sJ | j        d         J d S )NrG   Fmoe_every_n_layersr  max_trained_positionsr  r     z%unrecognized parameters: n_positions=z, max_trained_positions=activation_functionr  swiglucausalqkv_proj_biasmlp_fc1_biasmlp_fc2_biasprenormrotary_emb_fractionr  rotary_emb_interleavedrotary_emb_scale_base)popr8   r   r>   r   is_moerf   r  rr  
NOMIC_BERTrP   r  r   _is_tokenizer_xlmroberta_tokenizer_is_xlmrobertarb  rG   r   )	rq   r:   r<   r=   r  rG   nposmtpr  s	           r/   r   zNomicBertModel.__init__  s:   **Y--?,,Y>>G7;;';<<==<@Kg$/88T_MgE9PPgPPPP(,(E(E(G(G%( 	.++---L/1A1ABY[_1`1`c4<<C4KK*.DL''T\\cTkk*-DL''hThhcfhhiii@DYt|12f<<<<<Y |H%....|O,;;;;|N+;;;;|N+;;;; |I&%////|12c9999|45>>>>|34<<<<<r.   r\   r  c                z    | j         r|                                 S t                                                      S r`   )r  rx  r  r  r  s    r/   r  zNomicBertModel.set_vocab  s6    ( 	0--///ww  """r.   r   torch.Tensorrb   r   r   r   "Iterable[tuple[str, torch.Tensor]]c                n   d|v rg S d|v r=|                     | j        d         | j        d         | j        d                   }|dz  }d|v rS|                     | j        d         | j        d         | j        d                   }|                    dd	          }|dz  }|                     |          |fgS )
Nzmlp.experts.biaszmlp.experts.mlp.w1r  r  r  r   zmlp.experts.mlp.w2r   r   )r  rG   r  r   r   s       r/   r   zNomicBertModel.modify_tensors  s    %%I4''#m)DdlS\F]_c_klt_uvvJID4''#m)DdlS\F]_c_klt_uvvJ#--a33JID%%d++Z899r.   c                   t                                                       | j                            | j        d                    | j        rq| j                            | j        d                    | j                            | j        d                    | j                            | j        d                    d S d S )Nr  r  r  r]  )	r  r   rI   r  rG   r  add_moe_every_n_layersr  r  r  s    r/   r   z"NomicBertModel.set_gguf_parameters  s    ##%%%++DL9J,KLLL; 	N33DLAU4VWWW--dl=.IJJJ224<3LMMMMM	N 	Nr.   r>   c                    t          | j        dz            5 }t          j        |          }d d d            n# 1 swxY w Y   |d         d         }|dk    rdS |dk    rdS t	          d|           )	Nr  rs   rz   UnigramT	WordPieceFzunknown tokenizer: )r   r:   r   r   r   )rq   r   r  toktyps       r/   r  z'NomicBertModel._is_tokenizer_xlmroberta  s    $.#3344 	*!Yq\\N	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*(0Y4[  57v77888s   9= =)r:   r   r<   r;   r=   r   r  r   r8  )r   r  rb   r   r   r   r\   r  )r\   r>   )r$   r%   r&   rf   r  ry  rP   r   r  r   r   r  r9  r:  s   @r/   r  r    s        %J$= $= $= $= $= $=L# # # # # #
: : : : N N N N N9 9 9 9 9 9 9 9r.   NeoBERTNeoBERTLMHead NeoBERTForSequenceClassificationc                  @     e Zd Zej        j        Z fdZ fdZ xZ	S )NeoBertc                4   t                                                       | j                            t	          d| j        d         z  dz                       | j                            d           | j                            t          j	        j
                   | j                            dd          }| j                            |           t                              d|            | j                            t          j        j                   d S )Nr   r  r    r  r  r-  r  )r  r   rI   r  rQ   rG   r  r  rf   r  NONEr   r  rd   re   r.  r*  r,  )rq   r  r  s     r/   r   zNeoBert.set_gguf_parameters  s    ##%%% 	00QFY9Z5Z]^5^1_1_```++G444..t/C/HIIIL$$Z66	//	:::;	;;<<<))$*:*>?????r.   c                    |                     d          rg S |                     d          r
|dd          }t                                          |||          S )Nzdecoder.rv  r#   r  r  s       r/   r   zNeoBert.modify_tensors  sY    ??:&& 	I??8$$ 	8Dww%%j$<<<r.   )
r$   r%   r&   rf   r  NEO_BERTrP   r   r   r9  r:  s   @r/   r  r    s`        )J@ @ @ @ @= = = = = = = = =r.   r  XLMRobertaModel#XLMRobertaForSequenceClassificationc                  H     e Zd Zej        j        Z fdZd Zd fdZ	 xZ
S )r  c                b     t                      j        |i | |                                  d S r`   )r  r   rb  r  s      r/   r   zXLMRobertaModel.__init__  s7    $)&)))'')))))r.   c                .    |                                   d S r`   )rx  r   s    r/   r  zXLMRobertaModel.set_vocab      ""$$$$$r.   r   r   rb   r   r   r   r\   r   c                    |                     d          r
|dd          }|dk    r| j        || j        d d d f         }t                                          |||          S r  r  r  s       r/   r   zXLMRobertaModel.modify_tensors  r  r.   r  r  r:  s   @r/   r  r    so        %J* * * * *% % %= = = = = = = = = =r.   GemmaForCausalLMc                  :    e Zd Zej        j        Zd Zd ZddZ	dS )
GemmaModelc                   |                                   t          j        | j        dg d          }|                    dd           |                    dd           |                    dd	           |                    d
d           |                    dd           d |_        |                    | j                   | j                            d           d S )NF)r   r   r  fsepr  r  r   C   r   E   r  D   r  F   r  k   )	r  rf   r  r:   r  r  r  rI   r]  )rq   r  s     r/   r  zGemmaModel.set_vocab  s    %%''' )$.e@m@m@mo o o((2666((2666((2666((2666((3777&*#!!$"2333--e44444r.   c                   | j         }|d         }| j                            |d                    | j                            |d                    | j                            |           | j                            |d                    | j                            |d                    | j                            d|v r| j         d         n|d                    | j                            | j         d                    | j        	                    |d                    | j        
                    |d                    | j                            | j                   d S )	Nr  r  r  r  r  r  r  r  )rG   rI   r  r  r  r  r  r  r  r  r  r  r<   r  s      r/   r   zGemmaModel.set_gguf_parameters*  s[   ,12++G4M,NOOO--gm.DEEE((555009L1MNNN''0E(FGGG**RgkrRrRr4<8M+N+Nx  AV  yW  	X  	X  	X//^0LMMM''
(;<<<))'**=>>>&&tz22222r.   r   r   rb   r   r   r   r\   r   c                    ~|dk    r t                               d|d           g S |                    d          r|dz   }|                     |          |fgS Nlm_head.weightzSkipping get tensor 1 in safetensors so that convert can end normally.norm.weightr   rd   r  r  r   r   s       r/   r   zGemmaModel.modify_tensors9  q     ###LLiiiijjjI =='' 	(#aJ%%d++Z899r.   Nr  )
r$   r%   r&   rf   r  GEMMArP   r  r   r   r-   r.   r/   r  r    sP        &J5 5 5 3 3 3: : : : : :r.   r  Gemma2ForCausalLMc                  :    e Zd Zej        j        Zd Zd ZddZ	dS )Gemma2Modelc                b    |                                   | j                            d           d S r   r  rI   r]  r   s    r/   r  zGemma2Model.set_vocabM  1    %%'''--e44444r.   c                   | j         }|d         }| j                            |d                    | j                            |d                    | j                            |           | j                            |d                    | j                            |d                    | j                            d|v r| j         d         n|d                    | j                            | j         d                    | j        	                    |d                    | j        
                    |d                    | j                            | j                   | j                            | j         d	                    | j                            | j         d
                    | j                            | j         d                    d S )Nr  r  r  r  r  r  r  r  attn_logit_softcappingfinal_logit_softcappingr  )rG   rI   r  r  r  r  r  r  r  r  r  r  r<   add_attn_logit_softcappingadd_final_logit_softcappingr  r  s      r/   r   zGemma2Model.set_gguf_parametersR  s   ,12++G4M,NOOO--gm.DEEE((555009L1MNNN''0E(FGGG**RgkrRrRr4<8M+N+Nx  AV  yW  	X  	X  	X//^0LMMM''
(;<<<))'**=>>>&&tz22233L12	
 	
 	
 	44L23	
 	
 	
 	++DL9I,JKKKKKr.   r   r   rb   r   r   r   r\   r   c                    ~|dk    r t                               d|d           g S |                    d          r|dz   }|                     |          |fgS r  r  r   s       r/   r   zGemma2Model.modify_tensorsh  r  r.   Nr  )
r$   r%   r&   rf   r  GEMMA2rP   r  r   r   r-   r.   r/   r  r  I  sS        'J5 5 5
L L L,: : : : : :r.   r  Gemma3ForCausalLMGemma3ForConditionalGenerationc                  >    e Zd Zej        j        ZdZd Zd Z	ddZ
dS )Gemma3Modelr  c                b    |                                   | j                            d           d S r   r  r   s    r/   r  zGemma3Model.set_vocab}  r  r.   c                   | j         }|d         }| j                            |                    dd                     | j                            |d                    | j                            |           | j                            |d                    | j                            |                    dd                     | j                            | j                             dd	                     | j        	                    |                    d
d                     | j        
                    |                    d
d                     | j                            | j                   | j                            |                    dd                     |                    d          J | j                            |d                    | j                            |                    dd                     |                    d          e|d         d         dk    sJ | j                            t"          j        j                   | j                            |d         d                    d S d S )Nr  r  i   r  r  r  r  r  r-  r  r  r  g    .Ar  r  r  r!   r  r  r  r  )rG   rI   r  r   r  r  r  r  r  r  r  r  r<   r  r  r  r  rf   r  r  r  r  s      r/   r   zGemma3Model.set_gguf_parameters  s:   ,12 	++GKK8QSY,Z,Z[[[--gm.DEEE((555009L1MNNN''4I1(M(MNNN//0@0@QU0V0VWWW''J(D(DEEE))'++j#*F*FGGG&&tz222++GKKk,R,RSSS{{344<<<++G4D,EFFF**7;;7La+P+PQQQ;;~&&2>*;78CCCC2243G3NOOO44W^5LX5VWWWWW	 32r.   r   r   rb   r   r   r   r\   r   c                   ~d|v r|                     dd          }nV|                    d          s?|                    d          s*|                    d          s|                    d          rg S d|v r3|                                 }|d         }|d t          |                   }|                    d	          r
|| j        z   }|                     |          |fgS )
Nrw  r  r  r  multimodal_projector.vision_model.r  r   r  )r  rm  r  r   r  
norm_shiftr   )rq   r   rb   r   r  r  s         r/   r   zGemma3Model.modify_tensors  s    $$<< 1266DD__566 	$///:Z:Z 	??#:;;	?C?_?_	I !D((4466E1XF#LS[[L1J
 =='' 	6#do5J%%d++Z899r.   Nr  )r$   r%   r&   rf   r  GEMMA3rP   r  r  r   r   r-   r.   r/   r  r  x  sX        'JJ5 5 5
X X X2: : : : : :r.   r  c                  ,     e Zd Z fdZd ZddZ xZS )Gemma3VisionModelc                *   t                                                       | j        }| j                            t
          j        j                   | j                            |	                    dd                     | j        
                    d           | j        	                    dd          }t          |dz            }| j        d         }| j        d         }||z  |z  }|d	k    r"|d
k    r| j                            |           d S d S d S )Nr  r-  Timage_seq_lengthr  rk  rY  rZ  r   r!   )r  r   rG   rI   r  rf   r  r  r  r   r  r<  rQ   r  )rq   rG   r  
n_per_siderY  rZ  proj_scale_factorr  s          r/   r   z%Gemma3VisionModel.set_gguf_parameters  s   ##%%%,001I1PQQQ;;GKKHXZ^<_<_```,,T2223778JCPP)S011
\,/
\,/
':5*Dq  %6!%;%; >>?PQQQQQ ! %;%;r.   c                `    ~~~d|v rt           j        j        S d|v rt           j        j        S dS )Ninput_projectionr  Fr4  r   s        r/   r   z$Gemma3VisionModel.tensor_force_quant  s<    6%%,00T!!,00ur.   r   r   rb   r   r   r   r\   r   c                f   ~d|v rg S |                     d          s?|                     d          s*|                     d          s|                     d          rU|                    dd          }d|v r#t                              d	| d
           |dz   }|                     |          |fgS g S )Nzvision_model.head.r  r  r  r  _weightr   zsoft_emb_norm.weightzCorrecting norm value for 'r   r   )rm  r  rd   re   r   r   s       r/   r   z Gemma3VisionModel.modify_tensors  s    4''I??344 	>8X8X 	>??#:;;	>?C?_?_	> <<	955D
 &--A$AAABBB'!^
))$//<==	r.   r  )r$   r%   r&   r   r   r   r9  r:  s   @r/   r  r    s`        R R R R R$         r.   r  Gemma3nForConditionalGenerationc                       e Zd ZU ej        j        ZdZg Zde	d<   g Z
de	d<    fdZ fdZ fdZddZd fdZ xZS )Gemma3NModelr  list[Tensor]_altup_proj_altup_unembdc                d    t                      j        |i | | j        d         dk    s
J d            t          j                    t          j                    t          j                    g| _        t          j                    t          j                    t          j                    g| _        d S )Naltup_num_inputsr!   z/Current conversion only supports 4 altup inputs)r  r   rG   r   r   r  r  r  s      r/   r   zGemma3NModel.__init__  s    $)&)))|./14446g444LNNLNNLNN
 LNNLNNLNN
r.   c                H    t                                                       d S r`   )r  r  r  s    r/   r  zGemma3NModel.set_vocab  s    r.   c                P   t                                                       | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    g }| j        d         D ]}t          j	        j
                            dd          }|                    t          j        |t          j                            }|                    |                                           | j                            |           g }| j        d	         D ]}|                    |d
k               | j                            |           d S )Naltup_active_idxr  hidden_size_per_layer_inputnum_kv_shared_layersactivation_sparsity_patternr   r   r  layer_typessliding_attention)r  r   rI   add_altup_active_idxrG   add_altup_num_inputs$add_embedding_length_per_layer_inputadd_shared_kv_layersr   distributionsnormalNormalicdfr  r  rn  itemadd_activation_sparsity_scaleadd_sliding_window_pattern)rq   activation_sparsity_scaler   normal_diststd_multipliersliding_window_patterntr  s          r/   r   z Gemma3NModel.set_gguf_parameters  s   ##%%%--dl;M.NOOO--dl;M.NOOO==dlKh>ijjj--dl;Q.RSSS$&!;< 	D 	DA-4;;AqAAK(--el1EM.R.R.RSSN%,,^-@-@-B-BCCCC667PQQQ!#m, 	D 	DA"))!/B*BCCCC334JKKKKKr.   matricesr\   Tensor | Nonec                h    t          d |D                       }|sd S t          j        |d          S )Nc              3  F   K   | ]}|                                 d k    V  dS r   N)numel)ra   ms     r/   rc   z/Gemma3NModel._stack_matrices.<locals>.<genexpr>  s.      66aggii!m666666r.   r   r  )allr   r5  )rq   r  has_alls      r/   _stack_matriceszGemma3NModel._stack_matrices  s@    66X66666 	04;xQ////r.   r   r   rb   r   r   r   r   c                   |                     d          r|dz   }d|vrg S d|v r|                    d          }d|v r|| j        d<   n0d	|v r|| j        d
<   n!d|v r|| j        d<   nt          d|           |                     | j                  }||                     d          |fgS g S d|v r|                    d          }d|v r|| j        d<   n0d	|v r|| j        d
<   n!d|v r|| j        d<   nt          d|           |                     | j                  }||                     d          |fgS g S t                                          |||          S )N_scaler   rw  altup_unembed_projectionsr   )r   z.0.r   z.1.r   z.2.r   zUnknown name: z&model.altup_unembed_projections.weightaltup_projectionszmodel.altup_projections.weight)	r  r  r  r   r  r   r  r  r   )rq   r   rb   r   outr  s        r/   r   zGemma3NModel.modify_tensors  s   =="" 	$)#D D((I&$..#e44J}}(2"1%%$(2"1%%$(2"1%% !8$!8!8999&&t'9::C--.VWWY\]^^	$&&#e44J}}&0 ##$&0 ##$&0 ## !8$!8!8999&&t'788C--.NOOQTUVV	ww%%j$<<<r.   )r  r  r\   r  r  )r$   r%   r&   rf   r  GEMMA3NrP   r  r  r  r  r   r  r   r  r   r9  r:  s   @r/   r  r    s         (JJ "K"""""$M$$$$
 
 
 
 
    L L L L L&0 0 0 0)= )= )= )= )= )= )= )= )= )=r.   r  Starcoder2ForCausalLMc                  &    e Zd Zej        j        ZdS )StarCoder2ModelN)r$   r%   r&   rf   r  
STARCODER2rP   r-   r.   r/   r'  r'  G  s        +JJJr.   r'  Rwkv6ForCausalLMc                  J    e Zd ZU ej        j        Zd Zd Zi Z	de
d<   ddZdS )
Rwkv6Modelc                .    |                                   d S r`   r  r   s    r/   r  zRwkv6Model.set_vocabP  r  r.   c                Z   | j         d         }| j         d         }| j         d         }| j         d         }| j         d         }| j         d         | j         d         nt          |dz  dz  dz            }|d	k    rd
nd}|d	k    rdnd
}| j                            d           | j                            |           | j                            |           | j                            |           | j                            |           | j                            |           | j        	                    |           | j        
                    |           | j                            |           | j                            | j                   | j                            d           d S )Nr  	head_sizer  r  rescale_everyr  g      @rL  r  r  rJ     r   )rG   rQ   rI   r  r  r  r  add_rescale_every_n_layersadd_wkv_head_sizeadd_time_mix_extra_dimadd_time_decay_extra_dimr  r  r<   r  )	rq   rR   r/  r  r  rescale_every_n_layersr  time_mix_extra_dimtime_decay_extra_dims	            r/   r   zRwkv6Model.set_gguf_parametersS  s   l#67L-	l=1&:;!%o!>AENaAbAnDL)<==tw  zE  HK  zK  PR  yR  UW  yW  uX  uX#.$#6#6RRB&1T&9&9ssr 	++G444--k:::((555++N;;;334JKKK**9555//0BCCC112FGGG001BCCC&&tz222 	''*****r.   dict[int, dict[str, Tensor]]lerp_weightsr   r   rb   r   r   r   r\   r   c              #     K                         |          }|                    d          s|                    d          s|dz  }|                    d          s*|                    d          s|                    d          r|                    dd          }|                    d          r|                    dd	d          }|                    d
          sd|v r|                                }	  j        d         }|dk    rR|                    d          s|                    d          r(|                    d	t          |z            z            }n# t          $ r Y nw xY wd|v rd|vr	 | j	                 |<   n# t          $ r ||i j	        <   Y nw xY wt           fddD                       rBd d}t          j         fddD             d                              d          }||fV  d S ||fV  d S )Nr   r   time_mix_w1.weightztime_mix_decay_w1.weightztime_mix_decay_w2.weightr   r   time_mix_w2.weightr   ztime_mix_decay.weightlerpr0  ztime_mix_output.weightzchannel_mix_value.weighttime_mix_lerptime_mix_lerp_xc              3  h   K   | ],}d  d| dj                                                  v V  -dS )blk..time_mix_lerp_r   Nr:  rk   ra   r  r   rq   s     r/   rc   z,Rwkv6Model.modify_tensors.<locals>.<genexpr>  sQ      }}^_8#88a888D<Mc<R<W<W<Y<YY}}}}}}r.   )wr   rI  r   grB  .time_mix_lerp_fused.weightc                j    g | ]/}j                  d  d| d                             d          0S )rB  rC  r   r   )r:  rf  rE  s     r/   r  z-Rwkv6Model.modify_tensors.<locals>.<listcomp>  s]      $S  $S  $SstD$5c$:;`#;`;`VW;`;`;`$a$k$klm$n$n  $S  $S  $Sr.   r  )r   r  r  rf  r  rG   div_rQ   r   r:  r  r   r5  rf  )rq   r   rb   r   r   r6  r   s   `  `   r/   r   zRwkv6Model.modify_tensorsn  s     ''--!!),, 	"0A0A'0J0J 	"	!H122 	4h6G6GHb6c6c 	4gogxgx  zT  hU  hU 	4#--a33J122 	5#++Aq!44J455 	.89K9K#++--J	%)\/%B"%))$$%=>> Z(BSBSTnBoBo Z!+c#AW:W6X6X1X!Y!YJ 	 	 	D	
 ?(::?PX`?`?`@3=!#&x00 @ @ @*2J)?!#&&&@}}}}}c|}}}}} 'B#BBB{  $S  $S  $S  $S  $S  yR  $S  $S  $S  YZ  [  [  [  e  e  fg  h  h&&&&F$$$$$$s%   8A%E 
E+*E+9F
 
F#"F#Nr  )r$   r%   r&   rf   r  RWKV6rP   r  r   r:  r  r   r-   r.   r/   r+  r+  L  se         &J% % %+ + +2 24L3333$% $% $% $% $% $%r.   r+  RWKV6Qwen2ForCausalLMc                  D     e Zd Zej        j        Zd Zd Zd fdZ	 xZ
S )RWKV6Qwen2Modelc                z    	 |                                   d S # t          $ r |                                  Y d S w xY wr`   r  r   s    r/   r  zRWKV6Qwen2Model.set_vocab  r  r  c                   | j         d         }| j         d         }| j         d         }| j         d         }||z  }| j         d         }| j         d         }| j                             d|dk    rd	nd
          }| j                             d|dk    rdnd	          }	| j                            d           | j                            |           | j                            |           | j                            |           | j                            |           | j                            |	           | j        	                    |           | j        
                    | j                   | j                            |           | j                            d           | j                            |           | j                            d           d S )Nr  r  r  r  r  r  lora_rank_tokenshiftr  r  rL  lora_rank_decayrJ  r1  r   r   )rG   r   rI   r  r  r  r3  r4  r5  r  r  r<   r  add_token_shift_countr  r  )
rq   rR   r  r  r  r/  r  r  r7  r8  s
             r/   r   z#RWKV6Qwen2Model.set_gguf_parameters  s   l#67"l+@A"l+@Al=1#66	|N3 L)<=!\--.DK[_L_L_bbeghh#|//0A+Y]J]J]33ceff 	++G444--k:::((555**9555//0BCCC112FGGG001BCCC&&tz222 	//===..q111**+>??? 	''*****r.   r   r   rb   r   r   r   r\   r   c           	   #    K   t                                          |||          D ]\  }}d|v sd|v r|                    dd|j        d                   }t	          j        |d         |d         |d         |d         |d	         gd
                              d|j        d                   }d|v r"|                    dd|j        d                   }||fV  ||fV  d S )Ntime_mix_w1time_mix_w2r"   r  r    r   r   r   r!   r  r~  )r  r   r  r  r   r5  )rq   r   rb   r   r   r   r  s         r/   r   zRWKV6Qwen2Model.modify_tensors  s	     #gg44ZsKK 
	# 
	#NHd((MX,E,EyyB
277 {DGT!Wd1gtAwQ#PVWXXX]]^`bfblmobpqq8##99QDJrN;;D&&&&T"""""
	# 
	#r.   r  )r$   r%   r&   rf   r  
RWKV6QWEN2rP   r  r   r   r9  r:  s   @r/   rN  rN    se        +J# # #+ + +<# # # # # # # # # #r.   rN  Rwkv7ForCausalLMRWKV7ForCausalLMc                  ^    e Zd ZU ej        j        Zd Zd Zd Z	i Z
ded<   dZded<   ddZdS )
Rwkv7Modelc                .    |                                   d S r`   r-  r   s    r/   r  zRwkv7Model.set_vocab  r  r.   c                T    t          dt          ||z  |z  dz                      dz  S )Nr   rL  )r  r=  )rq   r  exponentr  s       r/   calc_lora_rankzRwkv7Model.calc_lora_rank  s.    1eK83j@2EFFGG"LLr.   c                   | j         d         }	 | j         d         }| j         d         }n*# t          $ r | j         d         }| j         d         }Y nw xY w| j         d         }| j         d         | j         d         n|dz  }	 | j         d	         | j         d	         n|                     |d
d          }| j         d         | j         d         n|                     |d
d          }| j         d         | j         d         n|                     |d
d          }| j         d         | j         d         n|                     |dd          }	n# t          $ r | j         d         | j         d         n|                     |d
d          }| j         d         | j         d         n|                     |d
d          }| j         d         | j         d         n|                     |d
d          }| j         d         | j         d         n|                     |dd          }	Y nw xY w| j                            d           | j                            |           | j                            |           | j                            |           | j                            |           | j        	                    |           | j        
                    |           | j                            |           | j                            |	           | j                            |           | j                            | j                   | j                            d           d S )Nr  r/  r  r  r  r  r  r!   rR  rk  g?lora_rank_iclrlora_rank_value_residual_mixg?lora_rank_gateg?g333333?decay_low_rank_dima_low_rank_dimv_low_rank_dimgate_low_rank_dimr1  r   )rG   r   r_  rI   r  r  r  r  r3  add_decay_lora_rankadd_iclr_lora_rank add_value_residual_mix_lora_rankadd_gate_lora_rankr  r  r<   r  )
rq   rR   r/  r  r  r  rR  ra  rb  rc  s
             r/   r   zRwkv7Model.set_gguf_parameters  s   l#67	6[1I!\*>?NN 	6 	6 	6Z0I!\*5NNN	6 l=1AENaAbAnDL)<==  vA  DE  vE		`AEN_A`Aldl+<==rv  sF  sF  GR  TW  Y\  s]  s]O?C|L\?]?iT\*:;;os  pC  pC  DO  QT  VY  pZ  pZN[_[g  iG  \H  \T4<8V+W+W  Z^  Zm  Zm  ny  {~  @C  ZD  ZD(?C|L\?]?iT\*:;;os  pC  pC  DO  QT  VY  pZ  pZNN 	` 	` 	`DHLQeDfDrdl+?@@x|  yL  yL  MX  Z]  _b  yc  ycO?C|L\?]?iT\*:;;os  pC  pC  DO  QT  VY  pZ  pZNMQ\ZjMkMw4<8H+I+I  ~B  ~Q  ~Q  R]  _b  dg  ~h  ~h(BF,ObBcBoT\*=>>uy  vI  vI  JU  WZ  \_  v`  v`NNN		` 	++G444--k:::((555++N;;;**9555,,_===++N;;;99:VWWW++N;;;001BCCC&&tz222 	''*****s#   * $AACE CHHr9  r:  Tr>   lora_needs_transposer   r   rb   r   r   r   r\   r   c              #  x   K   |                     dd                               dd          }|                     dd                               dd          }|                     dd	          }d
|v rd _        |                     dd          }|                     dd          }|                     dd          }|                     dd          }|                     dd          }d|v rd                     |          vrdk    rd S  j                            dd          }|rg dng d}d|v rd|v r2d d |                    t          |          d!d!d"          }|fV  ny	 | j                 |<   n# t          $ r ||i j        <   Y nw xY wt           fd#|D                       r/d d t          j         fd$|D             d%          }|fV  d S |                                }                     |                              d&          s                    d'          sd&z   j        r1t          fd(d)D                       r|                    dd!          }d*v r|                                }dk    rd+v r                     d+d,          |fV  |fV  d S )-Nblockslayersr  feed_forward	self_attnr  attnztime_mixer.r  z
_lora.loraFz_lora.lora.0.weightz1.weightz_lora.lora.2.weightz2.weightz_lora.lora.2.biasz0.weightfeed_forward_normln2g_normln_xzattention.vvaluer   wkv_has_gateT)r   rF  r   rI  arG  )r   rF  r   rI  ry  zattention.x_zattention.x_xrB  rH  r   r  c              3  f   K   | ]+}d  d| j                                                  v V  ,dS )r  .attention.x_NrD  rE  s     r/   rc   z,Rwkv7Model.modify_tensors.<locals>.<genexpr>!  sO      qqbc<s<<<<@QRU@V@[@[@]@]]qqqqqqr.   c                B    g | ]}j                  d  d|          S )r  r{  )r:  rE  s     r/   r  z-Rwkv7Model.modify_tensors.<locals>.<listcomp>#  s9    'y'y'yjk(9#(>?ds?d?dab?d?d(e'y'y'yr.   r  r   r   c              3  B   K   | ]}                     |          V  d S r`   )r  )ra   r  r   s     r/   rc   z,Rwkv7Model.modify_tensors.<locals>.<genexpr>-  sB       1 1)*!!!$$1 1 1 1 1 1r.   )r<  r=  ztime_mix_a1.weightztime_mix_a2.weightztime_mix_v1.weightztime_mix_v2.weightztime_mix_g1.weightztime_mix_g2.weightr_k
time_mix_a
time_mix_v)r  rl  r   rG   r   r  r   r:  r   r  r   r5  r  r  r  r  flatten)rq   r   rb   r   rx  	lerp_listr   r   s   `  `   @r/   r   zRwkv7Model.modify_tensors  s     ||Hh//77~NN||K55==fkRR||M2..4(-D%||1:>>||1:>>||/<<||/77||Hf--D  WD4H4H4N4N%N%NSVZ[S[S[ F|''==6Ba2222HaHaHa	?~55$&&B#BBB!))#i..!QCC&&&&&@3=D%c*400 @ @ @.2J-?D%c***@qqqqqgpqqqqq +FcFFFH ;'y'y'y'y'yox'y'y'y  @A  B  B  BD#T****F#++--J++D11H%%i00 &H4E4Eg4N4N &I%( 8S 1 1 1 1/1 1 1 . . 8 (11!Q77
  '//11
axxLH44  ''lCCZPPPPZ((((((s   .E? ?FFNr  )r$   r%   r&   rf   r  RWKV7rP   r  r_  r   r:  r  rl  r   r-   r.   r/   r[  r[    s         &J% % %M M M%+ %+ %+N 24L3333!%%%%%@) @) @) @) @) @)r.   r[  RwkvHybridForCausalLMc                  2    e Zd Zej        j        Zd Zd ZdS )ARwkv7Modelc                z    	 |                                   d S # t          $ r |                                  Y d S w xY wr`   r  r   s    r/   r  zARwkv7Model.set_vocabF  r  r  c                   | j         d         }| j         d         }| j         d         }| j         d         }| j         d         }| j         d         }| j         d         dk    sJ d	}d	}d
}	|rdnd}
| j                            d           | j                            |           | j                            |           | j                            |           | j                            |           | j                            |           | j                            |           | j        	                    |	           | j        
                    |
           | j                            |           | j                            | j                   | j                            d           | j                            d           d S )Nr  r  r/  r  r  rx  wkv_version   r  rL  rJ  r   r1  r   )rG   rI   r  r  r  r  r3  rh  ri  rj  rk  r  r  r<   rS  r  )rq   rR   r  r/  r  r  rx  rR  ra  rb  rc  s              r/   r   zARwkv7Model.set_gguf_parametersL  s   l#67l=1L-	|N3 L)<=|N3|M*a//// ')$ ,3! 	++G444--k:::((555//===**9555,,_===++N;;;99:VWWW++N;;;001BCCC&&tz222..q111 	''*****r.   N)	r$   r%   r&   rf   r  ARWKV7rP   r  r   r-   r.   r/   r  r  B  s<        'J# # #+ + + + +r.   r  MambaForCausalLMMambaLMHeadModelFalconMambaForCausalLMc                  P     e Zd Zej        j        Zd fdZd Zd Z	dZ
ddZ xZS )
MambaModelr:   r   c                    |                     dd           }|At          |dz  dd          5 }t          j        |          }d d d            n# 1 swxY w Y    t	                      j        |g|R d|i| d S )NrG   ru  r   r   r   )r  r   r   r   r  r   rq   r:   r  r  rG   r   r  s         r/   r   zMambaModel.__init__q  s    **Y--?i-/wGGG '1)A,,' ' ' ' ' ' ' ' ' ' ' ' ' ' 'ETEEE7EfEEEEE   AAAc                r   | j         d         }| j                             dd          }|| z   |z  }|| j         d<   | j        dz                                  r|                                  d S | j        dz                                  r|                                  d S |                     d|           d S )Nr  pad_vocab_size_multipler  r  r  r  )rG   r   r:   r   r  r  r%  rq   r  	pad_vocabs      r/   r  zMambaModel.set_vocaby  s    \,/
L$$%>BB	 "iZ/09<
%/\"N--6688 	<  """""n0099;; 	<))+++++ ##J
;;;;;r.   c                &   |                      ddg          }|                      ddgd          pd}|                      dd	gd          pd
|z  }|                      ddgd          pd}|                      ddgd          p|dz   }|                      ddgd          pd}d}|                      dgd          dv rd}|d
|z  k    sJ | j                            d           | j                            |           | j                            d           | j                            d           | j                            | j                   | j                            |           | j        	                    |           | j        
                    |           | j                            |           | j                            |           | j                            |           | j                            | j                   d S )Nr  r  conv_kerneld_convTr  r!   r  d_innerr   
state_sized_stater  time_step_rankdt_rankr  r  r  Fr  )falcon_mambar1  r   )r   rI   r  r  r  r  r  rR   r  r  r  r  r  add_ssm_dt_b_c_rmsr  r<   )rq   r  r  r  r  r  r  use_dt_b_c_norms           r/   r   zMambaModel.set_gguf_parameters  s>   ""M#CDD""M#Bd"SSXWX""$7#Cd"SSbWX[bWb""L#Cd"SSYWY '')9y(IY]'^^sdkordrbs'')=~(NY]'^^fbf\NT::>OOO"O!g+%%%%++E222--g66600333''***(()9:::,,V444++G444++G444//888//===++O<<<&&tz22222r.   Nr   r   rb   r   r   r   r\   r   c                v   |                      t          j        j                  }|                      t          j        j                  }|                     |          }|                    d          r2t                              d|z              t          j
        |           }|                     |t          j        j        |          r|                                }| j        C||k    r=t          j        | j        |          r"t                              | d| d           g S n||k    r|| _        ||fgS )Nr  A_log --> A ==> z is equivalent to z
, omitting)r   rf   r  r.  r,  r   r  rd   r  r   rS  r   r  r  	_tok_embdequalr'  s          r/   r   zMambaModel.modify_tensors  s2   --d.?.FGG//0A0LMM''--=="" 	0LL+h6777)J///J ''$2C2NPSTT 	.#++--J >%(k*A*A{4>:66 XX}XXXYYY	 &&'DN:&''r.   r:   r   r  )r$   r%   r&   rf   r  MAMBArP   r   r  r   r  r   r9  r:  s   @r/   r  r  m  s        &JF F F F F F< < <"3 3 3< I( ( ( ( ( ( ( (r.   r  Mamba2ForCausalLMc                  L     e Zd Zej        j        Zd fdZd Zd Z	ddZ
 xZS )Mamba2Modelr:   r   c                   |                     dd           }|At          |dz  dd          5 }t          j        |          }d d d            n# 1 swxY w Y    t	                      j        |g|R d|i| |                     g d          | _        |                     g dd	          p	d
| j        z  | _        |                     dgd	          pd| _	        d S )NrG   ru  r   r   r   )r  r  r  )mamba_d_ssmr  r  Tr  r   n_groupsr   )
r  r   r   r   r  r   r   r  r  n_groupr  s         r/   r   zMamba2Model.__init__  s4    **Y--?i-/wGGG '1)A,,' ' ' ' ' ' ' ' ' ' ' ' ' ' 'ETEEE7EfEEE''(I(I(IJJ''(W(W(Wbf'gg{kloso{k{''t'DDIr  c                   | j         d         }| j                             dd          }|| z   |z  }|| j         d<   | j        dz                                  r|                                  d S | j        dz                                  r%t          d| j        dz   d| j        dz             | j        dz                                  r|                                  d S |                     d	|           d S )
Nr  r  r  r  ztokenizer.model.v3zPlease rename z to r  r  )rG   r   r:   r   r  r   r  r%  r  s      r/   r  zMamba2Model.set_vocab  s;   \,/
L$$%>CC	 "iZ/09<
%/\"N..7799 		<))+++++n33<<>> 	<%  'Gt~H\7\  'G  'Gbfbp  tE  cE  'G  'G  H  H  Hn//88:: 	<  """"" ##J
;;;;;r.   c                   |                      ddgd          pd}|                      ddgd          pd}|                      d	d
gd          pd}|                      ddgd          pd}| j        t          j        j        k    r%| j        d| j        z  k    sJ | j        |z  dk    sJ | j                            d           | j        	                    | j                   | j        
                    d           | j                            d           | j                            | j                   | j                            |           | j                            | j                   | j                            |           | j                            | j        |z             | j                            | j                   | j                            |           | j                            | j                   d S )Nr  r  Tr  r!   r  r  rJ  mamba_d_headr  r  r  r  r  r   r   r1  )r   rP   rf   r  	FALCON_H1r  r  rI   r  r  r  r  r  rR   r  r  r  r  r  r  r  r  r<   )rq   r  r  r  r  s        r/   r   zMamba2Model.set_gguf_parameters  s   ""M8#<4"PPUTU""L9#=4"PPWTW##^Z$@4#PPVTV'')=~(NY]'^^fbf
 ?do777<1t|#33333<(*a////++E222--dl;;;00333''***(()9:::,,V444++DL999++G444//0HIII,,T\::://===&&tz22222r.   r   r   rb   r   r   r   r\   r   c              #     K   |                     d          s|                     d          r|                    d          }|                    d          r|                    d          d         dz   }                     |                               t          j        j                  r|	                                }nt           fdt          j        j        t          j        j        fD                       r |                    g |j        dR           }nO                     t          j        j                  r)|                     j         j         j        z  f          }|                    d	          r2t$                              d
z              t)          j        |           }|fV  d S )Nzmodel.backbonezmodel.lm_headrv  r  r   r  c              3  J   K   | ]}                     |d           V  dS )r  )r   Nr  )ra   r  r   r   rq   s     r/   rc   z-Mamba2Model.modify_tensors.<locals>.<genexpr>  sJ        q--h3r-JJ      r.   r   r  r  )rm  removeprefixr  r  r   r   rf   r  r  r  r  SSM_ASSM_Dr  r  SSM_NORMr  r  rd   r  r   rS  r  s   `  `@r/   r   zMamba2Model.modify_tensors  s     ??+,, 	/0P0P 	/$$X..D==$$ 	D??:..q1OCD''--''$2C2NPSTT 
	Z#++--JJ      ##U
      	Z $++,Bj.>,B,B,BCCJJ))(D4E4NPSTT 	Z#++T\4<4<;W,XYYJ=="" 	0LL+h6777)J///J$$$$$$r.   r  r  )r$   r%   r&   rf   r  MAMBA2rP   r   r  r   r   r9  r:  s   @r/   r  r    s        'J
J 
J 
J 
J 
J 
J< < <(3 3 36% % % % % % % %r.   r  JambaForCausalLMc                  b     e Zd ZU ej        j        ZddZd Zd Z	dZ
ded<   ddZ fdZ xZS )
JambaModelr\   r   c                    ~dS )NrB  r-   )rq   r  s     r/   r  zJambaModel.get_vocab_base_pre$  s
    wr.   c                    | j         dz                                  r|                                  d S |                                  d S )Nr  )r:   r   r  r  r   s    r/   r  zJambaModel.set_vocab)  sR    N..7799 	# ))+++++   """""r.   c                j  	
 |                      ddg          }|                      dgd          pd}| j        d         |z  }|                      dgd          pd	}|                      d
gd          p|dz   }|                      ddgd          pd}| j        d         
| j        d         | j        d         	d t                    D             	
fdt          | j                  D             z   }| j                            | j                   | j                            |                      ddg                     | j                            |           | j                            | j        d                    | j        	                    | j        d                    | j        
                    |           | j                            |           | j                            |           | j                            |           | j                            |           | j                            |           | j                            | j        d                    | j                            | j        d                    | j                            | j                   d S )Nr  mamba_d_modelr  Tr  r!   mamba_expandr  r  mamba_dt_rankr  r  r  r-  r  attn_layer_offsetattn_layer_periodc                    g | ]}d S r  r-   r)  s     r/   r  z2JambaModel.set_gguf_parameters.<locals>.<listcomp>@  s    222!A222r.   c                0    g | ]}|z
  z  d k    rnd S r  r-   )ra   r  attn_offsetattn_periodr  s     r/   r  z2JambaModel.set_gguf_parameters.<locals>.<listcomp>@  s=     6
 6
 6
IJ!k/[8A==II16
 6
 6
r.   r  r  r  r  r  r  )r   rG   r  rR   rI   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r<   )rq   r  r  r  r  r  r  n_kv_vecr  r  r  s           @@@r/   r   zJambaModel.set_gguf_parameters3  s   ""M?#CDD""N#3t"DDI,~.8""O#4t"DDJ ''(9D'II^wZ]~M^'')=~(NY]'^^fbfL!67	l#67l#6722u[11222 6
 6
 6
 6
 6
 6
NST_aeaqNrNr6
 6
 6
 
 	(()9:::++D,<,<>WY`=a,b,bccc--g66600>Q1RSSS''5J(KLLL**8444,,V444++G444++G444//888//===))$,}*EFFF..t|<Q/RSSS&&tz22222r.   Nr$  rh  r   r   rb   r   r   r   c           	   #    K   |                     dd          }|B| j        d         }| j        d         }||k    r||z
  |z  dk    s|                     dd          }d|v r| j        d	         }|J | j        #d
 t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rdD ]}g }t          |          D ]B}	d| d|	 d| d}
|                    | j        |         |
                    | j        |         |
= Ct          j        |d          }d| d| d}| 	                    |          }||fV  d S | 	                    |          }| 
                    |t          j        j        |          r|                                }|                    d          r2t                               d|z              t          j        |           }||fV  d S )NrP  z.feed_forward.expert_layer_offsetexpert_layer_periodr   z.experts.0.r   r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z-JambaModel.modify_tensors.<locals>.<listcomp>g  r*  r.   r    r  r  r   r  r  r  r  )r  rG   rh  r  rR   r   rn  r   r5  r   r   rf   r  r  r  r  rd   r  rS  )rq   r   rb   r   
moe_offset
moe_periodr  r  r6  r7  r8  r9  r   s                r/   r   zJambaModel.modify_tensorsU  sZ      ||G%566?&;<J&;<J:%%3+;z*IQ*N*N||M377 $t++]3I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-77 A / /C*,E$Y// 6 6 ] ] ]3 ] ]QT ] ] ]T]3%7%>??? M#.u55!&U!:!:!:J #Q#"P"PC"P"P"PK#33K@@H"J.....F''--''$2C2NPSTT 	.#++--J=="" 	0LL+h6777)J///J$$$$$$r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z.JambaModel.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  zJambaModel.prepare_tensors  r  r.   r7  r  )r$   r%   r&   rf   r  JAMBArP   r  r  r   rh  r  r   rK  r9  r:  s   @r/   r  r     s         &J   
# # #3 3 3@ 04H33334% 4% 4% 4%lD D D D D D D D Dr.   r  CohereForCausalLMc                  @     e Zd Zej        j        Z fdZ fdZ xZ	S )CommandR2Modelc                x     t                      j        |i | |                     ddg          | j        d<   d S )Nr  r  )r  r   r   rG   r  s      r/   r   zCommandR2Model.__init__  sI    $)&)))
 372B2BDVXqCr2s2s.///r.   c                    t                                                       | j                            | j        d                    | j                            t          j        j                   d S )Nrt  )	r  r   rI   rp  rG   r  rf   r  r  r  s    r/   r   z"CommandR2Model.set_gguf_parameters  sZ    ##%%%((m)DEEE..t/C/HIIIIIr.   )
r$   r%   r&   rf   r  	COMMAND_RrP   r   r   r9  r:  s   @r/   r  r    si        *Jt t t t tJ J J J J J J J Jr.   r  Cohere2ForCausalLMc                  6     e Zd Zej        j        Z fdZ xZS )Cohere2Modelc                    t                                                       | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j        d         }| j        d         }| j        d         }| j                            t          |||z  z                       | j        	                    t          j        j                   d S )Nrt  r  r  r|  r  r  )r  r   rI   rp  rG   r  rU  r~  rQ   r  rf   r  r  )rq   r|  r  r  r  s       r/   r   z Cohere2Model.set_gguf_parameters  s    ##%%%((m)DEEE++DL9I,JKKK''\(BCCC\,/
l=1"l+@A11#jKSfDf6g2h2hiii..t/C/HIIIIIr.   )	r$   r%   r&   rf   r  COHERE2rP   r   r9  r:  s   @r/   r  r    sK        (JJ J J J J J J J Jr.   r  OlmoForCausalLMOLMoForCausalLMc                  >     e Zd Zej        j        Z fdZdd
Z xZ	S )	OlmoModelc                    t                                                       | j                            d           | j                            d          }|| j                            |           d S d S )Nr  r  )r  r   rI   r  rG   r   r  )rq   r  r  s     r/   r   zOlmoModel.set_gguf_parameters  sp    ##%%%++D111<##J//**844444  r.   r   r   rb   r   r   r   r\   r   c                F   ~| j         d         }| j                             d          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                     |          |fgS r  r  r0  s         r/   r   zOlmoModel.modify_tensors  s    34L$$%:;;	==)) 	H#++JGGJ==)) 	K#++J	JJJ%%d++Z899r.   r  )
r$   r%   r&   rf   r  OLMOrP   r   r   r9  r:  s   @r/   r  r    sX         %J5 5 5 5 5: : : : : : : :r.   r  Olmo2ForCausalLMc                  &    e Zd Zej        j        ZdS )
Olmo2ModelN)r$   r%   r&   rf   r  OLMO2rP   r-   r.   r/   r  r    r  r.   r  OlmoeForCausalLMc                  X     e Zd ZU ej        j        Z fdZdZde	d<   ddZ
 fdZ xZS )
OlmoeModelc                    t                                                       | j                            d           | j                            d          x}| j                            |           d S d S )Nr  r  )r  r   rI   r  rG   r   r  )rq   r  r  s     r/   r   zOlmoeModel.set_gguf_parameters  sq    ##%%%//555))-888IE--i88888 FEr.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        |                     d          dk    r| j        d         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t          |          D ]B}d| d| d	| d
}	|                    | j        |         |	                    | j        |         |	= Ct          j        |d          }d| d| d
}
| 	                    |
          }|                    ||f           |S g S | 	                    |          |fgS )Nr  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z-OlmoeModel.modify_tensors.<locals>.<listcomp>  r*  r.   r    r  r  r  r   r   r   r  rT  r  s               r/   r   zOlmoeModel.modify_tensors  s   99Y2%%]3I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 D ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z.OlmoeModel.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  zOlmoeModel.prepare_tensors  r  r.   r  )r$   r%   r&   rf   r  OLMOErP   r   rh  r  r   rK  r9  r:  s   @r/   r  r    s         &J9 9 9 9 9 04H3333": ": ": ":JD D D D D D D D Dr.   r  JinaBertModelJinaBertForMaskedLMc                  6     e Zd Zej        j        Z fdZ xZS )JinaBertV2Modelc                   d}t          | j        dz  dd          5 }t          j        |          d         }d d d            n# 1 swxY w Y   |dk    r"t	                                                       d S |dk    r0|                                  | j                            d           d S t          d	| d
          )NBertTokenizerr  r   r   r   r  RobertaTokenizerr   z
Tokenizer z# is not supported for JinaBertModel)
r   r:   r   r   r  r  r  rI   r[  r   )rq   r  r   r  s      r/   r  zJinaBertV2Model.set_vocab  s	   )$.#::C'RRR 	>VW"ill+<=O	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> o--GG 222  """11!44444%&g?&g&g&ghhhs   AA	A	)	r$   r%   r&   rf   r  JINA_BERT_V2rP   r  r9  r:  s   @r/   r  r    sK        -Ji i i i i i i i ir.   r  OpenELMForCausalLMc                  p     e Zd Zej        j        Zedd            Z fdZ	d Z
d	 Zdd fdZddZ xZS )OpenELMModelrI  float | intdivisorrQ   r\   c                t    t          |t          | |dz  z             |z  |z            }|d| z  k     r||z  }|S )Nr   g?)r  rQ   )rI  r 	  new_vs      r/   _make_divisiblezOpenELMModel._make_divisible,  sI     GSWq[11W<wFGG37??WEr.   c                     t                      j        |i |  j        d         } j        d          j        d          _         j        d          _         j        d          _         fd|D              _        t           j        t                    r t           j        d         t                    sJ t           j        t                    r t           j        d         t                    sJ d S )Nffn_multipliersffn_dim_divisor	model_dimr  num_query_headsc                V    g | ]%}t                               |j        z            &S r-   )r  r	  _n_embd)ra   r  r	  rq   s     r/   r  z)OpenELMModel.__init__.<locals>.<listcomp>=  s@     %
 %
 %
 ((dl)BOTT%
 %
 %
r.   r   )
r  r   rG   r
	  r!  _num_query_headsr#  r   r   rQ   )rq   r  r  r	  r	  r  s   `   @r/   r   zOpenELMModel.__init__5  s   $)&)))'+|4E'F#|,=> L5(,^(D+/<8I+J%
 %
 %
 %
 %
-%
 %
 %
 $,d33^
4CUVWCXZ]8^8^^^^$/66d:dF[\]F^`c;d;ddddddr.   c                    	 |                                   d S # t          $ r% |                     d| j        d                    Y d S w xY w)Nr  r  )r  r  r%  rG   r   s    r/   r  zOpenELMModel.set_vocabE  se    	M))+++++  	M 	M 	M##Kl1KLLLLLL	Ms    +AAc                   | j         }| j        d         }d}| j        t          | j                  k    sJ | j        t          | j                  k    sJ | j        t          | j                  k    sJ | j                            | j                   | j        	                    | j        d                    | j        
                    |           | j                            | j                   | j                            | j                   | j                            | j                   | j                            | j        d                    | j                            d           | j                            t#          ||z                       | j                            |           | j                            |           | j                            | j                   d S )Nr  r  max_context_lengthrope_freq_constantr-  )r
	  rG   rR   r   r!  r	  r#  rI   r  r  r  r  r  r  r  r  r~  rQ   r  r  r  r<   )rq   r  r  r  s       r/   r   z OpenELMModel.set_gguf_parametersK  s   <
+3t'9#:#:::::3t'<#=#=====3t~#6#66666(()9:::++DL9M,NOOO--f55500@@@''(=>>>**4+=>>>++DL9M,NOOO//55511#g6H2I2IJJJ''111))(333&&tz22222r.   Frk   r   r   r>   r   c                j    d|v r| j         d         S t                                          ||          S )Nr  num_transformer_layers)rG   r  r   )rq   rk   r   r  s      r/   r   zOpenELMModel.find_hparama  s6    < 899ww""4222r.   r   r   rb   r   r   r   r   c              #  6  K   |{|d| dk    rq| j         |         }|                     t          j        j        |          |d |         fV  |                     t          j        j        |          ||d          fV  d S |                     |          |fV  d S )Nztransformer.layers.z.ffn.proj_1.weight)r#  r   rf   r  r  r  r   )rq   r   rb   r   r  s        r/   r   zOpenELMModel.modify_tensorsg  s       ?t'TS'T'T'TTT^C(F**4+<+EsKKZX_Y_X_M`aaaa**4+<+CSII:V\V]V]K^____F##D)):666666r.   )rI  r  r 	  rQ   r\   rQ   r  r  r  )r$   r%   r&   rf   r  OPENELMrP   r  r	  r   r  r   r   r   r9  r:  s   @r/   r  r  (  s        (J   \e e e e e M M M3 3 3,3 3 3 3 3 3 3	7 	7 	7 	7 	7 	7 	7 	7r.   r  ArcticForCausalLMc                  ^     e Zd ZU ej        j        Zd Z fdZdZ	de
d<   ddZ fdZ xZS )ArcticModelc                   ddl m} | j        dz  }|                                s1t                              d|            t          j        d            |            }|                    t          |                     | j
                            d|                                          }d t          |          D             }dg|z  }t          j        g|z  }t          |                                          D ]}|                    |          }	|	                    d	          }
|                    |          }t          j        }|                    |          rt          j        }ne|                    |          rt          j        }nC|                    |          rt          j        }n!|                    |          rt          j        }|
||<   |||<   |||<   | j        d
z  }|                                r0t7          |dd	          5 }t9          j        |          }d|v r|d         }|                                D ]\  }}t?          |          }||k    r$t                               d| d|dz
              >|d         }t          j!        }d}d|v r/|d         r'||d         k    rt          j        }nt          j        }d}t          "                    d| d| d| d|dd	           |                    d	          ||<   |||<   |||<   d d d            n# 1 swxY w Y   | j#        $                    d           | j#        %                    d           | j#        &                    |           | j#        '                    |           | j#        (                    |           tS          j*        | j        tW          |                    }|,                    | j#                   d S )Nr   r  r  r  r   r  c                @    g | ]}d | d                     d          S r  r  r  s     r/   r  z)ArcticModel.set_vocab.<locals>.<listcomp>  r  r.   r  r   r  r   r   r  r  r  r  r  r  r  zSetting added token z to 'z	' (type: z	, score: z.2frv   r  r  r  )-r  r  r:   r   rd   r  r
  r3  r  r   rG   r   r  r  r   r+   r  r  r  r'   r  r(   r  r)   r  r  r,   r   r   r   rl   rQ   r  r*   re   rI   r  r  r  r  r  rf   r  r   r  )rq   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  
token_jsontoken_content
token_typetoken_scorer  s                         r/   r  zArcticModel.set_vocabw  s    	988888*;;%%'' 	LL;>;;<<<HQKKK +*,,	s>22333\%%lI4H4H4J4JKK
VVE*DUDUVVV'j:56=>Ki224455 	) 	)H''11E<<((D&&x00E-4G""8,, 719$$X.. 719##H-- 718!!(++ 716#F8$F8!(HX !%1H H ((** 	7+S7CCC 7q(,	!%)-BBB+@AW+X(0D0J0J0L0L 7 7,*#&x==#z11"LL)m)m)m]gjk]k)m)mnnn$(29(=%<%I
&. &33I9N3,0Ek0RRR-D-L

-D-L
*-K  %J8  %J  %J-  %J  %Jbl  %J  %J  xC  %J  %J  %J  %J  K  K  K+8+?+?+H+Hx(-7*+6x((77 7 7 7 7 7 7 7 7 7 7 7 7 7 7: 	,,W555**9555''///))&111((222)$.#f++NNN!!$"233333s   DL''L+.L+c                    t                                                       | j        }| j                            |d                    | j                            |d         |d         z             d S )Nr  r  r  )r  r   rG   rI   rU  r~  r  s     r/   r   zArcticModel.set_gguf_parameters  si    ##%%%,''(=>>>11'-2HGTiLj2jkkkkkr.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        | j         d         }| j                             d          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                    d          dk    r| j         d         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |d	z  k    rg }d
D ]}g }	t          |          D ]B}
d| d|
 d| d}|	
                    | j        |         |                    | j        |         |= Ct          j        |	d          }d| d| d}|                     |          }|
                    ||f           |S g S |                     |          |fgS )Nr  r  r  r  rz  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z.ArcticModel.modify_tensors.<locals>.<listcomp>  r*  r.   r    r|  r  r  r   r   r   r  r  r  )rG   r   r  rI  rf  r.  rh  r  rR   r   rn  r   r5  r   )rq   r   rb   r   r  r  r  r  r  r6  r7  r8  r9  r   s                 r/   r   zArcticModel.modify_tensors  s   34L$$%:;;	==)) 	H#++JGGJ==)) 	K#++J	JJJ 99/00B66%89I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 . ; ;C*,E$Y// 6 6 a a as a aUX a a aT]3%7%>??? M#.u55!&U!:!:!:J"SC"S"Ss"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z/ArcticModel.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  zArcticModel.prepare_tensors   r  r.   r  )r$   r%   r&   rf   r  ARCTICrP   r  r   rh  r  r   rK  r9  r:  s   @r/   r	  r	  s  s         'JR4 R4 R4hl l l l l 04H3333+: +: +: +:ZD D D D D D D D Dr.   r	  DeepseekForCausalLMc                  v     e Zd ZU ej        j        Zd Z fdZdZ	de
d<   edd            ZddZ fdZ xZS )DeepseekModelc                z    	 |                                   d S # t          $ r |                                  Y d S w xY wr`   r  r   s    r/   r  zDeepseekModel.set_vocab  r  r  c                   t                                                       | j        }|                    d          x}|d         |d         z  }| j                            |           | j                            t          j        j	                   | j        
                    |d                    | j                            |d                    | j                            |d                    | j                            d           | j                            |d                    | j                            |d	                    d S )
Nr  r  r  first_k_dense_replacer  r   r  n_routed_expertsn_shared_experts)r  r   rG   r   rI   r~  r  rf   r  r  r  rU  r  add_expert_weights_scaler  r  )rq   rG   rc  r  s      r/   r   z!DeepseekModel.set_gguf_parameters  s-   ##%%%,J///H8}-9N1OOH11(;;;..t/C/HIII66w?V7WXXX''(=>>>77@W8XYYY11#666))'2D*EFFF009K1LMMMMMr.   Nr$  rh  r  r   r  rQ   r  r   c                    |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R                      dd                               | j                  S r  r  re  s      r/   rf  zDeepseekModel.permute%  rg  r.   r   rb   r   r   r\   r   c           	        | j         d         }| j                             d          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                    d          dk    r| j         d         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |d	z  k    rg }d
D ]}g }	t          |          D ]B}
d| d|
 d| d}|	
                    | j        |         |                    | j        |         |= Ct          j        |	d          }d| d| d}|                     |          }|
                    ||f           |S g S |                     |          |fgS )Nr  r  rx  ry  r
  r  r)	  c                    g | ]}i S r-   r-   r)  s     r/   r  z0DeepseekModel.modify_tensors.<locals>.<listcomp><  r*  r.   r    r  r  r  r   r   r   r  )rG   r   r  r%	  rf  r.  rh  r  rR   r   rn  r   r5  r   )rq   r   rb   r   r  r  r  r  r  r6  r7  r8  r9  r   s                 r/   r   zDeepseekModel.modify_tensors-  s   34L$$%:;;	==9:: 	K&..z66JJJ==9:: 	N&..z69MMJ 99]##r))%78I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 D ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z1DeepseekModel.prepare_tensors.<locals>.<listcomp>^  r  r.   r   r  r  r  s     r/   rK  zDeepseekModel.prepare_tensorsY  r  r.   r  r  )r$   r%   r&   rf   r  DEEPSEEKrP   r  r   rh  r  r  rf  r   rK  r9  r:  s   @r/   r%	  r%	  
  s         )J# # #N N N N N 04H3333) ) ) \)*: *: *: *:XD D D D D D D D Dr.   r%	  DeepseekV2ForCausalLMDeepseekV3ForCausalLMKimiVLForConditionalGenerationc                  ^     e Zd ZU ej        j        Zd Z fdZdZ	de
d<   ddZ fdZ xZS )DeepseekV2Modelc           	        	 |                                   d S # t          $ r Y nw xY wddlm} |                    | j        d          }|                     |          }|dk    rg }i }|j        j        }|	                                D ]\  }}||t                              |          <   t          |          dk    r6t                              |||          }	t          |	          dk    r@|                    d	                    t!          t          j        |	                               | j        d
         }
|j        }d i ||	                                D             }g }g }t'          |
          D ]}||vr>|                    d| d           |                    t(          j        j                   D||         }|                    |           ||                                v r%|                    t(          j        j                   |                    t(          j        j                   | j                            d           | j                            |           | j                            |           | j                            |           | j                            |           t)          j         | j        d          }|!                    | j                   d S tE          d|d          )Nr   r  Trr  r1  r   r  r   r
  r  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z-DeepseekV2Model.set_vocab.<locals>.<dictcomp>      kkk3C;S;kkkr.   r  r  r  Fr  zDeepseek pre-tokenizer z is not supported yet!)#r  r{  r  r   ry  r:   r  rs   _mergeable_ranksrl   r  r  r   r  rn  rB  r  rG   r  r  rf   r  r+   r  r)   r'   rI   r  r  r  r  r  r  r  r   rq   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                    r/   r  zDeepseekV2Model.set_vocabi  s   	  """F 	 	 	D	 	/.....!11$.TX1YY	((33YFE'o>O.4466 Z Zt@Di55e<<=u::??"MMv;;!##MM#((3y/NPV+W+W"X"XYYY l3J&5NkkGb%GbSaGbGhGhGjGjkkkM "F"$H:&& 
? 
?M))MM++++...OODN$9::::)!,EMM%(((N113333 (>???? (=>>>>00888..v666++F333,,X666--f555 -dn%PPPM%%d&677777%&`&`&`&`aaas    
%%c                   d| j         d<   t                                                       | j         }| j                            |d                    | j                            |d                    d|v r(|d          | j                            |d                    | j                            |d                    | j                            |d         |d         z              | j        	                    |d                    | j        
                    |d         |d         z              | j                            |d	                    | j                            |d
                    | j                            |d                    | j                            |d                    | j                            |d                    | j                            |d                    |d         dk    r*| j                            t$          j        j                   nN|d         dk    r*| j                            t$          j        j                   nt-          d|d                    | j                            |d                    | j                             d          pi }|                    d|                    d                    dk    rd|v r| j                            t$          j        j                   | j                            |d                    | j                            |d                    | j                            d|d         z             d S d S d S )Nr   r  r(	  r  r  r  r  r  
v_head_dimr   r)	  r*	  routed_scaling_factornorm_topk_probscoring_funcsigmoidsoftmax Unsupported scoring_func value: r  r  rz   r  r  r  r  mscale_all_dim)rG   r  r   rI   r  rU  r  r  r  r  add_key_length_mlaadd_value_length_mlar  r  r  r+	  add_expert_weights_normadd_expert_gating_funcrf   ExpertGatingFuncTypeSIGMOIDSOFTMAXr   r~  r   r  r  r  r  r  add_rope_scaling_yarn_log_mul)rq   rG   r  r  s      r/   r   z#DeepseekV2Model.set_gguf_parameters  sD    /0*+##%%%,66w?V7WXXX''(=>>>G##(>(J,,W]-CDDD))'.*ABBB 	''(?'J\B](]^^^))'.*ABBB++G4F,G'RdJe,efff--gl.CDDD77@W8XYYY))'2D*EFFF009K1LMMM11':Q2RSSS009I1JKKK>"i//33D4M4UVVVV^$	1133D4M4UVVVVY@WYYZZZ11':L2MNNN|''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;klll::3N^A_;_`````	 MLQiQir.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        d|v sd|v rg S |                     d          r|                    dd          }|                    d          r|                    dd          }| j        d         }t	          j        d|          }|r(t          |                    d	                    |k    rg S |                    d
          dk    r| j        d         }|J | j	        #d t          | j                  D             | _	        || j	        |         |<   t          | j	        |                   |dz  k    rg }dD ]}g }	t          |          D ]B}
d| d|
 d| d}|	                    | j	        |         |                    | j	        |         |= Ct          j        |	d          }d| d| d}|                     |          }|                    ||f           |S g S |                    d          r|                    dd          }|                    dd          }| j        d         }| j        d         }| j        d         }|j        d         |||z   z  k    sJ |                    |||z   |j        d                   }t          j        |||gd	          \  }}|                    d	d          }|                     |          |f|                     |          |fgS |                     |          |fgS )Nrn  rr  rw  r  r  r	  r  zmodel.layers.(\d+)r   r
  r  r)	  c                    g | ]}i S r-   r-   r)  s     r/   r  z2DeepseekV2Model.modify_tensors.<locals>.<listcomp>  r*  r.   r    r  r  r  r   r   r   r  zkv_b_proj.weight	kv_b_projk_b_projv_b_projr  r=	  r  r   )rm  r  r  rG   r  r  rQ   groupr.  rh  r  rR   r   rn  r   r5  r   r  r  r  r  )rq   r   rb   r   rR   r  r  r  r  r6  r7  r8  r9  r   name_kbname_vbr  r=	  r  kv_bk_bv_bs                         r/   r   zDeepseekV2Model.modify_tensors  s^   T!!%<%D%DI??,-- 	7<< 1266D ==233 	V<< 9;TUUD l#67.55 	SQ((K77I 99]##r))%78I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 D ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	 ==+,, 	ll;
;;Gll;
;;G%:;Il3J#|,>?#A&)zDT7T*UUUUU??9j;K.KZM]^`MabbD{4*:J)GQOOOHC--1%%C %%g..4%%g..4 
 %%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z3DeepseekV2Model.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  zDeepseekV2Model.prepare_tensors  r  r.   r  )r$   r%   r&   rf   r  	DEEPSEEK2rP   r  r   rh  r  r   rK  r9  r:  s   @r/   r6	  r6	  c  s          *J4b 4b 4bl(a (a (a (a (aT 04H3333G: G: G: G:RD D D D D D D D Dr.   r6	  Dots1ForCausalLMc                  L     e Zd Zej        j        Z fdZ fdZd
 fd	Z	 xZ
S )
Dots1Modelc                d     t                      j        |i | | j        d         | j        d<   d S )Nr)	  r  r  r  s      r/   r   zDots1Model.__init__"  s9    $)&)))&*l3E&F]###r.   c                   t                                                       | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j        d         dk    r+| j                            t          j
        j                   d S t          d| j        d                    )Nr(	  r*	  r>	  r?	  r@	  noaux_tcrC	  )r  r   rI   r  rG   r  r+	  rG	  rH	  rf   rI	  rJ	  r   r  s    r/   r   zDots1Model.set_gguf_parameters&  s    ##%%%66t|D[7\]]]00>P1QRRR11$,?V2WXXX00>N1OPPP<':5533D4M4UVVVVV^^@\^^___r.   r   r   rb   r   r   r   c                    |                     d          r|                    dd          }d|v r|                     |          |fgS t                                          |||          S )Nr  r	  shared_experts)r  r  r   r  r   r  s       r/   r   zDots1Model.modify_tensors2  sq    ==233 	V<< 9;TUUDt##))$//<==ww%%j$<<<r.   r  )r$   r%   r&   rf   r  DOTS1rP   r   r   r   r9  r:  s   @r/   r]	  r]	    s        &JG G G G G
` 
` 
` 
` 
`= = = = = = = = = =r.   r]	  PLMForCausalLMc                  N     e Zd Zej        j        Zd Z fdZddZ	 fdZ
 xZS )PLMModelc                .    |                                   d S r`   r  r   s    r/   r  zPLMModel.set_vocab>  r  r.   c                   t                                                       | j        }| j                            |d                    | j                            |d                    | j                            |d         |d         z              | j                            |d                    | j                            |d                    d S )Nr  r  r  r  r=	  )	r  r   rG   rI   rU  r  r  r  r~  r  s     r/   r   zPLMModel.set_gguf_parametersA  s    ##%%%,''(=>>>))'.*ABBB''0B(CgN`Fa(abbb))',*?@@@11':L2MNNNNNr.   r   r   rb   r   r   r   r\   r   c                2    |                      |          |fgS r`   r   r   s       r/   r   zPLMModel.modify_tensorsJ  s    %%d++Z899r.   c                H    t                                                       d S r`   r/  r  s    r/   rK  zPLMModel.prepare_tensorsM  r0  r.   r  )r$   r%   r&   rf   r  PLMrP   r  r   r   rK  r9  r:  s   @r/   rf	  rf	  :  s        $J  O O O O O: : : :" " " " " " " " "r.   rf	  T5WithLMHeadModelT5ForConditionalGenerationMT5ForConditionalGenerationUMT5ForConditionalGenerationc                  J     e Zd Zej        j        Z fdZd Zd Z	ddZ
 xZS )T5Modelc                H     t                      j        |i | d| _        d S r   r  r   shared_token_embeddings_foundr  s      r/   r   zT5Model.__init__X  -    $)&)))-2***r.   c           	     j
   dt           j        d<   ddlm} ddlm} | j        dz  }|                                s
| j        dz  }|                                st          d|           |                                }|	                    t          |d	                                                     |j        j        d
k    r!|j        dk    sJ |                                 S |j        j        dk    sJ |j        j        }|j        j        }|j        j        } |            }|                    t+          |                     | j                            d|                                          }	d t3          |	          D             }
dg|	z  }t4          j        g|	z  }t3          |                                          D ]}|                    |          }|                    d          }|                    |          }t4          j        }|                     |          rt4          j!        }ne|"                    |          rt4          j#        }nC|$                    |          rt4          j        }n!|%                    |          rt4          j&        }||
|<   |||<   |||<   | j        dz  }|                                rt          |dd          5 }tO          j(        |          }|D ]`}||         }||	k    r$tR          *                    d| d|	dz
              4|                    d          |
|<   d||<   t4          j+        ||<   a	 d d d            n# 1 swxY w Y   |	tY          |
          k    r|	tY          |
          z
  }tR          -                    d| d| d           t3          d|dz             D ]^}|
.                    t_          d| dd                     |.                    d           |.                    t4          j                   _| j0        1                    d           | j0        2                    d           | j0        3                    |
           | j0        4                    |           | j0        5                    |           | j0        6                    |           | j0        7                    |           |r| j0        8                    |           ts          j:        | j        tY          |
                    }|;                    | j0                   d S )Nrd  re  r   r  r/  r  spiece.modelr  r1  r   r   r  c                @    g | ]}d | d                     d          S r  r  r  s     r/   r  z%T5Model.set_vocab.<locals>.<listcomp>  r  r.   r  r   r  r   r   r  r  r  r  r  r  r  rj  r  r  <rk  rk  r  r  r0  r:   r   r  r4  r5  r   r  rn  r  rb   r  r6  r7  ro  rh  r  r   rG   r   r  r  r   r+   r  r  r  r'   r  r(   r  r)   r  r  r,   r   r   rd   r>  r*   r   r  rn  r  rI   r  r  r  r  r  r]  rs  rt  rf   r  r  rq   r  rs   r  r8  r9  rv  rh  r  r  r  r  r  r  r  r  r  r  r  r   r  r   r  r  r  s                            r/   r  zT5Model.set_vocab\      @H
;<888888BBBBBB*;; %%'' 	=!^n<N%%'' 	I#$G~$G$GHHH#..00++D,F,F,K,K,M,MNNN +6!;;!&*;;;;;00222&3>!CCCC(8I
0@Y2BW**,,	s>22333\%%lI4H4H4J4JKK
VVE*DUDUVVV'j:56=>Ki224455 	) 	)H''11E<<((D&&x00E-4G""8,, 719$$X.. 719##H-- 718!!(++ 716#F8$F8!(HX N-@@$$&& 	N'w??? 
N1$(IaLL!, N NC05H:--'kx'k'k[ehi[i'k'klll '*zz'':':F8$'.F8$)@)MHX&&N
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N 
N F##"S[[0ILLeyeeYbeeefff1i!m,, @ @eK1KKK'BBBCCCg&&& 7 >????,,T222**9555''///))&111((222--j999556HIII 	L556JKKK)$.#f++NNN!!$"233333   A8MM #M c                   |                      dgd          x}t                              d           d}| j                            |           | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j        	                    | j        d	                    | j        
                    | j        d
                    | j                            | j        d
                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j                   d S )Nr  Tr  JCouldn't find context length in config.json, assuming default value of 512r  r  d_ffr  r  d_kvr  relative_attention_num_bucketsdecoder_start_token_id)r   rd   r>  rI   r  r  rG   r  r  r  r  r  r  add_relative_attn_buckets_countr  add_decoder_start_token_idr  r<   rq   r  s     r/   r   zT5Model.set_gguf_parameters  s   %%}o%EEEENNNghhhE++E222--dl9.EFFF00f1EFFF((l)CDDD''[(ABBB''V(<===))$,v*>???++DL9M,NOOO88Ff9ghhh//=Q0RSSS33DLAY4Z[[[&&tz22222r.   r   r   rb   r   r   r   r\   r   c                    ~|dv r1| j         s
d}d| _         n t                              d|d           g S |                     |          |fgS N)zdecoder.embed_tokens.weightzencoder.embed_tokens.weightshared.weightr	  TzSkipping shared tensor r  rt	  rd   r  r   r   s       r/   r   zT5Model.modify_tensors  m     bbb5 &5922ptpppqqq	%%d++Z899r.   r  )r$   r%   r&   rf   r  T5rP   r   r  r   r   r9  r:  s   @r/   rq	  rq	  Q  sz        
 #J3 3 3 3 3\4 \4 \4|3 3 3": : : : : : : :r.   rq	  T5EncoderModelc                  J     e Zd Zej        j        Z fdZd Zd Z	ddZ
 xZS )r	  c                H     t                      j        |i | d| _        d S r   rs	  r  s      r/   r   zT5EncoderModel.__init__  ru	  r.   c           	     j
   dt           j        d<   ddlm} ddlm} | j        dz  }|                                s
| j        dz  }|                                st          d|           |                                }|	                    t          |d	                                                     |j        j        d
k    r!|j        dk    sJ |                                 S |j        j        dk    sJ |j        j        }|j        j        }|j        j        } |            }|                    t+          |                     | j                            d|                                          }	d t3          |	          D             }
dg|	z  }t4          j        g|	z  }t3          |                                          D ]}|                    |          }|                    d          }|                    |          }t4          j        }|                     |          rt4          j!        }ne|"                    |          rt4          j#        }nC|$                    |          rt4          j        }n!|%                    |          rt4          j&        }||
|<   |||<   |||<   | j        dz  }|                                rt          |dd          5 }tO          j(        |          }|D ]`}||         }||	k    r$tR          *                    d| d|	dz
              4|                    d          |
|<   d||<   t4          j+        ||<   a	 d d d            n# 1 swxY w Y   |	tY          |
          k    r|	tY          |
          z
  }tR          -                    d| d| d           t3          d|dz             D ]^}|
.                    t_          d| dd                     |.                    d           |.                    t4          j                   _| j0        1                    d           | j0        2                    d           | j0        3                    |
           | j0        4                    |           | j0        5                    |           | j0        6                    |           | j0        7                    |           |r| j0        8                    |           ts          j:        | j        tY          |
                    }|;                    | j0                   d S )Nrd  re  r   r  r/  r  rw	  r  r1  r   r   r  c                @    g | ]}d | d                     d          S r  r  r  s     r/   r  z,T5EncoderModel.set_vocab.<locals>.<listcomp>	  r  r.   r  r   r  r   r   r  r  r  r  r  r  r  rj  r  r  ry	  rz	  s                            r/   r  zT5EncoderModel.set_vocab  r{	  r|	  c                ~   |                      dgd          x}t                              d           d}| j                            |           | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j        	                    | j        d	                    | j        
                    | j        d
                    | j                            | j        d
                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j                   d S )Nr  Tr  r~	  r  r  r	  r  r  r	  r  r	  )r   rd   r>  rI   r  r  rG   r  r  r  r  r  r  r	  r  r  r<   r	  s     r/   r   z"T5EncoderModel.set_gguf_parametersC  s   %%}o%EEEENNNghhhE++E222--dl9.EFFF00f1EFFF((l)CDDD''[(ABBB''V(<===))$,v*>???++DL9M,NOOO88Ff9ghhh//=Q0RSSS&&tz22222r.   r   r   rb   r   r   r   r\   r   c                    ~|dv r1| j         s
d}d| _         n t                              d|d           g S |                     |          |fgS r	  r	  r   s       r/   r   zT5EncoderModel.modify_tensorsS  r	  r.   r  )r$   r%   r&   rf   r  	T5ENCODERrP   r   r  r   r   r9  r:  s   @r/   r	  r	    sx        *J3 3 3 3 3\4 \4 \4|3 3 3 : : : : : : : :r.   JAISLMHeadModelc                  T     e Zd Zej        j        Z fdZd Zd Z	ddZ
 fdZ xZS )	JaisModelc                    t                      j        |i | | j        d         dk    sJ | j        d         dk    sJ d| _        d| j        v r| j        d         | _        nd| j        v r| j        d         | _        nJ d| _        d	| j        v r,d
| j        v sJ | j        d	         | j        d
         z  | _        nd| j        v r| j        d         | _        nJ d| _        d S )Nr  r  position_embedding_typer  r  mup_embeddings_scaleembeddings_scaleFmup_output_alphamup_width_scalewidth_scaler  )r  r   rG   r	  r	  max_alibi_biasr  s      r/   r   zJaisModel.__init__i  s   $)&))) |12h>>>>|56'AAAA !$!T\11$(L1G$HD!!4<//$(L1C$DD!!5--$4444#|,>?$,O`BaaDdl**#|M:D5!r.   c                .    |                                   d S r`   r  r   s    r/   r  zJaisModel.set_vocab  r  r.   c                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j        d                    | j                             | j	                   d S )Nr  r  r  r  r  r  r  r   s    r/   r   zJaisModel.set_gguf_parameters  s    ((i)@AAA++DL,GHHH--dl8.DEEE00i1HIII''X(>???++DL9M,NOOO&&tz22222r.   r   r   rb   r   r   r   r\   r   c                @   ~g }|                     d          r|S |                     d          rdt          j        t          j        | j        d                             z  }t          |d                                                   }t          t          j        |          |z             | _        |S |                     d          r|	                    dd          }| 
                    |          }||                     t          j        j                  k    r |                    ||| j        z  f           n_||                     t          j        j                  k    r |                    ||| j        z  f           n|                    ||f           |S )Nr  zrelative_pe.slopesr   r  r   )r  r  r  z.c_fc2.weightr   )r  r  floorlog2rG   r  r  r=  r	  r  r   r   rf   r  r,  rn  r	  r.  r	  )rq   r   rb   r   r  n_head_closest_log2	first_valr   s           r/   r   zJaisModel.modify_tensors  s   ,. ==,(( 	N==.00 		
 #$tz$)DL<R2S2S'T'T"Tjm002233I#(9)=)=@S)S#T#T"TDN==^__ 	4#--a33J''--t..t/@/KLLLLNNHj43H&HIJJJJ001B1IJJJJNNHj43C&CDEEEENNHj1222r.   c                    t                                                       | j                            | j                   d S r`   )r  rK  rI   r  r	  r  s    r/   rK  zJaisModel.prepare_tensors  s9    !!!++D,?@@@@@r.   r  )r$   r%   r&   rf   r  JAISrP   r   r  r   r   rK  r9  r:  s   @r/   r	  r	  e  s        %J" " " " "8  3 3 3       DA A A A A A A A Ar.   r	  Glm4ForCausalLMGlm4vForConditionalGenerationc                  H     e Zd Zej        j        Zd Z fdZd fdZ	 xZ
S )	Glm4Modelc                Z   ddl m} |                    | j        d          }t	          j        | j        d          }|                                 \  }}}| j                            d           | j        	                    |           | j        
                    |           | j                            |           t	          j        | j        d          }|                    d|                                d                    |                    d	|                                d
                    |                    d|                                d                    |                    d|                                d                    |                    | j                   d S )Nr   r  Trr  r  r  r  r  r  <|user|>r  r  )r  r   ry  r:   rf   r  r!  rI   r  r  r  r  r  r  r  rq   r   r  r  r  r  r  s          r/   r  zGlm4Model.set_vocab  s   ......!11$.TX1YY	)$.dKKK#'#6#6#8#8 &,,V444**6222''///((222)$.dKKK((	0I0I0K0KO0\]]]((	0I0I0K0KJ0WXXX((	0I0I0K0KO0\]]]((	0I0I0K0KO0\]]]!!$"233333r.   c           	        t                                                       | j                            d          x}| j        d         | j        d         z  }| j                            t          || j                            dd          z                       | j                            d          pi }|                    d|                    d                    d	k    rod
|v rm| j                            t          j	        j
                   | j                            |d
                    | j                            |d                    d S d S d S )Nr  r  r  r!  rk  r  r  rz   r  r  r  )r  r   rG   r   rI   r~  rQ   r  rf   r  r  r  r  )rq   rc  r  r  s      r/   r   zGlm4Model.set_gguf_parameters  s?   ##%%%((444H=|M2dlCX6YYH11#hAQAQRiknAoAo6o2p2pqqq|''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;klllll MLQiQir.   r   r   rb   r   r   r   r\   r   c                    |                     d          rg S |                     d          r|                    dd          }t                                          |||          S )Nmodel.visual.model.language_model.rw  r  )rm  r  r  r   r  s       r/   r   zGlm4Model.modify_tensors  sa    ???++ 	7I__455 	7<< 1266Dww%%j$<<<r.   r  )r$   r%   r&   rf   r  GLM4rP   r  r   r   r9  r:  s   @r/   r	  r	    st        %J4 4 4 	m 	m 	m 	m 	m= = = = = = = = = =r.   r	  Glm4MoeForCausalLMc                  h     e Zd ZU ej        j        Z fdZd Z fdZ	dZ
ded<   ddZ fdZ xZS )Glm4MoeModelc                     t                      j        |i | | j        d         | j                            dd          z   | _        t          j        | j        | j                  | _        d S )Nr  num_nextn_predict_layersr   )	r  r   rG   r   rR   rf   r  rP   rS   r  s      r/   r   zGlm4MoeModel.__init__  se    $)&)))<(;<t|?O?OPjlm?n?nn24?DDTUUr.   c                   ddl m} |                    | j                  }t	          j        | j        d          }|                                 \  }}}| j                            d           | j        	                    |           | j        
                    |           | j                            |           |                    d|                                d                    |                    d|                                d	                    |                    d
|                                d                    |                    d|                                d                    t          |j        t                     r)d|j        v r |j                            dd          |_        |                    | j                   d S )Nr   r  Tr  r  r  [gMASK]r  r	  r  r  eom<|observation|>z visible_text(m.content).endswithz{{ visible_text(m.content) }}
{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}}z{% set content = visible_text(m.content) %}{{ content }}
{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not content.endswith("/nothink")) else '' -}})r  r   ry  r:   rf   r  r!  rI   r  r  r  r  r  r  r   r  r   r  r  r	  s          r/   r  zGlm4MoeModel.set_vocab  s   ......!11$.AA	)$.dKKK#'#6#6#8#8 &,,V444**6222''///((222 	((	0I0I0K0KI0VWWW((	0I0I0K0KJ0WXXX((	0I0I0K0KO0\]]]((	0I0I0K0KL]0^___ m1377 	I<^bob}<}<}*7*E*M*M } H+I +IM' 	!!$"233333r.   c           	        t                                                       | j                            d          x}| j        d         | j        d         z  }| j                            t          || j                            dd          z                       | j                            d          x}| j                            |           | j                            d          x}| j                            |           | j                            d          x}| j        	                    |           | j                            d	          x}| j        
                    |           | j                            t          j        j                   | j                            d
          x}| j                            |           | j                            d          x}| j                            |           | j                            d          x}| j                            |           d S d S )Nr  r  r  r!  rk  r)	  r   r*	  r(	  r>	  r?	  r	  )r  r   rG   r   rI   r~  rQ   r  r  r  r  rH	  rf   rI	  rJ	  r+	  rG	  add_nextn_predict_layers)
rq   rc  r)	  r   r*	  r(	  r>	  r?	  r	  r  s
            r/   r   z Glm4MoeModel.set_gguf_parameters  s7   ##%%%((444H=]+t|<Q/RR  	114<++,CSIIIJJ	
 	
 	

 !% 0 01C D DDQ--.>???%)\%5%56M%N%NN![;;<QRRR $ 0 01C D DDQ445EFFF%)\%5%56M%N%NN![::;PQQQ 	//0I0QRRR &*\%5%56M%N%NN![556KLLL #l../?@@@NM44^DDD )-(8(89S(T(TT$a556NOOOOO bar.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        |                     d          rg S |                     d          r|                    dd          }|dk    rd|vr|                     d          |fgS |                    d          d	k    r| j        d
         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t          |          D ]B}d| d| d| d}	|	                    | j        |         |	                    | j        |         |	= Ct          j        |d          }d| d| d}
|                     |
          }|	                    ||f           |S g S |                    d          r|                    dd          }|                     |          }||fgS )Nr	  r	  rw  r  zmodel.embed_tokens.weightz.layers.ztoken_embd.weightr
  r  r)	  c                    g | ]}i S r-   r-   r)  s     r/   r  z/Glm4MoeModel.modify_tensors.<locals>.<listcomp>:  r*  r.   r    r  r  r  r   r   r   r  r  r	  )rm  r  r   r.  rG   rh  r  rR   r   rn  r   r5  r  r  s               r/   r   zGlm4MoeModel.modify_tensors(  s3    ???++ 	7I__455 	7<< 1266D ...:T3I3I))*=>>
KLL 99]##r))%78I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 D ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	==233 	V<< 9;TUUD''--:&''r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z0Glm4MoeModel.prepare_tensors.<locals>.<listcomp>_  r  r.   r   r  r  r  s     r/   rK  zGlm4MoeModel.prepare_tensors[  sn    !!!=$BB$-BBBG7||a !B!B!BCCC	 %$  r.   r  )r$   r%   r&   rf   r  GLM4_MOErP   r   r  r   rh  r  r   rK  r9  r:  s   @r/   r	  r	    s         )JV V V V V4 4 44!P !P !P !P !PF 04H33331( 1( 1( 1(fD D D D D D D D Dr.   r	  GlmForCausalLMChatGLMModelChatGLMForConditionalGenerationc                  p    e Zd Zej        j        Zd Zed             Z	eddd            Z
d Zd ZddZdS )r	  c                   | j         }| j        }g }g }g }ddlm} |                    |d          }|                    dt          |                                                    }t          |                                	                                          |k     sJ g d}	g d|	z   }
t          |          D ]P}|                    |          }|dk    rd}n|d	k    rd
}n|dk    rd}|                    d          }d}t          |          dk    rA||j        j                                        k     r|j        j                            |          }||j        j                                        k    r||
v rt"          j        }nEt          |          dk    r&d| d                    d          }t"          j        }nt"          j        }|                    |           |                    |           |                    |           Vt"          j        }|j        j                            |          rt"          j        }n|j        j                            |          rt"          j        }nW|j        j                            |          rt"          j        }n+|j        j                            |          rt"          j        }|                    |           |                    |           |                    |           R| j                            d           | j                            d           | j                             |           | j        !                    |           | j        "                    |           tG          j$        | j         t          |                    }|%                    | j                   d S )Nr   r  Trr  padded_vocab_size)z
<|system|>r	  z<|assistant|>r	  )z[MASK]r	  z[sMASK]sopeopz<unk>r   z<bos>r   z<eos>r   r  r  r  r  zchatglm-spmr  )&r:   rG   r  r   ry  r   r   r  r  r  r  rp  r  r  sp_modelr  	get_scorer   r)   r+   r*   rn  r'   
is_unknownr(   
is_control	is_unusedis_byter,   rI   r  r  r  r  r  rf   r  r  )rq   r:   rG   r  r  r  r   r  r  role_special_tokensr  r  r  r  r  r  r  s                    r/   set_vocab_chatglm3zChatGLMModel.set_vocab_chatglm3h  s   N	,   ......!11)t1TT	[[!4c):M:M:O:O6P6PQQ
9&&((//1122Z????\\\GGGJ]]j)) )	% )	%H228<<E1}}QQ<<((DE 5zzQ8i.A.J.U.U.W.W#W#W!+4>>xHH9.7BBDDDDN**5=GGZZ1__-(---44W==D5<GG5BGd###e$$$(((-4G"+66x@@ 719$-88BB 719$-77AA 718$-55h?? 716MM$MM%   OOG$$$$,,W555 	**=999''///))&111((222)$.#f++NNN!!$"233333r.   c                    ddl m}  |            d                    fd|                     d          D                       S )Nr   r  r  c                :    g | ]}t          |                   S r-   r  r  s     r/   r  z6ChatGLMModel.token_bytes_to_string.<locals>.<listcomp>  r  r.   r  r  r  s     @r/   r  z"ChatGLMModel.token_bytes_to_string  r  r.   Nr  r  r  r  r  r   r\   r  c                n   d |D             }	 d }d }t          t          |d d         |dd                              D ]7\  }}|                     |d         |d         z             }||||k     r|}|}8||||k    rn3|J |d |         ||         ||dz            z   gz   ||dz   d          z   }|S )Nc                .    g | ]}t          |g          S r-   r  r  s     r/   r  z$ChatGLMModel.bpe.<locals>.<listcomp>  r  r.   Tr  r   r   r   r  r  s	            r/   r  zChatGLMModel.bpe  r  r.   c                   d| j                             dd          v r|                                  d S | j        }| j         }g }g }ddlm} |                    |d          }|                    d|d	                   }t          |                                	                                          |k     sJ | 
                                \  }}}| j                            d
           | j                            |           | j                            |           | j                            |           t!          j        | j        d          }	|	                    d|                                d                    |	                    d|                                d                    |	                    d|                                d                    |	                    | j                   d S )NzTHUDM/chatglm3-6b_name_or_pathr  r   r  Trr  r	  r  r  r  r  r  r  r	  r  )rG   r   r	  r:   r  r   ry  r  r  r  r!  rI   r  r  r  r  rf   r  r  r  r  )
rq   r:   rG   r  r  r   r  r  r  r  s
             r/   r  zChatGLMModel.set_vocab  s   $,"2"2?B"G"GGG##%%%FN	, ......!11)t1TT	[[!4W\5JKK
9&&((//1122Z????#'#6#6#8#8 &,,V444**6222''///((222)$.dKKK((	0I0I0K0KO0\]]]((	0I0I0K0KJ0WXXX((	0I0I0K0KO0\]]]!!$"233333r.   c           
        | j                             d| j                             d                    }| j                             d| j                             d                    }| j                             d| j                             d|                    }| j                            | j                             d|                     | j                            |           | j                            | j                             d| j                             d	d
|z                                 | j                            | j                             d| j         d                              | j                            |           | j                            |           | j        	                    | j                             dd                     | j        
                    | j                   d| j         v r| j         d         }n| j         d         | j         d         z  }| j                            t          || j                             dd          z                       | j                            d           d}d| j         v r|| j         d         z  }| j                            |           d S )Nr  r  r  r  multi_query_group_numr  r  r[  r  r!   r  r  layernorm_epsilonr  attention_dimr!  rk  FrQ  
rope_ratio)rG   r   rI   r  r  r  r  r  r  r  r  r<   r~  rQ   r   r  )rq   r  r  r  rc  	rope_freqs         r/   r   z ChatGLMModel.set_gguf_parameters  s   ,""=$,2B2B92M2MNN!!(DL,<,<=R,S,STTL$$%<dl>N>NOdfl>m>mnn	++DL,<,<\7,S,STTT--g666001A1ABSUYUaUeUefy{|  @G  |G  VH  VH  2I  2I  	J  	J  	J(()9)9,UhHi)j)jkkk''///**9555//0@0@ATUY0Z0Z[[[&&tz222dl**|O4HH|M2dlCX6YYH11#hAQAQRiknAoAo6o2p2pqqq**5111	4<''!DL$>>I++I66666r.   r   r   rb   r   r   r   c                    ~|                     d          s|                    d          rg S |                    d          }|                     |          |fgS )Nz.rotary_pos_emb.inv_freqzmodel.vision.ztransformer.)r  rm  r  r   r   s       r/   r   zChatGLMModel.modify_tensors  sa    ==344 	8X8X 	I  00%%d++Z899r.   r`   r  r  )r$   r%   r&   rf   r  CHATGLMrP   r	  r  r  r  r  r   r   r-   r.   r/   r	  r	  d  s        (JA4 A4 A4F R R \R
     \ 4 4 487 7 7.: : : : : :r.   NemotronForCausalLMc                  D     e Zd Zej        j        Zd Z fdZddZ	 xZ
S )NemotronModelc                    |                                   | j                            d           | j                            d           d S )Nr   r   )r  rI   r  r  r   s    r/   r  zNemotronModel.set_vocab  sH    %%'''))!,,,))!,,,,,r.   c                   t                                                       | j        }| j                            |d                    |                     g d          }| j                            |           |                     g d          }|                     ddg          }|                     ddg          }| j                            t          ||z            |z             d| j        vs| j        d         +| j        	                    t          j        j                   d S | j        	                    t          j        j                   | j                            | j        d	                    d S )
Nr  )r  r  r  r  )r!  r"  rope_percentr  r  r  r  r  r  )r  r   rG   rI   rU  r   r  r~  rQ   r  rf   r  r  r  r  )rq   rG   r  r  r  r  r  s         r/   r   z!NemotronModel.set_gguf_parameters  sb   ##%%%,''(=>>>%%&j&j&jkk
++J777 ""#X#X#XYY!!=(";<<!!#8("CDD11#g6F2G2G62QRRR --n1M1U2243G3LMMMMM2243G3NOOO44T\(5KLLLLLr.   r   r   rb   r   r   r   r\   r   c                f    |                     d          r|dz   }|                     |          |fgS )Nr  r   )r  r   r   s       r/   r   zNemotronModel.modify_tensors  s=    
 =='' 	(#aJ%%d++Z899r.   r  )r$   r%   r&   rf   r  NEMOTRONrP   r  r   r   r9  r:  s   @r/   r	  r	    sj        )J- - -
M M M M M*: : : : : : : :r.   r	  ExaoneForCausalLMc                  4    e Zd Zej        j        Zd ZddZdS )ExaoneModelc                   | j         }|d         dk    sJ |d         }|d         }|d         }|                    d|          }|d         }d|v r|d         nd	|z  }|d
         }| j                            |           | j                            |           | j                            |           | j                            |           | j                            |           | j                            |           | j        	                    |           | j        
                    | j                   | j                             d          x}	| j                            |	           |                     ddgd          }
|
|
nd}
| j                            t          |
|d         |d         z  z                       | j                             d          pi }|                    d|                    d                    dk    rOd|v rM| j                            t"          j        j                   | j                            |d                    d S d S d S )Nr  r  r  r  r  r  r  r  r!   r  r  r!  r"  Tr  r  r  r  rz   r  r  )rG   r   rI   r  r  r  r  r  r  r  r  r<   r  r   r~  rQ   r  rf   r  r  r  )rq   rG   r  r'  r  r  r  r  r  r  r#  r  s               r/   r   zExaoneModel.set_gguf_parameters,  s   ,-.&8888")*C"DM*	12	{{#8)DD !56<OSZ<Z<ZG$788`adm`m\*

 	--i888''	222**<888++,CDDD//???001BCCC((444&&tz222,**<888JE//
;;;((*A:)NY](^^)6)B11#mw}G]ahi~aG  7A  3B  3B  	C  	C  	C|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r\   r   c              #    K   |                      dgd          x}r|                    dd                                          dk    r| j                            dd          }| j                            d	          x}| j        d
         | j        d         z  }d|t	          j        d|dt          j                  |z  z  z  }|                    dd          }|                    dd          }|                    dd          }| j                            dd          }||z  }	||z  }
|	|
k    sJ g }|D ]{}dt          j        z  |z  }||
k     r|	                    d           0||	k    r|	                    |           L||z  |z
  ||z
  z  }|	                    dd|z
  |z  |z   z             || 
                    t          j        j                  t	          j        |t          j                  fV  d S d S d S r  r  r  s                  r/   r   z"ExaoneModel.generate_extra_tensorsO  r-  r.   Nr  )	r$   r%   r&   rf   r  EXAONErP   r   r   r-   r.   r/   r	  r	  (  sK        'J!M !M !MF     r.   r	  Exaone4ForCausalLMc                  D     e Zd Zej        j        Zd Z fdZddZ	 xZ
S )Exaone4Modelc                p   |                                  \  }}}| j                            d           | j                            |           | j                            |           | j                            |           t          j        | j        d          }|	                    | j                   d S r  r  r  s        r/   r  zExaone4Model.set_vocabr  r  r.   c           	        t                                                       | j        }| j                            |d                    |                    d          N| j                            |d                    d|v r+| j                            d |d         D                        nd|v rg }t          |d         t                    rSt          |d                   D ]=}|                    |d         |t          |d                   z           dk               >t          |d         t                    r=t          |d                   D ]'}|                    |dz   |d         z  d	k               (t          |          |d         k    r| j                            |           | j                            d
          pi }|                    d|                    d                    dk    rOd|v rM| j                            t          j        j                   | j                            |d                    d S d S d S )Nr  r  r  c                    g | ]}|d k    	S )r  r-   )ra   r  s     r/   r  z4Exaone4Model.set_gguf_parameters.<locals>.<listcomp>  s    <v<v<vZ[QBU=U<v<v<vr.   r  r  Lr   r   r  r  rz   r  r  )r  r   rG   rI   rU  r   r  r  r   r   r  rn  r   rQ   r  rf   r  r  r  )rq   rG   r  r  r  r  s        r/   r   z Exaone4Model.set_gguf_parameters|  s   ##%%%,''(=>>>;;'((4//8H0IJJJ'' ;;<v<v_fgt_u<v<v<vwwww)W44)+&g&>?EE L"7+>#?@@ L L.55g>V6WXY\_`g  iA  aB  ]C  ]C  YC  7D  HK  7K  L  L  L  Lg&>?EE h"7+>#?@@ h h.55q1uH`@a6aef6fgggg-..':M2NNN$??@VWWW|''77=2K)9)9&)A)ABBhNNS[_kSkSk2243G3NOOO44\(5KLLLLL ONSkSkr.   r\   r   c              #    K   |                      dgd          x}r|                    dd                                          dk    r| j                            dd          }| j                            d	          x}| j        d
         | j        d         z  }d|t	          j        d|dt          j                  |z  z  z  }|                    dd          }|                    dd          }|                    dd          }| j                            dd          }||z  }	||z  }
g }|D ]{}dt          j        z  |z  }||
k     r|	                    d           0||	k    r|	                    |           L||z  |z
  ||z
  z  }|	                    dd|z
  |z  |z   z             || 
                    t          j        j                  t	          j        |t          j                  fV  d S d S d S )Nr  Tr  r  r  r  r  r  r  r  r  r  r   r   r  r  g      0@r  r  r  r  r  r   r  r  s                  r/   r   z#Exaone4Model.generate_extra_tensors  s:     ++^,<t+LLL< 	R006688HDD|''h??<++J777C@,}5F[9\\CtQQem(T(T(TWZ(Z[\%))(D99"."2"23Dc"J"J#/#3#34F#L#L "&,"2"23UW["\"\#2_#D $36F$F!!! R RD$'kD0G!222$++A.... #333$++F3333"1G";o"MRbetRt!u$++A!f*1F1O,PQQQQ..t/@/KLLel[goto|N}N}N}~~~~~~5	 	DDr.   r  )r$   r%   r&   rf   r  EXAONE4rP   r  r   r   r9  r:  s   @r/   r	  r	  n  sr        (J4 4 4M M M M M2       r.   r	  GraniteForCausalLMc                  :     e Zd ZdZej        j        Z fdZ xZ	S )GraniteModelz'Conversion for IBM's GraniteForCausalLMc                D   | j                             dd          x}rt                              d|           t	                                                       | j                             d          x}r5| j                            |           t          	                    d|           | j                             d          x}r5| j        
                    |           t          	                    d|           | j                             d          x}r5| j                            |           t          	                    d	|           | j                             d
          x}r7| j                            |           t          	                    d|           dS dS )a  Granite uses standard llama parameters with the following differences:

        - No head_dim support
        - New multiplier params:
            - attention_scale
            - embedding_scale
            - residual_scale
        - logits_scaling
        r  Nz.Ignoring head_dim (%s) from config for Graniteattention_multiplierz$gguf: (granite) attention_scale = %sembedding_multiplierz$gguf: (granite) embedding_scale = %sresidual_multiplierz#gguf: (granite) residual_scale = %slogits_scalingz!gguf: (granite) logits_scale = %s)rG   r  rd   r>  r  r   r   rI   add_attention_scalere   rn  ro  rp  )rq   r  attention_scalerr  rs  logits_scaler  s         r/   r   z GraniteModel.set_gguf_parameters  s    |''
D9998 	WNNKXVVV##%%% #l../EFFF? 	Q00AAAKK>PPP"l../EFFF? 	Q00AAAKK>PPP!\--.CDDD> 	O//???KK=~NNN<++,<===< 	K,,\:::KK;\JJJJJ	K 	Kr.   )
r$   r%   r&   __doc__rf   r  GRANITErP   r   r9  r:  s   @r/   r	  r	    sQ        11(JK K K K K K K K Kr.   r	  GraniteMoeForCausalLMGraniteMoeSharedForCausalLMc                  F     e Zd ZdZej        j        Z fdZd fdZ	 xZ
S )GraniteMoeModelz*Conversion for IBM's GraniteMoeForCausalLMc                    t                                                       | j                            d          x}r7| j                            |           t                              d|           dS dS )zkGraniteMoeShared uses GraniteMoe parameters plus the following:
        - shared_intermediate_size
        shared_intermediate_sizez8gguf: (granitemoeshared) shared_feed_forward_length = %sN)r  r   rG   r   rI   r  rd   re   )rq   shared_feed_forward_lengthr  s     r/   r   z#GraniteMoeModel.set_gguf_parameters  s{     	##%%%)-)9)9:T)U)UU% 	pBBC]^^^KKRTnooooo	p 	pr.   r   r   rb   r   r   r   r\   r   c                V   |                     d          r| j        d         }|j        d         d|z  k    s
J d            |                    |d          \  }}|                     t
          j        j        |          |f|                     t
          j        j        |          |fgS t          | j        
                    d                    }|                     d          r| j        d	         }|j        d         d|z  k    s
J d
            |                    |d          \  }}|rN|                     t
          j        j        |          |f|                     t
          j        j        |          |fgS |                     t
          j        j        |          |f|                     t
          j        j        |          |fgS |s=|                     d          r(|                     t
          j        j        |          |fgS t!                                          |||          S )a/  In modeling_granitemoe, the JetMoe implementation of parallel experts
        is used. This essentially merges w1 and w3 into a single tensor with 2x
        the hidden size that is then split during forward. To keep compatibility
        with existing mixtral support, we pull them apart here.
        $block_sparse_moe.input_linear.weightr  r  r   z4Merged FFN tensor size must be 2 * intermediate_sizer  r  zshared_mlp.input_linear.weightr
  z;Merged FFN tensor size must be 2 * shared_intermediate_sizezshared_mlp.output_linear.weight)r  rG   r  r  r   rf   r  FFN_GATE_EXP
FFN_UP_EXPr>   r   FFN_GATE_SHEXPFFN_UP_SHEXPr  r  rE  r  r   )	rq   r   rb   r   ffn_dimgateuphas_expertsr  s	           r/   r   zGraniteMoeModel.modify_tensors  s/    ==?@@ 	l#67G#B'1w;6668n666!''R'88HD"(():)GMMtT(():)EsKKRP 
 4<++,?@@AA==9:: 	l#=>G#B'1w;6668u666!''R'88HD" ,,T->-MsSSUYZ,,T->-KSQQSUV 
 (():)CSII4P(():)A3GGL 
  	t}}-NOO 	(():)CSII:V  ww%%j$<<<r.   r  )r$   r%   r&   r
  rf   r  GRANITE_MOErP   r   r   r9  r:  s   @r/   r	
  r	
    sk        44,Jp p p p p%= %= %= %= %= %= %= %= %= %=r.   r	
  GraniteMoeHybridForCausalLMBambaForCausalLMc                  `     e Zd ZdZej        j        ZdZ fdZ	d Z
dd	ZddZd Zd Z xZS )GraniteHybridModelzzGraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM
    layers and optionally uses MoE w/ a shared expertTc                |    dg _          t                      j        |i |                                   _         fdt           j                  D              _                             ddg           _	                             dg           _
                             dg           j	        z   _        d S )Nmambac                &    g | ]}|j         v|S r-   _attn_layers)ra   r  rq   s     r/   r  z/GraniteHybridModel.__init__.<locals>.<listcomp>  s1     
 
 
))) )))r.   r  r  r  expand)hparam_prefixesr  r   get_attn_layersr 
  r  rR   _ssm_layersr   r  r  r  r  s   `  r/   r   zGraniteHybridModel.__init__  s     !(y$)&))) !0022
 
 
 
T-..
 
 
 ''	(BCC''55''
33dlBr.   c                   | j                             d          x}rd t          |          D             S | j                             dg           }|sm| j                             d          s
J d            | j                             d          
J d            fdt          | j                  D             }|S )	Nr  c                $    g | ]\  }}|d k    |S )r  r-   )ra   r  typs      r/   r  z6GraniteHybridModel.get_attn_layers.<locals>.<listcomp>)  s1       a+%% %%%r.   attn_layer_indicesr  z3Didn't find attn_layer_indices or attn_layer_periodr  z4No attention layer offset set with attn_layer_periodc                &    g | ]}|z  k    |S r-   r-   )ra   r  r  r  s     r/   r  z6GraniteHybridModel.get_attn_layers.<locals>.<listcomp>5  s2       {?k11 111r.   )rG   r   r  r  rR   )rq   r  attn_layersr  r  s      @@r/   r#
  z"GraniteHybridModel.get_attn_layers&  s   ,**=999; 	  )+ 6 6    l&&';R@@ 	,**+>??KUU UUU;,**+>??K**,b***     !122  K r.   rk   r   r\   r   c                    g }| j         D ]#|                    fd|D                        $t          |          |z   }t          j        | |g|R i |S )Nc              3  F   K   | ]}d                      |g          V  dS r   NrB  ra   r   pfxs     r/   rc   z1GraniteHybridModel.find_hparam.<locals>.<genexpr>>  G         #q""     r.   )r"
  extendr   r  r   )rq   rk   r  r  prefixedr0
  s        @r/   r   zGraniteHybridModel.find_hparam;  s    ' 	 	COO           DzzH$&tTCDCCCFCCCr.   r   r   rb   r   r   r   r   c                6   |                     d          sd|v rt                              | |||          S || j        v rt                              | |||          S || j        v rt                              | |||          S |                     |          |fgS )Nr
  
shared_mlp)r  r	
  r   r$
  r  r 
  r   r   s       r/   r   z!GraniteHybridModel.modify_tensorsE  s     MM@AA	Ot##"11$
D#NNN $"""--dJcJJJD%%%"11$
D#NNN%%d++Z899r.   c                2    t                                            j                                                 ddg                      j                                                 ddg                      j                             j                    j                             j	                    j        
                                         dg                                          ddg           fdt           j                  D             } j                            d	          x}r j                            |            j                            |           d
 j        d         v } j                            |           |s j                            d                                dgd          pd} j                            d          dv s
J d             j	        |z  dk    sJ d j	         d|             dS )ao  This method merges params from both parents and some that are
        specific to this model. The result is some duplication of how the params
        get set. The following warnings are expected during conversion:

        WARNING:Duplicated key name 'granitehybrid.attention.head_count_kv'
        WARNING:Duplicated key name 'granitehybrid.context_length'
        r  r  r  r  r  r  r  c                *    g | ]}|j         v rnd S r  r
  )ra   r  r  rq   s     r/   r  z:GraniteHybridModel.set_gguf_parameters.<locals>.<listcomp>j  s7     
 
 
?@Q$"333MM
 
 
r.   attn_rotary_embr
  r  r1  d_headTr  r  r  Nr  Only SILU activation supportedr   SSM inner size  not a multiple of head dim N)r	
  r   rI   r  r   r  r  r  r  r  r  r  rR   rG   r   r~  r  add_rope_scaling_finetunedr  )rq   head_count_kv_vecrc  use_roper9
  r  s   `    @r/   r   z&GraniteHybridModel.set_gguf_parametersU  s=    	++D111 	,,T-=-=}h>W-X-XYYY++D,<,<lI=V,W,WXXX,,T\:::++DL999 	//0@0@)0M0MNNN ((*?)MNN
 
 
 
 
DI$JZD[D[
 
 
 |''(9:::8 	@55h???**+<=== &o)FF33H=== 	7//666 !!8*t!<<B|--???Aa???|f$)))+oT\+o+ogm+o+o)))))r.   c                N    d| j         d<   t                              |            d S )Nr  r  )rG   r  r  r   s    r/   r  zGraniteHybridModel.set_vocab|  s)    23./d#####r.   rk   r   r\   r   r  )r$   r%   r&   r
  rf   r  GRANITE_HYBRIDrP   r  r   r#
  r   r   r   r  r9  r:  s   @r/   r
  r
    s        9 9/JLC C C C C(  *D D D D: : : : %p %p %pN$ $ $ $ $ $ $r.   r
  BailingMoeForCausalLMc                  v     e Zd ZU ej        j        Zd Z fdZdZ	de
d<   edd            ZddZ fdZ xZS )BailingMoeModelc                .    |                                   d S r`   r  r   s    r/   r  zBailingMoeModel.set_vocab  r  r.   c                N   t                                                       | j        }|                    d          x}|d         |d         z  }| j                            |           | j                            d          pi }|                    d|                    d                    dk    rnd|v rj| j                            t          j        j	                   | j        
                    |d                    | j                            |d	                    n)| j                            t          j        j                   | j                            |d
                    | j                            |d                    | j                            |d                    | j                            d           | j                            |d                    | j                            |d                    | j                            |d                    d S )Nr  r  r  r  r  rz   r  r  r  r(	  r  r   r  r  num_shared_expertsr?	  )r  r   rG   r   rI   r~  r  rf   r  r  r  r  r  r  rU  r  r+	  r  r  rG	  rb  s       r/   r   z#BailingMoeModel.set_gguf_parameters  s   ##%%%,J///H8}-9N1OOH11(;;;|''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;kllll2243G3LMMM66w?V7WXXX''(=>>>77@W8XYYY11#666))'-*@AAA009M1NOOO009I1JKKKKKr.   Nr$  rh  r  r   r  rQ   r  r   c                    |||k    r|} | j         |d| j        d         |z  dz  g| j        dd          R                      dd                               | j                  S r  r  re  s      r/   rf  zBailingMoeModel.permute  rg  r.   r   rb   r   r   r\   r   c           	     j   | j         d         }| j                             d          }| j         d         }| j                             d          x}||z  }|                     t          j        j                  }|                    d          r(|                     t          j        j        |          |fgS |                    d          r|                    ||z  ||z  ||z  gd          \  }	}
}|                     t          j        j	        |          t                              |	||          f|                     t          j        j        |          t                              |
||          f|                     t          j        j        |          |fgS |                    d	          d
k    r| j         d         }|J g }| j        #d t!          | j                  D             | _        || j        |         |<   t%          | j        |                   |dz  k    rdD ]}g }t!          |          D ]B}d| d| d| d}|                    | j        |         |                    | j        |         |= Ct)          j        |d          }d| d| d}|                     |          }|                    ||f           |S |                     |          }||k    rL| j                             d          r2|                                }|t)          j        |ddd          dz   z  }||fgS )Nr  r  r  r  zattention.dense.weightzquery_key_value.weightr  r  r
  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z2BailingMoeModel.modify_tensors.<locals>.<listcomp>  r*  r.   r    r  r  r  r   r   r   	norm_headr   T)pr  keepdimgHz>)rG   r   r   rf   r  r.  r  rD  r  r  rF
  rf  r  r  r.  rh  r  rR   r   rn  r   r5  r   r  norm)rq   r   rb   r   r  r  r  r  r(  r  r   rI  r  r  r  r6  r7  r8  r9  r   s                       r/   r   zBailingMoeModel.modify_tensors  s[   34L$$%:;;	m,((444H='H--d.?.FGG==122 '	,,T->-GMMzZ[[]]344 %	 &&(99x;OQZ]eQe'fln&ooGAq! (():)A3GGI`I`abdjlrIsIst(():)A3GGI`I`abdjluIvIvw(():)A3GGK 
 YY}%%++]3I???02G}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-77C ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::N''--{""t|'7'7'D'D"#))++J%*Z1!TJJJTQQJ:&''r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z3BailingMoeModel.prepare_tensors.<locals>.<listcomp>  r  r.   r   r  r  r  s     r/   rK  zBailingMoeModel.prepare_tensors  r  r.   r  r  )r$   r%   r&   rf   r  
BAILINGMOErP   r  r   rh  r  r  rf  r   rK  r9  r:  s   @r/   rF
  rF
    s         +J  L L L L L, 04H3333) ) ) \)8( 8( 8( 8(tD D D D D D D D Dr.   rF
  !ChameleonForConditionalGenerationChameleonForCausalLMc                  Z     e Zd Zej        j        Z fdZd ZddZ	e
d             Z xZS )ChameleonModelc                    t                                                       | j                            | j                            dd                     d S )N	swin_normF)r  r   rI   add_swin_normrG   r   r  s    r/   r   z"ChameleonModel.set_gguf_parameters  sH    ##%%%&&t|'7'7U'K'KLLLLLr.   c                .    |                                   d S r`   r  r   s    r/   r  zChameleonModel.set_vocab  r  r.   r   r   rb   r   r   r   r\   r   c                j   |                     d          rg S | j        d         }| j                            d          }| j                            d          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                    d          rt                              |||          }|                     |          |fgS )	Nzmodel.vqmodelr  r  r  rx  ry  )zq_norm.weightzq_norm.bias)zk_norm.weightzk_norm.bias)	rm  rG   r   r  rI  rf  rW
  r  r   )rq   r   rb   r   r  r  r  s          r/   r   zChameleonModel.modify_tensors  s'    ???++ 	I34L$$%:;;	\%%m44
==9:: 	H#++JGGJ==9:: 	K#++J	JJJ==9:: 	\';;JPZ[[J==9:: 	_';;J	S]^^J%%d++Z899r.   c                    ||z  }| d                              d|dz                                                                dd          } |                     |d          } | S )Nr   r   r   r  )r  r  r  repeat_interleave)r   r  r  r  s       r/   r  z"ChameleonModel._reverse_hf_permute  s`    (]''8q=99;;==EEaLL
11'1==
r.   r  )r$   r%   r&   rf   r  	CHAMELEONrP   r   r  r   r  r  r9  r:  s   @r/   rW
  rW
    s         *JM M M M M  : : : :,   \    r.   rW
  UltravoxModelc                  6     e Zd Zej        j        Z fdZ xZS )r`
  c                T     t                      j        |i | t          d          )NzUltravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to get the audio encoder, please use --mmproj argument)r  r   r   r  s      r/   r   zUltravoxModel.__init__  s:    $)&)))!  #}  ~  ~  	~r.   )	r$   r%   r&   rf   r  r  rP   r   r9  r:  s   @r/   r`
  r`
    sK        &J~ ~ ~ ~ ~ ~ ~ ~ ~r.   c                  >     e Zd ZdZdZ fdZ fdZd ZddZ xZ	S )WhisperEncoderModelFTc                     t                      j        |i | d| j        vrJd| j        vrC| j        d         | j        d<   | j        d         | j        d<   | j        d         | j        d<   d S d S d S )Nr  r  r  rG  rH  r  r  r  s      r/   r   zWhisperEncoderModel.__init__$  s    $)&))),,1DDL1X1X*.,y*ADL'04=N0ODL,-26,?X2YDL./// -,1X1Xr.   c                J   t                                                       | j                            t          j        j                   | j                            | j        d                    | j        	                    | j        
                    dd                     d S rJ  )r  r   rI   r  rf   r  QWEN2ArL  rG   rM  r   r  s    r/   r   z'WhisperEncoderModel.set_gguf_parameters+  s    ##%%%001I1PQQQ//^0LMMM::4<;K;KL\^b;c;cdddddr.   c                >    ~~~d|v rd|v rt           j        j        S dS r_  r`  r   s        r/   r   z&WhisperEncoderModel.tensor_force_quant1  ra  r.   r   r   rb   r   r   r   r\   r   c                    ~|                     d          rg S |                     d          rd|z   }d|v sd|v r|                    d          }|                     |          |fgS )Nrw  rr  zaudio.rc  rd  r  )rm  rf  r   r   s       r/   r   z"WhisperEncoderModel.modify_tensors7  s    ??,-- 	I ??233 	#d?D4<4#7#7#--b11J%%d++Z899r.   r  )
r$   r%   r&   r?  r@  r   r   r   r   r9  r:  s   @r/   rd
  rd
    s        Z Z Z Z Ze e e e e  : : : : : : : :r.   rd
  c                  &     e Zd ZdZdZ fdZ xZS )UltravoxWhisperEncoderModelFTc                    t                                                       | j                            t          j        j                   | j                            | j        d                    d S )Nstack_factor)	r  r   rI   r  rf   r  ULTRAVOXadd_audio_stack_factorr=  r  s    r/   r   z/UltravoxWhisperEncoderModel.set_gguf_parametersN  s[    ##%%%001I1RSSS//0B>0RSSSSSr.   r$   r%   r&   r?  r@  r   r9  r:  s   @r/   rk
  rk
  I  sM        T T T T T T T T Tr.   rk
  c                  &     e Zd ZdZdZ fdZ xZS )VoxtralWhisperEncoderModelFTc                    t                                                       | j                            t          j        j                   | j                            d           d S )Nr!   )r  r   rI   r  rf   r  VOXTRALro
  r  s    r/   r   z.VoxtralWhisperEncoderModel.set_gguf_parametersY  sS    ##%%%001I1QRRR//22222r.   rp
  r:  s   @r/   rr
  rr
  T  sD        3 3 3 3 3 3 3 3 3r.   rr
  FalconH1ForCausalLMc                  ^     e Zd Zej        j        Z fdZd fdZd Z	d fdZ
 fdZ xZS )FalconH1Modelc                   dg| _          t                      j        |i | t          | _        |                     dg          | _        |                     dg          | _        |                     dg          | _        d| _	        |                     dgd          | _
        |                     dgd          | _        |                     d	gd          | _        |                     d
gd          | _        |                     dgd          | _        |                     dgd          | _        |                     dg          | _        |                     dgd          | _        d S )Nr
  r  r  r9
  Tattention_in_multiplierr  attention_out_multiplierssm_in_multiplierssm_out_multipliermlp_multipliersssm_multipliersr  key_multiplier)r"
  r  r   rI  _transformer_model_classr   r  r  r9
  has_attentionry
  rz
  r{
  r|
  r}
  r~
  r  r
  r  s      r/   r   zFalconH1Model.__init__c  s    'y 	$)&))) )3% ''55''88&&z22 " (,'7'79R8S^b'7'c'c$(,(8(8:T9U`d(8(e(e%!%!1!13F2GRV!1!W!W"&"2"24H3ITX"2"Y"Y#//1B0Cd/SS#//1B0Cd/SS!%!1!13F2G!H!H"..0@/AD.QQr.   rk   r   r\   r   c                    g }| j         D ]#|                    fd|D                        $t          |          |z   } t                      j        |g|R i |S )Nc              3  F   K   | ]}d                      |g          V  dS r-
  r.
  r/
  s     r/   rc   z,FalconH1Model.find_hparam.<locals>.<genexpr>  r1
  r.   )r"
  r2
  r   r  r   )rq   rk   r  r  r3
  r0
  r  s        @r/   r   zFalconH1Model.find_hparam  s    ' 	 	COO           DzzH$"uww"49$999&999r.   c                .    |                                   d S r`   r  r   s    r/   r  zFalconH1Model.set_vocab  r  r.   r   r   rb   r   r   r   r   c                B   t          t                                          |||                    }|d         d         }d|v r|| j        d         z  }nd|v r|| j        d         z  }nd|v r|| j        z  | j        z  }nd|v r|| j        z  }nzd|v r|| j        z  }njd|v r|| j        z  }nZd	|v r|| j        z  }nJd
|v r|| j        z  }| j	        d         }| j	        d         }| j	        d         | j	        d         z  }|d |d d fxx         |d         z  cc<   ||d|z  d d fxx         |d         z  cc<   |d|z  d|z  |z   d d fxx         |d         z  cc<   |d|z  |z   d|z  d|z  z   d d fxx         |d         z  cc<   |d|z  d|z  z   d d d fxx         |d         z  cc<   nVd|v r|| j	        d         z  }nAd|v r|| j	        d         z  }n,d|v r(|
                    | j        | j        | j        z            }|d         d         |fg}|S )Nr   r   r  r  k_projq_projv_projo_projout_projin_projr~
  r  mamba_n_groupsr  r   r    r!   lm_headlm_head_multiplierembed_tokensr	  z
mamba.norm)r   r  r   r}
  r
  ry
  rz
  r|
  r{
  rG   r  r  r  )
rq   r   rb   r   r  r  zxbcdt_multipliersr  groups_time_state_sizer  s
            r/   r   zFalconH1Model.modify_tensors  s!   uww--j$DDEEA$t3A66FFD  d2155FFd11D4PPFFd::FFd::FFd;;FF4d55FF$d44F!%.?!@ $] ;%)\2B%CdlSbFc%c"%%%qqq()))-?-BB)))$Q):%::AAA=>>>BTUVBWW>>>1((->)>AW)WWYZYZYZZ[[[_qrs_tt[[[1((+AA!FWBWZ[^tZtBttvwvwvwwxxx  }O  PQ  }R  Rxxx1((1/E+EEFFIJJJN`abNccJJJJ$dl+?@@FFt##dl+ABBFFT!!^^DL$,$,2NOOFAJqM6*+r.   c                   t                                                       | j                            | j        d                    | j                            | j                   | j                            | j                            dd                     | j        	                    | j        d                    | j        
                    | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            | j        d                    | j                            d          d	v s
J d
            | j        | j        z  dk    sJ d| j         d| j                     | j                            |                     dg                     d S )Nr  r  r   r  r  r  r  r  r:
  r;
  r<
  r=
  r  )r  r   rI   rU  rG   r  rR   r  r   r  r  r  r  r  r  r9
  r  r   r  s    r/   r   z!FalconH1Model.set_gguf_parameters  s   ##%%% 	''\(BCCC(()9:::++DL,<,<=VXY,Z,Z[[[00>Q1RSSS 	''5J(KLLL**4<8M+NOOO''Z(@AAA))$,z*BCCC |--???Aa???|dk)Q...0y$,0y0ylplw0y0y... 	++D,<,<l^,L,LMMMMMr.   rB
  r  )r$   r%   r&   rf   r  r  rP   r   r   r  r   r   r9  r:  s   @r/   rw
  rw
  _  s        *JR R R R R8: : : : : :  $ $ $ $ $ $LN N N N N N N N Nr.   rw
  HunYuanMoEV1ForCausalLMc                  ^     e Zd ZU ej        j        Zd Z fdZdZ	de
d<   ddZ fdZ xZS )HunYuanMoEModelc           	        ddl m} |                    | j        d          }|                     |          }g }i }|j        }|                                D ]\  }}||t                              |          <   t          |          dk    r6t          
                    |||          }	t          |	          dk    r@|                    d                    t          t          j        |	                               | j        d	         }
|j        |
k    sJ |j        }d
 i ||                                D             }g }g }t#          |
          D ]}||vr>|                    d| d           |                    t$          j        j                   D||         }|                    |           ||                                v r%|                    t$          j        j                   |                    t$          j        j                   | j                            d           | j                            |           | j                            |           | j                            |           | j                            |           t%          j        | j        d          }|                    | j                   | j                             d           d S )Nr   r  Trr  r   r  r   r
  r  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z-HunYuanMoEModel.set_vocab.<locals>.<dictcomp>  s    ggg/?{C{gggr.   r  r  r  Fr  i )!r  r   ry  r:   r  r  rl   r  r  r   r  rn  rB  r  rG   r  r  r  rf   r  r+   r  r)   r'   rI   r  r  r  r  r  r  r  r  r;	  s                    r/   r  zHunYuanMoEModel.set_vocab  s   ......!11$.TX1YY	 ((33 #3*0022 	V 	VKE4<@E)11%8895zzQ]]?ED]IIF6{{achhs9+JF'S'STTUUU \,/
#z1111"1ggC^eC^~C^CdCdCfCfggg z"" 
	; 
	;A%%kQkkk*** 56666%a(e$$$--////OODN$:;;;;OODN$9:::: 	,,V444**6222''///((222))&111 )$.eLLL!!$"2333))&11111r.   c                  	 t                                                       | j        }| j                            |d                    | j                            |d                    |d         t          fdD                       sJ | j                            d                    |d         	t          	fd	D                       sJ | j                            	d                    |d         t          fd	D                       sJ | j        	                    d                    |
                    d
i           }|
                    d          dk    r|
                    dd          }|
                    dd          }|d         |d         z  }||||dz
  z  z  z  }| j                            |           | j                            t          j        j                   | j                            d           | j                            d           | j                            d           |dk    r|dk    r|dk    r| j        d         dv sJ d            d S d S )Nr  r  r   c              3  0   K   | ]}|d          k    V  dS r  r-   )ra   r   r   s     r/   rc   z6HunYuanMoEModel.set_gguf_parameters.<locals>.<genexpr>	  s.      PPQ1-a00PPPPPPr.   r   moe_topkc              3  0   K   | ]}|d          k    V  dS r  r-   )ra   topkr
  s     r/   rc   z6HunYuanMoEModel.set_gguf_parameters.<locals>.<genexpr>  s,      <<448A;&<<<<<<r.   num_shared_expertc              3  0   K   | ]}|d          k    V  dS r  r-   )ra   r   moe_shared_experts     r/   rc   z6HunYuanMoEModel.set_gguf_parameters.<locals>.<genexpr>  s.      HH1)!,,HHHHHHr.   r  rz   dynamicalpha  r  r  r  r  r   r      rJ  r  i   r
  dHunYuan dynamic RoPE scaling assumptions changed, please update the logic or context length manually)r  r   rG   rI   r  r  r  r  r  r  r   r  r  rf   r  r  r  r  r  )rq   rG   r  r
  r  r  scaled_baser   r
  r
  r  s          @@@r/   r   z#HunYuanMoEModel.set_gguf_parameters  s   ##%%%,))'-*@AAA>>wGZ?[\\\ '(? @PPPP:OPPPPPPPP778Ma8PQQQ:&<<<<8<<<<<<<<..x{;;;#$78HHHH6GHHHHHHHH001B11EFFF {{>266F##y00 !$$Wd33E;;|W55D=)W5J-KKC%C37O"<=K//<<<2243G3LMMM44Q777:::FFF//
;;; D==TW__UnHo  tK  IK  IK  IKv IK  IK  K 10 IK  IKr.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        |dk    r7| j                             dd          rt                              d           g S |                    d          dk    r| j         d         }|J | j        #d t          | j                  D             | _        || j        |         |<   t          | j        |                   |d	z  k    rg }d
D ]}g }t          |          D ]B}d| d| d| d}	|	                    | j        |         |	                    | j        |         |	= Ct          j        |d          }d| d| d}
|                     |
          }|	                    ||f           |S g S |                     |          |fgS )Nr  tie_word_embeddingsF+Skipping tied output layer 'lm_head.weight'r
  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z2HunYuanMoEModel.modify_tensors.<locals>.<listcomp>5  r*  r.   r    r  r  r  r   r   r   r  )rG   r   rd   re   r.  rh  r  rR   r   rn  r   r5  r   r  s               r/   r   zHunYuanMoEModel.modify_tensors*  s   ###| 5u== IJJJ	99]##r))]3I???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746C ; ;F*,E$Y// 6 6 W W W# W W W W WT]3%7%>??? M#.u55!&U!:!:!:J"S#"S"SF"S"S"SK#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z3HunYuanMoEModel.prepare_tensors.<locals>.<listcomp>R  r  r.   r   r  r  r  s     r/   rK  zHunYuanMoEModel.prepare_tensorsO  sl    !!!=$BB$-BBBG7||a !B!B!BCCC %$r.   r  )r$   r%   r&   rf   r  HUNYUAN_MOErP   r  r   rh  r  r   rK  r9  r:  s   @r/   r
  r
    s         ,J12 12 12f%w %w %w %w %wN 04H3333#: #: #: #:JD D D D D D D D Dr.   r
  HunYuanDenseV1ForCausalLMc                  D     e Zd Zej        j        Zd Z fdZddZ	 xZ
S )HunYuanModelc           	     Z   | j         dz                                  r|                                  d S ddlm} |                    | j         d          }|                     |          }g }i }|j        }|                                D ]\  }}||t          
                    |          <   t          |          dk    r6t                              |||          }	t          |	          dk    r@|                    d	                    t          t          j
        |	                               | j        d
         }
|j        |
k    sJ |j        }d i ||                                D             }g }g }t'          |
          D ]}||vr>|                    d| d           |                    t(          j        j                   D||         }|                    |           ||                                v r%|                    t(          j        j                   |                    t(          j        j                   | j                            d           | j                            |           | j                            |           | j                            |           | j                            |           t)          j         | j         d          }|!                    | j                   | j        d         dk    r| j        "                    d           d S d S )Nr  r   r  Trr  r   r  r   r
  r  c                    i | ]\  }}||	S r-   r-   r  s      r/   r  z*HunYuanModel.set_vocab.<locals>.<dictcomp>u  r9	  r.   r  r  r  Fr  r  r  i )#r:   r   r  r  r   ry  r  r  rl   r  r  r   r  rn  rB  r  rG   r  r  r  rf   r  r+   r  r)   r'   rI   r  r  r  r  r  r  r  r  r;	  s                    r/   r  zHunYuanModel.set_vocab[  s$   N--6688 4	:  """""222222%55dnX\5]]I ,,Y77F FE'7O.4466 Z Zt@Di55e<<=u::??"MMv;;!##MM#((3y/NPV+W+W"X"XYYY l3J':5555&5NkkGb%GbSaGbGhGhGjGjkkkM "F"$H:&& 
? 
?M))MM++++...OODN$9::::)!,EMM%(((N113333 (>???? (=>>>> 00888..v666++F333,,X666--f555 !-dn%PPPM%%d&6777|M*d22 11&99999 32r.   c                   t                                                       | j        }|                    di           }|                    d          dk    r|                    dd          }|                    dd          }|d         }||||d	z
  z  z  z  }| j                            |           | j                            t          j        j	                   | j        
                    d
           | j                            d           | j                            d           |dk    r| j        d         dv sJ d            d S d S )Nr  rz   r
  r
  2   r  r  r  r   r   r
  r  r
  r
  )r  r   rG   r   rI   r  r  rf   r  r  r  r  r  )rq   rG   r  r
  r  r  r
  r  s          r/   r   z HunYuanModel.set_gguf_parameters  sS   ##%%%, {{>266F##y00 !$$Wb11E;;|W55D*%C%C37O"<=K//<<<2243G3LMMM44Q777:::FFF//
;;; 7??t|4M'NRi'i'i'iv (j'ii 10 (j'ir.   r   r   rb   r   r   r   r\   r   c                    |dk    r7| j                             dd          rt                              d           g S |                     |          |fgS )Nr  r
  Fr
  )rG   r   rd   re   r   r   s       r/   r   zHunYuanModel.modify_tensors  s]    ###| 5u== IJJJ	%%d++Z899r.   r  )r$   r%   r&   rf   r  HUNYUAN_DENSErP   r  r   r   r9  r:  s   @r/   r
  r
  W  sk        .J5: 5: 5:nw w w w w0: : : : : : : :r.   r
  SmolLM3ForCausalLMc                  6     e Zd Zej        j        Z fdZ xZS )SmolLM3Modelc                   t                                                       ddlm} |                    | j                  }|j        7|j                            dd          }| j        	                    |           d S d S )Nr   r  z[:]r  )
r  r  r  r   ry  r:   r  r  rI   rX  )rq   r   r  r  r  s       r/   r  zSmolLM3Model.set_vocab  s     	/.....!11$.AA	".%3;;E2FFM..}===== /.r.   )	r$   r%   r&   rf   r  SMOLLM3rP   r  r9  r:  s   @r/   r
  r
    sB        (J> > > > > > > > >r.   r
  GptOssForCausalLMc                  Z     e Zd Zej        j        Zd ZddZdd
Z	ddZ
d Z fdZ xZS )GptOssModelc                x   |j         t          j        k    sJ |j        d         dk    sJ |dz  }|dz  }|dz  |dz	  z  }|}|                    dd          \  }}|dz                      dd          }|dz                      dd          }t          j        ||fd                              |j                  }|dz	                      dd          }	|dz                      dd          }
t          j        |	|
fd                              |j                  }||z  }|dz  }|dz  }|dz	  |dz  z  }|S )	Nr  r        r!   r   r  r   )r   r   r@  r  chunkr  r5  )rq   r  t_lot_hi	t_swappedblk_ablk_bblk_a0blk_a1blk_b0blk_b1r#  out_hout_ls                 r/   transform_nibble_layoutz#GptOssModel.transform_nibble_layout  sY   |u{****|B2%%%%}}QY419-	||A2|..u$,$$R++1*""2q))VV,!44499&,GG1*""2q))$,$$R++VV,!44499&,GGemd
d
zeqj)
r.   r   r   rn  r   r  c                   |j         t          j        k    sJ |j         t          j        k    sJ |                    d          }t	          |j                  dk    sJ t	          |j                  dk    sJ |                     |          }t          j        ||fd          }|j        d         |j        d         |j        d         dz  g}t          	                    d| d	| d
           |
                    |j        d         |j        d         |j        d         |j        d         z            }|                                }| j                            ||t          j        j                   d S )Nr  r!   r  r   r   r   rL  z	Repacked z with shape z and quantization MXFP4r    r  )r   r   r@  rf  r   r  r
  concatrd   re   r  r  rI   rD  rf   r  MXFP4)rq   r   rn  r  new_data	new_shapes         r/   repack_mxfp4zGptOssModel.repack_mxfp4  s`   |u{****|u{****!!"%%6<  A%%%%6<  A%%%%--f55< 0b999^A&q(98>!;Lr;QR	XXXiXXXYYY==!2HN14Ex~VWGX[c[ijk[lGlmm>>####Hh$B[Ba#bbbbbr.   r\   r   c                   t          j        d          }t          j        d          }|                                 D ])\  }}d|v r|}d|v rA|                     |                    dd                    }|                     |||           Rd|v r+|d d d d dd d d d f         |d d dd dd d d d f         }}d|v r|d d d d dd d f         |d d dd dd d f         }}|                     |                    d	d
                    }|                     |                    d	d                    }	|                     |||           |                     |	||           +g S )Nr   zmlp.experts.down_proj_blockszmlp.experts.down_proj_scales_scalesr   zmlp.experts.gate_up_proj_blocksr   zmlp.experts.gate_up_proj_scalesgate_up_proj_scalesr  r  )r   zerosr   r   r  r
  )
rq   blocks0blocks1rb   r   r   scales0scales1new_name_gatenew_name_ups
             r/   r   z"GptOssModel.generate_extra_tensors  s   +a..+a.. $ 0 0 2 2 	A 	AD*-55$/477//Y	0R0RSS!!(GZ@@@@2d::#-aaa1aaal#;Z14a4QRQRQRTUTUTU=V2d::#-aaa1aaai#8*QQQ1aaaZ:P $ 4 4T\\BWYk5l5l m m"224<<@UWg3h3hii!!-'BBB!!+w@@@	r.   r   rb   r   r   c                   ~d|v r|dz  }d|v r|                     d          r|                    dd          }nTd|vrNd|vrJt                              | d	           |                    dd
          }|                    dd          }ng S d|v rD|                     d          rt|                    dd          }|                    dd          }|dd d df         |ddd df         }}|                     |          |f|                     |          |fgS d|vrd|vrt                              | d	           |                    dd          }|                    dd          }|                    dd          }|d d d d dd d f         |d d dd dd d f         }	}|                     |          |f|                     |          |	fgS g S |                     |          |fgS )Nsinksr   r  _biasdown_proj_biaszdown_proj.bias_blocksr
  z- is not in MXFP4, performance may be degradedzdown_proj.weightr  r  r  gate_up_proj_biaszup_proj.biaszgate_proj.bias.r   r   r  r  )r  r  rd   r>  r  r   )
rq   r   rb   r   r  r  gate_proj_biasup_proj_biasr  r   s
             r/   r   zGptOssModel.modify_tensors   so   d??ID $}}W%% ||$46FGG$&&9D+@+@$UUUVVV||K1CDD'11"b99

 	 T!!}}W%% ,,':NKK LL)<>NOO	/9#sss(/CZPSUVUYXYUYPYEZ)))44nE))'22LA  $&&9D+@+@$UUUVVV,,~7GHH LL9KLL	'11"b99
3=aaa1aaai3H*UVUVUVXYX\[\X\^_^_^_U_J`. )))446FG))'22NC  	%%d++Z899r.   c                .    |                                   d S r`   r  r   s    r/   r  zGptOssModel.set_vocab.   r  r.   c                z   t                                                       | j                            | j        d                    | j                            | j        d                    | j                            d          pi }|                    d|                    d                    }|dk    sJ d|             | j                            t          j	        j
                   | j                            |d                    | j                            |                    d	d
                     d S )Nr  r  r  r  rz   r  z-GPT-OSS only supports yarn rope scaling, got r  r  r  )r  r   rI   r  rG   r  r   r  rf   r  r  r  r  )rq   r  r  r  s      r/   r   zGptOssModel.set_gguf_parameters1   s   ##%%%++DL9I,JKKK77EX8YZZZ|''77=2 $$[,2B2B62J2JKK	F"""$_T]$_$_"""..t/C/HIII00h1GHHH66|7G7GHjlp7q7qrrrrrr.   )r   r   rn  r   r  r   r  r  )r$   r%   r&   rf   r  GPT_OSSrP   r
  r
  r   r   r  r   r9  r:  s   @r/   r
  r
    s        (J  2c c c c   (*: *: *: *:X  
s 
s 
s 
s 
s 
s 
s 
s 
sr.   r
  Lfm2ForCausalLMLFM2ForCausalLMc                  D     e Zd Zej        j        Zd Z fdZddZ	 xZ
S )	LFM2Modelc                .   | j         d         }| j         d         }| j         d         }| j         d         }| j         d         }|r7t          d|z  dz            }|t          ||z            }|||z   dz
  |z  z  }| j                            |           d S )Nblock_ff_dimblock_auto_adjust_ff_dimblock_ffn_dim_multiplierblock_multiple_ofr   r    r   )rG   rQ   rI   r  )rq   r  auto_adjust_ff_dimffn_dim_multiplierr  s        r/   _add_feed_forward_lengthz"LFM2Model._add_feed_forward_lengthC   s    n-!\*DEn-!\*DEl#67 	OVa((F!-/&899 Vk%9A%=+$MNF0088888r.   c                     fd j         d         D              j         d<   t                                                        j                             j         d                     j                             j         d                     j                             j         d                                                      d S )Nc                :    g | ]}|d k    rj         d         ndS )full_attentionr  r   r   )ra   
layer_typerq   s     r/   r  z1LFM2Model.set_gguf_parameters.<locals>.<listcomp>V   sB     /
 /
 /
 4>AQ3Q3QDL.//WX/
 /
 /
r.   r  r  r  conv_L_cacher  )rG   r  r   rI   rU  add_shortconv_l_cacher  r
  r  s   `r/   r   zLFM2Model.set_gguf_parametersT   s    /
 /
 /
 /
"l=9/
 /
 /
*+
 	##%%%''\(BCCC..t|N/KLLL//Z0HIII%%'''''r.   r   r   rb   r   r   r   r\   r   c                d    d|v r|                     d          }|                     |          |fgS )Nz	conv.convr   )r  r   r   s       r/   r   zLFM2Model.modify_tensorsa   s<    $#++A..J%%d++Z899r.   r  )r$   r%   r&   rf   r  LFM2rP   r
  r   r   r9  r:  s   @r/   r
  r
  >   sg         %J9 9 9"( ( ( ( (: : : : : : : :r.   r
  SmallThinkerForCausalLMc                  X     e Zd ZU ej        j        Z fdZdZde	d<   ddZ
 fdZ xZS )SmallThinkerModelc                h   t                                                       | j                            d| j                            d                    x}| j                            |           | j                            d| j                            d                    x}| j                            |           | j                            d          x}Q| j                            |           | j                            |           t          
                    d|            | j                            d          r*| j                            t          j        j                   n)| j                            t          j        j                   | j                            d          pi }|                    d	|                    d
                    dk    rmd|v ri| j                            t          j        j                   | j                            |d                    | j                            |d                    | j                            d          }|rD|D ]C}|dk    r9| j                            d          }|r| j                            |            d S Bd S d S )Nr  moe_num_primary_expertsr  moe_num_active_primary_expertsmoe_ffn_hidden_sizer   moe_primary_router_apply_softmaxr  r  rz   r  r  r  sliding_window_layoutr   sliding_window_size)r  r   rG   r   rI   r  r  r  r  rd   re   rH	  rf   rI	  rK	  rJ	  r  r  r  r  r  r  )	rq   r  r  r   r  r  r  r  r  s	           r/   r   z%SmallThinkerModel.set_gguf_parametersm   s   ##%%%))-9I9IJc9d9deeeIr--i888"l../DdlFVFVWwFxFxyyyN  G22>BBB%)\%5%56K%L%LL!Y;;<QRRR445JKKKKKU>SUUVVVL?@@ 	W33D4M4UVVVV33D4M4UVVV |''77=2K)9)9&)A)ABBfLLQY]iQiQi2243G3LMMM44\(5KLLL::<Hj;klll $ 0 01H I I  	*  66%)\%5%56K%L%LN% L(;;NKKKEE	 	 	 r.   Nr$  rh  r   r   rb   r   r   r   r\   r   c           	        |                     d          dk    r<| j                            d| j                            d                    }|J | j        #d t	          | j                  D             | _        || j        |         |<   t          | j        |                   |dz  k    rg }dD ]}g }t	          |          D ]B}d| d	| d
| d}	|                    | j        |         |	                    | j        |         |	= Ct          j	        |d          }d| d	| d}
| 
                    |
          }|                    ||f           |S g S | 
                    |          |fgS )Nr  r  r  r  c                    g | ]}i S r-   r-   r)  s     r/   r  z4SmallThinkerModel.modify_tensors.<locals>.<listcomp>   r*  r.   r    )downr
  r
  r  r  r   r   r   r  )r.  rG   r   rh  r  rR   r   rn  r   r5  r   r  s               r/   r   z SmallThinkerModel.modify_tensors   s   99Y2%%((8H8HIb8c8cddI???}$ E EU43C-D-D E E E'1DM#t$4=%&&)a-7746 5 ; ;F*,E$Y// 6 6 d d ds d dU[ d d dT]3%7%>??? M#.u55!&U!:!:!:J"`#"`"`QW"`"`"`K#33K@@HNNHj#9::::	%%d++Z899r.   c                    t                                                       | j        6d | j        D             }t          |          dk    rt	          d|           d S d S )Nc                @    g | ]}|                                 D ]}|S r-   r<  r=  s      r/   r  z5SmallThinkerModel.prepare_tensors.<locals>.<listcomp>   r  r.   r   r  r  r  s     r/   rK  z!SmallThinkerModel.prepare_tensors   r  r.   r  )r$   r%   r&   rf   r  SMALLTHINKERrP   r   rh  r  r   rK  r9  r:  s   @r/   r   r   i   s         -J    > 04H3333": ": ": ":HD D D D D D D D Dr.   r   c                  N    e Zd Zej        j        ZdZdZdZ	dZ
edd	            Zd
S )rV  Mistralr  TFr  r   templates_dirr   c                   t           
J d            t          | j        t          t          f          sJ dt          | j                               | j        j        t           j        k    rdS | j        j        t           j        k    r| j	        t          j        k    rdS | j        j        t           j        k    r| j	        t          j        k    rdS | j        j        t           j        k    r| j	        t          j        k    rdS | j        j        t           j        k    r| j	        t          j        k    rdS | j        j        t           j        k    rd}nA| j        j        t           j        k    rd	}n$t!          d
| j	         d| j        j                   ||z  }|                                st%          d|           t'          |dd          5 }|                                }d d d            n# 1 swxY w Y   |S )Nzmistral_common is not installedz3Expected Tekkenizer or SentencePieceTokenizer, got z
mistral-v1z
mistral-v3zmistral-v3-tekkenz
mistral-v7zmistral-v7-tekkenz)Mistral-Small-3.2-24B-Instruct-2506.jinjaz)unsloth-mistral-Devstral-Small-2507.jinjazUnknown tokenizer type: z and version zTemplate file not found: r   r   r   )r   r   r  r   r   rz   r  v1v3rN  r   spmrY  v7v11v13r   r  r  r   r  )r  r  template_filer  r   r  s         r/   rW  z(MistralModel.get_community_chat_template   s-   ++-N+++%/J8N+OPP 	
 	
Y$uBWBWYY	
 	
P ?"&6&999<_$(8(;;;@TXlXp@p@p<_$(8(;;;@TXlXs@s@s&&_$(8(;;;@TXlXp@p@p<_$(8(;;;@TXlXs@s@s&&_$(8(<<<GMM_$(8(<<<GMMt8Ltt[`[j[rttuuu%5##%% 	Q#$O$O$OPPP-w777 	 1vvxxH	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  s   G..G25G2N)r  r   r  r   )r$   r%   r&   rf   r  r  rP   rK   r  rT   r  r  rW  r-   r.   r/   rV  rV     sQ        &JJGL   \  r.   rV  c                  8     e Zd ZdZdZdZ fdZdd fdZ xZS )PixtralModelPixtralr  Tc                   t                                                       | j                            t          j        j                   | j                            |                     dg                     | j        	                    | 
                    dg                     | j                            d           | 
                    dg          dk    r0| j                            | 
                    dg                     d S d S )Nr  r  Tmm_projector_idpatch_merger  )r  r   rI   r  rf   r  r  r  r   r  r`  r  r  r  s    r/   r   z PixtralModel.set_gguf_parameters   s   ##%%%001I1QRRR;;j\**	
 	
 	
 	++D,<,<l^,L,LMMM,,T222 ./00MAA::  "6!788     BAr.   r   rb   r   r   r   r\   c                h    |dk    rdS |dk    rdS t                                          ||          S )Nz#vision_language_adapter.w_in.weightzmm.1.weightz$vision_language_adapter.w_out.weightzmm.2.weight)r  r   )rq   rb   r   r  s      r/   r   zPixtralModel.map_tensor_name   s?    888 =;;; =ww&&t\:::r.   r  r  )	r$   r%   r&   rK   r  rT   r   r   r9  r:  s   @r/   r  r     sk        JG    "; ; ; ; ; ; ; ; ; ; ;r.   r  c                     e Zd ZU ej        Zded<   ded<   ej        ej        ej	        ej	        ej
        ej
        iZded<   ej        ej	        ej        ej        ej        ej        ej        ej
        ej        ej        ej        ej        dZded	<   ddZedd            Zedd            Zedd            Zedd            ZdS )rm   torch.dtyper   z
torch.Sizer  zdict[torch.dtype, type]
_dtype_map)F64r  r6  r4  I64I32I16U8I8BOOLF8_E4M3F8_E5M2zdict[str, torch.dtype]_dtype_str_mapr\   gguf.LazyNumpyTensorc                    | j         | j                 }t          j        t          j                            || j                  | fd           S )Nc                *    |                                  S r`   )r  r   s    r/   <lambda>z'LazyTorchTensor.numpy.<locals>.<lambda>-!  s    AGGII r.   metar  r  )r"  r   rf   LazyNumpyTensormeta_with_dtype_and_shaper  )rq   r   s     r/   r  zLazyTorchTensor.numpy(!  sM    
+#%??tzRR%%
 
 
 	
r.   tuple[int, ...]r   c                0    t          j        ||d          S )Nr3  )r  r   r   )r   empty)r   r   r  s      r/   r5  z)LazyTorchTensor.meta_with_dtype_and_shape0!  s    {U6BBBBr.   st_slicer   c                    | j         |                                         }t          |                                          } | |                     ||          |fd           }t          t          j        |          S )Nc                    | d d          S r`   r-   r0  s    r/   r1  z8LazyTorchTensor.from_safetensors_slice.<locals>.<lambda>8!  s    fghihihifj r.   r2  )r,  	get_dtypetuple	get_shaper5  r   r   r   )r   r9  r   r  rB   s        r/   r   z&LazyTorchTensor.from_safetensors_slice4!  sr    "8#5#5#7#78!&x'9'9';';!<!<s55eUCC8+\j\jkkkEL$'''r.   rp   gguf.utility.RemoteTensorc                    | j         |j                 |j        |                               } | ||ffd          }t	          t
          j        |          S )Nc                z    t          j        |                                                                         S )Nr  )r   
frombufferr   r  )r   r   r  s    r/   r1  z4LazyTorchTensor.from_remote_tensor.<locals>.<lambda>@!  s4    EDTUVU[U[U]U]ejDkDkDkDsDstyDzDz r.   r2  )r,  r   r  r5  r   r   r   )r   rp   r3  rB   r   r  s       @@r/   rn   z"LazyTorchTensor.from_remote_tensor;!  si    "=#67#,,UE::sM#3:z:z:z:z:z{{{EL$'''r.   r-   Nc                    ~|i }|t           j        j        u r|d                                         S  |                     |          |i |S r  )r   r   r  _wrap_fn)r   r  typesr  r  s        r/   __torch_function__z"LazyTorchTensor.__torch_function__C!  sR    >F5<%%%7==??"!s||D!!426222r.   )r\   r-  )r   r!  r  r6  r\   r   )r9  r   r\   r   )rp   r?  )r-   N)r$   r%   r&   r   r   _tensor_typer  r   r?  r  r@  r"  float64bfloat16int64int32int16int8r>   float8_e4m3fnfloat8_e5m2r,  r  r  r5  r   rn   rF  r-   r.   r/   rm   rm   !  se        <L 	rzrzRX+J     }}}{{{kj
&$. .N    $
 
 
 
 C C C [C ( ( ( [( ( ( ( [( 	3 	3 	3 [	3 	3 	3r.   rm   r\   argparse.Namespacec                 T   t          j        d          } |                     ddd           |                     dt          d	           |                     d
t          g ddd           |                     ddd           |                     dt          dd           |                     ddd           |                     ddd           |                     dt          d d           |                     ddd           |                     dt
          dd            |                     d!t          d"d#           |                     d$dd%           |                     d&dd'           |                     d(t          d)	           |                     d*dd+           |                     d,dd-           |                     d.dd/           |                     d0dd1           |                                 }|j        s|j        | 	                    d2           |S )3Nz5Convert a huggingface model to a GGML compatible file)descriptionz--vocab-only
store_truezextract only the vocab)actionhelpz	--outfilezSpath to write to; default: based on input. {ftype} will be replaced by the outtype.)rz   rU  z	--outtypef32f16bf16q8_0tq1_0tq2_0autorX  zoutput format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type)rz   choicesr  rU  z--bigendianz'model is executed on big endian machiners   zJdirectory containing model file or huggingface repository ID (if --remote)?)rz   rU  nargsz--use-temp-filez^use the tempfile library while processing (helpful when running out of memory, process killed)z	--no-lazyz\use more RAM by computing all outputs before writing (use in case lazy evaluation is broken)z--model-namezname of the model)rz   r  rU  z	--verbosezincrease output verbosityz--split-max-tensorsr   zmax tensors in each splitz--split-max-size0zmax size per split N(M|G)z	--dry-runzConly print out a split plan and exit, without writing any new filesz--no-tensor-first-splitz;do not add tensors to the first split (disabled by default)z
--metadataz9Specify the path for an authorship metadata override filez--print-supported-modelszPrint the supported modelsz--remoteaq  (Experimental) Read safetensors file remotely without downloading to disk. Config and tokenizer files will still be downloaded. To use this feature, you need to specify Hugging Face model repo name instead of a local directory. For example: 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set HF_TOKEN environment variable to your Hugging Face token.z--mmprojz(Experimental) Export multimodal projector (mmproj) for vision models. This will only work on some vision models. A prefix 'mmproj-' will be added to the output file name.z--mistral-formatz9Whether the model is stored following the Mistral format.z+the following arguments are required: model)
argparseArgumentParseradd_argumentr   r   rQ   
parse_argsprint_supported_modelsrs   r  )parserr  s     r/   re  re  P!  s7   $KM M MF
|%     $b     #'_'_'_in a     l6     cY    
 ,m     Lk     S$      L(     C(     c(     LR     !,J     4H     "<)     < A     < {     <H    
 D& D4:+=BCCCKr.   	split_strr   rQ   c                   |                      d          rt          | d d                   dz  }n|                      d          rt          | d d                   dz  dz  }nm|                      d          r!t          | d d                   dz  dz  dz  }n7|                                 rt          |           }nt          d|  d          |dk     rt          d|  d	          |S )
NKr  r
  MGzInvalid split size: z5, must be a number, optionally followed by K, M, or Gr   z, must be positive)r  rQ   	isnumericr   )rh  r   s     r/   split_str_to_n_bytesrn  !  s   # 	r	#2#$&			C	 	  r	#2#$&-			C	 	  r	#2#$&-4					 r	NNp	pppqqq1uuM	MMMNNNHr.   rG   rF   r  c                   |                      di           }|                      di           }d }|                      d          x}t          |          dk    r	|d         }n#d| v r| d                              dd          dz   }|t          j        k    r$|                     d          |d         d         }n3|t          j        k    r#|                     d          |d         d         }|t          d	          |S )
Nrw  rS  r  r   ssm_cfglayerMambaForCausalLMz#Failed to detect model architecture)r   r   r1   r2   r3   r   )rG   r  rw  rS  rx   archess         r/   r  r  !  s    ++mR00KKK44MD++o...;Faay	g		y!%%gw77-G Y^##(H(H(T?+A.	y'	'	'M,=,=o,N,N,Z_-a0|>???Kr.   r  c                 	   t                      } | j        rGt                              d           t                                           t          j        d           | j        r t          j
        t          j                   nt          j
        t          j                   | j        rI| j        }ddlm}  ||g d          }t#          |          }t                              d|            nd }t#          | j                  }|                                s2t                              d| d	           t          j        d
           t(          j        j        t(          j        j        t(          j        j        t(          j        j        t(          j        j        t(          j        j        t(          j        j        d}| j        dk    p
| j        dk    }| j        r0|r.t                              d           t          j        d
           | j         | j         }n.|r*t#          d|!                    dd          z   dz             }n|}t                              d|j"                    | j#        r$d|j"        vrt          $                    |d          }| j%        }tM          j'                    5  || j(                 }	| j#        rtR          j*        ntR          j+        }
t          ,                    ||          }|st[          ||
          }t                              d|            	 t          .                    ||
          }nw# t^          $ r5 t                              d| d           t          j        d
           Y n9w xY w| j#        r'|0                    d          
J d            tb          }ntd          } |||	|| j3        | j        | j4        | j5        | j6        | j        to          | j                  | j8        | j9        |          }| j:        rQt                              d           |;                                 t                              d|j<                    nnt                              d           |=                                 |r|j<        j>         t~          j@         n|j<        }t                              d|            d d d            d S # 1 swxY w Y   d S ) NzSupported models:r   )level)snapshot_download)LICENSEz*.jsonz*.mdz*.txtr  )repo_idallow_patternsz#Downloaded config and tokenizer to zError: z is not a directoryr   rV  ra  z*Error: Cannot use temp file when splittingz.//-z-{ftype}.ggufzLoading model: mmprojzmmproj-zModel architecture: r  zModel z is not supportedrF  z&This model does not support multimodal)
r?   rA   rU   rM   rK   rV   rW   rX   rY   rO   zExporting model vocab...z%Model vocab successfully exported to zExporting model...zModel successfully exported to )Are  rf  rd   r  r8   r  r
  r3  verboseloggingbasicConfigDEBUGINFOremoters   huggingface_hubrw  r   re   r  rf   r   r5  r   r   r7  r2  r3  r   rV   rW   rA   outfiler  rb   r}  r   mistral_formatr   inference_modeouttyper1   r3   r2   r   r  r  r   r   r  rV  	bigendianno_lazyrU  rK   rn  rX   no_tensor_first_splitrQ  r`  r=   ri  r  rk  sep)r  
hf_repo_idrw  	local_dirr:   	ftype_mapis_splitr=   rT   r  r  rG   model_architecturemodel_classmodel_instanceout_paths                   r/   mainr  !  s*   <<D" ()))))+++| 0'-00000',////{ 
%Z
555555%%TTTV V V	 OO	E)EEFFFF
$$	 =y===>>> !)!,".".#0#0"*0 0I %)GT-@C-GH h ABBB|L			 
 2 23 < <<NOO			
KK2).22333{ O9>))!88INNI+				 $F $F-)-HY%%).
((4EFF  	'!7!L!LKKC/ACCDDD'??@R_i?jj&   K&8KKKLLL [ 	';;/00<<>f<<<&KK&K$YY37>QUQc+/<7;}QUQ`7;7M4HI\4]4]gkgs7;7Q8B' ' ' ? 	FKK2333&&(((KKZ@XZZ[[[[KK,---  """GOm.29C26CCCUcUmHKKD(DDEEEI$F $F $F $F $F $F $F $F $F $F $F $F $F $F $F $F $F $Fs8   3A7S$+MS$<NS$NES$$S(+S(__main__)r\   rP  )rh  r   r\   rQ   )rG   rF   r  r1   r\   r   r8  )
__future__r   r  r  rb  r   r   rk  r  r
  enumr   pathlibr   hashlibr   typingr   r   r	   r
   r   r   r   r   r   r   	itertoolsr   r  r   r  r  r?  r   r   rk  rw   insertr   r  r  rf   
gguf.vocabr   r   %mistral_common.tokens.tokenizers.baser   +mistral_common.tokens.tokenizers.multimodalr   r   'mistral_common.tokens.tokenizers.tekkenr   .mistral_common.tokens.tokenizers.sentencepiecer   	getLoggerrd   r   r1   r4   r8   r{   r|   r  rz  r  r  r  r  r  r  r  r  r  rI  r  r  r  r  r  r  r  r4  rI  rX  rg  r  r  r  r  r  r  r  r  r%  rE  rg  rx  r  r  r  r  r  r  r  r  r  r!  r-  rD  rH  rz  r  r  r  r  r  r  r  r  r  r'  r+  rN  r[  r  r  r  r  r  r  r  r  r  r  r  r	  r%	  r6	  r]	  rf	  rq	  r	  r	  r	  r	  r	  r	  r	  r	  r	  r	
  r
  rF
  rW
  r`
  rd
  rk
  rr
  rw
  r
  r
  r
  r
  r
  r   rV  r  LazyBaserm   re  rn  r  r  r$   r-   r.   r/   <module>r     s$   # " " " " " 



        				 				 



                   u u u u u u u u u u u u u u u u u u u u u u u u       # # # # # #       "*$$HOOAss44>>09<==>>>  9 9 9 9 9 9 9 9 B B B B B B Q Q Q Q Q Q Q Q > > > > > >     
 
	<	(	(
    g          
 7:%6777lY lY lY lY lY lY lY lY^N< N< N< N< N<	 N< N< N<by9 y9 y9 y9 y9) y9 y9 y9x ())5 5 5 5 59 5 5 *)5p &555 5 5 5 5 5 5 655p $%%(( (( (( (( ((y (( (( &%((V &''J J J J J J J ('JD )+@AAL7 L7 L7 L7 L7I L7 L7 BAL7^ '((g
 g
 g
 g
 g
) g
 g
 )(g
T '992: 2: 2: 2: 2:) 2: 2: :92:j +,,3 3 3 3 3Y 3 3 -,3  *++< < < < <) < < ,+<~ )+EGfggV@ V@ V@ V@ V@I V@ V@ hgV@r #% kD kD kD kD kD kD kD kD\ &''
m 
m 
m 
m 
m 
m 
m ('
m #& N N N N N{ N N	 Nb 68YZZ    ;   [ZB 455'= '= '= '= '=* '= '= 65'=T 455       65, 677= = = = =J = = 87= '((m" m" m" m" m"	 m" m" )(m"` '((%% %% %% %% %%) %% %% )(%%P %&&1: 1: 1: 1: 1:	 1: 1: '&1:h %&&A A A A A	 A A '&AH ())3: 3: 3: 3: 3:9 3: 3: *)3:l )**1
 1
 1
 1
 1
I 1
 1
 +*1
h %&&&3 &3 &3 &3 &3	 &3 &3 '&&3R L"46Z[[A A A A A A A \[A< L!!BA BA BA BA BA BA BA "!BAJ N##aA aA aA aA aA aA aA $#aAH *+++: +: +: +: +:I +: +: ,++:\ -..SD SD SD SD SD} SD SD /.SDl %(	 : : : : :9 : : :4 N$EGkllN N N N N N N mlNb &''8= 8= 8= 8= 8=( 8= 8= ('8=v '((J J J J J J J )(JZ %&&"5 "5 "5 "5 "59 "5 "5 '&"5J )**FD FD FD FD FDI FD FD +*FDR &''' ' ' ' ' ' ' ('' )**P4 P4 P4 P4 P4M P4 P4 +*P4f %&&    	   '&> $%%2 2 2 2 2	 2 2 &%2. %&&n@ n@ n@ n@ n@I n@ n@ '&n@b '((5D 5D 5D 5D 5D- 5D 5D )(5Dp &'',( ,( ,( ,( ,( ,( ,( (',(^ ')<==e( e( e( e( e() e( e( >=e(P *++%( %( %( %( %(Y %( %( ,+%(P *++n> n> n> n> n>Y n> n> ,+n>b *++:: :: :: :: ::Y :: :: ,+::z K!24DFeffX4 X4 X4 X4 X4	 X4 X4 gfX4v %'>@eff= = = = =i = = gf=& N$FGG'= '= '= '= '=9 '= '= HG'=T $%%N9 N9 N9 N9 N9Y N9 N9 &%N9b I0RSS= = = = =i = = TS=6 %'LMM= = = = =i = = NM=0 &''/: /: /: /: /: /: /: ('/:d '((+: +: +: +: +:) +: +: )(+:\ ')IJJ8: 8: 8: 8: 8:) 8: 8: KJ8:v 4550 0 0 0 0 0 0 650f 566[= [= [= [= [=; [= [= 76[=| +,,, , , , ,i , , -,, &''E% E% E% E% E% E% E% ('E%P +,,2# 2# 2# 2# 2#j 2# 2# -,2#j &(:;;s) s) s) s) s) s) s) <;s)l +,,'+ '+ '+ '+ '+* '+ '+ -,'+T &(:<TUUR( R( R( R( R( R( R( VUR(j '((Y% Y% Y% Y% Y%) Y% Y% )(Y%x &''qD qD qD qD qD qD qD ('qDh '((J J J J JY J J )(J" ())J J J J J9 J J *)J" %&&
%&&: : : : :	 : : '& '&:4 &''' ' ' ' ' ' ' ('' &''8D 8D 8D 8D 8D 8D 8D ('8Dv O%:;;i i i i ii i i <;i" ())G7 G7 G7 G7 G79 G7 G7 *)G7T '((SD SD SD SD SD) SD SD )(SDl )**UD UD UD UD UDI UD UD +*UDp +,,
+,,
455uD uD uD uD uDi uD uD 65 -, -,uDp &''= = = = = = = ('=6 $%%" " " " "y " " &%", '((
011
122
233E: E: E: E: E:i E: E: 43 32 21 )(E:P $%%D: D: D: D: D:Y D: D: &%D:N %&&OA OA OA OA OA	 OA OA '&OAd %'FGG#= #= #= #= #=	 #= #= HG#=L ())AD AD AD AD AD9 AD AD *)ADH $n6WXXW: W: W: W: W:9 W: W: YXW:t )**%: %: %: %: %:I %: %: +*%:P '((B B B B B) B B )(BJ ())A A A A A9 A A *)AH ())K K K K K: K K *)KB +-JKK2= 2= 2= 2= 2=l 2= 2= LK2=j 13EFFr$ r$ r$ r$ r$o r$ r$ GFr$j +,,gD gD gD gD gDi gD gD -,gDT 788
*++% % % % %Y % % ,+ 98%P O$$~ ~ ~ ~ ~I ~ ~ %$~ 899&: &: &: &: &:+ &: &: :9&:R O$$T T T T T"5 T T %$T 5663 3 3 3 3!4 3 3 763 )**gN gN gN gN gNK gN gN +*gNT -..ID ID ID ID IDi ID ID /.IDX /00X: X: X: X: X:9 X: X: 10X:v ())> > > > >: > > *)> '((xs xs xs xs xs) xs xs )(xsv %&&
%&&&: &: &: &: &:	 &: &: '& '&&:R -..OD OD OD OD OD	 OD OD /.ODd& & & & &: & & &R; ; ; ; ;# ; ; ;BG3 G3 G3 G3 G3dm G3 G3 G3TP P P Pf   $   ,`F `F `F `FF zDFFFFF r.   