PiT ~IddlmZddlZddlZddlZddlmZmZmZm Z m Z ddl m Z m Z mZe rddl mZmZmZmZmZmZmZdZejdZeCejejejedz nejeZe eeZ ee Z!dZ"d Z#d Z$d Z%d Z&d Z'dZ(dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9d Z:d!Z;d"Zej?ej@ejAej@ejAZBej?ej@ejAZCge jD_EejFe jD_Ge DZHd%ZId&ZJd'ZKd(ZLd)ZMeLZNdZOeMZPd ZQed*eRZSejAZTed+eRZUejAZVed,eRZWejAZXed-eRZYejAZZed.eRZ[ejAZ\ej]Z^ej]Z_ej`e_Zaej]ZbdZc d Zd d Ze d Zf d/Zg d0Zh d Zi dZjd Zkd Zld Zmd/Znd0Zod Zpd ZqdZrdZsdZtdZudZvdZwdZxdZydZzdZ{dZ|dZ}dZ~dZdZdZdZdZd Zd!Zd"Zd#Zd$Zd1Zd2Zd3Zd4Zd5Zd&ZdZd xZZdxZZdxZZdZd Zd Zd Zd/Zd0Zd ZdZd Zd Zd/ZdZdZd2Zd6Zd7Zd8Zd9ZdZd Zd Zd Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZd Zd!Zd"Zd#Zd$Zd1Zd2Zd:Zd;Zdd?ejZej`eݦZGd@dAejZej`eߦZej?ej@ejejAZGdBdCejZdZd Zd Zd ZGdDdEejZGdFdGejZGdHdIejZGdJdKejZej?dejejejAZ GdLdMejZGdNdOejZej`eZGdPdQejZGdRdSejZejAZej`ejAZe!dTgeddVZe!dWgeddXZe!dYgeddZZe!d[gedd\Ze!d]gdd^ZdZd Zd Zd Zd/Zd0Ze!d_gdd`Ze!daejgddddZe!deejegeVddjZe!dkejegeVddlZe!dmej`ejejFegeVddqZe!dreVejgdddtZe!dueVgdddvZe!dweVgdddxZe!dyeVegeXdd{Z e!d|eVegeXdd}Z e!d~eXgdddZ e!dgej ddZ e!dgejFddZDe!dgejFddZe!dgej@ddZe!dgej@ddZe!dgej@ddZe!dgej@ddZe!deXgejddZe!deXgejddZe!deXgejddZe!deXgejddZe!deVgej]ddZe!deVgej]ddZe!deVgej]ddZe!deVgej]ddZe!deTgej]ddZe!deXgeVddZe!deXgeZddZe!deXgejddZe!deXge\ddZ e!deVgeTddZ!e!deVgejddZ"e!deVgej]ddZ#e!deVgej]ddZ$e!deVgej]ddZ%e!deVgej]ddZ&e!deVgej]ddZ'e!deVgej]ddZ(e!deVgejddZ)e!deVgejddZ*e!deVejgejddńZ+e!deTgejddȄZ,e!deTgej]ddʄZ-e!deVejejejFgej]ddЄZ.e!deVgej]dd҄Z/e!deVej]ejejFgej]ddքZ0e!deVej]ejejFgej]dd؄Z1e!deVejejFgej]ddۄZ2e!deVgej3dd݄Z4e!deVejgejddZ5e!deVgej3ddZ6e!deVgej@ddZ7e!deVgej@ddZ8e!deVgej]ddZ9e!deVgej@ddZ:e!deVgej@ddZ;e!dejejej`egejddZ<e!deVejgeddZ=e!degdddZ>e!deXeejgej]ddZ?e!deXegej]ddZ@e!deXgddǐdZAe!deXej`ejejFej]ej]ej]gej]dސdZBe!d eZej@gddߐd ZCe!d eZebe^e^gej@ddZDe!deZebebe^e^gdddZEe!deZebgdddZFe!deZebe^e^e^gdddZGe!deZebe^e^ejgdddZHe!deZebge^dd ZIe!d!eZebge^dd"ZJe!d#eZgej@dd$ZKe!d%eXgej]dʐd&ZLe!d'eXgej]dʐd(ZMe!d)eXgddǐd*ZNe!d+eXebe^e^gej@dd,ZOe!d-eXebebe^e^gddd.ZPe!d/eXebgddd0ZQe!d1eXebe^e^e^gddd2ZRe!d3eXebe^e^ejgddd4ZSe!d5eXebge^dd6ZTe!d7eXebge^dd8ZUe!d9eXgddǐd:ZVe!d;eXgej@dd<ZWe!d=eXgddǐd>ZXe!d?eXgejFdʐd@ZYe!dAeXgejFdʐdBZZe!dCeXej`ej[ejFgejFddGZ\e!dHeXej`ej[gejFddIZ]e!dJeXej`ej[ejFgejFddLZ^e!dMeXej`ej[gejFddNZ_e!dOeXejeaejFej`ejFgej@ddVZ`e!dWeXejeaejFej`ejFgej@ddXZae!dYeXejeaejFgej@dd\Zbe!d]eXejeaejFgej@dd^Zce!d_eXebgejFddaZde!dbeXej`ej[ejFebgejFddcZee!ddeXej`ej[ejFebgejFddfZfe!dgeXejebeaejFgejFddiZge!djeXejebeaejFej`ejFgejFddkZhe!dleaej]geddnZie!doej]ej]ej]geddsZje!dtegdddvZke!dweXegej]ddxZle!dyeXegej]ddzZme!d{eXej]ej]gddd~Zne!deXgej]dʐdZoe!deXgej]dʐdZpe!deXej@gdddZqe!deXej@gdddZre!deXej@gdddZse!deXeCejAgdddZte!deXgddǐdZue!deXgej`ejddZve!deXej]gej`ejddZwe!deXgej`ejddZxe!deXej]gej`ejddZye!deXebgej`ejddZze!deTe_gejddZ{e!deTe_gejddZ|e!deTe_gejddZ}e!deTe_gej@ddZ~e!deTe_gej@ddZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTgej@d dZe!deTgej@d dZe!deTgej@d dZe!deTge_d dZe!deTge_d dÄZe!deTge_d dńZe!deTge_d dDŽZe!deTge_d dɄZe!deTge_d d˄Ze!deTe_gejdd̈́Ze!deTe_gejddτZe!deTe_gejddфZe!deTe_gej@ddӄZe!deTe_gej@ddՄZe!deTge_dӐdׄZe!deTge_dӐdلZe!deTge_dӐdۄZe!deTge_dӐd݄Ze!deTge_dӐd߄Ze!deTge_dӐdZe!deTge_dӐdZe!deTgej@d dZe!deTgej@d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTge_d dZe!deTejej]eaej]ej@ej@gej]d dZe!deTe_ejej]ej]ej@gej]d dZe!deTej`e_ej]ejej]ej@ej@gej]d dZe!d ejej`eejFej@ejej]gej]ddZe!dej`ejejFgej]ddZejAZGddejZGddejZe r eeZej`eZej?ejeZej?dee_Zej?deeZej?deZej?eeZej?deZdߐefdefdefdefdefdefge_e!dej`eegedd"Ze!d#egejdd%Ze!d&ee_gddd'Ze!d(eegddd+Ze!d,egddd-Ze!d.egedd/Ze!d0egddd1Ze!d2egedd3Ze!d4eegddd6Ze!d7eej]gedd8Ze!d9egejdd:Ze!d;eej]gedd<Ze!d=gedd>Ze!d?ejgeddAZe!dBgeddCZe!dDej]geddFZe!dGejejFgeddJZe!dKejejFgeddLZe!dMejejFgeddNZe!dOejgeddQZe!dRejejejgeddTZe!dUejejejFejged dVZe!dWejged!dYZe!dZej]ejejejej]ged"d_Ze!d`ejejejged#daZe!dbeTejejged$deZe!dfeTejejej`ejejFej`e_ejFged%dkZe!dleTejejej`ejejFej`e_ejFged&doZe!dpej]ejejejged'duZe!dveTej]ejejej]ej]ej`ejejFged(d}Ze!d~ej]ej]eged)dZe!deTged*dZe!degejd+dZe!deeXej]ge_d,dZe!dejejFejejejgejd-dZe!dejejFejejejgejd.dZe!dgejd/dZe!dejAejAgdd0dZGddejZGddejZe!deXgeܦd1dZe!deXgddǐdZe!deXgddǐdZe!degeݦd2dZe!degdd3dZe!degdd3dZej?ej@ejAejAZe!dejAejAgej@d4dZGddejZe!deXeVegdd5dZe!deXejAejAejAej ejAejAgdd6dZdS(7) annotationsN)CallableUnionNewTypeOptional TYPE_CHECKING)load_shared_librarybyref"ctypes_function_for_shared_library) CtypesCData CtypesArray CtypesPointerCtypesVoidPointer CtypesRefCtypesPointerOrRefCtypesFuncPointerllamaLLAMA_CPP_LIB_PATHlib lialgginsggiqsgg llama_vocab_p llama_model_pllama_context_pllama_memory_tllama_kv_cache_p !"#@i$%&ic`eZdZUdZerded<ded<ded<defdejfdejfgZ dS)llama_token_datazUsed to store token data Attributes: id (llama_token): token id logit (float): log-odds of the token p (float): probability of the token llama_tokenidfloatlogitpN __name__ __module__ __qualname____doc__r__annotations__rGctypesc_float_fields_g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/llama_cpp/llama_cpp.pyrFrFsd//  { &.! fnHHHrVrFczeZdZUdZerded<ded<ded<ded<defdejfdej fdej fgZ d S) llama_token_data_arrayaUsed to sample tokens given logits Attributes: data (ctypes.Array[llama_token_data]): token data size (int): size of the array selected (int): index in the data array (i.e. not the token id) sorted (bool): whether the array is sortedzCtypesArray[llama_token_data]dataintsizeselectedboolsortedN) rMrNrOrPrrQllama_token_data_prRc_size_tc_int64c_boolrTrUrVrWrYrYs{66++++    #$ ! V^$ 6=! HHHrVrYc xeZdZUdZer#ded<ded<ded<ded <d ed <d ed <ded<dejfdeje fdejej fd eje fd ejejfd ejeje fdejej fgZdS) llama_batcha0Input data for llama_encode/llama_decode A llama_batch object can contain input about one or many sequences The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens Attributes: n_tokens (int): number of tokens token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL) embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL) pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output r[n_tokensCtypesArray[llama_token]tokenCtypesArray[ctypes.c_float]embdz#CtypesArray[CtypesArray[llama_pos]]poszCtypesArray[ctypes.c_int]n_seq_idz&CtypesArray[CtypesArray[llama_seq_id]]seq_idzCtypesArray[ctypes.c_int8]logitsN)rMrNrOrPrrQrRc_int32POINTERrGrS llama_pos llama_seq_idc_int8rTrUrVrWrere9s  + ''''))))0000++++6666**** V^$ .&.--. //0 y))* ^V^FN334 >6>.&.">">??@ >6>&-001HHHrVreceZdZUdejfdejfdejfdejdzfgZe rde d<de d<de d<d e d<d Sd S) llama_model_kv_override_valueval_i64val_f64val_boolval_strr@r[rIr^bytesN) rMrNrOrRrbc_doublercc_charrTrrQrUrVrWruruvs| FN# FO$ V]# FMC'( H  rVrucfeZdZUdejfdejdzfdefgZerde d<de d<de d<dSdS) llama_model_kv_overridetagkeyr@valuer[rzzUnion[int, float, bool, bytes]N) rMrNrOrRc_intr|rurTrrQrUrVrWr~r~sm    #$ /0H . ------..rVr~ceZdZUdZerFded<ded<ded<ded<ded <d ed <d ed <ded<ded<ded<ded<ded<ded<ded<dejfdejfdejfdej fd ejfd ej ej fd e fdejfdej e fdejfdejfdejfdejfdejfgZdS)llama_model_paramsaParameters for llama_model Attributes: devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights use_mmap (bool): use mmap if possible use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking)zCtypesArray[ctypes.c_void_p]devicesz-CtypesArray[llama_model_tensor_buft_override]tensor_buft_overridesr[ n_gpu_layers split_modemain_gpuri tensor_splitz(Callable[[float, ctypes.c_void_p], bool]progress_callbackctypes.c_void_pprogress_callback_user_dataz$CtypesArray[llama_model_kv_override] kv_overridesr^ vocab_onlyuse_mmap use_mlock check_tensorsuse_extra_buftsN)rMrNrOrPrrQrRc_void_prorrprSllama_progress_callbackr~rcrTrUrVrWrrsmVV$----LLLL 1111CCCC4444:::: FO$ &/2 ( v|$ V^$ 778 56 &8 (?@@A v}% V]# fm$ &-( FM*HHHrVrceZdZUdZerded<ded<ded<ded<ded<ded<ded <ded <ded <d ed <d ed<d ed<d ed<d ed<d ed<ded<d ed<ded<ded<ded<ded<ded<ded<ded<ded <ded!<ded"<ded#<ded$<ded%<dejfdejfdejfdejfdejfdejfd ej fd ej fd ej fd ej fdej fdej fdej fdej fdej fdejfdej fde fdej fdej fdej fde fdej fdejfd ejfd!ejfd"ejfd#ejfd$ejfd%ejfgZd&S)'llama_context_paramsaParameters for llama_context Attributes: n_ctx (int): text context, 0 = from model n_batch (int): logical maximum batch size that can be submitted to llama_decode n_ubatch (int): physical maximum batch size n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) attention_type (int): attention type to use for embeddings rope_freq_base (float): RoPE base frequency, 0 = from model rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model yarn_attn_factor (float): YaRN magnitude scaling factor yarn_beta_fast (float): YaRN low correction dim yarn_beta_slow (float): YaRN high correction dim yarn_orig_ctx (int): YaRN original context size defrag_thold (float): defragment the KV cache if holes/size > thold, <= 0 disabled (default) cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval type_k (int): data type for K cache type_v (int): data type for V cache abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention no_perf (bool): whether to measure performance timings op_offload (bool): offload host tensor operations to device swa_full (bool): use full-size SWA cache kv_unified (bool): use a unified buffer across the input sequences when computing the attention r[n_ctxn_batchn_ubatch n_seq_max n_threadsn_threads_batchrope_scaling_type pooling_typeattention_typerIrope_freq_baserope_freq_scaleyarn_ext_factoryarn_attn_factoryarn_beta_fastyarn_beta_slow yarn_orig_ctx defrag_tholdz'Callable[[ctypes.c_void_p, bool], bool]cb_evalrcb_eval_user_datatype_ktype_vz!Callable[[ctypes.c_void_p], bool]abort_callbackabort_callback_datar^ embeddings offload_kqv flash_attnno_perf op_offloadswa_full kv_unifiedN)rMrNrOrPrrQrRc_uint32rorrS ggml_backend_sched_eval_callbackrggml_abort_callbackrcrTrUrVrWrr s!!F   8888****  9999,,,,  &/" FO$ V_% fo& fn% FN+ fl+ & 6<( 6>* FN+ FN+ V^, 6>* 6>* &/* ( 45 fo. 6< 6< ./ 0 v}%  & v}% FM" v}% V]# v}%=HHHrVrcneZdZUdZerAded<ded<ded<ded<ded<ded <ded <ded <ded <d ed<d ed<d ed<d ed<dejfdejfdejfdejfdej fd ej fd ej fd ej fd ej fdej fdej fdej fdej fg Z dS)llama_model_quantize_paramsaParameters for llama_model_quantize Attributes: nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() ftype (int): quantize to this llama_ftype output_tensor_type (int): output tensor type token_embedding_type (int): token embeddings tensor type allow_requantize (bool): allow quantizing non-f32/f16 tensors quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored pure (bool): quantize all tensors to the default type keep_split (bool): quantize to the same number of shards imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides tensor_types (ctypes.c_void_p): pointer to vector containing tensor types prune_layers (ctypes.c_void_p): pointer to vector containing layer indices to prune r[nthreadftypeoutput_tensor_typetoken_embedding_typer^allow_requantizequantize_output_tensor only_copypure keep_splitrimatrixr tensor_types prune_layersN) rMrNrOrPrrQrRrorrcrrTrUrVrWrrs?$ &  !!!!$$$$     %%%%%%%%%%%% FN# &, v|, . V]+ !6=1 fm$  v}% FO$ ) ) )HHHrVrcFeZdZUdZer ded<ded<defdejfgZ dS)llama_logit_biaszjUsed to store logit bias Attributes: token (llama_token): token id bias (float): biasrGrhrIbiasNrLrUrVrWrrsT   +  HHHrVrc6eZdZUdZerded<dejfgZdS)llama_sampler_chain_paramszrParameters for llama_sampler_chain Attributes: no_perf (bool): whether to measure performance timingsr^rN) rMrNrOrPrrQrRrcrTrUrVrWrrsBBB   FM"HHHrVrc2eZdZdejfdejfgZdS)llama_chat_messagerolecontentN)rMrNrOrRc_char_prTrUrVrWrrs) ! FO$HHHrVrllama_model_default_paramsreturncdS)z&Get default parameters for llama_modelNrUrUrVrWrr CrVllama_context_default_paramscdS)z(Get default parameters for llama_contextNrUrUrVrWrr!rrV"llama_sampler_chain_default_paramscdS)z.Get default parameters for llama_sampler_chainNrUrUrVrWrr,rrV#llama_model_quantize_default_paramscdS)z/Get default parameters for llama_model_quantizeNrUrUrVrWrr7rrVllama_backend_initcdS)zMInitialize the llama + ggml backend Call once at the start of the programNrUrUrVrWrrE CrVllama_backend_freecdS)zACall once at the end of the program - currently only used for MPINrUrUrVrWrrcrrVllama_numa_initnumar[cdSNrU)rs rWrro CrVllama_load_model_from_file path_modelrzparamsOptional[llama_model_p]cdSrrUrrs rWrrrrVllama_model_load_from_filecdS)zLoad the model from a file If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf If the split file name does not follow this pattern, use llama_model_load_from_splitsNrUrs rWrr CrVllama_model_load_from_splitspaths List[bytes]n_pathscdS)znLoad the model from multiple splits (support custom naming scheme) The paths must be in the correct orderNrU)rrrs rWrr CrVllama_model_save_to_filemodelcdS)zSave the model to a fileNrU)rrs rWrrrrVllama_free_modelcdSrrUrs rWrrrrVllama_model_freecdSrrUrs rWrrrrVllama_init_from_modelOptional[llama_context_p]cdSrrUrrs rWrrrrVllama_new_context_with_modelcdSrrUrs rWrrrrV llama_freectxcdS)zFrees all allocated memoryNrUrs rWrrrrV llama_time_uscdSrrUrUrVrWrr rrVllama_max_devicescdSrrUrUrVrWrrCrVllama_max_parallel_sequencescdSrrUrUrVrWr r r rVllama_supports_mmapr^cdSrrUrUrVrWr r r rVllama_supports_mlockcdSrrUrUrVrWrr%r rVllama_supports_gpu_offloadcdSrrUrUrVrWrr+r rVllama_supports_rpccdSrrUrUrVrWrr1r rV llama_n_ctxcdSrrUrs rWrr7r rV llama_n_batchcdSrrUrs rWrr=r rVllama_n_ubatchcdSrrUrs rWrrCr rVllama_n_seq_maxcdSrrUrs rWrrIr rVllama_n_ctx_traincdSrrUrs rWrrOr rV llama_n_embdcdSrrUrs rWrrUr rV llama_n_layercdSrrUrs rWr!r![r rV llama_n_headcdSrrUrs rWr#r#ar rV llama_n_vocabcdSrrUrs rWr%r%gr rVllama_get_modelcdSrrUrs rWr'r'mr rVllama_get_memoryOptional[llama_memory_t]cdS)zGet the memory for the contextNrUrs rWr)r)s CrVllama_pooling_typecdSrrUrs rWr-r-zr rVllama_get_kv_selfOptional[llama_kv_cache_p]cdS)z0Get the KV cache for self-attention (DEPRECATED)NrUrs rWr/r/rrVllama_model_get_vocabOptional[llama_vocab_p]cdSrrUrs rWr2r2r rVllama_model_rope_typecdSrrUrs rWr5r5r rVllama_model_n_ctx_traincdSrrUrs rWr7r7r rVllama_model_n_embdcdSrrUrs rWr9r9r rVllama_model_n_layercdSrrUrs rWr;r;r rVllama_model_n_headcdSrrUrs rWr=r=r rVllama_model_n_head_kvcdSrrUrs rWr?r?r rVllama_model_n_swacdSrrUrs rWrArAr rV!llama_model_rope_freq_scale_trainrIcdSrrUrs rWrCrCr rVllama_model_n_cls_outcdS)zKReturns the number of classifier outputs (only valid for classifier models)NrUrs rWrErEr,rVllama_model_cls_labeliOptional[bytes]cdS)zNReturns label of classifier output by index. Returns None if no label providedNrU)rrHs rWrGrGr,rVllama_vocab_typevocabcdSrrUrLs rWrKrKr rVllama_vocab_n_tokenscdSrrUrNs rWrOrOr rVllama_model_meta_val_strrUnion[ctypes.c_char_p, bytes]bufbuf_sizecdS)z*Get metadata value as a string by key nameNrU)rrrSrTs rWrQrQ $CrVllama_model_meta_countcdS)z*Get the number of metadata key/value pairsNrUrs rWrWrWr,rVllama_model_meta_key_by_indexUnion[ctypes.c_int, int](Union[bytes, CtypesArray[ctypes.c_char]]cdS)zGet metadata key name by indexNrUrrHrSrTs rWrYrYrVrV!llama_model_meta_val_str_by_indexcdS)z'Get metadata value as a string by indexNrUr]s rWr^r^rVrVllama_model_descUnion[ctypes.c_size_t, int]cdS)z&Get a string describing the model typeNrU)rrSrTs rWr`r`4 CrVllama_model_sizecdS)z?Returns the total size of all the tensors in the model in bytesNrUrs rWrdrdEr,rVllama_model_chat_templatenamecdS)zsGet the default chat template. Returns None if not available If name is None, returns the default chat templateNrU)rrgs rWrfrfN CrVllama_model_n_paramscdS)z3Returns the total number of parameters in the modelNrUrs rWrjrjWr,rVllama_model_has_encodercdS)zOReturns true if the model contains an encoder that requires llama_encode() callNrUrs rWrlrl_r,rVllama_model_has_decodercdS)zNReturns true if the model contains a decoder that requires llama_decode() callNrUrs rWrnrngr,rVllama_model_decoder_start_tokencdS)zFor encoder-decoder models, this function returns id of the token that must be provided to the decoder to start generating output sequence. For other models, it returns -1. NrUrs rWrprpprrVllama_model_is_recurrentcdS)z?Returns true if the model is recurrent (like Mamba, RWKV, etc.)NrUrs rWrrrr|r,rVllama_model_is_diffusioncdS)zFReturns true if the model is diffusion-based (like LLaDA, Dream, etc.)NrUrs rWrtrtr,rVllama_model_quantize fname_inp fname_out/CtypesPointerOrRef[llama_model_quantize_params]cdS)zReturns 0 on successNrU)rwrxrs rWrvrv CrVllama_adapter_lora_init path_loraOptional[llama_adapter_lora_p]cdSrrU)rr}s rWr|r|rrVllama_adapter_lora_freeadapterllama_adapter_lora_pcdSrrU)rs rWrrrrVllama_set_adapter_lorascalecdS)zRAdd a loaded LoRA adapter to given context This will not modify model's weightNrU)rrrs rWrr CrVllama_rm_adapter_loracdS)zlRemove a specific LoRA adapter from given context Return -1 if the adapter is not present in the contextNrU)rrs rWrrrrVllama_clear_adapter_loracdS)z+Remove all LoRA adapters from given contextNrUrs rWrrrrVllama_apply_adapter_cvecrZ"CtypesPointerOrRef[ctypes.c_float]lenn_embdil_startil_endcdS)aApply a loaded control vector to a llama_context, or if data is NULL, clear the currently loaded vector. n_embd should be the size of a single layer's control, and data should point to an n_embd x n_layers buffer starting from layer 1. il_start and il_end are the layer range the vector should apply to (both inclusive) See llama_control_vector_load in common to load a control vector.NrU)rrZrrrrs rWrrs 6CrVllama_memory_clearmemcdS)zoClear the memory contents If data == true, the data buffers will also be cleared together with the metadataNrU)rrZs rWrr)rrVllama_memory_seq_rmrmUnion[llama_seq_id, int]p0Union[llama_pos, int]p1cdS)aRemoves all tokens that belong to the specified sequence and have positions in [p0, p1) Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails seq_id < 0 : match any sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrU)rrmrrs rWrr>s 0CrVllama_memory_seq_cp seq_id_src seq_id_dstcdS)zvCopy all tokens that belong to the specified sequence to another sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrU)rrrrrs rWrrb ,CrVllama_memory_seq_keepcdS)z?Removes all tokens that do not belong to the specified sequenceNrUrrms rWrr CrVllama_memory_seq_adddeltacdS)zAdds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrU)rrmrrrs rWrrrrVllama_memory_seq_divdcdS)zcInteger division of the positions by factor of `d > 1` p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrU)rrmrrrs rWrrrrVllama_memory_seq_pos_mincdS)zReturns the smallest position present in the memory for the specified sequence This is typically non-zero only for SWA caches Return -1 if the sequence is emptyNrUrs rWrr CrVllama_memory_seq_pos_maxcdS)ztReturns the largest position present in the memory for the specified sequence Return -1 if the sequence is emptyNrUrs rWrrrrVllama_memory_can_shiftcdS)z%Check if the memory supports shiftingNrU)rs rWrrr,rVllama_kv_self_n_tokenscdS)zTReturns the number of tokens in the KV cache (slow, use only for debug) (DEPRECATED)NrUrs rWrrrrVllama_kv_self_used_cellscdS)z0Returns the number of used KV cells (DEPRECATED)NrUrs rWrr rrVllama_kv_self_clearcdS)zClear the KV cache (DEPRECATED)NrUrs rWrrrrVllama_kv_self_seq_rmcdS)z(Remove tokens from KV cache (DEPRECATED)NrU)rrmrrs rWrr)rVrVllama_kv_self_seq_cpcdS)z$Copy tokens in KV cache (DEPRECATED)NrU)rrrrrs rWrrI (CrVllama_kv_self_seq_keepcdS)z5Keep only specified sequence in KV cache (DEPRECATED)NrUrrms rWrrerrVllama_kv_self_seq_addcdS)z8Add delta to sequence positions in KV cache (DEPRECATED)NrU)rrmrrrs rWrryrrVllama_kv_self_seq_divcdS)z2Divide sequence positions in KV cache (DEPRECATED)NrU)rrmrrrs rWrrrrVllama_kv_self_seq_pos_mincdS)zCReturns the smallest position in KV cache for sequence (DEPRECATED)NrUrs rWrrrrVllama_kv_self_seq_pos_maxcdS)zBReturns the largest position in KV cache for sequence (DEPRECATED)NrUrs rWrrrrVllama_kv_self_defragcdS)z$Defragment the KV cache (DEPRECATED)NrUrs rWrrr,rVllama_kv_self_can_shiftcdS)z rrVllama_set_state_datacdS)z=Set the state reading from the specified address (DEPRECATED)NrU)rrs rWrrR rrVllama_state_load_file path_session tokens_outrgn_token_capacityn_token_count_out#CtypesPointerOrRef[ctypes.c_size_t]cdSrrUrrrrrs rWrre &CrVllama_load_session_filecdSrrUrs rWrr rrVllama_state_save_filetokens n_token_countcdSrrUrrrrs rWrr "CrVllama_save_session_filecdSrrUrs rWrr rrVllama_state_seq_get_sizerrcdS)z@Get the exact size needed to copy the state of a single sequenceNrUrs rWrr rrVllama_state_seq_get_datacdS)z=Copy the state of a single sequence into the specified bufferNrU)rrr\rms rWrr rVrVllama_state_seq_set_data dest_seq_idcdS)z2Copy the sequence data into the specified sequenceNrU)rrr\rs rWrr rVrVllama_state_seq_save_filefilepathcdSrrU)rrrmrrs rWrr rrVllama_state_seq_load_filecdSrrU)rrrrrrs rWrr5 s *CrVllama_batch_get_onerfcdS)zReturn batch for single sequence of tokens NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it NrU)rrfs rWrrZ rrVllama_batch_initUnion[ctypes.c_int32, int]rjrcdS)aAllocates a batch of tokens on the heap that can hold a maximum of n_tokens Each token can be assigned up to n_seq_max sequence ids The batch has to be freed with llama_batch_free() If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) Otherwise, llama_batch.token will be allocated to store n_tokens llama_token The rest of the llama_batch members are allocated with size n_tokens All members are left uninitializedNrU)rfrjrs rWrry r{rVllama_batch_freebatchcdS)z9Frees a batch of tokens allocated with llama_batch_init()NrU)rs rWrr r,rV llama_encodecdS)zLProcess a batch of tokens using the encoder. 0 - success < 0 - errorNrUrrs rWrr rrV llama_decodecdS)aLProcess a batch of tokens. 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)NrUrs rWrr rrVllama_set_n_threadsrrcdS)zSet the number of threads used for decoding n_threads is the number of threads used for generation (single token) n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) NrU)rrrs rWr r  rrVllama_n_threadscdS)z?Get the number of threads used for generation of a single tokenNrUrs rWr r  r,rVllama_n_threads_batchcdS)zOGet the number of threads used for prompt and batch processing (multiple token)NrUrs rWr r  r,rVllama_set_embeddingsrcdS)z1Set whether the context outputs embeddings or notNrU)rrs rWrr r,rVllama_set_causal_attn causal_attncdS)zlSet whether to use causal attention or not If set to true, the model will only attend to the past tokensNrU)rrs rWrr rirVllama_set_warmupwarmupcdS)zSet whether the model is in warmup mode or not If true, all model tensors are activated during llama_decode() to load and cache their weights.NrU)rrs rWrr rirVllama_set_abort_callbackr!Callable[[ctypes.c_void_p], None]rrcdS)zSet abort callbackNrU)rrrs rWrr rcrVllama_synchronizecdS)zWait until all computations are finished This is automatically done when using one of the functions below to obtain the computation results and is not necessary to call it explicitly in most casesNrUrs rWrr rrVllama_get_logitsricdS)aZToken logits obtained from the last call to llama_decode() The logits for which llama_batch.logits[i] != 0 are stored contiguously in the order they have appeared in the batch. Rows: number of tokens for which llama_batch.logits[i] != 0 Cols: n_vocab Returns: Pointer to the logits buffer of shape (n_tokens, n_vocab)NrUrs rWrr* rcrVllama_get_logits_ithcdS)zNLogits for the ith token. Equivalent to: llama_get_logits(ctx) + i*n_vocabNrUrrHs rWrr> rrVllama_get_embeddingscdS)zDGet the embeddings for the input shape: [n_embd] (1-dimensional)NrUrs rWr!r!S rrVllama_get_embeddings_ithcdS)zPGet the embeddings for the ith sequence llama_get_embeddings(ctx) + i*n_embdNrUr s rWr#r#b rrVllama_get_embeddings_seqcdS)zGet the embeddings for a sequence id Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE shape: [n_embd] (1-dimensional)NrUrs rWr%r%t rrVllama_vocab_get_textrhUnion[llama_token, int]cdSrrUrLrhs rWr'r' rrVllama_vocab_get_scorecdSrrUr*s rWr+r+ rrVllama_vocab_get_attrcdSrrUr*s rWr-r- rrVllama_vocab_is_eogcdS)zXCheck if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)NrUr*s rWr/r/ rrVllama_vocab_is_controlcdS)z>Identify if Token Id is a control token or a render-able tokenNrUr*s rWr1r1 rrVllama_vocab_bosrGcdS)zbeginning-of-sentenceNrUrNs rWr3r3 r,rVllama_vocab_eoscdS)zend-of-sentenceNrUrNs rWr5r5 r,rVllama_vocab_eotcdS)z end-of-turnNrUrNs rWr7r7 r,rVllama_vocab_sepcdS)zsentence separatorNrUrNs rWr9r9 r,rVllama_vocab_nlcdS)z next-lineNrUrNs rWr;r; r,rVllama_vocab_padcdS)paddingNrUrNs rWr=r= r,rVllama_vocab_maskcdS)maskNrUrNs rWr@r@ r,rVllama_vocab_get_add_boscdSrrUrNs rWrCrC rrVllama_vocab_get_add_eoscdSrrUrNs rWrErE rrVllama_vocab_get_add_sepcdSrrUrNs rWrGrG rrVllama_vocab_fim_precdSrrUrNs rWrIrI rrVllama_vocab_fim_sufcdSrrUrNs rWrKrK rrVllama_vocab_fim_midcdSrrUrNs rWrMrM rrVllama_vocab_fim_padcdSrrUrNs rWrOrO) rrVllama_vocab_fim_repcdSrrUrNs rWrQrQ3 rrVllama_vocab_fim_sepcdSrrUrNs rWrSrS= rrVllama_token_get_textcdSrrUr*s rWrUrUH rrVllama_token_get_scorecdSrrUr*s rWrWrWT rrVllama_token_get_attrcdSrrUr*s rWrYrY_ rrVllama_token_is_eogcdSrrUr*s rWr[r[j rrVllama_token_is_controlcdSrrUr*s rWr]r]u rrVllama_token_boscdSrrUrNs rWr_r_ rrVllama_token_eoscdSrrUrNs rWrara rrVllama_token_eotcdSrrUrNs rWrcrc rrVllama_token_clscdSrrUrNs rWrere rrVllama_token_sepcdSrrUrNs rWrgrg rrVllama_token_nlcdSrrUrNs rWriri rrVllama_token_padcdSrrUrNs rWrkrk rrVllama_add_bos_tokencdSrrUrNs rWrmrm rrVllama_add_eos_tokencdSrrUrNs rWroro rrVllama_token_fim_precdSrrUrNs rWrqrq rrVllama_token_fim_sufcdSrrUrNs rWrsrs rrVllama_token_fim_midcdSrrUrNs rWruru rrVllama_token_fim_padcdSrrUrNs rWrwrw rrVllama_token_fim_repcdSrrUrNs rWryry rrVllama_token_fim_sepcdSrrUrNs rWr{r{ rrVllama_vocab_clscdSrrUrNs rWr}r} rrVllama_tokenizetexttext_len n_tokens_max add_specialUnion[ctypes.c_bool, bool] parse_specialcdS)aConvert the provided text into tokens. Args: vocab: The vocabulary to use for tokenization. text: The text to tokenize. text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. n_max_tokens: The maximum number of tokens to return. add_special: Allow adding special tokens if the model is configured to do so. parse_special: Allow parsing special tokens. Returns: Returns the number of tokens on success, no more than n_tokens_max Returns a negative number on failure - the number of tokens that would have been returned NrU)rLrrrrrrs rWrr, s LCrVllama_token_to_piece9Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]]lengthlstripspecialcdS)a(Token Id -> Piece. Uses the vocabulary in the provided context. Does not write null terminator to the buffer. User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. Args: vocab: The vocabulary to use for tokenization. token: The token to convert. buf: The buffer to write the token to. length: The length of the buffer. lstrip: The number of leading spaces to skip. special: If true, special tokens are rendered in the output.NrU)rLrhrSrrrs rWrra BCrVllama_detokenize text_len_maxremove_specialunparse_specialcdS)aConvert the provided tokens into text (inverse of llama_tokenize()). Args: vocab: The vocabulary to use for tokenization. tokens: The tokens to convert. n_tokens: The number of tokens. text: The buffer to write the text to. text_len_max: The length of the buffer. remove_special: Allow to remove BOS and EOS tokens if model is configured to do so. unparse_special: If true, special tokens are rendered in the output.NrU)rLrrfrrrrs rWrr rrVllama_chat_apply_templatetmplchatCtypesArray[llama_chat_message]n_msgadd_asscdS)ajApply chat template. Args: tmpl: Template to use. If None, uses model's default chat: Array of chat messages n_msg: Number of messages add_ass: Whether to end prompt with assistant token buf: Output buffer length: Buffer length Returns: Number of bytes written, or needed if buffer too small NrU)rrrrrSrs rWrr s DCrVllama_chat_builtin_templatesoutputCtypesArray[bytes]cdS)zGet list of built-in chat templates. Args: output: Output buffer to store template names. len: Length of the output buffer. Returns: Number of templates available. Returns a negative number on error. NrU)rrs rWrr .CrVceZdZdS)llama_sampler_iN)rMrNrOrUrVrWrr!sCrVrc:eZdZdejefdefgZdS) llama_samplerifacerN)rMrNrOrRrprllama_sampler_context_trTrUrVrWrr)s2 .&.112 '(HHHrVracceptapplyresetclonefreellama_sampler_initrctypes.POINTER(llama_sampler_i)rllama_sampler_pcdSrrU)rrs rWrrJrrVllama_sampler_namesmplcdSrrUrs rWrrVrrVllama_sampler_acceptcdSrrU)rrhs rWrr`rrVllama_sampler_applycur_p#CtypesArray[llama_token_data_array]cdSrrU)rrs rWrrjrrVllama_sampler_resetcdSrrUrs rWrrvrrVllama_sampler_clonecdSrrUrs rWrrrrVllama_sampler_freecdSrrUrs rWrrrrVllama_sampler_chain_initcdSrrU)rs rWrrrrVllama_sampler_chain_addchaincdSrrU)rrs rWrrrrVllama_sampler_chain_getcdSrrUrrHs rWrrrrVllama_sampler_chain_ncdSrrUrs rWrrrrVllama_sampler_chain_removecdSrrUrs rWrrrrVllama_sampler_init_greedycdSrrUrUrVrWrrr rVllama_sampler_init_distseedcdSrrU)rs rWrrr rVllama_sampler_init_softmaxcdSrrUrUrVrWrrr rVllama_sampler_init_top_kkcdSrrU)rs rWrrr rVllama_sampler_init_top_prKmin_keepcdSrrUrKrs rWrrrrVllama_sampler_init_min_pcdSrrUrs rWrrrrVllama_sampler_init_typicalcdSrrUrs rWrrrrVllama_sampler_init_temptcdSrrU)rs rWrrr rVllama_sampler_init_temp_extexponentcdSrrU)rrrs rWrrrrVllama_sampler_init_xtccdSrrU)rKrrrs rWrr%rrVllama_sampler_init_top_n_sigmancdSrrU)rs rWrr2rrVllama_sampler_init_mirostatn_vocabtauetamcdSrrU)rrrrrs rWrrBrrVllama_sampler_init_mirostat_v2cdSrrU)rrrs rWrrRrrVllama_sampler_init_grammar grammar_str grammar_rootcdSrrU)rLrrs rWrrbrrVllama_sampler_init_grammar_lazy trigger_wordsnum_trigger_wordstrigger_tokensnum_trigger_tokenscdSrrU)rLrrrrrrs rWrrvrrV(llama_sampler_init_grammar_lazy_patternstrigger_patternsnum_trigger_patternscdSrrU)rLrrrrrrs rWrrrrVllama_sampler_init_penaltiespenalty_last_npenalty_repeat penalty_freqpenalty_presentcdSrrU)rrrrs rWrrrcrVllama_sampler_init_dry n_ctx_traindry_multiplierdry_basedry_allowed_lengthdry_penalty_last_n num_breakerscdSrrU)rLrrrrr  seq_breakersr s rWrrs 2CrVllama_sampler_init_logit_bias n_logit_bias logit_biasCtypesArray[llama_logit_bias]cdSrrU)rrrs rWr r rrVllama_sampler_init_infillcdSrrUrNs rWrrrrVllama_sampler_get_seedcdSrrUrs rWrr rrVllama_sampler_sampleidxcdSrrU)rrrs rWrrrrVllama_split_path split_pathmaxlen path_prefixsplit_no split_countcdS)z-Build a split GGUF final path for this chunk.NrU)rrrrrs rWrr& CrVllama_split_prefix split_prefixcdS)z^Extract the path prefix from the split_path if and only if the split_no and split_count match.NrU)r"rrrrs rWr!r!9r rVllama_print_system_infocdSrrUrUrVrWr$r$Lr rV llama_log_set log_callbackOptional[CtypesFuncPointer] user_datacdS)z|Set callback for all future logging events. If this is not called, or NULL is supplied, everything is output on stderr.NrU)r'r)s rWr&r&TrrVceZdZdejfdejfdejfdejfdejfdejfdejfgZdS) llama_perf_context_data t_start_ms t_load_ms t_p_eval_ms t_eval_msn_p_evaln_evaln_reusedNrMrNrOrRr{rorTrUrVrWr,r,rs[ v' fo& ( fo& V^$ 6>" V^$HHHrVr,c2eZdZdejfdejfgZdS)llama_perf_sampler_data t_sample_msn_sampleNr4rUrVrWr6r6s) ( V^$HHHrVr6llama_perf_contextcdSrrUrs rWr9r9rrVllama_perf_context_printcdSrrUrs rWr;r;rrVllama_perf_context_resetcdSrrUrs rWr=r=rrVllama_perf_samplercdSrrUrs rWr?r?rrVllama_perf_sampler_printcdSrrUrs rWrArArrVllama_perf_sampler_resetcdSrrUrs rWrCrCrrVllama_opt_param_filter_alltensoruserdatacdSrrU)rFrGs rWrErErrVcXeZdZdejfdefdejfdejfdejfgZdS)llama_opt_paramsr param_filterparam_filter_ud get_opt_parsget_opt_pars_udN)rMrNrOrRrllama_opt_param_filterrrTrUrVrWrJrJsF ( /0 FO, ) FO, HHHrVrJllama_opt_initlctx lopt_paramscdSrrU)rQrrRs rWrPrPrrVllama_opt_epochdataset result_train result_eval idata_splitcallback_train callback_evalcdSrrU)rQrUrVrWrXrYrZs rWrTrTrrV)rr)rr)rr)rr)rr[)rrzrrrr)rrrr[rrrr)rr4rrz)rr4)rr4rrrr)rr5)rr[)rr^)rr5rr[)rr4rr[)rr3rr[)rr5rr)rr5rr*)rr5rr0)rr4rr3)rr4rrI)rr4rHr[rrI)rLr3rr[) rr4rrRrSrzrTr[rr[) rr4rHrZrSr[rTr[rr[)rr4rSr[rTrarr[)rr4rgrIrrI)rr4rr^)rwrzrxrzrryrr[)rr4r}rzrr~)rr)rr5rrrrIrr[)rr5rrrr[)rr5rZrrr[rr[rr[rr[rr[)rr6rZr^) rr6rmrrrrrrr^) rr6rrrrrrrr)rr6rmr) rr6rmrrrrrrr) rr6rmrrrrrrrZ)rr6rmrrr[)rr6rr^) rr5rmrrrrrrr^) rr5rrrrrrrr)rr5rmr) rr5rmrrrrrrr) rr5rmrrrrrrrZ)rr5rmrrr[)rr5rr^)rr5rrr\rarr[)rr5rrrr[)rr5rrr\rarr[)rr5rrrr[) rr5rrzrrgrrarrrr^) rr5rrzrrgrrarr^)rr5rmrrrr[) rr5rrr\rarmrrrr[) rr5rrr\rarrrrr[) rr5rrzrmrrrrgrrarr[)rr5rrzrrrrrgrrarrrr[)rrgrfrZrre)rfrrjrrrrre)rre)rr5rrerr[)rr5rrrr)rr5rr^)rr5rr^)rr5rr^)rr5rrrr)rr5rri)rr5rHrrri)rr5rmrrri)rLr3rhr(rrz)rLr3rhr(rrI)rLr3rhr(rr[)rLr3rhr(rr^)rLr3rrG)rLr3rr^)rLr3rrzrrZrrgrrZrrrrrr[)rLr3rhr(rSrrrZrrZrrrr[)rLr3rrgrfrZrrzrrZrrrrrr[)rrzrrrr[rr^rSrzrr[rr[)rrrrarr[)rrrrrr)rrrrz)rrrhr()rrrr)rr)rrrr)rrrr)rrrr)rrrHrrr)rrrr[)rr)rr[rr)rr[rr)rKrIrr[rr)rrIrr)rrIrrIrrIrr) rKrIrrIrr[rr[rr)rrIrr) rr[rr[rrIrrIrr[rr)rr[rrIrrIrr)rLr3rrzrrzrr)rLr3rrzrrzrrrr[rrgrr[rr)rLr3rrzrrzrrrr[rrgrr[rr) rr[rrIrrIrrIrr)rLr3rr[rrIrrIrr[r r[r r[rr)rr[rr[rrrr)rLr3rr)rrrr[)rrrr5rr[rr[) rrzrrarrzrrZrrZrr[) r"rzrrarrzrrZrrZrr[)rrz)r'r(r)r)rr5rr,)rrrr6)rr)rFrrGrrr^)rQr5rr4rRrJ)rQr5rUrrVrrWrrXr[rYrrZr( __future__rosrRpathlibtypingrrrrrllama_cpp._ctypes_extensionsr r r r r rrrrr_lib_base_nameenvironget_override_base_pathPathpathabspathdirname__file__ _base_path_libctypes_function GGML_TYPE_F32 GGML_TYPE_F16GGML_TYPE_Q4_0GGML_TYPE_Q4_1GGML_TYPE_Q5_0GGML_TYPE_Q5_1GGML_TYPE_Q8_0GGML_TYPE_Q8_1GGML_TYPE_Q2_KGGML_TYPE_Q3_KGGML_TYPE_Q4_KGGML_TYPE_Q5_KGGML_TYPE_Q6_KGGML_TYPE_Q8_KGGML_TYPE_IQ2_XXSGGML_TYPE_IQ2_XSGGML_TYPE_IQ3_XXSGGML_TYPE_IQ1_SGGML_TYPE_IQ4_NLGGML_TYPE_IQ3_SGGML_TYPE_IQ2_SGGML_TYPE_IQ4_XS GGML_TYPE_I8 GGML_TYPE_I16 GGML_TYPE_I32 GGML_TYPE_I64 GGML_TYPE_F64GGML_TYPE_IQ1_MGGML_TYPE_COUNT CFUNCTYPErcrrrrargtypesrarestypeLLAMA_MAX_DEVICESLLAMA_DEFAULT_SEEDLLAMA_TOKEN_NULLLLAMA_FILE_MAGIC_GGLALLAMA_FILE_MAGIC_GGSNLLAMA_FILE_MAGIC_GGSQLLAMA_SESSION_MAGICLLAMA_SESSION_VERSIONLLAMA_STATE_SEQ_MAGICLLAMA_STATE_SEQ_VERSIONr[r3llama_vocab_p_ctypesr4llama_model_p_ctypesr5llama_context_p_ctypesr6llama_memory_t_ctypesr7llama_kv_cache_p_ctypesrorqrGrp llama_token_prrLLAMA_VOCAB_TYPE_NONELLAMA_VOCAB_TYPE_SPMLLAMA_VOCAB_TYPE_BPELLAMA_VOCAB_TYPE_WPMLLAMA_VOCAB_TYPE_UGMLLAMA_VOCAB_TYPE_RWKVLLAMA_VOCAB_TYPE_PLAMO2LLAMA_VOCAB_PRE_TYPE_DEFAULTLLAMA_VOCAB_PRE_TYPE_LLAMA3!LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM#LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODERLLAMA_VOCAB_PRE_TYPE_FALCONLLAMA_VOCAB_PRE_TYPE_MPTLLAMA_VOCAB_PRE_TYPE_STARCODERLLAMA_VOCAB_PRE_TYPE_GPT2LLAMA_VOCAB_PRE_TYPE_REFACTLLAMA_VOCAB_PRE_TYPE_COMMAND_RLLAMA_VOCAB_PRE_TYPE_STABLELM2LLAMA_VOCAB_PRE_TYPE_QWEN2LLAMA_VOCAB_PRE_TYPE_OLMOLLAMA_VOCAB_PRE_TYPE_DBRXLLAMA_VOCAB_PRE_TYPE_SMAUGLLAMA_VOCAB_PRE_TYPE_POROLLAMA_VOCAB_PRE_TYPE_CHATGLM3LLAMA_VOCAB_PRE_TYPE_CHATGLM4LLAMA_VOCAB_PRE_TYPE_VIKINGLLAMA_VOCAB_PRE_TYPE_JAISLLAMA_VOCAB_PRE_TYPE_TEKKENLLAMA_VOCAB_PRE_TYPE_SMOLLMLLAMA_VOCAB_PRE_TYPE_CODESHELLLLAMA_VOCAB_PRE_TYPE_BLOOM!LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISHLLAMA_VOCAB_PRE_TYPE_EXAONELLAMA_VOCAB_PRE_TYPE_CHAMELEONLLAMA_VOCAB_PRE_TYPE_MINERVA"LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLMLLAMA_VOCAB_PRE_TYPE_GPT4OLLAMA_VOCAB_PRE_TYPE_SUPERBPELLAMA_VOCAB_PRE_TYPE_TRILLIONLLAMA_VOCAB_PRE_TYPE_BAILINGMOELLAMA_VOCAB_PRE_TYPE_LLAMA4LLAMA_VOCAB_PRE_TYPE_PIXTRALLLAMA_VOCAB_PRE_TYPE_SEED_CODERLLAMA_ROPE_TYPE_NONELLAMA_ROPE_TYPE_NORMLLAMA_ROPE_TYPE_NEOXGGML_ROPE_TYPE_NEOXLLAMA_ROPE_TYPE_MROPEGGML_ROPE_TYPE_MROPELLAMA_ROPE_TYPE_VISIONGGML_ROPE_TYPE_VISIONLLAMA_TOKEN_TYPE_UNDEFINEDLLAMA_TOKEN_TYPE_NORMALLLAMA_TOKEN_TYPE_UNKNOWNLLAMA_TOKEN_TYPE_CONTROLLLAMA_TOKEN_TYPE_USER_DEFINEDLLAMA_TOKEN_TYPE_UNUSEDLLAMA_TOKEN_TYPE_BYTELLAMA_TOKEN_ATTR_UNDEFINEDLLAMA_TOKEN_ATTR_UNKNOWNLLAMA_TOKEN_ATTR_UNUSEDLLAMA_TOKEN_ATTR_NORMALLLAMA_TOKEN_ATTR_CONTROLLLAMA_TOKEN_ATTR_USER_DEFINEDLLAMA_TOKEN_ATTR_BYTELLAMA_TOKEN_ATTR_NORMALIZEDLLAMA_TOKEN_ATTR_LSTRIPLLAMA_TOKEN_ATTR_RSTRIPLLAMA_TOKEN_ATTR_SINGLE_WORDLLAMA_FTYPE_ALL_F32LLAMA_FTYPE_MOSTLY_F16LLAMA_FTYPE_MOSTLY_Q4_0LLAMA_FTYPE_MOSTLY_Q4_1LLAMA_FTYPE_MOSTLY_Q8_0LLAMA_FTYPE_MOSTLY_Q5_0LLAMA_FTYPE_MOSTLY_Q5_1LLAMA_FTYPE_MOSTLY_Q2_KLLAMA_FTYPE_MOSTLY_Q3_K_SLLAMA_FTYPE_MOSTLY_Q3_K_MLLAMA_FTYPE_MOSTLY_Q3_K_LLLAMA_FTYPE_MOSTLY_Q4_K_SLLAMA_FTYPE_MOSTLY_Q4_K_MLLAMA_FTYPE_MOSTLY_Q5_K_SLLAMA_FTYPE_MOSTLY_Q5_K_MLLAMA_FTYPE_MOSTLY_Q6_KLLAMA_FTYPE_MOSTLY_IQ2_XXSLLAMA_FTYPE_MOSTLY_IQ2_XSLLAMA_FTYPE_MOSTLY_Q2_K_SLLAMA_FTYPE_MOSTLY_IQ3_XSLLAMA_FTYPE_MOSTLY_IQ3_XXSLLAMA_FTYPE_MOSTLY_IQ1_SLLAMA_FTYPE_MOSTLY_IQ4_NLLLAMA_FTYPE_MOSTLY_IQ3_SLLAMA_FTYPE_MOSTLY_IQ3_MLLAMA_FTYPE_MOSTLY_IQ2_SLLAMA_FTYPE_MOSTLY_IQ2_MLLAMA_FTYPE_MOSTLY_IQ4_XSLLAMA_FTYPE_MOSTLY_IQ1_MLLAMA_FTYPE_MOSTLY_BF16LLAMA_FTYPE_MOSTLY_TQ1_0LLAMA_FTYPE_MOSTLY_TQ2_0LLAMA_FTYPE_MOSTLY_MXFP4_MOELLAMA_FTYPE_GUESSED#LLAMA_ROPE_SCALING_TYPE_UNSPECIFIEDLLAMA_ROPE_SCALING_TYPE_NONELLAMA_ROPE_SCALING_TYPE_LINEARLLAMA_ROPE_SCALING_TYPE_YARN LLAMA_ROPE_SCALING_TYPE_LONGROPE!LLAMA_ROPE_SCALING_TYPE_MAX_VALUELLAMA_POOLING_TYPE_UNSPECIFIEDLLAMA_POOLING_TYPE_NONELLAMA_POOLING_TYPE_MEANLLAMA_POOLING_TYPE_CLSLLAMA_POOLING_TYPE_LASTLLAMA_POOLING_TYPE_RANK LLAMA_ATTENTION_TYPE_UNSPECIFIEDLLAMA_ATTENTION_TYPE_CAUSALLLAMA_ATTENTION_TYPE_NON_CAUSALLLAMA_SPLIT_MODE_NONELLAMA_SPLIT_MODE_LAYERLLAMA_SPLIT_MODE_ROW StructurerFr`rYllama_token_data_array_prSrreLLAMA_KV_OVERRIDE_TYPE_INTLLAMA_KV_OVERRIDE_TYPE_FLOATLLAMA_KV_OVERRIDE_TYPE_BOOLLLAMA_KV_OVERRIDE_TYPE_STRrur~rrrrllama_log_callbackrrllama_logit_bias_prrrllama_adapter_lora_p_ctypesrrrrrGGML_NUMA_STRATEGY_DISABLEDGGML_NUMA_STRATEGY_DISTRIBUTEGGML_NUMA_STRATEGY_ISOLATEGGML_NUMA_STRATEGY_NUMACTLGGML_NUMA_STRATEGY_MIRRORGGML_NUMA_STRATEGY_COUNTrrrrrrrrrrrrbrr r rrrrrrrrrrr!r#r%r'r)r-r/r2r5r7r9r;r=r?rArCrErGrKrOrQrWrYr^r`c_uint64rdrfrjrlrnrprrrtrvr|rrrrrrrrrrrrrrrrrrrrrrrrrrrrrc_uint8rrrrrrrrrrrrrrrrrrr r r rrrrrrrr!r#r%r'r+r-r/r1r3r5r7r9r;r=r@rCrErGrIrKrMrOrQrSrUrWrYr[r]r_rarcrergrirkrmrorqrsrurwryr{r}rrrrrrrrrllama_sampler_p_ctypesllama_sampler_i_namellama_sampler_i_acceptllama_sampler_i_applyllama_sampler_i_resetllama_sampler_i_clonellama_sampler_i_freerTrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrr!r$r&r,r6r9r;r=r?rArCrOrErJrPrTrUrVrWr.s B""""""   jnn%9::QdQl\W\"'//"'//(*C*CDD E E M Mr~ryr~@SsTsT >:6644T::L       $46#3 M6?FM6?$$ 'f&v}foFF#%!'**,, ###,.-- -- '+S11)3//7-s33 / N n {++ ~ @-'#1FV !$%!&'#!"!"!# " "   !#$&! !#!%'" " ""$ !"$-..*/00,133. ! !  ! &$  %^!')# !" #$ $D!"$$& "#v'($V^$455V-.*6>*@AA+&* M6>6?>!!!!!&"!!!T      FL    . . . . .f. . . .j22222)222Lccccc6+cccX&V%&,J.00000&"2000nv'"$V^$455     !1   &),fnV_==      "    (    )        !     \N   (  _()     _()   ""V^FO$$fo7IJ   6?+          /0   "/0        N    $b&/::;: /V_EEFE &FM::;: 'V];;<; -r6=AABA %r6=99:9 !7 8&/JJKJ #9":FOLLML !$:#;V_MMNM "%;$RSSTS #&<%=?TUUVU %(>'?NNON     (+?*@BVWWXW (+?*@&,OOPO *-A,BFNSSTS %(<'=v~NNON &)=(>OOPO %(<'=v~NNON (+?*@&.QQRQ $';&)?PPQP   N    ),@+A6>RRSR#   N    '   N    6?FO< N   #&:%;V_MMNM,/CV_.UW]Wfgghg'*>)?QQRQ*-A,BFMRRSR*-A,BFMRRSR%(<'=v~+.B-CV]SSTS+.B-CV]SSTS233  O$6?+    !   8&.I N   89 N      $v~&&  N    6FM*       M    4          $3\BD          .          *!6 Ey!6 Ey),A+BFMRRSR56!7 8&.23T    M    ,          "5|Dd"          0          ("8,!G"8,!G'*@)A4HHIH*-C,DfmTTUT'*@)A4HHIH'*@)A6?SSTS'*@)A6?SSTSv~&&  O     v~&& O ^V^FN;;V_M O   ^V^FN;;< O   v''  M    $v''  M        M        M    \* O   v~&&   O    (v~&&   O    "  O    $v''  O        2     .H+    $#k]D99:9"8+!FWWXW,"8+!FWWXW      "%;$EEFE/0.&.2P2P    "V^,FN6>""    34nfnV^6T6TV^,FN6>""   \*FN6>""   1;?2K@&.1;?/=v}3[A6="%9$:KHHIH "%9$:KHHIH "%9$:KHHIH "%9$:KHHIH !$8#9;GGHG "%9$:KHHIH #&:%;[IIJI  M     M     M                            ;' O   ;' N    ;' L    ;' M    ;' M                             M    M                          4   N    P  N    L{##   N    X)**   N    6"v'' N .!/f&F$3#M2O' 66'v'9OPP))$0F TT(( ":)(/EFF(()?AWXX'v'.DEE !" %& #$ #$ #$ !" V^O$$&=>    O    [)    56                   34    V^,    L     V^,   ,b2HIIJI *V_,=?UVVWV-r3IJJKJ+fn-=?UVVWV  ^V_%     ^V_%      ^V_%    *V^,<>TUUVU ! ^V^V^4    ^V^V_foF   $ ^   ! ^V_fnfnfnU   $ _fnfn5    6?FO<   %v''{##        ,.v''{##        &" ^V^V^V^D   (v''         $# ^V^%78        O    3V^D    _fov flS L      _fov flS L     *B@@A@ _fo&   2     f.   "f.                       *)&-&/ZZ  _fo& M   v'13CD              rV