
    
`i                         d dl mZmZ ddlmZ d dlZd dlZ	 	 	 ddej        dee         dee         deee	                  d	ef
d
Z
dS )    )ListOptional   )LLTokenizerNvocabn_vocab	eos_tokenslicesreturnc           	      f   t          j        |           }|t          j        |           }d}t          j        |dz             }g }t          |          D ]}t          j        | |||dd          }	|	dk     rt          d| d| d|	           |	|k    sJ t          |d|	                   }
t          j	        | |          }|t           j
        z  s9|t           j        z  r/|
                    d	          r|
                    d
          rd|
z   }
|                    |
           |;t          |          |k     r(|                    d           t          |          |k     (t          j        t           j         j        j        t          j                  j        }t+          j        || |||          S )a  
    Create a new tokenizer from a llama.cpp vocab object.
    This is an expensive operation (~1s), so the result should be cached.

    Args:
        vocab: llama_cpp.llama_vocab_p - the vocab object to use
        n_vocab: int - override the size of the vocabulary
        eos_token: int - override the EOS token
        slices: List[str] - configuration for slicer optimization; pass [] to disable,
            or None to use the default configuration
    Ni @  r   r   TzError writing token z to buffer of size z	. Error:    <   >       )tokens	vocab_ptrtokenize_fptrr	   r
   )	llama_cppllama_vocab_n_tokensllama_vocab_eosctypescreate_string_bufferrangellama_token_to_piece
ValueErrorbytesllama_token_get_attrLLAMA_TOKEN_ATTR_CONTROLLLAMA_TOKEN_ATTR_USER_DEFINED
startswithendswithappendlencast_libllama_tokenizec_void_pvaluer   from_llamacpp)r   r   r	   r
   ntok
buffer_lenbufferr   tokenntokattrfptrs                g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/llguidance/llamacpp.pylltokenizer_from_vocabr3      s   $ )%00D-e44	J(a88FFt  *
 
 q55fEffjffcdffgggJF2A2J-eU;; )44 	 9::	 t$$	  T""	 
 C-Cc&kkG##MM# &kkG## ;y*/>PPVD$   r   )NNN)typingr   r   r%   r   r   r   llama_vocab_pintstrr3    r   r2   <module>r9      s    ! ! ! ! ! ! ! !            "#"&	< <"<c]< }< T#Y	<
 < < < < < <r   