
    Pif                      d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m	Z	 d dl
mZ d dlmZmZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlZd dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% ddl&m'Z' dd	l(m)Z)m*Z* d
Z+dZ,dZ-dZ.dZ/dZ0dZ1dZ2 G d de          Z3 G d de4          Z5 G d de*          Z6ddZ7ddZ8ej9         G d d                      Z: G d de          Z; G d  d!e;          Z<dd%Z=dd)Z>dd-Z?	 ddd4Z@dd6ZAdd8ZBdd;ZCdd<ZD	 dddAZE	 dddBZFddFZGddIZHddLZIddPZJddQZKddRZLddSZMddTZNddUZOddVZPdddXZQ	 ddd[ZR	 ddd^ZSdd_ZT eTd`          ddc            ZU eTdd          dde            ZV eTdf          ddg            ZW eTdh          ddi            ZX eTdj          ddk            ZY eTdl          ddm            ZZ eTdn          ddo            Z[ eTdp          ddq            Z\ eTdr          dds            Z] eTdt          ddu            Z^ eTdv          ddw            Z_ eTdx          ddy            Z` eTdz          dd{            Za eTd|          dd}            Zb eTd~          dd            Zc eTd          dd            Zd eTd          dd            Ze eTd          dd            Zf eTd          dd            Zg eTd          dd            Zh eTd          dd            Zi eTd          dd            Zj eTd          dd            Zk e8d          dddddddddd.g ddddddd dddddfdd            Zl e8d           e8d          dddddddddd.g ddddddd dddddfdd                        Zm G d dæ          Zn G dĄ den          Zo G dƄ den          Zp G dȄ den          Zq G dʄ den          Zr G d̄ den          ZsesZt G d΄ den          Zu G dЄ den          Zv e8dҦ          dddddddddd.g ddddddd dddddddfddք            ZwdS )    )annotationsN)datetime)	ExitStack)
AnyDictIteratorListLiteralOptionalTupleUnionProtocolcast)ImmutableSandboxedEnvironment   )logger)suppress_stdout_stderr	Singletonz{% for message in messages %}{{'<|im_start|>' + message['role'] + '
' + message['content'] + '<|im_end|>' + '
'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
' }}{% endif %}<s>
<|im_end|>a  {{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}</s>a  {{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}a  {% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>

'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>

' }}{% endif %}c                  T    e Zd ZdZddddddddg ddddddddd	d
d
ddddddddd@d?ZdS )ALlamaChatCompletionHandlera  Base Protocol for a llama chat completion handler.

    Very generic protocol that can be used to implement any chat format.
    The only hard requirement is that it must return a ChatCompletion when
    stream=False and an iterator of ChatCompletionChunks when stream=True.N皙?ffffff?(   F        皙?皙?      ?r         @皙?)	functionsfunction_calltoolstool_choicetemperaturetop_ptop_kstreamstopseedresponse_format
max_tokenspresence_penaltyfrequency_penaltyrepeat_penaltymodel
logit_biasmin_p	typical_ptfs_zmirostat_modemirostat_taumirostat_etalogits_processorgrammarlogprobstop_logprobsllamallama.Llamamessages.List[llama_types.ChatCompletionRequestMessage]r#   2Optional[List[llama_types.ChatCompletionFunction]]r$   7Optional[llama_types.ChatCompletionRequestFunctionCall]r%   .Optional[List[llama_types.ChatCompletionTool]]r&   4Optional[llama_types.ChatCompletionToolChoiceOption]r'   floatr(   r)   intr*   boolr+   Optional[Union[str, List[str]]]r,   Optional[int]r-   9Optional[llama_types.ChatCompletionRequestResponseFormat]r.   r/   r0   r1   r2   Optional[str]r3   Optional[Dict[str, float]]r4   r5   r6   r7   r8   r9   r:   #Optional[llama.LogitsProcessorList]r;   Optional[llama.LlamaGrammar]r<   Optional[bool]r=   returniUnion[llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse]]c                   d S N )selfr>   r@   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   kwargss                                  o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/llama_cpp/llama_chat_format.py__call__z#LlamaChatCompletionHandler.__call__D   s
    R s    )<r>   r?   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   rF   r)   rG   r*   rH   r+   rI   r,   rJ   r-   rK   r.   rJ   r/   rF   r0   rF   r1   rF   r2   rL   r3   rM   r4   rF   r5   rF   r6   rF   r7   rG   r8   rF   r9   rF   r:   rN   r;   rO   r<   rP   r=   rJ   rQ   rR   __name__
__module____qualname____doc__rY   rU   rZ   rX   r   r   =   s        N N IMQU@DLP 02" $("%#& ##15!!@D04#'&*I) ) ) ) ) ) ) )rZ   r   c                      e Zd ZdS )+LlamaChatCompletionHandlerNotFoundExceptionN)r\   r]   r^   rU   rZ   rX   ra   ra   p   s        DrZ   ra   c                  :    e Zd ZU i Zded<   	 ddd
ZddZddZdS )"LlamaChatCompletionHandlerRegistryz%Dict[str, LlamaChatCompletionHandler]_chat_handlersFnamestrchat_handlerr   	overwriterH   c                V    |s|| j         v rt          d| d          || j         |<   d S )NzFormatter with name 'z>' is already registered. Use `overwrite=True` to overwrite it.rd   
ValueError)rV   re   rg   rh   s       rX    register_chat_completion_handlerzCLlamaChatCompletionHandlerRegistry.register_chat_completion_handlerw   sN      	TT%888llll   %1D!!!rZ   c                N    || j         v r
| j         |= d S t          d| d          )Nz(No formatter registered under the name 'z'.rj   )rV   re   s     rX   unregister_chat_handlerz:LlamaChatCompletionHandlerRegistry.unregister_chat_handler   s;    4&&&#D)))PPPPQQQrZ   rQ   c           
         	 | j         |         }|S # t          $ r; t          d| dt          | j                                                    d          w xY w)NzInvalid chat handler: z (valid formats: ))rd   KeyErrorra   listkeys)rV   re   rg   s      rX   #get_chat_completion_handler_by_namezFLlamaChatCompletionHandlerRegistry.get_chat_completion_handler_by_name   st    	.t4L 	 	 	=cccTEXE]E]E_E_@`@`ccc  	s
    AANF)re   rf   rg   r   rh   rH   re   rf   re   rf   rQ   r   )r\   r]   r^   rd   __annotations__rl   rn   rt   rU   rZ   rX   rc   rc   t   sq         <>N>>>>  	
1 
1 
1 
1 
1R R R R	 	 	 	 	 	rZ   rc   re   rf   rQ   c                D    t                                          |           S rT   )rc   rt   re   s    rX   get_chat_completion_handlerr{      s#    -//SS  rZ   c                     d fd}|S )Nfr   c                L    t                                          |            | S rT   )rc   rl   )r}   re   s    rX   	decoratorz3register_chat_completion_handler.<locals>.decorator   s%    *,,MMdTUVVVrZ   )r}   r   rU   re   r   s   ` rX   rl   rl      s)          rZ   c                  H    e Zd ZU dZded<   dZded<   dZded<   d	Zd
ed<   dS )ChatFormatterResponsea  Dataclass that stores completion parameters for a given chat format and
    create_chat_completion request.

    prompt contains the formatted prompt generated from the chat format and messages.
    stop contains the stop token or list of stop tokens to use for the chat format.rf   promptNrI   r+   z$Optional[llama.StoppingCriteriaList]stopping_criteriaFrH   added_special)r\   r]   r^   r_   rx   r+   r   r   rU   rZ   rX   r   r      s]         W W KKK,0D0000>BBBBBMrZ   r   c                      e Zd ZdZd
dZd	S )ChatFormattera  Base Protocol for a chat formatter. A chat formatter is a function that
    takes a list of messages and returns a chat format response which can be used
    to generate a completion. The response can also include a stop token or list
    of stop tokens to use for the completion.r@   rA   rW   r   rQ   r   c                   d S rT   rU   )rV   r@   rW   s      rX   rY   zChatFormatter.__call__   s	    
 !$rZ   Nr@   rA   rW   r   rQ   r   r[   rU   rZ   rX   r   r      s2        1 1
$ $ $ $ $ $rZ   r   c                  P    e Zd Z	 	 d d!dZed"d            Zdddddd#dZd$dZdS )%Jinja2ChatFormatterTNtemplaterf   	eos_token	bos_tokenadd_generation_promptrH   stop_token_idsOptional[List[int]]c                    || _         || _        || _        || _        |t	          |          nd| _        t          t          j                    dd          	                    | j                   | _
        dS )zAA chat formatter that uses jinja2 templates to format the prompt.NT)loadertrim_blockslstrip_blocks)r   r   r   r   setr   r   jinja2
BaseLoaderfrom_string_environment)rV   r   r   r   r   r   s         rX   __init__zJinja2ChatFormatter.__init__   s     !""%:"#1#=C4 	 :$&&
 
 
 +dm
$
$	 	rZ   r}   rQ   c                N    t          j                                        |           S rT   )r   nowstrftime)r}   s    rX   strftime_nowz Jinja2ChatFormatter.strftime_now   s    |~~&&q)))rZ   )r#   r$   r%   r&   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   rW   r   r   c                    dd} j                             | j         j        | j        |||| j        
  
        }d }	 j        d fd}
t          j        |
g          }	t          | j        g|	d          S )Nmessagerf   c                     t          |           rT   )rk   )r   s    rX   raise_exceptionz5Jinja2ChatFormatter.__call__.<locals>.raise_exception   s    W%%%rZ   )
r@   r   r   r   r   r#   r$   r%   r&   r   tokensnpt.NDArray[np.intc]logitsnpt.NDArray[np.single]rQ   rH   c                "    | d         j         v S )N)r   )r   r   rV   s     rX   stop_on_last_tokenz8Jinja2ChatFormatter.__call__.<locals>.stop_on_last_token   s     bzT%888rZ   T)r   r+   r   r   )r   rf   )r   r   r   r   rQ   rH   )
r   renderr   r   r   r   r   r>   StoppingCriteriaListr   )rV   r@   r#   r$   r%   r&   rW   r   r   r   r   s   `          rX   rY   zJinja2ChatFormatter.__call__   s    	& 	& 	& 	& "))nn+"&"<'#* * 
 
 !*9 9 9 9 9 9
 !& :<N;O P P$.!/	
 
 
 	
rZ   r   c                     t          |           S rT   ))chat_formatter_to_chat_completion_handlerrV   s    rX   to_chat_handlerz#Jinja2ChatFormatter.to_chat_handler	  s    8>>>rZ   )TN)
r   rf   r   rf   r   rf   r   rH   r   r   )r}   rf   rQ   rf   )r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   rW   r   rQ   r   )rQ   r   )r\   r]   r^   r   staticmethodr   rY   r   rU   rZ   rX   r   r      s         '+.2% % % % %. * * * \* IMQU@DLP)
 )
 )
 )
 )
 )
V? ? ? ? ? ?rZ   r   r<   (Optional[llama_types.CompletionLogprobs]"llama_types.ChatCompletionLogprobsc                j    | d S d t          | d         | d         | d                   D             d dS )Nc                \    g | ])\  }}}|d |d |                                 D             d*S )Nc                     g | ]\  }}||d dS )N)tokenlogprobbytesrU   ).0	top_tokentop_logprobs      rX   
<listcomp>zH_convert_text_completion_logprobs_to_chat.<locals>.<listcomp>.<listcomp>  s>     ! ! ! /	;	 "+#.!% ! ! !rZ   )r   r   r   r=   )items)r   r   r   r=   s       rX   r   z=_convert_text_completion_logprobs_to_chat.<locals>.<listcomp>  sj     
 
 
 15'< "! ! 3?2D2D2F2F! ! !	 
 
 
rZ   r   token_logprobsr=   )contentrefusal)zip)r<   s    rX   )_convert_text_completion_logprobs_to_chatr     si     t
 
 588JHUeLfhpq  iA  5B  5B
 
 
 !  rZ   
completionllama_types.Completionllama_types.ChatCompletionc           
         d| v sJ d| d         z   d| d         | d         dd| d	         d         d
         dt          | d	         d         d                   | d	         d         d         dg| d         dS )Nusagechatidchat.completioncreatedr2   r   	assistantchoicestextroler   r<   finish_reasonindexr   r<   r   r   objectr   r2   r   r   r   )r   s    rX    _convert_text_completion_to_chatr   &  s     j    z$''#i(G$ '))4Q7?  FjQZF[\]F^_iFjkk!+I!6q!9/!J 

 G$!  rZ   chunks4Iterator[llama_types.CreateCompletionStreamResponse])Iterator[llama_types.ChatCompletionChunk]c              #    K   t          |           D ]\  }}|dk    r'd|d         z   |d         |d         ddddid d d	gd
V  d|d         z   |d         |d         dd|d         d         d         d|d         d         d         ini t          |d         d         d                   |d         d         d         d	gd
V  d S )Nr   r   r   r2   r   chat.completion.chunkr   r   r   deltar<   r   r   r2   r   r   r   r   r   r   r   r<   )	enumerater   )r   ichunks      rX   '_convert_text_completion_chunks_to_chatr   >  sP      f%% %
 %
566uT{*w +1 "#"K" %))- 	    " 5;&7^Y'- 
 !+A.?G &uY'7':6'B    I%PYJZ[\J]^hJi j j%*9%5a%8%I 
 
 	
 	
 	
 	
%%
 %
rZ   Fcompletion_or_chunksaUnion[llama_types.CreateCompletionResponse, Iterator[llama_types.CreateCompletionStreamResponse]]r*   rH   ZUnion[llama_types.CreateChatCompletionResponse, Iterator[llama_types.ChatCompletionChunk]]c                J    |r| }t          |          S | }t          |          S rT   )r   r   )r   r*   r   r   s       rX   _convert_completion_to_chatr   i  s2      <G[6v>>>-A
/
;;;rZ   	tool_namec                ^    |s|}d|v sJ d z   dz   |d         z   }d|d         z   d|d         |d         d	d
d  |d         d	         d         d|d |d         d	         d         ddgdt          |d         d	         d                   ddg|d         d}|S |}d fd} ||          S )Nr   call__0__r   r   r   r   r2   r   r   r   r   re   	argumentsfunctionr   typer   r   r   r$   
tool_callsr<   r   r   r   r   r   rQ   8Iterator[llama_types.CreateChatCompletionStreamResponse]c              3    K   d}d }d }d }d }| D ]?}|rd|d         z   }|d         }|d         }dz   dz   |d         z   }|d||d	d d d
d d d ddgdV  d|d         z   d|d         |d         d	d t          |d         d	         d                   d d |d         d	         d         dd	|d|d         d	         d         ddgddgdV  d}|J d|d         z   d|d         |d         d	d t          |d         d	         d                   d d |d         d	         d         dd	|d|d         d	         d         ddgddgdV  A||||d||d	dd d d d d ddgdV  d S d S d S d S )NTr   r   r   r2   r   r   r   r   r   r   r   r   r<   r   r   r   r   r2   r   r   r<   r   r   r   r   r   r   r   Fr   r   )r   firstid_r   r2   tool_idr   r   s          rX   #_stream_response_to_function_streamzQ_convert_completion_to_chat_function.<locals>._stream_response_to_function_stream  s      ECGEG ] ] ; 5;.C#I.G!'NE-	9C?%+MG!"9#*!& *+15,0,7/35926	*" *"	
 
$    ( %uT{2"9#(#3!&w *+15,UV[\eVfghVijtVu,v,v,0/3095:95Ea5H5P6& 6& 67294>8A=B9=Ma=P4:>29. 9.	
)* 
)*3&*" *"	 $! ! ! ! !D "E*** 5;.5$Y/"7^ &'-1(QRWXaRbcdRefpRq(r(r(,+/,516y1A!1DV1L2" 2" 23.50:4=9>y9I!9LV9T5* 5*	%& %&
/"& &	       B 7#65;L5&" &'-9(,(,+/15.2	& &	
 
        #6#6;L;LrZ   )r   r   rQ   r   r   )r   r   r*   r   r   chat_completionr   r   s   `       rX   $_convert_completion_to_chat_functionr   z  sY     f;;O
*$$$$!I-3j6FF :d++'!),(  +#'$-)3I)>q)A&)I* * '.(2,51;I1Fq1I&1Q-" -" 	'   $ !J*U^J_`aJbcmJn o o%1+ 2  (=E
 E
@ G[{	 {	 {	 {	 {	 {	z 326:::rZ   chat_formatterc                N     d d d d ddddddg d d d ddddd	d
dd d d d d d dd> fd=}|S )?Nr   r   r   r   r    Fr   r   r   r!   r"   r#   r$   r%   r&   r'   r(   r)   r4   r5   r*   r+   r,   r-   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r3   r<   r=   r>   r?   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   r)   rG   r4   r5   r*   rH   r+   rI   r,   rJ   r-   rK   r.   r/   r0   r1   r6   r7   r8   r9   r2   rL   r:   rN   r;   rO   r3   rM   r<   rP   r=   rQ   rR   c                  '  (|||||          } | j         |j                            d          |j         d          }|j        L|g nt          |t                    r|gn|}t          |j        t                    r|j        n|j        g} || z   }d }!|j        |j        }!|"|d         dk    rt          || j
                  }|d |D             }|It          |t                    r|d	k    s|d
k    r|}t          |t                    rd|v rdd|d         id}d }"|t          |t                    r||d         d         't          'fd|D             d           }"|"t          d' d          |"d         d         }#	 t          j                            t#          j        |#          | j
                  }nv# t&          $ ri}$| j
        r(t)          t          |$          t*          j                   t          j                            t          j        | j
                  }Y d }$~$nd }$~$ww xY w | j        d+i d|d|d|d|d|	d|
d|r|nd d|d|d|d|d|d|d |d!|d"|d#|d$|d%|d&|d'|!d(|d)|}%|"|"d         d         }&t5          |&|%|          S t7          |%|*          S ),N)r@   r#   r$   r%   r&   utf-8T)add_bosspecialr   json_objectverbosec                    g | ]}d |dS r   r   r   rU   r   r   s     rX   r   z^chat_formatter_to_chat_completion_handler.<locals>.chat_completion_handler.<locals>.<listcomp>p  7       
  ' (   rZ   noneautore   r   r  c              3  @   K   | ]}|d          d         k    |V  dS r   re   NrU   r   tre   s     rX   	<genexpr>z]chat_formatter_to_chat_completion_handler.<locals>.chat_completion_handler.<locals>.<genexpr>  6      KKqQz]6-Bd-J-J-J-J-J-JKKrZ   Tool choice '' not found in tools.
parametersfiler   r'   r(   r)   r4   r5   r<   r*   r+   r,   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r   r;   r3   r*   rU   )tokenizer   encoder   r+   
isinstancerf   rr   r   _grammar_for_response_formatr	  dictnextrk   llama_grammarLlamaGrammarfrom_json_schemajsondumps	Exceptionprintsysstderrr   	JSON_GBNFcreate_completionr   r   ))r>   r@   r#   r$   r%   r&   r'   r(   r)   r4   r5   r*   r+   r,   r-   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r3   r<   r=   rW   resultr   rstopr   toolschemaer   r   re   r   s)                                          @rX   chat_completion_handlerzJchat_formatter_to_chat_completion_handler.<locals>.chat_completion_handler.  sL   L  '#
 
 
  M  )),,
 
 

 ;"22Zc5J5J+TD66PTD#-fk4#@#@SFKKv{mE%<D #/ & 8&?6+Bm+S+S2  G
   
 !*  E $--- ,''=F+B+B+-.. 6]3J3J&f 5!  #;-- $!z*62DKKKKEKKKTRRD| !L!L!L!LMMM*%l3F
'4EEJv&& F      = 3#a&&sz2222'4@@!+U] A    7u6  
  
  
6 
# 
 % 
 %	 

 % 
  i 
 &.7\\4 
 6 
  
  
 "z 
 .- 
 0/ 
 *> 
 % 
  (-! 
" &# 
$ &% 
& %' 
( .-) 
* 0/+ 
, G- 
. "z/ 
2 Z(0I7/   ++?OOOOs   8G 
H:AH55H:<r>   r?   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   rF   r)   rG   r4   rF   r5   rF   r*   rH   r+   rI   r,   rJ   r-   rK   r.   rJ   r/   rF   r0   rF   r1   rF   r6   rF   r7   rG   r8   rF   r9   rF   r2   rL   r:   rN   r;   rO   r3   rM   r<   rP   r=   rJ   rQ   rR   rU   )r   r3  s   ` rX   r   r   +  s     IMQU@DLP 02" $("%#& #!!#@D0415#'&*AMP MP MP MP MP MP MP MP^ #"rZ   pretrained_model_name_or_pathUnion[str, os.PathLike[str]]c                J    ddl m} |                    |           d
fd	}|S )Nr   AutoTokenizerr@   rA   rW   r   rQ   r   c                    d_                             | d          }t          |t                    sJ t	          |j        d          S )NFr  Tr   r+   r   )use_default_system_promptapply_chat_templater  rf   r   r   )r@   rW   r   	tokenizers      rX   format_autotokenizerz@hf_autotokenizer_to_chat_formatter.<locals>.format_autotokenizer  s\     /4	+33Hu3MM&#&&&&&$	 34
 
 
 	
rZ   r   )transformersr9  from_pretrained)r5  r9  r@  r?  s      @rX   "hf_autotokenizer_to_chat_formatterrC    sQ     +*****--.KLLI

 

 

 

 

 

  rZ   c                >    t          |           }t          |          S rT   )rC  r   )r5  r   s     rX   +hf_autotokenizer_to_chat_completion_handlerrE    s      88UVVN4^DDDrZ   Ttokenizer_configDict[str, Any]r   c                   t          | t                    sJ d| v sJ t          | d         t                    sJ | d         }d| v sJ t          | d         t                    sJ | d         d| v sJ t          | d         t                    sJ | d         t          dd                              |          dfd}|S )Nchat_templater   r   Tr   r   r@   rA   rW   r   rQ   r   c                    rg | t          j        dd          }                     |           }t          |gd          S )Nr    r   )r@   r   r   Tr<  )llama_types%ChatCompletionRequestAssistantMessager   r   )r@   rW   r   r   r   envr   s      rX   format_tokenizer_configzFhf_tokenizer_config_to_chat_formatter.<locals>.format_tokenizer_config  s     ! 	A$b  H   
 

 %I 6d
 
 
 	
rZ   r   )r  r!  rf   r   r   )rF  r   rI  rP  r   rO  r   s    `  @@@rX   %hf_tokenizer_config_to_chat_formatterrQ    s(    &-----.....&7=====$_5M*****&{3S99999 -I*****&{3S99999 -I
'   k-   

 
 
 
 
 
 
 
 
, #"rZ   c                B    t          | |          }t          |          S )N)r   )rQ  r   )rF  r   r   s      rX   .hf_tokenizer_config_to_chat_completion_handlerrS    s.     ;0E  N 5^DDDrZ   metadataDict[str, str]rL   c                    d| vrd S | d         t           k    rdS | d         t          k    s| d         t          k    rdS | d         t          k    rdS d S )Nztokenizer.chat_templatechatmlmistral-instructllama-3)CHATML_CHAT_TEMPLATEMISTRAL_INSTRUCT_CHAT_TEMPLATEMIXTRAL_INSTRUCT_CHAT_TEMPLATELLAMA3_INSTRUCT_CHAT_TEMPLATE)rT  s    rX   $guess_chat_format_from_gguf_metadatar^    sq     00t)*.BBBx 	*+/MMM-.2PPP!!)*.KKKy4rZ   r@   rA   c                @    | D ]}|d         dk    r|d         pdc S dS )zGet the first system message.r   systemr   rL  rU   )r@   r   s     rX   _get_system_messagera  1  sB      , ,6?h&&9%++++ '2rZ   role_mapList[Tuple[str, Optional[str]]]c                    g }| D ]P}|d         }||v rBt          |d         t                    r|d         nd}|                    ||         |f           Q|S )zMap the message roles.r   r   N)r  rf   append)r@   rb  outputr   r   r   s         rX   
_map_rolesrg  ;  su    
 /1F 5 5v8&01CS&I&IS	""t  MM8D>73444MrZ   system_messagesepsep2c                    ||g}| |z   }t          |          D ]H\  }\  }}| r|dk    r|pd}	||	||dz           z   z  }&|r|||z   dz   ||dz           z   z  }@||dz   z  }I|S )z(Format the prompt with the llama2 style.r   rL      r   )
rh  r@   ri  rj  sepsretr   r   r   ms
             rX   _format_llama2rr  K  s     ;D
3
C'11  ?D' 	a1ff2A1tAE{?"CC 	4'>C'$q1u+55CC4#:CCJrZ   c                R    | |z   }|D ]\  }}|r||dz   |z   |z   z  }||dz   z  }|S )z2Format the prompt with the add-colon-single style.: :rU   rh  r@   ri  rp  r   r   s         rX   _format_add_colon_singlerw  \  sX     3
C!  g 	4$;(3..CC4#:CCJrZ   c                    ||g}| |d         z   }t          |          D ]*\  }\  }}|r||dz   |z   ||dz           z   z  }"||dz   z  }+|S )z/Format the prompt with the add-colon-two style.r   rt  rl  ru  rn  )	rh  r@   ri  rj  ro  rp  r   r   r   s	            rX   _format_add_colon_twory  i  sz     ;D
47
"C'11  ?D' 	4$;(4A;66CC4#:CCJrZ   c                @    | }|D ]\  }}|r|||z   |z   z  }||z  }|S )z1Format the prompt with the no-colon-single style.rU   rv  s         rX   _format_no_colon_singler{  w  sI     C!  g 	4'>C''CC4KCCJrZ   c                R    | |z   }|D ]\  }}|r||dz   |z   |z   z  }||dz   z  }|S )z8Format the prompt with the add-colon-space-single style.rt  rU   rv  s         rX   _format_add_colon_space_singler}    sX     3
C!  g 	4$;(3..CC4$;CCJrZ   c                n    | dk    rdn| |z   dz   }|D ]!\  }}|r||dz   |z   |z   dz   z  }||dz   z  }"|S )z(Format the prompt with the chatml style.rL  
rU   rv  s         rX   _format_chatmlr    sp     "$$"".3*>*EC!  g 	4$;(3.55CC4$;CCJrZ   c                T    d}| r|| z  }|D ]\  }}|r||dz   dz   |z   z  }||z  }|S )z*Format the prompt with the chatglm3 style.rL  r  rm  rU   rv  s         rX   _format_chatglm3r    s`     C ~!  g 	4$;$w..CC4KCCJrZ   r	  c                X    t           j                            t           j        |           S Nr  )r#  r$  r   r,  r  s    rX   _grammar_for_jsonr    s*    %11 2   rZ   r1  fallback_to_jsonc                    	 t           j                            | |          S # t          $ r}|rt	          |          cY d }~S |d }~ww xY wr  )r#  r$  r%  r(  r  )r1  r	  r  r2  s       rX   _grammar_for_json_schemar    so    )::67:SSS    	$W555555555G	s    # 
AAAAAr-   /llama_types.ChatCompletionRequestResponseFormatc                    | d         dk    rd S d| v r)t          t          j        | d                   |          S t          |          S )Nr   r  r1  r  )r  r&  r'  r  )r-   r	  s     rX   r   r     sb     v-//t?""'Jx0117
 
 
 	
 !1111rZ   c                     d fd}|S )Nr}   r   c                j    t          |           }t                                          |           | S rT   )r   rc   rl   )r}   r3  re   s     rX   r   z'register_chat_format.<locals>.decorator  s<    "KA"N"N*,,MM)	
 	
 	
 rZ   )r}   r   rU   r   s   ` rX   register_chat_formatr    s)          rZ   zllama-2rW   r   c                    d}t          dd          }t          | |          }t          |           }|r|                    |          }t	          ||dd          dz   }t          |          S )	Nz([INST] <<SYS>>
{system_message}
<</SYS>>z	<s>[INST]z[/INST]userr   rh  rm  r   r   )r!  rg  ra  formatrr  r   )r@   rW   _system_template_roles	_messagesrh  _prompts          rX   format_llama2r    s    
 D{i888F8V,,I(22N P)000OO^YVDDyPG 0000rZ   rY  c                    t          ddd          }d}t          | |          }|                    |d         d f           t          d||          }t	          ||          S )	Nz,<|start_header_id|>system<|end_header_id|>

z*<|start_header_id|>user<|end_header_id|>

z/<|start_header_id|>assistant<|end_header_id|>

)r`  r  r   z
<|eot_id|>r   rL  r   r+   )r!  rg  re  r{  r   )r@   rW   r  _sepr  r  s         rX   format_llama3r    sy    
 ?;E  F
 D8V,,If[)40111%b)T::G d;;;;rZ   alpacac                    t          dd          }d}d}t          |           }t          | |          }t          ||||          }t	          |          S )Nz### Instruction### Responser  

r   r  )r!  ra  rg  ry  r   )r@   rW   r  r  _sep2rh  r  r  s           rX   format_alpacar     sa    
 (NCCCFDE(22N8V,,I#NItUKKG 0000rZ   qwenc                   t          dd          }t          |           pd}d}|                    |          }t          | |          }|                    |d         d f           d}t          |||          }d	}t          ||
          S )N<|im_start|>user<|im_start|>assistantr  You are a helpful assistant.#<|im_start|>system
{system_message}r  r   r   z<|endoftext|>r  )r!  ra  r  rg  re  r  r   )	r@   rW   r  rh  system_templater  r  r  r  s	            rX   format_qwenr    s    
 )5LMMMF(22T6TN<O$++>+JJN8V,,If[)40111D^Y==GE e<<<<rZ   vicunac                    d}t          dd          }d}d}|}t          | |          }|                    |d         d f           t          ||||          }t	          |          S )	NzA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.USER	ASSISTANTr  rm  r   r   r  )r!  rg  re  ry  r   )	r@   rW   _system_messager  r  r  rh  r  r  s	            rX   r  r    s|    
 sOv555FDE$N8V,,If[)40111#NItUKKG 0000rZ   oasst_llamac                   d}t          dd          }d}t          |           }|                    |          }t          | |          }|                    |d         d f           t          |||          }t          |          S )	Nz*[INST] <<SYS>>
{system_message}
<</SYS>>

<|prompter|><|assistant|>r  r   r  r   r  r!  ra  r  rg  re  r{  r   r@   rW   r  r  r  rh  r  r  s           rX   format_oasst_llamar  /  s    
 H~AAAFD(22N%,,N,KKN8V,,If[)40111%niFFG 0000rZ   z
baichuan-2c                   d}t          dd          }d}t          |           }|                    |          }t          | |          }|                    |d         d f           t          |||          }t          |          S )	N{system_message}z<reserved_106>z<reserved_107>r  rL  r  r   r  r  r  s           rX   format_baichuan2r  ?      
 *'3CDDDFD(22N%,,N,KKN8V,,If[)40111%niFFG 0000rZ   baichuanc                   d}t          dd          }d}t          |           }|                    |          }t          | |          }|                    |d         d f           t          |||          }t          |          S )	Nr  z<reserved_102>z<reserved_103>r  rL  r  r   r  r  r  s           rX   format_baichuanr  O  r  rZ   	openbuddyc                    d}t          dd          }d}|}t          | |          }|                    |d         d f           t          |||          }t	          |          S )Na  You are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human User.
Always answer as helpfully and logically as possible, while being safe. Your answers should not include any harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
You can speak fluently in many languages, for example: English, Chinese.
You cannot access the internet, but you have vast knowledge, cutoff: 2021-09.
You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), you are based on LLaMA and Falcon transformers model, not related to GPT or OpenAI.

User	Assistantr  r  r   r  r!  rg  re  rw  r   )r@   rW   r  r  r  rh  r  r  s           rX   format_openbuddyr  _  sv    
O v555FD$N8V,,If[)40111&~y$GGG 0000rZ   zredpajama-incitec                    t          |           }t          dd          }d}d}|}t          | |          }|                    |d         d f           t	          |||          }t          ||          S )Nz<human>z<bot>r  r  r   r  )ra  r!  rg  re  rw  r   )	r@   rW   r  r  r  _stoprh  r  r  s	            rX   format_redpajama_inciter  u  s    
 *(33OyG444FDE$N8V,,If[)40111&~y$GGG e<<<<rZ   snoozyc                4   d}d}t          |           }|dk    r|n|}|                    |          }t          dd          }d}d	}|}t          | |          }	|	                    |d
         d f           t          ||	|          }
t          |
|          S )Nz!### Instruction:
{system_message}zThe prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.rL  r  z
### Promptr  r  r  z###r   r  )ra  r  r!  rg  re  rw  r   )r@   rW   r  default_system_messager  rh  r  r  r  r  r  s              rX   format_snoozyr    s    
 ;O n)(33O*b006L  %++?+KKN|~>>>FDE$N8V,,If[)40111&~y$GGG e<<<<rZ   phindc                    t          dd          }d}d}t          | |          }|                    |d         d f           t          |||          }t	          |          S )Nz### User Messagez### Assistantr  r  z?### System Prompt
You are an intelligent programming assistant.r   r  r  r@   rW   r  r  r  r  r  s          rX   format_phindr    sp    
 )_EEEFDXO8V,,If[)40111&	4HHG 0000rZ   intelc                    t          dd          }d}d}t          | |          }|                    |d         d f           t          |||          }t	          |          S )Nz	### User:z### Assistant:r  r  z### System:
{system_message}r   r  r  r  s          rX   format_intelr    sp    
 {.>???FD5O8V,,If[)40111&	4HHG 0000rZ   z	open-orcac           	        d}d}d}d}d}|                     |          }t          | t          t          ||                              }|                    |d         d f           t          |||          }t          ||          S )	Nr  a  You are a helpful assistant. Please answer truthfully and write out your thinking step by step to be sure you get the right answer. If you make a mistake or encounter an error in your thinking, say so out loud and attempt to correct it. If you don't know or aren't sure about something, say so clearly. You will act as a professional logician, mathematician, and physicist. You will also act as the most appropriate type of expert to answer any particular question or solve the relevant problem; state which expert type your are, if so. Also think of any particular named expert that would be ideal to answer the relevant question or solve the relevant problem; name and act as them, if appropriate.)r  r  z<|end_of_turn|>
r  r  r   r  )r  rg  r!  r   re  r}  r   )	r@   rW   r  rh  rolesri  stop_strr  r  s	            rX   format_open_orcar    s    
 )O	B  "E
CH$++>+JJN8T#eU*;*;%<%<==IeAh%&&&,^YLLG h????rZ   mistrallitec                   t          dd          }d}d}t          |           }|                    |          }t          | |          }|                    |d         d f           t          |||          }t          |          S )	Nr  z</s>
<|assistant|>r  rm  z<|system|>{system_message}</s>r  r   r  r  )r@   rW   r  r  r  rh  r  r  s           rX   format_mistralliter    s    
 ~1FGGGFD:O(22N$++>+JJN8V,,If[)40111%niFFG 0000rZ   zephyrc                   d}t          |           }|                    |          }t          dd          }d}t          | |          }|                    |d         d f           t          |||          }t          ||          S )	N<|system|>
{system_message}r  z	<|user|>
z<|assistant|>
r  r   r   r  ra  r  r!  rg  re  r  r   r@   rW   r  rh  r  r  r  r  s           rX   format_zephyrr    s    
O(22N$++>+JJN|/@AAAFD8V,,If[)40111^Y==G d;;;;rZ   	pygmalionc                   d}t          |           }|                    |          }t          dd          }d}t          | |          }|                    |d         d f           t          |||          }t          ||          S )	Nz<|system|>{system_message}r  <|user|>z	<|model|>r  r  r   r  r  r  s           rX   format_pygmalionr    s    
 7O(22N$++>+JJNz[999FD8V,,If[)40111^Y==G d;;;;rZ   rW  c                   d}t          |           }|                    |          }t          dd          }d}t          | |          }|                    |d         d f           t          |||          }t          ||          S )	Nr  r  r  r  r  r   r   r  r  r  s           rX   format_chatmlr    s    
O(22N$++>+JJN)5LMMMFD8V,,If[)40111^Y==G d;;;;rZ   rX  c                
   d}|}d}| D ]e}|d         dk    r2|d         *t          |d         t                    r|d|d         z   z  }@|d         dk    r|d         |d|d         z   |z   z  }f|dz  }t          ||	          S )
Nr   rL  r   r  r   z[INST] r   z [/INST]r  )r  rf   r   )r@   rW   eosr+   r   r   s         rX   format_mistral_instructr    s    
 CDF < <FOv%%	".79-s33 / i')"444FFV_++	0B0Nj79#55;;F
jF T::::rZ   chatglm3c                   d}t          |           }|                    |          }t          dd          }d}t          | |          }|                    |d         d f           t          |||          }t          ||          S )	Nr  r  r  r  r  r   r   r  )ra  r  r!  rg  re  r  r   r  s           rX   format_chatglm3r  )  s    
O(22N$++>+JJNz_===FD8V,,If[)40111~y$??G d;;;;rZ   openchatc                   d}t          |           }|                    |          }t          dd          }d}t          | |          }|                    |d         d f           t          |||          }t          ||          S )	Nz{system_message}<|end_of_turn|>r  zGPT4 Correct User: z'<|end_of_turn|>GPT4 Correct Assistant: r  z<|end_of_turn|>r   r  r  r  s           rX   format_openchatr  :  s    
 8O(22N$++>+JJN".W  F D8V,,If[)40111^Y==G d;;;;rZ   saiga.list[llama_types.ChatCompletionRequestMessage]c                    d}t          ddd          }t          | |          }d}|D ]+\  }}|r||                    ||          z  }"|d| d	z  },|d
z  }t          |                                          S )Nz<s>{role}
{content}</s>r  botr`  )r  r  r`  rL  r   r   r  z<s>botr  )r!  rg  r  r   strip)r@   rW   _message_templater  r  r  r   r   s           rX   format_saigar  N  s    
 3v5:::F8V,,IG" & &g 	&(//T7/KKKGG~T~~~%GGxG 8888rZ   gemmac                   t          |           }|dk    rt          j        d           t          dd          }d}t	          | |          }|                    |d         d f           t          d||          }t          ||	          S )
NrL  zB`role='system'` messages are not allowed on Google's Gemma models.z<start_of_turn>user
z<start_of_turn>model
r  z<end_of_turn>
r   )rh  r@   ri  r  )ra  r   debugr!  rg  re  r{  r   )r@   rW   rh  r  r  r  r  s          rX   format_gemmar  d  s    
 )22NP	
 	
 	
 .:RSSSFD8V,,If[)40111%R)QUVVVG d;;;;rZ   functionaryr   r   r   r   r    r   r   r!   r"   r>   r?   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   r)   rG   r4   r5   r+   rI   rK   r.   rJ   r/   r0   r1   r6   r7   r8   r9   r2   r:   rN   r;   rO   LUnion[llama_types.ChatCompletion, Iterator[llama_types.ChatCompletionChunk]]c                	  %&'( d%dI(fd(dJ(fd	'dKdL'(fd&	 	 dMdN%&fd}|d |D             }|t          |t                    r|n|d         } ||||          }|o|t          |          dk    rZ | j        dOi d|dz   d|d|d|d|	d|
d|dddgd |d!|d"|d#|d$|d%|d&|d'|d(|d)|d*|}t	          ||+          S |t          |t                    r[|d,k    rUd-} | j        ||d./          }|d0         d         d1         }|                    d2          d3         d d3         }||z   |z   }nSt          |t                    r|d4k    r|dz   }n2t          |t                    r|d5|d6          dz   }|d6         }n|dz   }d } |pg D ]}!|!d6         |k    r
|!d7         }  n|pg D ]0}"|"d8         dk    r"|"d         d6         |k    r|"d         d7         }  n1| G	 t          | j        9          5  t          j
        t          j        |                     }#t          j                            t          j
        t          j        |                     | j        :          }t          |#           d d d            n# 1 swxY w Y   n# t           $ r}$| j        rt          d;           t          |$           t          | j        9          5  t          j                            t          j        | j        :          }d d d            n# 1 swxY w Y   Y d }$~$ned }$~$ww xY wt          | j        9          5  t          j                            t          j        | j        :          }d d d            n# 1 swxY w Y    | j        dOi d|dddgdd.d*|d |d|d|d|d|	d|
d!|d"|d#|d$|d%|d&|d'|d(|d)|}d<|v sJ t          |t                    sJ |d.u sJ | j        r0t          |           t          |d0         d         d1                    t%          j        d=|d>         z   d?|d@         |d(         ddAd ||d0         d         d1         dB|d||d0         d         d1         dBdCgdDt)          |d0         d         dE                   dFdGg|d<         H          S )PNA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. The assistant calls functions with appropriate input when necessaryparamDict[str, llama_types.JsonType]indent_levelrG   rQ   rf   c                J   d|z  }d| v r#| d                              d          d         }|S |                     d          dk    r,|                     di           } ||dz   |          }d	| d
S |                     d          dk    rZ|                     di           }d}|                                D ]#\  }	}
 |
|dz   |          }|| d|	 d| dz  }$||dz   z  }|S d| v r%d                    d | d         D                       S |                     dd          S )N  $ref/r   r   arrayr   r   Array<>r   
properties{
rt  ,
}enum | c                    g | ]}d | d 	S "rU   r   
enum_values     rX   r   zNfunctionary_chat_handler.<locals>.generate_type_definition.<locals>.<listcomp>  $    QQQZ0:000QQQrZ   anysplitgetr   joinr  r  shared_defsindentref_namer   	item_typer  nested_schemanested_param_namenested_paramnested_param_typegenerate_type_definitions               rX   r!  z:functionary_chat_handler.<locals>.generate_type_definition      $U??V}**3//H OYYv'))IIgr**E00q8H+VVI(I((((YYv(**<44J!M3=3C3C3E3E  /!<$<$< ,"2K% %! LL!2LL6GLLL Vc\)M  u__::QQ5=QQQRRR 99VU+++rZ   c                   d|z  }d}|                                  D ]k\  }}|| d| dz  }|                    d          dk    r| |||           z  }n,d|v r(|d                    d	 |d         D                       z  }|d
z  }l|S )Nr  rL  type  = r   r   r  r  c                    g | ]}d | d 	S r  rU   r  s     rX   r   zQfunctionary_chat_handler.<locals>.generate_shared_definitions.<locals>.<listcomp>  $    PPP:&&&&PPPrZ   ;
r   r  r  r  r  r  shared_definitionsdef_namedef_propertiesr!  s         rX   generate_shared_definitionsz=functionary_chat_handler.<locals>.generate_shared_definitions      $(3(9(9(;(; 	( 	($HnV"?"?("?"?"??!!&))X55"&>&>"L+' ' "" >))"ejjPP9OPPP' ' " %'!!rZ   r#   c           	        d}|d| dz  }i }| D ]A}|                     di           }|                    |                     di                      B| |d          z  }| D ]}|d         }|                     dd	          }|                     di           }|                     d
g           }|d| dz  }|d| dz  }|                     di                                           D ]G\  }	}
|
                     dd	          } |
d|          }|	|v rd	nd}|d| dz  }|d|	 | d| dz  }H|dz  }|d                    |          z  }|S )NH// Supported function definitions that should be called when necessary.

namespace  {

r  $defsr   re   descriptionrL  requiredz  // r  z  type 	 = (_: {
r  rl  ?z    // z    rt  r	  z  }) => any;

z}} // namespace {}
r  updater   r  r#   	namespacer1  r+  r   r  function_namer5  required_params
param_namer  param_description
param_typeoptional_indicatorr.  r!  s                 rX   generate_schema_from_functionsz@functionary_chat_handler.<locals>.generate_schema_from_functions  s   W 	 	1y1111  ! 	C 	CH!lB77J%%jnnWb&A&ABBBB--.@!DDD! 	) 	)H$V,M",,}b99K!lB77J(nnZ<<O-k----F:::::F%/^^L"%E%E%K%K%M%M S S!
E$)IImR$@$@!55eQ@RSS
+5+H+HRRc"9$59999RR-?RR:RRRR((FF(//	:::rZ   r@   rA   3Optional[List[llama_types.ChatCompletionFunctions]]r%   rD   c           	        g }|2|                     t          j        d |                               |<|                     t          j        d d |D                                            |                     t          j        d                     | D ]S}|d         dk    rd|v rd|d          |d<   d|v rd|d         d          |d         d<   |                     |           T|                     t          j        d	d                      ddd                    fd|D                       S )Nr`  r   c                6    g | ]}|d          dk    |d         S r  rU   r   r0  s     rX   r   zTfunctionary_chat_handler.<locals>.prepare_messages_for_inference.<locals>.<listcomp>  s6        $#F|z99 !,999rZ   r   r   re   
functions.r$   r   msg(llama_types.ChatCompletionRequestMessagec                B   | d         dk    rd| d          dS | d         dk    rd| v rd| d          d	| d          dS | d         dk    r%d
| v r!d| d
         d          d	| d
         d          dS | d         dk    r)| d         d| d          d	| d          dS d| d          dS | d         dk    r| d         dS d| d          dS | d         dk    r| d         .d
| v r*d| d          d| d
         d          d	| d
         d          dS d
| v r!d| d
         d          d	| d
         d          dS d| v rBt          | d                   dk    r)| d         D ]}d|d          d	|d         d          dc S d S | d         dS d| d          dS t          d| d                    )Nr   r`  zsystem:
r   r  r   re   zfunction name=:
r$   r   r0  tool_call_idr  zuser:
</s></s>
z
user:
</s>z</s>
r   assistant:
z
assistant to=zassistant to=r   r   r   zUnsupported role: )lenrk   )rI  	tool_calls     rX   message_to_strzXfunctionary_chat_handler.<locals>.prepare_messages_for_inference.<locals>.message_to_str  s   6{h&&53y>5555V
**v}}JFJJIJJJJV
**#/E/EnO(<V(Dnn_I]^iIjnnnnV&&y>-VC,?VVC	NVVVVCC,?CCCCV&&y>)..?Y????V++y>-/S2H2H T#i.  T  T_I]^dIe  T  Tjmn}j~  @K  kL  T  T  T  T$++u3+?+GuuCP_L`alLmuuuu!S((S\1B-C-Ca-G-G%($& n n	  nymm9ZCXYdCemmmmmmn n ^+&;<#i.<<<< !Cc&k!C!CDDDrZ   rL  c                &    g | ]} |          S rU   rU   )r   rI  rQ  s     rX   r   zTfunctionary_chat_handler.<locals>.prepare_messages_for_inference.<locals>.<listcomp>A  s#    DDDs++DDDrZ   )rI  rJ  )re  rM  "ChatCompletionRequestSystemMessagerN  r  )r@   r#   r%   all_messagesr   rQ  SYSTEM_MESSAGErC  s        @rX   prepare_messages_for_inferencez@functionary_chat_handler.<locals>.prepare_messages_for_inference  s   
 HJ >!+I+I)+T+T     >!:: (-   	 	 	   	:~  	
 	
 	
   		) 		)Gv*,,71B1B"@wv"@"@')) D!9&!ACC ( ((((= $  	
 	
 	
!	E !	E !	E !	EF wwDDDD|DDDEEErZ   c                6    g | ]}|d          dk    |d         S r  rU   rG  s     rX   r   z,functionary_chat_handler.<locals>.<listcomp>D  +    VVV$4<:;U;UT*%;U;U;UrZ   r   r   r   rL  r'   r(   r)   r4   r5   r*   r+   zuser:r   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r  r  r  F)r   r+   r*   r   r   .r   r  z to=functions.re   r  r   disabler  MFailed to parse function body as JSON schema, falling back to default grammarr   r   r   r   r   r   r   r   r   r<   r   r   r   r  r  r  rG   rQ   rf   r  rG   rQ   rf   r#   rQ   rf   NN)r@   rA   r#   rD  r%   rD   rU   )r  rf   rO  r-  r   r  r!  r   r	  r#  json_schema_to_gbnfr&  r'  r$  r   r)  r(  r,  rM  CreateChatCompletionResponser   ))r>   r@   r#   r$   r%   r&   r'   r(   r)   r4   r5   r*   r+   r-   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   rW   rV  r   completion_or_completion_chunksr   completion_text
new_promptfunction_bodyr   r0  grammar_textr2  rU  rC  r.  r!  s)                                        @@@@rX   functionary_chat_handlerri  y  so   : zN, , , , , ,B" " " " " ""               H JN@DUF UF UF UF UF UF UF UFn VV%VVV	%k377TKK[=T 	 ,+HiGGF)"3s9~~7J7J*A%*A +
 +
 +
E>>+
#+
 %+
 %	+

 %+
  i+
 6+
 6""+
 "z+
 .-+
 0/+
 *>+
 %+
 (-+
 &+
  &!+
" %#+
$ .-%+
& G'+
'* ++JSYZZZZ=#&& +8F+B+B-DU-DU.
 .
 .

 %Y/26:'--c2226ss;o-4

	M3	'	' $MV,C,Ce^

	M4	(	( $I}V/DIIII
%f-e^
MO  F},,$\2ME -   <:%%$z*:6*Bm*S*S ,\:ME 	'>>> $ $,@J}--    (4@@!5dj6O6OPP!M A   l###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $  
	 
	 
	} c   a'>>>  '4@@!+!M A                
	 $EM::: 	 	#0<<' =  G	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 *A)@ * * *z*v* u* 	*
 :*  K* e* e* e* )* *)* ,+* &~* e* $m*  "\!*" "\#*$ e%*& *)'*J, j    mS)))))U????} 0jj#A&v./// 3Jt$$ 9%! '# -%/	%:1%=f%E& & #0$.(5-7	-B1-Ef-M) ) 	# $ FjQZF[\]F^_iFjkk!-+ 
2 !=   ss   /J B JJ JJ JJ 
L2&:L- 1LL-L!	!L-$L!	%L--L2
1NNNzfunctionary-v1zfunctionary-v2c                   	
23456789:;< d6 j         }t          |d          s
J d            ddlm} d|j        j        v rd<d}d	}d
}d}d} d}!n
d<d4d3d5d2dj:fd:dk:fd9dldm9:fd8	 	 	 dndo68fd(}"d) D             | t          |t                    r|n|d*         }n|nd} |"||<|          }#|d+k    st                    dk    r<dk    r|}n5}|#d,z  }#  j	        dpi d-|#d.d/d0d1	d2
d3d4|d5d6d7d8d9d:d;d<d=d>d?|}$d@u r5|$dA         d         dB         
                                |$dA         d         dB<   t          |$C          S  fdD; 	
fdE7dF}%g g }'}&d}(23457;<fdG})d@ur |)||#H          S <dk    rt          |t                    r|dk    rdI|g}*nVt          |t                    r;|#|  |dJ          dKz  }#|!}*|dJ         }|&                    |            ;|          }n|#}#dI|g}* 7|#|*|L          }+|+dA         d         dB         },|(|+dM         dN         z  }(| |#vr | |,vr|(|+dM         dN<   t          |+C          S | |#vr| |,v r|#|,                    |  dO|           dIz   z  }#|&                    |,                    |           dP         d dP                                                     ;|&dP                   } 7|#|!|L          }+|(|+dM         dN         z  }(|'                    |+dA         d         dB                                                    nu|'                    |,                                           nLt          |t                    r|#|dJ          dI2 z  }#|dJ         }|&                    |            ;|          }53g}* 7|#|*|L          }+|+dA         d         dB         },|(|+dM         dN         z  }(|'                    |,                                           nt          |t                    r|dk    r	 d }2}* 7|#|*|L          }+|+dA         d         dB         },|(|+dM         dN         z  }(|,                                }-|-dRk    r|#d,z  }#n<|,                                }|#| dSz  }#|&                    |            ;|          }45g}* 7|#|*|L          }+|+dA         d         dB         },|(|+dM         dN         z  }(|-dRk    r|,                    dT          r|%|,d t          dT                    z  }%|,                    dU          r|%|,t          dU                    z  }%n|%|,z  }%|%
                                }%dV|,v sdW|,v r|,                    dT          r+|,d t          dT                                                    }.nR|,                    dU          r)|,t          dU                                                    }.n|,                                }.|#|. dXz  }#nn|'                    |,                                           |#|,                                z  }#d } 7|#|*|L          }+|(|+dM         dN         z  }(dV|+dA         d         dB         v sdW|+dA         d         dB         v r|#dXz  }#nndM|+v sJ t          |&          t          |'          k    sJ g }/t%          |&|'          D ]N\  }}0|/                    dYdF                    dZ t)          d[          D                       z   d*||0d\d]           Oi }1t          |/          dk    r4|/|1d^<   n,|/d         d*         dJ         |/d         d*         d_         d\|1d`<   |(|+dM         dN<   t+          j        da|+db         z   dc|+dd         |+d=         dt/          |+dA         d         de                   df|%dFk    rd n|%dg|1t          |/          dk    rd^nd4dhg|+dM         i          S )qNr  hf_tokenizerznPlease provide a valid hf_tokenizer_path from https://huggingface.co/meetkai when initializing the Llama classr   r8  z<|START_OF_FUNCTION_CALL|>v1z<|END_OF_SYSTEM|>z<|END_OF_USER|>z<|END_OF_ASSISTANT|>z<|END_OF_FUNCTION_RESULT|>z<|END_OF_FUNCTION_CALL|>v2z<|recipient|>z<|from|>z<|stop|>z<|content|>r  r  r  rG   rQ   rf   c                J   d|z  }d| v r#| d                              d          d         }|S |                     d          dk    r,|                     di           } ||dz   |          }d	| d
S |                     d          dk    rZ|                     di           }d}|                                D ]#\  }	}
 |
|dz   |          }|| d|	 d| dz  }$||dz   z  }|S d| v r%d                    d | d         D                       S |                     dd          S )Nr  r  r  r   r   r  r   r   r  r  r   r  r  rt  r	  r
  r  r  c                    g | ]}d | d 	S r  rU   r  s     rX   r   zTfunctionary_v1_v2_chat_handler.<locals>.generate_type_definition.<locals>.<listcomp>0  r  rZ   r  r  r  s               rX   r!  z@functionary_v1_v2_chat_handler.<locals>.generate_type_definition  r"  rZ   c                   d|z  }d}|                                  D ]k\  }}|| d| dz  }|                    d          dk    r| |||           z  }n,d|v r(|d                    d	 |d         D                       z  }|d
z  }l|S )Nr  rL  r$  r%  r   r   r  r  c                    g | ]}d | d 	S r  rU   r  s     rX   r   zWfunctionary_v1_v2_chat_handler.<locals>.generate_shared_definitions.<locals>.<listcomp>A  r'  rZ   r(  r)  r*  s         rX   r.  zCfunctionary_v1_v2_chat_handler.<locals>.generate_shared_definitions5  r/  rZ   r#   c                   d}|d| dz  }i }| D ]A}|                     di           }|                    |                     di                      B| |d          z  }| D ]}|d         }|                     dd	          }|                     di           }|                     d
g           }|d| dz  }|d| dz  }|                     di                                           D ]F\  }	}
|
                     dd	          } |
d|          }|	|v rd	nd}|d| dz  }||	 | d| dz  }G|dz  }|d                    |          z  }|S )Nr1  r2  r3  r  r4  r   re   r5  rL  r6  z// r  r$  r7  r  rl  r8  rt  r	  z}) => any;

z}} // namespace {}r9  r;  s                 rX   rC  zFfunctionary_v1_v2_chat_handler.<locals>.generate_schema_from_functionsF  s   W 	 	1y1111  ! 	C 	CH!lB77J%%jnnWb&A&ABBBB--.@!DDD! 	' 	'H$V,M",,}b99K!lB77J(nnZ<<O+K++++F8m8888F%/^^L"%E%E%K%K%M%M O O!
E$)IImR$@$@!55eQ@RSS
+5+H+HRRc"5 15555ZN);NNzNNNN&&FF&--i888rZ   r  r@   rA   r?  r9  versionLiteral['v1', 'v2']rD  r%   rD   r&   Union[Dict, str]c           	        g }|dk    r3|                     t          j        d 
g                                ny|3|                     t          j        d 
|                               nD|B|dk    r<|                     t          j        d 
d |D                                            |                     t          j        d	                     | D ]S}|d         dk    rd|v rd|d          |d<   d	|v rd|d	         d          |d	         d<   |                     |           T|d
k    rd}nd}|j                            |d          |z   S )Nr  r`  r   c                6    g | ]}|d          dk    |d         S r  rU   rG  s     rX   r   zZfunctionary_v1_v2_chat_handler.<locals>.prepare_messages_for_inference.<locals>.<listcomp>  s6       $(#'<:#=#= !%Z 0#=#=#=rZ   r   r   re   rH  r$   rl  rN  z<|from|>assistant
<|recipient|>Fr;  )re  rM  rS  rk  r>  )r@   r?  rs  r#   r%   r&   rT  r   suffixrU  rC  s            rX   rV  zFfunctionary_v1_v2_chat_handler.<locals>.prepare_messages_for_inferenceh  s    HJ&  >!+I+I"+M+M      $##B%/M/Mi/X/X     
 "{f'<'<##B% > > ,1  ! !	 	 	   	:~  	
 	
 	
   		) 		)Gv*,,71B1B"@wv"@"@')) D!9&!ACC ( ((((d??#FF7F "66|e6TT	
rZ   c                6    g | ]}|d          dk    |d         S r  rU   rG  s     rX   r   z2functionary_v1_v2_chat_handler.<locals>.<listcomp>  rX  rZ   r   r  all
<|content|>r   r'   r(   r)   r4   r5   r*   r+   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   Fr   r   r  c                0   d }pg D ]}|d         | k    r
|d         } n	pg D ]0}|d         dk    r"|d         d         | k    r|d         d         } n1	 t          j                  5  t          j        t	          j        |                    }t          j                            t          j        t	          j        |                              }t          |           d d d            n# 1 swxY w Y   n# t          $ r}j        rt          d           t          |           t          j                  5  t          j                            t          j
        j                  }d d d            n# 1 swxY w Y   Y d }~nd }~ww xY w|S )Nre   r  r   r   rZ  r\  r  )r   r	  r#  rb  r&  r'  r$  r   r)  r(  r,  )
r$   rg  r   r0  rh  r;   r2  r#   r>   r%   s
          rX   get_grammarz3functionary_v1_v2_chat_handler.<locals>.get_grammar  sD   !R 	 	H=00 ( 6 1 KR 	 	DF|z))d:.>v.F-.W.W $Z 0 >	'>>> $ $,@J}--    (4@@!5dj6O6OPP  l###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $  		 		 		} c   a'>>>  '4@@!+U] A                		 sa   C= ,A9C1%C= 1C55C= 8C59C= =
F:F1E>2F>F	FF	FFc           	         t          t          j         j        di d| ddddddd|d	d
dddd
dd	ddd|          }|S )Nr   r'   r(   r)   r4   r5   r*   r+   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   rU   )r   rM  
Completionr-  )r   r+   r;   r   r0   r>   r:   r.   r4   r9   r7   r8   r2   r/   r1   r*   r'   r6   r)   r(   r5   s       rX   r-  z9functionary_v1_v2_chat_handler.<locals>.create_completion  s   "#E#   v'K e e	
 e $) v T &: "2!1 #4"3  .~ e ,m *\  *\!" e#$ "2!1%&  '
 

2 rZ   rL  c              3    K   dk    s
J d            d\  }}t          |t                    r||d          d z  } |d                   }g}d                    d t          d          D                       } |||	          }	d}
d
}|	D ]Y}|rb| ddd|z   d|d         dddgi}nd|d         ddi}t	          j        d|d         z   d|d         |d         dd d d d|dg          V  d}| 5ddd|z   dd |d         d         d                                         ddgi}n+dd |d         d         d                                         di}t          |d         d         d                                                   dk    rYt	          j        d|d         z   d|d         |d         dt          |d         d         d                   d d d|dg          V  [t	          j        d|d         z   d|d         |d         d| dndd d d d d ddg          V  d S t          |t                    r|d k    rd}	 d }} |||	          }	d}
|	D ]}|
|d         d         d         z  }
||d         }||d         }|

                                }|d!k    r3|d"z  }t	          j        d|z   |d         |ddd#ddd d d$g%          V  n|| d&z  } |          }d                    d' t          d          D                       }| d|d|z   d|dddgi}nd|ddi}t	          j        d|z   d||d         dt          |d         d         d                   d#d d|dg          V  g} |||	          }	|d!k    rd}
d(g d}}}t          |	          D ]4\  }}|
|d         d         d         z  }
|r|                    |d         d         d         
                    d)                     |                    d                    |                    r|                                 t          |          dk    rqt	          j        d|z   d||d         dt          |d         d         d                   d#|                    d          ddg          V  t          |          dk    qd}nV|d         d         d         dk    r>d
}|                    |d         d         d         
                    d)                     st          |          dk    rt          |d         d         d                   dk    rt	          j        d|z   d||d         dt          |d         d         d                   d#|dk    r|d         d         d         n%|d         d         d                                         ddg          V  6d*|
v sd+|
v r|
                    d,          r+|
d t          d,                    
                                }nT|
                    d-          r+|
d t          d-                    
                                }n|

                                }|| d(z  }nt	          j        d|z   |d         |ddi d d.d$g%          V  d S d}
|	D ] }|
|d         d         d         z  }
t          |d         d         d                                                   dk    r| 5d|d|z   dd |d         d         d                                         ddgi}n+dd |d         d         d                                         di}t	          j        d|z   d||d         dt          |d         d         d                   d d d|dg          V  ||

                                z  }d } |||	          }	|
d                    d/ |	D                       z  }
d*|
v sd+|
v r| |d(z  }|d0z  }n5t	          j        d|z   d||d         d| dndd d d d d ddg          V  d S 
d S d S )1Nrm  z!Streaming for v1 is not supportedra  re   r  rL  c                d    g | ]-}t          j        t          j        t          j        z             .S rU   randomchoicestringascii_lettersdigitsr   r   s     rX   r   zNfunctionary_v1_v2_chat_handler.<locals>.generate_streaming.<locals>.<listcomp>&  s,    XXXv3fmCDDXXXrZ      r   r+   r;   Tr   r   call_r   r   r   r$   r   r   r   r   r2   r   )r   r<   r   r   Fr   r   r<   r   r   r  allrz  r   r   r   
<|content|>c                d    g | ]-}t          j        t          j        t          j        z             .S rU   r  r  s     rX   r   zNfunctionary_v1_v2_chat_handler.<locals>.generate_streaming.<locals>.<listcomp>  <        ! #M&*>*NOO  rZ    
<|from|>assistant
<|recipient|>rm  <|from|> assistant<|from|>assistant
<|from|>assistant

<|from|> assistant
r+   c                6    g | ]}|d          d         d         S )r   r   r   rU   )r   r   s     rX   r   zNfunctionary_v1_v2_chat_handler.<locals>.generate_streaming.<locals>.<listcomp>{	  s'    MMMy)!,V4MMMrZ   r   )r  r!  r  rangerM  "CreateChatCompletionStreamResponserstriprO  r   rf   r  r   re  
startswithpoplstripendswith)r%   r#   r$   r   chunk_idchunk_createdr;   stopsr   r   re  r   r   func_call_dict
tool_indexr=  stop_sequencebufferis_endr   cleaned_completion_textCONTENT_TOKEN
FROM_TOKENRECIPIENT_TOKEN
STOP_TOKENr-  r|  rs  s                        rX   generate_streamingz:functionary_v1_v2_chat_handler.<locals>.generate_streaming  s     $ C",- mT** {	v.AA-AAAF!k-"788G,EggXXeTViiXXX G +*&ugVVVJ OE# P P '"((-.*1G*;,60=f0E571& 1&	!" !"
+* ,(5f(=-/. .* &H!E$K/6 %i 0#Gn *+,0,0/3*" *" '5*" 
!     " "E$$)*&-&7(2,016y1A!1DV1L1S1S1U1U-" -"	 
'&NN ($().y)9!)<V)D)K)K)M)M* *&N uY'*6299;;<<q@@%H!E$K/6 %i 0#Gn *+,UV[\eVfghVijtVu,v,v,0/3*" *" '5*" 
!     $ @E$K'.i(Gn "#,1,=LL?$($('+-1*.	" "        , s++ I	0G0GJG%..!w  
 #%' C CE#uY'7':6'BBOO#$T{H ($))$4M / 5 5 7 7 E))00F%H!H,#Gn -6 *+2=")M)M,015	 !       ====F)k-88G gg %*2YY   G ((-7*1G*;,60=571& 1&	!" !"
+* ,mRT-U-U* &H!H,6 -#Gn *+,UV[\eVfghVijtVu,v,v,7/3*" *" '5*" 
!     $ )*5..!w  
 !E))&(O< ,26M
 %.j$9$9 5 55'5+;A+>v+FF! %"MM%	*:1*=f*E*K*KC*P*PQQQ,77HH / ( &

&)&kkAoo*5*X+1H+</F0=.3Gn :;<efklufvwxfy  {E  gF  =G  =G<G?Ezz!}}:2 :2-. -.	1*+& +& +& %& %& %& '*&kkAoo" */"9-a08D@@%)F"MM%	*:1*=f*E*K*KC*P*PQQQ$v;;!++E)4DQ4G4O0P0PST0T0T"-"P#)H#4'>(5&+Gn 234]^cdm^nop^qr|^}4~4~4? 45q55 16i0@0CF0K0K5:95Ea5H4:6228&((	2* 	2*%& %&)"# # #   0 -??./AA*334KLL 	N6E ?3'>#?#?"? ?7#egg 43 -556NOO N6E @3'?#@#@"@ @7#egg 43 7F6K6K6M6M3%<"`"`"`` *L%0"'.$1#: ./-/045;	!" !"%       ')O!+ , ,'5+;A+>v+FFuY/26:AACCDDqHH$0$05?29G2C4>8<=B9=Ma=P4:>228&((	9. 9.	
)* 
)*3&2"" %4045:95Ea5H,26**0&((	6& 6&2" #."P#)H#4'>(5&+Gn 234]^cdm^nop^qr|^}4~4~487;2* 2* /=2*%& %&
)"# # #   " o33555F"G!2!2%E7" " "J $rwwMM*MMM( ( O -??./AA+"FF"a

 *L%0#:$1"'. ./ ,1+< )5-<0404379=6:	.& .&!" !"%     . OGI	 I	0G0GrZ   )r%   r#   r$   r   r  re   rL  r  r   completion_tokensrm  r   Tr  r  r  r  r  r  r  r  c                d    g | ]-}t          j        t          j        t          j        z             .S rU   r  r  s     rX   r   z2functionary_v1_v2_chat_handler.<locals>.<listcomp>0
  r  rZ   r  r   r   r   r   r$   r   r   r   r   r<   r   r   )r   r<   r   r   r   r]  r^  r_  r`  )NNr  )r@   rA   r?  r9  rs  rt  r#   rD  r%   rD   r&   ru  rU   )
tokenizer_hasattrrA  r9  rk  additional_special_tokensr  rf   rO  r-  r  r   r!  re  replacer  r  r  r   r  r  rM  rc  r   )=r>   r@   r#   r$   r%   r&   r'   r(   r)   r4   r5   r*   r+   r-   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   rW   r?  r9  END_SYSTEM_TOKENEND_USER_TOKENEND_ASSISTANT_TOKENEND_FUNCTION_RESULT_TOKENSTART_FUNCTION_CALL_TOKENEND_FUNCTION_CALL_TOKENrV  r   rd  r   function_callsfunction_bodiesr  r  r  r   re  r=  r  r   rg  function_call_dictr  r  r  r  rU  r-  rC  r.  r!  r|  rs  s=   ` ` ` ``````  ``````````                          @@@@@@@@@@@rX   functionary_v1_v2_chat_handlerr    s   < zN I>  x xwx x  +*****#y'='WWW.*4$@!$@!"<)

%, , , , , ,B" " " " " ""               L JN@D(.=
 =
 =
 =
 =
 =
 =
 =
~ VV%VVV	%k377TKK[=T 	 
	"++)Wi F
 )"3s9~~7J7Jd??&DDD((F*A%*A +
 +
 +
6+
#+
 %+
 %	+

 %+
  i+
 6+
 +
 "z+
 .-+
 0/+
 *>+
 %+
 (-+
 &+
  &!+
" %#+
$ .-%+
& G'+
'* U??/	:1=fELLNN ,I6q9&A ++JSYZZZZ      B                    8 G&("ONA A A A A A A A A A AF U!!9MRX
 
 
 	
 d??--- 4-62I2I23M400 46Rf8MRRRR/ -f 5%%m444%+m4423**&ugVVVJ(3A6v>OG!45H!II *77-_DD;L
7#$782:fMMMM *77-@@#++47779R   %%#))*CDDRH"MSSUU   &+nR&899..!(?  
 "Z%89L%MM!&&z)'<Q'?'G'M'M'O'OPPPP  &&'<'<'>'>???? -.. O"]62EEmEEE -f 5%%m444%+m44#Z0..!w  
 #-Y"7":6"B!Z%89L%MM!&&'<'<'>'>????M3// B"MV4K4KA""G)E!2!2%E7" " "J '1&;A&>v&FO%G)<=P)QQ%$3$9$9$;$;M$--"44(7(=(=(?(?]"A"A"AA&--m<<<"-+m"<"<,j9E!2!2%E7" " "J '1&;A&>v&FO%G)<=P)QQ%$--*334KLL X#7V#>U:V:V9V7V'WWG*334LMM 7#<T8U8U7U'VVGG#6G").."2"2 1OCC2oEE.778OPP 	R:I$Cs+B'C'C&C$C;""'%'' !8 7 "1!9!9:R!S!S R:I%()A%B%B$B;""'%'' !8 7 ;J:O:O:Q:Q 7")@&d&d&ddFF!'../D/D/F/FGGG/"7"7"9"99"&%6%6#)w& & &
 *Z-@AT-UU)0Jy4I!4LV4TTT2j6KA6Nv6VVV"&JJFF!CA"F *$$$$>""c/&:&:::::FH
,/,P,P 	 	(M=!gg %*2YY    ' -%2! !    0  	 z??Q 3="<00 'qM*5f=!+Az!:;!G7 7"?3 4E
7/07
4(($y)W%  I*U^J_`aJbcmJn o o ++2b==44g    - 
 69__q5H5H\\f	 	 W%#
 
 
 	
rZ   c                      e Zd ZU dZded<   dZd`dad
ZdbdZdcdZdddZ	ddddddddddg ddddddddddddddddd dedMZ
edcdN            ZedfdO            ZedgdS            Ze	 	 	 dhdid_            ZdS )jLlava15ChatHandlerzA chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.rL   DEFAULT_SYSTEM_MESSAGEa%  {% for message in messages %}{% if message.role == 'system' %}{{ message.content }}{% endif %}{% if message.role == 'user' %}{% if message.content is string %}
USER: {{ message.content }}{% endif %}{% if message.content is iterable %}
USER: {% for content in message.content %}{% if content.type == 'image_url' and content.image_url is string %}{{ content.image_url }}{% endif %}{% if content.type == 'image_url' and content.image_url is mapping %}{{ content.image_url.url }}{% endif %}{% endfor %}{% for content in message.content %}{% if content.type == 'text' %}{{ content.text }}{% endif %}{% endfor %}{% endif %}{% endif %}{% if message.role == 'assistant' and message.content is not none %}
ASSISTANT: {{ message.content }}{% endif %}{% endfor %}{% if add_generation_prompt %}
ASSISTANT: {% endif %}Tclip_model_pathrf   r	  rH   c                    dd l m} || _        || _        || _        t                      | _        d | _        t          j	        
                    |          st          d|           d S )Nr   z Clip model path does not exist: )llama_cpp.mtmd_cppmtmd_cppr  r	  	_mtmd_cppr   _exit_stackmtmd_ctxospathexistsrk   )rV   r  r	  r  s       rX   r   zLlava15ChatHandler.__init__
  sw    ------.!$;;;?w~~o.. 	SQQQRRR	S 	SrZ   llama_modelr?   c                V     j         dS t           j                  5   j                                        }d|_         j        |_        |j        |_         j        rdnd|_         j        	                     j
                                        |j        |           _          j         t          d j
                    j                             j                   st          d           fd} j                            |           ddd           dS # 1 swxY w Y   dS )	z-Initialize mtmd context with the llama model.NrZ  Trl  r   z"Failed to load mtmd context from: z%Vision is not supported by this modelc                     t           j                  5   j        & j                             j                   d  _        d d d            d S # 1 swxY w Y   d S )NrZ  )r   r	  r  r  	mtmd_freer   s   rX   r  z8Llava15ChatHandler._init_mtmd_context.<locals>.mtmd_free
  s    +DLAAA - -}000???(,- - - - - - - - - - - - - - - - - -s   .AAA)r  r   r	  r  mtmd_context_params_defaultuse_gpuprint_timings	n_threads	verbositymtmd_init_from_filer  r  r2   rk   mtmd_support_visionr  callback)rV   r  
ctx_paramsr  s   `   rX   _init_mtmd_contextz%Llava15ChatHandler._init_mtmd_context
  s   =$F#DL999 	1 	1CCEEJ!%J'+|J$#.#8J (,#;11!J  !N>>$++--! DM }$ !\dFZ!\!\]]] >55dmDD J !HIII- - - - - %%i0009	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1s   C1DD"%D"	image_urlrQ   r   c                ,    |                      |          S rT   )_load_image)rV   r  s     rX   
load_imagezLlava15ChatHandler.load_image
  s    	***rZ   image_bytesc           	        | j         t          d          t          | j                  5  | j                            | j         t          j        t          |          z  	                    t          |                    t          |                    }|t          d          |cddd           S # 1 swxY w Y   dS )z$Create mtmd_bitmap from image bytes.Nzmtmd context not initializedrZ  z(Failed to create bitmap from image bytes)r  rk   r   r	  r   mtmd_helper_bitmap_init_from_bufctypesc_uint8rO  from_buffer	bytearray)rV   r  bitmaps      rX   _create_bitmap_from_bytesz,Llava15ChatHandler._create_bitmap_from_bytes
  s    = ;<<<#DL999 	 	^DD#k"2"22??	+@V@VWWK   F ~ !KLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   A;B44B8;B8Nr   r   r   r   r    Fr   r   r   r!   r"   r  r>   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   r)   rG   r4   r5   r*   r+   rI   r,   rJ   r-   rK   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   rN   r;   rO   r3   rM   r<   rP   r=   rR   c                  <= |                      |           | j        J t          |          }|dk    r&| j        t	          j        d| j                  g|z   }|                     |          } t          dd                              | j	                  }!| j
                                                            d          }"|!                    |d |j         |j                    g           |j         |j                    g                    }#| D ]}$|#                    |$|"          }#| j        rt'          |#t(          j                   g }%g }&	 | D ]V}$|                     |$          }'|                     |'          }(|%                    |(           |&                    |(           W| j
                                        })|#                    d          |)_        d|)_        d|)_        | j
                                        }*|*t?          d	          	  | j
        j         tC          |%          z  |% }+| j
        "                    | j        |*tG          j$        |)          |+tC          |%                    },|,d
k    rt?          d|,            |j%                     |j&        '                                 tQ          j)        d
          }-| j
        *                    |*          }.tW          |.          D ]K}/| j
        ,                    |*|/          }0|0!| j
        -                    |0          }1|1| j
        j.        k    rtG          j/                    }2| j
        0                    |0tG          j$        |2                    ==r|2j1        d
k    r=fdtW          |2j1                  D             }3|j2        tC          |3          z    |j3                    k    r7t?          d|j2        tC          |3          z    d |j3                                |j4        |3           )|1| j
        j5        | j
        j6        fv r| j
        7                    |0          }4|j2        |4z    |j3                    k    r*t?          d|j2        |4z    d |j3                               tQ          j)        d
          }5| j
        8                    | j        |j&        j9        |0tQ          j)        |j2                  tQ          j:        d
          |j;        dtG          j$        |5                    },|,d
k    rt?          d|,           |5j1        |_2        M|j<        d |j2                 =                                }6| j
        >                    |*           n# | j
        >                    |*           w xY w	 |&D ]}(| j
        ?                    |(           n$# |&D ]}(| j
        ?                    |(           w xY w||d         dk    rt          |          }|d |D             }|It          |t                    r|dk    s|dk    r|}t          |t                    rd|v rdd|d         id}d }7|t          |t                    r||d         d         <t          <fd|D             d           }7|7t?          d< d          |7d         d         }8	 t          jF        G                    t          jI        |8          |j                  }nv# t          $ ri}9|j        r(t'          t          |9          t(          j                   t          jF                            t          jK        |j                  }Y d }9~9nd }9~9ww xY w |jL        d5i d|6d|d |d!|	d"|
d#|d$|r|nd d%|d&|d'|d(|d)|d*|d+|d,|d-|d.|d/|d0|d1|d2|d3|}:|7|7d         d         };t          |;|:|          S t          |:|4          S )6NrL  r`  r   TrJ  r  )r@   r   r   r   r  zFailed to create input chunksr   z%Failed to tokenize input: error code c                     g | ]
}|         S rU   rU   )r   j
tokens_ptrs     rX   r   z/Llava15ChatHandler.__call__.<locals>.<listcomp>U  s    %W%W%Wjm%W%W%WrZ   zPrompt exceeds n_ctx: z > Fz%Failed to evaluate chunk: error code r   r  c                    g | ]}d |dS r  rU   r  s     rX   r   z/Llava15ChatHandler.__call__.<locals>.<listcomp>  r  rZ   r  r  re   r   r  c              3  @   K   | ]}|d          d         k    |V  dS r  rU   r  s     rX   r  z.Llava15ChatHandler.__call__.<locals>.<genexpr>  r  rZ   r  r  r  r  r   r'   r(   r)   r4   r5   r<   r*   r+   r,   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r3   r  rU   )Or  r  ra  r  rM  rS  get_image_urlsr   r   CHAT_FORMATr  mtmd_default_markerdecoder   
detokenize	token_eos	token_bosr  r	  r)  r*  r+  r  r  re  mtmd_input_textr  r   add_specialparse_specialmtmd_input_chunks_initrk   mtmd_bitmap_p_ctypesrO  mtmd_tokenizer  byrefreset_ctxkv_cache_clear	llama_cpp	llama_posmtmd_input_chunks_sizer  mtmd_input_chunks_getmtmd_input_chunk_get_typeMTMD_INPUT_CHUNK_TYPE_TEXTc_size_t mtmd_input_chunk_get_tokens_textvaluen_tokensn_ctxevalMTMD_INPUT_CHUNK_TYPE_IMAGEMTMD_INPUT_CHUNK_TYPE_AUDIOmtmd_input_chunk_get_n_tokensmtmd_helper_eval_chunk_singlectxllama_seq_idn_batch	input_idstolistmtmd_input_chunks_freemtmd_bitmap_freer   r  rf   r!  r"  r#  r$  r%  r&  r'  r(  r,  r-  r   r   )>rV   r>   r@   r#   r$   r%   r&   r'   r(   r)   r4   r5   r*   r+   r,   r-   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r3   r<   r=   rW   system_prompt
image_urlsr   media_markerr   r  bitmapsbitmap_cleanupr  r  
input_textr   bitmap_arrayr.  n_pastn_chunksr   r   
chunk_typen_tokens_outr   chunk_n_tokens
new_n_pastr   r0  r1  r2  r   r   re   r  s>                                                               @@rX   rY   zLlava15ChatHandler.__call__
  s	   P 	&&&}(((+H55B4#>#J>!4+F   	H ((22
0
 
 
 +d&
'
' 	 ~99;;BB7KK "&&e&(9(9':;;&e&(9(9':;;	  
 
 $ 	9 	9I<<	<88DD< 	)$SZ(((( d	8' . .	"ooi8877DDv&&&%%f---- 7799J"kk'22JO%)J"'+J$ ^::<<F~ !@AAAM> R Cc'll RU\]55ML,, LL  Q;;$%UV%U%UVVV 
))+++ #,Q//>@@HHx 1: 1:A N@@KKE} !%!I!I%!P!PJ!T^%NNN'-'8'8%)^%T%T!6<#=#=& &
 & /,*<q*@*@%W%W%W%WU<CU=V=V%W%W%WF$~F;kekmmKK&0$mU^cRXkk=Y$m$m^i^c^i^k^k$m$m'" '" !" 'EJv...#(RTXTbT~')-)U)UV[)\)\ >N:[U[]]JJ", l.9X l l]h]b]h]j]j l l# #  &/%8%;%;
!%!M!M M!JN!%/??%2155!M!"L44	" 	" "Q;;",-]U[-]-]"^"^^ *4)9 )95>)9:AACC 55f====55f===== ) 8 8//77778. 8 8//77778 &?6+Bm+S+S2?CCG   
 !*  E $--- ,''=F+B+B+-.. 6]3J3J&f 5!  #;-- $!z*62DKKKKEKKKTRRD| !L!L!L!LMMM*%l3F
'4EEJv&& F      = 3#a&&sz2222'4@@!+U] A    7u6  
  
  
6 
# 
 % 
 %	 

 % 
  i 
 &.7\\4 
 6 
  
  
 "z 
 .- 
 0/ 
 *> 
 % 
  (-! 
" &# 
$ &% 
& %' 
( .-) 
* G+ 
, "z- 
2 Z(0I7/   ++?OOOOs?   CW
 MV
 /W
 
V&&W
 
!W+8\ 
]:A]55]:c                0   |                      d          r4dd l}|                    |                     d          d                   }|S dd l}|j                            |           5 }|                                }|cd d d            S # 1 swxY w Y   d S )Nzdata:r   ,r   )r  base64	b64decoder  urllib.requestrequesturlopenread)r  r  r  urllibr}   s        rX   r  zLlava15ChatHandler._load_image  s     (( 	#MMM **9??3+?+?+BCCK!!!!''	22 #affhh"# # # # # # # # # # # # # # # # # #s   (BBBc                b   g }| D ]}|d         dk    r|d         |d         D ]}t          |t                    rrd|v rn|d         dk    rbt          |d         t                    r,d|d         v r"|                    |d         d                    n|                    |d                    |S )Nr   r  r   r   r  url)r  r!  re  )r@   r  r   r   s       rX   r  z!Llava15ChatHandler.get_image_urls  s     "
 	H 	HGv&((9%-&y1 	H 	HG!'400 HVw5F5F"6?k99 *7;+? F FH$)W[-A$A$A * 1 1'+2Fu2M N N N N * 1 1'+2F G G GrZ   r   r  	List[str]c                <   dd}g }| }|r |||          \  }}|e|c|dk    r|                     d|d|         f           |                     d	||         f           ||t          ||                   z   d         }n|                     d|f           d
}||S )z8This method is no longer used in the new implementation.srf   substrsr(  c                r    t          |          D ]&\  }}|                     |          }|dk    r||fc S 'dS )Nr   ra  )r   find)r*  r+  r   substrposs        rX   
find_firstz?Llava15ChatHandler.split_text_on_image_urls.<locals>.find_first  sL    &w// " "	6ffVnn"996MMM :rZ   Nr   r   r  rL  )r*  rf   r+  r(  )re  rO  )r   r  r0  
split_text	remainingr/  r   s          rX   split_text_on_image_urlsz+Llava15ChatHandler.split_text_on_image_urls  s    	 	 	 	 FH
	 
	Z	:66FC1=77%%vy#&?@@@!!;
1">???%cC
1,>,>&>&@&@A		!!69"5666	  
	 rZ   r  repo_idfilename	local_dir&Optional[Union[str, os.PathLike[str]]]local_dir_use_symlinksUnion[bool, Literal['auto']]	cache_dirrW   r   'Llava15ChatHandler'c                D   dd l ddlm} 	 ddlm}m}	 ddlm}
 n# t          $ r t          d          w xY w |
|            |	            }d |	                    |          D             }g }|D ]B} ||          
                    |          }|                    t          |                     Cfd|D             }t          |          dk    r*t          d| d	 d
t          j        |                     t          |          dk    r*t          d| d d
t          j        |                     |\  }t           ||          j                  } ||          j         |||t'          t(          t          |d f         |          |t'          t(          t          |d f         |                     |4 |||||t'          t(          t          |d f         |          d          }n t*          j                            |          } | dd|i|S )Nr   )Path)hf_hub_downloadHfFileSystem)validate_repo_idzrLlama.from_pretrained requires the huggingface-hub package. You can install it with `pip install huggingface-hub`.c                L    g | ]!}t          |t                    r|d          n|"S rz   )r  r!  )r   r  s     rX   r   z6Llava15ChatHandler.from_pretrained.<locals>.<listcomp>)  s?     
 
 
 'tT22<DLL
 
 
rZ   c                @    g | ]}                     |          |S rU   )fnmatch)r   r  r5  rC  s     rX   r   z6Llava15ChatHandler.from_pretrained.<locals>.<listcomp>4  s,    XXX4h8W8WX$XXXrZ   zNo file found in z that match z

Available Files:
r   zMultiple files found in z
 matching )r4  r5  	subfolderr6  r8  r:  T)r4  r5  rD  r6  r8  r:  local_files_onlyr  rU   )rC  pathlibr=  huggingface_hubr>  r?  huggingface_hub.utilsr@  ImportErrorlsrelative_tore  rf   rO  rk   r&  r'  parentre   r   r   r  r  r  )clsr4  r5  r6  r8  r:  rW   r=  r>  r?  r@  hffsfiles	file_listr  rel_pathmatching_filesmatching_filerD  
model_pathrC  s     `                 @rX   rB  z"Llava15ChatHandler.from_pretrained  s    	      	EEEEEEEE>>>>>>> 	 	 	I  	 	!!!|~~
 
((
 
 
  "	 	, 	,DtDzz--g66HS]]++++XXXXX9XXX~!##=G = = = =%)Z	%:%:= =  
 ~""97 9 9h 9 9%)Z%6%69 9  
 *]++233	4&&+ 	5dD19==#95dD19==	
 	
 	
 	
 (!##'=uS$_5yAA!%  JJ i::Js 
 
&

 
 	
s    7T)r  rf   r	  rH   )r  r?   )r  rf   rQ   r   )r  r   r4  )r@   rA   )r   rf   r  r(  )Nr  N)r4  rf   r5  rL   r6  r7  r8  r9  r:  r7  rW   r   rQ   r;  )r\   r]   r^   r  rx   r  r   r  r  r  rY   r   r  r  r3  classmethodrB  rU   rZ   rX   r  r  c
  s         	g     
	 F
S 
S 
S 
S 
S!1 !1 !1 !1F+ + + +   . IMQU@DLP 02" $("%#& #!!#@D0415#'&*CEP EP EP EP EP EPN 
# 
# 
# \
#    \$    \0 
 =A?E<@Q
 Q
 Q
 Q
 [Q
 Q
 Q
rZ   r  c                      e Zd ZdZdS )ObsidianChatHandlera<  {% for message in messages %}{% if message.role == 'system' %}<|im_start|>system
{{ message.content }}
###
{% endif %}{% if message.role == 'user' %}<|im_start|>user
{% if message.content is string %}{{ message.content }}{% endif %}{% if message.content is iterable %}{% for content in message.content %}{% if content.type == 'image_url' and content.image_url is string %}{{ content.image_url }}{% endif %}{% if content.type == 'image_url' and content.image_url is mapping %}{{ content.image_url.url }}{% endif %}{% endfor %}{% for content in message.content %}{% if content.type == 'text' %}{{ content.text }}{% endif %}{% endfor %}{% endif %}###
{% endif %}{% if message.role == 'assistant' %}<|im_start|>assistant
{{ message.content }}###
{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}Nr\   r]   r^   r  rU   rZ   rX   rX  rX  d  s        (	 KKrZ   rX  c                      e Zd ZdZdS )MoondreamChatHandlera  {% for message in messages %}{% if message.role == 'user' %}{% if message.content is iterable %}{% for content in message.content %}{% if content.type == 'image_url' %}{% if content.image_url is string %}{{ content.image_url }}

{% endif %}{% if content.image_url is mapping %}{{ content.image_url.url }}

{% endif %}{% endif %}{% endfor %}{% for content in message.content %}{% if content.type == 'text' %}Question: {{ content.text }}

{% endif %}{% endfor %}{% endif %}{% if message.content is string %}Question: {{ message.content }}

{% endif %}{% endif %}{% if message.role == 'assistant' %}Answer:{{ message.content }}

{% endif %}{% endfor %}{% if add_generation_prompt %}Answer:{% endif %}NrY  rU   rZ   rX   r[  r[    s        "	 KKrZ   r[  c                      e Zd ZdZdZdS )Llava16ChatHandlerzA chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. a  {% for message in messages %}{% if message.role == 'system' %}{{ message.content }}{% endif %}{% if message.role == 'user' %}{% if message.content is iterable %}{% for content in message.content %}{% if content.type == 'image_url' %}{% if content.image_url is string %}{{ content.image_url }}
{% endif %}{% if content.image_url is mapping %}{{ content.image_url.url }}
{% endif %}{% endif %}{% endfor %}{% for content in message.content %}{% if content.type == 'text' %}{{ content.text }}{% endif %}{% endfor %}{% endif %}{% if message.content is string %}{{ message.content }}{% endif %}{% endif %}{% if message.role == 'assistant' %}{{ message.content }}{% endif %}{% endfor %}{% if add_generation_prompt %}Answer:{% endif %}Nr\   r]   r^   r  r  rU   rZ   rX   r]  r]    s!         }%	 KKrZ   r]  c                      e Zd ZdZdZdS )NanoLlavaChatHandlerzAnswer the questionaM  {% for message in messages %}{% if message.role == 'system' %}<|im_start|>system
{{ message.content }}<|im_end|>{% endif %}{% if message.role == 'user' %}<|im_start|>user
{% if message.content is string %}{{ message.content }}{% endif %}{% if message.content is iterable %}{% for content in message.content %}{% if content.type == 'image_url' and content.image_url is string %}{{ content.image_url }}{% endif %}{% if content.type == 'image_url' and content.image_url is mapping %}{{ content.image_url.url }}{% endif %}{% endfor %}{% for content in message.content %}{% if content.type == 'text' %}{{ content.text }}{% endif %}{% endfor %}{% endif %}<|im_end|>{% endif %}{% if message.role == 'assistant' %}<|im_start|>assistant
{{ message.content }}<|im_end|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}Nr^  rU   rZ   rX   r`  r`    s          3(	 KKrZ   r`  c                      e Zd ZdZdZdS )Llama3VisionAlphaChatHandlerNa  {% for message in messages %}<|start_header_id|>{% if message.role == 'user' %}user<|end_header_id|>

{% if message.content is iterable %}{% for content in message.content %}{% if content.type == 'image_url' %}{% if content.image_url is string %}{{ content.image_url }}{% endif %}{% if content.image_url is mapping %}{{ content.image_url.url }}{% endif %}{% endif %}{% endfor %}{% for content in message.content %}{% if content.type == 'text' %}{{ content.text }}{% endif %}{% endfor %}{% endif %}{% if message.content is string %}{{ message.content }}{% endif %}{% endif %}{% if message.role == 'assistant' %}assistant<|end_header_id|>

{{ message.content }}{% endif %}<|eot_id|>{% endfor %}{% if add_generation_prompt %}<|start_header_id|>assistant<|end_header_id|>

{% endif %}r^  rU   rZ   rX   rb  rb  -  s          "&	 KKrZ   rb  c                      e Zd ZdZdZdS )MiniCPMv26ChatHandlerr  a  {% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}<|im_start|>system
You are a helpful assistant.<|im_end|>
{% endif %}<|im_start|>{{ message['role'] }}
{% if message['content'] is iterable %}{% for content in message['content'] %}{% if content.type == 'image_url' %}{% if content.image_url is string %}{{ content.image_url }}{% endif %}{% if content.image_url is mapping %}{{ content.image_url.url }}{% endif %}{% endif %}{% endfor %}{% for content in message['content'] %}{% if content.type == 'text' %}{{ content.text }}{% endif %}{% endfor %}{% endif %}{% if message['content'] is string %}{{ message['content'] }}{% endif %}<|im_end|>
{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}Nr^  rU   rZ   rX   rd  rd  b  s        ;	 KKrZ   rd  c                  &     e Zd ZdZdZ fdZ xZS )Qwen25VLChatHandlerr  aO  {% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
{{ self.DEFAULT_SYSTEM_MESSAGE }}<|im_end|>
{% endif %}<|im_start|>{{ message['role'] }}
{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
{% else %}{% for content in message['content'] %}{% if content['type'] == 'image_url' %}{% if content.image_url is string %}{{ content.image_url }}{% else %}{{ content.image_url.url }}{% endif %}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
{% endif %}{% endfor %}<|im_start|>assistant
c                   |d         } |j                      |j                                         d|_        t	          |d          r|j                            d           t	          | d          rd | _        d | _        | j	        rW|
                    dg           }t          |                     |                    }t          d| dt          j                    t!                      j        d	i |S )
Nr>   r   r  _last_image_embedr@   z$Minimal - Cleared state, processing z imagesr  rU   )r  r  r  r  r  r  fillrh  _last_image_hashr	  r  rO  r  r)  r*  r+  superrY   )rV   rW   r>   r@   image_count	__class__s        rX   rY   zQwen25VLChatHandler.__call__  s    w 	
!!###5+&& 	$O  ### 4,-- 	)%)D"$(D!< 	`zz*b11Hd11(;;<<KMMMMTWT^____  uww))&)))rZ   )r\   r]   r^   r  r  rY   __classcell__)rm  s   @rX   rf  rf    sH        ;
	" <* * * * * * * * *rZ   rf  zchatml-function-callingrP   r=   rR   c                  + d}t          t          j        ddg          t          j                                      |          }|d |D             }|It          |t                    r|dk    s|dk    r|}t          |t                    rd|v rd	d|d         id
}t          |t                    r|dgn
|r|dgz   ndg}|0t          |t                    r|dk    s|t          |          dk    r|	                    |g d d          }||d         dk    rt          |          }t           | j        dMi d|d|d|d|d|	d|
d|d|d|d|d|d|d|d|d|d |d!|d"|d#|d$|r|nd |%          S t          |t                    rh|d	         d         +t          +fd&|D             d           }|t          d'+ d(          |	                    ||dd          }|d)+ d*z  }	 t          j                            t%          j        |d	         d+                   | j        ,          }nl# t*          $ r_} t          j                            t          j        | j        ,          }| j        rt/          d-           t/          |            Y d } ~ nd } ~ ww xY w | j        dMi d|d|d|d|d|	d|
d|d|d|d|d|d|d|d|d|d |d!|d"|d#|}!t1          +|!|          S t          |t                    r|dk    sJ d.                    d/ |D                       }"d0|" d1}#d2|" d1}$|	                    ||dd          } | j        dMi d|ddd|d|d|	d|
dd3dd4gdd d|d|d|d|d|d|d |d!|d"|d#t          j                            |#| j        ,          }!|!}%|%d5         d         d6         }&d7|&v rt           | j        dMi d|d8z   d|d|d|d|	d|
d|ddgd$|r|nd dd d|d|d|d|d|d|d |d!|d"|d#t          j                            |$| j        ,          |%          S |&t          d)          d          +t          +fd9|D             d           }|sg }'g }(|)|d)+ d*z  }	 t          j                            t%          j        |d	         d+                   | j        ,          }nl# t*          $ r_} t          j                            t          j        | j        ,          }| j        rt/          d-           t/          |            Y d } ~ nd } ~ ww xY w | j        dMi d|d|d|d|d|	d|
dd3d|dd d|d|d|d|d|d|d |d!|d"|d#|}!t5          t6          j        |!          }!|'                    |!           |(                    +           ||!d5         d         d6         z  }|d1z  } | j        dMi d|d|d|d|d|	d|
dd3d|dd d|d|d|d|d|d|d |d!|d"|d#t          j                            |$| j        ,          })t5          t6          j        |)          })|)d5         d         d6         t          d)          d          +t          +fd:|D             d           }|)t          |'          d;k    rd<+|'d         d5         d         d6         d=ini }*d>|%d?         z   d@|%dA         |%d!         dBdt=          |%d5         d         d$                   dCd dD t?          tA          |(|'                    D             dE|*dFgtC          dG |'D                       tC          dH |'D                       tC          dI |'D                       dJdKS t          dL          )NNa  {% for message in messages %}<|im_start|>{{ message.role }}
{% if message.role == 'system' %}{{ message.content }}{% if tool_calls %}

You have access to the following functions:
{% for tool in tools %}
functions.{{ tool.function.name }}:
{{ tool.function.parameters | tojson }}
{% endfor %}

You can respond to users messages with either a single message or one or more function calls.

To respond with a message begin the message with 'message:', use the following format:

message:
<message>

To respond with one or more function calls begin the message with 'functions.<function_name>:', use the following format:

functions.<function_name>:
{ "arg1": "value1", "arg2": "value2" }
functions.<function_name>:
{ "arg1": "value1", "arg2": "value2" }{% endif %}<|im_end|>
{% endif %}{% if message.role == 'user' %}{{ message.content }}<|im_end|>
{% endif %}{% if message.role == 'assistant' %}{% if message.content and message.content | length > 0 %}{% if tool_calls %}message:
{% endif %}{{ message.content }}<|im_end|>
{% endif %}{% if 'tool_calls' in message %}{% for tool_call in message.tool_calls %}functions.{{ tool_call.function.name }}:
{{ tool_call.function.arguments }}{% endfor %}<|im_end|>
{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}htmlxml)
autoescape	undefinedc                    g | ]}d |dS r  rU   r  s     rX   r   z+chatml_function_calling.<locals>.<listcomp>  s7     
 
 

  #$ 
 
 
rZ   r  r  re   r   r  r   r   T)r@   r%   r   r   r   r  r   r'   r(   r)   r4   r5   r*   r+   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r<   r  c              3  @   K   | ]}|d          d         k    |V  dS r  rU   r   r0  r   s     rX   r  z*chatml_function_calling.<locals>.<genexpr>i  s7      MMdtJ'7'?9'L'LT'L'L'L'LMMrZ   zTool with name 'z' not found in toolsrH  rL  r  r  r\  r  c                2    g | ]}d |d         d          dS )z"functions.r   re   z:"rU   rG  s     rX   r   z+chatml_function_calling.<locals>.<listcomp>  s.    JJJD	7j)&1	7	7	7JJJrZ   z0root   ::= functions | "message:"
functions ::= r  z2root   ::= functions | "<|im_end|>"
functions ::= Fru  r   r   r   z	message:
c              3  @   K   | ]}|d          d         k    |V  dS r  rU   rv  s     rX   r  z*chatml_function_calling.<locals>.<genexpr>  7      QQ$4
+;F+Cy+P+P+P+P+P+PQQrZ   c              3  @   K   | ]}|d          d         k    |V  dS r  rU   rv  s     rX   r  z*chatml_function_calling.<locals>.<genexpr>0  ry  rZ   r   r$   r   r   r   r   r   r   r   c                |    g | ]9\  }\  }}d d| dz   |z   dz   |d         z   d||d         d         d         dd:S )	r  r   r   r   r   r   r   r   r   rU   )r   r   r   r   s       rX   r   z+chatml_function_calling.<locals>.<listcomp>Q  s     ' ' ' !;#:Iz '."*a((('+"+', #&'& #-T"2	'3
 )3,51;I1Fq1I&1Q-" -" ' ' 'rZ   )r   r   r   )r   r   r<   r   c              3  >   K   | ]}d |v r|d          d         ndV  dS )r   r  r   NrU   r   r   s     rX   r  z*chatml_function_calling.<locals>.<genexpr>g  sV       ) ) # #j00 #7+,?@@	) ) ) ) ) )rZ   c              3  >   K   | ]}d |v r|d          d         ndV  dS )r   prompt_tokensr   NrU   r}  s     rX   r  z*chatml_function_calling.<locals>.<genexpr>o  sQ       % %" =Dz<Q<QJw'88WX% % % % % %rZ   c              3  >   K   | ]}d |v r|d          d         ndV  dS )r   total_tokensr   NrU   r}  s     rX   r  z*chatml_function_calling.<locals>.<genexpr>s  sQ       $ $" <Cj;P;PJw'77VW$ $ $ $ $ $rZ   )r  r  r  r   z0Automatic streaming tool choice is not supportedrU   )"r   r   select_autoescapeStrictUndefinedr   r  rf   r!  rO  r   r   r   r-  r"  rk   r#  r$  r%  r&  r'  r	  r(  r,  r)  r   r  r   rM  CreateCompletionResponsere  r   r   r   sum),r>   r@   r#   r$   r%   r&   r'   r(   r)   r4   r5   r*   r+   r-   r.   r/   r0   r1   r6   r7   r8   r9   r2   r:   r;   r<   r=   rW   function_calling_templatetemplate_rendererr   r0  r2  r   function_namesinitial_gbnf_tool_grammarfollow_up_gbnf_tool_grammarr   r   completionscompletions_tool_nameresponser  r   s,                                              @rX   chatml_function_callingr    sW   F0	K f 6+VUO<<(   k+,,  
 

 &
 
 
  mS)) 	(V##}'>'>'KmT** 	v/F/F"M&1 K dC  	?|&*>T\N"" 	 	{C(( 	-8F-B-B=u::??"))"&	 * 
 
 &?6+Bm+S+S2?CCG*#E#   v'K e e	
 e $) v T &: "2!1 #4"3  .~ e ,m *\  *\!" e#$ "2!1%&  '( *2;t), /
 
 
 	
6 +t$$ 2

+F3	MMMMeMMMt
 
 <O	OOOPPP"))"&	 * 
 
 	-y----	#0AA
4
+L9::EM B  GG  	 	 	#0<<' =  G } c   a	  7u6  
  
  
6 
# 
 % 
 %	 

 % 
  i 
 6 
  
 "z 
 .- 
 0/ 
 *> 
 % 
 (- 
 & 
  &! 
" %# 
$ .-% 
& G' 
* 4+V
 
 	

 k3''AK6,A,A,AAZZJJEJJJ N	0)	0 	0 	0 
	0)	0 	0 	0   %%"	 &  F 352   vA e e	
 e ) u UU 4 *) ,+ &~ e $m "\  "\!" e#$ *)%& *66%u} 7 
 
 
'. 8LJi #F+DD*#E#   ,,'K e e	
 e $) v #^^ *2;t  4 "2!1 #4"3  .~ e ,m  *\!" *\#$ e%& "2!1'( &2>>/ ?   )0 3
 
 
 	
: S&&(()IQQQQ%QQQSWXXD R
BD+-191111F'4EEJtJ/=>> F      '4@@!+U] A   = g   !HHH $;5#: $ $ $v$'K$ e$ e	$
 e$ $)$ u$ T$  4$ "2!1$ #4"3$  .~$ e$ ,m$ *\$  *\!$" e#$$ "2!1%$&  '$ * $(46J$ $  3444!((333*95a8@@FdNF.u.   v'K e e	
 e $) u T  4 "2!1 #4"3  .~ e ,m *\  *\!" e#$ "2!1%& &2>>/ ?   'H. K@(KKH +A.v6s<7H7H7J7JKIQQQQ%QQQSW DM p ;1$$  %!,Q	!:1!=f!E" "   	" :d++'!),( &2 I*U^J_`aJbcmJn o o +#'' ' ?H #$9; G G? ?' ' '   ( -) 	 < &) ) ) '2) ) ) & & "% % %&1% % % " " !$ $ $&1$ $ $ ! ! E4
 4
 4	
l G
H
HHs4   =AI 
J+AJ&&J+0AS5 5
U?AUUrw   rv   )r<   r   rQ   r   )r   r   rQ   r   )r   r   rQ   r   ru   )r   r   r*   rH   rQ   r   )r   rf   r   r   r*   rH   )r   r   rQ   r   )r5  r6  rQ   r   )r5  r6  rQ   r   rU  )rF  rG  r   rH   rQ   r   )rF  rG  r   rH   rQ   r   )rT  rU  rQ   rL   )r@   rA   rQ   rf   )r@   rA   rb  rU  rQ   rc  )
rh  rf   r@   rc  ri  rf   rj  rf   rQ   rf   )rh  rf   r@   rc  ri  rf   rQ   rf   )r	  rH   )FT)r1  rf   r	  rH   r  rH   )r-   r  r	  rH   r   )r@   r  rW   r   rQ   r   )4r>   r?   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   rF   r)   rG   r4   rF   r5   rF   r*   rH   r+   rI   r-   rK   r.   rJ   r/   rF   r0   rF   r1   rF   r6   rF   r7   rG   r8   rF   r9   rF   r2   rL   r:   rN   r;   rO   rQ   r  )8r>   r?   r@   rA   r#   rB   r$   rC   r%   rD   r&   rE   r'   rF   r(   rF   r)   rG   r4   rF   r5   rF   r*   rH   r+   rI   r-   rK   r.   rJ   r/   rF   r0   rF   r1   rF   r6   rF   r7   rG   r8   rF   r9   rF   r2   rL   r:   rN   r;   rO   r<   rP   r=   rJ   rQ   rR   )x
__future__r   r  r*  r&  r  dataclassesr  r  r   
contextlibr   typingr   r   r   r	   r
   r   r   r   r   r   r   jinja2.sandboxr   numpynpnumpy.typingnptllama_cpp.llama_cppr  llama_cpp.llamar>   llama_cpp.llama_typesrM  llama_cpp.llama_grammarr#  _loggerr   _utilsr   r   rZ  CHATML_BOS_TOKENCHATML_EOS_TOKENr[  MISTRAL_INSTRUCT_BOS_TOKENMISTRAL_INSTRUCT_EOS_TOKENr\  r]  r   r(  ra   rc   r{   rl   	dataclassr   r   r   r   r   r   r   r   r   rC  rE  rQ  rS  r^  ra  rg  rr  rw  ry  r{  r}  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  ri  r  r  rX  r[  r]  r`  rb  Llama3VisionAlphard  rf  r  rU   rZ   rX   <module>r     sa   " " " " " " 				 



                                                   8 8 8 8 8 8           ' ' ' ' ' '       + + + + + + / / / / / /       5 5 5 5 5 5 5 5
 a    "A " #  "z  !l 
0 0 0 0 0 0 0 0f	 	 	 	 	) 	 	 	       B       
  
  
  
  
  
  
  
 $ $ $ $ $H $ $ $H? H? H? H? H?- H? H? H?V   2   0(
 (
 (
 (
` < < < < <"n; n; n; n;bR# R# R# R#j       2E E E E #'-# -# -# -# -#d #'E E E E E   .          "
 
 
 
   
 
 
 

 
 
 

 
 
 
        BF	 	 	 	 	 2 2 2 2 2$    i  1 1 1 ! 1  i  < < < ! <  h
1 
1 
1  
1 f= = = =  h1 1 1  1 m$$1 1 1 %$1 l##1 1 1 $#1 j!!1 1 1 "!1 k""1 1 1 #"1* ())= = = *)= h= = =  =* g
1 
1 
1 
1 g
1 
1 
1 
1 k""@ @ @ #"@4 m$$1 1 1 %$1 h< < <  <  k""< < < #"< h< < <  <  ()); ; ; *);( j!!< < < "!<  j!!< < < "!<& g9 9 9 9* g< < < <( "!-00 EIMQ<@HL,.QU $!"<@,03b b b b 10bJ "!"233!!"233 EIMQ<@HL,.QU $!"<@,03
 
 
 
 43 43
D~
 ~
 ~
 ~
 ~
 ~
 ~
 ~
B5 5 5 5 5, 5 5 5p' ' ' ' '- ' ' 'T- - - - -+ - - -`4 4 4 4 4- 4 4 4n. . . . .#5 . . .d 1 # # # # #. # # #L7* 7* 7* 7* 7*, 7* 7* 7*t "!";<< EIMQ<@HL,.QU $!"<@,0#"&7wI wI wI wI =<wI wI wIrZ   