
    *`icK              
       ,   d dl Z d dlmZmZmZ d dlmZmZmZ d dl	m
Z
mZmZmZmZmZmZmZmZmZmZ d dlmZmZ d dlmZmZmZ d dlmZmZ dZ G d	 d
eeeeeef                   Z  G d de           Z! G d de!          Z"dede fdZ#dede fdZ$dS )    N)GenericSequenceoverload)	TextChunk
ThinkChunkUserContentChunk)UATSAssistantMessageAssistantMessageTypeFinetuningAssistantMessageRolesSystemMessageSystemMessageTypeToolMessageToolMessageTypeUserMessageUserMessageType)ChatCompletionRequestInstructRequest)FunctionCallToolToolCall)InstructRequestTypeTokenizerVersionz

c                      e Zd ZU dZdZeed<   dZeed<   dee	         dee
         dee         dee         d	ee         f
d
Zed"d            Zdedz  defdZedeeez  ez           deeeez           z  fd            Zededefd            Zedee         defd            Zdeeeez  ez           z  ee         z  deeeez           z  fdZdee         dedz  fdZdee         dee         dee         fdZdedefdZdee         dee         fdZdee         de
fdZdee         de	fdZdee         de dz  dee         de!e         fdZ"dee         dee         fdZ#d e$e         defd!Z%dS )#InstructRequestNormalizera  Takes a [ChatCompletionRequest][mistral_common.protocol.instruct.request.ChatCompletionRequest] and normalizes
    it into an [InstructRequest][mistral_common.tokens.instruct.request.InstructRequest].

    The normalization process does several things such as:
    - Aggregate consecutive messages of the same role
    - Aggregate system prompts
    - Normalize json content
    - Normalize tool calls

    Examples:
        >>> normalizer = InstructRequestNormalizer.normalizer()
    F_system_prompt_in_begin_allow_tool_call_and_contentuser_message_classassistant_message_classtool_message_classsystem_message_classinstruct_request_classc                 L    || _         || _        || _        || _        || _        dS )a  Initializes the normalizer with the appropriate message classes.

        Args:
           user_message_class: The class for user messages.
           assistant_message_class: The class for assistant messages.
           tool_message_class: The class for tool messages.
           system_message_class: The class for system messages.
           instruct_request_class: The class for instruct requests.
        N)_user_message_class_assistant_message_class_tool_message_class_instruct_request_class_system_message_class)selfr   r    r!   r"   r#   s         ~/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/mistral_common/protocol/instruct/normalize.py__init__z"InstructRequestNormalizer.__init__0   s2    " $6 (?%#5 '=$%9"""    returnc            	      ~    t          t          t          t          t          t
          t          t          f                   S )zReturns a normalizer for the default instruct request.

        Examples:
            >>> normalizer = InstructRequestNormalizer.normalizer()
        )r   r   r
   r   r   r   r	   r    r-   r+   
normalizerz$InstructRequestNormalizer.normalizerH   s.     )D$J'
 
 	
r-   contentNc                     |t          |          dk    rdS 	 t          j        |          }t          j        |d          }n# t          j        $ r |}Y nw xY w|S )Nr   z{}F)ensure_ascii)lenjsonloadsdumpsJSONDecodeError)r*   r2   parsed_jsonnormalized_contents       r+   _normalize_json_contentz1InstructRequestNormalizer._normalize_json_contentW   sx    ?c'lla//4	)*W--K!%Ke!L!L!L# 	) 	) 	)!(	)!!s   *A AAc                     d S Nr0   r*   r2   s     r+   _aggregate_content_chunksz3InstructRequestNormalizer._aggregate_content_chunksb   s	     .1Sr-   c                     d S r>   r0   r?   s     r+   r@   z3InstructRequestNormalizer._aggregate_content_chunksf   s    >Acr-   c                     d S r>   r0   r?   s     r+   r@   z3InstructRequestNormalizer._aggregate_content_chunksh   s    DGCr-   c                    t          |t                    r|S t          |t                    sJ dt          |                       g }|D ]}t          |t                    rt	          |          }t          |t                    rW|r?t          |d         t                    r$|d         xj        t          |j        z   z  c_        }|                    |           t          |t                    r|                    |           t          dt          |                     t          |          dk    r(t          |d         t                    r|d         j        S |S )NzExpected list, got textzUnsupported chunk type    r   )
isinstancestrlisttyper   rE   CHUNK_JOIN_STRappendr   
ValueErrorr5   )r*   r2   aggregated_contentchunks       r+   r@   z3InstructRequestNormalizer._aggregate_content_chunksj   sq    gs## 	N'4((OO*OW*O*OOO(;= 	J 	JE%%% .!u---%++ 	J% 5*5G5KY*W*W 5&r*//>EJ3NN///&--e4444E:.. J"))%0000 !H4;;!H!HIII!""a''J7I!7Li,X,X'%a(--!!r-   messagesc                     g }|D ]M}|j         t          j        k    r6|j        r/|                     |j                  }|                    |           Nt          |          rt                              |          nd S r>   )	roler   systemr2   r@   rM   r5   rL   join)r*   rQ   system_promptmessagerO   s        r+   _aggregate_system_promptsz3InstructRequestNormalizer._aggregate_system_prompts   s~    #% 	9 	9G|u|+++%)%C%CGO%T%T"$$%7888585G5GQ~""=111TQr-   latest_call_idsc                 f   g }|D ]}t          || j                  s
J d            |j        }t          |t                    s$t                              d |D                       }|                     |          }|                    |                     ||j        |j	                             |S )zm
        We currently do not do any aggregation for tool messages, but we normalize the json content
        zExpected tool messagec                     g | ]	}|j         
S r0   rD   ).0rP   s     r+   
<listcomp>zFInstructRequestNormalizer._aggregate_tool_messages.<locals>.<listcomp>   s    .O.O.Oeuz.O.O.Or-   )r2   tool_call_idname)
rH   r'   r2   rI   rL   rU   r<   rM   r^   r_   )r*   rQ   rY   tool_messagesrW   r2   r;   s          r+   _aggregate_tool_messagesz2InstructRequestNormalizer._aggregate_tool_messages   s     02 
	 
	Ggt'?@@YYBYYY@oGgs++ Q(--.O.Ow.O.O.OPP!%!=!=g!F!F  ((.W=QX_Xd )      r-   	tool_callc                     |                      |j        j                  }t          t	          |j        j        |          |j                  S )N)r_   	arguments)functionid)r<   re   rd   r   r   r_   rf   )r*   rb   normalized_function_arumentss      r+   _normalize_tool_callz.InstructRequestNormalizer._normalize_tool_call   sO    '+'C'CIDVD`'a'a$!y'9'>Jfggg|
 
 
 	
r-   c                     g S r>   r0   r*   rQ   s     r+   _aggregate_system_messagesz4InstructRequestNormalizer._aggregate_system_messages   s    	r-   c                    g }g }d}d }|D ]}t          || j                  s
J d            | j        s |j        r|j        rt          d|           |j        r4|j        D ],}|                     |          }|                    |           -|j        x}	-|                    t          |	t                    r|	gn|	           ||j
        z  }t          |t                    r|||j        k    s
J d            |j        }|r|                     |          }
nd }
|                     |
|pd |          }|t          |d          r||_        |S )NFzExpected assistant messagezDTool calls and content cannot be used together in the same message. zEExpected weights of aggregated FinetuningAssistantMessage to be equal)r2   
tool_callsprefixweight)rH   r&   r   rm   r2   rN   rh   rM   extendrI   rn   r   ro   r@   hasattr)r*   rQ   messages_contentsrm   rn   ro   rW   rb   normalized_tool_callr2   rO   aggregated_messages               r+   _aggregate_assistant_messagesz7InstructRequestNormalizer._aggregate_assistant_messages   s   @B%'
# 	( 	(Ggt'DEEccGcccE4 s':L sQXQ` s !qho!q!qrrr! <!(!3 < <I+/+D+DY+O+O(%%&:;;;;"?*7!((j#6N6N)['T[\\\gn$F'#=>> (%!W^333_ 433 ! 	&!%!?!?@Q!R!R!%!::&!)T ; 
 
 '*<h"G"G(.%!!r-   c           	         g }g }|D ]}t          || j                  sJ dt          |                       t          |j        t                    r|                    |j                   f|j        D ]}t          |t                    r|                    |j                   2|r=|                    t          t          	                    |                               g }|                    |           |rt          	                    |          nd}|s|                     |          S |r#|                    t          |                     |                     |          S )z:
        Just coalesce neighboring blocks of text
        zExpected user message got rD    r2   )
rH   r%   rK   r2   rI   rM   r   rE   rL   rU   )r*   rQ   all_contenttext_chunksrW   rP   text_contents          r+   _aggregate_user_messagesz2InstructRequestNormalizer._aggregate_user_messages   s    /1!# 	2 	2Ggt'?@@nnBn_cdk_l_lBnBnnn@'/3// 
2""7?3333$_ 2 2E!%33 2#**5:6666& -'..yn>Q>QR]>^>^/_/_/_```*,K#**511112 <GN~**;777B 	B++L+AAA 	=yl;;;<<<'''<<<r-   rS   c                    |t           j        k    r|                     ||          S |t           j        k    r|                     |          gS |t           j        k    r|                     |          gS |                     |          S r>   )r   toolra   	assistantru   userr|   rk   r*   rQ   rS   rY   s       r+   _aggregate_rolez)InstructRequestNormalizer._aggregate_role   s    5:00?KKKU_$$66x@@AAUZ11(;;<<228<<<r-   c                 @   g }g }d }d }g }|D ]}t          |dd           }||j        k    s||k    r|                    |                     |||                     |t          j        k    rQ|d         }	t          |d         t                    sJ |	j        $|	j        D ]}
|	                    |
j
                   n$|t          j        k    r|                                 |                                 |}|j        }|	                    |           |                    |                     |||                     t          |          dk    s"| j        sE|d         j        t          j        k    r*|                    d|                     d                     |S )Nro   rF   r   rw   rx   )getattrrS   rp   r   r   r   rH   r
   rm   rM   rf   r~   clearr5   r   r   insertr%   )r*   rQ   aggregated_messagesmessages_to_aggregatecurrent_rolecurrent_weightrY   rW   
new_weightassistant_messagerb   s              r+   _aggregate_messagesz-InstructRequestNormalizer._aggregate_messages  s   *,,.%)'+%'   	2 	2G (D99Jw|++
n0L0L#**4+?+?@UWcet+u+uvvv5?22(;B(?%%&9"&=?OPPPPP(3?):)E A AI+229<@@@@!UZ//#))+++%++---'N"<L!((1111 	""4#7#78M|]l#m#mnnn "##q((, )1DQ1G1LPUPZ1Z1Z&&q$*B*B2*B*N*NOOO""r-   requestc                     |                      |j                  }|                     |j                  }|                     |||j        |j                  S )a  Converts a chat completion request to an instruct request.

        Args:
            request: The chat completion request to convert.

        Returns:
            The converted instruct request.

        Examples:
            >>> from mistral_common.protocol.instruct.messages import UserMessage, AssistantMessage
            >>> request = ChatCompletionRequest(
            ...     messages=[
            ...         UserMessage(content="Hello"),
            ...         AssistantMessage(content="Hi"),
            ...     ],
            ... )
            >>> normalizer = InstructRequestNormalizer.normalizer()
            >>> instruct_request = normalizer.from_chat_completion_request(request)
        )rQ   rV   available_toolscontinue_final_message)rX   rQ   r   r(   toolsr   )r*   r   rV   rQ   s       r+   from_chat_completion_requestz6InstructRequestNormalizer.from_chat_completion_request-  s^    ( 66w7GHH++G,<==++'#M#*#A	 , 
 
 	
r-   )r.   r   )&__name__
__module____qualname____doc__r   bool__annotations__r   rK   r   r   r   r   r   r,   staticmethodr1   rI   r<   r   rJ   r   r   r@   r	   rX   ra   r   rh   rk   ru   r|   r   r   r   r   r   r   r0   r-   r+   r   r      sl          %*T)))). $...: 1: "&&:!;: !1	:
 ##45: !%%8 9: : : :0 
 
 
 \
	"sTz 	"c 	" 	" 	" 	" 1C)Oj891	tI
*+	+1 1 1 X1 AAAAA XAGcGsGGG XG"T#	/J">??$s)K"	tI
*+	+" " " "8R$t* Rt R R R Rd dSVi \`ap\q    &
h 
8 
 
 
 
4: $GXBY    +"d4j +"EY +" +" +" +"Z=d = = = = =@=T
 =%$, =Y]^aYb =goptgu = = = ='#DJ '#4: '# '# '# '#R
4I$4O 
Tg 
 
 
 
 
 
r-   r   c            	           e Zd ZU dZdZeed<   dZeed<   edd            Z	de
e         de
e         fdZde
e         d	ed
z  de
e         dee         fdZde
e         ded
z  fdZdee         defdZd
S )InstructRequestNormalizerV7zvNormalizer for the v7 tokenizer.

    Examples:
        >>> normalizer = InstructRequestNormalizerV7.normalizer()
    Tr   r   r.   c            	      ~    t          t          t          t          t          t
          t          t          f                   S )zReturns a normalizer for the default instruct request

        Examples:
            >>> normalizer = InstructRequestNormalizerV7.normalizer()
        )r   r   r
   r   r   r   r	   r   r0   r-   r+   r1   z&InstructRequestNormalizerV7.normalizerV  s.     +D$J'
 
 	
r-   rQ   c                        fd|D             S )Nc                     g | ]E}t          |j                                                          |j                             FS )rx   )rH   r)   r@   r2   )r\   rW   r*   s     r+   r]   zJInstructRequestNormalizerV7._aggregate_system_messages.<locals>.<listcomp>f  s]     
 
 
'4#=>>
&&t/M/Mgo/^/^&__
 
 
r-   r0   rj   s   ` r+   rk   z6InstructRequestNormalizerV7._aggregate_system_messagese  s.    
 
 
 
#
 
 
 	
r-   rS   NrY   c                 b   |t           j        k    r|                     ||          S |t           j        k    r|                     |          gS |t           j        k    r|                     |          gS |t           j        k    r|                     |          S |t          |          dk    sJ g S )Nr   )
r   r~   ra   r   ru   r   r|   rT   rk   r5   r   s       r+   r   z+InstructRequestNormalizerV7._aggregate_rolel  s    5:00?KKKU_$$66x@@AAUZ11(;;<<U\!!228<<<<CMMQ$6$6$66Ir-   c                      t          d          )Nz&We should not aggregate system prompts)NotImplementedErrorrj   s     r+   rX   z5InstructRequestNormalizerV7._aggregate_system_promptsy  s    !"JKKKr-   r   c                 p    |                      |j                  }|                     |d|j                  S )a  Converts a chat completion request to an instruct request.

        Args:
            request: The chat completion request to convert.

        Returns:
            The converted instruct request.

        Examples:
            >>> from mistral_common.protocol.instruct.messages import UserMessage, AssistantMessage
            >>> request = ChatCompletionRequest(
            ...     messages=[
            ...         UserMessage(content="Hello"),
            ...         AssistantMessage(content="Hi"),
            ...     ],
            ... )
            >>> normalizer = InstructRequestNormalizerV7.normalizer()
            >>> instruct_request = normalizer.from_chat_completion_request(request)
        N)rQ   rV   r   )r   rQ   r(   r   )r*   r   rQ   s      r+   r   z8InstructRequestNormalizerV7.from_chat_completion_request|  s9    ( ++G,<==++XTcjcp+qqqr-   )r.   r   )r   r   r   r   r   r   r   r   r   r1   rJ   r	   r   rk   r   rI   r   r   rX   r   r   r   r0   r-   r+   r   r   L  s5          %)T((()- $---
 
 
 \

4: 
$GXBY 
 
 
 
T
 %$, Y]^aYb goptgu    L$t* Lt L L L Lr4I$4O rTg r r r r r rr-   r   c                   l     e Zd ZdZedd            Zdee         dee         dee	         f fdZ
 xZS )InstructRequestNormalizerV13zNormalizer for the v13 tokenizer.

    It reorders tool messages based on the tool call order.

    Examples:
        >>> normalizer = InstructRequestNormalizerV13.normalizer()
    r.   c            	      ~    t          t          t          t          t          t
          t          t          f                   S )z6Returns a normalizer for the default instruct request.)r   r   r
   r   r   r   r	   r   r0   r-   r+   r1   z'InstructRequestNormalizerV13.normalizer  s.     ,D$J'
 
 	
r-   rQ   rY   c                     t                                          ||          }d t          |          D             d t          |          D             |                    fd           |S )Nc                     i | ]\  }}||	S r0   r0   )r\   idxcall_ids      r+   
<dictcomp>zIInstructRequestNormalizerV13._aggregate_tool_messages.<locals>.<dictcomp>  s    [[[Ww[[[r-   c                 $    i | ]\  }}|j         |S r0   )r^   )r\   r   rW   s      r+   r   zIInstructRequestNormalizerV13._aggregate_tool_messages.<locals>.<dictcomp>  s!     h h h|sG!5s h h hr-   c                 r                         | j        pdt          d                    | j                 fS )Nnullinf)getr^   float)msgid_to_tool_call_idxid_to_tool_result_idxs    r+   <lambda>zGInstructRequestNormalizerV13._aggregate_tool_messages.<locals>.<lambda>  s6    #''(8(BFE%LLQQ%c&67 r-   )key)superra   	enumeratesort)r*   rQ   rY   r`   r   r   	__class__s       @@r+   ra   z5InstructRequestNormalizerV13._aggregate_tool_messages  s    /4ww/O/OPXZi/j/j[[	/@Z@Z[[[ h hyYfOgOg h h h     	 	
 	
 	
 r-   )r.   r   )r   r   r   r   r   r1   rJ   r	   rI   r   ra   __classcell__)r   s   @r+   r   r     s          
 
 
 \
d dSVi \`ap\q          r-   r   versionr.   c                 Z   | t           j        t           j        t           j        hv rt                                          S | t           j        t           j        hv rt                                          S | t           j	        k    rt                                          S t          d|            )aB  Gets the appropriate normalizer for the given tokenizer version.

    Args:
        version: The tokenizer version to get the normalizer for.

    Returns:
        The appropriate normalizer for the given tokenizer version.

    Examples:
        >>> normalizer = normalizer_for_tokenizer_version(TokenizerVersion.v1)
    zUnknown tokenizer version )r   v1v2v3r   r1   v7v11r   v13r   rN   )r   s    r+    normalizer_for_tokenizer_versionr     s     #&(8(;=M=PQQQ(33555	%(*:*>?	?	?*55777	$(	(	(+66888
;';;
<
<<r-   c                     | t           j        k    rt          }nB| t           j        k    rt          }n*| t           j        k    rt          }nt          d|            |                                S )NzUnsupported tokenizer version: )	r   r   r   r   r   r   r   rN   r1   )r   normalizer_clss     r+   get_normalizerr     sm    "%%%2	$'	'	'4	$(	(	(5D7DDEEE$$&&&r-   )%r6   typingr   r   r   &mistral_common.protocol.instruct.chunkr   r   r   )mistral_common.protocol.instruct.messagesr	   r
   r   r   r   r   r   r   r   r   r   (mistral_common.protocol.instruct.requestr   r   +mistral_common.protocol.instruct.tool_callsr   r   r   %mistral_common.tokens.tokenizers.baser   r   rL   r   r   r   r   r   r0   r-   r+   <module>r      s7    . . . . . . . . . .         
                          \ [ [ [ [ [ [ [ T T T T T T T T T T W W W W W W W Wl
 l
 l
 l
 l
O1?DUWjjkl
 l
 l
^	Er Er Er Er Er"; Er Er ErP    #>   D=.> =C\ = = = =*
', 
'1J 
' 
' 
' 
' 
' 
'r-   