*`iD NddlZddlmZddlmZmZmZmZddlZ ddl m Z ddl m Z mZmZmZddlmZddlmZmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,m-Z-dd l.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7ddl8m9Z9ddl:m;Z;Gdde1ee0e/e5efZ<GddeGdde>ee0e/e5efZ?Gdde?Z@Gdde@ZAGddeAZBdS)N)abstractmethod)AnyGenericSequenceoverload)Audio) InvalidAssistantMessageException InvalidMessageStructureExceptionInvalidRequestExceptionTokenizerException) FIMRequest) AudioChunk AudioURLChunk ContentChunk ImageChunk ImageURLChunk TextChunk ThinkChunkUserContentChunk)UATSAssistantMessageAssistantMessageType SystemMessage ToolMessage UserMessage)InstructRequest)ToolToolCall) StreamingModeTranscriptionRequest) AudioEncoderTranscriptionFormat) FIMRequestTypeInstructRequestTypeInstructTokenizerSpecialTokenPolicy SpecialTokens Tokenized TokenizedType TokenizerUserMessagePosition) ImageEncoder) Tekkenizerc eZdZdZ ddededzdedzffd ZededzfdZ de e fd Z e d edee e ffd Zed ed ede e fdZed ed edede e fdZedede e fdZde e e dzde ede de ddf dZede eddfdZd eeefdefdZdde e de dzde!fdZ"de e de!fdZ#xZ$S) InstructTokenizerBasezBase instruct tokenizer.N tokenizer image_encoder audio_encodercx||_||_||_t|||dS)zInitialize the instruct tokenizer. Args: tokenizer: The tokenizer to use. image_encoder: The image encoder to use if any. audio_encoder: The audio encoder to use. N)r0r1r2super__init__selfr0r1r2 __class__s }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/instruct.pyr5zInstructTokenizerBase.__init__9s>#** M=AAAAAreturnc|jSN)r1r7s r9 mm_encoderz InstructTokenizerBase.mm_encoderKs !!r:c|jjgS)zReturn the start tokens.)r0bos_idr>s r9startzInstructTokenizerBase.startRs%&&r:requestcd}d}tt|jD]$\}}t|tr |dkr|}|}%||fS)zFind the first and last user message in the request. Args: request: The request to search for user messages. Returns: The index of the first and last user message. )list enumeratemessages isinstancer)rC last_user_idxfirst_user_idximsgs r9find_first_last_userz*InstructTokenizerBase.find_first_last_userVsg 9W%56677 " "FAs#{++ "!R''%&N ! },,r:messageis_before_last_user_messagec td)zEncode a tool message. Raises: NotImplementedError: The tool message is not implemented for the base tokenizer. zTool message not implementedNotImplementedErrorr7rOrPs r9encode_tool_messagez)InstructTokenizerBase.encode_tool_messageis""@AAAr:continue_messagec td)zEncode an assistant message. Raises: NotImplementedError: The assistant message is not implemented for the base tokenizer. z!Assistant message not implementedrR)r7rOrPrVs r9encode_assistant_messagez.InstructTokenizerBase.encode_assistant_messagers""EFFFr:chunkc td)zEncode a think chunk. Raises: NotImplementedError: The think chunk is not implemented for the base tokenizer. zThink chunk not implementedrRr7rYs r9 encode_thinkz"InstructTokenizerBase.encode_think}s""?@@@r: tokenizedrH max_tokenslast_user_message_indexcdSr=)r7r]rHr^r_s r9_truncate_for_max_tokensz.InstructTokenizerBase._truncate_for_max_tokenss  r:cdSr=raclsrHs r9validate_messagesz'InstructTokenizerBase.validate_messagess  r:c 4g}g}d}g}||j||\}}t|jD]\}} |jr?|t |jdz kr$t | tstdt | tr\| | |j ||k||k|j d\} } } | | | | nt | tr|| ||k} nt | tr\|jo|t |jdz k} || ||k| } |t |jdz kr| }nJt | t"r|| } nt'dt)| || |j"|||j|j||}|D]}|| |t3|||t6j||| S) zEncode an instruct request. Args: request: The request to encode. Returns: The encoded tokens. Nz?Cannot continue final message if it is not an assistant messageT) system_promptforce_img_first)rVzUnknown message type special_token_policy)tokenstext prefix_idsimagesaudios)rfrHrNrGcontinue_final_messagelenrIrr rencode_user_messageavailable_toolsriextendrrUrXrencode_system_messager typeappendtruncate_at_max_tokensrbrBr(decoder&KEEP)r7rCrprqro tokens_listrKrJmsg_idxrM new_tokens new_images new_audiosrVrmtoks r9encode_instructz%InstructTokenizerBase.encode_instructs$& '+ .0  w/000)-(A(A'(J(J% %g&677# +# +LGS. G$4 5 5 999"3(899:7U#{++ N595M5M+},~-")"7$( 6N662 J  j))) j))))C-- N!55c7];RSS C!122 N#*#A#lwRUV]VfRgRgjkRkGk !::=0CS; c'"233a777!+JC// N!77<< ()Lc)L)LMMM   z * * * *  ) 5  ) ) .      # #C c"""V:L:QRR!     r:rmrlc:|j||S)aDecode tokens to a string. Args: tokens: The tokens to decode. special_token_policy: The policy to use for special tokens. Passing `None` will default to `self._special_token_policy` for [Tekkenizer][mistral_common.tokens.tokenizers.tekken.Tekkenizer] and `SpecialTokenPolicy.IGNORE` for [SentencePieceTokenizer][mistral_common.tokens.tokenizers.sentencepiece.SentencePieceTokenizer]. Note that passing `None` will be deprecated and `special_token_policy` will default to `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`. Returns: The decoded string. rk)r0r{)r7rmrls r9r{zInstructTokenizerBase.decodes ~$$VBV$WWWr:c6|j|Sr=)r0 _to_string)r7rms r9rz InstructTokenizerBase._to_strings~((000r:NNr=)%__name__ __module__ __qualname____doc__r*r,r!r5propertyr?rFintrB staticmethodrtuplerNrrboolrUrrXrr\rb classmethodrrfrr(rr&strr{r __classcell__r8s@r9r/r/4s$# .2-1 BBB$d*B$d* BBBBBB$"L4/"""X" 'tCy''''-o-%S/---\-$B;BUYB^bcf^gBBB^BG+GJNGbfG cGGG^GA*AcAAA^AS D()+,  "%    d    [ O  !5t!;<O  O O O O bXXT#YX>PSW>WXcfXXXX"1c1s11111111r:r/ceZdZdZ ddedeedzdedededzd ed e ee ee j ee ffd Zded ee fd Z dd eeezdededzd ed e ee ee j ee ff dZdeded ee fdZdededed ee fdZded ee fdZded efdZded efdZdS)InstructTokenizerV1zrInstruct tokenizer V1. This tokenizer has basic for messages. It does not support tools or image inputs. NFrOruis_lastis_firstrirjr;ct|jts Jd|j Jdd}|r|r|dz|jz}n|j}d|d}||dd \} } } | | | fS) arEncode a user message. Args: message: The message to encode. available_tools: Not used. is_last: Not used. is_first: Whether the message is the first one. system_prompt: The system prompt. force_img_first: Not used. Returns: The encoded tokens and empty list. "Message content must be normalizedNz(InstructTokenizerV1 cannot encode images z[INST] z [/INST]F)contentrri)rIrrr1encode_user_content) r7rOrurrrirjr message_txt curr_tokensimageaudios r9rtz'InstructTokenizerV1.encode_user_messages,'/3//UU1UUU/!))+U)))  &  &#f,w>GGoG1111 $($<$<[Z_os$<$t$t! UEE5((r:c:td|jj)Nz,System message encoding not implemented for )rSr8rr7rOs r9rwz)InstructTokenizerV1.encode_system_message&s !"jQUQ_Qh"j"jkkkr:rct|tsJ|r |r|dz|z}|j|dd}|ggfS)a*Encode a user content. Args: content: The content to encode. is_last: Whether the message is the last one. system_prompt: The system prompt. force_img_first: Not used. Returns: The encoded tokens and empty list. rFboseos)rIrr0encode)r7rrrirjrms r9rz'InstructTokenizerV1.encode_user_content)s`$'3'''''  7} 7#f,w6G&&wEu&EEr2~r:rPc td)zEncode a tool message. Raises: TokenizerException: The tool message is not implemented for this version. &Tools not implemented for tokenizer V1r rTs r9rUz'InstructTokenizerV1.encode_tool_messageCs !!IJJJr:rVct|ts J||j't|jdkrt d|r|jrt d|jrGt|jts Jd|j |jdd}nt |jd|j|js!|s| |j j |S) [Encode an assistant message. Args: message: The message to encode. is_before_last_user_message: Not used. continue_message: Whether to continue the message generation. Only use this if the assistant message is the last message. Returns: The encoded tokens. NrrU`continue_message` is only supported for assistant messages that have `prefix=False`.z4Message content must be a string for tokenizer < V13Frz // ) rIr tool_callsrsr prefixr rrr0rryeos_idr7rOrPrVrs r9rXz,InstructTokenizerV1.encode_assistant_messageKs'#344==g==4   )c'2D.E.E.I.I$%MNN N  S S2g _ Sgos33 k k5k k k3.//UPU/VVKK$%Q%QW=O%Q%QRR R~ 6&6 6   t~4 5 5 5r:rYc td)zEncode a think chunk. Raises: TokenizerException: The think chunk is not implemented for this version. z*Think not implemented for tokenizer < V13.rr[s r9r\z InstructTokenizerV1.encode_thinkis !!MNNNr:rCc:td|jj)zEncode a FIM request. Raises: TokenizerException: The FIM request is not implemented for this version. zFIM not available for r r0versionr7rCs r9 encode_fimzInstructTokenizerV1.encode_fimqs !!R$.:P!R!RSSSr:c:td|jj)Nz Transcription not available for rrs r9encode_transcriptionz(InstructTokenizerV1.encode_transcriptionys !\DNDZ!\!\]]]r:NF)rrrrrrFrrrrrnpndarrayrrtrrwrrrrUrrXrr\r r(rr rrar:r9rrsB%) %!)!)!)dd*!) !)  !) Tz !)!) tCy$rz*DK7 8!)!)!)!)Fl]ltCyllll%) % t,--Tz    tCy$rz*DK7 8 4K;KUYK^bcf^gKKKK+JNbf c<O*OcOOOOT*TTTTT^,@^Y^^^^^^r:rceZdZdZejZ d!dededzde dzffd Z d"de d e e dzd ed ed edzd edee ee eje effdZdedefdZdee ezdefdZdedeeeffdZdedede efdZdedeeeffdZde de efdZ!de de efdZ"de dedede efdZ#dede efdZ$de%de&fd Z'xZ(S)#InstructTokenizerV2z`Instruct tokenizer V2. This tokenizer adds supports to images, tools and FIM requests. Nr0r1r2ct||||jtjj|_|jtjj|_ |jtj j|_ |jtj j|_ |jtjj|_|jtjj|_|jtjj|_|jtjj|_|jtjj|_|jtjj|_dSInitialize the tokenizer. Args: tokenizer: The tokenizer to use. image_encoder: The image encoder to use. audio_encoder: The audio encoder to use. N)r4r5r0get_special_tokenr' begin_instvalue BEGIN_INSTend_instEND_INST begin_toolsBEGIN_AVAILABLE_TOOLS end_toolsEND_AVAILABLE_TOOLSbegin_tool_resultsBEGIN_TOOL_RESULTSend_tool_resultsEND_TOOL_RESULTSr TOOL_CALLSrBOSrPREFIXsuffixSUFFIXr6s r9r5zInstructTokenizerV2.__init__sK M=AAA.::=;S;YZZ889O9UVV %)^%E%EmF_Fe%f%f"#'>#C#CMD[Da#b#b "&."B"B=CcCi"j"j $ @ @A_Ae f f.::=;S;YZZ>33M4E4KLLn66}7K7QRR n66}7K7QRR r:FrOrurrrirjr;cd}||o|jtjkz}||o|jtjkz}g}|rP|rNd|D} |jt j| ddd} |jg| |j }| |j |||\} } } g||j }|j g}|| z|z}|| | fS)aEncode a user message. Args: message: The message to encode. available_tools: The list of available tools if any. is_last: Whether the message is the last one. is_first: Not used. system_prompt: The system prompt. force_img_first: Whether to force the image to be first. Returns: The encoded tokens and the list of images. Fc6g|]}|Sra) model_dump).0tools r9 z;InstructTokenizerV2.encode_user_message..s"CCC4T__&&CCCr: ensure_asciir)rrrirj)&_user_message_position_to_encode_toolsr+firstlastr0rjsondumpsrrrrrr)r7rOrurrrirjdo_encode_tools tools_tokenstoolstools_json_tokensrmrr prefix_tokens suffix_tokensrs r9rtz'InstructTokenizerV2.encode_user_messages , 8r)TXkXq)qr7p(SWjWo(op"$   CC?CCCE $ 5 5djUZ6[6[6[aflq 5 r r *"(L $77O'+ 8  u9,88  #f,}< E5((r:rcZ tj|S#tj$r|cYSwxYwr=)rloadsJSONDecodeErrorr7rs r9_parse_json_contentz'InstructTokenizerV2._parse_json_contents> :g&& &#   NNN s **ct|trdd|D}||S)Nrc3$K|] }|jV dSr=rnrrYs r9 z:InstructTokenizerV2._parse_tool_content..$>>Uej>>>>>>r:)rIrFjoinrrs r9_parse_tool_contentz'InstructTokenizerV2._parse_tool_contentsH gt $ $ ?gg>>g>>>>>G''000r: tool_messagecF|j||jdS)z8Bit of a hack due to the way tool results are tokenized.)namer)rrrr7rs r9_prepare_tool_resultz(InstructTokenizerV2._prepare_tool_results-!%// 0DEE   r:rPc|rgStj||gd}|jg|j|dd|j}|S)aEncode a tool message. Args: message: The message to encode. is_before_last_user_message: Whether the message is before the last user message. If true, the message is not encoded. Returns: The encoded tokens. Frrrrrrr0rrr7rOrPtool_result_strrs r9rUz'InstructTokenizerV2.encode_tool_messagesz ' I*d&?&?&H&H%IX]^^^  # ^ " "?5 " I I   ! r: tool_callcZ|jj||jjdS)z:Bit of a hack due to the way function calls are tokenized.r arguments)functionrrr)r7rs r9_prepare_function_callz*InstructTokenizerV2._prepare_function_calls3&+11)2D2NOO   r:c|js Jd|t|jts Jd|j|jdddS)Nz)Assistant message must have content. Got 3Message content must be a string for tokenizer < V7 Fr)rrIrr0rrstriprs r9(_encode_normal_content_assistant_messagez>>VX H K   K    T[[VhVm[-n-noooor:rr))rrrrr+rrr*r,r!r5rrFrrrrrrrrrtrrrrrdictrrUrrrr rrXrr r(rrrs@r9rr}s .A-E* .2-1 SSS$d*S$d* SSSSSS>%) %0)0)0)dd*0) 0)  0) Tz 0)0) tCy$rz*DK7 80)0)0)0)d33 13i+@1S1111   c3h    ;UY^bcf^g0  T#s(^    X@TXY]^aYbXXXX ?S X\]`Xa    "+"JN"bf" c""""HMcMd3iMMMM p*pppppppppr:rceZdZdZ ddededzdedzffd Zdede e e ffd Z d e de e e ffd Zd e d edeefdZd ed ededeeffd Zede ezezdeeeddffdZedeezdeeeejdffdZedeezdeeede ffdZde e!zdeeeejdze dzffdZde"e!deeeeejee ffdZ# dde ee$zdede dzdedeeeeejee ff fd Z%xZ&S)InstructTokenizerV3zxInstruct tokenizer V3. The only difference with V2 tokenizer is that it encodes the tool messages differently. Nr0r1r2cPt|||dS)r)r1r2N)r4r5r6s r9r5zInstructTokenizerV3.__init__Zs* -}]]]]]r:rr;c|jj||jjd}|jr|jdkr |j|d<|S)Nrnullid)rrrrr)r7r function_calls r9rz*InstructTokenizerV3._prepare_function_callisX&+11)2D2NOO  < /ILF22"+,M$ r:rch|j Jd||j|jdS)Nz7Tool message has to have the tool call id defined in v3)rcall_id) tool_call_idrrrs r9rz(InstructTokenizerV3._prepare_tool_resulttsE(446o444// 0DEE#0   r:rOrPctj||d}|jg|j|dd|j}|S)aEncode a tool message. Note: Same as [V2][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV2.encode_tool_message] but tools are not wrapped in a list and the history is also tokenized. Args: message: The message to encode. is_before_last_user_message: Whether the message is before the last user message. If true, the message is not encoded. Returns: The encoded tokens. Frrrrs r9rUz'InstructTokenizerV3.encode_tool_message|sh*T%>%>w%G%GV[\\\  # ^ " "?5 " I I   ! r:rVcJt|d|S)a Encode an assistant message. Note: Same as [V2][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV2.encode_assistant_message] but always encode the tool history. continue_message: Whether to continue the message generation. Only use this if the assistant message is the last message. Args: message: The message to encode. is_before_last_user_message: Not used. Returns: The encoded tokens. F)r4rX)r7rOrPrVr8s r9rXz,InstructTokenizerV3.encode_assistant_messages"$ww//@PQQQr:rYcdSr=rar[s r9_encode_content_chunkz)InstructTokenizerV3._encode_content_chunksjmjmr:cdSr=rar[s r9r&z)InstructTokenizerV3._encode_content_chunksnqnqr:cdSr=rar[s r9r&z)InstructTokenizerV3._encode_content_chunksililr:ct|tr |j|ddddfSt|tr%|j|jddddfSt|t r||ddfSt|ttfr5|j Jd| |}|j |j dfSt|ttfr5|j Jd||}|j d|jfSt#d|)NFrz+Make sure to define a image encoder at initz+Make sure to define a audio encoder at initzUnknown chunk type: )rIrr0rrrnrr\rrr1rmrrrr2rr)r7rY img_encodingaudio_encodings r9r&z)InstructTokenizerV3._encode_content_chunks_ eS ! ! =>((Eu(EEtTQ Q y ) ) =>((E(JJDRVV V z * * =$$U++T47 7  M: ; ; =%113`111--e44L& (:D@ @  M: ; ; =%113`111!//66N!($0DD D;E;;<< ?? D/3/I/I%/P/P, k1 k****#99%@@C MM, ' ' ' 'vu$$r:rr)'rrrrr*r,r!r5rrrrrrrrrFrrUrrXrrrrr&rrrrrrrrrr0rrrrs@r9rrRs.2-1 ^ ^ ^$d* ^$d* ^ ^ ^ ^ ^ ^  T#s(^      c3h    ;UY^bcf^g.R+RJNRbfR cRRRRRR(m3?Z+GmERVWZR[]acgRgLhmmmXm q: +Eq%PTUXPY[][egkPkJlqqqXq l: +El%PTUXPY[_afPfJglllXl=3+==%S SUS]`dSdfknrfrHrBs====,% -% tCy$rz*DK7 8%%%%(%) % 3%3%t,--3%3%Tz 3%  3% tCy$rz*DK7 8 3%3%3%3%3%3%3%3%3%3%r:rceZdZdZ d%dededzdedzddffd Zdeee dzd ee d e d e ddf d Z d e dee fdZ d&deeezdededzdedeee eejeeff fd Z d&d edeedzdedededzdedeee eejeefffd ZdedefdZdedefdZdeezdedefdZdedefdZe d ee!ddfdZ"e#d ee!defd Z$d e%d!edee fd"Z&d e d!ed#edee fd$Z'xZ(S)'InstructTokenizerV7a%Instruct tokenizer V7. The difference with V3 tokenizer is that it encodes the system prompts differently: - in V7 the system prompts are treated as separate SystemMessages - they are no longer prepended to the last user message - they are printed between special tokens Nr0r1r2r;ct||||jtjj|_|jtjj|_ |jtj j|_ d|_ |<|j js2|jtjj|_ dSdSdSr)r4r5r0rr' begin_systemr BEGIN_SYSTEM end_system END_SYSTEMbegin_tool_contentBEGIN_TOOL_CONTENT TRANSCRIBE audio_config is_streaming transcriber6s r9r5zInstructTokenizerV7.__init__s M=AAA N<<]=W=]^^.::=;S;YZZ"&."B"B=CcCi"j"j  $]-G-T $"n>>}?W?]^^DOOO % $ $ $r:tokenized_messagesrHr^r_c0tdD|z dtddffd }d}dkr|tkr|||dz }t|dz trl|tkrYt|ts>|||dz }|tkrt|t>dkr|tkdkrt ddS)Nc38K|]}|t|VdSr=)rs)rts r9rz?InstructTokenizerV7._truncate_for_max_tokens..6s(JJAMc!ffMMMMJJr:idxr;ct|trdS|krdS|}|Jt|zd|<dSr=)rIrrs)rKrr_rHto_droprGs r9dropz:InstructTokenizerV7._truncate_for_max_tokens..drop8sc(3-77 ---$S)C??? s3xx G&* s # # #r:rrhz+Input couldn't fit in truncate_at_max_token)sumrrsrIrr )r7rGrHr^r_rN current_idxrMs `` ` @r9rbz,InstructTokenizerV7._truncate_for_max_tokens*s`JJ&8JJJJJZW +c +d + + + + + + + + + kkkCMM99 D    1 K(;?3[AA %"CMM11*XkEZ\g:h:h1D%%%1$K"CMM11*XkEZ\g:h:h1 kkkCMM99 Q;;$%RSS S ;r:rOc|jg}t|jx}trt |g}|||dz }||j|S)zEncode a system message. Args: message: The message to encode. Returns: The encoded tokens. rr)r>rIrrrr0ryr@)r7rOrmrs r9rwz)InstructTokenizerV7.encode_system_messageSsq#$ 0g# 6 6 0 g.../G$--g66q99 do&&& r:FrrrirjcZ| Jdt|tr#t|||St |dko!t|dt t f}|r|r|d|dg}||\}}}|||fS)r2N?in Tokenizer V7 we don't encode system prompts in user messagesrrhr)rIrr4rrsrrr0) r7rrrirjr4rmrprr8s r9rz'InstructTokenizerV7.encode_user_contentds$$$&g$$$ gs # # P77..wOO O%(\\Q%6%n:gajS]_lRm;n;n"  /9 /qz71:.G $ ; ;G D Dvu$$r:rurc|| Jdt||||d|\}}} ||| fS)aEncode a user message. Args: message: The message to encode. available_tools: The list of available tools if any. is_last: Whether the message is the last one. is_first: Whether the message is the first one. system_prompt: Not used. force_img_first: Whether to force the image to be first. Returns: The encoded tokens and the list of images. NrS)rrrirj)r4rt) r7rOrurrrirjrmrprr8s r9rtz'InstructTokenizerV7.encode_user_messages`,$$&g$$$ % ; ;  + !<! ! vu$$r:rCcH|jJd|j|jjjtjkr||S|jjjtjkr||Std|jjjd)a Encodes an audio transcription request into a tokenized format. This method processes a transcription request containing audio data, encodes the user message, and returns the tokenized output. Args: request: The transcription request object containing the audio data to be encoded. Returns: Tokenized: The tokenized representation of the audio data, including processed audio and tokens Nz6Audio encoder must be defined, got self.audio_encoder=zxTranscription format should be one of 'instruct', 'streaming', got self.audio_encoder.audio_config.transcription_format=r3) r2rDtranscription_formatr"INSTRUCT_encode_instruct_transcription STREAMING_encode_streaming_transcriptionr rs r9rz(InstructTokenizerV7.encode_transcriptions!--/jUYUg/j/j---   * ?CVC_ _ _66w?? ?   , AEXEb b b77@@ @% H!.C H H H   r:c*|jtjksJd|j|jJ|jjd|}|tt|j ggddd\}}}g||}|j *d|j }||j |dd z }||jt||j || S) Nz=Request must not be in streaming mode, got request.streaming=z! needs to have a TRANSCRIBE token) input_audio)rT)rurrrizlang:Frrmrnrq) streamingrDISABLEDrCr8rrBrtrrrlanguager0rryr(r)r7rCrrmr7rlanguage_strings r9rXz2InstructTokenizerV7._encode_instruct_transcriptions2 M$:::: N':K N N;::**t~/F,i,i,i***33  !F!F!F G H H H 4  5$6#F#   '8g&688O dn++OE+RR RF do&&&T^-F-Fv-N-NW\]]]]r:ris_online_streamingc|jJd|jt|trtj|ntj|}|j||}t|j|j gS)NzFAudio encoder must be defined to encode audio, got self.audio_encoder=rb)rmrq) r2rIrr from_base64 from_bytes encode_audior(rmr)r7rrb_audio audio_encs r9 _encode_audioz!InstructTokenizerV7._encode_audios!-- W$BT W W.--.8s-C-C`"5)))IYZ_I`I`&33FPc3dd #O$    r:c`|jtjksJd|j||jj|jtjk}||jz}t|| |tj |j }|S)Nz9Request must be in streaming mode, got request.streaming=rdrkr])r^rr_rjrdataONLINErBrmr(r{r&r|rq)r7rCr]rms r9rZz3InstructTokenizerV7._encode_streaming_transcriptions M$:::: Jg6G J J;::&& M G4EI]4]'   00V:L:QRR#   r:c||r(td|DrtddSdS)Nc3@K|]}t|tVdSr=)rIrrrOs r9rz8InstructTokenizerV7.validate_messages..s,NN':g}55NNNNNNr:z9System messages are not yet allowed when audio is present) _has_audioanyrrds r9rfz%InstructTokenizerV7.validate_messagess^ >>( # # ^NNXNNNNN ^ !\]]] ^ ^ ^ ^r:c4td|DS)Nc3K|]Q}t|to7t|jtot d|jDVRdS)c3@K|]}t|tVdSr=)rIrrs r9rz;InstructTokenizerV7._has_audio...s,OOeJuj11OOOOOOr:N)rIrrrFrrrps r9rz1InstructTokenizerV7._has_audio..sy   w , , P7?D11 POOwOOOOO      r:)rr)rHs r9rqzInstructTokenizerV7._has_audios2  $       r:rPc$|jJt|jts Jd|j|jdd}|j|jdd}|jg||j}g|||j}|S)aEncode a tool message. Note: Same as [V3][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV3.encode_tool_message] but tools are not wrapped in a list and history is also tokenized Args: message: The message to encode. is_before_last_user_message: Not used. Returns: The encoded tokens. NrFr) r"rIrrr0rrrBr)r7rOrPtool_call_id_tokensrmrrs r9rUz'InstructTokenizerV7.encode_tool_messages#///'/3//UU1UUU/"n33G4HeY^3__&&wEu&MM  #    #      ! r:rVc|js|jstd||r|jrt dg}|jrmt |jt r||}n=t |jtr#|| |jdz }|jr|| |z }|js!|s| |j j |S)rrrr)rrr rr rIrr rFr0rryr0rrs r9rXz,InstructTokenizerV7.encode_assistant_message!s Nw'9 N$%L7%L%LMM M   2g  ? O'/3// O"KKGTT GOT22 Ot::7?KKANN   Q 4GGPP PK~ 6&6 6   t~4 5 5 5r:rr))rrrrr*r,r!r5rFrrrbrrwrrrrrrrrrrrtr r(rrXbytesrjrZrrrfrrqrrUrXrrs@r9r;r; s.2-1 ___$d*_$d* _  ______.'T cT!12'T+,'T 'T "% 'T  'T'T'T'TR]tCy*%) % %%t,--%%Tz %  % tCy$rz*DK7 8 %%%%%%J%) %!%!%!%dd*!% !%  !% Tz !%!% tCy$rz*DK7 8!%!%!%!%!%!%F ,@ Y    2^6J^y^^^^,  3;  T  i     7KPY$^d^^^^[^  T$Z D   \ ;UY^bcf^g> + JN bf  c        r:r;c beZdZdZ d dededzdedzddffd Zdede e fd Z xZ S) InstructTokenizerV11zInstruct tokenizer V11. The difference with V7 tokenizer is that it encodes tool calls differently: Tool call results are encoded as : - [begin tool call] call_name_tokens [call id] call_id_tokens [args] content tokens Nr0r1r2r;ct||||jtjj|_|jtjj|_ dSr=) r4r5r0rr'argsrARGSr!CALL_IDr6s r9r5zInstructTokenizerV11.__init__Ls_ M=AAAN44]5G5MNN ~77 8M8STT r:rOc |js Jd|g}|jD]}||}g}d|vr+|jg|j|ddd}||jg|j|ddd||j|jtj|ddddz }|S)Nr rFrrrr) rrrr0rrr~rr)r7rOrrpreparedidss r9rz= v13rc3$K|] }|jV dSr=rrs r9rz;InstructTokenizerV13.encode_tool_message..rr:Fr) r"rrIrrr0rrr)r7rOrPrrmrs r9rUz(InstructTokenizerV13.encode_tool_messages#//1e////'3'' ?gg>>g>>>>>G&&wEu&EE  #    ! r:rYc|j Jd|j Jd|j|jdd}|jg|}|jr||j|S)zEncode a thinking chunk. Args: chunk: The thinking chunk to encode. Returns: The encoded tokens. Nz2think tokens are not available for this tokenizer.Fr)rrr0rthinkingclosedry)r7rYrm think_tokenss r9r\z!InstructTokenizerV13.encode_thinks++-a+++~))+_)))&&u~5e&LL(262 < 0    / / /r:r)rrrrr+rrr*r,r!r5rrFrrrrrUrr\rrs@r9rrjs.A-F* .2-1 """$d*"$d* "  """"""$ ?S X\]`Xa    ;UY^bcf^g.*cr:r)Crabcrtypingrrrrnumpyrmistral_common.audiormistral_common.exceptionsr r r r #mistral_common.protocol.fim.requestr &mistral_common.protocol.instruct.chunkrrrrrrrr)mistral_common.protocol.instruct.messagesrrrrrr(mistral_common.protocol.instruct.requestr+mistral_common.protocol.instruct.tool_callsrr-mistral_common.protocol.transcription.requestrr &mistral_common.tokens.tokenizers.audior!r"%mistral_common.tokens.tokenizers.baser#r$r%r&r'r(r)r*r+&mistral_common.tokens.tokenizers.imager,'mistral_common.tokens.tokenizers.tekkenr-r/rrrr;r{rrar:r9rs 333333333333&&&&&& ;:::::                    EDDDDDFFFFFFFF]]]]]]]]TTTTTTTT                      @?????>>>>>>D1D1D1D1D1w2NMSgghD1D1D1N^^^^^7#6 Wk#kl^^^DRpRpRpRpRp!4nmUi!ijRpRpRpjt%t%t%t%t%!4nmUi!ijt%t%t%nxxxxx-xxxv #####.###LPPPPP/PPPPPr: