*`i5 ddlZddlmZmZddlmZddlmZddlm Z m Z ddl Z ddl mZmZddlmZddlmZdd lmZdd lmZdd lmZmZdd lmZdd lmZddlm Z ddl!m"Z"ddl#m$Z$Gdde%eZ&Gdde%eZ'Gdde(eZ)Gdde%eZ*GddeZ+GddeZ,e deZ-e deZ.e d e+Z/Gd!d"e e-e.e/efZ0dS)#N)ABCabstractmethod)Enum)Path)GenericTypeVar) ConfigDictField)Audio) MistralBase) FIMRequest)UserContentChunk)AssistantMessageType UserMessage)InstructRequest)Tool)TranscriptionRequest) AudioEncoder) ImageEncoderceZdZdZdZdZdS)UserMessagePositionzWhere to encode available toolsfirstlastN)__name__ __module__ __qualname____doc__rry/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/base.pyrrs)) E DDDrrceZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dS) SpecialTokensa(Enum of special tokens used in the tokenizer. Attributes: unk: The unknown token. bos: The beginning of string token. eos: The end of string token. begin_inst: The beginning of instruction token. end_inst: The end of instruction token. begin_tools: The beginning of tools token. end_tools: The end of tools token. begin_tool_results: The beginning of tool results token. end_tool_results: The end of tool results token. tool_calls: The tool calls token. img: The image token. pad: The pad token. img_break: The image break token. img_end: The image end token. prefix: The prefix token for FIM. middle: The middle token for FIM. suffix: The suffix token for FIM. begin_system: The beginning of system prompt token. end_system: The end of system prompt token. begin_tool_content: The beginning of tool content token. args: The args token. call_id: The call id token. audio: The audio token. begin_audio: The beginning of audio token. transcribe: The transcribe token. begin_think: The beginning of think token. end_think: The end of think token. Examples: >>> unk = SpecialTokens.unk zzzz[INST]z[/INST]z[AVAILABLE_TOOLS]z[/AVAILABLE_TOOLS]z[TOOL_RESULTS]z[/TOOL_RESULTS]z [TOOL_CALLS]z[IMG]zz [IMG_BREAK]z [IMG_END]z[PREFIX]z[MIDDLE]z[SUFFIX]z[SYSTEM_PROMPT]z[/SYSTEM_PROMPT]z[TOOL_CONTENT]z[ARGS]z [CALL_ID]z[AUDIO]z [BEGIN_AUDIO]z [TRANSCRIBE]z[THINK]z[/THINK]z[STREAMING_PAD]z[STREAMING_WORD]N)!rrrrunkboseos begin_instend_inst begin_tools end_toolsbegin_tool_resultsend_tool_results tool_callsimgpad img_breakimg_endprefixmiddlesuffix begin_system end_systembegin_tool_contentargscall_idaudio begin_audio transcribe begin_think end_think streaming_padstreaming_wordrrr r"r" s!!F C C CJH%K$I)(J C CIG F F F$L#J) DG E!KJKI%M'NNNrr"ceZdZdZdZdZdZdS)SpecialTokenPolicyzWhat to do with special tokens when encoding/decoding. Attributes: IGNORE: Ignore special tokens. KEEP: Keep special tokens. RAISE: Raise an error if special tokens are found. rN)rrrrIGNOREKEEPRAISErrr rArAcs)F D EEErrAceZdZdZdeddfdZedefdZddde fdZ ddde fd Z ddde fd Z ddde fd Z d Zd ZdZdZdZdZdS)TokenizerVersionaEnum of tokenizer versions. Allow to distinguish between different versions of the tokenizer and maintain backward compatibility. Attributes: v1: The first version of the tokenizer. v2: The second version of the tokenizer that includes special control tokens [INST], [\INST]. v3: The third version of the tokenizer that includes improved function calling. v7: The seventh version of the tokenizer that includes improved system prompt and function calling. v11: The eleventh version of the tokenizer that includes improved function calling. v13: The thirteenth version of the tokenizer that includes no call id tokenization and better prompt caching. Examples: >>> version = TokenizerVersion.v1 valuereturnctjd|std|dt||}||_|S)Nz^v\d+$zInvalid version format: z#. Must be 'v' followed by a number.)rematch ValueErrorstr__new___value_)clsrIobjs r rPzTokenizerVersion.__new__sPx 5)) dbbbbcc ckk#u%%  rc:t|jddS)NrB)intrIselfs r _version_numzTokenizerVersion._version_nums4:abb>"""rotherzstr | TokenizerVersioncjt|trt|}|j|jkSN isinstancerOrHrXrWrYs r __lt__zTokenizerVersion.__lt__s2 eS ! ! ,$U++E 5#555rcnt|trt|}|j|jkSdSr[r\r^s r __le__zTokenizerVersion.__le__< eS ! ! ;$U++E$(:: : ; ;rcnt|trt|}|j|jkSdSr[r\r^s r __gt__zTokenizerVersion.__gt__s< eS ! ! :$U++E$u'99 9 : :rcnt|trt|}|j|jkSdSr[r\r^s r __ge__zTokenizerVersion.__ge__rbrv1v2v3v7v11v13N)rrrrrOrPpropertyrUrXboolr_rardrfrgrhrirjrkrlrrr rHrHqs C$6#c###X#6466666 ;4;;;;; :4::::: ;4;;;;; B B B B C CCCrrHceZdZUdZedZeeed<dZ e dzed<dZ eedzed<e eZ eejed <e eZeeed <dS) TokenizedaA tokenized [`InstructRequest`][mistral_common.tokens.instruct.request]. Attributes: tokens: The token ids. text: The text representation of the tokens. prefix_ids: The prefix ids for FIM. images: The loaded images associated with the tokens. Examples: >>> tokenized = Tokenized(tokens=[1, 2, 3], text="Hello world", prefix_ids=[1], images=[]) T)arbitrary_types_allowedtokensNtext prefix_ids)default_factoryimagesaudios)rrrrr model_configlistrU__annotations__rsrOrtr rvnpndarrayrwr rrr rprps  :d;;;L ID#*#'JS D '''$uT:::FD :::%555FDK55555rrpc eZdZeedefdZedeefdZ ededefdZ eedefdZ eedefdZ eedefdZ eedefd Zed ed ed edeefd ZeddeededzdefdZed edefdZedeejzezdefdZeedefdZedeedefdZedeedefdZeedefdZdS) TokenizerrJcdS)z!Vocabulary size of the tokenizer.NrrVs r n_wordszTokenizer.n_wordsrcdS)z(All tokens in the vocabulary as strings.NrrVs r vocabzTokenizer.vocabrrtoken_idcdS)z$Convert a token id to the token str.Nr)rWrs r id_to_piecezTokenizer.id_to_piecerrcdS)z$id of the Beginning of String token.NrrVs r bos_idzTokenizer.bos_idrrcdS)zid of the End of String token.NrrVs r eos_idzTokenizer.eos_idrrcdS)zid of the Pad token.NrrVs r pad_idzTokenizer.pad_idrrcdS)zid of the Unk token.NrrVs r unk_idzTokenizer.unk_idrrsr$r%cdS)z(Convert a string to a list of token ids.Nr)rWrr$r%s r encodezTokenizer.encoderrNrrspecial_token_policycdS)aDecode the token ids to a string. Args: tokens: The token ids to decode. special_token_policy: The policy to use for special tokens. Passing `None` will default to `self._special_token_policy` for [Tekkenizer][mistral_common.tokens.tokenizers.tekken.Tekkenizer] and `SpecialTokenPolicy.IGNORE` for [SentencePieceTokenizer][mistral_common.tokens.tokenizers.sentencepiece.SentencePieceTokenizer]. Note that passing `None` will be deprecated and `special_token_policy` will default to `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`. Returns: The decoded string. NrrWrrrs r decodezTokenizer.decoderrcdS)zGet the id of a control token.Nr)rWrs r get_special_tokenzTokenizer.get_special_tokenrrtokencdS)z2Check if token id or token str is a special token.Nr)rWrs r is_specialzTokenizer.is_specialrrcdS)z!Get the version of the tokenizer.NrrVs r versionzTokenizer.versionrrcdS)z[DEPRECATED] Converts a list of token ids into a string, keeping special tokens. Use `decode` with `special_token_policy=SpecialTokenPolicy.KEEP` instead. This is a convenient method for debugging. NrrWrrs r to_stringzTokenizer.to_strings  rcdSr[rrs r _to_stringzTokenizer._to_string 47CrcdS)zThe file path of the tokenizer.NrrVs r file_pathzTokenizer.file_paths  rr[)rrrrmrrUrryrOrrrrrrrnrrArrr{integerrrHrrrrrrrr r~r~s  1111^X18tCy888^84C4C444^44444^X4....^X.$$$$^X$$$$$^X$77$7T7d3i777^7  T#Y >PSW>W cf   ^ .3.3...^.Bbj 03 6B4BBB^B1)111^X1 S  c   ^ 7c7s777^7  4   ^X   rr~InstructRequestType)boundFIMRequestType TokenizedTypecBeZdZUdZeed<edzed<edzed<dededzdedzddfdZe de de fd Z e de de fd Ze dd eed edzdefd Ze dede fdZe ddedeedzdedededzdedeeeeejeeffdZe ddeeezdededzdedeeeeejeeff dZe d eedefdZ dS)InstructTokenizerzBase class for instruct tokenizers. Attributes: tokenizer: The tokenizer to use. image_encoder: The image encoder to use if any. tokenizerN image_encoder audio_encoderrJcdS)zInitialize the instruct tokenizer. Args: tokenizer: The tokenizer to use. image_encoder: The image encoder to use if any. audio_encoder: The audio encoder to use if any. Nr)rWrrrs r __init__zInstructTokenizer.__init__'rrrequestcdS)zInstruct request to Tokenized object Args: request: The instruct request to encode. Returns: The tokenized instruct request. NrrWrs r encode_instructz!InstructTokenizer.encode_instruct2rrcdS)a Encodes an audio transcription request into a tokenized format. This method processes a transcription request containing audio data, encodes the user message, and returns the tokenized output. Args: request: The transcription request object containing the audio data to be encoded. Returns: Tokenized: The tokenized representation of the audio data, including processed audio and tokens Nrrs r encode_transcriptionz&InstructTokenizer.encode_transcription=s  rrrrcdS)aConvert token ids to string Args: tokens: The token ids to decode. special_token_policy: The policy to use for special tokens. Passing `None` will default to `self._special_token_policy` for [Tekkenizer][mistral_common.tokens.tokenizers.tekken.Tekkenizer] and `SpecialTokenPolicy.IGNORE` for [SentencePieceTokenizer][mistral_common.tokens.tokenizers.sentencepiece.SentencePieceTokenizer]. Note that passing `None` will be deprecated and `special_token_policy` will default to `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`. Returns: The decoded string. Nrrs r rzInstructTokenizer.decodeNrrcdS)zFIM request to Tokenized object Args: request: The FIM request to encode. Returns: The tokenized FIM request. Nrrs r encode_fimzInstructTokenizer.encode_fim_rrFmessageavailable_toolsis_lastis_first system_promptforce_img_firstcdS)aEncode a user message. Args: message: The user message to encode. available_tools: The available tools. is_last: Whether the message is the last one. is_first: Whether the message is the first one. system_prompt: The system prompt. force_img_first: Whether to force the image to be first. Returns: The encoded tokens and images. Nr)rWrrrrrrs r encode_user_messagez%InstructTokenizer.encode_user_messagejs . rcontentcdS)aIEncode a user content. Args: content: The user content to encode. is_last: Whether the content is the last one. system_prompt: The system prompt. force_img_first: Whether to force the image to be first. Returns: The encoded tokens and images. Nr)rWrrrrs r encode_user_contentz%InstructTokenizer.encode_user_contents & rcdSr[rrs r rzInstructTokenizer._to_stringrrr[)NF)!rrrrr~rzrrrrrrrrrryrUrArOrrrrrrntupler{r|r rrrrrrr rrs$&&&&$&&&&  "  3?$3F  WcfjWj        ': }   ^  ,@ ]   ^   T#Y >PSW>W cf   ^  . ]   ^ %) %   dd*     Tz    tCy$rz*DK7 8   ^ 0 %) %   t,--  Tz    tCy$rz*DK7 8    ^ (7c7s777^777rr)1rLabcrrenumrpathlibrtypingrrnumpyr{pydanticr r mistral_common.audior mistral_common.baser #mistral_common.protocol.fim.requestr &mistral_common.protocol.instruct.chunkr)mistral_common.protocol.instruct.messagesrr(mistral_common.protocol.instruct.requestr+mistral_common.protocol.instruct.tool_callsr-mistral_common.protocol.transcription.requestr&mistral_common.tokens.tokenizers.audior&mistral_common.tokens.tokenizers.imagerrOrr"rUrArHrpr~rrrrrrr rs ################&&&&&&&&&&&&&&++++++::::::CCCCCCEDDDDD<<<<<<NNNNNN????????????#t@(@(@(@(@(C@(@(@(F     d   55555sD555p66666 666*U U U U U U U U pg3?KKK)<<<y999 ~8~8~8~8~8 3^]Th hi~8~8~8~8~8r