*`i'TddlZddlZddlZddlmZddlmZddlZddl m Z ddl m Z m Z ddlmZmZmZddlmZmZejded ejded e rdd lmZd eezdefdZddeezdedefdZdeezdedzfdZGddeZdS)N)cached_property)Path)TokenizerException)assert_sentencepiece_installedis_sentencepiece_installed)SpecialTokenPolicy TokenizerTokenizerVersion) ImageConfigMultiModalVersiononcez%.*`get_control_token` is deprecated.*)actioncategorymessagez#.*`_control_tokens` is deprecated.*)SentencePieceProcessorpathreturnc<ttrtttj}tt jdgzfd|Ddgz}otfd|DS)z1Check if the given path is a SentencePiece model.c(g|]}D] }d|| S)z.model.).0vm mm_versionss /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/mistral_common/tokens/tokenizers/sentencepiece.py z$is_sentencepiece..)s4QQQQ[QQ ! Q  QQQQz.modelc3LK|]}j|VdSN)nameendswith)rsuffixrs r z#is_sentencepiece..+s3!T!T$)"4"4V"<"> Tc!T!T!T!T8!T!T!TTTTrFtokenizer_filenameraise_deprecatedcJt|}|dd}|dkr|dd}|dkr$|rtd|dtdS|tjvrtd |t|S) z3Get the version of the tokenizer from the filename..modelrrz4Make sure to rename your tokenizer file to end with z.v1.v1!Unrecognized tokenizer filename: )r&splitrr r()r.r/ _version_strs rget_spm_versionr8.s/00%++C004Lw#))#..q1 w  v$%t\n%t%t%tuu u %%%+777 !YEW!Y!YZZZ L ) ))rct|}|dd}|dksd|vrdSd|ddz}|tjvrt d|t|jS)z1Get the image config from the tokenizer filename.r1r2r3rNr5)r&r6r r(rconfig)r.r7_mm_version_strs rget_image_configr<Cs/00%++C004Lw#\"9"9tL..s33B77O/;;; !YEW!Y!YZZZ _ - - 44rc <eZdZdZd deezdedzddffd ZedefdZ edefdZ d ede fd Z d ede fd Z ede fd Zdeefd Zede fdZede fdZde ejzezdefdZedee fdZd edededee fdZd dee dedzdefdZde defdZdee dedefdZdee defdZdee defdZ ede fdZ!ede fdZ"xZ#S)!SentencePieceTokenizerzC[SentencePiece](https://github.com/google/sentencepiece) tokenizer.N model_pathtokenizer_versionrcttjjj_t j|s J|tt|tr|n| _ j j ksJfdt!jD_|pt'|d_t+|_t/dS)zInitialize the `SentencePieceTokenizer`. Args: model_path: The path to the `SentencePiece` model. tokenizer_version: The version of the tokenizer. If not provided, it will be inferred from the model path. ) model_filecDg|]}j|Sr_model id_to_piece)riselfs rrz3SentencePieceTokenizer.__init__..gs)OOOat{..q11OOOrF)r/N)rlogging getLogger __class____name___loggerosrisfilerr%r&as_posixrE vocab_sizeget_piece_sizerangen_words_vocabr8_versionr _file_pathsuper__init__)rHr?r@rKs` rrYzSentencePieceTokenizer.__init__Vs '(((()@AA w~~j))55:55),%/ C%@%@[zzjFYFYF[F[    {%%''4;+E+E+G+GGGGGOOOO5;N;NOOO *;*rzlq?r?r?r z** rc|jS)z The path to the tokenizer model.)rWrHs r file_pathz SentencePieceTokenizer.file_pathns rc|jS)zThe version of the tokenizer.)rVr[s rversionzSentencePieceTokenizer.versionss }rsc6|j|S)z+Get the special token for the given string.)rE piece_to_idrHr_s rget_special_tokenz(SentencePieceTokenizer.get_special_tokenxs{&&q)))rc`tjdt||S)NzC`get_control_token` is deprecated. Use `get_special_token` instead.)warningswarn FutureWarningrcrbs rget_control_tokenz(SentencePieceTokenizer.get_control_token|s* []jkkk%%a(((rc4|jS)z!Vocabulary size of the tokenizer.)rErQr[s rrTzSentencePieceTokenizer.n_wordss{%%'''rc|jS)z(All tokens in the vocabulary as strings.)rUr[s rvocabzSentencePieceTokenizer.vocabs {rc4|jS)z#The beginning of sentence token id.)rEbos_idr[s rrmzSentencePieceTokenizer.bos_id{!!###rc4|jS)zThe end of sentence token id.)rEeos_idr[s rrpzSentencePieceTokenizer.eos_idrnrtokenclt|ttjfr'|jt|St|t r4|j|}|j|Stdt|j )z7Return `True` if the passed `token` is a special token.zExpected int or str, got ) r%intnpintegerrE IsControlr&ra TypeErrortyperL)rHrq token_ints r is_specialz!SentencePieceTokenizer.is_specials ec2:. / / P;((U44 4 s # # P //66I;((33 3NU 8LNNOO OrcxtjdtfdtjDS)NzB`_control_tokens` is deprecated. Make use of `is_special` instead.cHh|]}j||Sr)rErvrtokrHs r z9SentencePieceTokenizer._control_tokens..s.QQQdk6K6KC6P6PQQQQr)rerfrgrSrTr[s`r_control_tokensz&SentencePieceTokenizer._control_tokenss; Z\ijjjQQQQuT\22QQQQrboseosct|tsJ|j|}|r |jg|}|r g||j}|S)aEncode the given string into a list of token ids. Args: s: The string to encode. bos: Whether to add the beginning of sentence token. eos: Whether to add the end of sentence token. Returns: The list of token ids. )r%r&rEencodermrp)rHr_rrts rrzSentencePieceTokenizer.encodesd!S!!!!!{))!,,  "!q!A  "!!!T[!Artokensspecial_token_policycT|5t|ts tdt|d|&t jdt tj}|tjtj fvr| ||S|j |S)aaDecode the given list of token ids into a string. Note: Using `special_token_policy=SpecialTokenPolicy.KEEP` will keep the special tokens and the normal tokens as SentencePiece pieces. Args: tokens: The list of token ids. special_token_policy: The policy to use for special tokens. If `None`, the default policy is `SpecialTokenPolicy.IGNORE`. Passing `None` is deprecated and will be changed to `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`. Returns: The decoded string. NzFExpected `special_token_policy` to be None or SpecialTokenPolicy, got r1zUsing the tokenizer's special token policy `None` is deprecated. It will be removed in 1.10.0. Please pass a special token policy explicitly. Future default will be SpecialTokenPolicy.IGNORE.) r%r ValueErrorrxrerfrgIGNOREKEEPRAISE_decode_with_special_tokensrEdecode)rHrrs rrzSentencePieceTokenizer.decodes +J?SUg4h4h +vY]^rYsYsvvv  ' MH    $6#< $6$;=O=U#V V V33F.s'%S%S%Sd&6&6s&;&;%S%S%Src:g|]}|Srrr}s rrzFSentencePieceTokenizer._decode_with_special_tokens..s'KKKd..s33KKKrr)rzrrrextendappendrFjoin)rHrr text_list curr_tokensr~s` rrz2SentencePieceTokenizer._decode_with_special_tokenss !#  ( (Cs## ('+=+CCC$%uvvv%$$%S%S%S%S{%S%S%STTT"$K  !1!1#!6!67777""3''''  M   KKKK{KKK L L Lwwy!!!rc`tjdt||S)z[DEPRECATED] Converts a list of token ids into a string, keeping special tokens. Use `decode` with `special_token_policy=SpecialTokenPolicy.KEEP` instead. This is a convenient method for debugging. z`to_string` is deprecated and will be removed in 1.10.0. Use `decode` with `special_token_policy=SpecialTokenPolicy.KEEP` instead.)rerfrg _to_stringrHrs r to_stringz SentencePieceTokenizer.to_strings7  \     v&&&rcD||tjS)N)r)rrrrs rrz!SentencePieceTokenizer._to_strings{{68J8O{PPPrc4|jS)zThe padding token id.)rEpad_idr[s rrzSentencePieceTokenizer.pad_idrnrc4|jS)zThe unknown token id.)rEunk_idr[s rrzSentencePieceTokenizer.unk_id rnrr )$rL __module__ __qualname____doc__r&rr rYpropertyr\r^rsrcrhrTr'rkrrmrprtruboolrzsetrrrrrFrrrrr __classcell__)rKs@rr>r>SsTNN3:BRUYBYei04X)X*3*3****)3)3))))((((X(tCy$$$$_$$$$$_$Pbj 03 6P4PPPPRSRRR_R$Td3i&$*$*T#Y$*>PSW>W$*cf$*$*$*$*L1C1C1111"$s)"Se"jm""""*'S 'c'''' QcQsQQQQ$$$$X$$$$$X$$$$$rr>)F)rIrNre functoolsrpathlibrnumpyrtmistral_common.exceptionsrmistral_common.importsrr%mistral_common.tokens.tokenizers.baserr r &mistral_common.tokens.tokenizers.imager r filterwarningsrg sentencepiecerr&rr-r8r<r>rrrrs %%%%%%888888]]]]]]]] RQQQQQQQ   4    25444444 U3: U$ U U U U**d *d*Wg***** 5t 5 d8J 5 5 5 5 |$|$|$|$|$Y|$|$|$|$|$r