PiCddlmZmZmZmZmZmZddlZddl m Z dZ Gdde Z dee effdZd eed fdeeeeffd Zd edefd ZdedeeeeffdZdS))AnyDictListMappingSetTupleN) BaseTokenizerzc eZdZdZddededefdZded eefd Z d eed efd Z dde ee fded e ee ffdZ ded efdZdS) CLIPTokenizerar Text tokenizer for CLIP. Based on the official implementation here: https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py Args: path (str): the path to the CLIP merges file max_seq_len (int): the context length (all CLIP models use 77) truncate (bool): whether to truncate the text when longer than max_seq_len MTpath max_seq_lentruncatec0||_||_t|_d|jD|_t |}t|j}| d|D| d|D| ddgdt|D|_ d|j D|_ dt|D|_ tjd tj|_|j d|_|j d|_|j|_ddd |_dS) Nci|]\}}|| Sr.0kvs t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torchtune/models/clip/_tokenizer.py z*CLIPTokenizer.__init__..!sHHHdaQHHHc"g|] }|tz Sr) WORD_BOUNDARY)rrs r z*CLIPTokenizer.__init__..&s777Aa-'777rc8g|]}d|S))join)rmerges rrz*CLIPTokenizer.__init__..'s"999bggenn999r<|startoftext|> <|endoftext|>ci|]\}}|| Srr)riwords rrz*CLIPTokenizer.__init__..*s@@@GAta@@@rci|]\}}|| Srrrs rrz*CLIPTokenizer.__init__..+s>>>A1>>>rci|]\}}|| Srr)rr$r s rrz*CLIPTokenizer.__init__..,sEEExq%%EEErz[<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+)r!r")rr_bytes_to_unicode byte_encoderitems byte_decoder _load_mergeslistvaluesextend enumerateencoderdecoder bpe_ranksrecompile IGNORECASEpat sot_token eot_token pad_tokencache)selfr rrmergesvocabs r__init__zCLIPTokenizer.__init__s&  -//HHd.?.E.E.G.GHHHd##T&--//00 77777888 99&999::: '9:::@@y/?/?@@@ >>););)=)=>>> EE9V3D3DEEE: n M   &78o6 1,   rtextreturncvt|}jg}tjj|D]}dfd|dD}|fd | dDt|j krn| jt|j kr*js Jd|dj }j|d<|S) z Given a string, return the encoded list of token ids. Args: text (str): The text to encode. Returns: List[int]: The encoded list of token ids. rc32K|]}j|VdSN)r))rbr<s r z'CLIPTokenizer.encode..Js+PPQD-a0PPPPPPrutf-8c32K|]}j|VdSrD)r1)r bpe_tokenr<s rrFz'CLIPTokenizer.encode..Ks;,5 Y'r zWTokenized text is larger than the maximum sequence length but truncate is set to False.N) _clean_textlowerr8r4findallr7rencoder/_bpesplitlenrappendr9r)r<r@tokenstokens` rrOzCLIPTokenizer.encode<s[4  &&((.!Z$//  EGGPPPP%,,w:O:OPPPPPE MM9=59I9I9O9OPS9T9T   6{{d.../ dn%%% v;;) ) )=  ,  =.d../FF2J rrTcdfd|D}tfd|DddtdS)z Given a list of token ids, return the decoded text, optionally including special tokens. Args: tokens (List[int]): The list of token ids to decode. Returns: str: The decoded text. rc*g|]}j|Sr)r2)rrUr<s rrz(CLIPTokenizer.decode..fs @@@ U+@@@rc*g|]}j|Sr)r+)rcr<s rrz(CLIPTokenizer.decode..hs!:::t(+:::rrGreplace)errorsrJ)r bytearraydecoderZr)r<rTr@s` rr]zCLIPTokenizer.decode\spww@@@@@@@AA ::::T::: ; ; VGIV . . W]C ( ( rFsample inferencec`|d}|||d<|S)a] Tokenize the "text" field in the sample. Args: sample (Mapping[str, Any]): A sample with a "text" field containing a string to tokenize inference (bool): Unused by this tokenizer Returns: Mapping[str, Any]: The sample with added "tokens" field and the "messages" field removed. r@rT)poprO)r<r^r_r@s r__call__zCLIPTokenizer.__call__ms0zz&!!;;t,,x rrUc|jvr j|St|dkr |tzSt|dd|dtzfz}t |} t |fd}|jvrn6|\}}g}d}|t|kr |||} |||| | }n-#t$r |||dYnwxYw|||krC|t|dz kr-||dz|kr| ||z|dz }n | |||dz }|t|kt|}t|dkrnt |}Ud |}|j|<|S) z@ Performs byte-pair encoding on a single token. NrKTcTj|tdS)Ninf)r3getfloat)pairr<s rz$CLIPTokenizer._bpe..s 1C1CD%PU,,1W1Wr)keyrrJ) r;rRrtuple _get_pairsminr3indexr/ ValueErrorrSr) r<rUr%pairsbigramfirstsecondnew_wordr$js ` rrPzCLIPTokenizer._bpe~s DJ  :e$ $ u::>>=( (U3B3Z  E"I $=#??4  $ %$W$W$W$WXXXFT^++#ME6HAc$ii-- 5!,,AOOD1I...AA!OODH---E 7e##CIIM(9(9d1q5kV>S>SOOEFN333FAAOODG,,,FA#c$ii--$??D4yyA~~t$$EI$ %Lxx~~  5 s*5C 'D  D N)r T)F)__name__ __module__ __qualname____doc__strintboolr?rrOr]rrrbrPrrrr r s    S s 4    @349@ T#Y 3    $.s   Q#a&&   r)r-rangeordrSdictzip)bscsrrEs rr(r(s  U3s88SXX\ * *++ uSYYD A .. / / 0 uSYYD A .. / / 0 AAAB A 4[[ B;; IIaLLL IIdQh    FA  "   B B  rr%.c~t}|d}|ddD]}|||f|}|S)z Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). rrlN)setadd)r%rr prev_charchars rrnrnsP EEEQIQRR 9d#$$$ Lrr@c.|ddS)zI Minimal version of CLIP's text cleaning via the `ftfy` package. u’')rZ)r@s rrLrLs <<s # ##rr cTg}t|d5}t|D]k\}}|}|dkr|ds|s7|t |l dddn #1swxYwY|S)NrG)encodingrz #version:)openr0strip startswithrSrmrQ)r r=fr$lines rr,r,s F dW % % %/ || / /GAt::<rsB 8777777777777777HHHHHH iiiiiMiiiX4S>( U38_ U38_)=    $c$c$$$$stE#s(O4r