
     `iw
                         d dl mZmZmZmZ 	 d dlZd dlmZ n# e$ r  ed          w xY wddl	m
Z
 ddlmZmZ ded	eeeeef                  fd
Zddeded	efdZ G d d          ZddgZdS )    )ListSetTupleUnionN)ExLlamaV2TokenizerzJexllamav2 is not installed. Please install it with "pip install exllamav2"   )CharacterLevelParser)TokenEnforcerTokenEnforcerTokenizerData	tokenizerreturnc                    | j                                         }t          | j                                                  }|                    | j        | j        | j        | j	        h           | 
                                }g }t          |          D ]F}||v r||         }t          |          dk    o|d         dk    }|                    |||f           G|S )Nr    )r   
vocab_sizesetextended_id_to_piecekeysupdatebos_token_ideos_token_idpad_token_idunk_token_idget_id_to_piece_listrangelenappend)r   r   all_special_idsid_to_pieceregular_tokens	token_idxdecodedis_word_start_tokens           {/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/lmformatenforcer/integrations/exllamav2.py_build_regular_tokens_listr$      s    $//11J)8==??@@OY3Y5KYMcene{}~~~0022KN:&& I I	''i(!'llQ.D71:3Dy'3FGHHHH    Fuse_bitmaskc                      t                     }dt          t                   dt          f fd}t	          || j        | j                  S )Ntokensr   c                 n    t          j        | t           j                  }                    |          S )N)dtype)torchtensorlongdecode)r(   r,   r   s     r#   _decodez4build_token_enforcer_tokenizer_data.<locals>._decode   s-    fEJ777'''r%   )r$   r   intstrr   r   actual_vocab_size)r   r&   r   r/   s   `   r#   #build_token_enforcer_tokenizer_datar3      si    /	::N(S	 (c ( ( ( ( ( ( &ngy?UWbdmd  A  A  Ar%   c                       e Zd ZU dZee         ed<   dedee	e
f         fdZdeddfd	Zd
ej        ddfdZd Zdeee         ee         f         fdZdS )ExLlamaV2TokenEnforcerFilterzkExLlamaV2Sampler.Settings.filters filter that uses the token enforcer to only allow format-complying tokenstoken_sequencecharacter_level_parsertokenizer_datac                     t          |t                    rt          |          }t          ||          | _        g | _        d S N)
isinstancer   r3   r
   token_enforcerr6   )selfr7   r8   s      r#   __init__z%ExLlamaV2TokenEnforcerFilter.__init__*   sF     n&899 	Q@PPN+N<RSS r%   
prefix_strr   Nc                     g | _         d S r:   )r6   )r=   r?   s     r#   beginz"ExLlamaV2TokenEnforcerFilter.begin2   s     r%   tokenc                 l    | j                             t          |d         d                              d S )Nr   )r6   r   r0   )r=   rB   s     r#   feedz!ExLlamaV2TokenEnforcerFilter.feed5   s/    ""3uQx{#3#344444r%   c                     | S r:    )r=   s    r#   clonez"ExLlamaV2TokenEnforcerFilter.clone8   s    r%   c                     | j                             | j                  j        }t	          |          t	                      fS r:   )r<   get_allowed_tokensr6   allowed_tokensr   )r=   rJ   s     r#   nextz!ExLlamaV2TokenEnforcerFilter.next;   s5    ,??@STTc>""CEE))r%   )__name__
__module____qualname____doc__r   r0   __annotations__r	   r   r   r   r>   r1   rA   r+   TensorrD   rG   r   r   rK   rF   r%   r#   r5   r5   &   s         uuI!)=!!&'9;U'U!V! ! ! !! ! ! ! ! !5%, 54 5 5 5 5  *eCHc#h./ * * * * * *r%   r5   r3   )F)typingr   r   r   r   r+   	exllamav2r   ImportErrorcharacterlevelparserr	   tokenenforcerr
   r   r0   r1   boolr$   r3   r5   __all__rF   r%   r#   <module>rY      sP   * * * * * * * * * * * *dLLL,,,,,,, d d d
+b
c
ccd 8 7 7 7 7 7 E E E E E E E E*< eCQTVZNF[A\     A A3E ATX Ae A A A A* * * * * * * *4 *+P
Qs   
 *