
     `iP                     J    	 d dl Z dZn# e$ r dZY nw xY w G d d          ZdS )    NTFc                   6    e Zd ZdedefdZd Zd ZdefdZdS )		TokenListuse_bitmask
vocab_sizec                     || _         |rt          st          d          | j         r/|dz   dz  }t          j        |t          j                  | _        d S g | _        d S )Nz'LMFormatEnforcer bitmasks require torch       )dtype)r   
_HAS_TORCH
ValueErrortorchzerosint32allowed_tokens)selfr   r   tensor_sizes       n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/lmformatenforcer/tokenlist.py__init__zTokenList.__init__   sp    & 	Hz 	HFGGG 	%%?r1K"'+k"M"M"MD"$D    c                     | j         r$|dz	  }|dz  }| j        |xx         d|z  z  cc<   d S | j                            |           d S N   r      )r   r   appendr   token_idelement_index	bit_indexs       r   r   zTokenList.append   se     	1$MM 4I...1	>B.....&&x00000r   c                    | j         rjt          |t          j                  r't          j                            | j        |           d S |D ]$}|dz	  }|dz  }| j        |xx         d|z  z  cc<   %d S | j                            |           d S r   )r   
isinstancer   Tensorbitwise_or_r   extend)r   	token_idsr   r   r   s        r   r#   zTokenList.extend   s     		2)U\22 K(()<iHHHHH ) K KH$,MM (4I'6661	>J6666K K
 &&y11111r   returnc                 d    | j         r!|dz  }|dz  }| j        |         d|z  z  dk    S || j        v S )Nr	   r   r   )r   r   r   s       r   is_token_allowedzTokenList.is_token_allowed)   sI     	3$NM 2I'6!y.IaOOt222r   N)	__name__
__module____qualname__boolintr   r   r#   r'    r   r   r   r      so        
%D 
%c 
% 
% 
% 
%1 1 1
2 
2 
23D 3 3 3 3 3 3r   r   )r   r   ImportErrorr   r-   r   r   <module>r/      sp   LLLJJ   JJJ(3 (3 (3 (3 (3 (3 (3 (3 (3 (3s   	 