
     `i!                     N   d dl Z d dlZd dlmZmZ d dlmZmZmZm	Z	m
Z
 ddlmZmZmZmZmZmZmZmZmZ dedefdZd	edefd
Ze G d d                      Z G d de j                  Z G d de          Z G d de          Z G d de          Z G d de          ZdS )    N)	dataclassfield)AnyHashableListOptionalTypeVar   )	COMPLETE_ALPHABETCONFIG_ENV_VAR_DEFAULT_ALPHABETWHITESPACE_CHARACTERS#DEFAULT_MAX_CONSECUTIVE_WHITESPACESDEFAULT_FORCE_JSON_FIELD_ORDER*CONFIG_ENV_VAR_MAX_CONSECUTIVE_WHITESPACES&CONFIG_ENV_VAR_STRICT_JSON_FIELD_ORDER$CONFIG_ENV_VAR_MAX_JSON_ARRAY_LENGTHDEFAULT_MAX_JSON_ARRAY_LENGTHsreturnc                 V    | o'|                                                                  dv S )N)true1)striplower)r   s    y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/lmformatenforcer/characterlevelparser.py_parse_boolr      s$    5!''))//##}45    env_varc                 |     t                    }|t          k    rt          n| fd}t          |          S )Nc                  p     t           j                            t                                         S N)osenvirongetstr)default_valr   parser_funcs   r   factory_funcz+_env_or_default_field.<locals>.factory_func   s+    {2:>>'3{3C3CDDEEEr   )default_factory)typeboolr   r   )r   r&   default_val_typer(   r'   s   ``  @r   _env_or_default_fieldr-      sb    K((!1T!9!9++?OKF F F F F F F....r   c                       e Zd ZU  eee          Zeed<    ee	e
          Zeed<   	  eee          Zeed<   	  eee          Zeed<   dS )CharacterLevelParserConfigalphabetmax_consecutive_whitespacesforce_json_field_ordermax_json_array_lengthN)__name__
__module____qualname__r-   r   r   r0   r%   __annotations__r   r   r1   intr   r   r2   r+   r   r   r3    r   r   r/   r/      s         ))*I*;= =Hc = = ='<'<=g=`(b (b b b bI#8#89_9W$Y $YD Y Y Y3!6!67[7T"V "V3 V V V r   r/   c                      e Zd ZdZddee         fdZej        de	dd fd            Z
ej        de	fd            Zej        defd	            Zdee         fd
Zdee         fdZedefd            Zej        defd            ZdS )CharacterLevelParserzCharacterLevelParser is an interface for classes that can parse strings one character at a time, and determine which characters are allowed at any specific timeNconfigc                 0    |pt                      | _        d S r!   )r/   _config)selfr<   s     r   __init__zCharacterLevelParser.__init__+   s    =!;!=!=r   new_characterr   c                     t                      )zAdd a character to the parser, and return a new parser that represents the state of the parser after the character has been added. This has to be
        an immutable operation - the original CharacterLevelParser (self) must not be modified.NotImplementedErrorr?   rA   s     r   add_characterz"CharacterLevelParser.add_character.   s     "###r   c                     t                      )zgReturn a string containing all characters that are allowed at the current point in the parsing process.rC   r?   s    r   get_allowed_charactersz+CharacterLevelParser.get_allowed_characters4        "###r   c                     t                      )zReturn True if the parser is in a state where it can end (potentially finished parsing the desired structure), and False otherwise.rC   rH   s    r   can_endzCharacterLevelParser.can_end9   rJ   r   c                     dS )zpOptional. Return a key that denotes that this state is a repeating state, full tree traversal should be avoided.Nr9   rH   s    r   shortcut_keyz!CharacterLevelParser.shortcut_key>       tr   c                     dS )z|Optional. Return a key that denotes that this state is a repeating state, and if it is visited again, results can be cached.Nr9   rH   s    r   	cache_keyzCharacterLevelParser.cache_keyB   rO   r   c                     | j         S r!   r>   rH   s    r   r<   zCharacterLevelParser.configF   s
    |r   
new_configc                     || _         | S r!   rS   )r?   rT   s     r   r<   zCharacterLevelParser.configJ   s    !r   r!   )r4   r5   r6   __doc__r   r/   r@   abcabstractmethodr%   rF   rI   r+   rL   r   rN   rQ   propertyr<   setterr9   r   r   r;   r;   (   sa        k  k> >x(BC > > > > 	$3 $3I $ $ $ $
 	$ $ $ $ $ 	$ $ $ $ $hx0    8H-     2    X ]!;    ]  r   r;   c                   F    e Zd ZdZdefdZdedefdZdefdZde	fdZ
d	S )
StringParsera  RegexParser is an example CharacterLevelParser that only allows an exact string. It is a debugging / learning tool
    to show how CharacterLevelParser works together with TokenizerPrefixTree to filter the allowed tokens (some of whom may contain multiple characters)stringc                     || _         d S r!   
target_str)r?   r]   s     r   r@   zStringParser.__init__S   s     r   rA   r   c                     | j                             |          r)t          | j         t          |          d                    S t	          d| j         d          d| d          )Nz
Expected 'r   z' but got '')r`   
startswithr\   len
ValueErrorrE   s     r   rF   zStringParser.add_characterV   sg    ?%%m44 	[M0B0B0C0C DEEEY$/!*<YYYYYZZZr   c                 .    | j         r| j         d         ndS )Nr    r_   rH   s    r   rI   z#StringParser.get_allowed_characters\   s    %)_<tq!!"<r   c                     | j          S r!   r_   rH   s    r   rL   zStringParser.can_end_   s    ?""r   N)r4   r5   r6   rV   r%   r@   r;   rF   rI   r+   rL   r9   r   r   r\   r\   P   s        \ \!s ! ! ! ![3 [3G [ [ [ [= = = = =# # # # # # #r   r\   c                   H    e Zd ZdZddefdZdedefdZdefdZ	defd	Z
d
S )ForceStopParserzbA simple parser that forbids any characters except the stop token. Used to force stop LM operationFallow_whitespacec                     || _         d S r!   )rk   )r?   rk   s     r   r@   zForceStopParser.__init__e   s     0r   rA   r   c                     | S r!   r9   rE   s     r   rF   zForceStopParser.add_characterg   s    r   c                 "    | j         rt          ndS Nrg   )rk   r   rH   s    r   rI   z&ForceStopParser.get_allowed_charactersi   s    (,(=E$$2Er   c                     dS )NTr9   rH   s    r   rL   zForceStopParser.can_endk   s    tr   N)F)r4   r5   r6   rV   r+   r@   r%   r;   rF   rI   rL   r9   r   r   rj   rj   c   s        ll1 1 1 1 1 13 3G    F F F F F      r   rj   c                       e Zd ZdZdee         fdZdedefdZdefdZ	de
fdZdee         fd	Zdee         fd
ZdS )UnionParserzWA parser that allows a string that would be allowed by any of several different parsersparsersc                     || _         d S r!   rs   r?   rs   s     r   r@   zUnionParser.__init__q       r   rA   r   c                     fd| j         D             }fd|D             }t          |          dk    r|d         S t          |          S )Nc                 @    g | ]}|                                 v |S r9   rI   .0parserrA   s     r   
<listcomp>z-UnionParser.add_character.<locals>.<listcomp>v   s0    rrrvRXRoRoRqRqAqAqFAqAqAqr   c                 :    g | ]}|                               S r9   )rF   r{   s     r   r~   z-UnionParser.add_character.<locals>.<listcomp>w   s'    [[[,,];;[[[r   r
   r   )rs   rd   rr   )r?   rA   relevant_parsersnext_parserss    `  r   rF   zUnionParser.add_charactert   sd    rrrrrrr[[[[JZ[[[|!!?"<(((r   c                     d                     d | j        D                       }d                     t          |                    S )Nrg   c                 6    g | ]}|                                 S r9   rz   r|   r}   s     r   r~   z6UnionParser.get_allowed_characters.<locals>.<listcomp>}   s$    VVVv688::VVVr   )joinrs   set)r?   alloweds     r   rI   z"UnionParser.get_allowed_characters|   s<    ''VVVVVWWwws7||$$$r   c                 >    t          d | j        D                       S )Nc                 6    g | ]}|                                 S r9   rL   r   s     r   r~   z'UnionParser.can_end.<locals>.<listcomp>   "    @@@FNN$$@@@r   )anyrs   rH   s    r   rL   zUnionParser.can_end   !    @@4<@@@AAAr   c                     t          d | j        D                       }t          |          dk    rt          t	          |                    S d S )Nc              3   >   K   | ]}|                                 V  d S r!   )rN   r   s     r   	<genexpr>z+UnionParser.shortcut_key.<locals>.<genexpr>   s.      "T"TV6#6#6#8#8"T"T"T"T"T"Tr   r
   )r   rs   rd   nextiter)r?   unique_shortcut_keyss     r   rN   zUnionParser.shortcut_key   sQ    ""T"Tt|"T"T"TTT#$$))122333tr   c                 |    t          d | j        D                       }t          d |D                       rd|fS d S )Nc              3   >   K   | ]}|                                 V  d S r!   rQ   r   s     r   r   z(UnionParser.cache_key.<locals>.<genexpr>   .      MMfv//11MMMMMMr   c              3      K   | ]}|d uV  	d S r!   r9   r|   keys     r   r   z(UnionParser.cache_key.<locals>.<genexpr>   &      993s$999999r   uniontuplers   allr?   all_cache_keyss     r   rQ   zUnionParser.cache_key   sO    MMMMMMM99.99999 	-^,,tr   N)r4   r5   r6   rV   r   r;   r@   r%   rF   rI   r+   rL   r   r   rN   rQ   r9   r   r   rr   rr   o   s        aa%9 :    )3 )3G ) ) ) )% % % % %B B B B Bhx0    8H-      r   rr   c                       e Zd ZdZdee         fdZdedefdZdefdZ	de
fdZdee         fd	Zdee         fd
ZdS )SequenceParserz0A parser that is a sequence of multiple parsers.rs   c                     || _         d S r!   ru   rv   s     r   r@   zSequenceParser.__init__   rw   r   rA   r   c                    g }t          | j                  D ]\  }}||                                v r||                    |          }|g| j        |dz   d          z   }t	          |          dk    r|                    |d                    n"|                    t          |                     |                                s nt	          |          dk    r|d         S t          |          S Nr
   r   )		enumeraters   rI   rF   rd   appendr   rL   rr   )r?   rA   legal_parsersidxr}   updated_parserr   s          r   rF   zSequenceParser.add_character   s     %T\22 		 		KC = = ? ???!'!5!5m!D!D ./$,s1uvv2FF|$$))!((a9999!(()E)EFFF>>## }"" ##=)))r   c                     t                      }| j        D ]?}|                    |                                           |                                s n@d                    |          S ro   )r   rs   updaterI   rL   r   )r?   allowed_charactersr}   s      r   rI   z%SequenceParser.get_allowed_characters   sl     UUl 	 	F%%f&C&C&E&EFFF>>## ww)***r   c                 >    t          d | j        D                       S )Nc                 6    g | ]}|                                 S r9   r   r   s     r   r~   z*SequenceParser.can_end.<locals>.<listcomp>   r   r   )r   rs   rH   s    r   rL   zSequenceParser.can_end   r   r   c                 t    t          | j                  dk    r| j        d                                         nd S r   )rd   rs   rN   rH   s    r   rN   zSequenceParser.shortcut_key   s3    14T\1B1Ba1G1Gt|A++---TQr   c                 |    t          d | j        D                       }t          d |D                       rd|fS d S )Nc              3   >   K   | ]}|                                 V  d S r!   r   r   s     r   r   z+SequenceParser.cache_key.<locals>.<genexpr>   r   r   c              3      K   | ]}|d uV  	d S r!   r9   r   s     r   r   z+SequenceParser.cache_key.<locals>.<genexpr>   r   r   sequencer   r   s     r   rQ   zSequenceParser.cache_key   sO    MMMMMMM99.99999 	0//tr   N)r4   r5   r6   rV   r   r;   r@   r%   rF   rI   r+   rL   r   rN   r   rQ   r9   r   r   r   r      s        ::%9 :    *3 *3G * * * *&+ + + + +B B B B BRhsm R R R R8H-      r   r   ) rW   r"   dataclassesr   r   typingr   r   r   r   r	   constsr   r   r   r   r   r   r   r   r   r%   r+   r   r-   r/   ABCr;   r\   rj   rr   r   r9   r   r   <module>r      s#   



 				 ( ( ( ( ( ( ( ( 9 9 9 9 9 9 9 9 9 9 9 9 9 94 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 463 64 6 6 6 6/3 / / / / /         % % % % %37 % % %P# # # # #' # # #&	 	 	 	 	* 	 	 	    &   B* * * * *) * * * * *r   