
    
`i]                     ,   d dl mZmZ d dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlZ	 d dlZn# e$ r Y nw xY werddlmZ ddlmZ dd	lmZmZmZm Z  dd
l!m"Z"m#Z#m$Z$ ddl%m&Z& d dl'm'Z' 	  e(e          Z)n# e*$ r dZ)Y nw xY w G d dee          Z+ G d de+          Z, G d de+          Z- G d de          Z. edd          Z/ G d de0          Z1 G d d          Z2 G d d          Z3 G d d          Z4d  Z5d! Z6 G d" d#          Z7d$e0fd%Z8 G d& d'          Z9 G d( d)          Z:ee1ge1f         Z; G d* d+e          Z<d5d-e	e.e0f         fd.Z= G d/ d0e<          Z> G d1 d2e>          Z? G d3 d4e<          Z@dS )6    )abstractmethodABCN)suppress)TypeVarTypeDictIterator
CollectionCallableOptional	FrozenSetAnyClassVarTYPE_CHECKINGoverload)
ModuleType   )	LexerConf)ParserState)classifyget_regexp_width	Serializelogger)UnexpectedCharactersLexErrorUnexpectedToken)TOKEN_DEFAULT_PRIORITY)copyFc            	          e Zd ZU dZeed<   ee         ed<   ee         ed<   ee         ed<   ddedee         dee         ddfd	Z	d
 Z
d Zd Zedefd            Zeedefd                        Zeedefd                        Zd ZdS )Patternz(An abstraction over regular expressions.valueflagsrawtype Nreturnc                 J    || _         t          |          | _        || _        d S N)r!   	frozensetr"   r#   )selfr!   r"   r#   s       ^/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/lark/lexer.py__init__zPattern.__init__)   s#    
u%%
    c                 D    t          |                                           S r(   )repr	to_regexpr*   s    r+   __repr__zPattern.__repr__.   s    DNN$$%%%r-   c                 T    t          t          |           | j        | j        f          S r(   )hashr$   r!   r"   r1   s    r+   __hash__zPattern.__hash__2   s!    T$ZZTZ8999r-   c                     t          |           t          |          k    o| j        |j        k    o| j        |j        k    S r(   )r$   r!   r"   r*   others     r+   __eq__zPattern.__eq__5   s7    DzzT%[[(dTZ5;-Fd4:Y^YdKddr-   c                     t                      r(   NotImplementedErrorr1   s    r+   r0   zPattern.to_regexp8   s    !###r-   c                     t                      r(   r;   r1   s    r+   	min_widthzPattern.min_width<        "###r-   c                     t                      r(   r;   r1   s    r+   	max_widthzPattern.max_widthA   r?   r-   c                 ,    | j         D ]}d|d|d}|S )Nz(?:))r"   )r*   r!   fs      r+   
_get_flagszPattern._get_flagsF   s.     	. 	.AA#$11eee,EEr-   )r%   N)__name__
__module____qualname____doc__str__annotations__r
   r   r   r,   r2   r5   r9   r   r0   propertyintr>   rA   rF   r%   r-   r+   r    r    !   s[        ..JJJc?	#
3- c *S/ Xc] ^b    
& & &: : :e e e $3 $ $ $ ^$ $3 $ $ $ ^ X$ $3 $ $ $ ^ X$    r-   r    c                   r    e Zd ZU dZdZee         ed<   defdZe	de
fd            Ze	de
fd            ZdS )	
PatternStr)r!   r"   r#   rK   r$   r&   c                 Z    |                      t          j        | j                            S r(   )rF   reescaper!   r1   s    r+   r0   zPatternStr.to_regexpQ   s     ry44555r-   c                 *    t          | j                  S r(   lenr!   r1   s    r+   r>   zPatternStr.min_widthT       4:r-   c                 *    t          | j                  S r(   rU   r1   s    r+   rA   zPatternStr.max_widthX   rW   r-   N)rG   rH   rI   __serialize_fields__r$   r   rK   rL   r0   rM   rN   r>   rA   r%   r-   r+   rP   rP   L   s         2D(3-63 6 6 6 6 3    X 3    X  r-   rP   c                   |    e Zd ZU dZdZee         ed<   defdZdZ	d Z
edefd            Zedefd	            ZdS )
	PatternRE)r!   r"   r#   _widthrR   r$   r&   c                 6    |                      | j                  S r(   )rF   r!   r1   s    r+   r0   zPatternRE.to_regexpb   s    tz***r-   Nc                 j    | j         &t          |                                           | _         | j         S r(   )r\   r   r0   r1   s    r+   
_get_widthzPatternRE._get_widthf   s+    ;*4>>+;+;<<DK{r-   c                 6    |                                  d         S Nr   r_   r1   s    r+   r>   zPatternRE.min_widthk         ##r-   c                 6    |                                  d         S )Nr   rb   r1   s    r+   rA   zPatternRE.max_widtho   rc   r-   )rG   rH   rI   rY   r$   r   rK   rL   r0   r\   r_   rM   rN   r>   rA   r%   r-   r+   r[   r[   ]   s         <D(3-+3 + + + + F  
 $3 $ $ $ X$ $3 $ $ $ X$ $ $r-   r[   c            	       l    e Zd ZU dZdZeefZee	d<   e
e	d<   ee	d<   efdede
deddfdZd	 Zdefd
ZdS )TerminalDefzA definition of a terminal)namepatternpriorityrg   rh   ri   r&   Nc                 n    t          |t                    s
J |            || _        || _        || _        d S r(   )
isinstancer    rg   rh   ri   )r*   rg   rh   ri   s       r+   r,   zTerminalDef.__init__}   s9    '7++44W44+	 r-   c                 P    t          |           j        d| j        d| j        dS )N(, rD   )r$   rG   rg   rh   r1   s    r+   r2   zTerminalDef.__repr__   s*    #Dzz222DIIIt|||LLr-   c                 j    | j                             d          r| j        j        p| j         S | j         S )N__)rg   
startswithrh   r#   r1   s    r+   	user_reprzTerminalDef.user_repr   s4    9%% 	<#0ty09r-   )rG   rH   rI   rJ   rY   rP   r[   __serialize_namespace__rK   rL   r    rN   r   r,   r2   rr   r%   r-   r+   rf   rf   t   s           8()3
IIIMMMDZ ! !S !7 !c !_c ! ! ! !M M M3      r-   rf   _TToken)boundc                       e Zd ZU dZdZdZeed<   ee	         ed<   e
ed<   ee	         ed<   ee	         ed<   ee	         ed	<   ee	         ed
<   ee	         ed<   e	 	 	 	 	 	 ddede
dee	         dee	         dee	         d	ee	         d
ee	         dee	         dd fd            Ze	 	 	 	 	 	 ddede
dee	         dee	         dee	         d	ee	         d
ee	         dee	         dd fd            Zd Zed fd	            Zeddee         dee
         dd fd            Zeddee         dee
         dd fd            Zd Zddee         dee
         dd fdZedee         dede
dd def
d            Zd Zd Zd Zd Zej        Z xZS ) ru   a  A string with meta-information, that is produced by the lexer.

    When parsing text, the resulting chunks of the input that haven't been discarded,
    will end up in the tree as Token instances. The Token class inherits from Python's ``str``,
    so normal string comparisons and operations will work as expected.

    Attributes:
        type: Name of the token (as specified in grammar)
        value: Value of the token (redundant, as ``token.value == token`` will always be true)
        start_pos: The index of the token in the text
        line: The line of the token in the text (starting with 1)
        column: The column of the token in the text (starting with 1)
        end_line: The line where the token ends
        end_column: The next column after the end of the token. For example,
            if the token is a single character with a column value of 4,
            end_column will be 5.
        end_pos: the index where the token ends (basically ``start_pos + len(token)``)
    )r$   	start_posr!   linecolumnend_line
end_columnend_posr$   r!   r$   rx   r!   ry   rz   r{   r|   r}   Nr&   c	                     d S r(   r%   )	clsr$   r!   rx   ry   rz   r{   r|   r}   s	            r+   __new__zToken.__new__   s	     	r-   type_c	                     d S r(   r%   )	r   r   r!   rx   ry   rz   r{   r|   r}   s	            r+   r   zToken.__new__   s	     r-   c                     d|v rEt          j        dt                     d|v rt          d          |                    d          |d<    | j        |i |S Nr   z(`type_` is deprecated use `type` insteadr$   zAError: using both 'type' and the deprecated 'type_' as arguments.)warningswarnDeprecationWarning	TypeErrorpop_future_new)r   argskwargss      r+   r   zToken.__new__   sg    fMDFXYYY cddd#ZZ00F6Ns////r-   c	                     t          t          |                               | |          }	||	_        ||	_        ||	_        ||	_        ||	_        ||	_        ||	_	        ||	_
        |	S r(   )superru   r   r$   rx   r!   ry   rz   r{   r|   r}   )r   r$   r!   rx   ry   rz   r{   r|   r}   inst	__class__s             r+   r   zToken._future_new   s_    UC  ((e44	"
	 $r-   c                     d S r(   r%   r*   r$   r!   s      r+   updatezToken.update       r-   c                     d S r(   r%   )r*   r   r!   s      r+   r   zToken.update   r   r-   c                     d|v rEt          j        dt                     d|v rt          d          |                    d          |d<    | j        |i |S r   )r   r   r   r   r   _future_update)r*   r   r   s      r+   r   zToken.update   sh    fMDFXYYY cddd#ZZ00F6N"t"D3F333r-   c                 ^    t                               ||n| j        ||n| j        |           S r(   )ru   new_borrow_posr$   r!   r   s      r+   r   zToken._future_update   s9    ##$DD$)&EEDJ
 
 	
r-   r   borrow_tc           
      b     | |||j         |j        |j        |j        |j        |j                  S r(   )rx   ry   rz   r{   r|   r}   )r   r   r!   r   s       r+   r   zToken.new_borrow_pos   sD    s5%!3X]HOU]Ufhph{  ~F  ~N  O  O  	Or-   c                 P    | j         | j        | j        | j        | j        | j        ffS r(   )r   r$   r!   rx   ry   rz   r1   s    r+   
__reduce__zToken.__reduce__   s&    DJ	SWS^ _``r-   c                 (    d| j         d| j        dS )NzToken(rn   rD   r~   r1   s    r+   r2   zToken.__repr__  s     "&)))TZZZ88r-   c                 Z    t          | j        | j        | j        | j        | j                  S r(   )ru   r$   r!   rx   ry   rz   )r*   memos     r+   __deepcopy__zToken.__deepcopy__  s"    TY
DNDIt{SSSr-   c                     t          |t                    r| j        |j        k    rdS t                              | |          S )NF)rk   ru   r$   rK   r9   r7   s     r+   r9   zToken.__eq__	  s;    eU## 		UZ(?(?5zz$&&&r-   )NNNNNNNN)rG   rH   rI   rJ   	__slots____match_args__rK   rL   r   rN   r   r   r   classmethodr   r   r   r   rt   r   r   r2   r   r9   r5   __classcell__)r   s   @r+   ru   ru      s         $ fI&N
III}JJJ
3-SMsmc] 
 (,"&$(&*(,%)    }	
 3- SM sm ! c] 
   X 
 (,"&$(&*(,%)
 

 
  }	

 3-
 SM
 sm
 !
 c]
 

 
 
 X
0 0 0      [  8C=  QX    X  HSM # RY    X4 4 4
 
8C= 
 
Y` 
 
 
 
 ODH OS O O OTV O O O [Oa a a9 9 9T T T' ' ' |HHHHHr-   c                   0    e Zd ZdZdZd Zd Zd	defdZdS )
LineCounterz>A utility class for keeping track of line & column information)char_posry   rz   line_start_posnewline_charc                 L    || _         d| _        d| _        d| _        d| _        d S )Nr   r   )r   r   ry   rz   r   )r*   r   s     r+   r,   zLineCounter.__init__  s-    (	r-   c                 z    t          |t                    st          S | j        |j        k    o| j        |j        k    S r(   )rk   r   NotImplementedr   r   r7   s     r+   r9   zLineCounter.__eq__  s9    %-- 	"!!}.Z43DHZ3ZZr-   Ttokenc                    |rV|                     | j                  }|r:| xj        |z  c_        | j        |                    | j                  z   dz   | _        | xj        t          |          z  c_        | j        | j        z
  dz   | _        dS )zConsume a token and calculate the new line & column.

        As an optional optimization, set test_newline=False if token doesn't contain a newline.
        r   N)countr   ry   r   rindexr   rV   rz   )r*   r   test_newlinenewliness       r+   feedzLineCounter.feed$  s    
  	Z{{4#455H Z		X%		&*mell4CT6U6U&UXY&Y#U#md&99A=r-   N)T)	rG   rH   rI   rJ   r   r,   r9   ru   r   r%   r-   r+   r   r     s`        DDNI     [ [ [> >% > > > > > >r-   r   c                       e Zd Zd Zd ZdS )UnlessCallbackc                     || _         d S r(   )scanner)r*   r   s     r+   r,   zUnlessCallback.__init__4  s    r-   c                 ^    | j                             |j        d          }|r
|\  }|_        |S ra   )r   matchr!   r$   )r*   tres_values       r+   __call__zUnlessCallback.__call__7  s3    l  !,, 	! NFAFr-   NrG   rH   rI   r,   r   r%   r-   r+   r   r   3  s2              r-   r   c                       e Zd Zd Zd ZdS )	CallChainc                 0    || _         || _        || _        d S r(   )	callback1	callback2cond)r*   r   r   r   s       r+   r,   zCallChain.__init__?  s    ""			r-   c                     |                      |          }|                     |          r|                     |          n|S r(   )r   r   r   )r*   r   t2s      r+   r   zCallChain.__call__D  s9    ^^A$(IIbMM9t~~a   r9r-   Nr   r%   r-   r+   r   r   >  s2          
: : : : :r-   r   c                 b    |                      |||          }|r|                    d          S d S ra   )r   group)re_regexpsr"   ms        r+   
_get_matchr   I  s9    		&!U##A wwqzz r-   c           
         t          | d           }t          |          dk    sJ |                                            t                      i }|                    t
          g           D ]}g }|                    t          g           D ]}|j        |j        k    r|j        j	        }	|	t          ||j                                        |	|          k    rD|                    |           |j        j        |j        j        k    r                    |           |r)t          t!          |||d|                    ||j        <   ܈fd| D             }
|
|fS )Nc                 *    t          | j                  S r(   )r$   rh   r   s    r+   <lambda>z _create_unless.<locals>.<lambda>O  s    4	?? r-      T)match_whole	use_bytesc                     g | ]}|v|	S r%   r%   ).0r   embedded_strss     r+   
<listcomp>z"_create_unless.<locals>.<listcomp>`  s#    DDD1Qm-C-CQ-C-C-Cr-   )r   rV   keyssetgetr[   rP   ri   rh   r!   r   r0   appendr"   addr   Scannerrg   )	terminalsg_regex_flagsr   r   tokens_by_typecallbackretokunlessstrtokr   new_terminalsr   s              @r+   _create_unlessr   N  sm   i)B)BCCN~!###^%8%8%:%:###EEMH##Ir22 ~ ~$((R88 	. 	.F%.00$AJsEM$;$;$=$=q-PPPPf%%%>'5=+>>>!%%f--- 	~#1'&-QTbfr{2|2|2|#}#}HUZ DDDD	DDDM(""r-   c                   "    e Zd ZddZd Zd ZdS )r   Fc                     || _         || _        || _        || _        || _        d | j         D             | _        |                     |t          |                    | _        d S )Nc                     h | ]	}|j         
S r%   rg   r   r   s     r+   	<setcomp>z#Scanner.__init__.<locals>.<setcomp>l  s    ===af===r-   )	r   r   r   r   r   allowed_types_build_mresrV   _mres)r*   r   r   r   r   r   s         r+   r,   zScanner.__init__e  s`    "*"&==dn===%%iY@@


r-   c                    | j         rdndg }|rd                    fd|d |         D                       }| j        r|                    d          }	 | j                            || j                  }n)# t          $ r |                     ||dz            cY S w xY w|	                    |           ||d          }||S )N$ |c              3   h   K   | ],}d |j         d|j                                        z   dV  -dS )z(?P<>rD   N)rg   rh   r0   )r   r   postfixs     r+   	<genexpr>z&Scanner._build_mres.<locals>.<genexpr>w  sL      xx^_^_9L9L9N9NQX9X9X9X Yxxxxxxr-   zlatin-1r   )
r   joinr   encoder   compiler   AssertionErrorr   r   )r*   r   max_sizemresrh   mrer   s         @r+   r   zScanner._build_mresp  s    )1##r 
	-iixxxxclmvnvmvcwxxxxxG~ 4!..33Bh&&w0BCC! B B B''	8q=AAAAAB KK!()),I  
	- s    A8 8#BBc                     | j         D ]8}|                    ||          }|r|                    d          |j        fc S 9d S ra   )r   r   r   	lastgroup)r*   textposr  r   s        r+   r   zScanner.match  sZ    : 	/ 	/C		$$$A /wwqzz1;..../	/ 	/r-   N)F)rG   rH   rI   r,   r   r   r%   r-   r+   r   r   d  sJ        	A 	A 	A 	A  &/ / / / /r-   r   rc                 2    d| v pd| v pd| v pd| v pd| v od| v S )zExpressions that may indicate newlines in a regexp:
        - newlines (\n)
        - escaped newline (\\n)
        - anything but ([^...])
        - any-char (.) when the flag (?s) exists
        - spaces (\s)
    
z\nz\sz[^z(?s.r%   )r  s    r+   _regexp_has_newliner    s>     19Z
ZeqjZDAIZ%1*BYQTXYQYZr-   c                   |    e Zd ZU dZdZeed<   eed<   ee	         ed<   d
dedee         dee	         fdZ
d Zd	 ZdS )
LexerStatezRepresents the current state of the lexer as it scans the text
    (Lexer objects are only instantiated per grammar, not per text)
    )r  line_ctr
last_tokenr  r  r  Nc                 |    || _         |p%t          t          |t                    rdnd          | _        || _        d S )N   
r
  )r  r   rk   bytesr  r  )r*   r  r  r  s       r+   r,   zLexerState.__init__  s<    	 [KD%9P9P0ZVZ$[$[$r-   c                     t          |t                    st          S | j        |j        u o| j        |j        k    o| j        |j        k    S r(   )rk   r  r   r  r  r  r7   s     r+   r9   zLexerState.__eq__  sG    %,, 	"!!yEJ&r4=EN+JrtbgbrOrrr-   c                 n     t          |           | j        t          | j                  | j                  S r(   )r$   r  r   r  r  r1   s    r+   __copy__zLexerState.__copy__  s*    tDzz$)T$-%8%8$/JJJr-   r   )rG   rH   rI   rJ   r   rK   rL   r   r   ru   r,   r9   r  r%   r-   r+   r  r    s           1I
III% %S %H[,A %T\]bTc % % % %
s s sK K K K Kr-   r  c                   V    e Zd ZdZdddefdZedddedd fd            Zd	 Z	d
 Z
eZdS )LexerThreadzTA thread that ties a lexer instance and a lexer state, to be used by the parser
    lexerLexerlexer_statec                 "    || _         || _        d S r(   )r  state)r*   r  r  s      r+   r,   zLexerThread.__init__  s    
 


r-   r  r&   c                 4     | |t          |                    S r(   r  )r   r  r  s      r+   	from_textzLexerThread.from_text  s    s5*T**+++r-   c                 B    | j                             | j        |          S r(   )r  lexr  )r*   parser_states     r+   r"  zLexerThread.lex  s    z~~dj,777r-   c                 b     t          |           | j        t          | j                            S r(   )r$   r  r   r  r1   s    r+   r  zLexerThread.__copy__  s&    tDzz$*d4:&6&6777r-   N)rG   rH   rI   rJ   r  r,   r   rK   r   r"  r  ru   _Tokenr%   r-   r+   r  r    s         !g !J ! ! ! ! ,g ,S ,] , , , [,8 8 88 8 8 FFFr-   r  c                   H    e Zd ZdZedededee         fd            Z	d Z
dS )r  zlLexer interface

    Method Signatures:
        lex(self, lexer_state, parser_state) -> Iterator[Token]
    r  r#  r&   c                     t           S r(   )r   )r*   r  r#  s      r+   r"  z	Lexer.lex  s    r-   c                      t          |          S )
Deprecatedr  )r*   r  s     r+   make_lexer_statezLexer.make_lexer_state  s    $r-   N)rG   rH   rI   rJ   r   r  r   r	   ru   r"  r*  r%   r-   r+   r  r    sf         
 z  %    ^         r-   r     terminal_to_regexpc                    |st           j                            |           }|rdnd}|                                |k    rd S t	          | d                                           D ]}|                    |d          D ]\  }}|j        |j        k    sJ |                    ||           d|j	         d|j	         d}	 |
                    |||                                          }	n# t          $ r d	}	Y nw xY w|rt          | d
|	           t          j        d||	           |                                |k    rt          j        d             d S ڌd S )Nr   g?c                     | j         S r(   )ri   r   s    r+   r   z)_check_regex_collisions.<locals>.<lambda>  s    
 r-   T)skip_markedzCollision between Terminals z and z. zONo example could be found fast enough. However, the collision does still existsr
  z5%s The lexer will choose between them arbitrarily.
%sz2Found 8 regex collisions, will not check for more.)interegular
Comparatorfrom_regexescount_marked_pairsr   valuescheckri   markrg   get_example_overlapformat_multiline
ValueErrorr   r   warning)
r,  
comparatorstrict_modemax_collisions_to_showmax_timer   abmessageexamples
             r+   _check_regex_collisionsrC    s    M +889KLL
  (qqSH $$&&*@@@,.B.BCCJJLL  $$U$== 	 	DAq:++++OOAq!!! MQVLL!&LLLGl$88AxHHYY[[ l l lkl  8'66W66777NSU\^efff,,..2HHHSTTT I	 s   ?)C))C87C8c                       e Zd ZU eeef         ed<   eddd            Zedde	de
defd	            Zd
e	de
dee         fdZdS )AbstractBasicLexerterminals_by_nameNconfr   r&   c                     d S r(   r%   )r*   rG  r;  s      r+   r,   zAbstractBasicLexer.__init__  r   r-   	lex_stater#  c                     d S r(   r%   )r*   rI  r#  s      r+   
next_tokenzAbstractBasicLexer.next_token  r   r-   r  c              #   ~   K   t          t                    5  	 |                     ||          V  # 1 swxY w Y   d S r(   )r   EOFErrorrK  )r*   r  r#  s      r+   r"  zAbstractBasicLexer.lex   st      h 	; 	;;ooe\:::::;	; 	; 	; 	; 	; 	; 	; 	; 	; 	;s   266r(   rG  r   r&   N)rG   rH   rI   r   rK   rf   rL   r   r,   r  r   ru   rK  r	   r"  r%   r-   r+   rE  rE    s         C,----    ^  J c U    ^; ;3 ;8E? ; ; ; ; ; ;r-   rE  c                       e Zd ZU ee         ed<   ee         ed<   ee         ed<   eee	f         ed<   eee	f         ed<   e
ed<   dddZd Zed             Zd Zddeded
efdZdS )
BasicLexerr   ignore_typesnewline_typesuser_callbacksr   rR   NrG  r   r&   c                 4   t          |j                  }t          d |D                       s
J |            |j        | _        |j        s@i }|D ]}|j                                        }	 | j                            ||j	                   n2# | j        j
        $ r  t          d|j        d|j                  w xY w|j        j        dk    r t          d|j        d|j        d          |j        j        dk    r|||<   t          |j                  d |D             k    s1t          d	t          |j                  d
 |D             z
  z            t"          rt%          |||j                   n|j        rt          d          t)          d |D                       | _        t)          |j                  | _        |                    d            || _        |j        | _        |j	        | _	        |j        | _        |j        | _        d | _        d S )Nc              3   @   K   | ]}t          |t                    V  d S r(   )rk   rf   r   s     r+   r   z&BasicLexer.__init__.<locals>.<genexpr>  s,      AA!:a--AAAAAAr-   zCannot compile token z: r   z,Lexer does not allow zero-width terminals. (rD   rR   c                     h | ]	}|j         
S r%   r   r   s     r+   r   z&BasicLexer.__init__.<locals>.<setcomp>#  s    (C(C(CA(C(C(Cr-   z$Ignore terminals are not defined: %sc                     h | ]	}|j         
S r%   r   r   s     r+   r   z&BasicLexer.__init__.<locals>.<setcomp>$  s    \w\w\whi]^]c\w\w\wr-   zUinteregular must be installed for strict mode. Use `pip install 'lark[interegular]'`.c              3   p   K   | ]1}t          |j                                                  (|j        V  2d S r(   )r  rh   r0   rg   r   s     r+   r   z&BasicLexer.__init__.<locals>.<genexpr>,  sB      &o&o!DWXYXaXkXkXmXmDnDn&oqv&o&o&o&o&o&or-   c                 j    | j          | j        j         t          | j        j                   | j        fS r(   )ri   rh   rA   rV   r!   rg   )xs    r+   r   z%BasicLexer.__init__.<locals>.<lambda>/  s.    qzkAI4G3G#aioJ^J^I^`a`f%g r-   )key)listr   all	re_modulerR   skip_validationrh   r0   r   r   errorr   rg   r>   r$   r   ignorehas_interegularrC  strictr)   rR  rQ  sort	callbacksrS  r   rF  _scanner)r*   rG  r;  r   r,  r   r   s          r+   r,   zBasicLexer.__init__  sK   ((	AAyAAAAALL9LLA.# 	x!# 
3 
3,,..XGOOFD,>????w} X X X"(AFFFAII#VWWWX 9&!++"(\]\b\b\bdedmdmdm#nooo9>T)),2&q)$$(C(C(C(C(CCCET[IYIY\w\wmv\w\w\wIwxyyy x'(:JTTTT xvwww '&o&oy&o&o&ooo%dk22gghhh""n!/!%!7s   , B/B<c                    t          | j        | j        | j        | j                  \  }| _        t          | j                                                  sJ | j        	                                D ]@\  }| j        v r(t          | j                 |fd          | j        <   6|| j        <   At          || j        | j        | j                  | _        d S )Nc                     | j         k    S r(   )r$   )r   r   s    r+   r   z+BasicLexer._build_scanner.<locals>.<lambda>?  s    TUTZ^cTc r-   )r   r   r   rR   r   r   r]  r4  rS  itemsr   r   rf  )r*   r   rE   r   s      @r+   _build_scannerzBasicLexer._build_scanner8  s    #1$.$BTVZV]_c_m#n#n 	4=4=''))*****+1133 	) 	)HE1%%'0u1EqJcJcJcJc'd'de$$'(e$$	4+=twWWr-   c                 F    | j         |                                  | j         S r(   )rf  rj  r1   s    r+   r   zBasicLexer.scannerE  s$    = !!!}r-   c                 8    | j                             ||          S r(   )r   r   )r*   r  r  s      r+   r   zBasicLexer.matchK  s    |!!$,,,r-   rI  r#  c           
      L   |j         }|j        t          |j                  k     rq|                     |j        |j                  }|sV| j        j        | j        z
  }|sdh}t          |j        |j        |j	        |j
        ||j        o|j        g|| j                  |\  }}|| j        v }d }	|r	|| j        v r"t          |||j        |j	        |j
                  }	|                    ||| j        v            |	|j	        |	_        |j
        |	_        |j        |	_        |	j        | j        v r | j        |	j                 |	          }	|s0t+          |	t                    st-          d|	z            |	|_        |	S |j        t          |j                  k     qt/          |           )Nz<END-OF-FILE>)allowedtoken_historyr  rF  z+Callbacks must return a token (returned %r))r  r   rV   r  r   r   r   rQ  r   ry   rz   r  rF  r   ru   r   rR  r{   r|   r}   r$   rk   r   rM  )
r*   rI  r#  r  r   rn  r!   r   ignoredr   s
             r+   rK  zBasicLexer.next_tokenN  s   %#in"5"555**Y^X->??C i,4t7HH 0./G*9>8;Lhm]e]l3:)J^Jydmdxcy1=QUQgi i i i LE5t00GA [et}44%(98=(/ZZMM%$*<!<===}%]
'$-	6T]**-af-a00A %a// Z&'TWX'XYYY+,I(H7 #in"5"555< tnnr-   r(   rN  )rG   rH   rI   r
   rf   rL   r   rK   r   	_Callbackr   r,   rj  rM   r   r   r  r   ru   rK  r%   r-   r+   rP  rP    s         +&&&&C.   S>!!!i((((3	>""""NNN( ( ( ( (TX X X   X
- - -   J  c  U            r-   rP  c            	           e Zd ZU eeef         ed<   eed<   eZee         ed<   ddddeee	e
         f         de	e
         d	d
fdZdeddd	ee         fdZd
S )ContextualLexerlexers
root_lexerrP  r%   rG  r   statesalways_acceptr&   Nc                    t          |j                  }|j        t          |          }||_        t          r1|j        s*t          j                            d |D                       }nd }i }i | _	        |
                                D ]\  }}	t          |	          }
	 ||
         }n# t          $ rt t          |	          t          |j                  z  t          |          z  }	t          |          }fd|	D             |_        |                     ||          }|||
<   Y nw xY w|| j	        |<   |j        |u sJ d|_        |                     ||          | _        d S )Nc                 B    i | ]}||j                                         S r%   )rh   r0   r   s     r+   
<dictcomp>z,ContextualLexer.__init__.<locals>.<dictcomp>  s)    =j=j=j[\aATATAVAV=j=j=jr-   c                 (    g | ]}|v |         S r%   r%   )r   nrF  s     r+   r   z,ContextualLexer.__init__.<locals>.<listcomp>  s+    'h'h'hQRVgQgQg(9!(<QgQgQgr-   T)r\  r   rF  r   rb  r_  r0  r1  r2  rt  ri  r)   KeyErrorr   ra  rP  ru  )r*   rG  rv  rw  r   	trad_confr;  lexer_by_tokensr  acceptsr[  r  
lexer_confrF  s                @r+   r,   zContextualLexer.__init__w  s   ((	 2JJ	'	 	4#7 	$/<<=j=j`i=j=j=jkkJJJDF$llnn 	' 	'NE7G$$C-', - - -g,,T[)9)99C<N<NN!)__
'h'h'h'hg'h'h'h
$
J??',$$$- "'DK"i////$(	!//)Z@@s   B((A;D&%D&r  r#  r   c              #   J  K   	 	 | j         |j                 }|                    ||          V  +# t          $ r Y d S t          $ r\}	 |j        }| j                            ||          }t          ||j        ||g| j        j	                  # t          $ r |w xY wd }~ww xY w)NT)r  ro  rF  )
rt  positionrK  rM  r   r  ru  r   rn  rF  )r*   r  r#  r  er  r   s          r+   r"  zContextualLexer.lex  s      	BL$9:&&{LAAAAAB  	 	 	DD# 	 	 	(3
22;MM%eQYl[eZfz~  {J  {\  ]  ]  ]  ]'   	s'   ,0 
B"	B"ABBBB")r%   )rG   rH   rI   r   rN   rE  rL   rP  r   r
   rK   r,   r  r	   ru   r"  r%   r-   r+   rs  rs  q  s         (())))""""+5J'(555A A[ A$sJsO7K2L A]ghk]l Atx A A A A<z  8TY?      r-   rs  )r+  )Aabcr   r   rR   
contextlibr   typingr   r   r   r	   r
   r   r   r   r   r   r   r   typesr   r   r0  ImportErrorcommonr   parsers.lalr_parser_stater   utilsr   r   r   r   
exceptionsr   r   r   grammarr   r   boolrb  	NameErrorr    rP   r[   rf   rt   rK   ru   r   r   r   r   r   r   r  r  r  rq  r  rC  rE  rP  rs  r%   r-   r+   <module>r     s
   $ # # # # # # # 				                                        	 	 	 	D	 7!!!!!!666666 @ @ @ @ @ @ @ @ @ @ @ @ G G G G G G G G G G + + + + + +      d;''OO   OOO( ( ( ( (i ( ( (V       "$ $ $ $ $ $ $ $.    )   0 WT!!!A A A A AC A A AH> > > > > > > >B       : : : : : : : :  
# # #,#/ #/ #/ #/ #/ #/ #/ #/L[3 [ [ [ [K K K K K K K K4       * eWe^$	         C       [#5E0F    @; ; ; ; ; ; ; ;"h h h h h# h h hV3 3 3 3 3e 3 3 3 3 3s!   ? AA;B BB