o
    bi]                     @   sN  d dl mZmZ d dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlZzd dlZW n	 eyG   Y nw erVddlmZ ddlmZ dd	lmZmZmZm Z  dd
l!m"Z"m#Z#m$Z$ ddl%m&Z& d dl'm'Z' ze(eZ)W n e*y   dZ)Y nw G dd deeZ+G dd de+Z,G dd de+Z-G dd deZ.edddZ/G dd de0Z1G dd dZ2G dd dZ3G dd dZ4d d! Z5d"d# Z6G d$d% d%Z7d&e0fd'd(Z8G d)d* d*Z9G d+d, d,Z:ee1ge1f Z;G d-d. d.eZ<d9d0e	e.e0f fd1d2Z=G d3d4 d4e<Z>G d5d6 d6e>Z?G d7d8 d8e<Z@dS ):    )abstractmethodABCN)suppress)TypeVarTypeDictIterator
CollectionCallableOptional	FrozenSetAnyClassVarTYPE_CHECKINGoverload)
ModuleType   )	LexerConf)ParserState)classifyget_regexp_width	Serializelogger)UnexpectedCharactersLexErrorUnexpectedToken)TOKEN_DEFAULT_PRIORITY)copyFc                	   @   s   e Zd ZU dZeed< ee ed< ee ed< ee ed< ddedee dee ddfd	d
Z	dd Z
dd Zdd ZedefddZeedefddZeedefddZdd ZdS )Patternz(An abstraction over regular expressions.valueflagsrawtype Nreturnc                 C   s   || _ t|| _|| _d S N)r   	frozensetr    r!   )selfr   r    r!   r#   r#   >/home/ubuntu/.local/lib/python3.10/site-packages/lark/lexer.py__init__)   s   

zPattern.__init__c                 C   s   t |  S r%   )repr	to_regexpr'   r#   r#   r(   __repr__.      zPattern.__repr__c                 C   s   t t| | j| jfS r%   )hashr"   r   r    r,   r#   r#   r(   __hash__2      zPattern.__hash__c                 C   s(   t | t |ko| j|jko| j|jkS r%   )r"   r   r    r'   otherr#   r#   r(   __eq__5   s   (zPattern.__eq__c                 C      t  r%   NotImplementedErrorr,   r#   r#   r(   r+   8   s   zPattern.to_regexpc                 C   r5   r%   r6   r,   r#   r#   r(   	min_width<      zPattern.min_widthc                 C   r5   r%   r6   r,   r#   r#   r(   	max_widthA   r9   zPattern.max_widthc                 C   s   | j D ]}d||f }q|S )Nz(?%s:%s))r    )r'   r   fr#   r#   r(   
_get_flagsF   s   
zPattern._get_flags)r#   N)__name__
__module____qualname____doc__str__annotations__r	   r   r   r)   r-   r0   r4   r   r+   propertyintr8   r:   r<   r#   r#   r#   r(   r   !   s&   
 $r   c                   @   sT   e Zd ZU dZdZee ed< defddZe	de
fddZe	de
fd	d
ZdS )
PatternStr)r   r    r!   rA   r"   r$   c                 C   s   |  t| jS r%   )r<   reescaper   r,   r#   r#   r(   r+   Q   s   zPatternStr.to_regexpc                 C   
   t | jS r%   lenr   r,   r#   r#   r(   r8   T      
zPatternStr.min_widthc                 C   rH   r%   rI   r,   r#   r#   r(   r:   X   rK   zPatternStr.max_widthN)r=   r>   r?   __serialize_fields__r"   r   rA   rB   r+   rC   rD   r8   r:   r#   r#   r#   r(   rE   L   s   
 rE   c                   @   s`   e Zd ZU dZdZee ed< defddZdZ	dd	 Z
edefd
dZedefddZdS )	PatternRE)r   r    r!   _widthrF   r"   r$   c                 C   s   |  | jS r%   )r<   r   r,   r#   r#   r(   r+   b   r.   zPatternRE.to_regexpNc                 C   s   | j d u rt|  | _ | j S r%   )rN   r   r+   r,   r#   r#   r(   
_get_widthf   s   
zPatternRE._get_widthc                 C      |   d S Nr   rO   r,   r#   r#   r(   r8   k      zPatternRE.min_widthc                 C   rP   )Nr   rR   r,   r#   r#   r(   r:   o   rS   zPatternRE.max_width)r=   r>   r?   rL   r"   r   rA   rB   r+   rN   rO   rC   rD   r8   r:   r#   r#   r#   r(   rM   ]   s   
 rM   c                	   @   sj   e Zd ZU dZdZeefZee	d< e
e	d< ee	d< efdede
deddfdd	Zd
d ZdefddZdS )TerminalDefzA definition of a terminal)namepatternpriorityrU   rV   rW   r$   Nc                 C   s(   t |ts	J ||| _|| _|| _d S r%   )
isinstancer   rU   rV   rW   )r'   rU   rV   rW   r#   r#   r(   r)   }   s   
zTerminalDef.__init__c                 C   s   dt | j| j| jf S )Nz
%s(%r, %r))r"   r=   rU   rV   r,   r#   r#   r(   r-      s   zTerminalDef.__repr__c                 C   s    | j dr| jjp| j S | j S )N__)rU   
startswithrV   r!   r,   r#   r#   r(   	user_repr   s   zTerminalDef.user_repr)r=   r>   r?   r@   rL   rE   rM   __serialize_namespace__rA   rB   r   rD   r   r)   r-   r[   r#   r#   r#   r(   rT   t   s   
 rT   _TToken)boundc                       s  e Zd ZU dZdZdZeed< ee	 ed< e
ed< ee	 ed< ee	 ed< ee	 ed	< ee	 ed
< ee	 ed< e						d'dede
dee	 dee	 dee	 d	ee	 d
ee	 dee	 dd fddZe						d'dede
dee	 dee	 dee	 d	ee	 d
ee	 dee	 dd fddZdd Zed' fdd	Zed(dee dee
 dd fddZed(dee dee
 dd fddZdd Zd(dee dee
 dd fddZedee dede
dd def
ddZdd  Zd!d" Zd#d$ Zd%d& ZejZ  ZS ))r^   a  A string with meta-information, that is produced by the lexer.

    When parsing text, the resulting chunks of the input that haven't been discarded,
    will end up in the tree as Token instances. The Token class inherits from Python's ``str``,
    so normal string comparisons and operations will work as expected.

    Attributes:
        type: Name of the token (as specified in grammar)
        value: Value of the token (redundant, as ``token.value == token`` will always be true)
        start_pos: The index of the token in the text
        line: The line of the token in the text (starting with 1)
        column: The column of the token in the text (starting with 1)
        end_line: The line where the token ends
        end_column: The next column after the end of the token. For example,
            if the token is a single character with a column value of 4,
            end_column will be 5.
        end_pos: the index where the token ends (basically ``start_pos + len(token)``)
    )r"   	start_posr   linecolumnend_line
end_columnend_posr"   r   r"   r`   r   ra   rb   rc   rd   re   Nr$   c	           	      C      d S r%   r#   )	clsr"   r   r`   ra   rb   rc   rd   re   r#   r#   r(   __new__   s   zToken.__new__type_c	           	      C   rg   r%   r#   )	rh   rj   r   r`   ra   rb   rc   rd   re   r#   r#   r(   ri      s   c                 O   B   d|v rt dt d|v rtd|d|d< | j|i |S Nrj   z(`type_` is deprecated use `type` insteadr"   zAError: using both 'type' and the deprecated 'type_' as arguments.)warningswarnDeprecationWarning	TypeErrorpop_future_new)rh   argskwargsr#   r#   r(   ri         c	           
         sF   t t| | |}	||	_||	_||	_||	_||	_||	_||	_	||	_
|	S r%   )superr^   ri   r"   r`   r   ra   rb   rc   rd   re   )
rh   r"   r   r`   ra   rb   rc   rd   re   inst	__class__r#   r(   rr      s   zToken._future_newc                 C   rg   r%   r#   r'   r"   r   r#   r#   r(   update      zToken.updatec                 C   rg   r%   r#   )r'   rj   r   r#   r#   r(   r{      r|   c                 O   rk   rl   )rm   rn   ro   rp   rq   _future_update)r'   rs   rt   r#   r#   r(   r{      ru   c                 C   s.   t |d ur|n| j|d ur|| S | j| S r%   )r^   new_borrow_posr"   r   rz   r#   r#   r(   r}      s   
zToken._future_updaterh   borrow_tc              	   C   s"   | |||j |j|j|j|j|jS r%   )r`   ra   rb   rc   rd   re   )rh   rj   r   r   r#   r#   r(   r~      s   "zToken.new_borrow_posc                 C   s   | j | j| j| j| j| jffS r%   )ry   r"   r   r`   ra   rb   r,   r#   r#   r(   
__reduce__   s   zToken.__reduce__c                 C   s   d| j | jf S )NzToken(%r, %r)rf   r,   r#   r#   r(   r-        zToken.__repr__c                 C   s   t | j| j| j| j| jS r%   )r^   r"   r   r`   ra   rb   )r'   memor#   r#   r(   __deepcopy__     zToken.__deepcopy__c                 C   s&   t |tr| j|jkrdS t| |S )NF)rX   r^   r"   rA   r4   r2   r#   r#   r(   r4   	  s   zToken.__eq__)NNNNNNNN)r=   r>   r?   r@   	__slots____match_args__rA   rB   r   rD   r   r   ri   classmethodrr   r{   r}   r   r]   r~   r   r-   r   r4   r0   __classcell__r#   r#   rx   r(   r^      s   
 	
	
"" 
$c                   @   s4   e Zd ZdZdZdd Zdd Zddefd	d
ZdS )LineCounterz>A utility class for keeping track of line & column information)char_posra   rb   line_start_posnewline_charc                 C   s"   || _ d| _d| _d| _d| _d S )Nr   r   )r   r   ra   rb   r   )r'   r   r#   r#   r(   r)     s
   
zLineCounter.__init__c                 C   s&   t |tstS | j|jko| j|jkS r%   )rX   r   NotImplementedr   r   r2   r#   r#   r(   r4     s   
zLineCounter.__eq__Ttokenc                 C   sb   |r| | j}|r|  j|7  _| j|| j d | _|  jt|7  _| j| j d | _dS )zConsume a token and calculate the new line & column.

        As an optional optimization, set test_newline=False if token doesn't contain a newline.
        r   N)countr   ra   r   rindexr   rJ   rb   )r'   r   test_newlinenewlinesr#   r#   r(   feed$  s   zLineCounter.feedN)T)	r=   r>   r?   r@   r   r)   r4   r^   r   r#   r#   r#   r(   r     s    r   c                   @      e Zd Zdd Zdd ZdS )UnlessCallbackc                 C   s
   || _ d S r%   )scanner)r'   r   r#   r#   r(   r)   4  s   
zUnlessCallback.__init__c                 C   s"   | j |jd}|r|\}|_|S rQ   )r   matchr   r"   )r'   tres_valuer#   r#   r(   __call__7  s   
zUnlessCallback.__call__Nr=   r>   r?   r)   r   r#   r#   r#   r(   r   3  s    r   c                   @   r   )	CallChainc                 C   s   || _ || _|| _d S r%   )	callback1	callback2cond)r'   r   r   r   r#   r#   r(   r)   ?  s   
zCallChain.__init__c                 C   s"   |  |}| |r| |S |S r%   )r   r   r   )r'   r   t2r#   r#   r(   r   D  s   
zCallChain.__call__Nr   r#   r#   r#   r(   r   >  s    r   c                 C   s    |  |||}|r|dS d S rQ   )r   group)re_regexpsr    mr#   r#   r(   
_get_matchI  s   
r   c              	      s   t | dd }t|dksJ | t  i }|tg D ]F}g }|tg D ]+}|j|jkr1q(|jj	}	|	t
||j |	|krS|| |jj|jjkrS | q(|rdtt|||d|d||j< q fdd| D }
|
|fS )Nc                 S   rH   r%   )r"   rV   r   r#   r#   r(   <lambda>O     
 z _create_unless.<locals>.<lambda>   T)match_whole	use_bytesc                    s   g | ]}| vr|qS r#   r#   .0r   embedded_strsr#   r(   
<listcomp>`      z"_create_unless.<locals>.<listcomp>)r   rJ   keyssetgetrM   rE   rW   rV   r   r   r+   appendr    addr   ScannerrU   )	terminalsg_regex_flagsr   r   tokens_by_typecallbackretokunlessstrtokr   new_terminalsr#   r   r(   _create_unlessN  s(   

r   c                   @   s&   e Zd Zd	ddZdd Zdd ZdS )
r   Fc                 C   sF   || _ || _|| _|| _|| _dd | j D | _| |t|| _d S )Nc                 S      h | ]}|j qS r#   rU   r   r#   r#   r(   	<setcomp>l      z#Scanner.__init__.<locals>.<setcomp>)	r   r   r   r   r   allowed_types_build_mresrJ   _mres)r'   r   r   r   r   r   r#   r#   r(   r)   e  s   zScanner.__init__c                    s   | j rdnd g }|rLd fdd|d | D }| jr#|d}z
| j|| j}W n ty>   | ||d  Y S w |	| ||d  }|s|S )N$ |c                 3   s(    | ]}d |j |j   f V  qdS )z
(?P<%s>%s)N)rU   rV   r+   r   postfixr#   r(   	<genexpr>w  s   & z&Scanner._build_mres.<locals>.<genexpr>zlatin-1r   )
r   joinr   encoder   compiler   AssertionErrorr   r   )r'   r   max_sizemresrV   mrer#   r   r(   r   p  s    

zScanner._build_mresc                 C   s4   | j D ]}|||}|r|d|jf  S qd S rQ   )r   r   r   	lastgroup)r'   textposr   r   r#   r#   r(   r     s   
zScanner.matchN)F)r=   r>   r?   r)   r   r   r#   r#   r#   r(   r   d  s    
r   rc                 C   s0   d| v pd| v pd| v pd| v pd| v od| v S )zExpressions that may indicate newlines in a regexp:
        - newlines (\n)
        - escaped newline (\\n)
        - anything but ([^...])
        - any-char (.) when the flag (?s) exists
        - spaces (\s)
    
z\nz\sz[^z(?s.r#   )r   r#   r#   r(   _regexp_has_newline  s   0r   c                   @   sb   e Zd ZU dZdZeed< eed< ee	 ed< ddedee dee	 fddZ
d	d
 Zdd ZdS )
LexerStatezRepresents the current state of the lexer as it scans the text
    (Lexer objects are only instantiated per grammar, not per text)
    )r   line_ctr
last_tokenr   r   r   Nc                 C   s,   || _ |ptt|trdnd| _|| _d S )N   
r   )r   r   rX   bytesr   r   )r'   r   r   r   r#   r#   r(   r)     s   
zLexerState.__init__c                 C   s2   t |tstS | j|ju o| j|jko| j|jkS r%   )rX   r   r   r   r   r   r2   r#   r#   r(   r4     s   
$zLexerState.__eq__c                 C   s   t | | jt| j| jS r%   )r"   r   r   r   r   r,   r#   r#   r(   __copy__  r   zLexerState.__copy__r   )r=   r>   r?   r@   r   rA   rB   r   r   r^   r)   r4   r   r#   r#   r#   r(   r     s   
  r   c                   @   sP   e Zd ZdZdddefddZedddedd fd	d
Zdd Z	dd Z
eZdS )LexerThreadzTA thread that ties a lexer instance and a lexer state, to be used by the parser
    lexerLexerlexer_statec                 C   s   || _ || _d S r%   )r   state)r'   r   r   r#   r#   r(   r)     s   
zLexerThread.__init__r   r$   c                 C   s   | |t |S r%   r   )rh   r   r   r#   r#   r(   	from_text  s   zLexerThread.from_textc                 C   s   | j | j|S r%   )r   lexr   )r'   parser_stater#   r#   r(   r     r   zLexerThread.lexc                 C   s   t | | jt| jS r%   )r"   r   r   r   r,   r#   r#   r(   r     r1   zLexerThread.__copy__N)r=   r>   r?   r@   r   r)   r   rA   r   r   r   r^   _Tokenr#   r#   r#   r(   r     s    r   c                   @   s6   e Zd ZdZedededee fddZ	dd Z
d	S )
r   zlLexer interface

    Method Signatures:
        lex(self, lexer_state, parser_state) -> Iterator[Token]
    r   r   r$   c                 C   s   t S r%   )r   )r'   r   r   r#   r#   r(   r     r|   z	Lexer.lexc                 C   s   t |S )
Deprecatedr   )r'   r   r#   r#   r(   make_lexer_state  s   zLexer.make_lexer_stateN)r=   r>   r?   r@   r   r   r   r   r^   r   r   r#   r#   r#   r(   r     s
    r      terminal_to_regexpc           
   
   C   s  |st j| }|rdnd}| |krd S t| dd  D ]_}|j|ddD ]U\}}|j|jks4J ||| d|j	 d|j	 d	}z|
||| }	W n ty[   d
}	Y nw |rgt| d|	 td||	 | |kr}td   d S q(qd S )Nr   g?c                 S   s   | j S r%   )rW   r   r#   r#   r(   r     s    z)_check_regex_collisions.<locals>.<lambda>T)skip_markedzCollision between Terminals z and z. zONo example could be found fast enough. However, the collision does still existsr   z5%s The lexer will choose between them arbitrarily.
%sz2Found 8 regex collisions, will not check for more.)interegular
Comparatorfrom_regexescount_marked_pairsr   valuescheckrW   markrU   get_example_overlapformat_multiline
ValueErrorr   r   warning)
r   
comparatorstrict_modemax_collisions_to_showmax_timer   abmessageexampler#   r#   r(   _check_regex_collisions  s0   
r  c                   @   sd   e Zd ZU eeef ed< eddddZedde	d	e
defd
dZde	d	e
dee fddZdS )AbstractBasicLexerterminals_by_nameNconfr   r$   c                 C   rg   r%   r#   )r'   r
  r   r#   r#   r(   r)     r|   zAbstractBasicLexer.__init__	lex_stater   c                 C   rg   r%   r#   )r'   r  r   r#   r#   r(   
next_token  r|   zAbstractBasicLexer.next_tokenr   c                 c   s2    t t
 	 | ||V  q1 sw   Y  d S r%   )r   EOFErrorr  )r'   r   r   r#   r#   r(   r      s   
zAbstractBasicLexer.lexr%   r
  r   r$   N)r=   r>   r?   r   rA   rT   rB   r   r)   r   r   r^   r  r   r   r#   r#   r#   r(   r    s   
 r  c                   @   s   e Zd ZU ee ed< ee ed< ee ed< eee	f ed< eee	f ed< e
ed< ddddZdd Zedd Zdd Zddeded
efddZdS )
BasicLexerr   ignore_typesnewline_typesuser_callbacksr   rF   Nr
  r   r$   c              	   C   st  t |j}tdd |D sJ ||j| _|jsi }|D ]?}|j }z
| j||j	 W n | jj
yA   td|j|jf w |jjdkrRtd|j|jf |jjdkr\|||< qt|jdd |D ksytd	t|jd
d |D   trt|||j n|jrtdtdd |D | _t|j| _|jdd d || _|j| _|j	| _	|j| _|j| _d | _d S )Nc                 s   s    | ]}t |tV  qd S r%   )rX   rT   r   r#   r#   r(   r     s    z&BasicLexer.__init__.<locals>.<genexpr>zCannot compile token %s: %sr   z3Lexer does not allow zero-width terminals. (%s: %s)rF   c                 S   r   r#   r   r   r#   r#   r(   r   #  r   z&BasicLexer.__init__.<locals>.<setcomp>z$Ignore terminals are not defined: %sc                 S   r   r#   r   r   r#   r#   r(   r   $  r   zUinteregular must be installed for strict mode. Use `pip install 'lark[interegular]'`.c                 s   s$    | ]}t |j r|jV  qd S r%   )r   rV   r+   rU   r   r#   r#   r(   r   ,  s   " c                 S   s"   | j  | jj t| jj | jfS r%   )rW   rV   r:   rJ   r   rU   )xr#   r#   r(   r   /  s   " z%BasicLexer.__init__.<locals>.<lambda>)key)listr   all	re_modulerF   skip_validationrV   r+   r   r   errorr   rU   r8   r"   r   ignorehas_interegularr  strictr&   r  r  sort	callbacksr  r   r	  _scanner)r'   r
  r   r   r   r   r   r#   r#   r(   r)     s@   

 
zBasicLexer.__init__c                    s   t | j| j| j| j\}| _t| j sJ | j	 D ]\ } | jv r6t
| j  | fdd| j < q|| j < qt|| j| j| j| _d S )Nc                    s
   | j  kS r%   )r"   r   rj   r#   r(   r   ?  r   z+BasicLexer._build_scanner.<locals>.<lambda>)r   r   r   rF   r   r   r  r   r  itemsr   r   r  )r'   r   r;   r#   r   r(   _build_scanner8  s   
"zBasicLexer._build_scannerc                 C   s   | j d u r	|   | j S r%   )r  r"  r,   r#   r#   r(   r   E  s   
zBasicLexer.scannerc                 C   s   | j ||S r%   )r   r   )r'   r   r   r#   r#   r(   r   K  s   zBasicLexer.matchr  r   c           
   
   C   s2  |j }|jt|jk r| |j|j}|s7| jj| j }|s!dh}t|j|j|j	|j
||jo1|jg|| jd|\}}|| jv }d }	|rI|| jv rTt|||j|j	|j
}	|||| jv  |	d ur|j	|	_|j
|	_|j|	_|	j| jv r{| j|	j |	}	|st|	tstd|	 |	|_|	S |jt|jk st| )Nz<END-OF-FILE>)allowedtoken_historyr   r	  z+Callbacks must return a token (returned %r))r   r   rJ   r   r   r   r   r  r   ra   rb   r   r	  r   r^   r   r  rc   rd   re   r"   rX   r   r  )
r'   r  r   r   r   r#  r   rj   ignoredr   r#   r#   r(   r  N  s<   

zBasicLexer.next_tokenr%   r  )r=   r>   r?   r	   rT   rB   r   rA   r   	_Callbackr   r)   r"  rC   r   r   r   r   r^   r  r#   r#   r#   r(   r    s   
 *
r  c                	   @   s|   e Zd ZU eeef ed< eed< eZee ed< ddddeee	e
 f de	e
 d	d
fddZdeddd	ee fddZd
S )ContextualLexerlexers
root_lexerr  r#   r
  r   statesalways_acceptr$   Nc              	      s  t |j}|j t|}||_tr |js tjdd |D }nd }i }i | _	|
 D ]B\}}	t|	}
z||
 }W n. tyg   t|	t|jB t|B }	t|} fdd|	D |_| ||}|||
< Y nw || j	|< q+|j|u suJ d|_| ||| _d S )Nc                 S   s   i | ]}||j  qS r#   )rV   r+   r   r#   r#   r(   
<dictcomp>  r   z,ContextualLexer.__init__.<locals>.<dictcomp>c                    s   g | ]
}| v r | qS r#   r#   )r   nr	  r#   r(   r     s    z,ContextualLexer.__init__.<locals>.<listcomp>T)r  r   r	  r   r  r  r   r   r   r(  r!  r&   KeyErrorr   r  r  r)  )r'   r
  r*  r+  r   	trad_confr   lexer_by_tokensr   acceptsr  r   
lexer_confr#   r.  r(   r)   w  s0   

zContextualLexer.__init__r   r   r   c                 c   s    z	 | j |j }|||V  q ty   Y d S  tyF } z!z|j}| j||}t||j||g| jj	d tyA   |w d }~ww )NT)r   r$  r	  )
r(  positionr  r  r   r   r)  r   r#  r	  )r'   r   r   r   er   r   r#   r#   r(   r     s$   zContextualLexer.lex)r#   )r=   r>   r?   r   rD   r  rB   r  r   r	   rA   r)   r   r   r^   r   r#   r#   r#   r(   r'  q  s   
 ,r'  )r   )Aabcr   r   rF   
contextlibr   typingr   r   r   r   r	   r
   r   r   r   r   r   r   typesr   rm   r   ImportErrorcommonr   parsers.lalr_parser_stater   utilsr   r   r   r   
exceptionsr   r   r   grammarr   r   boolr  	NameErrorr   rE   rM   rT   r]   rA   r^   r   r   r   r   r   r   r   r   r   r&  r   r  r  r  r'  r#   r#   r#   r(   <module>   sZ   8+ !& k