o
    i                     @   s2  d Z ddlZddlZddlZddlmZ ddlmZmZ ddl	Z	ddl
mZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZmZmZmZmZmZmZmZ dd
l m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddlm4Z4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z? ddl@mAZAmBZBmCZCmDZDmEZE eCddZFdgZGdZHdZIe2dZJi dddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:i d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_ZKi d`dadbdcdddedfdgdhdidjdkdldmdnddoddpdqdrdsdtdudLdKdvdwdxdydzd{d|d}d~eI dddddddddddddZLi ddgdddgdg dddgddvdgdddgdddgddxdgddgdg ddddgddgdddgdg ddg ddg dddzgi ddxgddxdzgdddgddgddgddgdddgdddgdddgdddgddgddgdg dâdddgddgddgdddgdgddgddgdd|gdМZMdZNdZOG ddԄ deAZPeFG ddք deDZQG dd؄ deBZRG ddڄ deAZSG dd܄ deDZTG ddބ deDZUG dd deDZVdd ZWdd ZXdd ZYeFG dd deDZZde$fddZ[G dd deEZ\G dd deDZ]dd Z^G dd dZ_eddZ`G dd dZaeadeGZbdd Zcdd ZdeFG dd deDZedd  Zfdd Zgddgfddgfdg dfd	g d
fddgfddgfdg dfdg dfddgfddgfddgfddgfddgfgZhdd Zid;dd Zjd!d" Zkd#d$ Zld;d%emdemdeee;emf  fd&d'Znd<d(d)Zod*d+ Zpd,d- ZqG d.d/ d/ZrG d0d1 d1Zsd2d3 Ztg fd4d5Zud6d7 Zvd8emdemfd9d:ZwdS (=  zCParses and compiles Lark grammars into an internal representation.
    N)
namedtuple)copydeepcopy)literal_eval)suppress)ListTupleUnionCallableDictOptionalSequence	Generator   )bfsloggerclassify_boolis_id_continueis_id_startbfs_all_uniquesmall_factors
OrderedSet)TokenTerminalDef
PatternStr	PatternREPattern)ParseTreeBuilder)ParsingFrontend)	LexerConf
ParserConf)RuleOptionsRuleTerminalNonTerminalSymbolTOKEN_DEFAULT_PRIORITY)classify
dedup_list)GrammarErrorUnexpectedCharactersUnexpectedToken
ParseErrorUnexpectedInput)TreeSlottedTree)TransformerVisitorv_argsTransformer_InPlaceTransformer_NonRecursiveT)inlinegrammarsz.larkimslux	__empty__.DOT,COMMA:COLON;	SEMICOLON+PLUS-MINUS*STAR/SLASH\	BACKSLASH|VBAR?QMARK!BANG@AT#HASH$DOLLAR%PERCENT^
CIRCUMFLEX&	AMPERSAND_
UNDERSCORE<LESSTHAN>MORETHAN=EQUAL"DBLQUOTE'QUOTE`	BACKQUOTE~TILDE(LPAR)RPAR{LBRACE}RBRACE[LSQB]RSQB
NEWLINEz
CRLFTABSPACE)	 _LPARz\(_RPARz\)_LBRAz\[_RBRAz\]_LBRACEz\{_RBRACEz\}OPz[+*]|[?](?![a-z_])_COLON_COMMA_ORz\|_DOTz\.(?!\.)_DOTDOTz\.\.RULE_MODIFIERSz(!|![?]?|[?]!?)(?=[_a-z])RULEz_?[a-z][_a-z0-9]*TERMINALz_?[A-Z][_A-Z0-9]*STRINGz"(\\"|\\\\|[^"\n])*?"i?z/(?!/)(\\/|\\\\|[^/])*?/[%s]*z(\r?\n)+\s*z(\r?\n)+\s*\|z[ \t]+z\s*//[^\n]*|\s*#[^\n]*z\\[ ]*\nz->z%ignorez	%overridez%declarez%extendz%importz[+-]?\d+)REGEXP_NL_NL_ORWSCOMMENTrJ   _TO_IGNORE	_OVERRIDE_DECLARE_EXTEND_IMPORTNUMBERstart_list_itemz_list _item)ruletermignoreimportdeclareoverrideextendr   r   zBrule_modifiers RULE template_params priority _COLON expansions _NLrule_modifiers priorityz_DOT NUMBERtemplate_paramsz _LBRACE _template_params _RBRACE_template_paramsz_template_params _COMMA RULE
expansions_expansions)aliasz_expansions _OR aliasz_expansions _NL_OR aliasz?aliaszexpansion _TO nonterminal	expansion
_expansionz_expansion exprz?expr)atomzatom OPzatom TILDE NUMBERz atom TILDE NUMBER _DOTDOT NUMBERz?atom)z_LPAR expansions _RPARmaybevaluer   )terminalnonterminalliteralrangetemplate_usager   r   z?namez?symbolr   z_LBRA expansions _RBRAr   zSTRING _DOTDOT STRINGr   z*nonterminal _LBRACE _template_args _RBRACE_template_argsz_template_args _COMMA valuer   zTERMINAL _COLON expansions _NLz*TERMINAL _DOT NUMBER _COLON expansions _NLr   z_OVERRIDE rulez_OVERRIDE termr   z_EXTEND rulez_EXTEND termr   z_IGNORE expansions _NLr   z_DECLARE _declare_args _NLr   )z_IMPORT _import_path _NLz._IMPORT _import_path _LPAR name_list _RPAR _NLz!_IMPORT _import_path _TO name _NL_import_path
import_lib
import_rel_import_argsz_DOT _import_argsnamez_import_args _DOT name
_name_listz_name_list _COMMA namesymbolz_declare_args symbolr   )	name_listr   _declare_argsr      2   c                   @   sp   e Zd ZdefddZdedefddZdeee	ef  de
e	d	d	f fd
dZde	fddZde	fddZd	S )FindRuleSizekeep_all_tokensc                 C   s
   || _ d S Nr   )selfr    r   N/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/lark/load_grammar.py__init__      
zFindRuleSize.__init__symreturnc                 C   sD   t |tr|jd S t |tr| jp|j S |tu rdS J |)Nr]   F)
isinstancer$   r   
startswithr#   r   
filter_out_EMPTY)r   r   r   r   r   _will_not_get_removed   s   

z"FindRuleSize._will_not_get_removedargsNc                 c   sD    |D ]}t |tr|V  qt |tr| |rdndV  qJ d S )Nr   r   )r   intr%   r   )r   r   ar   r   r   _args_as_int   s   

zFindRuleSize._args_as_intc                 C      t | |S r   )sumr   r   r   r   r   r   r         zFindRuleSize.expansionc                 C   r   r   )maxr   r   r   r   r   r      r   zFindRuleSize.expansions)__name__
__module____qualname__boolr   r%   r   r   r	   r   r   r   r   r   r   r   r   r   r      s    (		r   c                   @   s   e Zd Zdd ZdefddZdd Zded	efd
dZdd Z	dd Z
dededefddZdedefddZdefddZdS )EBNF_to_BNFc                 C   s"   g | _ i | _d| _d| _d | _d S )Nanonr   )	new_rulesrules_cacheprefixirule_optionsr   r   r   r   r      s
   
zEBNF_to_BNF.__init__innerc                 C   s$   d| j || jf }|  jd7  _|S )Nz
__%s_%s_%dr   )r   r   )r   r   new_namer   r   r   
_name_rule   s   zEBNF_to_BNF._name_rulec                 C   s*   t |}| j||| jf || j|< |S r   )r$   r   appendr   r   )r   keyr   r   tr   r   r   	_add_rule   s   
zEBNF_to_BNF._add_ruletype_exprc              
   C   s`   z| j | W S  ty/   | |}t|}tdtd|gtd||gg}| ||| Y S w )Nr   r   )r   KeyErrorr   r$   STr   )r   r   r   r   r   treer   r   r   _add_recurse_rule   s   

zEBNF_to_BNF._add_recurse_rulec              
   C   sn   ||||f}z| j | W S  ty6   | d||f }tdtd|g| |g|  g}| ||| Y S w )aN  Generate a rule that repeats target ``a`` times, and repeats atom ``b`` times.

        When called recursively (into target), it repeats atom for x(n) times, where:
            x(0) = 1
            x(n) = a(n) * x(n-1) + b

        Example rule when a=3, b=4:

            new_rule: target target target atom atom atom atom

        zrepeat_a%d_b%dr   r   )r   r   r   r   r   )r   r   btargetr   r   r   r   r   r   r   _add_repeat_rule  s   "zEBNF_to_BNF._add_repeat_rulec           	   	      s    |df}z| j | W S  tyB   | d |f }tdfddt D  fddt|D  }| ||| Y S w )a_  Creates a rule that matches atom 0 to (a*n+b)-1 times.

        When target matches n times atom, and target_opt 0 to n-1 times target_opt,

        First we generate target * i followed by target_opt, for i from 0 to a-1
        These match 0 to n*a - 1 times atom

        Then we generate target * a followed by atom * i, for i from 0 to b-1
        These match n*a to n*a + b-1 times atom

        The created rule will not have any shift/reduce conflicts so that it can be used with lalr

        Example rule when a=3, b=4:

            new_rule: target_opt
                    | target target_opt
                    | target target target_opt

                    | target target target
                    | target target target atom
                    | target target target atom atom
                    | target target target atom atom atom

        optzrepeat_a%d_b%d_optr   c                    s"   g | ]}t d  g| g qS r   r   .0r   )r   
target_optr   r   
<listcomp>4  s    z4EBNF_to_BNF._add_repeat_opt_rule.<locals>.<listcomp>c                    s&   g | ]}t d g  g|  qS r   r   r   )r   r   r   r   r   r   6  s    )r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   _add_repeat_opt_rule  s   z EBNF_to_BNF._add_repeat_opt_ruler   mnmxc                    s   |t k rtd fddt||d D S  }t|tD ]\}}| ||| }q||kr/|S || d }t|t} }	tdg }
|dd D ]\}}| |||	|
 }
| |||	 }	qG|d \}}| |||	|
 }
tdtd|g|
g gS )z\Generates a rule tree that repeats ``rule`` exactly between ``mn`` to ``mx`` times.
        r   c                    s   g | ]
}t d  g| qS r   r   )r   nr   r   r   r   @      z1EBNF_to_BNF._generate_repeats.<locals>.<listcomp>r   r   N)REPEAT_BREAK_THRESHOLDr   r   r   SMALL_FACTOR_THRESHOLDr   r   )r   r   r  r  	mn_targetr   r   diffdiff_factorsdiff_targetdiff_opt_targetr   r  r   _generate_repeats;  s"   "

zEBNF_to_BNF._generate_repeatsopc                 G   s   |j dkrtdg }td||gS |j dkr| d|S |j dkr1| d|}td|tdg gS |j dkrdt|d	krEt|d
  }}ntt|\}}||k sT|d
k r]td|||f | |||S J |)NrM   r   r   rA   plusrE   starrk   r   r   'Bad Range for %s (%d..%d isn't allowed))r   r   r   lenr   mapr)   r  )r   r   r  r   emptyr   r  r  r   r   r   r   Z  s    




zEBNF_to_BNF.exprc                 C   s:   | j o| j j}t||}tdtg| }td||gS )Nr   r   )r   r   r   	transformr   r   )r   r   r   	rule_sizer  r   r   r   r   w  s   zEBNF_to_BNF.maybeN)r   r   r   r   strr   r   r.   r   r   r   r   r  r   r   r   r   r   r   r   r      s    %r   c                   @   sB   e Zd ZedefddZdefddZdd Zdefdd	Zd
S )SimplifyRule_Visitorr   c                 C   s"   |  | jr	 |  | jsd S d S r   )expand_kids_by_datadatar   r   r   r   _flatten  s   zSimplifyRule_Visitor._flattenc                    sj     tjD ](\ }t|tr2|jdkr2d_ fddt|jD _   d S q
d S )Nr   c              	      s2   g | ]  td  fddtjD qS )r   c                    s    g | ]\}} |krn|qS r   r   )r   jother)r   optionr   r   r     s    z=SimplifyRule_Visitor.expansion.<locals>.<listcomp>.<listcomp>)visitr   	enumeratechildren)r   r   r   r   )r   r   r     s
    z2SimplifyRule_Visitor.expansion.<locals>.<listcomp>)r  r"  r#  r   r.   r  r(   )r   r   childr   r$  r   r     s   

zSimplifyRule_Visitor.expansionc                 C   sR   |j \}}|jdkr'g }|j d j D ]}|td||g qd|_||_ d S d S )Nr   r   r   )r#  r  r   r   )r   r   r   
alias_namealiasesr%  r   r   r   r     s   


zSimplifyRule_Visitor.aliasc                 C   s6   |  | tt|jt|jkrt|j|_d S d S r   )r  r  setr#  r(   )r   r   r   r   r   r     s   
zSimplifyRule_Visitor.expansionsN)	r   r   r   staticmethodr.   r  r   r   r   r   r   r   r   r  ~  s    	r  c                   @   s$   e Zd Zdd Zdd Zdd ZdS )RuleTreeToTextc                 C   s   |S r   r   )r   xr   r   r   r     s   zRuleTreeToText.expansionsc                 C   s   |d fS r   r   )r   symbolsr   r   r   r        zRuleTreeToText.expansionc                 C   s.   |\\}}}|d u sJ ||d|f||j fS )NrC   r   )r   r+  r   _aliasr   r   r   r   r     s   
zRuleTreeToText.aliasN)r   r   r   r   r   r   r   r   r   r   r*    s    r*  c                   @   s$   e Zd ZdZdd Zedd ZdS )PrepareAnonTerminalszfCreate a unique list of anonymous terminals. Attempt to give meaningful names to them when we add themc                 C   s8   || _ dd | j D | _dd |D | _d| _d | _d S )Nc                 S      h | ]}|j qS r   r.  r   tdr   r   r   	<setcomp>      z0PrepareAnonTerminals.__init__.<locals>.<setcomp>c                 S   s   i | ]}|j |qS r   )patternr2  r   r   r   
<dictcomp>      z1PrepareAnonTerminals.__init__.<locals>.<dictcomp>r   )	terminalsterm_setterm_reverser   r   )r   r9  r   r   r   r     s
   
zPrepareAnonTerminals.__init__c                 C   st  |j }|| jv r|j| j| jjkrtd| d }t|tr_z| j| j}W nL ty^   zt	| }W n  tyT   |rRt
|rRt|d rR| | jvrR| }Y nw || jv r\d }Y nw t|trp|| jv ro| j| j}nJ ||d u rd| j }|  jd7  _|| jvr|| jvsJ | j| t||}|| j|< | j| | jr| jjrdnt|t}t||dS )Nz+Conflicting flags for the same terminal: %sr   Fz	__ANON_%dr   r   )r   r;  flagsr6  r)   r   r   r   r   _TERMINAL_NAMESr   r   upperr:  r   r   addr   r9  r   r   r   r#   )r   pr   	term_nametermdefr   r   r   r   r6    sF   
&






zPrepareAnonTerminals.patternN)r   r   r   __doc__r   inline_argsr6  r   r   r   r   r0    s
    r0  c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	_ReplaceSymbolszHelper for ApplyTemplatesc                 C   s
   i | _ d S r   )namesr   r   r   r   r     r   z_ReplaceSymbols.__init__c                 C   sH   t |dkrt|d tr|d j| jv r| j|d j S | d|d S )Nr   r   r   )r  r   r%   r   rG  __default__)r   cr   r   r   r     s   *z_ReplaceSymbols.valuec                 C   sD   |d j }|| jv r| d| j| g|dd   d S | d|d S )Nr   r   r   )r   rG  rH  )r   rI  r   r   r   r   r     s   

"z_ReplaceSymbols.template_usageN)r   r   r   rD  r   r   r   r   r   r   r   rF    s
    rF  c                   @   s    e Zd ZdZdd Zdd ZdS )ApplyTemplateszIApply the templates, creating new rules that represent the used templatesc                 C   s   || _ t | _t | _d S r   )	rule_defsrF  replacerr(  created_templates)r   rK  r   r   r   r     s   zApplyTemplates.__init__c           	         s   |d j  |dd  }d ddd |D f }|| jvr^| j|  fdd| jD \\}}}}t|t|ks?J |t|}tt||| j	_
| j	| | j|g |t|f t|S )Nr   r   z%s{%s}r;   c                 s   s    | ]}|j V  qd S r   r.  )r   r   r   r   r   	<genexpr>  s    z0ApplyTemplates.template_usage.<locals>.<genexpr>c                 3   s     | ]}|d   kr|V  qdS )r   Nr   r   r   r.  r   r   rN    s    )r   joinrM  r@  rK  r  r   dictziprL  rG  r  r   r$   )	r   rI  r   result_name_nparamsr   optionsresult_treer   r.  r   r   	  s   

zApplyTemplates.template_usageN)r   r   r   rD  r   r   r   r   r   r   rJ    s    rJ  c                    s   t  fdd|D S )Nc                 3   s    | ]}  |V  qd S r   )rfind)r   rI  sr   r   rN    s    z_rfind.<locals>.<genexpr>)r   )rZ  choicesr   rY  r   _rfind     r\  c              
   C   s   d}t | }|D ]3}||7 }|dkr;zt|}W n ty%   td|  w |dkr/|d7 }n|dvr7|d7 }||7 }q|dddd	}d
| }zt|} W | S  tyb } zt| |d }~ww )Nr   rI   z/Literal ended unexpectedly (bad escaping): `%r`\\Uuxnftrz\"re   rg   z\'z	u'''%s''')iternextStopIterationr)   replacer   SyntaxError)rZ  wr   r  n2to_evaler   r   r   eval_escaping  s2   


ri  c                 C   s,  t | tsJ | j}t|dd }|dksJ ||d  }tdd |D s*J || jdkr7d|v r7td| jd	krHd|v rHd
|vrHtd|d | }|d |d kr\|d dv s^J |dd }t|}|dkrrtd|  | jdkr|dd}t	||| jdS | jd	krt
||| jdS J d)Nz/"r   r   c                 s   s    | ]}|t v V  qd S r   )	_RE_FLAGS)r   fr   r   r   rN  <      z&_literal_to_pattern.<locals>.<genexpr>r   ry   z*You cannot put newlines in string literalsr   r+  zLYou can only use newlines in regular expressions with the `x` (verbose) flagr  z"/r   z$Empty terminals are not allowed (%s)r^  rI   )rawFz:Invariant failed: literal.type not in ["STRING", "REGEXP"])r   r   r   r\  alltyper)   ri  rc  r   r   )r   v
flag_startr=  r+  rZ  r   r   r   _literal_to_pattern6  s,    

rr  c                   @      e Zd Zdd Zdd ZdS )PrepareLiteralsc                 C   s   t dt|gS )Nr6  )r   rr  )r   r   r   r   r   r   Y     zPrepareLiterals.literalc                 C   s   |j |j   krdksJ  J |jdd }|jdd }tt|tt|  kr1dks4J  J d||f }tdt|gS )Nr   r   r  z[%s-%s]r6  )ro  r   r  ri  r   r   )r   r   endregexpr   r   r   r   \  s    ,zPrepareLiterals.rangeN)r   r   r   r   r   r   r   r   r   rt  W      rt  r   c                 C   s
   t | dS )Nr   )r   )rw  	flags_setr   r   r   _make_joined_patterne  r   rz  c                   @   sf   e Zd Zdd Zdee defddZdee defdd	Zdefd
dZdd Z	dd Z
dd ZdS )TerminalTreeToPatternc                 C   s
   |\}|S r   r   )r   psrA  r   r   r   r6  i  s   zTerminalTreeToPattern.patternitemsr   c                 C   sH   |st dS t|dkr|d S ddd |D }t|dd |D S )Nr   r   r   c                 s       | ]}|  V  qd S r   	to_regexpr   r   r   r   rN  t  rl  z2TerminalTreeToPattern.expansion.<locals>.<genexpr>c                 S   r1  r   r=  r   r   r   r   r4  u  r5  z2TerminalTreeToPattern.expansion.<locals>.<setcomp>)r   r  rP  rz  )r   r}  r6  r   r   r   r   m  s   zTerminalTreeToPattern.expansionexpsc                 C   sP   t |dkr
|d S |jdd d dddd	 |D  }t|d
d |D S )Nr   r   c                 S   s   | j  | j t| j fS r   )	max_width	min_widthr  r   r+  r   r   r   <lambda>}  s    z2TerminalTreeToPattern.expansions.<locals>.<lambda>)r   z(?:%s)rK   c                 s   r~  r   r  r   r   r   r   rN    rl  z3TerminalTreeToPattern.expansions.<locals>.<genexpr>c                 S   r1  r   r  r   r   r   r   r4    r5  z3TerminalTreeToPattern.expansions.<locals>.<setcomp>)r  sortrP  rz  )r   r  r6  r   r   r   r   w  s
   z TerminalTreeToPattern.expansionsc                 C   s   |d d \}}|dkr:t |dkrdt|d  }n'tt|dd  \}}||k r3td|||f d||f }nt |dksBJ td| |f |jS )N   rk      z{%d}r  z{%d,%d}z(?:%s)%s)r  r   r  r)   r   r  r=  )r   r   r   r  r  r  r   r   r   r     s   zTerminalTreeToPattern.exprc                 C   s   |  |dg S )NrM   )r   )r   r   r   r   r   r     ru  zTerminalTreeToPattern.maybec                 C   s   t d)NzBAliasing not allowed in terminals (You used -> in the wrong place))r)   )r   r   r   r   r   r     r-  zTerminalTreeToPattern.aliasc                 C   s   |d S Nr   r   r   rp  r   r   r   r     r-  zTerminalTreeToPattern.valueN)r   r   r   r6  r   r   r   r   r   r   r   r   r   r   r   r   r{  h  s    
r{  c                   @   s   e Zd Zdd ZdS )ValidateSymbolsc                 C   s   |\}t |ttfsJ |S r   )r   r.   r%   r  r   r   r   r     s   zValidateSymbols.valueN)r   r   r   r   r   r   r   r   r    s    r  c                 C   s   t d| S )z#Deepcopy tree `t` without recursionF)r4   r  r   r   r   r   nr_deepcopy_tree  s   r  c                	   @   s   e Zd ZU eeeeeef f  ed< eeeeedf ee	f  ed< ee ed< deeeeedf ee	f  deeeeeef f  dee ddfddZ
deee ee ee f fd	d
ZdS )Grammar	term_defs.rK  r   r   Nc                 C   s   || _ || _|| _d S r   )r  rK  r   )r   rK  r  r   r   r   r   r     s   
zGrammar.__init__c           #         s  dd  j D }dd  jD }|D ]#\}\}}|d u rqt|d}t|dkr5|d js5td| qt t  fdd|D }	t	|	}
t t
  |
 t|9 t }g }d}|t|k r|| \}}}}|d7 }t|dkrwq^|r|jrtd	d
nd }||_||_||
_|}||}||||f |t|k sd||j7 }t|tdd |D ksJ dt }t }g }|D ]}|\}}}|| ||}t|D ]k\}\}}|r|drtd||f tdd |D }t|rt|pt }||_dd |D }n|}|D ]!}t|tsJ |jr3|r3|jr3t|t s0J d|_!qt"t#|||||}|| qqtt$|t|krt%|dd }|& D ].}t|dkr|d j'rytdd(dd |D  tdd |D t|ksJ q\tt)|}	 t|} dd |D dd |D O t*|fdd\}}!|!D ]	}"t+,d|" qt|| krnqdkrdd |D t*|	 fd d\}	}!|!rt+,d!d"d |!D  |	| j-fS )#Nc                 S   s$   g | ]\}\}}|t ||ffqS r   r  )r   r  r   rA  r   r   r   r        $ z#Grammar.compile.<locals>.<listcomp>c                 S   s$   g | ]\}}}}||t ||fqS r   r  )r   r  rA  r   or   r   r   r     r  r   r   r   zTerminals cannot be empty (%s)c                    s*   g | ]\}\}}|rt | ||qS r   )r   r  )r   r   	term_treer   )transformerr   r   r     s    Tr   c                 S   s   h | ]\}}}|qS r   r   )r   r   _t_or   r   r   r4    s    z"Grammar.compile.<locals>.<setcomp>zWhoops, name collisionr]   zkRule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)c                 s   s    | ]}|t kV  qd S r   r   r   r+  r   r   r   rN    rl  z"Grammar.compile.<locals>.<genexpr>c                 S   s   g | ]}|t kr|qS r   r  r  r   r   r   r     s    Fc                 S   s   | S r   r   r  r   r   r   r  	  s    z!Grammar.compile.<locals>.<lambda>zXRules defined twice: %s

(Might happen due to colliding expansion of optionals: [] or ?)r   c                 s   s    | ]}d | V  qdS )z
  * %sNr   r   r   r   r   rN    rl  c                 S   s   h | ]
}|j |j|jfqS r   )r   orderrV  )r   rr   r   r   r4    r  c                 S   s0   h | ]}|j D ]}t|tr||jkr|qqS r   )r   r   r$   origin)r   r  rZ  r   r   r   r4    s    
c                 S   s   h | ]}t |qS r   )r$   r   rZ  r   r   r   r4    r8  c                    s
   | j  v S r   )r  )r  )
used_rulesr   r   r       
 zUnused rule: %srE   c                 S   s(   h | ]}|j D ]
}t|tr|jqqS r   )r   r   r#   r   )r   r  r   r   r   r   r4  &  s    c                    s    | j v p| j  jv p| j v S r   )r   r   r  )r   terminals_to_keep
used_termsr   r   r  )  s     zUnused terminals: %sc                 S      g | ]}|j qS r   r.  rO  r   r   r   r   +  r5  ).r  rK  list	find_datar  r#  r)   rt  r{  r0  r  rJ  r   r   r!   r   r   r  r   r   r*  r  r!  r"  r   tupleanyr   empty_indicesr   r%   is_termr#   r   r"   r$   r(  r'   valuesr   rP  r   r   r   debugr   )#r   r   r  r  rK  r   r  r   r   r9  anon_tokens_transfebnf_to_bnfrulesr   rU  	rule_treerV  r   r   resrule_tree_to_textsimplify_rulecompiled_rulesrule_contentr   r   r  exp_optionsr   r   
duplicatesdupsrI  unusedr  r   )r   r  r  r  r  r   compile  s   



"



 
zGrammar.compile)r   r   r   r   r   r  r.   r   __annotations__r!   r   r   r"   r  r   r   r   r   r    s   
  J(r  PackageResourcezpkg_name pathc                   @   sr   e Zd ZU dZeed< ee ed< ddedee ddfddZd	d
 Zde	dee
f dedee
ef fddZdS )FromPackageLoadera  
    Provides a simple way of creating custom import loaders that load from packages via ``pkgutil.get_data`` instead of using `open`.
    This allows them to be compatible even from within zip files.

    Relative imports are handled, so you can just freely use them.

    pkg_name: The name of the package. You can probably provide `__name__` most of the time
    search_paths: All the path that will be search on absolute imports.
    pkg_namesearch_pathsr   r   Nc                 C   s   || _ || _d S r   )r  r  )r   r  r  r   r   r   r   A  s   
zFromPackageLoader.__init__c                 C   s   dt | j| j| jf S )Nz
%s(%r, %r))ro  r   r  r  r   r   r   r   __repr__E  s   zFromPackageLoader.__repr__	base_pathgrammar_pathc           	      C   s   |d u r| j }nt|tr|j| jkrt |jg}d }|D ]9}tj||}z	t	| j|}W n tyD } z|}W Y d }~qd }~ww t| j||rS|
 f  S df  S td|)Nr   z"Cannot find grammar in given paths)r  r   r  r  IOErrorpathosrP  pkgutilget_datadecode)	r   r  r  to_tryerrr  	full_pathtextrh  r   r   r   __call__H  s"   &
zFromPackageLoader.__call__)r  )r   r   r   rD  r  r  r   r   r  r	   r  r   r  r   r   r   r   r  3  s   
 
,r  larkc           
   
   C   s  	 d}|   D ]R\}}|d u rq|dD ]C}|j\}t|tr(td||f t|trQz| |j }W n tyB   td|j w |d usIJ ||jd< d}qt|t	sXJ qq|s]nq|   D ]!\}}|r|jD ]}dd |
 D }	t||	v rtd	| qkqbd S )
NTFr   z0Rules aren't allowed inside terminals (%s in %s)z!Terminal used but not defined: %sr   c                 S      g | ]}t |qS r   )idr  r   r   r   r     r8  z+resolve_term_references.<locals>.<listcomp>zNRecursion in terminal '%s' (recursion is only allowed in rules, not terminals))r}  r  r#  r   r$   r)   r#   r   r   r.   iter_subtreesr  )
	term_dictchangedr   
token_treeexpitem
term_valuer   r%  idsr   r   r   resolve_term_referencesd  s@   



r  c                 C   s0   t | tsJ |  rt| | ddS t| S Nr]   r<  )r   r  isupperr#   r   r$   rY  r   r   r   symbol_from_strcase  s   "r  c                   @   rs  )PrepareGrammarc                 C   s   t t||ddS r  )r#   r  r   r   r   r   r   r   r     r]  zPrepareGrammar.terminalc                 C   s
   t |jS r   )r$   r   r  r   r   r   r     r   zPrepareGrammar.nonterminalN)r   r   r   r   r   r   r   r   r   r    rx  r  c                 C   s"   | j dksJ dd | dD S )Nr   c                 S   s&   h | ]}| d d D ]}|jqqS )c                 S   s
   t | tS r   )r   r%   r  r   r   r   r    r  z._find_used_symbols.<locals>.<setcomp>.<lambda>)scan_valuesr   )r   r+  r   r   r   r   r4    s
    z%_find_used_symbols.<locals>.<setcomp>r   )r  r  r  r   r   r   _find_used_symbols  s   r  c                  C   s   zt jW S  tyP   dd t D } dd t D }dd |D }t|t }dd l	}t
| |g d}t||dg}d|_d	|_t||d t _t j Y S w )
Nc                 S   s   g | ]\}}t |t|qS r   )r   r   )r   r   r   r   r   r   r         z_get_parser.<locals>.<listcomp>c                 S   s,   g | ]\}}| d |t|d dfqS )rM   )expand1)lstripr!   r   )r   r   r+  r   r   r   r     s     c              	   S   sF   g | ]\}}}t |D ]\}}tt|d d | D |d|qqS )c                 S   r  r   )r  r  r   r   r   r     r8  z*_get_parser.<locals>.<listcomp>.<listcomp>N)r"  r"   r$   split)r   r  xsr  r   r+  r   r   r   r     s
    (r   )r   r   rJ   r   basiclalr)_get_parsercacheAttributeError	TERMINALSr}  RULESr   r   create_callbackrer   r    
lexer_typeparser_typer   )r9  r  callbackr  
lexer_confparser_confr   r   r   r    s&   
r  zIncorrect type of valueza: 1
zUnclosed parenthesisza: (
zUnmatched closing parenthesis)za: )
za: [)
za: (]
z5Expecting rule or terminal definition (missing colon))za
zA
za->
zA->
za A
z#Illegal name for rules or terminalszAa:
zAlias expects lowercase namez
a: -> "a"
zUnexpected colon)za::
za: b:
za: B:
za: "a":
zMisplaced operator)za: b??za: b(?)za:+
za:?
za:*
za:|*
z;Expecting option ("|") or a new rule or terminal definitionza:a
()
z"Terminal names cannot contain dotszA.B
z%Expecting rule or terminal definitionz"a"
z%import expects a namez%import "a"
z%ignore expects a valuez%ignore %import
c                 C   s*   |j | tdd}|r|S d|jv rdS d S )NT)use_acceptsr   zExpecting a value)match_examplesGRAMMAR_ERRORSexpected)parserh  errorr   r   r   _translate_parser_exception  s   
r  c              
   C   s   zt  | d |}W nC ty( } z|| }td|j|j||f d }~w tyN } z|| }tt  j|}|rItd||j|j|f  d }~ww t	 
|S )Nry   z1Unexpected input at line %d column %d in %s: 

%sz%s, at line %s column %s

%s)r  r  r*   get_contextr)   linecolumnr+   r  r  r  )r  r   r   r   rh  contextr  r   r   r   _parse_grammar  s"   

r  c                 C   sL   t | tr"tt j| }|r|S d| jp| j}dt| j	|f S t| S )Nz, z*Unexpected token %r. Expected one of: {%s})
r   r+   r  r  r  rP  acceptsr  r  token)r  error2r  r   r   r   _error_repr  s   
r  c                 C   s:   dd }t d| fg|D ]\}}||r||f  S qd S )Nc              	   s   sX    | \}}|  D ] }t|d}z||}W n	 ty    Y q	w ||f |fV  q	d S )Nr   )r[  r   
feed_tokenr,   )noder  rA  choicer   new_pr   r   r   expand  s   
z*_search_interactive_parser.<locals>.expandr   )r   )interactive_parser	predicater  r  rA  r   r   r   _search_interactive_parser  s   r
  r  c                    s^   g   fdd}t  j| d ||d}t dd }dd | D   D ]}d |d	 _q% S )
Nc                    s\     | t| f t| j dd \}}|D ]}| jt|d q| jtdd dS )Nc                 S   s   d|   v S )Nr   )r[  )rA  r   r   r   r    s    z7find_grammar_errors.<locals>.on_error.<locals>.<lambda>r   r   ry   T)r   r  r
  r  as_immutabler  r   )rh  
token_pathr]   
token_typeerrorsr   r   on_error  s   z%find_grammar_errors.<locals>.on_errorry   )r  c                 S   s
   | d j S r  )r  )rh  r   r   r   r    r  z%find_grammar_errors.<locals>.<lambda>c                 S   s   g | ]}|d  qS )r   r   )r   elr   r   r   r     r8  z'find_grammar_errors.<locals>.<listcomp>r   )r  r  r'   r  r  )r  r   r  _treeerrors_by_linerh  r   r  r   find_grammar_errors  s   
r  c                    s    fdd}|S )Nc                    sT   |  v r	 |  } n| d dkrd| dd  f } nd| f } d ur(| } | S )Nr   r]   z_%s__%sr   z%s__%sr   rY  r'  base_mangler   r   r   mangle  s   
z_get_mangle.<locals>.mangler   )r   r'  r  r  r   r  r   _get_mangle  s   r  c                 C   sT   |d u r| S t | } |  D ]}t|jD ]\}}t|tr&|||j|< qq| S r   )r   r  r"  r#  r   r%   renamed)r  r  r   r   rI  r   r   r   _mangle_definition_tree  s   
r  c           
   	   C   s   | j r| j \}d|v }|r|drtdd|v }nd}d}|j r+|j \}t|}	nd }	|d ur9dd |j D }|||t|||	|rG|dfS d dfS )	NrM   r]   z4Inlined rules (_rule) cannot use the ?rule modifier.rO   Fc                 S   r  r   )r   rO  r   r   r   r   0  r5  z$_make_rule_tuple.<locals>.<listcomp>)r   template_source)r#  r   r)   r   r!   )
modifiers_treer   rU  priority_treer   mr  r   rA  r   r   r   r   _make_rule_tuple  s&   

r  c                   @   s   e Zd ZdddZdS )
Definitionr   Nc                 C   s    || _ || _t|| _|| _d S r   )r  r   r  rU  rV  )r   r  r   rU  rV  r   r   r   r   7  s   

zDefinition.__init__r   N)r   r   r   r   r   r   r   r   r   6  s    r   c                   @   s\  e Zd ZU eed< eeeef  ed< e	eef ed< e	ee
f ed< ee ed< d.dedeeeeef   dee	eef  ddfd	d
Zdd Zdd Zd/ddddZd/ddZdd Zdd Zdd Zd0dededeeegef  ddfdd Zd!d" Zd1d#eed$f d%ee d&e	eef d'eeegef  ddf
d(d)Zd2d*d+Zdefd,d-ZdS )3GrammarBuilderglobal_keep_all_tokensimport_paths
used_files_definitions_ignore_namesFNr   c                 C   s*   || _ |pg | _|pi | _i | _g | _d S r   )r#  r$  r%  r&  r'  )r   r#  r$  r%  r   r   r   r   F  s
   


zGrammarBuilder.__init__c           	      G   sr   i }t |ddD ]&\}}|dkrdnt|}||d| < d|  |d| < }| |d| < qt|jdi |)	Nr   r   r   r   )r   r   ro  Typer   )r"  r  titler)   format)	r   r  msgrG  r   r   r   postfixlowercase_typer   r   r   _grammar_errorN  s   zGrammarBuilder._grammar_errorc                 C   sf   |r|d u r
d}|S t |tstdt|f |S |d u r"t }n	t |ts+td| jr1d|_|S )Nr   zBTerminal require a single int as 'options' (e.g. priority), got %sz1Rules require a RuleOptions instance as 'options'T)r   r   r)   ro  r!   r#  r   )r   r  rV  r   r   r   _check_optionsW  s   


zGrammarBuilder._check_optionsr   )r   c                C   sh   || j v r|s| |d| n	|r| |d| |dr$| |d| t|||| ||| j |< d S )Nz&{Type} '{name}' defined more than oncez+Cannot override a nonexisting {type} {name}__zDNames starting with double-underscore are reserved (Error at {name}))r&  r/  r   r   r0  )r   r   r  r  rU  rV  r   r   r   r   _defineg  s   

 zGrammarBuilder._definec                 C   s   || j vr| |d| | j | }||jkr| |d| t||jkr+| |d| |jd u r7| |d| |j}t|trD|jdksFJ |j	
d| d S )Nz6Can't extend {type} {name} as it wasn't defined beforezHCannot extend {type} {name} - one is a terminal, while the other is not.z6Cannot extend {type} with different parameters: {name}z,Can't extend {type} {name} - it is abstract.r   r   )r&  r/  r  r  rU  r   r   r.   r  r#  insert)r   r   r  r  rU  rV  dbaser   r   r   _extends  s   



zGrammarBuilder._extendc                 C   s   t |tr| j| d S t |tsJ |}|jdkrMt|jdkrM|j\}|jdkrMt|jdkrM|j\}|jdkrM|j\}t |trM| j|j	 d S dt| j }| j| t
d|td| j|< d S )Nr   r   r   r   z__IGNORE_%dT)rV  )r   r  r'  r   r.   r  r  r#  r#   r   r   r&   r&  )r   exp_or_namer   t2r  r   r   r   r   _ignore  s    


zGrammarBuilder._ignorec                 C   s6  t |jdkr|j\}}n|j\}d }t|tr(t|j}|j}tt||}n"t|jd d }|s=|j\}td| |jd }|j|pG|ji}|j	dkrRd }	nD|dkrnzt
jtjd j}
W n tym   d }
Y nw |}
|
rt|
trt|
jt
j|
jd }	nt
j|
d }	nt
jt
jj}	||	|fS )Nr   r  z&Nothing was imported from grammar `%s`r   <string>__main__r   )r  r#  r   r.   r  rQ  rR  r)   r   r  r  r  abspathsysmodules__file__r  r  r  r  curdir)r   stmtgrammar_name	path_nodearg1dotted_pathrG  r'  r   r  	base_filer   r   r   _unpack_import  s:   





zGrammarBuilder._unpack_importc                    s   |j dkrt|j \}}}}d}n|jd j}d}t|jdkr't|jd nt}|jd }d} d urCt fd	d
|D } |}t| }|||||fS )Nr   Fr   r   r  r   r  Tc                 3   s    | ]} |V  qd S r   r   )r   rA  r  r   r   rN    rl  z4GrammarBuilder._unpack_definition.<locals>.<genexpr>)	r  r  r#  r   r  r   r&   r  r  )r   r   r  r   rU  r  optsr  r   rH  r   _unpack_definition  s   
 

z!GrammarBuilder._unpack_definition<?>grammar_textrB  r  c              	   C   s  t ||}i }|jD ]:}|jdkrD| ||\}}}	z|| \}
}||
ks.J dd| ||	 W q
 tyC   ||	f||< Y q
w q
| D ]\}\}}	| |||	| qI|jD ]}|jdv rl| j	| 
||  q[|jdkr|j\}| j	| 
||ddi q[|jdkr|j\}| j| 
||  q[|jdkr|d u r| j|j  q[|jd	kr|jD ]$}t|tsJ |t|t}|d u r|j}n||j}| 	||d  qq[|jdkrq[J |dd | j D }t| d S )Nr   zInconsistent base_path for %s.r9   )r   r   r   Tr   r   r   Fc                 S   s   i | ]\}}|j r||jqS r   )r  r   )r   r   r4  r   r   r   r7    s    z/GrammarBuilder.load_grammar.<locals>.<dictcomp>)r  r#  r  rG  rP  updater   r}  	do_importr2  rJ  r6  r9  r   r%   r#   r   r&  r  )r   rL  rB  r  r   importsrA  rE  r  r'  import_base_pathimport_aliasesr  r   r  r   r  r   r   r   load_grammar  sX   


	









zGrammarBuilder.load_grammarc                    s8   fdd}t t||  fddj D _d S )Nc                    sD   z j |  }W n ty   g  Y S w |jrg S t|jt|j S r   )r&  r   r  r  r   r(  rU  )r   r4  r   r   r   rule_dependencies  s   z8GrammarBuilder._remove_unused.<locals>.rule_dependenciesc                    s   i | ]\}}| v r||qS r   r   )r   krp  )_usedr   r   r7    r  z1GrammarBuilder._remove_unused.<locals>.<dictcomp>)r(  r   r&  r}  )r   usedrS  r   )rU  r   r   _remove_unused  s   	zGrammarBuilder._remove_unusedrE  .r  r'  r  c              	   C   sx  |sJ t d|||}tjj| t }| j|d ur|gng  tg }|D ]}z/t|r5|||\}	}
n!tj||}	t|	dd}|	 }
W d    n1 sQw   Y  W n	 t
y`   Y q&w t|
}| j|	||krrtd|| j|	< t| j| j| j}||
|	| |t|| |jD ]}|| jv rtd||f q| jjdi |j  d S t|dd J d|f )	Nr1  utf8encodingz)Grammar file was changed during importingz5Cannot import '%s' from '%s': Symbol already defined.FzjCouldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for itr   )r  rP  r  r  EXTr$  stdlib_loadercallableopenreadr  sha256_digestr%  getRuntimeErrorr"  r#  rR  rW  r  r&  r)   rM  )r   rE  r  r'  r  r  r  r  sourcejoined_pathr  rk  hgbr   r   r   r   rN    s>   



zGrammarBuilder.do_importc                 C   sv  | j  D ]\}}|j}|j}t|D ]!\}}|| j v r$td||f ||d | v r4td||f q|d u r:q|dD ]H}|jd j}|jdd  }	||vr|| j vra| 	|j
d| | t|	t| j | jkrt| j | jt|	}
}| 	|j
d|
|f || q?t|D ]}|| j vr||vr| 	|j
d|| qqt| j | jstd	t| jt| j   d S )
Nz:Template Parameter conflicts with rule %s (in template %s)z0Duplicate Template Parameter %s (in template %s)r   r   r   z5Template '%s' used but not defined (in {type} {name})z]Wrong number of template arguments used for {name} (expected %s, got %s) (in {type2} {name2})z9{Type} '{name}' used but not defined (in {type2} {name2})z8Terminals %s were marked to ignore but were not defined!)r&  r}  rU  r   r"  r)   r  r#  r   r/  r  r  r  r(  
issupersetr'  )r   r   r4  rU  r  r   rA  tempr   r   r  actualr   r   r   validate>  sB   


zGrammarBuilder.validatec                 C   s   |    g }g }| j D ]-\}}|j|j|j}}}|jr1t|dks'J ||||ff q|||||f qt	||| j
S r  )rj  r&  r}  rU  r   rV  r  r  r   r  r'  )r   rK  r  r   r4  rU  r  rV  r   r   r   build^  s   zGrammarBuilder.build)FNNr!  )rK  Nr   )r   N)r   r   r   r   r  r   r	   r  r
   r   r   r   r   r/  r0  r2  r6  r9  rG  rJ  rR  rW  r   rN  rj  r  rk  r   r   r   r   r"  =  s&   
 8	
'*5B
$ r"  c              	   C   s   |   D ]d\}}d }t|tr0tj|r0t|dd}| }W d    n1 s*w   Y  n!t|trQt	t
 tj| d}W d    n1 sLw   Y  |d u rVqt|}||krhtd|   dS qdS )NrX  rY  zutf-8z"File %r changed, rebuilding ParserFT)r}  r   r  r  r  existsr^  r_  r  r   r  r  r  r  r`  r   info)file_hashesr  oldr  rk  currentr   r   r   verify_used_filesm  s&   


rq  c                 C   s$   t d|}|| d t|j S )zXReturns a list of paths to the lark grammars imported by the given grammar (recursively)Fr:  )r"  rR  r  r%  keys)grammarr$  builderr   r   r   list_grammar_imports  s   
ru  c                 C   s$   t ||}|| | | |jfS r   )r"  rR  rk  r%  )rs  rc  r$  r#  rt  r   r   r   rR    s   
rR  rZ  c                 C   s6   t jdkrtj| ddd S t| d S )zGet the sha256 digest of a string

    Supports the `usedforsecurity` argument for Python 3.9+ to allow running on
    a FIPS-enabled system.
    )r  	   rX  F)usedforsecurity)r=  version_infohashlibsha256encode	hexdigestrY  r   r   r   r`    s   
r`  r(  r   )xrD  ry  os.pathr  r=  collectionsr   r   r   r  astr   
contextlibr   typingr   r   r	   r
   r   r   r   r   utilsr   r   r   r   r   r   r   r   lexerr   r   r   r   r   parse_tree_builderr   parser_frontendsr   commonr   r    rs  r!   r"   r#   r$   r%   r&   r'   r(   
exceptionsr)   r*   r+   r,   r-   r   r.   r/   r   visitorsr0   r1   r2   r3   r4   rE  IMPORT_PATHSr[  rj  r   r>  r  r  r  r  r   r   r  r*  r0  rF  rJ  r\  ri  rr  rt  rz  r{  r  r  r  r  r  r\  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r   r"  rq  ru  rR  r`  r   r   r   r   <module>   sR   (( 
	
 !"#(	
!
"()+,./12579;<=ABCDFP -7!3 


-"

,  2