o
    V۷in                     @   s   d dl mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ erfddlmZ ddlmZ ddlmZ d dl
m Z  dd	l!m"Z" dd
l#m$Z$m%Z%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddlm2Z2 ddl3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z= ddl>m?Z? ddl!m@Z@mAZAmBZBmCZC ddlDmEZE zd dlFZFdZGW n eHy   dZGY nw G dd deZIG dd de(ZJh dZKdZLdZMedddZNG dd de(ZOdS )     )ABCabstractmethodN)TypeVarTypeListDictIteratorCallableUnionOptionalSequenceTupleIterableIOAnyTYPE_CHECKING
Collection   )InteractiveParser)	ParseTree)Transformer)Literal)ParsingFrontend)ConfigurationErrorassert_configUnexpectedInput)	SerializeSerializeMemoizerFSlogger)load_grammarFromPackageLoaderGrammarverify_used_filesPackageResourcesha256_digest)Tree)	LexerConf
ParserConf_ParserArgType_LexerArgType)Lexer
BasicLexerTerminalDefLexerThreadToken)ParseTreeBuilder)_validate_frontend_args_get_lexer_callbacks_deserialize_parsing_frontend_construct_parsing_frontend)RuleTFc                   @   s<   e Zd ZU edee dee fddZdZee	 e
d< dS )PostLexstreamreturnc                 C   s   |S N )selfr7   r:   r:   ?/home/ubuntu/vllm_env/lib/python3.10/site-packages/lark/lark.pyprocess)   s   zPostLex.processr:   always_acceptN)__name__
__module____qualname__r   r   r/   r=   r>   r   str__annotations__r:   r:   r:   r<   r6   (   s   
 r6   c                	   @   s6  e Zd ZU dZee ed< eed< eed< ded< eeef ed< eed< eeef ed	< eed
< e	ed< eed< e
eeegef  ed< eed< eed< ded< e
e ed< ded< eeeegef f ed< eed< eed< e
eegef  ed< ded< e
e ed< dZeree7 Zi ddddddddd	ddddddd dddddd dd d
ddddi dd!ddd"dd!g di d#Zeeef ed$< d%eeef d&dfd'd(Zd)ed&efd*d+Zd)ed,ed&dfd-d.Zd5d&eeef fd/d0Zed1eeef d2ee	eeef f d&d fd3d4ZdS )6LarkOptionsz$Specifies the options for Lark

    startdebugstrictzOptional[Transformer]transformerpropagate_positionsmaybe_placeholderscacheregexg_regex_flagskeep_all_tokens
tree_classparserlexerz0Literal["auto", "resolve", "explicit", "forest"]	ambiguitypostlexz-Optional[Literal["auto", "normal", "invert"]]prioritylexer_callbacks	use_bytesordered_setsedit_terminalszUList[Union[str, Callable[[Union[None, str, PackageResource], str], Tuple[str, str]]]]import_pathssource_patha7  
    **===  General Options  ===**

    start
            The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
    debug
            Display debug information and extra warnings. Use only when debugging (Default: ``False``)
            When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
    strict
            Throw an exception on any potential ambiguity, including shift/reduce conflicts, and regex collisions.
    transformer
            Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
    propagate_positions
            Propagates positional attributes into the 'meta' attribute of all tree branches.
            Sets attributes: (line, column, end_line, end_column, start_pos, end_pos,
                              container_line, container_column, container_end_line, container_end_column)
            Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating.
    maybe_placeholders
            When ``True``, the ``[]`` operator returns ``None`` when not matched.
            When ``False``,  ``[]`` behaves like the ``?`` operator, and returns no value at all.
            (default= ``True``)
    cache
            Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.

            - When ``False``, does nothing (default)
            - When ``True``, caches to a temporary file in the local directory
            - When given a string, caches to the path pointed by the string
    regex
            When True, uses the ``regex`` module instead of the stdlib ``re``.
    g_regex_flags
            Flags that are applied to all terminals (both regex and strings)
    keep_all_tokens
            Prevent the tree builder from automagically removing "punctuation" tokens (Default: ``False``)
    tree_class
            Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.

    **=== Algorithm Options ===**

    parser
            Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley").
            (there is also a "cyk" option for legacy)
    lexer
            Decides whether or not to use a lexer stage

            - "auto" (default): Choose for me based on the parser
            - "basic": Use a basic lexer
            - "contextual": Stronger lexer (only works with parser="lalr")
            - "dynamic": Flexible and powerful (only with parser="earley")
            - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible.
    ambiguity
            Decides how to handle ambiguity in the parse. Only relevant if parser="earley"

            - "resolve": The parser will automatically choose the simplest derivation
              (it chooses consistently: greedy for tokens, non-greedy for rules)
            - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
            - "forest": The parser will return the root of the shared packed parse forest.

    **=== Misc. / Domain Specific Options ===**

    postlex
            Lexer post-processing (Default: ``None``) Only works with the basic and contextual lexers.
    priority
            How priorities should be evaluated - "auto", ``None``, "normal", "invert" (Default: "auto")
    lexer_callbacks
            Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
    use_bytes
            Accept an input of type ``bytes`` instead of ``str``.
    ordered_sets
            Should Earley use ordered-sets to achieve stable output (~10% slower than regular sets. Default: True)
    edit_terminals
            A callback for editing the terminals before parse.
    import_paths
            A List of either paths or loader functions to specify from where grammars are imported
    source_path
            Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading
    **=== End of Options ===**
    FNearleyautoTr   )rM   rV   rW   rY   rZ   _plugins	_defaultsoptions_dictr8   c                 C   s   t |}i }| j D ]!\}}||v r&||}t|tr%|dvr%t|}n|}|||< qt|d tr;|d g|d< || jd< t| j	d | j	dkrR| j
rRtd|r\td|  d S )N)rK   rV   rI   rE   options)r[   lalrcykNr[   zCannot specify an embedded transformer when using the Earley algorithm. Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)zUnknown options: %s)dictr^   itemspop
isinstanceboolrB   __dict__r   rP   rH   r   keys)r;   r_   or`   namedefaultvaluer:   r:   r<   __init__   s&   


zLarkOptions.__init__rk   c              
   C   s2   z| j d | W S  ty } zt|d }~ww )Nr`   )rh   KeyErrorAttributeError)r;   rk   er:   r:   r<   __getattr__   s   zLarkOptions.__getattr__rm   c                 C   s    t || j d || j|< d S )Nz,%r isn't a valid option. Expected one of: %s)r   r`   ri   )r;   rk   rm   r:   r:   r<   __setattr__   s   zLarkOptions.__setattr__c                 C   s   | j S r9   r`   )r;   memor:   r:   r<   	serialize   s   zLarkOptions.serializedataru   c                 C   s   | |S r9   r:   )clsrw   ru   r:   r:   r<   deserialize   s   zLarkOptions.deserializer9   )r?   r@   rA   __doc__r   rB   rC   rg   r
   intr   r	   r   r)   r*   r6   r   r/   r-   OPTIONS_DOCr^   rn   rr   rs   rv   classmethodr5   ry   r:   r:   r:   r<   rD   /   s   
 M		
4rD   >
   rF   rL   rS   r]   rV   rO   rH   rM   rU   rI   )r\   normalinvertN)r\   resolveexplicitforest_TLark)boundc                   @   s  e Zd ZU dZeed< eed< ded< eed< eed< ded	< ee	 ed
< dGddZ
er3edej 7 ZdZdHdedefddZdIddZdJddZdKdee ddfddZedee defdd Zd!eeef d"eeee	ef f dedefd#d$Zd%ed&edefd'd(Zed)d* ZedLdee d+ed,e e defd-d.Z!ed/gfdee d0ed1ed2d3def
d4d5Z"d6d7 Z#dHd8edede$e% fd9d:Z&d;ede	fd<d=Z'dMd8e e d>e e dd?fd@dAZ(dMd8ed>e e dBdCddDfdEdFZ)dS )Nr   a}  Main interface for the library.

    It's mostly a thin wrapper for the many different parsers, and for the tree constructor.

    Parameters:
        grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax)
        options: a dictionary controlling various aspects of Lark.

    Example:
        >>> Lark(r'''start: "foo" ''')
        Lark(...)
    rZ   source_grammarr"   grammarr`   rQ   r   rP   	terminalsUnion[Grammar, str, IO[str]]r8   Nc              
      sp  t || _| jj}|rtrt}ntdt}| jjd u r/z|j| _W n ty.   d| _Y nw | jj| _z|j	}W n	 tyB   Y nw | }d }d }t
|tr]|| _| jjr_| s_td| jjrM| jjdkrntdd d fdd	| D }d
dlm}	 || |	 ttjd d  }
t|
}t
| jjtr| jj}n.| jjdurtdzt }W n ty   d}Y nw t d||gtjd d R   }| j}z\t|dK}t !d| t"|t# D ]}||= q|$ %d}t&'|}||(dkrt)|rt&'|}| j*|fi | 	 W d    W d S W d    n	1 s*w   Y  W n t+y:   Y n tyL   t ,d| || _Y nw t-|| j| jj.| jj/\| _0}nt
|t1seJ || _0| jj2dkr| jjdkr{d| j_2n/| jjdkr| jj3d urt 4d d| j_2nd| j_2n| jjdkrd| j_2nJ | jj| jj2}t
|t5rt6|t7sJ nt8|d | jj3d urd|v rtd| jj9dkr| jjdkrd | j_9nt8| jjd!d" | jj:dkrd#| j_:| jj:t;vr	td$| jj:t;f | jj9t<vrtd%| jj9t<f | jjd u r$d&}n| jj3d ur3t"| jj3j=}nt" }| j0>| jj?|\| _@| _A| _B| jjCrW| j@D ]	}| jC| qMd'd( | j@D | _D| jj:d)kr| jAD ]}|jj:d urz|jj: |j_:qj| j@D ]}|j: |_:qn#| jj:d u r| jAD ]}|jj:d urd |j_:q| j@D ]}d*|_:qtE| j@|| jB| jj3| jjF| jjG| jj| jjHd+| _I| jjr| J | _n|r| K | _2|r6t !d,| z9t|d-'}|d usJ |L|(dd  t&M|| | N|t# W d    W d S 1 sw   Y  W d S  tOy5 } zt ,d.|| W Y d }~d S d }~ww d S )/Nz?`regex` module must be installed if calling `Lark(regex=True)`.z<string>z/Grammar must be ascii only, when use_bytes=Truera   z+cache only works with parser='lalr' for now)rH   rS   rU   rX   r]    c                 3   s(    | ]\}}| vr|t | V  qd S r9   )rB   ).0kv
unhashabler:   r<   	<genexpr>7  s   & z Lark.__init__.<locals>.<genexpr>r   )__version__   Tz"cache argument must be bool or strunknownz/.lark_cache_%s_%s_%s_%s.tmprbzLoading grammar from cache: %s   
utf8z<Failed to load Lark from cache: %r. We will try to carry on.r\   
contextualr[   z~postlex can't be used with the dynamic lexer, so we use 'basic' instead. Consider using lalr with contextual instead of earleybasicdynamicrb   F)r   r   r   dynamic_completezGCan't use postlex with a dynamic lexer. Use basic or contextual insteadr   )r[   rb   zG%r doesn't support disambiguation. Use one of these parsers instead: %sr~   z.invalid priority option: %r. Must be one of %rz/invalid ambiguity option: %r. Must be one of %r*c                 S      i | ]}|j |qS r:   rk   r   tr:   r:   r<   
<dictcomp>      z!Lark.__init__.<locals>.<dictcomp>r   r   )rV   rG   zSaving grammar to cache: %swbz!Failed to save Lark to cache: %r.)PrD   r`   rL   
_has_regexImportErrorrerZ   rk   rp   readrf   rB   r   rV   isasciir   rK   rP   joinrd   r   r   sysversion_infor%   getpassgetuser	Exceptiontempfile
gettempdirr   openr   rF   set_LOAD_ALLOWED_OPTIONSreadlinerstrippickleloadencoder#   _loadFileNotFoundError	exceptionr    rY   rN   r   r"   rQ   rS   infotype
issubclassr+   r   rR   rT   _VALID_PRIORITY_OPTIONS_VALID_AMBIGUITY_OPTIONSr>   compilerE   r   rulesignore_tokensrX   _terminals_dictr'   rU   rM   rG   
lexer_conf_build_parser_build_lexerwritedumpsaveIOError)r;   r   r`   	use_regex	re_moduler   cache_fncache_sha256options_strr   susernameold_optionsfrk   file_sha256cached_used_filescached_parser_data
used_filesrQ   terminals_to_keepr   ruletermrq   r:   r   r<   rn     s"  





$

 
 	














(zLark.__init__z

)rP   r   r`   Fdont_ignorec                 C   s,   | j }|rddlm} ||}d|_t|S )Nr   )copyr:   )r   r   ignorer,   )r;   r   r   r   r:   r:   r<   r     s   zLark._build_lexerc                 C   sx   i | _ | jjdkr.t| j| jjpt| jj| jjdko| jjdk| jj	| _
| j
| jj| _ | j t| jj| j d S )Nr   ra   r   )
_callbacksr`   rR   r0   r   rO   r&   rI   rP   rJ   _parse_tree_buildercreate_callbackrH   updater2   r   r;   r:   r:   r<   _prepare_callbacks  s   
zLark._prepare_callbacksc                 C   sL   |    t| jj| jj t| j| j| jj}t	| jj| jj| j
|| jdS )Nrt   )r   r1   r`   rP   rQ   r(   r   r   rE   r4   r   )r;   parser_confr:   r:   r<   r     s   zLark._build_parserr:   exclude_optionsc                    sd   | j jdkr
td| ttg\}} r$ fdd|d  D |d< tj||d|tj	d dS )	zgSaves the instance into the given file object

        Useful for caching and multiprocessing.
        ra   z7Lark.save() is only implemented for the LALR(1) parser.c                    s   i | ]\}}| vr||qS r:   r:   )r   nr   r   r:   r<   r     s    zLark.save.<locals>.<dictcomp>r`   rw   ru   )protocolN)
r`   rP   NotImplementedErrormemo_serializer-   r5   rd   r   r   HIGHEST_PROTOCOL)r;   r   r   rw   mr:   r   r<   r     s   z	Lark.saverx   c                 C   s   |  | }||S )zfLoads an instance from the given file object

        Useful for caching and multiprocessing.
        __new__r   )rx   r   instr:   r:   r<   r     s   

z	Lark.loadrw   ru   c                 C   sN   t |d |}|jpi |_|jrtnt|_|j|_|j|_d|_	|j
|_
|S )Nr   T)r'   ry   rU   	callbacksrL   r   r   rV   rM   skip_validationrS   )r;   rw   ru   r`   r   r:   r:   r<   _deserialize_lexer_conf  s   zLark._deserialize_lexer_confr;   r   c                    s   t |tr|}nt|}|d }|d }|sJ t|ttdi  t|d }t|t	 tt
j@ r?tdt|t	 || t
| | _ fdd|d D | _d	| _t| jj| jj | |d
  | j| _| jj| _|   dd | jD | _t|d
  | j| j| j| _| S )Nru   rw   )r5   r-   r`   z6Some options are not allowed when loading a Parser: {}c                    s   g | ]}t | qS r:   )r5   ry   )r   rru   r:   r<   
<listcomp>  s    zLark._load.<locals>.<listcomp>r   z<deserialized>rP   c                 S   r   r:   r   r   r:   r:   r<   r   "  r   zLark._load.<locals>.<dictcomp>)rf   rc   r   r   r   ry   r5   r-   r   r   rD   r^   r   formatr   r`   r   rZ   r1   rP   rQ   r   r   r   r   r   r3   r   )r;   r   kwargsd	memo_jsonrw   r`   r:   r   r<   r     s:   



z
Lark._loadc                 K   s"   |  | }|j||dfi |S )Nr   r   )rx   rw   ru   r   r   r:   r:   r<   _load_from_dict,  s   
zLark._load_from_dictgrammar_filenamerel_toc                 K   s^   |rt j|}t j||}t|dd}| |fi |W  d   S 1 s(w   Y  dS )a&  Create an instance of Lark with the grammar given by its filename

        If ``rel_to`` is provided, the function will find the grammar filename in relation to it.

        Example:

            >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
            Lark(...)

        r   )encodingN)ospathdirnamer   r   )rx   r   r   r`   basepathr   r:   r:   r<   r   1  s   $z	Lark.openr   packagegrammar_pathsearch_pathszSequence[str]c                 K   sN   t ||}|d|\}}|d| |dg  |d | | |fi |S )ak  Create an instance of Lark with the grammar loaded from within the package `package`.
        This allows grammar loading from zipapps.

        Imports in the grammar will use the `package` and `search_paths` provided, through `FromPackageLoader`

        Example:

            Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...)
        NrZ   rY   )r!   
setdefaultappend)rx   r  r  r  r`   package_loader	full_pathtextr:   r:   r<   open_from_packageC  s   
zLark.open_from_packagec                 C   s   d| j | jj| jjf S )Nz(Lark(open(%r), parser=%r, lexer=%r, ...))rZ   r`   rP   rQ   r   r:   r:   r<   __repr__U  s   zLark.__repr__r
  c                 C   sP   t | dr|r| |}n| j}t||}|d}| jjr&| jj|S |S )a  Only lex (and postlex) the text, without parsing it. Only relevant when lexer='basic'

        When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore.

        :raises UnexpectedCharacters: In case the lexer cannot find a suitable match.
        rQ   N)	hasattrr   rQ   r.   	from_textlexr`   rS   r=   )r;   r
  r   rQ   lexer_threadr7   r:   r:   r<   r  Y  s   
zLark.lexrk   c                 C   s
   | j | S )z Get information about a terminal)r   )r;   rk   r:   r:   r<   get_terminalk  s   
zLark.get_terminalrE   r   c                 C   s   | j j||dS )a-  Start an interactive parsing session.

        Parameters:
            text (str, optional): Text to be parsed. Required for ``resume_parse()``.
            start (str, optional): Start symbol

        Returns:
            A new InteractiveParser instance.

        See Also: ``Lark.parse()``
        )rE   )rP   parse_interactive)r;   r
  rE   r:   r:   r<   r  o  s   zLark.parse_interactiveon_errorz+Optional[Callable[[UnexpectedInput], bool]]r   c                 C   s   | j j|||dS )a  Parse the given text, according to the options provided.

        Parameters:
            text (str): Text to be parsed.
            start (str, optional): Required if Lark was given multiple possible start symbols (using the start option).
            on_error (function, optional): if provided, will be called on UnexpectedToken error. Return true to resume parsing.
                LALR only. See examples/advanced/error_handling.py for an example of how to use on_error.

        Returns:
            If a transformer is supplied to ``__init__``, returns whatever is the
            result of the transformation. Otherwise, returns a Tree instance.

        :raises UnexpectedInput: On a parse error, one of these sub-exceptions will rise:
                ``UnexpectedCharacters``, ``UnexpectedToken``, or ``UnexpectedEOF``.
                For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``.

        )rE   r  )rP   parse)r;   r
  rE   r  r:   r:   r<   r  }  s   z
Lark.parse)r   r   r8   N)F)r8   N)r8   r   )r:   r9   )NN)*r?   r@   rA   rz   rB   rC   rD   r+   r   r-   rn   r|   __serialize_fields__rg   r,   r   r   r   r   r}   r   r   r   r   r   r{   r
   r5   r'   r   r   r   r   r   r  r  r   r/   r  r  r  r  r:   r:   r:   r<   r      sB   
 
 <

2
 
&* $)Pabcr   r   r   r   r   r   r   typesr   typingr   r   r   r   r   r	   r
   r   r   r   r   r   r   r   r   parsers.lalr_interactive_parserr   treer   visitorsr   r   parser_frontendsr   
exceptionsr   r   r   utilsr   r   r   r   r    r!   r"   r#   r$   r%   r&   commonr'   r(   r)   r*   rQ   r+   r,   r-   r.   r/   parse_tree_builderr0   r1   r2   r3   r4   r   r5   rL   r   r   r6   rD   r   r   r   r   r   r:   r:   r:   r<   <module>   sH    D  ?