o
    V۷iy                     @   s   d Z ddlZddlZddlZddlZddlmZmZm	Z	 dZ
ee
dddZdd	 Ze	d
dG dd deZdd ZdddZdddZdd Zedkrue Zeejdkraeej ed e Zeeej
ejejejd dS dS )z!Converts Nearley grammars to Lark    N)LarkTransformerv_argsa  
    start: (ruledef|directive)+

    directive: "@" NAME (STRING|NAME)
             | "@" JS  -> js_code
    ruledef: NAME "->" expansions
           | NAME REGEXP "->" expansions -> macro
    expansions: expansion ("|" expansion)*

    expansion: expr+ js

    ?expr: item (":" /[+*?]/)?

    ?item: rule|string|regexp|null
         | "(" expansions ")"

    rule: NAME
    string: STRING
    regexp: REGEXP
    null: "null"
    JS: /{%.*?%}/s
    js: JS?

    NAME: /[a-zA-Z_$]\w*/
    COMMENT: /#[^\n]*/
    REGEXP: /\[.*?\]/

    STRING: _STRING "i"?

    %import common.ESCAPED_STRING -> _STRING
    %import common.WS
    %ignore WS
    %ignore COMMENT

    earleybasic)parserlexerc                 C   s&   ddd | | } d| dd  S )N	_ws_maybe_ws)___n_$
__DOLLAR__)getreplacelower)name r   H/home/ubuntu/vllm_env/lib/python3.10/site-packages/lark/tools/nearley.py_get_rulename1   s   r   T)inlinec                   @   sl   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd ZdS )NearleyToLarkc                 C   s   d| _ i | _i | _i | _d S )Nr   )_countextra_rulesextra_rules_revalias_js_codeselfr   r   r   __init__7   s   
zNearleyToLark.__init__c                 C   s&   d| j  }|  j d7  _ || j|< |S )Nzalias_%d   )r   r   )r   coder   r   r   r   _new_function=   s   

zNearleyToLark._new_functionc                 C   sH   || j v r
| j | S dt| j }|| jvsJ || j|< || j |< |S )Nzxrule_%d)r   lenr   )r   ruler   r   r   r   _extra_ruleD   s   



zNearleyToLark._extra_rulec                 C   s   t |S Nr   )r   r   r   r   r   r$   N      zNearleyToLark.rulec                 C   s   dt ||f S )N!%s: %sr'   )r   r   expsr   r   r   ruledefQ      zNearleyToLark.ruledefc                 C   s   d||f }|  |S )Nz(%s)%sr%   )r   itemopr$   r   r   r   exprT   s   
zNearleyToLark.exprc                 C   s   d| S )Nz/%s/r   )r   rr   r   r   regexpX   r(   zNearleyToLark.regexpc                 C   s   dS )N r   r   r   r   r   null[   s   zNearleyToLark.nullc                 C   s
   |  |S r&   r-   )r   sr   r   r   string^   s   
zNearleyToLark.stringc                 G   sR   |d d |d }}|j r |j \}|dd }d| | }nd}d|| S )N   z-> r3    )childrenr"   join)r   xjsjs_codealiasr   r   r   	expansiona   s   zNearleyToLark.expansionc                 G   s   dd | S )Nz%sz
    |)r<   )r   r=   r   r   r   
expansionsk   s   zNearleyToLark.expansionsc                 G   s   d td |S N
)r<   filter)r   rulesr   r   r   startn   r,   zNearleyToLark.startN)__name__
__module____qualname__r   r"   r%   r$   r+   r0   r2   r4   r6   rA   rB   rG   r   r   r   r   r   5   s    

r   c                 C   s4  g }t | }|jD ]}|jdkri|j\}	}
|	dv re|	dkr |n|}tj||
dd }||vrd|| tj	|dd}|
 }W d    n1 sLw   Y  |t||||tjtj||7 }q
J |	|jd	kr~|j\}|d
d }|| q
|jdkrq
|jdkr||| q
td| |S )N	directive)builtinincluderL   r    r7   utf8encodingFr?   r8   r9   macror+   zUnknown statement: %s)nearley_grammar_parserparser;   dataospathr<   addcodecsopenread_nearley_to_larkabspathdirnameappend	transform	Exception)gbuiltin_pathn2lr?   folder_pathincludes	rule_defstree	statementrK   argfolderrV   ftextr!   r   r   r   r[   q   s4   





&


r[   Fc                    s6  dd l }g  d fdd	}dg}t }t| ||||t }	d|	}
|
dddd |j D  7 }
|d |  |d	t|
  |  |j D ]\}}|	d
||f  qJ|re||
d| n
||d| |d |jD ]
}|d||f  qv|d |  |d|  |d |d d S )Nr   c                    s   | r  |    d d S rC   )r^   )r=   	emit_coder   r   emit   s   
z-create_code_for_nearley_grammar.<locals>.emitzfunction id(x) {return x[0];}rD   c                 s   s    | ]}d | V  qdS )r)   Nr   ).0r.   r   r   r   	<genexpr>   s    z2create_code_for_nearley_grammar.<locals>.<genexpr>z"from lark import Lark, Transformerz
grammar = z
%s = (%s);z$class TransformNearley(Transformer):z"    %s = var.get('%s').to_python()z8    __default__ = lambda self, n, c, m: c if c else Nonez>parser = Lark(grammar, start="n_%s", maybe_placeholders=False)zdef parse(text):z;    return TransformNearley().transform(parser.parse(text))r3   r&   )js2pyr   r[   setr<   r   itemsreprr   r^   translate_js6translate_js)ra   rG   rb   rd   es6rr   ro   r?   rc   rf   lark_gr@   r!   r   rm   r   create_code_for_nearley_grammar   s4   
"

rz   c              	   C   sb   t j| dd}| }W d    n1 sw   Y  t||tj|dtjtj| |dS )NrN   rO   rL   )rx   )	rX   rY   rZ   rz   rU   rV   r<   r\   r]   )fnrG   nearley_librx   rk   grammarr   r   r   main   s   
,r~   c                  C   sJ   t jdd} | jddd | jddd | jdd	d | jd
ddd | S )NzSReads a Nearley grammar (with js functions), and outputs an equivalent lark parser.)descriptionnearley_grammarz/Path to the file containing the nearley grammar)help
start_rulez5Rule within the nearley grammar to make the base ruler|   zHPath to root directory of nearley codebase (used for including builtins)z--es6zEnable experimental ES6 support
store_true)r   action)argparseArgumentParseradd_argument)r   r   r   r   get_arg_parser   s   r   __main__r    )r{   rG   r|   rx   )F)__doc__os.pathrU   sysrX   r   larkr   r   r   r   rR   r   r   r[   rz   r~   r   rH   r   r#   argv
print_helpstderrexit
parse_argsargsprintr   r|   rx   r   r   r   r   <module>   s.    $;

'
 