o
    V۷i/                     @   s  d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZmZ d	d
 ZG dd dZG dd dZG dd dZG dd dZd/ddZdd ZG dd dZG dd deZdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Z d+d, Z!d-d. Z"dS )0z$This module implements a CYK parser.    )defaultdictN   )
ParseError)Token)Tree)TerminalNonTerminalSymbolc                 C   s   t | tsJ | j|jkS N)
isinstanceTnametype)ts r   F/home/ubuntu/vllm_env/lib/python3.10/site-packages/lark/parsers/cyk.pymatch   s   r   c                       sH   e Zd ZdZ fddZdd Zdd Zdd	 Zd
d Zdd Z	  Z
S )RulezContext-free grammar rule.c                    sV   t t|   t|tsJ |tdd |D sJ ||| _|| _|| _|| _	d S )Nc                 s   s$    | ]}t |tpt |tV  qd S r
   )r   NTr   .0xr   r   r   	<genexpr>   s   " z Rule.__init__.<locals>.<genexpr>)
superr   __init__r   r   alllhsrhsweightalias)selfr   r   r   r    	__class__r   r   r      s   
zRule.__init__c                 C   s$   dt | jddd | jD f S )Nz%s -> %s c                 s       | ]}t |V  qd S r
   strr   r   r   r   r   "       zRule.__str__.<locals>.<genexpr>)r'   r   joinr   r!   r   r   r   __str__!   s   $zRule.__str__c                 C      t | S r
   r&   r*   r   r   r   __repr__$      zRule.__repr__c                 C   s   t | jt| jfS r
   )hashr   tupler   r*   r   r   r   __hash__'   s   zRule.__hash__c                 C   s   | j |j ko| j|jkS r
   r   r   r!   otherr   r   r   __eq__*   s   zRule.__eq__c                 C   s
   | |k S r
   r   r3   r   r   r   __ne__-      
zRule.__ne__)__name__
__module____qualname____doc__r   r+   r-   r1   r5   r6   __classcell__r   r   r"   r   r      s    	r   c                   @   s0   e Zd ZdZdd Zdd Zdd Zdd	 Zd
S )GrammarzContext-free grammar.c                 C   s   t || _d S r
   )	frozensetrulesr!   r?   r   r   r   r   4   s   zGrammar.__init__c                 C      | j |j kS r
   )r?   r3   r   r   r   r5   7      zGrammar.__eq__c                 C   s"   dd tdd | jD  d S )N
c                 s   r%   r
   )reprr   r   r   r   r   ;   r(   z"Grammar.__str__.<locals>.<genexpr>)r)   sortedr?   r*   r   r   r   r+   :   s   "zGrammar.__str__c                 C   r,   r
   r&   r*   r   r   r   r-   =   r.   zGrammar.__repr__N)r8   r9   r:   r;   r   r5   r+   r-   r   r   r   r   r=   1   s    r=   c                   @   s"   e Zd ZdZdddZdd ZdS )	RuleNodez@A node in the parse tree, which also contains the full rhs rule.r   c                 C   s   || _ || _|| _d S r
   )rulechildrenr   )r!   rG   rH   r   r   r   r   r   E   s   
zRuleNode.__init__c                 C   s&   dt | jjddd | jD f S )NzRuleNode(%s, [%s])z, c                 s   r%   r
   r&   r   r   r   r   r   K   r(   z$RuleNode.__repr__.<locals>.<genexpr>)rD   rG   r   r)   rH   r*   r   r   r   r-   J   s   &zRuleNode.__repr__Nr   )r8   r9   r:   r;   r   r-   r   r   r   r   rF   B   s    
rF   c                       s8   e Zd ZdZ fddZdd Zdd Zdd	 Z  ZS )
ParserzParser wrapper.c                    sB   t t   dd |D  _ fdd|D }tt| _d S )Nc                 S   s   i | ]}||qS r   r   r   rG   r   r   r   
<dictcomp>T   s    z#Parser.__init__.<locals>.<dictcomp>c                    s   g | ]}  |qS r   )_to_rulerK   r*   r   r   
<listcomp>U       z#Parser.__init__.<locals>.<listcomp>)r   rJ   r   
orig_rulesto_cnfr=   grammarr@   r"   r*   r   r   R   s   zParser.__init__c                 C   sR   t |jtsJ tdd |jD sJ t|j|j|jjr$|jj|dS d|dS )z?Converts a lark rule, (lhs, rhs, callback, options), to a Rule.c                 s       | ]}t |tV  qd S r
   )r   r	   r   r   r   r   r   [       z"Parser._to_rule.<locals>.<genexpr>r   r   r    )r   originr   r   	expansionr   optionspriority)r!   	lark_ruler   r   r   rM   X   s   zParser._to_rulec                    st    sJ t   t|| j\}}t fdd|dt|d f D r'td|dt|d f   }| t|S )z(Parses input, which is a list of tokens.c                 3   s    | ]}|j  kV  qd S r
   )r   )r   rstartr   r   r   h   rT   zParser.parse.<locals>.<genexpr>r      zParsing failed.)r   _parserR   r   lenr   _to_tree
revert_cnf)r!   	tokenizedr]   tabletreesparser   r\   r   rf   a   s   &zParser.parsec                 C   sl   | j |jj }g }|jD ]}t|tr|| | qt|jt	s$J ||j qt
|j|}||_|S )z.Converts a RuleNode parse tree to a lark Tree.)rP   rG   r    rH   r   rF   appendra   r   r   r   rV   )r!   	rule_node	orig_rulerH   childr   r   r   r   ra   m   s   

zParser._to_tree)	r8   r9   r:   r;   r   rM   rf   ra   r<   r   r   r"   r   rJ   O   s    	rJ   c                 C   sb   t | tr"td|d  t| jj  | jD ]	}t||d  qd S td|d  t| j  d S )Nr$   r   r^   )	r   rF   printr'   rG   r   rH   print_parser   )nodeindentrj   r   r   r   rl   |   s   

rl   c              
   C   s  t t}t t}t| D ]I\}}|j D ]?\}}t||rT|D ]3}|||f | |j|||f vsA|j	|||f |j j	k rSt
|t|g|j	d|||f |j< q qqtdt| d D ]}	tt| |	 d D ]}t|d ||	 D ]}
||
d f}|
||	 d f}t|| || D ]g\}}|j|j|jfg D ]W}||||	 d f | || |j }|| |j }|j	|j	 |j	 }|j||||	 d f vs|||||	 d f |j j	k rt
|||g|d||||	 d f |j< qqqvqkq_||fS )z*Parses sentence 's' using CNF grammar 'g'.r   r   r^   )r   setdict	enumerateterminal_rulesitemsr   addr   r   rF   r   ranger`   	itertoolsproductnonterminal_rulesget)r   grd   re   iwterminalr?   rG   lpspan1span2r1r2r1_treer2_treerule_total_weightr   r   r   r_      s@   
$	 (r_   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )
CnfWrapperzkCNF wrapper for grammar.

  Validates that the input grammar is CNF and provides helper data structures.
  c                    s   t t|   || _|j| _tt| _tt| _| jD ]R}t	|j
ts'J |t|jdvr2tdt|jdkrMt	|jd trM| j|jd  | qt|jdkrjtdd |jD rj| jt|j | qJ |d S )N)r^   r   zCYK doesn't support empty rulesr^   r   r   c                 s   rS   r
   )r   r   r   r   r   r   r      rT   z&CnfWrapper.__init__.<locals>.<genexpr>)r   r   r   rR   r?   r   listrs   ry   r   r   r   r`   r   r   r   rg   r   r0   )r!   rR   r[   r"   r   r   r      s   


"zCnfWrapper.__init__c                 C   rA   r
   )rR   r3   r   r   r   r5      rB   zCnfWrapper.__eq__c                 C   s
   t | jS r
   )rD   rR   r*   r   r   r   r-      r7   zCnfWrapper.__repr__)r8   r9   r:   r;   r   r5   r-   r<   r   r   r"   r   r      s
    r   c                       s.   e Zd ZdZ fddZdd ZejZ  ZS )UnitSkipRulez@A rule that records NTs that were skipped during transformation.c                    s    t t| |||| || _d S r
   )r   r   r   skipped_rules)r!   r   r   r   r   r    r"   r   r   r      s   
zUnitSkipRule.__init__c                 C   s   t |t| o| j|jkS r
   )r   r   r   r3   r   r   r   r5      s   zUnitSkipRule.__eq__)	r8   r9   r:   r;   r   r5   r   r1   r<   r   r   r"   r   r      s
    r   c                 C   sV   g }t | tr|| j7 }|| t |tr||j7 }t| j|j|| j|j | jdS )NrU   )r   r   r   rg   r   r   r   r    )	unit_ruletarget_ruler   r   r   r   build_unit_skiprule   s   




r   c                 C   s6   | j D ]}t|jdkrt|jd tr|  S qdS )zDReturns a non-terminal unit rule from 'g', or None if there is none.r^   r   N)r?   r`   r   r   r   )r{   rG   r   r   r   get_any_nt_unit_rule   s
   
r   c                    sF    fdd| j D } fdd| j D }| fdd|D 7 }t|S )zFRemoves 'rule' from 'g' without changing the language produced by 'g'.c                    s   g | ]}| kr|qS r   r   r   rG   r   r   rN      s    z%_remove_unit_rule.<locals>.<listcomp>c                    s    g | ]}|j  jd  kr|qS rI   r2   r   r   r   r   rN      s     c                    s   g | ]}t  |qS r   )r   )r   refr   r   r   rN      rO   )r?   r=   )r{   rG   	new_rulesrefsr   r   r   _remove_unit_rule   s   r   c                 c   s    t | jd ddd | jD  }d| d }t| j| jd t|d g| j| jd	V  tdt	| jd
 D ]}tt|| | j| t||d  gddd	V  q8tt|t	| jd
  | jdd ddd	V  dS )z4Splits a rule whose len(rhs) > 2 into shorter rules.___c                 s   r%   r
   r&   r   r   r   r   r     r(   z_split.<locals>.<genexpr>z__SP_%sz_%dr   r^   rU   r   SplitN)
r'   r   r)   r   r   r   r   r    rv   r`   )rG   rule_str	rule_namer|   r   r   r   _split   s   $*22r   c              	      s   dd | j D }dd |D g }| j D ]? t jdkrNtdd  jD rNfdd	 jD }|t j| j jd
 |	 fdd
 D  q|  qt|S )z/Applies the TERM rule on 'g' (see top comment).c                 S   s&   h | ]}|j D ]	}t|tr|qqS r   )r   r   r   )r   rG   r   r   r   r   	<setcomp>  s   & z_term.<locals>.<setcomp>c              	   S   s,   i | ]}|t td t| |gdddqS )z__T_%sr   TermrU   )r   r   r'   )r   r   r   r   r   rL     s   , z_term.<locals>.<dictcomp>r^   c                 s   rS   r
   )r   r   r   r   r   r   r     rT   z_term.<locals>.<genexpr>c                    s$   g | ]}t |tr | jn|qS r   )r   r   r   r   )t_rulesr   r   rN     s   $ z_term.<locals>.<listcomp>rU   c                 3   s"    | ]\}}| j v r|V  qd S r
   )r   )r   kvr   r   r   r     s     )r?   r`   r   anyrg   r   r   r   r    extendrt   r=   )r{   all_tr   new_rhsr   )rG   r   r   _term	  s   
"r   c                 C   s>   g }| j D ]}t|jdkr|t|7 }q|| qt|S )z.Applies the BIN rule to 'g' (see top comment).r   )r?   r`   r   r   rg   r=   )r{   r   rG   r   r   r   _bin  s   
r   c                 C   s&   t | }|rt| |} t | }|s| S )z/Applies the UNIT rule to 'g' (see top comment).)r   r   )r{   nt_unit_ruler   r   r   _unit#  s   
r   c                 C   s   t tt| } t| S )z>Creates a CNF grammar from a general context-free grammar 'g'.)r   r   r   r   )r{   r   r   r   rQ   ,  s   rQ   c              
   C   sx   |st t| |||d||dS ||d j }t t| |d jg||dt|d j||dd  ||d j|d jg|dS )NrU   ro   r   r^   )rF   r   r   r   unroll_unit_skipruler    )r   orig_rhsr   rH   r   r    r   r   r   r   2  s   r   c                 C   s   t | tr| S | jjjdr| jd S g }tt| jD ]}t |t	r1|jjjdr1||j7 }q|
| qt | jtrPt| jj| jj| jj|| jj| jjS t	| j|S )zDReverts a parse tree (RuleNode) to its original non-CNF form (Node).__T_r   __SP_)r   r   rG   r   r   
startswithrH   maprb   rF   rg   r   r   r   r   r   r    )rm   rH   rj   r   r   r   rb   ?  s   

rb   rI   )#r;   collectionsr   rw   
exceptionsr   lexerr   treer   rR   r   r   r   r   r	   r   r   r=   rF   rJ   rl   r_   r   r   r   r   r   r   r   r   r   rQ   r   rb   r   r   r   r   <module>   s4    
-	3
	