o
    
i.z                     @   s  d dl mZ d dlmZ ddlmZmZ ddlmZ ddlm	Z	m
Z
mZmZ ddlZddlZddlZddlZddlZddlZd@de	e d	e
e d
efddZedZdd ZeG dd dZdd Ze Z[edZedZedZ edd e D Z!eg dZ"dde#de#de#ddZ$dd ge%d!d"e%d#d$Z&ed%Z'ed&Z(d'd(d)d*Z)eg d+Z*d,d-d.Z+d/d0d1d2d3Z,ed4Z-ed5Z.d6Z/e/d  Z0e/d Z1ed7Z2d8d9 Z3d:d; Z4G d<d= d=Z5G d>d? d?Z6dS )A   )data)MToken    )	dataclassreplace)	num2words)ListOptionalTupleUnionNtokensunkreturnc                 C   sH  dd | D }dd | D }dd | D }|d u rd }n&d}| D ]!}|j jr5|r5|d  s5|jr5|d7 }||jd u r=|n|j7 }q tddd	 | d d D | d j t| d
d dj| d j	|| d j
| d jtj| d j jd t|dkr~t|d nd |rt|nd dtdd | D | d j jd |v rd nt|ddS )Nc                 S       h | ]}|j jd ur|j jqS N)_stress.0tk r   =/home/ubuntu/.local/lib/python3.10/site-packages/misaki/en.py	<setcomp>        zmerge_tokens.<locals>.<setcomp>c                 S   r   r   )r   currencyr   r   r   r   r      r   c                 S   s   h | ]}|j jqS r   )r   ratingr   r   r   r   r            c                 s       | ]	}|j |j V  qd S r   text
whitespacer   r   r   r   	<genexpr>       zmerge_tokens.<locals>.<genexpr>c                 S   s   t dd | jD S )Nc                 s   s$    | ]}||  krd ndV  qdS )r      N)lowerr   cr   r   r   r$         " z1merge_tokens.<locals>.<lambda>.<locals>.<genexpr>)sumr"   )r   r   r   r   <lambda>   r   zmerge_tokens.<locals>.<lambda>keyr   r   c                 S   s   h | ]}|j jD ]}|qqS r   )r   	num_flags)r   r   r)   r   r   r   r   %   s    )is_headaliasr   r   r/   prespacer   )r"   tagr#   phonemesstart_tsend_tsr   )r   r2   isspacer4   r   joinr"   maxr3   r#   r5   r6   
Underscorer0   lenlistsortedmin)r   r   r   r   r   r4   r   r   r   r   merge_tokens   s6   $

r?   u
   AIOQWYʤʧc                 C   s   | rt dd | D S dS )Nc                 s   s     | ]}|t v r
d ndV  qdS )r&   r   N)
DIPHTHONGSr(   r   r   r   r$   -       z stress_weight.<locals>.<genexpr>r   )r+   psr   r   r   stress_weight,   s   rD   c                   @   s*   e Zd ZU dZee ed< dZeed< dS )TokenContextNfuture_vowelF	future_to)__name__
__module____qualname__rF   r	   bool__annotations__rG   r   r   r   r   rE   /   s   
 rE   c                      s    dd l d  fddS )Nr   u   ^['‘’]+|\p{Lu}(?=\p{Lu}\p{Ll})|(?:^-)?(?:\d?[,.]?\d)+|[-_]+|['‘’]{2,}|\p{L}*?(?:['‘’]\p{L})*?\p{Ll}(?=\p{Lu})|\p{L}+(?:['‘’]\p{L})*|[^-_\p{L}'‘’\d]|['‘’]+$c                    s     | S r   )findall)wordSUBTOKEN_REGEXregexr   r   r,   8   s    z'make_subtokenize_once.<locals>.<lambda>)rQ   compiler   r   rO   r   make_subtokenize_once5   s   
rS   z\[([^\]]+)\]\(([^\)]*)\)u   ',-._‘’/u   ;:,.!?—…"“”c                 c   s    | ]	}|d vr|V  qdS )u   "“”Nr   )r   pr   r   r   r$   A   r%   r$   ).,-LRB--RRB-``""'':$#NFP()i   i   )rW   rX   rY   rZ   r[   '   -   A   [   a   {   u#   bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθu   AIOWYiuæɑəɛɪɹʊʌ)dollarcent)poundpence)eurori   )r]      £u   €)stndrdthdotslash)rU   /percentandplusat)%&+@uH   AIOWYbdfhijklmnpstuvwzæðŋɑɔəɛɜɡɪɹɾʃʊʌʒʤʧˈˌθᵊᵻʔuD   AIQWYabdfhijklmnpstuvwzðŋɑɒɔəɛɜɡɪɹʃʊʌʒʤʧˈˌːθᵊu   ˌˈu    AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻc                    s  dd }|d u r
 S |dk r  td tdS |dks$|dv r.t v r.  td ttS |dv rPt fddtD rPt fd	dtD rJ S |t  S |d
krbt vrbt v rb  ttS |d
krt fddtD rt fddtD r~ S |t  S  S )Nc                    sh   t t|   fdd D }| D ]\}} | \}}|d |f |< qddd t D } | S )Nc                    s4   i | ]\}}|t v r|td d  |d D qS )c                 s   s     | ]\}}|t v r|V  qd S r   )VOWELS)r   jvr   r   r   r$   `   rA   z<apply_stress.<locals>.restress.<locals>.<dictcomp>.<genexpr>N)STRESSESnext)r   irT   ipsr   r   
<dictcomp>`   s   4 z2apply_stress.<locals>.restress.<locals>.<dictcomp>      ?r   c                 S   s   g | ]\}}|qS r   r   )r   r   rT   r   r   r   
<listcomp>d   r   z2apply_stress.<locals>.restress.<locals>.<listcomp>)r<   	enumerateitemsr8   r=   )rC   stressesr   r~   r   sr   r   r   restress^   s   zapply_stress.<locals>.restressr   r   )r         )r   r   r   c                 3       | ]}| vV  qd S r   r   r   r   rB   r   r   r$   l       zapply_stress.<locals>.<genexpr>c                 3   r   r   r   r   r   rB   r   r   r$   m   r   r   c                 3   r   r   r   r   rB   r   r   r$   r   r   c                 3   r   r   r   r   rB   r   r   r$   s   r   )r   PRIMARY_STRESSSECONDARY_STRESSallr   r}   )rC   r   r   r   rB   r   apply_stress]   s$   r   c                 C   s   t td| S )Nz^[0-9]+$)rK   rematchr"   r   r   r   is_digitx   s   r   c                   @   s   e Zd Zedd Zdd Zdd Zdd Zed	d
 Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zedd Zdd  Zd!d" Zed#d$ Zed%d& Zd'd( Zd)S )*Lexiconc                 C   sr   i }|   D ],\}}t|dk rq|| kr$|| kr#||| < q||  kr2||| < qi || S )Nr&   )r   r;   r'   
capitalize)dekr   r   r   r   grow_dictionary|   s   zLexicon.grow_dictionaryc                    s^  || _ d| _i | _i | _tjt|rdnd d}t	t
|| _W d    n1 s.w   Y  tjt|r;dnd d}t	t
|| _W d    n1 sUw   Y  tdd | j D shJ |rltnt | j D ]9}t|trt fdd|D sJ |qsd	|v sJ || D ]}|d u st fd
d|D sJ |qqsd S )N)r   r&   gbusz
_gold.jsonz_silver.jsonc                 s   s$    | ]}t |tpt |tV  qd S r   )
isinstancestrdictr   r   r   r   r$      r*   z#Lexicon.__init__.<locals>.<genexpr>c                 3       | ]}| v V  qd S r   r   r(   vocabr   r   r$      r   DEFAULTc                 3   r   r   r   r(   r   r   r   r$      r   )britishcap_stressesgoldssilvers	importlib	resources	open_textr   r   r   jsonloadr   valuesGB_VOCABUS_VOCABr   r   )selfr   rvsr   r   r   r   __init__   s*   
 &zLexicon.__init__c                    sH    fdd|D }d |v rdS t d|d}|td}t|dfS )Nc                    s$   g | ]}|  r j| qS r   )isalphar   getupperr(   r   r   r   r         $ z#Lexicon.get_NNP.<locals>.<listcomp>NNr   r   r      )r   r8   rsplitr   r   )r   rN   rC   r   r   r   get_NNP   s   zLexicon.get_NNPc                 C   s^  |dkr|t v r| t | d d|S |tv r | t| d d |S d|dv rA|dd rAtt|dtddk rA| 	|S |dv rQ|dkrMd	dfS d
dfS |dv rw|
dr_| 	|S |jd u sn|dksn|ru|dkru| jd dfS dS |dv r|dkr|
dr| 	|S dS |dkr|dkrt ddfS |dv rt|dkrdS |dv s|dkr|dv r| jd ddd|j dfS |d v s|d!kr|d"kr|jd u s|d!krtnd}|d# dfS |d$v s|d%kr|dkr|jd&krd'dfS d(dfS |d!krtd)|r| d*d d |S |d+v r-|d,v r$|jr$| jd- d. dfS | jd- d/ dfS d0S )1NADDr   rU   r   r-   r   )aADTu   ɐu   ˈA   )amAmAMNNr   r   )u   ɐmr   )anAnANr   )u   ɐnr   IPRP)byByBYADV)u   bˈIr   toToTOr   INr   u   təu   tʊ)NFT)inInr   NNPu   ɪn)theTheTHETu   ðiu   ðəz
(?i)vs\.?$versus)usedUsedUSED)VBDJJr   r   r   r   )ADD_SYMBOLSlookupSYMBOLSstripr   r   r;   r9   splitr   
startswithrF   r   r   r   get_parent_tagr   r   r   rG   )r   rN   r3   r   ctxr   r   r   get_special_case   sH   8




zLexicon.get_special_casec                 C   s\   | d u r| S |  drdS |  drdS |  ds|  dr dS |  ds*|  dr,dS | S )	NVBVERBr   NOUNr   RBADJr   )r   )r3   r   r   r   r      s   

zLexicon.get_parent_tagc                 C   s   || j v s|tv s|| jv rdS | rtdd |D sdS t|dkr'dS || kr6| | j v r6dS |dd  |dd   kS )NTc                 s       | ]	}t |tv V  qd S r   ordLEXICON_ORDSr(   r   r   r   r$      r%   z#Lexicon.is_known.<locals>.<genexpr>Fr   )r   r   r   r   r   r;   r   r'   )r   rN   r3   r   r   r   is_known   s   zLexicon.is_knownc                 C   s   d }||  kr|| jvr| }|dk}| j|d}}|d u r-|s-| j|d}}t|trQ|r@|jd u r@d|v r@d}n	||vrIt	|}|||d }|d u s[|rjt
|vrj| |\}}|d urj||fS t|||fS )Nr   r   r   Noner   )r   r   r'   r   r   r   r   rF   r   r   r   r   r   )r   rN   r3   r   r   is_NNPrC   r   r   r   r   r      s$   

zLexicon.lookupc                 C   sL   |sd S |d dv r|d S |d dv r"|| j rd d S d d S |d S )Nr   u   ptkfθr   u
   szʃʒʧʤ   ɪ   ᵻz)r   r   stemr   r   r   _s   s   z
Lexicon._sc                 C   s   t |dk s|dsdS |ds#| |d d |r#|d d }nH|ds8t |dkrI|drI|d	sI| |d d
 |rI|d d
 }n"t |dkri|d	ri| |d d d |ri|d d d }ndS | ||||\}}| ||fS )Nr   r   r   ssr   'sr   esiesy)r;   endswithr   r   r   r   rN   r3   r   r   r   r   r   r   r   stem_s  s   >.zLexicon.stem_sc                 C   s   |sd S |d dv r|d S |d dkr"|| j rd d S d d S |d dkr,|d S | j s5t|dk r9|d S |d	 tv rG|d d d
 S |d S )Nr   u
   pkfθʃsʧtr   r   r   r&   u   ɪdr  u   ɾᵻdu   ᵻdr   r;   US_TAUSr   r   r   r   _ed  s   zLexicon._edc                 C   s   t |dk s|dsdS |ds#| |d d |r#|d d }n#t |dkrD|drD|dsD| |d d |rD|d d }ndS | ||||\}}| ||fS )	Nr   r   r   ddr   edeedr  )r;   r  r   r   r  r	  r   r   r   stem_ed   s   4zLexicon.stem_edc                 C   sb   |sd S | j r|d dv rd S |d S t|dkr-|d dkr-|d tv r-|d d d S |d S )Nr   u   əːr   r  r  u   ɾɪŋu   ɪŋr  r   r   r   r   _ing,  s   $zLexicon._ingc                 C   s   t |dk s|dsdS t |dkr$| |d d |r$|d d }n4| |d d d |r9|d d d }nt |dkrVtd|rV| |d d |rV|d d }ndS | ||||\}}| ||fS )N   ingr   r  r   z ([bcdgklmnprstvxz])\1ing$|cking$)r;   r  r   r   searchr   r  r	  r   r   r   stem_ing;  s    ,zLexicon.stem_ingc           
         s  |  | \}}|d ur||fS | t|dkrr|dd rr|| krrdks4t|dkrr|| jvrr|| jvrr|| ksR|dd  |dd   krr| jv sp| jv spt fdd| j	| j
| jfD rr}| |r| | S |dr| |d d	 d
 r| |d d	 d
  S |dr| |d d r| |d d  S | 	| \}}|d ur||fS | 
| \}}|d ur||fS | |d u rdn \}	}|	d ur|	|fS dS )Nr   'r   r      c                 3   s"    | ]}| d  V  qdS )r   Nr   )r   fnr   r   r3   wlr   r   r$   S  s    
z#Lexicon.get_word.<locals>.<genexpr>zs'r  r  r   r   r   )r   r'   r;   r   r   r   r   r   anyr
  r  r  r   r   r  )
r   rN   r3   r   r   rC   r   r   r  r  r   r  r   get_wordI  s8   ((&" zLexicon.get_wordc                 C   sF   d| vrdS |  ddkrdS | dd }t|dk p"t|dhkS )NrU   Tr   Fr   r   )countr   r;   set)rN   centsr   r   r   is_currencyi  s   zLexicon.is_currencyc              	      s|  t d|}|r| nd }|r|d t|  n|}g |dr3dd d d  |dd  }d.fdd	 t|rR|tv rR t	t
|d	d
dd nspt|dkrp|tvrpt|rp t	t
|dd
dd n|sd|vr|dd}|d dkst|dkr fdd|D  nct|dkr|dsˈ |d  |d dkrdd dd   |d dd n7 |dd  dd n+ | n%|ddks|sd}|dddD ]3}|sn,|d dkst|dkrtdd |dd  D r fdd|D  n ||d d}qn|tv rt|rdd t|dddt| D }t|dkr_|d d dkrP|d d }n|d d dkr_|dd  }t|D ]@\}	\}}
|	dkrydd d d   ||	dkd t|dkr|
d kr|
d! d d d n|
d d d  qcnQt|rt	t
|d"d
}n=d|vrt	t
|dd|tv rd	nd"d
}n$|dd}|d dkrd#d$d%d |dd  D  }nt	t|} |dd std&d'|| d(S d$d)d D td*d D }|d+v r"|fS |d,v r.|fS |d-kr:|fS |fS )/Nz[a-z']+$-minusr   TFc              	      s   t d|r| ntt| }t|D ]R\}}|dksd v rJ|r8|dkr8t|dkr8|dkr8d v r8d q|d |d	krDd
nd d  q|dkrdd v rdrdd d d d d fd< qd S )Nz[^a-z]+rv   rz   r   r   oner   )u   ər   pointr  nr   u   ən)r   r   r   intr   r;   appendr   )numfirstescapesplitsr   w)r/   resultr   r   r   
extend_numz  s   ($ z&Lexicon.get_number.<locals>.extend_numordinal)r   )r-  r   yearrU   rV   r   r   0r   c                       g | ]} |d dqS Fr,  r   r   r(  r1  r   r   r         z&Lexicon.get_number.<locals>.<listcomp>00Or  r&   r7  c                 s   s    | ]}|d kV  qdS )r4  Nr   r8  r   r   r   r$     r   z%Lexicon.get_number.<locals>.<genexpr>c                    r5  r6  r   r8  r9  r   r   r     r:  c                 S   s$   g | ]\}}|rt |nd |fqS )r   )r)  )r   r+  unitr   r   r   r     r   rv   rk   r   cardinalzpoint r   c                 s   s    | ]	}t t|V  qd S r   )r   r)  r8  r   r   r   r$     r%   u   ❌zTODO:NUMr   c                 s   s    | ]\}}|V  qd S r   r   )r   rT   r   r   r   r   r$     r   c                 s   s    | ]\}}|V  qd S r   r   )r   r   r   r   r   r   r$     r   )r   r  )r  'dr  )TF)r   r  groupr;   r   r*  r   r   ORDINALSr   r)  
CURRENCIESr   r  r   r   r  r   r#  zipr   absr
  r8   floatprintr>   r   r  r  )r   rN   r   r0   r/   suffixr+  r,  pairsr   r=  r   r   )r1  r/   r0  r   r   
get_numberr  s   

 8&
F

("(


zLexicon.get_numberc                 C   sL   |s|S t |}|r| |d d d d d d nd }|r$| d| S |S )Nr   r   r   )rB  r   r
  )r   rC   r   r   r   r   append_currency  s
   
$zLexicon.append_currencyc                 C   s2   |   s| S t| }|t|krtt|S | S r   )isdigitunicodedatanumericr)  r   )r)   r(  r   r   r   numeric_if_needed  s   
zLexicon.numeric_if_neededc                    sp   t dd | D rdS ddddgtdR }|D ]}| |r*| d t|  }  nqt  fd	dt| D S )
Nc                 s   s    | ]}t | V  qd S r   r   r(   r   r   r   r$         z$Lexicon.is_number.<locals>.<genexpr>Fr  r?  r  r  r   c                 3   s8    | ]\}}t |p|d v p o|dko|dkV  qdS )z,.r   r$  NrO  )r   r   r)   r0   r   r   r$     s   6 )r   rA  r  r;   r   )rN   r0   suffixesr   r   rQ  r   	is_number  s   
zLexicon.is_numberc                 C   s  |j jd u r	|jn|j jtddtdd}td|}ddd |D }|| kr1d n
| j	t
|| k }| ||j||\}}|d urZt| ||j j|j j|fS t||j jr{| ||j j|j j|j j\}}t||j j|fS tdd |D sd	S d	S )
Ni   r  i   NFKCr   c                 s   s    | ]}t |V  qd S r   )r   rN  r(   r   r   r   r$     rP  z#Lexicon.__call__.<locals>.<genexpr>c                 s   r   r   r   r(   r   r   r   r$     r%   r   )r   r1   r"   r   chrrL  	normalizer8   r'   r   r)  r   r  r3   r   rJ  r   r   r   rS  r0   rI  r/   r   )r   r   r   rN   r   rC   r   r   r   r   __call__  s   2& zLexicon.__call__N)rH   rI   rJ   staticmethodr   r   r   r   r   r   r   r   r
  r  r  r  r  r  r#  rI  rJ  rN  rS  rW  r   r   r   r   r   {   s4    
&

 
Q


r   c                	   @   s   e Zd ZdddZedd Zded	ee fd
dZ	dee d	ee fddZ
edee d	eeeee f  fddZedd Zedd Zdded	eeee f fddZdS )G2PNF   ❓c                 C   sx   || _ || _d|rdnd }tj|stj| |rdnddg}tj||d| _t	|| _
|r4|nd | _|| _d S )Nen_core_web_trfsmtransformertok2vectagger)enable)versionr   spacyutil
is_packageclidownloadr   nlpr   lexiconfallbackr   )r   rb  r\  r   rj  r   name
componentsr   r   r   r     s   

zG2P.__init__c                 C   s  d}g }i }d}|   } t| D ]}|| ||  7 }|| ||    |d}t||d d dv r;dndd  rFt|}nP|dv rMd}nI|dkrTd	}nBt	|dkrt|d d
krt|d d
krt|d |dd  
d
 }n"t	|dkr|d dkr|d dkr|d |dd  
d }nd }|d ur||t	|< ||d7 }||d | }q|t	| k r|| |d  7 }|| |d    |||fS )Nr   r   r&   r   )r$  r{   )z0.5z+0.5r   z-0.5r   rt   r   r^   )lstrip
LINK_REGEXfinditerstartextendr   r@  r   r)  r;   rstripr*  end)r"   r0  r   featureslast_endmfr   r   r   
preprocess  s:   
$
$$

zG2P.preprocessr"   r   c                 C   s"  |  |}dd |D }|s|S tjj|dd |D }| D ]m\}}t|ts9t|ts9|dv s9J ||ft	t
|jj|kd D ]G\}	}
|
t|krQqFt|ts]|||
 j_qF|dr|	dk||
 j_|	dkrs|dnd||
 _d||
 j_qF|d	r|d	||
 j_qFq!|S )
Nc                 S   s.   g | ]}t |j|j|jt jd ddddqS )Tr   F)r0   r/   r2   )r"   r3   r#   r   )r   r"   tag_whitespace_r:   r   r   r   r   r      s    
z G2P.tokenize.<locals>.<listcomp>c                 S   s   g | ]}|j qS r   r   r   r   r   r   r   &  s    )r   r   r   rt   r   r  r^   )rh  rc  training	Alignmentfrom_stringsr   r   r   r)  r   npwherey2xr   r;   r   r   r   r0   rm  r4   r   r/   )r   r"   r   rt  docmutable_tokensalignr   r   r   r~   r   r   r   tokenize  s.   
("


zG2P.tokenizer   c                 C   sB   g }|D ]}|r|j jst| |g| jdn|}|| q|S )Nr   )r   r0   r?   popr   r*  )r   r   r0  r   r   r   r   	fold_left8  s
   &zG2P.fold_leftc              	      sT  g }d }t | D ]\}  jjd u r% jd u r% fddt jD }n g} j|d _t |D ]\}}|jjd usA|jd urBn|jdkrW|jtv rW|j}d|_d|j_	n|jdkri|jdv rid	|_d
|j_	n||jt
v rtdd |jD st|jddd |jD |_d|j_	nW|d ur|jdkrd }nK|d t|kr|d t| ks| |d  jdkr||j_n-d|  k rt|d k rn n|jdkr||d  jd ||d  jd   rd|j_|jjd us|jd ur|| q2|rt|d tr|d d jsd|j_|d | q2||jr|n|g q2qdd |D S )Nc                    s2   g | ]}t  |d tjd jj jjdddqS )r   TF)r0   r/   r   r2   )r"   r#   r   )r   r   r:   r   r/   r   )r   r  tokenr   r   r   E  s    
z"G2P.retokenize.<locals>.<listcomp>r   r]   r   r   r\   )r$  u   –u   —r   c                 s   s0    | ]}d t |   kodkn  V  qdS )rf   z   N)r   r'   r(   r   r   r   r$   V  s   . z!G2P.retokenize.<locals>.<genexpr>c                 s   s    | ]	}|t v r|V  qd S r   )PUNCTSr(   r   r   r   r$   W  r%   CDr   r   2r   Fc                 S   s.   g | ]}t |trt|d kr|d n|qS )r   r   )r   r<   r;   )r   r/  r   r   r   r   i  s   . )r   r   r1   r4   subtokenizer"   r#   r3   rB  r   
PUNCT_TAGSr   PUNCT_TAG_PHONEMESr   r8   r;   r   r   r*  r   r<   r0   )r   wordsr   r   tksr~   r   r   r  r   
retokenize?  sL   


"

2R&zG2P.retokenizec                 C   sL   | j }|rtdd |D |n|}|jdv p|jdko|jdv }t||dS )Nc                 3   s@    | ] t  fd dtttfD r tv rdn tv V  qdS )c                 3   s    | ]} |v V  qd S r   r   r   r)   r   r   r$   n  r   z.G2P.token_context.<locals>.<genexpr>.<genexpr>N)r  r}   
CONSONANTSNON_QUOTE_PUNCTS)r   r   r  r   r$   n  s   > z$G2P.token_context.<locals>.<genexpr>r   r   r   )rF   rG   )rF   r   r"   r3   rE   )r   rC   r  vowelrG   r   r   r   token_contextk  s   zG2P.token_contextc                 C   s  d dd | d d D | d j }d|v p%d|v p%tdd |D d	k}t| D ]9\}}|jd u r[|t| d	 krI|jtv rI|j|_d
|j_q*tdd |jD rZd|_d
|j_q*|dkrc||j_	q*|rhd S dd t| D }t|dkrt| |d d  jd	kr|d	 d }t
| | jd| | _d S t|dk stdd |D t|d	 d krd S t|d t|d  }|D ]\}}}t
| | jd| | _qd S )Nr   c                 s   r    r   r!   r   r   r   r   r$   t  r%   z%G2P.resolve_tokens.<locals>.<genexpr>r   r   rt   c                 S   s0   h | ]}|t vr| rd nt|rdndqS )r   r   r&   )SUBTOKEN_JUNKSr   r   r(   r   r   r   r   u  s   0 z%G2P.resolve_tokens.<locals>.<setcomp>r   r   c                 s       | ]}|t v V  qd S r   r  r(   r   r   r   r$   {  r   r   c                 S   s,   g | ]\}}|j rt|j v t|j |fqS r   )r4   r   rD   )r   r   r   r   r   r   r     s   , z&G2P.resolve_tokens.<locals>.<listcomp>r&   r   c                 s   s    | ]\}}}|V  qd S r   r   )r   br   r   r   r   r$     rP  )r8   r"   r;   r   r4   r  r   r   r   r2   r   r+   r=   )r   r"   r2   r   r   indicesr   r   r   r   resolve_tokensr  s6   &&

&.zG2P.resolve_tokensTc                    s  |dkrt jn|}|r||n|g i f\}}} |||} |}t |}t }ttt|D ]\}}t	|tsr|j
d u rQ t||jd|\|_
|_|j
d u ri jd uri t||jd\|_
|_t ||j
|}q3dt|}}	d}
||	k rtdd |||	 D rd }nt|||	 }|d u rdn ||\}}|d ur||| _
||| j_||d |	 D ]}d	|_
||_qt |||}|}	d}n6|d |	k r|d7 }n+|	d8 }	||	 }|j
d u rtd
d |jD rd	|_
d|j_n	 jd urd}
nd}||	k s|
r<t|} |\|d _
|d j_tdt|D ]}d	|| _
|d jj|| j_q(q3t | q3 fdd|D } jdkrg|D ]}|j
re|j
dddd|_
qSd	 fdd|D }||fS )NT)r   r   Fc                 s   s&    | ]}|j jd up|jd uV  qd S r   )r   r1   r4   r   r   r   r   r$     s   $ zG2P.__call__.<locals>.<genexpr>r   r   r   c                 s   r  r   r  r(   r   r   r   r$     r   r   c                    s(   g | ]}t |trt| jd n|qS )r  )r   r<   r?   r   r   r   r   r   r     s   ( z G2P.__call__.<locals>.<listcomp>z2.0u   ɾTu   ʔr  c                 3   s,    | ]}|j d u r jn|j |j V  qd S r   )r4   r   r#   r   r   r   r   r$     s   * )rY  rx  r  r  r  rE   reversedr<   r   r   r4   ri  r   r   r   rj  r  r;   r  r?   r   r"   ranger  rb  r8   )r   r"   rx  r   rt  r   r   r/  leftrightshould_fallbackr   rC   r   xr~   r0  r   r   r   rW    sv   








zG2P.__call__)NFFNrZ  )T)rH   rI   rJ   r   rX  rx  r   r   r   r  r  r   r  r  r  r
   rW  r   r   r   r   rY    s    

 (+

$rY  r   )7r   r   r  r   dataclassesr   r   r   typingr   r	   r
   r   importlib.resourcesr   r   numpyr~  r   rc  rL  r   r?   	frozensetr@   rD   rE   rS   r  rR   rn  r  r  r  r  rU  r  r  r   r  r  rB  rA  r   r   r   r   r   r   r   r}   r   r   r   rY  r   r   r   r   <module>   s^     

  v