o
    ۷i                     @   s   d dl Z d dlZd dlZi dddddddd	d
dddddddddddddddddddddddZd'd efd!d"Zd efd#d$ZG d%d& d&ZdS )(    Nu   œoeu   ŒOE   øo   ØO   æae   ÆAE   ßssu   ẞSSu   đdu   ĐD   ð   Ð   þth   Þu   łlu   ŁL sc                    s    d  fddtd| D S )z
    Replace any other markers, symbols, and punctuations with a space,
    and drop any diacritics (category 'Mn' and some manual mappings)
    r   c                 3   sX    | ]'}| v r
|n|t v rt | nt|d krdnt|d dv r&dn|V  qdS )Mnr   r   MSP N)ADDITIONAL_DIACRITICSunicodedatacategory.0ckeep O/home/ubuntu/vllm_env/lib/python3.10/site-packages/whisper/normalizers/basic.py	<genexpr>    s    
z0remove_symbols_and_diacritics.<locals>.<genexpr>NFKDjoinr   	normalize)r   r$   r%   r#   r&   remove_symbols_and_diacritics   s   
r,   c                 C   s   d dd td| D S )z[
    Replace any other markers, symbols, punctuations with a space, keeping diacritics
    r   c                 s   s*    | ]}t |d  dv rdn|V  qdS )r   r   r   N)r   r   r    r%   r%   r&   r'   6   s
    
z!remove_symbols.<locals>.<genexpr>NFKCr)   )r   r%   r%   r&   remove_symbols2   s   

r.   c                   @   s.   e Zd Zd
dedefddZdefddZd	S )BasicTextNormalizerFremove_diacriticssplit_lettersc                 C   s   |rt nt| _|| _d S )N)r,   r.   cleanr1   )selfr0   r1   r%   r%   r&   __init__=   s   

zBasicTextNormalizer.__init__r   c                 C   s`   |  }tdd|}tdd|}| |  }| jr'dtd|tj}tdd|}|S )Nz[<\[][^>\]]*[>\]]r   z\(([^)]+?)\)r   z\Xz\s+)	lowerresubr2   r1   r*   regexfindallU)r3   r   r%   r%   r&   __call__C   s   zBasicTextNormalizer.__call__N)FF)__name__
__module____qualname__boolr4   strr;   r%   r%   r%   r&   r/   <   s    r/   )r   )r6   r   r8   r   r@   r,   r.   r/   r%   r%   r%   r&   <module>   sN    	

