o
    
i                     @   s   d dl Z d dlZddlmZmZmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZ G dd deZdS )    N   )combine_passagessplit_long_passagessplit_text_passages)
get_piecesnormalize_letter_vinormalize_currency_vispell_acronyms_vinormalize_number_vinormalize_measurement_vinormalize_datenormalize_timenormalize_roman_numbersnormalize_abbreviations_vi)DEFAULT_PIECE_MAX_LENGTHDEFAULT_SENTENCE_MAX_LENGTH%opening_brackets_and_punctutations_repunctutations_rec                   @   s   e Zd Zd1ddZdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,efd-d.Zd,efd/d0Zd,S )2	ViCleanerTc                 C   s   || _ || _d S N)
clean_abbrclean_acronym)selfr   r    r!   M/home/ubuntu/.local/lib/python3.10/site-packages/misaki/vi_cleaner/cleaner.py__init__   s   zViCleaner.__init__c                 C   s   | ddS )N
 )replacer    textr!   r!   r"   
join_lines      zViCleaner.join_linesc                 C   s   | d\}}|S Nr   groups)r    matchspacepunctuationr!   r!   r"   '_collapse_whitespace_before_punctuation      z1ViCleaner._collapse_whitespace_before_punctuationc                 C   s   | d\}}|| S r+   r,   )r    r.   r0   r/   r!   r!   r"   "_collapse_whitespace_after_bracket    s   z,ViCleaner._collapse_whitespace_after_bracketc                 C   sV   t dd|}t t| j|}t t| j|}t dd|}t dd|}| }|S )Nz
(\s)\1{1,}z\1z\t+r%   )resubr   r1   r   r3   stripr'   r!   r!   r"   collapse_whitespace$   s   zViCleaner.collapse_whitespacec                 C   s   |  S r   )lowerr'   r!   r!   r"   	lowercase/      zViCleaner.lowercasec                 C   s   |  |}d| d }|S Nr%   )r7   r'   r!   r!   r"   clean_basic2   s   
zViCleaner.clean_basicc                 C   s   t d|S )NNFC)unicodedata	normalizer'   r!   r!   r"   normalize_ascii_vi7   r*   zViCleaner.normalize_ascii_vic                 C   s   t |}|S r   r   r'   r!   r!   r"   expand_abbreviations:   s   zViCleaner.expand_abbreviationsc                 C      t |S r   r   r'   r!   r!   r"   expand_acronyms>   r:   zViCleaner.expand_acronymsc                 C   rB   r   r   r'   r!   r!   r"   expand_roman_numbersA   r:   zViCleaner.expand_roman_numbersc                 C   s   t |}t|}|S r   r   r'   r!   r!   r"   expand_date_timeD   s   zViCleaner.expand_date_timec                 C   s   t jdd|t jdS )Nu   tháng bốnu
   tháng tư)flags)r4   r5   
IGNORECASEr'   r!   r!   r"   change_thang_bon_to_thang_tuI   s   z&ViCleaner.change_thang_bon_to_thang_tuc                 C   rB   r   r   r'   r!   r!   r"   expand_measurement_unitsL   r:   z"ViCleaner.expand_measurement_unitsc                 C   rB   r   r	   r'   r!   r!   r"   expand_currencyO   r:   zViCleaner.expand_currencyc                 C   rB   r   r   r'   r!   r!   r"   expand_numberR   r:   zViCleaner.expand_numberc                 C   rB   r   r   r'   r!   r!   r"   expand_letterU   r:   zViCleaner.expand_letterc                 C   s   t dd|}|S )Nz(/)u    trên r4   r5   r'   r!   r!   r"   normalize_splashX   r2   zViCleaner.normalize_splashc                 C   s   t dd|S )Nz([^\s])(-)([^\s])z\1 \3rM   r'   r!   r!   r"   remove_left_hyphen\   s   zViCleaner.remove_left_hyphenc                 C   s   dd t d|D S )Nc                 S   s   g | ]}|  qS r!   )r6   ).0er!   r!   r"   
<listcomp>`       z1ViCleaner.normalize_linebreak.<locals>.<listcomp>[\n]+)r4   splitr'   r!   r!   r"   normalize_linebreak_   s   zViCleaner.normalize_linebreakc                 C   s   |  |}d| d }| |}| jr| |}| |}| jr%| |}| |}| |}| 	|}| 
|}| |}| |}| |}|  |}| |}| |}|S r;   )r7   r@   r   rA   rD   r   rC   rE   rI   rJ   rK   rL   rN   rO   rH   r9   r'   r!   r!   r"   
clean_textb   s&   














zViCleaner.clean_textNc                 C   s   |d ur|n| j }tdd|}| |}g }g }|D ]$}t||}||7 }t|dkr8||d t| g7 }q|t|g7 }q||dt|d  fS )Nz(?<![.!?])[
]+z.
r   r   )r(   r4   r5   rV   r   len)r    r(   	maxLengthpassagesresultbreakspassagetempr!   r!   r"   split_sentencesy   s   

zViCleaner.split_sentencesc                    s   |d ur|n| j }t|ddd}t| } fdd|D }dd |D fddttD }g }|D ]}|D ]}	||	 q:q6t| }
|
|fS )NrT   Fz
	 c                    s   g | ]}t | qS r!   )r   rP   i)rZ   r!   r"   rR      s
    z,ViCleaner.split_passages.<locals>.<listcomp>c                 S   s   g | ]}t |qS r!   )rY   ra   r!   r!   r"   rR      rS   c                    s    g | ]}t  d |d  qS )Nr   )sumra   )sub_passages_lensr!   r"   rR      s    )r(   r   r   rangerY   appendr   )r    r(   rZ   r[   sub_passagescombined_sub_passagesr]   	flat_listsublistitemr\   r!   )rZ   rd   r"   split_passages   s"   




zViCleaner.split_passages)TT)__name__
__module____qualname__r#   r)   r1   r3   r7   r9   r<   r@   rA   rC   rD   rE   rH   rI   rJ   rK   rL   rN   rO   rV   rW   r   r`   r   rl   r!   r!   r!   r"   r      s0    
r   ) r4   r>   passage_utilsr   r   r   sentence_utilsr   	letter_vir   currency_vir
   
acronym_vir   numberical_vir   measurement_vir   datestime_vir   r   roman_number_vir   abbreviation_vir   	symbol_vir   r   r   r   objectr   r!   r!   r!   r"   <module>   s    