o
    ´©iÕ  ã                   @   sF   d dl Z ddedefdd„Zddefdd	„Zdefd
d„Zdd„ ZdS )é    Né   ÚwordsÚ
word_limitc                 C   s„   |dksJ ‚t | ƒ|kr| gS g }t | ƒ}|| }t|ƒD ]}| | || |d | … ¡ q|| dkr@| | || d … ¡ |S )Né   r   )ÚlenÚrangeÚappend)r   r   Ú	sentencesÚlengthÚsentence_lenÚi© r   úV/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/ct_transformer/utils.pyÚsplit_to_mini_sentence	   s    r   Útextc                 K   st  |r||   ¡ }g }g }g }d }|D ]9}t|ƒr'|dkr'| |¡ | d¡ g }nt|ƒs;|dkr;| |¡ | d¡ g }| |¡ t|ƒrGd}qd}q|rV| |¡ | |¡ g }	t||ƒD ]\}}|dkrk|	 |¡ q]|jt|ƒdd}
|	 |
¡ q]|	S g }|   ¡ }|D ]3}d}|D ]!}t| ¡ ƒdkr™||7 }qŠt|ƒdkr¦| |¡ d}| |¡ qŠt|ƒdkr·| |¡ q„|S )NÚChineseÚEnglishF)ÚHMMÚ r   r   )	ÚsplitÚ	isEnglishr   ÚzipÚextendÚcutÚjoin_chinese_and_englishr   Úencode)r   Újieba_usr_dictÚkwargsÚ
input_listÚtoken_list_allÚlangauge_listÚtoken_list_tmpÚlanguage_flagÚtokenÚresult_listÚseg_listr   ÚsegsÚsegÚcurrent_wordÚcr   r   r   Úsplit_words   sX   









€r*   c                 C   s   t  d| ¡rdS dS )Nz^[a-zA-Z']+$TF)ÚreÚsearch)r   r   r   r   r   R   s   r   c                 C   s8   d}| D ]}t |ƒr|d | }q|| }q| ¡ }|S )Nr   ú )r   Ústrip)r   Úliner#   r   r   r   r   Y   s   
r   )r   )N)r+   ÚlistÚintr   Ústrr*   r   r   r   r   r   r   Ú<module>   s
   ;