o
    iW                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ	 d dl
mZ d dlZd dlZd dlmZ d dlZd dlZd dlZd dlmZ d dlmZ eddG dd dejZdS )	    N)
Collection)nn)TextCleaner)tablespreprocessor_classesTextPreprocessRemovePunctuationc                       s$   e Zd Z fddZdd Z  ZS )r   c                    s   t    d S )N)super__init__)selfkwargs	__class__ ]/home/ubuntu/.local/lib/python3.10/site-packages/funasr/datasets/llm_datasets/preprocessor.pyr	      s   z(TextPreprocessRemovePunctuation.__init__c                 K   s4   t j}d}|| }tdt|}|d|S )Nu>   。？！，、；：“”‘’（）《》【】…—～·z[{}] )stringpunctuationrecompileformatescapesub)r
   textr   en_punctcn_punct	all_punctpunct_patternr   r   r   forward   s
   z'TextPreprocessRemovePunctuation.forward)__name__
__module____qualname__r	   r   __classcell__r   r   r   r   r      s    )osjsontorchloggingconcurrent.futures
concurrentlibrosatorch.distributeddistributeddisttypingr   
torchaudior   randomr   r   funasr.tokenizer.cleanerr   funasr.registerr   registerModuler   r   r   r   r   <module>   s$    
