o
    ॵi4                     @   s   d dl Z d dlZd dlZd dlmZmZmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZ d d	lmZ e Zejejd
G dd deZdS )    N)DictOptionalUnion)build_trainer)Trainers)	MsDataset)BaseTrainer)TRAINERS)DEFAULT_DATASET_NAMESPACEDEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISION	ModelFileTasksTrainerStages)
get_logger)module_namec                   @   s   e Zd ZdZddddeddddf	dedededed	eee	ef  d
ee dee
 dee
 dee dee fddZdd Zdd ZdddZdd Zdedeeef fddZdS )
ASRTrainerdataNFsmallmodelwork_dirdistributeddataset_typedata_dirmodel_revision
batch_bins	max_epochlrmate_paramsc                 K   sB  |st  j| _tj| jst| j n|| _tj| js(t| j dt	
d| j  tj| j| j| _d| _|| _|| _tj| jdd tj| jdd tj|r]|}n| ||}|| _tj| jd| _| | j| _d|vr| || j\| _| _n|d | _tj| j| j| j| j| j|||	|
d		| _d
S )a  ASR Trainer.

        Args:
            model (str) : model name
            work_dir (str): output dir for saving results
            distributed (bool): whether to enable DDP training
            dataset_type (str): choose which dataset type to use
            data_dir (str): the path of data
            model_revision (str): set model version
            batch_bins (str): batch size
            max_epoch (int): the maximum epoch number for training
            lr (float): learning rate
            mate_params (dict): for saving other training args
        Examples:

        >>> import os
        >>> from modelscope.metainfo import Trainers
        >>> from modelscope.msdatasets import MsDataset
        >>> from modelscope.trainers import build_trainer
        >>> ds_dict = MsDataset.load('speech_asr_aishell1_trainsets')
        >>> kwargs = dict(
        >>>     model='damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch',
        >>>     data_dir=ds_dict,
        >>>     work_dir="./checkpoint")
        >>> trainer = build_trainer(
        >>>     Trainers.speech_asr_trainer, default_args=kwargs)
        >>> trainer.train()

        z not existszSet workdir to  Tignore_errorsexist_okzconfiguration.jsonraw_data_dir)	modelscope_dictr   
output_dirr   r   r   r   r   r   N)tempfileTemporaryDirectorynamer   ospathexistsmakedirs	ExceptionloggerinfojoinDATA_DIRr   raw_dataset_pathr   r   shutilrmtreeget_or_download_model_dir	model_dir	model_cfg	parse_cfgcfg_dictload_dataset_raw_pathtrain_data_dirdev_data_dirr   trainer)selfr   r   r   r   r   r   r   r   r   r   kwargsr7    rA   Y/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/audio/asr_trainer.py__init__   sJ   )
zASRTrainer.__init__c                 C   sj  t j|}t }t|ddd}t|}|d d d |d< ||d< t j||d d |d	< t j||d d d
 |d
< t j|d|d< t j||d d d |d< t j|d|d< d|d d v ryt j||d d d |d< nd |d< d|d d v rt j||d d d |d< n|d	 |d< W d    |S W d    |S 1 sw   Y  |S )Nrzutf-8)encodingr   model_configmoder7   am_model_nameam_model_fileam_model_configzfinetune.yamlfinetune_configmvn_file	cmvn_fileseg_dictbpemodel
init_model)r*   r+   dirnamedictopenjsonloadr1   )r?   cfg_filecur_dirr:   fconfigrA   rA   rB   r9   q   sH   




zASRTrainer.parse_cfgc                 C   sT   d|vrt d|| j||dd}d|vrt d|| j||dd}||fS )Ntrainz*dataset {0} does not contain a train split)split
validationz(dataset {0} does not contain a dev split)r.   formatprepare_data)r?   datasetoutput_data_dirr<   r=   rA   rA   rB   r;      s   z ASRTrainer.load_dataset_raw_pathrZ   c                 C   s   t j||}tj|dd t j|dd t|| }tt j|dd}tt j|dd}t|D ]0}|| | }	|	d }
|	d }|	d	t j
|
|
gd
  |	d	t j
|
|gd
  q3|  |  |S )NTr    r"   zwav.scpwtextz
Audio:FILEz
Text:LABEL	
)r*   r+   r1   r4   r5   r-   lenrS   rangewritebasenameclose)r?   r_   out_base_dirr[   out_dirdata_cnt
fp_wav_scpfp_texticontentwav_filerb   rA   rA   rB   r^      s"   "zASRTrainer.prepare_datac                 O   s   | j   d S N)r>   run)r?   argsr@   rA   rA   rB   rZ      s   zASRTrainer.traincheckpoint_pathreturnc                 O   s   t rr   )NotImplementedError)r?   ru   rt   r@   rA   rA   rB   evaluate   s   zASRTrainer.evaluate)rZ   )__name__
__module____qualname__r2   r   strboolr   r   r   intfloatrR   rC   r9   r;   r^   rZ   r   rx   rA   rA   rA   rB   r      sN    	

V

r   )r*   r4   r'   typingr   r   r   rT   
funasr.binr   modelscope.metainfor   modelscope.msdatasetsr   modelscope.trainers.baser   modelscope.trainers.builderr	   modelscope.utils.constantr
   r   r   r   r   r   modelscope.utils.loggerr   r/   register_modulespeech_asr_trainerr   rA   rA   rA   rB   <module>   s    