o
    ॵi'                     @   sn  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZ d dlm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4m5Z5m6Z6 e&j7ej8dG dd de$Z9dS )    N)partial)ignore_patterns)CallableDictOptionalTupleUnion)distributed)nn)Dataset)model_file_download)Trainers)Model
TorchModel)	MsDataset)Preprocessor)OfaPreprocessor)
collate_fn)EpochBasedTrainer)TRAINERS)build_optimizer)is_parallel)Config)DEFAULT_MODEL_REVISION
ConfigKeysInvokeModeKeys	ModelFile   )(AdjustLabelSmoothedCrossEntropyCriterionget_schedulerecursive_overwrite)module_namec                       s   e Zd ZdZdddddddddedfdeeeej	e
f  dee
 dee dee d	eeeee
ef f  d
eeeef  deeeef  deeeee
ef f  deejjejjjf dee
 def fddZdefddZde
fddZdd Z  ZS )
OFATrainera  
    OFA trainer for MaaS.

    Args:
        model (`str`): A model dir or a model id to be loaded
        cfg_file (`str`, **optional**, default to `None`):
            A config dir
        cfg_modify_fn (`Callable`, **optional**, default to `None`):
            A function which can rebuild the config file.
        arg_parse_fn (`Callable`, **optional**, default to `None`):
            Same as ``parse_fn`` in :obj:`Config.to_args`.
        data_collator (`Callable`, **optional**, default to `None`):
            The function to use to form a batch from a list of elements
            of `train_dataset` or `eval_dataset`.
        train_dataset (:obj:`MsDataset` or :obj:`Dataset`, **optional**, default to `None`):
            Dataset for training.
        eval_dataset (:obj:`MsDataset` or :obj:`Dataset`, **optional**, default to `None`):
            Dataset for evaluation.
        preprocessor (:obj:`Preprocessor`, **optional**, default to `None`):
            The optional preprocessor.
            NOTE: If the preprocessor has been called before the dataset fed into this trainer by user's custom code,
            this parameter should be None, meanwhile remove the 'preprocessor' key from the cfg_file.
            Else the preprocessor will be instantiated from the cfg_file or assigned from this parameter and
            this preprocessing action will be executed every time the dataset's __getitem__ is called.
        model_revision (`str`, **optional**, default to `None`):
            The revision used when the model_name_or_path is
                a model id of the remote hub. default `None`.
        seed (`int`, **optional**, default to `42`):
            The optional random seed for torch, cuda, numpy and random.
    N)NN*   modelcfg_filecfg_modify_fnarg_parse_fndata_collatortrain_dataseteval_datasetpreprocessor
optimizersmodel_revisionseedc                    s  t j||
tjd}|j}|| _|dd}tj|dd t	 }|d urJ| 
|}tjtj|tj}tj|}||krDt|| |tj t||t| d tj|tj}| t|}|d ur| |}t|d}tjt||dd	 W d    n1 sw   Y  |d u rtjt|tj dd
tj!t|tj"dd
i}t#tj$dd}t%&t'||jj(j)|  }||jj* |jj+_,|j-|jj._-t/|jj.| _.|	d d u rt0||jj1d}n|	d }|	d d u rt2|jj+\}}|d ur|di d|i|}nd }n|	d }||f}	|d u rt3t4|j-j5|j-j6d}d|vr1|jdd r1|jj7|d< d|vrD|jddrD|jj8|d< d|d< t9 j:d|||||||||	|d
| d S )N)revision
invoked_bywork_dir	workspaceT)exist_ok)ignorew   )fpindent)	model_dirmode
no_collate
WORLD_SIZEr   r   )cfg	optimizer)pad_idxeos_idxlauncheruse_fp16F	to_tensor)
r%   r&   r(   r'   r)   r*   r+   r,   r-   r/    );r   from_pretrainedr   TRAINERr:   r'   getosmakedirssetget_config_filepathabspathjoinr   CONFIGURATIONshutilcopyaddr!   r   rebuild_configr   	from_fileopenjsondumpdictr   trainr   r   TRAINvalEVALintenvironmathceillen
dataloaderbatch_size_per_gpu
max_epochslr_schedulernum_train_steps	tokenizer	criterionr   r   r?   r    r   r   pad_token_ideos_token_idrB   rC   super__init__)selfr%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   kwargsr:   r2   ignore_file_setdstsrcr>   writer
world_sizeepoch_stepsr?   scheduler_classscheduler_argsrf   	__class__rE   c/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/multi_modal/ofa/ofa_trainer.pyrm   D   s   


	


zOFATrainer.__init__r>   c                 C   s   | j dur
|  |}|S )zB
        rebuild config if `cfg_modify_fn` is not `None`.
        N)r'   )rn   r>   rE   rE   rz   rT      s   

zOFATrainer.rebuild_configconfig_filec                 C   sr   t j|r|S t j}t|ddkr*|dd }|dd dd }n|}t}t	|t
j||d}|S )zC
        support local file/ url or model_id with revision
        #   r   =)	file_pathr0   	cache_dir)rI   rM   existstempfileTemporaryDirectorynamerb   splitr   r   r   rP   )rn   r{   	temp_namemodel_idr0   	file_namerE   rE   rz   rL      s   
zOFATrainer.get_config_filec                    s   | j st|r
|jn|}|  | ||\}}}d|i}d|vrsdg}tg }|D ] | fdd| D  q*i }	|D ],}
||
d}|durkt	
 rbt	 rb|j }t	|t	  |	|
| i q?| j|	 n| j|d  || _dS )a  
        A single training step.

        step 1. Let the model in a trainable state.
        step 2. Execute the criterion function.
        step 3. Update the logging variable's value.
        step 4. Update the training result.

        Args:
            model (:obj:`torch.nn.Module` or :obj:`TorchModel`): The model to be run.
            inputs (`dict`): model inputs.
        losslog_varsc                    s   g | ]} |v r|qS rE   rE   ).0keykey_prE   rz   
<listcomp>   s    z)OFATrainer.train_step.<locals>.<listcomp>N)_distr   modulerZ   ri   rK   updatekeysrH   distis_availableis_initializeddataclone
all_reducediv_get_world_sizeitem
log_buffertrain_outputs)rn   r%   inputsr   sample_sizelogging_outputr   default_keys_pattern
match_keysr   r   valuerE   r   rz   
train_step   s.   

zOFATrainer.train_step)__name__
__module____qualname____doc__r   r   r   r   r
   Modulestrr   r   r   r   r   r   torchoptim	Optimizerrf   _LRSchedulerr^   rm   r   rT   rL   r   __classcell__rE   rE   rx   rz   r#   #   sZ    !

	

dr#   ):r`   rI   rQ   r   	functoolsr   r   typingr   r   r   r   r   rW   r   r	   r   r
   torch.utils.datar   modelscope.hub.file_downloadr   modelscope.metainfor   modelscope.models.baser   r    modelscope.msdatasets.ms_datasetr   modelscope.preprocessors.baser   $modelscope.preprocessors.multi_modalr   *modelscope.preprocessors.ofa.utils.collater   modelscope.trainersr   modelscope.trainers.builderr   %modelscope.trainers.optimizer.builderr   "modelscope.trainers.parallel.utilsr   modelscope.utils.configr   modelscope.utils.constantr   r   r   r   r   ofa_trainer_utilsr   r    r!   register_moduleofar#   rE   rE   rE   rz   <module>   s8   