o
    ॵi;                     @   s:  d Z ddlmZ ddlZddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZm Z m!Z! ddl"m#Z# ddl$m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZA ddlBmCZCmDZDmEZEmFZFmGZG ddlHmIZI ddlJmKZK eK ZLG dd deZMdeeeNe
f  d e9d!eeejO  fd"d#ZPdeeeNe
f  d$eQd e9d!eeeNe
f  fd%d&ZRe;jSe)jTd'G d(d) d)e?ZUdS )*z PyTorch trainer for UniTE model.    N)ceil)mkdir)AnyCallableDictListOptionalTupleUnion)	DataFrame)pad)clip_grad_norm_)AdamW	Optimizer)BatchSampler
DataLoaderDatasetSamplerSequentialSamplerSubsetRandomSampler)SummaryWriter)tqdm)AutoTokenizer)MetricsTrainers)Metric)
MetricKeysbuild_metric)
TorchModel)InputFormat)UniTEForTranslationEvaluationcombine_input_sentences)	MsDataset)Preprocessor)TRAINERS)Hook)EpochBasedTrainer)
ConfigDict)
ConfigKeysFieldsModeKeys	ModelFileTrainerStages)create_device)
get_loggerc                   @   s4   e Zd ZdedefddZdd Zdefdd	Zd
S )$TranslationEvaluationTrainingSamplernum_of_samples batch_size_for_each_input_formatc           	      C   s
  || _ || _| j d | _| jd }td| j | j| j | f  || _ t| j   }t	 | _
t	 | _t	 | _ttjtjtjfD ]1\}}|| j }|| j }||| | j|< tt| j| | jdd| j|< t| j| | j
|< qCd| _|  dkrtdddS )	a  Build a sampler for model training with translation evaluation trainer.
        The trainer should derive samples for each subset of the entire dataset.

        Args:
            num_of_samples: The number of samples in total.
            batch_size_for_each_input_format: During training, the batch size for each input format

        Returns:
            A data sampler for translation evaluation model training.

           zn%d samples are given for training. Using %d samples for each input format. Leaving the last %d samples unused.T
batch_size	drop_lastr   zCThe dataset doesn't contain enough examples to form a single batch.z?Please reduce the batch_size or use more examples for training.N)r0   r1   $num_of_samples_for_each_input_formatloggerinfotorchrandpermcputolistdictsubset_iteratorssubset_samplersindices_for_each_input_format	enumerater   SRC_REFSRCREFr   r   iternum_of_sampled_batches__len__
ValueError)	selfr0   r1   num_of_samples_to_userandom_permutationsinput_format_indexinput_format	start_idxend_idx rP   j/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/nlp/translation_evaluation_trainer.py__init__,   sV   




z-TranslationEvaluationTrainingSampler.__init__c              	   c   s    	 z[| j |  kr7tjtjtjfD ]!}	 z	t| j|  W n ty1   t	| j
| | j|< Y nw qqd| _ t }ttjtjtjfD ]\}}|t| j| 7 }qD|  j d7  _ |V  W n
 tyg   Y d S w q)NTr      )rF   rG   r   rB   rC   rD   nextr>   StopIterationrE   r?   listrA   )rI   rM   outputinput_format_idxrP   rP   rQ   __iter__e   s8   

z-TranslationEvaluationTrainingSampler.__iter__returnc                 C   s   | j | j S N)r6   r1   rI   rP   rP   rQ   rG      s   z,TranslationEvaluationTrainingSampler.__len__N)__name__
__module____qualname__intrR   rY   rG   rP   rP   rP   rQ   r/   *   s    
9r/   batchpreprocessorrZ   c                    s>   t  }| d  D ] t fdd| D | < q	||}|S )Nr   c                 3   s    | ]}|  V  qd S r[   rP   .0xkeyrP   rQ   	<genexpr>   s    z,convert_csv_dict_to_input.<locals>.<genexpr>)r=   keysrV   )ra   rb   
input_dictrP   rf   rQ   convert_csv_dict_to_input   s
   rk   r4   c                    s  t  }t |d< jtjkr\ttjtjtj	fD ]B\}}|| }|| }| || }|d  |g| 7  < 
| t|}| D ]\}	}
|	| vrRt ||	< ||	 |
 qCqn<jtjkr|d  jgt|  7  < t| } |  D ]\}	}
|	| vrt ||	< ||	 |
 qyntdj tdd |d D  t fdd|d D |d< tj|d dd|d< t|d	 d
|d	< jtjkrt|d t |d< t|d t |d< t|d t |d< |S )NrM   z9During training, %s mode is not allowed for preprocessor.c                 s   s    | ]}| d V  qdS )N)sizerc   rP   rP   rQ   rh      s    z"data_collate_fn.<locals>.<genexpr>	input_idsc                 3   s.    | ]}t |d  |d fjdV  qdS )r   rl   )r   valueN)r   rm   pad_token_idrc   input_max_lengthsrb   rP   rQ   rh      s    
r   )dimscorerl   lp	raw_score
segment_id)r=   rV   moder*   TRAINrA   r   rB   rC   rD   change_input_formatrk   itemsri   appendEVALrM   lenrH   maxr9   catTensorviewsum)ra   r4   rb   output_dictrL   rM   rN   rO   batch_to_processrg   ro   rP   rq   rQ   data_collate_fn   sV   





r   )module_namec                       s   e Zd Z			ddeeeejje	f  dee	 de	f fddZ
ded	ef fd
dZd	efddZd	efddZdd Z  ZS )TranslationEvaluationTrainerNgpumodelcfg_filedevicec           	         sX    fdd} fdd}t j|t j|i}t j|g|R ||d| d _d _dS )a  Build a translation evaluation trainer with a model dir or a model id in the model hub.

        Args:
            model: A Model instance.
            cfg_file: The path for the configuration file (configuration.json).
            device: Used device for this trainer.

        c                       t |  jjj jdS N)r4   rb   )r   cfgtrainr4   train_preprocessorre   r\   rP   rQ   data_collator_for_train   
   zFTranslationEvaluationTrainer.__init__.<locals>.data_collator_for_trainc                    r   r   )r   r   
evaluationr4   eval_preprocessorr   r\   rP   rQ   data_collator_for_eval   r   zETranslationEvaluationTrainer.__init__.<locals>.data_collator_for_eval)r   data_collatorN)r(   r   valsuperrR   train_dataloadereval_dataloader)	rI   r   r   r   argskwargsr   r   r   	__class__r\   rQ   rR      s$   z%TranslationEvaluationTrainer.__init__r   rZ   c                    s    j jjjdkrt |S  jjj	 D ]}d|_
qtd t fddtd j jjD }| jjj	  j jjjd | jj	  j jjjd | jj	  j jjjd t| j jjj j jjj j jjj j jjjd}|S )	z/Sets the optimizers to be used during training.r   FzBuilding AdamW optimizer ...c                 3   sB    | ]} j jjj|   jjjj jjjj|  d V  qdS )paramslrN)	r   encoderlayer
parametersr   r   	optimizerplm_lrplm_lr_layerwise_decay)rd   ir\   rP   rQ   rh      s    
z?TranslationEvaluationTrainer.build_optimizer.<locals>.<genexpr>r   r   )r   betasepsweight_decay)r   r   r   typer   build_optimizerr   r   
embeddingsr   requires_gradr7   r8   rV   rangenum_hidden_layersr|   r   	estimatormlp_lrlayerwise_attentionr   r   r   r   )rI   r   paramlearning_rates_and_parametersr   r   r\   rQ   r      s8   









z,TranslationEvaluationTrainer.build_optimizerc                 C   s   t d | jd u r*t d| jjjj  tjt	
| j| jjjj| jjjjd| _t| jtt| j| jjjdd| jd d}t dt| j  |S )Nz$Building dataloader for training ...z"Reading train csv file from %s ...split)r1      batch_samplernum_workers
collate_fn	generatorReading done, %d items in total)r7   r8   train_datasetr   datasetr   namer"   loadospjoin	model_dirr   r   r/   r~   r4   train_data_collator)rI   r   rP   rP   rQ   get_train_dataloader&  s.   



	z1TranslationEvaluationTrainer.get_train_dataloaderc                 C   s   t d | jd u r*t d| jjjj  tjt	
| j| jjjj| jjjjd| _t| jtttdt| j| jjjddd| jd d}t d	t| j  |S )
Nz&Building dataloader for evaluating ...z!Reading eval csv file from %s ...r   r   Fr3   r   r   r   )r7   r8   eval_datasetr   r   validr   r"   r   r   r   r   r   r   r   r   r   r~   r   r4   eval_data_collator)rI   r   rP   rP   rQ   get_eval_data_loader>  s,   




z1TranslationEvaluationTrainer.get_eval_data_loaderc           
      C   s  d}t | jjdr| jjj}t| jfd| ji|}| tj t	 }t
jt
jt
jfD ]W}| j| | jrZddlm} ||| || j||| jjdd| jjdd| jd	 ndd
lm} ||| || j||| jd |D ]}	t |	drt|	jr|	  qqq+| tj |S )a   Evaluation loop used by `TranslationEvaluationTrainer.evaluate()`.

        The evaluation process of UniTE model should be arranged with three loops,
        corresponding to the input formats of `InputFormat.SRC_REF`, `InputFormat.REF`,
        and `InputFormat.SRC`.

        Here we directly copy the codes of `EpochBasedTrainer.evaluation_loop`, and change
        the input format during each evaluation subloop.
        Nvisualizationr   r   )multi_gpu_test	cache_dirgpu_collectF)r   metric_classesvis_closuretmpdirr   data_loader_iters_per_gpu)single_gpu_test)r   r   r   data_loader_itersclear)hasattrr   r   r   partialr   invoke_hookr,   
before_valr=   r   rB   rC   rD   r   rz   _dist#modelscope.trainers.utils.inferencer   updater   get_eval_iters_per_epochr   callabler   	after_val)
rI   data_loaderr   r   vis_cfgmetric_valuesrM   r   r   mrP   rP   rQ   evaluation_loopW  sd   

	z,TranslationEvaluationTrainer.evaluation_loop)NNr   )r]   r^   r_   r   r
   r   r9   nnModulestrrR   r'   r   r   r   r   r   r   __classcell__rP   rP   r   rQ   r      s"    -1r   )V__doc__os.pathpathr   randommathr   osr   typingr   r   r   r   r   r	   r
   r9   pandasr   torch.nn.functionalr   torch.nn.utilsr   torch.optimr   r   torch.utils.datar   r   r   r   r   r   torch.utils.tensorboardr   r   transformersr   modelscope.metainfor   r   modelscope.metricsr   modelscope.metrics.builderr   r   modelscope.models.baser   )modelscope.models.nlp.unite.configurationr   2modelscope.models.nlp.unite.translation_evaluationr    r!   modelscope.msdatasetsr"   modelscope.preprocessorsr#   modelscope.trainers.builderr$   modelscope.trainers.hooksr%   modelscope.trainers.trainerr&   modelscope.utils.configr'   modelscope.utils.constantr(   r)   r*   r+   r,   modelscope.utils.devicer-   modelscope.utils.loggerr.   r7   r/   r   r   rk   r`   r   register_moduletranslation_evaluation_trainerr   rP   rP   rP   rQ   <module>   s\   $ Z

3