o
    ॵis                     @   s   d dl Z d dlZd dlmZ d dlZd dlZd dlZd dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ G dd	 d	eZG d
d deZdS )    N)OrderedDict)tqdm)AdamWget_linear_schedule_with_warmup)MetricsTracker)	ModelFile)
get_loggerc                   @   sd   e Zd Z				dddZdd Z			dddZdd	 ZdddZdddZdddZ	dd Z
dS )TrainerNc                 C   sL  || _ || _|j| _|j| _|jjd dk| _|jjdd  | _|jj| _|jj| _|jj	| _	|jj
| _
|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|jj| _|| _ | jdkrm| j jn| j | _|| _|j| _|| _|| _|pt | _t | _t | _ t | _!t | _"t#| jrdnd| _$d| _%d| _&d S )Nr   -   infz-inf)'model	to_tensordo_traindo_inferr	   valid_metric_nameis_decreased_valid_metric
num_epochssave_dir	log_stepsvalid_stepssave_checkpointsave_summaryDatasetlearning_methodModelweight_decaywarmup_stepsbatch_size_labelbatch_size_nolabelgpulrmodule
func_modelreader	tokenizerlr_scheduler	optimizerr   loggerr   batch_metrics_tracker_labeltoken_metrics_tracker_labelbatch_metrics_tracker_nolabeltoken_metrics_tracker_nolabelfloatbest_valid_metricepoch	batch_num)selfr   r   configr$   r(   r&   r'    r3   h/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/nlp/space/trainer/intent_trainer.py__init__   sN   













zTrainer.__init__c                    s  ddg  fdd| j  D | jd fdd| j  D ddg}t|| jd}|| j }| jd	kr7| jnt|d
 }t|||d}|| _	|| _
| jd| j d | jd| j | jd| | jd| j | jd| j | jd| | jd| | jd dS )z
        Setup the optimizer and the learning rate scheduler.

        from transformers.Trainer

        parameters from cfg: lr (1e-3); warmup_steps
        biasznorm.weightc                    s*   g | ]\ }t  fd dD s|qS )c                 3       | ]}| v V  qd S Nr3   .0ndnr3   r4   	<genexpr>U       4Trainer.set_optimizers.<locals>.<listcomp>.<genexpr>anyr:   pno_decayr<   r4   
<listcomp>S       z*Trainer.set_optimizers.<locals>.<listcomp>)paramsr   c                    s*   g | ]\ }t  fd dD r|qS )c                 3   r7   r8   r3   r9   r<   r3   r4   r>   ]   r?   r@   rA   rC   rE   r<   r4   rG   [   rH           )r!   r   g?)num_warmup_stepsnum_training_stepsz***** Running training: z *****z  Num Epochs = %dzC  Num Training steps(one turn in a batch of dialogs) per epoch = %dz"  Batch size for labeled data = %dz$  Batch size for unlabeled data = %dz  Total optimization steps = %dz  Total warmup steps = %dz$************************************N)r   named_parametersr   r   r!   r   r   intr   r'   r&   r(   infor   r   r   )r1   num_training_steps_per_epochoptimizer_grouped_parametersr'   rL   rK   r&   r3   rE   r4   set_optimizersG   sR   	

	
zTrainer.set_optimizersc                 C   s0   | j | j }t|D ]}| j||||d q
d S )N)train_label_itertrain_nolabel_itervalid_label_itervalid_nolabel_iter)r   r/   rangetrain_epoch)r1   rS   rT   rU   rV   r   r/   r3   r3   r4   train   s   zTrainer.trainc                 C      t )!
        Train an epoch.
        NotImplementedError)r1   rS   rT   rU   rV   r3   r3   r4   rX      s   zTrainer.train_epochTc                 C   rZ   r8   r\   )r1   data_label_iterdata_nolabel_iter	need_saver3   r3   r4   evaluate      zTrainer.evaluatec                 C   rZ   r8   r\   )r1   	data_iternum_batchesr3   r3   r4   infer   rb   zTrainer.inferFc              	   C   s.  | j | j| j| j d}| jdur| j |d< | jrYtj	| j
d| j  d}t| j | | jd| d tj	| j
d| j  d}t|| | jd	| d |rtj	| j
tj}t| j | tj	| j
d
tj}t|| | jd| d| j  d| jd dS dS )z save )r/   r0   r.   r'   Nr&   state_epoch_z.modelzSaved model state to ''.trainzSaved train state to 'z{}.trainzSaved best model state to 'z' with new best valid metric =.3f)r/   r0   r.   r'   
state_dictr&   r   ospathjoinr   torchsaver   r(   rO   r   TORCH_MODEL_BIN_FILEformatr   upper)r1   is_besttrain_state
model_file
train_filebest_model_filebest_train_filer3   r3   r4   rp      sH   




zTrainer.savec                    s^    fdd} fdd} j jdu r jd dS  jr"|  dS  jr-|  |  dS dS )z load c                     sd  t j jj dd d} dt|  d v r6t }|  D ]\}}|d d dks+J |||dd  < q|} t }dd  j D }|  D ]\}}||v r|j	|| j	krt
|d	s^J | }tjj jj|| j	d
d}|dkr|||j	d  d < td| d n-|j	d |j	d k r|d |j	d  }td| d n||d |j	d < td| d |j|j}	}
t j||	|
d}|||< qG|||< qGtd| d qG|} |D ]A}|| vr|| jrtd| d tjj jj|| j	d
d}|| j|| j}	}
t j||	|
d| |< q|| | |< q܈ j|   jd jj d d S )Nc                 S      | S r8   r3   storagelocr3   r3   r4   <lambda>       z9Trainer.load.<locals>._load_model_state.<locals>.<lambda>map_locationzmodule.r      c                 S   s   i | ]\}}||qS r3   r3   )r:   nameparamr3   r3   r4   
<dictcomp>   s    z;Trainer.load.<locals>._load_model_state.<locals>.<dictcomp>numpy)scalesizefloat32zembedder.token_embedding.weightzpart of parameter(z) random normlize initializez) are dropped)dtypedevicez
parameter(zLoaded model state from 'z.model')ro   loadr#   init_checkpointlistkeysr   itemsrM   shapehasattrr   nprandomnormalinitializer_rangeastypeprintr   r   tensorrequires_gradload_state_dictr(   rO   )model_state_dictnew_model_state_dictkv
parametersr   r   arrzr   r   r1   r3   r4   _load_model_state   s   





z'Trainer.load.<locals>._load_model_statec               	      s    j j d} tj| rWtj| dd d}|d  _|d  _ j	d ur1d|v r1 j	
|d   jd urBd|v rB j
|d   jd	|  d
 j d jdd d S  jd d S )Nrh   c                 S   rz   r8   r3   r{   r3   r3   r4   r~   
  r   z9Trainer.load.<locals>._load_train_state.<locals>.<lambda>r   r/   r.   r'   r&   zLoaded train state from 'z' with (epoch-z best_valid_metric=rj   )zLoaded no train state)r#   r   rl   rm   existsro   r   r/   r.   r'   r   r&   r(   rO   )rw   train_state_dictr   r3   r4   _load_train_state  s*   

z'Trainer.load.<locals>._load_train_stateNzLoaded no model !!!)r#   r   r(   rO   r   r   )r1   r   r   r3   r   r4   r      s   C
zTrainer.load)NNNN)NNN)Tr8   )F)__name__
__module____qualname__r5   rR   rY   rX   ra   re   rp   r   r3   r3   r3   r4   r	      s     
2;



%r	   c                       s\   e Zd Zd fdd	Zdd Zdd Zdd	 Zdd
dZdd Z	dddZ	dd Z
  ZS )IntentTrainerNc                    s.   t t| |||| |jj| _|jj| _d S r8   )superr   r5   r   exampler	   can_norm)r1   r   r   r2   r$   	__class__r3   r4   r5   (  s   
zIntentTrainer.__init__c                 C   sD  t |d|kg}d| }d}t j|ddd d | d f }||jddd }|t | d t | }d}	|||	k  }
|||	k }|||	k  }|||	k }t|ra|
d|k nd}t|rp|d|k nd}|d	t||f 7 }|d
t||f 7 }t | jj	}|D ]\}\}}|d D ]
}||  d7  < qq||  }d\}}}t
|D ]B\}}t j|
|d  gdd}t|D ]}|| }||jddd }||d  9 }||jddd }q|d }| || kr|d7 }q|t|
 | t| }t|r|d|t|  7 }n|d7 }|d| 7 }|||fS )Nr   zoriginal acc: %s   axisT)r   keepdimsgffffff?rJ   z   (%s) confident acc: %sz   (%s) unconfident acc: %sintent_labelg      ?)r   r   r   r   z   new unconfident acc: %sz   no unconfident predictionsz   final acc: %s)r   meanargmaxsortsumloglenzerosr#   
num_intent	enumerateconcatenaterW   )r1   y_predy_trueex_data_iteracc_originalmessager   y_pred_topky_pred_uncertainty	thresholdy_pred_confidenty_pred_unconfidenty_true_confidenty_true_unconfidentacc_confidentacc_unconfidentprior_batch
batch_sizer   rightalphaitersiyYj	acc_finalr3   r3   r4   can_normalization-  sn    





zIntentTrainer.can_normalizationc           &   
      s  g }  j d7  _  j   j   j   j  t|}|dur)t|nd}t||}t|}	|dur<t|nd}
|durDdnd}t	d|d D ]a}t

 }g g g g g f\}}}}}g }z
t|	\}\}}W n ty   t|}	t|	\}\}}Y nw || || |d || |durz
t|
\}\}}W n ty   t|}
t|
\}\}}Y nw || || |d || t||||D ]r\}}}}t|t fdd| } jr!|r!|j| } jj||d |d	 d
d}t|t fdd| }| D ]	\} }!|!|| < q j |d<  j|d<  j|d||d}" j|"|d\}#}"||# ||" qt|dkrS|d n|d |d  }# jj|# j jd t

 | }$||$   jd7  _t|||D ]\}}"} j|"|||||d q~ jdkr|dur|dur| j dkr j||d qM j ||d}% j!||% d dS )r[   r   Nr   TFc                       | d   | d fS Nr   r   r   kvr   r3   r4   r~         z+IntentTrainer.train_epoch.<locals>.<lambda>r   idsintent)datasetlabelsindstaskc                    r   r   r   r   r   r3   r4   r~     r   r/   	num_steps)is_training
with_label	data_file)metricsr   )r'   r&   )r   batch_idr   rd   timesr   )r^   r_   )rc   r   )cur_valid_metric)"r/   r)   clearr*   r+   r,   r   maxiterrW   timenextStopIterationappendziptypemapr   r   data_file_to_datasetr$   retrieve_examplesr0   r   balance_metricsr#   	_optimizer'   r&   track_and_log_messager   ra   re   save_and_log_message)&r1   rS   rT   rU   rV   r   num_label_batchesnum_nolabel_batchesrd   train_label_iter_looptrain_nolabel_iter_loopreport_for_unlabeled_datar   
start_time
batch_listbatch_size_listwith_label_list	loss_listmetrics_listdata_file_listdata_file_labelbatch_labelr   data_file_nolabelbatch_nolabelr   r   r   r   r   current_datasetexample_batchr   r   r   losselapsedaccuracyr3   r   r4   rX   l  s  




















 	
zIntentTrainer.train_epochc                    s   g }t  H t|tfdd| }jj|d  fdd D   d }jr3||g7 }n|tj	|dd
 7 }W d    |S W d    |S 1 sQw   Y  |S )	Nc                    r   r   r   r   r   r3   r4   r~     r   z'IntentTrainer.forward.<locals>.<lambda>inputsc                    "   i | ]}| |     qS r3   cpudetachr   r:   r   resultr3   r4   r         z)IntentTrainer.forward.<locals>.<dictcomp>intent_probsr   r   )ro   no_gradr   r   r   r   re   r   r   r   tolist)r1   r   predr   r3   r  r1   r4   forward  s(   



zIntentTrainer.forwardc              
      s  j d tjjdj d}d}g g }}g g }}	t }
t	 , j
rt|ddD ]7\}\}}t|tfdd| }jj|d	  fd
d D  |t d  |	|d  7 }	q2tj|dd}jjry| n|}t|	d}	jjr|	 n|	}	j d|   t||dD ]\}\}}t|tfdd| }jj|d	  fdd D  j
rt d }jjr| n|}tj|| dd}t|djj}jjr| n|}| d|	!|dd|}|" # $ }n d }j%r*||g7 }||d " #  7 }n|t&j'|dd 7 }||d " #  7 }|d7 }||krL nqW d   n	1 sXw   Y  j%rt&(|}t&j)|dd}j*|||d\}}}t+||}|| |d}d| d| }nt,dd t-||D t.| }||d}d| }j d|  t/|d }t0j1||d!d" W d   n	1 sw   Y  d#j d$}d%t |
 d&}d|||g}j | |S )'z&
        Inference interface.
        zGeneration starts ...infer_z.result.jsonr   zBuilding train memory.)descc                    r   r   r   r   r   r3   r4   r~     r   z%IntentTrainer.infer.<locals>.<lambda>r  c                    r  r3   r  r  r  r3   r4   r     r  z'IntentTrainer.infer.<locals>.<dictcomp>featuresr   )dimzMemory size: )totalc                    r   r   r   r   r   r3   r4   r~   %  r   c                    r  r3   r  r  r  r3   r4   r   (  r  r   r   r   r   N)r   r   r   )r  pred_labelsr   z
Accuracy:    c                 s   s    | ]	\}}||kV  qd S r8   r3   )r:   rD   tr3   r3   r4   r>   T  s    z&IntentTrainer.infer.<locals>.<genexpr>)r  r+  zSaved inference results to w   )indentz[Infer][]zTIME-rj   )2r(   rO   rl   rm   rn   r   r/   r   ro   r!  r   r   r   r   r   r   re   r   
from_numpyr"  catr#   use_gpucuda
LongTensor	unsqueezer   softmaxmmr-  r   r   scatter_addrepeatr  r  r   r   r   r   arrayr   r   r   r   r   r   openjsondump)r1   rc   rd   r   infer_save_file	batch_cntr#  trueoutputsr   
begin_timer   r   r   memr(  probsr   r   r   r   r  infer_resultsmetrics_messagefpmessage_prefix	time_costr3   r$  r4   re      s   








9

 

zIntentTrainer.inferc              	   C   s  |r| j n| j}|r| jn| j}dd | D }|dd}	dd | D }
dd | D }||
| |||	 | jdkr|| j dkr| }| }|rUdnd}d	| j	 d
| d| d
| d	}dt
|| j d  | j d}d||||g}| j| d S d S d S )Nc                 S   s2   i | ]\}}|t |tjr|   n|qS r3   )
isinstancero   Tensorr  r  r   r:   r   r   r3   r3   r4   r   g  s     z7IntentTrainer.track_and_log_message.<locals>.<dictcomp>mlm_numr   c                 S   s   i | ]\}}d |vr||qS tokenr3   rN  r3   r3   r4   r   m      c                 S   s   i | ]\}}d |v r||qS rP  r3   rN  r3   r3   r4   r   n  rR  Labeled	Unlabeledz[Train][z][/r1  z	AVG_Time-rj   r,  )r)   r+   r*   r,   r   popupdater   valuer/   r   rn   r(   rO   )r1   r   r   r   rd   r   r   batch_metrics_trackertoken_metrics_trackerrO  batch_metricstoken_metricsbatch_metrics_messagetoken_metrics_messagelabel_prefixrJ  avg_timer   r3   r3   r4   r   a  s,    "z#IntentTrainer.track_and_log_messagec                 C   s   | j  }| j }d| j d}d|||g}| j| |r@| j }| j }d| j d}d|||g}| j| |d usFJ | j	rO|| j
k }n|| j
k}|rY|| _
| | d S )Nz[Valid][z
][Labeled]r,  z][Unlabeled])r)   summaryr*   r/   rn   r(   rO   r+   r,   r   r.   rp   )r1   r  r   r]  r^  rJ  r   rt   r3   r3   r4   r     s,   




z"IntentTrainer.save_and_log_messagec                 C   s,  | j dkr|D ]}|d urt|| j ksJ q|\}}}}}}	i }t|}||d< |}
|d ur`t|}t||| j   | }t|}||d< ||d< ||d< |
| jjrY|n|| jj  }
|d urut|}||d< |
|| jj  }
|	d urt|	}	|	|d< |
|	 }
|
|d< d|v sJ |d |fS )	Nr   intent_lossrO  	token_mlmmlmklconr  )	r    r   ro   r   r   r#   
token_loss	mlm_ratiokl_ratio)r1   r   r   metricrb  rd  rc  rO  re  rf  r  r3   r3   r4   r     s>   





zIntentTrainer.balance_metricsr8   )NN)r   r   r   r5   r   rX   r%  re   r   r   r   __classcell__r3   r3   r   r4   r   &  s    ? 
a 
r   )rl   r   collectionsr   r>  r   r   ro   r   transformers.optimizationr   r   5modelscope.trainers.nlp.space.metrics.metrics_trackerr   modelscope.utils.constantr   modelscope.utils.loggerr   objectr	   r   r3   r3   r3   r4   <module>   s     