o
    ॵiB                     @   s<  d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZ d dl	Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlm Z m!Z!m"Z" d d	l#m$Z$ d d
l%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ d dl,m-Z-m.Z. e&j/ej0dG dd de$Z1G dd dZ2dS )    N)CallableDictOptional)EasyDict)tqdm)Trainers)DBModel
DBModel_v2)boxes_from_bitmappolygons_from_bitmap)
DataLoaderImageDatasetQuadMeasurer)BaseTrainer)TRAINERS)DEFAULT_MODEL_REVISION	ModelFile)
get_logger)get_ranksynchronize)module_namec                       sl   e Zd Zddddefdededededef
 fdd	Zd
d Z	ddedeee	f fddZ
dd Z  ZS )OCRDetectionDBTrainerNTmodelcfg_fileload_pretrain
cache_pathmodel_revisionc           	         s  |dur|  ||| _|du rtj| jtj| _n|dur"|dus&J d|dur4|| _|dur4|| _t 	| j | j
}|rXd|v rK|d |j_ntj| j| j
jj|j_d| j
v rb| |}d|v rl|d |j_d|v rv|d |j_d|v r|d |j_d|v r|d |j_d	|v r|d	 |j_d
|v r|d
 |j_d|v r|d |j_d|v r|d |j_|jj| _t| j| _|| _
dS )aN   High-level finetune api for dbnet.

        Args:
            model: Model id of modelscope models.
            cfg_file: Path to configuration file.
            load_pretrain: Whether load pretrain model for finetune.
                if False, means training from scratch.
            cache_path: cache path of model files.
        Nz;cfg_file and cache_path is needed, if model is not providedpretrain_model	frameworkgpu_ids
batch_size
max_epochsbase_lrtrain_data_dirval_data_dirtrain_data_listval_data_list)get_or_download_model_dirr   ospathjoinr   CONFIGURATIONr   super__init__cfgtrainfinetune_pathr   weights_config_transformr   r    total_epochsr"   datasetr#   r$   r%   r&   len
world_size)	selfr   r   r   r   r   argskwargsr.   	__class__ c/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/cv/ocr_detection_db_trainer.pyr-   "   sZ   





zOCRDetectionDBTrainer.__init__c                 C   s   t | j}|jdd d S Nr   )
local_rank)	DBTrainerr.   r/   )r7   trainerr<   r<   r=   r/   c   s   
zOCRDetectionDBTrainer.traincheckpoint_pathreturnc                 O   s,   |d ur	|| j j_t| j }|jdd d S r>   )r.   testrB   r@   evaluate)r7   rB   r8   r9   	evaluaterr<   r<   r=   rE   g   s   

zOCRDetectionDBTrainer.evaluatec                 C   s   t i }|jj|_|jj|j_|j|_|j|_|j|_|j|_t	|jj
|jj_t	|jj
|jjj |jj_t	|jj
|jjj |jj_|jj|j_d|jj_d|jj_|jjj|jj_|S )N      )easydictr/   miscswork_dir
output_dirr   r4   
evaluationrD   r5   r   
dataloadernum_gpusbatch_size_per_gpur    workers_per_gpunum_workersr!   r3   	transform
collect_fn)r7   config
new_configr<   r<   r=   r2   p   s.   




z'OCRDetectionDBTrainer._config_transformN)__name__
__module____qualname__r   strboolr-   r/   r   floatrE   r2   __classcell__r<   r<   r:   r=   r      s2    A

	r   c                   @   s   e Zd Zdd Zdd Zdd Zd*dd	Zd
d Zdd Zd+ddZ	dd Z
d,ddZdd Zdd Zd,ddZd-ddZd d! Zd"d# Zd$d% Zd-d&d'Zd(d) ZdS ).r@   c                 C   s   |    || _|jj| _|jj| _d| _d| _	t
|jjdkr#d| _nd| _tj|jj|jj| _t dkr?tj| jdd ttj| jd| _| jd| j d S )Nr   rG   TFexist_okztrain_log.txtzcfg value:
{})init_devicer.   rJ   rL   dir_pathr/   r"   lr
current_lrtotalr5   r   distributedr(   r)   r*   exp_name	file_namer   makedirsr   loggerinfoformat)r7   r.   r<   r<   r=   r-      s   


zDBTrainer.__init__c                 C   s*   t j rt d| _d S t d| _d S )Ncudacpu)torchrm   is_availabledevicer7   r<   r<   r=   ra      s   
zDBTrainer.init_devicec                 C   s   t | j| j|}|S rW   )r	   rq   rf   )r7   r?   r   r<   r<   r=   
init_model   s   zDBTrainer.init_modelNc                 C   s0   d}t d|t| jjjd   |}|| j S )N?g      ?rG   )nppowerr]   r.   r/   r3   rc   )r7   epochstepfactorrater<   r<   r=   get_learning_rate   s
   
zDBTrainer.get_learning_ratec                 C   s*   |  ||}|jD ]}||d< q	|| _d S )Nrc   )r{   param_groupsrd   )r7   	optimizerrw   rx   rc   groupr<   r<   r=   update_learning_rate   s   


zDBTrainer.update_learning_ratec                 C   s    t j||d}|j|dd d S )Nmap_locationF)strict)ro   loadload_state_dict)r7   r   
model_pathrq   
state_dictr<   r<   r=   restore_model   s   zDBTrainer.restore_modely&1|?rt   -C6?c           
      C   s   g g g }}}| j  D ]9\}}t|dr#t|jtjr#||j t|tjs-d|v r4||j	 qt|drFt|j	tjrF||j	 qt
jj|||dd}	|	||d |	d|i |	S )NbiasbnweightT)rc   momentumnesterov)paramsweight_decayr   )r   named_moduleshasattr
isinstancer   nn	ParameterappendBatchNorm2dr   ro   optimSGDadd_param_group)
r7   rc   r   r   bn_groupweight_group
bias_groupkvr}   r<   r<   r=   create_optimizer   s$   zDBTrainer.create_optimizerc                 C   s<   || j jj dkr| jdt|  | ||| d S d S )Nr   zsave interval model for step )r.   rJ   save_intervalrj   rk   r[   
save_model)r7   r   rw   rx   r<   r<   r=   maybe_save_model   s   zDBTrainer.maybe_save_modelc                 C   sX   t |tr| D ]\}}| |||}| || q	d S | d||}| || d S )Nr   )r   dictitemsmake_checkpoint_namesave_checkpoint)r7   r   rw   rx   namenetcheckpoint_namer<   r<   r=   r      s   
zDBTrainer.save_modelc                 C   sL   t j| jdd t| t j| j| | j	dt j| j|  d S )NTr_   zsave_checkpoint to: )
r(   ri   rb   ro   saver   r)   r*   rj   rk   )r7   r   r   r<   r<   r=   r      s
   
zDBTrainer.save_checkpointc           
      C   s   t  | j}| }t| }tjtj	
| j|| jd}t| }d}tt|D ]}	|||	  |v rG||||	     |||	 < q-|| t| tj	
| j| d S )Nr   zmodel.module.)r   torq   r   listkeysro   r   r(   r)   r*   rb   setranger5   rn   r]   r   r   )
r7   finetune_model_nameinfer_model_nameinfer_modelmodel_state_dict
model_keys
saved_dict
saved_keysprefixir<   r<   r=   convert_model_for_inference   s$   


z%DBTrainer.convert_model_for_inferencec                 C   s.   |d u s|d u r|d }|S d |||}|S )Nz
_latest.ptz{}_epoch_{}_minibatch_{}.pt)rl   )r7   r   rw   rx   c_namer<   r<   r=   r      s
   zDBTrainer.make_checkpoint_nameFc                 C   sX   t ||jj|jj}t||jjd|d}t ||jj|jj}t||j	jd|d}||fS )NT)is_trainrf   F)
r   r4   r#   r%   r   r/   rN   r$   r&   rD   )r7   r.   rf   train_datasettrain_dataloadertest_datasettest_dataloaderr<   r<   r=   get_data_loader  s&   

zDBTrainer.get_data_loaderc                 C   sF  |  || _| | j| j\| _| _d| _| jjj	d ur4| j
d| jjj	  | | j| jjj	| j d}| | j}| j
d | j  	 | j
dt|  t| j| _| jD ]&}| ||| j | j| j|||| jd | | j|| j |  jd7  _q[|d7 }|| jjjkr| | jd | dd	 | j
d
 d S qH)Nr   zfinetune from zStart Training...TzTraining epoch )rw   rx   rG   zfinal.ptzpytorch_model.ptzTraining done)rs   r   r   r.   rf   train_data_loadervalidation_loadersstepsr/   r0   rj   rk   r   rq   r   rc   r[   r5   re   r   
train_stepr   r3   r   r   )r7   r?   rw   r}   batchr<   r<   r=   r/     sB   

zDBTrainer.trainc                 C   sR  |   |j|dd}t|dkr|\}}i }	nt|dkr#|\}}}	t|trLg }
td }| D ]\}}||	 7 }|

d||	  q5n|	 }|  |  || jjjj dkrt|trd|
}
dg d	|||
| j}| j| n| jd
||| | jf  |	 D ]\}}| jd||	 f  qd S d S )NTtraining      g        zloss_{0}:{1:.4f}r   	)z
step:{:6d}zepoch:{:3d}z{}z	lr:{:.4f}z)step: %6d, epoch: %3d, loss: %.6f, lr: %fz%s: %6f)	zero_gradforwardr5   r   r   ro   tensorrm   r   meanr   rl   backwardrx   r.   r/   rJ   print_interval_itersr*   rd   rj   rk   item)r7   r   r}   r   rw   rx   resultslpredmetricslinelosskeyl_vallog_infor   metricr<   r<   r=   r   @  sB   




zDBTrainer.train_stepc                 C   s>   t d t j rt d| _t d d S t d| _d S )Nztorch.FloatTensorrm   ztorch.cuda.FloatTensorrn   )ro   set_default_tensor_typerm   rp   rq   rr   r<   r<   r=   init_torch_tensorb  s
   

zDBTrainer.init_torch_tensorc                 C   s   |d }t |tr|d }n|}|| jjjk}g }g }t|dD ]1}	|d |	 \}
}|r<t||	 ||	 ||
\}}nt||	 ||	 ||
\}}|	| |	| q"||fS )a  
        batch: (image, polygons, ignore_tags
        batch: a dict produced by dataloaders.
            image: tensor of shape (N, C, H, W).
            polygons: tensor of shape (N, K, 4, 2), the polygons of objective regions.
            ignore_tags: tensor of shape (N, K), indicates whether a region is ignorable or not.
            shape: the original shape of images.
            filename: the original filenames of images.
        pred:
            binary: text region segmentation map, with shape (N, 1, H, W)
            thresh: [if exists] thresh hold prediction with shape (N, 1, H, W)
            thresh_binary: [if exists] binarized with threshhold, (N, 1, H, W)
        imagebinaryr   shape)
r   r   r.   rD   threshr   sizer   r
   r   )r7   r   _predis_output_polygonimagesr   segmentationboxes_batchscores_batchbatch_indexheightwidthboxesscoresr<   r<   r=   	representk  s*   



zDBTrainer.representc                 C   s&  |    | |}| || jjj| j | | j| j\| _	| _
t }|  t X g }tt| j
t| j
dD ]&\}}|j|dd}| ||| jjj}|j||| jjjdd}	||	 q:||}
|
 D ]\}}| jd||j|jf  qjW d    n1 sw   Y  | jd d S )N)re   Fr   g333333?)r   
box_threshz%s : %f (%d)zEvaluation done)r   rs   r   r.   rD   rB   rq   r   rf   r   r   r   evalro   no_gradr   	enumerater5   r   r   return_polygonvalidate_measurer   gather_measurer   rj   rk   avgcount)r7   r?   r   quad_measurerraw_metricsr   r   r   output
raw_metricr   r   r   r<   r<   r=   rE     sB   


zDBTrainer.evaluaterW   )r   rt   r   )NN)F)rX   rY   rZ   r-   ra   rs   r{   r   r   r   r   r   r   r   r   r   r/   r   r   r   rE   r<   r<   r<   r=   r@      s&    


	

-"
	$r@   )3copydatetimemathr(   timetypingr   r   r   numpyru   ro   torch.distributedrf   disttorch.multiprocessingmultiprocessingmptorch.nnr   rI   r   r   modelscope.metainfor   0modelscope.models.cv.ocr_detection.modules.dbnetr   r	   (modelscope.models.cv.ocr_detection.utilsr
   r   ?modelscope.msdatasets.dataset_cls.custom_datasets.ocr_detectionr   r   r   modelscope.trainers.baser   modelscope.trainers.builderr   modelscope.utils.constantr   r   modelscope.utils.loggerr   modelscope.utils.torch_utilsr   r   register_moduleocr_detection_dbr   r@   r<   r<   r<   r=   <module>   s2   g