o
    ॵicO                     @   s  d dl Z d dlZd dlmZ d dlZd dlmZmZm	Z	m
Z
mZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dl m!Z!m"Z"m#Z# d dl$m%Z% 					dddZ&ej'ej(dG dd deZ)dS )    N)CallableDictOptionalTupleUnion)nn)Dataset)snapshot_download)Trainers)
TorchModel)	MsDataset)Preprocessor)BaseTrainer)TRAINERS)DEFAULT_MODEL_REVISIONInvoke	ModelFile
get_loggerFc           "   
      s$  dd l }dd l}	ddlm}
m}m}m}m} ddlm	}m
}m} ddlm  ddlm}m} ddlm}m} t }t|ttfrB|n|g}jdd  fdd	|D }rod
d}||  |j gd|d} n*|dkr}|d |  } n|| jd} | jsddl m!}m"} ||dksJ d|| jj#}jdd u rdjj$dj_%|dt& |jj%t'| d |j(||dd}||_)dd }|d ur|d(i jj*|di}nrdjj*vr|d(i jj*}njj*}|j+jj,|jj-jj.jdd jdd d r(jj%d dkr(|/|
  |d urb |j0j1j2j0j1j3ddd }jd!i } jj%d d"k| d#< rS|n|}!|j/|!|fi | d$d% jj4rq|j5jj4dd& njj6r}|7jj6 d'd	 jj8D j_8|9|jj8 d S ))Nr   )DistSamplerSeedHookFp16OptimizerHookbuild_optimizerbuild_runnerget_dist_info)DistEvalHookDistOptimizerHookEvalHookbuild_dataloader)wrap_distributed_modelwrap_non_distributed_modelMMDataParallelMMDistributedDataParallelsamplerc                    s8   g | ]} |j jjj jjtjd jdqS )T)num_gpusdistround_upseedsampler_cfg)train
dataloaderbatch_size_per_gpuworkers_per_gpulengpu_idsr(   ).0dsr   cfgdistributedr)    e/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/cv/image_classifition_trainer.py
<listcomp>1   s    
ztrain_model.<locals>.<listcomp>find_unused_parametersF)
device_idsbroadcast_buffersr8   cpuzThe argument `device` is deprecated. To use cpu to train, please refers to https://mmclassification.readthedocs.io/en/latest/getting_started.html#train-a-modelr9   )__version__digit_version      rA   zLTo train with CPU, please confirm your mmcv version is not lower than v1.4.4runnerEpochBasedRunner)type
max_epochszVconfig is now expected to have a `runner` section, please set `runner` in your config.)modelbatch_processor	optimizerwork_dirloggermeta)default_argsfp16r4   rD   momentum_configcustom_hooks)custom_hooks_configTsamples_per_gpur-   r&   shuffler'   
evaluationIterBasedRunnerby_epochLOW)prioritymap_locationc                 S   s   g | ]}t |qS r5   )tuple)r0   flowr5   r5   r6   r7      s    r5   ):torchwarningsmmcv.runnerr   r   r   r   r   
mmcls.corer   r   r   mmcls.datasetsr   mmcls.utilsr   r    mmcv.parallelr"   r#   r   
isinstancelistr[   r*   getcudacurrent_devicewarningr;   r/   r9   mmcvr=   r>   rH   rE   rB   UserWarningdictrI   	timestampoptimizer_configregister_training_hooks	lr_configcheckpoint_config
log_configregister_hookrT   r+   r,   r-   resume_fromresume	load_fromload_checkpointworkflowrun)"rF   datasetr3   r4   val_datasetrm   devicerK   r]   r^   r   r   r   r   r   r   r   r   r   r    r"   r#   rJ   data_loadersr8   r=   r>   rH   rB   fp16_cfgrn   val_dataloadereval_cfg	eval_hookr5   r2   r6   train_model   s   







r   )module_namec                       s   e Zd Zddddddddeddfdeeeeje	f  dee	 dee
 deee
ee	e
f f  deeeef  d	eeeef  d
eeeee	ef f  deejjejjjf dee	 dedee
 f fddZdd Z	dde	dee	ef fddZ  ZS )ImageClassifitionTrainerN)NNr   rF   cfg_filearg_parse_fndata_collatortrain_dataseteval_datasetpreprocessor
optimizersmodel_revisionr(   cfg_modify_fnc           %         sf  ddl }ddl}ddlm}m} ddlm} ddlm}m	} ddl
m} ddlm} ddlm} ddl}|
| _|| j t|trW| j||	d	| _|du rVtj| jtj}n|dus_J d
tj|| _t || | j}d|v rz|d | _n	| jj !dd| _|"t#$| j | j|_tj| jtj%| _&d|v r|d |j _'n	|j !dd|j _'d|v r|d |j _(n|j !dddu rtj| jtj%|j _(d|v r|d |_)n|!dd|_)d|v r|d dd |_*ndg|_*d|v r|d du rdn|d |_+nd|_+|!dd|_,|dur||}d|vr.t-| jj ds'J d| jj j.| _.n|d | _.| j.|j _.|j !dddurH| j.|j j/_.d|v rsd}d|v rX|d nddi}||d fi | | \}}t0t1||_*nd}|d t23dt24 }t#| j| d }t|d!}t5 } | }!d"d#d$ |!6 D }"d%}#|7d&|# |" d" |#  |"| d'< |j8| d(< |7d)|  |7d*|j8  | j|_9|!d+d}$|7d,|j9 d-|$  ||j9|$d. |j9| d/< t#:|| d0< || _;|| _<|j=!d1ddu rd2| _>n|j=j>| _>|| jj?j@}|A  || _|j)| _)|| _B|| _?|| _C|| _D| | _E|| _FdS )3z High-level finetune api for Image Classifition.

        Args:
            model: model id
            model_version: model version, default is None.
            cfg_modify_fn: An input fn which is used to modify the cfg read out of the file.
        r   N)get_ms_dataset_rootget_classes)build_classifier)r   	init_dist)set_random_seed)collect_envr   )r   z?Config file should not be None if model is not from pretrained!rI   z
./work_dirrt   rv   r|   rg   r/   r@   rM   no_validateFrE   z+max_epochs is missing in configuration filerB   launcherTdist_paramsbackendnccl
modelscopez%Y%m%d_%H%M%Sz.log)log_file
c                 S   s   g | ]\}}| d | qS )z: r5   )r0   kvr5   r5   r6   r7   #  s    z5ImageClassifitionTrainer.__init__.<locals>.<listcomp>z=------------------------------------------------------------
zEnvironment info:
env_infoconfigzDistributed training: zConfig:
deterministiczSet random seed to z, deterministic: )r   r(   exp_namedata_prefix )Gr]   rj   /modelscope.models.cv.image_classification.utilsr   r   mmcls.modelsr   r_   r   r   
mmcls.apisr   rb   r   
mmcv.utilsr   3modelscope.models.cv.image_classification.backbones_seedrd   strget_or_download_model_dir	model_dirospathjoinr   CONFIGURATIONdirnamesuper__init__r3   rI   r*   rf   mkdir_or_existospabspathTORCH_MODEL_FILEeval_checkpoint_pathrt   rv   r|   r/   rM   r   hasattrrE   rB   re   rangetimestrftime	localtimerl   itemsinfopretty_textr(   basenamer   r   rz   r   rF   mm_modelinit_weightsr   r4   rm   rK   rJ   )%selfrF   r   r   r   r   r   r   r   r   r(   r   kwargsr]   rj   r   r   r   r   r   r   r   mmcv_get_loggerr   r3   r4   r   _
world_sizerm   r   rJ   rK   env_info_dictr   	dash_line_deterministic	__class__r5   r6   r      s   

















z!ImageClassifitionTrainer.__init__c              
   O   s  ddl m} ddlm}m}m} ddlm} | jd u rt	dd | j
jj_| j
t| jt| j | j
jjd u rP|| j}t|d}	t|	rM|	nd }
n| j
jj}
|| j| j
jj|
| jdg}t| j
jjdkr| jd u rut	d	| j
jj}|| j||
| jd}|| | jt || j
j!|d j"d
 d }| j
j#s|| j|| j
jj$|
| jd}t%| j|| j
| j&|| j'| j(dkrdnd| jd d S )Nr   )r=   )r   	MmDatasetpreprocess_transformsetup_multi_processeszBNot found train dataset, please set the 'train_dataset' parameter!classname.txtpipelineclassesr      DNot found evaluate dataset, please set the 'eval_dataset' parameter!)mmcls_versionr   CLASSESr;   rg   )r4   r{   rm   r|   rK   ))mmclsr=   r   r   r   r   rb   r   r   
ValueErrorr3   rF   r   
pretraineddumpr   r   rI   r   r   rz   r   existsr   r*   r   r.   rx   r   appendrK   updaterl   r   r   r   valr   r4   rm   r|   )r   argsr   r=   r   r   r   r   	data_rootclassname_pathr   datasetsval_data_pipeliner{   r5   r5   r6   r*   M  sv   






zImageClassifitionTrainer.traincheckpoint_pathreturnc           &      O   s  dd l }dd l}ddlm}m}m}m}	 ddlm}
 ddl	m
}m}m} ddlm}m} ddlm}m} ddlm} | jd u rCtdd | jjj_| jjjd u re|| j}t|d	}t|rb|nd }n| jjj}|| j|| jj j!|| j"d
}|
|| jj#j$j%| jj#j$j&| j'ddd}t()| j}| j*dd }|d ur|| |d u r|	| j+}|d ur||t,j-| j+|dd}n||| j.dd}n|||dd}d|*di v r|d d }nddlm/} | j01d |j2}| j's"| j3dkr|4 }n||| jj5d}|j6st78t7j9dksJ d||_2i }|||dd fi |}n||: |j:; gdd}|||d d}| \} }!| dkri }"t< }#| jj#*di }$d|$= v r]t>|$d |$d< nt?|dk rhd|$d< | jj#j@r|jA|| jj#j@|$|#d}%|"B|% |"S d S )Nr   )r   r   r   get_trained_checkpoints_namer   )r   rw   wrap_fp16_modelr!   )multi_gpu_testsingle_gpu_testr   r   r   r   FTrQ   rM   r;   rY   r   rK   )ImageNetzQClass names are not saved in the checkpoint's meta data, use imagenet by default.r<   r?   zKTo test with CPU, please confirm your mmcv version is not lower than v1.4.4)r9   r:   metric_optionstopk   )r@   )resultsmetricr   rJ   )Cr^   r]   r   r   r   r   r   ra   r   r_   r   rw   r   rc   r"   r#   r   r   r   rb   r   r   r   r3   rF   r   r   rz   r   r   r   r   r   r   r   rT   r+   r,   r-   r4   copydeepcopyrf   rI   r   r   r   r   rJ   ri   r   r|   r;   r/   r9   rj   r>   r=   rg   rh   r   keysr[   r.   metricsevaluater   )&r   r   r   r   r^   r]   r   r   r   r   r   r   rw   r   r"   r#   r   r   r   r   r   r   rz   data_loaderrF   r~   trained_checkpoints
checkpointr   r   show_kwargsoutputsrankr   r   rJ   r   eval_resultsr5   r5   r6   r     s   











z!ImageClassifitionTrainer.evaluate)N)__name__
__module____qualname__r   r   r   r   r   Moduler   r   r   r   r   r   r   r]   optim	Optimizerlr_scheduler_LRSchedulerintr   r*   floatr   __classcell__r5   r5   r   r6   r      sb    

	
 +H
r   )FNNNN)*r   r   os.pathr   r   r   typingr   r   r   r   r   numpynpr]   r   torch.utils.datar    modelscope.hub.snapshot_downloadr	   modelscope.metainfor
   modelscope.models.baser    modelscope.msdatasets.ms_datasetr   modelscope.preprocessors.baser   modelscope.trainers.baser   modelscope.trainers.builderr   modelscope.utils.constantr   r   r   modelscope.utils.loggerr   r   register_moduleimage_classificationr   r5   r5   r5   r6   <module>   s6   
 