o
    il%                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
l m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:m;Z; ej<ddddefddZ=dd Z<e>dkre=  dS dS )    N)BytesIO)nullcontext)
DictConfig	OmegaConf)autocast
GradScaler)DistributedDataParallel)FullyShardedDataParallel)Join)ShardedGradScaler)SummaryWriter)average_checkpoints)tables)optim_classes)Trainer)scheduler_classes)
initialize)download_model)mark_only_lora_as_trainable)set_all_random_seed)load_pretrained_model)prepare_model_dir)model_summary)	AutoModel)config_nameversion_basekwargsc                 C   st   |  ddrdd l}|  d| v sJ d| vr1td|  dd tdd	|  d	d
i| } tdi |  d S )NdebugFr   model
model_confz"download models from model hub: {}hubmsis_trainingT )getpdb	set_tracelogginginfoformatr   main)r   r%   r#   r#   D/home/ubuntu/.local/lib/python3.10/site-packages/funasr/bin/train.py
main_hydra(   s   r,   c                   K   s@  t | dd | dtjjjtjj_| dtjjjtjj_| ddtjj_| ddtjjj	_
ttjdd}|dkrDt  d	tjv oQttjd	 d
k}| dd}|s\|rmtj| dddd tj| td | dd}d| d< tdPi | }|s|rt dks|s|s|dkrtdPi |  |j} || d< | d }| d }|j}| d= | dd }|d urd|v rt|}t|ttfs|f}td| |D ]"}	| D ]\}
}|
|	d s|
|	krtd|
 d d|_ qq|dkrtt!|  |r||}t"||g| di ddd }n<|rPtd!fd"t#j$d#t%d$td%td&t%f
d'd(}t&j'|td)d*}tj| t(||d tj) d+}n
|j*| ddd,}t+|, j-| d< td- | d.d/}|t.v suJ t.|}||, fi | d0}td1 | d2d3}|t/v sJ t/|}||fi | d4}td5 tj0| d6 d7d8}|dPi | }t1dP|||| d | d9d:d;| d}|j2rt3|j2d<nd }|j4rt5|j2d<n|}|j6||||d= tj78| d9d>}tj9|dd? zt:|}W n   d }Y d@\}}t;|j<|j=D ]}t>? }t;|j@|jAD ]}t>? }|jB|||jCdA\}}|jD||||||||||jA|jCdB d|_Ct+|, j-}|jEdkr}tj-| tjF  W d    n	1 sxw   Y  t>? | dC }tdD| dE|dFdG|jA dH|jA|  dI|jA| | dFdJ|j=|  dK|j=| d
 |jA |jA | | dFdL q.d|_@|jG|||d
 |dM |H  d|_I|jJ|d
 ||||d= t>? }|| dC }tdD| dE|dFdG|j= dN|j=| | dFdL	 dO|_KdO|_Lq!|jMdkrtN|jO|jP |Q  d S )QNseedr   cudnn_enabledcudnn_benchmarkcudnn_deterministicTenable_tf32
LOCAL_RANK
WORLD_SIZE   use_fsdpFbackendncclzenv://)r6   init_methodz Build model, frontend, tokenizerdevicecudacpu	tokenizerfrontendr   freeze_param,zfreeze_param is not None: %s.zSetting z.requires_grad = False
train_conffind_unused_parameters)
device_idsrB   g    חAmodulerecursenonwrapped_numelmin_num_paramsreturnc                 S   s*   t |k}tdd |  D dk}|o|S )Nc                 S   s   h | ]}|j qS r#   )requires_grad).0pr#   r#   r+   	<setcomp>   s    z8main.<locals>.custom_auto_wrap_policy.<locals>.<setcomp>r4   )unwrapped_paramslen
parameters)rD   rE   rF   rG   is_largerequires_grad_uniformr#   r#   r+   custom_auto_wrap_policy~   s   z%main.<locals>.custom_auto_wrap_policyg     j@)rG   )auto_wrap_policymixed_precision	device_id)r9   zBuild optimoptimadam
optim_confzBuild scheduler	schedulerwarmuplrscheduler_confzBuild dataloaderdataset_conf
dataloaderDataloaderMapStyle
output_dirz./exp)
local_rankuse_ddpr5   r9   r_   )enabled)r   rV   rY   scalertensorboard)exist_ok)NN)data_split_i
start_step)r   rV   rY   rc   dataloader_traindataloader_valepochwriterrf   data_split_numrg   g      @zrank: z, time_escaped_epoch: z.3fz hours, estimated to finish z data_slices, remaining: z	 slices, z hours, epoch: z	 epochs, z hours
)r   ri   rj   rk   z epoch: g        r#   )Rr   r$   torchbackendscudnnrb   	benchmarkdeterministicr:   matmul
allow_tf32intosenvironr   printdistinit_process_group
set_devicer'   r(   r   get_rankr   r   r   eval
isinstancelisttuplenamed_parameters
startswithrI   r   DDPnnModulebool	functoolspartialFSDPcurrent_devicetonextrO   r9   r   r   dataloader_classesr   use_fp16r   r5   r   resume_checkpointpathjoinmakedirsr   rangestart_epoch	max_epochtimeperf_counterstart_data_split_irl   
build_iterrg   train_epochtypeempty_cachevalidate_epochstepstep_in_epochsave_checkpointtrain_acc_avgtrain_loss_avgrankr   r_   avg_nbest_modelclose) r   r`   ra   r5   r9   r   r<   r=   r>   tkrK   rR   my_auto_wrap_policyrV   optim_classrY   scheduler_classdataloader_classr]   trainerrc   tensorboard_dirrk   dataloader_trri   rj   time1rf   time_slice_itime_escapedtime2r#   r#   r+   r*   7   sx  









	
"
r*   __main__)?ru   sysrm   torch.nnr   hydrar'   r   argparseior   
contextlibr   torch.distributeddistributedrx   	omegaconfr   r   torch.cuda.ampr   r   torch.nn.parallelr   r   torch.distributed.fsdpr	   r   !torch.distributed.algorithms.joinr
   *torch.distributed.fsdp.sharded_grad_scalerr   tensorboardXr   'funasr.train_utils.average_nbest_modelsr   funasr.registerr   funasr.optimizersr   funasr.train_utils.trainerr   funasr.schedulersr   funasr.train_utils.initializer   'funasr.download.download_model_from_hubr   funasr.models.lora.utilsr   &funasr.train_utils.set_all_random_seedr   (funasr.train_utils.load_pretrained_modelr   funasr.utils.miscr    funasr.train_utils.model_summaryr   funasrr   r*   r,   __name__r#   r#   r#   r+   <module>   sL    R
