o ϯia@sddlZddlZddlZddlmZddlZddlZddlZddlm m Z ddl m Z ddl Z zddlZWn eyBdZYnwz ddlmmZWn eyYdZYnwzddlmZWn eymdZYnwddlmZmZmZddlmZddlmZddlmZmZm Z ddl!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+dd l(m,Z,m-Z-dd l.m/Z/ddZ0 dddZ1ddZ2d ddZ3ddZ4ddZ5ddZ6e7dkre5dSdS)!N)datetime) GradScaler)create_model_and_transforms trace_model create_model)get_data) parse_args) is_masterinit_distributed_deviceworld_info_from_env) setup_logging) cosine_lr)train_one_epochevaluate)get_tar_path_from_dataset_name dataset_split get_optimizer)load_pload_class_label) LinearProbec Cstt||D].}tjtj|jd|dr5ttj|jd|dtj|jd|ddqtjtj|jd|drUttj|jd|ddS)N epoch_top_.pt) reversedrangeospathexistsjoincheckpoint_pathrenameremove)argsstartidx all_idx_lenir&O/home/ubuntu/.local/lib/python3.10/site-packages/laion_clap/training/lp_main.pymaintain_ckpts's r(Tc Csrt|ttfrt|}t|||||dSt|tr,tt|}t|||||dSt|tt frdd| D}t | }t ||d}t |}||t ||d}|dd}||krh||fStt|D]} ||| || kr|| ||| <d||| <qntt|D]#} || rt|| t|t|tj|jd| d ||fSq||fSdS) z Record the top-k performance of the current epoch. current_top_k_metrics is a dictionary of the form: {1: top_1_ckpt_measure, 2: top_2_ckpt_measure, ...} )r"ckpt bignumbettercSi|]}|dqS)Fr&).0kr&r&r' Osz,update_top_k_performance..)reverseNTrr) isinstancelisttuplenpmeanupdate_top_k_performancedictvaluesfloatintkeyssortedcopydeepcopyappendrlenr(torchsaverrrr) new_metrics_inputscurrent_top_k_ckpt_metricsr"r)r* update_flag sorted_keys sorted_valuessorted_values_r%r&r&r'r65s^        r6cCs0|dp|dvp|dp|dp|dS)Nclap_model.transformerzclap_model.positional_embeddingzclap_model.text_projectionclap_model.token_embeddingclap_model.ln_finalzclap_model.logit_scale_t startswith)nr&r&r'is_pretrained_paramsps rP*cCs0t||tj||t||dSN)rA manual_seedr4randomseed)rUrankr&r&r' random_seedzsrWcsH|jdkrd|_d|_d|_ddt|}i}i}dd|D}|jr3td|D]}d|_ q-|j sd dfd dfd d|D}fd d|D} |j dur^d}d}nA|d j j |j} |jr3dD]} dD]} t|| | durt|| | t|| qqqmfdd|D} fdd|D}fdd|D}fdd|D}t| dd||jdg|j|j|jf|j|j|jd}t||j|j| }t|dd||jdg|j|j|jf|j|j|jd}t||j|j| }||d<||d<||d<||d<|jr2t j!||d}t j!||d}t j"|#ddt j$|ddt j$|ddnlt|dd| |jdg|j%|j&|j'f|j(|j)|jd|d<t|d|j%|j| |d<|jrt j!|d|d|d<t j"|#ddt j$|dddnfdd|D}t||j*|j&|j'f|j(d|jd}||d<|||fS)NadamrcSs |dS)N clap_modelrMrOpr&r&r's z%config_lp_optimizer..cSs:g|]\}}|ds|dvs|ds|dr|qS)rIrJrKrLrMr,rOr[r&r&r' sz'config_lp_optimizer..Freeze Text!!!!FcSs*|jdkpd|vpd|vpd|vpd|vS)Nbnlnbias logit_scale)ndimrZr&r&r'r\s cs || SrRr&rZexcluder&r'r\s c$g|]\}}||r|jr|qSr& requires_gradr]rfr&r'r^scrhr&rir]includer&r'r^$train)lrbeta1beta2epswd)_new _pretrainedc,g|]\}}||r|jrt|r|qSr&rjrPr]rfr&r'r^crvr&rwr]rkr&r'r^rxc,g|]\}}||r|jrt|s|qSr&rwr]rfr&r'r^rxcryr&rwr]rkr&r'r^rxg)params weight_decay)robetasrrmomentumoptimizer_nametextaudio)named_parameters) root_rankclapcs$g|]\}}||s|jr|qSr&rir])in_clapr&r'r^rmg?lp)+ optimizerrs wd_pretrainedwd_newr2r freeze_textlogginginforj lp_freeze train_data dataloader num_batchesepochs split_optgetattrsetattrr lr_pretrainedbeta1_pretrainedbeta2_pretrainedeps_pretrainedmomentum_pretrainedr warmuplr_new beta1_new beta2_neweps_new momentum_newhorovodhvdDistributedOptimizerbroadcast_parameters state_dictbroadcast_optimizer_staterorprqrrr}lp_lr)modeldatar"rr schedulertext_freeze_parametersr-gain_or_bias_params rest_params total_stepsxygain_or_bias_pretrained_paramsrest_pretrained_paramsgain_or_bias_new_paramsrest_new_paramspretrained_params_optimizerpretrained_params_schedulernew_params_optimizernew_params_scheduler lp_paramslp_optimr&)rgrrlr'config_lp_optimizers                     rc#s8 t}t|j|jdd|_t|jt|jtj |jtj |jt j|jt |j |_|jdurbdtdd|jd|jd|jd|jd|jg|_d |_t\|_|_|_|jrt|r|jD]2}t |D]+}t!j"#d |d|st!$d |d|t!%d |d|d |d|d q~qxd|_&t||j'drt!j"|j(|j}t!j$|dd|j'rd|jnd}t!j"|||_&d}t!j"#|j&r|d7}|dt)|}t!j$|dd|j'rd|jnd}t!j"|||_&t!j"#|j&s|j*rt+j,nt+j-|_.t/|j&|j.t0|}d|j1vp0d|j1v|_2d|j1vp|j?|j||j@|jAt!j";|j<d |jB|jC|jD|jEd% \} } tFtG|jH|_It+9d&|jJt+9d'|jKt+9d(|jLt+9d)|jIt+9d*|jMt+9d+|jNt+9d,|jOtP| |jJ|jKd-|jI|jLd.} | Q|} |j8rtR| SD] } | T| UqmWdn 1swY|jVrtW| |j|d/} t|rt+9d0t+9t)| t+9d1t!j"|j(|jd2} tX| d3-}tYtZ|D]}t[||}t+9d4|d5||\|d5|d6qWdn 1swY|jr|j8s|j]rtj^j_`| } i}|jar d|d7<tj^jbjc| f|gdd8|} td|| tFs*Jd9|jVr7d:vs7Jd;te| |\}}}|jd|vr|d>}|d?}|jstjtk|ldmd@rdAdB|lD}| n||jor|dur|lD]\}}|n||dCdDq|dur|n|dD|durdE|vr|n|dEt+9dF|jgdG|dHn| n|t+9dI|jgdG|dH|jprtqdJ|D]}d |_rqn t+9dKs|jgdtt_ud tt_v|j(o|j(wdLkot||_xd}|jxr6|j3r6t3dus0JdMt3y|j4}|j2rt|rt2dusHJdNt+*dOd:jzj{|_||j}duradPjzj{|_~t2jdQ|j|jgtZ|dR|j*rzt2j| ddSt2| t+*dTd:vrt| |||dS|dkrdPvr|jst| d|||jrdUdBt|jD}t||jD]}||jkrtqdV|D]}d |_rqt|rt+9dW|t| |||||||d}tfdXdYdZDr|jst| |||}|jr|j|jfd[d\|lD} |jxrd]dB|lD}!||j| d^}"|"|!|durB||"dE<||jksV|jdkrf||jdkrft|"t!j"|j5d_|d`|jrvt|"t!j"|j5da|jr|jst| |||"ddbq|j2rt|rt2dSdSdS)cN/-z%Y_%m_%d-%H_%M_%Slinear_probemodel_lr_b_j_p_Fz ./json_files/z,aws s3 cp s3://s-laion-audio/webdataset_tar/z/sizes.json ./json_files/z /sizes.json)localT)exist_okzout-zout.logrrwandball tensorboard checkpoints)ampfp16fp32rzIt is recommended to use AMP mixed-precision instead of FP16. FP16 support needs further verification and tuning, especially for train.zARunning in horovod mode with multiple processes / nodes. Device: z.Process (global: z, local z ), total .z=Running in distributed mode with multiple processes. Device: z&Running with a single process. Device zopenai cache dir: ) precisiondevicejitforce_quick_geluopenai_model_cache_dir skip_paramspretrained_audiopretrained_text enable_fusion fusion_typezlinear probe using mlp: zlinear probe using freeze: zlinear probe act layer: zlinear probe out ch: z,linear probe learning rate (if applicable): zlinear probe loss func: zlinear probe lp_metrics: i)mlpfreezein_chout_chact) batch_sizerzLinear Probe CLAP Model:zParams:z params.txtwz z:  static_graph) device_idsfind_unused_parametersz5At least one train or eval dataset must be specified.rnzCannot train with traced modelr) map_locationepochrmodulecSs"i|] \}}|tdd|qS)zmodule.N)r@r,r-vr&r&r'r.s"zmain.._rscalerz=> resuming checkpoint 'z ' (epoch )z=> loaded checkpoint 'r_z=> no checkpoint found at '{}'nonezPlease install tensorboard.zPlease install wandb.zStarting wandb.valr)projectnotesnametagsconfig)logzFinished loading wandb.cSr+)rr&)r,r%r&r&r'r.-sz8Text pretrained parameters are freezed since this epoch.z Start epoch c3s|]}|vVqdSrRr&)r,r)rr&r' =szmain..)rz imagenet-valz imagenet-v2cs$g|]\}}|vr|vr|qSr&r&r) top_k_dataset top_k_metricr&r'r^Bs zmain..cSs"i|] \}}|dd|qS)rr)rrr&r&r'r.Is)rrrepoch_rzepoch_latest.pt)r*)rtimesleepamodelreplacerTrUrArScudamanual_seed_allr4rclass_label_pathclass_index_dictrrrnowstrftimerorworkersr distributedr local_rankrV world_size remotedatar datasetnamesrrrrmakedirssystemlog_path log_locallogsstrdebugrDEBUGINFO log_levelr r report_torrtensorboard_pathr copy_codebasewarningrrr expanduserrrtmodel pretrained torchscriptrrrrrr@r2r; lp_out_chlp_mlprlp_actrlp_loss lp_metricsrtono_grad parametersset_ contiguoustraceropenr<varsrwrite use_bn_syncnn SyncBatchNormconvert_sync_batchnormddp_static_graphparallelDistributedDataParallelrrrresumeisfileloadnextiteritemsrNload_state_dictrrprintrjformatcudnn benchmark deterministiclower save_logs SummaryWriterr num_samplestrain_szval_dataval_szinit wandb_noteswatchrBrno_evalsave_top_performancerrfreeze_text_afterranytop_k_checkpoint_select_datasettop_k_checkpoint_select_metricrupdatesave_frequencysave_most_recentr6finish)#r" dataset_namesplit log_base_path log_filenamepostfixlog_base_path_newrdirnamerYclap_model_cfgrparam params_filefrrddp_argsrrrr start_epoch checkpointsdr-o_writerrDrcompleted_epochmetricsfiltered_metricsopt_dictcheckpoint_dictr&)rrrr'main%s.                                    "                rbcCsddlm}m}tj|j|jd}tj|r"t d|ddSt d|tj t }t dD]}tj |}q3||||d d d d t d dS)Nr)copytreeignore_patternscodez$Error. Experiment already exists at z). Use --name to specify a new experiment.r0zCopying codebase to rr r)ignorezDone copying code.r)shutilrcrdrrrr rrr3realpath__file__rrR)r"rcrd new_code_pathcurrent_code_pathrr&r&r'rns     r__main__)T)rQr)8rrrTrr=numpyr4rAtorch.backends.cudnnbackendsr5torch.cuda.amprrr ImportErrortorch.utils.tensorboardutilsr horovod.torchr clap_modulerrr training.datartraining.paramsrtraining.distributedr r r training.loggerr training.schedulerr training.lp_trainrrclap_module.utilsrrrrrclap_module.linear_proberr(r6rPrWrrbr__name__r&r&r&r'sb            ; 'K