o }oi l@sjddlZddlmZddlmZddlZddlmZmZddl m Z m Z ddl m Z ddlmZddlmZdd lmZdd lmZmZdd lmZdd lmZmZdd lmZddlm Z m!Z!ddl"m#Z#ddl$m%Z%ddl&m'Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6dej7j8_9ddZ:ddZ;ed dd?d!|jd@d ddA|jdBtgdCdDdEdF|jdGd ddA|jdHttt dIdJdF|jdKd ddLd!|jdMtddNdO|jdPtdQdRd|jdStdddTdU|jdVtd&dWd|jdXt dYdZd|jd[d d\d |jd]d dd^d!|jd_td`dad|jdbtdcddd|jdetdfdgd|jdhd ddid!|jdjd ddA|jdktdldmgdldndF|jdod ddpd!|jdqd ddrd!|jdsd ddtd!|jdutdfdvd|jdwtdxd|jdytdzd|jd{d dd|d!|jd}d dd~d!|jdt ddd|jdt ddd|jdtddd|jdd ddd!|jdtddfdd|jdtdddO|jdd ddd!|jdtdddfgdd|jdtdd|jdddddd|jdt ddd|jdt dd|jdd ddd!|jdd ddd!|jdt ddd|jdt ddd|jdd}|jdd ddA|jdd ddA| S)z(Parse arguments for Evo2 model training.z#Train a Hyena model using NeMo 2.0.) descriptionformatter_classT)requiredz-dz--dataset-configzCPath to the blended / weighted training dataset configuration YAML.)typehelpz --mock-data store_truezZTrain with Mock data (for testing/debugging), either set this or provide a dataset config.)actionr"z --dataset-dirzAbsolute path to the dataset directory. Defaults to using the absolute or relative paths (dataset_prefix) specified in the dataset config YAML.z --num-nodesz3Number of nodes to use for training, defaults to 1.)r!defaultr"z --devicesz5Number of devices to use for training, defaults to 1.z --seq-lengthi zTraining sequence lengthz--tensor-parallel-sizez+Order of tensor parallelism. Defaults to 1.z--pipeline-model-parallel-sizez-Order of pipeline parallelism. Defaults to 1.z--context-parallel-sizez,Order of context parallelism. Defaults to 1.z --no-wandbFzDisable Wandb logging)r$r&r"z--wandb-project nemo_evo2zWandb project namez--wandb-run-idNzWandb run identifierz --wandb-groupz3A unique string shared by all runs in a given groupz--wandb-job-typezA unique string representing a type of run, which is useful when you're grouping runs together into larger experiments using group.z--sequence-parallelz#Set to enable sequence parallelism.z--fp8zSet to enable FP8z--micro-batch-sizez,Micro-batch size for data-parallel training.z--global-batch-sizez\Global batch size for training. If set to None, infer it from the TP, CP, and PP parameters.z--grad-acc-batchesz/Number of batches to accumulate gradients over.z --max-stepsz*Number of training optimizer update steps.z--val-check-intervalzFNumber of steps between validation measurements and model checkpoints.z--grad-reduce-in-fp32zGradient reduce in FP32.z --fp8-wgradz?Faster option that is maybe less accurate (TBD) when using fp8.z--no-aligned-megatron-ddpz'Do not do aligned gradient updates etc.z--use-megatron-comm-overlap-8k)r$r&z--tp-comm-overlap-backend)ncclmpigloor(z4TP communication backend to use. Defaults to 'nccl'.)r!choicesr&r"z--align-param-gatherz --model-size7bzModel architecture to use, choose between 7b, 40b, or test (a sub-model of 4 layers, less than 1B parameters). '_arc_1m' models have GLU / FFN dimensions that support 1M context length when trained with TP<=8.z--add-bias-outputz?Add bias to the output layer to enable learning a simple prior.z--experiment-dirz4Directory to write model checkpoints and results to.)r!r r"z--limit-val-batcheszNumber of validation stepsz--log-every-n-stepsz Number of steps between logging.)r!r&r r"z --ckpt-dirzUDirectory to restore an initial checkpoint from. Use this for supervised fine-tuning.z--wdg{Gz?zWeight decay for optimizer.z--restore-optimizer-from-ckptzCRestore optimizer state from initial checkpoint. Defaults to False.z--no-average-in-collectivezSAvaerage optimizer state in collective rather than dividing by dp size and summing.z--seedizSet random seed for training.z --workersz*Number of workers to use for data loading.z --gc-intervalrzeSet to a value > 0 if you want to synchronize garbage collection, will do gc every gc-interval steps.z--enable-preemptionzTEnable preemption hooks. If enabled this will save a checkpoint whenver slurm exits.z--ckpt-async-savez --ckpt-format torch_distzarrzSpecify checkpoint format to use. Defaults to 'torch_dist', as 'zarr' is deprecated. Only use if resuming training from a zarr checkpoint.z--eod-pad-in-loss-maskzRDo not predict EOD/Pad tokens (typical default, but not default in original evo2).z--cross-entropy-loss-fusionzjUse the faster, but maybe less accurate fused form of cross entropy, which also has bf16 grads internally.z--no-fp32-residual-connectionzWIf set, turn off fp32 residual connections which may be faster but may impact accuracy.z--debug-ddp-parity-freqz:Set to value > 0 to debug DDP weight parity between ranks.z--hybrid-override-patternz]Override the hybrid override pattern in the config (specifies hyena layer ordering and type).z --num-layerszHIf set, override the number of layers specified in the requested config.z--tflops-callbackzGEnable tflops calculation callback for Hyena / Evo2. Defaults to False.z--log-parameters-and-shapesz8Log training parameters shapes and dtypes for debugging.z--lrga2U0*3?zLearning rate.z--min-lrgiUMu>z&Min learning rate in cosine annealing.z--warmup-stepsi z*Number of warmup steps in cosine annealingz--nsys-profilingaSEnable targeted `nsys` profiling on the training loop for a defined step range. To actually get profiling output you must run the whole program with `nsys`. For example: `nsys profile -s none -o output_report_name -t cuda,nvtx --force-overwrite true --capture-range=cudaProfilerApi --capture-range-end=stop [regular python command here]`z--nsys-start-stepz%Start nsys profiling after this step.)r!r r&r"z--nsys-end-stepz#End nsys profiling after this step.z--no-renormalize-lossz$Do not renormalize the loss weights.z --nsys-ranks+z&Enable nsys profiling for these ranks.)r!nargsr r&r"z,--activation-checkpoint-recompute-num-layersz5If set, override the default value set in the config.z--disable-checkpointing store_falsecreate_checkpoint_callbackz,Disable creating a ModelCheckpoint callback.)r$r&destr"z --clip-gradg?zGGrad clip value. Note that when using DDP this may need to be inflated.z--seq-len-interpolation-factorzAdjusts the linear scaling of ROPE (Rotary Position Embedding) for context extension. Set this factor relative to your base context length e.g., for an original context length of 8192 and an extended context length of 524288, use 524288/8192 = 64.z--overlap-param-gatherzOverlap the parameter gather with the optimizer step. This is currently disabled due to a NeMo bug when using DDP. Making this an option defaulting to False is a temporary solution until the bug is fixed.z--overlap-grad-reducez4Overlap the gradient reduce with the optimizer step.z--hidden-dropoutgz(Dropout probability for the hyena layersz--attention-dropoutz-Dropout probability for the attention layers.z--no-activation-checkpointingz$--selective-activation-checkpointing) argparseArgumentParserArgumentDefaultsHelpFormatteradd_mutually_exclusive_group add_argumentstrintsortedrkeysfloat parse_args)parser data_grouprecompute_grouprD^/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/gpt/model/test_hyena.pyr@6s>     r@cCst}td}|j}|jrt|j|j||j|d}n t|j |j }|j r&t nt }t|||j|j||j|j||j d }|jrDdddd}n|jrNdddd}n |jdurYd|ji}ni}|j|j|j|j|jrhdnd |jrnd nd |j|j |jd |}|jr|j|d <|jr|j|d<|jtvrtd|jt|jdUi|}t j!||j"d} t#ddt$t%g} |j&rt'|j(|j)dd d d d} | *| |j+r| *t,-|j.dkr| *t,j/|j.d|j0r| *t,1|j2rt3t4||d} | *| |jr|j5rt6} nt7} | *t8|j9| |j:dd |j;d|jrI|j?dur8|j@}n|j?}| *t,jA|jB||jCd dg}i}|jDs"|jEr"tFdGgd|jd|jHd|jId |jJd!|d"|jd#|jd$|jd%|jd&|jKd'|jd(|jLd)|jd*|jM d+|j d,|jd-|jNd.|jd/|jd0|jOd1|jPd2|jQd3|jRd4|jSd5|jTo|j5d6|jUd7|jVd8|jWd9|j5|jX|jY|jZ|jE|j)d:}|*|||d;<t[d<|*|t\dUd?|j)i|}t]d |jU|jV|jS|j;|jM d@}t^j_||jH|jI|jJt`ja|jd d |jb|jcdAdB }t^jd|je|jW|j@dC||| |jf|jgdd t^jhdDt`ja|jS|j5rdEnd|j5rdFndG|j5rdHndI|j5o|jTp|jdJ|j(|j&dK}|ji|d dLt^jjd d d |j)|jkrtl|jkd |jmdMnddN}|i|| tndO|jOdPdQ|jR|jNd d dR}to|j@|jQ|jPdS}tp|||jqdT}|r| |s| |dS)Vz#Main function to run Evo2 training.z byte-level) seq_lengthmicro_batch_sizeglobal_batch_size num_workers tokenizer) paths dataset_clsrFrGrHseedrIrJ eod_mask_lossN)recompute_granularityrecompute_methodrecompute_num_layers selectiverQweightednormalized_weightedFT) tp_comm_overlaprFhidden_dropoutattention_dropoutto_upperdistribute_saved_activationscross_entropy_loss_fusionfp32_residual_connectionadd_bias_outputhybrid_override_pattern num_layerszInvalid model size: )rJ) max_depth)every_n_train_stepsdirpath save_top_kalways_save_contextsave_optim_on_train_endsave_context_on_train_endr)intervalhyena)rUtp_comm_overlap_cfgtp_comm_bootstrap_backendwgrad_deferral_limit(overlap_param_gather_with_optimizer_stepalign_param_gather)gc_interval_traingc_interval_val) start_stepend_stepranks gen_shapez evo2-size-z-TPz-PPz-CPz-GBSz-MBSz-SkipLossRenormz-NOACz-SELACz-ACRNLz-PATz-F32Rz-FCEz-AICz-PEODz-BOz-GCLPz-HDOz-ADOz-LRz-MINLRz-WUSTEPSz-WDz-GRFP32z-FP8WGz-OGRz-OPGz-NODESz-FP8)namegroupjob_typeidprojectsave_dirwandbdummy)r| tensorboardlog_dir)check_for_nan_in_gradoverlap_grad_reduceoverlap_param_gathergrad_reduce_in_fp32roaverage_in_collectivelog_all) ddptensor_model_parallel_sizepipeline_model_parallel_sizecontext_parallel_sizepipeline_dtypesequence_parallelckpt_load_optimizerckpt_save_optimizerckpt_async_savesave_ckpt_formatckpt_load_strictnessgpuz bf16-mixedhybridr%max most_recent) precision params_dtyperfp8fp8_amax_history_lenfp8_amax_compute_algo fp8_wgrad)devices num_nodes max_steps acceleratorstrategyloggerrlog_every_n_stepslimit_val_batchesnum_sanity_val_stepsuse_distributed_samplerpluginsval_check_intervalenable_checkpointing)resume_if_exists)pathload_model_stateload_optim_state)rresume_ignore_no_checkpointresume_past_endresume_from_directoryrestore_configadamg?gffffff?) optimizerlr adam_beta1 adam_beta2 weight_decay clip_graduse_distributed_optimizerbf16)r warmup_stepsmin_lr)no_weight_decay_condrD)tr@rrH mock_datar rFrGworkersrdataset_config dataset_direod_pad_in_loss_maskrrr rMno_activation_checkpointing"selective_activation_checkpointing*activation_checkpoint_recompute_num_layersuse_megatron_comm_overlap_8krVrWno_renormalize_lossrrZno_fp32_residual_connectionr\r]r^ model_sizer ValueErrorr HyenaModelrJrrrr4rrexperiment_dirappendenable_preemption nl_callbacksPreemptionCallbackdebug_ddp_parity_freqDdpParityCheckerlog_parameters_and_shapesParameterDebuggertflops_callbackrrrrrrrUtp_comm_overlap_backendro gc_intervalGarbageCollectionCallbacknsys_profiling nsys_end_stepr NsysCallbacknsys_start_step nsys_ranksno_wandb wandb_projectrjointensor_parallel_sizerrrQr[no_average_in_collectiverrrrwdrrrrr wandb_groupwandb_job_type wandb_run_idrrrnlMegatronStrategytorchbfloat16r ckpt_formatTrainerrrrMegatronMixedPrecisionsetup AutoResumeckpt_dirrrestore_optimizer_from_ckptr rrhyena_no_weight_decay_cond_fnconnectfit)argsrJrHdatablended_dataset_configrLactivation_checkpointing_argsconfig_modifiers_init evo2_configmodelrcheckpoint_callbackflop_meas_callbackrkrloggersnemo_logger_kwargs wandb_logger tb_logger nemo_loggerrrtrainerresume opt_configschedoptrDrDrEmainjs              "               r__main__)=r6 dataclassesrtypingrrlightning.pytorch.callbacksrrlightning.pytorch.loggersrrmegatron.core.distributedrmegatron.core.optimizerr nemor rnemo.collectionsr nemo.collections.llm.gpt.datar r 3nemo.collections.llm.gpt.data.megatron.hyena.configr9nemo.collections.llm.gpt.data.megatron.hyena.evo2_datasetrr$nemo.collections.llm.gpt.model.hyenar;nemo.collections.llm.recipes.tp_overlap_configs.userbuffersrr3nemo.collections.nlp.modules.common.tokenizer_utilsrnemo.lightningrnemo.lightning.pytorchrr nemo.lightning.pytorch.callbacksr/nemo.lightning.pytorch.callbacks.flops_callbackr6nemo.lightning.pytorch.callbacks.megatron_comm_overlaprnemo.lightning.pytorch.optimr%nemo.lightning.pytorch.optim.megatronr'nemo.lightning.pytorch.strategies.utilsrnemo.utils.exp_managerr_dynamoconfigsuppress_errorsr@r__name__rDrDrDrEsH                   6,