o wiJ2@sddlmZmZddlmZddlZddlZddl m Z m Z ddl m Z ddlmZddlmZmZddlmZddlmZdd lmZdd lmZmZmZdd lmZdd lm Z dd l!m"Z"ddl#m$Z$ddl%m&Z&dZ'ej(j)e'dd2dej*ej+fddZ,ej(j)e e'ddddde ddfdee-de-de.de.ded e/d!e/dej0fd"d#Z1d$ej0dej0fd%d&Z2ej(j)e e'd  '  (  )   d3dee-d*e-de-de.de.d+ee-d,ee.d-ee/d!e/dej0fd.d/Z3d$ej0dej0fd0d1Z4dS)4)CallableOptionalN)finetunepretrain)MockDataModule)PackedSequenceSpecs) DeepSeekModelDeepSeekV3Config) PEFT_STR2CLS)trainer)default_finetune_recipe) default_logdefault_resumetensorboard_logger),distributed_fused_adam_with_cosine_annealing)DeepEPCallback)GarbageCollectionCallback)MegatronCommOverlapCallback)TimingCallback deepseek_v3nameFreturncCs.|r tjtddd}ntt}tjt|dS)ak Factory function to create a DeepSeek-V3 (671B) model configuration. Returns: run.Config[pl.LightningModule]: Configuration for the DeepSeek V3 model. Examples: CLI usage: $ nemo llm pretrain model=deepseek_v3 ... Python API usage: >>> model_config = model() >>> print(model_config) g?)mtp_num_layersmtp_loss_scaling_factor)config)runConfigr r)use_mtpconfr!e/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/llm/recipes/deepseek_v3.pymodel%s r#)targetrdefaultTdirr num_nodesnum_gpus_per_nodefnrperformance_modec Csvtj|t|tddd||ttgdtjtddddt||t|ddt d d t d }d |jj _ d |jj _ d|jj _d|jj _d |jj _d|jj j_d |jj_ddlm}tj|dd|jj_d|jj_ddg|jj_tt} d|jj_tj |jj_!tj"|jj_#tj"|jj_$tj"|jj_%tjt&ddd} tjt'dd} |jj()| |jj()| |jj()| |rt*|}|S)a Create a pre-training recipe for DeepSeek-V3 (671B) model. This function sets up a complete configuration for pre-training, including model, trainer, data, logging, optimization, and resumption settings. Args: dir (Optional[str]): Directory for saving logs and checkpoints. name (str): Name of the pre-training run. num_nodes (int): Number of compute nodes to use. num_gpus_per_node (int): Number of GPUs per node. performance_mode (bool): If true, enables optimizations for maximum performance. fn (Callable): The pre-training function to use. Returns: run.Partial: Partial configuration for pre-training. Examples: CLI usage: $ nemo llm pretrain --factory deepseek_v3 $ nemo llm pretrain --factory "deepseek_v3(num_nodes=128, name='my_deepseek_v3')" Python API usage: >>> recipe = pretrain_recipe(name="deepseek_v3_pretrain", num_nodes=128) >>> print(recipe) r@)tensor_parallelismpipeline_parallelismexpert_parallelismr)r* callbacksi) seq_lengthglobal_batch_sizemicro_batch_sizer)r(rrga2U0*3?)max_lr)r#r datalogoptimresumeNFr) timedelta<)minutes selective mla_up_proj layernormTgc_interval_traingc_interval_valtp_comm_overlap)+rPartialr#r rrrr rrrstrategy"num_layers_in_first_pipeline_stage!num_layers_in_last_pipeline_stage$virtual_pipeline_model_parallel_sizeexpert_tensor_parallel_sizetensor_model_parallel_sizeddpgrad_reduce_in_fp32r8ckpt save_top_kdatetimer=train_time_intervalrrecompute_granularityrecompute_modulesrr9use_precision_aware_optimizertorchfloat32main_params_dtypebfloat16main_grads_dtype exp_avg_dtypeexp_avg_sq_dtyperrr2append"pretrain_performance_optimizations) r(rr)r*r+rr,reciper=deepep_callbackgarbage_collection_callbackcomm_overlap_callbackr!r!r"pretrain_recipe<s`%                rfrbcC^t|jdr |jjdurg|j_tjtddd}tjtdd}|jj||gd|jj_ |S)a Create a performance-optimized pre-training recipe for DeepSeek-V3 (671B) model. This method enables performance optimizations that may not be suitable for all use cases. It builds upon the standard pre-training recipe and adds additional performance enhancements. Args: recipe (run.Partial): Base pre-train recipe to which performance optimizations will be added Returns: run.Partial: Partial configuration for performance-optimized pre-training. Note: Use this method with caution and only when you need maximum performance. It may not be suitable for all hardware configurations or use cases. r2Nr>rDFrG hasattrr r2rrrrextendpluginsrQrbrdrer!r!r"ras$ radeepseek-ai/DeepSeek-V3-BaserClora resume_path peft_schemer3packed_sequencec Cs|durd}|dur|dus|dkrd}n|dvrd}tt||||||} |dus4|dkrSd| jj_d| jj_d| jj_d | jj_d | jj_ d | j j _ nK|dvrt t|| _gd | j_d | j j _d | jj _d| jj_d| jj_d| jj_d| jj_d| jj_d| jj_ d| j j _ ntd||| jj _|| j_|rddi| j_t jt|d| j_|rt| } | S)a` Create a fine-tuning recipe for DeepSeek-V3 (671B) model. This function sets up a complete configuration for fine-tuning, including model, trainer, data, logging, optimization, and resumption settings. The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None. Args: dir (Optional[str]): Directory for saving logs and checkpoints. resume_path (str): Path to the NeMo checkpoint name (str): Name of the fine-tuning run. num_nodes (int): Number of compute nodes to use. num_gpus_per_node (int): Number of GPUs per node. peft_scheme (Optional[str]): Name of the peft scheme to use for fine-tuning. Allowed values: 'lora'/'dora'/'none'/None. seq_length (int): Maximum number of tokens per microbatch. packed_sequence (Optional[bool]): If true, fine-tuning sequences will be packed into batches up to the given maximum seq_length for better efficiency. By default, this value equals performance_mode. Returns: run.Partial: Partial configuration for fine-tuning. Examples: CLI usage: $ nemo llm finetune --factory deepseek_v3 $ nemo llm finetune --factory "deepseek_v3(num_nodes=5, name='my_deepseek_v3_finetune')" Python API usage: >>> recipe = finetune_recipe(name="deepseek_v3_finetune", num_nodes=6) >>> print(recipe) Note: This recipe uses the SQuAD dataset for fine-tuning. Be aware that fine-tuning the DeepSeek-V3 model requires substantial computational resources. Ninoner.)rndorarCrr'gh㈵>)linear_q_down_projlinear_q_up_projlinear_kv_down_projlinear_kv_up_proj linear_projFT g-C6?zUnrecognized peft scheme: pad_to_max_length)packed_sequence_size)lowerr r#r rJexpert_model_parallel_sizerOpipeline_model_parallel_sizerKrLr9rlrrrr pefttarget_modulesuse_distributed_optimizercross_entropy_loss_fusionsequence_parallel ValueErrorr3r7dataset_kwargsrpacked_sequence_specs"finetune_performance_optimizations) r(rorr)r*rpr3rqr,rbr!r!r"finetune_recipesH/                    rcCrg)a] Modify the given recipe to optimize settings for performance. This method enables performance optimizations that may not be suitable for all use cases. Intended to build upon the standard fine-tuning recipe. Args: recipe (run.Partial): Base fine-tuning recipe to which performance optimizations will be added Returns: run.Partial: Partial configuration for performance-optimized fine-tuning. Note: Use this method with caution and only when you need maximum performance. It may not be suitable for all hardware configurations or use cases. r2Nr>rDFrGrhrlr!r!r"r*s$ r)F) Nrmr%rCr'rnNNF)5typingrrlightning.pytorchpytorchplnemo_runrrYnemo.collections.llm.apirr"nemo.collections.llm.gpt.data.mockr-nemo.collections.llm.gpt.data.packed_sequencer'nemo.collections.llm.gpt.model.deepseekrr nemo.collections.llm.peftr %nemo.collections.llm.recipes.deepseekr -nemo.collections.llm.recipes.finetune_defaultr (nemo.collections.llm.recipes.log.defaultr rr'nemo.collections.llm.recipes.optim.adamr'nemo.lightning.pytorch.callbacks.deepepr3nemo.lightning.pytorch.callbacks.garbage_collectionr6nemo.lightning.pytorch.callbacks.megatron_comm_overlaprnemo.utils.exp_managerrNAMEclifactoryrLightningModuler#strintboolrIrfrarrr!r!r!r"s             b)  a