o wi#@sddlmZddlmZddlZddlZddlm Z ddl m Z ddl m ZddlmZmZddlmZddlmZmZdd lmZdd lmZdd lmZmZmZdd lm Z dd l!m"Z"ddl#m$Z$dZ%ej&j'e%ddej(ej)fddZ*          d0de+de+deej,dee+de+de-de+d e+d!e+d"ee.ej(e dej(ej/fd#d$Z0ej&j'ee%d%dd&ddefd'ee1d(e1de+d e+dej2f d)d*Z3ej&j'ee%d%  &   + d1d'ee1d(e1de+d e+d,ee1d-e-dej2fd.d/Z4dS)2)OptionalN)Callback)DistributedDataParallelConfig) lightning)finetunepretrain)MockDataModule)MistralConfig7B MistralModel) PEFT_STR2CLS)default_finetune_recipe) default_logdefault_resumetensorboard_logger),distributed_fused_adam_with_cosine_annealing) bf16_mixed)TimingCallback mistral_7bnamereturncCstjtttdS)a^ Factory function to create a Mistral 7B model configuration. Returns: run.Config[pl.LightningModule]: Configuration for the Mistral 7B model. Examples: CLI usage: $ nemo llm pretrain model=mistral ... Python API usage: >>> model_config = model() >>> print(model_config) )config)runConfigr r rrd/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/llm/recipes/mistral_7b.pymodel&srFdtensor_parallelismpipeline_parallelismpipeline_parallelism_typevirtual_pipeline_parallelismcontext_parallelismsequence_parallelism num_nodesnum_gpus_per_node max_steps callbacksc Csbtjtj||||||ddddtjtdddddd } tjtjdd| |ddd||t| d d d } | S) a Configure the NeMo Lightning Trainer for Mistral 7B model. This function sets up the distributed training strategy and other training parameters. Args: tensor_parallelism (int): Degree of tensor model parallelism. pipeline_parallelism (int): Degree of pipeline model parallelism. pipeline_parallelism_type (Optional[torch.dtype]): Data type for pipeline parallelism. virtual_pipeline_parallelism (Optional[int]): Size of virtual pipeline parallelism. context_parallelism (int): Degree of context parallelism. sequence_parallelism (bool): Whether to use sequence parallelism. num_nodes (int): Number of compute nodes to use. num_gpus_per_node (int): Number of GPUs per node. max_steps (int): Maximum number of training steps. callbacks (Optional[list[run.Config[Callback]]]): List of callback configurations. Returns: run.Config[nl.Trainer]: Configuration for the NeMo Lightning Trainer. Examples: CLI usage: $ nemo llm pretrain trainer=mistral ... Python API usage: >>> trainer_config = trainer(num_nodes=2, num_gpus_per_node=8) >>> print(trainer_config) T)check_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gather) tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelgradient_as_bucket_viewckpt_include_optimizerckpt_async_saveckpt_parallel_loadddpgpur2 Fi) acceleratoraccumulate_grad_batchesr*deviceslimit_test_batcheslimit_val_batcheslog_every_n_stepsr)r'pluginsstrategyuse_distributed_samplerval_check_interval)rrnlMegatronStrategyrTrainerr) r!r"r#r$r%r&r'r(r)r*rEtrainerrrrrK9sJ(rK)targetrdefaultdirrcCs^tj|ttdddddd||ttgd tjtddddt||t|d d t d d t d S)a Create a pre-training recipe for Mistral 7B model. This function sets up a complete configuration for pre-training, including model, trainer, data, logging, optimization, and resumption settings. Args: dir (Optional[str]): Directory for saving logs and checkpoints. name (str): Name of the pre-training run. num_nodes (int): Number of compute nodes to use. num_gpus_per_node (int): Number of GPUs per node. fn (Callable): The pre-training function to use. Returns: run.Partial: Partial configuration for pre-training. Examples: CLI usage: $ nemo llm pretrain --factory mistral $ nemo llm pretrain --factory "mistral(num_nodes=2, name='my_mistral_pretrain')" Python API usage: >>> recipe = pretrain_recipe(name="mistral_pretrain", num_nodes=2) >>> print(recipe) rNrF) r!r"r#r$r%r&r'r(r*ii) seq_lengthglobal_batch_sizemicro_batch_sizer)rNrrga2U0*3?)max_lr)rrKdatalogoptimresume) rPartialrrKrrrr rrr)rNrr'r(fnrrrpretrain_recipes&  rYlora peft_schemepacked_sequencecCs~ttd|||||}|dus|dkr!d|jj_d|jj_|S|dvr8t t ||_ d|jj_|St d|) a9 Create a fine-tuning recipe for Mistral 7B model. This function sets up a complete configuration for fine-tuning, including model, trainer, data, logging, optimization, and resumption settings. The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None. Args: dir (Optional[str]): Directory for saving logs and checkpoints. name (str): Name of the fine-tuning run. num_nodes (int): Number of compute nodes to use. num_gpus_per_node (int): Number of GPUs per node. peft_scheme (Optional[str]): Name of the peft scheme to use for fine-tuning. Allowed values: 'lora'/'dora'/'none'/None. packed_sequence (Optional[bool]): Packing multiple training sequences into one long sequence for training efficiency. Default sequence length is 2048. Returns: run.Partial: Partial configuration for fine-tuning. Examples: CLI usage: $ nemo llm finetune --factory mistral $ nemo llm finetune --factory "mistral(num_nodes=2, name='my_mistral_finetune')" Python API usage: >>> recipe = finetune_recipe(name="mistral_finetune", num_nodes=2) >>> print(recipe) Note: This recipe uses the SQuAD dataset for fine-tuning. zmistralai/Mistral-7B-v0.3Nnonergh㈵>)rZdorag-C6?zUnrecognized peft scheme: )r rlowerrKrEr/rUrlrrrr peft ValueError)rNrr'r(r[r\reciperrrfinetune_recipes$)    rd) rrNNrFrrr N)NrMrrrZF)5typingrlightning.pytorchpytorchplnemo_runrtorch$lightning.pytorch.callbacks.callbackrmegatron.core.distributedrnemorrHnemo.collections.llm.apirr"nemo.collections.llm.gpt.data.mockr&nemo.collections.llm.gpt.model.mistralr r nemo.collections.llm.peftr -nemo.collections.llm.recipes.finetune_defaultr (nemo.collections.llm.recipes.log.defaultr rr'nemo.collections.llm.recipes.optim.adamr6nemo.collections.llm.recipes.precision.mixed_precisionrnemo.utils.exp_managerrNAMEclifactoryrLightningModulerintdtypeboollistrJrKstrrWrYrdrrrrs               Q  1