o }oiN8@sddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z ddlmZddlZddlZddlmZddlmZddlmZddlmZddlmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0dd lm1Z1m2Z2m3Z3m4Z4m5Z5m6Z6dd l7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@ddlAmBZBe rddlCmDZDddlEmFZFe ZGe eeHfZIejJjKdd     dde ejLeIfdejMde4dee e2ejNe2fdee e1ejNe1fde e3de eGde e ee fdefdd ZOejJjKdd   dde ejLeIfdejMde4dee e2ejNe2fdee e1ejNe1fde e3defd!d"ZPejJjKdd     dde ejLeIfdejMde4dee e2ejNe2fdee e1ejNe1fde e3d#e e ee fde eGdefd$d%ZQejJjKdd     ddejLdejMde4dee e2ejNe2fdee e1ejNe1fde e3de eGde e ee fdefd&d'ZRejJjKd(dd) * * * * +   ,dd-eHd.eHd/e*d0eSd1eSd2eSd3eSd4eSdejMdBd5eHdBd6eTdeHfd7d(ZUejJjKd8dd)      dd9eId:eIdejMde4d;e eIdee e2ejNe2fdee e1ejNe1fde e3de eGde e ee fdefdeHd?e)d@eSdAeSdBeSdBdCeSdBd0eSdBd1eSdBdDee e+ejNe+fdEe dBd5eHdBd6eTdFeTdefdGd=ZWejJjKdd  H  I * J K L  M L N * * * * * * *  O P Q M M M ,dd-e eIdReHdSe eHdTeHdUe eSdVeSdWeSdXeHdYe eIdZeTd[eHd\eSd]eSd1eSd^eSd_eSd`eSdaeSdbeSdce eHddeSdeeSdfeSdgeTdheTdieTd6eTf6djdkZXejJjKdde$dldmdfdne%doe$dpe#dBdeYfdqdrZZejJjKdsdd)  ,ddejLdteHdue eIdveTdef dwdxZ[dyeIdzeHde6j\fd{d|Z]ejJjKd}dd)dd,e]dfdyeIdzeHdue eIdveTd~e eeHge6j\fdeYeHe fdefddZ^ejJjKddd)dddej_d,ddddd,ddMf dyeIdej4de e`eHde e`eHde e ejMeHfdejadeTdfeSde eSdeSde ddeTdue eIdieTde`e deHffddZbdejLdejMdeGddfddZcdejLdejMde4de e2de e1de e3de eGde e ee fde fddZdddZe     ddejLdejMde4de e2de e1de e3de eGde e ee fddfddZfdddZgde ejLeIffddZhdS)N)deepcopy)Path) TYPE_CHECKINGAnyCallableOptionalUnion)parallel_state)Console)all_gather_object) Annotated)GPTModelHFAutoModelForCausalLM) AdapterConfigEvaluationConfigEvaluationTargetMisconfigurationError) DistillationGPTModel ExportConfig PruningConfigQuantizationConfig Quantizerprune_gpt_modelsave_pruned_model#set_modelopt_spec_if_exists_in_ckpt2setup_trainer_and_restore_model_with_modelopt_spec) AutoResume NeMoLoggerOptimizerModuleTrainer-configure_no_restart_validation_training_loopio)NEMO_MODELS_CACHE)ckpt_to_context_subdir)PEFT JitTransformModelTransform)logging)is_global_rank_zero)CommonInferenceParams)InferenceRequestllm) namespacemodeldatatrainerlogresumeoptim tokenizermodel_transformreturnc Csft|}|r|jr|jjrt||jjn |jrt||jt||||||||d}||||jS)a Trains a model using the specified data and trainer, with optional tokenizer, source, and export. Args: model (Union[pl.LightningModule, AnyPath]): The model to be trained or a path to the NeMo 2 checkpoint. data (pl.LightningDataModule): The data module containing training data. trainer (Trainer): The trainer instance configured with a MegatronStrategy. log (NeMoLogger): A nemologger instance. resume (Optional[Union[AutoResume, Resume]]): Resume training from a checkpoint. optim (Optional[OptimizerModule]): The optimizer module to be used. If not provided, the default optimizer from the model will be used. tokenizer (Optional[TokenizerType]): Tokenizer setting to be applied. Can be 'data' or 'model' or an instance of TokenizerSpec. export (Optional[str]): Filename to save the exported checkpoint after training. model_transform (Optional[Union[Callable[[nn.Module], nn.Module], PEFT]]): A model transform to be applied. Returns ------- Path: The directory path where training artifacts are saved. Examples -------- >>> from nemo.collections import llm >>> from nemo import lightning as nl >>> model = llm.MistralModel() >>> data = llm.SquadDataModule(seq_length=4096, global_batch_size=16, micro_batch_size=2) >>> precision = nl.MegatronMixedPrecision(precision="bf16-mixed") >>> trainer = nl.Trainer(strategy=nl.MegatronStrategy(tensor_model_parallel_size=2), plugins=precision) >>> llm.train(model, data, trainer, tokenizer="data") PosixPath('/path/to/log_dir') r-r.r/r0r1r2r3r4)_load_model_from_pathrestore_configpathrresume_from_path_setupfitexp_dir r-r.r/r0r1r2r3r4 app_stater@L/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/llm/api.pytrainHs$+  rBc Cs2t|}t||||||dt||||||ddS)a Pretrains a model using the specified data and trainer, with optional logging, resuming, and optimization. This function is a wrapper around the `train` function, specifically configured for pretraining tasks. Note, by default it will use the tokenizer from the model. Args: model (Union[pl.LightningModule, AnyPath]): The model to be pretrained or a path to the NeMo 2 checkpoint. data (pl.LightningDataModule): The data module containing pretraining data. trainer (Trainer): The trainer instance configured with a MegatronStrategy. log (NeMoLogger): A nemologger instance. resume (Optional[AutoResume]): Resume training from a checkpoint. optim (Optional[OptimizerModule]): The optimizer module to be used. If not provided, the default optimizer from the model will be used. Returns: Path: The directory path where pretraining artifacts are saved. Examples: >>> from nemo.collections import llm >>> from nemo import lightning as nl >>> model = llm.MistralModel() >>> data = llm.PretrainingDataModule(paths=[...], seq_length=4096, global_batch_size=16, micro_batch_size=2) >>> precision = nl.MegatronMixedPrecision(precision="bf16-mixed") >>> trainer = nl.Trainer(strategy=nl.MegatronStrategy(tensor_model_parallel_size=2), plugins=precision) >>> llm.pretrain(model, data, trainer) PosixPath('/path/to/log_dir') )r0r1r2r.)r-r.r/r0r1r2r3r7_validate_configrB)r-r.r/r0r1r2r@r@rApretrains%rEpeftc Cs6t|}t|||||||dt||||||||dS)a Finetunes a model using the specified data and trainer, with optional logging, resuming, and PEFT. Note, by default it will use the tokenizer from the model. Args: model (Union[pl.LightningModule, AnyPath]): The model to be finetuned. data (pl.LightningDataModule): The data module containing finetuning data. trainer (Trainer): The trainer instance configured with a MegatronStrategy. log (NeMoLogger): A nemologger instance. resume (Optional[AutoResume]): Resume training from a checkpoint. optim (Optional[OptimizerModule]): The optimizer module to be used. If not provided, the default optimizer from the model will be used. peft (Optional[PEFT]): A PEFT (Parameter-Efficient Fine-Tuning) configuration to be applied. tokenizer (Optional[TokenizerType]): Tokenizer setting to be applied. Can be 'data' or 'model' or an instance of TokenizerSpec. If 'data' uses the data loader's tokenizer instead of the tokenizer from the model checkpoint, which is useful for expanding vocabulary or adding special tokens (such as chat template tokens). Returns: Path: The directory path where finetuning artifacts are saved. Examples: >>> from nemo.collections import llm >>> from nemo import lightning as nl >>> model = llm.MistralModel() >>> data = llm.SquadDataModule(seq_length=4096, global_batch_size=16, micro_batch_size=2) >>> precision = nl.MegatronMixedPrecision(precision="bf16-mixed") >>> trainer = nl.Trainer(strategy=nl.MegatronStrategy(tensor_model_parallel_size=2), plugins=precision) >>> llm.finetune(model, data, trainer, peft=llm.peft.LoRA()]) PosixPath('/path/to/log_dir') )r0r1r2r4r6rC)r-r.r/r0r1r2rFr3r@r@rAfinetunes+rGc Cs*t||||||||d}||||jS)a Validates a model using the specified data and trainer, with optional logging, resuming, and model transformations. Args: model (pl.LightningModule): The model to be validated. data (pl.LightningDataModule): The data module containing validation data. trainer (Trainer): The trainer instance configured with a MegatronStrategy. log (NeMoLogger): A nemologger instance. resume (Optional[AutoResume]): Resume from a checkpoint for validation. optim (Optional[OptimizerModule]): The optimizer module to be used. If not provided, the default optimizer from the model will be used. tokenizer (Optional[TokenizerType]): Tokenizer setting to be applied. Can be 'data' or 'model' or an instance of TokenizerSpec. model_transform (Optional[Union[Callable[[nn.Module], nn.Module], PEFT]]): A model transform to be applied. Returns: Path: The directory path where validation artifacts are saved. Examples: >>> from nemo.collections import llm >>> from nemo import lightning as nl >>> model = llm.MistralModel() >>> data = llm.SquadDataModule(seq_length=4096, global_batch_size=16, micro_batch_size=2) >>> precision = nl.MegatronMixedPrecision(precision="bf16-mixed") >>> trainer = nl.Trainer(strategy=nl.MegatronStrategy(tensor_model_parallel_size=2), plugins=precision) >>> llm.validate(model, data, trainer, tokenizer="data") PosixPath('/path/to/log_dir') r6)r;validater=r>r@r@rArHs' rHprune)namer,Fnemo_checkpoint save_pathpruning_configdevices num_nodestp_sizepp_sizenum_train_samplestokenizer_path legacy_ckptc Cs|dur|j|jksJd||j} n|} t|||||d| | ddd| | | dddid \} } t| ||| t| |t}|d |d |S) a Prunes a model using the specified data and trainer. Currently only supports GPT models. Args: nemo_checkpoint (str): The path to the NeMo checkpoint to be pruned. save_path (str): The path to save the pruned NeMo checkpoint. pruning_config (PruningConfig): The pruning configuration. devices (int): The number of devices to use for pruning. num_nodes (int): The number of nodes to use for pruning. tp_size (int): The tensor parallel size. pp_size (int): The pipeline parallel size. num_train_samples (int): Number of training samples for importance estimation using forward pass. data (pl.LightningDataModule): The data module for forward pass. Required if not dropping layers. tokenizer_path (str): Path to the tokenizer if not using model's tokenizer. legacy_ckpt (bool): If True, allow loading ckpt saved with older version of TE. Use for cases like missing state dict keys ending with `_extra_state`. Returns: str: The path to the pruned NeMo checkpoint. Examples: >>> from nemo.collections import llm >>> from nemo.collections.llm.modelopt.prune import PruningConfig >>> data = llm.PretrainingDataModule( paths=["1.0", "path/to/tokenized/data"], seq_length=256, global_batch_size=1, micro_batch_size=1, ) >>> llm.prune( nemo_checkpoint="path/to/llama3.1-8b", save_path="path/to/pruned_llama_model", pruning_config=PruningConfig(target_ffn_hidden_size=9216, target_hidden_size=3072), data=data ) Nz3Global batch size must be equal to micro batch sizeTF)sequence_parallelreplace_progress_bar) max_stepslimit_val_batchesval_check_intervalrW) model_pathtensor_model_parallel_sizepipeline_model_parallel_sizerPrQinference_onlyrUrVstrategy_kwargstrainer_kwargsmodel_config_overridesu8[green]✓ Pruning succeded, pruned checkpoint saved to [/green])global_batch_sizemicro_batch_sizerrrr print)rMrNrOrPrQrRrSrTr.rUrVstepsr-r/consoler@r@rArI/s,3    distillstudent_model_pathteacher_model_pathdistillation_config_pathc Cstjt|dd} tjt|dd} t| tsJdt| ts$Jd|dur>> from nemo.collections import llm >>> from nemo import lightning as nl >>> student = "/path/to/student/nemo/ckpt" # <-- change me >>> teacher = "/path/to/teacher/nemo/ckpt" # <-- change me >>> data = llm.SquadDataModule(seq_length=4096, global_batch_size=16, micro_batch_size=2) >>> precision = nl.MegatronMixedPrecision(precision="bf16-mixed") >>> trainer = nl.Trainer(strategy=nl.MegatronStrategy(tensor_model_parallel_size=2), plugins=precision) >>> llm.distill(student, teacher, data, trainer, tokenizer="model") PosixPath('/path/to/log_dir') r-subpathziszgenerate..cSsg|]}|dqS)inputr@r$r@r@rAr'jsz1Either prompts or input_dataset must be provided.)r9r/rrrc3s|] }t|VqdSN)rtokenize)r%r)mcore_tokenizerr@rA xszgenerate..rK num_replicasrank)r-r3rrrrrrcsg|] }r |jn|qSr@)generated_text)r%r)r r@rAr')groupcSsg|] }|D]}|qqSr@r@)r%sublistresultr@r@rAr'r1wr label prediction)r(r7 zPredictions written to )#nemo.collections.llmr!rqr test_pathopen readlinesrsetup_model_and_tokenizernum_tokens_to_generatemaxinference_wrapper_configrinference_contextmax_sequence_lengthstrategyrdistributed_sampler_kwargsrminrr r get_data_parallel_groupr(ziprdictpopr/writer"dumpsr'r)r9r/rrrrrrrrrr rrr! input_pathfdatasetinputsinference_wrapped_modelmax_seq_lengthinputs_on_this_dp_rankdp_sizedp_rank chunk_size start_idxend_idxresults_on_this_dp_rankgathered_resultsr&predr@)r+r rAr s| X         cCs|dkr t|d|jdS|dkrt|d|jdSz!ddlm}t||r5t|d|t|d|WdStd|tyFtdw)Nr.r3r-r) TokenizerSpecz2Expected TokenizerSpec or 'data' or 'model', got: zTokenizerSpec is not available) _set_with_ior31nemo.collections.common.tokenizers.tokenizer_specr[rqrr)r-r.r3r[r@r@rA_use_tokenizers    r^c Cs2t||pt}|rt|tr|jrtdd|j_|j|t |ddt t ddd} |dur6||||r=| ||rEt ||||rMt |d|t |ddrptdd|jDspt|tri|j|n|jtd} t|jD]\} } t| tr| dusJ|j| } qw| dur|j| | S) Nz8Disabling try_restore_best_ckpt restoration for adaptersFresume_if_existsrt)r_ task_configr4css|]}t|tVqdSr))rqr&)r%cbr@r@rAr,sz_setup..)r rrqr$ckptr'rtry_restore_best_ckptsetuprrrBconnectr^r\any callbacksr&append enumerater%rI) r-r.r/r0r1r2r3r4_logr?jit_cbirar@r@rAr;s@              r;cCs@t|||t|drt|drt|j|t|jdSdSdS)Nrt)setattrhasattrrtr)objattrvaluer@r@rAr\s r\cCst|drgsz'_build_directory_tree..)keyz [bold cyan]z /[/bold cyan])z.jsonz.jsonlz[yellow]z [/yellow])z.ptz.binz.ckptz.nemoz [magenta]z [/magenta])z.pyz.shz[green]rcz[white]z[/white]) rich.treerrrJsortediterdirraddr suffix)r9treerritemsitembranchr@r@rAr ^s"     r cCs t|trtjt|dd}|Sr)rqAnyPathr!rpr#)r-r@r@rAr7{s r7)NNNNN)NNN)NNNNr-)rKrKrKrKrLNNF)NNNNNN) rKrKNNNNNNNFF)NrNrrKrrrNTrrrKrKrKrKrKrKrKNrrrTTTF)NF)NN)irr"multiprocessingrzcopyrpathlibrtypingrrrrrlightning.pytorchpytorchplnemo_runrr megatron.corer rich.consoler torch.distributedr typing_extensionsr nemo.lightning lightningrxr9r r#nemo.collections.llm.evaluation.apirrrrnemo.collections.llm.modeloptrrrrrrrrrrrrrr r!nemo.lightning.baser"nemo.lightning.ckpt_utilsr# nemo.lightning.pytorch.callbacksr$r%r& nemo.utilsr'nemo.utils.get_rankr(/megatron.core.inference.common_inference_paramsr))megatron.core.inference.inference_requestr* TokenizerTyperrcli entrypointLightningModuleLightningDataModuleConfigrBrErGrHintboolrIrirwrrHrr ModelConnectorrrbfloat16listrrr^r;r\rDr r7r@r@r@rAs         ,          C  2  8 6    N   P      o       T Q T  J        " 2  f