o }oi>@sddlmZddlmZddlmZmZmZmZddl Z ddl m Z ddl m Z ddl mZmZmZddlmZdd lmZmZmZdd lmZdd lmZerhdd lmZdd lmZddlm Z ddl!m"Z"eGdddeZeGdddeZ#eGdddeZ$eGdddeZ%eGdddeZ&eGdddeZ'GdddeZ(e)e(dGdd d ej*d!e(fZ+e,e(dGd"d#d#ej*e(d!fZ-gd$Z.dS)%) dataclass)Path) TYPE_CHECKING AnnotatedCallableOptionalN)nn) squared_relu) GPTConfigGPTModeltorch_dtype_from_mcore_config)Config)OptimizerModuleioteardown) TransformFns) dtype_from_hfNemotronConfigNemotronForCausalLM AutoTokenizer) TokenizerSpecc@s"eZdZUdZdZeed<eZe ed<dZ eed<dZ e ed<dZ e ed <d Zeed <d Zeed <d Zeed<dZe ed<dZe ed<dZe ed<dZe ed<dZe ed<dZe ed<dZeed<dZeed<dZeed<dZeed<dZeed<d Zeeed!<d"Z eeed#<d$Z!eed%<d&S)'rzQ Configuration class for the Nemotron Config, inheriting from GPTConfig. LayerNorm normalizationactivation_funcropeposition_embedding_typeF#share_embeddings_and_output_weightsadd_bias_linearghidden_dropoutattention_dropoutg?rotary_percentTmasked_softmax_fusionpersist_layer_normbias_dropout_add_fusionlayernorm_zero_centered_gammacross_entropy_loss_fusionapply_rope_fusion num_layers seq_length hidden_size$ffn_hidden_sizenum_attention_headsnum_query_groups kv_channelsS!uq?init_method_stdN)"__name__ __module__ __qualname____doc__rstr__annotations__r rrrrboolr r!floatr"r#r$r%r&r'r(r)r+intr-r/r1r3r5rr7r9rCrC[/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/llm/gpt/model/nemotron.pyr%s0                    rc@sveZdZUdZdZeed<dZeed<dZeed<dZ eed <d Z eed <d Z eed <dZ e eed<dZeed<dS)Nemotron3Config4BzZ Configuration class for the Nemotron3 4B Config, inheriting from NemotronConfig. r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9N)r:r;r<r=r+rBr?r-r/r1r3r5r7rr9rArCrCrCrDrEGs       rEc@szeZdZUdZdZeed<dZeed<dZeed<dZ eed<dZ eed <d Z e eed <d Z e eed <d Zeed<d S)Nemotron3Config8BzZ Configuration class for the Nemotron3 8B Config, inheriting from NemotronConfig. r*r+r,r-r/i@r1r3Nr5r7g{Gz?r9r:r;r<r=r+rBr?r-r/r1r3r5rr7r9rArCrCrCrDrFW      rFc@szeZdZUdZdZeed<dZeed<dZeed<dZ eed <d Z eed <d Z e eed <d Z e eed<dZeed<d S)Nemotron3Config22Bz[ Configuration class for the Nemotron3 22B Config, inheriting from NemotronConfig. (r+r,r-r/`r10r3Nr5r7gMb?r9rGrCrCrCrDrIgrHrIc@szeZdZUdZdZeed<dZeed<dZeed<dZ eed <d Z eed <d Z e eed <dZ e eed<dZeed<dS)Nemotron4Config15Bz[ Configuration class for the Nemotron4 15B Config, inheriting from NemotronConfig. r*r+r,r-rKr/rLr1rMr3r4r5Nr7r8r9rGrCrCrCrDrNwrHrNc@szeZdZUdZdZeed<dZeed<dZeed<dZ eed <dZ eed <d Z e eed <d Z e eed<dZeed<d S)Nemotron4Config340Bz\ Configuration class for the Nemotron4 340B Config, inheriting from NemotronConfig. `r+r,r-iHr/i r1r3r4r5Nr7g o_y?r9rGrCrCrCrDrOrHrOc sfeZdZdZ    d deeeeefdeededdee e j ge j fffdd Z Z S) NemotronModelz Nemotron model implementation based on the GPT model architecture. This class provides a high-level interface for Nemotron models, implementing the specific architecture and settings needed for Nemotron models. Nconfigoptim tokenizerrmodel_transformcstj|pt|||ddS)N)rSrTrU)super__init__r)selfrRrSrTrU __class__rCrDrWszNemotronModel.__init__)NNNN)r:r;r<r=rrrr rrrModulerW __classcell__rCrCrYrDrQs rQhfc@sXeZdZdZdefddZdedefddZdd Ze dd d Z e de fd dZ dS)HFNemotronImporterz Importer for converting Hugging Face Nemotron models to NeMo format. This class handles the conversion of Hugging Face's NemotronForCausalLM models to NeMo's Nemotron format, including weight mapping and configuration translation. returncCst|j|jdS)z Initialize a NeMo NemotronModel instance. Returns: NemotronModel: Initialized NeMo Nemotron model with the appropriate configuration and tokenizer. )rT)rQrRrTrXrCrCrDinitszHFNemotronImporter.init output_pathcCspddlm}td|jt|dd}|}||}||||||td|t ||~~|S)z Apply the conversion from HF to NeMo format. Args: output_path: Path where the converted model will be saved Returns: Path: Path to the saved NeMo model rrz!Start converting Nemotron model..auto torch_dtypez1Converted Nemotron model to Nemo, model saved to ) transformersrprintfrom_pretrainedr>ra nemo_setup convert_state nemo_saver)rXrbrsourcetargettrainerrCrCrDapplys     zHFNemotronImporter.applyc CsBddddddddd d d d }tjd dtjdg}tj||||dS)aS Convert state dict from HF format to NeMo format. Maps the weights from the HF model to the NeMo model according to the appropriate mapping scheme. Args: source: Source HF model target: Target NeMo model Returns: The result of applying the transforms embedding.word_embeddings.weight2decoder.layers.*.self_attention.linear_proj.weight&decoder.layers.*.mlp.linear_fc1.weight&decoder.layers.*.mlp.linear_fc2.weight)rXrrCrCrDrTs zHFNemotronImporter.tokenizercCsddlm}|t|}dd}tdid|jd|jd|jd|jd |jd |j d |j d |j d |j d|j d||jdddt|tjkdt|tjkdt|dt|dd}|S)z Create a NeMo NemotronConfig from the HF model config. Translates the HF configuration parameters to the equivalent NeMo configuration. Returns: NemotronConfig: NeMo configuration for Nemotron models rrcSs(d}||dkr|d}||dks|S)Nr6rrC) vocab_sizebaserCrCrDmake_vocab_size_divisible_bys   z?HFNemotronImporter.config..make_vocab_size_divisible_byr+r/r1r3r9r-layernorm_epsilonr5 rotary_baser#rrFfp16bf16 params_dtyper7head_dimNrC)rfrrhr>num_hidden_layersr/intermediate_sizer3initializer_rangemax_position_embeddingsnorm_epsnum_key_value_heads rope_thetapartial_rotary_factorrrtorchfloat16bfloat16getattr)rXHFNemotronConfigrlroutputrCrCrDrR sJ       zHFNemotronImporter.configN)r_r) r:r;r<r=rQrarrorjpropertyrTrrRrCrCrCrDr^s )  r^rc@sTeZdZdZejfdddZdedefddZd d Z e d d Z e dddZ dS)HFNemotronExporterz Exporter for converting NeMo NemotronModel to Hugging Face format. This class handles the conversion of NeMo's NemotronModel to Hugging Face's NemotronForCausalLM format, including weight mapping and configuration translation. r_rcCsRddlm}ddlm}||j|j|dWdS1s"wYdS)z Initialize a HF NemotronForCausalLM instance. Args: dtype: Data type for model parameters Returns: NemotronForCausalLM: Initialized HF Nemotron model r)AutoModelForCausalLM)no_init_weightsrdN)rfrtransformers.modeling_utilsr from_configrR)rXdtyperrrCrCrDra?s $zHFNemotronExporter.initrbcCsP|t|\}}|t|j}|||}|}|||j||S)N) nemo_loadr>rar rRrjcpusave_pretrainedrT)rXrbrl_rmrCrCrDroOs   zHFNemotronExporter.applyc Cs^ddddddddd d }tjd d tjd tjddtjd tjddtjd g}tj||||dS)ac Convert state dict from NeMo format to HF format. Maps the weights from the NeMo model to the HF model according to the appropriate mapping scheme. Args: source: Source NeMo model target: Target HF model Returns: The target model with weights transferred from source r|r}r~rrrrrr) rqrrrsrtrurvrwrxryrrrrzrrpr{r)rrr split_qkv prune_paddingrrrCrCrDrjZs6  z HFNemotronExporter.convert_statecCstt|jjjS)z Get the tokenizer from the NeMo model. Returns: TokenizerSpec: Tokenizer from the NeMo model )r load_contextr>modelrTr`rCrCrDrTszHFNemotronExporter.tokenizerrcCsrddlm}tjt|dd}||j|j|j|j|j dur |j n|j|j|j |j |j |j |j|j|j|jjd S)zCreate a HF NemotronConfig from the NeMo model config. Translates the NeMo configuration parameters to the equivalent HF configuration. Returns: HFNemotronConfig: HF configuration for Nemotron models rrz model.config)subpathN) rr/rr3rtie_word_embeddingsrrrrrrr)rfrrrr>r+r/r1r3r7rr-r9rr5rr#rTr)rXrrlrCrCrDrRs&   zHFNemotronExporter.configN)r_r)r_r) r:r;r<r=rrrarrorjrrTrRrCrCrCrDr6s 1  r)rrErFrIrNrOrQ)/ dataclassesrpathlibrtypingrrrrrr"nemo.collections.llm.fn.activationr #nemo.collections.llm.gpt.model.baser r r nemo.collections.llm.utilsr nemo.lightningrrrnemo.lightning.io.staternemo.lightning.pytorch.utilsrrfrrrrr1nemo.collections.common.tokenizers.tokenizer_specrrErFrIrNrOrQmodel_importerModelConnectorr^model_exporterr__all__rCrCrCrDsH           !