o wi@sddlmZddlmZddlmZmZmZmZddl m Z ddl Z ddl Z ddlmmZddlmZmZddlmZddlmZddlmZdd lmZdd lmZdd lm Z dd l!m"Z"dd l mZddl#m$Z$ddl%m&Z&ddl'm(Z(m)Z)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5ddl6m7Z7gdZ8d(ddZ9dee:e j;ffddZZ?Gd"d#d#eZ@Gd$d%d%e jAe,j>e,jBe&jCZDed&d'ZEd%d!gZFdS)))contextmanager) dataclass)CallableDictListOptionalN)InferenceParamsdist_checkpointingparallel_state)tensor_parallel) ModelType) LLaVAModel)OptimizerConfig)PackedSeqParams)TransformerConfig)nn) TokenizerSpec)fn) AudioToken ImageTokenMultiModalSampleConfig)io)ckpt_to_weights_subdir)$MaskedTokenLossReductionWithLossMask)MegatronOptimizerModuleOptimizerModule)logging)AppState) num_layers hidden_sizenum_attention_headsnum_query_groupsffn_hidden_size kv_channelshidden_dropoutattention_dropoutfp32_residual_connection(apply_residual_connection_post_layernormlayernorm_epsilonlayernorm_zero_centered_gammaadd_bias_linear add_qkv_biasgated_linear_unitactivation_funcactivation_func_fp8_input_storenum_moe_expertsrotary_interleaved window_size normalization qk_layernorm test_modecalculate_per_token_loss seq_lengthFcCsr|dur7t|jddd}tjd|t|dddd|sdd ini}d d |d D}|j||d dSdS)z Restores model weights from a checkpoint. Args: model: The model to restore weights for. checkpoint_path: Path to the checkpoint. strict: Whether to restore weights even if they are not the same. Nmodule.)prefix state_dictF) is_savingsharded_state_dictcheckpoint_dirvalidate_access_integritystrictlog_allcSsi|] \}}|d|qS)r8) removeprefix).0kvrG]/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/avlm/model/base.py Ysz)restore_model_weights..r;)rArG)dictr>r loadritemsload_state_dict)modelcheckpoint_pathrAr>loaded_state_dictrGrGrHrestore_model_weightsHs  rQreturncs ddlm}t|}t|trt|dkr|d}n|}td|r-d| r6d| dd}fd d | D}|durdd D]}t ||d}|durct |||jd d qM||d<tdkrd}d|vr|ddur|d}||d<|S)zAVLM Data Steprr )tokensattention_maskimagesnum_image_tiles image_sizesaudios audio_lengths) position_ids)labels loss_maskpacked_seq_paramsNcs2i|]\}}||vr|dur|jddndqS)NT non_blocking)cuda)rDkeyval required_keysrGrHrIs z"avlm_data_step..) cu_seqlens_q cu_seqlens_kvcu_seqlens_q_paddedcu_seqlens_kv_paddedTr_r]num_valid_tokens_in_ub) megatron.corer next isinstancetuplelensetupdateis_pipeline_first_stageis_pipeline_last_stagegetrLgetattrsetattrrapsget_context_parallel_world_sizesum)dataloader_iterr batch_batchr^attrvaluerkrGrdrHavlm_data_step]s@         rc Csh|d|d|dd|dd|dd|d|dd|d |d d|d dd }|d i|S)zAVLM Forward SteprTr[r]NrUr\rVrWrYrZr^) input_idsr[r]rUr\rVrWrYrZr^rG)ru)rNr| forward_argsrGrGrHavlm_forward_steps      rc@sxeZdZUdZdZeeed<dZeeed<dZ eeed<dZ eeed<dZ eeed<dZ e ed <d Zeed <d Zeed <d Zeed<dZeed<dZeed<dZeed<dZeed<dZeeed<dZeeed<dZeeed<dZeeed<dZeeed<dZe ed<dZe ed<dZe ed<dZe ed<dZ e ed <e!Z"e#ed!<e$Z%e#ed"<d#d$Z&d)d'd(Z'dS)* AVLMConfigzAVLM Model Base ConfigNlanguage_transformer_configvision_transformer_configvision_projection_configaudio_transformer_configaudio_projection_configTdrop_vision_class_tokenvision_feature_layeraudio_feature_layerr$encoder_pipeline_model_parallel_sizerj"encoder_tensor_model_parallel_sizerr!ir7language_model_from_pretrainedvision_model_from_pretrained!vision_projection_from_pretrainedaudio_model_from_pretrained audio_projection_from_pretrainedFfreeze_language_modelfreeze_vision_modelfreeze_vision_projectionfreeze_audio_modelfreeze_audio_projectionforward_step_fn data_step_fncCs>|jdurtD] }t||t|j|q|jdusJddS)NFz$AVLM does not return normalized loss)rMODEL_CONFIG_ATTRrwrvr6)selfr~rGrGrH __post_init__s zAVLMConfig.__post_init__rRMCoreAVLMModelc Csd|j_|j|j_|j|j_|jdur|j|j_|j|j_|jdur,|j|j_|j|j_|j|j_|j |j_ |j dks?Jdt ||t t t t pUt |j k|jd}|S)NFrzPAVLM `encoder_pipeline_model_parallel_size` has bug for now. Fix will come soon.)config tokenizer pre_process post_process add_encoder add_decoderr)r#scatter_embedding_sequence_paralleltensor_model_parallel_sizesequence_parallelrrrrpipeline_model_parallel_sizecontext_parallel_sizerrrxrsrt get_pipeline_model_parallel_rankr)rrrNrGrGrHconfigure_models2             zAVLMConfig.configure_model)rRr)(__name__ __module__ __qualname____doc__rrr__annotations__rrrrrboolrintrrrrr!r7rstrrrrrrrrrrrrrrrrrrGrGrGrHrs:                rcsLeZdZdZ      d1dededededed ed ed dffd d ZddZddZ ddZ ddddde j dde j dddf dejdejdeejdeejdeejdeejdeeedeedeejdeeedeedeed eed!eed ejfd"d#Zd$ed%ed&ed'ed(ef d)d*Zd2d+d,Zd-d.Zd/d0ZZS)3rzAVLM Model Base Model ClassNTrrrrrrrrRcsntt|j|d|j}|j} |j} |j} |j} ||_||_ ||_ ||_ ||_ d|_ d|_d|_d|_d|_d|_|j|_|j|_|j|_|j|_d|_|j ry|j|||d|_|jj|_|jj|_|jdk|_ t!|j|j"t#$d|j"n|j"durt%j&t'id|j"dd|j r| dur| |_| |_||_(t!|j|j)t#$d|j)| durt*} |j+r|j+,d rt-| d| |_t!|j|j+t#$d |j+Wdn1swYn| |_t!|j|j+t#$d |j+| |_|j.|j/|j0|j1|j2|j3d t4j5|_6t7| d |_8| j9|_:|r3| j;r5|j:| j<8_:dSdSdS) NrF)rrrrjz%Restored language model weights from r:r=z#Restored vision model weights from z.nemoz"Restored audio model weights from )rrrrr image_size)=superMCoreLLaVAModel__init__rrrrrrrrrrencoder_hidden_state vision_modelvision_projection audio_modelaudio_projectionlanguage_modelrsequence_parallel_lmtp_comm_overlaptp_comm_overlap_lmrcontext_parallel_lmrtensor_model_parallel_size_lm#share_embeddings_and_output_weightsrmax_sequence_length_language_max_sequence_lengthr_language_is_pipeline_parallelrQrrinfor rKrJ_drop_vision_class_tokenrrrendswithtemporary_model_parallel_sizefreezerrrrrr encoder_or_decoder model_typehasattrvision_model_from_hfnum_image_embeddings_per_tile _img_seq_lenadd_class_tokenclass_token_len)rrrrrrrrrrrrr app_state __class__rGrHrs             zMCoreAVLMModel.__init__c Csh|jsJd|js |S|r|S||k} ||k} |} |dur&t| | f|} |dur2t| | f|} | S)zk Combine image_embeddings, audio_embeddings, and language_embeddings into a single tensor. z>input text preprocessing is only needed for the language modelN)rrtorch index_put) rrimage_embeddingsaudio_embeddingslanguage_embeddingsimage_token_indexaudio_token_indexuse_inference_kv_cacher^image_token_maskaudio_token_maskcombined_embeddingsrGrGrHcombine_embeddingscsz!MCoreAVLMModel.combine_embeddingsc Cs|jr|j}n?|jdks|jrA|jr|jr|j}n-||\}}|dur*|jd}n |dur4|jd}nd}|d|||}n|||fS|jdkrN|j}|duru|jd|kru||jd}tj|d|ft j d}tj|d|fdd}|dur|jd|kr||jd}ddd|f} t j j j|| ddd}|duo|jdk} | r|jd|kr|jd|jd } ||jd } | | ksJd ||jd<||jd<t| |j|_t| |j|_|||fS) z Pad the sequence (labels, loss_mask, combined_embeddings, packed_seq_params) to the language model's max sequence length. combined_embeddings's shape is [batch_size, seq_len, attention head, embed_dim] rjNr)rconstant)padmoderthdrzW`language_max_sequence_length` needs to increase for sequence packing to work properly.)rrrrr_get_shard_factorshapeFrrignore_place_holderrr functional contiguous qkv_formatrfrhrimax max_seqlen_q max_seqlen_kv) rrr\r]r^ max_seq_len shard_factor_pad_size pad_shapepacked_sequence last_seqlenlast_seqlen_paddedrGrGrH pad_sequencesT           zMCoreAVLMModel.pad_sequencec CsN|duo |jd|jk}|r$|ddd|jf}|ddd|jf}|dur;|jd|jkr;|ddd|jf}|duoC|jdk}|r|jd|jkr|jd|j}|jd|jd}|jd|jd} || } || 8}|j|jd<|j|jd<|dkr|jd|8<|jd|8<|jd|jdksJd|||fS)z Truncate the sequence (labels, loss_mask, combined_embeddings) to the language model's max sequence length. combined_embeddings's shape is [batch_size, seq_len, attention head, embed_dim] NrjrrrrzLwith packed sequence, the truncation can only truncate on the last sequence.)rrrrhrfrirg) rrr\r]r^truncate_labelsr truncate_lenfinal_seq_len_paddedfinal_seq_len_unpadded final_paddingrGrGrHtruncate_sequences0   z MCoreAVLMModel.truncate_sequencerr[r]rUr\rVrWrrYrZrinference_paramsruntime_gather_outputr^c Cs| duod| jv}|rd}d}n|jr|jdur|duo"|jddk}|s(d}nm|t|jj}|jrL|j |_|j|dd}|d|j j }n |j||j j dd}|j rot |jd d}|dd|dddf}|ddd }||}| dur||k| jd <| jd | jd<nd}|jdur| duo| jddk}|sd}nQ|j| | ddd \}}|d dd}||}| dur|| k| jd <d| jvr| jd| jd 7<n| jd | jd<nd}n |jr|j\}}nd\}}|js||fSd}|jrm|}d||dk<|jj||d}|dd}|dur:|jd}|ddd d|}|dura|jd|jd}}t|d|j|dk}||}|}| |||||| ||}|!||||\}}}|"||||\}}}|dur|dd}|dur|}|j#dks|j$r|j#dkr|dd}|%||||\}}}}|jdd|||| | |d}|dus|dur|S||fS)a7Forward function of the LLaVA model. Args: images (torch.Tensor): input image of shape [num_tiles, img_h, img_w]. num_tiles means the number of image tiles in this batch. input_ids (torch.Tensor): input text ids [batch, text_seq_len]. position_ids (torch.Tensor): input text position ids [batch, text_seq_len]. attention_mask (torch.Tensor): Attention mask for the language model [batch, 1, combined_seq_len, combined_seq_len]. labels (torch.Tensor): Optional target text labels [batch, combined_seq_len]. loss_mask (torch.Tensor): Text loss mask [batch, text_seq_len]. inference_params (InferenceParams): Inference-time parameters including KV cache. num_image_tiles (list of int): Number of tiles per image. Default 1 tile per image. image_token_index (int): ID for input images. Default None means `image_token_index` arg in the constructor will be used. runtime_gather_output (bool): Gather output at runtime. Default None means `parallel_output` arg in the constructor will be used. packed_seq_params (PackedSeqParams): Dict with padded token information. Required for using SP/CP with padding mask type. Returns: output (torch.Tensor): Loss of shape [b, s] if labels are provided, otherwise logits of shape [b, s, vocab_size]. loss_mask (torch.Tensor): Loss mask expanded to combined sequence length. Shape [b, s]. Nmedia_tokens_countrT)output_hidden_statesrrj)num_unused_layersrimage_tokens_count) input_signalinput_signal_lengthprocessed_signalprocessed_signal_lengthaudio_tokens_count)NN)rr[)rr[rU decoder_inputr\rrr^)&key_value_memory_dictrrrtorm parametersdtyperevalrrrrvpermuterrrzrrrrrcloner embedding transposereshaperarange unsqueezedevicerrrrr!_process_embedding_token_parallel)rrr[r]rUr\rVrWrrYrZrrrr^rrr has_imagesr has_audiosaudio_embedding_lensrinput_ids_textr embed_dimaudio_embeddings_max_seq_lenrnonpadded_masknonpadded_audio_embeddingsoutputrGrGrHforwards.                   zMCoreAVLMModel.forwardrrrrrc Csg}|r|jdur||j|r|jdur||j|r)|jdur)||j|r6|jdur6||j|rC|jdurC||j|D] }|D]}d|_qKqEdS)aFreeze model modules. Make specific modules non-trainable by setting requires_grad to False. Args: freeze_language_model (bool): Freeze the language model module. freeze_vision_model (bool): Freeze the vision model module. freeze_vision_projection (bool): Freeze the vision projection module. freeze_audio_model (bool): Freeze the audio model module. freeze_audio_projection (bool): Freeze the audio projection module. NF)rappendrrrrr requires_grad) rrrrrrmodulesmoduleparamrGrGrHrs       zMCoreAVLMModel.freezecCst|ts|g}t|dksJd|jr"|jr"|j|ddS|jr/|j|ddS|jr9|d|_dS|j |ddS)zSet model chunk input tensor.rjz.input_tensor should only be length 1 for llavarN) rnlistrprrrset_input_tensorrrr)r input_tensorrGrGrHr/s  zMCoreAVLMModel.set_input_tensorcCs|dur |jdkr d}nd}|jdkr"|jr"|j|jd}||fS|jdkr0|jd}||fS|jr:|j}||fSd}d}||fS)z&Get shard factor of sequence dimensionNrrjrr )rrrr)rr^seq_dimrrGrGrHrs z MCoreAVLMModel._get_shard_factorc Cs|js |js ||||fS|jr`|jdkr"|jr"|j|jd}d}n|jdkr/|jd}d}n|jr7|j}d}|j||dksNJd|d|jd||jr`|jr`|j||jks`Jd|jdkrt}|jrr| d|i|jr}| ||d |d us|j d krdd l m }||}n?zdd l } Wnty} ztd | d } ~ wwt} t} |D]\} }| |j|d| | }|d||| <q|jr|d}|dd}|jr|d}|d}|jr|jrt|}||||fS)z7Processes the input data for model parallelism support.rjr rz'Sequence length should be divisible by z2 for Sequence/Context parallelism z with dim zhTP Comm overlap either requires Vision+Text token length == language_max_sequence_lengthr) new_labels new_loss_maskNsbhd)get_batch_on_this_cp_rankztPlease update Transformer Engine to >= 1.10 to use Context Parallel with THD format datar2r3)rrrrrrrrrJrrrmegatron.training.utilsr5transformer_engine_torchModuleNotFoundErrorrerrorrxryget_context_parallel_rankrLthd_get_partitioned_indicesrhsize index_selectrrr #scatter_to_sequence_parallel_region)rrr2r3r^rr1r|r5texecp_sizecp_rankrbdataindexrGrGrHr&s           z0MCoreAVLMModel._process_embedding_token_parallel)NTTTTTrRN)rrrrrrrrrrrrtoken_idrrTensorrrrrr(rr/rr __classcell__rGrGrrHrs d$D,       l $rcszeZdZdZ   d)dedeededdeeej gej fffdd Z d*d d Z ddddde j ddej dddf d ejdejdeejdeejdeejdeejdeeedeedeejdeeedeedeedeedeed ejfddZd eeejffddZd ejfdd Zd+d ejfd!d"Zd+d ejfd#d$Zed efd%d&Zed efd'd(Z Z!S), AVLMModelz Lightning Wrapper for AVLM ModelNroptimrrmodel_transformcsPt||_||_|pttdddd|_|j|||_d|_ d|_ dS)Ng-C6?T)lruse_distributed_optimizerr) rrrrrrrJconnectrK_training_loss_reduction_validation_loss_reduction)rrrJrrKrrGrHrys   zAVLMModel.__init__rRcCs"t|ds|j|j|_dSdS)Nr,)rrrrr,rrGrGrHrs zAVLMModel.configure_modelrr[r]rUr\rVrWrrYrZrrrr^cCs*|j||||||||| | | | | |d}|S)N)rr[r]rUr\rVrWrrYrZrrrr^)r,)rrr[r]rUr\rVrWrrYrZrrrr^ output_tensorrGrGrHr(s"zAVLMModel.forwardcCs |j|SN)rr)rr{rGrGrH data_steps zAVLMModel.data_stepcCs|j||SrS)rr)rr|rGrGrH forward_stepszAVLMModel.forward_stepcC ||SrSrUrr| batch_idxrGrGrH training_steps zAVLMModel.training_stepcCrVrSrWrXrGrGrHvalidation_steps zAVLMModel.validation_stepcCs|jst|_|jSrS)rOrrQrGrGrHtraining_loss_reductionsz!AVLMModel.training_loss_reductioncCs|js tdd|_|jS)NT)r[)rPrrQrGrGrHvalidation_loss_reductions z#AVLMModel.validation_loss_reduction)NNNrErS)"rrrrrrrrrModulerrrrFrrrGrrrrrr(rrrTrUrZr[propertyrr\r]rHrGrGrrHrIvs        (rIccs*|j}||_z dVW||_dS||_w)zE Context manager to temporarily set the model parallel size. N)model_parallel_size)r temp_valueoriginal_valuerGrGrHrs r)F)G contextlibr dataclassesrtypingrrrrlightning.pytorchpytorchLrtorch.distributedtorch.nn.functionalrrrrlrr r rxr megatron.core.enumsr +megatron.core.models.multimodal.llava_modelrrmegatron.core.optimizerrmegatron.core.packed_seq_paramsr,megatron.core.transformer.transformer_configr1nemo.collections.common.tokenizers.tokenizer_specrnemo.collections.llmr/nemo.collections.multimodal.data.energon.configrrrnemo.lightningrnemo.lightning.io.plr nemo.lightning.megatron_parallelrnemo.lightning.pytorch.optimrr nemo.utilsrnemo.utils.app_staterrrQrrGrrIOMixinrrLightningModuleConnectorMixinFNMixinrIr__all__rGrGrGrHsT                   ?K~d