o }oi,@sddlmZddlmZmZmZmZddlmZ ddl Z ddl Z ddl m Z mZddl mZddl mZddlmZddlmZdd lmZdd lmZdd lmZdd l mZdd lmZddl m!Z!ddl"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-ddl.m/Z/gdZ0d%ddZ1dee2e j3ffddZ4de j3fddZ5eGdddee&j6Z7Gd d!d!eZ8Gd"d#d#e j9e&j6e&j:e!j;Z]sz)restore_model_weights..r>)rDrJ)dictrArloadritemsload_state_dict)modelcheckpoint_pathrDrAloaded_state_dictrJrJrKrestore_model_weightsLs  rTreturncs ddlm}t|}t|trt|dkr|d}n|}td|r-d| r6d| dd}fd d | D}|durdd D]}t ||d}|durct |||jd d qM||d<tdkrd}d|vr|ddur|d}||d<|S)zNeva Data Steprr )tokensattention_maskmedianum_media_tiles) position_ids)labels loss_maskpacked_seq_paramsNcs@i|]\}}||vr|durt|tr|n|jddndqS)NT non_blocking) isinstancelistcuda)rGkeyval required_keysrJrKrLsz"neva_data_step..) cu_seqlens_q cu_seqlens_kvcu_seqlens_q_paddedcu_seqlens_kv_paddedTr_r]num_valid_tokens_in_ub) megatron.corer nextratuplelensetupdateis_pipeline_first_stageis_pipeline_last_stagegetrOgetattrsetattrrcpsget_context_parallel_world_sizesum)dataloader_iterr batch_batchr^attrvaluermrJrfrKneva_data_stepas@        rc Csb|d|d|d|dd|dd|dd|dd|d d|d dd }|d i|S) zNeva Forward SteprYrWr[rXNr]r\rZimage_token_maskr^) images input_idsr[rXr]r\num_image_tilesrr^rJ)rv)rQr} forward_argsrJrJrKneva_forward_steps       rc@s"eZdZUdZdZeeed<dZeeed<dZ eeed<dZ e ed<dZ e ed <d Ze ed <d Ze ed <d Ze ed<dZe ed<dZe ed<dZeeed<dZeeed<dZeeed<dZe ed<dZe ed<dZe ed<eZeed<eZeed<ddZd#dee dd fd!d"Z dS)$ NevaConfigzNeva Model Base ConfigNlanguage_transformer_configvision_transformer_configvision_projection_configTdrop_vision_class_tokenvision_feature_layerr$encoder_pipeline_model_parallel_sizerl"encoder_tensor_model_parallel_sizerrir7language_model_from_pretrainedvision_model_from_pretrained!vision_projection_from_pretrainedFfreeze_language_modelfreeze_vision_modelfreeze_vision_projectionforward_step_fn data_step_fncCs0|jdurtD]}t||t|j|qdSdSN)rMODEL_CONFIG_ATTRrxrw)selfrrJrJrK __post_init__s zNevaConfig.__post_init__vp_stagerUMCoreNevaModelc Csd|j_|j|j_|j|j_|j|j_|j|j_|j|j_|j|j_ |jdkrO|jdks1Jd|j|j_|j|j_|j|j_|j dkrO|j |j_|j |j_t ||t j d|dt j d|dt j d|dt j d|dpqt |jk|j|d}|S)NFzPNEVA `encoder_pipeline_model_parallel_size` has bug for now. Fix will come soon.rrlz&ViT can only live on 1 pipeline stage.)ignore_virtualr)config tokenizer pre_process post_process add_encoder add_decoderrr)r#scatter_embedding_sequence_paralleltensor_model_parallel_sizesequence_parallelrrpipeline_model_parallel_sizecontext_parallel_sizerrrryrtru get_pipeline_model_parallel_rankr)rrrrQrJrJrKconfigure_models:                 zNevaConfig.configure_modelr)!__name__ __module__ __qualname____doc__rrr__annotations__rrrboolrintrrrrr7rstrrrrrrrrrrrrrrJrJrJrKrs,             rcs eZdZdZ       d%dededededed ed ed eed dffd d Zdddddde dddf de j de j dee j dee j dee j dee j dee dee edeedeedee j deed e j fddZd&ddZdd Zd!d"Zd#d$ZZS)'rzNeva Model Base Model ClassNTrrrrrrrrrUc stt|j|d|j} |j} |j} ||_||_||_||_ ||_ ||_ d|_ d|_ d|_d|_| j|_| j|_| j|_| j|_d|_|j rq| j||||d|_|jj|_|jj|_| jdk|_t|j|jt !d|jn|jdurt"j#t$id|jdd|jr| |_ | |_||_%t|j |j&t !d|j&|j'|j(|j)|j*d t+j,|_-t.| d |_/| j0|_1|r| j2r|j1| j38_1dSdSdS) NrF)rrrrrlz%Restored language model weights from r=r@z#Restored vision model weights from )rrr image_size)4superMCoreLLaVAModel__init__rrrrrrrrrencoder_hidden_state vision_modelvision_projectionlanguage_modelrsequence_parallel_lmtp_comm_overlaptp_comm_overlap_lmrcontext_parallel_lmrtensor_model_parallel_size_lmr8rmax_sequence_length_language_max_sequence_lengthr_language_is_pipeline_parallelrTrrinforrNrM_drop_vision_class_tokenrfreezerrrr encoder_or_decoder model_typehasattrvision_model_from_hfnum_image_embeddings_per_tile _img_seq_lenadd_class_tokenclass_token_len) rrrrrrrrrrrr __class__rJrKrsf         zMCoreNevaModel.__init__rr[r]rXrr\inference_paramsrimage_token_indexruntime_gather_outputrr^c  Cs|duod|jv} |duot|dk}| rd}n|jr4|s4t|j}tjg|j|j d ddd}no|jr|r| t|jj}|j r]|j |_|j|dd}|d|jj}n |j||jj dd }|jrt|jd d}|dd|dddf}|ddd }||}|dur|jd|jd|jd<n|j}|js|Sd}|jr|}d||dk<|jj||d }|dd}|durtj|jdtj|j d}nt|t rtj|tj|j d}|!|||||| | ||| \}}}}|j"dks|j#r)|j"dkr| dur| j$d kr|dd}|%|||| \}}}} |jdd||||| | d}|dus@|durB|S||fS)aForward function of the LLaVA model. Args: images (torch.Tensor): input image of shape [num_tiles, img_h, img_w]. num_tiles means the number of image tiles in this batch. input_ids (torch.Tensor): input text ids [batch, text_seq_len]. position_ids (torch.Tensor): input text position ids [batch, text_seq_len]. attention_mask (torch.Tensor): Attention mask for the language model [batch, 1, combined_seq_len, combined_seq_len]. labels (torch.Tensor): Optional target text labels [batch, combined_seq_len]. loss_mask (torch.Tensor): Text loss mask [batch, text_seq_len]. inference_params (InferenceParams): Inference-time parameters including KV cache. num_image_tiles (list of int): Number of tiles per image. Default 1 tile per image. image_token_index (int): ID for input images. Default None means `image_token_index` arg in the constructor will be used. runtime_gather_output (bool): Gather output at runtime. Default None means `parallel_output` arg in the constructor will be used. image_token_mask (torch.Tensor): Tensor indicating the location of image token index in input_ids. packed_seq_params (PackedSeqParams): Dict with padded token information. Required for using SP/CP with padding mask type. Returns: output (torch.Tensor): Loss of shape [b, s] if labels are provided, otherwise logits of shape [b, s, vocab_size]. loss_mask (torch.Tensor): Loss mask expanded to combined sequence length. Shape [b, s]. Nimage_tokens_countrdtypedeviceT)output_hidden_statesrl)num_unused_layersr)rr[thd)rr[rX decoder_inputr\rrr^)&key_value_memory_dictrqrror parameterstorchtensorrrreshapetorevalrrrrwpermute contiguousrshaperrrcloner embedding transposeonesrrarb_preprocess_datarr qkv_format!_process_embedding_token_parallel)rrr[r]rXrr\rrrrrr^use_inference_kv_cache has_imagesimage_embeddings vision_paramrlanguage_embeddingsinput_ids_textcombined_embeddings final_labelsfinal_loss_maskfinal_attention_maskoutputrJrJrKforwardCs-              zMCoreNevaModel.forwardcCst|ts|g}t|dksJd|jr"|jr"|j|ddS|jr/|j|ddS|jr9|d|_dS|j |ddS)zSet model chunk input tensor.rlz.input_tensor should only be length 1 for llavarN) rarbrqrrrset_input_tensorrrr)r input_tensorrJrJrKrs  zMCoreNevaModel.set_input_tensorc 8Cs|jsJd|js|js|||| fS|r|||| fS|j} |j\} } |du}|r;|j|jks;Jd|jd|j| duoC| jdk}t#||k}tj|dd}|j | dd}tj d d |D|j d }|| || }| }|| \}}|d |||}|jr||jkr|j}|jr|jr||jkr|j}nt}t||d ||}|}|r| jd|kr| jd| jd }|| jd }||ksJd|| jd<|| jd<t || j| _t || j| _t||k\}}|}|| d ||<tj|d ddd }|||f}|r5|d } | dk}!| |!} ||!}"|d }#|#dk}$|#|$}#tj| |fdtj|j d}%d|%||f<|dddfd }&d|%tj ||&j d !| d |&"d k<Wdn 1spwYd}'|jr|jd}(tj#| ||(|j$|j d}'|||f|'||f<|%d dd&d|('|'|%<d\})}*|rtj| |ft(|j$|j d})tj| |fd|j$|j d}*||"|#f|)|"| f<|||f|*||f<|d }+t t)|+},||,df|)|,|+f<d|*|%<t|\}-}.|.d }/|/dk}0|-|0}1|/|0}2||1|2f}2d|*|1|2f<|'dur9|r9|'jdd|)jkr4|*jks9JdJd|oC|)jd |jk}3|3r]|)ddd|jf})|*ddd|jf}*|'durz|'*d d'}'|'jd|jkrz|'d|j}'|r| jd|jkr| jd|j}4| jd| jd }5| jd| jd }6|5|6}7|4|78}4|j| jd<|j| jd<|4dkr| jd|48<| j+d|48<| jd| jd ksJd|'|)|*| fS)aPreprocess input data before input to language model. This function is adopted from https://github.com/huggingface/transformers/blob/85817d98fb60977c97e3014196a462b732d2ed1a/src/transformers/models/llava_next/modeling_llava_next.py#L409 for our input data conventions. image_token_index = -200 indicates the image position in the input_ids = [0, 1, -200, 2, 3] and labels = [1, -200, 2, 3, 4], for example. We want to replace the image position (-200) with image_embeddings and return the following: - final_embeddings = [0, 1, image_embeddings, 2, 3], - final_labels = [1, -100, 2, 3, 4] - final_loss_mask = [1, 0, 0, 1, 1] This function handles samples without images (text-only sample). It also handles samples with images that are split into multiples tiles. If pipeline parallelism is not used, then self.pre_process and self.post_process are both True and we update both input embeddings, labels and loss masks (if available). If pipeline parallelism is used, then we do the following - the first language model chunk has self.pre_process = True and self.post_process = False. We update input embeddings. - the middle language model chunk(s) has self.pre_process = False and self.post_process = False. We don't need to update anything. - the last language model chunk has self.pre_process = False and self.post_process = True. We update labels and loss mask. TODO: This function should adjust the attention mask too. Currently, we assume the language model uses a causal mask. Returns: final_embedding (torch.Tensor): image and text embeddings [combined_seq_len, b, h]. final_labels (torch.Tensor): labels for image and text positions [b, combined_seq_len]. final_loss_mask (torch.Tensor): loss mask [b, combined_seq_len]. z>input text preprocessing is only needed for the language modelNzmismatching labels shape z and loss mask shape rr)dimrcSsg|]}|qSrJ)r{)rGxrJrJrK =sz3MCoreNevaModel._preprocess_data..)rrlrzW`language_max_sequence_length` needs to increase for sequence packing to work properly.TrFr)NNz*unexpected shapes after data preprocessingzLwith packed sequence, the truncation can only truncate on the last sequence.),rrrrrrrno_gradr{splittolistrrmax_get_shard_factorrrrrry$get_tensor_model_parallel_world_sizerrhrjrk max_seqlen_q max_seqlen_kvwherercumsumfullrarangerepeat unsqueezezerosrrrrrrqrri)8rrrrr]r\rrrrXr^ img_seq_len batch_size text_seq_len has_labelspacked_sequencernum_images_per_samplenum_image_tiles_batchseq_lens max_seq_len shard_factor_padded_seq_len tp_world_size last_seqlenlast_seqlen_padded batch_indicesnon_image_indicesimage_token_mask_lensnew_position_idstext_position_idslabel_text_position_idsvalid_label_text_position_idslabel_batch_indiceslabel_non_image_indicesvalid_label_non_image_indices images_maskfirst_padding_idxfinal_embedding embed_dimrrlabel_extra_text_position_ids batch_rangebatch_image_indices image_indicesbefore_image_indicesvalidvalid_batch_image_indicesvalid_before_image_indicestruncate_labels truncate_lenfinal_seq_len_paddedfinal_seq_len_unpadded final_paddingrJrJrKrs0            R     &     zMCoreNevaModel._preprocess_datacCs|dur |jdkr d}nd}|jdkr"|jr"|j|jd}||fS|jdkr0|jd}||fS|jr:|j}||fSd}d}||fS)z&Get shard factor of sequence dimensionNrrlrr)rrrr)rr^seq_dimrrJrJrKrs z MCoreNevaModel._get_shard_factorc Cs|js |js ||||fS|jr?||\}}|j||dks-Jd|d|jd||jr?|jr?|j||jks?Jd|jdkrt}|jrQ| d|i|jr\| ||d|d use|j d kr}dd l m }|jrx|d dd|d<||}n?zdd l} Wnty} ztd | d } ~ wwt} t} |D]\} }| |j|d| | }|d||| <q|jr|d}| dd}|jr|d }|d}|jr|jr|d ur|j dkr| dd}t|}||||fS)z7Processes the input data for model parallelism support.rz'Sequence length should be divisible by z2 for Sequence/Context parallelism z with dim zhTP Comm overlap either requires Vision+Text token length == language_max_sequence_lengthrlr) new_labels new_loss_maskNsbhd)get_batch_on_this_cp_rankztPlease update Transformer Engine to >= 1.10 to use Context Parallel with THD format datar=r>r)rrrrrrrrrMrsrmegatron.core.utilsr@rtransformer_engine_torchModuleNotFoundErrorrerrorryrzget_context_parallel_rankrOthd_get_partitioned_indicesrjsize index_selectrr #scatter_to_sequence_parallel_region)rrr=r>r^rr<r}r@texecp_sizecp_rankrddataindexrJrJrKrs|         z0MCoreNevaModel._process_embedding_token_parallel)NTTTTTN)rUN)rrrrrrrrrrrTensorrrrrrrrr __classcell__rJrJrrKrs  R       wrcsleZdZdZ   d(dedeededdeeej gej fffdd Z d)d ee d dfd d Z dddddde dddf dejdejdeejdeejdeejdeejdeedeee dee deedeejdeed ejfddZd eeejffddZd ejfddZd)d ejfd d!Zd)d ejfd"d#Zed efd$d%Zed efd&d'ZZS)* NevaModelz Lightning Wrapper for Neva ModelNroptimrrmodel_transformcsPt||_||_|pttdddd|_|j|||_d|_ d|_ dS)Ng-C6?T)lruse_distributed_optimizerr) rrrrrrrSconnectrT_training_loss_reduction_validation_loss_reduction)rrrSrrTrrJrKrHs   zNevaModel.__init__rrUcCs&t|ds|jj|j|d|_dSdS)Nmodule)r)rrrrrZ)rrrJrJrKrXs zNevaModel.configure_modelrr[r]rXrr\rrrrrr^c Cs&|j||||||||| | | | d } | S)N) rrr[r]rXr\rrrrrr^)rZ)rrr[r]rXrr\rrrrrr^ output_tensorrJrJrKr]szNevaModel.forwardcCs |j|Sr)rr)rr|rJrJrK data_step~s zNevaModel.data_stepcCs|j||Sr)rr)rr}rJrJrK forward_stepszNevaModel.forward_stepcC ||Srr]rr} batch_idxrJrJrK training_steps zNevaModel.training_stepcCr^rr_r`rJrJrKvalidation_steps zNevaModel.validation_stepcCs|jst|_|jSr)rXrrrJrJrKtraining_loss_reductionsz!NevaModel.training_loss_reductioncCs|js tdd|_|jS)NT)rc)rYrrdrJrJrKvalidation_loss_reductions z#NevaModel.validation_loss_reduction)NNNr) rrrrrrrrrModulerrrrrrPrrrrrrrr\r]rbrcpropertyrrerfrQrJrJrrKrREsx       !rR)rRrrr)F)> dataclassesrtypingrrrrlightning.pytorchpytorchLrtorch.distributedrnrrr ryr megatron.core.enumsr +megatron.core.models.multimodal.llava_modelr rmegatron.core.optimizerrmegatron.core.packed_seq_paramsr,megatron.core.transformer.transformer_configrr1nemo.collections.common.tokenizers.tokenizer_specrnemo.collections.llmr0nemo.collections.vlm.neva.data.multimodal_tokensrrnemo.lightningrnemo.lightning.io.plr nemo.lightning.megatron_parallelrnemo.lightning.pytorch.optimrr nemo.utilsrrrTrrPrrIOMixinrrLightningModuleConnectorMixinFNMixinrR__all__rJrJrJrKsF                 #8GW ]