o wi@sFddlmZmZddlmZmZmZmZmZddl m Z ddl Z ddl Z ddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lm Z dd l!m"Z"ddl m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9ddl:m;Z;ddlm?Z?ddl@mAZAddlBmCZCmDZDddlEmFZFdeeGe jHffdd ZIde jHfd!d"ZJd#d$ZKeGd%d&d&e e?jLZMeGd'd(d(e e?jLZNeGd)d*d*e e?jLZOGd+d,d,eZPGd-d.d.e jQe?jLe?jRe'jSZTgd/ZUdS)0) dataclassfield)CallableDictListOptionalTupleN)dist_checkpointingparallel_state) ModelType)InferenceParams) LLaVAModel)OptimizerConfig)#scatter_to_sequence_parallel_region) ModuleSpec)TransformerConfig)get_batch_on_this_cp_rank)nn) TokenizerSpec)fn) quick_gelu)get_packed_seq_params) Qwen2Config)get_layer_spec_te)MODEL_CONFIG_ATTRrestore_model_weights) IGNORE_INDEXIMAGE_TOKEN_INDEXVIDEO_TOKEN_INDEX)Qwen2VisionModelQwen25VisionModel)MultimodalProjectorConfig)get_image_sequence_length)io)$MaskedTokenLossReductionWithLossMask)MegatronOptimizerModuleOptimizerModule)loggingreturncsddlm}t|}t|trt|dkr|d}n|}t|dkr)dn |dkr2d|r;d| rDd fd d | D}t |}|S) zQwen2VL Data Steprr qwen2-vl) input_ids pixel_valuesimage_grid_thwpixel_values_videosvideo_grid_thw qwen25-vl)r,r-r.r/r0second_per_grid_ts) position_ids)labels loss_maskcs2i|]\}}||vr|dur|jddndqS)NT) non_blocking)cuda).0keyval required_keysd/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/vlm/qwen2vl/model/base.py Ws z%qwen2vl_data_step..) megatron.corer next isinstancetuplelensetupdateis_pipeline_first_stageis_pipeline_last_stageitemsr)dataloader_iter model_versionr batch_batchoutputr=r;r>qwen2vl_data_step2s.     rOc Cst|d|dd|dd|dd|dd|dd|dd|ddd }d |vr3t||d <|d i|S) Nr,r-r.r/r0r2r5r4)r,r-r.r/r0r2r5r4 cu_seqlenspacked_seq_paramsr=)getr)modelrL forward_argsr=r=r>qwen2vl_forward_step`s        rUcCsdSNr=)selftensorr=r=r>set_input_tensorqsrYc@seZdZUdZdZeed<dZeed<dZ eed<dZ eed <dZ eed <d Z eed <d Z eed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<eZeed<d Zeed!<d Zeed"<dZeed#<dZ eed$<dZ!eed%<dZ"eed&<dZ#eed'<d(Z$e%ed)<dZ&eed*<d+Z'eed,<d-Z(e)ed.<d/Z*e%ed0<d5d3d4Z+d-S)6Qwen2VLVisionConfigzQwen2VL Vision Model ConfigFadd_class_tokenclass_token_len patch_dimPimg_himg_w num_layersnum_attention_headsTadd_bias_linear add_qkv_bias embed_dim hidden_sizespatial_merge_sizespatial_patch_sizetemporal_patch_sizehidden_dropoutattention_dropoutiffn_hidden_sizegated_linear_unitactivation_funcP kv_channelsnum_query_groupslayernorm_zero_centered_gammaapply_query_key_layer_scalingbias_activation_fusionbias_dropout_fusionattention_softmax_in_fp32 LayerNorm normalizationapply_rope_fusionư>layernorm_epsilonNtransformer_layer_specr+rKr)r c CJ|j}t|ts tdd}t|||j|j|j|j|j |j |j |j d }|SNT)is_vit)r[r]r_rormrnrarb) rrBrrr r[r]r_rormrnrarbrWrrSr=r=r>configure_model    z#Qwen2VLVisionConfig.configure_model)r)r ),__name__ __module__ __qualname____doc__r[bool__annotations__r]intr_rarbrdrfrgrhrjrkrmrnrorqfloatrrrsrtrrurrwrxryrzr{r|r}rstrrrrrrKrr=r=r=r>rZvsD                                rZc@seZdZUdZdZeed<dZeed<dZ eed<dZ eed <dZ eed <d Z eed <d Z eed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<ejjjZeed<d Z eed!<d Z!eed"<dZ"eed#<dZ#eed$<dZ$eed%<dZ%eed&<dZ&eed'<d(Z'e(ed)<dZ)eed*<d+Z*eed,<d-Z+e,ed.<e-d/d0d1Z.e/eed2<d3Z0e(ed4<d9d7d8Z1d-S):Qwen25VLVisionConfigzQwen2.5VL Vision Model ConfigFr[r\r]r^r_r`rarbrcrdrerfTrgrhrirjrkrlrmrnrorprqrri\ rsrtrurvrwrxryrzr{r|r}RMSNormrrrrNrcCsgdS)N)r=r=r=r=r>szQwen25VLVisionConfig.)default_factoryfullatt_block_indexesr1rKr)r!c Crr) rrBrrr!r[r]r_rormrnrarbrr=r=r>rrz$Qwen25VLVisionConfig.configure_model)r)r!)2rrrrr[rrr]rr_rarbrdrfrgrhrjrkrmrnrorqrrrrsrttorchr functionalsilururrwrxryrzr{r|r}rrrrrrrrrrKrr=r=r=r>rsF                               rc@s&eZdZUdZdZeeed<dZee e Bed<dZ ee ed<dZ eed<dZeed <d Zeed <d Zeed <d Zeed<dZeed<dZeed<dZeeed<dZeeed<dZeeed<dZeed<dZeed<dZeed<eZeed<e Z!eed<ddZ"d"deeddfd d!Z#dS)# Qwen2VLConfigzQwen2VL Model Base ConfigNlanguage_transformer_configvision_transformer_configvision_projection_configFdrop_vision_class_tokenvision_feature_layerr$encoder_pipeline_model_parallel_sizer\"encoder_tensor_model_parallel_sizerdrfi seq_lengthlanguage_model_from_pretrainedvision_model_from_pretrained!vision_projection_from_pretrainedfreeze_language_modelfreeze_vision_modelfreeze_vision_projectionforward_step_fn data_step_fncCsD|jdur tD] }t||t|j|qd|j_gd|j_dSdS)Nmrope)rer)rrsetattrgetattrposition_embedding_type mrope_section)rWattrr=r=r> __post_init__s zQwen2VLConfig.__post_init__vp_stager)MCoreQwen2VLModelc CsZd|j_|j|j_|j|j_|j|j_|j|j_|j|j_|j|j_|jdkrN|jdks0Jd|j|j_|j|j_|j|j_|j dkrN|j |j_|j |j_gd}|j|j|jfD]}|D] }t ||t ||q^qZ|j |j_ d|j_ d|j_ |p{d}t ||tjd|dpt|jktjd|dtjd|dtjd|dpt|jk|j|d}|S)NFrr\z&ViT can only live on 1 pipeline stage.) cross_entropy_loss_fusionenable_cuda_graphuse_te_rng_trackergradient_accumulation_fusionr{r|masked_softmax_fusionr}roverlap_p2p_commbatch_p2p_comm)ignore_virtualr)config tokenizer pre_process post_process add_encoder add_decoderrr)r#scatter_embedding_sequence_paralleltensor_model_parallel_sizesequence_parallelcontext_parallel_sizerrpipeline_model_parallel_sizerrrrtp_comm_overlaprpsrG get_pipeline_model_parallel_rankrHr)rWrr config_attrsrrrSr=r=r>rsR                  zQwen2VLConfig.configure_modelrV)$rrrrrrrrrrZrrr"rrrrrrrdrfrrrrrrrrrUrrrOrrrr=r=r=r>rs,              rcseZdZdZ       d(dedededed ed ed ed eed dffdd Z     d)dee j dee j dee j dee j dee j d e e j e j ff ddZ           d*de j dee j dee j dee j dee j deedee j dee jdee j dee j deedee jd e j fddZ        d+de j dee j dee j d ee j d!ee j d"ee j dee j d#eedee j fd$d%Zd,d&d'ZZS)-rzQwen2VL Model Base Model ClassNTFrrrrrrrrr)c stt|j|d|j} |j} |j} | j|_|jdusJ||_||_||_ ||_ ||_ ||_ d|_ d|_d|_d|_| j|_| j|_| j|_d|_|j rx| j||||d|_|jj|_|jj|_| jdk|_t|j|jt d|jn|jdurt!j"t#id|jdd|j r| |_| |_||_$t|j|j%t d|j%|j&|j'|j(|j)d t*j+|_,t-| j.| j/| j0| | j1d |_2dS) NrF)rrrrr\z%Restored language model weights from ) state_dict)sharded_state_dictcheckpoint_dirvalidate_access_integrityz#Restored vision model weights from )rrr)rarbr_r[r])3superMCoreLLaVAModel__init__rrrrKrrrrrrencoder_hidden_state vision_modelvision_projectionlanguage_modelrsequence_parallel_lmrtp_comm_overlap_lmrcontext_parallel_lm#share_embeddings_and_output_weightsrmax_sequence_length_language_max_sequence_lengthr_language_is_pipeline_parallelrrr(infor loaddict_drop_vision_class_tokenrfreezerrrr encoder_or_decoder model_typer#rarbr_r] _img_seq_len) rWrrrrrrrrrrr __class__r=r>r]sr        zMCoreQwen2VLModel.__init__r,r.r0r2attention_maskc2 Cs d}t}t}d} d} |dur|}g} |dus|dur$|} |dur*t| }tjd|jd|jd|j|j d} d\}}| | j }t | D]\}}|||dk}d\}}t || k d}||d}||k}||k}|}g}d}||}}t||D]/}||vr|dkr|||}nt|d}||vr|dkr|||}nt|d}||kr||d||d||d}} }!d}"|d7}|d8}|}#n/||d||d||d}} }!|jd kr|dur||}"nd }"|d7}|d8}|}#|| ||!|}$}%}&|#|}'t|dkr0|d dnd}(|t|'dd dd |(|jd kr\t|$d dd |%|&})n%|jd krt|$d d}*|*d |%|&}+|+|"| },|,}-|-})t|%dd d|$d |&}.t|&ddd |$|%d }/|t|)|.|/g|'|(|#|$|%|&}q|t|krt|dkr|d dnd}(t||}'|t|'dd dd |(tj|dd dd }0|0 | j | d|||dkf<| |0dt| |qJtj| |j d d} | | fS|dure|!d d} | "|dkd| ddd d  |j } | jddddjd ddd}1|1d|jd } | | fStj|jd|j dddd d|jdd } tj#|jddg|j |jd} | | fS)aX Calculate the 3D rope index based on image and video's temporal, height and width in LLM. Explanation: Each embedding sequence contains vision embedding and text embedding or just contains text embedding. For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. Examples: input_ids: [T T T T T], here T is for text. temporal position_ids: [0, 1, 2, 3, 4] height position_ids: [0, 1, 2, 3, 4] width position_ids: [0, 1, 2, 3, 4] For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part and 1D rotary position embedding for text part. Qwen2-VL and Qwen25-VL has differnt type: Qwen2-VL Examples: Assume we have a video input with 3 temporal patches, 2 height patches and 2 width patches. input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. vision temporal position_ids: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] text temporal position_ids: [3, 4, 5, 6, 7] text height position_ids: [3, 4, 5, 6, 7] text width position_ids: [3, 4, 5, 6, 7] Qwen25-VL Examples: Temporal (Time): 3 patches, representing different segments of the video in time. Height: 2 patches, dividing each frame vertically. Width: 2 patches, dividing each frame horizontally. We also have some important parameters: fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each second. tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal tokens" are conceptually packed into a one-second interval of the video. In this case, we have 25 tokens per second. So each second of the video will be represented with 25 separate time points. It essentially defines the temporal granularity. temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. interval: The step size for the temporal position IDs, calculated as tokens_per_second * temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be have a difference of 50 in the temporal position IDs. input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] text temporal position_ids: [101, 102, 103, 104, 105] text height position_ids: [101, 102, 103, 104, 105] text width position_ids: [101, 102, 103, 104, 105] Here we calculate the text start position_ids as the max vision position_ids plus 1. Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): The temporal, height and width of feature shape of each image in LLM. video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): The temporal, height and width of feature shape of each video in LLM. second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. Returns: position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) rlidPNr*rr\dtypedevice)rrr1g?r+)dim.)rF)keepdimT)rr)$rrcpucloner ones_likeonesshaperrto enumerateargwheresqueezesumtolistrangeindexrDrKitemmaxappendarangeviewexpandflattenlongstackcatreshaperX unsqueezecumsum masked_fill_zeros)2rWr,r.r0r2rrmimage_token_idvideo_token_idvision_start_token_idtokens_per_secondmrope_position_deltastotal_input_idsr3 image_index video_indexiinput_ids_item _input_ids image_nums video_numsvision_start_indices vision_tokens input_tokensllm_pos_ids_listst remain_images remain_videos_ed_imageed_videothwsecond_per_grid_ted llm_grid_t llm_grid_h llm_grid_wtext_lenst_idxt_index range_tensorexpanded_range time_tensortime_tensor_longh_indexw_index llm_positionsmax_position_idsr=r=r>get_rope_indexsM                    "$ $  """ $      z MCoreQwen2VLModel.get_rope_indexr3r5r4inference_paramsr-r/runtime_gather_outputc  s|duod|jv} |du}|du}d}| rd}n|jr |s d}n|jr|r|t|jj}|jjrPt |j|| d}Wdn1sJwYn|j|| d}|j dkr`|jj nd}|j rt|jdd|dddddf}|j dkrfdd|D}nd}||}|j dkrt |}||ddf}|dur|jd|jd |jd <n|j}d}|jr|r|t|jj}|jjrt |j|| d}Wdn1swYn|j|| d}||}|js|Sd}|jd}||jkrH|ddd|jf}|dur$|ddddd|jf}|durH|jd|jkrH|ddd|jf}|ddd|jf}|jrp||jkrp|j|}t jj|d |f}|durpt jj|d |f}|dur|dur||| | | |\}}|jr|jrt}t|jd|d|||jd}|d krt jj|d |f}|durt jj|d |f}|}d ||d k<|j j!|dd }|"dd #}|j$|||||||d \}}}}|j d|||||| d}|dus|dur|S||#fS)arForward function of the Qwen2VL model. Args: input_ids (torch.Tensor): input text ids [batch, decoder_seq_len]. attention_mask (torch.Tensor): Attention mask for the language model [batch, 1, combined_seq_len, combined_seq_len]. position_ids (torch.Tensor): input text position ids [batch, decoder_seq_len]. loss_mask (torch.Tensor): Text loss mask [batch, decoder_seq_len]. labels (torch.Tensor): Optional target text labels [batch, combined_seq_len]. inference_params (InferenceParams): Inference-time parameters including KV cache. pixel_values (torch.Tensor): input image of shape [images_total_patches, num_channels * temporal_size * patch_size * patch_size]. pixel_values_videos (torch.Tensor): input video of shape [videos_total_patches, num_channels * temporal_size * patch_size * patch_size]. image_grid_thw (torch.Tensor): The temporal, height and width of feature shape of each image. Shape [num_images, 3]. video_grid_thw (torch.Tensor): The temporal, height and width of feature shape of each video. Shape [num_videos, 3]. runtime_gather_output (bool): Gather output at runtime. Default None means `parallel_output` arg in the constructor will be used. Returns: output (torch.Tensor): Loss of shape [b, s] if labels are provided, otherwise logits of shape [b, s, vocab_size]. loss_mask (torch.Tensor): Loss mask expanded to combined sequence length. Shape [b, s]. Nimage_tokens_count)grid_thwr1r]r\csg|] }|kr|qSr=r=)r8idxr]r=r> sz-MCoreQwen2VLModel.forward..rlmedia_tokens_countr)r,r3)r5r4language_embeddingsimage_embeddingsvideo_embeddingsr)r,r3r decoder_inputr4r>r?)%key_value_memory_dictrrrAr parametersrrrrno_gradrK window_indexrrrargsortrrrrrrrpadr=rrr$get_tensor_model_parallel_world_sizerrr embedding transpose contiguous_preprocess_data)rWr,rr3r5r4r>r-r/r.r0r?r2use_inference_kv_cache has_images has_videosrGrMreverse_indicesrHrFlanguage_seq_lenpadded_seq_lenr' tp_world_sizeinput_ids_textcombined_embeddings final_labelsfinal_loss_maskfinal_attention_maskrNr=rCr>forwardrs)                &      zMCoreQwen2VLModel.forwardrFrGrHrUc Cs(|jsJd|js|jsdS|r|||| fS|j\} } |du} | r4|j|jks4Jd|jd|j|du} |du}d}|jr|}| r||tk}|jd}||kr`td|d||tkd  | |j }| |j |j }| ||}|r|tk}|jd}||krtd |d||tkd  | |j }| |j |j }| ||}d \}}|jr| r|jr|jd |jkr|j}tj| |ft|j |j d }tj| |fd|j |j d }|ddddf|ddd|jd f<|ddddf|ddd|jd f<n||}}|dur?|dur?|jdd|jkr:|jks?JdJd|durh|jd |jkrX|ddd|jf}|d d}|jrht|}|duot|jd |jk}|r|ddd|jf}|ddd|jf}|||| fS)a MCoreQwen2VLModel uses its own version of _preprocess_data instead of MCoreLLaVAModel's (in megatron-lm/megatron/core/models/multimodal/llava_model.py) This function handles several data preprocess requirements: - merge image and/or video embeddings into language embedding - padding inputs variables (e.g. labels/loss masks) for pipeline_parallel case - truncate inputs variables (e.g. labels/loss masks) if exceeding max seq length This function won't shift labels as forward() and _preprocess_data() in MCoreQwen2VLModel expect labels from input arguments already handle this shift. About merging image/video embeddings: language_embeddings may include num of imgage_token placeholders, and this function will put each imgage_token from image_embeddings into placeholder within language_embeddings(1:1 mapping), when image_embeddings/video_embeddings is available and it's the 1st pipeline_parallel stage z>input text preprocessing is only needed for the language model)NNNNNzmismatching labels shape z and loss mask shape rz6Image features and image tokens do not match: tokens: z , features rz6Video features and video tokens do not match: tokens: )NNr\rrlz*unexpected shapes after data preprocessing)rrrrrrr ValueErrorr expand_asrrrmasked_scatterrrrrfullrrRrSrr)rWr,r5r4rFrGrHr3rUr batch_sizerY has_labelsrVrWfinal_embeddingn_image_tokensn_image_features image_maskn_video_tokensn_video_features video_maskr^r_ max_seq_lentruncate_labelsr=r=r>rT-s      *, &  z"MCoreQwen2VLModel._preprocess_datacCst|ts|g}t|dksJd|jr"|jr"|j|ddS|jr/|j|ddS|jr9|d|_dS|j |ddS)zSet model chunk input tensor.r\z.input_tensor should only be length 1 for llavarN) rBlistrDrrrrYrrr)rW input_tensorr=r=r>rYs  z"MCoreQwen2VLModel.set_input_tensor)NTTTTFN)NNNNN) NNNNNNNNNNN)NNNNNNFN)r)N)rrrrrrrrrr LongTensorTensorrr=r FloatTensorrarTrY __classcell__r=r=rr>rZs  V E      ?  rcsneZdZdZ   d(dededeededdeee j ge j ff fd d Z d)d ee d dfd dZ          d*dejdeejdeejdeejdeejdeedeejdeejdeejdeejdeejd ejfddZd eeejffddZd ejfddZd)d ejfd d!Zd)d ejfd"d#Zed efd$d%Zed efd&d'ZZS)+ Qwen2VLModelz#Lightning Wrapper for Qwen2VL ModelNrrKoptimrrmodel_transformcsht||_||_|pttdddd|_|j|||_d|_ d|_ ||_ |j dvs2JddS)Ng-C6?T)lruse_distributed_optimizerr)r+r1z3model_version only supports qwen2-vl and qwen25-vl.) rrrrr&rrxconnectry_training_loss_reduction_validation_loss_reductionrK)rWrrKrxrryrr=r>rs  zQwen2VLModel.__init__rr)cCs&t|ds|jj|j|d|_dSdS)Nmodule)r)hasattrrrrr)rWrr=r=r>rs zQwen2VLModel.configure_modelr,rr3r5r4r>r-r/r.r0r2c Cs$|j||||||||| | | d } | S)N) r,rr3r5r4r>r-r/r.r0r2)r) rWr,rr3r5r4r>r-r/r.r0r2 output_tensorr=r=r>raszQwen2VLModel.forwardcCs|j||jSrV)rrrK)rWrJr=r=r> data_step szQwen2VLModel.data_stepcCs|j||SrV)rr)rWrLr=r=r> forward_stepszQwen2VLModel.forward_stepcC ||SrVrrWrL batch_idxr=r=r> training_steps zQwen2VLModel.training_stepcCrrVrrr=r=r>validation_steps zQwen2VLModel.validation_stepcCs|jst|_|jSrV)r}r%rWr=r=r>training_loss_reductionsz$Qwen2VLModel.training_loss_reductioncCs|js tdd|_|jS)NT)r)r~r%rr=r=r>validation_loss_reduction's z&Qwen2VLModel.validation_loss_reduction)NNNrV) NNNNNNNNNN)rrrrrrrr'rrModulerrrrrtrsr rurarrrrrpropertyr%rrrvr=r=rr>rwsx     rw)rwrrOrU)V dataclassesrrtypingrrrrrlightning.pytorchpytorchLrtorch.distributedr@r r rmegatron.core.enumsr megatron.core.inference_paramsr +megatron.core.models.multimodal.llava_modelrrmegatron.core.optimizerrmegatron.core.tensor_parallelr$megatron.core.transformer.spec_utilsr,megatron.core.transformer.transformer_configrmegatron.core.utilsrr1nemo.collections.common.tokenizers.tokenizer_specrnemo.collections.llmr"nemo.collections.llm.fn.activationr#nemo.collections.llm.gpt.model.baser$nemo.collections.llm.gpt.model.qwen2r nemo.collections.vlm.layer_specsr$nemo.collections.vlm.neva.model.baserr3nemo.collections.vlm.qwen2vl.data.multimodal_tokensrrr)nemo.collections.vlm.qwen2vl.model.visionr r!nemo.collections.vlm.visionr" nemo.collections.vlm.vision.baser#nemo.lightningr$ nemo.lightning.megatron_parallelr%nemo.lightning.pytorch.optimr&r' nemo.utilsr(rrtrOrUrYIOMixinrZrrrLightningModuleConnectorMixinFNMixinrw__all__r=r=r=r>s^                       .9:n{ _