o }oi6f@sddlZddlZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl m mZddlmZmZmZddlmZddlmZddlmZmZz ddlmZmZmZWneyqdd l m!Z!dZdZ"dZe!#d Ynwdd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6ddl7m8Z8ddl m9Z9m Z erddl:m;Z;zddlmWneyddl?m@Z@dZ=e@Z>YnwddZAde j9de j9deBdeBd eeeBf d!d"ZCd#e.fd$d%ZD       d;d&e9d'e9d(e9d)e9d*e9d+e9d,ed-ed.eeBfd/d0ZEGd1d2d2e,ZFGd3d4d4e j jGZHGd5d6d6eZIGd7d8d8e5ZJGd9d:d:e,ZKdS)<N) nullcontext) TYPE_CHECKINGListOptionalTupleUnion)InferenceParamsparallel_statetensor_parallel)get_bias_dropout_add)PackedSeqParams) SelfAttentionSelfAttentionSubmodules)TEColumnParallelLinearTEDotProductAttentionTERowParallelLinear)loggingzFailed to import Transformer Engine dependencies. `from megatron.core.transformer.custom_layers.transformer_engine import *`If using NeMo Run, this is expected. Otherwise, please verify the Transformer Engine installation.) AttnMaskType) IdentityOp)MLP MLPSubmodules)MegatronModule) ModuleSpec build_module)TransformerBlock)TransformerConfig)TransformerLayerTransformerLayerSubmodules)make_viewless_tensor)Tensornn)CrossAttentionVisionConfig)TEDelayedScalingTENormT)WrappedTorchLayerNormFcCst|tjjr |S||fS)z( Convert an input to a 2-tuple. ) isinstance collectionsabcIterable)xr*\/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/mllama/model/vision.py to_2tupleIsr,r)ar_idsntok num_chunkssupported_aspect_ratiosc Csg}|j}|D]C}||d}tj||jd|f|jd} d| d|d|dd|f<| ||jd|d} | | j} | d} || qt | |t |j }|S)a Build attention masks for a vision encoder to handle padding and token alignment. Args: x (torch.Tensor): Input tensor of shape (batch_size, sequence_length). ar_ids (torch.Tensor): Aspect ratio IDs for masking. ntok (int): Number of tokens. num_chunks (int): Number of chunks in the data. supported_aspect_ratios (List[List[int]]): List of supported aspect ratios. Returns: torch.Tensor: Tensor containing the attention mask. )devicerN) dtypetorchonesshaper2viewT unsqueezeappendstacktofinfomin) r)r-r.r/r0masksr4ar_idarxmask_ir*r*r+build_encoder_attention_maskRs     rDreturnc CsNttttdtjitttt t t ddt ttt t tt ddt d}tt|dS)z@ Create a specification for an image transformer layer. attn_mask_type) linear_qkvcore_attention linear_proj q_layernorm k_layernorm)moduleparams submodules) linear_fc1 linear_fc2)rLrN)input_layernormself_attention self_attn_bdapre_mlp_layernormmlpmlp_bda)rr#rSelfAttentionNoBiasrno_maskrrrrrr rrImageTransformerLayer)image_transformer_submodulesr*r*r+ get_image_transformer_layer_specqs2  r[ hidden_statesattention_maskcontext context_maskrotary_pos_embattention_biasinference_paramspacked_seq_paramsreturn_intermediatec  Cs<|js|j}t|ddd}|jjrt} nt} |jj reddl } |jj dkr0| j j j j} n|jj dkr=| j j j j} ntdt|j| dd|jj fd } d}tr[tjdd }| jjd| |d }nt}| ok||jjd kr|jr| dus~Jd |j|||||||d}nYg}t|jD]Q\}}| dur|| vr|||j#|||||||||d\}}|dus|jjr|jrJWdn1swYt !r|jj"r|j#dur|#|}q|j$dur|$|}t|ddd}| dur |t j%|ddfWdS|WdS1swYdS)z Perform a forward pass through the transformer layers with optional intermediate outputs. Override regular MCore transformer layer forward pass. Tinp requires_grad keep_graphrNe4m3hybridz3E4M3 and HYBRID are the only supported FP8 formats.F)config fp8_formatoverride_linear_precision)with_context_parallel)enabled fp8_recipe fp8_groupfullzQConfig `return_intermediate` cannot be used with `recompute_granularity='full'`. )r\r]r^r_r`rarc)r\r]r^r_r`rarbrcr3dim)& pre_process input_tensorrrksequence_parallelr get_cuda_rng_trackerforkrfp8transformer_enginecommonrecipeFormatE4M3HYBRID ValueErrorr" fp8_wgradr model_parallel_is_initializedget_amax_reduction_grouppytorch fp8_autocastrecompute_granularitytraining_checkpointed_forward enumeratelayersr;offload_contextenable_cuda_graphr5is_grad_enabledcpu_offloading#group_prefetch_offload_commit_asyncfinal_layernormr<)selfr\r]r^r_r`rarbrcrd rng_contextr{rlrprq fp8_contextintermediate_hidden_statesl_nolayerr*r*r+ forward_with_return_intermediates              5&rcs|eZdZdZ ddedededeeeeeffdeeeeeffdee d d ffd d Z d e j d e j fddZ ZS)ColumnParallelConv2dPatcha Conv2D Patching layer with model parallelism. Applies convolution in a column-parallel fashion. Args: config (TransformerConfig): Configuration object for the layer. in_channels (int): Number of input channels. out_channels (int): Number of output channels. kernel_size (Union[int, Tuple[int, int]]): Size of the convolution kernel. stride (Union[int, Tuple[int, int]]): Stride of the convolution. bias (Optional[bool], default=False): Whether to include a bias term. Input: torch.Tensor: Input tensor of shape (batch_size, in_channels, width, height). Output: torch.Tensor: Output tensor of shape (batch_size, num_tokens, out_channels). Frk in_channels out_channels kernel_sizestridebiasrENc shtj|dt|tr||f}tjj||d|_t||d|d|||j |j j ddddd |_ dS)Nrk)rrrr1Fconv1)rrk init_method gather_output skip_bias_add is_experttp_comm_buffer_name) super__init__r%intr5r Unfold_unfoldrrkr_linear)rrkrrrrr __class__r*r+rs  z"ColumnParallelConv2dPatch.__init__r)cCs6||}|ddd}t||jj}t|}|S)Forward.rr1)rpermuteFlinearrweightr (gather_from_tensor_model_parallel_regionrr)r*r*r+forward(s  z!ColumnParallelConv2dPatch.forwardF)__name__ __module__ __qualname____doc__rrrrrboolrr5rr __classcell__r*r*rr+rs&rcsJeZdZdZ d dedeffdd Zdejdejd ejfd d Z Z S) PrecomputedTilePositionEmbeddingz Module to compute positional embeddings for tiles with optional gating. Args: config (TransformerConfig): Configuration object. gated (bool, default=False): Whether to apply gating to the embeddings. Frkgatedcsbt|j|_|j|_|j|_t|jd|j|j|_||_|r/t t d|_ dSdS)Nr1) rr max_num_tiles hidden_sizemax_aspect_ratio_idr Embedding embeddingr Parameterr5zerosgate)rrkrrr*r+r:s z)PrecomputedTilePositionEmbedding.__init__r\aspect_ratio_idsrEcCs>||}|d|jd|j}|jr||j}||}|S)rr3r1)rreshaperrrrtanh)rr\r embeddingsr*r*r+rIs z(PrecomputedTilePositionEmbedding.forwardr) rrrrrrrr5rrrr*r*rr+r1s $rcs4eZdZdZejfdededeffdd Z Z S)rWar Self-attention layer implementation without bias. Args: config (TransformerConfig): Configuration for the transformer. submodules (SelfAttentionSubmodules): Submodules required for self-attention. layer_number (int): The layer number in the transformer stack. attn_mask_type (AttnMaskType): Type of attention mask to apply. rkrN layer_numberc stjd ||||d|t|j|jj|jd|j|j|jjdddddd |_t|j |j|jj|j|jj dddddd |_ dS) N)rkrNrrFrFqkv)rkrrrrrrTproj)rkrrinput_is_parallelrrrr*) rrrrGrkrquery_projection_sizekv_projection_sizerrIoutput_layer_init_method)rrkrNrrFkwargsrr*r+r`s@   zSelfAttentionNoBias.__init__) rrrrrpaddingrrrrrr*r*rr+rWUsrWc sNeZdZdZ  d dedededeffdd Z      d d d Z Z S)rYa Transformer layer adapted for processing image data with optional gating. Args: config (TransformerConfig): Transformer configuration object. submodules (TransformerLayerSubmodules): Submodules to use in the layer. layer_number (int, default=1): Layer number in the transformer. hidden_dropout (float, optional): Dropout rate for hidden layers. r1NrkrNrhidden_dropoutc shtjd||||d||jj|_|jr2ttjd|jjd|_ ttjd|jjd|_ dSdS)N)rkrNrrr1)r4r*) rrrkrr rr5r params_dtype gate_attngate_ffn)rrkrNrrrrr*r+rs zImageTransformerLayer.__init__c sf|} ||} |j| |||||d} |jsdn|jt| ts%Jdtfdd| D} |||j |j j | | |j }Wdn1sMwY|} | |} || } |jscdn|jt| tsqJdtfdd| D} |||j |j j | | |j }Wdn1swYt||jd d }|j jr|j r|S||fS) r)r]rbr`rarcr1z:`attention_output_with_bias` needs to be tuple for gating.c3$|] }|dur |ndVqdSNr*.0output) _gate_attnr*r+  z0ImageTransformerLayer.forward..Nz4`mlp_output_with_bias` needs to be tuple for gating.c3rrr*r) _gate_ffnr*r+rrTre)rQrRrrrr%tuplebias_dropout_add_exec_handlerrSrrkbias_dropout_fusionrrTrUrrVrrgexternal_cuda_graph)rr\r]r^r`rarbrcrresidualinput_layernorm_outputattention_output_with_biaspre_mlp_layernorm_outputmlp_output_with_biasrr*)rrr+rsR        zImageTransformerLayer.forward)r1N)NNNNNN) rrrrrrrfloatrrrr*r*rr+rYs(rYc steZdZdZ      dddd ed ed ed ed ef fdd ZddZddZde j de j de j fddZ Z S) VisionEncoderag Vision encoder module for processing image inputs with patch-based embeddings. Args: config ('CrossAttentionVisionConfig'): Configuration object for the encoder. image_size (int, default=560): Input image size. patch_size (int, default=14): Size of patches extracted from the image. in_channels (int, default=3): Number of input channels. pre_process (bool, default=True): Whether to preprocess input. post_process (bool, default=True): Whether to postprocess output. return_intermediate (Optional[bool]): Whether to return intermediate layers. 0TNrkr! image_size patch_sizerru post_processc stj|d||_t||_t||_|jd|jd|jd|jdf|_||_||_|j j |_ |j |_ |j }t |||||dd|_|d} t| t||_t| t|jd|jdd||_t||d|_t||d|_t|j td|j|jd|_tt|j|j_t |j } |j j!| _"d | _#t| td|j|jd|_$t%|d d |_&t%|d d |_'t(|j d|j |jd|jdd||_)tt*d|_+dS) Nrrr1F)rkrrrrrg)rkr)rkspecpost_layer_normrurT)rkr),rrrdr,rr grid_sizerurrkrrrrrr rr5randnclass_embeddingpositional_embedding LayerNormImplln_postln_prerr[ transformertypes MethodTyperrcopydeepcopynum_global_layers num_layersrglobal_transformerrpre_tile_pos_embedpost_tile_pos_embedrgated_tile_positional_embeddingrgated_positional_embedding_gate) rrkrrrrurrdwidthscale global_configrr*r+rsn    ,  (zVisionEncoder.__init__cCsx|j\}}}}|||||}||jd|j}|||||}||}|||||}||j|}|S)z>Apply regular position embedding and tile positonal embedding.r1)r7r8rrrrr)rr)rbszr/ num_tokensrttile_position_embeddingr*r*r+apply_positional_embeddingFs z(VisionEncoder.apply_positional_embeddingc CsDtj|j|jtj|jdd|jd|j|jd|gdd}|S)zConcat class embedding tokens.rr1r3)r4r2rs)r5catrr=r4rr7r2rr*r*r+apply_class_embeddingQs "z#VisionEncoder.apply_class_embeddingimagesr-rEcCs|jdkrd}|j\}}}}}n |j\}}}}}}|||||||}|||d}|||||||} || } | j\} } } | |||| | } || |} | |||| | } || } | d7} | |||| | } || |} || } d| jddd} ddd| f}tj | |ddd} | ||d| } t | || ||j j }| dd} |j| d ||jd \} }| dd|dd} }|| } | |||| | | } || |} | |||| | | } | dd} |j| d |d } | dd} | |||| | | } | d d d d d | f} | |||| | } ||||| | d}|d d d d d | f}||||| d}tj| |gdd } | S) rr1rconstant)modevaluer3N)r\r]rard)r\r]rars)ndimr7rrrrrrrpadr8rDrkr0 transpose contiguousrrdrrrr5r)rrr-num_concurrent_mediar r/nchwhr)_r.rtnpadr attn_biasint_xr*r*r+r]sb         "   zVisionEncoder.forward)rrrTTN) rrrrrrrrrr5rrrr*r*rr+rs0F $ r)NNNNNNN)Lr&rr contextlibrtypingrrrrrr5torch.nn.functionalr functionalr megatron.corerr r (megatron.core.fusions.fused_bias_dropoutr megatron.core.packed_seq_paramsr #megatron.core.transformer.attentionr r:megatron.core.transformer.custom_layers.transformer_enginerrr ImportError nemo.utilsrr#warningmegatron.core.transformer.enumsr%megatron.core.transformer.identity_oprmegatron.core.transformer.mlprr megatron.core.transformer.moduler$megatron.core.transformer.spec_utilsrr+megatron.core.transformer.transformer_blockr,megatron.core.transformer.transformer_configr+megatron.core.transformer.transformer_layerrrmegatron.core.utilsrrnemo.collections.vlmr!r"HAVE_TEr*megatron.core.transformer.torch_layer_normr$r,rrDr[rrModulerrWrYrr*r*r*r+s                #  l5$7f