o
    }oi                     @   sJ  d dl mZ d dlmZmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZmZ d dlmZ d d	lmZmZ zd d
lmZmZmZmZmZ dZW n ey[   dZY nw zd dlmZ dZ eZ!W n ey   d dl"Z"d dl#m$Z$ e"%d e$Z!Y nw defddZ&ddefddZ'dde(defddZ)defddZ*dS )    )get_bias_dropout_add)ColumnParallelLinearRowParallelLinear)SelfAttentionSelfAttentionSubmodules)DotProductAttention)AttnMaskType)
IdentityOp)MLPMLPSubmodules)
ModuleSpec)TransformerLayerTransformerLayerSubmodules)TEColumnParallelLinearTEDotProductAttentionTELayerNormColumnParallelLinearTENormTERowParallelLinearTF)FusedLayerNormN)WrappedTorchLayerNormz6Apex is not installed. Falling back to Torch LayerNormreturnc                 C   sx   | rt jnt j}|dkrt}n|dkrt}ntd|tdd}ttt	|tt
d|ittttttddt||td	d
S )zTransformer Layer Spec	LayerNormRMSNormzunknown normalizationFuse_teattn_mask_type
linear_qkvcore_attentionlinear_projq_layernormk_layernormmoduleparams
submodules)input_layernormself_attentionself_attn_bdapre_mlp_layernormmlpmlp_bdar#   r%   )r   no_maskcausalLNImplr   RuntimeErrorget_mlp_module_specr   r   r   r   r   r   r   r   r	   r   )is_vitnormalizationr   normr*    r5   T/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/layer_specs.pyget_layer_spec4   s8   

r7   c                 C   sL   | rt jnt j}t }tttttd|itt	t
tttddtt|tddS )z$Transformer Layer Spec w/ TE Modulesr   r   r"   )r'   r(   r)   r*   r+   r,   )r   r-   r.   get_norm_mlp_module_spec_ter   r   r   r   r   r   r   r   r	   r   )r2   r   r*   r5   r5   r6   get_layer_spec_teW   s,   r9   r   c                 C   s$   t tt| rtnt| rtntddS )zMLP Submodule Spec
linear_fc1
linear_fc2r,   )r   r
   r   r   r   r   r   r   r5   r5   r6   r1   r   s   

r1   c                   C   s   t ttttddS )zNorm + MLP Submodule Specr:   r,   )r   r
   r   r   r   r5   r5   r5   r6   r8   ~   s   
r8   )F)T)+(megatron.core.fusions.fused_bias_dropoutr   $megatron.core.tensor_parallel.layersr   r   #megatron.core.transformer.attentionr   r   /megatron.core.transformer.dot_product_attentionr   megatron.core.transformer.enumsr   %megatron.core.transformer.identity_opr	   megatron.core.transformer.mlpr
   r   $megatron.core.transformer.spec_utilsr   +megatron.core.transformer.transformer_layerr   r   +megatron.core.extensions.transformer_enginer   r   r   r   r   HAVE_TEImportError&megatron.core.fusions.fused_layer_normr   	HAVE_APEXr/   warnings*megatron.core.transformer.torch_layer_normr   warnr7   r9   boolr1   r8   r5   r5   r5   r6   <module>   s:   
	#