o
    i
                     @   sx   d dl mZ d dlZddlmZ ddlmZ ddlmZm	Z	 G dd	 d	eZ
G d
d deZG dd de	Zg dZdS )    )OptionalN   )Cache   )Gemma2Config)Gemma2DecoderLayerGemma2ForCausalLMc                   @      e Zd ZdS )VaultGemmaConfigN__name__
__module____qualname__ r   r   n/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/vaultgemma/modular_vaultgemma.pyr
          r
   c                       s   e Zd Z fddZ						ddejdeejejf deej deej d	ee	 d
ee
 dee
 deej deejeeejejf  f fddZ  ZS )VaultGemmaDecoderLayerc                    s   t  jdi | | `| `d S )Nr   )super__init__post_attention_layernormpost_feedforward_layernorm)selfsuper_kwargs	__class__r   r   r      s   zVaultGemmaDecoderLayer.__init__NFhidden_statesposition_embeddingsattention_maskposition_idspast_key_valuesoutput_attentions	use_cachecache_positionreturnc	                 K   st   |}
|  |}| jd||||||||d|	\}}|
| }|}
| |}| |}|
| }|f}|r8||f7 }|S )N)r   r   r   r   r   r    r!   r"   r   )input_layernorm	self_attnpre_feedforward_layernormmlp)r   r   r   r   r   r   r    r!   r"   kwargsresidualself_attn_weightsoutputsr   r   r   forward#   s.   
	



zVaultGemmaDecoderLayer.forward)NNNFFN)r   r   r   r   torchTensortupler   
LongTensorr   boolFloatTensorr,   __classcell__r   r   r   r   r      s6    		r   c                   @   r	   )VaultGemmaForCausalLMNr   r   r   r   r   r4   K   r   r4   )r
   r4   VaultGemmaModelVaultGemmaPreTrainedModel)typingr   r-   cache_utilsr   gemma2.configuration_gemma2r   gemma2.modeling_gemma2r   r   r
   r   r4   __all__r   r   r   r   <module>   s   .