o
    ¹ÈÏiò
  ã                   @   sx   d dl mZ d dlZddlmZ ddlmZ ddlmZm	Z	 G dd	„ d	eƒZ
G d
d„ deƒZG dd„ de	ƒZg d¢ZdS )é    )ÚOptionalNé   )ÚCacheé   )ÚGemma2Config)ÚGemma2DecoderLayerÚGemma2ForCausalLMc                   @   ó   e Zd ZdS )ÚVaultGemmaConfigN©Ú__name__Ú
__module__Ú__qualname__© r   r   úe/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/vaultgemma/modular_vaultgemma.pyr
      ó    r
   c                       sž   e Zd Z‡ fdd„Z						ddejdeejejf deej deej d	ee	 d
ee
 dee
 deej deejeeejejf  f fdd„Z‡  ZS )ÚVaultGemmaDecoderLayerc                    s   t ƒ jdi |¤Ž | `| `d S )Nr   )ÚsuperÚ__init__Úpost_attention_layernormÚpost_feedforward_layernorm)ÚselfÚsuper_kwargs©Ú	__class__r   r   r      s   zVaultGemmaDecoderLayer.__init__NFÚhidden_statesÚposition_embeddingsÚattention_maskÚposition_idsÚpast_key_valuesÚoutput_attentionsÚ	use_cacheÚcache_positionÚreturnc	                 K   st   |}
|   |¡}| jd||||||||dœ|	¤Ž\}}|
| }|}
|  |¡}|  |¡}|
| }|f}|r8||f7 }|S )N)r   r   r   r   r   r    r!   r"   r   )Úinput_layernormÚ	self_attnÚpre_feedforward_layernormÚmlp)r   r   r   r   r   r   r    r!   r"   ÚkwargsÚresidualÚself_attn_weightsÚoutputsr   r   r   Úforward#   s.   
ø	
÷


zVaultGemmaDecoderLayer.forward)NNNFFN)r   r   r   r   ÚtorchÚTensorÚtupler   Ú
LongTensorr   ÚboolÚFloatTensorr,   Ú__classcell__r   r   r   r   r      s6    	÷þýüûúùø	÷õr   c                   @   r	   )ÚVaultGemmaForCausalLMNr   r   r   r   r   r4   K   r   r4   )r
   r4   ÚVaultGemmaModelÚVaultGemmaPreTrainedModel)Útypingr   r-   Úcache_utilsr   Úgemma2.configuration_gemma2r   Úgemma2.modeling_gemma2r   r   r
   r   r4   Ú__all__r   r   r   r   Ú<module>   s   .