o
    Gik                     @   sh   d dl Z d dlmZ d dlmZ d dlmZmZmZ ddl	m
Z
mZ ddlmZ G dd dee
eZdS )	    N)ModuleUtilsMixin)T5BlockT5ConfigT5LayerNorm   )ConfigMixinregister_to_config)
ModelMixinc                       sV   e Zd Ze	ddededededededed	ed
edef fddZdd Z	  Z
S )SpectrogramNotesEncoderF
max_length
vocab_sized_modeldropout_rate
num_layers	num_headsd_kvd_fffeed_forward_proj
is_decoderc                    s   t    t||| _t||| _d| jj_tj|d| _	t
|||||||	|
dd	}t | _t|D ]}t|}| j| q5t|| _tj|d| _d S )NF)p)	r   r   r   r   r   r   r   r   is_encoder_decoder)super__init__nn	Embeddingtoken_embedderposition_encodingweightrequires_gradDropoutdropout_prer   
ModuleListencodersranger   appendr   
layer_normdropout_post)selfr   r   r   r   r   r   r   r   r   r   t5configlyr_numlyr	__class__ v/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.pyr      s,   



z SpectrogramNotesEncoder.__init__c           	      C   s   |  |}|jd }tj||jd}|| |7 }| |}| }| ||}| j	D ]	}|||d }q+| 
|}| ||fS )N   )devicer   )r   shapetorcharanger0   r   r    sizeget_extended_attention_maskr"   r%   r&   )	r'   encoder_input_tokensencoder_inputs_maskx
seq_lengthinputs_positionsinput_shapeextended_attention_maskr*   r-   r-   r.   forwardE   s   




zSpectrogramNotesEncoder.forward)F)__name__
__module____qualname__r   intfloatstrboolr   r=   __classcell__r-   r-   r+   r.   r
      s2    	
*r
   )r2   torch.nnr   transformers.modeling_utilsr   "transformers.models.t5.modeling_t5r   r   r   configuration_utilsr   r   modelsr	   r
   r-   r-   r-   r.   <module>   s   