o i@sbddlmZddlZddlZddlmZddlmZddlmZGdddZ Gd d d eZ dS) )OptionalN)Tensor)Module)Errorsc@s\eZdZUeed<eeed<defddZedefddZde fdd Z ed d Z d S) AttentionMask bool_mask _logit_maskcCs2|jtjkr td||_tjttd|_ dS)Nz7Expected the attention mask to be of dtype 'torch.bool') dtypetorchbool ValueErrorrjitannotaterrr )selfrrY/home/ubuntu/.local/lib/python3.10/site-packages/curated_transformers/models/attention.py__init__s zAttentionMask.__init__returncCs4|jdurd|jd|_|j}|dusJ|S)Ng?g)r rint)r logit_maskrrrrs  zAttentionMask.logit_maskcCs |jSN)rdimrrrrr s zAttentionMask.dimcCs|jjSr)rshaperrrrr#szAttentionMask.shapeN) __name__ __module__ __qualname__r__annotations__rrpropertyrrrrrrrrr s   rc sFeZdZdddeffddZdededed ed ef d d ZZS) ScaledDotProductAttentiong?) dropout_probr!csttjj|d|_dS)N)p)superrr nnDropoutdropout)rr! __class__rrr*s z"ScaledDotProductAttention.__init__kqv attn_maskrc Csz|dkr td|jd}||dd}|t|}|j\}}||j|dd|7}|jdd} | | |} | S)zw Shapes: k, q, v - (batch, heads, seq_len, width) attn_mask - (batch, seq_len) rz@The attention mask must be a 2D-tensor of shape [batch, seq_len])r) rr r transposemathsqrtrviewsoftmaxr&) rr)r*r+r, model_dim attn_scoresbatchseq_len attn_weights attn_valuesrrrforward.s    z!ScaledDotProductAttention.forward) rrrfloatrrrr; __classcell__rrr'rr )sr ) typingrr1r rtorch.nnrerrorsrrr rrrrs