o
     i                     @   s:   d dl mZmZmZ d dlZedddZG dd dZdS )    )OptionalTypeTypeVarNSelfAttentionMask)boundc                   @   sH  e Zd ZdZd)dejdefddZdejfdd	Ze	d
e
e dejdefddZe	d
e
e dejdefddZe				d*d
e
e dedee deej deej defddZ	d+dedee dd fddZdd Zedd Zedd Zedd  Zed!d" Zed#d$ Zd%d& Z	d,deej deej dd fd'd(ZdS )-r   a!  
    Holds an attention mask, along with a couple of helpers and attributes.

    .. note: this is an additive mask, meaning that coefficients which should be computed hold the '0.' value,
        and coefficients which should be skipped hold the '-inf' value. Any other value is possible if the purpose
        is to bias the attention computation for instance

    .. note: the attention mask dimensions are expected to be `[batch, to_sequence, from_sequence]`,
        `[to_sequence, from_sequence]`, or anything broadcastable in between
    Fadditive_mask	is_causalc                 C   sX   |  s	J |j|jrJ |jdkr|d}|| _|| _|jd | _|jd | _	d S )N   r      )
is_floating_pointdtyperequires_gradndim	unsqueezevaluesr	   shapeseq_len
to_seq_len)selfr   r	    r   `/home/ubuntu/.local/lib/python3.10/site-packages/xformers/components/attention/attention_mask.py__init__   s   


zAttentionMask.__init__returnc                 C   s   | j tdkS )z`
        .. warning: we assume here that True implies that the value should be computed
        -inf)r   floatr   r   r   r   to_bool&   s   zAttentionMask.to_boolclsxc                 C   sJ   |j tjksJ tj|tj|jd}||d || td | |S )z
        Create an AttentionMask given a boolean pattern.
        .. warning: we assume here that True implies that the value should be computed
        r   device        r   r   torchbool
empty_liker   r!   masked_fill_r   r   r   r   r   r   	from_bool,   s
   zAttentionMask.from_boolc                 C   sR   |j tjkrJ tj|tj|jd}| }||d || td | |S )zP
        Create an AttentionMask given a multiplicative attention mask.
        r    r"   r   r#   r(   r   r   r   from_multiplicative:   s   z!AttentionMask.from_multiplicativeNr   r   r!   r   c                 C   s8   |s|}t jt j||||dtd dd}| |ddS )Nr!   r   r   r   )diagonalT)r   r	   )r$   triuonesr   )r   r   r   r!   r   r   r   r   r   make_causalI   s   zAttentionMask.make_causalc                 C   s.   |s|}t | jddd|d|f | jdS )z`
        Return a cropped attention mask, whose underlying tensor is a view of this one
        Nr	   )r   r   r	   )r   r   r   r   r   r   	make_cropZ   s
   zAttentionMask.make_cropc                 C   s   d| j  dt| j S )NzAttentionMask - causal z - mask )r	   strr   r   r   r   r   __repr__h   s   zAttentionMask.__repr__c                 C      | j jS N)r   r!   r   r   r   r   r!   k      zAttentionMask.devicec                 C   s   dS )NFr   r   r   r   r   	is_sparseo   s   zAttentionMask.is_sparsec                 C   s   t | jjS r5   )lenr   r   r   r   r   r   r   s   s   zAttentionMask.ndimc                 C   r4   r5   )r   r   r   r   r   r   r   w   r6   zAttentionMask.dtypec                 C   r4   r5   )r   r   r   r   r   r   r   {   r6   zAttentionMask.shapec                 C   s   t | j|j ddS )NFr0   )r   r   )r   otherr   r   r   __add__   s   zAttentionMask.__add__c                 C   s   |d u st |tjsJ |d u st |tjsJ |d us"|d us"J |r)|| jks+|s6|r2|| jks4|s6| S t| jj||d| jS )Nr+   )
isinstancer$   r!   r   r   r   tor	   )r   r!   r   r   r   r   r<      s   zAttentionMask.to)F)NNNr5   )NN)__name__
__module____qualname____doc__r$   Tensorr%   r   r   classmethodr   r   r)   r*   intr   r!   r   r/   r1   r3   propertyr7   r   r   r:   r<   r   r   r   r   r      sj    





)typingr   r   r   r$   r   r   r   r   r   r   <module>   s   