o
    ॵi6                     @   sj   d dl Z d dlmZ d dlm  mZ d dlmZ d dlm	Z	m
Z
 G dd dejZG dd dejZdS )    N)	rearrange)score_computation_opvalue_aggregation_opc                       sH   e Zd Zg ddddf fdd	Zdd Z	dd	d
ZdddZ  ZS )QTAttA    r   r   r   NF皙?c                    s&   t    || _|| _|| _|| _d S )N)super__init__use_dropouttopksnheaddim)selfr   r   r   scaler   attention_dropout	__class__ p/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/ops/quadtree_attention/modules/quadtree_attention.pyr
      s
   
	
zQTAttA.__init__c                 C   s   |j \}}}}|j d | j }	t|d|d| j|	}t|d|d| j|	}t|d|d| j|	}td||}
d|	d  }tj||
 dd}tj|d|d	d
\}}t|}|j	d|t
| d}td|| |}||||fS )N   b c h w -> b (h w) cnlhd,nshd->nlsh      ?      ?r   Tr   klargestr   indexsrcnlsh,nshd->nlhd)shaper   r   viewtorcheinsumsoftmaxtopk	ones_likescatter
zeros_likefloat)r   querykeyvaluer*   bschwcur_dimQKsoftmax_tempA
topk_scoretopk_idxmaskmessager   r   r   process_coarse_level   sH   



zQTAttA.process_coarse_levelc	              	   C   s(  |j \}	}
}}|j d | j }t|d|	d| j|}t|d|	d| j|}||	|
|d d|d d}t|d|	dd| j|}g }|d }dD ]}dD ]}|d | | |d  | }|| qNqJtj|d	d
}t|| ||	d|d | j}t|d|dd}d|d  }tj	|| dd
}|
d
d}|| |	dd|d | j}||	dd|d | jddddd}tj|d|dd\}}|st|}|jd|t| d}t|| | |}nt|| |}|stj||dd}t|d|d dd}t|d|d dd}||||fS )Nr   r   r      "b c h t1 w t2-> b (h w) (t1 t2) c    r   r   r      r   zn l w (k f) h -> n l w k f h)r   fr   r   r   Tr   r!   r"   r   *b (h w) (t1 t2) k nh -> b (h t1 w t2) k nhr4   t1)r%   r   r   r&   appendr'   stackr   
contiguousr)   	unsqueezereshaperepeatr*   r+   r,   r-   r.   r   gather)r   r/   r0   r1   r:   topk_pos	topk_prevr*   finalr2   r3   r4   r5   r6   
idx_gatherxyidxr7   r8   r9   r;   r<   r=   r   r   r   process_fine_level<   s   	




zQTAttA.process_fine_levelc                 C   s:  |d j d }g }| jd }ttt|t|t|D ]V\}	\}
}}|j \}}}}|	dkr;| |
|||\}}}}n#|}| j|	 }|	t|d krLdnd}| |
|||||||\}}}}|| |durrt	
|| || g}qd}t|D ]!\}	}|	dkr|}qy|d| }t|ddd||	  j d d}qy|S )	Multi-head quadtree attention
        Args:
            queries: Query pyramid [N, C, H, W]
            keys: Key pyramid [N, C, H, W]
            values: Value pyramid [N, C, H, W]
        Returns:
            message: (N, C, H, W)
        r   r   TFNr?   (b (H W) (t1 t2) h d -> b (H t1 W t2) h drH   t2H)r%   r   	enumeratezipreversedr>   lenrW   rI   r'   rJ   rL   r   )r   querieskeysvaluesq_maskkv_maskr2   messagesr*   ir/   r0   r1   r3   r4   r5   r9   r=   r:   r;   rQ   rR   rP   final_messagemr   r   r   forward   sL   



zQTAttA.forwardFNN__name__
__module____qualname__r
   r>   rW   rj   __classcell__r   r   r   r   r      s    '
Kr   c                       sH   e Zd Zg ddddf fdd	Zdd Z	ddd	ZdddZ  ZS )QTAttBr   Fr   c                    sh   t    || _|| _| _ | _|| _|r&t fddt	|D | _
| dtt| d S )Nc              
      s.   g | ]}t j    d dd  dqS )rC   r   )kernel_sizestridepaddinggroups)nnConv2d).0_r   r   r   r   
<listcomp>   s    z#QTAttB.__init__.<locals>.<listcomp>weight)r	   r
   r   r   r   r   leperw   
ModuleListrangeget_vsregister_parameter	Parameterr'   randn)r   r   r   r   r   r   r   r~   r   r{   r   r
      s   

	zQTAttB.__init__c                 C   s   |j \}}}}|j d | j }	t|d|d| j|	}t|d|d| j|	}t|d|d| j|	}td||}
d|	d  }tj||
 dd}tj|d|d	d
\}}td||}||||fS )Nr   r   r   r   r   r   r   r   Tr   r$   )r%   r   r   r&   r'   r(   r)   r*   )r   r/   r0   r1   r*   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r=   r   r   r   r>      s@   


zQTAttB.process_coarse_levelc	              	   C   s  |j \}	}
}}|j d | j }t|d|	d| j|}t|d|	d| j|}||	|
|d d|d d}t|d|	dd| j|}|d }g }dD ]}dD ]}|d | | |d  | }|| qNqJtj|d	d
}t|| ||	d|d | j}d|d  }tj	|| dd
}|
|	dd|d | j}||	dd|d | jddddd}tj|d|dd\}}t|| |}tj||dd}t|d|d dd}t|d|d dd}||||fS )Nr   r   r   r?   r@   rA   rB   r   rC   r   r   r   r   Tr   rE   rF   rG   )r%   r   r   r&   rI   r'   rJ   r   rK   r)   rM   rN   r*   r   rO   )r   r/   r0   r1   r:   rP   rQ   r*   rR   r2   r3   r4   r5   r6   rS   rT   rU   rV   r7   r8   r9   r;   r=   r   r   r   rW      s   	



zQTAttB.process_fine_levelNc                 C   s  |d j d }g }| jd }ttt|t|t|D ]R\}	\}
}}|j \}}}}|	dkr;| |
|||\}}}}n#|}| j|	 }|	t|d krLdnd}| |
|||||||\}}}}|| t	
|| || g}qd}t	j| jdd}t|D ]u\}	}| jr||	d   j dd \}}| j|	 ||	d   }|	dkr| jrt|d| jd	}|| ||	  }q}|||	  }q}| jrt|d
| jddd}|d|| ||	   }n|d|||	   }t|ddd||	  j d d}q}|S )rX   r   r   TFr   r   Nzb (hd d) H W -> b (H W) hd d)hdz.b (hd d) (H t1) (W t2) -> b (H W) (t1 t2) hd dr?   )r   rH   r[   rY   rZ   )r%   r   r]   r^   r_   r>   r`   rW   rI   r'   rJ   r)   r}   r~   r   r   r   rL   )r   ra   rb   rc   rd   re   r2   rf   r*   rg   r/   r0   r1   r3   r4   r5   r9   r=   r:   r;   rQ   rR   rP   rh   r}   ri   r\   Wr~   r   r   r   rj   .  sr   



zQTAttB.forwardrk   rl   rm   r   r   r   r   rr      s    !
?rr   )r'   torch.nnrw   torch.nn.functional
functionalFeinops.einopsr   >modelscope.ops.quadtree_attention.functions.quadtree_attentionr   r   Moduler   rr   r   r   r   r   <module>   s    /