o
    Ti                     @   s4   d dl mZ d dlZddlmZ G dd deZdS )    )abstractmethodN   )HybridEngineContainerc                       sh   e Zd Z fddZedd Zd fdd	Z fdd	Zd
d Zdd Z	dddZ
 fddZ  ZS )HybridSplitQKVContainerc                    s   t  |||| |   d S N)superset_attention	set_q_k_v)selfqkvwqkvbdense_wdense_b	__class__ i/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/module_inject/containers/features/split_qkv.pyr      s   z%HybridSplitQKVContainer.set_attentionc                 C   s   t d)a4  
        In `set_q_k_v`, it is necessary to populate the following variables (where appropriate)
        for the given model:
            self.qw: q weight
            self.qb: q bias
            self.kw: k weight
            self.kb: k bias
            self.vw: v weight
            self.vb: v bias
        zA set_q_k_v() function must be defined in the model container                                     in order to set the unfused q, k, and v tensors.)NotImplementedError)r
   r   r   r   r	      s   z!HybridSplitQKVContainer.set_q_k_vFc                    s   | j jjd u rV| j jj| jf| j jj| jf| j jj| jf| j jj	| j
f| j jj| jf| j jj| jfg}|D ]\}}|d urQ|j|d | jjd |j  |||dnd }q5d S t | d S )Nr   )int8allocate_tensor)module	attention	attn_qkvwattn_qwqwattn_qbqbattn_kwkwattn_kbkbattn_vwvwattn_vbvbcopyshapemp_sizer   attention_qkv_mp)r
   
mp_replacereversed_dimparamsdstsrcr   r   r   r(   !   s$   z(HybridSplitQKVContainer.attention_qkv_mpc                    sp   t    | jjj| jf| jjj| jf| jjj| j	f| jjj
| jf| jjj| jf| jjj| jfg}| | d S r   )r   release_qkvr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   _release_params)r
   split_qkv_paramsr   r   r   r.   3   s   
	z#HybridSplitQKVContainer.release_qkvc                 C   s  | j j| jjd | j jd < | jj| jj| j jd d| j jd  < | jj| jjd| j jd  d < | j j| jj| jjg}| jjd | j jd  | j _| jj| j jd d| j jd   | j_| jjd| j jd  d  | j_| jd ur| jj| jjd | j jd < | jj| jj| j jd d| j jd  < | j	j| jjd| j jd  d < |
| jj| jj| j	jg | jjd | j jd  | j_| jj| j jd d| j jd   | j_| jjd| j jd  d  | j	_|D ]}~qd S Nr      )r   datar   r&   r   r"   r   r   r    r$   extendr
   qkv_datar3   r   r   r   	reset_qkv@   s$   ( ( 
( ( z!HybridSplitQKVContainer.reset_qkvc                 C   sf  | j jjdu r6tj| jjd d | jjd | jj| jjd| j j_tj| jjd d | jj| jjd| j j_	| jj
| j jjj
d| jjd < | jj
| j jj	j
d| jjd < | jj
| j jjj
| jjd d| jjd  < | jj
| j jj	j
| jjd d| jjd  < | jj
| j jjj
d| jjd  d< | jj
| j jj	j
d| jjd  d< | jj
| jj
| jj
| jj
| jj
| jj
g}| j jjj
d| jjd  | j_
| j jj	j
d| jjd  | j_
| j jjj
| jjd d| jjd   | j_
| j jj	j
| jjd d| jjd   | j_
| j jjj
d| jjd  d | j_
| j jj	j
d| jjd  d | j_
|D ]}~q,dS )zj
        WIP - experimental and likely to be changed/improved.
        Unused by keeping for now.
        Nr      )dtypedevicer2   )r   r   r   torchemptyr   r&   r9   r:   	attn_qkvbr3   r   r   r    r"   r$   r5   r   r   r   reset_qkv_experimentalY   s@   
  ,,$$  ,,$$z.HybridSplitQKVContainer.reset_qkv_experimentalc                 C   sf  | j | jj_| j| jj_|s| j| jj_| j| jj_	| jd | j
jd d d f | j
_| j| j
jd d| j
jd  d d f | j_| j| j
jd d d d d f | j_| jd ur| jd | j
jd  | j_| j| j
jd d| j
jd   | j_| j| j
jd d d  | j_d S d S | j
| jj_| j| jj_| j| jj_| j| jj_| j| jj_| j| jj_d S r1   )r   r   r   attn_owr   attn_obr   r   r   r=   r   r&   r3   r   r"   r   r    r$   r   r   r   r   r!   r#   )r
   
Z3_enabledr   r   r   set_attn_params_wo_copy~   s&   ".&
&"z/HybridSplitQKVContainer.set_attn_params_wo_copyc                    s0   t   }|| j| j| j| j| j| jg |S r   )	r   get_attn_paramsr4   r   r   r   r    r"   r$   )r
   r+   r   r   r   rC      s   
"z'HybridSplitQKVContainer.get_attn_params)F)__name__
__module____qualname__r   r   r	   r(   r.   r7   r>   rB   rC   __classcell__r   r   r   r   r      s    

%r   )abcr   r;   hybrid_enginer   r   r   r   r   r   <module>   s   