o
    ߥi0                     @   sr   d dl Z d dlZd dlmZ d dlm  mZ d dlmZ dd Z	dd Z
G dd dejZG d	d
 d
ejZdS )    N)	rearrangec                 C   sz   t jdd | D dd} t j| t|d ddd}t jdd |D dd}t j|t|d dd}tj|||d	}|S )
z
        weight list: list of conv1d weight ([out, in] * a)
        bias list: list of conv1d bias ([out] * a)
        agg: aggreagtion weights (a)
        x: input tensor (b, in, n)

        return output in (b, n, out)
    c                 S      g | ]}| d qS r   	unsqueeze.0w r
   ^/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/cv/vision_middleware/vim.py
<listcomp>       z_agg_conv1d.<locals>.<listcomp>r   dimz
a -> a 1 1   c                 S   r   r   r   r   r
   r
   r   r      r   a -> a 1weightbias)torchcatsumr   r   Fconv1dweight_list	bias_listaggxr   r   r
   r
   r   _agg_conv1d   s   
r   c                 C   sx   t jdd | D dd} t j| t|d dd}t jdd |D dd}t j|t|d dd}tj|||ddd	}|S )
z
        weight list: list of conv2d weight ([out, in, m, n] * a)
        bias list: list of conv2d bias ([out] * a)
        agg: aggregation weights (a)
        x: input tensor (b, in, p, q)

        return output in (b, out, p, q)
    c                 S   r   r   r   r   r
   r
   r   r   /   r   z_agg_conv2d.<locals>.<listcomp>r   r   za -> a 1 1 1 1c                 S   r   r   r   r   r
   r
   r   r   4   r   r      r   r   stridepadding)r   r   r   r   r   conv2dr   r
   r
   r   _agg_conv2d%   s   

r%   c                   @   s   e Zd ZdejfddZdS )	QuickGELUr   c                 C   s   |t d|  S )NgZd;?)r   sigmoid)selfr   r
   r
   r   forward@   s   zQuickGELU.forwardN)__name__
__module____qualname__r   Tensorr)   r
   r
   r
   r   r&   >   s    r&   c                       s<   e Zd Z fddZdd Zdd Zdd Zd	d
 Z  ZS )ViMc                    sj   t    t | _t | _t | _t | _t | _	t | _
t | _d| _g | _i | _i | _d S Nr   )super__init__r&   actnnParameterListadapter_conv_weightadapter_conv_biasadapter_up_weightadapter_up_biasadapter_down_weightadapter_down_biasnum_modules	task_listagg_weights	agg_algos)r(   	__class__r
   r   r1   F   s   







zViM.__init__c                 C   s   t || _|D ]D}| jt|d  | jt|d  | jt|d  | jt|d  | j	t|d  | j
t|d  qd S )Nzadapter_conv.weightzadapter_conv.biaszadapter_up.weightzadapter_up.biaszadapter_down.weightzadapter_down.bias)lenr;   r5   appendr3   	Parameterr6   r7   r8   r9   r:   )r(   vim_list
state_dictr
   r
   r   register_ViMZ   s*   
zViM.register_ViMc                 C   s8   |j d | jks
J | j| || j|< || j|< d S r/   )shaper;   r<   rB   r=   r>   )r(   	task_namer=   agg_algor
   r
   r   register_taskl   s   
zViM.register_taskc                 C   s@   || j v sJ | j| }|dkr| || j| S td|)NzEns-MoEz6Aggregation algorithm [{}] is currently not supported!)r<   r>   forward_ens_moer=   NotImplementedErrorformat)r(   r   rH   rI   r
   r
   r   r)   s   s   
zViM.forwardc                 C   s  |}|j d }|jt|d |ddd\}}|d | }|d | }tj|dd}	tj|dd}
|
d||	}|j \}}}|	ddd}d }t
| jD ]}|| dkrtj|| j| d| j| d}| |}tt|dd }|d d d d dd f |d||}tj|| j| | j| ddd	}t|d
}|d d d d d df |ddd}tj|| j| | j| ddd	}t|d}tj||gdd}| |}tj|| j| d| j| d}|d u r|||  }qM||||  7 }qM|	dddS )Nr   r    r   T)requires_gradr   r   r!   zb o p q -> b o (p q)zb o 1 1 -> b o 1)rG   topkminsizer   softmaxr   
zeros_likescatterpermuteranger;   r   r9   r   r:   r2   intmathsqrtreshaper$   r5   r6   r   r   r7   r8   )r(   r   r   logitsk
top_logitstop_indicestop_k_logitstop_k_indicestop_k_gateszerosgatesNBCoutputix_downnum_patch_sidex_patchx_clsx_upr
   r
   r   rK   ~   st   



&

zViM.forward_ens_moe)	r*   r+   r,   r1   rF   rJ   r)   rK   __classcell__r
   r
   r?   r   r.   D   s    r.   )rY   r   torch.nnr3   torch.nn.functional
functionalr   einopsr   r   r%   Moduler&   r.   r
   r
   r
   r   <module>   s   