o
    -i,@                     @   s0  d dl mZ d dlZd dlmZ d dlmZ d dlmZm	Z	 d dlm
Z d dlmZ d dlmZ d dlmZmZmZ d d	lmZ d d
lmZ G dd dejZG dd dejZG dd dejZG dd dejZG dd dejZG dd deZ
G dd dejZG dd dejZG dd dejZ dS )    )IterableN)
SwinConfig)SwinEmbeddingsSwinPatchMerging)	SwinLayer)meshgrid)
get_act_fn)ColumnParallelLinearQKVParallelLinearRowParallelLinear)QuantizationConfig)default_weight_loaderc                       s   e Zd Z		ddedededededB ded	df fd
dZdd Zd	e	j
fddZ		dde	j
de	jdB dedB d	ee	j
df fddZ  ZS )SwinSelfAttentionN configdim	num_headswindow_sizequant_configprefixreturnc                    s  t    || dkrtd| d| d|| _t|| | _| j| j | _t|tr.|n||f| _	| jd | _
ttd| j	d  d d| j	d  d  || _t| j	d }t| j	d }tt||gdd	}	t|	d}
|
d d d d d f |
d d d d d f  }|ddd }|d d d d df  | j	d d 7  < |d d d d df  | j	d d 7  < |d d d d df  d| j	d  d 9  < |d
}tj|dd| _t|| j| j|j|| dd| _d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()g            ij)indexingF)requires_gradz.qkv)hidden_size	head_sizetotal_num_headsbiasr   r   )super__init__
ValueErrornum_attention_headsintattention_head_sizeall_head_size
isinstancer   r   scalenn	Parametertorchzerosrelative_position_bias_tablearangestackr   flattenpermute
contiguoussumrelative_position_indexr
   qkv_biasqkv)selfr   r   r   r   r   r   coords_hcoords_wcoordscoords_flattenrelative_coordsr6   	__class__ \/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/swin.pyr#      sR   
	$,((,
zSwinSelfAttention.__init__c                 C   s6   |  d d | j| jf }||}|ddddS )Nr   r   r   r      )sizer%   r'   viewr3   )r9   xnew_x_shaperA   rA   rB   transpose_for_scoresQ   s   
z&SwinSelfAttention.transpose_for_scoresc                 C   s\   | j | jd }|| jd | jd  | jd | jd  d}|ddd }|dS )Nr   r   r   r   )r/   r6   rE   r   r3   r4   	unsqueeze)r9   relative_position_biasrA   rA   rB   _get_rel_pos_biasY   s   

z#SwinSelfAttention._get_rel_pos_biasFhidden_statesattention_maskoutput_attentions.c                 C   s  |j \}}}| |\}}|jddd\}	}
}| |
}
| |}| |	}	|  }|d urY|j d }|d|d|||| || j||}||dd }|d| j||}t	j
jj|	|
||dd}d }|dddd }| d d	 | jf }||}|r||f}|S |f}|S )
NrC   r   )r   r   r           )	attn_mask	dropout_pr   )shaper8   chunkrH   rK   rE   expandr%   rI   r-   r+   
functionalscaled_dot_product_attentionr3   r4   rD   r(   )r9   rL   rM   rN   
batch_sizer   num_channels
qkv_output_query_layer	key_layervalue_layerattention_scores
mask_shapeattention_mask_expandedcontext_layerattention_probsnew_context_layer_shapeoutputsrA   rA   rB   forwarde   sR   







zSwinSelfAttention.forwardNr   NF)__name__
__module____qualname__r   r&   r   strr#   rH   r-   TensorrK   FloatTensorbooltuplerf   __classcell__rA   rA   r?   rB   r      s>    9r   c                       sX   e Zd Z		ddedededB deddf
 fdd	Zd
ej	dej	dej	fddZ
  ZS )SwinSelfOutputNr   r   r   r   r   r   c                    s&   t    t|||| dd| _d S )N.dense)
input_sizeoutput_sizer   r   )r"   r#   r   denser9   r   r   r   r   r?   rA   rB   r#      s   
zSwinSelfOutput.__init__rL   input_tensorc                 C   s   |  |\}}|S Nrv   )r9   rL   rx   r[   rA   rA   rB   rf      s   zSwinSelfOutput.forwardrg   ri   rj   rk   r   r&   r   rl   r#   r-   rm   rf   rq   rA   rA   r?   rB   rr      s*    rr   c                       sv   e Zd Z		ddedededededB ded	df fd
dZ		ddej	dej
dB dedB d	eej	 fddZ  ZS )SwinAttentionNr   r   r   r   r   r   r   r   c                    sJ   t    t|||||| dd| _t|||| dd| _t | _d S )Nz.selfr   r   .output)r"   r#   r   r9   rr   outputsetpruned_heads)r9   r   r   r   r   r   r   r?   rA   rB   r#      s   
	zSwinAttention.__init__FrL   rM   rN   c                 C   s4   |  |||}| |d |}|f|dd   }|S )Nr   r   )r9   r   )r9   rL   rM   rN   self_outputsattention_outputre   rA   rA   rB   rf      s   zSwinAttention.forwardrg   rh   )ri   rj   rk   r   r&   r   rl   r#   r-   rm   rn   ro   rp   rf   rq   rA   rA   r?   rB   r|      s:    r|   c                       R   e Zd Z		ddedededB deddf
 fdd	Zd
ej	dej	fddZ
  ZS )SwinIntermediateNr   r   r   r   r   r   c                    s<   t    t|t|j| || dd| _t|j| _d S Nrs   r}   )	r"   r#   r	   r&   	mlp_ratiorv   r   
hidden_actintermediate_act_fnrw   r?   rA   rB   r#      s   
zSwinIntermediate.__init__rL   c                 C   s   |  |\}}| |}|S ry   )rv   r   r9   rL   r[   rA   rA   rB   rf      s   
zSwinIntermediate.forwardrg   r{   rA   rA   r?   rB   r      s    r   c                       r   )
SwinOutputNr   r   r   r   r   r   c                    s0   t    tt|j| ||| dd| _d S r   )r"   r#   r   r&   r   rv   rw   r?   rA   rB   r#      s   
zSwinOutput.__init__rL   c                 C   s   |  |\}}|S ry   rz   r   rA   rA   rB   rf      s   zSwinOutput.forwardrg   r{   rA   rA   r?   rB   r      s    r   c                       sP   e Zd Z				ddedededed	ed
ededB deddf fddZ  Z	S )r   rO   r   Nr   r   r   input_resolutionr   drop_path_rate
shift_sizer   r   r   c	           	         sj   t  j||||||d t|||| j|| dd| _t|||| dd| _t|||| dd| _d S )N)r   r   r   r   r   r   z
.attention)r   r   r   z.intermediater}   r~   )	r"   r#   r|   r   	attentionr   intermediater   r   )	r9   r   r   r   r   r   r   r   r   r?   rA   rB   r#      s,   	zSwinLayer.__init__)rO   r   Nr   )
ri   rj   rk   r   r&   floatr   rl   r#   rq   rA   rA   r?   rB   r      s0    	
r   c                       s   e Zd Z			ddedededededee d	edB d
edB de	ddf fddZ
		ddejdeeef dedB dedB deej f
ddZ  ZS )	SwinStageNr   r   r   r   depthr   	drop_path
downsampler   r   r   c
           
   	      sl   t     | _| _t fddt|D | _|d ur.|tjd| _	nd | _	d| _
d S )Nc                    sH   g | ] }t  | |d  dkrdn jd   d| dqS )r   r   z.blocks.)r   r   r   r   r   r   r   r   )r   r   .0	layer_idxr   r   r   r   r   r   r   rA   rB   
<listcomp>5  s    z&SwinStage.__init__.<locals>.<listcomp>)r   
norm_layerF)r"   r#   r   r   r+   
ModuleListrangeblocks	LayerNormr   pointing)
r9   r   r   r   r   r   r   r   r   r   r?   r   rB   r#   %  s   


zSwinStage.__init__FrL   input_dimensionsrN   always_partitionc                 C   s   |\}}t | jD ]\}}|||||}	|	d }q	|}
| jd ur:|d d |d d }}||||f}| |
|}n||||f}||
|f}|rO||	dd  7 }|S )Nr   r   r   )	enumerater   r   )r9   rL   r   rN   r   heightwidthilayer_modulelayer_outputs!hidden_states_before_downsamplingheight_downsampledwidth_downsampledoutput_dimensionsstage_outputsrA   rA   rB   rf   N  s0   

zSwinStage.forward)NNr   FF)ri   rj   rk   r   r&   listr   r   r   rl   r#   r-   rm   rp   ro   rf   rq   rA   rA   r?   rB   r   $  sL    		
-
r   c                       sx   e Zd Z		ddedededB deddf
 fdd	Z	
	
ddej	de
eef dedB dedB de
ej	 f
ddZ  ZS )SwinEncoderNr   r   	grid_sizer   r   r   c                    sn   t    t j_ _dd tjd jt	 jddD t
 fddtjD _d S )Nc                 S   s   g | ]}|  qS rA   )item)r   rF   rA   rA   rB   r     s    z(SwinEncoder.__init__.<locals>.<listcomp>r   cpu)devicec                    s   g | ]L}t  t jd |  d d |  d d |  f j|  j| t jd| t jd|d   |jd k rCtnd d| d	qS )r   r   r   Nz.layers.)	r   r   r   r   r   r   r   r   r   )r   r&   	embed_dimdepthsr   r5   
num_layersr   r   r   dprr   r   r   r9   rA   rB   r     s,    )r"   r#   lenr   r   r   r-   linspacer   r5   r+   r   r   layers)r9   r   r   r   r   r?   r   rB   r#   v  s   

zSwinEncoder.__init__FrL   r   rN   r   c           	      C   sF   t | jD ]\}}|||||}|d }|d }|d |d f}q|S )Nr   r   rR   r   )r   r   )	r9   rL   r   rN   r   r   r   r   r   rA   rA   rB   rf     s   zSwinEncoder.forwardrg   r   )ri   rj   rk   r   r&   r   rl   r#   r-   rm   rp   ro   rf   rq   rA   rA   r?   rB   r   u  s6    .
r   c                	       s   e Zd ZU eed< 		ddededB deddf fdd	Z		dd
ej	dB de
dB deej fddZdeeeejf  dee fddZ  ZS )	SwinModelconfig_classNr   r   r   r   r   c                    s`   t    || _t|j| _t|jd| jd   | _t	|| _
t|| j
j|| dd| _d S )Nr   r   z.encoderr}   )r"   r#   r   r   r   r   r&   r   num_featuresr   
embeddingsr   
patch_gridencoder)r9   r   r   r   r?   rA   rB   r#     s   

zSwinModel.__init__pixel_valuesrN   c                 C   s"   |  |\}}| j|||d}|S )N)rN   )r   r   )r9   r   rN   embedding_outputr   encoder_outputsrA   rA   rB   rf     s   zSwinModel.forwardweightsc                 C   s   g d}t |  }t }|D ]9\}}|D ]\}}}	||vrq|||}|| }
|
j}||
||	  n|| }
t|
dt}||
| || q|S )N))r8   queryq)r8   keyk)r8   valuevweight_loader)dictnamed_parametersr   replacer   getattrr   add)r9   r   stacked_params_mappingparams_dictloaded_paramsnameloaded_weight
param_nameweight_nameshard_idparamr   rA   rA   rB   load_weights  s"   
zSwinModel.load_weightsrg   )NN)ri   rj   rk   r   __annotations__r   rl   r#   r-   rn   ro   rp   rm   rf   r   r   r   rq   rA   rA   r?   rB   r     s.   
 
,r   )!collections.abcr   r-   torch.nnr+   transformersr   &transformers.models.swin.modeling_swinr   r   r   HFSwinLayertransformers.pytorch_utilsr   %vllm.model_executor.layers.activationr   !vllm.model_executor.layers.linearr	   r
   r   'vllm.model_executor.layers.quantizationr   -vllm.model_executor.model_loader.weight_utilsr   Moduler   rr   r|   r   r   r   r   r   rA   rA   rA   rB   <module>   s*    $%QB