o
    پi                     @   s   d dl Z d dlmZmZmZmZmZ d dlZd dlm	  m
Z d dlm	Z	 d dlmZ d dlmZ d dlmZmZ d dlmZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dl m!Z! G dd deZ"G dd de	j#Z$e$gZ%dS )    N)IterableListOptionalSetTuple)nn)POINTSV15ChatConfig)QuantizationConfig)/MultiModalityDataPaddingPatternMultimodalTokensgeneral_mm_embed_routine)ModalityMultimodalDataItemMultimodalInputs)ForwardBatch)default_weight_loader)Qwen2ForCausalLM)Qwen2VisionPatchMergerQwen2VisionTransformer)
add_prefixc                       sZ   e Zd Z			ddededee deddf
 fd	d
Zde	j
de	j
de	j
fddZ  ZS )$Qwen2VisionTransformerForNavitPOINTSư>N vision_confignorm_epsquant_configprefixreturnc                    s   t  j||||d d S )N)r   r   r   )super__init__)selfr   r   r   r   	__class__ U/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/models/points_v15_chat.pyr      s   
z-Qwen2VisionTransformerForNavitPOINTS.__init__xgrid_thwc                 C   s   |j | j| jd}| |}| |}tj||fdd}| | f}t	|d d df |d d df  |d d df j
dtjd}t|dd	d}|d}| jD ]	}||||d
}qT|S )N)devicedtypedim      r   )r*   r'   )r+   r   constant)
cu_seqlensposition_embeddings)tor&   r'   patch_embedrot_pos_embtorchcatcossinrepeat_interleavecumsumint32Fpad	unsqueezeblocks)r   r$   r%   rotary_pos_embembr/   r.   blkr"   r"   r#   forward)   s   

,

z,Qwen2VisionTransformerForNavitPOINTS.forward)r   Nr   )__name__
__module____qualname__r   floatr   r	   strr   r3   TensorrA   __classcell__r"   r"   r    r#   r      s,    r   c                	       s   e Zd Z		ddedee deddf fddZd	ee	 d
e
fddZdee dejfddZ	dd	ejdejdedefddZdeeeejf  fddZ  ZS )POINTSV15ChatModelNr   configr   r   r   c                    s   t    d|j_d|_|| _|| _t|j}dg|_	t
||td|d| _t|j|td|d| _t|jjd|td	|d
| _d S )Nflash_attention_2Fr   llm)rJ   r   r   vision_encoder)r   r   i   vision_projector)d_modelcontext_dimr   r   )r   r   
llm_config_attn_implementation_attn_implementation_autosetrJ   r   copydeepcopyarchitecturesr   r   rL   r   r   rM   r   hidden_sizerN   )r   rJ   r   r   kwargsrQ   r    r"   r#   r   F   s.   
zPOINTSV15ChatModel.__init__	input_ids	mm_inputsc                 C   s   t  }|||S )N)r
   pad_input_tokens)r   rY   rZ   patternr"   r"   r#   pad_input_idsh   s   z POINTSV15ChatModel.pad_input_idsitemsc                 C   s   t jdd |D dd| jj}t jdd |D dd}| dks)J | | dks5J | | j||d}| |}|S )Nc                 S      g | ]}|j qS r"   )feature.0itemr"   r"   r#   
<listcomp>m       z8POINTSV15ChatModel.get_image_feature.<locals>.<listcomp>r   r)   c                 S   r_   r"   )image_grid_thwra   r"   r"   r#   rd   p   re   r,   )r%   )r3   r4   typerM   r'   concatr*   rN   )r   r^   pixel_valuesrf   image_featuresr"   r"   r#   get_image_featurel   s   
z$POINTSV15ChatModel.get_image_featureF	positionsforward_batchget_embeddingc                 C   s    t ||| jtj| ji|d}|S )N)rY   rm   language_modeldata_embedding_funcsrl   )r   rL   r   IMAGErk   )r   rY   rl   rm   rn   hidden_statesr"   r"   r#   rA   y   s   
zPOINTSV15ChatModel.forwardweightsc              	   C   s   g d}t |  }t }|D ]i\}}d|v rq|D ](\}}}	||vr$q|||}|dr4||vr4q|| }
|
j}||
||	  n5d|v rM|dd}z|drY||vrYW q|| }
W n tyl   t|   w t	|
dt
}||
| qd S )N))qkv_projq_projq)rt   k_projk)rt   v_projv)gate_up_proj	gate_projr   )r{   up_projr+   zrotary_emb.inv_freqz.biasrM   z	attn.qkv.zattn.qkv_proj.weight_loader)dictnamed_parameterssetreplaceendswithr~   KeyErrorprintkeysgetattrr   )r   rs   stacked_params_mappingparams_dictloaded_paramsnameloaded_weight
param_nameweight_nameshard_idparamr~   r"   r"   r#   load_weights   s<   
zPOINTSV15ChatModel.load_weights)Nr   )F)rB   rC   rD   r   r   r	   rF   r   r   intr   r]   r   r3   rG   rk   r   boolrA   r   r   r   rH   r"   r"   r    r#   rI   E   s2    "
$rI   )&rT   typingr   r   r   r   r   r3   torch.nn.functionalr   
functionalr:   "sglang.srt.configs.points_v15_chatr   *sglang.srt.layers.quantization.base_configr	   sglang.srt.managers.mm_utilsr
   r   "sglang.srt.managers.schedule_batchr   r   r   ,sglang.srt.model_executor.forward_batch_infor   $sglang.srt.model_loader.weight_utilsr   sglang.srt.models.qwen2r   sglang.srt.models.qwen2_vlr   r   sglang.srt.utilsr   r   ModulerI   
EntryClassr"   r"   r"   r#   <module>   s"    +
u