o پi] @s ddlZddlmZddlmZddlmZmZmZm Z m Z ddl Z ddl m Z ddlm mZddlmZddlmZzddlmZWn eyPdZYnwddlmZ  d1d e jd e jd e jd ee jd ee jf ddZ  d1d e jd e jd e jd ee jd ee jde jf ddZeedZddZde jde jde jdee je jffddZ Gddde j!Z"Gddde j!Z#Gdd d e j!Z$Gd!d"d"e j!Z%Gd#d$d$e j!Z&Gd%d&d&e j!Z' 'd2d(e jd)e jd*e(e)e)fdee jfd+d,Z*Gd-d.d.e j!Z+Gd/d0d0eZ,dS)3N)deepcopy)cached_property)ListOptionalSequenceTupleUnion)ACT2FN)PreTrainedModel)flash_attn_varlen_func) MoonViTConfigqkv q_cu_seqlens k_cu_seqlensc Cs$tdurtd||kr |kr dks%JdJd|d|jdks2Jd|d|jdkrG|jdksLJdJd|jtjtjfvs^Jd |jd |d d|dd }|d d|dd }t|||||||d d }|j dd}|S)aMulti-head attention using flash attention 2. This function is used to handle the case where the query, key, and value are packed. Args: q, k, v: tensor of shape (tot_seqlens, num_heads, head_dim). q_cu_seqlens (torch.Tensor): cumulative sequence lengths of q. The first element should be 0 and the last element should be q.shape[0]. k_cu_seqlens (torch.Tensor): cumulative sequence lengths of k. The first element should be 0 and the last element should be k.shape[0]. Returns: output: shape (batch_size, seqlen, dim) or (tot_seqlens, dim) if packing, where dim = num_heads * head_dim NzWflash_attn is not installed, this function needs flash_attn_varlen_func from flash_attnq, k, v must have 3 dimsr#q_cu_seqlens must sum to q.shape[0]z#k_cu_seqlens must sum to k.shape[0]zunsupported dtype z for multihead attnF)causal) start_dim) r ImportErrordimshapedtypetorchbfloat16float16maxitemflatten)r rrrr max_seqlen_q max_seqlen_kattn_outr'U/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/models/kimi_vl_moonvit.pymultihead_attention?s>:$    r)returnc Cs||kr|krdksJdJd|d|jdks*Jd|jd}tjd||g|jtjd}tdt|D]}d|d ||d||||d||f<qC|dd}|dd}|dd}t j ||||d d }|dd}| |d}|S) aMulti-head attention using torch scaled dot product attention. This function is used to handle the case where the query, key, and value are packed. Args: q, k, v: tensor of shape (tot_seqlens, num_heads, head_dim). q_cu_seqlens (torch.Tensor): cumulative sequence lengths of q. The first element should be 0 and the last element should be q.shape[0]. k_cu_seqlens (torch.Tensor): cumulative sequence lengths of k. The first element should be 0 and the last element should be k.shape[0]. Returns: output: shape (batch_size, seqlen, dim) or (tot_seqlens, dim) if packing, where dim = num_heads * head_dim rrrrrrdevicerT.g) dropout_p) rrrzerosr,boolrangelen transposeFscaled_dot_product_attentionreshape) r rrrr seq_lengthattention_maski attn_outputr'r'r(sdpa_attentionss*:      r:)flash_attention_2sdpacCs|j|jdksJ|j|jf|jdd|jddks&J|j|jf|jdd|jdks:J|j|jf|jtjksEJ|jdS)Nrrr)ndimrrr complex64)x freqs_cisr'r'r(_apply_rope_input_validations ,(rBxqxkrAcCst||t|||d}t|jg|jddddR}t|jg|jddddR}t||d}t||d}| || |fS)a Args: (The leading dimensions of all inputs should be the same) xq: query, tensor of shape (..., num_heads, head_dim) xk: key, tensor of shape (..., num_heads, head_dim) freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64. It contains the precomputed cis(freqs) for each position in the 2D grid. Returns: xq_out, xk_out: tensors of shape (..., num_heads, head_dim) rNrr=) rB unsqueezerview_as_complexfloatviewr view_as_realr#type_as)rCrDrAxq_xk_xq_outxk_outr'r'r( apply_ropes  ,,rOc sZeZdZ ddededededdf fdd Zd d Zd ejd ejdejfddZ Z S)Learnable2DInterpPosEmbbicubicheightwidthrinterpolation_moder*Ncs>t||_||_||_tt||||_ | dSN) super__init__rRrSrTnn Parameterremptyweightreset_parameters)selfrRrSrrT __class__r'r(rWs  z Learnable2DInterpPosEmb.__init__cCstj|jdSrU)rXinitnormal_r[r]r'r'r(r\sz(Learnable2DInterpPosEmb.reset_parametersr@grid_hwscCsg}|D]4}||jjddkr||jjddq|tj|jdd||j d ddjddq|t |}|S)Nrr)end_dim)r=rrr)sizemode)rr=r) tolistr[rappendr#r3 interpolatepermuterErTsqueezercat)r]r@rcpos_embsroutr'r'r(forwards"  zLearnable2DInterpPosEmb.forward)rQ) __name__ __module__ __qualname__intstrrWr\rTensorro __classcell__r'r'r^r(rPs $rPc sheZdZ    ddededeeeeeffdedef fd d Zd ejd ejd ejfddZ Z S)MoonVisionPatchEmbedrryryout_dimin_dim patch_sizepos_emb_height pos_emb_widthcstt|ttfsJdt|t|tr||f}t|dks+Jd|||_tj ||||d|_ t |||d|_ dS)NzInvalid patch_size type: r=z,Expected patch_size to be a tuple of 2, got ) kernel_sizestride)rRrSr) rVrW isinstancersrtyper1r|rXConv2dprojrPpos_emb)r]rzr{r|r}r~r^r'r(rWs$    zMoonVisionPatchEmbed.__init__r@grid_hwr*cCs(|||dd}|||}|S)z Args: x (L, Channels): input tensor grid_hw (N, 2): grid height and width Returns: (L, Cout) tensor rr)rrHrer)r]r@rr'r'r(ros zMoonVisionPatchEmbed.forward)rrxryry) rprqrrrsrrrWrrurorvr'r'r^r(rws"$rwcseZdZdZ ddededeffdd Zd d Zed ej fd d Z dej d ej fddZ dej dej d ej fddZ Z S) Rope2DPosEmbaR2D rotary position embedding with multi-resolution support. This class is intended to be used in the following way: 1. Before training, create an instance of Rope2DPosEmb. This instance will hold the precomputed cis. 2. Before each forward pass, call `get_freqs_cis_by_*` to get the `freqs_cis` tensor for this iteration. 3. During the forward pass, pass the `freqs_cis` tensor to each attention layer, and call `apply` just before each attention operation. The rope is shared across all attention layers and all heads. Refs: - RoFormer: https://arxiv.org/abs/2104.09864 - VisionLLaMA: https://arxiv.org/abs/2403.00522 - https://github.com/Meituan-AutoML/VisionLLaMA/blob/main/dit/models.py Args: dim (int): usually the multi-head attention dimension, should be divisible by 4 (TODO: relax this constraint if needed) max_height (int): the maximum height of the 2D grid max_width (int): the maximum width of the 2D grid theta_base (float): the base of the theta device (str): the device to store the precomputed cis 'cudar max_height max_widthcsBt||_|jddksJd||_||_||_||_dS)Nrzdim must be divisible by 4)rVrWrrr theta_baser,)r]rrrrr,r^r'r(rW&s  zRope2DPosEmb.__init__cCs$d|jd|jd|jd|jS)Nzdim=z , max_height=z , max_width=z , theta_base=)rrrrrbr'r'r( extra_repr1s$zRope2DPosEmb.extra_reprr*c Cs|j|j}td||j}||j}||j}td|jdd|jd|j}d|j||j}t ||}t ||}t t ||} t t ||} tj | j dd| j ddgdd} | |j|jd} | S)aCalculate the cis(freqs) for each position in the 2D grid. Return: complex tensor of shape (max_height, max_width, dim//2) and value: height axis: ret[h, w, 2*i] = cis(h * theta_base**(-4*i/dim)) weight axis: ret[h, w, 2*i+1] = cis(w * theta_base**(-4*i/dim)) with (i in [0, dim//4)) note: `cis` is a mathematical notation defined by cis x = cos x + i sin x, rrNg?rr)rrrarangerGtor,rrouterpolar ones_likerlrEr5) r]Nflat_posx_posy_pos dim_rangefreqsx_freqsy_freqsx_cisy_cisrAr'r'r(precomputed_freqs_cis4s   (z"Rope2DPosEmb.precomputed_freqs_cisrccsP|}tfdd|DsJ|jjftjfdd|Ddd}|S)z Args: grid_hws (torch.Tensor): containing list of (height, width) or (t, height, width) tuples. Returns: freqs_cis: tensor of shape (sum(t * height * width), dim//2) c3sH|]\}}d|kojknod|kojknVqdS)rN)rr.0hwrbr'r( Ys8 z8Rope2DPosEmb.get_freqs_cis_by_seqlens..cs4g|]\}}jd|d|fdjdqS)Nrr=)rr5rrrbr'r( as"z9Rope2DPosEmb.get_freqs_cis_by_seqlens..rr)rgallrrrrl)r]rcshapesrAr'rbr(get_freqs_cis_by_seqlensQs    z%Rope2DPosEmb.get_freqs_cis_by_seqlenspos_idx pos_idx_maskcCs|jdd|jkr|jddkr|j|jdks!J|j|jf|jtjks,J|j|j|jdf}tj|tj|jd}|j |d||d|f||<|S)a Args: pos_idx: tensor of shape (..., 2), It contains the (h, w) position indices of each 2D token. pos_idx_mask: a mask of shape (...), the leading dimensions should be the same as pos_idx. Rope will only be applied to the tokens with True mask. `freqs_cis` for the tokens with False mask with be ones. Return: freqs_cis: tensor of shape (..., dim//2) Nrr=r)rr,).r).r) rr>rrr/ronesr?r,r)r]rrshprAr'r'r(get_freqs_cis_by_idxis   z!Rope2DPosEmb.get_freqs_cis_by_idx)rr)rprqrr__doc__rsrWrrrrurrrrvr'r'r^r(rs* rcsBeZdZdZd deeffdd Zdejdejfdd Z Z S) MLP2zn Args: dims: [in_dim, hidden_dim, out_dim] bias: whether to use bias in linear layer. Tdimscstt|dks Jtj|d|d|d|_tj|d|d|d|_||_|j|jfD]}tjj |j t d|j d|jdurMtj|jq0dS)Nrrrbiasr=)std)rVrWr1rXLinearfc0fc1 activationr` trunc_normal_r[mathsqrt in_featuresrzeros_)r]rrrmr^r'r(rWs  z MLP2.__init__r@r*cCs||}||}||SrU)rrr)r]r@r'r'r(ros   z MLP2.forward)T) rprqrrrlistrsrWrrurorvr'r'r^r(rs rc seZdZdejdddededededef fd d Z dd e j d e j de e j fddZ dde j d e j de e j d fde j fddZZS)MoonVitEncoderLayerr;F)attn_implementationr attn_bias num_heads hidden_dimmlp_dimrrcst||_||_|j|j|_||_t||_t||_ t |||g||_ tj ||d|d|_ tj |||d|_dS)Nrr)rVrWrrhidden_size_per_attention_headrrX LayerNormnorm0norm1rmlprwqkvwo)r]rrrrrrr^r'r(rWs   zMoonVitEncoderLayer.__init__Nr@ cu_seqlensrope_freqs_cisc Cs||}|ddd|j|jf}|j|}tj|dd\}}}t|||\}}t|j } | |||||d} | | } | S)z} Args: x (torch.Tensor): (batch_size, seqlen, hidden_dim) cu_seqlens (torch.Tensor): Nrrr)rr) rrerrrHrunbindrOVL_VISION_ATTENTION_FUNCTIONSrr) r]r@rrxqkv qkv_shaperCrDxv attn_funcr&r'r'r(attention_qkvpackeds     z'MoonVitEncoderLayer.attention_qkvpacked hidden_statesr*cCsF|}||}|j|||d}||}|}|||}||}|S)a Args: hidden_states: non-packed (B, N, D) or packed (L, D). if non-packed, seqlens should be None, if packed, seqlens should be set Returns: output: same shape of input, non-packed (B, N, D) for non-packed input, (L, D) for packed input r)rrrr)r]rrrresidualr&r'r'r(ros zMoonVitEncoderLayer.forwardrU)rprqrrr3gelursrtr/rWrrurrrrorvr'r'r^r(rsB $ rcsJeZdZdedededdffdd Zdejd ejdejfd d ZZ S) MoonVitEncoderr num_layers block_cfgr*NcsRttdddd|_tfddt|D|_t||_ dS)Nrricsg|] }tdiqS)r')r)r_rr'r(rsz+MoonVitEncoder.__init__..) rVrWrrope_2drX ModuleListr0blocksrfinal_layernorm)r]rrrr^rr(rWs zMoonVitEncoder.__init__rrcCs|jj|d}ttjd|j|jd|dddf|dddff}|jdtjd}t |j D] \}}||||d}q1| |}|S)N)rcrr+r)rrr) rrrrlr.r,rcumsumint32 enumeraterr)r]rrrlengthsrrblockr'r'r(ros zMoonVitEncoder.forward) rprqrrrsdictrWrrurorvr'r'r^r(rs"rr=r=r@rmerge_kernel_sizecCs|d}g}d}|D]L}|d|d}}|||||} |\} } || || } } | | | | | |}|ddddd}|| | | | d}|||||7}q |S)Nrrrr=rr)rergrHrj contiguousrh)r@rrd_modeloutputspre_sumx_shaperRrSseq kernel_height kernel_width new_height new_width reshaped_seq padded_seqr'r'r( patch_mergers$    rc sXeZdZ   ddedeeefdededef fd d Zd ej d ej fd dZ Z S)MoonVitVLProjectorrh㈵> in_channelsr hidden_actln_epsrzcsjt||d|d|_tjj||d|_tj|j|jdd|_t||_ tj|j|dd|_ dS)Nrr)epsTr) rVrW hidden_sizerXrpre_normrlinear_1r actlinear_2)r]rrrrrzr^r'r(rW7s  zMoonVitVLProjector.__init__rr*cCs6||d|j}||}||}||}|S)Nr)rrHrrrr)r]rr'r'r(roGs    zMoonVitVLProjector.forward)rrr) rprqrrrsrrtrGrWrrurorvr'r'r^r(r5s  rcsTeZdZeZdZdgZdZdZdeffdd Z de j de j d e j fd d Z Z S) MoonVitPretrainedModelmoonvitPackingTransformerTconfigc sddlm}tj|g|Ri|t|}|j|_|j|_t|j|j|j |j d|_ t |j|j |j|j|j|d|jdd|_dS)Nr)GELUTanh)rzr|r}r~T)rrrrrr)rrr)transformers.activationsrrVrWrrr|rwrinit_pos_emb_heightinit_pos_emb_width patch_embedrnum_hidden_layersnum_attention_headsintermediate_size_attn_implementationencoder)r]rinputskwargsrr^r'r(rWVs,  zMoonVitPretrainedModel.__init__ pixel_valuesrr*cCs,|||}|||}t|||jd}|S)z Args: pixel_values (torch.Tensor): The input pixel values. grid_hw (torch.Tensor): The grid height and width. Returns: torch.Tensor: The output tokens. )r)r rrr)r]rrrr'r'r(roqs zMoonVitPretrainedModel.forward)rprqrrr config_class model_type_no_split_modules_supports_flash_attn_2_supports_sdparWrrurorvr'r'r^r(rOsr)NN)r)-rcopyr functoolsrtypingrrrrrrtorch.nnrXtorch.nn.functional functionalr3r r transformers.modeling_utilsr flash_attn.flash_attn_interfacer rsglang.srt.configsr rur)r:rrBtuplerOModulerPrwrrrrrrsrrrr'r'r'r(s+        8 + #,uR,