o پiB@stUddlZddlmZddlmZddlmZddlZddlm Z ddl m m Z ddl mZddlmZddlmZddlmZdd lmZmZmZdd lmZeeeefBZeed <eeeefej BZ!eed <d dZ"e"dZ#e"dZ$e"dZ%e"dZ&e"Z'Gddde j(Z)Gddde j(Z*Gddde j(Z+Gddde j,Z-Gddde j(Z.Gddde j(Z/dS)N)Iterable)repeat) TypeAlias) rearrange)PretrainedConfig)BaseModelOutput)QuantizationConfig)default_weight_loaderreplace_prefixreplace_substrings)InternVisionEncoder input_dim_tnorm_tcsfdd}|S)Ncs*t|trt|tst|Stt|SN) isinstancerstrtupler)xnK/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/models/radio.pyparse)sz_ntuple..parser)rrrrr_ntuple(s rc sTeZdZ    ddededededBdedBf fd d Zd ejfd d ZZ S)ClsTokenrTNndim num_tokensenabledregister_multiple num_registerscst||_||_d|_||_|r6|r||_n |r"||||_|d}tt ||j|||_ nd|_ |j|j|_ dS)Nr) super__init__rr!r#r nn Parametertorchrandntoken num_patches)selfrr r!r"r#scale __class__rrr&9s$ zClsToken.__init__rcCs@|jdur|S|jd|jddd}tj||gdd}|S)Nrrdim)r+ unsqueezeexpandshaper)cat)r-rr+rrrforwardYs zClsToken.forward)rTNN) __name__ __module__ __qualname__intboolr&r)Tensorr8 __classcell__rrr/rr8s" rcseZdZ            d7dededed ed ed ed edBd ededededBdedBdeffdd Zdej dej fddZ e ddZ e ddZ e ddZe dd Ze d!d"Zd#ej d$ejfd%d&Zd'ej d(ej fd)d*Zdej dej fd+d,Z  d8d-ej d.ej dBd/eeefdBdej fd0d1Z  d8d2ed.ej dBd/eeefdBdej fd3d4Zd2edeeeffd5d6ZZS)9ViTPatchGeneratorTFNr patch_size embed_dim input_dimsabs_posnormalize_patches cls_tokenmax_input_dims pos_dropoutreturn_pos_encnum_cls_tokensr"r# patch_biascsTtt|tr||f}|dur|}t|tr||f}tfdd|D}||k|_||_| |_t||d}|_ ||_ ||_ |d|_ |d|_ tfdd|D|_|j |j |_||_t|_t|fd| i||_|r|d}ttjd|j|fi|||_t|| || | d |_|rt||_dSt|_dS) Nc3s&|]}tt|VqdSr)r<mathceil.0drBrr s z-ViTPatchGenerator.__init__..)devicedtyperrc3s|]}|VqdSrrrOrRrrrSsbiasr$)r r!r"r#)r%r&rr<rcpe_moderIrJdictrBrErCnum_rowsnum_colsrDr,rH Im2Patches im_to_patchesViTPatchLinearembedderr'r(r)r* pos_embedrrG LayerNormIdentitypatch_normalizer)r-rBrCrDrErFrGrHrIrJrKr"r#rLrTrUfactoryr.r/rRrr&js\        zViTPatchGenerator.__init__rreturncCsL||}|j||jddd\}}||}||}|jr$||fS|S)Nr) input_size) embed_patches apply_pos_encr6rGrbrJ)r-rpatchespos_encrrrr8s   zViTPatchGenerator.forwardcC|jjSr)rGr!r-rrrapply_cls_tokenz!ViTPatchGenerator.apply_cls_tokencCrjr)rGr rkrrrrKrmz ViTPatchGenerator.num_cls_tokenscCrjr)rGr,rkrrrnum_cls_patchesrmz!ViTPatchGenerator.num_cls_patchescCrjr)rGr#rkrrrr#rmzViTPatchGenerator.num_registerscCs |j|jSr)rKr#rkrrrnum_skips zViTPatchGenerator.num_skip src_embed targ_embedcCs|j|jkr8tt|jd}|d|jdksJdt|d||d}tj||j|jfdddd }t|d }|j |dS) Nrrz*Unable to interpolate non-square embeddingzb (h w) c -> b c h w)hwbicubicTFsizemode align_corners antialiaszb c h w -> b (h w) c) r6r<rMsqrtrF interpolaterYrZdatacopy_)r-rprqsrc_sizerrr _load_embeds"   zViTPatchGenerator._load_embedsrc_proj_weighttarg_proj_weightcCs|j|jkr=tt|jdd}|dd|jdks!Jdt|dd||d}tj||j|jfddd d }t|d }|j |dS) Nrrrz+Unable to interpolate non-square patch sizezb (c h w) -> b c h w)crrrsrtTFruzb c h w -> b (c h w)) r6r<rMrzrr{r|rBr}r~)r-rrsrc_patch_sizerrr_load_projections.   z"ViTPatchGenerator._load_projectioncCs||}||}|Sr)r\r^)r-rrhrrrrfs  zViTPatchGenerator.embed_patchesrh patch_idxsrecCsr|js|S||jd||}|jr1|jdkr1tj|jddd|j|jd|jk}t ||d}n|}|||fS)Nrr)rUrT) rE get_pos_encr6trainingrIr)randrUrTwhere)r-rhrrerikeeps pos_enc_droprrrrgs zViTPatchGenerator.apply_pos_enc batch_sizecs||durj}n tfdd|D}||}|dur|S|ddd|jd}tj||jdddd|d}|S)Nc3s|]}|jVqdSrrRrOrkrrrS$sz0ViTPatchGenerator.get_pos_enc..r1rr)r3index)rDr_get_pos_embeddingsr4r5r6r)gather)r-rrrerDr_exp_patch_idxsrrkrrs zViTPatchGenerator.get_pos_enccs<|j|jfkr |jS|jd|j|jddddd}fdd}|jr|jrtd}t j |dd|j d d||}t d }| }t t j |dd|j d |||} || } |d| } t j| | gdd dd} t j |ddd|j d d| } t jddd|j d d |dd}t jddd|j d ddddf|dd}t j||gdd }|| | }|ddtj||ddd|dddd|j}nt}tj|||fddd|j}||}n||}|jddkrtj|ddd|j}|dddd}|S)Nrr1rrrcs\d|jdkr|dddddf}d|jdkr,|dddddf}|S)Nr.rr1)r6)r_rDrr window_select:s z.window_selectg?)rTg?r2)stepsrTNNbilinearzerosT)gridrw padding_moderx)rvrxrwr)rYrZr_reshapepermuterWrrMrzr)rrTlogexpstackclamp_linspacer5mul_sub_r{ grid_samplefloattorUmaxr|r6flatten)r-rrDr_r min_scaler. aspect_min aspect_maxaspectscale_xscale_yscale_xypos_xylin_xlin_ylin_xygrid_xymax_dimrrrr2s        z%ViTPatchGenerator._get_pos_embeddings) TFFNrAFrNNFNNr)r9r:r;r<r r=rr&r)r>r8propertyrlrKrnr#ror'r(rrrfrrgrrr?rrr/rr@is     G        "r@cs8eZdZdeffdd ZdejdejfddZZS)r[rBcst||_dSrr%r&rB)r-rBr/rrr&s  zIm2Patches.__init__rrdcCsb|jdkr|d}|ddd}|S|jd|j}|jd|j}t|d||j||jd}|S)Nrrrrr1z*b c (py yy) (px xx) -> b (py px) (c yy xx))pyyypxxx)rBrrr6r)r-rrhrrrrrr8s  zIm2Patches.forward) r9r:r;r<r&r)r>r8r?rrr/rr[sr[cs,eZdZddededeffdd ZZS)r]FrBrCrVc s,tjd|d|fd|i|||_dS)NrrrVr)r-rBrCrVrcr/rrr&s" zViTPatchLinear.__init__)F)r9r:r;r<r=r&r?rrr/rr]s$r]csveZdZddgiZ  ddededBddffdd ZdeeeefBfd d Z d d Z d e j de j fddZZS)RadioInternVisionModelqkvNconfig quant_configrdcszt||_|t|j|j\|_|_|_ t t |j |j|j}t |j|j|j|d|jd|_t||d|_dS)NT)rDrHrGr"rr)r%r&r_init_img_size to_2tuplerB image_sizeimg_size grid_sizer,r<round max_img_sizer@ hidden_size reg_tokenspatch_generatorr encoder)r-rrrr/rrr&s"   zRadioInternVisionModel.__init__rcCsF|durdSt|}tddt||D}|d|d}|||fS)N)NNNcSsg|]\}}||qSrr)rPsprrr sz9RadioInternVisionModel._init_img_size..rr)rrzip)r-rBrrr,rrrrs  z%RadioInternVisionModel._init_img_sizecCs|jSr) embeddingsrkrrrget_input_embeddingssz+RadioInternVisionModel.get_input_embeddingsrcCs:|jdusJ||}|jj|d}t|tsJ|jS)N) inputs_embeds)rrr8rrlast_hidden_state)r-r hidden_statesencoder_outputsrrrr8s  zRadioInternVisionModel.forwardr)r9r:r;packed_modules_mappingrrr&r<rrrr)r> FloatTensorr8r?rrr/rrsrcseZdZddgiZ ddededBddffdd Z  ddejdBd ejdBdej fd d Z de e fd d Z dejfddZZS) RadioModelrNrrrdcs"t||_t||d|_dS)Nr)r%r&rrmodel)r-rrr/rrr&s  zRadioModel.__init__ pixel_values pixel_embedscCs||}||Sr)r_extract_final)r-rryrrrr8s  zRadioModel.forwardc Csdddd}ddi}t}t|}t|trt|}nt|}|D].\}}|ds/q%t||}t||}|rS||vrS||} t | dt } | | || |q%|S)Nz attn.attnqkv_projzencoder.layers)attnrblocksz radio_model. weight_loader) setrXnamed_parametersrlistitems startswithr r getattrr add) r-weightsremap_substringsremap_prefixes loaded_params params_dict weights_listnameweightparamrrrr load_weightss.          zRadioModel.load_weightsrcCs0t|jdd}|dur|dd|jdf}|S)Nr)rrro)r-r patch_genall_featrrrrszRadioModel._extract_finalrr)r9r:r;rrrr&r)r>rr8rrrrr?rrr/rrs,  r)0rMcollections.abcr itertoolsrtypingrr)torch.nnr'torch.nn.functional functionalr{einopsr transformersrtransformers.modeling_outputsr*sglang.srt.layers.quantization.base_configr$sglang.srt.model_loader.weight_utilsr r r sglang.srt.models.internvlr r<rr __annotations__rr>rr to_1tupler to_3tuple to_4tuple to_ntupleModulerr@r[Linearr]rrrrrrs:           11