o پiM@sPddlZddlZddlmZddlmZmZmZmZm Z m Z ddl Z ddl m Z ddl mZmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZmZmZdd lm Z m!Z!dd l"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,e-e.Z/ee,Z0Gddde Z1Gddde j2Z3GdddeZ4e4Z5dS)N) lru_cache)IterableListOptionalSetTuple TypedDict)nn) Gemma3ConfigPreTrainedModel)TritonAttnBackend) Gemma3RMSNorm)LogitsProcessor)QuantizationConfig))MultiModalityDataPaddingPatternTokenPairsgeneral_mm_embed_routine)MultimodalDataItemMultimodalInputsflatten_nested_list) ForwardBatch ForwardMode)default_weight_loadermaybe_remap_kv_scale_name)Gemma3ForCausalLM)SiglipVisionModel) add_prefix) get_processorc@seZdZUejed<dS)Gemma3ImagePixelInputs pixel_valuesN)__name__ __module__ __qualname__torchTensor__annotations__r%r%O/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/models/gemma3_mm.pyr7s rcs<eZdZdZdeffdd ZdejdejfddZZ S) Gemma3MultiModalProjectorz Projector for Gemma3 multimodal.configcsttt|jj|jj|_ t |jj|jj d|_ t |jj|jj|_t |jd|_|j|j|_tj|j|jd|_dS)N)epsg?) kernel_sizestride)super__init__r Parameterr"zeros vision_config hidden_size text_configmm_input_projection_weightr layer_norm_epsmm_soft_emb_normint image_size patch_sizepatches_per_imagemm_tokens_per_imagetokens_per_sider* AvgPool2davg_pool)selfr( __class__r%r&r-?s"    z"Gemma3MultiModalProjector.__init__vision_outputsreturnc Csv|j\}}}|dd}||||j|j}|}||}|d}|dd}||}t ||j }| |S)N) shape transposereshaper9 contiguousr=flattenr5r"matmulr3type_as) r>rA batch_size seq_lengthr1reshaped_vision_outputspooled_vision_outputsnormed_vision_outputsprojected_vision_outputsr%r%r&forwardUs        z!Gemma3MultiModalProjector.forward) rr r!__doc__r r-r"r#rR __classcell__r%r%r?r&r'<sr'csTeZdZeZ gdZdddddddZgd d d gd Zgd ZiZ gZ dZ e dZ  d4dedeededdffdd ZdeededeefddZdedejdejfddZdejfd d!Zd"d#Zd$ee fd%d&Z!e" d5dej#d'ejded(ejd)e$de%f d*d+Z&d,ede'fd-d.Z(d/d0Z)d1e*e+eejffd2d3Z,Z-S)6Gemma3ForConditionalGeneration)z .gate_proj.z .down_proj.z .up_proj.z.q_proj.z.k_proj.z.v_proj.z.o_proj.z .out_proj.)qkv_projr)rVrC)rVrD) gate_up_projr)rWrC)projr)q_projk_projv_proj gate_projup_projout_proj)rYrZr[r\r])rVrW)rVo_projrW down_projTzd^language_model\.model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)Nr( quant_configprefixrBcstj|d||_||_t|ds|jj|_t|ds!|jj|_t|j |t d|d|_ t ||_ |jj|_t|j|t d|d|_|jjjrWt|dd }|jjj|9_|dS) N)r(num_hidden_layersr1 vision_tower)r(rbrclanguage_model)rc logit_scaleg?)r,r-r(rbhasattrr2rdr1rr0rrer'multi_modal_projector vocab_sizerrflogits_processorrggetattr post_init)r>r(rbrcrgr?r%r&r-s.         z'Gemma3ForConditionalGeneration.__init__ input_ids image_inputscCs.|j}|j}||fg}t|}|||}|S)z Pad input IDs with image tokens.) im_start_id im_end_idrpad_input_tokens)r>rnrorprqmedia_token_pairspatternidsr%r%r& pad_input_idss   z,Gemma3ForConditionalGeneration.pad_input_ids forward_batch mask_dtypec CsTt|jtr|jtjksJg}tj|jdtj |j d}t |jD]n}tj |j ||j ||j|||j d}|d|j|j|d}|j|}|jD]-} | r|| jD]#\} } | |j|kr{d|| |j|| d|j|| | df<qXqO|||||||d<q"|rtj|dd} ||jj_| |jj_dSdSdS)z.Prepare attention masks for multimodal inputs.rC)dtypedevice)diagonalrdimN) isinstance attn_backendr forward_moderEXTENDr"r/rLint32rzrangeemptyextend_seq_lensextend_prefix_lensfill_tril mm_inputsmm_itemsis_imageoffsetsappendrInelementcatforward_metadata mask_indptr custom_mask) r>rwrnrxbidirectional_attn_masks_listbidirectional_attn_mask_indptribidirectional_attn_maskrmm_itemim_beginim_endbidirectional_attn_masksr%r%r&prepare_attn_maskssl       )z1Gemma3ForConditionalGeneration.prepare_attn_maskscC |jSN)rfget_input_embeddingsr>r%r%r&r z3Gemma3ForConditionalGeneration.get_input_embeddingscCr)zX This value is used to initialize attention backends in `ForwardBatch`. )rf!get_attention_sliding_window_sizerr%r%r&rs z@Gemma3ForConditionalGeneration.get_attention_sliding_window_sizeitemsc CsFtdd|D}g}|D]}|dkr*|jd|jjjkr*|||jj q |dkr6| d}n|dkrB| d}n|dkrPt d|jg}|jd}t |D]!}|||d }|j|jj |jd }|j|d } || q[|rtj|dd } || } || q |rtj|dd Stjg|jj d S)a1 Projects the last hidden state from the vision model into language model space. Supports both raw image pixel values and precomputed embeddings. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). cSsg|]}|jqSr%)feature).0itemr%r%r& #szDGemma3ForConditionalGeneration.get_image_feature..rzUnexpected pixel_values shape: rC)rzry)rr|)rz)rr}rEr(r2r1rtorfrzsqueeze unsqueeze ValueErrorrreryr"rritensor) r>rall_pixel_valuesfinal_features_listpixel_values_batchbatch_vision_outputsrLr pixel_value vision_outputvision_outputs_catprojected_featuresr%r%r&get_image_featuresD               z0Gemma3ForConditionalGeneration.get_image_feature positions input_embedskwargsc Ks~|d7}|dur|jj|jkr||jjk}|}d||<n|}|jtjkr3|r3|j||t j dt |||j ||d}|S)a# labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`. logits_to_keep (`int` or `torch.Tensor`, *optional*): If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that token can save memory, which becomes pretty significant for long sequences or large vocabulary size. If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. This is useful when using packed tensor format (single dimension for batch and sequence length). Returns: Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/Gemma3-test-224px-hf") >>> processor = AutoProcessor.from_pretrained("google/Gemma3-test-224px-hf") >>> prompt = "answer en Where is the cow standing?" >>> url = "https://huggingface.co/gv-hf/Gemma3-test-224px-hf/resolve/main/cow_beach_1.png" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, text=prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(**inputs, max_length=30) >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "answer en Where is the cow standing?\nbeach" ```rCNr)rx)rnrwrfmultimodal_modelr) r(image_token_indexrjclonerrrcontains_image_inputsrr"boolrrf) r>rnrrwrrspecial_image_mask llm_input_idshsr%r%r&rRSs00  z&Gemma3ForConditionalGeneration.forward module_namecCst|j|S)z5Skip vision tower and multi_modal_projector for LoRA.)r lora_patternmatch)r>rr%r%r&should_apply_lorasz0Gemma3ForConditionalGeneration.should_apply_loracCrr)rf tie_weightsrr%r%r&rrz*Gemma3ForConditionalGeneration.tie_weightsweightscCs gd} t|}t}|D]t\}}d|vr't|||fg}||q|D](\}} } | |vr3q)|| |}|drC||vrCq)||} | j} | | || n-d|vr\|dd}|drf||vrfqt ||}|durpq||} t | dt } | | || |q| |} | r |S)N)) .qkv_projz.q_projq)rz.k_projk)rz.v_projv)rWr]rC)rWr\rrfz.bias vision_modelz.self_attn.out_projz.self_attn.proj weight_loader)dictnamed_parameterssetr load_weightsupdatereplaceendswithrrrlraddkeys)r>rstacked_params_mapping params_dict loaded_paramsname loaded_weightcausal_loaded_params param_name weight_nameshard_idparamrunloaded_paramsr%r%r&rsL           z+Gemma3ForConditionalGeneration.load_weights)Nrar).rr r!r config_class#default_bitsandbytes_target_modules#bitsandbytes_stacked_params_mappingpacked_modules_mappingsupported_lora_modulesembedding_modulesembedding_padding_modules supports_lorarecompilerrrstrr-rr6rrvrr"r#ryrr Embeddingrrrrno_grad LongTensorobjectrrRrrrrrrrTr%r%r?r&rUos   %   ;9 O$rU)6loggingr functoolsrtypingrrrrrrr"r transformersr r *sglang.srt.layers.attention.triton_backendr sglang.srt.layers.layernormr "sglang.srt.layers.logits_processorr*sglang.srt.layers.quantization.base_configrsglang.srt.managers.mm_utilsrr"sglang.srt.managers.schedule_batchrrr,sglang.srt.model_executor.forward_batch_inforr$sglang.srt.model_loader.weight_utilsrrsglang.srt.models.gemma3_causalrsglang.srt.models.sigliprsglang.srt.utilsr&sglang.srt.utils.hf_transformers_utilsr getLoggerrloggercached_get_processorrModuler'rU EntryClassr%r%r%r&s6            3w