o -iQ@sUdZddlmZmZmZddlmZmZmZddl Z ddl m Z ddl m Z ddlmZddlmZddlmZdd lmZmZdd lmZdd lmZdd lmZmZmZdd lm Z ddl!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8ee9Z:Gddde+Z;e;ZZ?Gddde j>Z@Gddde$ZAGdd d e"eAZBGd!d"d"e#eAZCejDeCeAeBd#Gd$d%d%e j>e/e.e0ZEdS)&zInference-only BAGEL model compatible with HuggingFace weights. BAGEL is a unified multimodal model for image understanding and generation. For vLLM, we focus on the image understanding (vision-to-text) capabilities. )IterableMappingSequence)AnyLiteral TypeAliasN) VllmConfig)BaseDummyOptions) init_logger) get_act_fn)ColumnParallelLinearRowParallelLinear)QuantizationConfig)MULTIMODAL_REGISTRY)MultiModalDataDictMultiModalFieldConfigMultiModalKwargsItems)MultiModalDataItems)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoPromptReplacement)IntermediateTensors)BagelProcessor) TensorSchema)MultiModalEmbeddings SupportsLoRASupportsMultiModal SupportsPP)SiglipVisionModel)AutoWeightsLoaderStageMissingLayer WeightsMapperinit_vllm_registered_model maybe_prefixc@s(eZdZUdZeded<ejed<dS)BagelImagePixelInputsz Dimensions: - bn: Batch size * number of images - c: Number of channels (3) - h: Height of each image - w: Width of each image pixel_valuestypeN)__name__ __module__ __qualname____doc__r__annotations__torchTensorr0r0]/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/model_executor/models/bagel.pyr&<s  r&BagelImageInputsc s\eZdZdZ   ddedededed edBd ef fd d Zd ej dej fddZ Z S)BagelVisionMLPz"MLP connector for vision features.gelu_pytorch_tanhN in_featureshidden_features out_features act_layer quant_configprefixcsLtt||d||dd|_t||_t||d||dd|_dS)NTz.fc1)biasr:r;z.fc2)super__init__r fc1r actr fc2)selfr6r7r8r9r:r; __class__r0r1r>Os   zBagelVisionMLP.__init__xreturncCs*||\}}||}||\}}|SN)r?r@rA)rBrE_r0r0r1forwardis zBagelVisionMLP.forward)r4Nr5) r)r*r+r,intstrrr>r.r/rI __classcell__r0r0rCr1r3Ls&r3cszeZdZdZdedeffdd Zededefdd Zedefd d Zedefd d Z de j de j fddZ Z S)PositionEmbeddingzA2D position embedding for vision tokens using sin-cos embeddings.max_num_patch_per_side hidden_sizecs@t||_||_|||}|jdt|dddS)N pos_embedF) persistent) r=r>rNrO_get_2d_sincos_pos_embedregister_bufferr. from_numpyfloat)rBrNrOrPrCr0r1r>ss    zPositionEmbedding.__init__ embed_dim grid_sizecCsdddl}|j||jd}|j||jd}|||}|j|dd}|dd||g}t||}|S)z(Generate 2D sin-cos position embeddings.rNdtypeaxisr)numpyarangefloat32meshgridstackreshaperM"_get_2d_sincos_pos_embed_from_grid)rVrWnpgrid_hgrid_wgridrPr0r0r1rRs z*PositionEmbedding._get_2d_sincos_pos_embedcCsVddl}|ddks Jt|d|d}t|d|d}|j||gdd}|S)z2Generate 2D sin-cos position embeddings from grid.rNr\rrZ)r]rM"_get_1d_sincos_pos_embed_from_grid concatenate)rVrgrdemb_hemb_wembr0r0r1rcs  z4PositionEmbedding._get_2d_sincos_pos_embed_from_gridcCsddl}|ddks J|j|d|jd}||d}dd|}|d}|d ||}||}||}|j||gd d }|S) z(Generate 1D sin-cos position embeddings.rNr\rXg@g?i'zm,d->mdrrZ)r]r^float64rbeinsumsincosri)rVposrdomegaoutemb_sinemb_cosrlr0r0r1rhs     z4PositionEmbedding._get_1d_sincos_pos_embed_from_grid position_idsrFcCs||jj}|j|S)z Args: position_ids: Flattened position IDs, shape (N,) where each ID corresponds to a position in the flattened grid Returns: Position embeddings of shape (N, hidden_size) )torPdevice)rBrwr0r0r1rIs zPositionEmbedding.forward)r)r*r+r,rJr> staticmethodrRrcrhr.r/rIrLr0r0rCr1rMps rMc@sxeZdZdZdedefddZdeee dBffddZ d e d eee fdeee ffd d Z d e de de fddZ dS)BagelProcessingInfoz'Processing information for BAGEL model.kwargsrFcKsHddlm}||jjj|jjj|jjjd}|}td||d|S)Nr)cached_get_image_processor)revisiontrust_remote_code)image_processor tokenizerr0) !vllm.transformers_utils.processorr}ctx model_configmodelr~r get_tokenizerr)rBr|r}rrr0r0r1get_hf_processors z$BagelProcessingInfo.get_hf_processorNcCsddiS)Nimager0)rBr0r0r1get_supported_mm_limitssz+BagelProcessingInfo.get_supported_mm_limitsseq_len mm_countscCs|}|jd}d|iS)Nr\r) get_hf_configvit_max_num_patch_per_side)rBrr hf_configmax_num_patchesr0r0r1get_mm_max_tokens_per_items z.BagelProcessingInfo.get_mm_max_tokens_per_item image_width image_heightcCs,|}|j}|j}||}||}||SrG)r vit_config patch_size)rBrrrrr num_patches_h num_patches_wr0r0r1get_num_image_tokenss z(BagelProcessingInfo.get_num_image_tokens) r)r*r+r,objectrrrrKrJrrrr0r0r0r1r{s$    r{c @s\eZdZdZdeeefdefddZ d dedeeefdeeefdBde fd d Z dS) BagelDummyInputsBuilderz-Build dummy inputs for BAGEL model profiling.rrFcCs|dd}d|S)Nrr <|image_pad|>)get)rBr num_imagesr0r0r1get_dummy_texts z&BagelDummyInputsBuilder.get_dummy_textNr mm_optionsc CsJ|dd}|j}|j}|j}|r|dnd}d|j||||diS)Nrr)widthheightr overrides)rinforr image_size_get_dummy_images) rBrrrrrrrimage_overridesr0r0r1get_dummy_mm_datas  z)BagelDummyInputsBuilder.get_dummy_mm_datarG) r)r*r+r,rrKrJrr rrr0r0r0r1rs  rc @seZdZdZdededeeefdeeefdef ddZ dedeee fd e de e fd d Zd e deeefdeeeffd dZdS)BagelMultiModalProcessorz%Multimodal processor for BAGEL model. prompt_textmm_itemshf_processor_mm_kwargstokenization_kwargsrFcCsdS)NFr0)rBrrrrr0r0r1_hf_processor_applies_updatessz6BagelMultiModalProcessor._hf_processor_applies_updates out_mm_kwargscsX|j|j}|ddurtddtffdd }tdg|dgS) z=Replace image placeholders with the correct number of tokens.rNz=Image token '<|image_pad|>' not found in tokenizer vocabularyitem_idxcsjd}g|S)Nr\)r)r num_tokensrimage_token_idr0r1get_replacement_bagel-s  zKBagelMultiModalProcessor._get_prompt_updates..get_replacement_bagelr)modalitytarget replacement)rrr get_vocabr ValueErrorrJr)rBrrrrrr0rr1_get_prompt_updatess  z,BagelMultiModalProcessor._get_prompt_updates hf_inputscCsdtdiS)Nr'r)rbatched)rBrrr0r0r1_get_mm_fields_config;s z.BagelMultiModalProcessor._get_mm_fields_configN)r)r*r+r,rKrrrboolrrrrrrrrr0r0r0r1rs:        r)r dummy_inputscs eZdZdZeddddddZeded ed ed Bfd d Z ddde deffddZ de d e d BfddZde d eejdffddZde d efddZ d)dejdejded Bd ejd Bde d ejeBf d!d"Zd#ejd ejd Bfd$d%Zd&eeeejfd eefd'd(ZZS)*BagelForConditionalGenerationz BAGEL: A unified multimodal model for image understanding and generation. For vLLM, we focus on the image understanding (vision-to-text) capabilities. The image generation part is not supported in vLLM. language_model. vit_model. connector.vit_pos_embed.)rrrr)orig_to_new_prefixrirFNcCs|drdStd)Nrrz Only image modality is supported) startswithr)clsrrr0r0r1get_placeholder_str^s z1BagelForConditionalGeneration.get_placeholder_strr5)r; vllm_configr;c st|jj}|j}|jj}t|jdkr"tdt|jd||_ ||_| |t ||j t |ddgd|_Wdn1sFwY|jr|j}|jdkr^tdd |_t|d sktd d |_||d 2t||t |dd|_|jj}|j j}t||||j|t |dd|_t|j|d|_Wdn1swYnt d|_t d|_t d|_|jj!|_!dS)N BagelConfigzExpected BagelConfig, got z0. Make sure the model config is properly loaded.language_modelQwen2ForCausalLM)rrr; architectureszZOverriding vit_config.num_hidden_layers from 27 to 26 to match the Bagel model checkpoint.vision_use_headz_Setting vit_config.vision_use_head to False as it is not present in the Bagel model checkpoint.Fr vit_model)configr:r; connector)r6r7r8r9r:r;)rNrO image_tower)"r=r>rrr:multimodal_configr(r)rr_mark_language_modelr$ llm_configr%r visual_undrnum_hidden_layersloggerwarninghasattrr_mark_tower_modelr rrOr3 connector_actrrMr vit_pos_embedr"make_empty_intermediate_tensors) rBrr;rr:rrvit_hidden_sizellm_hidden_sizerCr0r1r>esr           z&BagelForConditionalGeneration.__init__r|cKs$|dd}|dur dStd|dS)Nr')r(r')popr&)rBr|r'r0r0r1_parse_and_validate_image_inputs z=BagelForConditionalGeneration._parse_and_validate_image_input image_input.cCs|d}|jdkr|j\}}}}}||||||}||}||} | j\}} } |jjj} |jjj} | | }t j || j d}t j || j d}|dddf|jj | }|d|d }||}||| | }|| j }| |} t| S)z:Process image inputs through vision encoder and connector.r')ryNrrm)ndimshaperbrrrrrrr.r^ryrflatten unsqueezeexpandrrxtuple)rBrr' batch_sizerchannelsrrvision_features vision_embeds num_patchesrOrrnum_patches_per_sideh_coordsw_coordsrw pos_embedsr0r0r1_process_image_inputs.         z2BagelForConditionalGeneration._process_image_inputcKs&|jdi|}|durgS||S)z%Get multimodal embeddings from input.Nr0)rr)rBr|rr0r0r1embed_multimodals z.BagelForConditionalGeneration.embed_multimodal input_ids positionsintermediate_tensors inputs_embedscKs$|durd}|jj||||d}|S)anRun forward pass for BAGEL. Args: input_ids: Flattened (concatenated) input_ids corresponding to a batch. positions: Flattened (concatenated) position ids corresponding to a batch. intermediate_tensors: Intermediate tensors from prior forward pass. inputs_embeds: Optional tensor of input embeddings. N)rrrr)rr)rBrrrrr| hidden_statesr0r0r1rIsz%BagelForConditionalGeneration.forwardrcCs |j|SrG)rcompute_logits)rBrr0r0r1rs z,BagelForConditionalGeneration.compute_logitsweightsc sgd}ddg}g}|D]Z\}tfdd|Drq tfdd|Dr(q dvr_|jdkr_|jd }|jd }|jjj}|jjj} || ||kr_||||| }|d d d d }| |fq t |d gd } | j ||j dS)zLoad weights from checkpoint.)moe_genlatent_pos_embedllm2vaevae2llm time_embedderzdecoder.zencoder.c3s|]}|vVqdSrGr0).0skipnamer0r1 4sz=BagelForConditionalGeneration.load_weights..c3s|]}|VqdSrG)r)rr;rr0r1r6szpatch_embedding.weightr\rrzvit_pos_embed.pos_embed) skip_prefixes)mapper)anyrrrrr num_channelsrbpermute contiguousappendr! load_weightshf_to_vllm_mapper) rBrgeneration_keywords vae_prefixesfiltered_weightstensor out_channelsr6r in_channelsloaderr0rr1rs.      z*BagelForConditionalGeneration.load_weights)NN)r)r*r+r,r#r classmethodrKrJrrr>rr2rrr.r/rrrrrIrrsetrrLr0r0rCr1rEsX P    2   ,r)Fr,collections.abcrrrtypingrrrr.torch.nnnn vllm.configrvllm.config.multimodalr vllm.loggerr %vllm.model_executor.layers.activationr !vllm.model_executor.layers.linearr r 'vllm.model_executor.layers.quantizationrvllm.multimodalrvllm.multimodal.inputsrrrvllm.multimodal.parservllm.multimodal.processingrrrr vllm.sequencer(vllm.transformers_utils.processors.bagelrvllm.utils.tensor_schemar interfacesrrrrsiglipr utilsr!r"r#r$r%r)rr&r2r-Moduler3rMr{rrregister_processorrr0r0r0r1sJ             $N2 5