o -i<@sddlmZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlmZddlmZddlmZddlmZmZd d lmZmZmZmZmZmZmZd d lm Z d d l!m"Z"m#Z#m$Z$m%Z%errdd l&m'Z'mZm(Z(ddl)m*Z*ee+Z,e de-ddZ.e de$dZ/e de$ddZ0Gddde e0Z1Gddde e/Z2Gddde e/Z3eddGddde e/Z4Gdd d Z5d!S)")Mapping) dataclass)Lock) TYPE_CHECKINGGenericLiteralProtocolTypeVarcast)BaseDummyOptions)ObservabilityConfig) init_logger) TokenizerLikecached_tokenizer_from_config)BaseMultiModalProcessorCacheBaseMultiModalReceiverCacheMultiModalProcessorOnlyCacheMultiModalProcessorSenderCacheMultiModalReceiverCacheShmObjectStoreReceiverCacheShmObjectStoreSenderCache)MultiModalInputs)BaseDummyInputsBuilderBaseMultiModalProcessorBaseProcessingInfoInputProcessingContext) ModelConfigr VllmConfig)SupportsMultiModalNr)bound_I_I_coT)r! covariantc@s"eZdZdZdedefddZdS)ProcessingInfoFactory Constructs a [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor] instance from the context. ctxreturncCdSN)selfr'r+r+U/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/multimodal/registry.py__call__0szProcessingInfoFactory.__call__N)__name__ __module__ __qualname____doc__rr#r.r+r+r+r-r%)sr%c@s&eZdZdZdedeefddZdS)DummyInputsBuilderFactoryz Constructs a [`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder] instance from the context. infor(cCr)r*r+)r,r4r+r+r-r.=sz"DummyInputsBuilderFactory.__call__N)r/r0r1r2r"rr.r+r+r+r-r36sr3c @s<eZdZdZdddedeededBdeefdd ZdS) MultiModalProcessorFactoryr&Ncacher4 dummy_inputsr7r(cCr)r*r+)r,r4r8r7r+r+r-r.Gsz#MultiModalProcessorFactory.__call__) r/r0r1r2r"rrrr.r+r+r+r-r5@s r5)frozenc@sNeZdZUeeed<eeed<eeed<dddede dBfdd Z dS) _ProcessorFactoriesr4 processorr8Nr6r'r7cCs$||}||}|j|||dS)Nr6r4r8r;)r,r'r7r4dummy_inputs_builderr+r+r-build_processorVs  z#_ProcessorFactories.build_processor) r/r0r1r%r"__annotations__r5r3rrr>r+r+r+r-r:Ps   r:c@s0eZdZdZdddeeefdBfddZdddefdd Z dddd ddd e dBd eee fdBd e dBdeee ff ddZ dddddd e dBd e dBdeee ffddZdeedeedeefddZd9ddZ  d:ddd ddedBdefddZ d;ddddd ddedBdefd d!Z d;ddd"ddd ddedBd e dBdeef d#d$Z d;dddd%ddd&eee fdBd e dBd e dBdedBdef d'd(Zddde fd)d*Zd+d,ded-fd.d/Z d+d,de dBfd0d1Z!d+d,de"dBfd2d3Z#d+d,de$dBfd4d5Z%d+d,d6e&de$dBfd7d8Z'dS)<MultiModalRegistryzL A registry that dispatches data processing according to the model. model_configrr(Ncs6jsdSfddjjD}t|dkr|SdS)z Extract multimodal dummy options from model config. Returns None if no configurable options are found, otherwise returns a mapping of modality names to their dummy options. Ncs&i|]}j|dur|qSr*)multimodal_configget_dummy_options).0mrAoptr+r- ss z:MultiModalRegistry._extract_mm_options..r)rBlimit_per_promptlen)r,rA mm_optionsr+rFr-_extract_mm_optionsfs   z&MultiModalRegistry._extract_mm_optionscsP|jsdS|j|dd}|}|tfdd|Dr&tddSdS)z Checks if the model supports multimodal inputs. Returns True if the model is multimodal with any non-zero supported modalities, otherwise returns False, effectively running in text-only mode. FN tokenizerc3s|] }|dkVqdS)rN)get_limit_per_prompt)rDmodality mm_configr+r- s  z@MultiModalRegistry.supports_multimodal_inputs..zcAll limits of multimodal modalities supported by the model are set to 0, running in text-only mode.T)is_multimodal_model_create_processing_infoget_supported_mm_limitsget_multimodal_configalllogger info_once)r,rAr4supported_modalitiesr+rQr-supports_multimodal_inputs{s z-MultiModalRegistry.supports_multimodal_inputs)r7profiler_limitsobservability_configr7r]r^cs|jsiS|j|||d}|dur|j}dd|D|jj|jd}|dur5fdd|DS|j||d}dd|d DS) z Get the maximum number of tokens per data item from each modality based on underlying model configuration. r6NcSsi|] \}}|dkr|dqS)rrr+)rDrPlimitr+r+r-rHszJMultiModalRegistry.get_max_tokens_per_item_by_modality..)seq_len mm_countscs&i|]\}}|ddkr||qS)r)get)rDrP max_tokensrar+r-rHs )rar;cSs$i|]\}}|tdd|DqS)css|]}|jVqdSr*)get_num_embeds)rDitemr+r+r-rSszTMultiModalRegistry.get_max_tokens_per_item_by_modality...)sum)rDrP placeholdersr+r+r-rHsmm_placeholders)rTcreate_processorallowed_mm_limitsitemsr4get_mm_max_tokens_per_item max_model_lenget_dummy_mm_inputs)r,rAr7r]r^r;max_tokens_per_item mm_inputsr+rdr-#get_max_tokens_per_item_by_modalitys4   z6MultiModalRegistry.get_max_tokens_per_item_by_modality)r7r^cCs |jsiS|j|||d}|jS)z Get the maximum number of multi-modal input instances for each modality that are allowed per prompt for a model class. r6)rTrjrk)r,rAr7r^r;r+r+r-get_mm_limits_per_prompts  z+MultiModalRegistry.get_mm_limits_per_promptr;r4r8cs dtdtffdd }|S)a/ Register a multi-modal processor to a model class. The processor is constructed lazily, hence a factory method should be passed. When the model receives multi-modal data, the provided function is invoked to transform the data into a dictionary of model inputs. model_clsr(cs,d|jvr td|td|_|S)N_processor_factoryzfModel class %s already has a multi-modal processor registered to %s. It is overwritten by the new one.r<)__dict__rYwarningr:ru)rtr8r4r;r,r+r-wrappers z6MultiModalRegistry.register_processor..wrapper)r )r,r;r4r8ryr+rxr-register_processorsz%MultiModalRegistry.register_processorrcCs0ddlm}||\}}t|dsJtd|S)Nr)get_model_architecturerur) vllm.model_executor.model_loaderr{hasattrr )r,rAr{rt_r+r+r-_get_model_clss   z!MultiModalRegistry._get_model_clszObservabilityConfig | NonerNcCs|durt|}t|||dS)N)r^)rr)r,rAr^rNr+r+r-_create_processing_ctxs z)MultiModalRegistry._create_processing_ctxrMcCs(||}|j}||||}||Sr*)rrurr4)r,rAr^rNrt factoriesr'r+r+r-rUs  z*MultiModalRegistry._create_processing_info)rNr7cCsB|js t|jd||}|j}||||}|j||dS)zT Create a multi-modal processor for a specific model and tokenizer. z is not a multimodal modelr6)rT ValueErrormodelrrurr>)r,rAr^rNr7rtrr'r+r+r-rjs   z#MultiModalRegistry.create_processor)r7r^r;rac Cs|j}|dur|j|||d}|dur|j}|jj||||d}|j|j|j|j |j d}|d} t | } | |krE| dg|| |S)z Create dummy data for profiling the memory usage of a model. The model is identified by `model_config`. Nr6)r`rarK)promptmm_datahf_processor_mm_kwargstokenization_kwargsprompt_token_idsr) rnrjrkr8get_dummy_processor_inputsrLapplyrrrrrJextend) r,rArar7r^r;r`processor_inputsrqr total_lenr+r+r-ro2s.z&MultiModalRegistry.get_dummy_mm_inputscCsD|jsdS||}|sdSt|dksJdtt|}||S)zY Get the maximum length of the encoder input for encoder-decoder models. rrzdEncoder-decoder models are expected to implement the multimodal interface with at most one modality.)is_encoder_decoderrrrJnextiter)r,rArcfirst_modalityr+r+r-get_encdec_max_encoder_len\s  z-MultiModalRegistry.get_encdec_max_encoder_len vllm_configr)Nprocessor_onlylrushmcCs`|j}||s dS|}|jdkrdS|j}|jdko$|jdkp$|j}|s)dS|}|jS)Nrrr) rAr\rWmm_processor_cache_gbparallel_config_api_process_countdata_parallel_sizedata_parallel_external_lbmm_processor_cache_type)r,rrArRris_ipc_supportedr+r+r-_get_cache_typeps    z"MultiModalRegistry._get_cache_typecCsX||}|dur dS|dkrt|jS|dkrt|jS|dkr%t|Std|)z4Return a `BaseMultiModalProcessorCache`, if enabled.NrrrUnknown cache type: )rrrArrrr,r cache_typer+r+r-processor_cache_from_configs   z.MultiModalRegistry.processor_cache_from_configcCs ||}|dur dSt|jS)z4Return a `MultiModalProcessorOnlyCache`, if enabled.N)rrrArr+r+r- processor_only_cache_from_configs  z3MultiModalRegistry.processor_only_cache_from_configcCs6||}|dvr dS|dkrt|jStd|)z>Return a `BaseMultiModalReceiverCache` for the engine process.)NrrNrr)rrrArrr+r+r-!engine_receiver_cache_from_configs  z4MultiModalRegistry.engine_receiver_cache_from_configshared_worker_lockcCs6||}|dvr dS|dkrt||Std|)z>Return a `BaseMultiModalReceiverCache` for the worker process.)NrrNrr)rrr)r,rrrr+r+r-!worker_receiver_cache_from_configs  z4MultiModalRegistry.worker_receiver_cache_from_config)rArr(r)NNr*)(r/r0r1r2rstrr rLboolr\rintr rrrsr5r"r%r3rzrrrrrrUrrjrrorrrrrrrrLockTyperr+r+r+r-r@as    4   "     *      r@N)6collections.abcr dataclassesrmultiprocessing.synchronizerrtypingrrrrr r vllm.config.multimodalr vllm.config.observabilityr vllm.loggerr vllm.tokenizersrrr7rrrrrrrinputsr processingrrrr vllm.configrr%vllm.model_executor.models.interfacesrr/rYtyper r"r#r%r3r5r:r@r+r+r+r-s0       $