o i@szUddlmZmZddlmZmZddlmZmZddl m Z ddl m Z m Z ddlmZddlmZmZmZmZmZmZmZmZddlZdd lmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&erddl'Z'ddl(Z'ddl)m*Z*ddl+m,Z,n e"de-dZ'e.Z,edZ/edej0dfZ1ee2d< ee3dej0de3ej0e3dfZ4ee2d< ee3e5ej0dfZ6ee2d< ee1de&e1fZ7ee2d< ee4de8e4e.e9efffZ:ee2d< ee6e8ej0e5fdfZ;ee2d< e/e3e/dBBdBZe=e>BZ? eGd!d"d"ed#d$Z@ee9ed?d?eZNe d'd'd=Gd@dAdAeNZOe d'd'd=GdBdCdCeNZPe d'd'd=GdDdEdEeNZQe d'd(GdFdGdGZRGdHdIdIee9eMfZSedJeSeSdBeSdKZTGdLdMdMee9eeTfZUeUeSeUeSdBBZVee2dN<e.e9e3e9fZW ee9eeCfZXee2dO< GdPdQdQe,ZYGdRdSdSeYZZdS)T)ABCabstractmethod)UserDict defaultdict)MappingSequence) dataclass)cached_propertypartial) accumulate) TYPE_CHECKINGAnyLiteral TypeAlias TypedDictUnioncastfinalN)Image)TypeVar) is_list_of) LazyLoaderjson_map_leaves)MediaWithBytes) BatchFeature) _InputOptionstorch_Tr torch.Tensor HfImageItem HfVideoItem HfAudioItem ImageItem VideoItem AudioItem ModalityDatac@s2eZdZUdZeded<eed<edBed<dS)VisionChunkImagez.Represents an image wrapped as a vision chunk.imagetypeNuuid)__name__ __module__ __qualname____doc__r__annotations__rstrr2r2L/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/multimodal/inputs.pyr(os  r(c@sFeZdZUdZeded<eeed<edBed<eed<e ed<dS)VisionChunkVideoz'Represents a video chunk with metadata. video_chunkr*Nr+prompt video_idx) r,r-r.r/rr0listrr1intr2r2r2r3r4ws     r4c@sHeZdZUdZeeed< eeed< eeed< ee ed<dS)MultiModalDataBuiltinsz7Type annotations for modality types predefined by vLLM.r)videoaudio vision_chunkN) r,r-r.r/r'r$r0r%r& VisionChunkr2r2r2r3r:s     r:F)totalMultiModalDataDictMultiModalUUIDDictT)frozenc@seZdZUdZeed< eed< dZded< edej dBfdd Z defd d Z d ed ede eeffddZ dee eeffddZdedefddZdS)PlaceholderRangea Placeholder location information for multi-modal data. Example: Prompt: `AAAA BBBB What is in these images?` Images A and B will have: ``` A: PlaceholderRange(offset=0, length=4) B: PlaceholderRange(offset=5, length=4) ``` offsetlengthNztorch.Tensor | Noneis_embedreturncCs|jdurdS|jjddS)Nrdim)rFcumsumselfr2r2r3 embeds_cumsumszPlaceholderRange.embeds_cumsumcCs|jdur|jSt|jdS)N)rMrEr9rKr2r2r3get_num_embedss zPlaceholderRange.get_num_embeds start_idxend_idxcCsJ|jdur ||fS|dkrt|j|dnd}t|j|d}||fS)a Returns the starting and ending indices of the embeddings of encoder outputs in the range of [start_idx, end_idx) in the placeholders. For example, given: PlaceholderRange(offset=2, length=5, is_embed=[False, True, False, True, True]) If start_idx=3 and end_idx=5, the output is (1, 3) because we want to get the second and the third embeddings from the encoder output. Nrr)rMr9)rLrPrQembeds_start_idxembeds_end_idxr2r2r3get_embeds_indices_in_ranges z,PlaceholderRange.get_embeds_indices_in_rangecCs|jdur|j|j|jdfgS|j}ttj||dddk}ttj||dddk}tj ||fdd|j}dd| DS) aExtract the start and end indices of the embedded region in prompt. For example, given `PlaceholderRange(offset=2, length=5)` and `is_embed = [False, True, False, True, True]`, the output is `[(1 + offset, 1 + offset), (3 + offset, 4 + offset)]`. Returns: A tuple `(start, end)` representing the start and end indices (inclusive) of the embedded region. Returns full placeholder range if `is_embed` is `None`. Nr)prepend)appendrNrHcSsg|]}t|qSr2)tuple).0xr2r2r3 z9PlaceholderRange.extract_embeds_range..) rFrDrEr9rnonzerodiff new_zerosflattenstacktolist)rLmask_istartsendsrangesr2r2r3extract_embeds_ranges  z%PlaceholderRange.extract_embeds_rangeothercCsbt||jsdS|j|jf|j|jfksdS|jdur |jduS|jdur*|jduSt|j|jSNF) isinstance __class__rDrErFnested_tensors_equal)rLrgr2r2r3__eq__s     zPlaceholderRange.__eq__)r,r-r.r/r9r0rFr rTensorrMrOrWrTr8rfobjectboolrlr2r2r2r3rCs(    rC NestedTensors)r .abrGcCst|tjrt|tjot||St|tjr$t|tjo#t||St|tr:t|to9tddt||DSt|trPt|toOtddt||DS||kS)ze Equality check between [`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects. cs|] \}}t||VqdSNrk)rXa_b_r2r2r3 $ z'nested_tensors_equal..csrsrtru)rXrwrvr2r2r3rx(ry)rirrmequalr8allziprqrrr2r2r3rks    rktensorsdevicecsdur|Stfdd|S)Ncst|tjr |jddS|S)NT)r non_blocking)rirrmto)rYrr2r39s z%_nested_tensors_h2d..r)r~rr2rr3_nested_tensors_h2d0s  rBatchedTensorInputscstfddDS)zq Equality check between [`BatchedTensorInputs`][vllm.multimodal.inputs.BatchedTensorInputs] objects. c3s*|]}|vot||VqdSrtrurXkr}r2r3rxMs(z(batched_tensors_equal..)r{r}r2r}r3batched_tensors_equalHsrc@sjeZdZUdZded< eed< eed< eed< dZedBed< ed e dd e efd d Z dS) MultiModalFeatureSpecz Represents a single multimodal input with its processed data and metadata. Used to track multimodal data through processing and caching. A request containing multiple multimodal items will have one `MultiModalFeatureSpec` per item. zMultiModalKwargsItem | Nonedatamodality identifier mm_positionNmm_hashfeatureskeyscCsZttttft}|D]}|j}|dur(|D]}||vr'||||jqq t|Srt)rr1r8rprrVdict)rrkwargsfitemrr2r2r3 gather_kwargsqsz#MultiModalFeatureSpec.gather_kwargs) r,r-r.r/r0r1rCr staticmethodr8setrr2r2r2r3rPs  rc@s8eZdZUdZeed< ded< dedefddZd S) MultiModalFieldElemz Represents a processed keyword argument to pass to a model for a [`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]. rBaseMultiModalFieldfieldrgrGcCsbt||jsdS|jdur|jdu}n|jdur|jdu}nt|j|j}|o0t|jt|juSrh)rirjrrkr*r)rLrg data_equalr2r2r3rls     zMultiModalFieldElem.__eq__N) r,r-r.r/rpr0rnrorlr2r2r2r3rs  r)rBkw_onlyc @seZdZUdZdZeed< ddZede de de d e e fd d Z ed ee d ed e fddZddddee dejjd ed e fddZdS)rz Defines how to interpret tensor data belonging to a keyword argument for [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems], and vice versa. F keep_on_cpucs&tt|ddtdtffdd }|S)N)rrrGcs |dS)Nrr2rrr2r3factorys z3BaseMultiModalField._field_factory..factory)r rrp)rLrr2rr3_field_factorys z"BaseMultiModalField._field_factoryrkeyrrGcCt)a  Construct [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem] instances to represent the provided data. This is the inverse of [`reduce_data`][vllm.multimodal.inputs.BaseMultiModalField.reduce_data]. NotImplementedErrorrLrrrr2r2r3 build_elemsszBaseMultiModalField.build_elemsbatch pin_memorycCrrtrrLrrr2r2r3 _reduce_datasz BaseMultiModalField._reduce_dataNrrelemsrcCstdd|D}tt|dkrtd||dur|jrd}|r&|jr&d}dd|D}|j||d }t||d S) z Merge the data from multiple instances of [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]. This is the inverse of [`build_elems`][vllm.multimodal.inputs.BaseMultiModalField.build_elems]. cSsg|]}t|jqSr2)r*rrXrr2r2r3rZz3BaseMultiModalField.reduce_data..rz#Cannot merge different field_types=NcpuFcSsg|]}|jqSr2rrXelemr2r2r3rZs)rr)lenr ValueErrorrrr)rLrrr field_typesroutr2r2r3 reduce_datas  zBaseMultiModalField.reduce_data)r,r-r.r/rror0rrr1rprrrr8rrtypesDevicerr2r2r2r3rsD      rc @sHeZdZdZdedededeefddZde ed e defd d Z d S) MultiModalBatchedFieldzo Info: [`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched] rrrrGcs|fdd|DS)Ncsg|]}|qSr2r2r field_factoryr2r3rZr[z6MultiModalBatchedField.build_elems..)rrr2rr3rsz"MultiModalBatchedField.build_elemsrrcst|dkrTt|tjddrTtttj|}t|dkr%|ddS|djt fdd|DrTtj t|g|djR|dj |dj |d}tj ||dS|S) Nrr{checkrc3s|]}|jkVqdSrtshaper first_shaper2r3rxz6MultiModalBatchedField._reduce_data..dtyperr)r)rrrrmrr8 unsqueeze contiguousrr{emptyrrr`)rLrrrr2rr3rs  z#MultiModalBatchedField._reduce_dataN) r,r-r.r/r1rprrrr8rorr2r2r2r3rs$  rc @sneZdZUdZeeeeeBed<dZeed<de de de dee fd d Z d e e d ede fd dZdS)MultiModalFlatFieldz Info: [`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat] [`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes] slicesrrIrrrrGcsB|t|jtddsttjsJdfdd|jDS)Nr{rz,torch.Tensor is required for multiple slicescsg|] }tt|qSr2)rslice)rXsrrr2r3rZ2sz3MultiModalFlatField.build_elems..)rrrrrirrmrr2rr3r's zMultiModalFlatField.build_elemsrrc s t|dkrt|tjddrtttj|}t|dkr"|dS|j|jdkt|djdtjffdd |dt fdd |Dr{\}}t fd d |D}tj g|||R|dj |dj |d }tj||j|d S|dj}g}t|D]#kr|t fd d |Dq|tfdd |Dqtj||dj |dj |d }d} |D]5} g} t|D]kr| t| | | jq| td| jq| |t| <| | j7} q|S|jdksJddd|DS)Nrr{rrtensorcs |jd|jddfSNrr)rrHr2r3_shape_before_afterDs z=MultiModalFlatField._reduce_data.._shape_before_afterc3s|] }|kVqdSrtr2r)rrr2r3rxIsz3MultiModalFlatField._reduce_data..c3|]}|jVqdSrtrrrHr2r3rxKrr)rIrc3rrtrrXtdr2r3rxarc3rrtrrrr2r3rxcrz$dim == 0 is required for nested listcSsg|] }|D]}|qqSr2r2)rXrer2r2r3rZ~z4MultiModalFlatField._reduce_data..)rrrrmrr8rrIrr{sumrrrconcatndimrangerVmaxzerosrrW) rLrr shape_before shape_after shape_concatrr max_sizes concat_offsetrrr2)rrrIrr3r4sV       z MultiModalFlatField._reduce_dataN)r,r-r.r/rrr0rIr9r1rprrr8rorr2r2r2r3rs(    rc @sReZdZUdZeed<dedededee fddZ d e ed e defd d Z d S)MultiModalSharedFieldzm Info: [`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared] batch_sizerrrrGcCs|}||g|jSrt)rr)rLrrrrr2r2r3rsz!MultiModalSharedField.build_elemsrrcCs|dS)Nrr2rr2r2r3rsz"MultiModalSharedField._reduce_dataN)r,r-r.r/r9r0r1rprrrr8rorr2r2r2r3rs&   rc @seZdZUedddedefddZe ddddedeeeeeBd e defd d Z e dddded d d e defddZ edddede defddZ e ed<eed<dededeefddZdS)MultiModalFieldConfigFrrrcCstt|d|dS)aN Defines a field where an element in the batch is obtained by indexing into the first dimension of the underlying data. Args: modality: The modality of the multi-modal item that uses this keyword argument. keep_on_cpu: Whether to keep this field on the CPU for the model inputs. Example: ``` Input: Data: [[AAAA] [BBBB] [CCCC]] Output: Element 1: [AAAA] Element 2: [BBBB] Element 3: [CCCC] ``` rrr)rr)rrr2r2r3batchedszMultiModalFieldConfig.batchedrrrIcCstt|||d|dS)a Defines a field where an element in the batch is obtained by slicing along the first dimension of the underlying data. Args: modality: The modality of the multi-modal item that uses this keyword argument. slices: For each multi-modal item, a slice (dim=0) or a tuple of slices (dim>0) that is used to extract the data corresponding to it. dim: The dimension to extract data, default to 0. keep_on_cpu: Whether to keep this field on the CPU for the model inputs. Example: ``` Given: slices: [slice(0, 3), slice(3, 7), slice(7, 9)] Input: Data: [AAABBBBCC] Output: Element 1: [AAA] Element 2: [BBBB] Element 3: [CC] ``` ``` Given: slices: [ (slice(None), slice(0, 3)), (slice(None), slice(3, 7)), (slice(None), slice(7, 9))] dim: 1 Input: Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]] Output: Element 1: [[A],[A],[A]] Element 2: [[B],[B],[B],[B]] Element 3: [[C],[C]] ``` )rrIrr)rr)rrrIrr2r2r3flats5zMultiModalFieldConfig.flat size_per_itemr csV|jdkr td|jdgt|fddtt|D}tj|||dS)a Defines a field where an element in the batch is obtained by slicing along the first dimension of the underlying data. Args: modality: The modality of the multi-modal item that uses this keyword argument. size_per_item: For each multi-modal item, the size of the slice that is used to extract the data corresponding to it. dim: The dimension to slice, default to 0. keep_on_cpu: Whether to keep this field on the CPU for the model inputs. Example: ``` Given: size_per_item: [3, 4, 2] Input: Data: [AAABBBBCC] Output: Element 1: [AAA] Element 2: [BBBB] Element 3: [CC] ``` ``` Given: size_per_item: [3, 4, 2] dim: 1 Input: Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]] Output: Element 1: [[A],[A],[A]] Element 2: [[B],[B],[B],[B]] Element 3: [[C],[C]] ``` Info: [`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat] rz7size_per_item should be a 1-D tensor, but found shape: rcs6g|]}tdddft||dfqSr)r)rXirI slice_idxsr2r3rZ6s z9MultiModalFieldConfig.flat_from_sizes..)rIr)rrrr rrrr)rrrIrrr2rr3flat_from_sizess 5  z%MultiModalFieldConfig.flat_from_sizesrcCstt||d|dS)a Defines a field where an element in the batch is obtained by taking the entirety of the underlying data. This means that the data is the same for each element in the batch. Args: modality: The modality of the multi-modal item that uses this keyword argument. batch_size: The number of multi-modal items which share this data. keep_on_cpu: Whether to keep this field on the CPU for the model inputs. Example: ``` Given: batch_size: 4 Input: Data: [XYZ] Output: Element 1: [XYZ] Element 2: [XYZ] Element 3: [XYZ] Element 4: [XYZ] ``` )rrr)rr)rrrr2r2r3sharedCs#zMultiModalFieldConfig.sharedrrrrGcCs|j|j||Srt)rrr)rLrrr2r2r3rqsz!MultiModalFieldConfig.build_elemsN)r)r,r-r.rr1rorrrr9rrrrr0rprrr2r2r2r3rs^  = H *rc@s:eZdZdZed defddZdeee ffddZ d S) MultiModalKwargsItemz A dictionary of processed keyword arguments to pass to the model, corresponding to a single item in [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]. rnbytescCs*ttj|tjdtddd}td|iS)zConvenience class for testing.)rr)r)rrdummy)rrruint8rr)rmm_elemr2r2r3rs  zMultiModalKwargsItem.dummyrGcCsdd|DS)NcSsi|]\}}||jqSr2r)rXrrr2r2r3 sz1MultiModalKwargsItem.get_data..)itemsrKr2r2r3get_dataszMultiModalKwargsItem.get_dataN)r) r,r-r.r/rr9rrr1rprr2r2r2r3rys r_I)defaultcsxeZdZdZedddeeeffddZdede e ffd d Z dd d Z dddde jjdedefddZZS)MultiModalKwargsItemsa A dictionary of processed multi-modal inputs by modality. For example, given a processor that processes images into `pixel_values` and `image_grid_thw`, and audios into `input_audio_features`, a prompt with 2 images and 1 audio will be processed into a `MultiModalKwargsItems` with the following structure: ```python MultiModalKwargsItems( { "image": [ # For the first image MultiModalKwargsItem({"pixel_values": ..., "image_grid_thw": ...}), # For the second imgae MultiModalKwargsItem({"pixel_values": ..., "image_grid_thw": ...}), ], "audio": [ # For the first audio MultiModalKwargsItem({"input_audio_features": ...}), ], } ) ``` Unlike HF processing which returns all items in a single dictionary with batched keyword arguments, we split up the items because some of them may already be cached. Also, items from multiple requests may be batched together to improve throughput, using the logic defined by the [`BaseMultiModalField`][vllm.multimodal.inputs.BaseMultiModalField] for each keyword argument. hf_inputsr config_by_keyc sttttfttttft}|D]%\}}||}|dur<|||}t |dkr<||<||j  |qttt t f}|D]?\}} fdd| DddD} t t| dkrttd|d| tt| } fdd t| D||<qJt|S) Nrcsi|]}||qSr2r2r) elems_by_keyr2r3rrz8MultiModalKwargsItems.from_hf_inputs..cSi|] \}}|t|qSr2rrXrvr2r2r3rrrz0Cannot merge different batch sizes for modality=z! Found: batch_sizes=cs&g|]tfddDqS)csi|] \}}||qSr2r2rrr2r3rrzCMultiModalKwargsItems.from_hf_inputs...)rr)rX)elems_in_modalityrr3rZsz8MultiModalKwargsItems.from_hf_inputs..)rr1rrrrrgetrrraddr8rvaluesrnextiterrr) rrkeys_by_modalityrconfigrritems_by_modalityrr batch_sizesrr2)rrr3from_hf_inputss2     z$MultiModalKwargsItems.from_hf_inputsrrGcs0||vrtd|dt|t|S)Nz Modality z" not found. Available modalities: )KeyErrorrrsuper __getitem__)rLrrjr2r3rs  z!MultiModalKwargsItems.__getitem__+MultiModalKwargsItems[MultiModalKwargsItem]cCsF|D]\}}t|D]\}}|durtd|d|dq q|S)NzFound empty mm_items[z][])r enumerate RuntimeError)rLrrrrr2r2r3 require_datasz"MultiModalKwargsItems.require_dataNFrrrc sddlm|}fdd|D}i}|D]"\}}t|dkr8dd|D}td|d||dq|S) zAConstruct a dictionary of keyword arguments to pass to the model.r)group_and_batch_mm_itemsc s6i|]\}}t|dkr|dd|dDqS)rcSsg|]\}}|qSr2r2)rX_rr2r2r3rZsz=MultiModalKwargsItems.get_data...rr)rXrrrrrr2r3rs  z2MultiModalKwargsItems.get_data..cSrr2r)rXrbatchesr2r2r3rszNSome modalities cannot be merged into a single batch (num_batches_by_modality=)r)utilsrrrrrupdate) rLrrr batches_by_modalityout_datarrnum_batches_by_modalityr2rr3rs$    zMultiModalKwargsItems.get_data)rGr)r,r-r.r/rrr1rr rrrrrrrrorr __classcell__r2r2rr3rs&#  #  rMultiModalKwargsOptionalItemsMultiModalPlaceholderDictc@sJeZdZUdZeded< eeed< eed< e ed< e ed<dS) MultiModalInputsz Represents the outputs of [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor], ready to be passed to vLLM internals. multimodalr*prompt_token_ids mm_kwargs mm_hashesmm_placeholdersN) r,r-r.r/rr0r8r9r"MultiModalHashesr#r2r2r2r3r$)s   r$c@seZdZUdZeeed<dS)MultiModalEncDecInputsaV Represents the outputs of [`EncDecMultiModalProcessor`][vllm.multimodal.processing.EncDecMultiModalProcessor] ready to be passed to vLLM internals. Note: Even text-only encoder-decoder models are currently implemented as multi-modal models for convenience. (Example: https://github.com/vllm-project/bart-plugin) encoder_prompt_token_idsN)r,r-r.r/r8r9r0r2r2r2r3r+Cs  r+)[abcrr collectionsrrcollections.abcrr dataclassesr functoolsr r itertoolsr typingr r rrrrrrnumpynp PIL.Imagertyping_extensionsrvllm.utils.collection_utilsrvllm.utils.import_utilsrvllm.utils.jsontreermediarr torch.types%transformers.feature_extraction_utilsrvllm.inputs.datarglobalsrrndarrayr!r0r8r"floatr#r$rWr1r%r&r'r(r4r>r:r@rArCrprorkrrrrrrrrrrrrrrrr"r*r#r$r+r2r2r2r3s  (                d  . $ M ) d]