o -iC @s$ddlZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z mZddlmZddlZddlm ZddlZddlmZmZdd lmZmZddlmZdd lmZm Z dd l!m"Z"dd l#m$Z$d dl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+e rd dl,m-Z-m.Z.m/Z/ne Z-e Z.e Z/e"e0Z1eej2dZ3e4e3j5edZ6e$Z7e74dGdddZ8dddej9de:de;de;fddZdd%d d!ejd"e;de;de;fd&d'Z?d(dd)ej@de;de;fd*d+ZAd(dd)ej@de;de;fd,d-ZBd.e/deCeDe;e:ffd/d0ZEdd1d2d3eCe.d4ejFjGd5eHdeeDe;e:e-fddffd6d7ZI dDd8e;d9eJe;e fdBdeDej9e:eKBffd:d;ZL dDdd?ZM dDd@e;dAeJe;e fdBdeDej@eJe;e fffdBdCZNdS)EN) Generator)ThreadPoolExecutor)groupby)Path) TYPE_CHECKINGAnyTypeVar) url2pathname)ImageUnidentifiedImageError)Url parse_url)HTTPConnectionglobal_http_connection) init_logger)ExtensionManager)AudioEmbeddingMediaIO AudioMediaIOImageEmbeddingMediaIO ImageMediaIOMediaIO VideoMediaIO)BatchedTensorInputsMultiModalKwargsItemMultiModalPlaceholderDict) max_workers_Mhttpc seZdZdefddddeeeeeffdBdededeedBddf fd d Z d e d e e de fd dZ d e d e e de fddZd e ddfddZddded e e dedBde fddZddded e e dedBde fddZdedeejeeBffddZdedeejeeBffddZdd d!ed"edejfd#d$Zdd d!ed"edejfd%d&Zdd d'ed"edeejeeefffd(d)Zdd d'ed"edeejeeefffd*d+Zd,ede j!fd-d.Z"d,ede j!fd/d0Z#Z$S)1MediaConnectorN)allowed_local_media_pathallowed_media_domainsmedia_io_kwargs connectionr!r"returncs|t|r |ni|_||_|r.t|}|s!td|d|s-td|dnd}||_|dur9g}||_ dS)aj Args: media_io_kwargs: Additional args passed to process media inputs, keyed by modalities. For example, to set num_frames for video, set `--media-io-kwargs '{"video":{"num_frames":40}}'` connection: HTTP connection client to download media contents. allowed_local_media_path: A local directory to load media files from. allowed_media_domains: If set, only media URLs that belong to this domain can be used for multi-modal inputs. z/Invalid `--allowed-local-media-path`: The path z does not exist.z must be a directory.N) super__init__r#r$rexists ValueErroris_dirr!r")selfr#r$r!r"allowed_local_media_path_ __class__R/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/multimodal/utils.pyr';s0   zMediaConnector.__init__url_specmedia_ioc CsT|jpd}|dd\}}|dd\}}|d}|dkr$d}t||||S)Nr ,r;/base64z,Only base64 data URLs are supported for now.)pathsplitlstripNotImplementedError load_base64) r+r1r2 url_spec_path data_specdata media_type data_typemsgr/r/r0_load_data_urljs   zMediaConnector._load_data_urlcCsh|j}|dur td|jpd}|jpd}tt||}||jvr/td|d|d| |S)Nz=Cannot load local files without `--allowed-local-media-path`.r zThe file path z2 must be a subpath of `--allowed-local-media-path z`.) r! RuntimeErrorr7netlocrr resolveparentsr) load_file)r+r1r2r!r<url_spec_netlocfilepathr/r/r0_load_file_url{s   zMediaConnector._load_file_urlcCs2|jr|j|jvrtd|jd|jdSdS)Nz1The URL must be from one of the allowed domains: z. Input URL domain: )r"hostnamer))r+r1r/r/r0$_assert_url_in_allowed_media_domainss z3MediaConnector._assert_url_in_allowed_media_domains fetch_timeouturlrNcCs~t|}|jr#|jdr#|||j}|j||tjd}||S|jdkr.| ||S|jdkr9| ||Sd}t |Nr)timeoutallow_redirectsr>filez0The URL must be either a HTTP, data or file URL.) r scheme startswithrLr$ get_bytesenvsVLLM_MEDIA_URL_ALLOW_REDIRECTS load_bytesrBrJr))r+rOr2rNr1r$r>rAr/r/r0 load_from_urls       zMediaConnector.load_from_urlc st|}t}|jr3|jdr3|||j}|j||tj dIdH}| t |j |}|IdHS|jdkrF| t |j ||}|IdHS|jdkrY| t |j||}|IdHSd} t| rP)r asyncioget_running_looprTrUrLr$async_get_bytesrWrXrun_in_executorglobal_thread_poolrYrBrJr)) r+rOr2rNr1loopr$r>futurerAr/r/r0load_from_url_asyncs2         z"MediaConnector.load_from_url_async audio_urlcCs*tdi|jdi}|j||tjdS)z( Load audio from a URL. audiorMNr/)rr#getrZrWVLLM_AUDIO_FETCH_TIMEOUTr+rcaudio_ior/r/r0 fetch_audios zMediaConnector.fetch_audiocs2tdi|jdi}|j||tjdIdHS)z8 Asynchronously fetch audio from a URL. rdrMNr/)rr#rerbrWrfrgr/r/r0fetch_audio_asyncs z MediaConnector.fetch_audio_asyncRGB image_mode image_urlrmc CsXtdd|i|jdi}z |j||tjdWSty+}ztt||d}~ww)z Load a PIL image from an HTTP or base64 data URL. By default, the image is converted into RGB format. rmimagerMNr/) rr#rerZrWVLLM_IMAGE_FETCH_TIMEOUTr r)strr+rnrmimage_ioer/r/r0 fetch_images  zMediaConnector.fetch_imagec s`tdd|i|jdi}z |j||tjdIdHWSty/}ztt||d}~ww)z Asynchronously load a PIL image from an HTTP or base64 data URL. By default, the image is converted into RGB format. rmrorMNr/) rr#rerbrWrpr r)rqrrr/r/r0fetch_image_asyncs   z MediaConnector.fetch_image_async video_urlcCsHtdd|i|jdi}t|fi|jdi}|j||tjdS)z= Load video from an HTTP or base64 data URL. rmrovideorMNr/)rr#rerrZrWVLLM_VIDEO_FETCH_TIMEOUTr+rwrmrsvideo_ior/r/r0 fetch_video.s  zMediaConnector.fetch_videocsPtdd|i|jdi}t|fi|jdi}|j||tjdIdHS)z Asynchronously load video from an HTTP or base64 data URL. By default, the image is converted into RGB format. rmrorxrMNr/)rr#rerrbrWryrzr/r/r0fetch_video_asyncBs   z MediaConnector.fetch_video_asyncr>cCt}|d|S)z2 Load image embedding from a URL. r )rr;)r+r>image_embedding_ior/r/r0fetch_image_embeddingX z$MediaConnector.fetch_image_embeddingcCr~)z2 Load audio embedding from a URL. r )rr;)r+r>audio_embedding_ior/r/r0fetch_audio_embeddingcrz$MediaConnector.fetch_audio_embedding)%__name__ __module__ __qualname__rdictrqrrlistr'r rrrBrJrLintrZrbtuplenpndarrayfloatrirjr rurvnptNDArrayr|r}torchTensorrr __classcell__r/r/r-r0r9s /   # $        rWAVformatrd sampling_raterr%cCst}|j||f|dS)zEncode audio as base64.) audio_format)r encode_base64)rdrrrhr/r/r0encode_audio_base64osrcC4t|||d}tjd|d}d|d|S)zEncode audio as a data URL.r.rddata:;base64,)r mimetypes types_maprelower)rdrr audio_b64mimetyper/r/r0encode_audio_urlzsrrkrmrrormcCst|d}|j||dS)z Encode a pillow image to base64 format. By default, the image is converted into RGB format before being encoded. rl) image_format)rr)rormrrsr/r/r0encode_image_base64s rPNGcCr)z| Encode a pillow image as a data URL. By default, the image is converted into RGB format before being encoded. rrrorr)rrrrer)rormr image_b64rr/r/r0encode_image_urls rJPEGframescCst}t|}|j||dS)N) video_format)rrr)rrrsr{r/r/r0encode_video_base64srcCsDt||d}|dkrd}n tjd|d}d|d|S)Nrjpegz video/jpegrrxrr)rrrrre)rr video_b64rr/r/r0encode_video_urls  r mm_positionscCs0dd|D}t|ddd}dd|DS)a/ Given a `MultiModalPlaceholderDict`, output a sequence of keys to sort the dictionary by `offset` (starting index in the input sequence) in ascending order. Returns: A list of `(modality, idx)`, which can be used to access an item by `mm_positions[modality][idx]`. css0|]\}}t|D] \}}|||fVq qdSN) enumerate).0modalityitemsidxitemr/r/r0 s z'argsort_mm_positions..cSs |djS)N)offset)xr/r/r0s z&argsort_mm_positions..keycSsg|] \}}}||fqSr/r/)rrr_r/r/r0 sz(argsort_mm_positions..)rsorted)r flat_itemssorted_flat_itemsr/r/r0argsort_mm_positionss  rFdevice pin_memory mm_kwargsrrc cs\ddlm}t|dddD]\}}t|}||}|j||d}|t||fVqdS)aGroup consecutive `MultiModalKwargsItem`s from `mm_kwargs` with the same modality together into the same `MultiModalKwargs` instance. Args: mm_kwargs: List of `MultiModalKwargsItem`. device: The device to place the grouped tensors on. pin_memory: Whether to pin memory for faster host-to-device transfer. Yields: A tuple `(modality, num_items, grouped_kwargs)`. r)MultiModalKwargsItemscSs|jSr)r)rr/r/r0rsz-group_mm_kwargs_by_modality..rrN)vllm.multimodal.inputsrrrfrom_seqget_datalen) rrrrrr items_lstmm_kwargs_itemsmm_kwargs_datar/r/r0group_mm_kwargs_by_modalitys  rrcaudio_io_kwargscC&|sdnd|i}t|dd}||S)a+ Args: audio_url: URL of the audio file to fetch. audio_io_kwargs: Additional kwargs passed to handle audio IO. Warning: This method has direct access to local files and is only intended to be called by user code. Never call this from the online server! Nrdr5r#r!)rri)rcrr#media_connectorr/r/r0ri   rirnimage_io_kwargscCr)a+ Args: image_url: URL of the image file to fetch. image_io_kwargs: Additional kwargs passed to handle image IO. Warning: This method has direct access to local files and is only intended to be called by user code. Never call this from the online server! Nror5r)rru)rnrr#rr/r/r0rurrurwvideo_io_kwargscCr)a+ Args: video_url: URL of the video file to fetch. video_io_kwargs: Additional kwargs passed to handle video IO. Warning: This method has direct access to local files and is only intended to be called by user code. Never call this from the online server! Nrxr5r)rr|)rwrr#rr/r/r0r|rr|r)Or[atexitrcollections.abcrconcurrent.futuresr itertoolsrpathlibrtypingrrrurllib.requestr numpyr numpy.typingrrPILr r urllib3.utilr r vllm.envsrWvllm.connectionsrr vllm.loggerrvllm.utils.registryrmediarrrrrrinputsrrrrloggerVLLM_MEDIA_LOADING_THREAD_COUNTr_registershutdownrMEDIA_CONNECTOR_REGISTRYrrrrqrrrrrrrrrrtypesDeviceboolrrrrirur|r/r/r/r0s             ;