o
    i*+                     @   s"  d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
Z
 d dlZd dlmZ d dlZd dlmZmZ d dlmZmZ d dlmZ d dlmZmZ d d	lmZ d
dlmZmZ d
dl m!Z! d
dl"m#Z#m$Z$ d
dl%m&Z& edZ'eej(dZ)e*e)j+ e Z,e,*dG dd dZ-dS )    N)ThreadPoolExecutor)Path)AnyTypeVar)url2pathname)ImageUnidentifiedImageError)Url	parse_url)HTTPConnectionglobal_http_connection)ExtensionManager   )AudioEmbeddingMediaIOAudioMediaIO)MediaIO)ImageEmbeddingMediaIOImageMediaIO)VideoMediaIO_M)max_workershttpc                       s  e Zd Zdefddddeeeeef f dB dededee dB ddf
 fd	d
Z	de
dee defddZde
dee defddZde
ddfddZdddedee dedB defddZdddedee dedB defddZdedeejeeB f fddZdedeejeeB f fddZdd d!ed"edejfd#d$Zdd d!ed"edejfd%d&Zdd d'ed"edeejeeef f fd(d)Zdd d'ed"edeejeeef f fd*d+Zd,ede j!fd-d.Z"d,ede j!fd/d0Z#  Z$S )1MediaConnectorN )allowed_local_media_pathallowed_media_domainsmedia_io_kwargs
connectionr   r   returnc                   s|   t    |r	|ni | _|| _|r.t|}| s!td| d| s-td| dnd}|| _|du r9g }|| _	dS )aj  
        Args:
            media_io_kwargs: Additional args passed to process media
                             inputs, keyed by modalities. For example,
                             to set num_frames for video, set
                             `--media-io-kwargs '{"video":{"num_frames":40}}'`
            connection: HTTP connection client to download media contents.
            allowed_local_media_path: A local directory to load media files from.
            allowed_media_domains: If set, only media URLs that belong to this
                                   domain can be used for multi-modal inputs.
        z/Invalid `--allowed-local-media-path`: The path z does not exist.z must be a directory.N)
super__init__r   r   r   exists
ValueErroris_dirr   r   )selfr   r   r   r   allowed_local_media_path_	__class__ U/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/multimodal/media/connector.pyr    &   s0   


zMediaConnector.__init__url_specmedia_ioc           	      C   sT   |j pd}|dd\}}|dd\}}|d}|dkr$d}t||||S )Nr   ,r   ;/base64z,Only base64 data URLs are supported for now.)pathsplitlstripNotImplementedErrorload_base64)	r$   r*   r+   url_spec_path	data_specdata
media_type	data_typemsgr(   r(   r)   _load_data_urlU   s   

zMediaConnector._load_data_urlc                 C   sh   | j }|d u rtd|jpd}|jpd}tt|| }|| jvr/td| d| d|	|S )Nz=Cannot load local files without `--allowed-local-media-path`.r   zThe file path z2 must be a subpath of `--allowed-local-media-path z`.)
r   RuntimeErrorr0   netlocr   r   resolveparentsr"   	load_file)r$   r*   r+   r   r5   url_spec_netlocfilepathr(   r(   r)   _load_file_urlf   s   


zMediaConnector._load_file_urlc                 C   s2   | j r|j| j vrtd| j  d|j d S d S )Nz1The URL must be from one of the allowed domains: z. Input URL domain: )r   hostnamer"   )r$   r*   r(   r(   r)   $_assert_url_in_allowed_media_domains|   s   z3MediaConnector._assert_url_in_allowed_media_domainsfetch_timeouturlrG   c                C   s~   t |}|jr#|jdr#| | | j}|j||tjd}||S |jdkr.| 	||S |jdkr9| 
||S d}t|Nr   )timeoutallow_redirectsr7   filez0The URL must be either a HTTP, data or file URL.)r
   scheme
startswithrE   r   	get_bytesenvsVLLM_MEDIA_URL_ALLOW_REDIRECTS
load_bytesr;   rC   r"   )r$   rH   r+   rG   r*   r   r7   r:   r(   r(   r)   load_from_url   s    



zMediaConnector.load_from_urlc          
         s   t |}t }|jr3|jdr3| | | j}|j||tj	dI d H }|
t|j|}|I d H S |jdkrF|
t| j||}|I d H S |jdkrY|
t| j||}|I d H S d}	t|	rI   )r
   asyncioget_running_looprM   rN   rE   r   async_get_bytesrP   rQ   run_in_executorglobal_thread_poolrR   r;   rC   r"   )
r$   rH   r+   rG   r*   loopr   r7   futurer:   r(   r(   r)   load_from_url_async   s2   







z"MediaConnector.load_from_url_async	audio_urlc                 C   s*   t di | jdi }| j||tjdS )z(
        Load audio from a URL.
        audiorF   Nr(   )r   r   getrS   rP   VLLM_AUDIO_FETCH_TIMEOUTr$   r\   audio_ior(   r(   r)   fetch_audio   s   zMediaConnector.fetch_audioc                    s2   t di | jdi }| j||tjdI dH S )z8
        Asynchronously fetch audio from a URL.
        r]   rF   Nr(   )r   r   r^   r[   rP   r_   r`   r(   r(   r)   fetch_audio_async   s   z MediaConnector.fetch_audio_asyncRGB)
image_mode	image_urlre   c             
   C   sX   t dd|i| jdi }z
| j||tjdW S  ty+ } ztt||d}~ww )z
        Load a PIL image from an HTTP or base64 data URL.

        By default, the image is converted into RGB format.
        re   imagerF   Nr(   )	r   r   r^   rS   rP   VLLM_IMAGE_FETCH_TIMEOUTr   r"   strr$   rf   re   image_ioer(   r(   r)   fetch_image   s   zMediaConnector.fetch_imagec             
      s`   t dd|i| jdi }z| j||tjdI dH W S  ty/ } ztt||d}~ww )z
        Asynchronously load a PIL image from an HTTP or base64 data URL.

        By default, the image is converted into RGB format.
        re   rg   rF   Nr(   )	r   r   r^   r[   rP   rh   r   r"   ri   rj   r(   r(   r)   fetch_image_async   s    z MediaConnector.fetch_image_async	video_urlc                C   sH   t dd|i| jdi }t|fi | jdi }| j||tjdS )z=
        Load video from an HTTP or base64 data URL.
        re   rg   videorF   Nr(   )r   r   r^   r   rS   rP   VLLM_VIDEO_FETCH_TIMEOUTr$   ro   re   rk   video_ior(   r(   r)   fetch_video  s   	zMediaConnector.fetch_videoc                   sP   t dd|i| jdi }t|fi | jdi }| j||tjdI dH S )z
        Asynchronously load video from an HTTP or base64 data URL.

        By default, the image is converted into RGB format.
        re   rg   rp   rF   Nr(   )r   r   r^   r   r[   rP   rq   rr   r(   r(   r)   fetch_video_async-  s   z MediaConnector.fetch_video_asyncr7   c                 C      t  }|d|S )z2
        Load image embedding from a URL.
        r   )r   r4   )r$   r7   image_embedding_ior(   r(   r)   fetch_image_embeddingC     z$MediaConnector.fetch_image_embeddingc                 C   rv   )z2
        Load audio embedding from a URL.
        r   )r   r4   )r$   r7   audio_embedding_ior(   r(   r)   fetch_audio_embeddingN  ry   z$MediaConnector.fetch_audio_embedding)%__name__
__module____qualname__r   dictri   r   r   listr    r	   r   r   r;   rC   rE   intrS   r[   tuplenpndarrayfloatrb   rc   r   rm   rn   nptNDArrayrt   ru   torchTensorrx   r{   __classcell__r(   r(   r&   r)   r   $   s    
/


#
$






r   ).rT   atexitconcurrent.futuresr   pathlibr   typingr   r   urllib.requestr   numpyr   numpy.typingr   r   PILr   r   urllib3.utilr	   r
   	vllm.envsrP   vllm.connectionsr   r   vllm.utils.registryr   r]   r   r   baser   rg   r   r   rp   r   r   VLLM_MEDIA_LOADING_THREAD_COUNTrX   registershutdownMEDIA_CONNECTOR_REGISTRYr   r(   r(   r(   r)   <module>   s4   