o
    }oi                     @   s   d dl mZ d dlZd dlZd dlZd dlmZ d dl	m
Z
 dZzd dlmZ W n ey3   dZY nw zd dlmZ W n eyN   d dlZed	 Y nw G d
d dZdS )    )BytesION)Image)str_list2numpyT)ModelClientF)VideoReaderz;The package `decord` was not installed in this environment.c                   @   sL   e Zd ZdZdd Zdd Zdd Zdd	 Z	
		
				
		dddZdS )NemoQueryMultimodala  
    Sends a query to Triton for Multimodal inference

    Example:
        from nemo.deploy.multimodal import NemoQueryMultimodal

        nq = NemoQueryMultimodal(url="localhost", model_name="neva", model_type="neva")

        input_text = "Hi! What is in this image?"
        output = nq.query(
            input_text=input_text,
            input_media="/path/to/image.jpg",
            max_output_len=30,
            top_k=1,
            top_p=0.0,
            temperature=1.0,
        )
        print("prompts: ", prompts)
    c                 C   s   || _ || _|| _d S )N)url
model_name
model_type)selfr   r	   r
    r   [/home/ubuntu/.local/lib/python3.10/site-packages/nemo/deploy/multimodal/query_multimodal.py__init__;   s   
zNemoQueryMultimodal.__init__c                 C   s>  | j dkrt|}dd |D }t|S | j dks| j dkr:t|}dd |D }| |}| ||}t|S | j dv rn|dsI|d	r\tj|d
d}t	
t|jd}nt	
|d}tjt|ddS | j dkrtj|tjd\}}	tj|gtjd}
tjt|ggtjd}|
|dS td| j  )zSetup input mediaz
video-nevac                 S      g | ]}|  qS r   asnumpy.0fr   r   r   
<listcomp>D       z3NemoQueryMultimodal.setup_media.<locals>.<listcomp>litavitac                 S   r   r   r   r   r   r   r   r   H   r   )nevavilamllamahttphttps   )timeoutRGBr   axissalmdtype)input_signalinput_signal_lengthzInvalid model type )r
   r   nparray	frame_lenget_subsampled_frames
startswithrequestsgetr   openr   contentconvertexpand_dimssfreadfloat32lenint32RuntimeError)r   input_mediavrframessubsample_len
sub_framesresponsemediawaveformsample_rater&   r'   r   r   r   setup_media@   s,   






zNemoQueryMultimodal.setup_mediac                 C   sL   d}t ||krt |S tttt || }tttt || S )zGet frame len   )r6   intr(   ceilfloatround)r   r;   
max_frames	subsampler   r   r   r*   [   s
   zNemoQueryMultimodal.frame_lenc                    s8   t t dt d |t} fdd|D }|S )zGet subsampled framesr      c                    s   g | ]} | qS r   r   )r   ir;   r   r   r   g   r   z=NemoQueryMultimodal.get_subsampled_frames.<locals>.<listcomp>)r(   rG   linspacer6   astyperD   )r   r;   r<   idxr=   r   rL   r   r+   d   s   "z)NemoQueryMultimodal.get_subsampled_framesrJ                    ?      N@Nc                 C   s  t |g}d|i}| |}t|tr|| ntj|tjddddddddf |jd dd|d< |durFtj	|j|tj
d|d< |durVtj	|j|tj
d|d< |durftj	|j|tj
d|d	< |durvtj	|j|tjd|d
< |durtj	|j|tjd|d< |durtj	|j|tjd|d< |	durtj	|j|	tj
d|d< |durtj|d}t	|jd t|f||d< t| j| j|
d5}|jdi |}|jjd j}|tjkrtj|d dd}|W  d   S |d W  d   S 1 sw   Y  dS )z	Run query
input_textNr   r!   r9   r$   
batch_sizemax_output_lentop_ktop_ptemperaturerepetition_penalty	num_beamszutf-8	lora_uids)init_timeout_soutputsbytesr   )r   rB   
isinstancedictupdater(   repeatnewaxisshapefullint_singlecharencoder6   r   r   r	   infer_batchmodel_configr^   r%   bytes_decoderN   )r   rT   r9   rU   rV   rW   rX   rY   rZ   r[   init_timeoutr\   promptsinputsr?   clientresult_dictoutput_type	sentencesr   r   r   queryj   s@   


:
&zNemoQueryMultimodal.query)	rJ   rP   rJ   rQ   rR   rR   rJ   rS   N)	__name__
__module____qualname____doc__r   rB   r*   r+   rv   r   r   r   r   r   &   s     	
r   )ior   numpyr(   r-   	soundfiler3   PILr   nemo.deploy.utilsr   use_pytritonpytriton.clientr   	Exceptiondecordr   loggingwarningr   r   r   r   r   <module>   s&   