o
    ۷i2                     @  s   d dl mZ d dlZd dlZd dlmZ d dlmZmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZ d dlmZmZ d dlmZmZm Z  d dl!m"Z" d dl#m$Z$ ee%Z&G dd dZ'dS )    )annotationsN)
HTTPStatus)Anycast)HTTPExceptionRequest)Image)EngineClient)init_logger)	AsyncOmni)
parse_size)	VideoDataVideoGenerationRequestVideoGenerationResponse)decode_input_referenceencode_video_base64)OmniDiffusionSamplingParamsOmniSamplingParamsOmniTextPrompt)LoRARequest)stable_lora_int_idc                   @  s   e Zd ZdZ		d2d3ddZe	d4d5ddZ	d4ddd6ddZd7ddZe	d8ddZ
e	d9d$d%Zd:d)d*Ze	d;d-d.Zd<d0d1ZdS )=OmniOpenAIServingVideoz@OpenAI-style video generation handler for omni diffusion models.Nengine_clientr	   
model_name
str | Nonestage_configslist[Any] | NonereturnNonec                 C  s   || _ || _|| _d S N)_engine_client_model_name_stage_configs)selfr   r   r    r$   `/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm_omni/entrypoints/openai/serving_video.py__init__"   s   
zOmniOpenAIServingVideo.__init__diffusion_enginestrc                 C  s   | |||dS )N)r   r   r$   )clsr'   r   r   r$   r$   r%   for_diffusion,   s
   z$OmniOpenAIServingVideo.for_diffusion)input_reference_bytesrequestr   raw_requestRequest | Noner+   bytes | Noner   c             
     s6  |j rttjjdddt j }| |}|j	d ur/|d ur/|j	|kr/t
d|j	| d|ji}|jd ur>|j|d< d }zt|j|}W n ty_ } zttjjt|d|d }~ww |d urjd|i|d< t|jd	}	| |\}
}}}|
d ur|d ur|
|	_||	_|d ur||	_|d ur||	_|jd ur|j|	_|jd ur|j|	_|jd ur|j|	_|jd ur|j|	_|jd ur|j|	_|jd ur|j|	_t
d
|j|	j |j d ur|j |	j!d< | "|j#|	 t
d|	j|	j|	j|	j | $||	||I d H }| %|}|pd  fdd|D }t&t't(( |dS )Nz0Streaming video generation is not supported yet.status_codedetail
video_gen_zVModel mismatch: request specifies '%s' but server is running '%s'. Using server model.promptnegative_promptimagemulti_modal_data)num_outputs_per_promptz.Boundary ratio parse: request=%s gen_params=%s
flow_shiftzAVideo sampling params: steps=%s guidance=%s guidance_2=%s seed=%s   c                   s   g | ]}t t| d dqS ))fps)b64_json)r   r   ).0video
output_fpsr$   r%   
<listcomp>   s    z:OmniOpenAIServingVideo.generate_videos.<locals>.<listcomp>)createddata))streamr   r   BAD_REQUESTvalueuuiduuid4hex_resolve_model_namemodelloggerwarningr4   r5   r   input_reference
ValueErrorr(   r   n_resolve_video_paramswidthheight
num_framesr;   num_inference_stepsguidance_scaleguidance_scale_2true_cfg_scaleseedboundary_ratioinfor9   
extra_args_apply_loralora_run_generation_extract_video_outputsr   inttime)r#   r,   r-   r+   
request_idr   r4   input_imageexc
gen_paramsrR   rS   rT   r;   resultvideos
video_datar$   r?   r%   generate_videos9   s   












z&OmniOpenAIServingVideo.generate_videosc                 C  sP   | j r| j S |d u rd S t|jjdd }|r&t|dd r&|j}|r&|d jS d S )Nopenai_serving_modelsbase_model_pathsr   )r!   getattrappstaterl   name)r#   r-   serving_models
base_pathsr$   r$   r%   rJ      s   
z*OmniOpenAIServingVideo._resolve_model_name5tuple[int | None, int | None, int | None, int | None]c                 C  s   | j p| jr
| jj nd }| jp| jr| jjnd }| jp#| jr"| jjnd }| jp/| jr.| jjnd }| j}| jr=t| j\}}|d u rCd}|d u rS|d urSt|t| }||||fS )Nr:   )	rR   video_paramsrS   rT   r;   secondssizer   ra   )r,   rR   rS   rT   r;   ru   r$   r$   r%   rQ      s   z,OmniOpenAIServingVideo._resolve_video_params	lora_bodyr   rf   r   c                 C  s  | d u rd S t | tsttjjdd| dp!| dp!| d}| dp5| dp5| dp5| d	}| d
}|d u rD| d}| d}|d u rR| d}|d u r^|r^tt|}|rb|sjttjjddt	t|t
|t||_|d urt||_d S d S )Nz'Invalid lora field: expected an object.r0   rp   	lora_nameadapter
local_pathpath	lora_pathlora_local_pathscale
lora_scaleint_idlora_int_idz5Invalid lora object: both name and path are required.)
isinstancedictr   r   rE   rF   getr   r(   r   ra   lora_requestfloatr   )rw   rf   rx   r|   r   r   r$   r$   r%   r]      s@   





z"OmniOpenAIServingVideo._apply_lorar4   r   rc   c                   sh  t | jd}td| js|rt|jjdd rn	 ndnd|t| jj	 | jp:|r2t|jjdd nd p:t| jdd }|sKt | jdsKt
tjjdd|rt|D ]$}t |dr]|d	d
}nt|d	d
}|dkrst
tjjd| ddqOtt| j}	t|	dd }
t|
tr fdd|
D }n g}d }|	j|||d2 z3 d H W }|}q6 |d u rt
tjjdd|S )N
stage_listzMVideo generation routing: stage_configs=%s, has_stage_list=%s, engine_type=%sr   presentmissingzCStage configs not found. Start server with an omni diffusion model.r0   r   
stage_typellm	diffusionz8Video generation only supports diffusion stages, found 'z' stage.c                   s   g | ]} qS r$   r$   )r=   _rf   r$   r%   rA     s    z:OmniOpenAIServingVideo._run_generation.<locals>.<listcomp>)r4   rc   sampling_params_listz3No output generated from video generation pipeline.)hasattrr    rL   r[   r"   rm   rn   ro   type__name__r   r   SERVICE_UNAVAILABLErF   r   r   r   r   listgenerateINTERNAL_SERVER_ERROR)r#   r4   rf   rc   r-   has_stage_listr   stager   r   r   r   rg   outputr$   r   r%   r_      sr   
	


z&OmniOpenAIServingVideo._run_generationrh   	list[Any]c                   s  d u rg S t drjdkrfddtjd D S ttr~s'g S d }t |dr_|jdkr_g }D ]# t  drW jdkrW| fddt jd D  q9|  q9|S t|trfS t |drs|jdkrsgS t|tjr|gS S gS )Nndim   c                      g | ]} | qS r$   r$   r=   i)rh   r$   r%   rA         zCOmniOpenAIServingVideo._normalize_video_outputs.<locals>.<listcomp>r   c                   r   r$   r$   r   )itemr$   r%   rA   #  r      )	r   r   rangeshaper   r   extendappendr   )rh   first	flattenedr$   )r   rh   r%   _normalize_video_outputs  s.   
$
z/OmniOpenAIServingVideo._normalize_video_outputsrg   c                 C  s   d }t |dr|jr|j}n1t |dr?|j}t|tr%|dr%|d }nt |dr1|jr1|j}nt |dr?|jr?|jd}|d u rQt |drQ|jrQ|jd}| |}|s`tt	j
jdd|S )Nimagesrequest_outputmultimodal_outputr>   z,No video outputs found in generation result.r0   )r   r   r   r   r   r   r   r   r   r   r   rF   )r#   rg   rh   r   
normalizedr$   r$   r%   r`   0  s(   


z-OmniOpenAIServingVideo._extract_video_outputs)NN)r   r	   r   r   r   r   r   r   r   )r'   r	   r   r(   r   r   r   r   )r,   r   r-   r.   r+   r/   r   r   )r-   r.   r   r   )r,   r   r   rs   )rw   r   rf   r   r   r   )
r4   r   rf   r   rc   r(   r-   r.   r   r   )rh   r   r   r   )rg   r   r   r   )r   
__module____qualname____doc__r&   classmethodr*   rj   rJ   staticmethodrQ   r]   r_   r   r`   r$   r$   r$   r%   r      s*    

W
"Cr   )(
__future__r   rb   rG   httpr   typingr   r   fastapir   r   PILr   vllm.engine.protocolr	   vllm.loggerr
    vllm_omni.entrypoints.async_omnir   ,vllm_omni.entrypoints.openai.image_api_utilsr   ,vllm_omni.entrypoints.openai.protocol.videosr   r   r   ,vllm_omni.entrypoints.openai.video_api_utilsr   r   vllm_omni.inputs.datar   r   r   vllm_omni.lora.requestr   vllm_omni.lora.utilsr   r   rL   r   r$   r$   r$   r%   <module>   s$   