o
    ॵi^                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlZd dlm	Z	m
Z
mZmZ d dlZd dlZd dlZd dlm  mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z m!Z! d d	l"m#Z# d d
l$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z, d dl-m.Z. dZ/e. Z0dd Z1dd Z2d%ddZ3	d&ddZ4	d&ddZ5dd Z6dd Z7d d! Z8d"gZ9e#j:e,j;ej;d#G d$d" d"e!Z<dS )'    N)AnyDictOptionalUnion)	make_grid)	Pipelines)do_scene_detect) VFINetForVideoFrameInterpolation)
OutputKeys)InputPipeline)	PIPELINES)	LoadImage)VideoReader)Config)	ModelFileTasks)
get_logger).mp4z.movc                 C   s4   | d } t g ddddd| }| |8 } | S )Ng     o@g~jt?g/$?g rh?      torchTensorviewtype_as)
img_tensormean r   n/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/cv/video_frame_interpolation_pipeline.py	img_trans"   s   r!   c                 C   s(   t g ddddd| }| | S )Nr   r   r   r   )xr   r   r   r    add_mean*   s    r#       c                 C   sL   |d | d | }|d | d | }d|| d|| f}t | |} | S )Nr   r   )Fpad)r   heightwidthpad_numphpwpaddingr   r   r    img_padding/   s
   r-   Fc                 C   s   |r|d8 }n|d8 }||krM|d | }t |d}|d |	d k r%|}nd| |	d k r0|}n
|| ||||||}dt| }|
| ||	7 }||ks|
|fS Nr      r      roundr#   append)flow_10flow_12flow_21flow_23img1img2inter_model
read_countinter_countdeltaoutputsstart_end_flagtoutputr   r   r    do_inference_lowers7   s$   


rB   c              
   C   s   |r|	d8 }	n|	d8 }	|
|	krO|
d |	 }t |d}|d |d k r%|}nd| |d k r0|}n|| ||||||||	}dt| }|| |
|7 }
|
|	ks||
fS r.   r1   )r4   r5   r6   r7   r8   r9   img1_upimg2_upr:   r;   r<   r=   r>   r?   r@   rA   r   r   r    do_inference_highersZ   s$   


rE   c           $      C   s8  |
|  d|
|  d}}d }d }d }d }t u ||k r7|
| }t|||}|	| |t| |d7 }t|dkr|d }|d }|	d }|	d }| ||ddd\}}| ||ddd\}}|||||d\}}t|d d d d d|d|f |d d d d d|d|f |	d d d d d d|d|f |	d d d d d d|d|f |d< |d rtd d d d |||||||dd\}}ntd ||d |||||||dd\}}t|d	kr3|d u s|d u rY|d }|	d }| ||ddd\}}| ||ddd\}}|||||d\}}t|d d d d d|d|f |d d d d d|d|f |	d d d d d d|d|f |	d d d d d d|d|f |d< |d }|	d } | || ddd\}}!| | |ddd\}}"||||!|"d\}}#t|d d d d d|d|f |#d d d d d|d|f |	d d d d d d|d|f |	d d d d d d|d|f |d< |d rtd d d d |||||||\}}n+|d s|d rtd ||d |||||||\}}nt||||!|||||||\}}|	d |		d |}|}|}| }|}|}|#}|}|!}|"}|d |d< |d |d< d
|d< ||k s"|dkr|		d |	d t|dksOJ |d rgtd d d d |||||||dd\}}n"td ||d |||||||dd\}}W d    |S W d    |S W d    |S 1 sw   Y  |S )Nr/   r   r   r      Titers	test_moder?      F)
sizer   no_gradr-   r3   r!   lenr   rB   pop)$
flow_modelrefine_modelr:   	video_lenr;   r<   r=   scene_change_flagimg_tensor_listimg_ori_listinputsr>   r'   r(   r4   r5   r6   r7   imgimg0r8   img0_oriimg1_ori_
flow_01_up
flow_10_upflow_01r9   img2_ori
flow_12_up
flow_21_upimg3img3_ori
flow_23_up
flow_32_upflow_32r   r   r    inference_lowers   sh  



2""


""

2""






s


 
 
 
   rg   c           *      C   s  |
|  dd dks|
|  dd dkrtd|
|  dd |
|  dd }}d }d }d }d }g }t  ||k r|
| }t||d |d dd}tj|ddd	d
}|t| |	| |t| |d7 }t	|dkr"|d }|d }|	d }|	d }|d }|d }| ||ddd\}}| ||ddd\}}|||||d\}}t
|d d d d d|d|f |d d d d d|d|f |	d d d d d d|d|f |	d d d d d d|d|f |d< |d rtd d d d |||||||||dd\}}ntd ||d |||||||||dd\}}t	|dkr|d u s3|d u r|d }|	d } |d }!| || ddd\}}"| | |ddd\}}#||||"|#d\}}t
|d d d d d|d|f |d d d d d|d|f |	d d d d d d|d|f |	d d d d d d|d|f |d< |d }$|	d }%|d }&| | |%ddd\}}'| |%| ddd\}}(|||$|'|(d\}})t
|d d d d d|d|f |)d d d d d|d|f |	d d d d d d|d|f |	d d d d d d|d|f |d< |d r)td d d d ||||!|||||\}}n/|d s3|d rFtd ||d ||||!|||||\}}nt||||'||||!|||||\}}|d |d |	d |}|$}| }|%} |!}|&}!|}|}|)}|#}|'}"|(}#|d |d< |d |d< d	|d< ||k sB|dkr|	d |d t	|dksJ |d rtd d d d ||||!|||||dd\}}n$td ||d ||||!|||||dd\}}W d    |S W d    |S W d    |S 1 sw   Y  |S )Nr/   r   r   z#Video width and height must be even@   )r)   g      ?bilinearF)scale_factormodealign_cornersr   rF   TrG   rJ   rK   )rL   RuntimeErrorr   rM   r-   r%   interpolater3   r!   rN   r   rE   rO   )*rP   rQ   r:   rR   r;   r<   r=   rS   rT   rU   rV   r>   r'   r(   r4   r5   r6   r7   img_up_listimg_uprW   rX   r8   rY   rZ   img0_uprC   r[   r\   r]   r^   r9   r_   rD   r`   ra   rb   rc   img3_uprd   re   rf   r   r   r    inference_highers$  s  



2""



""

2""








  



 
 , 
 , 
 ,  ,rs   c                 C   s   dd |   D S )Nc                 S   s&   i | ]\}}d |v r| d d|qS )zmodule. )replace).0kvr   r   r    
<dictcomp>  s    zconvert.<locals>.<dictcomp>)items)paramr   r   r    convert  s   r|   VideoFrameInterpolationPipeline)module_namec                	       s   e Zd ZdZ	ddeeef f fddZddede	d	e
eef fd
dZde
eef d	e
eef fddZde
eef d	e
eef fddZ  ZS )r}   aa   Video Frame Interpolation Pipeline.

    Examples:

    >>> from modelscope.pipelines import pipeline
    >>> from modelscope.utils.constant import Tasks
    >>> from modelscope.outputs import OutputKeys

    >>> video = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/videos/video_frame_interpolation_test.mp4'
    >>> video_frame_interpolation_pipeline = pipeline(Tasks.video_frame_interpolation,
    'damo/cv_raft_video-frame-interpolation')
    >>> result = video_frame_interpolation_pipeline(video)[OutputKeys.OUTPUT_VIDEO]
    >>> print('pipeline: the output video path is {}'.format(result))
    Nmodelc                    sl   t  jd||d| tj rtd| _ntd| _| jj| _| j	| j | j
  td d S )N)r   preprocessorcudacpuz#load video frame-interpolation doner   )super__init__r   r   is_availabledevice_devicer   nettoevalloggerinfo)selfr   r   kwargs	__class__r   r    r     s   


z(VideoFrameInterpolationPipeline.__init__r   inputout_fpsreturnc           	      C   s   t |tr
t|}nt |trt|d }g }|D ]}|| q|j}t|D ]\}}t|	 
ddd }|d||< q(t |trLd| }nt |trid|v r\|d | }nd|v re|d }nd| }|||dS )Nvideor/   r   r   interp_ratior   )r   fpsr   )
isinstancestrr   dictr3   r   	enumerater   
from_numpycopypermutefloat	unsqueeze)	r   r   r   video_readerrV   framer   irW   r   r   r    
preprocess  s(   






z*VideoFrameInterpolationPipeline.preprocessc                 C   s  |d }|d }|d }t |}| jj}| jjj}d}d}	|| }
g d}g }g }g }|| d|| d}}|dksB|d	krW| jjj}t||||||	|
|||||}n| jjj}t||||||	|
|||||}t	t |D ]}|| d d d d d|d|f ||< qq||d
S )Nr   r   r   r   )FFFr/   r   i  i 
  )rA   r   )
rN   r   flownetinternetifnetrL   internet_Dsrs   rg   range)r   r   rV   r   r   rR   rP   rQ   r;   r<   r=   rS   rT   rU   r>   r'   r(   r:   r   r   r   r    forward)  s@   


*
z'VideoFrameInterpolationPipeline.forwardrV   c                 K   s   | dd }| dd}|d u rtjddj}|d d jdd  \}}tjd	 }t|||d
 ||f}tt	|d D ] }	|d |	 }
|
d 
ddd   }
||
tj q=|  |rtddksoJ d|d d d }d| d| }tj|dd tj|iS tj|iS )Noutput_videodemo_serviceTr   )suffixrA   r   mp4vr   r   r/   zffmpeg -versionz"ffmpeg is not installed correctly!z_web.mp4z
ffmpeg -i z -vcodec h264 -crf 5 )shell)gettempfileNamedTemporaryFilenameshapecv2VideoWriter_fourccVideoWriterr   rN   r   byter   numpywriteastypenpuint8releaseossystem
subprocesscallr
   OUTPUT_VIDEO)r   rV   r   output_video_pathr   hwfourccvideo_writerr   rW   output_video_path_for_webconvert_cmdr   r   r    postprocessM  s6   


z+VideoFrameInterpolationPipeline.postprocess)N)r   )__name__
__module____qualname____doc__r   r	   r   r   r   r   r   r   r   r   r   __classcell__r   r   r   r    r}     s    
 "*$)r$   )F)=globmathr   os.pathpathospr   r   typingr   r   r   r   r   r   r   r   torch.nn.functionalnn
functionalr%   torchvision.utilsr   modelscope.metainfor   Kmodelscope.models.cv.video_frame_interpolation.utils.scene_change_detectionr   Smodelscope.models.cv.video_frame_interpolation.VFINet_for_video_frame_interpolationr	   modelscope.outputsr
   modelscope.pipelines.baser   r   modelscope.pipelines.builderr   modelscope.preprocessorsr   modelscope.preprocessors.cvr   modelscope.utils.configr   modelscope.utils.constantr   r   modelscope.utils.loggerr   VIDEO_EXTENSIONSr   r!   r#   r-   rB   rE   rg   rs   r|   __all__register_modulevideo_frame_interpolationr}   r   r   r   r    <module>   sV   

0
% & A