o
    ॵi9                     @   sZ  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlm  m  m  mZ d dlmZ d dlmZmZ d dl m!Z!m"Z"m#Z# d d	l$m%Z% d d
l&m'Z' d dl(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 e2 Z3e,j4e.j5ejdG dd de*Z6dS )    N)partial)AnyDict)VideoCap)Image)	Pipelines)CenterCropV3random_resize)make_irregular_maskmake_rectangle_maskmake_uncrop)	rand_name)
OutputKeys)InputPipeline)	PIPELINES)Tasks)device_placement)
get_logger)module_namec                       s   e Zd ZdZdef fddZdedeeef fddZ	deeef deeef fd	d
Z
deeef deeef fddZdd Z				dddZ  ZS )VideoComposerPipelinea   Video Composer Pipeline.

    Examples:

    >>> from modelscope.pipelines import pipeline
    >>> from modelscope.utils.constant import Tasks
    >>> pipe = pipeline(
            task=Tasks.text_to_video_synthesis,
            model='buptwq/videocomposer',
            model_revision='v1.0.1')
    >>> inputs = {'Video:FILE': 'path/input_video.mp4',
                  'Image:FILE': 'path/input_image.png',
                  'text': 'the text description'}
    >>> output = pipe(inputs)
    modelc                    s  t  j|d |dd| _tj| jst| j |dd| _|dg d| _	|ddg| _
|d	d
d
d
d
d| _t| j	}t| j
}| j	d||  |  | _| jt| j | _|dd| _|dd| _|dg d| _|dg d| _|dd| _|dg d| _|dg d| _|dd| _|dd| _|dd| _|dd| _ttj| jd t tj| j| jd!g| _ tt!"t#t$| jd t| jt g| _%tt!j&| jd t!'| jg| _(t!t)| jt! t!j| j| jd!g| _*d"S )#z
        use `model` to create a videocomposer pipeline for prediction
        Args:
            model: model id on modelscope hub.
        r   log_dirz./video_outputsfeature_framerate   
frame_lens)   r   r   r   feature_frameratesbatch_sizes   )14816r   
resolution   image_resolutionmean)      ?r)   r)   stdvit_image_size   vit_mean)g3<4'?gwgM?gy{ ?vit_std)gB91?gwt.?g	U?z
kwargs.popi  	visual_mvF	max_wordsi  
mvs_visual)size)r(   r*   N)+super__init__popr   ospathexistsmakedirsr   r   r   r   len
max_framesstr
batch_sizer%   r'   r(   r*   r+   r-   r.   	misc_sizer/   r0   r1   dataComposeCenterCropV2ToTensor	Normalizeinfer_transTLambdar   r	   misc_transformsResize
CenterCropmv_transformsr   vit_transforms)selfr   kwargsl1l2	__class__ k/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/multi_modal/videocomposer_pipeline.pyr4   7   sl   




zVideoComposerPipeline.__init__inputreturnc              
   C   s  |d }|d }|d }d }| j }tj|rpz| || j || j\}}}	}
}W n tyo } z;tjd	||dd t
d| j| j}t
| jd| j| j}	t
| jd| j| j}
t
| jd| j| j}W Y d }~n=d }~ww td	 t
d| j| j}t
d| j| j}t
| jd| j| j}	t
| jd| j| j}
t
| jd| j| j}t }|d
k rtdd}n|dk rtdd}ntdd}t
tj|| j| jftjdd }|dj| jdd}|d||	d|
d||d|d|d}|S )Nz
Video:FILEtextz
Image:FILEz&{} get frames failed... with error: {}Tflush      z7The video path does not exist or no video dir provided!gffffff?i   g?)interpolationr   )repeatsdim)	ref_framecap_txt
video_data	misc_datar   maskmv_datastyle_image)r   r6   r7   r8   video_data_preprocessr1   	Exceptionloggerinfoformattorchzerosr+   r;   r'   r>   randomr
   r   r   
from_numpycv2resizeINTER_NEAREST	unsqueezefloatrepeat_interleave)rL   rT   	video_keyr_   rd   total_framesr   r^   	vit_imager`   ra   rc   e_prb   video_inputrR   rR   rS   
preprocessy   s   







z VideoComposerPipeline.preprocessc                 C   s
   |  |S )Nr   )rL   rT   rR   rR   rS   forward   s   
zVideoComposerPipeline.forwardinputsc                 K   sJ   | dd }d}|d urtjddj}d}|rtj|d iS tj|d iS )Noutput_videoF.gif)suffixT
video_pathvideo)gettempfileNamedTemporaryFilenamer   OUTPUT_VIDEO)rL   r}   post_paramsoutput_video_pathtemp_video_filerR   rR   rS   postprocess   s   z!VideoComposerPipeline.postprocessc                    sx  |}t dD ].}z| j|||d\} W  n ty4 } ztjd||dd W Y d }~qd }~ww t|}tt	|dk|t
| | jk@ d }	tj|	}
t
|
|
| j } fdd	|D  fd
d	|D t|rfdd	|D }| jd |dd  d }tj| jd stj| jd dd td| tj||dd t dk}tt d }|r | }| |}| jd d  }| jdd  |}|   | n	td| j| j}t| jd| j | j }t| jd| j | j }t| jd| j!| j!}|r0 |d t df< ||d t df< |d t df< |} ~|||||fS )N   )input_videofpsr/   z={} read video frames and motion vectors failed with error: {}TrW   Ir   c              
      s2   g | ]}t  | d d d d d d df qS )N)r   	fromarray.0i)framesrR   rS   
<listcomp>   s   2 z?VideoComposerPipeline.video_data_preprocess.<locals>.<listcomp>c                    s    g | ]}t  | d qS ))rZ   r   r    )rj   rm   	transposer   )mvsrR   rS   r      s     c                    s2   g | ]} | d d d d d d df  dqS )Nr   uint8)astyper   )r1   rR   rS   r      s    *z/visual_mv//r   r   exist_okz&save motion vectors visualization to :   )r   rZ   rY   .)"rangeextract_motion_vectorsrf   rg   errorri   r:   npwherearrayaranger;   rl   choicerj   stackr   splitr6   r7   r8   r9   rh   imageiomimwriteintrK   rG   rD   rJ   rk   r+   r'   r>   )rL   rt   r   ru   r/   filenamerx   frame_typesrw   start_indexsstart_indexindicesimagesr7   have_framesmiddle_indixr^   rv   misc_imgs_np	misc_imgsr`   rc   ra   rR   )r   r   r1   rS   re      s   




z+VideoComposerPipeline.video_data_preprocessr   Fc           $      C   s  |rt  d}dD ]}tjtjd| |dd qt }tj|dd | |dd  }	t	
|}
|
t	j}|
t	j}|| | d	krSt|d
}n
td	||  d
 }d| d| d|	 }tj|	rst|	 tj|ddd t }||	}|std|	 d}g }g }g }g }g }	 |rtjd|dd t }| \}}}}}t }|| }|| |s|rtd ntj|  j!tj"d}|rt#||}d}|j!d
 |j!d kr|j!d
 |j!d  d }|rt	$tjdd| d|d d || f  ||d d || f  n5|j!d |j!d
  d }|rCt	$tjdd| d||| d d f  |||| d d f  |j!d d \} }!t| |!df}"|d d ddf %d|!d
 | d
 f}#|d d dd
f |d d ddf  |d d dd f  |"|#d d d
f |#d d df f< |d
7 }|| || ||" q|rtdt&| |'  tj|	rt|	 ||||fS ) Nz%Y-%m-%dT%H:%M:%S)r   motion_vectorszout-Tr   r   r   r   r   r    z%ffmpeg -threads 8 -loglevel error -i z' -filter:v                         fps=z -c:v mpeg4 -f rawvideo x   )argsshelltimeoutzCould not open zFrame:  )endzNo frame read. Stopping.)dtypeFrZ   z./mv_visual/zframe-z.jpgr      )r   r   	   zaverage dt: )(datetimenowstrftimer6   r9   r7   joinr   r   rn   VideoCapturer   CAP_PROP_FRAME_COUNTCAP_PROP_FPSmaxr   r8   remove
subprocessrunr   openRuntimeErrorrg   rh   timeperf_counterreadappendwarningr   rk   copyshaper   draw_motion_vectorsimwriteclipr(   release)$rL   r   r   dumpverboser/   r   childtemp	tmp_videovideocapture
frames_num	fps_video
ffmpeg_cmdcapretsteptimesr   r   r   r1   tstartframer   
frame_type	timestamptendtelapsed
frame_savew_halfh_halfhwmvpositionrR   rR   rS   r     s    





(




6
z,VideoComposerPipeline.extract_motion_vectors)r   FFF)__name__
__module____qualname____doc__r<   r4   r   r   r   r{   r|   r   re   r   __classcell__rR   rR   rP   rS   r   $   s    B"F

Hr   )7r6   rl   r   r   r   	functoolsr   typingr   r   rn   r   numpyr   rj   torchvision.transforms
transformsrE   mvextractor.videocapr   PILr   0modelscope.models.multi_modal.videocomposer.datamodelsmulti_modalvideocomposerr?   modelscope.metainfor   ;modelscope.models.multi_modal.videocomposer.data.transformsr   r	   ;modelscope.models.multi_modal.videocomposer.ops.random_maskr
   r   r   7modelscope.models.multi_modal.videocomposer.utils.utilsr   modelscope.outputsr   modelscope.pipelines.baser   r   modelscope.pipelines.builderr   modelscope.utils.constantr   modelscope.utils.devicer   modelscope.utils.loggerr   rg   register_moduletext_to_video_synthesisr   rR   rR   rR   rS   <module>   s<   