o
    ߥi0+                     @   s   d Z ddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 dejj_d\ZZdZdZd	ZdZd
d ZG dd deZG dd deZe
e e gZdd Zdd Zdd Zdd Zdd Zdd ZdS )z VideoInpaintingProcess
The implementation here is modified based on STTN,
originally Apache 2.0 License and publicly available at https://github.com/researchmm/STTN
    N)Image)
transformsF)   `   i,        c                 C   sd   t | }| \}}|du rd}d\}}}nd }|jdd \}}|t j}|  ||||fS )NFdecode_error)r   r   r   r      )cv2VideoCapturereadshapegetCAP_PROP_FPSrelease)video_input_pathvideo_inputsuccessframer   whfps r   d/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/cv/video_inpainting/inpainting.pyvideo_process   s   
r   c                   @   s   e Zd ZdddZdd ZdS )StackFc                 C   
   || _ d S N)roll)selfr   r   r   r   __init__)      
zStack.__init__c                 C   s   |d j }|dkrdd |D }d}|dkr"tjdd |D ddS |d	kr<| jr5tjd
d |D ddS tj|ddS td| )Nr   1c                 S   s   g | ]}| d qS )L)convert).0imgr   r   r   
<listcomp>/   s    z"Stack.__call__.<locals>.<listcomp>r#   c                 S   s   g | ]}t |d qS )r	   )npexpand_dimsr%   xr   r   r   r'   2   s    r	   )axisRGBc                 S   s.   g | ]}t |d d d d d d df qS )N)r(   arrayr*   r   r   r   r'   5   s   . zImage mode )moder(   stackr   NotImplementedError)r   	img_groupr0   r   r   r   __call__,   s   
zStack.__call__N)F)__name__
__module____qualname__r    r4   r   r   r   r   r   '   s    
r   c                   @   s"   e Zd ZdZdddZdd ZdS )	ToTorchFormatTensorz Converts a PIL.Image (RGB) or numpy.ndarray (H x W x C) in the range [0, 255]
    to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0] Tc                 C   r   r   )div)r   r9   r   r   r   r    A   r!   zToTorchFormatTensor.__init__c                 C   s   t |tjrt|dddd }n'ttj	|
 }||jd |jd t|j}|dddd }| jrG| d}|S | }|S )Nr	      r         )
isinstancer(   ndarraytorch
from_numpypermute
contiguous
ByteTensorByteStoragefrom_buffertobytesviewsizelenr0   	transposer9   float)r   picr&   r   r   r   r4   D   s    zToTorchFormatTensor.__call__N)T)r5   r6   r7   __doc__r    r4   r   r   r   r   r8   =   s    
r8   c                 C   s  | j \}}}| dk r*| dt|dt|fdt|dt|gdt|dt|gfS tt| dkd }tt| dkd }tt| dkd }tt| dkd }||||g}tt|d d d }	tt|d d d }
||	k r||	k r||
k r||
k r| d |	d |
d d f }d|	d|
f}nI||	k r||	k r||
kr||
kr| d |	||
 |d d f }d|	||
 t|f}n||	kr||	kr||
k r||
k r| ||	 |d |
d d f }||	 t|d|
f}n||	kr&||	kr&||
kr&||
kr&| ||	 |||
 |d d f }||	 t|||
 t|f}n||	k rP||	k rP||
k rP||
krP| d |	d d d d f }d|	dt|f}n||	k rz||	krz||
k rz||
k rz| d d d |
d d f }dt|d|
f}nl||	kr||	kr||
k r||
kr| ||	 |d d d d f }||	 t|dt|f}n<||	k r||	kr||
kr||
kr| d d ||
 |d d f }dt|||
 t|f}n| }dt|dt|f}||d  }||d  }||d  }||d  }|||||||gfS )Nr<   r   r;   r	      )r   allintr(   minwheremaxround)maskorig_horig_w_hshewswecrop_boxmask_hmask_w	crop_maskres_pixabcdr   r   r   get_crop_mask_v1S   sd   

   ("((((rf   c                 C   s,   g }t d|tD ]}|| vr|| q|S )Nr   )range
ref_lengthappend)neighbor_idslength	ref_indexir   r   r   get_ref_index   s   
rn   c                 C   s   g }t d|  t| dd dd dd }t| dd dd dd dd }t| }t|d	}t|dk	tj
}d| }t|d |d D ]}|t|d
  qY|S )Nzmask_path: {}/r.   mask_r;   rX   r   .r#   r<   )printformatrP   splitr   openr(   r/   r$   astypeuint8rg   ri   	fromarray)mpathmasksstartendmrm   r   r   r   read_mask_oneImage   s   &*
r~   c                 C   s(   d}| dkr
d} d}|dkrd}d}|S )NF   Ti  r   )r   r   	is_resizer   r   r   
check_size   s   r   c           	      C   s   t | }|  g }g }g }g }|D ]?}|dd }|t|dd  |t|dd dd  |t j| | t	t j| |}|| q||||fS )Nrp   r;   rX   r   rq   )
oslistdirsortrt   ri   rP   pathjoinr
   imread)		mask_path
mask_namesabs_mask_path	mask_list
begin_listend_list	mask_namemask_name_tmprU   r   r   r   get_mask_list   s   
"r   c           @      C   s4  t |}t jd }t |||||f}	t|\}
}}}g }t|
D ]p\}}t|| }t|\}}}}|jdd \}}t	|jd |jd }|| }|| }t
d| t
d| t||d tD ]*}t }|t |krwt}n|| d }t|D ]}| \}}|| qg } |D ]&}!t|!}!|!|d |d |d |d f}"|"ttftj}#| |# qt| dd d }$dd	 | D } g }%||| || |  D ]!}&|&|d |d |d |d f}'|'ttftj}(|%|( qd
d	 |%D })t|%d}%tj r|$ |% }$}%d g| }*|   t / |$d|%   }+|+ |dtt}+| j!"|+}+|+# \}},}-}.|+ d||,|-|.}+W d    n	1 sYw   Y  td|t$D ]}!dd	 tt%d|!t$ t&||!t$ d D }/t'|/|}0t  | j!(|+d|/|0 d d d d d d f |%d|/|0 d d d d d d f }1t)| j!*|1d t+|/d d d d d d f , }2|2d d }2|2- .dddd/ d }2tt+|/D ]A}3|/|3 }4t01|2|3 2t0j3|)|4  | |4 d|)|4    }5|*|4 d u r|5|*|4< q|*|4 2t0j4d |52t0j4d  |*|4< qW d    n	1 s-w   Y  qdt
dt |  t|D ]N}!t01|*|! 2t0j3|)|!  | |! d|)|!    }6|rgt |6||f}6||! }7|\}8}9}:};|\}<}=}>}?|6|=|?|<|>f |7|9|;|8|:f< |	5|7 qAg }qgq"|6  d S )Nmp4vr   r	   r;   z	begin: {}zend: {}r:   c                 S   s   g | ]}t |t jqS r   )r(   r/   rv   rw   )r%   fr   r   r   r'      s    z/inpainting_by_model_balance.<locals>.<listcomp>c                 S   s*   g | ]}t t |d kt jdqS )r   r	   )r(   r)   r/   rv   rw   )r%   r}   r   r   r   r'      s    c                 S   s   g | ]}|qS r   r   )r%   rm   r   r   r   r'     s    r<   g      ?zinpainting time:)7r
   r   VideoWriter_fourccVideoWriterr   	enumerater~   rf   r   r   rr   rs   rg   	MAX_frametimer   ri   r   rx   cropresizer   r   NEAREST_to_tensors	unsqueezer?   cudais_availableevalno_gradrK   rG   modelencoderrH   neighbor_striderS   rQ   rn   infertanhdecoderrI   detachcpurA   numpyr(   r/   rv   rw   float32writer   )@r   video_inputPathr   video_savePathr   w_orih_ori	video_orifourcc
video_saver   r   r   r   img_npyindexrU   rz   ra   crop_for_oriimgcrop_for_inpimgr^   r_   r   beginr|   rm   
begin_timevideo_lengthframe_countrX   r   frames_tempr   i_temprb   
feats_temp
masks_tempr}   m_temprc   binary_masks_tempcomp_frames	feats_outrd   feat_hfeat_wrj   ref_ids	pred_featpred_imgjidxr&   compcomplete_framea1b1c1d1a2b2c2d2r   r   r   inpainting_by_model_balance   s   






  "

$Tr   ) rM   r   r   r
   r   r(   r?   PILr   torchvisionr   backendscudnnenabledr   r   rh   r   default_fpsr   r   objectr   r8   Composer   rf   rn   r~   r   r   r   r   r   r   r   <module>   s0    
5