o ॵi%@sddlZddlmZddlmZmZddlZddlZddl Z ddl Z ddl m Z ddl mZddlmZddlmZddlmZmZddlmZdd lmZdd lmZmZdd lmZeZej ej!ej!d Gd ddeZ"dS)N)AnyDict)tqdm) Pipelines) KNetTrack) OutputKeys)InputPipeline) PIPELINES)Config) ModelFileTasks) get_logger) module_namecseZdZdZdeffdd ZdedeeeffddZ deeefdeeeffd d Z d eeefdeeeffd d Z Z S)!VideoInstanceSegmentationPipelinea Video Instance Segmentation Pipeline. Examples: >>> from modelscope.pipelines import pipeline >>> detector = pipeline('video-instance-segmentation', 'damo/cv_swinb_video-instance-segmentation') >>> detector("http://www.modelscope.cn/api/v1/models/damo/cv_swinb_video-instance-segmentation/repo?Revision=master" >>> "&FilePath=resources/kitti-step_testing_image_02_0000.mp4") >>> { >>> "boxes": [ >>> [ >>> [ >>> 0, >>> 446.9007568359375, >>> 36.374977111816406, >>> 907.0919189453125, >>> 337.439208984375, >>> 0.333 >>> ], >>> [ >>> 1, >>> 454.3310241699219, >>> 336.08477783203125, >>> 921.26904296875, >>> 641.7871704101562, >>> 0.792 >>> ] >>> ], >>> [ >>> [ >>> 0, >>> 446.9007568359375, >>> 36.374977111816406, >>> 907.0919189453125, >>> 337.439208984375, >>> 0.333 >>> ], >>> [ >>> 1, >>> 454.3310241699219, >>> 336.08477783203125, >>> 921.26904296875, >>> 641.7871704101562, >>> 0.792 >>> ] >>> ] >>> ], >>> "masks": [ >>> [ >>> [ >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> ..., >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False] >>> ], >>> [ >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> ..., >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False] >>> ] >>> ], >>> [ >>> [ >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> ..., >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False] >>> ], >>> [ >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> ..., >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False], >>> [False, False, False, ..., False, False, False] >>> ] >>> ] >>> ] >>> } >>> modelc stjd|dd|td|t|tj}t|tj}td|t ||_ | dd|_ t||_tj|t|jd}|j|d|j|j|_td d |_tgd tj|_tgd tj|_d|_d S)z use `model` to create a video panoptic segmentation pipeline for prediction Args: model: model id on modelscope hub. F)r auto_collatezloading model from zloading config from max_video_framesi) map_location state_dictzload model done g33333^@gR]@gRY@g(\2M@g(\L@gL@N)super__init__loggerinfoospjoinr TORCH_MODEL_FILE CONFIGURATIONr from_filecfggetrrrtorchloaddeviceload_state_dicttoevalpad_size_divisornparrayfloat32meanstdto_rgb)selfrkwargs model_path config_path checkpoint __class__rp/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/cv/video_instance_segmentation_pipeline.pyr|s$     z*VideoInstanceSegmentationPipeline.__init__inputreturnc Cst|tstdt|g}g}g}g}t|}|tj|_|tj |_ d}| r| \}} |s8n||j kr>n}t| d} t| |j|j|j} tj| |jdd} ddtjgdtjdtjgd tjdd d dd d } | j| d <| j| d<| j| d<|| d<|dkrttt| gdg|jg}| gg}|t| gd|| |d7}| s/t|gg}|gg}|||t||j|d}|S)z^ Read video and process into 'imgs', 'img_metas', 'ref_img', 'ref_img_metas' z input should be a str, but got r)iih)pad_valFNr)dtyperT)r/r0r1)flipflip_direction img_norm_cfgvideo_id is_video_data ori_shape img_shape pad_shapeframe_id)rrH) video_nameimgs img_metasref_img ref_img_metas) isinstancestr TypeErrortypecv2 VideoCapturer$ CAP_PROP_FPSfpsCAP_PROP_FRAME_COUNT frame_countisOpenedreadrmmcvimresize imnormalizer/r0r1impad_to_multipler+r,r-r.shaper% from_numpy transposer)r'append)r2r:rJrKref_imgsrMcap frame_idxretframe resize_frame norm_frame pad_frame ref_img_metaresultrrr9 preprocesss|          ,z,VideoInstanceSegmentationPipeline.preprocessc Csg}g}tB|d}|d}|d}|d}|j||||d}tt|dD]} ||d| d||d| dq*Wdn1sMwY||d } | S) a7 Segmentation Instance (bounding boxes or masks) in the video passed as inputs. Args: input (`Video`): The pipeline handles two types of images: - A string containing an HTTP(S) link pointing to a video - A string containing a local path to a video The pipeline accepts a single video as input. Return: A dictionary of result. If the input is a video, a dictionary is returned. The dictionary contain the following keys: - **boxes** (`List[float]) -- The bounding boxes [index, x1, y1, x2, y2, score] of instance in each frame. - **masks** (`List[List[bool]]`, optional) -- The instance mask [[False,...,False],...,[False,...,False]] rJrKrLrM)rLrMrrHN)boxesmasks)r%no_gradrrangelenra) r2r: bbox_results mask_resultsrJrKrLrM segm_resultsiioutputrrr9forwards& z)VideoInstanceSegmentationPipeline.forwardinputscCs|S)Nr)r2rxrrr9 postprocesssz-VideoInstanceSegmentationPipeline.postprocess) __name__ __module__ __qualname____doc__rOrrrrrlrwry __classcell__rrr7r9rs ^"J*.r)#osos.pathpathrtypingrrrRrZnumpyr,r%rmodelscope.metainfor;modelscope.models.cv.video_instance_segmentation.video_knetrmodelscope.outputsrmodelscope.pipelines.baserr modelscope.pipelines.builderr modelscope.utils.configr modelscope.utils.constantr r modelscope.utils.loggerrrregister_modulevideo_instance_segmentationrrrrr9s,