o ॵi7@s`ddlZddlmZddlZddlmZmZmZm Z ddl Z ddl Z ddl m ZddlmmZddlZddlZddl mZddlmZddlmZddlmZddlmZddlm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-e .de-Z/dddZ0dddZ1dddZ2dddZ3e)j4e+j5ej5dGddde&Z6dS)N)AnyDictListUnion) animation)writers)MultipleLocator) Pipelines)KeypointsTypes) OutputKeys)pipeline)InputModelPipelineTensor) PIPELINES)Tasks) get_loggerAggc Csfddgddgddgddgd d gd d gd dgddgddgd dgddgdd gdd gdd gd dgg}|j\}}t||f}|D] }||d||d<q=|dkrtddgddgddgddgd d gd d gd d gddgddgd dgddgddgdd gddgd dgg}td}||dddf||dddf<|d|dd|d<|d|dd|d <|}|S)Nr    r)r$r?)shapenpzerosarray) joints joints_nbr lst_mappingsnbrdim h36m_jointsmappinglst_mappings_17h36m_joints_17r4f/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/cv/body_3d_keypoints_pipeline.pyconvert_2_h36m!s,*$  "  r6.@c Cs&|dur|S|d|d\}}|dks|dkr|St|}t|}|dks*|dkr,|S|| } || } t|D]V} t| t|| d|| d} t| t|| d|| d} d| || d| || d|| d<d| || d| || d|| d<q:|S)Nrrg?)lenranger(expabs)cur_ptspre_ptsbboxsmooth_xsmooth_ywhsize_presize_curfactor_xfactor_yiw_xw_yr4r4r5 smooth_pts9s"   ((,.rJcCst|jd|jdks Jg}d}t|jdD]}t|||||dddd||}|||}qt|S)Nrr)r'r9rJreshapeappendr(r*)lst_kps lst_bboxesr?r@lst_smoothed_kpsprev_ptsrGsmoothed_cur_kpsr4r4r5 smoothingQs   rScCsh|}|}|jd|jdksJt||}g}t|jdD]}t|||d}||q"|S)Nrr,)squeezer'rSr9r6rM)rNrOr, keypointsrGh36m_joints_2dr4r4r5convert_2_h36m_data`s  rX) module_namecseZdZdeffdd ZdedeeeffddZdeeefdeeeffdd Z deeefdeeeffd d Z d e ee j ffd dZddZZS)Body3DKeypointsPipelinemodelc stjdd|i||j|_|jd|_ttj|jt j r#dndd|_ t |jjjjdr;|jjjjj|_dS|jjjjj|_dS) ajHuman body 3D pose estimation. Args: model (str): model id on modelscope hub. kwargs (dict, `optional`): Extra kwargs passed into the preprocessor's constructor. Example: >>> from modelscope.pipelines import pipeline >>> body_3d_keypoints = pipeline(Tasks.body_3d_keypoints, model='damo/cv_hdformer_body-3d-keypoints_video') >>> test_video_url = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/videos/Walking.54138969.mp4' >>> output = body_3d_keypoints(test_video_url) >>> print(output) r[z*damo/cv_hrnetv2w32_body-2d-keypoints_imagegpucpu)r[device MAX_FRAMENr4)super__init__r[keypoint_model_3devalhuman_body_2d_kps_det_pipeliner rbody_2d_keypointstorchcuda is_availablehuman_body_2d_kps_detectorhasattrcfgINPUTr_ max_frame)selfr[kwargs __class__r4r5rass  z Body3DKeypointsPipeline.__init__inputreturncCsf||_||j}dt|krddd}|Sg}g}t|D]Q\}}||}g|dkr:dd|d}|S|dd} |dd} t|dd} | | | t t|  d | g|d |j kronqt| t|d d f} t| t|d f} t |jjjjdr|jjjjjn|jjjjj}t| | |d}t|}d|d}|S)NrFzget video frame failed.successmsgboxesz%fail to detect person at image frame rVscoresrKrrrr IN_NUM_JOINTSrTT)ru input_2d_pts) video_urlread_video_framesr8 enumeraterigetr(r*maxrMlistrLrmrjrbrkr[MODELryn_jointsrX)rnrr video_framesres all_2d_posesall_boxes_with_socrerGframekps_2dboxposescoreall_2d_poses_np all_boxes_np joint_numkps_2d_h36m_17r4r4r5 preprocesss`             z"Body3DKeypointsPipeline.preprocesscCsN|ds ddd}|S|d}|j|}|j|}tddifi|}|S)NruFzpreprocess failed.rtrzT)rbrforwarddict)rnrrrrzoutputsr4r4r5rs   zBody3DKeypointsPipeline.forwardcKs|dd}|durtjddj}tjgtjgtj|i}|ds(|j|tj<|S|t j }|j d}d|jjvrI|||||tj<||tj<|j|tj<|S)N output_videoz.mp4)suffixrurrender)r~tempfileNamedTemporaryFilenamer KEYPOINTS TIMESTAMPS OUTPUT_VIDEOr{r POSES_CAMERAdatar]numpyrbrkkeysrender_prediction timestamps)rnrrrooutput_video_pathrposes pred_3d_poser4r4r5 postprocesss(       z#Body3DKeypointsPipeline.postprocessr{cCsdd}g}g|_t|trt|}|std|n|}|tj|_ |j dus1|j dkr7td|d} | \}}|sCn|j |||j d|d 7}| |||j kr^nq:| |S) aRead video from local video file or from a video stream URL. Args: video_url (str or cv2.VideoCapture): Video path or video stream. Raises: Exception: Open video fail. Returns: [nd.array]: List of video frames. cSs.t|d\}}t|d\}}d|||f}|S)N<z%02d:%02d:%06.3f)divmod)secondsmsrBtimer4r4r5timestamp_formatszCBody3DKeypointsPipeline.read_video_frames..timestamp_formatz1modelscope error: %s cannot be decoded by OpenCV.Nrz/modelscope error: %s cannot get video fps info.T)rr)r isinstancestrcv2 VideoCaptureisOpened Exceptionr~ CAP_PROP_FPSfpsreadrMrmrelease)rnr{rframescap frame_idxretrr4r4r5r|sB       z)Body3DKeypointsPipeline.read_video_framescsjdgdddgddgddgddgddgddgdd gdd gd d gd d gd d gd dgddgd dgddgd dggt}tj|dd|td}j|j |j | d d ddddddd|jjjj}|jjjj}||ddddf}ddddf}ddddf} ||| d\} fdd} | ||| } fdd} tj|| |j| | fd }td!}||jid"d#}|j||d$dS)%zrender predict result 3d poses. Args: pose3d_cam_rr (nd.array): [frame_num, joint_num, joint_dim], 3d pose joints output_video_path (str): output path for video Returns: r)r#rrrrr!rrrrrr!rrr r#rrr%rrr"F)auto_add_to_figurer&XYZrKNzr.c si}tD]8\}}|d|d}}|vrd}nd}j||||g||||g||||gd|d} | d||<q|S)zrender bones in skeleton Args: xs (nd.array): [joint_num, joint_channel] ys (nd.array): [joint_num, joint_channel] zs (nd.array): [joint_num, joint_channel] rrredbluer) linewidthcolor)r}plot) xsyszsbonesidxedgeindex1index2 edge_colorconnect)axedges left_pointsr4r5 renderBones<sz>Body3DKeypointsPipeline.render_prediction..renderBonesc s|dddf}|dddf}|dddf}tD];\}}|d|d}} |||| f} |||| f} |||| f} ||| ||| ||| dq|||||dd|dkrxtd|d||fS) aMupdate animation Args: frame_idx (int): frame index points (mpl_toolkits.mplot3d.art3d.Line3D): skeleton points ploter bones (dict[int, mpl_toolkits.mplot3d.art3d.Line3D]): connection ploter Returns: tuple: points and bones ploter Nrrrzdz rendering /)r} set_xdata set_ydataset_3d_propertiesset_dataloggerinfo) rpointsrrrrrrrrx1x2y1y2z1z2)r frame_num pose3d_cam_rrr4r5updateUs     z9Body3DKeypointsPipeline.render_prediction..update)figfuncrintervalfargsffmpegi)rmetadatabitrate)writer)r'pltfigurep3Axes3Dadd_axesrxaxisset_major_locatoryaxiszaxis set_xlabel set_ylabel set_zlabelset_xlimset_ylimset_zlimrbrkrazimelev view_initrr FuncAnimationrrsave)rnrrrx_major_locatorrrxyrrrrraniWriterrr4)rrrrrr5rsT &$                z)Body3DKeypointsPipeline.render_prediction)__name__ __module__ __qualname__rrar rrrrrrrrr|r __classcell__r4r4rpr5rZos"," 1rZ)r)r7r7)7datetimeos.pathpathosprtypingrrrrr matplotlibmatplotlib.pyplotpyplotrmpl_toolkits.mplot3d.axes3dmplot3daxes3drrr(rfrmatplotlib.animationrmatplotlib.tickerrmodelscope.metainfor Bmodelscope.models.cv.body_3d_keypoints.canonical_pose.body_3d_poser modelscope.outputsr modelscope.pipelinesr modelscope.pipelines.baser rrrmodelscope.pipelines.builderrmodelscope.utils.constantrmodelscope.utils.loggerruserr6rJrSrXregister_modulebody_3d_keypointsrZr4r4r4r5s>