o
    ॵiL                     @   sr  d dl Z d dlZd dlZd dlmZmZ d dlZd dlZd dlZ	d dl
Zd dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z# d dl$m%Z% d d	l&m'Z' d d
l(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6 ej7dkrej8j9Ze:  e6 Z;e,j<e1j=ej=dG dd de*Z>dS )    N)AnyDict)loadmatsavemat)	Pipelines)LargeBaseLmkInfer)align_for_lm	align_img	draw_lineenlarged_bboximage_warp_grid1	load_lm3dmesh_to_stringread_objresize_on_long_sidespread_flow	write_obj)Model)
OutputKeys)pipeline)InputPipeline)	PIPELINES)	LoadImage)	ModelFileTasks)create_devicedevice_placement)
get_loggerz2.0)module_namec                       s   e Zd Zdedef fddZdedeeef fddZd!ddZ	dd Z
d"ddZdd Zdd Zdd Zd#ddZdd Zdeeef deeef fddZdeeef fdd Z  ZS )$FaceReconstructionPipelinemodeldevicec              
      s  t  j||d |}tj|d}tj|tj}d|v r!d| _n|| _| j | _tj|d}t	
|| jdk| _td| jd| _d}tjtjtj||d	d
}| j| | j  t| j}| j| | j| | j  | j  | jjdd tjtjdd}	tj|	st|	 ttj|dd|	 ttj|dd|	 ttj|dd|	 tjtj j!dd| _"t#j$dd}
d|
j%_&d|
j%_'t#( }t#j)||
d| _*| j*+ Z |+ E t#j,-tj|dd(}t#. }|/|0  | j*j1+  t#j2|dd | j*3t#4  W d   n	1 s$w   Y  W d   n	1 s4w   Y  W d   n	1 sDw   Y  d| _5t6|| _7t8d9|| _:t;| j<}|| _dS )a_  The inference pipeline for face reconstruction task.

        Args:
            model (`str` or `Model` or module instance): A model instance or a model local dir
                or a model id in the model hub.
            device ('str'): device str, should be either cpu, cuda, gpu, gpu:X or cuda:X.

        Example:
            >>> from modelscope.pipelines import pipeline
            >>> test_image = 'data/test/images/face_reconstruction.jpg'
            >>> pipeline_faceRecon = pipeline('face-reconstruction',
                model='damo/cv_resnet50_face-reconstruction')
            >>> result = pipeline_faceRecon(test_image)
            >>> mesh = result[OutputKeys.OUTPUT]['mesh']
            >>> texture_map = result[OutputKeys.OUTPUT_IMG]
            >>> mesh['texture_map'] = texture_map
            >>> write_obj('hrn_mesh_mid.obj', mesh)
        )r!   r"   assetsgpucudazlarge_base_net.pth   )max_sizer"   z,retinaface_resnet50_2020-07-20_old_torch.pthcpu)map_location	image_res~z.cache/torch/hub/checkpointsface_alignmentzs3fd-619a316812.pthz3DFAN4-4a694010b9.zipzdepth-6c4283c0e0.zipF)
flip_inputT)allow_soft_placementg?)graphconfigzsegment_face.pbrb )nameNi   z {}/assets/BBRegressorParam_r.mat)=super__init__ospathjoinr   TORCH_MODEL_FILEdevice_name_lowerr   model_preloadlarge_base_lmks_modelr   detectortorchloaddirnameload_state_dictevalr"   r!   
set_devicesetupparallelize
set_render
expanduserexistsmakedirsshutilcopyr-   FaceAlignmentLandmarksTypeTHREE_Dlm_sesstfConfigProtogpu_optionsper_process_gpu_memory_fractionallow_growthGraphSession	face_sess
as_defaultgfile	FastGFileGraphDefParseFromStringreadr0   import_graph_defrunglobal_variables_initializertex_sizer   lm3d_stdr   formatalign_paramsr   device_name)selfr!   r"   
model_root
bfm_foldercheckpoint_pathlmks_cpkt_pathdetector_ckpt_name
state_dictsave_ckpt_dirr1   g1f	graph_def	__class__ h/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/cv/face_reconstruction_pipeline.pyr6   ,   s   









z#FaceReconstructionPipeline.__init__inputreturnc                 C   s<   t |}t|jdkrt|tj}|t}d|i}|S )N   img)	r   convert_to_ndarraylenshapecv2cvtColorCOLOR_GRAY2BGRastypefloat)rh   rw   rz   resultru   ru   rv   
preprocess   s   

z%FaceReconstructionPipeline.preprocessT   c                 C   sV  t j|dd d df }|j\}}|d |d d df  |d d df< t|||\}	}
}}	t||||d| d d\}	}}}	| jj| jjddt	
|
id	}|rtjt	
|
d
 tjddddd}
tjt	
|d
 tjddddd}tjt	
|d
 tjdd d d d d d f }t|d}t|d}|
||||fS )N.   g     Y@   )target_sizerescale_factorzoutput_alpha:0zinput_image:0)	feed_dict     o@dtypery   r   )PILImage	fromarraysizer	   rY   ra   r0   get_tensor_by_namenparrayr@   tensorfloat32permute	unsqueeze)rh   rz   lmrd   	to_tensorr+   imWH_im_lrlm_lrim_hdlm_hdmask_lrru   ru   rv   	read_data   sL   
$
z$FaceReconstructionPipeline.read_datac                 C   s   t t|tjS N)r@   r   r   r   r   r   )rh   labelru   ru   rv   parse_label   s   z&FaceReconstructionPipeline.parse_labelNc                 C   s   t ||| j\}}}|dkrd S t|g dtj}|dd d d d d d df }||d }|d d d df | }|d d df |d  |d d df< |d d df |d  |d d df< |S )Nr   )r   r   r      r   ry   r   )r   rf   r   reshaper   r   get_landmarks_from_image)rh   rz   rQ   five_points	input_imgscalebboxlandmarkru   ru   rv   prepare_data   s   
$$z'FaceReconstructionPipeline.prepare_datac                 C   sz   | dddd   d d }|tj}| j|dd}tj	t
|d tjd dddd}|| jj}|S )	Nr   ry   r   r   r   gQ?)degreer   )r   detachr(   numpyr   r   uint8fat_facer@   r   r   r   r   tor!   r"   )rh   input_img_tensorr   input_img_for_textureinput_img_for_texture_tensorru   ru   rv   get_img_for_texture   s&   z.FaceReconstructionPipeline.get_img_for_texturec           !   
   C   s  d}d}g }t |t j}| j|}g }|D ]$}|d dkr! n||d d |d d |d d |d d	 d
 q|D ]}	|	d }
|	d }|	d }|	d }||
 d }|| d }||
 d }|| d }t||| }||d  }
||d  }|
}|}|
| }|| }|j\}}}td|
 }td| }td|
}
td|}td|| }td|| }t||}t||}|t	|t	|t	|
t	|f }|dks|dks|dks|dkrt j
|t	|t	|t	|t	|t jdd}t |||f}t|| j| jdk}|| }td}tdD ]'} |d | d d  | | ||  d< |d | d d  | | ||  d< qt|d d df }
t|d d df }t|d d df }t|d d df }||
 d }|| d }||
 d }|| d }t||| }||d  }
||d  }|
}|}|
| }|| }|j\}}}td|
 }td| }td|
}
td|}td|| }td|| }t||}t||}|t	|t	|t	|
t	|f }|dks|dks|dks|dkrt j
|t	|t	|t	|t	|t jdd}t |||f}t|| j| j dk}|| }td}tdD ]'} |d | d d  | | ||  d< |d | d d  | | ||  d< q/|| q>||fS )Nr   g?scorer   r   r   r   ry   r   )x1y1x2y2r   r   r   r   )g\(Y@gR1]@gQ^@)valuer%   )j   ry   r   )r~   r   COLOR_BGR2RGBr?   predict_jsonsappendmaxr}   minintcopyMakeBorderBORDER_CONSTANTresizer   	infer_imgr>   r;   r   zerosranger<   )!rh   img_bgr
INPUT_SIZEENLARGE_RATIO	landmarks	rgb_imageresultsboxesannodetect_resultr   r   r   r   whcxcysztrans_x1trans_y1heightwidthr   dxdyedxedycrop_img	base_lmks	inv_scaleaffine_base_lmksidxru   ru   rv   
infer_lmks   s   









$ 




$(
z%FaceReconstructionPipeline.infer_lmksc              	   C   sL  |  |\}}t|}g dg dg dg dg dg dg dg dg}g }tt|D ]/}t|| d	 || d
 || d || d g|jd |jd d}dd |D }|| q.g }t|jd D ]}||d d d d f }	g }
t|jd |jd ftj	}|| }|d |d  }|d |d  }||k r|n|}|d }|dkrd}|d }|d dkr|n|d }t
|D ]L\}}t|jd |jd ftj	}t||	|d |d  d||d  t|||fd}|dkrt||	|d |d  d|d |d  |
| qt|||fd}|
| ||
 qg|d |fS )N)r   !   F)r   &   F)*   /   F)3   7   F)9   @   F)B   J   T)K   S   T)T   `   Tr   r   r   r   r   r   g      ?c                 S   s   g | ]}t |qS ru   )r   ).0xru   ru   rv   
<listcomp>h  s    z@FaceReconstructionPipeline.find_face_contour.<locals>.<listcomp>ry   r   
   )   r   r   )r   r   r   r   r|   r   r}   r   r   r   	enumerater
   r~   GaussianBlur)rh   imager   r   args	roi_bboxsiroi_bboxpeople_mapsr   maps
whole_maskroi_boxroi_box_widthroi_box_heightshort_side_length
line_widthkernel_sizegaussian_kerneltargmaskru   ru   rv   find_face_contourY  sb   
(

z,FaceReconstructionPipeline.find_face_contour皙?c                 C   s  t |d\}}| |\}}|d }|d }tj|jd |jd dftjd}|d |d  d |d |d	  d g}	tt|d |d	  t|d |d  }
d|jd |	d  d  }d|jd |	d  d  }|
d d|	d d  d|	d d  ||g}t|}t	|}t
d
|| }t|||f}||t	|	d |d  t	|	d |d  t	|	d |d  t	|	d |d  f< |t||f d }|d }|dt||fd   }t||jd |jd f}|| }t|d |d |dg d\}}}}}|S )Ni   r   r   ry   )r}   r   r   r   r   r   d   r   r   g      ?).r   ).r   )r   r   r   r   )r   r  r   r   r}   r   r   absr   r   r   r~   r   dstackr   )rh   rz   r   _imgr   contour_mapsr   contour_mapFlow
box_center
box_lengthvalue_1value_2
value_listflow_box_lengthsfinter_face_mapspred	top_boundbottom_bound
left_boundright_boundru   ru   rv   r     sh   $"
z#FaceReconstructionPipeline.fat_facec                 C   s   |j d dks|j d dkrt|d\}}| |\}}|d u s*t|j d dkr,i S g }|d }dD ]}||| d || d g q4t|}| j|| j|d}| j||| j	dd\}}}	}
}||	||
|d	}| j
| | j
 }|S )
Nr   i  r   i  )r   r   6   r   Z   )r   r&   r*   )imgsimgs_hdlmslms_hd	face_mask)r}   r   r   r   r   r   r   rQ   r   rd   r!   set_input_basepredict_results_base)rh   rz   r   boxr   r   r   	im_tensor	lm_tensorim_hd_tensorlm_hd_tensorr  dataoutputru   ru   rv   predict_base  s.    


z'FaceReconstructionPipeline.predict_basec           
      C   s   |d    tj}t|tj}|}| |}| 	|d }|d ||d |d |d |d |d |d |d	 |d
 d
}| j
| | j
  | j
dd | j
 }|d }	|d |d |d d}tjd tj|	tj|iS )Nrz   r   input_img_hdr.  gt_lmcoeffsposition_maptexture_maptex_valid_maskde_retouched_albedo_map)
r   input_img_for_texr9  r.  r:  r;  r<  r=  r>  r?  T)	visualize	face_mesh	vis_image
frame_list)meshrC  rD  )r(   r   r   r   r   r~   r   COLOR_RGB2BGRr8  r   r!   set_input_hrnget_edge_points_horizontalsave_results_hrnr   
OUTPUT_OBJ
OUTPUT_IMGOUTPUT)
rh   rw   r   	bgr_imagerz   base_model_outputr@  	hrn_inputr   r=  ru   ru   rv   forward  s>   


z"FaceReconstructionPipeline.forwardc           
      K   s   | dd}|tj }|tj }|tj }|r,t }t|d }|jdd}|	| tj|tj|tj|r;d i}	|	S |i}	|	S )NrenderFrE  zutf-8)encoding)
getr   rJ  rK  rL  ioBytesIOr   encodewrite)
rh   inputskwargsrQ  
output_objr=  r   mesh_str
mesh_bytesr   ru   ru   rv   postprocess  s"   




z&FaceReconstructionPipeline.postprocess)Tr   r   )r  )__name__
__module____qualname__strr6   r   r   r   r   r   r   r   r   r   r  r   r8  rP  r]  __classcell__ru   ru   rs   rv   r    (   s    Y

 

87"!+r    )?rT  r7   rL   typingr   r   r~   r-   r   r   	PIL.Imager   
tensorflowrR   r@   scipy.ior   r   modelscope.metainfor   Rmodelscope.models.cv.face_reconstruction.models.facelandmark.large_base_lmks_inferr   .modelscope.models.cv.face_reconstruction.utilsr   r	   r
   r   r   r   r   r   r   r   r   >modelscope.models.cv.skin_retouching.retinaface.predict_singler   modelscope.outputsr   modelscope.pipelinesr   modelscope.pipelines.baser   r   modelscope.pipelines.builderr   modelscope.preprocessorsr   modelscope.utils.constantr   r   modelscope.utils.devicer   r   modelscope.utils.loggerr   __version__compatv1disable_eager_executionloggerregister_moduleface_reconstructionr    ru   ru   ru   rv   <module>   s>   4
