o
    ॵiT                     @   s  d dl Z d dlZd dlZd dlmZmZ d dlZd dlZd dlZ	d dl
Zd dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d d	l(m)Z) d d
l*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z;m<Z< d dl=m>Z> zd dl?m@Z@ W n eAy   d dl?mBZ@ Y nw ejCdkrejDjEZeF  e> ZGe2jHe9jIejIdG dd de0ZJdS )    N)AnyDict)loadmatsavemat)	Pipelines)LargeBaseLmkInfer)POSalign_for_lm	draw_lineenlarged_bbox
extract_5pimage_warp_grid1	load_lm3dmesh_to_stringread_objresize_n_crop_imgresize_on_long_sidespread_flow	write_obj)HeadSegmentor)TexProcesser)Model)
OutputKeys)pipeline)InputPipeline)	PIPELINES)	LoadImage)Config)	ModelFileTasks)create_devicedevice_placement)
get_logger)get_dir)_get_torch_homez2.0)module_namec                       s   e Zd Zd!dedef fddZdedeeef fdd	Z	
		d"ddZ				
	
	d#ddZ
d$ddZdd Zdd Zd%ddZdeeef deeef fddZdeeef fdd Z  ZS )&HeadReconstructionPipelineFmodeldevicec              
      s  t  j||d |}tj|d}tj|tj}tj|tj}t	d|  t
|| _|| _d|v r:d| _n|| _| j | _tj|d}t|| jdk| _td| jd| _d	}	tjtjtj||	d
d}
| j|
 | j  t| j}| j| | j| | j  | j  | j  t  }tj|d}tj!|st"| t#$tj|dd| t#$tj|dd| t#$tj|dd| t%j&t%j'j(dd| _)t*j+dd}d|j,_-d|j,_.t*/ }t*j0||d| _1| j12 Z |2 E t*j34tj|dd(}t*5 }|6|7  | j1j82  t*j9|dd | j1:t*;  W d   n	1 s:w   Y  W d   n	1 sJw   Y  W d   n	1 sZw   Y  t<|d| _=t>|d| _?t@|| _AtBdC|| _DtE| jF}|| _dS )aa  The inference pipeline for head reconstruction task.

        Args:
            model (`str` or `Model` or module instance): A model instance or a model local dir
                or a model id in the model hub.
            device ('str'): device str, should be either cpu, cuda, gpu, gpu:X or cuda:X.

        Example:
            >>> from modelscope.pipelines import pipeline
            >>> test_image = 'data/test/images/face_reconstruction.jpg'
            >>> pipeline_headRecon = pipeline('head-reconstruction',
                model='damo/cv_HRN_head-reconstruction')
            >>> result = pipeline_headRecon(test_image)
            >>> mesh = result[OutputKeys.OUTPUT]['mesh']
            >>> texture_map = result[OutputKeys.OUTPUT_IMG]
            >>> mesh['texture_map'] = texture_map
            >>> write_obj('head_reconstruction.obj', mesh)
        )r(   r)   assetszloading config from gpucudazlarge_base_net.pthi   )max_sizer)   z,retinaface_resnet50_2020-07-20_old_torch.pthcpu)map_locationcheckpointsface_alignmentzs3fd-619a316812.pthz3DFAN4-4a694010b9.zipzdepth-6c4283c0e0.zipF)
flip_inputT)allow_soft_placementg?)graphconfigzsegment_face.pbrb )nameN)
model_rootz {}/assets/BBRegressorParam_r.mat)Gsuper__init__ospathjoinr   TORCH_MODEL_FILECONFIGURATIONloggerinfor   	from_filecfghair_texdevice_name_lowerr   model_preloadlarge_base_lmks_modelr   detectortorchloaddirnameload_state_dictevalr)   r(   
set_devicesetupparallelize
set_renderr$   existsmakedirsshutilcopyr1   FaceAlignmentLandmarksTypeTHREE_Dlm_sesstfConfigProtogpu_optionsper_process_gpu_memory_fractionallow_growthGraphSession	face_sess
as_defaultgfile	FastGFileGraphDefParseFromStringreadr4   import_graph_defrunglobal_variables_initializerr   head_segmentorr   tex_processorr   lm3d_stdr   formatalign_paramsr!   device_name)selfr(   r)   rE   r9   
bfm_foldercheckpoint_pathconfig_pathlmks_cpkt_pathdetector_ckpt_name
state_dicthub_dirsave_ckpt_dirr5   g1f	graph_def	__class__ h/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/cv/head_reconstruction_pipeline.pyr;   6   s   










z#HeadReconstructionPipeline.__init__inputreturnc                 C   sR   t |trt|}t|jdkrt|tj}|	t
}n|	t
}d|i}|S )N   img)
isinstancestrr   convert_to_ndarraylenshapecv2cvtColorCOLOR_GRAY2BGRastypefloat)rs   r   r   resultr   r   r   
preprocess   s   


z%HeadReconstructionPipeline.preprocessN      l@     Y@c                 C   s   |j \}}|jd dkrt|}	n|}	t|	 | \}
}|| }t|||
|||d\}}}t||||
d d |
d d g}||||fS )ax  
        Return:
            transparams        --numpy.array  (raw_W, raw_H, scale, tx, ty)
            img_new            --PIL.Image  (target_size, target_size, 3)
            lm_new             --numpy.array  (68, 2), y direction is opposite to v direction
            mask_new           --PIL.Image  (target_size, target_size)

        Parameters:
            img                --PIL.Image  (raw_H, raw_W, 3)
            lm                 --numpy.array  (68, 2), y direction is opposite to v direction
            lm3D               --numpy.array  (5, 3)
            mask               --PIL.Image  (raw_H, raw_W, 3)
        r      )target_sizemask   )sizer   r   r   	transposer   nparray)rs   r   lmlm3Dr   r   rescale_factorw0h0lm5ptsimg_newlm_newmask_newtrans_paramsr   r   r   	align_img   s   

$z$HeadReconstructionPipeline.align_imgT        R@c	                 C   s  t j|dd d df }	|	j\}
}|d |d d df  |d d df< t j|}t j|dd d df }| |	||\}}}}| j|	||||d\}}}}| j|||||| d d\}}}}| jj| jjddt	
|id	}|rtjt	
|d
 tjddddd}tjt	
|d
 tjddddd}tjt	
|d
 tjdd d d d d d f }|d urtjt	
|d
 tjdd d d d d d f nd }t|d}t|d}tjt	
|d
 tjddddd}t|d}||||||||fS )N.r   )r   r      )r   r   zoutput_alpha:0zinput_image:0)	feed_dict     o@)dtyper   r   )PILImage	fromarrayr   r   rc   rk   r4   get_tensor_by_namer   r   rK   tensorfloat32permute	unsqueeze)rs   r   r   ro   	to_tensor	image_resimg_fat	head_maskr   imWHim_fat_im_lr_coefflm_lr_coeffim_lrlm_lrmask_lr_headim_hdlm_hdmask_lrr   r   r   	read_data   st   

$

z$HeadReconstructionPipeline.read_datac                 C   s   t ||| j\}}}|dkrd S t|g dtj}|dd d d d d d df }||d }|d d d df | }|d d df |d  |d d df< |d d df |d  |d d df< |S )Nr   )r   r   r      r   r   r   )r	   rq   r   reshaper   r   get_landmarks_from_image)rs   r   r[   five_points	input_imgscalebboxlandmarkr   r   r   prepare_data   s   
$$z'HeadReconstructionPipeline.prepare_datac           !   
   C   s  d}d}g }t |t j}| j|}g }|D ]$}|d dkr! n||d d |d d |d d |d d	 d
 q|D ]}	|	d }
|	d }|	d }|	d }||
 d }|| d }||
 d }|| d }t||| }||d  }
||d  }|
}|}|
| }|| }|j\}}}td|
 }td| }td|
}
td|}td|| }td|| }t||}t||}|t	|t	|t	|
t	|f }|dks|dks|dks|dkrt j
|t	|t	|t	|t	|t jdd}t |||f}t|| j| jdk}|| }td}tdD ]'} |d | d d  | | ||  d< |d | d d  | | ||  d< qt|d d df }
t|d d df }t|d d df }t|d d df }||
 d }|| d }||
 d }|| d }t||| }||d  }
||d  }|
}|}|
| }|| }|j\}}}td|
 }td| }td|
}
td|}td|| }td|| }t||}t||}|t	|t	|t	|
t	|f }|dks|dks|dks|dkrt j
|t	|t	|t	|t	|t jdd}t |||f}t|| j| j dk}|| }td}tdD ]'} |d | d d  | | ||  d< |d | d d  | | ||  d< q/|| q>||fS )Nr   g?scorer   r   r   r   r   r   )x1y1x2y2r   r   r   r   )g\(Y@gR1]@gQ^@)valuer,   )j   r   r   )r   r   COLOR_BGR2RGBrJ   predict_jsonsappendmaxr   minintcopyMakeBorderBORDER_CONSTANTresizer   	infer_imgrI   rF   r   zerosrangerG   )!rs   img_bgr
INPUT_SIZEENLARGE_RATIO	landmarks	rgb_imageresultsboxesannodetect_resultr   r   r   r   whcxcysztrans_x1trans_y1heightwidthr   dxdyedxedycrop_img	base_lmks	inv_scaleaffine_base_lmksidxr   r   r   
infer_lmks  s   









$ 




$(
z%HeadReconstructionPipeline.infer_lmksc              	   C   sL  |  |\}}t|}g dg dg dg dg dg dg dg dg}g }tt|D ]/}t|| d	 || d
 || d || d g|jd |jd d}dd |D }|| q.g }t|jd D ]}||d d d d f }	g }
t|jd |jd ftj	}|| }|d |d  }|d |d  }||k r|n|}|d }|dkrd}|d }|d dkr|n|d }t
|D ]L\}}t|jd |jd ftj	}t||	|d |d  d||d  t|||fd}|dkrt||	|d |d  d|d |d  |
| qt|||fd}|
| ||
 qg|d |fS )N)r   !   F)r  &   F)*   /   F)3   7   F)9   @   F)B   J   T)K   S   T)T   `   Tr   r   r   r   r   r   g      ?c                 S   s   g | ]}t |qS r   )r   ).0xr   r   r   
<listcomp>  s    z@HeadReconstructionPipeline.find_face_contour.<locals>.<listcomp>r   r   
   )   r  r  )r  r   r   r   r   r   r   r   r   uint8	enumerater
   r   GaussianBlur)rs   imager   r   args	roi_bboxsiroi_bboxpeople_mapsr   maps
whole_maskroi_boxroi_box_widthroi_box_heightshort_side_length
line_widthkernel_sizegaussian_kernelr   argr   r   r   r   find_face_contour  sb   
(

z,HeadReconstructionPipeline.find_face_contour{Gz?c                 C   s  t |d\}}| |\}}|d }|d }tj|jd |jd dftjd}|d |d  d |d |d	  d g}	tt|d |d	  t|d |d  }
d|jd |	d  d  }d|jd |	d  d  }|
d d|	d d  d|	d d  ||g}t|}t	|}t
d
|| }t|||f}||t	|	d |d  t	|	d |d  t	|	d |d  t	|	d |d  f< |t||f d }|d }|dt||fd   }t||jd |jd f}|| }t|d |d |dg d\}}}}}|S )Ni   r   r   r   )r   r   r   r   r   r   d   r   r   g      ?).r   ).r   )r   r   r   r   )r   r)  r   r   r   r   r   absr   r   r   r   r   dstackr   )rs   r   degree_imgr   contour_mapsr   contour_mapFlow
box_center
box_lengthvalue_1value_2
value_listflow_box_lengthsfinter_face_mapspred	top_boundbottom_bound
left_boundright_boundr   r   r   fat_face  sh   $"
z#HeadReconstructionPipeline.fat_facec              	   C   s  |d    tj}t|tj}|}|jd dks#|jd dkr*t	|d\}}| 
|\}}|d u s?t|jd dkrAi S | |}g }	|d }dD ]}
|	||
 d ||
 d g qNt|	}	| j|| j|	d}	| j|d }| j||	| j||d\}}}}}}}}||||||||d	}| j| |  }|d usJ |d
 tj}| jj|| jd}|d |d d |d |d |d |d}d|i}tjd tj|tj|iS )Nr   r   i  r   i  )r  r  6   r  Z   )r   )r   r   )imgsimgs_hdlmslms_hd	face_maskr   
imgs_coeff	lms_coefftex_map)rE   vertices	trianglesuvsfaces_uvnormals)rK  facesUVsrN  rO  texture_mapmesh)r.   numpyr   r   r  r   r   COLOR_RGB2BGRr   r   r  r   r@  r   r   r[   rm   processr   ro   r(   	set_inputr   rn   post_process_texturerE   r   
OUTPUT_OBJ
OUTPUT_IMGOUTPUT)rs   r   r   	bgr_imager   r   boxr   fatbgrr   r  r   	im_tensor	lm_tensorim_hd_tensorlm_hd_tensorr   im_colm_codataoutputrJ  	head_meshr   r   r   forward	  s`   
 



z"HeadReconstructionPipeline.forwardc           
      K   s   | dd}|tj }|tj }|tj }|r,t }t|d }|jdd}|	| tj|tj|tj|r;d i}	|	S |i}	|	S )NrenderFrS  zutf-8)encoding)
getr   rY  rZ  r[  ioBytesIOr   encodewrite)
rs   inputskwargsri  
output_objrR  r   mesh_str
mesh_bytesr   r   r   r   postprocessN  s"   




z&HeadReconstructionPipeline.postprocess)F)Nr   r   )Tr   NNr   )N)r*  )__name__
__module____qualname__r   r;   r   r   r   r   r   r   r   r  r)  r@  rh  ru  __classcell__r   r   r   r   r'   2   s(    b
*

4 

8"7Er'   )Krl  r<   rV   typingr   r   r   r1   rT  r   	PIL.Imager   
tensorflowr\   rK   scipy.ior   r   modelscope.metainfor   Rmodelscope.models.cv.face_reconstruction.models.facelandmark.large_base_lmks_inferr   .modelscope.models.cv.face_reconstruction.utilsr   r	   r
   r   r   r   r   r   r   r   r   r   r   Amodelscope.models.cv.head_reconstruction.models.head_segmentationr   =modelscope.models.cv.head_reconstruction.models.tex_processorr   >modelscope.models.cv.skin_retouching.retinaface.predict_singler   modelscope.outputsr   modelscope.pipelinesr   modelscope.pipelines.baser   r   modelscope.pipelines.builderr   modelscope.preprocessorsr   modelscope.utils.configr   modelscope.utils.constantr   r    modelscope.utils.devicer!   r"   modelscope.utils.loggerr#   	torch.hubr$   BaseExceptionr%   __version__compatv1disable_eager_executionrA   register_modulehead_reconstructionr'   r   r   r   r   <module>   sN   <
