o
    ߥis#                     @   s   d Z ddlmZ ddlZddlmZ ddl	Z
ddlmZmZ ddlmZmZ ddlmZ g dZG dd	 d	Zd
d Zdd Z				d#ddZ				d#ddZd$ddZd%ddZd&ddZd'ddZd d! Zed"krp	 dS dS )(zv
CVPR 2020 submission, Paper ID 6791
Source code for 'Learning to Cartoonize Using White-Box Cartoon Representations'
    N)Paralleldelayed)colorsegmentation   )disc_sn)gjtY@g`"1]@gQ^@c                   @   sP   e Zd ZdddZdddZdd Zd	d
 Zdd Zdd Zdd Z	dd Z
dS )Vgg19Nc                 C   s"   t j|ddd | _td d S )Nlatin1T)encodingallow_picklezFinished loading vgg19.npy)nploaditem	data_dictprint)selfvgg19_npy_path r   U/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/cv/cartoon/loss.py__init__   s   zVgg19.__init__Fc                 C   s  |d d }t jdd|d\}}}t jd|td  |td  |td  gd}| |d| _t j| j| _| | jd	| _	t j| j	| _
| | j
d
| _| | jd| _t j| j| _| | jd| _t j| j| _| | jd| _| | jd| _t j| j| _| | jd| _t j| j| _| | jd| _t j| j| _| | jd| _t j| j| _| | jd| _| | jd| _t j| j| _| | jd| _t j| j| _| | jd| _t j| j| _ | | j d| _!t j| j!| _"| | j"d| _#| j!S )Nr   g     _@   )axisnum_or_size_splitsvaluer      )r   valuesconv1_1conv1_2pool1conv2_1conv2_2pool2conv3_1conv3_2conv3_3conv3_4pool3conv4_1conv4_2conv4_3conv4_4pool4)$tfsplitconcatVGG_MEAN
conv_layerr   nnrelurelu1_1r   relu1_2max_poolr   r   relu2_1r    relu2_2r!   r"   relu3_1r#   relu3_2r$   relu3_3r%   relu3_4r&   r'   relu4_1r(   relu4_2r)   relu4_3r*   relu4_4r+   )r   rgb
include_fc
rgb_scaledbluegreenredbgrr   r   r   build_conv4_4   sL   zVgg19.build_conv4_4c                 C   s   t jj|g dg dd|dS )N)r   r   r   r   SAME)ksizestridespaddingname)r,   r1   r5   )r   bottomrL   r   r   r   r5   J   s   zVgg19.max_poolc                 C   sl   t |' | |}t jj||g ddd}| |}t j||}|W  d    S 1 s/w   Y  d S )Nr   r   r   r   rH   )rK   )r,   variable_scopeget_conv_filterr1   conv2dget_biasbias_add)r   rM   rL   filtconvconv_biasesbiasr   r   r   r0   R   s   

$zVgg19.conv_layerc           
      C   s   t |< |  }d}|dd  D ]}||9 }qt |d|g}| |}| |}t jt 	|||}	|	W  d    S 1 sDw   Y  d S )Nr   )
r,   rO   	get_shapeas_listreshapeget_fc_weightrR   r1   rS   matmul)
r   rM   rL   shapedimdxweightsbiasesfcr   r   r   fc_layerZ   s   


$zVgg19.fc_layerc                 C      t j| j| d ddS )Nr   filterrL   r,   constantr   r   rL   r   r   r   rP   g      zVgg19.get_conv_filterc                 C   rf   )Nr   rc   rh   ri   rk   r   r   r   rR   j   rl   zVgg19.get_biasc                 C   rf   )Nr   rb   rh   ri   rk   r   r   r   r\   m   rl   zVgg19.get_fc_weightN)F)__name__
__module____qualname__r   rG   r5   r0   re   rP   rR   r\   r   r   r   r   r      s    

.r   c                 C   s   t t| d}||}||}||}|  dd  \}}	}
tj||}t	|||	 |
  }tj||}t	|||	 |
  }|| }|S )Nz	vgg19.npyr   )
r   ospjoinrG   rY   rZ   r,   lossesabsolute_differencereduce_mean)	model_dirinput_phototransfer_resinput_superpixel	vgg_model	vgg_photo
vgg_outputvgg_superpixelhwc	abs_photo
photo_lossabs_superpixelsuperpixel_losslossr   r   r   content_lossq   s   


r   c                 C   sv   t ||ddd}t | | ddd}t|| \}}tt||dddd\}}tt||dddd\}}	|	| }
|| }|
|fS )	N   g?)repsr   T	disc_gray)scalepatchrL   	disc_blur)guided_filtercolor_shift
lsgan_lossr   )input_cartoonoutput_cartoon	blur_fakeblur_cartoon	gray_fakegray_cartoond_loss_grayg_loss_grayd_loss_blurg_loss_blur
sty_g_loss
sty_d_lossr   r   r   
style_loss   s,   

r       Fdiscriminatorc                 C   sz   | |||||dd}| |||||dd}t j|}t j|}t t | }	t t |t d|   }
|
|	fS )NFrL   r   reuseTg      ?)r,   r1   sigmoidru   log)r   realfaker   channelr   rL   
real_logit
fake_logitr   r   r   r   r   gan_loss   s    r   c                 C   sf   | |||||dd}| |||||dd}t |d d }	dt |d d t |d   }
|
|	fS )NFr   Tr   r   g      ?)r,   ru   )r   r   r   r   r   r   rL   r   r   g_lossd_lossr   r   r   r      s   r   c              	   C   s   |    dd \}}t| d d |d d d d d f | d d d || d d d d f  d }t| d d d d |d d d f | d d d d d || d d f  d }|| d| |  }|S )Nr   r   r   )rY   rZ   r,   ru   )imagek_sizer~   r   tv_htv_wtv_lossr   r   r   total_variation_loss   s   FFr   {Gz?c                 C   s   t | }tt jd|d |d df| jd|}t| || }t||| }t| | || ||  }t| |  || ||  }	||	|  }
||
|  }t|
|| }t||| }||  | }|S )Nr   r   )dtype)r,   r^   tf_box_filteronesr   )ra   yr   r   x_shapeNmean_xmean_ycov_xyvar_xAbmean_Amean_boutputr   r   r   r      s   
 r   uniformc                 C   s  t j| ddd\}}}t j|ddd\}}}|dkr9t jjdgddd}	tjjdgddd}
tjjdgd	dd}n"|d
kr[t jjdgddd}	t jjdgddd}
t jjdgddd}|	| |
|  ||  |	|
 |  }|	| |
|  ||  |	|
 |  }||fS )Nr   )r   r   normalr   gv/?g?)r^   meanstddevgbX9?gA`"?r   gy&1?g1Zd?)r^   minvalmaxvalg^I+?gCl?gFx?gK7?)r,   r-   randomr   r   r   )image1image2modeb1g1r1b2g2r2b_weightg_weightr_weightoutput1output2r   r   r   r      s"   

r      c                    s@   fdd t | d }t|d fdd| D }t |S )Nc                    s,   t j|  ddddd}tj|| ddd} | S )Nr   
   T)
n_segmentssigmacompactnessconvert2labstart_labelavgr   )kindbg_label)r   slicr   	label2rgb)r   	seg_label)seg_numr   r   process_slic   s   z'simple_superpixel.<locals>.process_slicr   )n_jobsc                 3   s    | ]	}t  |V  qd S rm   )r   ).0r   )r   r   r   	<genexpr>  s    
z$simple_superpixel.<locals>.<genexpr>)r   r^   r   array)batch_imager   num_job	batch_outr   )r   r   r   simple_superpixel   s   
r   c                 C   sv   |    d }dd| d d  }|td| d d| d |df }t|tj}tj	| |g dd}|S )NrX   r   r   rN   rH   )
rY   rZ   r   r   r   astypefloat32r,   r1   depthwise_conv2d)ra   r   chweight
box_kernelr   r   r   r   r     s   &r   __main__)r   r   Fr   )r   )r   )r   )r   )__doc__os.pathpathrq   numpyr   scipy.statsstatsst
tensorflowr,   joblibr   r   skimager   r   networkr   r/   r   r   r   r   r   r   r   r   r   r   rn   r   r   r   r   <module>   s<    ^






	