o ߥi@s.ddlZddlZddlZddlmZddlZddlZddlmZddl Z ddlm Z ddl m Z ddl mZddlmZdejd<d d gZdhd d ZddZddZddZddZdiddZddZdiddZddZdjdd Zdjd!d"Zd#d$Zd%d&Zd'd(Z did)d*Z!did+d,Z"d-d.Z#d/d0Z$d1ej%d2d2fd3d4Z&dkd5d6Z'did7d8Z(d9d:Z)e*d1d1ge*d;d;gdd?Z+d@dAZ,dBdCZ-dDdEZ.dhdFdGZ/dhdHdIZ0dhdJdKZ1dhdLdMZ2dldQdRZ3dmdSdTZ4dmdUdVZ5dndYdZZ6dnd[d\Z7d]d^Z8d_d`Z9dodbdcZ:dpddd Z;dmdedfZrr"r'r@r))rrCrDrrr ycbcr2rgbOs       rHcCr0)zxbgr version of rgb2ycbcr only_y: only return Y channel Input: uint8, [0, 255] float, [0, 1] r )r3r2r1r4r8r7r5r9r<rArrr bgr2ycbcrfrFrIcCsz|dkr|dkrdd|D}dd|DS|dkr,|dkr,dd|D}dd|DS|d kr;|d kr;d d|DS|S) Nr graycSg|] }t|tjqSr)rrCOLOR_BGR2GRAY.0rrrr z#channel_convert..cSg|] }tj|ddqSr rrrrMrrrrOrPycSsg|]}t|ddqS)T)rB)rIrMrrrrOscSrQrRrSrMrrrrOrPr RGBcSrKr)rrCOLOR_GRAY2BGRrMrrrrOrPr)in_ctar_typeimg_list gray_listy_listrrrchannel_convertsr\cCs|j|jks td|jdd\}}|||||||f}|||||||f}|tj}|tj}t||d}|dkrLtdSdtdt |S)N+Input images must have the same dimensions.r rinfr ) shape ValueErrorr>rfloat64meanfloatmathlog10sqrt)img1img2borderhwmserrrcalculate_psnrs   rnc Cs|j|jks td|jdd\}}|||||||f}|||||||f}|jdkr9t||S|jdkr|jddkrog}tdD]}|t|dddd|f|dddd|fqKt|S|jddkrtt |t |SdStd)zMcalculate SSIM the same outputs as MATLAB's img1, img2: [0, 255] r]Nr r r zWrong input image dimensions.) r`rarssimrangeappendrarrayrcsqueeze)rhrirjrkrlssimsirrrcalculate_ssims"     6rvcCs:d}d}|tj}|tj}tdd}t||}t|d|ddddf}t|d|ddddf}|d}|d} ||} t|dd|ddddf|} t|dd|ddddf| } t||d|ddddf| } d| |d| ||| || | |}|S) Ng(\@gzGBM@ ?r ) r>rrbrgetGaussianKernelouter transposefilter2Drc)rhriC1C2kernelwindowmu1mu2mu1_sqmu2_sqmu1_mu2 sigma1_sq sigma2_sqsigma12ssim_maprrrros,   &&&    rocCsnt|}|d}|d}d|d|d|dk|d|d|d|d|dk|dk|S)Nr r rxg@r )torchabstype_as)xabsxabsx2absx3rrrcubics  0rcCs|dkr |r ||}td||}||ddd|}t||d}t|d} ||d|| td| d| d| || } ||d|| | } |dkrc|rc|t| |} nt| } t| d|d} | | || } t| dkd}tj |dddds| dd| d} | dd| d} tj |dddds| dd| d} | dd| d} | } | } | d}| |}| |d} | | t|t|fS)Nr ?r rgư>)rel_tolry)rlinspacefloorreceilviewexpandrsumisclosenarrow contiguousminmaxint) in_length out_lengthscaler kernel_width antialiasingruleftPindicesdistance_to_centerweights weights_sumweights_zero_tmp sym_len_s sym_len_errrcalculate_weights_indicess<     rcCsR|dkrdnd}|r|d|\}}}|t||t||}}} d} d} t|||| | |\} } }}t|| || | |\}}}}t|||||}|d|| ||ddd|ddf}t |dddd }| d|}|dd| ||dd| dddf}t |dddd }| d|}|d||| |t|||}| d} t |D].}t| |d}t |D]}||||| ddfdd| ||||ddf<qqt|||||}|d|| ||ddddd|f}t |dddd }| d|}|dd| ||dddd| df}t |dddd }| d|}|d||| |t||| }|d} t | D],}t||d}t |D]}||dd||| f||||dd|f<qqs|r||S) Nr TFrrrr ry)dim unsqueeze_sizererrr FloatTensorrcopy_arangelong index_selectrprr~mvsqueeze_)rrr need_squeezein_Cin_Hin_Wout_Cout_Hout_Wrr weights_H indices_H sym_len_Hs sym_len_He weights_W indices_W sym_len_Ws sym_len_Weimg_aug sym_patchinv_idx sym_patch_invout_1ruidxj out_1_augout_2rrrimresizesx                   rcCs`t|}|dkr dnd}|r|d|\}}}|t||t||}}} d} d} t|||| | |\} } }}t|| || | |\}}}}t|||||}| d|| ||d|ddddf}t |dddd }| d|}| dd| ||| dddddf}t |dddd }| d|}| d||| |t|||}| d} t|D].}t| |d}t|D]}|||| dd|fdd| |||dd|f<qqt|||||}| d|| ||ddd|ddf}t |dddd }| d|}| dd| ||dd| dddf}t |dddd }| d|}| d||| |t|| |}|d} t| D],}t||d}t|D]}|dd||| |f|||dd||f<qqx|r||S) Nr TFrrrr ry)r from_numpyrrrrerrrrrrrrrprr~rrnumpy)rrrrrrrrrrrrrrrrrrrrrrrrrrurrrrrrr imresize_np]sz                 rcCsB|jdd\}}t|}|d|||d|||dfS)zq Args: img: numpy image, WxH or WxHxC sf: scale factor Return: cropped image Nr .)r`rcopy)rsfrlrkimrrr modcrop_nps &rcCs|jd}td|dd|df}t|D])}t|D]"}|d|d||d|d||f|||f|7<qq|d}||| || f}||S)zLCalculate the X4 kernel from the X2 kernel (for proof see appendix in paper)rr r )r`rzerosrpr)kk_sizebig_krccrop cropped_big_krrranalytic_kernels   B rc Csttt|t| gt|t|ggtddg}t|d|dg|d|d gg}t|dgd|gg}tt||tj|}tddg||d}|S)aM generate an anisotropic Gaussian kernel Args: ksize : e.g., 15, kernel size theta : [0, pi], rotation angle range l1 : [0.1,50], scaling of eigenvalues l2 : [0.1,l1], scaling of eigenvalues If l1 = l2, will get an isotropic Gaussian kernel. Returns: k : kernel ?rr )rccovr)rr?rrcossinlinalginvgm_blur_kernel) ksizethetal1l2vVDSigmarrrranisotropic_Gaussians  (rc Cs~|dd}t||g}t|D]$}t|D]}||d}||d}tjj||g||d|||f<qq|t|}|S)N@rr )rcr)rrrpstatsmultivariate_normalpdfr) rcrrcenterrrTrcycxrrrrs      rc Cs|jdd\}}|dd}td|dtd|d}}|r)||}||} n||}||} t|d|d}t| d|d} |jdkrQt||||| }|jdkr{t|jdD]} t|||dddd| f|| |dddd| f<q]|S) zshift pixel for super-resolution with different scale factors Args: x: WxHxC or WxH sf: scale factor upper_left: shift direction Nr r rrrr ry)r`rrr(rrrp) rr upper_leftrkrlshiftxvyvx1y1rurrr shift_pixels     8rcCs|jdd\}}|jddd|jddd}}tjjj|||||fdd}|d|dd}|dd|jd|jd}|dd|jd|jd}tjjj||ddd ||d }||||jd|jd}|S) z2 x: image, NxcxHxW k: kernel, Nx1xhxw Nr r ry replicate)padmoder r)biasstridepaddinggroups)r`rnn functionalr repeatrconv2d)rrnrp1p2rrrblurs&rr333333?g$@cCs|tj||}|tj||}tjtj}| tjj||d}t||g} tt|t| gt|t|gg} | | | j} tj | ddddddf} |dd|d} | dddddf} t t |dt |d\}}t ||gddddddddf}|| }|dddd}tdt|| |d|}|t|}|S)z" # modified version of https://github.com/assafshocher/BlindSR_dataset_generator # Kai Zhang # min_var = 0.175 * sf # variance of the gaussian kernel will be sampled between min_var and max_var # max_var = 2.5 * sf r Nrr rr r)rrandomrandpidiagrrrrTrrmeshgridrpstackr~exprsr)r scale_factormin_varmax_var noise_levellambda_1lambda_2rnoiseLAMBDAQSIGMA INV_SIGMAMUXYZZZZZ_t raw_kernelrrrr gen_kernels&   ($r4c Cs||g}|ddd|dddg}|}tt|d |ddt|d |dd\}}|||| d||}t|}d||ttj|k<| }|dkrb||}|S)Nrrrr r ) rrrr!scipyfinfordepsrr) hsizesigmasizstdrrTargrksumhrrrfspecial_gaussianIs 0 r>cCsbtdt|dgg}||d}d||d}|||g|d|d|g|||gg}t|}|S)Nrr )rrrrr)alphah1h2rkrrrfspecial_laplacianXs  $ rCcOs0|dkr t|i|S|dkrt|i|SdS)Ngaussian laplacian)r>rC) filter_typeargskwargsrrrfspecialas rIcCst|d|d}|S)z Args: x: HxWxC image, [0, 1] sf: down-scale factor Return: bicubicly downsampled LR image r )r)r)rrrrrbicubic_degradationhsrJcCs,tjj|tj|dddd}t||d}|S)a blur + bicubic downsampling Args: x: HxWxC image, [0, 1] k: hxw, double sf: down-scale factor Return: downsampled LR image Reference: @inproceedings{zhang2018learning, title={Learning a single convolutional super-resolution network for multiple degradations}, author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, pages={3262--3271}, year={2018} } r rwrapr r)rfiltersconvolverrrJrrrrrrsrmd_degradationts rQcCs,t||d}tjj|tj|dddd}|S)a bicubic downsampling + blur Args: x: HxWxC image, [0, 1] k: hxw, double sf: down-scale factor Return: downsampled LR image Reference: @inproceedings{zhang2019deep, title={Deep Plug-and-Play Super-Resolution for Arbitrary Blur Kernels}, author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, pages={1671--1681}, year={2019} } rMr rrKrL)rJrrNrOrrrPrrrdpsr_degradations rRcCs:tjj|tj|dddd}d}||d||d|dfS)z blur + downsampling Args: x: HxWxC image, [0, 1]/[0, 255] k: hxw, double sf: down-scale factor Return: downsampled LR image r rrKrLrN.)rrNrOrr)rrrstrrrclassical_degradations rTr2 c Cs|ddkr |d7}t|||fd}||}t|d|k}|d}t|||fd}|||}t|dd}||d||S)aUSM sharpening. borrowed from real-ESRGAN Input image: I; Blurry image: B. 1. K = I + weight * (I - B) 2. Mask = 1 if abs(I - B) > threshold, else: 0 3. Blur mask: 4. Out = Mask * K + (1 - Mask) * I Args: img (Numpy array): Input image, HWC, BGR; float32, [0, 1]. weight (float): Sharp weight. Default: 1. radius (float): Kernel size of Gaussian blur. Default: 50. threshold (int): r rr r")r GaussianBlurrrr>r() rweightradius thresholdrresidualmask soft_maskKrrradd_sharpenings   r`cCsd|}dd|}|d}|d}tdkr8|t}|t}ttdddttj||d }ntd tddd|t}tjj|tj |dd d d }|S)N@r皙?rrr rwr rrrrrDrmirrorrL rrrandintrrrIrrNrOrrrwd2wdrrrrrr add_blur_1s(     rjcCstj}|dkrtdd}n|dkrtd|d}nd}tj|t||jdt||jdftgdd }t |d d}|S) N皙?r r gffffff?rrrr r r  interpolationr) rrruniformrresizerr`choicer()rrrnumsf1rrr add_resizes $ rtr c Cst||}tj}|dkr!|tjd|d|jtj}n^|dkr@|tjd|dg|jdddRtj}n?|d}ttjd}t tjdd}t t t |||}|tj gdt |d||jddtj}t|d d }|S) Nrrr 皙?r r r rrrrr)rrfrrnormalr`r>r"rrr?r~rrr( r noise_level1 noise_level2r%rrLrUconvrrradd_Gaussian_noises2    rc Cs(t||}t|dd}t}|dkr)||tjd|d|jtj7}nb|dkrJ||tjd|dg|jdddRtj7}nA|d}ttj d }t tj d d }t t t |||}||tj gd t|d||jddtj7}t|dd}|S) Nrrrrr rvr r r rw)rrfrr(rxr`r>r"rrrr?r~rrryrrradd_speckle_noises4  "rcCst|dddd}ddtd}tdkr,tj||tj|}n9t|ddd fgd }t|dddd}tj||tj||}||ddddtjf7}t|d d }|S) Nr rrWrVr rr.r )gA`"?gbX9?gv/?rr) rr(r)rpoissonr>r"r?newaxis)rvalsimg_gray noise_grayrrradd_Poisson_noises  rcCs\tdd}tt|tj}td|ttj|g\}}t |d}tt |tj }|S)NP_z.jpgr ) rrfrrr* COLOR_RGB2BGRimencoderIMWRITE_JPEG_QUALITYimdecoder$r)rquality_factorresultencimgrrradd_JPEG_noise(s  r@c Cs|jdd\}}td||}td||}|||||||ddf}t||t||}} |||||| | ||ddf}||fS)Nr r)r`rrfr) lqhqr lq_patchsizerkrlrnd_hrnd_wrnd_h_Hrnd_w_Hrrr random_crop2s"rc Cst|}d\}}}|jdd\}}|d|||d|||df}|jdd\}} |dkrnt|krntjdkr_tj|td|jdtd|jdft gd d }nt |dd }t |d d }d}t t dd} | d| d} } | | kr| | | | | | <| | <| D]} | dkrt||d}q| dkr|jd|jd}}tdkrtdd|}tj|td||jdtd||jdft gd d }n2tddtdd|}t||}||}tjj|tj|dddd}|dd|dd|df}t |d d }q| dkrCtj|td||td||ft gd d }t |d d }q| dkrPt|ddd}q| dkr`t|kr`t|}qt|}t|}|S) This is the variant of the degradation model of BSRGAN from the paper "Designing a Practical Degradation Model for Deep Blind Image Super-Resolution" sf: scale factor isp_model: camera ISP model Returns img: low-quality patch, size: lq_patchsizeXlq_patchsizeXC, range: [0, 1] hq: corresponding high-quality patch, size: (lq_patchsizexsf)X(lq_patchsizexsf)XC, range: [0, 1] ?g?rNr .rrr rrlrmTrrr rMrkrDru皙?rrrdrLrzr{rz)r$r`rrrrrrprrqrr(samplerpindexrjrorIrrrrNrOrrrr*imager isp_model_ jpeg_prob scale2_probrAw1rkrl shuffle_orderidx1idx2ruabrsr k_shiftedrrrr >sv *"            cCsd|}dd|}tdkr2|t}|t}tdtdddttj||d}ntd dtddd|t}tjj|tj |dd d d }|S) Nrarrbrr rwr rcrDrrdrLrergrrr add_blur_2s$      rc Cst|}d\}}}|jdd\}}|d|||d|||df}|jdd\}} |dkrnt|krntjdkr_tj|td|jdtd|jdft gd d }nt |dd }t |d d }d}t t dd} | d| d} } | | kr| | | | | | <| | <| D]} | dkrt||d}q| dkrt||d}q| dkr&|jd|jd}}tdkrtdd|}tj|td||jdtd||jdft gd d }n2tddtdd|}t||}||}tjj|tj|dddd}|dd|dd|df}t |d d }q| dkrNtj|td||td||ft gd d }t |d d }q| dkr[t|ddd}q| dkrkt|krkt|}qt|}t|}|S)rrNr .rrr rrlrmTrrrr rMg?rDrurrrrdrLrrz)r$r`rrrrrrprrqrr(rrprrrorIrrrrNrOrrrr*rrrrr sz *"            )r )T)r)r)rrUrV)r)r ru)rr)rN)>reosrrrrr5 scipy.statsrrrscipy.interpolater scipy.linalgrtorchvision.utilsrenviron__all__rr$r*r-r/rErHrIr\rnrvrorrrrrrrrrrrrrr4r>rCrIrJrQrRrTr`rjrtrrrrrr rr rrrrsx             9 JL      /         K