o oiZ@s UddlmZmZmZmZmZddlZddlmm Z ddlmZddl m Z ddl mZmZmZmZmZddlmZmZddlmZddlmZd Zeed <d d gZeeed <gdZeeed<gdZeeed<eeedZ ee!eefed<dddDZ"ee!e!fed<d6de#dee!effddZ$de#de#defdd Z%Gd!d"d"ej&Z'Gd#d$d$ej&Z(Gd%d&d&ej&Z)d'e!d(ee!efdefd)d*Z*Gd+d,d,ej&Z+Gd-d.d.ej&Z,Gd/d0d0ej&Z-d'e!d1e!dee!effd2d3Z.Gd4d5d5ej&Z/dS)7)AnyDictListTupleUnionN)nn)pi)Tensorcossintensorzeros)GaussianBlur2dSpatialGradient)cart2pol)create_meshgridg;f?sqrt2gpu?g "? COEFFS_N1_K1)"g#[?x@ٔ+?K '"? COEFFS_N2_K8)rrrgˈfF? COEFFS_N3_K8)xyrhophithetaCOEFFScCsi|] }|d|dqS)zChttps://github.com/manyids2/mkd_pytorch/raw/master/mkd_pytorch/mkd-z-64.pth).0krrF/home/ubuntu/.local/lib/python3.10/site-packages/kornia/feature/mkd.py %s r!)cartpolarconcaturls patch_sizereturncCs^t||dd}|ddddddf}|ddddddf}t||\}}||||d}|S)z1Get cartesian and polar parametrizations of grid.Theightwidthnormalized_coordinatesrN)xyrhophi)rr)r'kgridr.r/r0r1 grid_dictrrr get_grid_dict+s r4d1d2cCsFtjt|t|dd\}}tj||gdddd}|tjS)z&Get order for doing kronecker product.ij)indexingdim)torchmeshgridarangestackreshapetoint64)r5r6grid_d1grid_d2 kron_orderrrr get_kron_order5s  rGcsBeZdZdZd fdd ZdedefddZdefd d ZZ S) MKDGradientsaModule, which computes gradients of given patches, stacked as [magnitudes, orientations]. Given gradients $g_x$, $g_y$ with respect to $x$, $y$ respectively, - $\mathbox{mags} = $\sqrt{g_x^2 + g_y^2 + eps}$ - $\mathbox{oris} = $\mbox{tan}^{-1}(\nicefrac{g_y}{g_x})$. Args: patch_size: Input patch size in pixels. Returns: gradients of given patches. Shape: - Input: (B, 1, patch_size, patch_size) - Output: (B, 2, patch_size, patch_size) Example: >>> patches = torch.rand(23, 1, 32, 32) >>> gradient = MKDGradients() >>> g = gradient(patches) # 23x2x32x32 r(Ncs$td|_tdddd|_dS)N:0yE>diffr-F)modeorder normalized)super__init__epsrgradself __class__rr rOTs zMKDGradients.__init__r.cCst|tstdt|t|jdkstd|j|| }|dddddddddf}|dddddddddf}tj t |||j dd}|S)N Input type is not a Tensor. Got -Invalid input shape, we expect Bx1xHxW. Got: rr-r:) isinstancer TypeErrortypelenshape ValueErrorrQr=catrrP)rSr.grads_xygxgyr/rrr forwardZs  ""zMKDGradients.forwardcCs|jjSN)rU__name__rRrrr __repr__fszMKDGradients.__repr__r(N) re __module__ __qualname____doc__rOr rcstrrf __classcell__rrrTr rH<s  rHc sreZdZdZdedeeeeefeeeefdffddffdd Z d e de fd d Z de fd d Z ZS)VonMisesKernelaModule, which computes parameters of Von Mises kernel given coefficients, and embeds given patches. Args: patch_size: Input patch size in pixels. coeffs: List of coefficients. Some examples are hardcoded in COEFFS, Returns: Von Mises embedding of given parametrization. Shape: - Input: (B, 1, patch_size, patch_size) - Output: (B, d, patch_size, patch_size) Examples: >>> oris = torch.rand(23, 1, 32, 32) >>> vm = VonMisesKernel(patch_size=32, ... coeffs=[0.14343168, ... 0.268285, ... 0.21979234]) >>> emb = vm(oris) # 23x7x32x32 r'coeffs.r(Ncst||_t|}|d|t|d}||_d|d|_t dd||g}t |d}| ddd}t d|dg}t ||d|d<t |dd||dd<| ddd}|d||d||d|dS)Nrnr-r9r<emb0frangeweights)rNrOr'r register_bufferr\ndr=onesr?rAr sqrt)rSr'rnb_coeffsrsrorprqrTrr rOs"     zVonMisesKernel.__init__r.cCst|tstdt|t|jdkr|jddkr$td|jt|jts3tdt||j| | dddd}|j ||}t |}t |}tj|||gdd}|j|}|S)NrVrWr-rXzEmb0 type is not a Tensor. Got rr:)rYr rZr[r\r]r^rorBrepeatsizerpr r r=r_rq)rSr.rorpemb1emb2 embeddingrrr rcs   zVonMisesKernel.forwardc Cs.|jjd|jd|jd|jd|jd S)N (patch_size=z, n=z, d=z , coeffs=))rUrer'rsrtrnrRrrr rfs.zVonMisesKernel.__repr__)rerhrirjintrrfloatrrOr rcrkrfrlrrrTr rmjs >rmcsbeZdZdZddededdffdd Zd edefd d Zd edefddZ de fddZ Z S)EmbedGradientsa=Module that computes gradient embedding, weighted by sqrt of magnitudes of given patches. Args: patch_size: Input patch size in pixels. relative: absolute or relative gradients. Returns: Gradient embedding. Shape: - Input: (B, 2, patch_size, patch_size) - Output: (B, 7, patch_size, patch_size) Examples: >>> grads = torch.rand(23, 2, 32, 32) >>> emb_grads = EmbedGradients(patch_size=32, ... relative=False) >>> emb = emb_grads(grads) # 23x7x32x32 r&Fr'relativer(Ncst||_||_d|_t|tdd|_t||dd}t |dddddddf|dddddddf\}}| d|dS) NrIrr'rnTr)rr-r1) rNrOr'rrPrmrkernelrrrr)rSr'rr2_r1rTrr rOs >zEmbedGradients.__init__magscCst||j}|S)z@Embed square roots of magnitudes with eps for numerical reasons.)r=rvrP)rSrrrr emb_magsszEmbedGradients.emb_magsgradscCst|tstdt|t|jdkstd|j|ddddddddf}|ddddddddf}|jrH||j |}| || |}|S)NrVrWz-Invalid input shape, we expect Bx2xHxW. Got: r-) rYr rZr[r\r]r^rr1rBrr)rSrrorisr/rrr rcs   zEmbedGradients.forwardcCs|jjd|jd|jdS)Nr}z , relative=r~)rUrer'rrRrrr rfszEmbedGradients.__repr__)r&F) rerhrirjrboolrOr rrcrkrfrlrrrTr rs  r kernel_typegridsc sdtttdtdd|dkrd}ddg}n |dkr"d }d d g}t|}||d jd }fdd|D}dd|D}t|t|d}t|t|d}|||d } |||d} t |j |j } | d | ddd f| d | dddf} | S)z.cSs&i|]\}}||ddqS)r) unsqueezerrrrr r!s&rr-N) rrlistkeysr]itemsrmrsqueezerGrt index_select) rrcoeffs_params_rr' grids_normedvm_avm_bemb_aemb_brFspatial_kernelrrr spatial_kernel_embeddings$  0rc seZdZdZ     ddededed ed ed d f fd d Zded e fddZ d e e e ffddZ de d e fddZ d efddZZS)ExplicitSpacialEncodingaModule that computes explicit cartesian or polar embedding. Args: kernel_type: Parametrization of kernel ``'polar'`` or ``'cart'``. fmap_size: Input feature map size in pixels. in_dims: Dimensionality of input feature map. do_gmask: Apply gaussian mask. do_l2: Apply l2-normalization. Returns: Explicit cartesian or polar embedding. Shape: - Input: (B, in_dims, fmap_size, fmap_size) - Output: (B, out_dims, fmap_size, fmap_size) Example: >>> emb_ori = torch.rand(23, 7, 32, 32) >>> ese = ExplicitSpacialEncoding(kernel_type='polar', ... fmap_size=32, ... in_dims=7, ... do_gmask=True, ... do_l2=True) >>> desc = ese(emb_ori) # 23x175x32x32 r#r&Tr fmap_sizein_dimsdo_gmaskdo_l2r(Nc st|dvrt|d||_||_||_||_||_t||_ d|_ t |j|j }|jr=|j dd|_ ||j }| d|d|jd|_|j|j|_|j|_|\}}| d|| d|dS) N)r#r"z" is not valid, use polar or cart).r)sigmaembrr{idx1)rNrONotImplementedErrorrrrrrr4gridgmaskr get_gmaskrrrr]d_embout_dimsodims init_kron) rSrrrrrrr{rrTrr rO$s*      z ExplicitSpacialEncoding.__init__rcCs6|jd|jd}td|d|d}|S)zCompute Gaussian mask.r0r<r9)rmaxr=exp)rSrnorm_rhorrrr rLsz!ExplicitSpacialEncoding.get_gmaskcCsLt|j|j}tjt|j}t|d|dddf}||dddffS)z3Initialize helper variables to calculate kronecker.r-Nr) rGrrr=jitannotater rr)rSkron_embr{rrr rRsz!ExplicitSpacialEncoding.init_kronr.cCst|tstdt|t|jdk|jd|jkBs)td|jd|jtj t|j }t |d|}||j }|jdd}|jrMtj|dd}|S)NrVrWr-z!Invalid input shape, we expect Bxz xHxW. Got: )r9r:)rYr rZr[r\r]rr^r=rrrrr{sumrF normalize)rSr.rrzoutputrrr rcYs   zExplicitSpacialEncoding.forwardcCs>|jjd|jd|jd|jd|jd|jd|jdS)Nz (kernel_type=z , fmap_size= , in_dims=z , out_dims=z , do_gmask=z, do_l2=r~)rUrerrrrrrrRrrr rffs z ExplicitSpacialEncoding.__repr__)r#r&rTT)rerhrirjrkrrrOrr rrrrcrfrlrrrTr rs0( rcseZdZdZ   ddedeeeeeeffdfded ed ed e d dffd d Z deeeeeffd dfddZ dddZ dddZ dddZdddZded efddZd efddZZS) WhiteningaBModule, performs supervised or unsupervised whitening. This is based on the paper "Understanding and Improving Kernel Local Descriptors". See :cite:`mukundan2019understanding` for more details. Args: xform: Variant of whitening to use. None, 'lw', 'pca', 'pcaws', 'pcawt'. whitening_model: Dictionary with keys 'mean', 'eigvecs', 'eigvals' holding Tensors. in_dims: Dimensionality of input descriptors. output_dims: (int) Dimensionality reduction. keval: Shrinkage parameter. t: Attenuation parameter. Returns: l2-normalized, whitened descriptors. Shape: - Input: (B, in_dims, fmap_size, fmap_size) - Output: (B, out_dims, fmap_size, fmap_size) Examples: >>> descs = torch.rand(23, 238) >>> whitening_model = {'pca': {'mean': torch.zeros(238), ... 'eigvecs': torch.eye(238), ... 'eigvals': torch.ones(238)}} >>> whitening = Whitening(xform='pcawt', ... whitening_model=whitening_model, ... in_dims=238, ... output_dims=128, ... keval=40, ... t=0.7) >>> wdescs = whitening(descs) # 23x128 (ffffff?xformwhitening_modelNr output_dimskevaltr(cst||_||_||_||_d|_t||}||_t j t |dd|_ t j t |ddd|fdd|_t j t |d|dd|_|durS||dSdS)NrT) requires_grad)rNrOrrrrpvalminrr Parameterr meanr=eyeevecsruevalsload_whitening_parameters)rSrrrrrrrTrr rOs  &zWhitening.__init__cCs|jdkrdnd}||}|d|j_|dddd|jf|j_|dd|j|j_|j|j|j|j d}||jdS)Nlwpcareigvecseigvals)rrpcawspcawt) rrdatarrr _modify_pca _modify_lw _modify_pcaws _modify_pcawt)rSralgowh_model modificationsrrr rs z#Whitening.load_whitening_parameterscCs d|_dS)zModify powerlaw parameter.g?N)rrRrrr rs zWhitening._modify_pcacCsdS)zNo modification required.NrrRrrr rszWhitening._modify_lwcCs>|j|j}d||j|}|jtt|d|j_dS)zShrinkage for eigenvalues.r-N)rrrr=diagpowr)rSalpharrrr rs  zWhitening._modify_pcawscCs,d|j}|jtt|j||j_dS)zAttenuation for eigenvalues.rN)rrr=rrrr)rSmrrr rs "zWhitening._modify_pcawtr.cCszt|tstdt|t|jdkstd|j||j}||j}t |t t ||j }tj|ddS)NrVr9z)Invalid input shape, we expect NxD. Got: r-r:)rYr rZr[r\r]r^rrr=signrabsrrrrSr.rrr rcs   zWhitening.forwardcCs&|jjd|jd|jd|jdS)Nz(xform=r, output_dims=r~)rUrerrrrRrrr rfs&zWhitening.__repr__)rrrrg)rerhrirjrkrrr rrrOrrrrrrcrfrlrrrTr rrs6("     rc sfeZdZdZ     ddeded ed ed ed d f fdd Zded efddZd efddZ Z S) MKDDescriptoraModule that computes Multiple Kernel local descriptors. This is based on the paper "Understanding and Improving Kernel Local Descriptors". See :cite:`mukundan2019understanding` for more details. Args: patch_size: Input patch size in pixels. kernel_type: Parametrization of kernel ``'concat'``, ``'cart'``, ``'polar'``. whitening: Whitening transform to apply ``None``, ``'lw'``, ``'pca'``, ``'pcawt'``, ``'pcaws'``. training_set: Set that model was trained on ``'liberty'``, ``'notredame'``, ``'yosemite'``. output_dims: Dimensionality reduction. Returns: Explicit cartesian or polar embedding. Shape: - Input: :math:`(B, in_{dims}, fmap_{size}, fmap_{size})`. - Output: :math:`(B, out_{dims}, fmap_{size}, fmap_{size})`, Examples: >>> patches = torch.rand(23, 1, 32, 32) >>> mkd = MKDDescriptor(patch_size=32, ... kernel_type='concat', ... whitening='pcawt', ... training_set='liberty', ... output_dims=128) >>> desc = mkd(patches) # 23x128 r&r$rlibertyrr'r whitening training_setrr(Ncs@t||_||_||_||_d|d|_td|j|jfd|_t |_ d}d}|jdkr4||gn|jg|_ d|_ |d |d i}i|_ |j D]$} t||| d } t| || jjd } t| | |j | <|j | j 7_ qHt||j |_|jdurtjjt|jtd d} | |} t|| |j |jd|_|j|_ |dS)Nffffff?@r replicater#r"r$rTFr'rrrrcpu map_locationrr)rNrOr'rrrrr smoothingrH gradientsparametrizationsrfeatsrrrrtr Sequentialrrr=hubload_state_dict_from_urlr%devicerwhitening_layereval)rSr'rrrrpolar_scart_srelative_orientationsparametrizationgradient_embeddingspatial_encodingwhitening_modelsrrTrr rOs@       zMKDDescriptor.__init__patchescCst|tstdt|t|jdkstd|j||}||}g}|j D]}|j | |j | |j ||q,tj|dd}tj|dd}|jdurZ||}|S)NrVrWrXr-r:)rYr rZr[r\r]r^rrrrrBrappendr=r_rrrr)rSrgfeaturesrr/rrr rc5s      zMKDDescriptor.forwardc Cs6|jjd|jd|jd|jd|jd|jd S)Nr}z, kernel_type=z , whitening=z, training_set=rr~)rUrer'rrrrrRrrr rfPs zMKDDescriptor.__repr__)r&r$rrr) rerhrirjrrkrOr rcrfrlrrrTr rs, 2rrcCs&tjjt|tdd}||}|S)zLoad whitening model.rr)r=rrr%r)rrrrrrr load_whitening_model[sr c sXeZdZdZ     ddeded ed ed ed d f fdd Zded efddZZ S)SimpleKDz+Example to write custom Kernel Descriptors.r&r#rrrr'rrrrr(Nc st|dk}d|d}||_td||fd}t} t||d} t||| jjd} t |t ||| j |d} t || | | | |_dS) Nr#rrrrrrr)rNrOr'rrHrrrrtrr rrrr ) rSr'rrrrrrrroriesewhrTrr rOes   zSimpleKD.__init__r.cCs ||Srd)r rrrr rc}s zSimpleKD.forward)r&r#rrr) rerhrirjrrkrOr rcrlrrrTr r bs*r )r&)0typingrrrrrr=torch.nn.functionalr functionalrkornia.constantsr kornia.corer r r r r kornia.filtersrrkornia.geometry.conversionsr kornia.utilsrrr__annotations__rrrrrkr%rr4rGModulerHrmrrrrrr r rrrr s8       .F9jrw