o
    oiZr                  
   @   sJ  d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZ d dlZd dlm  mZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z# d dl$m%Z%m&Z& d d	l'm(Z( zd d
l)m*Z* W n e+y   dZ*Y nw e*se,edrdZ-ndZ-dd Z.e(ej/ddededefddZ0dede1deeef fddZ2dedefddZ3dededefddZ4G d d! d!eZ5G d"d# d#eZ6G d$d% d%eZ7G d&d' d'eZ8G d(d) d)eZ9G d*d+ d+eZ:d,ed-ed.edefd/d0Z;G d1d2 d2eZ<d3ed4e=deeeeef fd5d6Z>G d7d8 d8eZ?dS )9    N)Path)SimpleNamespace)AnyCallableClassVarDictListOptionalSequenceTuple)nn)Module
ModuleListTensorarangeconcatenatecoseinsumones	ones_likesinstackwherezeros)KORNIA_CHECK)laf_to_three_points	scale_laf)
custom_fwd)FlashCrossAttentionscaled_dot_product_attentionTFc                 C   s   t t| ||S )z'Clamp a value to lie within [min, max].)minmax)xmin_max_ r%   L/home/ubuntu/.local/lib/python3.10/site-packages/kornia/feature/lightglue.py
math_clamp:   s   r'   )cast_inputskptssizereturnc                 C   sp   t |tjrt|d }| | d }|dj | d }| |dddf  |ddddf  } | S )zNormalize tensor of keypoints.N      )
isinstancetorchSizetensorfloattor!   values)r)   r*   shiftscaler%   r%   r&   normalize_keypoints?   s   &r7   r"   lengthc                 C   s   || j d kr| t| dddf tjdfS tg | j dd || j d  | j d R | j| jd}t| |gdd}tg |j dd dR tj| jd	}d
|dd| j d ddf< ||fS )zPad tensor to desired length..Nr-   )dtypedevicer:   dim)r:   r=   T)	shaper   r/   boolr   r=   r:   r   r   )r"   r8   padymaskr%   r%   r&   pad_to_lengthJ   s   <(rE   c                 C   s6   |  dd} | jdd\}}t| |fddjddS )zApply half rotation.r;   )r;   r,   r>   r9   	start_dim)	unflattenunbindr   flatten)r"   x1x2r%   r%   r&   rotate_halfU   s   rM   freqstc                 C   s   || d  t || d   S )zApply rotary embedding.r   r-   )rM   )rN   rO   r%   r%   r&   apply_cached_rotary_emb\   s   rP   c                       sJ   e Zd Zddededee deddf
 fdd	Zd
edefddZ  Z	S )"LearnableFourierPositionalEncodingN      ?Mr?   F_dimgammar+   c                    sX   t    |d ur|n|}|| _tj||d dd| _tjj| jjj	d| jd d d S )Nr,   Fbiasr   r9   )meanstd)
super__init__rU   r   LinearWrinitnormal_weightdata)selfrS   r?   rT   rU   	__class__r%   r&   r[   b   s
   
"z+LearnableFourierPositionalEncoding.__init__r"   c                 C   s>   |  |}t|t|}}t||gdd}|jdddS )zEncode position vector.r   r,   r;   r>   )r]   r   r   r   	unsqueezerepeat_interleave)rb   r"   	projectedcosinessinesembr%   r%   r&   forwardi   s   
z*LearnableFourierPositionalEncoding.forward)NrR   )
__name__
__module____qualname__intr	   r2   r[   r   rl   __classcell__r%   r%   rc   r&   rQ   a   s    (rQ   c                       sD   e Zd Zdeddf fddZdededeeef fdd	Z  ZS )
TokenConfidencer?   r+   Nc                    s(   t    tt|dt | _d S )Nr-   )rZ   r[   r   
Sequentialr\   Sigmoidtokenrb   r?   rc   r%   r&   r[   r   s   
zTokenConfidence.__init__desc0desc1c                 C   sT   | j d jj}|j}|  | |d||  | |d|fS )zGet confidence tokens.r   r;   )ru   r`   r:   detachr3   squeeze)rb   rw   rx   r:   
orig_dtyper%   r%   r&   rl   v   s
   zTokenConfidence.forward)	rm   rn   ro   rp   r[   r   r   rl   rq   r%   r%   rc   r&   rr   q   s    &rr   c                       sJ   e Zd Zdeddf fddZddededed	ee def
d
dZ  ZS )	Attentionallow_flashr+   Nc                    sb   t    |rtstjddd |ot| _ttd| _|r#t	r#t	 | _
| jr/tjj| d S d S )NzcFlashAttention is not available. For optimal speed, consider installing torch >= 2.0 or flash-attn.r,   
stacklevelr   )rZ   r[   FLASH_AVAILABLEwarningswarnenable_flashhasattrFhas_sdpr   flash_r/   backendscudaenable_flash_sdp)rb   r}   rc   r%   r&   r[      s   

zAttention.__init__qkvrD   c           
      C   sD  | j rZ|jjdkrZ| jr,dd |||fD }tj|d|i|j}|d u r(|S | S t	|d u  dd |||fD \}}}| 
| t||gd }|dd	|j S | jryd
d |||fD }tj|d|i}|d u ru|S | S |jd d }td||| }|d ur|| td  t|d}	td|	|S )Nr   c                 S   s   g | ]}|   qS r%   )half
contiguous.0r"   r%   r%   r&   
<listcomp>       z%Attention.forward.<locals>.<listcomp>	attn_maskc                 s   s     | ]}| d d V  qdS )r9   re   N)	transposer   r   r%   r%   r&   	<genexpr>   s    z$Attention.forward.<locals>.<genexpr>r,   r9   re   c                 S   s   g | ]}|  qS r%   )r   r   r%   r%   r&   r          r;         z...id,...jd->...ijinfz...ij,...jd->...id)r   r=   typer   r   r   r3   r:   
nan_to_numr   r   r   r   r   cloner@   r   masked_fillr2   softmax)
rb   r   r   r   rD   argsmssimattnr%   r%   r&   rl      s&   zAttention.forwardN)	rm   rn   ro   rA   r[   r   r	   rl   rq   r%   r%   rc   r&   r|      s    ,r|   c                       sV   e Zd Zddededededdf
 fd	d
Z	ddededee defddZ  Z	S )	SelfBlockFT	embed_dim	num_headsflashrW   r+   Nc              	      s   t    || _|| _t| j| dkd | j| | _tj|d| |d| _t	|| _
tj|||d| _ttd| d| tjd| ddt td| || _d S )Nr   z0Embed dimension should be dividable by num_heads   rV   r,   Telementwise_affine)rZ   r[   r   r   r   head_dimr   r\   Wqkvr|   
inner_attnout_projrs   	LayerNormGELUffn)rb   r   r   r   rW   rc   r%   r&   r[      s   


zSelfBlock.__init__r"   encodingrD   c           
      C   s   |  |}|d| jddfdd}|d |d |d }}}t||}t||}| j||||d}| |ddjd	d
}	|| t	||	gd S )Nr;   r   r-   r,   .r   .r-   ).r,   )rD   r9   rF   )
r   rH   r   r   rP   r   r   rJ   r   r   )
rb   r"   r   rD   qkvr   r   r   contextmessager%   r%   r&   rl      s   


zSelfBlock.forwardFTr   )
rm   rn   ro   rp   rA   r[   r   r	   rl   rq   r%   r%   rc   r&   r      s    $r   c                       s~   e Zd Zddededededdf
 fd	d
Zdedededeeef fddZ	ddedede
e deeef fddZ  ZS )
CrossBlockFTr   r   r   rW   r+   Nc              	      s   t    || _|| }|d | _|| }tj|||d| _tj|||d| _tj|||d| _t	td| d| tj
d| ddt td| || _|r[tr[td| _d S d | _d S )Nr   rV   r,   Tr   )rZ   r[   headsr6   r   r\   to_qkto_vto_outrs   r   r   r   r   r|   r   )rb   r   r   r   rW   dim_head	inner_dimrc   r%   r&   r[      s"   


zCrossBlock.__init__funcx0rK   c                 C   s   ||||fS r   r%   )rb   r   r   rK   r%   r%   r&   map_   s   zCrossBlock.map_rD   c              	      s     j||\}}   j||\}} fdd||||fD \}}}} jd urK|jjdkrK ||||} ||||d urG|ddnd }	nT| jd  | jd  }}td||}
|d urn|
	| t
d }
tj|
dd	}tj|
dd dd	}td
||}td|dd|}	|d ur| |	 }}	  dd ||	\}}	   j||	\}}	| t||gd }| t||	gd }||fS )Nc                 3   s*    | ]}| d  jd fddV  qdS )r;   r-   r,   N)rH   r   r   )r   rO   rb   r%   r&   r      s   ( z%CrossBlock.forward.<locals>.<genexpr>r   r;   r9   g      ?zbhid, bhjd -> bhijr   r>   zbhij, bhjd -> bhidzbhji, bhjd -> bhidc                 S   s   |  ddjddS )Nr-   r,   r9   rF   )r   rJ   )rO   r%   r%   r&   <lambda>   r   z$CrossBlock.forward.<locals>.<lambda>)r   r   r   r   r=   r   r   r6   r   r   r2   r   r   r   r   r   r   r   )rb   r   rK   rD   qk0qk1v0v1m0m1r   attn01attn10r%   r   r&   rl      s*   "&zCrossBlock.forwardr   r   )rm   rn   ro   rp   rA   r[   r   r   r   r   r	   rl   rq   r%   r%   rc   r&   r      s    $"0r   c                       sv   e Zd Z fddZ		ddededededee d	ee d
efddZdededededed	ed
efddZ  ZS )TransformerLayerc                    s.   t    t|i || _t|i || _d S r   )rZ   r[   r   	self_attnr   
cross_attn)rb   r   kwargsrc   r%   r&   r[      s   
zTransformerLayer.__init__Nrw   rx   	encoding0	encoding1mask0mask1r+   c                 C   sH   |d ur|d ur|  ||||||S | ||}| ||}| ||S r   )masked_forwardr   r   )rb   rw   rx   r   r   r   r   r%   r%   r&   rl      s
   	zTransformerLayer.forwardc                 C   sZ   || dd@ }|| dd@ }|| dd@ }| |||}| |||}| |||S )Nr;   r9   )r   r   r   )rb   rw   rx   r   r   r   r   rD   r%   r%   r&   r     s   zTransformerLayer.masked_forwardNN)	rm   rn   ro   r[   r   r	   rl   r   rq   r%   r%   rc   r&   r      sD    
r   r   z0z1c           
      C   s   | j \}}}t|t|dd }t| d}t| dd ddd}| ||d |d fd}	|| | |	ddd|d|f< t|d |	dddddf< t|d |	dddddf< |	S )z<Create the log assignment matrix from logits and similarity.r-   r,   r;   r9   r   N)r@   r   
logsigmoidr   log_softmaxr   new_fullrz   )
r   r   r   br   ncertaintiesscores0scores1scoresr%   r%   r&   sigmoid_log_double_softmax  s    "$$r   c                       sV   e Zd Zdeddf fddZdededeeef fdd	Zd
edefddZ  Z	S )MatchAssignmentr?   r+   Nc                    s8   t    || _tj|ddd| _tj||dd| _d S )Nr-   TrV   )rZ   r[   r?   r   r\   matchability
final_projrv   rc   r%   r&   r[   *  s   
zMatchAssignment.__init__rw   rx   c                 C   sp   |  ||  |}}|j\}}}||d  ||d  }}td||}| |}| |}	t|||	}
|
|fS )z)Build assignment matrix from descriptors.g      ?zbmd,bnd->bmn)r   r@   r   r   r   )rb   rw   rx   mdesc0mdesc1_dr   r   r   r   r%   r%   r&   rl   0  s   

zMatchAssignment.forwarddescc                 C   s   t | |dS )Nr;   )r/   sigmoidr   rz   )rb   r   r%   r%   r&   get_matchability;  s   z MatchAssignment.get_matchability)
rm   rn   ro   rp   r[   r   r   rl   r   rq   r%   r%   rc   r&   r   )  s    r   r   thc                 C   s  | ddddddf  d| ddddddf  d}}|j|j}}t|jd |jdd }t|jd |jdd }||d|k}||d|k}	|j }
|
d}t	||
|}t	|	|d||}|||k@ }|	|d|@ }t	||d}t	||d}||||fS )z;Obtain matches from a log assignment matrix [Bx M+1 x N+1].Nr;   r,   r-   r=   r   )
r!   indicesr   r@   r=   gatherr4   exp
new_tensorr   )r   r   max0max1r   r   indices0indices1mutual0mutual1max0_expzeromscores0mscores1valid0valid1r%   r%   r&   filter_matches?  s   B

r   c                       s  e Zd ZU dddddddddddd	d
ddZeeeef  ed< dddddZ	eeeef  ed< ddgZ
eee  ed< dZee ed< dZee ed< ddddddddddddd ddd!ddd"d#ddddd$d%d#ddddd$d%d&ddddd'd(	Zeeeef  ed)< dHd)ed+df fd,d-Z	/dId0ed1ee d+dfd2d3Zd4ed+efd5d6Zd4ed+efd7d8Zd9ed+efd:d;Zd<ed=ed9ed+efd>d?Zd@edAed9edBed+ef
dCdDZdEejd+efdFdGZ  ZS )J	LightGlue	lightglue   FrR   	      Tgffffff?gGz?皙?N)name	input_dimdescriptor_dimadd_scale_oriadd_laf
scale_coefn_layersr   r   mpdepth_confidencewidth_confidencefilter_thresholdweightsdefault_confr;         )cpumpsr   r   pruning_keypoint_thresholdsimage0image1required_data_keysz
v0.1_arxivversionzFhttps://github.com/cvg/LightGlue/releases/download/{}/{}_lightglue.pthurlsuperpoint_lightglue)r  r  dedodeb_lightgluededodeg_lightgluedisk_lightglue   aliked_lightgluesift_lightglue)r  r  r
  
doghardnetgUUUUUU?)r  r  r  r  r
  r  keynet_affnet_hardnet_lightglue)r  r  r  r  r  )	
superpointdedodebdedodegdiskalikedsiftdog_affnet_hardnetr%  keynet_affnet_hardnetfeaturesr'  r+   c                    s  t    td+i i j| _ |d ur6t|tj v d j| 	 D ]
\}}t
 || q+tjjo>jj   j jkrStj j jdd_nt _ j j }tdd j  d j  ||_ j j j}t fddt|D _tfddt|D _tfd	dt|d
 D _dtfddtjjD  d }|d ur j dj d dd}|dkrd}|dv rd}d}	n|dv rd}d}	n|dv rd}d}	nj!"j|}	t#j$j%|	|d}n jd ur t&t'j(}
|
djj d }
t#j)t*|
dd}|r`tjjD ]/}d | d!| d"ffd#d$|	 D }d%| d!| d&ffd'd$|	 D }q)j+|d(d) t,d* d _-d S ),NzFeatures keys are wrongTrV   r,   r  c                    s   g | ]	}t  jqS r%   )r   r   r   r   )confr   hr%   r&   r         z&LightGlue.__init__.<locals>.<listcomp>c                       g | ]}t  qS r%   )r   r0  r   r%   r&   r     r   c                    r4  r%   )rr   r0  r5  r%   r&   r     r   r-   confidence_thresholdsc                    s   g | ]}  |qS r%   )confidence_threshold)r   ir   r%   r&   r     s    r   z.pth.-r-  r%  )r.  z"keynet_affnet_hardnet_lightlue.pthzKhttp://cmp.felk.cvut.cz/~mishkdmy/models/keynet_affnet_hardnet_lightlue.pth)r(  zdedodeb_lightglue.pthz>http://cmp.felk.cvut.cz/~mishkdmy/models/dedodeb_lightglue.pth)r)  zdedodeg_lightglue.pthz>http://cmp.felk.cvut.cz/~mishkdmy/models/dedodeg_lightglue.pth)	file_namezweights/r  )map_locationz
self_attn.ztransformers.z
.self_attnc                       i | ]
\}}|j   |qS r%   replacer   r   r   patternr%   r&   
<dictcomp>      z&LightGlue.__init__.<locals>.<dictcomp>zcross_attn.z.cross_attnc                    r=  r%   r>  r@  rA  r%   r&   rC    rD  F)strictzLoaded LightGlue modelr%   ).rZ   r[   r   r  r1  r   listr/  keysitemssetattrr
  r  r  r	  r   r\   
input_projIdentityr   rQ   posencr  r   rangetransformerslog_assignmenttoken_confidenceregister_bufferr   r  r  r?  r  formatr/   hubload_state_dict_from_urlr   __file__parentloadstrload_state_dictprintstatic_lengths)rb   r/  conf_r   r   r   r   
state_dictfnamer  pathr8  rc   )r1  r   r2  rB  rb   r&   r[     sj   

  


zLightGlue.__init__reduce-overheadr  i   i   r  i   r  moder[  c                 C   sV   | j jdkrtjddd t| j jD ]}tj| j| j	|dd| j| _	q|| _
d S )Nr;   z9Point pruning is partially disabled for compiled forward.r,   r~   T)rb  	fullgraph)r1  r  r   r   rM  r  r/   compilerN  r   r[  )rb   rb  r[  r8  r%   r%   r&   rd    s   
zLightGlue.compilera   c                 C   s@   t j| jjdd | |W  d   S 1 sw   Y  dS )a  Match keypoints and descriptors between two images.

        Input (dict):
            image0: dict
                keypoints: [B x M x 2]
                descriptors: [B x M x D]
                image: [B x C x H x W] or image_size: [B x 2]
            image1: dict
                keypoints: [B x N x 2]
                descriptors: [B x N x D]
                image: [B x C x H x W] or image_size: [B x 2]
        Output (dict):
            log_assignment: [B x M+1 x N+1]
            matches0: [B x M]
            matching_scores0: [B x M]
            matches1: [B x N]
            matching_scores1: [B x N]
            matches: List[[Si x 2]], scores: List[[Si]]
        r   )enableddevice_typeN)r/   autocastr1  r  _forward)rb   ra   r%   r%   r&   rl     s   $zLightGlue.forwardc           6         s  | j D ]}t||v d| d q|d |d d d }}|j\}}}|j\}}}|j}	dd}
}|
d urC|
nd jdd  d d d	 }
|d urW|nd jdd  d d d	 }t||
 }t|| }tt|d	k	 ot|d
k	 d tt|d	k	 ot|d
k	 d | j
jrt|gfdddD  d	}| j
jdkr|d | j
j |d< t|gfdddD  d	}| j
jdkr|d | j
j |d< n[| j
jr;td | j
j}td | j
j}t|}t|}t|t|d |
 |jt|d |
 |jgd	}t|t|d | |jt|d | |jgd	}d   }d   }t|jd	 | j
jkd t|jd	 | j
jkd t rp| }| }d\}}t|| | jo t| jk}|rt fdd| jD }t||\}}t||\}}t||\}}t||\}}| |}| |}| |}| |}| j
jdk}| j
jdko| }|  |	}|rt!d||	dd  }t!d||	dd  }t"|}t"|}d\}}t#| j
j$D ]}| j%| ||||||d\}}|| j
j$d
 krq|rG| j&| ||\}}| '|dd |d d f |dd |d d f ||| rG n~|r|jd |kr| j(| )|} | *|| |}!t+|!d
 }"|,d
|"}|,d
|"}|,d|"}|d d |f  d
7  < |r|jd |kr| j(| )|}#| *||#|}$t+|$d
 }%|,d
|%}|,d
|%}|,d|%}|d d |f  d
7  < q|dd |d d f |dd |d d f }}| j(| ||\}&}t-|&| j
j.\}'}(})}*g g }+},t#|D ]7}-|'|- d	k}.t+|.d }/|'|- |. }0|r||-|/f }/||-|0f }0|+/t0|/|0gd	 |,/|)|- |.  q|rtj1||fd	|'j|'jd}1tj1||fd	|(j|(jd}2t+|'d	kd	|2d
|'j3dd|1d d |f< t+|(d	kd	|2d
|(j3dd|2d d |f< t4||f|)jd}3t4||f|*jd}4|)|3d d |f< |*|4d d |f< |1|2|3|4f\}'}(})}*nt"|)| j
j$ }t"|*| j
j$ }|&|'|(|)|*|d
 |+|,||d
}5|5S ) NzMissing key z in datar  r  	keypoints
image_sizeimager9   r;   r-    c                       g | ]	} |  d qS r;   rf   r   r   )data0r%   r&   r     r3  z&LightGlue._forward.<locals>.<listcomp>)scalesorisrR   ).r9   c                    rm  rn  ro  rp  )data1r%   r&   r     r3  lafsr   r   descriptorsz7Descriptor dimension does not match input dim in configr   c                    s   g | ]}| kr|qS r%   r%   rp  )cr%   r&   r   D  r   r   r   )r   r   .r<   )r    )
rO  matches0matches1matching_scores0matching_scores1stopmatchesr   prune0prune1)5r  r   r@   r=   getr7   r   r/   allitemr1  r
  r   r  r  r   r   r3   r:   ry   r   r  is_autocast_enabledr   r!   r[  r    rE   rJ  rL  r  r  pruning_min_kptsr   r   rM  r  rN  rP  check_if_stoprO  r   get_pruning_maskr   index_selectr   r  appendr   fullr   clampr   )6rb   ra   keykpts0kpts1r   r   r   r   r=   size0size1laf0laf1rw   rx   r   r   
do_compileknr   r   do_early_stopdo_point_pruning
pruning_thind0ind1r~  r  token0token1r8  r   
prunemask0keep0r   
prunemask1keep1r   r   r   r   r   r}  mscoresr   validm_indices_0m_indices_1m0_m1_	mscores0_	mscores1_predr%   )rw  rq  rt  r&   rh  	  s  
((**
	






:.
,,zLightGlue._forwardlayer_indexc                 C   s.   ddt d| | jj   }tt|ddS )zScaled confidence threshold.g?r  g      r   r-   )mathr   r1  r  r    r!   )rb   r  	thresholdr%   r%   r&   r7    s   zLightGlue.confidence_thresholdconfidencesr   c                 C   s.   |d| j j k}|dur||| j| kO }|S )z$Mask points which should be removed.r-   N)r1  r  r6  )rb   r  r   r  keepr%   r%   r&   r    s   zLightGlue.get_pruning_maskconfidences0confidences1
num_pointsc                 C   s<   t ||gd}| j| }d||k   |  }|| jjkS )zEvaluate stopping condition.r;   rR   )r   r6  r2   sumr1  r  )rb   r  r  r  r  r  r  ratio_confidentr%   r%   r&   r    s   
zLightGlue.check_if_stopr=   c                 C   s,   | j jrtr|jdkr| jd S | j|j S )Nr   r   )r1  r   r   r   r  )rb   r=   r%   r%   r&   r    s   
zLightGlue.pruning_min_kpts)r'  )r`  ra  )rm   rn   ro   r  r   r   rX  r   __annotations__r  r  r   r  r  r/  r[   r
   rp   rd  dictrl   rh  r2   r7  r   r  r  r/   r=   r  rq   r%   r%   rc   r&   r  R  s   
 		3?
 
r  )@r  r   pathlibr   typesr   typingr   r   r   r   r   r	   r
   r   r/   torch.nn.functionalr   
functionalr   kornia.corer   r   r   r   r   r   r   r   r   r   r   r   r   kornia.core.checkr   kornia.feature.lafr   r   kornia.utils._compatr   flash_attn.modules.mhar   ModuleNotFoundErrorr   r   r'   float32r7   rp   rE   rM   rP   rQ   rr   r|   r   r   r   r   r   r2   r   r  r%   r%   r%   r&   <module>   sH   (<

(!1""