o پiK @sdZddlZddlZddlmZmZddlZddlmZddl mm Z ddl m Z ddlmZddlmZddlmZeejd ddkZ  d4d eeefd eeeefd edejfddZdeeefdeedffddZ  d5dedefddZdeeefdeedffddZGdddejZ !d6d"eeefd#eeeffd$d%Z!Gd&d'd'ejZ" d7d(ed)eefd*d+Z#d,ed-ed.ejd/ejdejf d0d1Z$Gd2d3d3ejZ%dS)8zf Relative position embedding modules and functions Hacked together by / Copyright 2022 Ross Wightman N)OptionalTuple)ndgrid)RegularGridInterpolator)Mlp) trunc_normal_TIMM_USE_SCIPY_INTERPFq_sizek_size class_tokenreturncCs`|dusJdttt|dt|dd}|dddddf|dddddf}|ddd}|dddddf|dd7<|dddddf|dd7<|dddddfd|dd9<d|ddd|dd}|d}|rt|gd}||dddf<|d|dddf<|d|d<| S)Nz-Different q & k sizes not currently supportedrr)rrrrrr) torchstackrarangeflattenpermutesumFpad contiguous)r r r coordsrelative_coordsnum_relative_distancerelative_position_indexrM/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/pos_embed_rel.pygen_relative_position_indexs*,&&*   r new_window_sizenew_bias_shape.cCs|ddd|dddf}|jdkrI|\}}}|j\}}} ||dkr,||dks.J||ks6| |krGtjjj|d|dddd}|S|jdksPJ|\} }|j\} }| |d|d} t| | d} | | f} | d|dks| d|dkr| r|| dddf}|d| ddf}nd}tjjj| dd dd | d| df|ddd d | |  dd}|durtj ||fdd }|S) NrrrbicubicF)sizemode align_corners?rdim) ndimshapernn functional interpolate unsqueezesqueezeint transposereshapeviewcat) rel_pos_biasr!r"dst_size_dst_hdst_wnum_attn_headssrc_hsrc_w dst_num_pos src_num_posnum_extra_tokenssrc_size extra_tokensrrr resize_rel_pos_bias_table_simpleIsJ        rDr$T interpolation antialiasc Cs|\}}|\}}||ksJ||krM|j}|}t|d} t|d} tj|ddd|| | | | f||d} | ||dd} | || S|S)z Resample relative position bias table suggested in LeVit Adapted from: https://github.com/microsoft/Cream/blob/main/TinyViT/utils.py r(rr)r%r&rF) r%dtypefloatr2rr/rr5to) position_bias_tablenew_sizerErFL1nH1L2nH2 orig_dtypeS1S2$relative_position_bias_table_resizedrrrresize_rel_pos_bias_table_levitxs&     rTcstrddlm}|ddd|dddf}|jdkr?d}|\}}}||dkr0||dks2J|j\} } } | | f} d} n(|jdksFJ|\}}|j\}} ||d|d}t||d} | | f} d} | d|dksx| d|dkrh|r|| d d d f}|d | d d f}nd }d d fd d }|| d|d}|| d|d}t|t|g}|dd}|dd}t| |dd}t| |dd}t ||}g}t | D]^}| r|d d |f | d| d }n ||d d d d f }tr%|j |||dd}t||||j}nt||}|||j}| r=| dd}||q| rNtj|dd}ntj|dd}|d urh| s_Jtj||fdd}|S)a Resize relative position bias table using more advanced interpolation. Modified from code in Microsoft Unilm (https://github.com/microsoft/unilm) repo (BeiT, BeiT-v2, etc). https://github.com/microsoft/unilm/blob/5255d52de86dad642810f5849dd357769346c1d7/beit/run_class_finetuning.py#L351 Args: rel_pos_bias: new_window_size: new_bias_shape: Returns: r)r/rrr#Fr(TNcSs|d||d|S)N?r)arnrrrgeometric_progressionsz8resize_rel_pos_bias_table..geometric_progressionc sd\}}||dkr)||d}d||d}||dkr!|}n|}||dks g}d}t|dD]}|||||d7}q3ddt|D} | dg|S) N)g)\(?g?gư>@rrcSsg|]}| qSrr).0r9rrr sz._calc..r)rangeappendreversed) srcdstleftrightqgpdiscurir_idsrYrr_calcs     z(resize_rel_pos_bias_table.._calcrZg?rUcubic)kindrr)) _USE_SCIPYscipyr/r+r,r2rtensorrrr]r5rHinterp2dnumpyTensorrrIdevicerr^r6)r7r!r"r/r8rAr9r:r;r<r=r>rBhas_flat_shaper?r@rCrkyxyxtytxdydxdyxall_rel_pos_biasrhzfrWrrjrresize_rel_pos_bias_tablesh      "     &     rcPeZdZdZdfdd ZddZdejfdd Zdd e ejfd d Z Z S) RelPosBiasz_ Relative Position Bias Adapted from Swin-V1 relative position bias impl, modularized. rcst|dks J||_|d|d|_|j|fd|f|_d|ddd|ddd|}tt|||_ |j dt |j|dkd ddd | dS) Nrrrr#r)r rF persistent)super__init__ window_size window_area bias_shaper- Parameterrzerosrelative_position_bias_tableregister_bufferr r5 init_weights)selfr num_heads prefix_tokensr __class__rrrs  ( zRelPosBias.__init__cCst|jdddSNg{Gz?)std)rrrrrrr"szRelPosBias.init_weightsr cCs0|j|j}||jddd}|dS)Nrrr)rrr5rrr0rrrelative_position_biasrrrget_bias%s zRelPosBias.get_biasNshared_rel_poscC ||SNrrattnrrrrforward+ zRelPosBias.forwardrr __name__ __module__ __qualname____doc__rrrrsrrr __classcell__rrrrr s rrswinwin_sizepretrained_win_sizecCsr|dvsJt|dd |dtj}t|dd |dtj}tt||}|ddd}|dkr|ddkrj|dddddf|dd<|dddddf|dd<n&|dddddf|dd<|dddddf|dd<|d9}t|t d| t d}|St|t d| }|S)N)rcrrrrrrU) rrrIfloat32rrrrsignlog2absmathlog)rrr&relative_coords_hrelative_coords_wrelative_coords_tablerrrgen_relative_log_coords/s, "" &(&&   rcsReZdZdZ     dfdd Zd ejfd d Zdd eejfddZ Z S) RelPosMlpz Log-Coordinate Relative Position MLP Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883) This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw') rrrrcst||_|jd|jd|_||_||_|jfd|f|_|dkr2t|_ d|_ d}n t |_ d|_ d}t d||tj |dd |_|jd t|d d d |jdt|||dd d dS)Nrrrr)TFT)g?g)hidden_features out_features act_layerbiasdroprrFrrel_coords_log)r&)rrrrrrrr-Sigmoidbias_act bias_gainIdentityrReLUmlprr r5r)rrr hidden_dimrr&pretrained_window_sizemlp_biasrrrrRs>       zRelPosMlp.__init__r cCs||j}|jdur|d|j|j}||j}|ddd}||}|jdur1|j|}|j r@t ||j d|j dg}| d S)Nrrrr)rrrr5rrrrrrrrr0rrrrrr~s      zRelPosMlp.get_biasNrcCrrrrrrrrrzRelPosMlp.forward)rrrrrr) rrrrrrrsrrrrrrrrrLs, rlengthmax_relative_positioncCsv|dur|d}d|d}t|||}t|D]}t|D]}|||}t|||kr0qd||||f<qq|S)aGenerate a one_hot lookup tensor to reindex embeddings along one dimension. Args: length: the length to reindex to. max_relative_position: the maximum relative position to consider. Relative position embeddings for distances above this threshold are zeroed out. Returns: a lookup Tensor of size [length, length, vocab_size] that satisfies ret[n,m,v] = 1{m - n + max_relative_position = v}. Nrr)rrr]r)rr vocab_sizeretrhrwvrrrgenerate_lookup_tensors    rheightwidth height_lookup width_lookupcCs8td||}td||}||}||jd||S)a\Reindex 2d relative position bias with 2 independent einsum lookups. Adapted from: https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py Args: relative_position_tensor: tensor of shape [..., vocab_height, vocab_width, ...]. height: height to reindex to. width: width to reindex to. height_lookup: one-hot height lookup width_lookup: one-hot width lookup Returns: reindexed_tensor: a Tensor of shape [..., height * width, height * width, ...] z nhw,ixh->nixwznixw,jyw->nijxyr)reinsumr4r,)relative_position_tensorrrrrreindexed_tensorarearrrreindex_2d_einsum_lookupsrcr) RelPosBiasTfz Relative Position Bias Impl (Compatible with Tensorflow MaxViT models) Adapted from: https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py rcst|dks J||_|d|d|_||_d|dd}d|dd}|j||f|_tt |j|_ |j dt |ddd|j dt |ddd| dS)NrrrrFrr)rrrrrrr-rrrrrrr)rrrr vocab_height vocab_widthrrrrs   zRelPosBiasTf.__init__cCstjj|jdddSr)r-initnormal_rrrrrrszRelPosBiasTf.init_weightsr cCs"t|j|jd|jd|j|jS)Nrr)rrrrrrrrrrszRelPosBiasTf.get_biasNrcCrrrrrrrrrzRelPosBiasTf.forwardrrrrrrrrs  r)NF)r$T)rrr)&rrostypingrrrtorch.nnr-torch.nn.functionalr.rgridrr/rrr weight_initrr2environgetrnboolrsr rDstrrTrModulerrrrrrrrrrs       4   2    t%   E