o i&@sddlZddlmZddlZddlZddlmZmZm Z m Z ddZ ddddd Z d d d d Z ddddZddZddZdejdejfddZdededejfddZdededejfddZdedejfd d!ZdJd"d#ZdJd$d%ZdJd&d'ZdKd(d)ZdJd*d+Zd,d-ZdLd.d/ZdMd1d2Zd3d4Z d5ejd6edejfd7d8Z!d9ed:ej"dejfd;d<Z#d=ed6ed>efd?d@Z$d=ed6ed>efdAdBZ%d=ed6ed>efdCdDZ&d=ed6ed>efdEdFZ'dGejd6efdHdIZ(dS)NN)List)BigBirdSparsityConfigBSLongformerSparsityConfigFixedSparsityConfigVariableSparsityConfigcGsdd|D}tj|S)NcSsg|]}t|qS)torcharange).0srrd/home/ubuntu/.local/lib/python3.10/site-packages/xformers/components/attention/attention_patterns.py sz%_generate_nd_grid..)rmeshgrid)sizescoordsrrr _generate_nd_grids r@pweightscGsj|dur dt|}t|t|ksJt|}ddt||D}tj|dd}tj|||d}|S)NcSsg|] \}}||qSr)flatten)r iwrrr r sz%local_nd_distance..r)dimr)lenrziprstackfloatcdist)rrrgriddrrr local_nd_distances r$rsigmacGs,t|ddid}td|d|}|S)Nrrgg)r$rexp)r&rr#rrr local_nd_gaussian_distribution&sr)rcGst|d|i}||kS)Nrr$)distancerrr#rrr local_nd_pattern,sr,cGst|ddi}|dkS)Nrrr'r*)rr#rrr axial_nd_pattern1sr-cCs~tj|tjd}|dkr,|}||}tjj||| dd}t |}n tj | |dd}d| d|<|S)NdtypeiF)rreplace) replacementT) r zeros_likeboolnumeldoublesumnprandomchoicer as_tensor multinomialview) dist_matrixnnzattidxsrrr &random_pattern_from_probability_matrix7s   rBattention_query_maskreturncCsB|jdksJ|jtjksJ|dddf}||ddB}|S)Nrr)ndimr/rr4 transpose)rCmaskrrr global_token_patternLs rH attn_sizesparsitycCs0d|kr dksJJt|||k}|S)Nrr)rrand)rIrJrGrrr random_patternTsrL window_sizecCs.|ddks Jd|dd}t||ddS)Nr'rzFThe window size is assumed to be odd (counts self-attention + 2 wings)g?r+rr,)rIrM h_win_sizerrr local_1d_pattern[s  rQcCsttj||tjd}|S)Nr.)rtrilonesr4)rIrGrrr causal_1d_patterncsrTcCt|||dd}|S)Nrrrr*HWrr#rrr horizontal_axial_2d_distanceirZcCrU)Nrrrr*rWrrr vertical_axial_2d_distancenr[r]cCt|||dS)Nrr*)rXrYrrrr local_2d_distancesr_cCr^)Nr%)r))rXrYr&rrr local_2d_gausian_distributionwr`racCst||||dS)NrNrO)rXrYr+rrrr local_2d_pattern{srbcCs t||S)N)r-)rXrYrrr axial_2d_patterns rccCs:||dksJ||dksJd|kr|ks"JdJdt||\}}|d|d}}t||dk}|||}|||}t||\} } | |} |d| } | || } | || } t||gd} t| | gd}tj| |ddd}|dddf|dddfk}|S)Nrz shift_size must in 0-window_sizeg?r'rr)rintrrrr r!argmin)rXrYrM shift_sizerjextragrid_hgrid_wiijjr offset input_coordsanchors_coords anchor_idrGrrr swin_attention_patterns$$       rqr'cCsDt||ddd}t||ddd}||dk||dk@}|S)z Returns a 2d pattern that samples 1 every k elements in the attention mask. Can be seen as a form of downsampling, where every pixel attends to a downsampled version of the input. rrVrr\r)r$floor)rXrYkd_hd_wr#rrr dilated_2d_patterns rvcCstj|d|||f|j|jd}tt|jddD]-\}\}}}|dd||||d||||d|f|dd|ddddf<q|S)z> Block sparsify a tensor, given a mask and block size r)r/deviceT)as_tupleNr) remptysizer7r/rw enumeraternonzero)xrG block_sizeretidxhrrgrrr block_sparsify_tensors"rrGr~cCs|jdks Jdd}|jdkr|d}d}|jd|dkr)|jd|dks-Jdtjjj|tj||d}|tj }|rH| d|S) z Given a mask pattern and blocksize, return the corresponding layout which makes sure that all the positives in the mask are covered r'z/We're expecting [Heads, Seq, Seq] or [Seq, Seq]FrTrz1We're only handling masks divisible by block_size) kernel_sizestride) rE unsqueezeshapernn functional max_pool2dtor longsqueeze_)rGr~_should_squeezelayoutrrr pattern_to_layouts  &  r threshold mask_shapecsvdtffdd |d}|d}t|}|ddt|dd|dd}||d|}||kS)a1 Use the additive bias computation from ALiBi_ to generate a mask. Note that this mask can in turn be used to generate a blocksparse attention computation layout .. note: mask_shape is expected to hold the [heads, seq, seq] dimensions .. _ALiBi: https://arxiv.org/pdf/2108.12409.pdf ncsjdtdttfdd}t|r||Sdtt|}||d|dddd||S)NrrDcs6ddt|d  fddt|DS)Nr'csg|]}|qSrr)r rratiostartrr r szTalibi_pattern..get_slopes..get_slopes_power_of_2..)mathlog2range)rrrr get_slopes_power_of_2sz@alibi_pattern..get_slopes..get_slopes_power_of_2r'r)rdrr rr is_integerrr)rrclosest_power_of_2 get_slopesrr rs z!alibi_pattern..get_slopesrrr2)rdrTensorrr expandr=)rrmaxpos attn_headsslopesalibirrr alibi_patterns r num_headsseq_lencCt||d}||SN)rr~)r make_layoutrr~rconfigrrr quick_fixed_layout  rcCrr)rrrrrr quick_variable_layoutrrcCrr)rrrrrr quick_bigbird_layoutrrcCrr)rrrrrr quick_bslongformer_layoutrrrcCst|t||S)z create a pattern of shape [heads, seq, seq] out of a blocksparse layout of shape [heads, seq/block_size, seq/block_size] )rkronrS)rr~rrr layout_to_pattern"sr)rr)r)r'))rtypingrnumpyr8r-xformers.components.attention.sparsity_configrrrrrr$r)r,r-rBrrHrdr rLrQrTrZr]r_rarbrcrqrvrrSizerrrrrrrrrr s>           1