o ۾i<+@sdZddlZddlmZddlmZddlmZer!ddlmZddl m Z ddl m Z d d l mZmZmZmZe eZGd d d eZGd ddeZdS)z$Attention layer with FlashAttention.N) AttentionType)#is_flash_attn_varlen_func_available)%triton_reshape_and_cache_flash_diffkv)flash_attn_varlen_func) init_logger)get_kv_cache_layout)FlashAttentionBackendFlashAttentionImplFlashAttentionMetadatacascade_attentionc@seZdZUdZeed<ededdfddZede fddZ ede d fd d Z e dd edededede de edff ddZe ddede edffddZdS)FlashAttentionDiffKVBackend head_size_vreturnNcCs ||_dSN)r)clsrr`/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/attention/backends/flash_attn_diffkv.pyset_head_size_v s z+FlashAttentionDiffKVBackend.set_head_size_vcCsdS)NFLASH_ATTN_DIFFKVrrrrrget_name$z$FlashAttentionDiffKVBackend.get_namer cCstSr)FlashAttentionDiffKVImplrrrr get_impl_cls(rz(FlashAttentionDiffKVBackend.get_impl_clsauto num_blocks block_size num_kv_heads head_sizecache_dtype_str.cCs&|ddkr td||||tjfS)Nrz$Block size must be a multiple of 16.) ValueErrorr r)rrrrr rrrget_kv_cache_shape.s z.FlashAttentionDiffKVBackend.get_kv_cache_shapeFinclude_num_layers_dimensioncCsVt}|dkr |r dS|dkrd}|S|dkr|rdS|dkr#d}|Std|d) NNHD)rr)rrr&r'HND)rr'rr&r()rr&rr'zUnknown cache layout format .)rr")r$ cache_layout stride_orderrrrget_kv_cache_stride_order?s  z5FlashAttentionDiffKVBackend.get_kv_cache_stride_order)r)F)__name__ __module__ __qualname__rint__annotations__ classmethodr staticmethodstrrtypertupler#boolr-rrrrr s<    r c@sfeZdZ   ddejjdejdejdejdejdedejdBd ejdBd ejdBd ejfd d ZdS)rNlayerquerykeyvaluekv_cache attn_metadataoutput output_scaleoutput_block_scalerc  Csf|dusJd|jdusJd|dus| durtd|dur&|dS|j} |j} | tjtjfvrN||d| |d| |d| |d| ||S|dd|j f} |d|j df} |j dur{|dur{|dur{t ||||j |j |j|j|j drt|j }| |} | |} |jsP|j}|j}|j}|j}|j}|j}|jdd|jf}|jdkr|j|d| |d| |d| | | |d| ||j ||j ||j |d |S|j!durt"|j!nd}t#d+id |d| d | d | d |d| d|d|d|d|d|j$d|j%d|j&d|d|d|j'd|d|jd|j |d|j |d|j |d|j(d|j)|St*|d| |d| | | fid|jd |jd!|j+d"|j,d#|j-d$|jd|j$d|j&d%|j!d&|j'd|jd'|j.d(|j(d|jd)|j/d*|jd|jd|jd|jd|j)|S),aForward pass with FlashAttention. Args: query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size_v] kv_cache: shape = [num_blocks, block_size, num_kv_heads, head_size + head_size_v] attn_metadata: Metadata for attention. Returns: shape = [num_tokens, num_heads * head_size_v] NOTE: FP8 quantization, flash-attn expect the size of {q,k,v}_descale to be (num_sequences, num_kv_heads). We use torch's .expand() to avoid duplicating values NzOutput tensor must be provided.z$FlashAttention version not detected.zEfused output quantization is not yet supported for FlashAttentionImplr.fp8r) q_descale k_descale v_descaleqkvout cu_seqlens_q max_seqlen_q seqused_k max_seqlen_k softmax_scalecausal alibi_slopes window_size block_tablesoftcapscheduler_metadata fa_versionrCrDrE num_splitss_aux cu_query_lens max_query_lencu_prefix_query_lensprefix_kv_lenssuffix_kv_lens max_kv_lensliding_windowlogits_soft_capcommon_prefix_lenmax_num_splitsprefix_scheduler_metadatasuffix_scheduler_metadatar)0vllm_flash_attn_versionNotImplementedErrorfill_ attn_typenum_actual_tokensr ENCODER_ONLYENCODER_forward_encoder_attentionrkv_sharing_target_layer_namer slot_mappingkv_cache_dtype_k_scale_v_scale startswithr get_fp8_dtype_for_flashattnview use_cascadequery_start_locseq_lensrY max_seq_lenrRrTshaperdcp_world_size_forward_with_dcp_q_scaleexpandr^listrscalerOrPr_rasinksr rZr[r\r`rb)selfr9r:r;r<r=r>r?r@rArgrh key_cache value_cachedtyperJrLrKrMrRrT descale_shapesliding_window_sizerrrforwardXs<                                        z FlashAttentionDiffKVImpl.forward)NNN) r.r/r0torchnnModuleTensorr rrrrrrWs2    r)__doc__rvllm.v1.attention.backendr#vllm.v1.attention.backends.fa_utilsr4vllm.v1.attention.ops.triton_reshape_and_cache_flashrr vllm.loggerr vllm.v1.attention.backends.utilsr flash_attnr r r r r.loggerr rrrrrs      ;