o پil9@sdUddlmZddlmZddlZddlmZddlmZzddl Z dZ Wn e y2dZ d Z Ynwda iZ eejejfed <d ejfd d ZdejjfddZGdddeZddddddd ddddddddddejdejdejdedeedeedeejdejdeejdeedeedeed eejd!eejd"eejd#eejd$eejd%eejd&eejd'eejf(d(d)Ze r>e je jjgd*e jed+ddddddd ddddddddd,dejdejdejdedeedeedeejdeejdeejdeedeed eejd!eejd"eejd#eejd$eejd%eejd&eejd'eejf&d-d.Z ddddddddddddd/ dejdejdejded0ejd1ededejdejdeejd2eded3eejd4eejd5eejd eejd!eejd"eejd#eejd$eejd%eejd&eejd'eejd6e!ejejff0d7d8Z"edddddddddddd ddd9dejdejdejded0ejd1ededejdejdeejd2eded3eejd4eejd5eejd eejd!eejd"eejd#eejd$eejd%eejd&eejd:ed;ee#d'eejd6e!ejeejff4d)Enum)OptionalN)flashinfer_api)get_cudnn_fmha_gen_moduleTF_dummy_scale_tensorsdevicecCs>t|}|durtjdg|tjddddd}|t|<|S)Ng?r dtyper)rgettorchtensorfloat32reshape)r trL/home/ubuntu/.local/lib/python3.10/site-packages/flashinfer/cudnn/prefill.py_get_dummy_scale_tensors  rstreamcCs"tdurtatt|jtS)N) _cudnn_handlecudnn create_handle set_stream cuda_stream)rrrr_create_cudnn_handlesrc@sleZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZdZdZdZdZdZdZdZdS)UIDsrrrde23456iiN)__name__ __module__ __qualname__RESERVED_INVALID_UIDQ_UIDK_UIDV_UIDACTUAL_SEQ_LENS_Q_UIDACTUAL_SEQ_LENS_KV_UIDBLOCK_TABLES_UIDBLOCK_TABLES_K_UIDBLOCK_TABLES_V_UID RAGGED_Q_UID RAGGED_O_UIDRAGGED_STATS_UID RAGGED_K_UID RAGGED_V_UIDO_UID STATS_UID Q_SCALE_UID K_SCALE_UID V_SCALE_UID S_SCALE_UID S_DESCALE_UID O_SCALE_UID S_AMAX_UID O_AMAX_UIDrrrrr)s2r)max_token_seq_qmax_sequence_kvactual_seq_lens_q block_tables page_sizebottom_right_causal_mask return_lsebatch_offsets_qbatch_offsets_obatch_offsets_kbatch_offsets_vbatch_offsets_statsoutlse o_data_typeqk_cachev_cachescalerKrLrMactual_seq_lens_kvrNrOrPrQrRrSrTrUrVrWrXrYcCs|jd}|dkr|jd|jd}}n|dkr(|jd|jd}}|dkr:|jd|jd}}n|dkrK|jd|jd}}|durT|jd} |||j||||||||du| | | f}|S)Nrrrr)shapedimr )rZr[r\r]rKrLrMr^rNrOrPrQrRrSrTrUrVrWrXrYgraph_bh_qod_qkh_kvd_vokeyrrr_sdpa_prefill_key_fnKs6      rh) heur_modes)key_fn)rKrLrMr^rNrPrQrRrSrTrUrVrWrXrYc=Csttj|j}|jd}|}|}tjst dtj |j }tj |j }tj |j }|dur7|j }tj |}|tj j ksI|tj jkrXtdkrXt dtt|\}}|dkrz|jd|jd}}|\}} }!n!|dkr|jd|jd}}|\}} }!ntd |j|jd ||||f||| ||!f|d }"|tj j ks|tj jkr'|jd d d tj jd }#|jdd d tj jd }$|jdd d tj jd }%|jdd d tj jd }&|jdd d tj jd }'|jdd d tj jd }(|#tjj|$tjj|%tjj|&tjj|'tjj|(tjj| dur=|| })|)tj j|"!|)|dkrY|dusMJd|jd|jd}*}+n|dkrl|jd|jd}*}+ntd|j|dkr|\}} }!|jd||*||f|*||| ||!f|d },| dur|| }-|-tj"j|,!|-|dksJd|\}} }!|jd||*||+f|*|+|| ||!f|d }.|dur||}/|/tj#j|.!|/n|dkr |jd|j||d },|jd|j||d }.|"tj$j|,tj%j|.tj&j|durJ|'|jdd|jdd}0||0}1|1tj(j||0}2|2tj)j|dur`||}3|3*d|3tj+j|durv||}4|4*d|4tj,j|duo~|du}5|tj j-ks|tj j.kr|j/d|"|,|.|dur|3nd|dur|4nd|5|| | |dur|1nd|dur|2nd|dur|ndtj jd\}6}7n|tj j ks|tj jkrc|j0d/id |"d|,d|.d|#d|$d |%d!|&d"|'d#|(d$d%d&|d'| d(|5d)|dur |3ndd*|dur|4ndd+|dur|1ndd,|dur'|2ndd-|dur1|nd\}6}7}8}9|8tj1j2d.3d 4d 5tj j|9tj6j2d.3d 4d 5tj j| dury|| }:|:tj7j|6!|:|dur||};|;tj8j|7!|;|6tj9j2d%3||||+g4||+||+|+|dg5|| r|7tj:j2| 5tj j3|||dg4||d|dg|"|,|.|6g}<| r|<;|7|dur|<;|3|dur|<;|4||rArBappend)=rZr[r\r]rKrLrMr^rNrPrQrRrSrTrUrVrWrXrYhandlerb graph_s_qo graph_s_kvcudnn_q_data_typecudnn_k_data_typecudnn_v_data_typecudnn_o_data_typeg_rcrds_strideh_strided_stridecudnn_q cudnn_q_scale cudnn_k_scale cudnn_v_scale cudnn_s_scalecudnn_s_descale cudnn_o_scaleragged_qrerf cudnn_k_cacheragged_k cudnn_v_cacheragged_vnd_block_tablescudnn_k_block_tablescudnn_v_block_tablescudnn_actual_seq_lens_qcudnn_actual_seq_lens_kv padding_maskOStatsamax_samax_oragged_o ragged_statstensors_to_returnrrr_build_prefill_graphs                                             #               &r) rNrnrorprRrSrTrUrVrWrXrYworkspace_buffermax_token_per_sequencecausalrnrorpreturncCstdid|d|d|d|d|d|d|d|d | d | d | d |d |d|d|d|d|d|d|\}}tjj|tjj|tjj|tjj|i}|dur\||tjj<|durf||tjj<|durp||tj j<|durz||tj j<|dur||tj j<|dur||tj j<| dur| |tj j<| |tjj<| r||tjj<|dur||tjj<| durt|j}| |tjj<||tjj<||tjj<||tjj<| dur| |tjj<|dur||tjj<ttj|j}|j|||d| r||fS|dfS)NrZr[r\r]rKrLrMr^rNrPrQrRrSrTrUrVrWrXrY) workspacerr)rrr4rr5r6rAr7r8r<r=r?r@r:r;rBr>rr rCrFrGrHrDrErr rrexecute)rZr[r\r]rrrLrMr^rNrrQrnrorprRrSrTrUrVrWrXrYrtensorsvar_mapdummy_scale_tensorrrrr_batch_prefill_with_kv_caches                      r)rNrnrorprRrSrTrUrVrWrXis_cuda_graph_compatiblebackendrYrrc"Cs@|jd}|jd}|dkr|jd|jd}}n|dkr-|jd|jd}}|dkr9|jd}n |dkrD|jd}| rV|durVtj||||jtjd}|durf|j|||fkrftd|durm|j}|dur|||f}tj||j|d}tr|d krt d'id |d |d |d |d|d|d|d|d|d| d| d| d| d| d|d|d|d|d|d|d|d|d |S| sJd!|d"kr| dus|d#kr| dusJd$|dur|}|j |jd%d&}|j |jd%d&} t j }!|!|||||||||||| | | | ||dddd|||fS)(a3 Performs batched prefill attention with paged KV cache using cuDNN. Args: q: Query tensor of shape (Total number of tokens, num_heads_qo, head_dim) k_cache: Key cache tensor of shape (total_num_pages, num_heads_kv, page_size, head_dim) if paged kv cache is enabled else (Total sequence length of kv, num_heads_kv, d_qk) v_cache: Value cache tensor of shape (total_num_pages, num_heads_kv, page_size, head_dim) if paged kv cache is enabled else (Total sequence length of kv, num_heads_kv, d_vo) scale: Scaling factor for attention scores, typically 1/sqrt(head_dim) workspace_buffer: Workspace buffer for cuDNN operations. Scales with batch size. 128 MB should be sufficient for most cases max_token_per_sequence: Maximum number of tokens per query sequence (s_qo_max) max_sequence_kv: Maximum number of tokens per key/value sequence (s_kv_max) actual_seq_lens_q: Actual number of tokens per query sequence shape (batch_size,) on cpu or device (cpu if cuda_graph is False) actual_seq_lens_kv: Actual sequence lengths for key/values per batch, shape (batch_size,) on CPU or device (cpu if cuda_graph is False) block_tables: Page table mapping for KV cache, shape (batch_size, num_pages_per_seq) on GPU causal: Whether to apply causal masking return_lse: Whether to return log-sum-exp values (must be True) out: Optional pre-allocated output tensor lse: Optional pre-allocated tensor for log-sum-exp values if return_lse is True else returns None is_cuda_graph_compatible: Whether the prefill operation is compatible with CUDA graph q_scale: Optional scale tensor for query tensor of shape (1, 1, 1, 1) on GPU k_scale: Optional scale tensor for key tensor of shape (1, 1, 1, 1) on GPU v_scale: Optional scale tensor for value tensor of shape (1, 1, 1, 1) on GPU batch_offsets_q: Optional batch offsets for query tensor of shape (batch_size,) on GPU batch_offsets_o: Optional batch offsets for output tensor of shape (batch_size,) on GPU batch_offsets_k: Optional batch offsets for key tensor of shape (batch_size,) on GPU batch_offsets_v: Optional batch offsets for value tensor of shape (batch_size,) on GPU o_data_type: Optional data type for output tensor Returns: Output tensor of shape (batch_size * seq_len_q, num_heads_qo, head_dim) If return_lse is True, also returns log-sum-exp tensor of shape (batch_size, seq_len_q, num_heads_qo) Note: Query and KV heads can have different sizes (num_heads_qo >= num_heads_kv) When using cuda graph, actual_seq_lens_q and actual_seq_lens_kv must be on the same device as q Head dimension of query and key must be 128 or 192 Head dimension of value and output must be 128 rrrrr_Nr zAlse must have shape (num_sequences, max_token_per_sequence, h_qo)cubinrZr[r\r]rrrLrMr^rNrrQrnrorprRrSrTrUrVrWrXrYz)Currently only supports return_lse = TrueziCurrently only supports if d_qk = 192 and block_tables is None or d_qk = 128 and block_tables is not NoneT) non_blockingr) r`rar emptyr rrr CUDNN_AVAILABLErtorprefill)"rZr[r\r]rrrLrMr^rNrrQrnrorprRrSrTrUrVrWrXrrrY num_tokens num_sequencesrcrdrf out_shapeactual_seq_lens_q_gpuactual_seq_lens_kv_gpurun_funcrrr!cudnn_batch_prefill_with_kv_cache1s B               r)%enumrtypingrr api_loggingrutilsrrr Exceptionrrdictr Tensor__annotations__rrStreamrrfloatintboolr rhjit heur_modeA graph_cachertuplerstrrrrrrs      (      9      S      c