o ۾if!@sLUddlmZddlmZddlZddlZddlmZddl m Z eGdddZ e e Z eed<d ed ed ed efd dZdededed efddZde deded e fddZ d*dedejdeded ede eeBdBd ee dBe dBffddZdejd ed ejfd!d"Zd#e d$e d e fd%d&Zde e d'e d e e fd(d)ZdS)+) dataclass) TypeAliasN)ParallelConfig)CommonAttentionMetadatac@s>eZdZUeed<eed<defddZedefddZ dS) UBatchSlice request_slice token_slicereturncCs |jj|jjkp|jj|jjkSN)rstartstoprselfrO/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/ubatch_utils.pyis_emptyszUBatchSlice.is_emptycCs|jj|jjSr )rr r r rrr num_tokensszUBatchSlice.num_tokensN) __name__ __module__ __qualname__slice__annotations__boolrpropertyintrrrrrr s r UBatchSlicesorig_num_tokenspadded_num_tokens num_ubatchesr cCs|||d|kS)Nr)rrrrrris_last_ubatch_empty sr configruniform_decodecCs"|jsdS|r ||jkS||jkS)NF) use_ubatchingdbo_decode_token_thresholddbo_prefill_token_threshold)r!rr"rrrcheck_ubatch_thresholds&s   r& ubatch_slicesnum_total_tokensnum_reqs_paddedcCs<|d}t|jj|}t|jj|}|ddt||gS)N)rrr rr)r'r(r) last_slicepadded_last_request_slicepadded_last_token_slicerrr_pad_out_ubatch_slices3s  r. should_ubatchnum_scheduled_tokensnum_tokens_padded split_pointcs|sdSdurt||fddtd|D}tjt|dtjd}tj|tj|dddg}d} ||dg} | D],} t| | } ttj|| d d d} ttj|| d d }t| |}| t || | } q@t |||}t d d |D|ksJ||fS)N)NNcsg|]}|qSrr).0ir2rr Msz.maybe_create_ubatch_slices..r)dtype)r7outrr*right)sideleftcss|]}|jVqdSr )r)r3srrr psz-maybe_create_ubatch_slices..) rrangenpzeroslenint32cumsumr searchsortedappendrr.sum)r/r0r1r)rr2token_split_points cu_num_tokensr' start_token all_points end_tokenr req_startreq_stop req_sliceubatch_slices_paddedrr5rmaybe_create_ubatch_slices?s,   rPquery_start_locrcCs||j|jd||jS)z Creates a new query_start_loc that corresponds to the requests in request_slice. Note: This function creates a new tensor to hold the new query_start_locs. This will break cudagraph compatibility. r)r r )rQrrrrslice_query_start_locsus rR ubatch_slice attn_metadatac Cs|r Jd|d|j}|j}|j}|j}|j}|jd}|jd}|||kr8||dks=JdJd|||k} |||ddk} t||} t|j|} t| dksgJdt| | r|||} | dd| 8<| dd| 8<|j |}|j |}| r||d|j} | d| 8<| d| 8<| }| }|d| 8<|d| 8<t | }|j|}|j|j}|j|j}t t t| dd| dd}|d kr|j}|j|}|j|}t| | |||||||||d S) z{ This function creates a new CommonAttentionMetadata that corresponds to the requests included in ubatch_slice z Ubatch slice z is emptyrz*Token slice start outside of first requestz3query_start_loc must have at least 2 elements, got Nr*r) rQquery_start_loc_cpuseq_lensnum_reqsnum_actual_tokens max_query_len max_seq_lenblock_table_tensor slot_mapping _seq_lens_cpu_num_computed_tokens_cpu)rrrrVr r rRrQrArW seq_lens_cpuclonermaxnum_computed_tokens_cputorchabsitemrZr\r]r)rSrTrr start_locs first_req first_toklast_reqlast_toksplits_first_requestsplits_last_requestrVrQtokens_skippedrWr`r[rc num_requestsrYrZr\r]rrr_make_metadata_with_slicesx             &  rpcommon_attn_metadatacCs"g}|D] }|t||q|S)z Creates a new CommonAttentionMetadata instance that corresponds to the requests for each UBatchSlice in ubatch_slices. Note: This function does not modify common_attn_metadata )rErp)r'rqresultsrSrrrsplit_attn_metadatas rsr ) dataclassesrtypingrnumpyr?rd vllm.configrvllm.v1.attention.backendrrlistrrrrr r&r.ndarraytuplerPTensorrrRrprsrrrrs        6  _