o پi|@sRddlmZddlmZddlmZ ddlZddlZddlZddl Z ddl m Z ddl m Z mZddlmZmZmZmZmZmZmZddlZeeZddlmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/dd l0m1Z1m2Z2erdd l3m4Z4Gd ddZ5GdddZ6d5ddZ7d5ddZ8d6ddZ9d7d8ddZ:d9d#d$Z;d:d)d*ZZ?e?@e$e5gd.dd/d0e?@e$e5gd.dd/d0e?@e$e5gd1dd/d0e?@e$e5gd2dd/d0e?@e$e5gd3dd/d0e?AeBe?Ce&e5gd4dd/d0dSdS);) annotations)CacheInitParams)convert_to_bigram_keyN) defaultdict) lru_cachepartial) TYPE_CHECKINGAnyIteratorListOptionalTupleUnion) MEDIUM_GPUAllBlocksCleared BlockRemoved BlockStored)BasePrefixCache EvictParams EvictResult InsertParams InsertResultMatchPrefixParams MatchResult)EvictionStrategy FIFOStrategy FILOStrategy LFUStrategy LRUStrategy MRUStrategyPriorityStrategy) get_hash_strhash_str_to_int64)Reqc@sDeZdZ  ddd d Zdd dZdddZdddZdddZdS) RadixKeyNF token_ids List[int] extra_key Optional[str] is_bigramboolcCs||_||_||_dSN)r%r'r))selfr%r'r)r-T/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/mem_cache/radix_cache.py__init__Ds zRadixKey.__init__returnintcC t|jSr+)lenr%r,r-r-r.__len__Q zRadixKey.__len__ Iterator[int]cCr2r+)iterr%r4r-r-r.__iter__Tr6zRadixKey.__iter__idxUnion[int, slice] 'RadixKey'cCs0t|trt|j||jSt|j|g|jSr+) isinstanceslicer$r%r')r,r:r-r-r. __getitem__Ws zRadixKey.__getitem__strcCs>|jdd}d|jd|t|jdkrddSddS)N zRadixKey(extra_key=z , token_ids=z...))r%r'r3)r,previewr-r-r.__repr__\s0zRadixKey.__repr__)NF)r%r&r'r(r)r*)r0r1)r0r7)r:r;r0r<)r0r@)__name__ __module__ __qualname__r/r5r9r?rEr-r-r-r.r$Cs  r$c@sleZdZdZdd ddZed d Zed d Zd dZddZ d!ddZ e ddd"ddZ d#ddZ dS)$TreeNoderNid Optional[int]priorityr1cCsztt|_d|_d|_d|_d|_t|_ t|_ d|_ d|_ d|_ d|_||_|dur1tjn||_tjd7_dSNr)rrIchildrenparentkeyvaluelock_reftime monotoniclast_access_time creation_time hit_counthost_ref_counter host_value hash_valuerLcounterrJ)r,rJrLr-r-r.r/es   zTreeNode.__init__cCs |jduSr+)rRr4r-r-r.evicted| zTreeNode.evictedcCs |jduSr+)rZr4r-r-r.backupedr^zTreeNode.backupedcCs|jd7_dS)z%Protect the host value from eviction.rNN)rYr4r-r-r. protect_hostszTreeNode.protect_hostcCs$|jdkr|jd8_dStd)z2Release the host value, allowing it to be evicted.rrNz'Host reference counter is already zero.N)rY RuntimeErrorr4r-r-r. release_hosts zTreeNode.release_hostr0r(cCs&|jdus t|jdkrdS|jdS)z5Returns the hash value of the last page in this node.Nr)r[r3r4r-r-r.get_last_hash_values zTreeNode.get_last_hash_valuerN)maxsizenode List[str]cCs(|dus |jdur gS||j|jSr+)r[get_prefix_hash_valuesrP)r,rfr-r-r.rhszTreeNode.get_prefix_hash_valuesother 'TreeNode'cCs |j|jkSr+)rV)r,rir-r-r.__lt__s zTreeNode.__lt__Nr)rJrKrLr1)r0r()rfrIr0rg)rirj)rFrGrHr\r/propertyr]r_r`rbrdrrhrkr-r-r-r.rIas     rIkey0key1cCs(|j|jkrtd|jd|jdS)NzG_key_match should be run on the same extra key, but got key0.extra_key=z != key1.extra_key=)r' ValueError)rnror-r-r._check_extra_keys rqcCs@t||d}t|j|jD]\}}||kr|S|d7}q|SrM)rqzipr%)rnroik0k1r-r-r._key_match_page_size1s  rv page_sizer1cCsft||tt|t|}d}||kr1|j||||j|||kr) |S||7}||ks|Srl)rqminr3r%)rnrorwmin_lenrsr-r-r._key_match_pageds $rzrNrQcCs>|dkr |jd}n t|jd|}|jdur|S|j|fS)NrNr)r%tupler')rQrw plain_keyr-r-r. get_child_keys    r}rfrjr0rgcCsg}d}|jdur%|jjdur%t|jjdkr%t|jjdkr%|jjd}tdt|j|D]}|jj|||}|s=q.t||d}|||}q.|S)zCompute SHA256-based hash values for position-aware identification. Args: node: The TreeNode to compute hash values for page_size: The page size for chunking tokens Returns: List of SHA256 hex strings, one per page Nrrc) prior_hash)rPr[r3rQranger%r!append)rfrw hash_values parent_hashstart page_tokenshash_valr-r-r.compute_node_hash_valuess     rchild_hash_valueOptional[List[str]] split_len/tuple[Optional[List[str]], Optional[List[str]]]cCsB|durdS|dkr |}n||}|d|}||d}||fS)apSplit hash_value between parent and child nodes during node splitting. Args: child_hash_value: The hash_value list from the child node being split split_len: The length at which to split (in tokens) page_size: The page size for calculating number of pages Returns: Tuple of (new_node_hash_value, updated_child_hash_value) N)NNrNr-)rrrw split_pages new_node_hash child_hashr-r-r.split_node_hash_values   rc@s(eZdZd]ddZe    d^d_ddZddZ d`daddZdbddZdcd!d"Z ddd$d%Z dedfd*d+Z dgdhd,d-Z d.d/Z d0d1Zdid4d5Zdjd8d9Zdjd:d;Zdd?Zd@dAZdkdBdCZdldFdGZdmdndJdKZdodMdNZdOdPZdjdQdRZdSdTZdjdUdVZdjdWdXZdYdZZd[d\ZdS)p RadixCacheparamsrcCsL|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_g|_ |j r,| |jr5|jj |_ nt d|_ |jdkrGt|_t|_ntt|jd|_tt|jd|_|jdkrat|_n;|jdkrkt|_n1|jdkrut|_n'|jdkrt|_n|jdkrt|_n|jd krt|_n td |jd t|_|dS) NcpurN)rwlrulfufifomrufilorLzUnknown eviction policy: zF. Supported policies: 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority'.)disablereq_to_token_pooltoken_to_kv_pool_allocatorrwenable_kv_cache_eventsis_eagledisable_finished_inserteviction_policylowerkv_event_queueenable_metricsinit_metrics_collectordevicetorchrv key_match_fnr}get_child_key_fnrrzreviction_strategyrrrrr rpsetevictable_leavesreset)r,rr-r-r.r/sH                  zRadixCache.__init__FNrNrr*mock_allocator Optional[Any]rwr1rr0cCst|d|||d}t|S)z?Init a radix cache without memory pools for simulation purpose.N)rrrrwr)rr)r,rrrwrrr-r-r.create_simulated5s zRadixCache.create_simulatedcCsbttj d|_tgdd|j_g|j_g|j_d|j_g|j_ d|_ d|_ |j |dS)NrLr%r'rNr)rIsysre root_noder$rQrRrZrSr[evictable_size_protected_size_rclear_record_all_cleared_eventr4r-r-r.rIs  zRadixCache.resetrQr$rROptional[torch.Tensor]'Tuple[RadixKey, Optional[torch.Tensor]]cCs8|jr|jst|j|_|dur|dt|}||fSr+)rr)rr%r3)r,rQrRr-r-r.maybe_bigram_convertVs  zRadixCache.maybe_bigram_convertrrcs|j}|\}}fdd}jst|dkr|Sjdkr1t|jj}|d|}t|dkr:|Sj|\}}|rKt|}n tj dtj j d}t |||dS) aFind the longest cached prefix of ``key`` in the radix tree. The logical namespace for prefix matching is determined by both the token id sequence and the optional ``extra_key`` carried by ``RadixKey``. Entries that share identical leading token ids but have *different* ``extra_key`` values are intentionally kept disjoint and never share prefix nodes. This is useful to: * Isolate KV cache lines for different LoRA / adapter IDs. * Separate requests that intentionally should not share state (e.g., different sampling salt, cache version, or retrieval augmentation context) by supplying a distinct ``extra_key``. Args: params (MatchPrefixParams): Parameters containing the lookup key with a list of token ids and an optional ``extra_key`` namespace tag. If ``page_size > 1`` the length is internally truncated to a multiple of ``page_size`` before matching. Passing an empty key returns an empty result with the root as the last node. Returns: MatchResult: ``device_indices`` is a 1-D ``torch.int64`` tensor of the concatenated KV cache indices corresponding to the longest cached prefix (may be length 0). ``last_device_node`` and ``last_host_node`` (currently the same) are the tree node objects representing the terminal node of the matched prefix. This method may mutate internal structure by splitting an existing node if the match ends inside a stored segment. Internal updates: * Refreshes access metadata (timestamps) used by the configured eviction strategy. * If the lookup ends inside a stored segment the node is split once to expose a precise boundary; this structural refinement improves subsequent match efficiency and does not duplicate data. cs"ttjdtjjdjjdS)Nrdtyperdevice_indiceslast_device_nodelast_host_node)rremptyint64rrr-r4r-r.empty_match_resultsz3RadixCache.match_prefix..empty_match_resultrrNNrrr) rQrrr3rw_match_prefix_helperrrcatrrrr)r,rrQ_rpage_aligned_lenrR last_noder-r4r. match_prefix`s&%      zRadixCache.match_prefixrrcCsh|jrtddS|j}|j}|j}|durtj|jtjd}| ||\}}| |j |||}t|dS)Nr) prefix_len)r) rrrQrRrLrtensorr%rr_insert_helperr)r,rrQrRrLrr-r-r.inserts  zRadixCache.insertlistcCs.|jdkr|St||j|j}|d|S)NrN)rwr3)r,rQrr-r-r._page_align_keyss  zRadixCache._page_align_keysTreqr# is_insertc Cs>|jrd}|}|jr|jj|jd|f}|j|dS|j|j d|}|jj|jdt |f}|j r=t |n|}| |}|dt |jtjdd}t||j|j d}|r~t|ddpdd} |t||| d} | j} |j||j| n |j||jt ||j|t |d||jdS) zCache request when it finishes.FNTrcopyr)rLr)rQrRrL)rpop_committed_kv_cacherr req_to_token req_pool_idxrfreeorigin_input_ids output_idsr3rrrtorrr$r'getattrrrrcache_protected_len dec_lock_refr) r,rrkv_committed_len kv_indicesr%keysvalues radix_keyrLresultnew_prefix_lenr-r-r.cache_finished_reqs>     zRadixCache.cache_finished_reqc Cs|jrdS|j}|jj|jdt|f}|jrt|n|}||}|dt|j t j dd}t ||j |jd}|t|||t|ddpGdd}|j} |j||j| |t|d} | j| j} } t| t|ks~Jd t| d t||j|jt|jt| f| |jdt| |_||j|| t| t|krt | |t| dg|_n| |_| |_dS) z$Cache request when it is unfinished.NTrrrLr)rQrRchunkedrLrQzlen(new_indices)=z , len(keys)=) rfill_idsrrrr3rrrrrrr$r'rrrrrrrrrrrwriter>rr inc_lock_refrprefix_indices) r,rrr%rrrrrr match_result new_indices new_last_noder-r-r.cache_unfinished_reqsP  ,    zRadixCache.cache_unfinished_reqcCs$||jdtd|dS)Nrz #tokens: ) _print_helperrprint total_sizer4r-r-r. pretty_print.szRadixCache.pretty_printcCs|Sr+)_total_size_helperr4r-r-r.r2szRadixCache.total_sizerrc sjrtSt}|j}tj}fdd|D}t|d}||krot |rot |\}}j |j |t |j 7}|t |jjdkrb|jjdkrbj|j} t|| |jf|||krot |s*||t|dS)Ncsg|] }j||fqSr-)r get_priority).0rfr4r-r. <sz$RadixCache.evict..r)num_tokens_evicted)rrrT perf_counter num_tokensrrheapqheapifyr3heappoprrrR _delete_leafrPrOrSrrheappush_record_remove_eventupdate_eviction_metrics) r,r start_timerleaves eviction_heap num_evicted _priorityx new_priorityr-r4r.evict5s,      zRadixCache.evictrfrIcCs|jrdSd}||jkr@|jdkr,|jt|j8_|jt|j7_|t|j8}|jd7_|||j}||jks |SrM rrrSrr3rQr_update_leaf_statusrPr,rfdeltar-r-r.rRs    zRadixCache.inc_lock_refcCs|jrdSd}||jkrN|jdkr,|jt|j7_|jt|j8_|t|j7}|jd8_|||jdurF||jusFJd|j}||jks |S)NrrNz-This request holds the node from another treer r r-r-r.ras"       zRadixCache.dec_lock_refcC|jSr+)rr4r-r-r.evictable_sizetszRadixCache.evictable_sizecCr r+)rr4r-r-r.protected_sizewszRadixCache.protected_sizecs(gdfdd |jtS)NrfrIcs,|jD]\}}|j|qdSr+)rOitemsrrR)rfrchild _dfs_helperrr-r.r~s  z2RadixCache.all_values_flatten.._dfs_helperrfrI)rrrr4r-rr.all_values_flatten{s  zRadixCache.all_values_flattenc Cst}||_||}g}t|dkrj||jvrj|j|}||_||j|}|t|jkrF| |j||}| |j |} ||fS| |j |}||d}t|r]||}t|dkrj||jvs||fSrl) rTrUrVrr3rOrrrQ _split_noderrR) r,rfrQ access_time child_keyrRrrnew_noder-r-r.rs*      zRadixCache._match_prefix_helperrrcCst|jd}|||d|i|_|j|_|j|_|jd||_|jd||_||_|j|d|_|j|d|_||jj||<t |j ||j \|_ |_ |S)Nr) rIrLrrOrPrSrQrRclonerr[rw)r,rQrrrr-r-r.rs   zRadixCache._split_noderrLc Csp|durd}t}||_t|j||_t|dkrdS||}d}t|dkr||jvr|j|}||_| |j |}||7}||d}||d}|t|j krh| |j ||} t| j|| _| }nt|j||_t|rx||}t|dkr||jvs0t|rt |d} || _ || _ || _| |j|<|jt|7_|||| || |S)Nrr)rTrUrVmaxrLr3rrOrrrQrrIrPrrRrr _record_store_event) r,rfrQrRrLrrtotal_prefix_lengthrrr-r-r.rsF            zRadixCache._insert_helperindentcCs||fg}|rP|\}}td|t|j|jjddd|j|jD]#\}}|||df|| |jksKJd|d| |jq(|sdSdS)z1Prints the radix tree in a human-readable format. NrAzr=zkey=z#, self.get_child_key_fn(child.key)=) poprr3rQr%rSrOrrr)r,rfrstack current_nodecurrent_indentrQrr-r-r.rs"    zRadixCache._print_helpercCsl||j}|jj|d}||ksJd||jt|j8_||jvr.|j|| |jdS)Nz parent does not have child key, ) rrQrPrOr!rr3rremover )r,rfrQvr-r-r.rs   zRadixCache._delete_leafcCsz|js|jdkr||jvr|j|dS|jD]}|js-||jvr*|j|dSq||jvr;|j|dSdSrl)r]rSrr%rOradd)r,rfrr-r-r.r s     zRadixCache._update_leaf_statuscCsPd}|jg}|r&|}|t|j7}|jD] }|jrq||q|s|Srl)rr!r3rRrOrr]r)r,rr"r#rr-r-r.rs zRadixCache._total_size_helperc Cs|jrm|jdurt||j|_d}|jdur2|j|jkr2|jjdur2t|jjdkr2t|jjd}d}tdt|j |jD]0}|j j |||j}|sNq>t|j|}|j t |g||t|dtd|}|d7}q>dSdS)Nrrc) block_hashesparent_block_hashr% block_sizelora_idmediumrN)rr[rrwrPrr3r"rrQr%rrrr)r,rfr) page_indexrr block_hashr-r-r.rs8   zRadixCache._record_store_eventcCs|jrB|jdurt||j|_d}tdt|j|jD](}|jj|||j}|s+qt|j|}|j t |gt d|d7}qdSdS)Nr)r(r,rN) rr[rrwrr3rQr%r"rrrr)r,rfr-rrr.r-r-r.rAs   zRadixCache._record_remove_eventcCs|jr |jtdSdSr+)rrrrr4r-r-r.rVsz$RadixCache._record_all_cleared_eventcCs|jsgS|j}g|_|S)zsAtomically takes all events and clears the queue. Returns: A list of KV cache events. )rr)r,eventsr-r-r. take_eventsZs zRadixCache.take_events)rr)FNrNF) rr*rrrwr1rr*r0rr+)rQr$rRrr0r)rrr0r)rrr0r)rQrr0r)T)rr#rr*)F)rr#)rrr0rr)rfrIrQr$)rQr$rrIrr1r)rfrIrQr$rLr1)rfrIrr1) rFrGrHr/ classmethodrrrrrrrrrrrrrrrrrrrrrr rrrrr0r-r-r-r.rsF / H   /A      -  & r__main__)rNr rr)rNr )rNr r4r5) rA )rNr r3 )rnr$ror$)rnr$ror$rwr1)rN)rQr$rwr1)rfrjrwr1r0rg)rrrr1rwr1r0r)D __future__r&sglang.srt.mem_cache.cache_init_paramsrsglang.srt.mem_cache.utilsrrloggingrrT collectionsr functoolsrrtypingrr r r r r rr getLoggerrFlogger#sglang.srt.disaggregation.kv_eventsrrrr&sglang.srt.mem_cache.base_prefix_cacherrrrrrr!sglang.srt.mem_cache.evict_policyrrrrrrr $sglang.srt.mem_cache.hicache_storager!r""sglang.srt.managers.schedule_batchr#r$rIrqrvrzr}rrrrtreerrrrr-r-r-r.sf   $ $$    ?  ! f