o
    TiM                     @   s   d dl mZ d dlmZ d dlmZmZ d dlmZ ddl	m
Z
 G dd deZG d	d
 d
eZ	 G dd deZG dd deZG dd deZdS )    )Enum)Tuple)PositiveIntmodel_validator)DeepSpeedConfigModel   )	DtypeEnumc                   @   s   e Zd ZdZ	 dZdS )KVCacheTypedenselocalN)__name__
__module____qualname__DENSELOCAL r   r   a/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/inference/v2/ragged/manager_configs.pyr	      s
    r	   c                   @   sj   e Zd ZU ejZeed< 	 dZeed< 	 dZ	e
ed< 	 ee
e
e
f ed< 	 ejZeed< 	 dZe
ed	< d
S )KVCacheConfigtype   
block_size   num_allocation_groupscache_shapecache_dtype@   max_blocks_per_allocation_groupN)r   r   r   r	   r   r   __annotations__r   intr   r   r   r   fp16r   r   r   r   r   r   r      s   
 r   c                   @   s   e Zd ZdZdZ	 dZdS )AllocationModezQ
    Helper class to describe memory allocation strategies for the KV-cache.
    reserveallocateN)r   r   r   __doc__RESERVEALLOCATEr   r   r   r   r    h   s    r    c                   @   s(   e Zd ZU ejZeed< dZeed< dS )MemoryConfigmodei ʚ;sizeN)	r   r   r   r    r$   r'   r   r(   r   r   r   r   r   r&   x   s   
 r&   c                   @   sv   e Zd ZU dZeed< 	 dZeed< 	 dZeed< 	 dZeed< 	 e	 Z
e	ed	< 	 d
Zeed< 	 edddd ZdS )DSStateManagerConfigi   max_tracked_sequencesi   max_ragged_batch_sizei   max_ragged_sequence_counti    max_contextmemory_configFoffloadafter)r'   c                 C   s,   | j | jks
J d| j | jksJ d| S )NzAmax_ragged_sequence_count must be less than max_tracked_sequenceszAmax_ragged_sequence_count must be less than max_ragged_batch_size)r,   r*   r+   )selfr   r   r   #max_ragged_sequence_count_validator   s   z8DSStateManagerConfig.max_ragged_sequence_count_validatorN)r   r   r   r*   r   r   r+   r,   r-   r&   r.   r/   boolr   r2   r   r   r   r   r)      s   
 r)   N)enumr   typingr   pydanticr   r   deepspeed.runtime.config_utilsr   inference_utilsr   r	   r   r    r&   r)   r   r   r   r   <module>   s   2