o
    `۷iH                     @   sJ  d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZd dl	m
Z
 d dlmZmZmZ d dlmZ eeZe Ze Ze Zded	efd
dZG dd deZG dd deZG dd deZdefddZdefddZG dd dZ G dd dZ!ej"ej#fdeded	e!fddZ$dd  Z%	!	"	#	$	d'd	efd%d&Z&dS )(    N)defaultdict)Enum)List)
node_stats)ActorIDJobIDTaskID)node_stats_to_dict
object_refreturnc                 C   s"   |  dr
t| S tjj| S )a  Decode objectRef bytes string.

    gRPC reply contains an objectRef that is encodded by Base64.
    This function is used to decode the objectRef.
    Note that there are times that objectRef is already decoded as
    a hex string. In this case, just convert it to a binary number.
    =)endswithbase64standard_b64decoderay_commonutilshex_to_binary)r
    r   P/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/dashboard/memory_utils.pydecode_object_ref_if_needed   s   

r   c                   @   s   e Zd ZdZdZdZdS )SortingType         N)__name__
__module____qualname__PIDOBJECT_SIZEREFERENCE_TYPEr   r   r   r   r   %   s    r   c                   @   s   e Zd ZdZdZdS )GroupByTypenodestack_traceN)r   r   r   NODE_ADDRESSSTACK_TRACEr   r   r   r   r!   +   s    r!   c                   @   s$   e Zd ZdZdZdZdZdZdZdS )ReferenceTypeACTOR_HANDLEPINNED_IN_MEMORYLOCAL_REFERENCEUSED_BY_PENDING_TASKCAPTURED_IN_OBJECTUNKNOWN_STATUSN)	r   r   r   r'   r(   r)   r*   r+   r,   r   r   r   r   r&   0   s    r&   sort_byc                 C   s:   |   } | dkrtjS | dkrtjS | dkrtjS td)z0Translate string input into SortingType instancer   r   r    z\The sort-by input provided is not one of                PID, OBJECT_SIZE, or REFERENCE_TYPE.)upperr   r   r   r    	Exception)r-   r   r   r   get_sorting_type:   s   r0   group_byc                 C   s,   |   } | dkrtjS | dkrtjS td)z0Translate string input into GroupByType instancer$   r%   zUThe group-by input provided is not one of                NODE_ADDRESS or STACK_TRACE.)r.   r!   r$   r%   r/   )r1   r   r   r   get_group_by_typeJ   s   r2   c                   @   sz   e Zd ZdedededefddZdefdd	Zd
e	defddZ
defddZdefddZdd Zdd Zdd ZdS )MemoryTableEntryr
   node_address	is_driverpidc                C   s   || _ || _|| _|dd| _| jdkrd| _t|ddd | _t|dd	| _|d
d| _t	| jdkr=d| _t
t|d | _t|dd| _t|dd| _t|dd| _dd |dg D | _|  | _d S )N
taskStatus?NIL-attemptNumberr   r   
objectSizecallSitez	<Unknown>disabledobjectIdlocalRefCountpinnedInMemoryFsubmittedTaskRefCountc                 S   s   g | ]	}t t|qS r   )r   	ObjectRefr   .0r
   r   r   r   
<listcomp>r   s    z-MemoryTableEntry.__init__.<locals>.<listcomp>containedInOwned)r5   r6   r4   gettask_statusintattempt_numberobject_size	call_sitelenr   rD   r   r
   local_ref_countboolpinned_in_memorysubmitted_task_ref_countcontained_in_owned_get_reference_typereference_type)selfr
   r4   r5   r6   r   r   r   __init__Y   s*   


zMemoryTableEntry.__init__r   c                 C   s>   | j s| jdkr| jdkrt| jdkrdS | j rdS dS )Nr   FT)rR   rP   rS   rO   rT   r
   is_nilrW   r   r   r   is_validx   s   


zMemoryTableEntry.is_validgroup_by_typec                 C   s0   |t jkr| jS |t jkr| jS td| d)Nzgroup by type  is invalid.)r!   r$   r4   r%   rN   
ValueError)rW   r\   r   r   r   	group_key   s
   

zMemoryTableEntry.group_keyc                 C   s`   |   rtjjS | jrtjjS | jdkrtjjS | jdkr!tj	jS t
| jdkr,tjjS tjjS )Nr   )_is_object_ref_actor_handler&   r'   valuerR   r(   rS   r*   rP   r)   rO   rT   r+   r,   rZ   r   r   r   rU      s   

z$MemoryTableEntry._get_reference_typec                 C   sV   | j  }tt d }tt d }|d | }||||  }|dkr)|dks)dS dS )N   ffffffffffffffffffffffffffffffffffffffffTF)r
   hexTASKID_BYTES_SIZEACTORID_BYTES_SIZEJOBID_BYTES_SIZE)rW   object_ref_hextaskid_random_bits_sizeactorid_random_bits_sizerandom_bitsactor_random_bitsr   r   r   r`      s   

z,MemoryTableEntry._is_object_ref_actor_handlec                 C   sT   | j  | j| j| j| j| j| j| j| j	| j
| jdd | jD | jr&ddS ddS )Nc                 S      g | ]}|  qS r   )re   rE   r   r   r   rG      s    z,MemoryTableEntry.as_dict.<locals>.<listcomp>DriverWorker)r
   r6   node_ip_addressrM   rV   rN   rJ   rL   rP   rR   rS   rT   type)r
   re   r6   r4   rM   rV   rN   rJ   rL   rP   rR   rS   rT   r5   rZ   r   r   r   as_dict   s$   zMemoryTableEntry.as_dictc                 C      |   S N__repr__rZ   r   r   r   __str__      zMemoryTableEntry.__str__c                 C      t |  S ru   strrs   rZ   r   r   r   rw         zMemoryTableEntry.__repr__N)r   r   r   dictr|   rQ   rK   rX   r[   r!   r_   rU   r`   rs   rx   rw   r   r   r   r   r3   X   s"    
r3   c                   @   s   e Zd Zejejfdee dedefddZ	dedefddZ
defd	d
Zdd ZdefddZdefddZdd Zdee fddZdd Zdd ZdS )MemoryTableentriesr\   sort_by_typec                 C   sV   || _ i | _tt| _|r|r| || d S |r | | d S |r)| | d S d S ru   )tablegroupr   rK   summarysetup	_group_by_sort_by)rW   r   r\   r   r   r   r   rX      s   
zMemoryTable.__init__c                 C   s4   |  || | j D ]}|  q|   | S )zSetup memory table.

        This will sort entries first and group them after.
        Sort order will be still kept.
        )r   r   r   values	summarize)rW   r\   r   group_memory_tabler   r   r   r      s
   
zMemoryTable.setupentryc                 C   s   | j | d S ru   )r   append)rW   r   r   r   r   insert_entry      zMemoryTable.insert_entryc                 C   s   d}d}d}d}d}d}| j D ]G}|jdkr||j7 }|jtjjkr'|d7 }q|jtjjkr3|d7 }q|jtjjkr?|d7 }q|jtjjkrK|d7 }q|jtj	jkrV|d7 }q||||||d| _
| S )Nr   r   )total_object_sizetotal_local_ref_counttotal_pinned_in_memorytotal_used_by_pending_tasktotal_captured_in_objectstotal_actor_handles)r   rM   rV   r&   r)   ra   r(   r*   r+   r'   r   )rW   r   r   r   r   r   r   r   r   r   r   r      s8   






zMemoryTable.summarizesorting_typec                 C   sp   |t jkr| jjdd d | S |t jkr | jjdd d | S |t jkr0| jjdd d | S td| d)Nc                 S      | j S ru   )r6   r   r   r   r   <lambda>      z&MemoryTable._sort_by.<locals>.<lambda>)keyc                 S   r   ru   )rM   r   r   r   r   r     r   c                 S   r   ru   )rV   r   r   r   r   r     r   zGive sorting type: r]   )r   r   r   sortr   r    r^   )rW   r   r   r   r   r     s   


zMemoryTable._sort_byc                 C   st   i | _ tt}| jD ]}||| | q
| D ]\}}t|ddd| j |< q| j  D ]\}}|  q/| S )zbGroup entries and summarize the result.

        NOTE: Each group is another MemoryTable.
        Nr\   r   )	r   r   listr   r_   r   itemsr   r   )rW   r\   r   r   r_   r   r   r   r   r   r     s   

zMemoryTable._group_byc                 C   s   | j dd | j D dS )Nc                 S   s"   i | ]\}}||  |jd qS ))r   r   )get_entriesr   )rF   r_   r   r   r   r   
<dictcomp>4  s    z'MemoryTable.as_dict.<locals>.<dictcomp>)r   r   )r   r   r   rZ   r   r   r   rs   1  s
   zMemoryTable.as_dictr   c                 C   s   dd | j D S )Nc                 S   rn   r   )rs   )rF   r   r   r   r   rG   >      z+MemoryTable.get_entries.<locals>.<listcomp>)r   rZ   r   r   r   r   =  r   zMemoryTable.get_entriesc                 C   rz   ru   r{   rZ   r   r   r   rw   @  r}   zMemoryTable.__repr__c                 C   rt   ru   rv   rZ   r   r   r   rx   C  ry   zMemoryTable.__str__N)r   r   r   r!   r$   r   r   r   r3   rX   r   r   r   r   r   rs   r~   r   rw   rx   r   r   r   r   r      s&    
!r   workers_statsc                 C   sv   g }| D ]-}|d }| ddk}|d }| dg }|D ]}	t|	|||d}
|
 r0||
 qqt|||d}|S )Nr6   
workerTypeDRIVER	ipAddress
objectRefs)r
   r4   r5   r6   r   )rI   r3   r[   r   r   )r   r1   r-   memory_table_entriescore_worker_statsr6   r5   r4   object_refsr
   memory_table_entrymemory_tabler   r   r   construct_memory_tableG  s*   
	r   c                 C   sV   t t}dddddd}| d D ]}|d }|d	krd
}|||d    |7  < q|S )z]Returns dictionary mapping reference type
    to memory usage for a given memory table group.r   r   r   r   r   )r)   r(   r*   r+   r'   r   rM   r=   r   rV   )r   rK   )r   d
table_namer   sizer   r   r   track_reference_sizeb  s   r   r$   r   TBc                    s  dd l }|dj}d}ddddd}	g }
|  D ]1}|d	 s qztt|d
 |d }W n	 ty6   Y qw |
|d  t|t	u rHd|v sJJ qt
|t|}}t|
|| }d|v rdd|v sfJ d}|j dd|j dd}}g d}d}g d}d}||kr|rd}|d| d| d|d ur|nd d7 }|d  D ]^\}}|d }t|}| D ]-\}}|dkrt||	|  d|  ||< qt|d|| |	|   d| d ||< q|d| d | d!7 }||j| 7 }||j|  d" 7 }|d#| d | d!7 }||j| 7 }d}|d$ D ]|d ur+||kr+ n݈d% d&krAtd% |	|  d|  nd'd%< d}||kr|rd( td) dkr`d*gd)< n fd+d,tdtd)  D d)< d-fd.d,tdtd/ D d/< ttd) td/ }n|d"7 }d0 d1 d2 d) d/ d3 d% d4 d5 g	}tt|D ]&t| ts| g|< | d6d, t|t|  D  qt|D ]fd7d,|D }||j| 7 }q|d"7 }|d7 }qq|d87 }|S )9Nr   )P         r   i  i@B i ʚ;)r   KBMBGBAliveNodeManagerAddressNodeManagerPortcoreWorkersStatsr   r    _ )zMem Used by ObjectszLocal ReferencesPinnedzUsed by taskzCaptured in ObjectszActor Handlesz/{:<19}  {:<16}  {:<12}  {:<13}  {:<19}  {:<13}
)	z
IP Addressr   Typez	Call SiteStatusAttemptSizezReference Typez
Object RefzI{:<13} | {:<8} | {:<7} | {:<9} | {:<9} | {:<8} | {:<8} | {:<14} | {:<10}
zC{:<15}  {:<5}  {:<6}  {:<22}  {:<14}  {:<8}  {:<6}  {:<18}  {:<56}
zGrouping by z...        Sorting by z...        Display allz entries per group...


r   z, ()z--- Summary for z: z ---

z--- Object references for r   rM   r=   r8      rN   r?   c                    s    g | ]}d  ||   qS )rN   r   rF   i)call_site_lengthr   r   r   rG         z"memory_summary.<locals>.<listcomp>   c                    s    g | ]} d  ||  qS )rJ   r   r   )r   task_status_lengthr   r   rG     r   rJ   rq   r6   rr   rL   rV   r
   c                 S   s   g | ]}d qS )r   r   )rF   xr   r   r   rG     s    c                    s   g | ]}|  qS r   r   )rF   elem)r   r   r   rG     r   zmTo record callsite information for each ObjectRef created, set env variable RAY_record_ref_creation_sites=1

)shutilget_terminal_sizecolumns
node_tabler	   r   RuntimeErrorextendrr   r~   r2   r0   r   rs   namelowerreplacer   r   r|   formatr   rO   rangemax
isinstancer   )stater1   r-   	line_wrapunitnum_entriesr   r   line_wrap_thresholdunitsr   rayletstatsr   memsummary_labelssummary_stringobject_ref_labelsobject_ref_stringr   r   r   ref_sizekvn	num_linesobject_ref_valuesrowr   )r   r   r   r   r   memory_summaryv  s   	


 ,

r   )r$   r   Tr   N)'r   loggingcollectionsr   enumr   typingr   r   ray._private.internal_apir   ray._rayletr   r   r   ray.dashboard.utilsr	   	getLoggerr   loggerr   rf   rg   rh   r|   bytesr   r   r!   r&   r0   r2   r3   r   r$   r   r   r   r   r   r   r   r   <module>   sN    

t}
