o
    `۷i                  	   @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z
 e eZdZdee ded	e
d
ee fddZdee ded	e
d
ee fddZdS )    N)Iterable)TaskContext)BlockBlockAccessor	DataBatch)CheckpointConfigcheckpointed_idsblockstask_contextcheckpoint_configreturnc                 #   sj    ddl m} |||jt  dtdtf fdd}| D ]}||}t|}| dkr2|V  qdS )zbFor each block, filter rows that have already been checkpointed
    and yield the resulting block.r   BatchBasedCheckpointFilterblockr   c                       j |  dS )N)r   r   )filter_rows_for_block)r   r   ckpt_filter N/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/data/checkpoint/util.py	filter_fn      z6filter_checkpointed_rows_for_blocks.<locals>.filter_fnN)%ray.data.checkpoint.checkpoint_filterr   kwargsCHECKPOINTED_IDS_KWARG_NAMEr   r   	for_blocknum_rows)r	   r
   r   r   r   r   filtered_blockbar   r   r   #filter_checkpointed_rows_for_blocks   s   

r   batchesc                 #   sT    ddl m} |||jt  dtdtf fdd}| D ]	}||}|V  qdS )zdFor each batch, filter rows that have already been checkpointed
    and yield the resulting batches.r   r   batchr   c                    r   )N)r!   r   )filter_rows_for_batch)r!   r   r   r   r   :   r   z7filter_checkpointed_rows_for_batches.<locals>.filter_fnN)r   r   r   r   r   )r    r
   r   r   r   r!   filtered_batchr   r   r   $filter_checkpointed_rows_for_batches,   s   
r$   )loggingtypingr   4ray.data._internal.execution.interfaces.task_contextr   ray.data.blockr   r   r   ray.data.checkpoint.interfacesr   	getLogger__name__loggerr   r   r$   r   r   r   r   <module>   s2    

