o
    bi                     @   sx   d dl mZmZmZmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ er2d dlZG dd	 d	eZdS )
    )TYPE_CHECKINGIteratorListUnion)BlockOutputBufferOutputBlockSizeOption)_check_import)Block)DataContext)FileBasedDatasourceNc                       sR   e Zd ZdZdgZdeeee f f fddZddded	e	e
 fd
dZ  ZS )AvroDatasourcez#A datasource that reads Avro files.avropathsc                    s&   t  j|fi | t| ddd d S )Nfastavro)modulepackage)super__init__r   )selfr   file_based_datasource_kwargs	__class__ a/home/ubuntu/.local/lib/python3.10/site-packages/ray/data/_internal/datasource/avro_datasource.pyr      s   zAvroDatasource.__init__fzpyarrow.NativeFilepathreturnc           	      c   s    dd l }||}t }t|jd}t|}|D ]}|| | r.|	 V  | s%q|
  | rB|	 V  | s7d S d S )Nr   )target_max_block_size)r   readerr
   get_currentr   r   r   addhas_nextnextfinalize)	r   r   r   r   r   ctxoutput_block_size_optionoutput_bufferrecordr   r   r   _read_stream   s$   



zAvroDatasource._read_stream)__name__
__module____qualname____doc___FILE_EXTENSIONSr   strr   r   r   r	   r(   __classcell__r   r   r   r   r      s    "	r   )typingr   r   r   r    ray.data._internal.output_bufferr   r   ray.data._internal.utilr   ray.data.blockr	   ray.data.contextr
   )ray.data.datasource.file_based_datasourcer   pyarrowr   r   r   r   r   <module>   s    