o
    bi                     @   sX   d dl mZmZmZ d dlmZ d dlmZ d dlm	Z	 er"d dl
Z
G dd de	ZdS )    )TYPE_CHECKINGIteratorList)DelegatingBlockBuilder)Block)FileBasedDatasourceNc                       sX   e Zd ZdZdZddddee dedef fd	d
Zdddede	e
 fddZ  ZS )TextDatasourcez4Text datasource, for reading and writing text files.textFzutf-8)drop_empty_linesencodingpathsr
   r   c                   s$   t  j|fi | || _|| _d S )N)super__init__r
   r   )selfr   r
   r   file_based_datasource_kwargs	__class__ a/home/ubuntu/.local/lib/python3.10/site-packages/ray/data/_internal/datasource/text_datasource.pyr      s   
zTextDatasource.__init__fzpyarrow.NativeFilepathreturnc           	      c   sd    |  }t }|| j }|D ]}| jr| dkrq| j|i}|| q|	 }|V  d S )N )
readallr   decoder   
splitlinesr
   strip_COLUMN_NAMEaddbuild)	r   r   r   databuilderlineslineitemblockr   r   r   _read_stream   s   

zTextDatasource._read_stream)__name__
__module____qualname____doc__r   r   strboolr   r   r   r&   __classcell__r   r   r   r   r      s    "r   )typingr   r   r   +ray.data._internal.delegating_block_builderr   ray.data.blockr   )ray.data.datasource.file_based_datasourcer   pyarrowr   r   r   r   r   <module>   s    