o
    .iu                     @   sP   d dl mZmZ ddlmZmZmZ ddlmZ ddl	m
Z
 G dd de
Zd	S )
    )CallableOptional   )Features
NamedSplitSplit)	Generator   )AbstractDatasetInputStreamc                       sd   e Zd Zddddddejfdedee dede	de	dee
 d	ee d
ef fddZdd Z  ZS )GeneratorDatasetInputStreamNF	generatorfeatures	cache_dirkeep_in_memory	streaming
gen_kwargsnum_procsplitc	           
         s>   t  jd|||||d|	 td|||||d|	| _d S )N)r   r   r   r   r   )r   r   r   r   r    )super__init__r   builder)
selfr   r   r   r   r   r   r   r   kwargs	__class__r   I/home/ubuntu/.local/lib/python3.10/site-packages/datasets/io/generator.pyr   	   s$   z$GeneratorDatasetInputStream.__init__c                 C   sd   | j r| jj| jjjd}|S d }d }d }d }| jj||||| jd | jj| jjj|| jd}|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r   	in_memory)	r   r   as_streaming_datasetconfigr   download_and_preparer   
as_datasetr   )r   datasetr   r   r   r    r   r   r   read&   s$   z GeneratorDatasetInputStream.read)__name__
__module____qualname__r   TRAINr   r   r   strbooldictintr   r   r'   __classcell__r   r   r   r   r      s4    	r   N)typingr   r    r   r   r   $packaged_modules.generator.generatorr   abcr
   r   r   r   r   r   <module>   s
    