o
    پi                   
   @   s   d Z ddlZddlZddlmZmZ ddlmZ ddlZddl	m
Z ddlmZ zddlZddlmZ ddlmZ W n eyL Z zed edZ[ww d	d
lmZ d	dlmZ d	dlmZ eejddZG dd deZ dS )z' Dataset reader for HF IterableDataset
    N)repeatchain)Optional)Image)split_dataset_by_node)	SplitInfozDPlease install Hugging Face datasets package `pip install datasets`.   )load_class_map)Reader)SharedCountHFIDS_SHUFFLE_SIZEi   c                !       s   e Zd Z													
			d.dedee dededededededee dededededee dee def  fddZdd Z		d/dee fd d!Z
d"d# Zd$d% Zd&d' Zd(d) Zd0d*d+Zd0d,d-Z  ZS )1ReaderHfidsNtrainFr   r   *   imageRGBlabel namerootsplitis_training
batch_sizedownloadrepeatsseed	class_map	input_keyinput_img_mode
target_keytarget_img_modeshuffle_sizenum_samplestrust_remote_codec                    sj  t    || _|| _|| _|| _|| _|| _|| _|pt	| _
|
| _|| _|| _|| _tj|||d| _|r;| j  d }| jjjrZ|| jjjv rZt| jjj| trZ| jjj| }|r`|| _n|rj|jrj|j| _ntdd| _|	r|t|	| _d| _ni | _d| _d| _t  rt! rt" dkrt# | _t" | _d | _$d| _%d| _&d| _'d| _(d | _)t* | _+d S )N)	cache_dirr#   zDataset length is unknown, please pass `num_samples` explicitly. The number of steps needs to be known in advance for the learning rate scheduler.FTr   r   ),super__init__r   r   r   r   r   r   common_seedSHUFFLE_SIZEr!   r   r   r   r    datasetsload_dataset_builderbuilderdownload_and_prepareinfosplits
isinstancer   r"   num_examples
ValueErrorremap_classr	   class_to_idx	dist_rankdist_num_replicasdistis_availableis_initializedget_world_sizeget_rankworker_info	worker_idnum_workersglobal_worker_idglobal_num_workersdsr   epoch)selfr   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   
split_info	__class__ R/home/ubuntu/.local/lib/python3.10/site-packages/timm/data/readers/reader_hfids.pyr&      s`   







zReaderHfids.__init__c                 C   s   || j _d S N)rA   value)rB   countrF   rF   rG   	set_epochp   s   zReaderHfids.set_epochr=   c                 C   s2   | j d urd S |d ur|| _| j| j | _d S d S rH   )r@   r=   r5   r?   )rB   r=   rF   rF   rG   set_loader_cfgt   s   
zReaderHfids.set_loader_cfgc                 C   s   | j du r+tjj }|dur|| _ |j| _|j| _| j| j | _	| j
| j | j | _| jr>| jj| jd}|j| j	d}n| jj| jd}| jrR|j| j| jd}t|| j
| jd| _dS )z8 Lazily initialize worker (in worker processes)
        N)r   )
num_shards)r   buffer_size)rank
world_size)r;   torchutilsdataget_worker_infoidr<   r=   r5   r?   r4   r>   r   r+   
as_datasetr   to_iterable_datasetas_streaming_datasetr   shuffler'   r!   r   r@   )rB   r;   datasetr@   rF   rF   rG   
_lazy_init~   s   
	zReaderHfids._lazy_initc                 C   sh   t d| j| j t | j| j }| js| jdkrt|}| jr0| jd ur0t|| j | j }t	|S )Nr   )
maxr   r"   r?   r5   r   mathceilr   int)rB   num_worker_samplesrF   rF   rG   _num_samples_per_worker   s   
z#ReaderHfids._num_samples_per_workerc                 c   s    | j d u r
|   | j | jj |  }d}| jr$tt	| j }nt
| j }|D ]P}|| j }| jrA|j| jkrA|| j}|| j }| jr`t|tjsSJ d|j| jkr_|| j}n| jrh| j| }||fV  |d7 }| jr{||kr{ d S q+d S )Nr   z7target_img_mode is specified but target is not an imager   )r@   r[   rK   rA   rI   ra   r   r   from_iterabler   iterr   r   modeconvertr   r    r/   r   r2   r3   )rB   target_sample_countsample_countds_itersample
input_datatarget_datarF   rF   rG   __iter__   s6   





zReaderHfids.__iter__c                 C   s   |   | j }|S rH   )ra   r=   )rB   r"   rF   rF   rG   __len__   s   zReaderHfids.__len__c                 C   s   J d)NFzNot supportedrF   )rB   indexbasenameabsoluterF   rF   rG   	_filename   s   zReaderHfids._filenamec                 C   s   | j du r	|   g }| j D ]/}d|v r|d }nd|v r"|d }nd|v r+|d }nd|v r4|d }nJ d|| q|S )z0 Return all filenames in dataset, overrides baseN	file_namefilenamerU   image_idFzNo supported name field present)r@   r[   append)rB   ro   rp   namesri   r   rF   rF   rG   	filenames   s   





zReaderHfids.filenames)Nr   Fr   Fr   r   Nr   r   r   r   NNFrH   )FF)__name__
__module____qualname__strr   boolr_   dictr&   rK   rL   r[   ra   rl   rm   rq   rw   __classcell__rF   rF   rD   rG   r      sx    	
R

!	
r   )!__doc__r]   os	itertoolsr   r   typingr   rQ   torch.distributeddistributedr6   PILr   r)   datasets.distributedr   datasets.splitsr   ImportErroreprintr   r	   readerr
   shared_countr   r_   environgetr(   r   rF   rF   rF   rG   <module>   s,    