o
    پi#                     @   s   d Z ddlZddlZddlmZ ddlZddlmZm	Z	 ddl
mZ e  ddlmZmZmZ ddlmZ dd	lmZmZ G d
d deZdS )zDataLoader for video datasets.    N)
ThreadPool   )cpugpu)try_import_mxnet)
DataLoaderdefault_batchify_fn_thread_worker_initializer)sampler)is_np_shapeis_np_arrayc                   @   sD   e Zd Zdddddddddddde gfddZdd Zd	d
 ZdS )ShardedDataLoaderNFr   x   c                 C   s  || _ |	| _|
| _|| _|| _|| _|dksJ d|t|}t	|t
| }|g|d  }||t| g7 }|d |ksBJ tdg| | _|	rTtd d}	|d u r|d u r`td|d u rv|rntt|}ntt|}n|r|tdt|||r|nd	}n|d us|s|d us|d urtd
|| _|dkr|nd| _|d u rt| _n|| _d | _d | _t|dkr|d t kr|dkrt||||||||	|
|||d| _d S |dkrt| jtt  t! fd| _d S d S )Nr   z"timeout must be positive, given {}   zpin_memory not supported.Fz>batch_size must be specified unless batch_sampler is specifiedz5shuffle must not be specified if sampler is specifiedkeepz`batch_size, shuffle, sampler and last_batch must not be specified if batch_sampler is specified.)
batch_sizeshufflebatch_sampler
last_batchbatchify_fnnum_workers
pin_memorypin_device_idprefetchthread_pooltimeout)initializerinitargs)"_dataset_pin_memory_pin_device_id_thread_pool_timeout	_ctx_listformatlenmathceilfloatsumnpcumsum_bs_listwarningswarn
ValueError_samplerRandomSamplerSequentialSamplerBatchSampler_batch_sampler_num_workersr   _batchify_fn_iter_poolr   r   r   r	   r   r   )selfdatasetr   r   r
   r   r   r   r   r   r   r   r   r   ctx_listnum_ctx
bs_per_ctxbs_list r@   J/home/ubuntu/.local/lib/python3.10/site-packages/decord/data/dataloader.py__init__   sf   
"
zShardedDataLoader.__init__c              
   #   s   j dkrMjD ]A}g }tjjd d jdd  D ](\ }}|t|kr+ n||t|t| }| fdd|D  q|V  q	d S j	d urej
D ]}t|ts_|g}|V  qUd S jd uslJ jD ]U}g }tjjd d jdd  D ]+\ }}|t|kr n||t|t| }|jjj fdd|D  qfdd|D }jj|}|V  qod S )Nr   r   r   c                    s   g | ]	}j | f qS r@   )r   .0idxctxr:   r@   rA   
<listcomp>X   s    z.ShardedDataLoader.__iter__.<locals>.<listcomp>c                    s   g | ]}| fqS r@   r@   rC   )rG   r@   rA   rH   g   s    c                    s   g | ]}|  jqS r@   )getr#   )rD   rr:   r@   rA   rH   h   s    )r6   r5   zipr$   r-   r&   minappendr7   r8   iter
isinstancelistr9   	map_asyncr   __getitem__map)r:   batchretstartendindices
async_retstmp_retsr@   rF   rA   __iter__P   s:   

,"



,(zShardedDataLoader.__iter__c                 C   s
   t | jS )N)r&   r5   rK   r@   r@   rA   __len__l   s   
zShardedDataLoader.__len__)__name__
__module____qualname__r   rB   r\   r]   r@   r@   r@   rA   r      s    
?r   )__doc__r'   r.   multiprocessing.poolr   numpyr+   ndarrayr   r   bridge.mxnetr   mxnet.gluon.data.dataloaderr   r   r	   mxnet.gluon.datar
   r1   
mxnet.utilr   r   objectr   r@   r@   r@   rA   <module>   s    