o
    ίi                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ e eZeG dd dZdd Z		ddeeef fddZdS )    N)	dataclass)partial)AnyDictOptional)MllmPaddingCollator)DatasetMixerPersistentDataLoader)VisionPreprocessor)get_image_transform)get_region_transform)get_video_transformc                   @   s   e Zd ZU dZeed< dZeed< dZeed< dZ	e
e ed< dZe
e ed	< dZe
e ed
< dZe
e ed< dZe
e ed< dZe
e ed< dZe
e ed< dZe
e ed< dZe
e ed< dZe
e ed< dZe
e ed< dS )DataloadArgszdummy_image:1,dummy_multi_image:1,dummy_image_region:1,dummy_video:1,dummy_text:1,dummy_stc_RDCap:1,dummy_stc_RCap:1,dummy_stc_RTLoc:1datamix   
batch_size*   seedN	image_resmax_num_tilesvision_input_typenum_workerstokenizer_pathtokenizer_nameconversation_format
patch_sizeseq_lenmax_video_framesFshow_first_batch)__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   r   r   r   r   r   r   r   bool r&   r&   H/home/ubuntu/.local/lib/python3.10/site-packages/core/data/dataloader.pyr      s    
 
r   c              
   C   sb   | d ur/|| v r/| d |ks| d |kr+t d| d  d| d| d  d|  d S | | S d S )Nr   
world_sizezpCheckpoint resumed with different number of total dataloader workers. Dataloaders have been reset. num_workers: z -> z, world_size: )loggerwarning)	positionsrankworkersr(   r&   r&   r'   get_rank_position&   s    r.   dataset_configsc              
   C   s   | j }| j}| j}| j}	ttt|||dt|dt d||	d}
t	| j
||| j|
g|d}t|| j| jt|| jdt||| j|d}|S )N)r   r   r   )r   )imagevideoregion)	transform	tokenizerr   )global_rankr(   r   preprocessorsr/   )r   )
collate_fnr+   )r   r   r   r   r   r
   r   r   r   r   r   r   r	   r   r   r   r   r.   )argsdp_rankdp_world_sizer/   r4   r+   r   r   r   r   preprocessordataset
dataloaderr&   r&   r'   get_dataloader3   sL   	

r>   )NN)loggingdataclassesr   	functoolsr   typingr   r   r   core.data.data_collatorsr   core.data.data_mixerr   r	   core.data.preprocessorr
   core.transforms.image_transformr    core.transforms.region_transformr   core.transforms.video_transformr   	getLoggerr   r)   r   r.   r"   r>   r&   r&   r&   r'   <module>   s&   

