o
    }oi_                     @   s   d dl mZmZmZ d dlmZ d dlZd dl	Z	d dl
mZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ G d	d
 d
ejZddejfddZG dd deZdS )    )DictListOptionalN)EVAL_DATALOADERSTRAIN_DATALOADERS)Image)data)
DataLoaderDataset)IMAGE_TOKEN_INDEX)MegatronDataSamplerc                       s   e Zd ZdZ												d$d	ed
ededededeee  dedededededef fddZd%de	ddfddZ
defddZdefddZdefd d!Zdefd"d#Z  ZS )&Qwen2VLMockDataModulezK
    A mock data module for Qwen2VL training, validation, and testing.
       N      '  TF
seq_length	tokenizerimage_processormicro_batch_sizeglobal_batch_sizerampup_batch_sizenum_train_samplesnum_val_samplesnum_test_samplesnum_workers
pin_memorypersistent_workersc                    s|   t    || _|| _|| _|| _|| _|	| _|
| _|| _	|| _
|d ur(|d us,J d|| _|| _t| j|||d| _d S )Nz+please assign tokenizer and image_processor)seq_lenr   r   r   )super__init__r   r   r   r   r   r   r   r   r   r   r   r   data_sampler)selfr   r   r   r   r   r   r   r   r   r   r   r   	__class__ Z/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/qwen2vl/data/mock.pyr    "   s&   
zQwen2VLMockDataModule.__init__ stagereturnc                 C   sR   t | j| jd| j| j| _t | j| jd| j| j| _t | j| jd| j| j| _	d S )Ntrainvalidtest)
_Qwen2VLMockDatasetr   r   r   r   	_train_dsr   _validation_dsr   _test_ds)r"   r(   r%   r%   r&   setupF   s   
zQwen2VLMockDataModule.setupc                 C      t | ds	|   | | jS )Nr.   )hasattrr1   _create_dataloaderr.   r"   r%   r%   r&   train_dataloaderR      
z&Qwen2VLMockDataModule.train_dataloaderc                 C   r2   )Nr/   )r3   r1   r4   r/   r5   r%   r%   r&   val_dataloaderX   r7   z$Qwen2VLMockDataModule.val_dataloaderc                 C   r2   )Nr0   )r3   r1   r4   r0   r5   r%   r%   r&   test_dataloader^   r7   z%Qwen2VLMockDataModule.test_dataloaderc                 K   s"   t |f| j| j| j|jd|S )N)r   r   r   
collate_fn)r	   r   r   r   r:   )r"   datasetkwargsr%   r%   r&   r4   d   s   z(Qwen2VLMockDataModule._create_dataloader)r   NNr   r   Nr   r   r   r   TF)r'   )__name__
__module____qualname____doc__intr   r   boolr    strr1   r   r6   r   r8   r9   r	   r4   __classcell__r%   r%   r#   r&   r      sX    
	
$r         num_channelsc                 C   s.   t jjd| ||ft jdg}dd |D }|S )z+This function prepares a list of PIL images   sizedtypec              	   S   s    g | ]}t t|d dqS )r   )r   	fromarraynpmoveaxis).0xr%   r%   r&   
<listcomp>s   s     z(prepare_image_inputs.<locals>.<listcomp>)rN   randomrandintuint8)rG   widthheightimage_inputsr%   r%   r&   prepare_image_inputsp   s   rY   c                       s   e Zd Z	ddededededdf
 fdd	Zdefd
dZdedejfddZ	de
eejf fddZdd Zdd Z  ZS )r-   *   namenum_samplesr   seedr)   Nc                    s   t    || _|| _|j| _|| _tjt	ddd\| _
| _|| _|| _tj| jtjd| _tj	| jtjd| _d| _d| _d S )N8   rF      )rK   z<|image_pad|>)r   r    r[   r   
vocab_sizer   rN   rS   choicearangeimage_widthimage_heightlengthr]   torchonesfloat	loss_maskint64position_idsimage_tokenspatial_merge_size)r"   r   r   r[   r\   r   r]   r#   r%   r&   r    x   s   
	
z_Qwen2VLMockDataset.__init__c                 C   s   | j S )N)re   r5   r%   r%   r&   __len__   s   z_Qwen2VLMockDataset.__len__idxc                 C   s,   t jj| j| d}|j| j| jgt jdS )Nr]   rI   )rN   rS   default_rngr]   integersr`   r   rj   )r"   ro   np_genr%   r%   r&   	_get_text   s   z_Qwen2VLMockDataset._get_textc                 C   s   t d| j| j}| j|dd}|j}|jd }| | jd  }| j| dk r+t	dt
jj| j| d}t|j| j| jd	 gt
jd
}d}	t||	|	| < |d d }
| }d}|	| | }d|d |< |d	d  }|
||| j|dS )NrE   pt)return_tensorsr   r_      z4need to adjust self.seq_length or image_token_amountrp      rI      rL      i)	input_idspixel_valuesimage_grid_thwri   labels)rY   rd   rc   r   r|   r}   prodrm   r   
ValueErrorrN   rS   rq   r]   rf   
from_numpyrr   r`   rj   r   cloneri   )r"   ro   rX   prcocess_outr|   r}   image_token_amountrs   tokensimg_start_idxr{   r~   generation_prompt_sizeprompt_end_idxr%   r%   r&   __getitem__   s.   
"z_Qwen2VLMockDataset.__getitem__c                 C   s   t j|}d|d< |S )z
        A default implementation of a collation function.
        Users should override this method to define custom data loaders.
        Nattention_mask)r   
dataloaderdefault_collate)r"   batchcollated_batchr%   r%   r&   _collate_fn   s   z_Qwen2VLMockDataset._collate_fnc                 C   s
   |  |S )a  Method that user pass as functor to DataLoader.

        The method optionally performs neural type checking and add types to the outputs.

        Please note, subclasses of Dataset should not implement `input_types`.

        # Usage:
        dataloader = torch.utils.data.DataLoader(
                ....,
                collate_fn=dataset.collate_fn,
                ....
        )

        Returns
        -------
            Collated batch, with or without types.
        )r   )r"   r   r%   r%   r&   r:      s   
z_Qwen2VLMockDataset.collate_fn)rZ   )r=   r>   r?   rC   rA   r    rn   rN   ndarrayrt   r   rf   Tensorr   r   r:   rD   r%   r%   r#   r&   r-   w   s$    %	r-   )rE   rF   rF   )typingr   r   r   lightning.pytorchpytorchplnumpyrN   rf   !lightning.pytorch.utilities.typesr   r   PILr   torch.utilsr   torch.utils.datar	   r
   3nemo.collections.vlm.qwen2vl.data.multimodal_tokensr   nemo.lightning.pytorch.pluginsr   LightningDataModuler   rU   rY   r-   r%   r%   r%   r&   <module>   s   S