o
    }oi                     @   s   d dl mZmZmZ d dlmZ d dlZd dl	Z	d dl
mZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d
ZdZdZG dd dejZddejfddZ G dd deZ!dS )    )DictListOptionalN)EVAL_DATALOADERSTRAIN_DATALOADERS)Image)data)
DataLoaderDataset)Gemma3ImageProcessor)MegatronDataSampler)loggingi   i     c                       s   e Zd ZdZ											d#d	ed
edededeee  dedededededef fddZd$de	ddfddZ
defddZdefddZdefdd Zdefd!d"Z  ZS )%Gemma3VLMockDataModulezL
    A mock data module for Gemma3VL training, validation, and testing.
    N         '  TF	tokenizer
seq_lengthmicro_batch_sizeglobal_batch_sizerampup_batch_sizenum_train_samplesnum_val_samplesnum_test_samplesnum_workers
pin_memorypersistent_workersc                    s   t    || _|| _|| _|| _|| _|| _|| _|	| _	|
| _
|| _|d u r9td ddlm} |p7|d| _tttddddd	d
| _t| j|||d| _d S )NzATokenizer is not provided! Fall back to `'google/gemma-3-4b-it'`.r   )AutoTokenizerzgoogle/gemma-3-4b-it)heightwidthFr   r   g333333?)sizedo_pan_and_scanpan_and_scan_min_crop_sizepan_and_scan_max_num_crops"pan_and_scan_min_ratio_to_activate)seq_lenr   r   r   )super__init__r   r   r   r   r   r   r   r   r   r   r   warning=nemo.collections.common.tokenizers.huggingface.auto_tokenizerr   r   
IMAGE_SIZEimage_processorr   data_sampler)selfr   r   r   r   r   r   r   r   r   r   r   r   	__class__ [/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/gemma3vl/data/mock.pyr)   '   s8   

zGemma3VLMockDataModule.__init__ stagereturnc                 C   sR   t | j| jd| j| j| _t | j| jd| j| j| _t | j| jd| j| j| _	d S )Ntrainvalidtest)
_Qwen2VLMockDatasetr   r-   r   r   	_train_dsr   _validation_dsr   _test_ds)r/   r5   r2   r2   r3   setupU   s   
zGemma3VLMockDataModule.setupc                 C      t | ds	|   | | jS )Nr;   )hasattrr>   _create_dataloaderr;   r/   r2   r2   r3   train_dataloadera      
z'Gemma3VLMockDataModule.train_dataloaderc                 C   r?   )Nr<   )r@   r>   rA   r<   rB   r2   r2   r3   val_dataloaderg   rD   z%Gemma3VLMockDataModule.val_dataloaderc                 C   r?   )Nr=   )r@   r>   rA   r=   rB   r2   r2   r3   test_dataloaderm   rD   z&Gemma3VLMockDataModule.test_dataloaderc                 K   s"   t |f| j| j| j|jd|S )N)r   r   r   
collate_fn)r	   r   r   r   rG   )r/   datasetkwargsr2   r2   r3   rA   s   s   z)Gemma3VLMockDataModule._create_dataloader)Nr   r   r   Nr   r   r   r   TF)r4   )__name__
__module____qualname____doc__r   intr   boolr)   strr>   r   rC   r   rE   rF   r	   rA   __classcell__r2   r2   r0   r3   r   "   sR    
	
.r         num_channelsc                 C   s.   t jjd| ||ft jdg}dd |D }|S )z+This function prepares a list of PIL images   )r"   dtypec              	   S   s    g | ]}t t|d dqS )r   )r   	fromarraynpmoveaxis).0xr2   r2   r3   
<listcomp>   s     z(prepare_image_inputs.<locals>.<listcomp>)rY   randomrandintuint8)rT   r!   r    image_inputsr2   r2   r3   prepare_image_inputs   s   rb   c                       s   e Zd Z	ddededededdf
 fdd	Zdefd
dZdedejfddZ	de
eejf fddZdd Zdd Z  ZS )r:   *   namenum_samplesr   seedr6   Nc                    s`   t    || _|| _|jd | _|| _|| _|| _tj	| jtj
d| _tj| jtjd| _d S )N
   rV   )r(   r)   rd   r   
vocab_sizer-   lengthrf   torchonesfloat	loss_maskarangeint64position_ids)r/   r   r-   rd   re   r   rf   r0   r2   r3   r)      s   
	z_Qwen2VLMockDataset.__init__c                 C   s   | j S )N)rj   rB   r2   r2   r3   __len__   s   z_Qwen2VLMockDataset.__len__idxc                 C   s.   t jj| j| d}|jd| j| jgt jdS )Nrf   r   lowhighr"   rV   )rY   r^   default_rngrf   integersri   r   rp   )r/   rs   np_genr2   r2   r3   	_get_text   s   z_Qwen2VLMockDataset._get_textc                 C   s   t dtt}| j|dd}|jtj}tjj	| j
| d}t|jd| j| jd gtjd}d}t|||t < |d d	 }tj| jtjd
}	| }
d}|t | }d|
d |< |
dd  }
||	|| j|
dS )NrR   pt)return_tensorsrt   r      ru      rW   rh      i)	input_idsrq   pixel_valuesrn   labels)rb   r,   r-   r   tork   bfloat16rY   r^   rx   rf   
from_numpyry   ri   r   rp   IMAGE_TOKEN_INDEXIMAGE_TOKENSro   clonern   )r/   rs   ra   prcocess_outr   rz   tokensimg_start_idxr   rq   r   generation_prompt_sizeprompt_end_idxr2   r2   r3   __getitem__   s,   z_Qwen2VLMockDataset.__getitem__c                 C   s   t j|}d|d< |S )z
        A default implementation of a collation function.
        Users should override this method to define custom data loaders.
        Nattention_mask)r   
dataloaderdefault_collate)r/   batchcollated_batchr2   r2   r3   _collate_fn   s   z_Qwen2VLMockDataset._collate_fnc                 C   s
   |  |S )a  Method that user pass as functor to DataLoader.

        The method optionally performs neural type checking and add types to the outputs.

        Please note, subclasses of Dataset should not implement `input_types`.

        # Usage:
        dataloader = torch.utils.data.DataLoader(
                ....,
                collate_fn=dataset.collate_fn,
                ....
        )

        Returns
        -------
            Collated batch, with or without types.
        )r   )r/   r   r2   r2   r3   rG      s   
z_Qwen2VLMockDataset.collate_fn)rc   )rJ   rK   rL   rP   rN   r)   rr   rY   ndarrayr{   r   rk   Tensorr   r   rG   rQ   r2   r2   r0   r3   r:      s$    "	r:   )rR   rS   rS   )"typingr   r   r   lightning.pytorchpytorchplnumpyrY   rk   !lightning.pytorch.utilities.typesr   r   PILr   torch.utilsr   torch.utils.datar	   r
   transformersr   nemo.lightning.pytorch.pluginsr   
nemo.utilsr   r   r,   r   LightningDataModuler   r`   rb   r:   r2   r2   r2   r3   <module>   s"   ]