o
wi?3 @ s d dl Z d dlZd dlZd dlZd dlmZmZmZmZm Z d dl
m Z d dl
Z
d dlm mZ d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dl m Z d dl m Z m Z d d l!m"Z"m#Z# d d
l$m%Z% G d d d e#Z&G d
d de j'Z(dS ) N)AnyDictListOptionalSequence)EVAL_DATALOADERSTRAIN_DATALOADERS)data)
DataLoaderdefault_collate) get_ltor_masks_and_position_idscreate_vision_mask_tensor)
DataConfigImageDataConfig) IGNORE_INDEXLazySupervisedDataset)MegatronDataSamplerc s` e Zd ZdZ fddZdeeejf fddZ dd Z
d e e deeejf fd
d Z Z
S )
MLlamaDatasetz#Dataset for supervised fine-tuning.c s* | drt |||| n| drt d ||| td |jdkr|j}t|dD ]W}t |}g |d< |d D ]A} t
d| d }
|
D ])} | d
d
d } tj|| }
tj|
sltd |
qJ|d | qJt
d
d| d | d< q>| j| q/ntd| d|| _d S )Nz.jsonz.jsonlz,Loading image inputs from SteerLM Dataset...imager
conversationsz ![](["\']([^"\']+)["\'])
zFormatting of z is not supported in MLlama.)endswithsuper__init__loggingwarning
media_type image_folderopenjsonloadsrefinditergroupsplitospathjoinisfileappendsublist_data_dict
ValueErrorsequence_length)self data_path data_config tokenizerimage_processorr3 r# linerecordturnmatchesmatch
image_name
image_path __class__ g/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/vlm/mllama/data/preloaded.pyr ' s0
zMLlamaDataset.__init__returnc C s^ | j | }| j|| jdkd}|dd}| |\}}| |}tdi |||d}|S )Nplain) use_plainr z <|image|>)tokenslabelsrB )r1 _apply_prompt_templates
conv_templatereplace_tokenize_and_label_process_imagesdict)r4 isourcer rG rH
image_dict data_dictrB rB rC __getitem__Q s
zMLlamaDataset.__getitem__c C s g }d|v r3t |d ts|d g|d< |d D ]}| j|}|d u r-td| d || qt|dkrL| jj |dd}dd |
D }|S t t
d d
d | jjd | jjd
t jdgt jddgd}|S )Nr zImage z could not be found!r pt)return_tensorsc S s" i | ]
\}}|d v r||d qS ) pixel_valuesaspect_ratio_ids num_tilesr rB ).0kvrB rB rC
l s z1MLlamaDataset._process_images..r heightwidth)dtyperV )
isinstancelist image_loader
open_imager r! r/ lenr8
preprocessitemsrN torchzerossizetensorlong)r4 rP images
image_filer rQ rB rB rC rM _ s. z MLlamaDataset._process_images instancesc C s | j }tdd |D d d d d }|| jkr)td| d| j d | j}tdd |D }|D ]o}||d jd
}t|d d
|fd d
|d < t|d d
|fd t |d < ||d
jd
}t|d
d
d
d
d
d
d
d
d
d
|f
d d
|d
< t|d d
t|d d
fd d
|d< tt
|d d
t|d d
fd d
|d< q4dd |D }t |} | j
}
| d } | d } t| |
j|j|j|jd\}
}}d|| d
k < | | | d
|| d |
| d ||d } | S )Nc s | ]
}|d j d V qdS )rG r NshaperZ instancerB rB rC | z+MLlamaDataset.collate_fn..r @ zTruncating sequence length z to .c s rr )rW r Nrs ru rB rB rC rw rx rG r constantrH rW rX rY c S s g | ] }t |d dqS )rG i r
ru rB rB rC
s z,MLlamaDataset.collate_fn..)r eod_token
eod_mask_lossreset_attention_maskreset_position_idsg ) rG rH batch_images batch_masks
num_chunksattention_maskrX loss_mask position_ids)r6 maxr3 r r!
seq_lengthrt Fpadr rj rm r r7 r eos_token_idr~ r r )r4 rq r6 max_lenmax_num_concurrent_mediarv pad_lenpad_num_imagesr batchr7 rG rH r r r rB rB rC
collate_fnz sX "
zMLlamaDataset.collate_fn)__name__
__module__ __qualname____doc__r r strrj TensorrS rM r r
__classcell__rB rB r@ rC r $ s *(r c % s e Zd Zdeddddddddddddddfd eee B d
eee d ee d e d
ee dedede de de de de de de
de
de
de ddf$ fdd
Z d.d eddfd d Z de
fd!d"Zdefd#d$Zdefd%d&Zdefd'd(Zdeeef fd)d*Zd+eeef ddfd,d-Z ZS )/MLlamaPreloadedDataModuleNi r^ i' TFi pathsweightsr6 r decoder_seq_lengthr7 r8 micro_batch_sizeglobal_batch_sizenum_train_samplesnum_val_samplesnum_test_samples num_workers
pin_memorypersistent_workersuse_packed_sequenceseedrD c s& t t|ttfs|g}|d ur%t|t|ks J t|dkr%d }|| _|| _|| _|| _ || _
|| _ | | _ || _
|| _|
| _| | _| | _|
| _|| _|| _|| _|| _d| _|| _
|| _|d u si|d u rtd ddlm} | d}|p}|j
| _
|p|j| _t | j | j
|| dd| _ d S )Nr r zbProcessor and tokenizer are not provided! Fall back to `meta-llama/Llama-3.2-11B-Vision-Instruct`.)
AutoProcessorz(meta-llama/Llama-3.2-11B-Vision-Instructcyclic)seq_lendecoder_seq_lenr r dataloader_type) r r rc rd tuplerg r r r6 r r r r r7 r8 r r r r r r r r init_global_stepr r! transformersr from_pretrainedr data_sampler)r4 r r r6 r r r7 r8 r r r r r r r r r r r processorr@ rB rC r sT
z"MLlamaPreloadedDataModule.__init__ stagec C sd t | jdks J d| jrd S t| jd | j| j| j| j| _t| jd | j| j| j| j| _ d S )Nr z,not yet support blend dataset in MLlama 2.0!r )
rg r r r r6 r7 r8 r _train_ds_validation_ds)r4 r rB rB rC setup s
z MLlamaPreloadedDataModule.setupc C | | jS N)_create_dataloaderr r4 rB rB rC train_dataloader z*MLlamaPreloadedDataModule.train_dataloaderc C r r )r r r rB rB rC val_dataloader r z(MLlamaPreloadedDataModule.val_dataloaderc C r r )r _test_dsr rB rB rC test_dataloader
r z)MLlamaPreloadedDataModule.test_dataloaderc K s@ | j j| _| j| j_t|f| j| j| jt|dt j
j d|S )Nr )r r r r ) trainer global_stepr r r
r r r getattrr
dataloaderr )r4 datasetkwargsrB rB rC r
s
z,MLlamaPreloadedDataModule._create_dataloaderc C s | j | jj| j }d|iS )zCalled when saving a checkpoint, implement to generate and save datamodule state.
Returns:
A dictionary containing datamodule state.
consumed_samples)r compute_consumed_samplesr r r )r4 r rB rB rC
state_dict s z$MLlamaPreloadedDataModule.state_dictr c C sp zddl m} W n ty ddlm} Y nw |d }|| j_|| j_d| _|dur6|}|j|dd dS dS )zCalled when loading a checkpoint, implement to reload datamodule state given datamodule stat
Args:
state_dict: the datamodule state returned by ``state_dict``.
r )#_GLOBAL_NUM_MICROBATCHES_CALCULATORr r NF)r consistency_check) (apex.transformer.pipeline_parallel.utilsr ModuleNotFoundErrornemo.lightning.apex_utilsr init_consumed_samplesprev_consumed_samples
if_first_stepupdate)r4 r r r num_microbatch_calculatorrB rB rC load_state_dict# s
z)MLlamaPreloadedDataModule.load_state_dict)r )r r r r r r r floatr intboolr r r r r r r r
r r r r r r rB rB r@ rC r sz
D "
r ))r% r r+ r' typingr r r r r lightning.pytorchpytorchplrj torch.nn.functionalnn
functionalr !lightning.pytorch.utilities.typesr r torch.utilsr torch.utils.datar
r 2nemo.collections.nlp.modules.common.megatron.utilsr 'nemo.collections.vlm.mllama.model.utilsr %nemo.collections.vlm.neva.data.configr r (nemo.collections.vlm.neva.data.preloadedr r nemo.lightning.pytorch.pluginsr r LightningDataModuler rB rB rB rC s&