o
    wi                     @   s  d dl mZ d dlZd dlZd dlmZ d dlmZ	 d dl
mZ d dlmZmZ dZdd	 Zd
d Zedkre	e e	dkrd dlZe Zejddd ejddd ejddd ejddd ejdddgd ejdedd ejdedd ejddd ejd d!d"d# e ZdZejdurd$ej d%d&d Zeeje d'ej! d(d)ZdZ"d*Z#ej$%ejZ&ej'j(ej$ejej)d+ee&ej*ej+ej,ej!ej-ej.ej/ddd,dd-d de"e#eej0d.e1ej2j3d/d0dd1 dS dS dS )2    )versionN)WandbLogger)Version)	lightning)llmvlmz/home/TestData/vlm/rdr-itemsc                  C   s$   dd l } t| drt| jS tdS )Nr   __version__torch)r	   hasattrstrr   r   )r	    r   _/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/tests/collections/vlm/hf/sft_fsdp2.pyget_torch_version_str   s   

r   c                    s6   t jfdd t jtd|| fdddS )Nc                    s   dd }g }g }t || D ]}||j|d ddd ||d 7 }q|||ddd	}|d
 tj|d
< |d  }d|t| < ||d< |S )Nc                 S   sH   d}dd|dd| d dgddd| d dgdg}|| d gd	S )
Nz$Describe accurately the given image.usertext)typer   image)r   r   )rolecontent	assistant)conversationimagesr   )sampleinstructionr   r   r   r   fmt)   s   z2mk_hf_vlm_dataset.<locals>.collate_fn.<locals>.fmtr   F)tokenizeadd_generation_promptr   Tpt)r   r   paddingreturn_tensorspixel_values	input_idsilabels)mapappendapply_chat_templatetor	   bfloat16cloneisin)examples	processorr   r   r   examplebatchr"   )skipped_tokensr   r   
collate_fn(   s.   z%mk_hf_vlm_dataset.<locals>.collate_fnz
train[:10]c                    s    | dS )N)r+   r   )x)r/   r+   r   r   <lambda>T   s    z#mk_hf_vlm_dataset.<locals>.<lambda>)splitmicro_batch_sizeglobal_batch_sizer/   )r   HFAutoModelForImageTextToTextextract_skipped_token_idsHFDatasetDataModule	DATA_PATH)r+   mbsgbsr   )r/   r+   r.   r   mk_hf_vlm_dataset%   s   'r;   __main__z2.4z--modelzQwen/Qwen2-VL-2B-Instruct)defaultz	--devices   z--mbs   z--gbsz--acceleratorgpu)r=   choicesz--max-stepsd   )r   r=   z--wandb-projectz--disable-ckptstore_false)actionz
--use-4bitzLoad model in 4bit
store_true)helprD   _/_dev_strat_fsdp2)projectnameF)load_in_4bit)data_parallel_sizetensor_parallel_sizeg        )devices	max_stepsacceleratorstrategylog_every_n_stepslimit_val_batchesnum_sanity_val_stepsaccumulate_grad_batchesgradient_clip_valuse_distributed_samplerloggerenable_checkpointinggh㈵>)lr)modeldatatraineroptimlog)4importlib.metadatar   fiddlefdlr	   lightning.pytorch.loggersr   packaging.versionr   
PkgVersionnemor   nlnemo.collectionsr   r   r8   r   r;   __name__argparseArgumentParserparseradd_argumentintr   
parse_argsargswandbwandb_projectjoinr^   r2   rQ   	grad_clipuse_dist_sampr5   configure_processorr+   apifinetuneuse_4bitr9   r:   TrainerrR   rS   FSDP2Strategydisable_ckptbuildadampytorch_adam_with_flat_lrr   r   r   r   <module>   sp   	3

