o
    }oiF                     @   s0  d dl Zd dlmZ d dlZd dlmZ d dlm	Z
 d dlmZmZ dZdd Zd;d	d
Zedkrd dlZe Zejddd ejdedg dd ejdded ejddd ejddd ejdddgd ejdedd ejdedd ejdd d! ejd"d#d$d% e ZdZejdurd&ejd'd(d Zeeje d)ej d*ej  d+Zd,Z!ej d-krdZ!dZ"ej#$ejZ%ej#ejej&d.Zej'j(eee%ej)ej*e
j+ejej,ej-eej eejdd/d0dd1d d2e!e"eej.d3d4e/ej0j1d5d6dej2j3d7gd8ej&rej4ndd9d: dS dS )<    N)WandbLogger)	lightning)llmvlmz/home/TestData/vlm/rdr-itemsc                    s6   t jfdd t jtd|| fdddS )Nc                    s   dd }g }g }t || D ]}||j|d ddd ||d 7 }q|||ddd	}|d
 tj|d
< |d  }d|t| < ||d< |S )Nc                 S   sH   d}dd|dd| d dgddd| d dgdg}|| d gd	S )
Nz$Describe accurately the given image.usertext)typer   image)r   r	   )rolecontent	assistant)conversationimages )sampleinstructionr   r   r   T/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/vlm/hf/peft_hf.pyfmt   s   z2mk_hf_vlm_dataset.<locals>.collate_fn.<locals>.fmtr   F)tokenizeadd_generation_promptr   Tpt)r   r   paddingreturn_tensorspixel_values	input_idsilabels)mapappendapply_chat_templatetotorchbfloat16cloneisin)examples	processorr   r   r   examplebatchr   )skipped_tokensr   r   
collate_fn   s.   z%mk_hf_vlm_dataset.<locals>.collate_fnz
train[:10]c                    s    | dS )N)r%   r   )x)r)   r%   r   r   <lambda>I   s    z#mk_hf_vlm_dataset.<locals>.<lambda>)splitmicro_batch_sizeglobal_batch_sizer)   )r   HFAutoModelForImageTextToTextextract_skipped_token_idsHFDatasetDataModule	DATA_PATH)r%   mbsgbsr   )r)   r%   r(   r   mk_hf_vlm_dataset   s   'r5   Fc                 C   sj   | dkrt jjd|j|ddS | dkrt jj|j|ddS | dkr1tj|| d|j|dd	S td
)Nautozcuda:0adapter_only)devicecheckpoint_ioddp)r:   fsdp2   )data_parallel_sizetensor_parallel_sizer:   zEncountered unknown strategy)pl
strategiesSingleDeviceStrategymake_checkpoint_ioDDPStrategynlFSDP2StrategyNotImplementedError)strategymodeldevices	num_nodesr8   r   r   r   make_strategyM   s    


rL   __main__z--modelzQwen/Qwen2-VL-2B-Instruct)defaultz
--strategyr6   )r6   r;   r<   )r   rN   choicesz	--devicesr=   )rN   r   z--mbsz--gbsz--acceleratorgpu)rN   rO   z--max-stepsd   )r   rN   z--wandb-projectz--disable-ckptstore_false)actionz
--use-4bitzLoad model in 4bit
store_true)helprS   _/_dev_strat_)projectnameg      ?fsdp)load_in_4bitTr7   g           bf16)rJ   	max_stepsacceleratorrH   log_every_n_stepslimit_val_batchesnum_sanity_val_stepsaccumulate_grad_batchesgradient_clip_valuse_distributed_samplerloggerenable_checkpointing	precisiongh㈵>)lrz*_proj   )target_modulesdim
lora_dtype)rI   datatraineroptimlogpeft)F)5fiddlefdllightning.pytorchpytorchr@   r    lightning.pytorch.loggersr   nemor   rE   nemo.collectionsr   r   r2   r5   rL   __name__argparseArgumentParserparseradd_argumentstrint
parse_argsargswandbwandb_projectjoinrI   r,   rJ   rH   	grad_clipuse_dist_sampr/   configure_processorr%   use_4bitapifinetuner3   r4   Trainerra   rb   disable_ckptbuildadampytorch_adam_with_flat_lrru   LoRAr!   r   r   r   r   <module>   s~   
3



