o
    wi                     @   s  d dl mZ d dlmZ d dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# dZ$dZ%ej&j'e$ddej(ej) fddZ*ej&j'ej+e$d					d-dee, de,de-de-dee, dej.fd d!Z/e0d"krd#d$iZ1e/d%d&Z2d'e2j3_4d'e2j3_5d(e2j3_6de2j7_8de2j3j9_:ej;d%d)e1d*Z<eje2e<d+d, dS dS ).    )OptionalN)DistributedDataParallelConfig)OptimizerConfig)Qwen2VLImageProcessor)	lightning)llmvlm)AutoTokenizer)tensorboard_logger)
bf16_mixed)Qwen2VLMockDataModule)MegatronCommOverlapCallback)CosineAnnealingScheduler)MegatronOptimizerModule)TimingCallback
qwen2vl_2bzQwen/Qwen2-VL-2B-Instructnamereturnc                   C   s   t jtjt tjdS )ag  
    Factory function to create a Qwen2VL 2B model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the Qwen2VL 2B model model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=qwen2vl_2b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    )config)runConfigr   Qwen2VLModelQwen2VLConfig2B r   r   d/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/vlm/recipes/qwen2vl_2b.pymodel)   s   r   )targetr   default      nonedirr   	num_nodesnum_gpus_per_nodepeft_schemec                 C   s  t jtjddddtjt jtddddddd}t jtjdd|ddd|t |dt t	t jt
dd	gd
}t tt}t t}d}	t jtj|	d}
t tj}t jtjd|j|
j|jd}t jtj|
||ddd}t jtj||d}t jtjddd}t jtddddddd}t jt|jdddd}t t||}t jtj||t jt|	dd||ddtj| |t |dd||d }|d!u s|! d"krd|j"j#_$d#|j%j&_'|S |! d$krt jtj(g d%d&|_)d'|j%j&_'|S t*d(| ))a  
    Create a fine-tuning recipe for Qwen2VL 2B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory qwen2vl_2b

        Python API usage:
            >>> recipe = finetune_recipe(name="qwen2vl_2b_finetune", num_nodes=1)
            >>> print(recipe)

    Note:
        This recipe uses the Mock dataset for fine-tuning.
    r   r   T)check_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gatheraverage_in_collective)tensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizesequence_parallelpipeline_dtypeddpgpu
   i  )tp_comm_overlap)acceleratoraccumulate_grad_batchesdeviceslimit_val_batcheslog_every_n_steps	max_stepsr#   pluginsstrategyval_check_interval	callbacksi   )
seq_length	mcore_mlp)projector_type
input_sizehidden_sizeffn_hidden_sizeF)language_transformer_configvision_transformer_configvision_projection_configfreeze_language_modelfreeze_vision_model)	tokenizer)resume_if_existsresume_ignore_no_checkpointadamg>g?gffffff?)	optimizerlr
adam_beta1
adam_beta2use_distributed_optimizerbf16gHz>)r9   warmup_stepsconstant_stepsmin_lr         )r>   global_batch_sizemicro_batch_sizerI   image_processornum_workersr   )r"   r   r
   )r   trainerdatalogoptimresumeNr!   gh㈵>lora)
linear_qkvlinear_proj
linear_fc1
linear_fc2)target_modulesg-C6?zUnrecognized peft scheme: )+r   r   nlMegatronStrategytorchbfloat16r   Trainerr   r   r   r	   HF_MODEL_NAMEr   r   Qwen2Config1P5Br   Qwen2VLVisionConfigMultimodalProjectorConfigrC   rB   Qwen2VLConfigr   
AutoResumer   r   r9   r   Partialfinetuner   default_logr
   lowerr]   r;   r+   r`   r   rN   LoRApeft
ValueError)r"   r   r#   r$   r%   r;   r]   rI   r[   max_sequence_lengthrD   rE   rF   qwen2vl_configr   nemo_resume
opt_configschedoptreciper   r   r   finetune_recipe<   s   $
				


	r   __main__CUDA_VISIBLE_DEVICESz0,1rW   )r$   r2   g        torchrun)ntasks_per_nodelauncherenv_varsqwen2vl_2b_finetune)executorr   )Nr   r   r    r!   )=typingr   lightning.pytorchpytorchplnemo_runr   rj   megatron.core.distributedr   megatron.core.optimizerr   transformersr   nemor   rh   nemo.collectionsr   r   =nemo.collections.common.tokenizers.huggingface.auto_tokenizerr	   (nemo.collections.llm.recipes.log.defaultr
   6nemo.collections.llm.recipes.precision.mixed_precisionr   &nemo.collections.vlm.qwen2vl.data.mockr   6nemo.lightning.pytorch.callbacks.megatron_comm_overlapr   nemo.lightning.pytorch.optimr   %nemo.lightning.pytorch.optim.megatronr   nemo.utils.exp_managerr   NAMErm   clifactoryr   LightningModuler   rt   strintrs   r   __name__r   r   r]   r9   r<   r7   r^   rY   r;   r+   LocalExecutorr   r   r   r   r   <module>   sj    

