o
    wi5                     @   s  d dl mZ d dlmZ d dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ dZejj eddej!ej" fddZ#ejj ej$ed					ddee% de%de&de&dee% dej'fddZ(dS )    )OptionalN)DistributedDataParallelConfig)	lightning)llmvlm)nemo_resume)tensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)MockDataModule)MegatronCommOverlapCallback)TimingCallback
llava15_7bnamereturnc                   C   s   t jtjt tjdS )ak  
    Factory function to create a Llava 1.5 7B model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the Llava 1.5 7B model model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=llava15_7b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    )config)runConfigr   
LlavaModelLlava15Config7B r   r   d/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/vlm/recipes/llava15_7b.pymodel$   s   r   )targetr   default      nonedirr   	num_nodesnum_gpus_per_nodepeft_schemec                 C   s$  t jtjddddtjt jtddddddd}t jtjdd|ddd|t |d	t t	t jt
dd
gd}t jtjt |t jtdddddddtj| |t|ddtddddtdd}|du sf| dkrrd|jj_d|jj_|S | dkrt jtjdg dd|_d|jj_|S td | )!a5  
    Create a fine-tuning recipe for Llava1.5 7B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory llava15_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava15_7b_finetune", num_nodes=1)
            >>> print(recipe)

    Note:
        This recipe uses the SQuAD dataset for fine-tuning. For more information
        on fine-tuning LLMs with NeMo, see the fine-tuning guide in the
        `examples/llm/finetune/` directory.
    r   r   T)check_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gatheraverage_in_collective)tensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizesequence_parallelpipeline_dtypeddpgpu
   iF  i  )tp_comm_overlap)acceleratoraccumulate_grad_batchesdeviceslimit_val_batcheslog_every_n_steps	max_stepsr    pluginsstrategyval_check_interval	callbacksi         N   )
seq_lengthglobal_batch_sizemicro_batch_size	tokenizerimage_processornum_workersr   )r   r   r   gh㈵>gH׊>   )max_lrmin_lrwarmup_stepszllava-hf/llava-1.5-7b-hf)r   trainerdatalogoptimresumer   loraF)z*.language_model.*.linear_qkvz*.language_model.*.linear_qz*.language_model.*.linear_kvz*.language_model.*.linear_projz*.language_model.*.linear_fc1z*.language_model.*.linear_fc2)freeze_vision_modeltarget_modulesg-C6?zUnrecognized peft scheme: )r   r   nlMegatronStrategytorchbfloat16r   Trainerr
   r   r   Partialr   finetuner   r   default_logr   r	   r   lowerrH   r8   r(   rK   r   lrr   LoRApeft
ValueError)r   r   r    r!   r"   r8   rH   reciper   r   r   finetune_recipe7   s|   &	


r^   )Nr   r   r   r   ))typingr   lightning.pytorchpytorchplnemo_runr   rR   megatron.core.distributedr   nemor   rP   nemo.collectionsr   r   -nemo.collections.llm.recipes.finetune_defaultr   (nemo.collections.llm.recipes.log.defaultr   'nemo.collections.llm.recipes.optim.adamr	   6nemo.collections.llm.recipes.precision.mixed_precisionr
   #nemo.collections.vlm.neva.data.mockr   6nemo.lightning.pytorch.callbacks.megatron_comm_overlapr   nemo.utils.exp_managerr   NAMEclifactoryr   LightningModuler   rV   strintrU   r^   r   r   r   r   <module>   sH   