o
    }oi                     @   s  d dl mZ d dlmZ d dlZd dlZd dlm	Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ dZejjeddejej fddZejjej ed					ddee! de!de"de"dee! dej#fddZ$dS )    )OptionalN)	lightning)llmvlm)nemo_resume)tensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)MockDataModule)TimingCallback
mllama_11bnamereturnc                   C   s   t jtjt tjdS )au  
    Factory function to create a Llama-3.2-Vision 11B model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the Llama-3.2-Vision 11B model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=mllama_11b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    )config)runConfigr   MLlamaModelMLlamaConfig11BInstruct r   r   [/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/recipes/mllama_11b.pymodel"   s   r   )targetr   default      loradirr   	num_nodesnum_gpus_per_nodepeft_schemec                 C   s  t jtjdddtjd}t jtjdd|ddd|t |dt tgd	}t j	t
jt |t jtd
dddddddt
j| |t|ddtddddtdd}|du sV| dkrbd|jj_d|jj_|S | dkr	 t jtjdg ddddd d!|_d|jj_|S td"| )#a6  
    Create a fine-tuning recipe for Llama3.2 11B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory mllama_11b

        Python API usage:
            >>> recipe = finetune_recipe(name="mllama_11b_finetune", num_nodes=1)
            >>> print(recipe)

    Note:
        This recipe uses the SQuAD dataset for fine-tuning. For more information
        on fine-tuning LLMs with NeMo, see the fine-tuning guide in the
        `examples/llm/finetune/` directory.
    r   r   )tensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizepipeline_dtypegpu   
   iF  d   )acceleratoraccumulate_grad_batchesdeviceslimit_val_batcheslog_every_n_steps	max_stepsr   pluginsstrategyval_check_interval	callbacksi  i   i  )0  r3   )
seq_lengthdecoder_seq_lengthglobal_batch_sizemicro_batch_size
vocab_size	crop_sizenum_workersr   )r   r   r   g-C6?gH׊>   )max_lrmin_lrwarmup_stepsz(meta-llama/Llama-3.2-11B-Vision-Instruct)r   trainerdatalogoptimresumeNnonegh㈵>r   T)
linear_qkvlinear_q	linear_kvr       g?pre)freeze_vision_modeltarget_modulesdimalphadropoutdropout_positionzUnrecognized peft scheme: )r   r   nlMegatronStrategytorchbfloat16Trainerr	   r   Partialr   finetuner   r
   default_logr   r   r   lowerr?   r0   r!   rB   r   lrr   LoRApeft
ValueError)r   r   r   r   r    r0   r?   reciper   r   r   finetune_recipe5   sr   &




r^   )Nr   r   r   r   )%typingr   lightning.pytorchpytorchplnemo_runr   rR   nemor   rP   nemo.collectionsr   r   -nemo.collections.llm.recipes.finetune_defaultr   (nemo.collections.llm.recipes.log.defaultr   'nemo.collections.llm.recipes.optim.adamr   6nemo.collections.llm.recipes.precision.mixed_precisionr	   %nemo.collections.vlm.mllama.data.mockr
   nemo.utils.exp_managerr   NAMEclifactoryr   LightningModuler   rV   strintrU   r^   r   r   r   r   <module>   sD   