o
    }oiG                     @   sX  d dl mZ d dlmZ d dlZd dlZd dlm	Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ dZejjedeejfdejej fddZ ejjej!ed					ddee" de"de#de#dee" dej$fddZ%ejjej&ed					d dee" de"de#de#dee" dej$fddZ'dS )!    )OptionalN)	lightning)llmvlm)nemo_resume)tensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)LlavaNextMockDataModule)TimingCallbackllava_next_7bnamereturnc                 C   s   t jtj| dS )ai  
    Factory function to create a LlavaNext 7B model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the Llava Next 7B model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=llava_next_7b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    config)runConfigr   LlavaNextModelr    r   ^/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/recipes/llava_next_7b.pymodel"   s   r   )targetr   default      nonedirr   	num_nodesnum_gpus_per_nodepeft_schemec           
      C   s.  t jtjdddtjd}t jtjdd|ddd|t |dt tgd	}dd
l	m
} ddlm} t jtjtt jtjddddd|t jtdddddddtj| |t|ddtddddtdd}	|du sk| dkrwd|	jj_d|	jj_|	S | dkrt jtjdg dd |	_ d!|	jj_|	S t!d"| )#al  
    Create a fine-tuning recipe for LlavaNext 7B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory llava_next_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava_next_7b_finetune", num_nodes=1)
            >>> print(recipe)
       r   r   tensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizepipeline_dtypegpu
   F    acceleratoraccumulate_grad_batchesdeviceslimit_val_batcheslog_every_n_steps	max_stepsr   pluginsstrategyval_check_interval	callbacksAutoProcessorAutoTokenizerFT)freeze_language_modelfreeze_vision_modelfreeze_vision_projectionr      r      N
seq_lengthglobal_batch_sizemicro_batch_size	tokenizerimage_processornum_workersr   r   r   r   h㈵>gH׊>   max_lrmin_lrwarmup_stepsz llava-hf/llava-v1.6-vicuna-7b-hf)r   trainerdatalogoptimresumer   lora)z*.language_model.*.linear_qkvz*.language_model.*.linear_qz*.language_model.*.linear_kvz*.language_model.*.linear_projz*.language_model.*.linear_fc1z*.language_model.*.linear_fc2)r;   target_modulesg-C6?zUnrecognized peft scheme: )"r   r   nlMegatronStrategytorchbfloat16Trainerr	   r   transformersr7   =nemo.collections.common.tokenizers.huggingface.auto_tokenizerr9   Partialr   finetuner   r   LlavaNextConfig7Br
   default_logr   r   r   lowerrM   r3   r#   rP   r   lrLoRApeft
ValueError)
r   r   r   r   r    r3   rM   r7   r9   reciper   r   r   finetune_recipe5   sx   !
	


re   language_model_from_pretrainedc           
      C   s   t jtjdddtjd}t jtjdd|ddd|t |dt tgd	}dd
l	m
} ddlm} t jtjtt jtjddd|dd|t jtdddddddtj| |t|ddtddddd}	|	S )a   
    Create a Pre-training recipe for Llava1.6 7B model.

    This function sets up a complete configuration for pre-training, including
    model, trainer, data, logging, optimization, and resumption settings.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm pretrain --factory llava_next_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava_next_7b_pretrain", num_nodes=1)
            >>> print(recipe)
    r!   r   r   r"   r'   r(   r)   r*   r+   r6   r8   TF)r:   r;   r<   rf   r   r=   r   r>   Nr?   r   rF   gMbP?rG   rH   rI   )r   rM   rN   rO   rP   )r   r   rT   rU   rV   rW   rX   r	   r   rY   r7   rZ   r9   r[   r   pretrainr   r   r]   r
   r^   r   r   )
r   r   r   r   rf   r3   rM   r7   r9   rd   r   r   r   pretrain_recipe   s`    
		rh   )Nr   r   r   r   )Nr   r   r   N)(typingr   lightning.pytorchpytorchplnemo_runr   rV   nemor   rT   nemo.collectionsr   r   -nemo.collections.llm.recipes.finetune_defaultr   (nemo.collections.llm.recipes.log.defaultr   'nemo.collections.llm.recipes.optim.adamr   6nemo.collections.llm.recipes.precision.mixed_precisionr	   nemo.collections.vlmr
   nemo.utils.exp_managerr   NAMEclifactoryr   r]   LightningModuler   r\   strintr[   re   rg   rh   r   r   r   r   <module>   sj   $j