o
    }oi|"                     @   s  d dl mZ d dlmZ d dlZd dlZd dlm	Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ dZdd Zejjede ej!fdej ej" fddZ#ejjej$ed							d%de%dee% de%de&de&dee% dee' dej(fd d!Z)ejjej*ed						d&dee% de%de&de&d"ee% dee' dej(fd#d$Z+dS )'    )OptionalN)	lightning)avlmllm)AVLMMockDataModule)AutoTokenizer)LoRA)tensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)TimingCallbackavlm_8bc                  C   s   ddl m}  | dS )z_
    Create an image processor for the AVLM 8B model.
    This helps by pass fiddle.config
    r   AutoProcessorzopenai/clip-vit-large-patch14)transformersr   from_pretrainedr    r   Y/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/avlm/recipes/avlm_8b.pycreate_image_processor#   s   
r   namereturnc                 C   s   t jtj| dS )aX  
    Factory function to create a AVLM 8B model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the AVLM 8B model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=avlm_8b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    config)runConfigr   	AVLMModelr   r   r   r   model-   s   r   )targetr    default      nonecheckpoint_pathdirr   	num_nodesnum_gpus_per_nodepeft_schemefreeze_modulesc           
      C   sL  t jtjdddtjd}t jtjdd|ddd|t |dt tgd	}t j	t
jtt jtj|d
 |d |d |d |d dd|t jtdddt tdt tddt
j||t|ddtddddt jtjt jtj| ddd}	|du s|| d krd|	jj_d|	jj_|	S | d!krt jtg d"d#|	_d$|	jj_|	S t d%| )&ag  
    Create a fine-tuning recipe for AVLM 8B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory llava_next_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava_next_7b_finetune", num_nodes=1)
            >>> print(recipe)
       r!   r   tensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizepipeline_dtypegpu
   F    acceleratoraccumulate_grad_batchesdeviceslimit_val_batcheslog_every_n_steps	max_stepsr&   pluginsstrategyval_check_interval	callbacksfreeze_language_modelfreeze_vision_modelfreeze_audio_modelfreeze_vision_projectionfreeze_audio_projection)r?   r@   rA   rB   rC   r       r"      meta-llama/Meta-Llama-3-8B
seq_lengthglobal_batch_sizemicro_batch_size	tokenizerimage_processornum_workersr   r%   r   r	   h㈵>gH׊>   max_lrmin_lrwarmup_steps)path)restore_config)r   trainerdatalogoptimresumeNr#   lora)z*.language_model.*.linear_qkvz*.language_model.*.linear_qz*.language_model.*.linear_kvz*.language_model.*.linear_projz*.language_model.*.linear_fc1z*.language_model.*.linear_fc2)target_modulesg-C6?zUnrecognized peft scheme: )!r   r   nlMegatronStrategytorchbfloat16Trainerr   r   Partialr   finetuner   r   AVLMConfig8Br   r   r   default_logr	   r
   
AutoResumeRestoreConfiglowerrW   r<   r,   rZ   r   lrr   peft
ValueError)
r$   r%   r   r&   r'   r(   r)   r<   rW   reciper   r   r   finetune_recipe@   s|   #


	


rn   language_model_from_pretrainedc           	      C   s   t jtjdddtjd}t jtjdd|ddd|t |dt tgd	}t j	t
jtt jtj||d
 |d |d |d |d dd|t jtdddt tdt tddt
j| |t|ddtddddd}|S )a  
    Create a Pre-training recipe for AVLM 8B model.

    This function sets up a complete configuration for pre-training, including
    model, trainer, data, logging, optimization, and resumption settings.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm pretrain --factory llava_next_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava_next_7b_pretrain", num_nodes=1)
            >>> print(recipe)
    r*   r!   r   r+   r0   r1   r2   r3   r4   r?   r@   rA   rB   rC   )ro   r?   r@   rA   rB   rC   r   rD   r"   rF   rG   r   rN   gMbP?rO   rP   rQ   )r   rW   rX   rY   rZ   )r   r   r^   r_   r`   ra   rb   r   r   rc   r   pretrainr   r   re   r   r   r   rf   r	   r
   )	r%   r   r&   r'   ro   r)   r<   rW   rm   r   r   r   pretrain_recipe   s`   !

	rq   )r   Nr    r!   r"   r#   N)Nr    r!   r"   NN),typingr   lightning.pytorchpytorchplnemo_runr   r`   nemor   r^   nemo.collectionsr   r   nemo.collections.avlmr   =nemo.collections.common.tokenizers.huggingface.auto_tokenizerr   nemo.collections.llm.peftr   (nemo.collections.llm.recipes.log.defaultr	   'nemo.collections.llm.recipes.optim.adamr
   6nemo.collections.llm.recipes.precision.mixed_precisionr   nemo.utils.exp_managerr   NAMEr   clifactoryr   re   LightningModuler   rd   strintdictrc   rn   rp   rq   r   r   r   r   <module>   s   
$m