o
    wi_$                     @   s  d dl mZ d dlmZ d dlZd dlZd dlm	Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ dZdd Z ej!j"ede#ej$fdej#ej% fddZ&ej!j"ej'edddddddddddddfde(dee( de(d e)d!e)d"ee( d#ee* dej+fd$d%Z,ej!j"ej-eddddddddddddfdee( de(d e)d!e)d&ee( d#ee* dej+fd'd(Z.dS ))    )OptionalN)	lightning)avlmllm)AVLMMockDataModule)AutoTokenizer)LoRA)nemo_resume)default_resumetensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)TimingCallbackavlm_8bc                  C   s   ddl m}  | dS )z_
    Create an image processor for the AVLM 8B model.
    This helps by pass fiddle.config
    r   AutoProcessorzopenai/clip-vit-large-patch14)transformersr   from_pretrainedr    r   b/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/avlm/recipes/avlm_8b.pycreate_image_processor$   s   
r   namereturnc                 C   s   t jtj| dS )aX  
    Factory function to create a AVLM 8B model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the AVLM 8B model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=avlm_8b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    config)runConfigr   	AVLMModelr   r   r   r   model.   s   r   )targetr    default      noneFTfreeze_language_modelfreeze_vision_modelfreeze_audio_modelfreeze_vision_projectionfreeze_audio_projectioncheckpoint_pathdirr   	num_nodesnum_gpus_per_nodepeft_schemefreeze_modulesc           
      C   s:  t jtjdddtjdd}t jtjdd|ddd|t |d	t tgd
}t j	t
jtt jtj|d |d |d |d |d dd|t jtdddt tdt tddt
j||t|ddtddddt| d}	|du ss| dkrd|	jj_d|	jj_|	S | d krt jtg d!d"|	_d#|	jj_|	S td$| )%ag  
    Create a fine-tuning recipe for AVLM 8B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory llava_next_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava_next_7b_finetune", num_nodes=1)
            >>> print(recipe)
       r#   r   Ftensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizepipeline_dtypeckpt_async_savegpu
   F    acceleratoraccumulate_grad_batchesdeviceslimit_val_batcheslog_every_n_steps	max_stepsr.   pluginsstrategyval_check_interval	callbacksr'   r(   r)   r*   r+   r&   r       r$      meta-llama/Meta-Llama-3-8B
seq_lengthglobal_batch_sizemicro_batch_size	tokenizerimage_processornum_workersr   r-   r   r   h㈵>gH׊>   max_lrmin_lrwarmup_stepsr   trainerdatalogoptimresumeNr%   lora)z*.language_model.*.linear_qkvz*.language_model.*.linear_qz*.language_model.*.linear_kvz*.language_model.*.linear_projz*.language_model.*.linear_fc1z*.language_model.*.linear_fc2)target_modulesg-C6?zUnrecognized peft scheme: ) r   r   nlMegatronStrategytorchbfloat16Trainerr   r   Partialr   finetuner   r   AVLMConfig8Br   r   r   default_logr   r   r	   lowerrZ   rE   r4   r]   r   lrr   peft
ValueError)
r,   r-   r   r.   r/   r0   r1   rE   rZ   reciper   r   r   finetune_recipeA   sx   )	


	


ro   language_model_from_pretrainedc           	      C   s   t jtjdddtjdd}t jtjdd|ddd|t |d	t tgd
}t j	t
jtt jtj||d |d |d |d |d dd|t jtdddt tdt tddt
j| |t|ddtddddt d}|S )a  
    Create a Pre-training recipe for AVLM 8B model.

    This function sets up a complete configuration for pre-training, including
    model, trainer, data, logging, optimization, and resumption settings.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm pretrain --factory llava_next_7b

        Python API usage:
            >>> recipe = finetune_recipe(name="llava_next_7b_pretrain", num_nodes=1)
            >>> print(recipe)
    r2   r#   r   Fr3   r9   r:   r;   r<   r=   r'   r(   r)   r*   r+   )rp   r'   r(   r)   r*   r+   r   rH   r$   rJ   rK   r   rR   gMbP?rS   rT   rU   rY   )r   r   ra   rb   rc   rd   re   r   r   rf   r   pretrainr   r   rh   r   r   r   ri   r   r   r
   )	r-   r   r.   r/   rp   r1   rE   rZ   rn   r   r   r   pretrain_recipe   sd   '	

	rr   )/typingr   lightning.pytorchpytorchplnemo_runr   rc   nemor   ra   nemo.collectionsr   r   nemo.collections.avlmr   =nemo.collections.common.tokenizers.huggingface.auto_tokenizerr   nemo.collections.llm.peftr   -nemo.collections.llm.recipes.finetune_defaultr	   (nemo.collections.llm.recipes.log.defaultr
   r   'nemo.collections.llm.recipes.optim.adamr   6nemo.collections.llm.recipes.precision.mixed_precisionr   nemo.utils.exp_managerr   NAMEr   clifactoryr   rh   LightningModuler   rg   strintdictrf   ro   rq   rr   r   r   r   r   <module>   s   
$q