o
    wit                     @   s   d dl mZ d dlmZ d dlZd dlZd dlm	Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d	Zejjed
dejej fddZejjejed				ddee dedededejf
ddZ dS )    )OptionalN)	lightning)llmvlm)tensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)TimingCallbackclip_b32namereturnc                   C   s   t jtjt tjdS )a]  
    Factory function to create a Clip B32 model configuration.

    Returns:
        run.Co  nfig[pl.LightningModule]: Configuration for the Clip B32 model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=clip_b32 ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    )config)runConfigr   	CLIPModelCLIPConfigB32 r   r   b/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/vlm/recipes/clip_b32.pymodel    s   r   )targetr   default      dirr   	num_nodesnum_gpus_per_nodec                 C   s   t jtjdddtjd}t jtjd|ddd|t |dt tgd}t j	t
jt |t jtjd	d
dddddt
j| |t|ddtddddddt jtjdddd}|S )a  
    Create a fine-tuning recipe for Clip B32 model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory clip_b32

        Python API usage:
            >>> recipe = finetune_recipe(name="clip_b32", num_nodes=1)
            >>> print(recipe)

    Note:
        This recipe uses the Mock dataset for fine-tuning. For more information
        on fine-tuning LLMs with NeMo, see the fine-tuning guide in the
        `scripts/vlm/` directory.
    r   r   )tensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizepipeline_dtypegpu
   i  i  )
acceleratordeviceslimit_val_batcheslog_every_n_steps	max_stepsr   pluginsstrategyval_check_interval	callbacksP         N   )
seq_lengthglobal_batch_sizemicro_batch_size	tokenizerimage_processornum_workersr   )r   r   r   gMbP?gh㈵>i  g?g\(\?)max_lrmin_lrwarmup_steps
adam_beta1
adam_beta2FT)resume_if_existsresume_ignore_no_checkpoint)r   trainerdatalogoptimresume)r   r   nlMegatronStrategytorchbfloat16Trainerr   r	   Partialr   pretrainr   r   ClipMockDataModuledefault_logr   r   
AutoResume)r   r   r   r   r)   r=   reciper   r   r   pretrain_recipe3   sb   $	rM   )Nr   r   r   )!typingr   lightning.pytorchpytorchplnemo_runr   rD   nemor   rB   nemo.collectionsr   r   (nemo.collections.llm.recipes.log.defaultr   'nemo.collections.llm.recipes.optim.adamr   6nemo.collections.llm.recipes.precision.mixed_precisionr   nemo.utils.exp_managerr	   NAMEclifactoryr   LightningModuler   rH   strintrG   rM   r   r   r   r   <module>   s:   