o
    }oi<                     @   s  d dl mZmZ d dlmZ d dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZm Z m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- dZ.ej/j0e.ddej1ej2 fddZ3												dCde4d e4d!eej5 d"ee4 d#e4d$e6d%e4d&e4d'e4d(e4d)e4d*ee7ej1e
   dej1ej8 fd+d,Z9ej/j0ee.d-dd.ddd/efd0ee: d1e:d'e4d(e4d2e6d3edej;fd4d5Z<d6ej;dej;fd7d8Z=ej/j0ee.d-		9	.			:			/dDd0ee: d;e:d1e:d'e4d(e4d<ee: d=ee4 d>ee6 d2e6dej;fd?d@Z>d6ej;d<e:dej;fdAdBZ?dS )E    )CallableOptionalN)Callback)DistributedDataParallelConfig)	lightning)finetunepretrain)MockDataModule)PackedSequenceSpecs)Llama4Experts128Config
LlamaModel)PEFT_STR2CLS)default_finetune_recipe)default_logdefault_resumetensorboard_logger),distributed_fused_adam_with_cosine_annealing)
bf16_mixed)GarbageCollectionCallback)MegatronCommOverlapCallback)MegatronTokenDropCallback)TimingCallbackllama4_e128namereturnc                   C   s   t jtt tdS )a  
    Factory function to create a Llama4 128-Experts (Maverick) model configuration.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the Llama4 128-Experts (Maverick) model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=llama4_e128 ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    )config)runConfigr   r    r   r   \/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/llm/recipes/llama4_e128.pymodel*   s   r!         T   @      { tensor_parallelismpipeline_parallelismpipeline_parallelism_typevirtual_pipeline_parallelismcontext_parallelismsequence_parallelismexpert_tensor_parallelismexpert_model_parallelism	num_nodesnum_gpus_per_node	max_steps	callbacksc                 C   sj   t jtj| |||||||dddt jtddddddddd}t jtjdd||	dd	d
|
|t |ddd}|S )az  
    Configure the NeMo Lightning Trainer for Llama4 128-Experts (Maverick) model.

    This function sets up the distributed training strategy and other training parameters.

    Args:
        tensor_parallelism (int): Degree of tensor model parallelism.
        pipeline_parallelism (int): Degree of pipeline model parallelism.
        pipeline_parallelism_type (Optional[torch.dtype]): Data type for pipeline parallelism.
        virtual_pipeline_parallelism (Optional[int]): Size of virtual pipeline parallelism.
        context_parallelism (int): Degree of context parallelism.
        sequence_parallelism (bool): Whether to use sequence parallelism.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.
        max_steps (int): Maximum number of training steps.
        callbacks (Optional[list[run.Config[Callback]]]): List of callback configurations.

    Returns:
        run.Config[nl.Trainer]: Configuration for the NeMo Lightning Trainer.

    Examples:
        CLI usage:
            $ nemo llm pretrain trainer=llama4_e128 ...

        Python API usage:
            >>> trainer_config = trainer(num_nodes=2, num_gpus_per_node=8)
            >>> print(trainer_config)

    Note:
        For more information on distributed training strategies, refer to the
        NeMo documentation on multi-GPU and multi-node training.
    Toptim_grads_params)check_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gatheraverage_in_collectivedata_parallel_sharding_strategyN)tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelexpert_tensor_parallel_sizeexpert_model_parallel_sizegradient_as_bucket_viewckpt_async_saveckpt_parallel_loadddpfsdpgpur#   2       
   Fi  )acceleratoraccumulate_grad_batchesr3   deviceslimit_test_batcheslimit_val_batcheslog_every_n_stepsr2   r0   pluginsstrategyuse_distributed_samplerval_check_interval)r   r   nlMegatronStrategyr   Trainerr   )r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   rS   trainerr   r   r    rY   =   sR   .	rY   )targetr   defaultFdirr   performance_modefnc                 C   sb   t j|t t||t tgdt jtddddt| |t|ddt	dd	t
 d
}|r/t|}|S )a  
    Create a pre-training recipe for Llama4 128-Experts (Maverick) model.

    This function sets up a complete configuration for pre-training, including
    model, trainer, data, logging, optimization, and resumption settings.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the pre-training run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.
        performance_mode (bool): If true, enables optimizations for maximum performance.
        fn (Callable): The pre-training function to use.

    Returns:
        run.Partial: Partial configuration for pre-training.

    Examples:
        CLI usage:
            $ nemo llm pretrain --factory llama4_e128 ...
            $ nemo llm pretrain --factory "llama4_e128(num_nodes=2, name='my_pretrain')"

        Python API usage:
            >>> recipe = pretrain_recipe(name="llama4_e128_pretrain", num_nodes=2)
            >>> print(recipe)

    Note:
        For more details on pre-training LLMs with NeMo, see the pre-training
        guide in the `examples/llm/pretrain/` directory.
    )r0   r1   r3   i    i   r#   )
seq_lengthglobal_batch_sizemicro_batch_sizer   )r\   r   r   ga2U0*3?)max_lr)r!   rY   datalogoptimresume)r   Partialr!   rY   r   r   r	   r   r   r   r   "pretrain_performance_optimizations)r\   r   r0   r1   r]   r^   reciper   r   r    pretrain_recipe   s    '
rj   ri   c                 C   sZ   | j jsg | j _tjtddd}tjtdd}tt}| j j|||g d| j j_	| S )a  
    Create a performance-optimized pre-training recipe for Llama4 128-Experts (Maverick) model.

    This method enables performance optimizations that may not be suitable for all use cases.
    It builds upon the standard pre-training recipe and adds additional performance enhancements.

    Args:
        recipe (run.Partial): Base pre-train recipe to which performance optimizations will be added

    Returns:
        run.Partial: Partial configuration for performance-optimized pre-training.

    Note:
        Use this method with caution and only when you need maximum performance.
        It may not be suitable for all hardware configurations or use cases.
    d   )gc_interval_traingc_interval_valTtp_comm_overlapF)
rY   r3   r   r   r   r   r   extendrR   r6   )ri   garbage_collection_callbackmcomm_overlap_callbacktoken_drop_callbackr   r   r    rh      s,   
rh   -meta-llama/Llama-4-Maverick-17B-128E-Instructloraresume_pathpeft_schemer_   packed_sequencec	           
      C   s@  |du r|}|du r|rdnd}t t || ||||}	|du s%| dkr:d|	jj_d|	jj_d|	jj_d|	jj	_
nC| dv rvd	|	jj_d
|	jj_d
|	jj_d|	jj_tt|  |	_d
|	j_d|	j_d|	jj	_d|	jj	_d|	jj	_
ntd| ||	jj	_||	j_|rdd	i|	j_tjt|d|	j_|rt|	|}	|	S )aL  
    Create a fine-tuning recipe for Llama4 128-Experts (Maverick) model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.
        peft_scheme (Optional[str]): Name of the peft scheme to use for fine-tuning.
            Allowed values: 'lora'/'dora'/'none'/None.
        seq_length (int): Maximum number of tokens per microbatch.
        packed_sequence (Optional[bool]): If true, fine-tuning sequences will be packed into batches up to the given
            maximum seq_length for better efficiency. By default, this value equals performance_mode.
        performance_mode (bool): If true, enables optimizations for maximum performance.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory llama4_e128

        Python API usage:
            >>> recipe = finetune_recipe(name="llama4_e128_finetune", num_nodes=2)
            >>> print(recipe)

    Note:
        This recipe uses the SQuAD dataset for fine-tuning. For more information
        on fine-tuning LLMs with NeMo, see the fine-tuning guide in the
        `examples/llm/finetune/` directory.
    Ni   i   noner"   rJ   gh㈵>)ru   doraTr&      Fg-C6?zUnrecognized peft scheme: pad_to_max_length)packed_sequence_size)r   r!   lowerrY   rS   r;   !expert_tensor_model_parallel_sizerB   re   r   lrr@   r<   r   r   r   peftdimalphause_distributed_optimizercross_entropy_loss_fusion
ValueErrorr_   rc   dataset_kwargsr
   packed_sequence_specs"finetune_performance_optimizations)
r\   rv   r   r0   r1   rw   r_   rx   r]   ri   r   r   r    finetune_recipe   sL   1	










r   c                 C   s   d| j j_t| j dr| j jdu rg | j _|du s| dkr.tjtdddddd| j j_	ndg| j
_d| j j_| j jtjtdd	 | j jtt | j jttd
d
 | S )a  
    Modify the given recipe to optimize settings for performance.

    This method enables performance optimizations that may not be suitable for all use cases.
    Intended to build upon the standard fine-tuning recipe.

    Args:
        recipe (run.Partial): Base fine-tuning recipe to which performance optimizations will be added
        peft_scheme (Optional[str]): Name of the peft scheme to use for fine-tuning.
            Allowed values: 'lora'/'dora'/'none'/None.

    Returns:
        run.Partial: Partial configuration for performance-optimized fine-tuning.

    Note:
        Use this method with caution and only when you need maximum performance.
        It may not be suitable for all hardware configurations or use cases.
    r#   r3   Nry   TF)r5   r6   r7   r8   r9   
linear_qkvrn   rk   )rY   rS   r;   hasattrr3   r~   r   r   r   rF   r   target_modulesrR   r6   appendr   r   r   )ri   rw   r   r   r    r   d  s:   

	
r   )r"   r#   NNr#   Tr"   r$   r%   r&   r'   N)	Nrt   r[   r#   r&   ru   NNF)@typingr   r   lightning.pytorchpytorchplnemo_runr   torch$lightning.pytorch.callbacks.callbackr   megatron.core.distributedr   nemor   rV   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr	   -nemo.collections.llm.gpt.data.packed_sequencer
   $nemo.collections.llm.gpt.model.llamar   r   nemo.collections.llm.peftr   -nemo.collections.llm.recipes.finetune_defaultr   (nemo.collections.llm.recipes.log.defaultr   r   r   'nemo.collections.llm.recipes.optim.adamr   6nemo.collections.llm.recipes.precision.mixed_precisionr   3nemo.lightning.pytorch.callbacks.garbage_collectionr   6nemo.lightning.pytorch.callbacks.megatron_comm_overlapr   /nemo.lightning.pytorch.callbacks.moe_token_dropr   nemo.utils.exp_managerr   NAMEclifactoryr   LightningModuler!   intdtypeboollistrX   rY   strrg   rj   rh   r   r   r   r   r   r    <module>   s   	


[:-	
c