o
    }oi                     @   s   d dl Z d dlZd dlmZ dZdZdZdZe jdd Z	e j
je j
d	edd
dfedd
dfedg ddfedddfedddfedddfedddfgdd ZdS )    N)FLOPsMeasurementCallbacka  
    run:
        name: train_llama2_7b_tp1_pp1_FP8_1node_15steps
    trainer:
        num_nodes: 1
        devices: 8
        precision: bf16
    exp_manager:
        explicit_log_dir: "results/logs"
    model:
        micro_batch_size: 1
        global_batch_size: 128
        encoder_seq_length: 4096
        max_position_embeddings: 4096
        num_layers: 32
        hidden_size: 4096
        ffn_hidden_size: 11008
        num_attention_heads: 32
a  
    run:
      name: train_nemotron_8b_tp2_pp1_FP8_8node_20steps
    trainer:
      num_nodes: 8
      devices: 8
      precision: bf16
    exp_manager:
      explicit_log_dir: null
    model:
      micro_batch_size: 4
      global_batch_size: 256
      encoder_seq_length: 4096
      max_position_embeddings: 4096
      num_layers: 32
      hidden_size: 4096
      ffn_hidden_size: 16384
      num_attention_heads: 32
      fp8: true
a  
    run:
        name: unsupported_model
    trainer:
        num_nodes: 1
        devices: 8
        precision: bf64
    exp_manager:
        explicit_log_dir: null
    model:
        micro_batch_size: 1
        global_batch_size: 1
        encoder_seq_length: 1
        max_position_embeddings: 1
        num_layers: 1
        hidden_size: 1
        ffn_hidden_size: 1
        num_attention_heads: 1
z
    run:
        name: null
c                 C   s
   t | S )N)yaml	safe_load)cfg r   ^/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/common/test_perf_metrics.pymodel_configW   s   
r   z0cfg, model_name, train_step_time, expected_value   gGzw@llama2)r	   r	   r	   r	   g(\?gp=
@   z[Failed to extract valid model name from or missing FLOPs calculations for unsupported_modelunknown_modelzWFailed to extract valid model name from or missing FLOPs calculations for unknown_modelzNFailed to extract valid model name from or missing FLOPs calculations for Nonec                 C   s*  t |ttfrit| |d}||}|tj|ddksJ |d u ri| d d dd| d d< t| |d}||}|tj|ddksEJ | d d dd| d d< t| |d}||}|tj|ddksiJ t |trt| |d}tj	t
|d	 ||}W d    d S 1 sw   Y  d S d S )
N)
model_nameg-C6?)relrunname_. )match)
isinstanceintfloatr   eval_tflops_per_sec_per_gpupytestapproxreplacestrraisesKeyError)r   r   train_step_timeexpected_valueflops_callbacktflops_per_sec_per_gpur   r   r   r    test_eval_tflops_per_sec_per_gpu\   s&   



"r#   )r   r   ,nemo.collections.common.metrics.perf_metricsr   LLAMA2_CFG_STRNEMOTRON_CFG_STRUNSUPPORTED_MODEL_CFG_STRNULL_MODEL_CFG_STRfixturer   markunitparametrizer#   r   r   r   r   <module>   sD   



