o
    }oi                     @   s`   d dl mZ d dlZd dlZd dlmZ d dlmZ	 d dl
mZ G dd dZG dd dZdS )	    N)OptimizerConfig)	lightning)llmc                   @   s   e Zd Zdd ZdS )DummyTokenizerc                 C   s
   d| _ d S )Ni0u  )
vocab_size)self r   Y/home/ubuntu/.local/lib/python3.10/site-packages/tests/lightning/test_precision_plugin.py__init__   s   
zDummyTokenizer.__init__N)__name__
__module____qualname__r
   r   r   r   r	   r      s    r   c                   @   s8   e Zd ZdZejddd Zejddd ZdS )TestMegatronMixedPrecisionz0Unit tests for the MegatronMixedPrecision class.GPUc              
   C   s   G dd dt j}|dddt jdddddt jdd	d
ddd}t jtddddtjdd}t	 }d|_
tj|t |d}|j| dS )&Test __init__ with default parameters.c                       s(   e Zd Zdejddf fddZ  ZS )zPTestMegatronMixedPrecision.test_precision_plugin_fp8_passed.<locals>.TrainerHookmodelreturnNc                    sP   |j jdksJ |j jd u sJ t | |j jdksJ |j jdks&J d S )NFe4m3T)configbf16fp8superconnect)r   r   	__class__r   r	   r   %   s
   zXTestMegatronMixedPrecision.test_precision_plugin_fp8_passed.<locals>.TrainerHook.connect)r   r   r   plLightningModuler   __classcell__r   r   r   r	   TrainerHook$   s     r      gpuTFtensor_model_parallel_sizesequence_parallelckpt_load_optimizerckpt_save_optimizer
bf16-mixedr   )	precisionr           r   devicesaccelerator	max_stepsstrategypluginslimit_val_batchesnum_sanity_val_stepsadamh㈵>	optimizerlruse_distributed_optimizerfp16params_dtyper   	tokenizeroptimN)nlTrainerMegatronStrategyMegatronMixedPrecisionMegatronOptimizerModuler   torchfloat32r   Llama2Config7B
num_layers
LlamaModelr   r-   r   )r   r   trainerr<   r   r   r   r   r	    test_precision_plugin_fp8_passed    s8   	z;TestMegatronMixedPrecision.test_precision_plugin_fp8_passedc              
   C   s   t jdddt jdddddt jdddd	d
}t jtddddtjdd}t	 }d|_
d|_d|_tj|t |d}|j| |jjdusIJ |jjdusQJ |jjdksYJ |jjdksaJ |jjdksiJ |jjdksqJ dS )r   r   r    TFr!   r&   )r'   r(   r   r)   r1   r2   r3   r9   r:   N)r=   r>   r?   r@   rA   r   rB   rC   r   rD   rE   r7   r   rF   r   r-   r   r   )r   rG   r<   r   r   r   r   r	   /test_precision_plugin_precision_params_overrideI   sF   
	zJTestMegatronMixedPrecision.test_precision_plugin_precision_params_overrideN)	r   r   r   __doc__pytestmarkrun_only_onrH   rI   r   r   r   r	   r      s    


(r   )lightning.pytorchpytorchr   rK   rB   megatron.core.optimizerr   nemor   r=   nemo.collectionsr   r   r   r   r   r   r	   <module>   s   