o
    }oi                     @   s   d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ dd	 Zd
d Zdd Zejjdddd Zejjdddd Zdd Zejjdddd Zedkrne  dS dS )    N)OptimizerConfig)	lightning)llm)PreTrainingDataModule)get_nmt_tokenizer)DdpParityCheckerc                  C   s@   t jdd} | jdtdd | jdtdd | jdtd	d | S )
Nz&Train a small GPT model using NeMo 2.0)descriptionz--data-pathzPath to data file)typehelpz--vocab-pathzPath to vocab filez--merges-pathzPath to merges file)argparseArgumentParseradd_argumentstr)parser r   [/home/ubuntu/.local/lib/python3.10/site-packages/tests/lightning/test_ddp_parity_checker.pymake_parser   s
   r   c                    "   G  fdddt | }|| _| S )Nc                       s"   e Zd Zdd fddZ  ZS )z"wrap_config.<locals>.ConfigWrapperNreturnMCoreGPTModelc                    s   t t |S N)make_byzantine_model_wrappersuperconfigure_model)self	tokenizervp_stage	__class__trainerr   r   r   (   s   z2wrap_config.<locals>.ConfigWrapper.configure_modelr   )r   r   )__name__
__module____qualname__r   __classcell__r   r   r   r   ConfigWrapper'   s    r&   r	   r   )configr   r&   r   r$   r   wrap_config&   s   r)   c                    r   )Nc                       s   e Zd Z fddZ  ZS )z4make_byzantine_model_wrapper.<locals>.ByzantineModelc              
      s   t  j|i |}t @ dd l}ttjd }|dkr&|W  d    S jj	j
jD ]}|jD ]}|d D ]}||dd q7q1q,W d    |S 1 sPw   Y  |S )Nr   
LOCAL_RANK   params)r   forwardtorchno_gradrandomintosenvironstrategymodeloptim_optimizersparam_groupsfill_uniform)r   anskwargsr0   rankoptgparamr   r   r   r-   1   s$   




z<make_byzantine_model_wrapper.<locals>.ByzantineModel.forward)r    r!   r"   r-   r#   r   r$   r%   r   ByzantineModel0   s    rA   r'   )r5   r   rA   r   r$   r   r   /   s   r   ztested with GH)reasonc                 C   s6   t jdd}t|| }t j|||d}| || d S N   )
num_layers)r   r6   )r   Llama2Config7Br)   
LlamaModelfitr   
ddp_parityr6   datar   r(   r5   r   r   r   test_failingC   s   
rL   c                 C   s,   t jdd}t j|||d}| || d S rC   )r   rF   rG   rH   rI   r   r   r   test_workingK   s   rM   c                 C   s   t d}tjdddtjdddtjdddd	dd |gd

}tjtdddddtjdd}t	dd| j
| jd}t| jddd|d}|||||fS )Nr+   rD      gpuF)ckpt_load_optimizerckpt_save_optimizerz
bf16-mixed)	precisionr   )
devices	max_stepsacceleratorr4   pluginslimit_val_batchesnum_sanity_val_stepslog_every_n_stepslogger	callbacksadamgh㈵>T)	optimizerlruse_distributed_optimizerfp16bf16params_dtype)r(   megatronGPT2BPETokenizer)
vocab_filemerges_filei       i  )paths
seq_lengthglobal_batch_sizeseedr   )r   nlTrainerMegatronStrategyMegatronMixedPrecisionMegatronOptimizerModuler   r.   float32r   
vocab_pathmerges_pathr   	data_path)argsrJ   r   r6   r   rK   r   r   r   make_trainer_optimR   sP   
rv   c                  C   sb   t   } t| \}}}}}t||||| |jdkrtdztt|   td W d S     )NTzDDP parity checking failed.z&DDP parity checking worked as expected)r   
parse_argsrv   rL   should_stop
ValueErrorrM   print)ru   r   rJ   r6   rK   r   r   r   r   main   s   

r{   __main__)r   r2   pytestr.   megatron.core.optimizerr   nemor   rl   nemo.collectionsr   nemo.collections.llm.gpt.datar   3nemo.collections.nlp.modules.common.tokenizer_utilsr    nemo.lightning.pytorch.callbacksr   r   r)   r   markskiprL   rM   rv   r{   r    r   r   r   r   <module>   s.   		

.

