o
    }oi
                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlZd dlZd dlm	Z	 d dlm
Z
 e d d ZeedeZed	 Zeed
eZed ZeedeZeddu r]edd 	ddededefddZdde	dee
j ddfddZddgZdS )    N)Path)Optional)Trainer)nnz.cachenemo	NEMO_HOMEdatasetsNEMO_DATASETS_CACHEmodelsNEMO_MODELS_CACHETOKENIZERS_PARALLELISMTrue   
vocab_sizemake_vocab_size_divisible_byreturnc              	   C   sT   ddl m} |}|| j }|| d | | }|d| d| d||  d |S )z^returns `vocab size + padding` to make sure sum is dividable by `make_vocab_size_divisible_by`r   )logging   zPadded vocab_size: z, original vocab_size: z, dummy tokens: .)
nemo.utilsr   tensor_model_parallel_sizeinfo)configr   r   r   aftermultiple r   G/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/base.pyget_vocab_size%   s   
r   trainermodelc                 C   s|   t j rddlm} |  t j  |   |dur3t	 D ]}zt 
|r,|jr,~W q    Y q t  t j  dS )zGDestroys distributed environment and cleans up cache / collects garbager   )parallel_stateN)torchdistributedis_initializedmegatron.corer    destroy_model_paralleldestroy_process_group	_teardowngcget_objects	is_tensoris_cudacollectcudaempty_cache)r   r   r    objr   r   r   teardown7   s   

r0   )r   )N)r(   ospathlibr   typingr   r!   torch.distributedlightning.pytorchr   r   homeDEFAULT_NEMO_CACHE_HOMEgetenvNEMO_CACHE_HOMEDEFAULT_NEMO_DATASETS_CACHEr	   DEFAULT_NEMO_MODELS_CACHEr   putenvintr   Moduler0   __all__r   r   r   r   <module>   s4   
