o
    }oiU                     @   s~   d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ ddefd	d
ZG dd de	ZdddZdS )    N)Path)sleep)LightningEnvironment)DDPStrategyStrategyRegistry)loggingFreturnc              	   C   sR  |du rt t  d}n|}|durt ||}n|}tj|s)tj|dd t ||}tj|r_td| d |rStd td|  t	| ntd	|  t
|S | | }td
| dt
|  d}	d}
|	|
k r|	d7 }	zt|t
| tj|r|W S W dS    td|	 d|
  td Y qttd)a  
    Helper function to download pre-trained weights from the cloud
    Args:
        url: (str) URL of storage
        filename: (str) what to download. The request will be issued to url/filename
        subfolder: (str) subfolder within cache_dir. The file will be stored in cache_dir/subfolder. Subfolder can
            be empty
        cache_dir: (str) a cache directory where to download. If not present, this function will attempt to create it.
            If None (default), then it will be $HOME/.cache/torch/NeMo
        refresh_cache: (bool) if True and cached file is present, it will delete it and re-fetch

    Returns:
        If successful - absolute local path to the downloaded file
        else - empty string
    Nz.cache/torch/NeMoT)exist_okzFound existing object .zAsked to refresh the cache.zDeleting file: zRe-using file from: zDownloading from: z to r          z$Download from cloud failed. Attempt z of g?z5Not able to download url right now, please try again.)r   joinpathhomeospathexistsmakedirsr   inforemovestrwgetdownloadr   
ValueError)urlfilename	subfolder	cache_dirrefresh_cachecache_locationdestinationdestination_filewget_uriimax_attempts r%   D/home/ubuntu/.local/lib/python3.10/site-packages/nemo/utils/cloud.pymaybe_download_from_cloud   sB   
r'   c                   @   s&   e Zd Zedd Zejdd ZdS )SageMakerDDPStrategyc                 C   s   t  }dd |_dd |_|S )Nc                   S      t tjd S )N
WORLD_SIZEintr   environr%   r%   r%   r&   <lambda>[       z:SageMakerDDPStrategy.cluster_environment.<locals>.<lambda>c                   S   r)   )NRANKr+   r%   r%   r%   r&   r.   \   r/   )r   
world_sizeglobal_rankselfenvr%   r%   r&   cluster_environmentX   s   

z(SageMakerDDPStrategy.cluster_environmentc                 C   s   d S Nr%   r3   r%   r%   r&   r6   _   s   N)__name__
__module____qualname__propertyr6   setterr%   r%   r%   r&   r(   W   s
    
r(   c                  C   s   t jdtddd ddd} ddd	}|  tjd
rAtjdrAddlm  m  m	} ddl
}|  | r=|   | S |   dS )z
    Helper function to initiate sagemaker with NeMo.
    This function installs libraries that NeMo requires for the ASR toolkit + initializes sagemaker ddp.
    smddpF)namestrategyprocess_group_backendfind_unused_parametersr   Nc                   S   s   t d d S )NzIchmod 777 /tmp && apt-get update && apt-get install -y libsndfile1 ffmpeg)r   systemr%   r%   r%   r&   _install_system_librariesr   s   z7initialize_sagemaker.<locals>._install_system_librariesc                  S   s   ddl m}  dd }|| _dS )z
        Patches torchmetrics to not rely on internal state.
        This is because sagemaker DDP overrides the `__init__` function of the modules to do automatic-partitioning.
        r   )Metricc                 S   s   | j jt| g}tt|S r7   )	__class__r8   idhashtuple)r4   	hash_valsr%   r%   r&   __new_hash__|   s   zHinitialize_sagemaker.<locals>._patch_torch_metrics.<locals>.__new_hash__N)torchmetricsrD   __hash__)rD   rJ   r%   r%   r&   _patch_torch_metricsu   s   
z2initialize_sagemaker.<locals>._patch_torch_metricsr0   r*   r   r   N)r   registerr(   r   r-   get,smdistributed.dataparallel.torch.distributeddataparalleltorchdistributed,smdistributed.dataparallel.torch.torch_smddpinit_process_groupget_local_rankbarrier)rC   rM   distsmdistributedr%   r%   r&   initialize_sagemakere   s$   

r[   )NNFrN   )r   pathlibr   timer   r   &lightning.pytorch.plugins.environmentsr   lightning.pytorch.strategiesr   r   
nemo.utilsr   r   r'   r(   r[   r%   r%   r%   r&   <module>   s   =