o
    }oi                  #   @   s   d dl Z d dlZd dlmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ e deZdZd	d
g i g dedddf
dededededededededee deeef dee dededededee dejf"ddZdS )    N)DictList)get_nemorun_home)SlurmTemplate)DEFAULT_NEMO_CACHE_HOME)logging	NEMO_HOMEz
#!/usr/bin/env bash
set -euo pipefail

# NOTE: DO NOT change the single quotes to double quotes.
bash -c '{{ pre_cmds }} {{ command }}'
z00:30:00znvcr.io/nvidia/nemo:devgpuaccount	partitionlog_dirnodesnum_gpus_per_node
time_limitcontainer_imagecustom_mountscustom_env_varscustom_srun_argshf_token	nemo_home	wandb_keynetworkcustom_bash_cmdsreturnc              	   C   s  dddddddd}|du rg n|}g }g }|
  dg }|t kr+|d| d t|d	kr>td
| td | 	 dvrHd|d< ||d< |durT||d< | 	 dkrbd|d< d|d< |t
kru||d< || d| g |dur||dd ||	O }|| d}|dkr|dkrtdd	dD ]}|| d	kr|} nq| 	 dkrdnd}d| d| d}|| ttdd |id!}tjd5i d"|d#|d$tjtj|d%d&d'|d(|d)|d*|d+|d,|d-|d.dd/d0d1t d2|d3|d4|}|S )6z
    Slurm cluster definition with appropriate cluster params and NeMo container params needed for pre-training
    and fine-tuning experiments
    1False0)TORCH_NCCL_AVOID_RECORD_STREAMSTRANSFORMERS_OFFLINETOKENIZERS_PARALLELISMNCCL_NVLS_ENABLENVTE_FLASH_ATTNNVTE_FUSED_ATTNNEMO_LOG_MEMORY_USAGENz
--mpi=pmixz
Run `export NEMORUN_HOME=z2` in your shell environment and rerun this script.r   
   )b200zexpandable_segments:TruePYTORCH_CUDA_ALLOC_CONFNEMORUN_HOMEWANDB_API_KEYgb200PHBNCCL_NET_GDR_LEVELNCCL_NET_GDR_C2Cr   :)HF_TOKENr            z'numactl --cpunodebind=$((SLURM_LOCALID/z)) --membind=$((SLURM_LOCALID/z))pre_cmdsz ; )template_inlinetemplate_varsr
   r   tunnelexperiments)job_dirr   ntasks_per_noder   container_mountsenv_vars	srun_argstimemem	exclusiveTpackagersegmentr   launcher )copyr   appendlenr   errorjoinsysexitlowerr   extendupdateranger   INLINE_TEMPLATErunSlurmExecutorLocalTunnelospathGitArchivePackager)r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   PERF_ENV_VARSerr_msgsmountsr=   rB   segment_candidatenuma_divisornuma_cmdrC   executorrD   rD   Q/home/ubuntu/.local/lib/python3.10/site-packages/scripts/performance/executors.pyslurm_executor'   s   





	
r_   )rT   rJ   typingr   r   nemo_runrQ   nemo_run.configr    nemo_run.core.execution.launcherr   nemo.lightning.baser   
nemo.utilsr   getenvDEFAULT_NEMO_HOMErP   strintrR   r_   rD   rD   rD   r^   <module>   sn   	

