o rri'@sddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z ddl m Z mZdZGdddZGd d d Zd e d e d edefddZdS)N)Path)loggerutils)DelayedSubmissionJobPathsSUBMITIT_PREEMPT_SIGNALcseZdZUdZejedZiZ e j e j e e fed<de jddffdd Zd2d d Zede fd d Zedefd dZdefddZede fddZede je fddZede fddZede fddZede je fddZede je fddZ ede!fddZ"ede!fdd Z#ede!fd!d"Z$ede!fd#d$Z%ede!fd%d&Z&de fd'd(Z'ede jfd)d*Z(d+ed,e)ddfd-d.Z*d/e!ddfd0d1Z+Z,S)3JobEnvironmentaVDescribe the environment inside which the job is running. This includes job id, number of GPUs available, ... This class can only be instantiated from a running submitit job. @plugin-dev: default implementation look for information into environment variables. Override _env to map environment variable to each property. USR2_envargsreturncs0|turtj|g|RSddlm}|S)Nr)plugins)r super__new__rget_job_environment)clsr r __class__Q/home/ubuntu/.local/lib/python3.10/site-packages/submitit/core/job_environment.pyr&s zJobEnvironment.__new__NcCs||_dSN)nameclusterselfrrr__init__.szJobEnvironment.__init__cCs*|j}|dr|dtd }|S)Nr )__name__endswithlenlower)rnrrrr1s zJobEnvironment.namecCstjd}t||j|jdS)zlProvides the paths used by submitit, including stdout, stderr, submitted_pickle and folder. SUBMITIT_FOLDER)job_idtask_id)osenvironrr$ global_rank)rfolderrrrpaths8s zJobEnvironment.pathscCstjdd|kS)zTests if we are running inside this environment. @plugin-dev: assumes that the SUBMITIT_EXECUTOR variable has been set to the executor name SUBMITIT_EXECUTORr)r&r'getrrrrr activated@szJobEnvironment.activatedcCstSr)socket gethostnamerrrrhostnameHzJobEnvironment.hostnamecCs|jgSr)r0rrrr hostnamesLr1zJobEnvironment.hostnamescCs|jr |jd|jS|jS)N_) array_job_id array_task_id raw_job_idrrrrr$PszJobEnvironment.job_idcCstj|jdS)Nr$)r&r'r rrrrr6WszJobEnvironment.raw_job_idcC&d}||jvr dStj|j|dS)Nr4r r&r'r,rr"rrrr4["zJobEnvironment.array_job_idcCr7)Nr5r8r9rrrr5`r:zJobEnvironment.array_task_idcCttj|jddS)z!Total number of tasks for the job num_tasksrintr&r'r,r rrrrr<ezJobEnvironment.num_taskscCr;)z!Total number of nodes for the job num_nodesrr=rrrrr@jr?zJobEnvironment.num_nodescCr;)zId of the current nodenoderr=rrrrrAor?zJobEnvironment.nodecCr;)zGlobal rank of the taskr(rr=rrrrr(tr?zJobEnvironment.global_rankcCr;)z/Local rank of the task, ie on the current node. local_rankrr=rrrrrByr?zJobEnvironment.local_rankcs^fdddD}d}jjjjg}|fddt||D7}d|}d|dS) Ncs g|] }|dt|qS)=getattr).0r"rrr s z+JobEnvironment.__repr__..)r$r0)rBrAr(cs,g|]\}}|dt|d|dqS)rC()rD)rFr"trrrrGs,z, zJobEnvironment(rI)r<r@zipjoin)rinfonamestotalsinfo_strrrr__repr__~s   zJobEnvironment.__repr__cCs8d|j}tt|d}|durtd|dtd|S)NSIGzUnknown signal z*, you may need to unset or update env var z (Eg: USR2))USR_SIGrEsignal RuntimeError_PREEMPT_SIG_ENV)rroutrrr_usr_sigs  zJobEnvironment._usr_sigr* submissioncCsBt|||}ttj|jttj|jt||jdS)zSet up signals handler for the current executable. The default implementation checkpoint the given submission and requeues it. @plugin-dev: Should be adapted to the signals used in this cluster. N) SignalHandlerrTSIGTERMbypassSIGCONTrXcheckpoint_and_try_requeue)rr*rYhandlerrrr_handle_signalss zJobEnvironment._handle_signals countdowncCsdS)zRequeue the current job. @plugin-dev:Must be overridden by JobEnvironment implementations. Use self.job_id to find what need to be requeued. Nr)rrarrr_requeueszJobEnvironment._requeuer N)-r __module__ __qualname____doc__r&r'r,rVrSr tpClassVarDictstr__annotations__Anyrr classmethodrpropertyrr*boolr-r0Sequencer2r$r6Optionalr4r5r>r<r@rAr(rBrQrXrr`rb __classcell__rrrrr sL     r c@seZdZdedededdfddZdefdd Zdd e d e j e j ddfd d Zdd e d e j e j ddfddZdd e d e j e j ddfddZdddZdS)rZenv job_pathsdelayedr NcCs*||_||_||_t|_t|_dSr)rs _job_paths_delayedr get_logger_loggertime _start_time)rrsrtrurrrrs  zSignalHandler.__init__cCst|j}|jjd}t|d|d}||k}|r1|jd|ddd|ddd|S|jd|ddd|ddd|S) N<g?iXzJob has timed out. Ran z.0fz minutes out of requested z minutes.zJob has not timed out. Ran )rzr{rw _timeout_minminryrM)rwalltime max_walltimeguaranteed_walltime timed_outrrr has_timed_outs zSignalHandler.has_timed_outsignumframecCs|jdt|jdS)NzBypassing signal )rywarningrTSignalsr)rrrrrrr\szSignalHandler.bypassc Cs|}|rdnd}|jdt|jdtd|d|jj }|dkr4|j d|d dS|j }|j |}d }t |jd rMt||jj|}n|rQd }|dkrWd }|rjd|d} |j | t| |j||dS)Nz timed-out preemptedCaught signal  on z: this job is .rz;Not checkpointing nor requeuing since I am a slave (procid=).r checkpointz timed-out and not checkpointableztimed-out too many timeszJob not requeued because: )rryrrTrrr.r/rsr(rMrw_timeout_countdownhasattrfunction _checkpointrvsubmitted_picklerUncompletedJobErrorrb_exit) rrrrcaseprocidrurano_requeue_reasonmessagerrrr^s0 "       z(SignalHandler.checkpoint_and_try_requeuecCs||jdt|jdt|jj}|r$|jd|ddS|j }t |j dr8t |j |j j|j j|dS)Nrrz-Not checkpointing since I am a slave (procid=rr)ryrMrTrrr.r/rsr(rwrrrrvrrr)rrrrrurrrcheckpoint_and_exits$  z!SignalHandler.checkpoint_and_exitcCs|jdtddS)Nz,Exiting gracefully after preemption/timeout.)ryrMsysexitrrrrrs zSignalHandler._exitrrc)rrdrer rrrrorr>rgrqtypes FrameTyper\r^rrrrrrrZs   !rZrufilepathrar cCsjtd|}|durdS||j|t|}||WddS1s.wYdS)zCall the checkpoint method and dump the updated delayed. Returns: -------- no_requeue_reason: str a string explaining while there was no requeuing, else empty string if requeuing works zCalling checkpoint method.Nz!checkpoint function returned Noner) rrxrM_checkpoint_function set_timeoutr}rtemporary_save_pathdump)rurra ckpt_delayedtmprrrrs   r)r&rTr.rrzrtypingrgpathlibrrrrrrrVr rZr>rjrrrrrs U