o $iL @sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZmZddlZddlZddlmZddlmZddlmZmZmZddlZdZd Zd d gZGd d d eZGdddeZ GdddeZ!GdddZ"GdddZ#GdddZ$  dHde$de ee%de%de$fddZ& dIde$de%de$fd d!Z'de$fd"d#Z(  $dJde$d%e e ee%e)fd&e)fd'd(Z*de$d)e"fd*d+Z+dKd-e e%fd.d/Z, 0dLd1e#d)e"d2e%de e%fd3d4Z-de$d1e#d)e"fd5d6Z.de$d)e"fd7d8Z/de$d9ee#d)e"fd:d;Z0de$d9ee#d)e"fde%dee e%e%e%e e%e e%ffd?d@Z2     dMdAe e%dBe e%dCe e%dDe e%dEe e%f dFdGZ3dS)NN)ThreadPoolExecutor)contextmanager)ListOptionalSequenceTuple) cli_logger)_get_node_provider)NODE_KIND_HEADNODE_KIND_WORKERTAG_RAY_NODE_KINDubuntu~/ray_bootstrap_key.pemz%~/.ssh/ray-autoscaler_2_us-west-2.pemc@ eZdZdS) CommandFailedN__name__ __module__ __qualname__rra/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/autoscaler/_private/cluster_dump.pyrrc@r)LocalCommandFailedNrrrrrrrrc@r)RemoteCommandFailedNrrrrrr#rrc@sLeZdZ      d dedededededeeeeeff d d ZdS) GetParametersTNlogs debug_statepip processesprocesses_verboseprocesses_listcCs(||_||_||_||_||_||_dSN)rrrrr r!)selfrrrrr r!rrr__init__(s   zGetParameters.__init__)TTTTTN) rrrboolrrrstrr$rrrrr's(rc @s<eZdZdZ    d dededed eed ef d d ZdS)NodezNode (as in "machine")rrNFhostssh_userssh_keydocker_containeris_headcCs"||_||_||_||_||_dSr")r(r)r*r+r,)r#r(r)r*r+r,rrrr$<s  z Node.__init__)rrNF)rrr__doc__r&rr%r$rrrrr'9s"r'c@sleZdZdZddeefddZeddZdd Z d d Z d d Z ddZ e ddedeefddZdS)ArchiveaArchive object to collect and compress files into a single file. Objects of this class can be passed around to different data collection functions. These functions can use the :meth:`subdir` method to add files to a sub directory of the archive. NfilecCs,|p tjdddd|_d|_t|_dS)N ray_logs_.tar.gzprefixsuffix)tempfilemkstempr/tar threadingLock_lock)r#r/rrrr$TszArchive.__init__cCs t|jSr")r%r8r#rrris_openYs zArchive.is_opencCst|jd|_dS)Nzw:gz)tarfileopenr/r8r<rrrr?]sz Archive.opencCs|jd|_dSr")r8closer<rrrr@`s  z Archive.closecCs ||Sr")r?r<rrr __enter__dszArchive.__enter__cCs |dSr")r@)r#exc_typeexc_valexc_tbrrr__exit__hs zArchive.__exit__/subdirrootc#s0tjGfddd}|VdS)a Open a context to add files to the archive. Example: .. code-block:: python with Archive("file.tar.gz") as archive: with archive.subdir("logfiles", root="/tmp/logs") as sd: # Will be added as `logfiles/nested/file.txt` sd.add("/tmp/logs/nested/file.txt") Args: subdir: Subdir to which to add files to. Calling the ``add(path)`` command will place files into the ``subdir`` directory of the archive. root: Root path. Files without an explicit ``arcname`` will be named relatively to this path. Yields: A context object that can be used to add files to the archive. cs0eZdZeddedeeffdd ZdS)z Archive.subdir.._ContextNpatharcnamecsPtj|}|ptjtj|}jjj||dj dS)NrJ) osrIabspathjoinrelpathr;acquirer8addrelease)rIrJrHr#rGrrrQs  z$Archive.subdir.._Context.addr")rrr staticmethodr&rrQrrSrr_Contexts&rUN)rLrIrM)r#rGrHrUrrSrrGks  zArchive.subdirr")rF)rrrr-rr&r$propertyr=r?r@rArErrGrrrrr.Ks r./tmp/ray/session_latestarchiveexcludesession_log_dirreturnc s|js||p g}tjtj|d}|jd|d9}t|D]*\}}}|D]"}tj||}tjj||dt fdd|DrGq*| |q*q#Wd|S1sYwY|S)aCopy local log files into an archive. Args: archive: Archive object to add log files to. exclude (Sequence[str]): Sequence of regex patterns. Files that match any of these patterns will not be included in the archive. session_dir: Path to the Ray session files. Defaults to ``/tmp/ray/session_latest`` Returns: Open archive object. rrH)startc3s|] }t|VqdSr")rematch).0patternrel_pathrr sz%get_local_ray_logs..N) r=r?rLrIrN expanduserrGwalkrOanyrQ) rXrYrZsdrHdirsfilesr/ file_pathrrbrget_local_ray_logss$    rl session_dircCsz|js|tj|}tj|d}tj|std|jd|d}| |Wd|S1s6wY|S)zCopy local log files into an archive. Args: archive: Archive object to add log files to. session_dir: Path to the Ray session files. Defaults to ``/tmp/ray/session_latest`` Returns: Open archive object. zlogs/debug_state.txtz No `debug_state.txt` file found.r\N) r=r?rLrIrerNexistsrrGrQ)rXrmdebug_state_filerhrrrget_local_debug_states    rqc Cs|js|zddlm}Wntyddlm}Ynwtd?}|D] }||dgq)| | d}| |j dWdn1sNwYWd|SWd|S1sfwY|S)zGet currently installed pip packages and write into an archive. Args: archive: Archive object to add meta files to. Returns: Open archive object. r)freezewt rnzpip_packages.txtN) r=r?pip._internal.operationsrr ImportErrorpip.operationsr6NamedTemporaryFile writelinesflushrGrQname)rXrrfplinerhrrrget_local_pip_packagess*       r~Frverbosec Cs|s ddlm}|}g}tgdD]Q}z>|0d|}||r(|n |ddddd| |j | d |fWdn1sMwYWqt yd}zt ||d}~wwi}|D]'\} }|D] \} } | rz| d } nt|} | | vr| d |vr| || d <qoqitd B} |D] }| t|d gq| |d}|| j dWdn1swYWd|SWd|S1swY|S)a7Get the status of all the relevant ray processes. Args: archive: Archive object to add process info files to. processes: List of processes to get information on. The first element of the tuple is a string to filter by, and the second element is a boolean indicating if we should filter by command name (True) or command line including parameters (False) verbose: If True, show entire executable command line. If False, show just the first term. Returns: Open archive object. r) RAY_PROCESSES)pidr{cmdlinestatus z--r5N) executabler{rrr{rrsrtmetazprocess_info.txt)!ray.autoscaler._private.constantsrpsutil process_iteroneshotrNrappendsplitr{rr Exceptionr subprocess list2cmdliner6rxvaluesryyamldumprzrGrQ)rXrrr process_infosprocessrexcrelevant_processes process_dictkeyword filter_by_cmdcorpusr|r}rhrrrget_local_ray_processessb             r parametersc Cs.|js||jr)zt|dWnty(}z t|WYd}~nd}~ww|jrKzt|dWntyJ}z t|WYd}~nd}~ww|j rmzt |dWntyl}z t|WYd}~nd}~ww|j rz t ||j |jdW|Sty}z t|WYd}~|Sd}~ww|S)a'Get all local data. Gets: - The Ray logs of the latest session - The currently installed pip packages Args: archive: Archive object to add meta files to. parameters: Parameters (settings) for getting data. Returns: Open archive object. )rXN)rXrr)r=r?rrlrrerrorrrqrr~rrr!r )rXrrrrrget_all_local_data6sJr'itemscCs|d||S)Nr)rN)rquotesrrr_wraphsrray remote_node script_pathc Csddddd|j|jd|jg}|jr|dd|jg7}|d d g}||jr(d gnd g7}||jr3d gndg7}||jr>dgndg7}||jrIdgndg7}|jr[||jrWdgndg7}|ddt |ddg7}|j skdnd}t d|jt jd|d|jdddd }t|d!-}z tj||tjd"Wntjy}z td#d$||d%}~wwWd%|S1swY|S)&aCreate an archive containing logs on a remote node and transfer. This will call ``ray local-dump --stream`` on the remote node. The resulting file will be saved locally in a temporary file and returned. Args: remote_node: Remote node to gather archive from. script_path: Path to this script on the remote node. parameters: Parameters (settings) for getting data. Returns: Path to a temporary file containing the node's collected data. sshz-o StrictHostKeyChecking=noz-o UserKnownHostsFile=/dev/nullz-o LogLevel=ERRORz-i@dockerexecz local-dumpz--streamz--logsz --no-logsz --debug-statez--no-debug-statez--pipz--no-pipz --processesz--no-processesz--processes-verbosez--no-proccesses-verbosez /bin/bashz-c")rnodeheadz"Collecting data from remote node: ray__r1r2r5wb)stdoutstderrz(Gathering logs from remote node failed: rN)r*r)r(r+rrrrr rr,rprintr6r7r?r check_callsysrCalledProcessErrorrrN) rrrcmd collect_cmdcattmpr|rrrr'create_and_get_archive_from_remote_nodelsZ  "  rcCst||}|js ||jsdnd}|jdtj|d}|j|d|d|j ddWd |S1s9wY|S) a%Create and get data from remote node and add to local archive. Args: archive: Archive object to add remote data to. remote_node: Remote node to gather archive from. parameters: Parameters (settings) for getting data. Returns: Open archive object. rrrnr\rrr1rKN) rr=r?r,rGrLrIdirnamerQr()rXrrrrrhrrr+create_and_add_remote_data_to_local_archives   rcCst }t||Wdn1swY|js||jdtj|jd}|j |jddWdn1s>wYt |j|S)zCreate and get data from this node and add to archive. Args: archive: Archive object to add remote data to. parameters: Parameters (settings) for getting data. Returns: Open archive object. Nrnr\zlocal_node.tar.gzrK) r.rr=r?rGrLrIrr/rQremove)rXrlocal_data_archiverhrrr*create_and_add_local_data_to_local_archives  r remote_nodescCsZ|js|ttd}|D] }|jt|||dqWd|S1s&wY|S)agCreate an archive combining data from the remote nodes. This will parallelize calls to get data from remote nodes. Args: archive: Archive object to add remote data to. remote_nodes (Sequence[Node]): Sequence of remote nodes. parameters: Parameters (settings) for getting data. Returns: Open archive object. ) max_workers)rXrrN)r=r?rMAX_PARALLEL_SSH_WORKERSsubmitr)rXrrexecutorrrrrcreate_archive_for_remote_nodess    rc Csr|js|zt||Wnty%}z t|WYd}~nd}~wwt|||tdt|d|S)ajCreate an archive combining data from the local and remote nodes. This will parallelize calls to get data from remote nodes. Args: archive: Archive object to add data to. remote_nodes (Sequence[Node]): Sequence of remote nodes. parameters: Parameters (settings) for getting data. Returns: Open archive object. Nz#Collected data from local node and z remote nodes.) r=r?rrrrrrlen)rXrrrrrr)create_archive_for_local_and_remote_nodess rcluster_configc sddlm}td|tj|}tt | }||dd}t |d|d t ti} t ti}fdd ||D}|d d }|d d }d }|dd } | ra| dd }|dd } ||||| fS)a[Get information from Ray cluster config. Return list of host IPs, ssh user, ssh key file, and optional docker container. Args: cluster_config: Path to ray cluster config. Returns: Tuple of list of host IPs, ssh user name, ssh key file path, optional docker container name, optional cluster name. r)_bootstrap_configz6Retrieving cluster information from ray cluster file: T)no_config_cacheprovider cluster_namecsg|]}|qSr) external_ip)r`rrrr @sz4get_info_from_ray_cluster_config..authr)ssh_private_keyNrcontainer_name) ray.autoscaler._private.commandsrrrrLrIrer safe_loadr?readr non_terminated_nodesr r r get) rrconfig head_nodes worker_nodeshostsr)r*r docker_configrrrr get_info_from_ray_cluster_config"s&        rclusterr(r)r*rcCs"|s|stjd}tj|r|}td|dn|r$tj|}d}|rPt|\}}} } }|p4|}|p8| }|p<| }|rD|dn|} | sOtd|n |rX|d} ntd|sit }td|d |st D]} tj| } tj| r| }td |d nqm|| ||||fS) ziParse command line arguments. Note: This returns a list of hosts, not a comma separated string! z~/ray_bootstrap_config.yamlz Detected cluster config file at z@. If this is incorrect, specify with `ray cluster-dump `N,z6Invalid cluster file or cluster has no running nodes: z` or `--host`.zUsing default SSH user `z9`. If this is incorrect, specify with `--ssh-user `zAuto detected SSH key file: z6. If this is incorrect, specify with `--ssh-key `) rLrIrerorwarningrrrDEFAULT_SSH_USERDEFAULT_SSH_KEYS)rr(r)r*rbootstrap_configrhukdrcand_key cand_key_filerrr_info_from_paramsNsV         r)NrW)rW)NF)r)r)NNNNN)4rLr^rrr>r6r9concurrent.futuresr contextlibrtypingrrrrrr"ray.autoscaler._private.cli_loggerr!ray.autoscaler._private.providersr ray.autoscaler.tagsr r r rrrr RuntimeErrorrrrrr'r.r&rlrqr~r%rrrrrrrrrrrrrrs    M  '  A2 @    # -