o `۷i@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZmZddlZddlmZddlmZmZddlmZmZmZddlmZmZmZmZmZdd l m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'e(e)Z*d Z+ej,-ej,.ej,/e0d Z1dZ2dZ3dddZ4ddZ5ddZ6ddZ7de8fddZ9de:de e:e;ffddZGd#d$d$e'Z?Gd%d&d&e'Z@dS)'N)getuserquote)DictListOptionalTuple)%DEFAULT_OBJECT_STORE_MAX_MEMORY_BYTES)cf cli_logger)AUTOSCALER_NODE_SSH_INTERVAL_SAUTOSCALER_NODE_START_WAIT_S&DEFAULT_OBJECT_STORE_MEMORY_PROPORTION)check_bind_mounts_cmdcheck_docker_imagecheck_docker_running_cmddocker_start_cmdswith_docker_exec)LogTimer)ProcessRunnerErroris_output_redirectedrun_cmd_redirected)CommandRunnerInterface z_kubernetes/kubectl-rsync.shT)use_login_shells silent_rsynccCtdS)Nr_configr!r!\/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/autoscaler/_private/command_runner.pyis_rsync_silent1r#cC |td<dS)zChoose whether to silence rsync output. Most commands will want to list rsync'd files themselves rather than print the default rsync spew. rNrvalr!r!r"set_rsync_silent5s r(cCr)Nrrr!r!r!r"is_using_login_shells>r$r)r'cCr%)aChoose between login and non-interactive shells. Non-interactive shells have the benefit of receiving less output from subcommands (since progress bars and TTY control codes are not printed). Sometimes this can be significant since e.g. `pip install` prints hundreds of progress bar lines when downloading. Login shells have the benefit of working very close to how a proper bash session does, regarding how scripts execute and how the environment is setup. This is also how all commands were ran in the past. The only reason to use login shells over non-interactive shells is if you need some weird and non-robust tool to work. Args: val: If true, login shells will be used to run all commands. rNrr&r!r!r"set_using_login_shellsBs r*cmdenvironment_variablescCsPg}|D]\}}tj|dd}d|t|}||qd|}||S)a7Prepend environment variables to a shell command. Args: cmd: The base command. environment_variables (Dict[str, object]): The set of environment variables. If an environment variable value is a dict, it will automatically be converted to a one line yaml string. ),:) separatorsz export {}={};)itemsjsondumpsformatrappendjoin)r+r, as_stringskeyr'sall_varsr!r!r"_with_environment_variablesVs   r;cCsd|d}ddddt|gS)NzEsource ~/.bashrc; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && ()bashz--loginz-c-ir)r+force_interactiver!r!r"_with_interactiveis r@c@s$eZdZdddZddddZdS) SSHOptionsNcKsV||_dtjddddd|_|r#tjdkrn |jdd|d d |j|dS) Nnoyesrr)StrictHostKeyCheckingUserKnownHostsFileIdentitiesOnlyExitOnForwardFailureServerAliveIntervalServerAliveCountMaxwin32autoz{}/%C10s) ControlMaster ControlPathControlPersist)ssh_keyosdevnullarg_dictsysplatformupdater4)selfrP control_pathkwargsr!r!r"__init__rs$ zSSHOptions.__init__<timeoutcCsFd||jd<|jrd|jgng}|dddd|jDDS)Nz{}sConnectTimeoutr>cSg|] }|D]}|qqSr!r!).0yxr!r!r" sz2SSHOptions.to_ssh_options_list..css,|]\}}|durdd||gVqdS)Nz-oz{}={}r4)r`kvr!r!r" sz1SSHOptions.to_ssh_options_list..)r4rSrPr1)rWr]ssh_key_optionr!r!r"to_ssh_options_listszSSHOptions.to_ssh_options_listN)__name__ __module__ __qualname__rZrir!r!r!r"rAqs $rAc@seZdZddZddZddZddZ d&d eed e d e d e fddZ       d'de ede d e de ee e e fd e de eeefdedede d e defddZddZd(d d!Zd(d"d#Zd$d%ZdS))SSHCommandRunnerc Cst|}tt} tjdkrd} nd| dt|dt} ||_ ||_ ||_ ||_ ||_ ||_|d|_|d|_| |_d|_|dd|_t|j|j|jd|_dS)NrJz/tmp/ray_ssh_{}/{}ssh_private_keyssh_userssh_proxy_command ProxyCommand)hashlibsha256encode hexdigestrrTrUr4HASH_MAX_LENGTH cluster_name log_prefixprocess_runnernode_iduse_internal_ipprovidergetrorpssh_control_pathssh_iprqrA ssh_options) rWrzr|r~ auth_configryr{r}ssh_control_hash ssh_user_hashrr!r!r"rZs.     zSSHCommandRunner.__init__cCs"|jr |j|jS|j|jSrj)r}r~ internal_ipr| external_iprWr!r!r" _get_node_ipszSSHCommandRunner._get_node_ipcCs|}|durtd||St}tdbt|kra|j|jsi|}|durSSHCommandRunner._create_rsync_filter_args..cSsg|] }dd|gqS)z--filterzdir-merge,- {}rd)r`rr!r!r"rcs cSr_r!r!)r` args_listargr!r!r"rcs)r)rWoptionsrsync_excludes rsync_filters exclude_args filter_argsr!r!r"_create_rsync_filter_argssz*SSHCommandRunner._create_rsync_filter_argscCs||pi}tjdkr(ddg}||jjdd7}||d|j|j|g7}n.dg}|dt d g|jjddg7}|d g7}||j |d 7}||d|j|j|g7}t d t d ||j|tddSNrJscpz-rrr\z{}@{}:{}rsyncz--rshrz-avzrrrrrrTrUrrir4rprr list2cmdlinerr rr rr6rr#rWsourcetargetrrr!r!r" run_rsync_ups$  zSSHCommandRunner.run_rsync_upcCs|tjdkr$ddg}||jjdd7}|d|j|j||g7}n.dg}|dt d g|jjddg7}|d g7}||j |d 7}|d|j|j||g7}t d t d ||j|tddSrrrr!r!r"run_rsync_downs"  zSSHCommandRunner.run_rsync_downcCs*|jr d|j|j|jSd|j|jS)Nz&ssh -o IdentitiesOnly=yes -i {} {}@{} z ssh -o IdentitiesOnly=yes {}@{} )ror4rprrr!r!r"remote_shell_command_strs z)SSHCommandRunner.remote_shell_command_str)FFF) NrFNFNrKr0FFrj)rkrlrmrZrrrrrboolrrintrrobjectrrrrrr!r!r!r"rnsl& B    P   rnc@s2eZdZddZ         d1deed ed ed eee eefd ed ee ee fdedededefddZ d2ddZ d2ddZddZddZddZd3ddZd ed!e eefdefd"d#Zd$ed%e eefd&efd'd(Zd)eedeefd*d+Zd)eedeefd,d-Zd.edefd/d0ZdS)4DockerCommandRunnercKsPtdi||_|d|_||_d|_d|_|dd}|r#d|_dSd|_dS)Ncontainer_nameF use_podmanpodmandockerr!)rnssh_command_runnerr docker_confighome_dir initializedr docker_cmd)rWr common_argsrr!r!r"rZs  zDockerCommandRunner.__init__NrFrKr0r+r]rrrr,rrrrc Cs|dkrt|r||jdkrdn|j}|rt||}||jkr@|j|dd}tr2dt|}t|g|j t|jdd}| rF|d7}|j j ||||||d S) NrKrhostT)any_charrrwith_interactiverr)r]rrrr) rfindrr;_docker_expand_userr)r6r@rrrr) rWr+r]rrrr,rrrr!r!r"rs>   zDockerCommandRunner.runc Cs.|pi}tj||jj|d}tj|d}t j dkr3| dd}| dd}| dd}|jj d|d|jj d|td|jj|||d|r|d d stj|rd|d 7}td tj||g|jt|jd d}|jj d||j||j||tddSdSdS)N/rJ\ mkdir -p && chown -R rrrdocker_mount_if_possibleFz/.z mkdir -p {}rrz){} && rsync -e '{} exec -i' -avz {} {}:{})rQpathr6_get_docker_host_mount_locationrrylstripdirnamerstriprTrUreplacerrpr#r_check_container_statusrisdirrr4rrr)r)rWrrrhost_destinationhost_mount_locationprefixr!r!r"rs\          z DockerCommandRunner.run_rsync_upc Cs|pi}tj||jj|d}tj|d}| dd}|jj d|d|jj d|t d|ddkr@|d7}| d d s^|jj d |j|j||| ddt d|jj|||d dS) Nrrrrrr.rFz,rsync -e '{} exec -i' -avz --delete {}:{} {}r)rQrr6rrryrrrrrrpr#rr4rrrr)rWrrr host_sourcerhost_mount_location_unixr!r!r"rIs<     z"DockerCommandRunner.run_rsync_downcCs.|jdddd}|d|j|jS)Nrzssh -tt z {} exec -it {} /bin/bash )rrrstripr4rr)rW inner_strr!r!r"rks z,DockerCommandRunner.remote_shell_command_strcCsd}|jjd|jd|ddd}|}||vs d|vrF|jdkr*gd}nd d g}t|jd |jd d |dSdS)NNoExistz command -v z || echo ''Trr)z2curl -fsSL https://get.docker.com -o get-docker.shzsudo sh get-docker.shzsudo usermod -aG docker $USERz sudo systemctl restart docker -fzsudo apt-get updatezsudo apt-get -y install podmanz not installed. You can install z@ by adding the following commands to 'initialization_commands': r) rrrdecoderloggererror capitalizer6)rWno_existoutputcleaned_outputinstall_commandsr!r!r"_check_docker_installedus&    z+DockerCommandRunner._check_docker_installedcCsF|jrdS|jjt|j|jddd}d|vo"d|vS)NTr utf-8truezno such object) rrrrrrr rlower)rWrr!r!r"rs  z+DockerCommandRunner._check_container_statuscCs~|d}|dkr=|jdur$|jj|jd|jdddd|_|r/|d|jd S|s=|d kr=|d|jd S|S) N~rz exec z printenv HOMETr rz~/rrr) rrrrrrr rr)rWstringruser_posr!r!r"rs    z'DockerCommandRunner._docker_expand_userimagecleaned_bind_mountsc sd}jtjjdddd}||kr!tdj||jtjjdddd}z+t |}dd|D}fd d| D}||} | r\d}t d | W|SW|St j yntd Y|Sw) NFTr)rrrz[A container with name {} is running image {} instead of {} (which was provided in the YAML)cSsh|] }|ddqS) Destinationr)r)r`mntr!r!r" szLDockerCommandRunner._check_if_container_restart_is_needed..csh|] }|dqS)r)rr)r`rrr!r"rsz~This Docker Container is already running. Restarting the Docker container on this node to pick up the following file_mounts {}z`Unable to check if file_mounts specified in the YAML differ from those on the running container.)rrrrr rr rrr2loadskeysrJSONDecodeErrorr) rWrrre_init_required running_imagemounts active_mountsactive_remote_mountsrequested_remote_mountsunfulfilled_mountsr!rr"%_check_if_container_restart_is_neededs`       z9DockerCommandRunner._check_if_container_restart_is_neededas_head file_mounts sync_run_yetcCsBddg}|j|r dndd|jd}||jddr4|s'Jd |jd |j|d d n||jd |d|jd||}|D]}||dqLd}|} d} | r}| ||} | r}|jd|j } t d| |j| d d | r| r|sdS|j j|jdd|dd} d} zt| D]}|dr|dd} nqWntjy}z td| |d}~ww|jdg|j|rdnddg}t|j j|||j ||||j j| |j}|j|d d d}|D]}||vr|s|||||j dj|jtj| |j j|!dd|j |"|dz|d |d!|Wqt#y|jd"|ddd#}|dd$}|dd%}|dd&}|jd'ddd#}t$d(|d)|d*|d+|d,|d-|d|d.Yqwqd|_%|S)/Nz~/ray_bootstrap_config.yamlz~/ray_bootstrap_key.pemheadworker_imagerpull_before_runTz@Image must be included in config if pull_before_run is specifiedz {} pull {}r)rz image inspect z 1> /dev/null 2>&1 || z pull Fz stop zExecuting Docker command: %srz"inspect -f '{{json .Config.Env}}' r z/rootzHOME=rzHUnable to deserialize `image_env` to Python object. The `image_env` is: run_options _run_optionsz5rsync -e '{cmd} exec -i' -avz {src} {container}:{dst}rr)r+src containerdstzcat z1 >/dev/null 2>&1 || sudo chown $(id -u):$(id -g) zls -l rrrwhoamizFile (z) is owned by user:z and group:z with permissions (z). The current user (z) does not have permission to read these files, and Ray may not be able to autoscale. This can be resolved by installing `sudo` in your container, or adding a command like 'chown z' to your `setup_commands`.)&rrrrr4rcopypoprr*rr inforr rr2r  startswithsplitr"r rrrp_configure_runtime_auto_configure_shmryrrQrr6rrr Exceptionrr)rWr+r,r-BOOTSTRAP_MOUNTSspecific_imagerrdocker_run_executedcontainer_runningrequires_re_initdocker_stop_cmd image_envhome_directoryenv_varruser_docker_run_options start_commandmount lsl_string permissionsownerr current_userr!r!r"run_inits         4zDockerCommandRunner.run_initr2c Cs|jdr|S|jj|jdddd}d|vrNz|jjddd|d gWStyM}zt d t d ||WYd}~Sd}~ww|S) N#disable_automatic_runtime_detectionrzinfo -f '{{.Runtimes}}' Tr znvidia-container-runtimez nvidia-smiFz--runtime=nvidiaz7NVIDIA Container Runtime is present, but no GPUs found.znvidia-smi error: ) rrrrrr rr@r rdebug)rWr2runtime_outputrr!r!r"r>s*  z&DockerCommandRunner._configure_runtimec Cs|jdr|S|D]}d|vrtd||Sq z5|jjddd}tdd| d Dd  d }|d }t |t d t }|d|dgWSt yn}ztd||WYd}~Sd}~ww)Ndisable_shm_size_detectionz --shm-sizez;Bypassing automatic SHM-Detection because of `run_option`: zcat /proc/meminfo || trueTr cSsg|]}d|vr|qS) MemAvailabler!)r`lnr!r!r"rcsz;DockerCommandRunner._auto_configure_shm..rrrig?z --shm-size='zb'z5Received error while trying to auto-compute SHM size )rrr r;rrr rrr=minrr r@r)rWr2run_opt shm_outputavailable_memoryavailable_memory_bytesshm_sizerr!r!r"r?sH  z'DockerCommandRunner._auto_configure_shmrycCsddlm}||S)z0Return the docker host mount directory location.r)get_docker_host_mount_location)ray.autoscaler.sdkr^)rWryr^r!r!r"rs z3DockerCommandRunner._get_docker_host_mount_location) NrFNFNrKr0Frj)F)rkrlrmrZrrrrrrrrrrrrrrrr*rQr>r?rr!r!r!r"rsp    / 4"    8  "r)Artr2loggingrQrrTrgetpassrshlexrtypingrrrrrray._private.ray_constantsr "ray.autoscaler._private.cli_loggerr r !ray.autoscaler._private.constantsr r rray.autoscaler._private.dockerrrrrr!ray.autoscaler._private.log_timerr.ray.autoscaler._private.subprocess_output_utilrrrray.autoscaler.command_runnerr getLoggerrkr rxrr6rabspath__file__ KUBECTL_RSYNCMAX_HOME_RETRIESHOME_RETRY_DELAY_Sr r#r(r)rr*rrr;r@rArnrr!r!r!r"sJ        39