o $i>A@sddlZddlZddlZddlZddlZddlZddlZddlmm Z ddl m m Z ddlm mZddlmZmZddlmZddlmZddlmZddlmZddlmZddlmZdd lm Z m!Z!e"e#Z$Gd d d Z%e#d krej&d dZ'e'j(dde)dde'j(dde)dde'j(dde)dde'j(dde*dde'j(dde*dde'j(dde*dde'j(dde)ddde'j(d d!e*e j+d"de'j(d#de)dd$de'j(d%d!d&d'e j,e j-e j.d(e'j(d)d!e)e j/e j0de'j(d*d!e)e j1d+2e j1de'j(d,de*d-de'j(d.de*d/de'j(d0de)dd1de'j(d2de)dd3de'j(d4de)dd5de'j(d6d7d8d9e'j(d:d7d;d9e'j(dd!e)d?d@de'j(dAd!e)d?dBde'3Z4znej5dCkrne4j6ndZ7ej5dCkrye4j8ndDZ9ee4j:e4j;e4je4j?e4j@e7e9eZAe%e4jBe4jCe4jDe4jEe4jFe4jGe4jHe4jRsz)DashboardAgent.__init__..zget self node info) description max_attempts max_backoff_s)ipminimal gcs_addresscluster_id_hexrrr grpc_portmetrics_export_portnode_manager_portevents_export_addr listen_portrrosenvironrmetrics_collection_disabledrserver http_serverr rr is_head_nodeis_head_init_non_minimal)rnode_ip_addressr*r(r)r'r+r,r-r.r rrrrrr node_inforrr__init__sF     zDashboardAgent.__init__cCsdddlm}ddlm}ddlm}ddlm}ddlm }t j j dkrNt j j dkrNddl}||jd td |Wdn1sHwYn|g}|r]|||j|d d tfd tffd|_z||jt|j|jt|js||jd|jWntytdd|_d|_Yn wtdt|j|j||j|j|_dS)Nr)aio)is_token_auth_enabled)$AsyncAuthenticationServerInterceptor)add_port_to_grpc_server)HttpServerAgent ignore)category)zgrpc.so_reuseportrzgrpc.max_send_message_lengthzgrpc.max_receive_message_length) interceptorsoptionsz 127.0.0.1:zVFailed to add port to grpc server. Agent will stay alive but disable the grpc service.z Dashboard agent grpc address: %s) grpcr:0ray._private.authentication.authentication_utilsr;Bray._private.authentication.grpc_authentication_server_interceptorr<ray._private.tls_utilsr=ray.dashboard.http_server_agentr>sys version_infomajorminorwarningscatch_warnings simplefilterDeprecationWarning init_grpc_aioappendr2rrr&r*r Exceptionlogger exceptioninfor.r3)raiogrpcr;r<r=r>rNrCrrrr6\sX              z DashboardAgent._init_non_minimalcCsRg}ttj}|D]}tdtjj|||}||q tdt||S)zLoad dashboard agent modules.zLoading %s: %szLoaded %d modules.)dashboard_utilsget_all_modulesDashboardAgentModulerUrW__name__rSlen)rmodulesagent_cls_listclscrrr _load_moduless  zDashboardAgent._load_modulescCs|jsJd|jjS)Nz=Accessing unsupported API (HttpServerAgent) in a minimal ray.)r3 http_sessionrrrrrcs zDashboardAgent.http_sessionreturncCs|jSr)rrrrr get_node_idszDashboardAgent.get_node_idc sjr jIdH}d}jr>z j|IdHWnty=}ztd|dd}WYd}~nd}~ww|rjsEdnjj}jsNdnj}j j t j j tj||gdtjd}j j t jjtj ||gdtjd}t||IdHfdd|D}tjd vrfd d } tjj | t} || jr|jn d d } || tj|IdHjr߈jIdHdSdS)NTz,Failed to start HTTP server with exception: zB. The agent will stay alive but the HTTP service will be disabled.F) namespacecsg|]}|jqSr)runr2).0mrrr sz&DashboardAgent.run..)win32cygwincs$tdjdjd|dS)NzTerminated Raylet: ip=z , node_id=z. )rUrWr&r)msgrrrcallbacksz$DashboardAgent.run..callbackcs tdIdHq)NTi)asynciosleeprrrr wait_foreversz(DashboardAgent.run..wait_forever)r2startrbr3rTrUrV http_portr*rasync_internal_kv_putdashboard_consts#DASHBOARD_AGENT_ADDR_NODE_ID_PREFIXrencodejsondumpsr& ray_constantsKV_NAMESPACE_DASHBOARDDASHBOARD_AGENT_ADDR_IP_PREFIXrpgatherrJplatformrrlooprSwait_for_terminationcleanup) rr^launch_http_serverertr*put_by_node_id put_by_iptasksrocheck_parent_taskrrrrrrhs\       zDashboardAgent.run)r\ __module__ __qualname__r{#DEFAULT_DASHBOARD_AGENT_LISTEN_PORTboolstrr9r6rbpropertyrcrerhrrrrr s6   CG  r __main__zDashboard agent.)r#z--node-ip-addressTzthe IP address of this node.)requiredtypehelpz --gcs-addresszThe address (ip:port) of GCS.z--cluster-id-hexzThe cluster id in hex.z--metrics-export-portz.The port to expose metrics through Prometheus.z --grpc-portz9The port on which the dashboard agent will receive GRPCs.z--node-manager-portz-The port to use for starting the node managerz--object-store-namez#The socket name of the plasma store)rrdefaultrz --listen-portFz!Port for HTTP server to listen onz --raylet-namez%The socket path of the raylet processz--logging-levelcCst|Sr)logging getLevelNameupper)srrrr Bsr )rrrchoicesrz--logging-formatz--logging-filenamezJSpecify the name of log file, log to stdout if set empty, default is "{}".z--logging-rotate-bytesz,Specify the max bytes for rotating log file.z--logging-rotate-backup-countz-Specify the backup count of rotated log file.z --log-dirz"Specify the path of log directory.z --temp-dirz?Specify the path of the temporary directory use by Ray process.z --session-dirz!Specify the path of this session.z --minimal store_truezMinimal agent only contains a subset of features that don't require additional dependencies installed when ray is installed by `pip install 'ray[default]'`.)actionrz--disable-metrics-collectionzCIf this arg is set, metrics report won't be enabled from the agent.z--session-namezThe current Ray session name.z--stdout-filepathz,The filepath to dump dashboard agent stdout.z--stderr-filepathz,The filepath to dump dashboard agent stderr.rlr") logging_levellogging_formatrfilename max_bytes backup_count) rrrr+r,r.rrr rcCstdttjdS)Nz#Exiting with SIGTERM immediately...)rUwarningr/_exitsignalSIGTERMrrrrsigterm_handlers rz6Agent is working abnormally. It will exit immediately.)Yargparserpryrr/rrJray._private.ray_constants_privater{ray.dashboard.consts dashboardconstsrvray.dashboard.utilsutilsrYray._common.network_utilsrrray._common.retryrray._common.utilsr ray._privaterray._private.process_watcherrrray._private.ray_loggingr ray._rayletr r getLoggerr\rUr ArgumentParserparser add_argumentrintr LOGGER_LEVELLOGGER_LEVEL_CHOICESLOGGER_LEVEL_HELP LOGGER_FORMATLOGGER_FORMAT_HELPDASHBOARD_AGENT_LOG_FILENAMEformat parse_argsargsrlogging_rotate_byteslogging_rotation_byteslogging_rotate_backup_countlogging_rotation_backup_countrrrlogging_filename redirect_stdout_stderr_if_neededstdout_filepathstderr_filepathrr7r*r(r)r'rrr+r,r.rrr ragentradd_signal_handlerrrun_until_completerhrTrVexitrrrrs        p