o biHm @sddlZddlZddlZddlZddlZddlmZmZddlm Z m Z m Z ddl Z ddl mZddlZddlmmZddlmZmZmZddlmZddlmZmZmZddlmZmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*m+Z+m,Z,m-Z-m.Z.e/e0Z1dZ2dZ3ddZ4ddZ5ddefddZ6 dde7de e7e fde e e7e ffddZ8Gddde(Z9dS)N) OrderedDict defaultdict)AnyDictList)ServiceResource)"CLOUDWATCH_AGENT_INSTALLED_AMI_TAGCLOUDWATCH_AGENT_INSTALLED_TAGCloudwatchHelper bootstrap_aws)boto_exception_handler client_cacheresource_cache)cf cli_logger)BOTO_CREATE_MAX_RETRIESBOTO_MAX_RETRIES)LogTimer)NodeLaunchException) NodeProvider)TAG_RAY_CLUSTER_NAMETAG_RAY_LAUNCH_CONFIGTAG_RAY_NODE_KINDTAG_RAY_NODE_NAMETAG_RAY_USER_NODE_TYPEcCst|vr |t|d<|t=|S)z=Convert the Ray node name tag to the AWS-specific 'Name' tag.Namertagsr!]/home/ubuntu/.local/lib/python3.10/site-packages/ray/autoscaler/_private/aws/node_provider.py to_aws_format, r#cCsd|vr |d|t<|d=|S)z=Convert the AWS-specific 'Name' tag to the Ray node name tag.rrrr!r!r"from_aws_format5r$r%returncCs|pi}td||fi|S)z3Make client, retrying requests up to `max_retries`.ec2)rregion max_retriesaws_credentialsr!r!r"make_ec2_resource>sr,r)r+cCstg}|pi}td|tfi|}|}|t|dd|vr8|j|dd}|t|dd|vs"|S)aqGet all instance-types/resources available in the user's AWS region. Args: region: the region of the AWS provider. e.g., "us-west-2". Returns: final_instance_types: a list of instances. An example of one element in the list: {'InstanceType': 'm5a.xlarge', 'ProcessorInfo': {'SupportedArchitectures': ['x86_64'], 'SustainedClockSpeedInGhz': 2.5},'VCpuInfo': {'DefaultVCpus': 4, 'DefaultCores': 2, 'DefaultThreadsPerCore': 2, 'ValidCores': [2], 'ValidThreadsPerCore': [1, 2]}, 'MemoryInfo': {'SizeInMiB': 16384}, ...} r' InstanceTypes NextToken)r.)rrdescribe_instance_typesextendcopydeepcopy)r)r+final_instance_typesr'instance_typesr!r!r"list_ec2_instancesDsr5c@s eZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ deeeffddZedeeeefdeeeefddfddZddZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zed*d+Zed,eeefdeeeffd-d.ZdS)/AWSNodeProvidericCst||||dd|_|d}t|dt|d|_t|dd|d|_i|_t t |_ d|_ t |_|jt |_|jt |_t |_i|_dS)Ncache_stopped_nodesTr+r)r(r)r__init__getr7r,rr' ec2_fail_fast tag_cacherdicttag_cache_pendingbatch_thread_count threadingEventbatch_update_donesetready_for_new_batchLocktag_cache_lock count_lock cached_nodes)selfprovider_config cluster_namer+r!r!r"r8fs.         zAWSNodeProvider.__init__cCst|}dddgddt|jgdg}|D]\}}|d||gdqtdt|jj j |d}Wdn1sCwY|D]}|j |j vrSqJt dd |jD|j |j <qJd d |D|_d d |DS) Ninstance-state-namependingrunningrValuestag:{}z+Failed to fetch running instances from AWS.FilterscSi|] }|d|dqSKeyValuer!.0xr!r!r" z8AWSNodeProvider.non_terminated_nodes..cSi|]}|j|qSr!idrXnoder!r!r"rZcSg|]}|jqSr!r]r_r!r!r" z8AWSNodeProvider.non_terminated_nodes..)r#formatrrJitemsappendr listr' instancesfilterr^r;r%r rG)rH tag_filtersfilterskvnodesr`r!r!r"non_terminated_nodess2   z$AWSNodeProvider.non_terminated_nodescCs||}|jddkS)NrrM_get_cached_nodestaterHnode_idr`r!r!r" is_runnings zAWSNodeProvider.is_runningcCs||}|jd}|dvS)Nr)rMrLrq)rHrur`rsr!r!r" is_terminateds  zAWSNodeProvider.is_terminatedcCsR|j|j|}|j|i}t|fi|WdS1s"wYdSN)rEr;r=r9r<)rHrud1d2r!r!r" node_tagss  $zAWSNodeProvider.node_tagscC$||}|jdur||}|jSrx)rrpublic_ip_address _get_nodertr!r!r" external_ip   zAWSNodeProvider.external_ipcCr|rx)rrprivate_ip_addressr~rtr!r!r" internal_iprzAWSNodeProvider.internal_ipcCsDd}|j$|jsd}|j|j|j|j||Wdn1s,wY|rTtt |j| |j Wdn1sOwY|j |j d7_ Wdn1siwY|j|j "|j d8_ |j dkr|j WddSWddS1swYdS)NFTrr)rEr=rCwaitclearrAupdatetimesleepTAG_BATCH_DELAY_update_node_tagsrBrFr>)rHrur is_batching_threadr!r!r" set_node_tagss4        "zAWSNodeProvider.set_node_tagscCs`tt}|jD]\}}|D] }|||q|j||q tt|_||dSrx) rrhr=rfrgr;rr< _create_tags)rH batch_updatesrur rYr!r!r"rs  z!AWSNodeProvider._update_node_tagsc Cs~|D]8\\}}}d|||}td||tkrd}|jjjj|||dgdWdn1s7wYqdS)NzSet tag {}={} on {}zAWSNodeProvider: {}rrT) ResourcesTags)rfrerrr'metaclient create_tags)rHrrmrnnode_idsmr!r!r"rs  zAWSNodeProvider._create_tagsr&c Csttt|}i}|jrdddgddt|jgddt |t gddt |t gdg}t |vrG| dt |t gdt |jjj|dd|}dd |D}d d |D}|rtd t|td -|D]"}tdd |jD|j|j<|jddkrtd|j|qvWdn1swY|jjjj|d|D]} || |q|t|8}i} |r| |||} |} | !| | S)zCreates instances. Returns dict mapping instance id to ec2.Instance object for the created instances. rKstoppedstoppingrNrPrQNcSrbr!r]rXnr!r!r"rc&rdz/AWSNodeProvider.create_node..cSr\r!r]rr!r!r"rZ'raz/AWSNodeProvider.create_node..zsReusing nodes {}. To disable reuse, set `cache_stopped_nodes: False` under `provider` in the cluster configuration.zStopping instances to reusecSrSrTr!rWr!r!r"rZ5r[rzWaiting for instance {} to stop InstanceIds)"rsortedr1r2rfr7rerrJrrrrgrhr'rirjrprint render_listgroupr%r r;r^rswait_until_stoppedrrstart_instancesrlen _create_noder) rH node_configr countreused_nodes_dictrl reuse_nodesreuse_node_idsr`rucreated_nodes_dictall_created_nodesr!r!r" create_nodesf     zAWSNodeProvider.create_node tag_specsuser_tag_specsNcCs|D]?}|ddkr<|dD],}d}|ddD]}|d|dkr,d}|d|d<nq|s:|dd|g7<qq||g7}qd S) aZ Merges user-provided node config tag specifications into a base list of node provider tag specifications. The base list of node provider tag specs is modified in-place. This allows users to add tags and override values of existing tags with their own, and only applies to the resource type "instance". All other resource types are appended to the list of tag specs. Args: tag_specs (List[Dict[str, Any]]): base node provider tag specs user_tag_specs (List[Dict[str, Any]]): user's node config tag specs ResourceTypeinstancerFrrUTrVNr!)rr user_tag_specuser_tagexiststagr!r!r"_merge_tag_specsHs     z AWSNodeProvider._merge_tag_specsc Csni}t|}|}t|jdg}|D] \}}|||dqt|jdr9| |} | r9| t ddgd|dg} | dg} t | | |d} |d|| d d } i}ttt| }td|dD]}zud |vr|d }|d dt||d <n| | t| }||d<||d<|jjd i|}dd|D}tjd||d,|D]}d}|jr|jdp|}tjd|jt|jd|ddqWdW|S1swYW|Stj j!y4}zC| d7} ||kr$zt"|j#dd|j#ddt$%d}Wnt&yt'(d|Ynwtj)d|dnt(d|WYd}~qkd}~ww|S)!NrTagentTruer)rrTagSpecifications SubnetIdsr)MinCountMaxCountrrNetworkInterfacesSecurityGroupIdsnetwork_interfacesSubnetId subnet_idcSr\r!r]rr!r!r"rZraz0AWSNodeProvider._create_node..zLaunched {} nodes)_tagsrLMessagezLaunched instance {}r)rsinfoErrorCode)category description src_exc_infozCouldn't parse exception.z2Failed to launch instances. Max attempts exceeded.)excz3create_instances: Attempt failed with {}, retrying.r!)*r#r1rrJrfrgr cloudwatch_config_existsrI_check_ami_cwa_installationr0r r9r6rpoprmaxrrrangestrr:create_instancesrr state_reasonr instance_idr<rsbotocore exceptions ClientErrorrresponsesysexc_info Exceptionloggerwarningabort)rHrr rrconf tag_pairsrmrn cwa_installedrr subnet_ids subnet_idxcli_logger_tags max_triesattemptnet_ifsrcreatedrrrr!r!r"ris              zAWSNodeProvider._create_nodecCsf||}|jr-|jrtdtd||dStdtd||dS|dS)NzTerminating instance {} ,(cannot stop spot instances, only terminate)zStopping instance {} f(to terminate instead, set `cache_stopped_nodes: False` under `provider` in the cluster configuration)) rrr7spot_instance_request_idrrrdimmed terminatestoprtr!r!r"terminate_nodes(   zAWSNodeProvider.terminate_nodecCsj|jjjj|dgd}d}|d}|r3t|dks%Jdt|d|dd d }t|vr3d }|S) NImageId)ImageIdsFImagesrz9Expected to find only 1 AMI with the given ID, but found .rrT)r'rrdescribe_imagesr9rr)rHconfigrrimages image_namer!r!r"rs z+AWSNodeProvider._check_ami_cwa_installationc Cs|sdS|jjjj}|jjjj}|g|gi}|jr\g}g}|D]}||jr-||g7}q||g7}q|rCt dt dt ||rSt dt dt ||||<|||<n|||<|j durh|j nt|}|D]\} } tdt| |D] } | | | | |dq|qpdS)NzStopping instances {} rzTerminating instances {} rrr)r'rrterminate_instancesstop_instancesr7rrrrrrrrmax_terminate_nodesrrfr) rHrterminate_instances_funcstop_instances_funcnodes_to_terminatespot_ids on_demand_idsrurterminate_funcrostartr!r!r"terminate_nodessP         zAWSNodeProvider.terminate_nodesc Cs|i||jvr|j|StttdD],}t|jjj|gd}t |dkr/|dSt d|t |t |dtt t qtd|)z7Refresh and get info for this node, updating the cache.rrrzAttempt to fetch EC2 instances that have instance ID {}. Got {} matching EC2 instances. Will retry after {} second. This is retry number {}, and the maximum number of retries is {}.zInvalid instance id {})rprGrrrrhr'rirjrrrLIST_RETRY_DELAY_SECrrAssertionErrorre)rHruattemptsmatchesr!r!r"r~Ms"      zAWSNodeProvider._get_nodecCs||jvr |j|S||S)z>Return node info from cache if possible, otherwise fetches it.)rGr~)rHrur!r!r"rris   z AWSNodeProvider._get_cached_nodecCst|Srxr )cluster_configr!r!r"bootstrap_configpsz AWSNodeProvider.bootstrap_configrcCsd|vr|St|}t|dd|dd}dd|D}|d}|d}|D]}||dd }||vr||d d }d |i}||krg||d d} t| dd} dtj} t| | } | |d<tjj D]} | ||} | ||}| r| || <|rd|d|<qm|||di|||dikr|||d<td||q*td|d|ddd|S)z=Fills out missing "resources" field for available_node_types.available_node_typesproviderr)r+cSsi|]}|d|qS) InstanceTyper!)rXrr!r!r"rZs zJAWSNodeProvider.fillout_available_node_types_resources..head_node_typerrVCpuInfo DefaultVCpusCPU MemoryInfo SizeInMiBirmemoryzaccelerator_type: resourcesz#Updating the resources of {} to {}.zInstance type z! is not available in AWS region: r)r1r2r5r9int ray_constants&DEFAULT_OBJECT_STORE_MEMORY_PROPORTIONray_private acceleratorsget_all_accelerator_managers!get_ec2_instance_num_accelerators!get_ec2_instance_accelerator_typeget_resource_namerrdebugre ValueError)rinstances_listinstances_dictrr  node_type instance_typecpusautodetected_resources memory_totalpropmemory_resourcesaccelerator_managernum_acceleratorsaccelerator_typer!r!r"&fillout_available_node_types_resourcests          z6AWSNodeProvider.fillout_available_node_types_resources)__name__ __module__ __qualname__rr8rprvrwr{rrrrrrrrr staticmethodrrrrrrr~rrrr)r!r!r!r"r6csF!%  L  };   r6rx):r1loggingrr?r collectionsrrtypingrrrrboto3.resources.baserrray._private.ray_constantsrr8ray.autoscaler._private.aws.cloudwatch.cloudwatch_helperrr r "ray.autoscaler._private.aws.configr !ray.autoscaler._private.aws.utilsr rr"ray.autoscaler._private.cli_loggerrr!ray.autoscaler._private.constantsrr!ray.autoscaler._private.log_timerr$ray.autoscaler.node_launch_exceptionrray.autoscaler.node_providerrray.autoscaler.tagsrrrrr getLoggerr*rrrr#r%r,rr5r6r!r!r!r"sF