o xi;@sdZddlmZddlZddlZddlmZmZ ddl Z ddl m Z ddl m Z ddlmZd d lmZmZd d lmZd d lmZmZmZmZmZd dlmZmZmZee Z!GdddeZ"GdddeZ#d8ddZ$ d9d:d*d+Z% d9d;d1d2Z&dJs"z2SagemakerSubmittedRun.get_logs..z(Failed to handle logs for training job: z with error ) rrdescribe_log_streamsrlenwandbtermwarnget_log_eventsjoin exceptionsResourceNotFoundException Exceptionr)r#r6 describe_reslog_namer:reser&r&r'get_logs2sL      zSagemakerSubmittedRun.get_logsboolcsT |IdHj}ttd|jd||dvr! |dkStdIdHq)NTz Training job z status: )stoppedfailedfinishedrG) get_statusstater8termlogr rasynciosleep)r# status_stater&r&r'waitWszSagemakerSubmittedRun.waitcs@|IdH}|jdkr|jj|jd|IdHdSdS)NrTrainingJobName)rIrJrstop_training_jobrrO)r#statusr&r&r'cancelbs  zSagemakerSubmittedRun.cancelrcst|jj}||jdIdHd}|dks|dkr"td|_|jS|dkr.td|_|jS|dkr:td |_|jS|d krCtd |_|jS) NrPTrainingJobStatus CompletedStoppedrGFailedrFStoppingstopping InProgressr)rrdescribe_training_jobrrr")r#r\ job_statusr&r&r'rIis&    z SagemakerSubmittedRun.get_statusN)rrrrrrrr)rr)rr))rrD)rr)rr) __name__ __module__ __qualname____doc__r!propertyr(rCrOrTrI __classcell__r&r&r$r'rs   % rcs,eZdZdZdfd d ZdddZZS)SageMakerRunnerz?Runner class, uses a project to create a SagemakerSubmittedRun.apirbackend_configdict[str, Any] environmentrregistryr rrcst||||_||_dS)a;Initialize the SagemakerRunner. Arguments: api (Api): The API instance. backend_config (Dict[str, Any]): The backend configuration. environment (AwsEnvironment): The AWS environment. Raises: LaunchError: If the runner cannot be initialized. N)r r!rirj)r#rfrgrirjr$r&r'r!~s zSageMakerRunner.__init__launch_projectr image_urirAbstractRun | Nonec sltd|jd}|durtd|jdid}|dur,|ds,d|}|jIdH}t |j dIdH}| }|d }td ||j IdH} t ||j|| } | d} d} z| d } Wnty} ztd t| d WYd} ~ nd} ~ ww|diddurt||j| |j|jt|jj|did|}td|t||| | IdH}|jtr|IdH|Std|jp|}g}|dur||j7}||j7}|rd|}tt d|ntt dt||j| ||jt|jj||}td|t||| | IdH}|jtr4|IdH|S)aRun a project on Amazon Sagemaker. Arguments: launch_project (LaunchProject): The project to run. Returns: Optional[AbstractRun]: The run instance. Raises: LaunchError: If the launch is unsuccessful. zusing AWSSagemakerRunner sagemakerNDNo sagemaker args specified. Specify sagemaker args in resource_argsrunners3_output_pathzs3://stsAccountzUsing account ID logsz0Failed to connect to cloudwatch logs with error z, logs will not be availableAlgorithmSpecification TrainingImagez:Launching sagemaker job on user supplied image with args: zConnecting to sagemaker client z,Launching run on sagemaker with entrypoint: zALaunching run on sagemaker with user-provided entrypoint in imagez#Launching sagemaker job with args: )!_loggerinfo resource_argsgetrrg startswithri get_sessionrrget_caller_identity get_partition get_role_arnr>r8r9rbuild_sagemaker_args_apioverride_entrypoint override_argsrr%r_launch_sagemaker_jobrrOget_job_entry_pointcommandr;rKr )r#rkrlgiven_sagemaker_argsdefault_output_pathsessionr caller_id account_id partitionrole_arnsagemaker_clientrrBsagemaker_argsrun entry_point command_args command_strr&r&r'rs                   zSageMakerRunner.run) rfrrgrhrirrjr rr)rkr rlrrrm)r_r`rarbr!rrdr&r&r$r're{srealgorithm_specificationdict[str, Any] | Nonerlr)entrypoint_command list[str]argslist[str] | NonerrhcCsP|dur |dd}n|r||d<|r||d<|r||d<|ddur&td|S)aLCreate an AWS AlgorithmSpecification. AWS Sagemaker algorithms require a training image and an input mode. If the user does not specify the specification themselves, define the spec minimally using these two fields. Otherwise, if they specify the AlgorithmSpecification set the training image if it is not set. NFile)rvTrainingInputModervContainerEntrypointContainerArgumentsz'Failed determine tag for training imager)rrlrrr&r&r',merge_image_uri_with_algorithm_specifications  rrkr rfrrrrEntryPoint | Nonemax_env_lengthintrcCsi}||} | d} | durtd| ddur&|dur&d|i|d<n| d|d<|ddur8tdtt| dpA|j} | |d<|rL|jng} t| d| d|| ||d<||d <d d | D} i| |}|d dur|td |ddurtd| d|di}| ||}i||}||d<|dg}| d|jd||d<| dd| dd| dddd |D}|S)NrnroOutputDataConfig S3OutputPathzKSagemaker launcher requires an OutputDataConfig Sagemaker resource argumentrQrurRoleArncSsi|] \}}t||qSr&)r)r3keyitemr&r&r' Ysz(build_sagemaker_args..ResourceConfigz>Sagemaker launcher requires a ResourceConfig resource argumentStoppingConditionzASagemaker launcher requires a StoppingCondition resource argument EnvironmentriTags WandbRunId)KeyValue EcrRepoNameregionprofilecSsi|] \}}|dur||qSr^r&)r3kvr&r&r'r}s) fill_macrosr{rrrrun_idrritemsget_env_vars_dictappendpop)rkrfrrrrrlrrrzrr entry_cmdcamel_case_args given_env calced_env total_envtags filtered_argsr&r&r'r%s~          rrrrrrc s|dp|j}t|j}|d i|IdH}|ddur#tdt|||}ttd|dd|j j d|j j d|}ttd||S) NrQTrainingJobArnz:Failed to create training job when submitting to SageMakerzRun job submitted with arn: zhttps://z..console.aws.amazon.com/sagemaker/home?region=z#/jobs/zSee training job status at: r&) r{rrcreate_training_jobrrr8rKr meta region_name) rkrrrrrresprurlr&r&r'rs  rrgrrcCst|dp |d}|dur|did}|dus t|ts$td|d|dr/|Sd|d|d|S) z?Get the role arn from the sagemaker args or the backend config.rrNrpzpAWS sagemaker require a string RoleArn set this by adding a `RoleArn` key to the sagemakerfield of resource_argszarn:z:iam::z:role/)r{ isinstancerrr|)rrgrrrr&r&r'rsr) rrrlr)rrrrrrhr^)rkr rfrrrrrrrrrrlrrr)rrh) rkr rrhrrrrrr) rrhrgrhrrrrrr)(rb __future__rrLloggingtypingrrboto3r8wandb.apis.internalr,wandb.sdk.launch.environment.aws_environmentrwandb.sdk.launch.errorsr _project_specr r registry.abstractr utilsr rrrrabstractrrr getLoggerr_rxrrerrrrr&r&r&r's0      \ ' a