o
    xi                     @  s   d dl mZ d dlZd dlZd dlmZ 	 d dlm	Z	 d dl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZmZmZ dZeeZdZ G dd deZ!G dd deZ"	d"d#d d!Z#dS )$    )annotationsN)AnyF)
aiplatform)Api)
get_module   )LaunchProject)GcpEnvironment)LaunchError)AbstractRegistry)MAX_ENV_LENGTHSPROJECT_SYNCHRONOUSevent_loop_thread_exec   )AbstractRunAbstractRunnerStatusz https://console.cloud.google.comzwandb-run-idc                   @  s   e Zd ZdddZeddd	Zd ddZedddZedddZedddZ	dddZ
d!ddZd"ddZd#ddZdS )$VertexSubmittedRunjobr   returnNonec                 C  s
   || _ d S N)_job)selfr    r   Y/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sdk/launch/runner/vertex_runner.py__init__   s   
zVertexSubmittedRun.__init__strc                 C     | j jS r   )r   namer   r   r   r   id    s   zVertexSubmittedRun.id
str | Nonec                   s   d S r   r   r    r   r   r   get_logs%   s   zVertexSubmittedRun.get_logsc                 C  r   r   )r   display_namer    r   r   r   r   )      zVertexSubmittedRun.namec                 C  r   r   )r   locationr    r   r   r   
gcp_region-   r%   zVertexSubmittedRun.gcp_regionc                 C  r   r   )r   projectr    r   r   r   gcp_project1   r%   zVertexSubmittedRun.gcp_projectc                 C  s    t  d| j d| j d| j S )Nz/vertex-ai/locations/z
/training/z	?project=)GCP_CONSOLE_URIr'   r!   r)   r    r   r   r   get_page_link5   s    z VertexSubmittedRun.get_page_linkboolc                   s&   | j  I d H  |  I d H jdkS )Nfinished)r   wait
get_statusstater    r   r   r   r.   8   s   zVertexSubmittedRun.waitr   c                   sV   t | jj}|dkrtdS |dkrtdS |dkrtdS |dkr'tdS td	S )
NzJobState.JOB_STATE_SUCCEEDEDr-   zJobState.JOB_STATE_FAILEDfailedzJobState.JOB_STATE_RUNNINGrunningzJobState.JOB_STATE_PENDINGstartingunknown)r   r   r0   r   )r   	job_stater   r   r   r/   =   s   zVertexSubmittedRun.get_statusc                   s   | j   d S r   )r   cancelr    r   r   r   r6   I   s   zVertexSubmittedRun.cancelN)r   r   r   r   )r   r   )r   r"   )r   r,   )r   r   )r   r   )__name__
__module____qualname__r   propertyr!   r#   r   r'   r)   r+   r.   r/   r6   r   r   r   r   r      s    




r   c                      s,   e Zd ZdZd fddZdddZ  ZS )VertexRunnerz<Runner class, uses a project to create a VertexSubmittedRun.apir   backend_configdict[str, Any]environmentr	   registryr   r   r   c                   s   t  || || _|| _dS )z#Initialize a VertexRunner instance.N)superr   r?   r@   )r   r<   r=   r?   r@   	__class__r   r   r   P   s   
zVertexRunner.__init__launch_projectr   	image_urir   AbstractRun | Nonec                   sD  | |}|d}|s|d}|std|di }|di }| jt }|jp/| }g }	|dur;|	|j7 }	|	|j7 }	|j	| j
t| jj d}
|dg }|sXtd	|D ]+}|d
setd|	|d
 d< |d
 dg }|dd |
 D  ||d
 d< qZ|dstdtd t|||| j|I dH }|S )zRun a Vertex job.vertexz
gcp-vertexz}No Vertex resource args specified. Specify args via --resource-args with a JSON file or string under top-level key gcp_vertexspecrunN)r<   max_env_lengthworker_pool_specszVertex requires at least one worker pool spec. Please specify a worker pool spec in resource arguments under the key `vertex.spec.worker_pool_specs`.container_speczVertex requires a container spec for each worker pool spec. Please specify a container spec in resource arguments under the key `vertex.spec.worker_pool_specs[].container_spec`.commandenvc                 S  s   g | ]	\}}||d qS ))r   valuer   ).0keyrO   r   r   r   
<listcomp>   s    z$VertexRunner.run.<locals>.<listcomp>staging_bucketzVertex requires a staging bucket. Please specify a staging bucket in resource arguments under the key `vertex.spec.staging_bucket`.zLaunching Vertex job...)fill_macrosgetr
   r=   r   override_entrypointget_job_entry_pointrM   override_argsget_env_vars_dict_apir   rC   r7   extenditems_loggerinfolaunch_vertex_jobr?   )r   rD   rE   full_resource_argsresource_args	spec_argsrun_argssynchronousentry_point	entry_cmdenv_varsworker_specsrH   rN   submitted_runr   r   r   rI   \   sf   










zVertexRunner.run)
r<   r   r=   r>   r?   r	   r@   r   r   r   )rD   r   rE   r   r   rF   )r7   r8   r9   __doc__r   rI   __classcell__r   r   rB   r   r;   M   s    r;   rD   r   rb   r>   rc   r?   r	   rd   r,   r   c                   s  zm|  I d H  tdd}t|j}||j|j|d| I d H dI d H  |di }| j|t	< |j
| j|d|d|d|d	}t|d
|d|d|dd|d|d|d|ddd}	W n ty }
 ztd|
 d }
~
ww |rt|j}|di |	ddiI d H  nt|j}|di |	I d H  t|}d}t|jdd st|I d H  td|d }t|jdd r|S )Nzgoogle.cloud.aiplatformz=VertexRunner requires google.cloud.aiplatform to be installedrS   )r(   r&   rS   credentialslabelsrK   base_output_direncryption_spec_key_name)r$   rK   rn   ro   rm   timeoutservice_accountnetworkenable_web_accessF
experimentexperiment_runtensorboardrestart_job_on_worker_restart)rp   rq   rr   rs   rt   ru   rv   rw   zFailed to create Vertex job: syncTr   r      r   r   )verifyr   r   initr(   regionrU   get_credentialsrun_idWANDB_RUN_ID_KEY	CustomJobr   dict	Exceptionr
   rI   submitr   getattr_gca_resourceasynciosleepmin)rD   rb   rc   r?   rd   r   r{   rm   r   execution_kwargserI   r   ri   intervalr   r   r   r_      sf   





r_   )F)rD   r   rb   r>   rc   r>   r?   r	   rd   r,   r   r   )$
__future__r   r   loggingtypingr   google.cloudr   wandb.apis.internalr   
wandb.utilr   _project_specr   environment.gcp_environmentr	   errorsr
   registry.abstractr   utilsr   r   r   abstractr   r   r   r*   	getLoggerr7   r]   r   r   r;   r_   r   r   r   r   <module>   s(    
1`