o
    -wi                     @   s  d dl Z d dlZd dlmZmZmZ 	 d dlm	Z	 d dl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZ dZeeZdZ G dd deZ!G dd deZ"	ddedee#ef dee#ef dede$de!fddZ%dS )    N)AnyDictOptionalF)
aiplatform)Api)
get_module   )LaunchProject)GcpEnvironment)LaunchError)AbstractRegistry)MAX_ENV_LENGTHSPROJECT_SYNCHRONOUSevent_loop_thread_exec   )AbstractRunAbstractRunnerStatusz https://console.cloud.google.comzwandb-run-idc                   @   s   e Zd ZdeddfddZedefddZdee fdd	Z	edefd
dZ
edefddZedefddZdefddZdefddZdefddZdddZdS )VertexSubmittedRunjobreturnNc                 C   s
   || _ d S N)_job)selfr    r   b/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sdk/launch/runner/vertex_runner.py__init__   s   
zVertexSubmittedRun.__init__c                 C      | j jS r   )r   namer   r   r   r   id   s   zVertexSubmittedRun.idc                    s   d S r   r   r   r   r   r   get_logs#   s   zVertexSubmittedRun.get_logsc                 C   r   r   )r   display_namer   r   r   r   r   '      zVertexSubmittedRun.namec                 C   r   r   )r   locationr   r   r   r   
gcp_region+   r#   zVertexSubmittedRun.gcp_regionc                 C   r   r   )r   projectr   r   r   r   gcp_project/   r#   zVertexSubmittedRun.gcp_projectc                 C   s    t  d| j d| j d| j S )Nz/vertex-ai/locations/z
/training/z	?project=)GCP_CONSOLE_URIr%   r    r'   r   r   r   r   get_page_link3   s    z VertexSubmittedRun.get_page_linkc                    s&   | j  I d H  |  I d H jdkS )Nfinished)r   wait
get_statusstater   r   r   r   r+   6   s   zVertexSubmittedRun.waitc                    sV   t | jj}|dkrtdS |dkrtdS |dkrtdS |dkr'tdS td	S )
NzJobState.JOB_STATE_SUCCEEDEDr*   zJobState.JOB_STATE_FAILEDfailedzJobState.JOB_STATE_RUNNINGrunningzJobState.JOB_STATE_PENDINGstartingunknown)strr   r-   r   )r   	job_stater   r   r   r,   ;   s   zVertexSubmittedRun.get_statusc                    s   | j   d S r   )r   cancelr   r   r   r   r4   G   s   zVertexSubmittedRun.cancel)r   N)__name__
__module____qualname__r   r   propertyr2   r    r   r!   r   r%   r'   r)   boolr+   r   r,   r4   r   r   r   r   r      s    r   c                
       sX   e Zd ZdZdedeeef dede	ddf
 fdd	Z
d
ededee fddZ  ZS )VertexRunnerz<Runner class, uses a project to create a VertexSubmittedRun.apibackend_configenvironmentregistryr   Nc                    s   t  || || _|| _dS )z#Initialize a VertexRunner instance.N)superr   r=   r>   )r   r;   r<   r=   r>   	__class__r   r   r   N   s   
zVertexRunner.__init__launch_project	image_uric                    sD  | |}|d}|s|d}|std|di }|di }| jt }|jp/| }g }	|dur;|	|j7 }	|	|j7 }	|j	| j
t| jj d}
|dg }|sXtd	|D ]+}|d
setd|	|d
 d< |d
 dg }|dd |
 D  ||d
 d< qZ|dstdtd t|||| j|I dH }|S )zRun a Vertex job.vertexz
gcp-vertexz}No Vertex resource args specified. Specify args via --resource-args with a JSON file or string under top-level key gcp_vertexspecrunN)r;   max_env_lengthworker_pool_specszVertex requires at least one worker pool spec. Please specify a worker pool spec in resource arguments under the key `vertex.spec.worker_pool_specs`.container_speczVertex requires a container spec for each worker pool spec. Please specify a container spec in resource arguments under the key `vertex.spec.worker_pool_specs[].container_spec`.commandenvc                 S   s   g | ]	\}}||d qS ))r   valuer   ).0keyrL   r   r   r   
<listcomp>   s    z$VertexRunner.run.<locals>.<listcomp>staging_bucketzVertex requires a staging bucket. Please specify a staging bucket in resource arguments under the key `vertex.spec.staging_bucket`.zLaunching Vertex job...)fill_macrosgetr   r<   r   override_entrypointget_job_entry_pointrJ   override_argsget_env_vars_dict_apir   rA   r5   extenditems_loggerinfolaunch_vertex_jobr=   )r   rB   rC   full_resource_argsresource_args	spec_argsrun_argssynchronousentry_point	entry_cmdenv_varsworker_specsrE   rK   submitted_runr   r   r   rF   Z   sf   










zVertexRunner.run)r5   r6   r7   __doc__r   r   r2   r   r
   r   r   r	   r   r   rF   __classcell__r   r   r@   r   r:   K   s(    
r:   rB   r_   r`   r=   ra   r   c                    s  zm|  I d H  tdd}t|j}||j|j|d| I d H dI d H  |di }| j|t	< |j
| j|d|d|d|d	}t|d
|d|d|dd|d|d|d|ddd}	W n ty }
 ztd|
 d }
~
ww |rt|j}|di |	ddiI d H  nt|j}|di |	I d H  t|}d}t|jdd st|I d H  td|d }t|jdd r|S )Nzgoogle.cloud.aiplatformz=VertexRunner requires google.cloud.aiplatform to be installedrP   )r&   r$   rP   credentialslabelsrH   base_output_direncryption_spec_key_name)r"   rH   rk   rl   rj   timeoutservice_accountnetworkenable_web_accessF
experimentexperiment_runtensorboardrestart_job_on_worker_restart)rm   rn   ro   rp   rq   rr   rs   rt   zFailed to create Vertex job: syncTr   r      r   r   )verifyr   r   initr&   regionrR   get_credentialsrun_idWANDB_RUN_ID_KEY	CustomJobr   dict	Exceptionr   rF   submitr   getattr_gca_resourceasynciosleepmin)rB   r_   r`   r=   ra   r   rx   rj   r   execution_kwargserF   r   rf   intervalr   r   r   r\      sf   





r\   )F)&r   loggingtypingr   r   r   google.cloudr   wandb.apis.internalr   
wandb.utilr   _project_specr	   environment.gcp_environmentr
   errorsr   registry.abstractr   utilsr   r   r   abstractr   r   r   r(   	getLoggerr5   rZ   r|   r   r:   r2   r9   r\   r   r   r   r   <module>   s>    
1`

