o
    ni                      @   s  d dl Z d dlZd dlmZmZmZ 	 d dlm	Z	 d dl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZ dZeeZdZ G dd deZ!G dd deZ"	ddedee#ef dee#ef dede$de!fddZ%dS )    N)AnyDictOptionalF)
aiplatform)Api)
get_module   )LaunchProject)GcpEnvironment)LaunchError)AbstractRegistry)MAX_ENV_LENGTHSPROJECT_SYNCHRONOUSevent_loop_thread_exec   )AbstractRunAbstractRunnerStatusz https://console.cloud.google.comzwandb-run-idc                   @   s   e Zd ZdeddfddZedefddZdee fdd	Z	edefd
dZ
edefddZedefddZdefddZdefddZdefddZdddZdS )VertexSubmittedRunjobreturnNc                 C   s
   || _ d S N)_job)selfr    r   c/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/launch/runner/vertex_runner.py__init__   s   
zVertexSubmittedRun.__init__c                 C      | j jS r   )r   namer   r   r   r   id   s   zVertexSubmittedRun.idc                    s   d S r   r   r   r   r   r   get_logs#   s   zVertexSubmittedRun.get_logsc                 C   r   r   )r   display_namer   r   r   r   r   '      zVertexSubmittedRun.namec                 C   r   r   )r   locationr   r   r   r   
gcp_region+   r#   zVertexSubmittedRun.gcp_regionc                 C   r   r   )r   projectr   r   r   r   gcp_project/   r#   zVertexSubmittedRun.gcp_projectc                 C   s   dj t| j| j| jdS )NzN{console_uri}/vertex-ai/locations/{region}/training/{job_id}?project={project})console_uriregionjob_idr&   )formatGCP_CONSOLE_URIr%   r    r'   r   r   r   r   get_page_link3   s   z VertexSubmittedRun.get_page_linkc                    s&   | j  I d H  |  I d H jdkS )Nfinished)r   wait
get_statusstater   r   r   r   r/   ;   s   zVertexSubmittedRun.waitc                    sV   t | jj}|dkrtdS |dkrtdS |dkrtdS |dkr'tdS td	S )
NzJobState.JOB_STATE_SUCCEEDEDr.   zJobState.JOB_STATE_FAILEDfailedzJobState.JOB_STATE_RUNNINGrunningzJobState.JOB_STATE_PENDINGstartingunknown)strr   r1   r   )r   	job_stater   r   r   r0   @   s   zVertexSubmittedRun.get_statusc                    s   | j   d S r   )r   cancelr   r   r   r   r8   L   s   zVertexSubmittedRun.cancel)r   N)__name__
__module____qualname__r   r   propertyr6   r    r   r!   r   r%   r'   r-   boolr/   r   r0   r8   r   r   r   r   r      s    r   c                
       sX   e Zd ZdZdedeeef dede	ddf
 fdd	Z
d
ededee fddZ  ZS )VertexRunnerz<Runner class, uses a project to create a VertexSubmittedRun.apibackend_configenvironmentregistryr   Nc                    s   t  || || _|| _dS )z#Initialize a VertexRunner instance.N)superr   rA   rB   )r   r?   r@   rA   rB   	__class__r   r   r   S   s   
zVertexRunner.__init__launch_project	image_uric                    sD  | |}|d}|s|d}|std|di }|di }| jt }|jp/| }g }	|dur;|	|j7 }	|	|j7 }	|j	| j
t| jj d}
|dg }|sXtd	|D ]+}|d
setd|	|d
 d< |d
 dg }|dd |
 D  ||d
 d< qZ|dstdtd t|||| j|I dH }|S )zRun a Vertex job.vertexz
gcp-vertexz}No Vertex resource args specified. Specify args via --resource-args with a JSON file or string under top-level key gcp_vertexspecrunN)r?   max_env_lengthworker_pool_specszVertex requires at least one worker pool spec. Please specify a worker pool spec in resource arguments under the key `vertex.spec.worker_pool_specs`.container_speczVertex requires a container spec for each worker pool spec. Please specify a container spec in resource arguments under the key `vertex.spec.worker_pool_specs[].container_spec`.commandenvc                 S   s   g | ]	\}}||d qS ))r   valuer   ).0keyrP   r   r   r   
<listcomp>   s    z$VertexRunner.run.<locals>.<listcomp>staging_bucketzVertex requires a staging bucket. Please specify a staging bucket in resource arguments under the key `vertex.spec.staging_bucket`.zLaunching Vertex job...)fill_macrosgetr   r@   r   override_entrypointget_job_entry_pointrN   override_argsget_env_vars_dict_apir   rE   r9   extenditems_loggerinfolaunch_vertex_jobrA   )r   rF   rG   full_resource_argsresource_args	spec_argsrun_argssynchronousentry_point	entry_cmdenv_varsworker_specsrI   rO   submitted_runr   r   r   rJ   _   sf   










zVertexRunner.run)r9   r:   r;   __doc__r   r   r6   r   r
   r   r   r	   r   r   rJ   __classcell__r   r   rD   r   r>   P   s(    
r>   rF   rc   rd   rA   re   r   c                    s  zm|  I d H  tdd}t|j}||j|j|d| I d H dI d H  |di }| j|t	< |j
| j|d|d|d|d	}t|d
|d|d|dd|d|d|d|ddd}	W n ty }
 ztd|
 d }
~
ww |rt|j}|di |	ddiI d H  nt|j}|di |	I d H  t|}d}t|jdd st|I d H  td|d }t|jdd r|S )Nzgoogle.cloud.aiplatformz=VertexRunner requires google.cloud.aiplatform to be installedrT   )r&   r$   rT   credentialslabelsrL   base_output_direncryption_spec_key_name)r"   rL   ro   rp   rn   timeoutservice_accountnetworkenable_web_accessF
experimentexperiment_runtensorboardrestart_job_on_worker_restart)rq   rr   rs   rt   ru   rv   rw   rx   zFailed to create Vertex job: syncTr   r      r   r   )verifyr   r   initr&   r)   rV   get_credentialsrun_idWANDB_RUN_ID_KEY	CustomJobr   dict	Exceptionr   rJ   submitr   getattr_gca_resourceasynciosleepmin)rF   rc   rd   rA   re   r   r|   rn   r   execution_kwargserJ   r   rj   intervalr   r   r   r`      sf   





r`   )F)&r   loggingtypingr   r   r   google.cloudr   wandb.apis.internalr   
wandb.utilr   _project_specr	   environment.gcp_environmentr
   errorsr   registry.abstractr   utilsr   r   r   abstractr   r   r   r,   	getLoggerr9   r^   r   r   r>   r6   r=   r`   r   r   r   r   <module>   s>    
6`

