o
    xiv]                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZ d dlZd dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlm Z  d dl!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/ e"ddd d dl0Z1d dl0m2Z2 e3e4Z5dZ6ej78ddZ9ej78dZ:e:rej78dd;dndZ<ej78dZ=ej>?d re@d ZAeAB C ZDW d   n1 sw   Y  nd!ZDd2d$d%ZE	d3d4d.d/ZFG d0d1 d1eZGdS )5    )annotationsN)Any)JobAndRunStatusTracker)AbstractBuilderregistry_from_uri)AbstractEnvironment)AzureEnvironment)AbstractRegistry)AzureContainerRegistry)ElasticContainerRegistry)GoogleArtifactRegistry)
get_module   )
EntryPointLaunchProject)LaunchError)
LOG_PREFIXget_kube_context_and_api_client$warn_failed_packages_from_build_logs   )_WANDB_DOCKERFILE_NAME)BuildContextManagerkubernetes_asynciozkKaniko builder requires the kubernetes_asyncio package. Please install it with `pip install wandb[launch]`.)required)clienti  !WANDB_LAUNCH_SERVICE_ACCOUNT_NAMEdefaultWANDB_LAUNCH_KANIKO_PVC_NAME"WANDB_LAUNCH_KANIKO_PVC_MOUNT_PATHz/kaniko/WANDB_LAUNCH_KANIKO_AUTH_SECRETz7/var/run/secrets/kubernetes.io/serviceaccount/namespacewandbjobclient.V1Jobc                 C  s$   z| j jjjW S  ty   Y d S w N)spectemplatemetadatanameAttributeError)r"    r*   [/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sdk/launch/builder/kaniko_builder.pyget_pod_name_safeD   s
   r,   batch_clientclient.BatchV1Apijob_namestrdeadline_secs
int | Nonereturnboolc                   s   t   }	 | |tI d H }|jjd ur|jjdkrdS |jjd ur:|jjdkr:tt d|jj d|  dS t	t d |d urPt   | |krPdS t
dI d H  q)NTr   z
Build job z failed Fz$Waiting for build job to complete...   )timeread_namespaced_job_status	NAMESPACEstatus	succeededfailedr!   	termerrorr   termlogasynciosleep)r-   r/   r1   
start_timer"   r*   r*   r+   _wait_for_completionK   s   rA   c                   @  s   e Zd ZU dZdZded< ded< ded< ded< ded	< 	
					d?d@ddZe		dAdBddZdCd d!Z	dCd"d#Z
dDd(d)ZdEd+d,ZdFd/d0Z	dGdHd7d8ZdId=d>ZdS )JKanikoBuilderz1Builds a docker image for a project using Kaniko.kanikor0   build_job_namebuild_context_storez
str | Nonesecret_name
secret_keyimagewandb-launch-container-build &gcr.io/kaniko-project/executor:v1.11.0Nenvironmentr   registryr	   configdict | Nonec	           	      C  s>   || _ || _|| _|d| _|| _|| _|| _|pi | _dS )a  Initialize a KanikoBuilder.

        Arguments:
            environment (AbstractEnvironment): The environment to use.
            registry (AbstractRegistry): The registry to use.
            build_job_name (str, optional): The name of the build job.
            build_context_store (str, optional): The name of the build context store.
            secret_name (str, optional): The name of the secret to use for the registry.
            secret_key (str, optional): The key of the secret to use for the registry.
            verify (bool, optional): Whether to verify the functionality of the builder.
                Defaults to True.
        r   N)	rL   rM   rD   rstriprE   rF   rG   rH   kaniko_config)	selfrL   rM   rD   rE   rF   rG   rH   rN   r*   r*   r+   __init__h   s   zKanikoBuilder.__init__Tdictverifyr4   loginr3   r   c              
   C  s   | ddkrtd| dd}|du rtstd| dd	}| d
d}| dd}	| dd}
| d}|dur@t|}| di }| ||||||	|
|dS )a  Create a KanikoBuilder from a config dict.

        Arguments:
            config: A dict containing the builder config. Must contain a "type" key
                with value "kaniko".
            environment: The environment to use for the build.
            registry: The registry to use for the build.
            verify: Whether to verify the builder config.

        Returns:
            A KanikoBuilder instance.
        typerC   zFBuilder config must include 'type':'kaniko' to create a KanikoBuilder.zbuild-context-storerJ   Na.  You must specify a build context store for kaniko builds. You can set builder.build-context-store in your agent config to a valid s3, gcs, or azure blog storage URI. Or, configure a persistent volume claim through the agent helm chart: https://github.com/wandb/helm-charts/tree/main/charts/launch-agentzbuild-job-namerI   zsecret-namez
secret-keyzkaniko-imagerK   destinationzkaniko-config)rE   rD   rF   rG   rH   rN   )getr   PVC_MOUNT_PATHr   )clsrN   rL   rM   rU   rV   rE   rD   rF   rG   kaniko_image	image_urirQ   r*   r*   r+   from_config   s:   
zKanikoBuilder.from_configNonec                   s$   | j r| j| j I dH  dS dS )z}Verify that the builder config is valid.

        Raises:
            LaunchError: If the builder config is invalid.
        N)rE   rL   verify_storage_urirR   r*   r*   r+   rU      s   zKanikoBuilder.verifyc                 C  s   dS )zLogin to the registry.Nr*   ra   r*   r*   r+   rV      s    zKanikoBuilder.loginr/   corev1_clientclient.CoreV1Api
repositoryc                   s   | j  I d H \}}t| d|  d}tjddtjd| t	ddt
d| j  I d H  d	|iiiid
d}|t	|I d H  d S )N:zutf-8v1	ConfigMapdocker-config-)r(   	namespaceconfig.jsonauthsauthT)api_versionkindr'   data	immutable)rM   get_username_passwordbase64	b64encodeencodedecoder   V1ConfigMapV1ObjectMetar8   jsondumpsget_repo_uricreate_namespaced_config_map)rR   r/   rb   rd   usernamepasswordencodedecr_config_mapr*   r*   r+   _create_docker_ecr_config_map   s(   	z+KanikoBuilder._create_docker_ecr_config_mapr   c                   s(   | j r|d| tI d H  d S d S )Nrh   )rF   delete_namespaced_config_mapr8   )rR   r/   r   r*   r*   r+   _delete_docker_ecr_config_map   s   
z+KanikoBuilder._delete_docker_ecr_config_maprun_idcontext_pathc              
     s   t jdd}tjj|dd}|j|dd W d    n1 s!w   Y  |  td u rM| j d| d}| j	d u r@t
d	| j	|j|I d H  |S t d| d}z	t|j| W n tyu } zt
d
t d| |d }~ww d| dS )NF)deletezw:gz)fileobjmode.)arcnamer   z.tgzz*No environment specified for Kaniko build.z.Error copying build context to PVC mounted at : ztar:///context/)tempfileNamedTemporaryFiletarfileTarFileopenaddcloserZ   rE   rL   r   upload_filer(   shutilcopy	Exception)rR   r   r   context_filecontext_tgzrX   er*   r*   r+   _upload_build_context   s0   
z#KanikoBuilder._upload_build_contextlaunch_projectr   
entrypointr   job_trackerJobAndRunStatusTracker | Nonec                   s&  |   I d H  t|d}|d\}}|j}| j I d H }|d | }	| s@tr5t	d|	 d n| j
|	I d H r@|	S td|	 d tt|jI d H \}
}t|}t|}| j d| }| ||I d H }| |||	|||I d H }t	t d	|  z;zt| jtrtstjtjd
| ddtd| jj ddiiid}|d|I d H  | jr|  |||I d H  |!t"|I d H }t#||dt$ I d H s|r|%d d| d}t&|}|r|dt" d| d7 }t'|z<|j(t"d| dI d H }t)|j*dkrt'd| dt)|j* |j*d j+j,}|-|t"I d H }t.||	|j/| W n  t'yU } zt0t d| d |  W Y d }~nd }~ww W n t'yp } zt1t d!| d"  d }~ww W t	t d# z/t| jtrts|2d
| dI d H  | jr| 3||I d H  |4|t"I d H  W |	S  t'y } zt56  t7d$| |d }~ww t	t d# z.t| jtrts|2d
| dI d H  | jr| 3||I d H  |4|t"I d H  W w  t'y } zt56  t7d$| |d }~ww )%N)r   rC   re   z"Skipping check for existing image z due to custom dockerconfig.zBuilding image z...-zCreated kaniko job rh   )r(   rj   credHelpersz.azurecr.iozacr-env)r'   ro   r!      buildz(Failed to build image in kaniko for job r   z! View logs with `kubectl logs -n  z`.z	job-name=)ri   label_selectorr   zExpected 1 pod for job z, found r   z"Failed to get logs for kaniko job r   z.Exception when creating Kubernetes resources: 
zCleaning up resourcesz.Exception during Kubernetes resource clean up )8rU   r   create_build_contextr   rM   rz   build_requiredDOCKER_CONFIG_SECRETr!   r=   check_image_exists_loggerinfor   
kubernetesresource_argsr   
BatchV1Api	CoreV1ApirD   r   _create_kaniko_jobr   
isinstancer
   rv   rw   rx   ry   registry_namer{   rF   r   create_namespaced_jobr8   rA   _DEFAULT_BUILD_TIMEOUT_SECSset_err_stager,   r   list_namespaced_podlenitemsr'   r(   read_namespaced_pod_logr   apitermwarnr<   r   r   delete_namespaced_job	traceback	print_excr   )rR   r   r   r   build_contex_managerr   	image_tagr   repo_urir]   _
api_clientbatch_v1core_v1rD   build_context	build_jobdockerfile_config_mapk8s_jobmsgpod_namepods_from_joblogsr   r*   r*   r+   build_image  s  


















zKanikoBuilder.build_imager   build_context_pathcore_clientdict[str, Any]c              
     sZ  t | j}|di }|di }	|di }
|
di }|di }|di }|di }|dg }|dp>i g}t|dkrItd|d	 }|d
g }|dg }|dg }trs|ddtid |ddd t	| j
t	| jkrtdt| jtr|d| jjd t| jtrz|ddI d H  W n ty } ztd|d }~ww |dddddid tr|dtddd gd!d" |dd#d nj| j
rE| jrE|d$d%d&| id' |d$d#d t| jtrd(}d)}n%t| jtrd*}d}|d+d,d ntt d-t| j d.| j  || j
| j
| j|d gd!d" || j
|d/d0 t| jtrdtsd|d$d%d&| id' |d$d1d |}|d2rr|d2d3}|t|d4|d2d3d5d6d7}|D ]}|d8d\}}|||< qd9d: |  D }||d< d;|d< t| jtrtsd4|d<< |d=d>|d=< |d?t!|d?< |d@t"|d@< |
dAd	|
dA< d;|	d< |dBt#|dB< |d%||d%< dC|dD< dE|dF< ||d< ||d< |d%dG|d%< |dH| j$|dH< ||d
< ||d< |g|d< ||d< ||d< ||
d< |	|d< ||d< |
|d< |S )INr'   labelsr%   r&   volumes
containersr   z<Multiple container configs not supported for kaniko builder.r   volumeMountsenvargsz
kaniko-pvc	claimName)r(   persistentVolumeClaimz/context)r(   	mountPathzmBoth secret_name and secret_key or neither must be specified for kaniko build. You provided only one of them.
AWS_REGION)r(   valuezazure-storage-access-keyr!   zSecret azure-storage-access-key does not exist in namespace wandb. Please create it with the key password set to your azure storage access key.AZURE_STORAGE_ACCESS_KEYsecretKeyRefr}   )r(   key)r(   	valueFromzkaniko-docker-configz.dockerconfigjsonrj   )r   path)
secretNamer   )r(   secretz/kaniko/.dockerzdocker-configr(   rh   )r(   	configMapz
/root/.awscredentialsz/kaniko/.config/gcloudGOOGLE_APPLICATION_CREDENTIALSz"/kaniko/.config/gcloud/config.jsonzAAutomatic credential handling is not supported for registry type z. Build job: T)r(   r   readOnlyz/kaniko/.docker/zhttps://rJ   trueredofalse)z	--contextz--dockerfilez--destinationz--cachez--cache-repoz--snapshot-modez--compressed-caching=c                 S  s   g | ]\}}| d | qS )r   r*   ).0arg_name	arg_valuer*   r*   r+   
<listcomp>*  s    z4KanikoBuilder._create_kaniko_job.<locals>.<listcomp>launchzazure.workload.identity/userestartPolicyNeveractiveDeadlineSecondsserviceAccountNamebackoffLimitri   zbatch/v1
apiVersionJobrn   zwandb-container-buildrH   )%r   deepcopyrQ   rY   r   r   rZ   appendPVC_NAMEr4   rF   rG   r   rM   r   regionrL   r   read_namespaced_secretr   r   r   r!   r   r   rW   rD   r
   
startswithreplacer   splitr   r   SERVICE_ACCOUNT_NAMEr8   rH   )rR   r/   rd   r   r   r   r   r"   job_metadata
job_labelsjob_specpod_templatepod_metadata
pod_labelspod_specr   r   	containervolume_mountsr   custom_argsr   
mount_pathr   rX   r   
custom_argr   r   parsed_argsr*   r*   r+   r     sT  		

	

z KanikoBuilder._create_kaniko_job)rI   rJ   rJ   rJ   rK   N)rL   r   rM   r	   rD   r0   rE   r0   rF   r0   rG   r0   rH   r0   rN   rO   )TT)rN   rT   rL   r   rM   r	   rU   r4   rV   r4   r3   r   )r3   r_   )r/   r0   rb   rc   rd   r0   r3   r_   )r/   r0   r   rc   r3   r_   )r   r0   r   r0   r3   r0   r$   )r   r   r   r   r   r   r3   r0   )r/   r0   rd   r0   r   r0   r   r0   r   rc   r3   r   )__name__
__module____qualname____doc__rW   __annotations__rS   classmethodr^   rU   rV   r   r   r   r   r   r*   r*   r*   r+   rB   ]   s6   
  
7
	


}rB   )r"   r#   r$   )r-   r.   r/   r0   r1   r2   r3   r4   )H
__future__r   r>   rr   r   rx   loggingosr   r   r   r6   r   typingr   r!   )wandb.sdk.launch.agent.job_status_trackerr   !wandb.sdk.launch.builder.abstractr   r   %wandb.sdk.launch.environment.abstractr   .wandb.sdk.launch.environment.azure_environmentr   "wandb.sdk.launch.registry.abstractr	   2wandb.sdk.launch.registry.azure_container_registryr
   4wandb.sdk.launch.registry.elastic_container_registryr   2wandb.sdk.launch.registry.google_artifact_registryr   
wandb.utilr   _project_specr   r   errorsr   utilsr   r   r   r   r   context_managerr   r   r   r   	getLoggerr  r   r   environrY   r   r   rP   rZ   r   r   existsr   freadstripr8   r,   rA   rB   r*   r*   r*   r+   <module>   sj    


