o
    -wi+Y                     @   s  d Z ddlZddlZddlZddlZddlZddlmZmZm	Z	m
Z
mZmZmZmZmZmZ ddlZddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ dd	lmZ d
dl m!Z! e"e#Z$ermddl%m&Z& dZ'dZ(dZ)ed Z*G dd dZ+e+ddde+ddddZ,G dd deZ-G dd deZ.G dd deZ/G dd deZ0G dd dedd Z1G d!d" d"eZ2d#ee.e/e0f d$ee+ fd%d&Z3G d'd( d(Z4dS ))zjob builder.    N)
TYPE_CHECKINGAnyCallableDictListLiteralOptionalTuple	TypedDictUnion)InternalArtifact)Artifact)TypeRegistry)Api)
DIFF_FNAMEMETADATA_FNAMEREQUIREMENTS_FNAME)make_artifact_name_safe   )SettingsStatic)ArtifactRecordzrequirements.frozen.txtwandb-job.jsonjob)logwarnerrorc                   @   sT   e Zd ZdededefddZdefddZd	d defd
dZd	e	defddZ
dS )Versionmajorminorpatchc                 C   s   || _ || _|| _d S N_major_minor_patch)selfr   r   r    r&   [/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sdk/internal/job_builder.py__init__,   s   
zVersion.__init__returnc                 C   s   | j  d| j d| j S )N.r!   r%   r&   r&   r'   __repr__1   s   zVersion.__repr__otherc                 C   sL   | j |j k rdS | j |j kr$| j|jk rdS | j|jkr$| j|jk r$dS dS )NTFr!   r%   r-   r&   r&   r'   __lt__4   s   zVersion.__lt__c                 C   s2   t |tstS | j|jko| j|jko| j|jkS r    )
isinstancer   NotImplementedr"   r#   r$   r.   r&   r&   r'   __eq__?   s   


zVersion.__eq__N)__name__
__module____qualname__intr(   strr,   boolr/   objectr2   r&   r&   r&   r'   r   +   s
    r      )
dockerfilebuild_contextc                   @      e Zd ZU eed< eed< dS )GitInforemotecommitNr3   r4   r5   r7   __annotations__r&   r&   r&   r'   r>   P      
 r>   c                   @   sB   e Zd ZU eed< ee ed< eed< ee ed< ee ed< dS )GitSourceDictgit
entrypointnotebookr<   r;   N)	r3   r4   r5   r>   rB   r   r7   r8   r   r&   r&   r&   r'   rD   U      
 rD   c                   @   sB   e Zd ZU eed< ee ed< eed< ee ed< ee ed< dS )ArtifactSourceDictartifactrF   rG   r<   r;   N)r3   r4   r5   r7   rB   r   r8   r   r&   r&   r&   r'   rI   ]   rH   rI   c                   @   s   e Zd ZU eed< dS )ImageSourceDictimageNrA   r&   r&   r&   r'   rK   e   s   
 rK   c                   @   s\   e Zd ZU eed< eed< eeeef ed< e	ee
f ed< e	ee
f ed< ee ed< dS )JobSourceDict_versionsource_typesourceinput_typesoutput_typesruntimeN)r3   r4   r5   r7   rB   r   rD   rI   rK   r   r   r   r&   r&   r&   r'   rM   i   s   
 rM   F)totalc                   @   r=   )ArtifactInfoForJobidnameNrA   r&   r&   r&   r'   rU   r   rC   rU   rP   r)   c                 C   s4   d}| D ]}t |}|r|du s||k r|}q|S )zJGet the minimum supported wandb version the source dict of wandb-job.json.N)!SOURCE_KEYS_MIN_SUPPORTED_VERSIONget)rP   min_seenkeynew_verr&   r&   r'   !get_min_supported_for_source_dictw   s   
r]   c                   @   sh  e Zd ZU eed< ee ed< ee ed< eeeef  ed< eeeef  ed< ee	 ed< e
ed< ee ed< ee ed	< ee ed
< ee ed< e
ed< e
ed< dUdede
fddZdeeef ddfddZdeeef ddfddZede
fddZejde
ddfddZedeeef fdd Zedeeef fd!d"Zd#eddfd$d%Zd&ee d'd(ddfd)d*Zd+ed,eeef deee ee f fd-d.Zd/ed0eddfd1d2Zd+ed,eeef deee ee f fd3d4Zd,eeef deeef fd5d6Zd7edefd8d9Zd+ed,eeef dee fd:d;Z de
fd<d=Z!de
fd>d?Z"d@ed+ee d,eeef dee#eeedf ee f fdAdBZ$			dVdCe%dDee dEee dFee dee& f
dGdHZ'd,eeef dee fdIdJZ(d@ed,eeef dee fdKdLZ)dee fdMdNZ*d,eeef de
fdOdPZ+de
fdQdRZ,d,eeef de
fdSdTZ-dS )W
JobBuilder	_settings_metadatafile_path_requirements_path_config_summary_logged_code_artifact_disable_partial_source_id_aliases_job_seq_id_job_version_alias_is_notebook_run_verboseFsettingsverbosec                 C   sl   || _ d | _d | _d | _d | _d | _d | _d | _|jp|j	| _
d | _g | _|j| _|  | _|| _d| _d S )NF)r_   r`   ra   rb   rc   rd   rh   ri   disable_job_creationx_disable_machine_infore   rf   rg   
job_source_source_type_get_is_notebook_runrj   rk   _partial)r%   rl   rm   r&   r&   r'   r(      s    

zJobBuilder.__init__configr)   Nc                 C   
   || _ d S r    )rb   )r%   rt   r&   r&   r'   
set_config      
zJobBuilder.set_configsummaryc                 C   ru   r    )rc   )r%   rx   r&   r&   r'   set_summary   rw   zJobBuilder.set_summaryc                 C   s   | j S r    re   r+   r&   r&   r'   disable   s   zJobBuilder.disablevalc                 C   ru   r    rz   )r%   r|   r&   r&   r'   r{      s   
c                 C      t | j S r    )r   type_ofrb   to_jsonr+   r&   r&   r'   rQ         zJobBuilder.input_typesc                 C   r}   r    )r   r~   rc   r   r+   r&   r&   r'   rR      r   zJobBuilder.output_types	source_idc                 C   ru   r    )rf   )r%   r   r&   r&   r'   set_partial_source_id   rw   z JobBuilder.set_partial_source_idresrJ   r   c              
   C   s   |j dkr`|d ur`z<|d d d u rd| _n'|d d d |d kr/d|d d d  | _nd|d d d d  | _|d d | _W n ty_ } ztd	|  W Y d }~nd }~ww |j d
krv|d urxt|d |jd| _d S d S d S )Nr   artifactSequencelatestArtifactv0rV   vversionIndexr   z+Malformed response from ArtifactSaver.save code)rV   rW   )	typeri   rh   KeyError_loggerinforU   rW   rd   )r%   r   rJ   er&   r&   r'   _handle_server_artifact   s(   
z"JobBuilder._handle_server_artifactprogram_relpathmetadatac                 C   sf  | di }| d}| d}| d}|d usJ |d us!J | jrtjtjt tj|s7dS |d u sA| jj	d u rHt
d dS | jj	d usPJ tjtjt| jj	||}tj|}|dr|d}d	}	|D ]
}
|
dkr~|	d
7 }	qtd|d|	 d  }n|}| ||}||d|| j| d| dd}| | d| }||fS )NrE   r?   r@   rootNN#target path does not exist, exitingz../r   r      )r?   r@   r<   r;   )rE   rF   rG   r<   r;   _)rY   rj   ospathexistsjoingetcwdbasenamer_   x_jupyter_rootr   r   relpathr7   normpath
startswithsplit_get_entrypoint_make_job_name)r%   r   r   git_infor?   r@   r   full_program_path
split_path
count_dotsprF   rP   rW   r&   r&   r'   _build_repo_job_source   sN   





z!JobBuilder._build_repo_job_sourcemessagelevelc                 C   sr   d }|dkrt | tj}n|dkrt | tj}n|dkr(t | tj}| jr5|d ur7|| d S d S d S )Nr   r   r   )	r   r   wandbtermlogwarningtermwarnr   	termerrorrk   )r%   r   r   log_funcr&   r&   r'   _log_if_verbose  s   


zJobBuilder._log_if_verbosec           	      C   s   t | jtsJ | jr<|  s<tj|t }tj	|s;tj	tj
|s5td | dd dS tj
|}n|}| ||}|| jd| jd  |d|dd	}| jd
 d^}}| |}||fS )Nr   zNo program path found when generating artifact job source for a non-colab notebook run. See https://docs.wandb.ai/guides/launch/create-jobr   r   zwandb-artifact://_id/rV   r<   r;   )rF   rG   rJ   r<   r;   rW   :)r0   rd   dictrj   _is_colab_runr   r   r   r   r   r   r   r   r   r   rY   r   r   )	r%   r   r   full_program_relpathrF   rP   artifact_basenamer   rW   r&   r&   r'   _build_artifact_job_source  s0   

z%JobBuilder._build_artifact_job_sourcec                 C   sz   | d}t|tsJ |}d|v r0|dd }td|r0|d| d}|  j|g7  _d|i}| |}||fS )Ndockerr   z([a-zA-Z0-9_\-\.]+) rL   )	rY   r0   r7   r   re	fullmatchreplacerg   r   )r%   r   
image_nameraw_image_nametagrP   rW   r&   r&   r'   _build_image_job_sourceB  s   

z"JobBuilder._build_image_job_source	input_strc                 C   s   | j jr| j jS td| S )zCUse job name from settings if provided, else use programmatic name.zjob-)r_   job_namer   )r%   r   r&   r&   r'   r   Y  s   zJobBuilder._make_job_namec                 C   s2   | j r|dr|d }|S tjtj|g}|S )NrF   )rs   rY   r   r   r   sys
executable)r%   r   r   rF   r&   r&   r'   r   `  s   
zJobBuilder._get_entrypointc                 C      t | jdot| jjS )N_jupyter)hasattrr_   r8   r   r+   r&   r&   r'   rr   p     zJobBuilder._get_is_notebook_runc                 C   r   )N_colab)r   r_   r8   r   r+   r&   r&   r'   r   s  r   zJobBuilder._is_colab_runrO   c                 C   s   d}|dkr|  |pd|\}}n"|dkr | |pd|\}}n|dkr1| |r1| |\}}nd}|du rE|rC| d| dd d	S ||fS )
zConstruct a job source dict and name from the current run.

        Args:
            source_type (str): The type of source to build the job from. One of
                "repo", "artifact", or "image".
        Nrepor   rJ   rL   zSource type is set to 'z' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-jobr   r   )r   r   _has_image_job_ingredientsr   r   )r%   rO   r   r   rP   rW   r&   r&   r'   _build_job_sourcev  s.   


zJobBuilder._build_job_sourceapir<   r;   
base_imagec                 C   s  t d | jdurd| ji| jd}|| j| dS tjtj	| j
jts1| dd dS |  }|du rG| d| j
j dd dS |d	}|du rX| d
d dS t| j }t| j }	d}
d}| |}|s| j
js~| j
js~| jr| dd dS | ||}| js|dkr|s| dd dS | |||\}}
|du rdS |r||d< |r||d< |r||d< t| D ]}|| du r|| qtt |pd||||	|d}|dusJ |
dusJ t!|
t"}t d |#d}|$t%j&|dd W d   n	1 sw   Y  |j'tj	| j
jtt(d |dkrFtjtj	| j
jt)rF|j'tj	| j
jt)t)d |S )a  Build a job artifact from the current run.

        Args:
            api (Api): The API object to use to create the job artifact.
            build_context (Optional[str]): Path within the job source code to
                the image build context. Saved as part of the job for future
                builds.
            dockerfile (Optional[str]): Path within the build context the
                Dockerfile. Saved as part of the job for future builds.
            base_image (Optional[str]): The base image used to run the job code.

        Returns:
            Optional[Artifact]: The job artifact if it was successfully built,
            otherwise None.
        z Attempting to build job artifactNz@wandb.config)rQ   rR   zhNo requirements.txt found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-jobr   z/Ensure read and write access to run files dir: zf, control this via the WANDB_DIR env var. See https://docs.wandb.ai/guides/track/environment-variablespythonzrNo python version found in metadata, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-jobz/No source type found, not creating job artifactrL   zdNo program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-jobr<   r;   r   r   )rN   rO   rP   rQ   rR   rS   zadding wandb-job metadata filer      )indent)rW   r   )*r   r   rf   rQ   rR   update_artifact_metadatar   r   r   r   r_   	files_dirr   r   _handle_metadata_filerY   r   r~   rb   r   rc   _get_source_typer   rp   rq   _get_program_relpathrs   r   listkeyspopr7   r]   r   JOB_ARTIFACT_TYPEnew_filewritejsondumpsadd_fileFROZEN_REQUIREMENTS_FNAMEr   )r%   r   r<   r;   r   new_metadatar   rS   rQ   rR   rW   source_inforO   r   rP   r[   rJ   fr&   r&   r'   build  s   




	


zJobBuilder.buildc                 C   s`   | j r| j S | |rtd dS |  rtd dS | |r)td dS td d S )Nzis repo sourced jobr   zis artifact sourced jobrJ   zis image sourced jobrL   zno source found)rq   _has_git_job_ingredientsr   r   _has_artifact_job_ingredientsr   r%   r   r&   r&   r'   r   ;  s   





zJobBuilder._get_source_typec                 C   s`   | j rtd |d}|s| dd |S |dks!| jjdkr+|dp*|dS |dS )Nzrun is notebook based runprogramzaNotebook 'program' path not found in metadata. See https://docs.wandb.ai/guides/launch/create-jobr   rJ   codePathLocalcodePath)rj   r   r   rY   r   r_   rp   )r%   rO   r   r   r&   r&   r'   r   N  s   


zJobBuilder._get_program_relpathc                 C   sd   t jt j| jjtr0tt j| jjt}t	|}W d    |S 1 s)w   Y  |S d S r    )
r   r   r   r   r_   r   r   openr   load)r%   r   r   r&   r&   r'   r   e  s   
z JobBuilder._handle_metadata_filec                 C   s@   | di }| jr| dd u rdS | dd uo| dd uS )NrE   r   Fr?   r@   )rY   rj   )r%   r   r   r&   r&   r'   r   o  s   z#JobBuilder._has_git_job_ingredientsc                 C   s
   | j d uS r    )rd   r+   r&   r&   r'   r   u  rw   z(JobBuilder._has_artifact_job_ingredientsc                 C   s   | dd uS )Nr   )rY   r   r&   r&   r'   r   x  s   z%JobBuilder._has_image_job_ingredients)F)NNN).r3   r4   r5   r   rB   r   r7   r   r   rU   r8   r   r(   rv   ry   propertyr{   setterrQ   rR   r   r   r	   rD   r   	LOG_LEVELr   rI   r   rK   r   r   r   rr   r   r   r   r   r   r   r   r   r   r   r   r   r&   r&   r&   r'   r^      s   
 


6

&






4
 



r^   )5__doc__r   loggingr   r   r   typingr   r   r   r   r   r   r   r	   r
   r   r   &wandb.sdk.artifacts._internal_artifactr   wandb.sdk.artifacts.artifactr   wandb.sdk.data_types._dtypesr   wandb.sdk.internal.internal_apir   wandb.sdk.lib.filenamesr   r   r   
wandb.utilr   settings_staticr   	getLoggerr3   r   wandb.proto.wandb_internal_pb2r   r   	JOB_FNAMEr   r   r   rX   r>   rD   rI   rK   rM   rU   r]   r^   r&   r&   r&   r'   <module>   sL    0

 
	
