o
    xil                     @  s  U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlm Z  ddl!m"Z"m#Z# e$e%e# de%e" Z&erd dl'm(Z( e$dZ)dZ*dZ+dZ,e$d-e,e*gZ.e$dZ/e$dZ0e$dZ1dZ2e$dZ3e$dZ4e$dZ5e$dej6Z7e$dZ8e$dZ9e$dZ:e$d Z;d!Z<d"Z=d#Z>e?e@ZAejBd$d%d& d'ZCed(d) ZDd*eEd+< d,eDd-< d.ZFdd1d2ZGdd5d6ZHdd:d;ZIdd<d=ZJdd>d?ZKdd@dAZLddCdDZMddGdHZNddPdQZOddRdSZPddWdXZQ	dddddeZRddfdgZSddidjZT	kdddndoZUddtduZVddwdxZWdd{d|ZXdd}d~ZYdddZZdddZ[dddZ\dddZ]dddZ^dddZ_dddZ`dddZadddZbddddZcdddZddddZedddZfdddZgdddZhdddZidddZjdddZkdS )    )annotationsN)defaultdict)Iterator)TYPE_CHECKINGAnycast)util)Api)LaunchError)GitReference)WandbReference)Config   )FAILED_PACKAGES_POSTFIXFAILED_PACKAGES_PREFIXz(.*))JobAndRunStatusTrackerzA^[^/|^~|^\.].*(git|bitbucket|dev\.azure\.com|\.visualstudio\.com)z)^https?://[0-9]+(?:\.[0-9]+){3}(:[0-9]+)?z^[a-zA-Z0-9_.-]+$z^https?://(api.)?wandb|z^https?://ap\w.qa.wandbz^https?://ap\w.wandb.testz^https?://localhostzWANDB_API_KEY=\w+(-\w+)?z\$\{(\w+)\}zG^(?:https://)?([\w]+)\.azurecr\.io/(?P<repository>[\w\-]+):?(?P<tag>.*)zx^(?:https://)?(?P<account>[\w-]+)\.dkr\.ecr\.(?P<region>[\w-]+)\.amazonaws\.com/(?P<repository>[\.\/\w-]+):?(?P<tag>.*)$z^(?:https://)?(?P<region>[\w-]+)-docker\.pkg\.dev/(?P<project>[\w-]+)/(?P<repository>[\w-]+)/?(?P<image_name>[\w-]+)?(?P<tag>:.*)?$zs3://([^/]+)(/(.*))?zgs://([^/]+)(?:/(.*))?z:^https://([^\.]+)\.blob\.core\.windows\.net/([^/]+)/?(.*)$z%^arn:([^:]+):[^:]*:[^:]*:[^:]*:[^:]*$SYNCHRONOUSz"~/.config/wandb/launch-config.yamlzmodel-registryzlaunch:magenta)fg c                   C  s   dS )Ni   r   r   r   J/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sdk/launch/utils.py<lambda>Z   s    r   zdict[str, int]MAX_ENV_LENGTHSi   SageMakerRunnerz
/mnt/wandbreturnr   c               
   C  s   t jd} | du r.d} d}	 t jd| }|du rn	| |7 } |d7 }q|dk r.tdt }zt| }W n tjyM } ztd	| |d}~ww || |S )
a  Load wandb config from WANDB_CONFIG environment variable(s).

    The WANDB_CONFIG environment variable is a json string that can contain
    multiple config keys. The WANDB_CONFIG_[0-9]+ environment variables are
    used for environments where there is a limit on the length of environment
    variables. In that case, we shard the contents of WANDB_CONFIG into
    multiple environment variables numbered from 0.

    Returns:
        A dictionary of wandb config values.
    WANDB_CONFIGN r   TWANDB_CONFIG_r   zBNo WANDB_CONFIG or WANDB_CONFIG_[0-9]+ environment variables foundzFailed to parse WANDB_CONFIG: )	osenvirongetr
   r   jsonloadsJSONDecodeErrorupdate)
config_stridxchunkwandb_config
env_configer   r   r   load_wandb_config`   s0   
r-   funcr   c                   s   d fdd}|S )	aC  Wrapper for running any function in an awaitable thread on an event loop.

    Example usage:
    ```
    def my_func(arg1, arg2):
        return arg1 + arg2


    future = event_loop_thread_exec(my_func)(2, 2)
    assert await future == 4
    ```

    The returned function must be called within an active event loop.
    argsr   kwargsr   c                    s2   t  }tt|d  fddI d H }|S )Nc                     s    i S Nr   r   )r/   r.   r0   r   r   r      s    z9event_loop_thread_exec.<locals>.wrapper.<locals>.<lambda>)asyncioget_event_loopr   r   run_in_executor)r/   r0   loopresultr.   )r/   r0   r   wrapper   s   z'event_loop_thread_exec.<locals>.wrapperN)r/   r   r0   r   r   r   r   )r.   r8   r   r7   r   event_loop_thread_exec   s   r9   uristrboolc                 C  s,   t | pt| pt| pt| d uS r1   )_WANDB_URI_REGEXmatch_WANDB_DEV_URI_REGEX_WANDB_LOCAL_DEV_URI_REGEX_WANDB_QA_URI_REGEXr:   r   r   r   _is_wandb_uri   s   
rC   c                 C     t t| S r1   )r<   r?   r>   rB   r   r   r   _is_wandb_dev_uri      rE   c                 C  rD   r1   )r<   r@   r>   rB   r   r   r   _is_wandb_local_uri   rF   rG   c                 C  rD   r1   )r<   _GIT_URI_REGEXr>   rB   r   r   r   _is_git_uri   rF   rI   sc                 C  s   t ttd| S )NWANDB_API_KEY)r;   resubAPI_KEY_REGEX)rJ   r   r   r   sanitize_wandb_api_key   s   rO   job
str | Nonec                 C  s"   |  d}t|dkr|d S d S )N/   r   )splitlen)rP   	job_partsr   r   r   get_project_from_job   s   
rW   apir	   projectentitylaunch_configdict[str, Any] | Nonetuple[str | None, str]c           
      C  s   d }| d ur#t | rt| \}}}nt| r"tjtj| d }n|d ur+t|}|d u r>d }|r8|d}|p=|p=d}|d u rGt	||}d}	t
 dkrWtjjdkrWd}	tt |	 d| |rfd| nd  ||fS )	Nr   rY   r   WindowszUTF-8u   🚀 zLaunching run into rR   )rC   parse_wandb_urirI   r    pathsplitextbasenamerW   r"   get_default_entityplatformsystemsysstdoutencodingwandbtermlog
LOG_PREFIX)
r:   rP   rX   rY   rZ   r[   
source_uri_config_projectprefixr   r   r   set_project_entity_defaults   s.   	

 rp   c                 C  s   d }|r	| d}|p| jS )NrZ   )r"   default_entity)rX   r[   config_entityr   r   r   rc      s   

rc   launch_specdict[str, Any]Nonec                 C  s4   |  dr|  drtd | d d S d S d S )Nresource_argstemplate_variablesznLaunch spec contains both resource_args and template_variables, only one can be set. Using template_variables.)r"   ri   termwarnpop)rs   r   r   r   %strip_resource_args_and_template_vars   s   rz   namedocker_imageresourceentry_pointlist[str] | Noneversionrv   run_id
repositoryauthorsweep_idc                 C  sh  |dur|ni }| dur| |d< |dur||d< t | |||||\}}||d< |r-||d< ||d< |r7||d< d|vr?i |d< |rG||d d	< |rM||d
< d|vrY|rU|nd|d< d|vrai |d< |	ri|	|d d< d|vrqi |d< t|d dg tstd|
r|
|d< |r||d d< |dur||d< |r|pi }|dr||d d< nd|i|d< t| |S )z6Construct the launch specification from CLI arguments.Nr:   rP   rZ   r   rY   r{   dockerr|   r   r}   gitr   	overridesr/   z'override args must be a list of stringsrv   r~   r   registryurl)rp   
isinstancer"   listr
   rz   )r:   rP   rX   r{   rY   rZ   r|   r}   r~   r   rv   r[   r   r   r   r   rs   r   r   r   construct_launch_spec   s`   
r   c                 C  s8   |  d}|  di  d}t|t|krtdd S )NrP   r   r|   zHExactly one of job or docker_image must be specified in the launch spec.)r"   r<   r
   )rs   rP   r|   r   r   r   validate_launch_spec_sourceB  s   
r   tuple[str, str, str]c                 C  s>   t | }|r|jr|jr|jstd|  |j|j|jfS )z9Parse wandb uri to retrieve entity, project and run name.zTrouble parsing wandb uri )r   parserZ   rY   r   r
   )r:   refr   r   r   r_   K  s   
r_   requirements.local.txtdirfilenamec              
   C  s   z,t j}tt j| |d}tjddg||d W d    |W S 1 s%w   Y  |W S  tjyH } zt	d|  W Y d }~d S d }~ww )Nwpipfreeze)envrg   zCommand failed: )
r    r!   openr`   join
subprocesscallCalledProcessErrorri   	termerror)r   r   r   fr,   r   r   r   get_local_python_depsS  s   
r   req_1	list[str]req_2dict[str, str]c                 C  s   ddd}z
|| }||}W n t tttfy& } ztd| d}~ww g }t| t| A D ]}|| q5i }|D ]\}	}
||	du rQ|
||	< qAd	|
 d
||	  ||	< qA|S )zIReturn a list of pip requirements that are not in req_1 but are in req_2.reqr   r   r   c                 S  sP  t  }| D ]}d }d }|drqd|v sd|v r-|dd }|dd dd }ncd	|v rH|d	}|d  }|d dd  }nHd
|v rc|d
}|d  }|d dd  }n-d|v r~|d}|d  }|d dd  }ntt|d ur|}ntd| |d urtt|sJ d| |||< q|S )N#zgit+zhg+z#egg=r   @r   z==z>=>z,Unable to parse pip requirements file line: zInvalid pip package name )	dict
startswithrT   lowerstriprL   r>   _VALID_PIP_PACKAGE_REGEX
ValueError)r   dline_name_version_sr   r   r   
_parse_reqc  s>   



z)diff_pip_requirements.<locals>._parse_reqz"Failed to parse pip requirements: Nvz and v)r   r   r   r   )	AssertionErrorr   
IndexErrorKeyErrorr
   setitemsappendr"   )r   r   r   
req_1_dict
req_2_dictr,   diffitempretty_diffr{   r   r   r   r   diff_pip_requirements`  s"   
$
r   requirements_filec                 C  s   | durUt j|| }t|}|  }W d   n1 s w   Y  t|}|durUt j||}t|}|  }W d   n1 sIw   Y  t|| dS t	d dS )zEWarn if local python dependencies differ from wandb requirements.txt.Nz,Unable to validate local python dependencies)
r    r`   r   r   read
splitlinesr   r   _loggerwarning)r   r   requirements_pathr   wandb_python_depslocal_python_filelocal_python_deps_pathlocal_python_depsr   r   r   validate_wandb_python_deps  s   


r   patch_stringdst_dirc                 C  s   t d ttj|dd}||  W d   n1 sw   Y  ztddd| dd	dg W dS  tj	yB   t
d
w )z$Applies a patch file to a directory.zApplying diff.patchz
diff.patchr   Npatchz-sz--directory=z-p1z-iz/Failed to apply diff.patch associated with run.)r   infor   r    r`   r   writer   
check_callr   ri   Error)r   r   fpr   r   r   apply_patch  s"   



r   c                 C  sF   t d t||}|du rtd| ||  |du r!|j}|S )zClones the git repo at ``uri`` into ``dst_dir``.

    checks out commit ``version``. Assumes authentication parameters are
    specified by the environment, e.g. by a Git credential helper.
    zFetching git repoNzUnable to parse git uri: )r   r   r   r
   fetchr   )r   r:   r   r   r   r   r   _fetch_git_repo  s   
	

r   fnameproject_dirc                 C  s   t jdd}t jdd}td | dd}ttj	|| 9}|
| |j}|jD ]$}|jdkrS|jd}g }	|D ]}
|
d	sL|	|
 q@d	|	|_q/W d    n1 s^w   Y  | }||\}}ttj	||d
}|| W d    |S 1 sw   Y  |S )N	nbconvertz@nbformat and nbconvert are required to use launch with notebooksnbformatzConverting notebook to scriptz.ipynb.pycode
!zw+)ri   r   
get_moduler   r   replacer   r    r`   r   readsr   
NO_CONVERTcells	cell_typesourcerT   r   r   PythonExporterfrom_notebook_node
writelines)r   r   r   r   new_namefhnbcellsource_linesmodified_linesr   exporterr   metar   r   r   "convert_jupyter_notebook_to_script  s<   





r   maybe_snake_strc                 C  s*   d| vr| S |  d}ddd |D S )Nrm   r   c                 s  s     | ]}|r
|  nd V  qdS )rm   N)title).0xr   r   r   	<genexpr>  s    z to_camel_case.<locals>.<genexpr>)rT   r   )r   
componentsr   r   r   to_camel_case  s   
r   build_configregistry_configc                 C  s<   |  di }| di }|r|r||krtdd S d S d S )Ncredentialsz-registry and build config credential mismatch)r"   r
   )r   r   build_config_credentialsregistry_config_credentialsr   r   r   #validate_build_and_registry_configs  s   r   
kubernetestuple[Any, Any]c           	        s   | d}d }|d ustjtjdre| j|\}}d }| dr@|d }|D ]}|d |kr7|} nq+td| d|}t	dd | j
||d I d H  | jj||d d	I d H }||fS | j  | jj }||fS )
N
configFilez~/.kube/configcontextr{   zSpecified context z was not found.awsclizoawscli is required to load a kubernetes context from eks. Please run `pip install wandb[launch]` to install it.)r  )r"   r    r`   exists
expanduserconfiglist_kube_config_contextsr
   r   r   load_kube_confignew_client_from_configload_incluster_configclient
api_client	ApiClient)	r   rv   config_filer  all_contextsactive_contextcontext_namecr  r   r   r   get_kube_context_and_api_client  s<   



r  default_launch_config%tuple[dict[str, Any], dict[str, Any]]c                 C  sn   i }|d u r| d ur|  di }n|d ur|}i }|d u r(| d ur(|  di }n|d ur.|}t|| ||fS )Nbuilderr   )r"   r   )r  r   r   resolved_build_configresolved_registry_configr   r   r   !resolve_build_and_registry_config/  s   
r  c                 C  s   | j  }|stddS )zCheck if a user is logged in.

    Raises an error if the viewer doesn't load (likely a broken API key). Expected time
    cost is 0.1-0.2 seconds.
    zCould not connect with current API-key. Please relogin using `wandb login --relogin` and try again (see `wandb login --help` for more options)T)rX   viewerr
   )rX   resr   r   r   check_logged_inB  s   
r  c                 C  s.   |  dd }tdd|}|d d }|S )Nrm   -z
[^a-z\.\-]r      )r   r   rL   rM   )r{   respr   r   r   make_name_dns_safeS  s   r!  valuec                 C  sT   |  dd }tdd|}tdd|}|dd d}|s(td|  |S )	a!  Return a Kubernetes label/identifier safe string (DNS-1123 label).

    See:
    https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names

    Rules:
    - lowercase alphanumeric and '-'
    - must start and end with an alphanumeric
    - max length 63
    rm   r  z[^a-z0-9\-]r   z-+N?   z$Invalid value for Kubernetes label: )r   r   rL   rM   r   r
   )r"  safer   r   r   make_k8s_label_safe[  s   r%  log	image_urijob_trackerJobAndRunStatusTracker | Nonec                 C  sf   t | }|r/d|d d| d}t| |d ur1|j|dd}||jdd| d S d S d S )	Nz*Failed to install the following packages: r   z for image: z,. Will attempt to launch image without them.zfailed-packages.logr   z>Some packages were not successfully installed during the buildbuild)	FAILED_PACKAGES_REGEXsearchgroupri   rx   saversave_contentsupdate_run_queue_item_warningrun_queue_item_id)r&  r'  rX   r(  r>   _msgr  r   r   r   $warn_failed_packages_from_build_logsu  s    

r3  Fshould_raisec              	   C  sN   t d ztddd| g W dS  tjtfy&   |r t d Y dS w )zsCheck if a specific image is already available.

    Optionally raises an exception if the image is not found.
    z Checking if base image exists...r   imageinspectTz/Base image not found. Generating new base imageF)r   r   r   runDockerErrorr   )r|   r4  r   r   r   docker_image_exists  s   

r9  c              
   C  s@   zt dd| g W dS  t jy } ztd| d}~ww )z Pull the requested docker image.r   pullzDocker server returned error: N)r   r7  r8  r
   )r|   r,   r   r   r   pull_docker_image  s   r;  originalsub_dictdict[str, str | None]c                   s   t  fdd| S )a  Substitute macros in a string.

    Macros occur in the string in the ${macro} format. The macro names are
    substituted with their values from the given dictionary. If a macro
    is not found in the dictionary, it is left unchanged.

    Args:
        original: The string to substitute macros in.
        sub_dict: A dictionary mapping macro names to their values.

    Returns:
        The string with the macros substituted.
    c                   s   t  | d| dS )Nr   r   )r;   r"   r-  )r>   r=  r   r   r     s    zmacro_sub.<locals>.<lambda>)MACRO_REGEXrM   )r<  r=  r   r?  r   	macro_sub  s   rA  r   c                   sT   t | tr
t|  S t | tr fdd| D S t | tr( fdd|  D S | S )a  Recursively substitute macros in a parsed JSON or YAML blob.

    Macros occur in strings at leaves of the blob in the ${macro} format.
    The macro names are substituted with their values from the given dictionary.
    If a macro is not found in the dictionary, it is left unchanged.

    Arguments:
        source: The JSON or YAML blob to substitute macros in.
        sub_dict: A dictionary mapping macro names to their values.

    Returns:
        The blob with the macros substituted.
    c                   s   g | ]}t | qS r   recursive_macro_sub)r   r   r?  r   r   
<listcomp>  s    z'recursive_macro_sub.<locals>.<listcomp>c                   s   i | ]
\}}|t | qS r   rB  )r   keyr"  r?  r   r   
<dictcomp>  s    z'recursive_macro_sub.<locals>.<dictcomp>)r   r;   rA  r   r   r   )r   r=  r   r?  r   rC    s   




rC  runqueuefieldsr   c              
   C  s   i }i }| j D ]}t|d ||d < q|D ]]}|d}t|dkr+td| |\}}||vr?td| j d| d||i }	|	d	}
z|
d
krTt|}n|
dkr\t	|}W n t
yo   td| d|
 dw |||< q|S )Nschemar{   =   zL--set-var value must be in the format "--set-var key1=value1", instead got: zQueue z does not support overriding .typeintegernumberz
Value for z must be of type )rw   r#   r$   rT   rU   r
   r{   r"   intfloatr   )rG  rH  rw   variable_schemastvfieldfield_partsrE  valrI  
field_typer   r   r   %fetch_and_validate_template_variables  s8   




rX  
entrypointc                 C  sD   | sdS | d  ds| d  dr| d S t| dk rdS | d S )zGet the entrypoint file from the given command.

    Args:
        entrypoint (List[str]): List of command and arguments.

    Returns:
        Optional[str]: The entrypoint file if found, otherwise None.
    Nr   r   z.shrK  r   )endswithrU   )rY  r   r   r   get_entrypoint_file  s   	r[  tuple[str, str]c                  C  sJ   t j d d} | d }t| dkrd| d d n|d }||fS )Nr   rL  rK  z.0)rf   r   rT   rU   r   )full_versionmajorr   r   r   r   get_current_python_version  s   &r_  rootdict | listIterator[dict]c                 c  s    t | tr.|  D ]!\}}|dkrt |tr|E dH  q
t |ttfr+t|E dH  q
dS t | tr?| D ]}t|E dH  q5dS dS )zYield all container specs in a manifest.

    Recursively traverses the manifest and yields all container specs. Container
    specs are identified by the presence of a "containers" key in the value.
    
containersN)r   r   r   r   yield_containers)r`  kr   r   r   r   r   rd    s    



rd  c                 C  s   t | tr| D ]}t| qd S t | tsd S | d}t |tr0|d }r0tt||d< t| D ]}|d }rEtt||d< q4|  D ]\}}t |ttfrZt| qJ|dkrit |trit|| |< qJd S )Nmetadatar{   )	r   r   sanitize_identifiers_for_k8sr   r"   r%  r;   rd  r   )r`  r   rf  r{   	containerrE  r"  r   r   r   rg    s(   




rg  )r   r   )r.   r   r   r   )r:   r;   r   r<   )rJ   r;   r   r;   )rP   r;   r   rQ   )r:   rQ   rP   rQ   rX   r	   rY   rQ   rZ   rQ   r[   r\   r   r]   )rX   r	   r[   r\   )rs   rt   r   ru   r1   )"r:   rQ   rP   rQ   rX   r	   r{   rQ   rY   rQ   rZ   rQ   r|   rQ   r}   rQ   r~   r   r   rQ   rv   r\   r[   r\   r   rQ   r   rQ   r   rQ   r   rQ   r   rt   )r:   r;   r   r   )r   )r   r;   r   r;   r   rQ   )r   r   r   r   r   r   )r   rQ   r   r;   r   ru   )r   r;   r   r;   r   ru   )r   r;   r:   r;   r   rQ   r   rQ   )r   r;   r   r;   r   r;   )r   r;   r   r;   )r   rt   r   rt   r   ru   )r   r   rv   rt   r   r  )r  r\   r   r\   r   r\   r   r  )rX   r	   r   r<   )r{   r;   r   r;   )r"  r;   r   r;   )
r&  r;   r'  r;   rX   r	   r(  r)  r   ru   )F)r|   r;   r4  r<   r   r<   )r|   r;   r   ru   )r<  r;   r=  r>  r   r;   )r   r   r=  r>  r   r   )rG  r   rH  r   r   rt   )rY  r   r   rQ   )r   r\  )r`  ra  r   rb  )r`  r   r   ru   )l
__future__r   r2   r#   loggingr    rd   rL   r   rf   collectionsr   collections.abcr   typingr   r   r   clickri   wandb.dockerr   r   wandb.apis.internalr	   wandb.sdk.launch.errorsr
   wandb.sdk.launch.git_referencer    wandb.sdk.launch.wandb_referencer   wandb.sdk.wandb_configr   "builder.templates._wandb_bootstrapr   r   compileescaper+  )wandb.sdk.launch.agent.job_status_trackerr   rH   _VALID_IP_REGEXr   _VALID_WANDB_REGEXr   r=   rA   r?   r@   rN   r@  "AZURE_CONTAINER_REGISTRY_URI_REGEX$ELASTIC_CONTAINER_REGISTRY_URI_REGEX
IGNORECASEGCP_ARTIFACT_REGISTRY_URI_REGEX	S3_URI_RE
GCS_URI_REAZURE_BLOB_REGEXARN_PARTITION_REPROJECT_SYNCHRONOUSLAUNCH_CONFIG_FILELAUNCH_DEFAULT_PROJECT	getLogger__name__r   stylerk   r   __annotations__CODE_MOUNT_DIRr-   r9   rC   rE   rG   rI   rO   rW   rp   rc   rz   r   r   r_   r   r   r   r   r   r   r   r   r  r  r  r!  r%  r3  r9  r;  rA  rC  rX  r[  r_  rd  rg  r   r   r   r   <module>   s    






$

	





!

S
		

9






)








"

