o niZd%@sdUddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZddlZddlZddlmZddlmZddlmZddlmZddlmZddlmZdd lm Z d d l!m"Z"m#Z#e$e%e#d e%e"Z&e rdd l'm(Z(e$dZ)dZ*dZ+dZ,e$d-e,e*gZ.e$dZ/e$dZ0e$dZ1dZ2e$dZ3e$dZ4e$dZ5e$dej6Z7e$dZ8e$dZ9e$dZ:e$dZ;dZe?e@ZAejBd"d#d$d%ZCe d&d'ZDe eEeFfeGd(<d)eDd*<d+ZHd,e fd-d.ZId/e d,e fd0d1ZJd2eEd,eKfd3d4ZLd2eEd,eKfd5d6ZMd2eEd,eKfd7d8ZNd2eEd,eKfd9d:ZOd;eEd,eEfdeEd,eeEfd?d@ZQd2eeEd>eeEdAedBeeEdCeeEdDee eEe fd,eeeEeEffdEdFZRdAedDee eEe ffdGdHZSdIe eEe fd,dfdJdKZT dd2eeEd>eeEdAedLeeEdBeeEdCeeEdMeeEdNeeEdOeeeEdPeeEdQee eEe fdDee eEe fdReeEdSeeEdTeeEdUeeEd,e eEe ff"dVdWZUdIe eEe fd,dfdXdYZVd2eEd,eeEeEeEffdZd[ZW \dd]eEd^eEd,eeEfd_d`ZXdaeeEdbeeEd,e eEeEffdcddZYdeeeEd]eEd,dfdfdgZZdheEdieEd,dfdjdkZ[dieEd2eEdPeeEd,eeEfdldmZ\dneEdoeEd,eEfdpdqZ]dreEd,eEfdsdtZ^due eEe fdve eEe fd,dfdwdxZ_dye dQe eEe fd,ee e ffdzd{Z`d|ee eEe fduee eEe fdvee eEe fd,ee eEe fe eEe fffd}d~ZadAed,eKfddZbdLeEd,eEfddZcdeEdeEdAededd,df ddZdddMeEdeKd,eKfddZedMeEd,dfddZfdeEde eEeeEfd,eEfddZgde de eEeeEfd,e fddZhde deid,e eEe ffddZjdeeEd,eeEfddZkd,eeEeEffddZldS)N) defaultdict) TYPE_CHECKINGAnyDictListOptionalTuplecast)util)Api) LaunchError) GitReference)WandbReference)Config)FAILED_PACKAGES_POSTFIXFAILED_PACKAGES_PREFIXz(.*))JobAndRunStatusTrackerzA^[^/|^~|^\.].*(git|bitbucket|dev\.azure\.com|\.visualstudio\.com)z)^https?://[0-9]+(?:\.[0-9]+){3}(:[0-9]+)?z^[a-zA-Z0-9_.-]+$z^https?://(api.)?wandb|z^https?://ap\w.qa.wandbz^https?://ap\w.wandb.testz^https?://localhostzWANDB_API_KEY=\w+(-\w+)?z \$\{(\w+)\}zG^(?:https://)?([\w]+)\.azurecr\.io/(?P[\w\-]+):?(?P.*)zx^(?:https://)?(?P[\w-]+)\.dkr\.ecr\.(?P[\w-]+)\.amazonaws\.com/(?P[\.\/\w-]+):?(?P.*)$z^(?:https://)?(?P[\w-]+)-docker\.pkg\.dev/(?P[\w-]+)/(?P[\w-]+)/?(?P[\w-]+)?(?P:.*)?$zs3://([^/]+)(/(.*))?zgs://([^/]+)(?:/(.*))?z:^https://([^\.]+)\.blob\.core\.windows\.net/([^/]+)/?(.*)$z%^arn:([^:]+):[^:]*:[^:]*:[^:]*:[^:]*$ SYNCHRONOUSz"~/.config/wandb/launch-config.yamlzmodel-registryzlaunch:magenta)fg cCsdS)NirrrT/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/launch/utils.pyWsrMAX_ENV_LENGTHSiSageMakerRunnerz /mnt/wandbreturnc Cstjd}|dur.d}d} tjd|}|durn ||7}|d7}q|dkr.tdt}zt|}WntjyM}ztd ||d}~ww|||S) aLoad wandb config from WANDB_CONFIG environment variable(s). The WANDB_CONFIG environment variable is a json string that can contain multiple config keys. The WANDB_CONFIG_[0-9]+ environment variables are used for environments where there is a limit on the length of environment variables. In that case, we shard the contents of WANDB_CONFIG into multiple environment variables numbered from 0. Returns: A dictionary of wandb config values. WANDB_CONFIGNrT WANDB_CONFIG_rzBNo WANDB_CONFIG or WANDB_CONFIG_[0-9]+ environment variables foundzFailed to parse WANDB_CONFIG: ) osenvirongetr rjsonloadsJSONDecodeErrorupdate) config_stridxchunk wandb_config env_configerrrload_wandb_config]s0  r/funccsdtdtdtffdd }|S)aCWrapper for running any function in an awaitable thread on an event loop. Example usage: ``` def my_func(arg1, arg2): return arg1 + arg2 future = event_loop_thread_exec(my_func)(2, 2) assert await future == 4 ``` The returned function must be called within an active event loop. argskwargsrcs2t}tt|dfddIdH}|S)NcsiSNrr)r1r0r2rrrsz9event_loop_thread_exec..wrapper..)asyncioget_event_loopr rrun_in_executor)r1r2loopresultr0)r1r2rwrappers z'event_loop_thread_exec..wrapper)r)r0r:rr9revent_loop_thread_execsr;uricCs,t|pt|pt|pt|duSr3)_WANDB_URI_REGEXmatch_WANDB_DEV_URI_REGEX_WANDB_LOCAL_DEV_URI_REGEX_WANDB_QA_URI_REGEXr<rrr _is_wandb_uris rCcCtt|Sr3)boolr?r>rBrrr_is_wandb_dev_urirFcCrDr3)rEr@r>rBrrr_is_wandb_local_urirGrHcCrDr3)rE_GIT_URI_REGEXr>rBrrr _is_git_urirGrJscCstttd|S)N WANDB_API_KEY)strresub API_KEY_REGEX)rKrrrsanitize_wandb_api_keysrQjobcCs"|d}t|dkr|dSdS)N/r)splitlen)rR job_partsrrrget_project_from_jobs  rXapiprojectentity launch_configc Csd}|dur#t|rt|\}}}nt|r"tjtj|d}n|dur+t|}|dur>d}|r8|d}|p=|p=d}|durGt ||}d} t dkrWt j jdkrWd} tt| d||rfd|nd||fS) NrrZr WindowszUTF-8u🚀 zLaunching run into rS)rCparse_wandb_urirJr"pathsplitextbasenamerXr$get_default_entityplatformsystemsysstdoutencodingwandbtermlog LOG_PREFIX) r<rRrYrZr[r\ source_uri_config_projectprefixrrrset_project_entity_defaultss.     rocCsd}|r |d}|p |jS)Nr[)r$default_entity)rYr\ config_entityrrrrbs  rb launch_speccCs8|ddr|ddrtd|ddSdSdS)N resource_argstemplate_variablesznLaunch spec contains both resource_args and template_variables, only one can be set. Using template_variables.)r$rhtermwarnpop)rrrrr%strip_resource_args_and_template_varssrwname docker_imageresource entry_pointversionrsrun_id repositoryauthorsweep_idcCsh| dur| ni}|dur||d<|dur||d<t|||||| \}}||d<|r-||d<||d<|r7||d<d|vr?i|d<|rG||dd <|rM||d <d |vrY|rU|nd|d <d |vrai|d <| ri| |d d <d|vrqi|d<t|ddgtstd| r| |d<|r||dd<| dur| |d<| r| pi} | dr| | dd<nd| i| d<t||S)z6Construct the launch specification from CLI arguments.Nr<rRr[rrZrxdockerryrrzgitr| overridesr1z'override args must be a list of stringsrsr{r}registryurl)ro isinstancer$listr rw)r<rRrYrxrZr[ryrzr{r|rsr\r}r~rrrrrrrconstruct_launch_specs`     rcCs8|d}|did}t|t|krtddS)NrRrryzHExactly one of job or docker_image must be specified in the launch spec.)r$rEr )rrrRryrrrvalidate_launch_spec_sourceAs rcCs>t|}|r|jr|jr|jstd||j|j|jfS)z9Parse wandb uri to retrieve entity, project and run name.zTrouble parsing wandb uri )rparser[rZr}r )r<refrrrr^Ks r^requirements.local.txtdirfilenamec Csz,tj}ttj||d}tjddg||dWd|WS1s%wY|WStjyH}zt d|WYd}~dSd}~ww)Nwpipfreeze)envrfzCommand failed: ) r"r#openr_join subprocesscallCalledProcessErrorrh termerror)rrrfr.rrrget_local_python_depsSs rreq_1req_2c Csdttdtttffdd}z ||}||}Wnttttfy0}ztd|d}~wwg}t| t| AD]}| |q?i}|D]\} } | | dur[| || <qKd| d|| || <qK|S) zIReturn a list of pip requirements that are not in req_1 but are in req_2.reqrcSsPt}|D]}d}d}|drqd|vsd|vr-|dd}|dddd}ncd |vrH|d }|d}|ddd}nHd |vrc|d }|d}|ddd}n-d |vr~|d }|d}|ddd}ntt|dur|}ntd ||durtt|sJd ||||<q|S)N#zgit+zhg+z#egg=r@rz==z>=>z,Unable to parse pip requirements file line: zInvalid pip package name ) dict startswithrUlowerstriprNr>_VALID_PIP_PACKAGE_REGEX ValueError)rdline_name_version_srrr _parse_reqcsB       z)diff_pip_requirements.._parse_reqz"Failed to parse pip requirements: Nvz and v) rrMrAssertionErrorr IndexErrorKeyErrorr setitemsappendr$) rrr req_1_dict req_2_dictr.diffitem pretty_diffrxr|rrrdiff_pip_requirements`s"$    rrequirements_filecCs|durUtj||}t|}|}Wdn1s wYt|}|durUtj||}t|}|}Wdn1sIwYt||dSt ddS)zEWarn if local python dependencies differ from wandb requirements.txt.Nz,Unable to validate local python dependencies) r"r_rrread splitlinesrr_loggerwarning)rrrequirements_pathrwandb_python_depslocal_python_filelocal_python_deps_pathlocal_python_depsrrrvalidate_wandb_python_depss   r patch_stringdst_dircCstdttj|dd }||Wdn1swYztddd|dd dgWdStj yBt d w) z$Applies a patch file to a directory.zApplying diff.patchz diff.patchrNpatchz-sz --directory=z-p1z-iz/Failed to apply diff.patch associated with run.) rinforr"r_rwriter check_callrrhError)rrfprrr apply_patchs"    rcCsFtdt||}|durtd||||dur!|j}|S)zClones the git repo at ``uri`` into ``dst_dir``. checks out commit ``version``. Assumes authentication parameters are specified by the environment, e.g. by a Git credential helper. zFetching git repoNzUnable to parse git uri: )rrr r fetchr)rr<r|rrrr_fetch_git_repos  rfname project_dircCs tjdd}tjdd}td|dd}ttj ||9}| | |j }|j D]$}|jdkrS|jd}g} |D] } | d sL| | q@d | |_q/Wdn1s^wY|} | |\} } ttj ||d }|| Wd|S1swY|S) N nbconvertz@nbformat and nbconvert are required to use launch with notebooksnbformatzConverting notebook to scriptz.ipynb.pycode !zw+)rhr get_modulerrreplacerr"r_rreadsr NO_CONVERTcells cell_typesourcerUrrPythonExporterfrom_notebook_node writelines)rrrrnew_namefhnbcell source_linesmodified_linesrexporterrmetarrr"convert_jupyter_notebook_to_scripts<           rmaybe_snake_strcCs*d|vr|S|d}ddd|DS)Nrlr css |] }|r |ndVqdS)rlN)title).0xrrr sz to_camel_case..)rUr)r componentsrrr to_camel_cases r build_configregistry_configcCs<|di}|di}|r|r||krtddSdSdS)N credentialsz-registry and build config credential mismatch)r$r )rrbuild_config_credentialsregistry_config_credentialsrrr#validate_build_and_registry_configss  r kubernetesc s|dd}d}|dustjtjdrf|j|\}}d}|drA|d}|D] }|d|kr8|}nq,td|d|}t dd|j ||dIdH|jj ||dd IdH}||fS|j |j j}||fS) N configFilez~/.kube/configcontextrxzSpecified context z was not found.awsclizoawscli is required to load a kubernetes context from eks. Please run `pip install wandb[launch]` to install it.)r)r$r"r_exists expanduserconfiglist_kube_config_contextsr r rload_kube_confignew_client_from_configload_incluster_configclient api_client ApiClient) rrs config_filer all_contextsactive_context context_namecrrrrget_kube_context_and_api_clients<       r default_launch_configcCsni}|dur|dur|di}n|dur|}i}|dur(|dur(|di}n|dur.|}t||||fS)Nbuilderr)r$r)r rrresolved_build_configresolved_registry_configrrr!resolve_build_and_registry_config/s rcCs|j}|s tddS)zCheck if a user is logged in. Raises an error if the viewer doesn't load (likely a broken API key). Expected time cost is 0.1-0.2 seconds. zCould not connect with current API-key. Please relogin using `wandb login --relogin` and try again (see `wandb login --help` for more options)T)rYviewerr )rYresrrrcheck_logged_inBs rcCs.|dd}tdd|}|dd}|S)Nrl-z [^a-z\.\-]r )rrrNrO)rxresprrrmake_name_dns_safeSs rlog image_uri job_trackerrcCsft|}|r/d|dd|d}t||dur1|j|dd}||jdd|dSdSdS) Nz*Failed to install the following packages: rz for image: z,. Will attempt to launch image without them.zfailed-packages.logrz>Some packages were not successfully installed during the buildbuild) FAILED_PACKAGES_REGEXsearchgrouprhrusaver save_contentsupdate_run_queue_item_warningrun_queue_item_id)rrrYrr>_msgrrrr$warn_failed_packages_from_build_logs[s  r%F should_raisec Csbtdz tddd|gWdStjtfy0}z|r |tdWYd}~dSd}~ww) zsCheck if a specific image is already available. Optionally raises an exception if the image is not found. z Checking if base image exists...rimageinspectTz/Base image not found. Generating new base imageNF)rrrrun DockerErrorr)ryr&r.rrrdocker_image_existsns  r+c Cs@z tdd|gWdStjy}ztd|d}~ww)z Pull the requested docker image.rpullzDocker server returned error: N)rr)r*r )ryr.rrrpull_docker_image~s r-originalsub_dictcstfdd|S)aSubstitute macros in a string. Macros occur in the string in the ${macro} format. The macro names are substituted with their values from the given dictionary. If a macro is not found in the dictionary, it is left unchanged. Args: original: The string to substitute macros in. sub_dict: A dictionary mapping macro names to their values. Returns: The string with the macros substituted. cst|d|dS)Nrr)rMr$r)r>r/rrrszmacro_sub..) MACRO_REGEXrO)r.r/rr0r macro_subs r2rcsTt|tr t|St|trfdd|DSt|tr(fdd|DS|S)aRecursively substitute macros in a parsed JSON or YAML blob. Macros occur in strings at leaves of the blob in the ${macro} format. The macro names are substituted with their values from the given dictionary. If a macro is not found in the dictionary, it is left unchanged. Arguments: source: The JSON or YAML blob to substitute macros in. sub_dict: A dictionary mapping macro names to their values. Returns: The blob with the macros substituted. csg|]}t|qSrrecursive_macro_sub)rrr0rr sz'recursive_macro_sub..csi|] \}}|t|qSrr3)rkeyvaluer0rr sz'recursive_macro_sub..)rrMr2rrr)rr/rr0rr4s     r4runqueuefieldsc Csi}i}|jD] }t|d||d<q|D]]}|d}t|dkr+td||\}}||vr?td|jd|d||i} | d } z| d krTt|}n| d kr\t |}Wnt yotd |d | dw|||<q|S)Nschemarx=zL--set-var value must be in the format "--set-var key1=value1", instead got: zQueue z does not support overriding .typeintegernumberz Value for z must be of type ) rtr%r&rUrVr rxr$intfloatr) r9r:rtvariable_schemastvfield field_partsr6valr; field_typerrr%fetch_and_validate_template_variabless8        rJ entrypointcCsD|sdS|dds|ddr|dSt|dkrdS|dS)zGet the entrypoint file from the given command. Args: entrypoint (List[str]): List of command and arguments. Returns: Optional[str]: The entrypoint file if found, otherwise None. Nrrz.shr=r)endswithrV)rKrrrget_entrypoint_files  rMcCsJtjdd}|d}t|dkrd|ddn|d}||fS)Nrr>r=z.0)rer|rUrVr) full_versionmajorr|rrrget_current_python_versions&rPr3)r)F)mr4r%loggingr"rcrNrre collectionsrtypingrrrrrrr clickrh wandb.dockerrr wandb.apis.internalr wandb.sdk.launch.errorsr wandb.sdk.launch.git_referencer wandb.sdk.launch.wandb_referencerwandb.sdk.wandb_configr"builder.templates._wandb_bootstraprrcompileescaper)wandb.sdk.launch.agent.job_status_trackerrrI_VALID_IP_REGEXr_VALID_WANDB_REGEXrr=rAr?r@rPr1"AZURE_CONTAINER_REGISTRY_URI_REGEX$ELASTIC_CONTAINER_REGISTRY_URI_REGEX IGNORECASEGCP_ARTIFACT_REGISTRY_URI_REGEX S3_URI_RE GCS_URI_REAZURE_BLOB_REGEXARN_PARTITION_REPROJECT_SYNCHRONOUSLAUNCH_CONFIG_FILELAUNCH_DEFAULT_PROJECT getLogger__name__rstylerjrrMrB__annotations__CODE_MOUNT_DIRr/r;rErCrFrHrJrQrXrorbrwrrr^rrrrrrrrr rrrr%r+r-r2r4rrJrMrPrrrrs  $             $  !       S   & 9 "       )  ""  "