o -wi%@sUdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZmZmZmZddlZddlmZmZddlmZdd lmZmZmZdd lmZmZmZGd d d eZ dddZ!GdddZ"da#de$d<dddZ%dS)zArtifact cache.) annotationsN)Path)NamedTemporaryFile)IOContextManagerIteratorProtocol)envutil)files_in)B64MD5ETag b64_to_hex_id) FilePathStrStrPathURIStrc@seZdZd d ddZdS) Opener.modestrreturnContextManager[IO]cCsdSN)selfrrrd/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/sdk/artifacts/artifact_file_cache.py__call__szOpener.__call__N).)rrrr)__name__ __module__ __qualname__rrrrrrsrrintcCstjddf}tt|S)Nz-czimport os; print(os.umask(22)))sys executabler subprocess check_output) umask_cmdrrr_get_sys_umask_threadsafes r%c@steZdZd,ddZd-d d Zd.ddZ d/d0ddZ   d1d2d!d"Zd3d#d$Zd4d%d&Z d/d5d(d)Z d6d*d+Z dS)7ArtifactFileCache cache_dirrrNonecCs<t||_|jd|_|jd|_|t|_d|_dS)Nobjtmp)r _cache_dir_obj_dir _temp_dir_ensure_write_permissionsr% _sys_umask_override_cache_path)rr'rrr__init__&s    zArtifactFileCache.__init__b64_md5r sizer tuple[FilePathStr, bool, Opener]cCsX|jdur d}t|j}nd}t|}|jd|dd|dd}|j|||dS)NTFmd5 skip_cache)r0rrr,_check_or_create)rr2r3r8pathhex_md5rrrcheck_md5_obj_path2s  "z$ArtifactFileCache.check_md5_obj_pathurlretagr cCs|jdur d}t|j}n,d}tt|dt|d}|jd|dd|dd}|j|||dS)NTFzutf-8r>r6r7) r0rhashlibsha256encodedigest hexdigestr,r9)rr=r>r3r8r:hexhashrrrcheck_etag_obj_pathAs  "z%ArtifactFileCache.check_etag_obj_pathFr:rr8boolcCs8|j|||d}|o|j|k}tt|||fS)Nr7)_openeris_filestatst_sizerr)rr:r3r8openerhitrrrr9Tsz"ArtifactFileCache._check_or_createN target_size int | None remove_temptarget_fraction float | Nonec Cs|dur |dur d}|dur|durtd|dur"|dkr"td|dur2|dks.|dkr2tdd}d}d}t|jD]'}|j}||7}|r`z t|j||7}Wq=ty_Yq=w||7}q=|rst dt |dg} t|j D]} || j7}| | qz|durt||}|dusJt| d d d D]*}||kr|Szt|jWn tyYnw||j8}||j7}q||krt d |jd |S)aClean up the cache, removing the least recently used files first. Args: target_size: The target size of the cache in bytes. If the cache is larger than this, we will remove the least recently used files until the cache is smaller than this size. remove_temp: Whether to remove temporary files. Temporary files are files that are currently being written to the cache. If remove_temp is True, all temp files will be removed, regardless of the target_size or target_fraction. target_fraction: The target fraction of the cache to reclaim. If the cache is larger than this, we will remove the least recently used files until the cache is smaller than this fraction of its current size. It is an error to specify both target_size and target_fraction. Returns: The number of bytes reclaimed. Nrz3Cannot specify both target_size and target_fractionz target_size must be non-negativez'target_fraction must be between 0 and 1zCache contains zO of temporary files. Run `wandb artifact cleanup --remove-temp` to remove them.cSs |jSr)rIst_atime)xrrrs z+ArtifactFileCache.cleanup..)keyz"Failed to reclaim enough space in zU. Try running `wandb artifact cache cleanup --remove-temp` to remove temporary files.) ValueErrorr r-rIrJosremover:OSErrorwandbtermwarnr to_human_sizer,appendrsorted termerrorr+) rrMrOrPbytes_reclaimed total_size temp_sizeentryr3entries file_entryrrrcleanup[s`          zArtifactFileCache.cleanupcCst|jdS)z@Return the number of bytes of free space in the cache directory.r6)shutil disk_usager+)rrrr _free_spaceszArtifactFileCache._free_spacecCsf||krdStd|jdd||krdS|jdd||kr1ttjd|jdS)zIf a `size` write would exceed disk space, remove cached items to make space. Raises: OSError: If there is not enough space to write `size` bytes, even after removing cached items. Nz3Cache size exceeded. Attempting to reclaim space...g?)rPr)rMzInsufficient free space in )rjr[r\rgrZerrnoENOSPCr+)rr3rrr_reserve_spaces      z ArtifactFileCache._reserve_spacercs tjdd fdd }|S) Nwrrr Iterator[IO]c3sd|vr tdrj}nj}|jdddt||dd}z$|V|t|j dj @jjdddt |j WdSt yWt |j w)Naz)Appending to cache files is not supportedTparentsexist_okF)dirrdeletei)rWparentrmr-mkdirrcloserXchmodnamer/replace ExceptionrY)rtemp_dir temp_filer:rr3r8rr atomic_opens&   z.ArtifactFileCache._opener..atomic_open)rn)rrrro) contextlibcontextmanager)rr:r3r8rrrrrGszArtifactFileCache._openerc Cs~z(|jjdddt|jd}|dWdWdS1s!wYWdSty>}z td|jd|d}~ww)z9Raise an error if we cannot write to the cache directory.Trq)rtswandbNzUnable to write to z5. Ensure that the current user has write permissions.)r-rwrwritePermissionErrorr+)rferrrr.s & z+ArtifactFileCache._ensure_write_permissions)r'rrr()r2r r3rrr4)r=rr>r r3rrr4)F)r:rr3rr8rFrr4)NFN)rMrNrOrFrPrQrrrr)r3rrr()r:rr3rr8rFrr)rr() rrrr1r<rEr9rgrjrmrGr.rrrrr&%s     S  r&zArtifactFileCache | None_artifact_file_cachecCs*td}tdustj|krt|atS)N artifacts)r get_cache_dirrr+r&)r'rrrget_artifact_file_caches rr)rr&)&__doc__ __future__rrrkr?rXrhr"r pathlibrtempfilertypingrrrrr[r r wandb.sdk.lib.filesystemr wandb.sdk.lib.hashutilr r rwandb.sdk.lib.pathsrrrrr%r&r__annotations__rrrrrs.       K