o iҼ @sddlZddlZddlZddlmZddlmZmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZmZddlZdd lmZmZmZdd lmZdd l m!Z!d d l"m#Z#d dl$m%Z%m&Z&d dl'm(Z(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1d dl2m3Z3m4Z4m5Z5e6dej7Z8eGdddZ9Gdddej:Z;GdddejGdddejZ$dedefd?d@Z%e&dfdPfdAdB Z'e(dCdDZ)dEdFZ*dGdHZ+Z,S)Q HfFileSystema% Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: token (`str` or `bool`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . Usage: ```python >>> from huggingface_hub import HfFileSystem >>> fs = HfFileSystem() >>> # List files >>> fs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> fs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` r/hfN)endpointtoken block_sizerDrErFcsHtj|i||p tj|_||_t||d|_||_i|_ i|_ dS)N)rDrE) super__init__rENDPOINTrDrEr _apirF_repo_and_revision_exists_cachedircache)r7rDrErFargsstorage_options __class__r9r:rHcs  zHfFileSystem.__init__r'r(r)r.c Cs|||f|jvrqz |jj|||tjdWnKttfy=}zd|f|j|||f<d|f|j||df<WYd}~n8d}~wty`}zd|f|j|||f<d|j||df<WYd}~nd}~wwd|j|||f<d|j||df<|j|||fS)N)r)r'timeoutF)TN)rKrJ repo_inforHF_HUB_ETAG_TIMEOUTrr#r)r7r'r(r)er9r9r:_repo_and_revision_existys"   z%HfFileSystem._repo_and_revision_existpathcCspdttdttdttfdd}||}|std|dddtjvr@d|vr2td|dd \}}tj|}ntj }| ddkrd |vr|d d \}}d|vrt |}|d ury|d | fvryt d |d}| }n |dd \}}nd }|t||}||||\} } | st|| nd }d|dd d } d|dd d } |dd} d|dd d }| }| }||||\} } | st| ttfr| }|}||||\} }| st|| n1t|| n+|}d }d |vr|d d \}}|t||}nd }||||\} }| s%td|d ur,|ntj}t|||||dS)a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. revision_in_pathr)r.cSs:|dur|dur||krtd|d|d|S|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)rWr)r9r9r:%_align_revision_in_path_with_revisionszHHfFileSystem.resolve_path.._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r1rrr0Nr/)r-)r r@_strip_protocolNotImplementedErrorsplitrr2valuesREPO_TYPES_MAPPINGREPO_TYPE_MODELcountSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprrU_raise_file_not_foundjoin isinstancerr#r5r&)r7rVr)rYr'r(rWmatchr*repo_and_revision_existerrrepo_id_with_namespacepath_in_repo_with_namespacerepo_id_without_namespacepath_in_repo_without_namespace_r9r9r: resolve_pathsx        zHfFileSystem.resolve_pathcCs|s|j|jdS||}|}|r'|j|d||}|s|jsE|j|j|j dfd|j|j|j |j fddSdS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) rLclearrKrrr;pop_parentr*r'r(r))r7rV resolved_pathr9r9r:invalidate_caches   zHfFileSystem.invalidate_cacherbmodeHfFileSystemFilecKsj|dur|n|j}|dur||d<d|vrtd|dkr)t||f||d|St||f||d|S)NrFaz/Appending to remote files is not yet supported.r)ryr))rFr\HfFileSystemStreamFilerz)r7rVryr)rFkwargsr9r9r:_openszHfFileSystem._openc KsR|j||d}|jj|j|j|j|j|j|d|dd|j | ddS)Nr)commit_messagecommit_description)r*r(rEr'r)rrrV) rrrJ delete_filer*r(rEr'r)r3rwr;)r7rVr)r}rvr9r9r:_rms zHfFileSystem._rmF recursivemaxdepthc sj||d}j||||d}fdd|D}dd|D} d|d} | |r+dnd 7} | |d ur9d |dnd 7} jj|j|jj| |j|d | |d dj | dd S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrr)cs"g|] }|s|jqSr9)isdirrrr*).0rVr7r9r: C"z#HfFileSystem.rm..cSsg|]}t|dqS))r*)r)rr*r9r9r:rDzDelete  z recursively r/Nz up to depth rr)r(r'rE operationsr)rrr) rr expand_pathrJ create_commitr(r'rEr)r3rwr;) r7rVrrr)r}rvpaths paths_in_reporrr9rr:rm&s"   zHfFileSystem.rmTdetailrefreshc s|j|d}|z|jf||d|}Wn4tyM|js(td|j|f||d|}fdd|D}t|dkrKtdYnw|rR|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `List[Union[str, Dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rr)Ncg|] }|dkr|qSnamer9rorr9r:rwz#HfFileSystem.ls..rcSsg|]}|dqSrr9rr9r9r:rzs)rrr;_ls_treerr*rgrulen)r7rVrrr)r}rvoutr9rr:lsTs    zHfFileSystem.ls expand_infoc sj||d}|}t|j|j|jd|jd}g} |jvr|sj|} | | g} |rwdt fdd| D} | rw| \} |dusN|kru| djvr]| | dnj| d} | | | fdd| D| s@g}|rfd d| D}|r| s|r|rt j | |}|d s||ks|t| |vr|d n||durt|dd }||8}fd d| D} tjD]}|d r߈j|dqψjd| j|d |||d | Sjj|j|j|||j|jd}|D]U}|d |j }t|tr(||jd|j|j |j!|j"d}n |dd|j#|j!d}|d}j$|g ||t|dd |dusX|kr]| |q | S)Nrr/r*r-rZcs g|] }|ddkr|fqStype directoryr9r path_infodepthr9r:rs z)HfFileSystem._ls_tree..rcs$g|]}|ddkrd|fqS)rrrr9rrr9r:rs   cs&g|]}|ddur|dqS) last_commitNr)rurrr9r:rs&r1cs"g|] }|dds|qS)rr1) startswithr) common_pathr9r:rrT)rrr)rr)rexpandr)r'filersizerblob_idlfsrsecurityrrrrrtree_idr)%rrr;r&r'r(r)r-rLextendrpopleftappendosrV commonprefixendswithrr4rurralistrrtrrJlist_repo_treer*rir"rrrrrr setdefault)r7rVrrr)rrrv root_pathrcached_path_infosdirs_not_in_dircache dirs_to_visitdir_infodirs_not_expanded common_prefixcommon_path_depth cached_pathtreer cache_pathcache_path_info parent_pathr9)rrr7r:r|s       ,   zHfFileSystem._ls_treec/s>|j||dd}tj|g|Ri|EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[Tuple[str, List[str], List[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r)rN)rrr3r;rGwalk)r7rVrMr}rOr9r:rs $zHfFileSystem.walkc s,|j||dd}tj|fi|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `List[str]`: List of paths matching the pattern. r)r)rrr3r;rGglobr7rVr}rOr9r:rs zHfFileSystem.globwithdirsc s"|dur |dkr td|j||d}|}z|j|fd||j|d|Wn-tyUz|j|fd|i|dd krD|iiniWn tyRiYnwYn+w|s`d d Dn|j|fd|ji|} | dd krw| gnd dDt} |s| Sfdd| DS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[List[str], Dict[str, Dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rT)rrr)rr)rrcSsg|] }|ddkr|qSrr9rr9r9r:rErz%HfFileSystem.find..rcSsi|]}|d|qSrr9rr9r9r: Jrz%HfFileSystem.find..ci|]}||qSr9r9)rrrr9r:rOr) rXrrr;rr)rinfoFileNotFoundErrorsorted) r7rVrrrrr)r}rvrnamesr9rr:finds@      zHfFileSystem.findpath1path2c Ks,|j||d}|j||d}|j|jko|j|jk}|rEd|d|}|jj|j|j|j|d||ddt|j|j|jdgdn?|j |d |jd } | } Wd n1s]wYd|d|}|jj | |j|j|j |j|j|d||dd |j |d |j |d d S) a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rzCopy z to rrr/)src_path_in_repor* src_revision)r(r'r)rrrrxNpath_or_fileobjr*r(rEr'r)rrr)rrr'r(rJrr)r3rr*openread upload_filerErwr;) r7rrr)r}resolved_path1resolved_path2 same_reporfcontentr9r9r:cp_fileQsF     zHfFileSystem.cp_filecKs*|j|fii|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rTrdater)r7rVr}rr9r9r:modifieds zHfFileSystem.modifiedc  s|j|d}||dd}|js=dddd|r<|jj|j|j|jdd }idt |j |j |j d d nd| }|sP||jvrP||||jvrlfd d |j|D} | shtd| d|s|dus||rوrوddur|jj|j|j||j|jd} | std| d} t|j|j|jd|jd} t| tr| d| j| jd| j| j| j| jdn| d| jdd| j| jd|sهfdddDdusJS)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `Dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rrFrrN)rrrr)r'r))oidtitler)rrcrrr9rrr9r:rrz%HfFileSystem.info..r)rr)r'r/rr1rrrcrr9r9)rkrr9r:rrz%HfFileSystem.info..)rrr)rrr;r3r*rJlist_repo_commitsr(r'r)r! commit_idr created_atrurLrrgget_paths_infor&r-rir"rVrrrrrr) r7rVrr)r}rvrrrout1 paths_inforrr9)rrVr:rs            zHfFileSystem.infocKs<z|ddr |||j|fi|WdSYdS)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)r3rwrrr9r9r:existss  zHfFileSystem.existscCs*z ||ddkWStyYdSw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rrF)rOSErrorr7rVr9r9r:rs   zHfFileSystem.isdircCs"z ||ddkWSYdS)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rrFrrr9r9r:isfile&s zHfFileSystem.isfilecCsB||}t|j|j|j|j|jd}||r|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r'r)rDz /resolve/z/tree/r) rrrr(r*r'r)rDrreplace)r7rVrvurlr9r9r:r8s  zHfFileSystem.urlc sb|d}t|dh}t|ttfrt|dkr)tj||f||d|St |r0|}n| |r>t j |dddSt|t tfrPt j t j|ddd}|dur]t|d}d}|} |j||d } |j||d d } || z2tt| j| j| j| j|jd ||| d|jt|tr|jndd | | W|r|!dSdS|r|!ww) aC Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r)r)callbackoutfileT)exist_okNFwbrrr(r)filenamer'rD)r temp_filedisplayed_filename expected_size resume_sizeheaders _tqdm_bar)"r3setkeysrirrrrGget_filerrrmakedirsr@rrVdirnamertellrrrset_sizerrr(r)r*r'rDrJ_build_hf_headerstqdmseekclose) r7rpathlpathrrr}r)unhandled_kwargs close_file initial_posresolve_remote_pathrrOr9r:rOsR       zHfFileSystem.get_filecCtd)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.r\rr9r9r: transactions zHfFileSystem.transactioncCr)z@Begin write transaction for deferring files, non-context versionrrrr9r9r:start_transactionszHfFileSystem.start_transactioncCs"tt||j|j|j|jdffS)N)rLrK) make_instancer storage_argsrNrLrKrr9r9r: __reduce__szHfFileSystem.__reduce__N)rxNN)FNN)TFN)FFNFN)NFFFN)FNr.N)-r<r=r>r? root_markerprotocolr r@rboolintrHr ExceptionrUr&rrrwr~rrr r rrrr rrrrrrrrrrrrrpropertyrrr __classcell__r9r9rOr:rB:s%  _  / + ,v ?8(bD  rBcseZdZddededeeffdd ZfddZd ed ed e fd d Z dddZ dde d dfddZ dfdd Zd efddZZS)rzNfsrVr)c stz |j||d|_Wnty'}zd|ddvr"t|d|d}~wwtj||jfi||dS)Nrwryr/B. Make sure the repository and revision exist before writing data.)rrrvrr3rGrHr;)r7r'rVr)r}rTrOr9r:rHszHfFileSystemFile.__init__ct|dsdStSNrvhasattrrG__del__rrOr9r:r.  zHfFileSystemFile.__del__startendr.cCsjdd|d|di|jj}t|jj|jj|jj|jj|jj d}t d||t j d}t ||jS)Nrangezbytes=-rrGET)rrQ)r'rJr rrvr(r)r*r'rDr%rHF_HUB_DOWNLOAD_TIMEOUTr$r)r7r0r1rrrr9r9r: _fetch_ranges zHfFileSystemFile._fetch_rangecCstjddd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerrr9r9r:_initiate_uploadsz!HfFileSystemFile._initiate_uploadFfinalc Cs|jd|j}|j||rN|j|jjj|jj |j j |j j |jj |j j|j j|jd|jddt|jj |jj|j ddSdS)Nrrrrr)bufferrrrwriterr'rJrrrvr*r(rEr'r)r}r3rremoverwr;)r7r=blockr9r9r: _upload_chunks(        zHfFileSystemFile._upload_chunkrcs|jdkr;|dus |dkr;|jdkr;|jj|jddd}|}|jt|7_|WdS1s6wYt|S)a)Read remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rxNrr)rF)rylocr'rrVrrrG)r7lengthrrrOr9r:rs$ zHfFileSystemFile.readcC|j|jSrr'rrVrr9r9r:rzHfFileSystemFile.urlrr)Fr)r<r=r>rBr@r rHr.r#bytesr7r<r"rBrrr&r9r9rOr:rzs   rzc seZdZ    ddedededeed ed ef fd d Zdd edefddZddefddZ defddZ fddZ ddZ Z S)r|rxNrnoner'rVryr)rF cache_typec s|dkr td||dkrtd|d|vr"td|dz |j||d|_Wn"tyN}zd|d d vrDt|d |WYd}~nd}~ww|jdd |_tj||jf|||d |d|_ |dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rJz?HfFileSystemStreamFile only supports cache_type='none' but got r(z;HfFileSystemStreamFile only supports reading but got mode=''rryr/r))rrryrFrK) rXrrrvrr3r;detailsrGrHresponse) r7r'rVryr)rFrKr}rTrOr9r:rHs8   zHfFileSystemStreamFile.__init__rCwhencecCs2|dkr |dkr dS||jkr|dkrdStd)NrrzCannot seek streaming HF file)rCrX)r7rCrPr9r9r:rs zHfFileSystemStreamFile.seekrrDc CsN|dkr|fnd}|jdur5t|jj|jj|jj|jj|jjd}t d||jj dt j d|_t|jzd|jj_|jjj|}WnXty|jt|jj|jj|jj|jj|jjd}t d|dd|ji|jj dt j d|_t|jzd|jj_|jjj|}Wn ty|jwYnw|jt|7_|S) Nrr9rr4T)rstreamrQRangez bytes=%d-)rOrrvr(r)r*r'r'rDr%rJr rr5r$rawdecode_contentrr$rrCr)r7rD read_argsrrr9r9r:r%s`          zHfFileSystemStreamFile.readr.cCrErrFrr9r9r:rVrGzHfFileSystemStreamFile.urlcr*r+r,rrOr9r:r.Yr/zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSr)reopenr'rVry blocksizecacherrr9r9r:r_sz!HfFileSystemStreamFile.__reduce__)rxNrrJ)rrH)r<r=r>rBr@r r#rHrrrr.rr&r9r9rOr:r|s.1 r|r)r.cCst|r|St|Sr)rbrj safe_quoterr9r9r:r6csr6scCs t|ddS)Nr/)safe)r)rZr9r9r:rYgs rYrVrlcCsN|}t|tr |d}nt|tr|d}n t|tr"|d}t||)Nz (repository not found)z (revision not found)z (invalid repository id))rirrr#r)rVrlmsgr9r9r:rgks       rgr'ryrFrKcCs|j||||dS)NrM)r)r'rVryrFrKr9r9r:rVvsrVcCs0||i|}|D] \}}t|||q |Sr)itemssetattr)clsrMr}instance_cache_attributes_dictr'attr cached_valuer9r9r:rzsr)Hrrer: collectionsr dataclassesrrr itertoolsrpathlibrtypingrr r r r r rr urllib.parserrfsspecfsspec.callbacksrrr fsspec.utilsrrequestsrr/r _commit_apirrerrorsrrr file_downloadrrhf_apir r!r"utilsr#r$r%compileVERBOSErbr&AbstractFileSystemrBspecAbstractBufferedFilerzr|r@r6rYr$rgr#rVrr9r9r9r:sP    (    }Me