o ߥiE @sddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z ddl mZmZddlmZddlmZddlmZdd lmZmZmZmZmZdd lmZd d lmZm Z m!Z!eZ"eddd ddddddf de#de e#de e#edfde e e e#fde e$de ede e e#e e#fde e e#e e#fde e#de e e e#e#fde e e e#e#fde#fddZ%edddd dddddf de#de e#de e#edfde e#de e e e#fde e$de ede e e#e e#fde e e#e e#fde e e e#e#fde e e e#e#fde#fddZ&deddd ddddddd d e#d!e e#de e#de e#edfde e e e#fde e$de ede e e#e e#fde e e#e e#fde e#de e e e#e#fde e e e#e#ffd"d#Z'd$e#fd%d&Z(d'e e#e e#ffd(d)Z)d'e e#fd*d+Z*dedddddfd,e e#d-ed.e#d e#d/ed0e#d1e#d!e e#de e#de ede e e#e e#fde e e#e e#fde e e e#e#fde e e e#e#ffd2d3Z+dS)4N) CookieJarPath)DictListOptionalUnion)HubApiModelScopeConfig)InvalidParameter)ModelFileSystemCache)model_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT) get_logger)$create_temporary_directory_and_cache download_fileget_file_download_urlFmodel_idrevision cache_dir user_agentlocal_files_onlycookiesignore_file_patternallow_file_pattern local_dirallow_patternsignore_patternsreturnc Cs t|t||||||||| | d S)a Download all files of a repo. Downloads a whole snapshot of a repo's files at the specified revision. This is useful when you want all files from a repo, because you don't know which ones you will need a priori. All files are nested inside a folder in order to keep their actual filename relative to that folder. An alternative would be to just clone a repo but this would require that the user always has git and git-lfs installed, and properly configured. Args: model_id (str): A user or an organization name and a repo name separated by a `/`. revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a commit hash. NOTE: currently only branch and tag name is supported cache_dir (str, Path, optional): Path to the folder where cached files are stored, model will be save as cache_dir/model_id/THE_MODEL_FILES. user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string. local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the local cached file if it exists. cookies (CookieJar, optional): The cookie of the request, default None. ignore_file_pattern (`str` or `List`, *optional*, default to `None`): Any file pattern to be ignored in downloading, like exact file names or file extensions. allow_file_pattern (`str` or `List`, *optional*, default to `None`): Any file pattern to be downloading, like exact file names or file extensions. local_dir (str, optional): Specific local directory path to which the file will be downloaded. allow_patterns (`str` or `List`, *optional*, default to `None`): If provided, only files matching at least one pattern are downloaded, priority over allow_file_pattern. For hugging-face compatibility. ignore_patterns (`str` or `List`, *optional*, default to `None`): If provided, files matching any of the patterns are not downloaded, priority over ignore_file_pattern. For hugging-face compatibility. Raises: ValueError: the value details. Returns: str: Local folder path (string) of repo snapshot Note: Raises the following errors: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) if `use_auth_token=True` and the token cannot be found. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if ETag cannot be determined. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) if some parameter value is invalid repo_typerrrrrrrr r"r!)_snapshot_downloadr) rrrrrrrrr r!r"r'T/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/hub/snapshot_download.pysnapshot_downloads:r) dataset_idc Cs t|t||||||||| | d S)a Download raw files of a dataset. Downloads all files at the specified revision. This is useful when you want all files from a dataset, because you don't know which ones you will need a priori. All files are nested inside a folder in order to keep their actual filename relative to that folder. An alternative would be to just clone a dataset but this would require that the user always has git and git-lfs installed, and properly configured. Args: dataset_id (str): A user or an organization name and a dataset name separated by a `/`. revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a commit hash. NOTE: currently only branch and tag name is supported cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset will be save as cache_dir/dataset_id/THE_DATASET_FILES. local_dir (str, optional): Specific local directory path to which the file will be downloaded. user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string. local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the local cached file if it exists. cookies (CookieJar, optional): The cookie of the request, default None. ignore_file_pattern (`str` or `List`, *optional*, default to `None`): Any file pattern to be ignored in downloading, like exact file names or file extensions. Use regression is deprecated. allow_file_pattern (`str` or `List`, *optional*, default to `None`): Any file pattern to be downloading, like exact file names or file extensions. allow_patterns (`str` or `List`, *optional*, default to `None`): If provided, only files matching at least one pattern are downloaded, priority over allow_file_pattern. For hugging-face compatibility. ignore_patterns (`str` or `List`, *optional*, default to `None`): If provided, files matching any of the patterns are not downloaded, priority over ignore_file_pattern. For hugging-face compatibility. Raises: ValueError: the value details. Returns: str: Local folder path (string) of repo snapshot Note: Raises the following errors: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) if `use_auth_token=True` and the token cannot be found. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if ETag cannot be determined. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) if some parameter value is invalid r$)r&r) r*rrr rrrrrr!r"r'r'r(dataset_snapshot_downloadcs;r+) r%rrrrrrrr r!r"repo_idr%c Cs,|st}|tvrtd|tft|| ||d\} } |r2t| jdkr'tdtd|| Sdt j |di}dt j vrGtt|d <t}|durRt }g}|tkr|j|||d }|d }dt j vrk|ni|d d i}| jdur}| j|d<|j||d|durdn||d}t|| | ||dd||||||| | dng|tkrt|\}}|st}||||}d}d} |j|||dd||d}d|vr|ddkstd||d|dfdS|dd}t|| | ||||||||||| | dt||krn|d7}q| j|dt j| S)Nz'Invalid repo type: %s, only support: %s)r rr%rzCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z6We can not confirm the cached file is for revision: %sz user-agent)rCI_TESTzsnapshot-identifier)rrRevisionSnapshotTruecached_model_revisionTF)rr recursive use_cookiesheaders)r%rrrrr"r!rd/) dataset_name namespacer root_pathr2 page_number page_sizeCodez=Get dataset: %s file list failed, request_id: %s, message: %s RequestIdMessageDataFiles) revision_info) rrr rlen cached_files ValueErrorloggerwarningget_root_locationr get_user_agentosenvironstruuiduuid4r get_cookiesget_valid_revision_detailr1get_model_files_download_file_listsrr rdataset_download_statisticslist_repo_treeprintsave_model_versionpathjoin)r,r%rrrrrrrr r!r"temporary_cache_dircacher4_api repo_filesrevision_detailsnapshot_headergroup_or_ownernamer:r;files_list_treer'r'r(r&s          %r&patterncCs&zt|WdStyYdSw)NTF)recompile BaseException)rbr'r'r(_is_valid_regex3s   rfpatternscCs*t|tr|g}|durdd|D}|S)NcSs"g|] }|ds |n|dqS)r6*)endswith).0itemr'r'r( ?sz'_normalize_patterns..) isinstancerL)rgr'r'r(_normalize_patterns;s rncCs0|durg}|D] }t|r||q|SdS)N)rfappend)rgregex_patternsrkr'r'r(_get_valid_regex_patternEs rqr\rZrYapir`r_c st|}t| } t| } t| } t| }|D]ddkrqzU|r/tfdd|Dr/Wq| r>tfdd| Dr>Wq|rMtfdd|DrMWq| dur`| r`tfdd| Ds`Wq| durs| rstfd d| DssWqWnty}z td |WYd}~nd}~ww|rtj d }t d |d q|t krt |d| d}n|t kr|jd||| d}t||||| qdS)NTypetreecg|] }td|qSrfnmatchrjrb repo_filer'r(rlmz(_download_file_lists..crurrvrxryr'r(rlsr{cs g|] }t|dduqS)NameN)rcsearchrxryr'r(rlysc3 |] }td|VqdSrNrvrxryr'r(   z'_download_file_lists..c3r~rrvrxryr'r(rrz The file pattern is invalid : %sr|zFile z$ already in cache, skip downloading!r)r file_pathr) file_namer7r8r)rnrqany ExceptionrFrGexistsrJrWbasenamedebugrrrget_dataset_file_urlr)r\rZrYr,rrr`r_r4r%rrrrr!r"ignore_regex_patternerurlr'ryr(rRPsv        rR),rwrJrcrMhttp.cookiejarrpathlibrtypingrrrrmodelscope.hub.apir r modelscope.hub.errorsr modelscope.hub.utils.cachingr modelscope.hub.utils.utilsr modelscope.utils.constantrrrrrmodelscope.utils.loggerr file_downloadrrrrFrLboolr)r+r&rfrnrqrRr'r'r'r(sB           K     M