o .i{ @sddlZddlZddlmZddlmZddlmZmZddl m Z m Z m Z ddl Z ddlmZddlmZddl mZdd lmZdd lmZd d lmZd d lmZd dlmZd dlmZd dlm Z d dlm!Z"d dl#m$Z$m%Z%m&Z&m'Z'm(Z(d dl)m*Z*m+Z+e e,e-e-fe,e-e,dfZ.e-ej/Z0e 1e2Z3Gddde-Z4Gddde5Z6dZ7ej/ddgej8gdej9gdiZ:dZ;ejgd"Z?nejgd%Z?nd&d'gZ>gd(Z?ej/ej8ej9gZ@d)d*e@DZAd+d*e@DZBej/d,giZCe7gZDeBeAeCgZEd-ZFgd.ZGd/e-d0eHfd1d2ZId3e eJeKe-fd0eJe-e eKe-d4fffd5d6ZLd7e-d/e-d0eHfd8d9ZMd7e-d/e-d0eHfd:d;ZNdZO  dTd/e-d?e-d@e eKe-dAe ed0eKe-f dBdCZPdUd?e-dAe ed0eJe-eKe-ffdDdEZQ dUdFe-dAe ed0e.fdGdHZR  dTdIeKe-dAe edJe eSd0eKe.fdKdLZTGdMd4d4eKe-ZUGdNdOdOeJe-eUfZVGdPdQdQeKe-ZWGdRdSdSeJe-eWfZXdS)VN)partial) has_magic)PathPurePath)CallableOptionalUnion) url_to_fs)HTTPFileSystem) HfFileSystem)version) thread_map)config)DownloadConfig) _split_re)Split)logging)tqdm)!_prepare_path_and_storage_options is_local_pathis_relative_path xbasenamexjoin)glob_pattern_to_regexstring_to_dictc@ eZdZdS)UrlN__name__ __module__ __qualname__rrrG/home/ubuntu/.local/lib/python3.10/site-packages/datasets/data_files.pyr"rc@r)EmptyDatasetErrorNrrrrr#r%&r$r%zFdata/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*traintraining) validationvaliddevval)testtestingeval evaluationz-._ 0-9z2023.9.0z**[{sep}/]{keyword}[{sep}]*z{keyword}[{sep}]*)z {keyword}/**z{keyword}[{sep}]*/**z**[{sep}/]{keyword}/**z**[{sep}/]{keyword}[{sep}]*/**z 2023.12.0z**/*[{sep}/]{keyword}[{sep}]*)z{keyword}/**/*z{keyword}[{sep}]*/**/*z**/*[{sep}/]{keyword}/**/*z"**/*[{sep}/]{keyword}[{sep}]*/**/*z**/{keyword}[{sep}]*z**/*[{sep}]{keyword}[{sep}]*)z**/{keyword}/**z**/{keyword}[{sep}]*/**z**/*[{sep}]{keyword}/**z**/*[{sep}]{keyword}[{sep}]*/**cC i|] }|ddt|DqS)cS$g|]}tD] }|j|tdqqS)keywordsep)"KEYWORDS_IN_FILENAME_BASE_PATTERNSformatNON_WORDS_CHARS.0r3patternrrr# M .SPLIT_KEYWORDSr9splitrrr# L rBcCr0)cSr1r2)"KEYWORDS_IN_DIR_NAME_BASE_PATTERNSr6r7r8rrr#r;Ur<r=r>r@rrr#rBTrCz**z*[])z README.mdz config.jsonzdataset_info.jsonzdataset_infos.jsonzdummy_data.zipzdataset_dict.jsonr:returncstfddtDS)Nc3s|]}|vVqdSNr)r9wilcard_characterr:rr# tsz%contains_wildcards..)anyWILDCARD_CHARACTERSrHrrHr#contains_wildcardsssrLpatterns DataFilesListcCst|trdd|DSt|trt|giSt|trntdd|Drj|D]"}t|trCt|dkrCd|vrCt|dttfsJt d|q(d d |D}tt |t|krct d |d d|DSt|iSt t|S) a/ Take the data_files patterns from the user, and format them into a dictionary. Each key is the name of the split, and each value is a list of data files patterns (paths or urls). The default split is "train". Returns: patterns: dictionary of split_name -> list of patterns cSs*i|]\}}t|t|tr|n|gqSrstr isinstancelist)r9keyvaluerrr#rBs*z%sanitize_patterns..css|]}t|tVqdSrF)rQdictr9r:rrr#rIz$sanitize_patterns..rApathz]Expected each split to have a 'path' key which can be a string or a list of strings, but got cSsg|]}|dqSrArrVrrr#r;z%sanitize_patterns..z*Some splits are duplicated in data_files: cSs6i|]}t|dt|dtr|dn|dgqS)rArYrOrVrrr#rBs() rQrUitemsrPSANITIZED_DEFAULT_SPLITrRrJlenget ValueErrorsetsanitize_patterns)rMr:splitsrrr#rbws2      rbmatched_rel_pathcCs<ddt|jjD}ddt|jjD}t|t|kS)u When a path matches a pattern, we additionnally check if it's inside a special directory we ignore by default (if it starts with a double underscore). Users can still explicitly request a filepath inside such a directory if "__pycache__" is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── __pycache__ └── b.txt >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*") False >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*") False cSg|] }|dr|qS__ startswithr9partrrr#r;z6_is_inside_unrequested_special_dir..cSrerfrhrjrrr#r;rl)rparentpartsr^)rdr:data_dirs_to_ignore_in_pathdata_dirs_to_ignore_in_patternrrr#"_is_inside_unrequested_special_dirsrqcCs8ddt|jD}ddt|jD}t|t|kS)u: When a path matches a pattern, we additionnally check if it's a hidden file or if it's inside a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot. Users can still explicitly request a filepath that is hidden or is inside a hidden directory if the hidden part is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*") False base directory: ./ └── .hidden_dir └── a.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*") False base directory: ./ └── .hidden_dir └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*") False cS(g|]}|drt|dhks|qS.rirarjrrr#r; zS_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir..cSrrrsrurjrrr#r;rv)rrnr^)rdr:hidden_directories_in_pathhidden_directories_in_patternrrr#?_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dirs5rypattern_resolverc sztD]qdd}z||}Wn tyYqwt|dkrst|D]}tt|tt}|dus8J|dq%t ddDrTt dt d d fd d t Dt d dt D}fdd|DSqtD]<g}D]&\}}|D]}z||}Wn tyYqwt|dkr||nqq~|rfdd|DSqvtd|d|)a+ Get the default pattern from a directory or repository by testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. z{split}*rNrAcss|] }tt| VqdSrF)rematchrr@rrr#rIsz+_get_data_files_patterns..zSplit name should match 'z '' but got 'z'.csg|] }|vrt|qSrrPr@)rcrr#r;z,_get_data_files_patterns..cSsh|]}t|qSrr~r@rrr# r[z+_get_data_files_patterns..csi|] }|j|dgqS)rZ)r6r@) split_patternrr#rBrz,_get_data_files_patterns..csi|]}||qSrrr@) patterns_dictrr#rB$szCouldn't resolve pattern z with resolver )ALL_SPLIT_PATTERNSreplaceFileNotFoundErrorr^rarrraddrJr`rDEFAULT_SPLITSsortedALL_DEFAULT_PATTERNSr\append) rzr: data_filespp_parts sorted_splitsnon_empty_splitsrArMr)rrrcr#_get_data_files_patternssL         r base_pathallowed_extensionsdownload_configc s~t|r t||}nt|rtj|dtj}nd}t||d\}}t|fi|\}t t t |ht |j tr@|j n|j d}|dkrM|dndi}|dkratjtdkrad|d <fd d |j|fd d i|D}durfdd |D} t| t|krtt |t | } td|d| n|} | sd|d} dur| dt7} t| | S)a Resolve the paths and URLs of the data files from the pattern passed by the user. You can use patterns to resolve multiple local files. Here are a few examples: - *.csv to match all the CSV files at the first level - **.csv to match all the CSV files at any level - data/* to match all the files inside "data" - data/** to match all the files inside "data" and its subdirectories The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix other than a forward slash /. More generally: - '*' matches any character except a forward-slash (to match just the file or directory name) - '**' matches any character including a forward-slash / Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested. The same applies to special directories that start with a double underscore like "__pycache__". You can still include one if the pattern explicilty mentions it: - to include a hidden file: "*/.hidden.txt" or "*/.*" - to include a hidden directory: ".hidden/*" or ".*/*" - to include a special directory: "__special__/*" or "__*/*" Example:: >>> from datasets.data_files import resolve_pattern >>> base_path = "." >>> resolve_pattern("docs/**/*.py", base_path) [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py'] Args: pattern (str): Unix pattern or paths or URLs of the data files to resolve. The paths can be absolute or relative to base_path. Remote filesystems using fsspec are supported, e.g. with the hf:// protocol. base_path (str): Base path to use when resolving relative paths. allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions). For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"] download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. Returns: List[str]: List of paths or URLs to the local or remote files that match the patterns. rrfilez://hfz0.20.0F expand_infocspg|]4\}}|ddks|dr6tjtj|r6t|vrt|st|s|r2|n|qS)typerislink) r_osrYisfilerealpathrrqryri)r9filepathinfo)files_to_ignore fs_patternprotocol_prefixrr#r;gs* z#resolve_pattern..detailTNcs8g|]}tfddt|dddDr|qS)c3s|] }d|vVqdS)rtNr)r9suffixrrr#rIssz-resolve_pattern...rtrN)rJrrA)r9rrrr#r;ps &z Some files matched the pattern 'z-' but don't have valid data file extensions: zUnable to find ''z with any supported extension )rrrrrY splitdriver4rr raFILES_TO_IGNORErrQprotocolrPrHF_HUB_VERSIONr parseglobr\r^rRloggerrr) r:rrrstorage_optionsfsr glob_kwargs matched_pathsoutinvalid_matched_files error_msgr)rrrrr#resolve_pattern(sB0   rcCs:tt||d}zt|WStytd|ddw)uA Get the default pattern from a directory testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. Some examples of supported patterns: Input: my_dataset_repository/ ├── README.md └── dataset.csv Output: {'train': ['**']} Input: my_dataset_repository/ ├── README.md ├── train.csv └── test.csv my_dataset_repository/ ├── README.md └── data/ ├── train.csv └── test.csv my_dataset_repository/ ├── README.md ├── train_0.csv ├── train_1.csv ├── train_2.csv ├── train_3.csv ├── test_0.csv └── test_1.csv Output: {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'], 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]} Input: my_dataset_repository/ ├── README.md └── data/ ├── train/ │ ├── shard_0.csv │ ├── shard_1.csv │ ├── shard_2.csv │ └── shard_3.csv └── test/ ├── shard_0.csv └── shard_1.csv Output: {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...], 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]} Input: my_dataset_repository/ ├── README.md └── data/ ├── train-00000-of-00003.csv ├── train-00001-of-00003.csv ├── train-00002-of-00003.csv ├── test-00000-of-00001.csv ├── random-00000-of-00003.csv ├── random-00001-of-00003.csv └── random-00002-of-00003.csv Output: {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']} In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. )rrzThe directory at z doesn't contain any data filesN)rrrrr%)rrresolverrrr#get_data_patternss T  r data_filec Cst||d\}}t|fi|^}}t|tr"||}|j|jfSt|trR|t j rRtt j |j d}d|t t j dd ddd}||}|j|jfS||}dD]}||vrht||fSqYdS) Nr)endpointtokenzhf://rz /resolve/@)ETagetagmtimer)rr rQr resolve_pathrepo_idrevisionr rir HF_ENDPOINTrr^rrrP) rrrr_ resolved_pathhffsrrSrrr#_get_single_origin_metadatas    $   rr max_workerscCs:|dur|ntj}ttt|d||tdt|dkpddS)NrzResolving data files)r tqdm_classdescdisable)r&HF_DATASETS_MULTITHREADING_MAX_WORKERSr rrhf_tqdmr^)rrrrrr#_get_origin_metadatas rcs0eZdZdZdeedeeddffdd Zdd d Ze   dd eed e j j d e ede eede eddf ddZe   dd eed e ede eede eddf ddZe   dd eed e ede eede eddf ddZdddde eede eeddfddZZS)rNa List of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns: - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, DataFilesList has an additional attribute ``origin_metadata``. It can store: - the last modified time of local files - ETag of remote files - commit sha of a dataset repository Thanks to this additional attribute, it is possible to hash the list and get a different hash if and only if at least one file changed. This is useful for caching Dataset objects that are obtained from a list of data files. rorigin_metadatarENct|||_dSrF)super__init__r)selfrr __class__rr#rs  zDataFilesList.__init__othercCtg|||j|jSrF)rNrrrrrr#__add__zDataFilesList.__add__rM dataset_inforrrcCs6d|jd|jd|p dd}|j||||dS)Nzhf://datasets/r/rrrr)idsharstrip from_patterns)clsrMrrrrrrr# from_hf_repos$ zDataFilesList.from_hf_repocCs,|dur|nt}|j||||dSNr)rresolveas_posixr)rrMrrrrrr#from_local_or_remote-sz"DataFilesList.from_local_or_remotec Cst|dur|nt}g}|D]}z |t||||dWqty.t|s,Yqwt||d}|||SNrr)rrrextendrrrr)rrMrrrrr:rrrr#r:s&   zDataFilesList.from_patterns extensions file_namesrrcsg|rddd|D}td|d|r2ddd|D}td|drBtfd d |D|jd Stt||jd S) N|cs|]}t|VqdSrFr|escape)r9extrrr#rIYrWz'DataFilesList.filter..z.*(z )(\..+)?$csrrFr)r9fnrrr#rI\rWz.*[\/]?(z)$cs&g|]tfddDrqS)c3s|]}|VqdSrF)r}rVrrr#rI`rWz2DataFilesList.filter...)rJ)r9rMrr#r;`s&z(DataFilesList.filter..)r)joinrr|compilerNrrR)rrr ext_pattern fn_patternrrr#filterTszDataFilesList.filter)rrNrErNNNN)r r!r"__doc__rRrPSingleOriginMetadatarr classmethodhuggingface_hubhf_api DatasetInforrrrrr __classcell__rrrr#rNsv"           c@s0eZdZdZe   ddeeeeee ffde ede eede e ddf dd Z e   ddeeeeee ffd e jjde ede eede e ddf d d Ze   ddeeeeee ffde ede eede e ddf d dZdddde eede eeddfddZdS) DataFilesDicta Dict of split_name -> list of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns : - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, each list is a DataFilesList. It is possible to hash the dictionary and get a different hash if and only if at least one file changed. For more info, see [`DataFilesList`]. This is useful for caching Dataset objects that are obtained from a list of data files. Changing the order of the keys of this dictionary also doesn't change its hash. NrMrrrrEcC@|}|D]\}}t|tr|ntj||||d||<q|Sr)r\rQrNrrrMrrrrrSpatterns_for_keyrrr#rw z"DataFilesDict.from_local_or_remoterc CsB|}|D]\}}t|tr|n tj|||||d||<q|S)N)rrrr)r\rQrNr) rrMrrrrrrSrrrr#rs  zDataFilesDict.from_hf_repocCrr)r\rQrNrrrrr#rrzDataFilesDict.from_patternsrrrcCs2t|}|D] \}}|j||d||<q |S)Nr)rr\r)rrrrrSdata_files_listrrr#rs zDataFilesDict.filterr)r r!r"rrrUrPrrRrNrrrrrrrrrrrrr#rgsr        rcseZdZdZdeedeeeeffdd ZddZe ddeedeeed dfd d Z dd ed ee d dfddZ deed dfddZ ZS)DataFilesPatternsListz List of data files patterns (absolute local paths or URLs). For each pattern there should also be a list of allowed extensions to keep, or a None ot keep all the files for the pattern. rMrcrrF)rrr)rrMrrrr#rs  zDataFilesPatternsList.__init__cCrrF)rNrrrrr#rrzDataFilesPatternsList.__add__NrEcCs|||gt|SrF)r^)rrMrrrr#rsz#DataFilesPatternsList.from_patternsrrrNc Cs|dur|nt}g}t||jD]\}}z |t||||dWqty4t|s2Yqwt ||d}t ||Sr) rrrziprrrrrrrN)rrrrr:rrrrr#rs&   zDataFilesPatternsList.resolvercst|fdd|jDS)Ncsg|]}|qSrr)r9rrrr#r;r[z;DataFilesPatternsList.filter_extensions..)rr)rrrr r#filter_extensionssz'DataFilesPatternsList.filter_extensionsrF)r r!r"rrRrPrrrrrrrr rrrrr#rs4   rc@sveZdZdZe ddeeeefdeeeddfddZ dded ee dd fd d Z d eeddfddZ dS)DataFilesPatternsDictz[ Dict of split_name -> list of data files patterns (absolute local paths or URLs). NrMrrEcCs<|}|D]\}}t|tr|ntj||d||<q|S)Nr)r\rQrr)rrMrrrSrrrr#rsz#DataFilesPatternsDict.from_patternsrrrcCs,t}|D] \}}|||||<q|SrF)rr\r)rrrrrSdata_files_patterns_listrrr#rszDataFilesPatternsDict.resolvercCs.t|}|D] \}}||||<q |SrF)rr\r )rrrrSr rrr#r s z'DataFilesPatternsDict.filter_extensionsrF) r r!r"rrrUrPrRrrrrr rrrr#r s(    r )NNrF)Yrr| functoolsrrrpathlibrrtypingrrrr fsspec.corer fsspec.implementations.httpr r packagingr tqdm.contrib.concurrentr rrdownloadrnamingrrcrutilsrrrutils.file_utilsrrrrrutils.py_utilsrrtuplerPrTRAINr] get_loggerr rrrr%SPLIT_PATTERN_SHARDED VALIDATIONTESTr?r7FSSPEC_VERSIONrr5rDr"DEFAULT_PATTERNS_SPLIT_IN_FILENAME"DEFAULT_PATTERNS_SPLIT_IN_DIR_NAMEDEFAULT_PATTERNS_ALLrrrKrboolrLrUrRrbrqryrrrrintrrNrrr rrrr#s                      0&!,>/  (\]  a]5