o 8wi%{ @sddlZddlZddlmZddlmZddlmZmZddl m Z m Z m Z ddl Z ddlmZddl mZddlmZdd lmZd d lmZd d lmZd d lmZd dlmZd dlmZd dlmZ d dl!m"Z"m#Z#m$Z$m%Z%m&Z&d dl'm(Z(e e)e*e*fe)e*e)dfZ+e*ej,Z-e.e/Z0Gddde*Z1Gddde2Z3dZ4ej,ddgej5gdej6gdiZ7dZ8ej9e:dkrdd gZ;gd!Zd*d)e=DZ?ej,d+giZ@e4gZAe?e>e@gZBd,ZCgd-ZDd.e*d/eEfd0d1ZFd2e eGeHe*fd/eGe*e eHe*d3fffd4d5ZId6e*d.e*d/eEfd7d8ZJd6e*d.e*d/eEfd9d:ZKd;e e*geHe*fd/eGe*eHe*ffde*d?e eHe*d@e ed/eHe*f dAdBZMdTd>e*d@e ed/eGe*eHe*ffdCdDZN dTdEe*d@e ed/e+fdFdGZO  dSdHeHe*d@e edIe ePd/eHe+fdJdKZQGdLd3d3eHe*ZRGdMdNdNeGe*eRfZSGdOdPdPeHe*ZTGdQdRdReGe*eTfZUdS)UN)partial) has_magic)PathPurePath)CallableOptionalUnion) url_to_fs) HfFileSystem)version) thread_map)config)DownloadConfig) _split_re)Split)logging)tqdm)!_prepare_path_and_storage_options is_local_pathis_relative_path xbasenamexjoin)string_to_dictc@ eZdZdS)UrlN__name__ __module__ __qualname__rrrP/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/datasets/data_files.pyr!rc@r)EmptyDatasetErrorNrrrrr!r#%r"r#zFdata/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*traintraining) validationvaliddevval)testtestingeval evaluationz-._ 0-9z2023.9.0z**[{sep}/]{keyword}[{sep}]*z{keyword}[{sep}]*)z {keyword}/**z{keyword}[{sep}]*/**z**[{sep}/]{keyword}/**z**[{sep}/]{keyword}[{sep}]*/**z 2023.12.0z**/*[{sep}/]{keyword}[{sep}]*)z{keyword}/**/*z{keyword}[{sep}]*/**/*z**/*[{sep}/]{keyword}/**/*z"**/*[{sep}/]{keyword}[{sep}]*/**/*z**/{keyword}[{sep}]*z**/*[{sep}]{keyword}[{sep}]*)z**/{keyword}/**z**/{keyword}[{sep}]*/**z**/*[{sep}]{keyword}/**z**/*[{sep}]{keyword}[{sep}]*/**cC i|] }|ddt|DqS)cS$g|]}tD] }|j|tdqqS)keywordsep)"KEYWORDS_IN_FILENAME_BASE_PATTERNSformatNON_WORDS_CHARS.0r1patternrrr! L .SPLIT_KEYWORDSr7splitrrr! K r@cCr.)cSr/r0)"KEYWORDS_IN_DIR_NAME_BASE_PATTERNSr4r5r6rrr!r9Tr:r;r<r>rrr!r@SrAz**z*[])z README.mdz config.jsonzdataset_info.jsonzdataset_infos.jsonzdummy_data.zipzdataset_dict.jsonr8returncstfddtDS)Nc3s|]}|vVqdSNr)r7wildcard_characterr8rr! ssz%contains_wildcards..)anyWILDCARD_CHARACTERSrFrrFr!contains_wildcardsrsrJpatterns DataFilesListcCst|trdd|DSt|trt|giSt|trntdd|Drj|D]"}t|trCt|dkrCd|vrCt|dttfsJt d|q(d d |D}tt |t|krct d |d d|DSt|iSt t|S) a/ Take the data_files patterns from the user, and format them into a dictionary. Each key is the name of the split, and each value is a list of data files patterns (paths or urls). The default split is "train". Returns: patterns: dictionary of split_name -> list of patterns cSs*i|]\}}t|t|tr|n|gqSrstr isinstancelist)r7keyvaluerrr!r@s*z%sanitize_patterns..css|]}t|tVqdSrD)rOdictr7r8rrr!rGz$sanitize_patterns..r?pathz]Expected each split to have a 'path' key which can be a string or a list of strings, but got cSsg|]}|dqSr?rrTrrr!r9z%sanitize_patterns..z*Some splits are duplicated in data_files: cSs6i|]}t|dt|dtr|dn|dgqS)r?rWrMrTrrr!r@s() rOrSitemsrNSANITIZED_DEFAULT_SPLITrPrHlenget ValueErrorsetsanitize_patterns)rKr8splitsrrr!r`vs2      r`matched_rel_pathcCs<ddt|jjD}ddt|jjD}t|t|kS)u When a path matches a pattern, we additionally check if it's inside a special directory we ignore by default (if it starts with a double underscore). Users can still explicitly request a filepath inside such a directory if "__pycache__" is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── __pycache__ └── b.txt >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt") True >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*") False >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*") False cSg|] }|dr|qS__ startswithr7partrrr!r9z6_is_inside_unrequested_special_dir..cSrcrdrfrhrrr!r9rj)rparentpartsr\)rbr8data_dirs_to_ignore_in_pathdata_dirs_to_ignore_in_patternrrr!"_is_inside_unrequested_special_dirsrocCs8ddt|jD}ddt|jD}t|t|kS)u9 When a path matches a pattern, we additionally check if it's a hidden file or if it's inside a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot. Users can still explicitly request a filepath that is hidden or is inside a hidden directory if the hidden part is mentioned explicitly in the requested pattern. Some examples: base directory: ./ └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*") False base directory: ./ └── .hidden_dir └── a.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*") False base directory: ./ └── .hidden_dir └── .hidden_file.txt >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*") False >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*") True >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*") False cS(g|]}|drt|dhks|qS.rgr_rhrrr!r9 zS_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir..cSrprqrsrhrrr!r9rt)rrlr\)rbr8hidden_directories_in_pathhidden_directories_in_patternrrr!?_is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dirs5rwpattern_resolverc svtD]odd}z||}Wn tyYqwt|dkrqt|D]}tt|t}|dus6J|dq%tddDrRt dt d d fd d t Dt d dt D}fdd|DSqt D]<g}D]&\}}|D]}z||}Wn tyYqwt|dkr||nqq||rfdd|DSqttd|d|)a+ Get the default pattern from a directory or repository by testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. z{split}*rNr?css|] }tt| VqdSrD)rematchrr>rrr!rGsz+_get_data_files_patterns..zSplit name should match 'z '' but got 'z'.csg|] }|vrt|qSrrNr>)rarr!r9z,_get_data_files_patterns..cSsh|]}t|qSrr|r>rrr! rYz+_get_data_files_patterns..csi|] }|j|dgqS)rX)r4r>) split_patternrr!r@r}z,_get_data_files_patterns..csi|]}||qSrrr>) patterns_dictrr!r@#szCouldn't resolve pattern z with resolver )ALL_SPLIT_PATTERNSreplaceFileNotFoundErrorr\r_rraddrHr^rDEFAULT_SPLITSsortedALL_DEFAULT_PATTERNSrZappend) rxr8 data_filespp_parts sorted_splitsnon_empty_splitsr?rKr)rrrar!_get_data_files_patternssL         r base_pathallowed_extensionsdownload_configc s~t|r t||}nt|rtj|dtj}nd}t||d\}}t|fi|\}t t t |ht |j tr@|j n|j d}|dkrM|dndi}|dkratjtdkrad|d <fd d |j|fd d i|D}durfdd |D} t| t|krtt |t | } td|d| n|} | sd|d} dur| dt7} t| | S)a Resolve the paths and URLs of the data files from the pattern passed by the user. You can use patterns to resolve multiple local files. Here are a few examples: - *.csv to match all the CSV files at the first level - **.csv to match all the CSV files at any level - data/* to match all the files inside "data" - data/** to match all the files inside "data" and its subdirectories The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix other than a forward slash /. More generally: - '*' matches any character except a forward-slash (to match just the file or directory name) - '**' matches any character including a forward-slash / Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested. The same applies to special directories that start with a double underscore like "__pycache__". You can still include one if the pattern explicitly mentions it: - to include a hidden file: "*/.hidden.txt" or "*/.*" - to include a hidden directory: ".hidden/*" or ".*/*" - to include a special directory: "__special__/*" or "__*/*" Example:: >>> from datasets.data_files import resolve_pattern >>> base_path = "." >>> resolve_pattern("docs/**/*.py", base_path) [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py'] Args: pattern (str): Unix pattern or paths or URLs of the data files to resolve. The paths can be absolute or relative to base_path. Remote filesystems using fsspec are supported, e.g. with the hf:// protocol. base_path (str): Base path to use when resolving relative paths. allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions). For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"] download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. Returns: List[str]: List of paths or URLs to the local or remote files that match the patterns. rrfilez://hfz0.20.0F expand_infocspg|]4\}}|ddks|dr6tjtj|r6t|vrt|st|s|r2|n|qS)typerislink) r]osrWisfilerealpathrrorwrg)r7filepathinfo)files_to_ignore fs_patternprotocol_prefixrr!r9fs* z#resolve_pattern..detailTNcs8g|]}tfddt|dddDr|qS)c3s|] }d|vVqdS)rrNr)r7suffixrrr!rGrsz-resolve_pattern...rrr N)rHrr?)r7rrrr!r9os &z Some files matched the pattern 'z-' but don't have valid data file extensions: zUnable to find ''z with any supported extension )rrrrrW splitdriver2rr r_FILES_TO_IGNORErrOprotocolrNrHF_HUB_VERSIONr parseglobrZr\rPloggerrr) r8rrrstorage_optionsfsr glob_kwargs matched_pathsoutinvalid_matched_files error_msgr)rrrrr!resolve_pattern'sB0   rcCs:tt||d}zt|WStytd|ddw)uA Get the default pattern from a directory testing all the supported patterns. The first patterns to return a non-empty list of data files is returned. Some examples of supported patterns: Input: my_dataset_repository/ ├── README.md └── dataset.csv Output: {'train': ['**']} Input: my_dataset_repository/ ├── README.md ├── train.csv └── test.csv my_dataset_repository/ ├── README.md └── data/ ├── train.csv └── test.csv my_dataset_repository/ ├── README.md ├── train_0.csv ├── train_1.csv ├── train_2.csv ├── train_3.csv ├── test_0.csv └── test_1.csv Output: {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'], 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]} Input: my_dataset_repository/ ├── README.md └── data/ ├── train/ │ ├── shard_0.csv │ ├── shard_1.csv │ ├── shard_2.csv │ └── shard_3.csv └── test/ ├── shard_0.csv └── shard_1.csv Output: {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...], 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]} Input: my_dataset_repository/ ├── README.md └── data/ ├── train-00000-of-00003.csv ├── train-00001-of-00003.csv ├── train-00002-of-00003.csv ├── test-00000-of-00001.csv ├── random-00000-of-00003.csv ├── random-00001-of-00003.csv └── random-00002-of-00003.csv Output: {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']} In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. )rrzThe directory at z doesn't contain any data filesN)rrrrr#)rrresolverrrr!get_data_patternss T  r data_filec Cst||d\}}t|fi|^}}t|tr"||}|j|jfS|tj rMttj |j d}d|t tj dd ddd}||}|j|jfS| |}dD]}||vrct||fSqTdS) Nr)endpointtokenzhf://r z /resolve/@)ETagetagmtimer)rr rOr resolve_pathrepo_idrevisionrgr HF_ENDPOINTrr\rrrN) rrrr_ resolved_pathhffsrrQrrr!_get_single_origin_metadatas     $   rr max_workerscCs:|dur|ntj}ttt|d||tdt|dkpddS)NrzResolving data files)r tqdm_classdescdisable)r&HF_DATASETS_MULTITHREADING_MAX_WORKERSr rrhf_tqdmr\)rrrrrr!_get_origin_metadatas rcs0eZdZdZdeedeeddffdd Zdd d Ze   dd eed e j j d e ede eede eddf ddZe   dd eed e ede eede eddf ddZe   dd eed e ede eede eddf ddZdddde eede eeddfddZZS)rLa List of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns: - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, DataFilesList has an additional attribute ``origin_metadata``. It can store: - the last modified time of local files - ETag of remote files - commit sha of a dataset repository Thanks to this additional attribute, it is possible to hash the list and get a different hash if and only if at least one file changed. This is useful for caching Dataset objects that are obtained from a list of data files. rorigin_metadatarCNct|||_dSrD)super__init__r)selfrr __class__rr!rs  zDataFilesList.__init__othercCtg|||j|jSrD)rLrrrrrr!__add__zDataFilesList.__add__rK dataset_inforrrcCs6d|jd|jd|p dd}|j||||dS)Nzhf://datasets/r/rrrr)idsharstrip from_patterns)clsrKrrrrrrr! from_hf_repos$ zDataFilesList.from_hf_repocCs,|dur|nt}|j||||dSNr)rresolveas_posixr)rrKrrrrrr!from_local_or_remote,sz"DataFilesList.from_local_or_remotec Cst|dur|nt}g}|D]}z |t||||dWqty.t|s,Yqwt||d}|||SNrr)rrrextendrrrr)rrKrrrrr8rrrr!r9s&   zDataFilesList.from_patterns extensions file_namesrrcsg|rddd|D}td|d|r2ddd|D}td|drBtfd d |D|jd Stt||jd S) N|cs|]}t|VqdSrDrzescape)r7extrrr!rGXrUz'DataFilesList.filter..z.*(z )(\..+)?$csrrDr)r7fnrrr!rG[rUz.*[\/]?(z)$cs&g|]tfddDrqS)c3s|]}|VqdSrD)r{rTrrr!rG_rUz2DataFilesList.filter...)rH)r7rKrr!r9_s&z(DataFilesList.filter..)r)joinrrzcompilerLrrP)rrr ext_pattern fn_patternrrr!filterSszDataFilesList.filter)rrLrCrLNNN)rrr __doc__rPrNSingleOriginMetadatarr classmethodhuggingface_hubhf_api DatasetInforrrrrr __classcell__rrrr!rLsv"           c@s0eZdZdZe   ddeeeeee ffde ede eede e ddf dd Z e   ddeeeeee ffd e jjde ede eede e ddf d d Ze   ddeeeeee ffde ede eede e ddf d dZdddde eede eeddfddZdS) DataFilesDicta Dict of split_name -> list of data files (absolute local paths or URLs). It has two construction methods given the user's data files patterns : - ``from_hf_repo``: resolve patterns inside a dataset repository - ``from_local_or_remote``: resolve patterns from a local path Moreover, each list is a DataFilesList. It is possible to hash the dictionary and get a different hash if and only if at least one file changed. For more info, see [`DataFilesList`]. This is useful for caching Dataset objects that are obtained from a list of data files. Changing the order of the keys of this dictionary also doesn't change its hash. NrKrrrrCcC@|}|D]\}}t|tr|ntj||||d||<q|Sr)rZrOrLrrrKrrrrrQpatterns_for_keyrrr!rv z"DataFilesDict.from_local_or_remoterc CsB|}|D]\}}t|tr|n tj|||||d||<q|S)N)rrrr)rZrOrLr) rrKrrrrrrQrrrr!rs  zDataFilesDict.from_hf_repocCrr)rZrOrLrrrrr!rrzDataFilesDict.from_patternsrrrcCs2t|}|D] \}}|j||d||<q |S)Nr)rrZr)rrrrrQdata_files_listrrr!rs zDataFilesDict.filterr)rrr rrrSrNrrPrLrrrrrrrrrrrrr!rfsr        rcseZdZdZdeedeeeeffdd ZddZe ddeedeeed dfd d Z dd ed ee d dfddZ deed dfddZ ZS)DataFilesPatternsListz List of data files patterns (absolute local paths or URLs). For each pattern there should also be a list of allowed extensions to keep, or a None ot keep all the files for the pattern. rKrcrrD)rrr)rrKrrrr!rs  zDataFilesPatternsList.__init__cCrrD)rLrrrrr!rrzDataFilesPatternsList.__add__NrCcCs|||gt|SrD)r\)rrKrrrr!rsz#DataFilesPatternsList.from_patternsrrrLc Cs|dur|nt}g}t||jD]\}}z |t||||dWqty4t|s2Yqwt ||d}t ||Sr) rrrziprrrrrrrL)rrrrr8rrrrr!rs&   zDataFilesPatternsList.resolvercst|fdd|jDS)Ncsg|]}|qSrr)r7rrrr!r9rYz;DataFilesPatternsList.filter_extensions..)rr)rrrrr!filter_extensionssz'DataFilesPatternsList.filter_extensionsrD)rrr rrPrNrrrrrrrr rrrrr!rs4   rc@sveZdZdZe ddeeeefdeeeddfddZ dded ee dd fd d Z d eeddfddZ dS)DataFilesPatternsDictz[ Dict of split_name -> list of data files patterns (absolute local paths or URLs). NrKrrCcCs<|}|D]\}}t|tr|ntj||d||<q|S)Nr)rZrOrr)rrKrrrQrrrr!rsz#DataFilesPatternsDict.from_patternsrrrcCs,t}|D] \}}|||||<q|SrD)rrZr)rrrrrQdata_files_patterns_listrrr!r szDataFilesPatternsDict.resolvercCs.t|}|D] \}}||||<q |SrD)rrZr )rrrrQr rrr!r s z'DataFilesPatternsDict.filter_extensionsrD) rrr rrrSrNrPrrrrr rrrr!r s(    r )NNrD)Vrrz functoolsrrrpathlibrrtypingrrrr fsspec.corer r packagingr tqdm.contrib.concurrentr rrdownloadrnamingrrarutilsrrrutils.file_utilsrrrrrutils.py_utilsrtuplerNrTRAINr[ get_loggerrrrrr#SPLIT_PATTERN_SHARDED VALIDATIONTESTr=r5FSSPEC_VERSIONrr3rBr"DEFAULT_PATTERNS_SPLIT_IN_FILENAME"DEFAULT_PATTERNS_SPLIT_IN_DIR_NAMEDEFAULT_PATTERNS_ALLrrrIrboolrJrSrPr`rorwrrrrintrrLrrr rrrr!s                      0&!,>/  (\]  a]5