o .iL@sTdZddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z ddl Z ddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZmZee Z!eGdddZ"eGdddZ#Gddde$Z%Gddde$Z&eGdddZ'eGdddZ(Gddde)e*e(fZ+dS)awDatasetInfo record information we know about a dataset. This includes things that we know about the dataset statically, i.e.: - description - canonical location - does it have validation and tests splits - size - etc. This also includes the things that can and should be computed once we've processed the dataset as well: - number of examples (in each split) - etc. N) dataclass)Path)ClassVarOptionalUnion) url_to_fs) DatasetCardDatasetCardData)config)Features) SplitDict)Version) get_logger)asdict unique_valuesc@&eZdZUdZeed<dZeed<dS)SupervisedKeysDatainputoutputN)__name__ __module__ __qualname__rstr__annotations__rrrA/home/ubuntu/.local/lib/python3.10/site-packages/datasets/info.pyr7 rc@r)DownloadChecksumsEntryDatarkeyvalueN)rrrr rrr!rrrrr=rrc@eZdZdZdS)MissingCachedSizesConfigErrorz;The expected cached sizes of the download file are missing.Nrrr__doc__rrrrr#Cr#c@r")NonMatchingCachedSizesErrorz/The prepared split doesn't have expected sizes.Nr$rrrrr'Gr&r'c@sLeZdZUdZeeed<dZeeed<ddZ e deddfdd Z dS) PostProcessedInfoNfeaturesresources_checksumscCs0|jdurt|jtst|j|_dSdSdSN)r) isinstancer from_dictselfrrr __post_init__PszPostProcessedInfo.__post_init__post_processed_info_dictreturnc4ddt|D|difdd|DS)NcSh|]}|jqSrname.0frrr Wz.PostProcessedInfo.from_dict..ci|] \}}|vr||qSrrr8kv field_namesrr Xz/PostProcessedInfo.from_dict..r dataclassesfieldsitems)clsr1rr@rr-U zPostProcessedInfo.from_dict) rrrr)rr rr*dictr0 classmethodr-rrrrr(Ks r(c@seZdZUdZejedZeed<ejedZ eed<ejedZ eed<ejedZ eed<dZ e eed<dZe eed <dZe eed <dZe eed <dZe eed <dZe eed <dZe eeefed<dZe eed<dZe eed<dZe eed<dZe eed<dZe eed<dZ e eed<gdZ!e"e#eed<ddZ$d6de efddZ%d7ddZ&dd Z'e(d!e#dfd"d#Z)e(d8d$ede ed%dfd&d'Z*e(d(ed%dfd)d*Z+d9d:d-d.Z,d;d/d0Z-d%efd1d2Z.e(d3ed%dfd4d5Z/dS)< DatasetInfoa Information about a dataset. `DatasetInfo` documents datasets, including its name, version, and features. See the constructor arguments and properties for a full list. Not all fields are known on construction and may be updated later. Attributes: description (`str`): A description of the dataset. citation (`str`): A BibTeX citation of the dataset. homepage (`str`): A URL to the official homepage for the dataset. license (`str`): The dataset's license. It can be the name of the license or a paragraph containing the terms of the license. features ([`Features`], *optional*): The features used to specify the dataset's column types. post_processed (`PostProcessedInfo`, *optional*): Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index. supervised_keys (`SupervisedKeysData`, *optional*): Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS). builder_name (`str`, *optional*): The name of the `GeneratorBasedBuilder` subclass used to create the dataset. Usually matched to the corresponding script name. It is also the snake_case version of the dataset builder class name. config_name (`str`, *optional*): The name of the configuration derived from [`BuilderConfig`]. version (`str` or [`Version`], *optional*): The version of the dataset. splits (`dict`, *optional*): The mapping between split name and metadata. download_checksums (`dict`, *optional*): The mapping between the URL to download the dataset's checksums and corresponding metadata. download_size (`int`, *optional*): The size of the files to download to generate the dataset, in bytes. post_processing_size (`int`, *optional*): Size of the dataset in bytes after post-processing, if any. dataset_size (`int`, *optional*): The combined size in bytes of the Arrow tables for all splits. size_in_bytes (`int`, *optional*): The combined size in bytes of all files associated with the dataset (downloaded files + Arrow files). **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`]. )default_factory descriptioncitationhomepagelicenseNr)post_processedsupervised_keys builder_name dataset_name config_nameversionsplitsdownload_checksums download_sizepost_processing_size dataset_size size_in_bytes)rVrZr\r)rX_INCLUDED_INFO_IN_YAMLcCs|jdurt|jtst|j|_|jdur$t|jts$t|j|_|jdurCt|jtsCt|jtr Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") >>> ds.info.write_to_directory("/path/to/directory/") ``` wb) pretty_printN) ropen posixpathjoinr DATASET_INFO_FILENAME _dump_inforQLICENSE_FILENAME _dump_license)r/dataset_info_dirrdrbfs_r9rrrwrite_to_directorys "zDatasetInfo.write_to_directorycCs*|tjt||r dnddddS)zQDump info in `file` file-like object open in bytes mode (to support remote files)Nindentutf-8)writejsondumpsrencode)r/filerdrrrris*zDatasetInfo._dump_infocCs||jddS)zTDump license in `file` file-like object open in bytes mode (to support remote files)rsN)rtrQrw)r/rxrrrrkszDatasetInfo._dump_license dataset_infoscsddDtdkrtfddDrdSdtddD}dtddD}dtd dD}dtd dD}d}d}|||||||d S) NcSsg|] }|dur|qSr+)copyr8 dset_inforrr z*DatasetInfo.from_merge..rc3s|] }d|kVqdS)rNrr{ryrr sz)DatasetInfo.from_merge..z cs|]}|jVqdSr+)rNr8inforrrrcsrr+)rOrrrrrrcsrr+)rPrrrrrrcsrr+)rQrrrrrr)rNrOrPrQr)rS)lenallrgrstrip)rHryrNrOrPrQr)rSrrr from_merges""zDatasetInfo.from_mergerlr2cCst|fi|pi^}}td||std|jt|tjddd }t |}Wdn1s7wY| |S)a Create [`DatasetInfo`] from the JSON file in `dataset_info_dir`. This function updates all the dynamically generated fields (num_examples, hash, time of creation,...) of the [`DatasetInfo`]. This will overwrite all previous metadata. Args: dataset_info_dir (`str`): The directory containing the metadata file. This should be the root directory of a specific dataset version. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the file-system backend, if any. Example: ```py >>> from datasets import DatasetInfo >>> ds_info = DatasetInfo.from_directory("/path/to/directory/") ``` zLoading Dataset info from zECalling DatasetInfo.from_directory() with undefined dataset_info_dir.rrsencodingN) rloggerr ValueErrorrerfrgr rhruloadr-)rHrlrbrmrnr9dataset_info_dictrrrfrom_directorys  zDatasetInfo.from_directoryrcr3)NcSr4rr5r7rrrr:r;z(DatasetInfo.from_dict..cr<rrr=r@rrrBrCz)DatasetInfo.from_dict..rrD)rHrrr@rr-rIzDatasetInfo.from_dictTother_dataset_infocs.|j}|jdifdd|jDdS)Ncs(i|]\}}|dus s|t|qSr+rzdeepcopyr= ignore_nonerrrB!s   z&DatasetInfo.update..r)__dict__updaterG)r/rr self_dictrrrrs   zDatasetInfo.updatecCs |jdidd|jDS)NcSi|] \}}|t|qSrrr=rrrrB)r~z$DatasetInfo.copy..r) __class__rrGr.rrrrz(s zDatasetInfo.copycCsfi}t|}|D](}||jvr0t||}t|dr |||<qt|dr,|||<q|||<q|S)N _to_yaml_list_to_yaml_string)rr^getattrhasattrrr)r/ yaml_dictrr r!rrr _to_yaml_dict+s    zDatasetInfo._to_yaml_dict yaml_datacs~t|}|ddurt|d|d<|ddur%t|d|d<ddt|D|difdd|DS)Nr)rXcSr4rr5r7rrrr:@r;z.DatasetInfo._from_yaml_dict..cr<rrr=r@rrrBArCz/DatasetInfo._from_yaml_dict..r) rzrgetr _from_yaml_listr rErFrG)rHrrr@r_from_yaml_dict9s  zDatasetInfo._from_yaml_dict)FN)Fr+)T)rrL)r2rL)0rrrr%rEfieldrrNrrOrPrQr)rr rRr(rSrrTrUrVrWrrrXrJrYrZintr[r\r]r^rrar0rorirkrKrrr-rrzrrrrrrrL[sF - !  rLc@sNeZdZd dddZedddZededdfd d Zdeddfd d ZdS)DatasetInfosDictFr2Nc Cs i}tj|tj}tj|tj}|s||}||tj|rPt |ddd}dd| D}t j |||r=dnddWdn1sKwYtj|r_t |} | j} nd} t} |r|| | duryt dt| d n| } | t|dSdS) NwrsrcSsi|] \}}|t|qSr)rr8rVr|rrrrBOsz7DatasetInfosDict.write_to_directory..rprqz--- z --- )ospathrgr DATASETDICT_INFOS_FILENAMEREPOCARD_FILENAMErrexistsrerGrudumprrdatar to_dataset_card_datarsaver) r/dataset_infos_dir overwriterdtotal_dataset_infosdataset_infos_pathdataset_readme_pathr9dataset_infos_dict dataset_carddataset_card_datarrrroEs0      z#DatasetInfosDict.write_to_directorycCstd|tjtj|tjr(t t |tjj }d|vr(| |Stjtj|tj r`ttj|tj dd}|ddt |DWdS1sYwYdS|S)NzLoading Dataset Infos from dataset_inforsrcSrr)rLr-)r8rVrrrrrBms z3DatasetInfosDict.from_directory..)rrrrrrgr rrrrrfrom_dataset_card_datarrerurG)rHrrr9rrrras  $zDatasetInfosDict.from_directoryrcCslt|dttfr3t|dtr|dd|dDSt|d}|ddd|_||j|iS|S)NrcSs i|] }|ddt|qS)rVdefault)rrLr)r8dataset_info_yaml_dictrrrrBzs z;DatasetInfosDict.from_dataset_card_data..rVr)r,rrarJrLrrV)rHrrrrrrus z'DatasetInfosDict.from_dataset_card_datacCs6|rd|vrt|dtr|ddd|di}nd|vr/t|dtr/dd|dD}ni}i|dd|D}|D]\}}||d<qBt|dkrutt||d<|d dd}|dkrsd|i|d|d<dSdSg|d<t |D]\}}| ddd|i|}|d |qdSdS)NrrVrcSsi|]}|d|qS)rVr)r8config_metadatarrrrBsz9DatasetInfosDict.to_dataset_card_data..cSsi|] \}}||qSr)rrrrrrBsr ) r,rJrrarGrnextitervaluespopsortedappend)r/rdataset_metadata_infosrrVdset_info_yaml_dictrrrrrs@     !z%DatasetInfosDict.to_dataset_card_data)FF)r2N)r2r) rrrrorKrr rrrrrrrDs  r),r%rzrErurrfrpathlibrtypingrrrfsspec fsspec.corerhuggingface_hubrr rr r)r rXr utilsr utils.loggingrutils.py_utilsrrrrrr Exceptionr#r'r(rLrJrrrrrrs>        i