o ॵi@szddlZddlZddlmZmZmZmZmZmZm Z m Z m Z ddl Z ddlmZmZmZmZddlmZddlmZddlmZddlmZddlmZmZmZmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl0m1Z1ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlZ>ddl?m@Z@e@ZAdefddZBGdddZCdS)N) AnyCallableDictIterableListMappingOptionalSequenceUnion)Dataset DatasetDictIterableDatasetIterableDatasetDict)_PACKAGED_DATASETS_MODULES)is_relative_path)DatasetRepository)DatasetContextConfig)LocalDataLoaderManagerLocalDataLoaderTypeRemoteDataLoaderManagerRemoteDataLoaderType)ExternalDatasetNativeIterableDataset)build_custom_dataset)DatasetDeleteManager)load_dataset_with_ctx)DatasetUploadManager)build_preprocessor)Config ConfigDict)MS_DATASETS_CACHE) DEFAULT_DATASET_NAMESPACEDEFAULT_DATASET_REVISION ConfigFieldsDatasetFormations DownloadModeHubsModeKeysTasks UploadMode)is_tf_availableis_torch_available) get_loggerreturncCsJ|durg}|St|tr|g}|Stt|t|kr#td||S)Nz"List columns contains duplicates: ) isinstancestrlenset ValueError)parar4T/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/ms_dataset.py format_list's r6c'@s>eZdZUdZdZdZeed< d\dee e e e fde efddZdd Zd d Zd d ZeddZeddZe d\dee ee fdedeedffddZe d\dee ee e e efdedeedffddZeedeej dddde!j"e#dde$ddfdeee%fde ede ede ede ede ede ed e ed!e eee&ee'eeee&efffd"e e!d#e ed$e e(d%e e)d&e e$d'e ed(e e(deede ff"d)d*Z*eeeddd+e+j,fd,ed-edede ede ed.e e)d/e e)d0e e(d1e e+ddfd2d3Z-eeddfd4ed5ed6e ed7e ed8e eddf d9d:Z.eedddfd4ed;ed6e ed7e ed8e edZ/eeefd,edede ede edef d?d@Z0     +d]dAeee1efdBee2e1e2fdCedDe3dEe(f dFdGZ4       +d^dHe)dIe(dBee2e1e2fdAeee1efdJe2dKe(dLe5ee6fdMeee1efdNe(fdOdPZ7de fdQdRZ8dSe5eefde fdTdUZ9  +d_dBee2e1e2fdAeee1efdEe(fdVdWZ:  +  d`dHe)dIe(dBee2e1e2fdKe(dNe(dMeee1efdAeee1effdXdYZ;  dad&e$fdZd[Zs zMsDataset.__init__ccs0|jD]}|jdur||jVq|VqdSN)r;r:)rAitemr4r4r5__iter__Js   zMsDataset.__iter__cCs |j|SrCr;)rAkeyr4r4r5 __getitem__Q zMsDataset.__getitem__cC t|jSrC)r0r;rAr4r4r5__len__TrIzMsDataset.__len__cCs|jSrCrFrKr4r4r5r9WszMsDataset.ds_instancecCst|jtr |jjSdSrC)r.r;r config_kwargsrKr4r4r5rM[s zMsDataset.config_kwargshf_dsr-cstdtt|tr|St|tr4t|dkr(tt | Sfdd| DSt|t r=|St dt|)z @deprecated This method is deprecated and may be removed in future releases, please use `to_ms_dataset()` instead. z@from_hf_dataset is deprecated, please use to_ms_dataset instead.ci|] \}}||qSr4r4.0kvclsr:r4r5 rz-MsDataset.from_hf_dataset..z2"hf_ds" must be a Dataset or DatasetDict, but got )warningswarnDeprecationWarningr.r r r0r?nextitervaluesitemsrr=type)rVrNr:r4rUr5from_hf_datasetbs     zMsDataset.from_hf_datasetcst|tr |St|tr.t|dkr"tt|Sfdd|DSt|t r7|St|t r@|St|t rI|St|t rmt|dkratt|Sfdd|DSt dt|)z&Convert input to `MsDataset` instance.rOcrPr4r4rQrUr4r5rWrXz+MsDataset.to_ms_dataset..crPr4r4rQrUr4r5rWrXz8"ds_instance" must be a Dataset or DatasetDict, but got )r.r r r0r?r\r]r^r_rrr rr=r`)rVr9r:r4rUr5 to_ms_datasetzs&        zMsDataset.to_ms_datasetFrO dataset_name namespaceversionhub subset_namesplitdata_dir data_files download_mode cache_dir use_streamingstream_batch_size custom_cfgtokendataset_info_onlyc!KsP|rddlm}|}||t| ptj} t|ptj}t|ts1t|t s1t dt |t|t rJ|dur| j?Sd)a Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. Args: dataset_name (str): Path or name of the dataset. The form of `namespace/dataset_name` is also supported. namespace(str, optional): Namespace of the dataset. It should not be None if you load a remote dataset from Hubs.modelscope, namespace (str, optional): Namespace of the dataset. It should not be None if you load a remote dataset from Hubs.modelscope, target (str, optional): Name of the column to output. version (str, optional): Version of the dataset script to load: subset_name (str, optional): Defining the subset_name of the dataset. data_dir (str, optional): Defining the data_dir of the dataset configuration. I data_files (str or Sequence or Mapping, optional): Path(s) to source data file(s). split (str, optional): Which split of the data to load. hub (Hubs or str, optional): When loading from a remote hub, where it is from. default Hubs.modelscope download_mode (DownloadMode or str, optional): How to treat existing datasets. default DownloadMode.REUSE_DATASET_IF_EXISTS cache_dir (str, Optional): User-define local cache directory. use_streaming (bool, Optional): If set to True, no need to download all data files. Instead, it streams the data progressively, and returns NativeIterableDataset or a dict of NativeIterableDataset. stream_batch_size (int, Optional): The batch size of the streaming data. custom_cfg (str, Optional): Model configuration, this can be used for custom datasets. see https://modelscope.cn/docs/Configuration%E8%AF%A6%E8%A7%A3 token (str, Optional): SDK token of ModelScope. dataset_info_only (bool, Optional): If set to True, only return the dataset config and info (dict). **config_kwargs (additional keyword arguments): Keyword arguments to be passed Returns: MsDataset (MsDataset): MsDataset object for a certain dataset. r)HubApiz.dataset_name must be `str` or `list`, but got Nr:)r:/rOzUThe dataset_name should be in the form of `namespace/dataset_name` or `dataset_name`.) rcrdrergrhr:rfrirjrkcache_root_dirrmrnroT) load_dataset)namerh streamingrk)rcrd)pathrvrirjrhrlr<download_configrkrevisionrprwrqtrust_remote_code)VirgoDownloader)VirgoDatasetConfig) CACHE_HOMEvirgorfdatasetszPlease adjust input args to specify a loading mode, we support following scenes: loading from local disk, huggingface hub and modelscope hub.r4)@modelscope.hub.apirrloginr%REUSE_DATASET_IF_EXISTSr& modelscoper.r/r>r=r`r from_dictr7rbosrx expanduserexistsrcountrhstriprrisdirisfilerrurHF_DATA_LOADERr8to_custom_datasetr@ huggingfacervalueget_dataset_id_and_typer$generalrrrMS_DATA_LOADERdataset_context_configr-modelscope.msdatasets.data_loader.data_loaderr|modelscope.utils.constantr}r!default_virgo_namespacerdr"default_dataset_versionrer modelscope.utils.config_dsr~joinrtprocessdataset)!rcrdr:rerfrgrhrirjrkrlrmrnrorprqrMrrapi dataset_inst is_local_pathdataset_name_splitrru_apidataset_id_on_hub dataset_type dataset_resremote_dataloader_managerr|r}r~virgo_downloaderr4r4r5loads*9                  &         zMsDataset.loadT object_namelocal_file_path num_processes chunksizefilter_hidden_files upload_modec Cs tdt|s tdt|||d} t|ptj}tj |r+| j |||ddStj |r>| j ||||||ddSt|d)z @deprecated This method is deprecated and may be removed in future releases, please use git command line instead. Hupload is deprecated, please use git command line to upload the dataset.zobject_name cannot be empty!rcrdre)rrr)object_dir_namelocal_dir_pathrrrrz& is not a valid file path or directoryN) rYrZr[r2rr) OVERWRITErrxruploadr upload_dir) rrrcrdrerrrr_upload_managerr4r4r5r^s:   zMsDataset.uploaddataset_work_dir dataset_idrz auth_tokengit_pathcCsRtdtt|||||d}|}|rtd|dStd|dS)aClone meta-file of dataset from the ModelScope Hub. Args: dataset_work_dir (str): Current git working directory. dataset_id (str): Dataset id, in the form of your-namespace/your-dataset-name . revision (str, optional): revision of the model you want to clone from. Can be any of a branch, tag or commit hash auth_token (str, optional): token obtained when calling `HubApi.login()`. Usually you can safely ignore the parameter as the token is already saved when you login the first time, if None, we will use saved token. git_path (str, optional): The git command line path, if None, we use 'git' Returns: None r repo_work_dirrrzrrzAlready cloned repo to: {}zRepo dir already exists: {}N) rYrZr[rcloneloggerinfoformatwarning)rrrzrr_repoclone_work_dirr4r4r5 clone_metas"zMsDataset.clone_metacommit_messageforcecCs&t|d|||d}|j|||ddS)aUUpload meta-file of dataset to the ModelScope Hub. Please clone the meta-data from the ModelScope Hub first. Args: dataset_work_dir (str): Current working directory. commit_message (str): Commit message. revision(`Optional[str]`): revision of the model you want to clone from. Can be any of a branch, tag or commit hash auth_token(`Optional[str]`): token obtained when calling `HubApi.login()`. Usually you can safely ignore the parameter as the token is already saved when you log in the first time, if None, we will use saved token. git_path:(`Optional[str]`): The git command line path, if None, we use 'git' force (Optional[bool]): whether to use forced-push. Returns: None r)rbranchrN)rpush)rrrzrrrrr4r4r5 upload_metaszMsDataset.upload_metacCs0t|||d}|j|d}td|d|S)as Delete object of dataset. Please log in first and make sure you have permission to manage the dataset. Args: object_name (str): The object name of dataset to be deleted. Could be a name of file or directory. If it's directory, then ends with `/`. For example: your-data-name.zip, train/001/img_001.png, train/, ... dataset_name (str): Path or name of the dataset. namespace(str, optional): Namespace of the dataset. version (str, optional): Version of the dataset. Returns: res_msg (str): Response message. r)rzObject z successfully removed!)rdeleterr)rrcrdre_delete_managerresp_msgr4r4r5rs  zMsDataset.deletecolumns preprocessors task_name data_config to_tensorcKsztstdt|jtr |d|i||jjt||S|dur,|j|||dS|j |jj d||d|jS)aFCreate a torch.utils.data.Dataset from the MS Dataset. The torch.utils.data.Dataset can be passed to torch.utils.data.DataLoader. Args: preprocessors (Callable or List[Callable], default None): (list of) Preprocessor object used to process every sample of the dataset. The output type of processors is dict, and each (numeric) field of the dict will be used as a field of torch.utils.data.Dataset. columns (str or List[str], default None): Dataset column(s) to be loaded (numeric data only if `to_tensor` is True). If the preprocessor is None, the arg columns must have at least one column. If the `preprocessors` is not None, the output fields of processors will also be added. task_name (str, default None): task name, refer to :obj:`Tasks` for more details data_config (ConfigDict, default None): config dict for model object. Attributes of ConfigDict: `preprocessor` (Callable, List[Callable], optional): preprocessors to deal with dataset `type` (str): the type of task `split_config` (dict, optional): get the split config for ExternalDataset `test_mode` (bool, optional): is test mode or not to_tensor (bool, default None): whether convert the data types of dataset column(s) to torch.tensor or not. format_kwargs: A `dict` of arguments to be passed to the `torch.tensor`. Returns: :class:`torch.utils.data.Dataset` z>The function to_torch_dataset requires pytorch to be installed preprocessorN)rrtorch)r`r format_kwargs) r+ ImportErrorr.r;rupdaterMr!_to_torch_dataset_with_processors reset_format set_format)rArrrrrrr4r4r5to_torch_datasets"!   zMsDataset.to_torch_dataset batch_sizeshuffle collate_fndrop_remaindercollate_fn_args label_colsprefetchc Csjtstd|dur|j||||| ||dS|dur"tddS|j|jj|||||||| dS)aCreate a tf.data.Dataset from the MS Dataset. This tf.data.Dataset can be passed to tf methods like model.fit() or model.predict(). Args: batch_size (int): Number of samples in a single batch. shuffle(bool): Shuffle the dataset order. preprocessors (Callable or List[Callable], default None): (list of) Preprocessor object used to process every sample of the dataset. The output type of processors is dict, and each field of the dict will be used as a field of the tf.data. Dataset. If the `preprocessors` is None, the `collate_fn` shouldn't be None. columns (str or List[str], default None): Dataset column(s) to be loaded. If the preprocessor is None, the arg columns must have at least one column. If the `preprocessors` is not None, the output fields of processors will also be added. collate_fn(Callable, default None): A callable object used to collect lists of samples into a batch. If the `preprocessors` is None, the `collate_fn` shouldn't be None. drop_remainder(bool, default None): Drop the last incomplete batch when loading. collate_fn_args (Dict, optional): A `dict` of arguments to be passed to the`collate_fn`. label_cols (str or List[str], defalut None): Dataset column(s) to load as labels. prefetch (bool, default True): Prefetch data. Returns: :class:`tf.data.Dataset` z?The function to_tf_dataset requires Tensorflow to be installed.N)rrrrz?The `preprocessors` and the `collate_fn` should`t be both None.)rrrr)r*r_to_tf_dataset_with_processorsrerrorr;r to_tf_dataset) rArrrrrrrrrr4r4r5r4s<$  zMsDataset.to_tf_datasetcCs|j|jSrC)r;rrKr4r4r5 to_hf_datasetvs zMsDataset.to_hf_datasetcolumn_mappingcCs|j|j|S)a Rename columns and return the underlying hf dataset directly TODO: support native MsDataset column rename. Args: column_mapping: the mapping of the original and new column names Returns: underlying hf dataset )r;rrename_columns)rArr4r4r5 remap_columnszs zMsDataset.remap_columnsc st|tr|n|g}tfdd|jjDg}g}|ritt|jfddD}|D]}|dd| Dq3dd} |D]} | || sct d| d | | qL| | qLd dl Gfd d d jjj} | |j||||S) Ncg|]}|vr|qSr4r4rRrG)rr4r5 z?MsDataset._to_torch_dataset_with_processors..csi|] }|t|qSr4nparrayrRrS)sampler4r5rWrXz?MsDataset._to_torch_dataset_with_processors..cSi|] \}}|t|qSr4rrQr4r4r5rWscSs t|jtjpt|jtjSrC)r issubdtypedtypeintegerfloating)rr4r4r5is_numpy_numberszDMsDataset._to_torch_dataset_with_processors..is_numpy_numberzData of column z is non-numeric, will be removedrcs>eZdZdeffdd ZddZfddZdd ZZS) zAMsDataset._to_torch_dataset_with_processors..MsMapDatasetrcs4tt||_||_||_||_||_||_dSrC) superr7rBrpreprocessor_listrretained_numeric_columnsretained_unumeric_columnsr)rArrrrrr __class__r4r5rBs  zJMsDataset._to_torch_dataset_with_processors..MsMapDataset.__init__cSrJrC)r0rrKr4r4r5rLrIzIMsDataset._to_torch_dataset_with_processors..MsMapDataset.__len__cs|jr|S|SrC)r as_tensor)rAxrr4r5type_converters zPMsDataset._to_torch_dataset_with_processors..MsMapDataset.type_convertercsxj|fddjD}jD]&}|D]\}}jr'|jvr/|||<q|jvr8|||<qq|S)Ncs,i|]}jr |jvr||qSr4)rrrr item_dictrAr4r5rWs  zaMsDataset._to_torch_dataset_with_processors..MsMapDataset.__getitem__..)rrrr_rrrr)rAindexresrrSrTr4rr5rHs     zMMsDataset._to_torch_dataset_with_processors..MsMapDataset.__getitem__) __name__ __module__ __qualname__rrBrLrrH __classcell__r4rrr5 MsMapDatasets  r )r.r>r6r;r<r?r\r]rr_rrappendrutilsdatar ) rArrrrrr sample_res processorrrSr r4)rrrr5rsF        &z+MsDataset._to_torch_dataset_with_processorsc sHt|tr|n|gtt|}tt|fddjjDddljj t j t jt jd}|rF|jt jd}dfdd dd jdjgd fd d } dd lm} |j| | d}rfdd} || }n t |dkr|dd}|dkr|j||d}|r|| }|S)Ncrr4r4r)cols_to_retainr4r5rrz.r)r) buffer_sizeFcs`tfddD}D]}|dd|jDq|r(|Stt|S)Ncs"i|] }|tj|qSr4)rrr;r)irAr4r5rWs"zJMsDataset._to_tf_dataset_with_processors..func..cSrr4rrQr4r4r5rWs )intrr;r_tupler>r^)r return_dictrr)rretained_columnsrArr5funcs  z6MsDataset._to_tf_dataset_with_processors..funcT)input_signaturecs:j|gfddDdfddtDS)Ncsg|] }j|jqSr4)dtypesas_dtyper)rRval)tfr4r5rs zTMsDataset._to_tf_dataset_with_processors..fetch_function..)inpToutcsi|] \}}||qSr4r4)rRrrGoutputr4r5rW szTMsDataset._to_tf_dataset_with_processors..fetch_function..)numpy_functionr^ enumerater)rrrr r5fetch_functions z@MsDataset._to_tf_dataset_with_processors..fetch_function)AUTOTUNE)num_parallel_callscsVfdd|D}t|dkrtt|}t|dkr'tt|}||fS)Ncsi|] \}}|vr||qSr4r4)rRrGtensorrr4r5rWsz_MsDataset._to_tf_dataset_with_processors..split_features_and_labels..rO)r_r0r\r]r^) input_batchlabelsr(r4r5split_features_and_labelss   zKMsDataset._to_tf_dataset_with_processors..split_features_and_labelsrOcSstt|SrC)r\r]r^)rr4r4r5sz:MsDataset._to_tf_dataset_with_processors..)r)F)r.r>r6r1r;r<r? tensorflowr r from_tensor_slicesraranger0int64rfunction TensorSpectensorflow.data.experimentalr%mapbatchr) rArrrrrrr tf_datasetr$r%r+r4)rrrrrrrArr5rsJ        z(MsDataset._to_tf_dataset_with_processorsc Ksptstd|s dSd|_|durd|vr|d}|tjkr"dnd}|d|}|durBt|tj r=t |j j dnt dd}| t |d |j}d |vrV|d }t|}d |vrd|d }|durwt|d rw|j} | rwt| |}t|jtr| t |d | |jjt||jd|_dS|dur|dd} |j|| d|_dS|j|jjdddS)aConvert the input datasets to specific custom datasets by given model configuration and preprocessor. Args: custom_cfg (Config): The model configuration for custom datasets. preprocessor (Preprocessor, Optional): Preprocessor for data samples. mode (str, Optional): See modelscope.utils.constant.ModeKeys Returns: `MsDataset` z?The function to_custom_dataset requires pytorch to be installedNTmodetrainrzdataset.)r`)r7taskfieldr)r)cfgrr)rrr)r+rr@getr'TRAINsafe_gethasattrr#modelrr`rdictr9popr(find_field_by_taskrrr.r;rrMrrrr) rArorr7kwargs ds_cfg_keydata_cfgr field_namepreprocessor_cfgrr4r4r5r'sZ        zMsDataset.to_custom_datasetrC)NNNNT)NNNNNNT)NT)NTNN)NN)=rrr__doc__r;r8r__annotations__r r r rrrr/rBrErHrLpropertyr9rM classmethodr rArarrb staticmethodr!r"r&rr%rr rr>r rboolrrr)rrrrrrrrrrrrrrrrrr4r4r4r5r71s                H  B &    6   B V Rr7)DrrYtypingrrrrrrrr r numpyrrr r r rdatasets.packaged_modulesrdatasets.utils.file_utilsrmodelscope.hub.repositoryr4modelscope.msdatasets.context.dataset_context_configr5modelscope.msdatasets.data_loader.data_loader_managerrrrr!modelscope.msdatasets.dataset_clsrr9modelscope.msdatasets.dataset_cls.custom_datasets.builderr(modelscope.msdatasets.utils.delete_utilsr,modelscope.msdatasets.utils.hf_datasets_utilr(modelscope.msdatasets.utils.upload_utilsrmodelscope.preprocessorsrmodelscope.utils.configrrrr rr!r"r#r$r%r&r'r(r)modelscope.utils.import_utilsr*r+modelscope.utils.loggerr,rr6r7r4r4r4r5s0,          ,