o ॵi<0@sddlZddlZddlZddlmZddlZddlZddlmZddl m Z ddl m Z ddl m Z mZmZmZddlmZddlmZmZeZGd d d eZGd d d eZGd ddeZdS)N)islice)IterableDataset)tqdm)MaxComputeUtil)DEFAULT_MAXCOMPUTE_ENDPOINTEXTENSIONS_TO_LOADMaxComputeEnvsVirgoDatasetConfig) get_logger)fetch_csv_with_url valid_urlc@s0eZdZdZddZddZddZdd Zd S) ExternalDatasetz"Dataset class for custom datasets.c s0||_t||_|jd|jid|_dd|jD|_i|_d}|jD]U\}t t rt j rt }tdd|D}d|vrLq*t|dkrfdt}td |d |d q*t|d }|tvrqq*fd d|D}||j|<q*|rt|}tj|fd|ji||_dSdS)N split_configcSsi|]\}}|gqSr).0k_rr]/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/dataset_cls/dataset.py sz,ExternalDataset.__init__..cSs"g|] }tj|ddqS).)ospathsplitextstripr file_namerrr +sz,ExternalDataset.__init__..,zSplit-z has been ignored, please flatten your folder structure, and make sure these files have same extensions. Supported extensions: z .rcsg|] }tj|qSr)rrjoinr split_dirrrr=s  data_files)split_path_dictcopydeepcopy config_kwargsupdatespec_extension_datasetitemssplit_data_files custom_map isinstancestrrrisdirlistdirsetlenr!rkeysloggererrorlistgetdatasets load_dataset) selfr%r(file_ext split_namesplit_file_namesset_files_extssupported_extssplit_file_pathsrr"r__init__sV        zExternalDataset.__init__cCs|jst|jS|jSN)r*r3r%__len__r;rrrrDHs zExternalDataset.__len__cCs|js |j|S|j|SrC)r*r%r8 __getitem__)r;itemrrrrFNs  zExternalDataset.__getitem__ccsL|js|jD] \}}||fVq dS|jD] \}}||fVqdSrC)r*r%r+)r;rvrrr__iter__Ts  zExternalDataset.__iter__N)__name__ __module__ __qualname____doc__rBrDrFrIrrrrr s . r csLeZdZdZdfdd ZddZddZd d Zd d ZdddZ Z S)NativeIterableDatasetz&The modelscope iterable dataset class.rcstj|||d||_dS)N) ex_iterableinfosplit)superrBstream_batch_size)r;rOrPrQrS __class__rrrB`s zNativeIterableDataset.__init__ccs:t|j|jddd|jddD] }||}|VqdS)NF batch_sizedrop_last_batchzOverall progressT)desctotal dynamic_ncols)riterrSn_shards_download_item)r;rGretrrrrIds zNativeIterableDataset.__iter__cC|jSrC)r]rErrrrDoszNativeIterableDataset.__len__ccst|tr|}|d}d}n |j}|j}|j}|dur$|dkr$tdtt|jddd|||ddd D] }| |}|Vq5dS) z` Returns the item at index `index` in the dataset. Slice indexing is supported. rNrzstep must be positiveFrVzSlicing progressT)rYr[) r.intstartstopstep ValueErrorrrr\r^)r;indexrbrcrdrGr_rrrrFrs*  z!NativeIterableDataset.__getitem__c Csi}t|trVz4|D],\}}|||<|dr8|jjd}||}t|tr-|g}|||<||| d<q W|St yU}zt ||}WYd}~|Sd}~ww|}|S)Nz:FILE dl_manager) r.dictr+endswith _ex_iterablekwargsr8download_and_extractr/r Exceptionr5r6)r;rGr_rrHrg ex_cache_patherrrr^s.     z$NativeIterableDataset._download_itemcCsFg}|dkr|Sd}|D]}||kr|S|||d7}q|S)z Returns the first n rows of the dataset. Args: n (int): Number of rows to return. Returns: list: The list of results, e.g. [{'id': 'abc123', 'text': 'hello world'}, ...] rr)rIappend)r;nresiter_numrGrrrheads   zNativeIterableDataset.head)r)rp) rJrKrLrMrBrIrDrFr^ru __classcell__rrrTrrN]s rNc@sfeZdZdZddZddZddZdd Zed e j fd d Z d dZ e d efddZddZdS) VirgoDatasetaDataset class for Virgo. Attributes: _meta_content (str): Virgo meta data content, could be a url that contains csv file. _data_type (int): Virgo dataset type, 0-Standard virgo dataset; Others-User define dataset (to be supported) Examples: >>> from modelscope.msdatasets.dataset_cls.dataset import VirgoDataset >>> input_kwargs = {'metaContent': 'http://xxx-xxx/xxx.csv', 'samplingType': 0} >>> virgo_dataset = VirgoDataset(**input_kwargs) >>> print(virgo_dataset[1]) >>> print(len(virgo_dataset)) >>> for line in virgo_dataset: >>> print(line) Note: If you set `download_virgo_files` to True by using MsDataset.load(dataset_name='your-virgo-dataset-id', hub=Hubs.virgo, download_virgo_files=True), you can get the cache file path of the virgo dataset, the column name is `cache_file`. >>> if virgo_dataset.download_virgo_files: >>> print(virgo_dataset[1].get('cache_file')) cKsd|_d|_d|_d|_d|_||_t|_|j t j d|_|j t j d|_| |d|_d|_d|_d|_d|_|j dd|_|j dd|_|j dd|_|jrj|j|j|j\|_|_dSdS)NrrFodps_batch_sized odps_limitodps_drop_last) _meta_content data_typeodps_table_nameodps_table_partition _odps_utilsr(pd DataFrame_metapopr meta_content sampling_type_check_variables _parse_metameta_content_cache_filevirgo_cache_dirdownload_virgo_filesodps_table_insodps_reader_insrxrzr{get_table_reader_ins)r;rkrrrrBs8 zVirgoDataset.__init__c CsB|jrtj|j||j|j|j|jjj|jjj dS|j j | S)N)readerrf batch_size_inlimit_in drop_last_in partitionscolumns) rrgen_reader_itemrxrzr{r table_schemarnamesrilocto_dict)r;rfrrrrFszVirgoDataset.__getitem__cCs$t|jtr |jddSt|jS)N odpsCountr)r.rrhr8r3rErrrrDs  zVirgoDataset.__len__ccsh|jr#tj|j|j|j|j|jjj|jjj d}|D]}|VqdS|j D] \}}| Vq(dS)N)rrrrrr) rrgen_reader_batchrxrzr{rrrrriterrowsr)r;odps_batch_databatchrrowrrrrIs  zVirgoDataset.__iter__returncCr`)z Virgo meta data. Contains columns: id, meta_info, analysis_result, external_info and cache_file (if download_virgo_files is True). )rrErrrmetaszVirgoDataset.metacCspt|jtrt|jrt|j}||_dSt|jtr6|j|_|jdd|_|jdd|_ | |_ dSd)N odpsTableNamerodpsTablePartitionz%The meta content must be url or dict.) r.r|r/r r rrhr8r~r_get_odps_infor)r;meta_content_dfrrrrs   zVirgoDataset._parse_metac Cstjtjd}tjtjd}tjtjd}tjtjt}|r&|r&|s;t dtjdtjdtjdtjd t ||||S)z Get MaxComputeUtil instance. Args: None Returns: MaxComputeUtil instance. rz&Please set MaxCompute envs for Virgo: z, z6(default: http://service-corp.odps.aliyun-inc.com/api)) renvironr8r ACCESS_IDACCESS_SECRET_KEY PROJECT_NAMEENDPOINTrrer) access_id access_key proj_nameendpointrrrr's"    zVirgoDataset._get_odps_infocCsR|jsd|jdvr d|jdkrt|jsd|jdkr%t|jts'ddSdS) zCheck member variables in this class. 1. Condition-1: self._meta_content cannot be empty 2. Condition-2: self._meta_content must be url when self._data_type is 0 z"Them meta content cannot be empty.)rrzFSupported samplingType should be 0 or 1, others are not supported yet.rz1The meta content must be url when data type is 0.rz2The meta content must be dict when data type is 1.N)r|r}r r.rhrErrrrAs zVirgoDataset._check_variablesN)rJrKrLrMrBrFrDrIpropertyrrrr staticmethodrrrrrrrrws   rw)r&mathr itertoolsrr9pandasrrr,modelscope.msdatasets.utils.maxcompute_utilsrmodelscope.utils.constantrrrr modelscope.utils.loggerr modelscope.utils.url_utilsr r r5objectr rNrwrrrrs      F\