o
    ॵi#                     @   s   d dl Z d dlZd dlmZ d dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZmZmZ G d	d
 d
eZdS )    N)defaultdict)FileLock)HubApi)DatasetContextConfig)DataMetaConfig)get_dataset_filesget_target_dataset_structure)DatasetFormationsDatasetPathNameDownloadModec                   @   s\   e Zd ZdZdefddZdddZd	d
 ZdddZdd Z	de
de
de
de
fddZdS )DataMetaManagerzData-meta manager.dataset_context_configc                 C   s   || _ t | _d S N)r   r   api)selfr    r   `/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/meta/data_meta_manager.py__init__   s   zDataMetaManager.__init__returnNc              
   C   sN  | j j}| j j}| j j}| j j}| j j}| j j}| j j}tj	
||||}tj	
|tj}	| j jp3t }
|s:tj}n|}|sBtj}n|}tj tj | tj | tj | tj | d
}tj	
||}tj|dd |tjkrtj	|	rt|	r| |	\}}nwt|d tj|	dd | ||||	\}}W d    n1 sw   Y  nP|tjkrtj	|	rt|	rtj|	dd t|d tj|	dd | ||||	\}}W d    n1 sw   Y  ntdtjj dtjj d| d	|	|
_||
_||
_ d
|v r|d
 }t!|dkr|d |
_"|
| j _|| j _#|| j _$d S )Nz.lockT)exist_ok)	lock_file)ignore_errorsz"Expected values of download_mode: z or z
, but got  ..pyr   )%r   dataset_name	namespacedownload_modeversioncache_root_dirsubset_namesplitospathjoinr
   	META_NAMEdata_meta_configr   LOCK_FILE_NAME_ANYLOCK_FILE_NAME_DELIMITERmakedirsr   REUSE_DATASET_IF_EXISTSexistslistdir_fetch_meta_from_cacher   _fetch_meta_from_hubFORCE_REDOWNLOADshutilrmtree
ValueErrorvaluemeta_cache_dirdataset_scriptsdataset_formationlendataset_py_scriptdataset_version_cache_root_dirglobal_meta_lock_file_path)r   r   r   r   r   r   r   r    r8   r3   r%   lock_subset_name
lock_splitlock_file_namelock_file_pathr4   r5   tmp_py_scriptsr   r   r   fetch_meta_files   s   






z DataMetaManager.fetch_meta_filesc              	   C   sF  | j j}| j j}| j j}| j j}| j j}| j jpt }d }d }|j}	|	r*t	|	dkr.t
dd|	v r8|	d d }|	d D ]'}
|
| drct|
dd}t|}W d    n1 s\w   Y   nq<|ss|sst
d| d| d	|ry||_n$t|||\}}t|||| j |\}}}}||_||_||_||_||_|| j _d S )
Nr   zFCannot find dataset meta-files, please fetch meta from modelscope hub.r   z.jsonzutf-8)encodingzFile z
.json and z4.py not found,please specify at least one meta-file.)r   r   r   r    r   r   r%   r   r4   r6   FileNotFoundErrorendswithopenjsonloadr7   r   r   meta_data_fileszip_data_filesmeta_args_mapmeta_type_maptarget_dataset_structure)r   r   r   r    r   r   r%   dataset_jsonr7   r4   	json_pathdataset_json_filetarget_subset_namerJ   meta_mapfile_mapargs_maptype_mapr   r   r   parse_dataset_structurei   sR   z'DataMetaManager.parse_dataset_structurec                 C   s6   | j j}t| j j}| jj||d}| j j| d S )N)
dataset_idr   )r   r   intr   r   get_virgo_metaconfig_kwargsupdate)r   virgo_dataset_idr   meta_contentr   r   r   fetch_virgo_meta   s   z DataMetaManager.fetch_virgo_metac                 C   s   t t}d }t|D ]'}tj|d }|tjjkr&t	tj|d }q|| 
tj|| q|sCttjj dtjj d|t|fS )Nr   z! file does not exist, please use r   )r   listr!   r+   r"   splitextr	   formation_mark_extr2   rU   appendr#   rA   r   r.   )r   r3   local_pathsdataset_typemeta_file_namefile_extr   r   r   r,      s"   
z&DataMetaManager._fetch_meta_from_cacher   r   revisionr3   c           
      C   sF   | j ||\}}| j ||||}| j ||||||\}}	||	fS r   )r   get_dataset_id_and_typeget_dataset_meta_file_list"get_dataset_meta_files_local_paths)
r   r   r   re   r3   rT   rb   	file_listra   r5   r   r   r   r-      s   
z$DataMetaManager._fetch_meta_from_hub)r   N)__name__
__module____qualname____doc__r   r   r?   rS   r[   r,   strr-   r   r   r   r   r      s    
N
/
r   )r!   r/   collectionsr   rD   datasets.utils.filelockr   modelscope.hub.apir   4modelscope.msdatasets.context.dataset_context_configr   +modelscope.msdatasets.meta.data_meta_configr   )modelscope.msdatasets.utils.dataset_utilsr   r   modelscope.utils.constantr	   r
   r   objectr   r   r   r   r   <module>   s   