o
    ߥi                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dlZd dlZd dlmZ d d	lmZmZ d d
lm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z: d dl;m<Z< d dl=m>Z> d dl?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZK d dlLmMZM ddlNmOZOmPZPmQZQ eM ZRG dd dZSG dd dZTdS )    N)defaultdict)
HTTPStatus)	CookieJar)
expanduser)DictListOptionalTupleUnion)	urlencode)Session)HTTPAdapterRetry)API_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_CREDENTIALS_PATHMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_REQUEST_IDONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODDatasetVisibilityLicensesModelVisibility)InvalidParameterNotExistErrorNotLoginExceptionNoValidRevisionErrorRequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper)
Repository)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATDatasetFormationsDatasetMetaFormatsDatasetVisibilityMapDownloadChannelDownloadMode	ModelFileVirgoDatasetConfig)
get_logger   )get_endpointget_release_datetimemodel_id_to_group_owner_namec                   @   s:  e Zd ZdZdefdee fddZdefddZe	j
ejdd	fd
edee dee dee dee defddZd
efddZd
efddZefd
edee defddZe	j
ejdddedddf	d
ededee dee dee dee dee dee dee deeee ef  deeeee f  fdd Z	!	"dd#ed$ee d%ee defd&d'Z	(dd)eeef defd*d+Z		(dd
ed,ee d)eeef dee fd-d.Z		(dd
ed,ee d)eeef dee fd/d0Zd1d2 Z		dd
ed3ee fd4d5Z		dd
ed3ee fd6d7Z 	(dd
ed)eeef de!ee ee f fd8d9Z"	(dd
ed)eeef de!ee ee f fd:d;Z#edd(d(i fd
edee d<ee d=ee d)eeef d>ee dee fd?d@Z$d	eje%j
d	fdAedBedee dee dee dCee defdDdEZ&dFdG Z'dAedBefdHdIZ(	(	J	KddLededMedNe)d=ef
dOdPZ*	Q	!	JddAedBededRed=ed$ed%efdSdTZ+dAedBedUedefdVdWZ,e-dXedYefdZd[Z.dAedBededYedXed\e/fd]d^Z0e-d_e1j2fd`daZ3e4d(dQfdbedAedBedee dcee ddee fdedfZ5e4fdbedAedBedee fdgdhZ6e4fdAedBedee fdidjZ7e4fdAedBedkedee fdldmZ8ddUednedefdodpZ9dAedBededqefdrdsZ:dtdu Z;dvedAedBededef
dwdxZ<dvedAedBededef
dydzZ=d{d| Z>ddAedBed}eddfd~dZ?dd Z@dBedAedefddZAdS )HubApizModel hub api interface.
    Nendpointc                 C   s   |dur|nt  | _dt i| _t | _tddddddd}t|d}| j	d	| | j	d
| t
D ]}t| j|tjt| j||d q4dS )u   The ModelScope HubApi。

        Args:
            endpoint (str, optional): The modelscope server http|https address. Defaults to None.
        N
user-agent   r8   )i  i  i  i  F)totalreadconnectbackoff_factorstatus_forcelistrespect_retry_after_header)max_retrieszhttp://zhttps://)timeout)r9   r=   ModelScopeConfigget_user_agentheadersr   sessionr   r   mountr   setattr	functoolspartialgetattr)selfr=   rG   retryadaptermethod rU   F/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/hub/api.py__init__@   s.   

zHubApi.__init__access_tokenc                 C   s   | j  d}| jj|d|i| | jd}t| | }t| |t t	 }|j
}t| t| t|t t |t t  |t t	 |fS )a  Login with your SDK access token, which can be obtained from
           https://www.modelscope.cn user center.

        Args:
            access_token (str): user access token on modelscope.

        Returns:
            cookies: to authenticate yourself to ModelScope open-api
            git_token: token to access your git repository.

        Note:
            You only have to login once within 30 days.
        z/api/v1/loginAccessToken)jsonrJ   )r=   rK   postbuilder_headersrJ   r'   rZ   r(   r   r   cookiesrH   
save_tokensave_cookiessave_user_infor   r   )rQ   rX   pathrdtokenr]   rU   rU   rV   login\   s,   




zHubApi.login model_id
visibilitylicensechinese_nameoriginal_model_idreturnc              
   C   s   |du rt dt }|du rtd| j d}t|\}}	||	||||tjddd}
| j	j
||
|| | jd}t|||
 t|  t  d	| }|S )
a  Create model repo at ModelScope Hub.

        Args:
            model_id (str): The model id
            visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
            license (str, optional): license of the model, default none.
            chinese_name (str, optional): chinese name of the model.
            original_model_id (str, optional): the base model id which this model is trained from

        Returns:
            Name of the model created

        Raises:
            InvalidParameter: If model_id is invalid.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nzmodel_id is required!)Token does not exist, please login first.z/api/v1/modelsMODELSCOPE_TRAIN_IDrf   )PathNameChineseName
VisibilityLicenseOriginalModelIdTrainId)rZ   r]   rJ   /)r   rH   get_cookies
ValueErrorr=   r;   osenvirongetrK   r[   r\   rJ   r$   r(   rZ   r9   )rQ   rg   rh   ri   rj   rk   r]   ra   owner_or_groupnamebodyrb   model_repo_urlrU   rU   rV   create_model   s2   	
zHubApi.create_modelc                 C   sZ   t  }|du rtd| j d| }| jj||| | jd}t| t	|
  dS )zDelete model_id from ModelScope.

        Args:
            model_id (str): The model id.

        Raises:
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nrm   /api/v1/models/r]   rJ   )rH   rw   rx   r=   rK   deleter\   rJ   r'   r(   rZ   )rQ   rg   r]   ra   rb   rU   rU   rV   delete_model   s   
zHubApi.delete_modelc                 C   s   | j  d| dS )Nr   z.gitr=   )rQ   rg   rU   rU   rV   get_model_url   s   zHubApi.get_model_urlrevisionc                 C   s   t  }t|\}}|r| j d| d| d| }n| j d| d| }| jj||| | jd}t|t	|| |j
tjkrTt| rL| t S t| t t| dS )an  Get model information at ModelScope

        Args:
            model_id (str): The model id.
            revision (str optional): revision of model.

        Returns:
            The model detail information.

        Raises:
            NotExistError: If the model is not exist, will throw NotExistError

        Note:
            model_id = {owner}/{name}
        r   rv   z
?Revision=r   N)rH   rw   r;   r=   rK   r{   r\   rJ   r%   loggerstatus_coder   OKr&   rZ   r   r   r   r'   )rQ   rg   r   r]   r|   r}   ra   rb   rU   rU   rV   	get_model   s   

zHubApi.get_modelzupload model	model_dircommit_messagetagignore_file_pattern
lfs_suffixc              
      s  |du rt d|du rt dtj|rtj|r t dtj|tj}tj|s5t| dt	
 }|du rAtdt|}|
du rLg }
t|
trT|
g}
z| j|d W n& ty   |du sk|du rot dtd	|  | j|||||	d
 Y nw t }t }zzt||d}||}||vrtd|  ||| ||| t|}|D ]"  d dkrtj| }tj|rt| qtj|dd q|D ]5  d dkrt fdd|
D rqtj| }tj|r	t |tj|  qt!|| q|s!t"j"# $d}d||f }|dur<t|tr/|gn|}|D ]}|%| q3|j&|||d |durO|'|| W n	 tyY    w W tj|dd dS tj|dd w )a^
  Upload model from a given directory to given repository. A valid model directory
        must contain a configuration.json file.

        This function upload the files in given directory to given repository. If the
        given repository is not exists in remote, it will automatically create it with
        given visibility, license and chinese_name parameters. If the revision is also
        not exists in remote repository, it will create a new branch for it.

        This function must be called before calling HubApi's login with a valid token
        which can be obtained from ModelScope's website.

        If any error, please upload via git commands.

        Args:
            model_id (str):
                The model id to be uploaded, caller must have write permission for it.
            model_dir(str):
                The Absolute Path of the finetune result.
            visibility(int, optional):
                Visibility of the new created model(1-private, 5-public). If the model is
                not exists in ModelScope, this function will create a new model with this
                visibility and this parameter is required. You can ignore this parameter
                if you make sure the model's existence.
            license(`str`, defaults to `None`):
                License of the new created model(see License). If the model is not exists
                in ModelScope, this function will create a new model with this license
                and this parameter is required. You can ignore this parameter if you
                make sure the model's existence.
            chinese_name(`str`, *optional*, defaults to `None`):
                chinese name of the new created model.
            commit_message(`str`, *optional*, defaults to `None`):
                commit message of the push request.
            tag(`str`, *optional*, defaults to `None`):
                The tag on this commit
            revision (`str`, *optional*, default to DEFAULT_MODEL_REVISION):
                which branch to push. If the branch is not exists, It will create a new
                branch and push to it.
            original_model_id (str, optional): The base model id which this model is trained from
            ignore_file_pattern (`Union[List[str], str]`, optional): The file pattern to ignore uploading
            lfs_suffix (`List[str]`, optional): File types to use LFS to manage. examples: '*.safetensors'.

        Raises:
            InvalidParameter: Parameter invalid.
            NotLoginException: Not login
            ValueError: No configuration.json
            Exception: Create failed.
        Nzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.z# must contain a configuration.json.zMust login before upload!)rg   zAvisibility and license cannot be empty if want to create new repozCreate new model %s)rg   rh   ri   rj   rk   )r   
clone_fromzCreate new branch %sr   .T)ignore_errorsc                    s   g | ]
}t | d uqS N)research).0patternfrU   rV   
<listcomp>d  s    z%HubApi.push_model.<locals>.<listcomp>z%Y_%m_%d_%H_%M_%Sz$[automsg] push model %s to hub at %s)r   local_branchremote_branch)(r   ry   ra   existsisfilejoinr5   CONFIGURATIONrx   rH   rw   r    listdir
isinstancestrr   	Exceptionr   infor   tempfilemkdtempr)   r*   get_remote_branches
new_branchcheckoutremoveshutilrmtreeanyisdircopytreecopydatetimenowstrftimeadd_lfs_typepushtag_and_push)rQ   rg   r   rh   ri   rj   r   r   r   rk   r   r   cfg_filer]   files_to_savetmp_dirgit_wrapperrepobranchesfiles_in_reposrcdatelfs_suffix_listsuffixrU   r   rV   
push_model   s   ;






"zHubApi.push_modelr8   
   r|   page_number	page_sizec                 C   s   t  }| j d}| jj|d|||f || | jd}t|t|| |j	t
jkr@t| r8| t }|S t| t t| dS )a  List models in owner or group.

        Args:
            owner_or_group(str): owner or group.
            page_number(int, optional): The page number, default: 1
            page_size(int, optional): The page size, default: 10

        Raises:
            RequestError: The request error.

        Returns:
            dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
        r   z.{"Path":"%s", "PageNumber":%s, "PageSize": %s})datar]   rJ   N)rH   rw   r=   rK   putr\   rJ   r%   r   r   r   r   r&   rZ   r   r"   r   r'   )rQ   r|   r   r   r]   ra   rb   r   rU   rU   rV   list_models~  s$   
zHubApi.list_modelsFuse_cookiesc                 C   s6   d }t |tr|}|S |rt }|d u rtd|S )Nrm   )r   r   rH   rw   rx   )rQ   r   r]   rU   rU   rV   _check_cookie  s   
zHubApi._check_cookiecutoff_timestampc                 C   s.   | j |||d}|rdd |D }|S g }|S )3  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )rg   r   r   c                 S      g | ]}|d  qS RevisionrU   r   xrU   rU   rV   r         z/HubApi.list_model_revisions.<locals>.<listcomp>)list_model_revisions_detail)rQ   rg   r   r   tags_detailstagsrU   rU   rV   list_model_revisions  s   zHubApi.list_model_revisionsc           	      C   sz   |  |}|du rt }| j d| d| }| jj||| | jd}t|t|| |	 }t
| |t }|d d S )r   Nr   z/revisions?EndTime=%sr   RevisionMapTags)r   r:   r=   rK   r{   r\   rJ   r%   r   rZ   r(   r   )	rQ   rg   r   r   r]   ra   rb   rc   r   rU   rU   rV   r     s   


z"HubApi.list_model_revisions_detailc                 C   s"   |D ]}|d |kr|  S qd S )Nr   rU   )rQ   detailsr}   itemrU   rU   rV   get_branch_tag_detail  s
   zHubApi.get_branch_tag_detailr]   c                    s8  t   tttj  }| j||d u rdn|d\}}|r&dd |D ng }|r1dd |D ng } |t kro|d u rFt}t	
d|  ||vrV||vrVtd||f | ||}	|	d u rf| ||}	t	
d|  |	S |d ur||v r| ||}	t	d	|  |	S t|d
kr|d u s|tkrt}ntd||f | ||}	|	S |d u r|r fdd|D ng }
t|
d
kr|
d
 d }|
d
 }	nt}| ||}	dd| }t	d|  t	d|  |	S ||vr|tkrt	d | ||}	ndd| }td|||f | ||}	t	
d|  |	S )NFr   c                 S   r   r   rU   r   rU   rU   rV   r     r   z4HubApi.get_valid_revision_detail.<locals>.<listcomp>c                 S   r   r   rU   r   rU   rU   rV   r     r   zAModel revision not specified, use default: %s in development modez$The model: %s has no revision : %s .z!Development mode use revision: %sz9Using branch: %s as version is unstable, use with cautionr   z#The model: %s has no revision: %s !c                    s   g | ]
}|d   kr|qS )	CreatedAtrU   r   release_timestamprU   rV   r     s    r   z[%s],z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z0The model: %s has no revision: %s valid are: %s!z%Use user-specified model revision: %s)r:   introundr   r   	timestamp#get_model_branches_and_tags_detailsr   r.   r   r   r   r   warninglenr   )rQ   rg   r   r]   current_timestampall_branches_detailall_tags_detailall_branchesall_tagsrevision_detailrevisions_detailvlrU   r   rV   get_valid_revision_detail  sv   
'


z HubApi.get_valid_revision_detailc                 C   s   | j |||dd S )N)rg   r   r]   r   )r   )rQ   rg   r   r]   rU   rU   rV   get_valid_revision,  s   zHubApi.get_valid_revisionc                 C   st   |  |}| j d| d}| jj||| | jd}t|t|| | }t	| |t
 }|d d |d d fS )q  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        r   z
/revisionsr   r   Branchesr   )r   r=   rK   r{   r\   rJ   r%   r   rZ   r(   r   )rQ   rg   r   r]   ra   rb   rc   r   rU   rU   rV   r   4  s   


z*HubApi.get_model_branches_and_tags_detailsc                 C   sF   | j ||d\}}|rdd |D ng }|rdd |D ng }||fS )r   )rg   r   c                 S   r   r   rU   r   rU   rU   rV   r   _  r   z6HubApi.get_model_branches_and_tags.<locals>.<listcomp>c                 S   r   r   rU   r   rU   rU   rV   r   a  r   )r   )rQ   rg   r   branches_detailtags_detailr   r   rU   rU   rV   get_model_branches_and_tagsN  s   
z"HubApi.get_model_branches_and_tagsroot	recursiverJ   c                 C   s   |rd| j |||f }nd| j ||f }| |}|dur$|d|  }|du r+| jn|}tt j|d< | jj|||d}	t	|	t
|| |	 }
t|
 g }|
t d D ]}|d d	ksd|d d
kreqV|| qV|S )a`  List the models files.

        Args:
            model_id (str): The model id
            revision (Optional[str], optional): The branch or tag name.
            root (Optional[str], optional): The root path. Defaults to None.
            recursive (Optional[str], optional): Is recursive list files. Defaults to False.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            headers: request headers

        Returns:
            List[dict]: Model file list.
        z7%s/api/v1/models/%s/repo/files?Revision=%s&Recursive=%sz+%s/api/v1/models/%s/repo/files?Recursive=%sNz&Root=zX-Request-IDr   Filesrp   z
.gitignorez.gitattributes)r=   r   rJ   r   uuiduuid4hexrK   r{   r%   r   rZ   r(   r   append)rQ   rg   r   r   r   r   rJ   ra   r]   rb   rc   filesfilerU   rU   rV   get_model_filese  s0   

zHubApi.get_model_filesdataset_name	namespacedescriptionc                 C   s   |d u s|d u rt dt }|d u rtd| j d}d |fd |fd |fd |fd |fd |fd}	| jj||	|| | jd}
t	|
||	 t
|
  | j d| d| }td|  |S )	Nz(dataset_name and namespace are required!rm   /api/v1/datasets)rp   rq   Ownerrs   rr   Description)r   r]   rJ   z
/datasets/rv   zCreate dataset success: )r   rH   rw   rx   r=   rK   r[   r\   rJ   r$   r(   rZ   r   r   )rQ   r  r  rj   ri   rh   r  r]   ra   r   rb   dataset_repo_urlrU   rU   rV   create_dataset  s0   	
zHubApi.create_datasetc                 C   sL   | j  d}i }| jj||| | jd}t| | t }dd |D S )Nr  )paramsrJ   c                 S   r   )rp   rU   r   rU   rU   rV   r     r   z(HubApi.list_datasets.<locals>.<listcomp>)r=   rK   r{   r\   rJ   r'   rZ   r   )rQ   ra   r
  rb   dataset_listrU   rU   rV   list_datasets  s   

zHubApi.list_datasetsc           	      C   sb   | j  d| d| }t }| jj||d}| }t||| |d d }|d d }||fS )z Get the dataset id and type. /api/v1/datasets/rv   r]   DataIdTyper=   rH   rw   rK   r{   rZ   r#   )	rQ   r  r  datahub_urlr]   rb   resp
dataset_iddataset_typerU   rU   rV   get_dataset_id_and_type  s   zHubApi.get_dataset_id_and_typed   Truedataset_hub_idfiles_metadatarG   c                 C   s^   | j  d| d}|d|d}t }|rd|d< | jj||||d}	|	 }
t||
|	 |
S )z$
        Get dataset infos.
        r  
/repo/treeN)r   Root	RecursiveTblobs)r
  r]   rG   r  )rQ   r  r   r  rG   r   r  r
  r]   rb   r  rU   rU   rV   get_dataset_infos  s   	zHubApi.get_dataset_infosT	root_pathc                 C   s   | j ||d\}}	|rdnd}| j d| d}
|r|nd|r!|nd|||d}t }| jj|
||d	}| }t|
|| |S )
N)r  r  r  Falser  r  masterrv   )r   r  r  
PageNumberPageSize)r
  r]   )r  r=   rH   rw   rK   r{   rZ   r#   )rQ   r  r  r   r!  r   r   r   r  r  r  r
  r]   rb   r  rU   rU   rV   list_repo_tree  s   	

zHubApi.list_repo_treer  c           
      C   s   | j  d| d| }t }| jj||| | jd}| }t||| |d }	|	du r<t	d| d| d| d	|	d
 }	|	S )z( Get the meta file-list of the dataset. r  z/repo/tree?Revision=r   r  Nz'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existr   )
r=   rH   rw   rK   r{   r\   rJ   rZ   r#   r   )
rQ   r  r  r  r   r  r]   rb   r  	file_listrU   rU   rV   get_dataset_meta_file_list  s"   
z!HubApi.get_dataset_meta_file_listr  meta_cache_dirc                 C   sX   t j|t|  tjj }t|d}|d W d   dS 1 s%w   Y  dS )z
        Dump the data_type as a local file, in order to get the dataset
         formation without calling the datahub.
        More details, please refer to the class
        `modelscope.utils.constant.DatasetFormations`.
        wz3*** Automatically-generated file, do not modify ***N)	ry   ra   r   r   r0   formation_mark_extvalueopenwrite)r  r)  dataset_type_file_pathfprU   rU   rV   dump_datatype_file  s   "zHubApi.dump_datatype_filer'  c              
   C   s   t t}t|}t| }	t }
tj||d |D ]r}|d }tj	
|d }||	v r| j d| d| d| d| 	}| jj||
d}t| tj	||}tj	|ritd	| d
| d || | qt|d}||j W d    n1 sw   Y  || | q||fS )N)r  r)  ro   r  rv   /repo?Revision=
&FilePath=r  zReusing dataset z's python file ()wb)r   listr0   r1   rH   rw   r<   r1  ry   ra   splitextr=   rK   r{   r'   r   r   r   r   r   r-  r.  content)rQ   r  r  r   r)  r  r'  local_pathsdataset_formationdataset_meta_formatr]   	file_info	file_path	extensionr  rb   
local_pathr   rU   rU   rV   "get_dataset_meta_files_local_paths  s:   z)HubApi.get_dataset_meta_files_local_pathsi   c                    s  ddl }ddlm} ddl}tj||| jdd }|t	j
kr.tj|r.t| tj|r>td|  |S t }td tj| |dd	}t|jd
d}	||	dd}
 fdd}d}t|dQ}||D ]D}|
t| | drdd |D }t|dkrqp|dkrd}nd}||}|j|d|dd |d7 }qp|D ]	}||d  qqpW d   n1 sw   Y  |
  |S )zO
        Fetch the meta-data files from the url, e.g. csv/jsonl files.
        r   N)tqdmzUTF-8encodingzReusing cached meta-data file: zLoading meta-data file ...T)r]   streamzcontent-length)r@   dynamic_ncolsc                 3   sH    g }|   D ]}|d}|| t| kr|V  g }q|V  d S )Nutf-8)
iter_linesdecoder   r   )r  
chunk_datar   
chunk_sizerU   rV   	get_chunkR  s   


z3HubApi.fetch_meta_files_from_url.<locals>.get_chunkajsonlc                 S   s   g | ]}|  rt|qS rU   )striprZ   loads)r   linerU   rU   rV   r   a  s    z4HubApi.fetch_meta_files_from_url.<locals>.<listcomp>F\)indexheader
escapecharr8   
)hashlibrB  pandasry   ra   r   md5encode	hexdigestr4   FORCE_REDOWNLOADr   r   r   r   rH   rw   requestsr{   r   rJ   r-  updater   endswith	DataFrameto_csvr.  close)urlout_pathrL  moderX  rB  pdr]   response
total_sizeprogressrM  iter_numr   chunkwith_headerchunk_dfrR  rU   rK  rV   fetch_meta_files_from_url;  sH    





z HubApi.fetch_meta_files_from_url	file_nameviewextension_filterc           	      C   sJ   |r|r|s
t dd|||d}t|}| j d| d| d| }|S )Nz:Args (file_name, dataset_name, namespace) cannot be empty!SDK)Sourcer   FilePathViewr  rv   /repo?)rx   r   r=   )	rQ   rp  r  r  r   rq  rr  r
  file_urlrU   rU   rV   get_dataset_file_urls  s   	zHubApi.get_dataset_file_urlc              	   C   s>   |rt j|d tv r| j d| d| d| d| 	}|S )Nr2  r  rv   r3  r4  )ry   ra   r8  r/   r=   )rQ   rp  r  r  r   rU   rU   rV   get_dataset_file_url_origin  s   z"HubApi.get_dataset_file_url_originc                 C   s&   | j  d| d| d| }| |S )Nr  rv   /ststoken?Revision=)r=   datahub_remote_call)rQ   r  r  r   r  rU   rU   rV   get_dataset_access_config  s   
z HubApi.get_dataset_access_configcheck_cookiec           	      C   sh   | j  d| d| d| }|r| jdd}nt }| jj||| | jd}| }t	| |d S )Nr  rv   r{  Tr   rd  r]   rJ   r  )
r=   r   rH   rw   rK   r{   r\   rJ   rZ   r(   )	rQ   r  r  r~  r   r  r]   rb   r  rU   rU   rV   !get_dataset_access_config_session  s   
z(HubApi.get_dataset_access_config_sessionversionc           
      C   s   t jtjd}|stdtj | d}tjt	
 }t||d}t|d}| jj|||| | jdd}| }	|	d d	krJtd
|	 |	d S )z.
        Get virgo dataset meta info.
        rf   z"Virgo endpoint is not set in env: z/data/set/download)	dataSetIddataSetVersion)r   i  )rd  rZ   r]   rJ   rG   coder   zFailed to get virgo dataset: r   )ry   rz   r{   r6   env_virgo_endpointRuntimeErrorr^  utilsdict_from_cookiejarrH   rw   dictrK   r[   r\   rJ   rZ   )
rQ   r  r  virgo_endpointvirgo_dataset_urlr]   dataset_infor   rb   r  rU   rU   rV   get_virgo_meta  s,   

zHubApi.get_virgo_metazip_file_namec                 C   s   | j  d| d| }t }| jj||| | jd}| }t| |d }	t	|	d }
| d| }| jj||| | jd}| }t| |d }|
d d | d | d | }||d	< |S )
Nr  rv   r  r  rr   r{  z	-unzipped_Dir)
r=   rH   rw   rK   r{   r\   rJ   rZ   r(   r2   )rQ   r  r  r   r  r  r]   rb   r  r   rh   datahub_sts_urlr_stsresp_stsdata_stsfile_dirrU   rU   rV   &get_dataset_access_config_for_unzipped  s&   



 z-HubApi.get_dataset_access_config_for_unzippedc           
      C   sd   | j  d| d| d| d| d| d| }t }| jj||dd}	|	 }	t|	 |	d	 }	|	S )
Nr  rv   z/oss/tree/?MaxLimit=
&Revision=z&Recursive=z&FilterDir=i  )rd  r]   rG   r  )r=   rH   rw   rK   r{   rZ   r(   )
rQ   r  r  	max_limitis_recursiveis_filter_dirr   rd  r]   r  rU   rU   rV   list_oss_dataset_objects  s   zHubApi.list_oss_dataset_objectsobject_namec              	   C   n   |r|r|r|st d| j d| d| d| d| 	}t }| jj||d}| }t| |d }|S )NArgs cannot be empty!r  rv   z
/oss?Path=r  rd  r]   Messagerx   r=   rH   rw   rK   r   rZ   r(   rQ   r  r  r  r   rd  r]   r  rU   rU   rV   delete_oss_dataset_object  s   "z HubApi.delete_oss_dataset_objectc              	   C   r  )Nr  r  rv   z/oss/prefix?Prefix=z/&Revision=r  r  r  r  rU   rU   rV   delete_oss_dataset_dir
  s   zHubApi.delete_oss_dataset_dirc                 C   s>   t  }| jj||dt  id}| }t||| |d S )Nr>   r   r  )rH   rw   rK   r{   rI   rZ   r#   )rQ   rd  r]   rb   r  rU   rU   rV   r|    s   
zHubApi.datahub_remote_calluse_streamingc              
   C   s(  t ddk}|r|r|s|szdt }| j d| d| d}| jj||| | jd}t	| t
jj}d}	tt jv rAt jt }tt jv rKt jt }	| j d| d| d| d	|	 	}
| jj|
|| | jd}| }t| W d S  ty } zt| W Y d }~d S d }~ww d S d S d S d S )
NCI_TESTr  r  rv   z/download/increaser   rf   z/download/uv/z?user=)ry   getenvrH   rw   r=   rK   r[   r\   rJ   r'   r3   LOCALr,  r   rz   r   rZ   r(   r   r   error)rQ   r  r  r  
is_ci_testr]   download_count_urldownload_count_respchannel	user_namedownload_uv_urldownload_uv_resperU   rU   rV   dataset_download_statistics#  s<   







z"HubApi.dataset_download_statisticsc                 C   s   t tt ji|S r   )r   r   r   r   r   )rQ   rJ   rU   rU   rV   r\   @  s   zHubApi.builder_headersc                 C   s   | j  d| d| dS )Nr  rv   rw  r   )rQ   r  r  rU   rU   rV   get_file_base_pathD  s   zHubApi.get_file_base_path)r8   r   )F)NFNN)Fr  r  )Tr8   r  )r8   )B__name__
__module____qualname____doc__r   r   r   rW   re   r   PUBLICr   	APACHE_V2r   r   r   r   r,   r   r-   r
   r   r   r  r   boolr   r   r   r   r   r   r   r	   r   r   r  r   r	  r  r  floatr   r&  r(  staticmethodr1  r7  rA  r4   REUSE_DATASET_IF_EXISTSro  r+   ry  rz  r}  r  r  r  r  r  r  r|  r  r\   r  rU   rU   rU   rV   r<   =   sX   
)
4
)	

 

&





D






2
&	


!<
"








r<   c                   @   s   e Zd ZeeZdZdZdZdZ	dZ
edd Zedefdd	Zed
d Zedd ZedefddZededefddZedeeef fddZedee fddZeddeeedf defddZdS )rH   r]   	git_tokenuserrK   Fc                   C   s   t jtjdd d S )NT)exist_ok)ry   makedirsrH   path_credentialrU   rU   rU   rV   make_sure_credential_path_existQ  s   z0ModelScopeConfig.make_sure_credential_path_existc                 C   sR   t   ttjt jt jd}t	| | W d    d S 1 s"w   Y  d S )Nzwb+)
rH   r  r-  ry   ra   r   r  COOKIES_FILE_NAMEpickledump)r]   r   rU   rU   rV   r_   U  s   
"zModelScopeConfig.save_cookiesc                  C   s   t jtjtj} t j| rHt| d,}t	|}|D ]}|
 r6tjs6dt_td  W d    d S q|W  d    S 1 sCw   Y  d S )NrbTzAuthentication has expired, please re-login with modelscope login --token "YOUR_SDK_TOKEN" if you need to access private models or datasets.)ry   ra   r   rH   r  r  r   r-  r  load
is_expiredcookie_expired_warningr   r   )cookies_pathr   r]   cookierU   rU   rV   rw   ]  s$   


 zModelScopeConfig.get_cookiesc                  C   s   t jtjtj} d}t j| r2t| d}t|	 
 dd}|W  d    S 1 s-w   Y  |dks<t|dkrbtt j}t  t| d}|| W d    |S 1 s]w   Y  |S )Nrf   r  rG  rC      w+)ry   ra   r   rH   r  USER_SESSION_ID_FILE_NAMEr   r-  r   readlinerP  r   r   r   r   r  r.  )session_path
session_idr   wfrU   rU   rV   get_user_session_ido  s$   
 
z$ModelScopeConfig.get_user_session_idrd   c                 C   sP   t   ttjt jt jd}||  W d    d S 1 s!w   Y  d S )Nr  )	rH   r  r-  ry   ra   r   r  GIT_TOKEN_FILE_NAMEr.  rd   r   rU   rU   rV   r^     s   
"zModelScopeConfig.save_tokenr  
user_emailc                 C   sX   t   ttjt jt jd}|d| |f  W d    d S 1 s%w   Y  d S )Nr  z%s:%s)	rH   r  r-  ry   ra   r   r  USER_INFO_FILE_NAMEr.  )r  r  r   rU   rU   rV   r`     s   
"zModelScopeConfig.save_user_inforl   c                  C   s~   z4t tjtjtjddd} |  }|dd |dd fW  d    W S 1 s-w   Y  W dS  t	y>   Y dS w )Nrb   rG  rC  :r   r8   r  )
r-  ry   ra   r   rH   r  r  rA   splitFileNotFoundError)r   r   rU   rU   rV   get_user_info  s"   
$	zModelScopeConfig.get_user_infoc                  C   sh   d} z't tjtjtjddd}| } W d   W | S 1 s"w   Y  W | S  ty3   Y | S w )z
        Get token or None if not existent.

        Returns:
            `str` or `None`: The token, `None` if it doesn't exist.

        Nrb   rG  rC  )	r-  ry   ra   r   rH   r  r  rA   r  r  rU   rU   rV   	get_token  s&   	

zModelScopeConfig.get_tokenN
user_agentc                 C   s   d}t tjv rtjt  }d}ttjv rtjt }ddlm} d|t t	 t t
 ||f }t| trH|dddd |  D  7 }|S t| trS|d|  7 }|S )	a  Formats a user-agent string with basic info about a request.

        Args:
            user_agent (`str`, `dict`, *optional*):
                The user agent info in the form of a dictionary or a single string.

        Returns:
            The formatted user-agent string.
        customunknownr   )__version__zSmodelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%sz; c                 s   s"    | ]\}}| d | V  qdS )rv   NrU   )r   kvrU   rU   rV   	<genexpr>  s     z2ModelScopeConfig.get_user_agent.<locals>.<genexpr>)r   ry   rz   r   
modelscoper  platformpython_versionrH   r  	processorr   r  r   itemsr   )r  envr  r  uarU   rU   rV   rI     s,   




	 
zModelScopeConfig.get_user_agentr   )r  r  r  r   r   r  r  r  r  r  r  r  r  r   r_   rw   r  r   r^   r`   r	   r  r   r  r
   r   rI   rU   rU   rU   rV   rH   I  s2    


$rH   )Ur   rN   ry   r  r  r   r   r   r   collectionsr   httpr   http.cookiejarr   os.pathr   typingr   r   r   r	   r
   urllib.parser   rZ   r^  r   requests.adaptersr   r   modelscope.hub.constantsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   modelscope.hub.errorsr   r   r    r!   r"   r#   r$   r%   r&   r'   r(   modelscope.hub.gitr)   modelscope.hub.repositoryr*   modelscope.utils.constantr+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   modelscope.utils.loggerr7   utils.utilsr9   r:   r;   r   r<   rH   rU   rU   rU   rV   <module>   sJ   D48        