o ߥih@sddlZddlZddlZddlmZmZmZddlm Z ddl m Z ddl m Z ddlmZeZejjddZeZeZeZdad d dd de fd d Zdd d d dd de fddZdd d dd de fddZddZGdddZejfddZddZ dS)N)ManagerProcessValue)HubApi)ModelVisibility)DEFAULT_REPOSITORY_REVISION) get_logger) max_workersTc Csz.t} | || j|||stjntj||||||d |p d}td|d|WdStyO} zt d|d|d| WYd} ~ dSd} ~ ww) N) visibility chinese_namecommit_messagetagoriginal_model_idignore_file_patternrevisionzNo commit messagez!Successfully upload the model to z with message: Tz#Error happens when uploading model z: F) rlogin push_modelrPUBLICPRIVATEloggerinfo Exceptionerror) repo_name output_dirtokenprivaterr source_reporrapier"N/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/hub/push_to_hub.py_api_push_to_hubs8   r$c Cs|dur tjd}|durtjd}|dusJ|dus"Jdtj|s*Jdt|vsAdt|vsAdt|vsAJtd|d |d |t|D]} t ||||||||| rdd SqSd S) a Args: repo_name: The repo name for the modelhub repo output_dir: The local output_dir for the checkpoint token: The user api token, function will check the `MODELSCOPE_API_TOKEN` variable if this argument is None private: If is a private repo, default True retry: Retry times if something error in uploading, default 3 commit_message: The commit message tag: The tag of this commit source_repo: The source repo (model id) which this model comes from ignore_file_pattern: The file pattern to be ignored in uploading. revision: The branch to commit to Returns: The boolean value to represent whether the model is uploaded. NMODELSCOPE_API_TOKENUPLOAD_IGNORE_FILE_PATTERNUEither pass in a token or to set `MODELSCOPE_API_TOKEN` in the environment variables.configuration.jsonconfiguration.yamlconfiguration.yml Uploading  to  with message TF) osenvirongetpathisdirlistdirrrranger$) rrrrretryrrrrrir"r"r# push_to_hub9s(     r8c Cs|dur tjd}|durtjd}|dusJ|dus"Jdtj|s*Jdt|vsAdt|vsAdt|vsAJtd|d |d |t t ||||||||| S) az Args: repo_name: The repo name for the modelhub repo output_dir: The local output_dir for the checkpoint token: The user api token, function will check the `MODELSCOPE_API_TOKEN` variable if this argument is None private: If is a private repo, default True commit_message: The commit message tag: The tag of this commit source_repo: The source repo (model id) which this model comes from ignore_file_pattern: The file pattern to be ignored in uploading revision: The branch to commit to Returns: A handler to check the result and the status Nr&r'r(r)r*r+r,r-r.) r/r0r1r2r3r4rr _executorsubmitr$) rrrrrrrrrr"r"r#push_to_hub_asyncfs     r;c Cs d|_|}t|d|_|ddsO|dd}|d}ztdi||r6tj|r6t |Wnt yN}z t |WYd}~nd}~wwdSq)NTFdone delete_dirrr") valuer1rrpopr8r/r2existsshutilrmtreerr)qbitemr=rr!r"r"r# submit_tasks&     rFc@seZdZdZdZdS)UploadStrategycancelwaitN)__name__ __module__ __qualname__rHrIr"r"r"r#rGsrGcKs|dur t|dksJdtdurta|tvr:tt|<tddt|<ttt|t|fd}| |t |<t|}t|}| ddrO| |dS|j re|tjkretd| dd dS| |dS) Nrz"Please specify a valid queue name!rDF)targetargsr<z:Another uploading is running, this uploading with message rz will be canceled.)len_managerr_queuesQueuer_flagsrrFstart_tasksr1putr>rGrHrr) queue_namestrategykwargsprocessqueueflagr"r"r#push_to_hub_in_queues8   r]cCs8t|d}|dur dS|t|t|dS)N)rUr?joinrQrS)rWrZr"r"r# wait_for_dones  r_)!concurrent.futures concurrentr/rAmultiprocessingrrrmodelscope.hub.apirmodelscope.hub.constantsrmodelscope.utils.constantrmodelscope.utils.loggerrrfuturesProcessPoolExecutorr9dictrQrSrUrPr$r8r;rFrGrHr]r_r"r"r"r#sT     % / (