o ॵiu@sddlZddlZddlZddlZddlmZddlZddlZddl m Z ddl m Z ddl m Z ddlmZddlZddlmZddlmZmZmZmZmZmZmZmZmZmZmZddlm Z m!Z!dd l"m#Z#dd l$m%Z%e e&Z'd Z(e')j*d Z+e#Z,d Z-ddgZ.gdZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;dZd Z?d!Z@d"ZAd#ZBd$d%ZCeCZDGd&d'd'eEZFGd(d)d)eEZGeGZHd*eIdeeJe fd+dfd,d-ZKd?d/d0ZLd@d1d2ZMd3d4ZNd5d6ZOdd.e,e0fd7d8ZPdAd9d:ZQdBd;d<ZReSd=kr ePd d>ZTeUeTdSdS)CN)datetime)reduce)Path)Union)version) CustomDatasetsHeadsHooks LR_SchedulersMetricsModels Optimizers Pipelines Preprocessors TaskModelsTrainers)FieldsTasks)get_modelscope_cache_dir) default_groupTregister_module modelscope.)modelsmetrics pipelines preprocessorstrainers msdatasets exporters ast_indexer decoratorsexpress from_importsimportsfilepathmodelscope_pathrmd5index files_mtime requirementsmodule class_name group_key module_name module_cls TEMPLATE_PATHzast_index_file.pycCstd}|tj|S)Nzmodelscope.ast)logging getLoggersetLevelINFO) ast_loggerr7N/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/utils/ast_utils.pyget_ast_logger;s  r9c @s&eZdZd)ddZdedefddZdejdefdd Z de ejd fdefd d Z d*dejdede fddZ d*dejdedefddZddZde ejdeffddZ d+de ejdefdedede fddZdejde fddZddZd e d!ede fd"d#Zd$edefd%d&Zd'd(ZdS), AstScanningreturnNcC t|_t|_g|_g|_dSN)dict result_importresult_from_importresult_decoratorr#selfr7r7r8__init__F zAstScanning.__init__nodecCst|tjo t|tj Sr=) isinstanceastAST expr_contextrCrFr7r7r8 _is_sub_nodeLs  zAstScanning._is_sub_nodecCsT|jD]$}t||}||rdSt|ttfr'|D] }||r&dSqqdS)NFT)_fieldsgetattrrLrGlisttuple)rCrFfieldattrvalr7r7r8_is_leafPs    zAstScanning._is_leafstrcCs"trt|jdks |dkrdSdS)N FunctionDefTF)SKIP_FUNCTION_SCANNINGtype__name__rKr7r7r8_skip_function\szAstScanning._skip_functionTn show_offsetscCs|r|j|jS|jSr=) _attributesrM)rCr[r\r7r7r8rMbs zAstScanning._fieldscCs^t}t|tjr-t}|j||dD]}|jt|||d}|||<q||t|j<|S|S)Nr\) r>rGrHrIrM_leafrNrXrY)rCrFr\output local_dictrQ field_outputr7r7r8r_hs   zAstScanning._leafcCr<r=)r>r?r@rAresult_expressrBr7r7r8_refreshurEzAstScanning._refreshcCs||j|ddddS)Nz F)indentr\) _setup_global scan_importrKr7r7r8scan_ast{szAstScanning.scan_astparent_node_namecs|dur|S|rj|dS ddttjdtfdtdtffdd }t}t|jdkrat |d }|d krad d g|}t |d d t |d }|durYt |d |nt |d ||j |dD]U} t || } | gkryg|| <n| |rqht| trt| d krt| d tjr| d r|| d } | || <nQt| trt} | D].} || t| j} t| j}|dks|dks|dks|dkr|| vrg| |<| || q| || <nt| tjr|| }||| <n| || <t|jdkst|jdkrt|jdkrX| d krtj|| <| dkrXt|| dtrHg}|| dD]}|dd}||q1|j|d <n|| dd}|gj|d <t|jdkr|| d}t|tr~|D]}|dj|dd<qnn |j|| dd<d| kr| gkr| D] }t |t|jqj| | gkrt| jdkr|dkrj| qhtjtjtjtjiS)Nr^rielrjr;csj||dS)N)r\rj)rg)rkrjrCr\r7r8 _scan_imports z-AstScanning.scan_import.._scan_import ImportFromlevelrrrr,Importnamesaliasnamedecorator_listCallExpr)ri)rTr_rrHrIrUr>rXrYrNjoinsetattrrMrZrGrOlenappendr@r? CLASS_NAMErsrAextendrc IMPORT_KEYFROM_IMPORT_KEY DECORATOR_KEY EXPRESS_KEY)rCrFr\rjrmoutputsro path_levelr/rQrR local_outel_dictrkrsr` item_nameitem local_name final_dictr7rlr8rgs                         zAstScanning.scan_importcsjdtjdtfdddtdtffdd }dtjdtffdd }|j}||j}||j}|||fS) NrFr;cSsVd\}}}t|jdkrt|d}t|d}t|d}t|jdkr't|d}||fS)N)NNN AttributevalueidrRName)rXrYrN)rFrrrRr7r7r8_get_attribute_items     z9AstScanning._parse_decorator.._get_attribute_itemnodescs`g}|D])}t|jdkr||jdfqt|jdkr&||jdfq||q|S)NStrConstant)rXrYrzsr)rresultrFrr7r8_get_args_namesz4AstScanning._parse_decorator.._get_args_namecsg}|D]C}t|jdkrGt|d}t|jdkr&|t|d|jdfqt|jdkr:|t|d|jdfq|t|df|q|S)Nkeywordrrargr)rXrYrNrzrr)rrrFattribute_noderr7r8_get_keyword_names& z7AstScanning._parse_decorator.._get_keyword_name)rHrIrPrOfuncargskeywords)rCrFrr functions args_list keyword_listr7rr8_parse_decorators     zAstScanning._parse_decoratorcCsH|durdS|dkr tS|d}t|dkr|Stt|d|dS)Nrrrr)rsplitryrNeval)rCkey_item split_listr7r7r8_get_registry_values  zAstScanning._get_registry_value parsed_inputr-c Cs|\}}}t|dkr dS|dg}t|dkr#t|dkr#|tt|dkr4t|dkr4||t|dkrbt|dkrbd}|D]}|\} } } | tkrX|| | f|}qD|durb||t|dkrm|t|D]}|\} } } | tkr}| }qo|| | fqo|D]-}|dur|dq|ddur||dqt|tr||q|d |q|d| |d| |dfS)zformat registry information to a tuple indexer Return: tuple: (MODELS, Tasks.text-classification, Models.structbert) rNrr) REGISTER_MODULEryrzr GROUP_KEYremove MODULE_CLSrGrUrwr) rCrr-rrrr`remove_group_itemrkeyrsrRr7r7r8_registry_indexer!sH               zAstScanning._registry_indexerrcCsrg}|D]2}t|jdkrqt|td}t|d}t|ddtkr"q||}|||}d|ur6||q|S)zparse the AST nodes of decorators object to registry indexer Args: nodes (list): list of AST decorator nodes Returns: list: list of registry indexer ruNrrR)rXrYrNr{rrrrz)rCrresultsrFr-r parse_outputr)r7r7r8parse_decoratorsUs      zAstScanning.parse_decoratorscCs|t|ddd }|}Wdn1swYd|}t|}|j|dd}||t|t<||t |t <|t |t |S)Nrutf8)encodingriFr^) rdopen readlinesrwrHparsergrrrr|)rCfilecodedatarFr`r7r7r8 generate_astns   zAstScanning.generate_astr;N)T)Tri)rY __module__ __qualname__rDobjectboolrLrHrIrTrrZrPrMrUr_rdrhrgrrrrOrrr7r7r7r8r:Ds2    j, 4 r:c@seZdZdddZ d dededefddZd d Zd edefd d Z d!ddZ d"ddZ ddZ ddZ ddZddZdeefddZeedfddZdS)#FilesAstScanningr;NcCst|_g|_g|_dSr=)r: astScaner file_dirsrequirement_dirsrBr7r7r8rD}s zFilesAstScanning.__init__import_package current_pathc Cs|tdrtdd|ddddS|tdrV|d}|d}d}t|D] \}}|dkr<|}nq0|d| }|||ddd|dS|S)z Args: import_package (str): relative import or abs import current_path (str): path/to/current/file r/rrN.pyri) startswithIGNORED_PACKAGESMODELSCOPE_PATHrwr enumerater|) rCrrcurrent_path_listimport_package_listror)r abs_path_listr7r7r8_parse_import_paths&   z#FilesAstScanning._parse_import_pathcCsdSr=r7)rCimport_abs_pathr7r7r8_traversal_importsz"FilesAstScanning._traversal_import scan_resultcCsg}|t|t|t|tt|D]\}}d|ddkr.d||<q|dd||<qt}|D]}tD] }| |rM| |qAq=tt|t|S)aparse import and from import dicts to a third party package list Args: scan_result (dict): including the import and from import result Returns: list: a list of package ignored 'modelscope' and relative path import rirr) r|rOr}keysr~rrsetrradd)rCrr`r)rignoredignored_packager7r7r8 parse_imports    zFilesAstScanning.parse_importFcCsbg|_|dus t|dkr|j||ddS|D]}tj||}tj|r.|j||dqdS)Nr include_init)rry_traversal_filesospathrwisdir)rCr check_sub_dirrrsub_dirr7r7r8traversal_filess z FilesAstScanning.traversal_filescCst|}|D]Q}|jdkr|sq|jdr|jdks(|jds(|jdr)q|r6|j|j|dq|rH|jdrH|j |jq|rXd|jvrX|j |jqdS)Nz __init__.py__z.jsonz.mdrr requirement) rscandirrsrendswithis_dirrris_filerrzr)rCrrdir_listrr7r7r8rs&   z!FilesAstScanning._traversal_filescCsz|j|}Wn8ty@}z,t|j}td|d|djd|djd|djdt |j d|d|d d}~ww| |}|t |fS) NzDuring ast indexing the file z', a related error excepted in the file z at line: z: "z" with error msg: "z: z'", please double check the origin file z- to see whether the file is correctly edited.) rr Exception traceback extract_tb __traceback__filenamelinenolinerXrYrr)rCrr`edetail import_listr7r7r8_get_single_file_scan_results.  z-FilesAstScanning._get_single_file_scan_resultc CsJt}|D]}||tD]}t|t||tt||ti||<q q|Sr=)r>r FILE_NAME_KEYr} MODULE_KEY)rC forward_indexinverted_indexr)rr7r7r8_inverted_indexs   z FilesAstScanning._inverted_indexcCs,t}|D] \}}|t||t<q|Sr=)r>itemsr}r)rCr module_importr) value_dictr7r7r8_module_importszFilesAstScanning._module_importcCs d|vr|d=d|vr|d=|S)N) OPTIMIZERSdefaultrs) LR_SCHEDULERrrsr7)rCrr7r7r8_ignore_useless_keyss z%FilesAstScanning._ignore_useless_keyscCst}|dur ||_n|||td|d|t}|jD]'}||dd}|tj ddd}| |\} } t | t | t |i||<q#||} || } ||} t| t| i} tdt| d t|d | S) athe entry method of the ast scan method Args: target_file_list can override the dir and folders combine target_dir (str, optional): the absolute path of the target directory to be scanned. Defaults to None. target_folder (list, optional): the list of sub-folders to be scanned in the target folder. Defaults to SCAN_SUB_FOLDERS. Returns: dict: indexer of registry NzAST-Scanning the path "z!" with the following sub folders rrrrizScanning done! A number of z. components indexed or updated! Time consumed r)timerrloggerinfor>rfindreplaceospseprrr}rrrr INDEX_KEYREQUIREMENT_KEYry)rCtarget_file_list target_dirtarget_foldersstartrrr&r/rtrinverted_index_with_resultsrr)r7r7r8get_files_scan_resultss@      z'FilesAstScanning.get_files_scan_resultsc Csg|_|rt|tr||_n|||g}t}|jD]}tj|}|||||<qt dd|d}t | } | |fS)NcSst|t|Sr=rU)xyr7r7r8?sz2FilesAstScanning.files_mtime_md5..ri)rrGrOrr>rrgetmtimerzrhashlibr(encode hexdigest) rC target_pathtarget_subfolder file_listr*files_mtime_dictrmtime result_strr(r7r7r8files_mtime_md50s      z FilesAstScanning.files_mtime_md5rr=NFF)rYrrrDrUrrr>rOrrrrrrrrSCAN_SUB_FOLDERSrrr7r7r7r8r{s4      3robjr;cCsdtj|}|rtj|stj|ddt|d}||WddS1s+wYdS)a Write data to a given ``filepath`` with 'wb' mode. Note: ``write`` will create a directory if the directory of ``filepath`` does not exist. Args: obj (bytes): Data to be written. filepath (str or Path): Path to write data. T)exist_okwbN)rrdirnameexistsmakedirsrwrite)r#r&r&fr7r7r8 ensure_writeGs   "r+FcCsdd|tD|t<ddlm}||t<tj|d\|t<|t<t |t <t |}|r8|t t}t||dd|tD|t<dS)NcSsi|] \}}t||qSr7r.0kvr7r7r8 \sz_save_index..r __version__rcSi|] \}}t||qSr7rH literal_evalr,r7r7r8r0g )r rmodelscope.versionr2 VERSION_KEY file_scannerrMD5_KEYFILES_MTIME_KEYras_posixMODELSCOPE_PATH_KEYjsondumpsrr1r+r)r) file_pathr with_templater2 json_indexr7r7r8 _save_indexZs      rDcCsrt|d }|}Wdn1swY|r%|tt}t|}dd|t  D|t <|S)NrbcSr4r7r5r,r7r7r8r0ur7z_load_index..) rreaddecoderr1rr=r?loadsr r)rArBr* bytes_index wrapped_indexr7r7r8 _load_indexms      rKc Cs0|t}tt|t|}tt|t|}g}|D]}||vr/||||kr/||q||||t|dkrg}g}|tD]} |t| t|vrd|| ||t| tqH|D]} |t| =qg|D] } | |t vr~|t | =qqt |} |t | t|t  | t dS)Nr) r<rOrrzr|ryr rrr r:rupdate) r)r*origin_files_mtime new_files removed_files updated_filesrremove_index_keysremove_requirement_keysr updated_indexr7r7r8 _update_index|s<          rTcCsFttttjd}d}ttt}|||kr!dSdS)Nz%Y-%m-%d %H:%M:%Si3TF)introundrstrptimer__release_datetime__ timestampnow)release_timestampSECONDS_PER_YEARcurrent_timestampr7r7r8__is_develop_models r^c Cs:td|}td|}tj||}d}|r#tdt|}|Strtd|tj |rkt |}tj |d\} } |}ddl m } |t| krT|t| ksZ|t| krjtd t|| t|||ntd |d t|}t|||td |td |tdt|td|St}|S)aget the index from scan results or cache Args: file_list: load indexer only from the file lists if provided, default as None force_rebuild: If set true, rebuild and load index, default as False, indexer_file_dir: The dir where the indexer file saved, default as INDEXER_FILE_DIR indexer_file: The indexer file name, default as INDEXER_FILE Returns: dict: the index information for all registered modules, including key: index, requirements, files last modified time, modelscope home path, version and md5, the detail is shown below example: { 'index': { ('MODELS', 'nlp', 'bert'):{ 'filepath' : 'path/to/the/registered/model', 'imports': ['os', 'torch', 'typing'] 'module': 'modelscope.models.nlp.bert' }, ... }, 'requirements': { 'modelscope.models.nlp.bert': ['os', 'torch', 'typing'], 'modelscope.models.nlp.structbert': ['os', 'torch', 'typing'], ... }, 'files_mtime' : { '/User/Path/To/Your/Modelscope/modelscope/preprocessors/nlp/text_generation_preprocessor.py': 16554565445, ... },'version': '0.2.3', 'md5': '8616924970fe6bc119d1562832625612', 'modelscope_path': '/User/Path/To/Your/Modelscope' } MODELSCOPE_CACHEMODELSCOPE_INDEX_FILENz4Force rebuilding ast index from scanning every file!zLoading ast index from r3rr1z}Updating the files for the changes of local files, first time updating will take longer time! Please wait till updating done!zNo valid ast index found from z$, generating ast index from scratch!z,Loading done! Current index file version is z , with md5 z and a total number of z components indexed)rgetenvrrwrrr:rr^r'rKrr8r2r9r;rTrDryr load_from_prebuilt) r force_rebuildindexer_file_dir indexer_file cache_dir index_filerAr)rJr(r*r2r7r7r8 load_indexsV $               rhcCsH|durtjd}tj|t}tj|r t|dd}|Sd}|S)NrTrB) presolveparentsrrrw TEMPLATE_FILEr'rK)rA local_pathr)r7r7r8rbs  rbcCsTt|d}|durtjd}tj|t}t||ddtj |s(t d|S)NrcrTriz;The index file is not create correctly, please double check) rhrjrkrlrrrwrmrDr'r)rArcr)rnr7r7r8generate_ast_templates  rp__main__ror r!r=)NT)VrHrr2ros.pathrrrrr functoolsrpathlibrtypingrr?rrmodelscope.metainforrr r r r r rrrrmodelscope.utils.constantrrmodelscope.utils.file_utilsrmodelscope.utils.registryr__file__rjrWrkrlrINDEXER_FILE_DIRrrr" INDEXER_FILErrr~r}rr>r9r;r r<r rr{r MODULE_NAMErr1rmr9rrr:rr:bytesrUr+rDrKrTr^rhrbrprYr)printr7r7r7r8s      4  9J  " Q