o im @sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)ddl*Z*ddl+Z+ddl,Z,ddl-Z-ddl*m.Z.m/Z/ddl0m1Z1dd l2m3Z3m4Z4dd l5m6Z6m7Z7dd l8m9Z9m:Z:m;Z;mZ>m?Z?zddl@ZAWn eBydZAYnwdd l8mCZCmDZDmEZEd dlFmGZGd dlHmIZImAZAmJZJmKZKd dlLmMZMmNZNmOZOd dlPmQZQerd dlRmSZSmTZTd dlUmVZVmWZWd dlXmYZYe+Ze+j[j\Z]dZ^gdZ_gdZ`iddgddgddgddgd d!gd"d#gd$d%d&gd'd(gd)d*d+gd,d-gd.d/d0gd1d2d3gd4d5gd6d7gd8d9gd:d;dd?gid@dAgdBdCgdDdEdFgdGdHgdIdJgdKdLgdMdNdOgdPdQgdRdSdTgdUdVgdWdXgdYdZd[gd\d]d^gd_d`gdadbgdcddgdedfgidgdhgdidjgdkdlgdmdngdodpgdqdrgdsdtdugdvdwgdxdygdzd{d|gd}d~gddgdddgddgddgddgdgdiddgddgddgdddgddgdddgdddgddgddgddgddgddgddgddgddgddgddgdgdgdgddgdgdZaebdZcedZeeefegdecheeGdddZiGddde-jjZjGdddekZlGdddemZndeodepfddńZqdeode#eofddDŽZrdeode'dfddʄZsdeode'dddfdd΄ZtdedefddфZude(eoefde(ekemffddӄZvdedefddքZwenZxdexexexeld؜de(eoefde(depfde(eoeeofde(eoeeofde(eoeeofde(eeoefe:fddfddZydexexexeld؜deode(depfde(eoeeofde(eoeeofde(eoeeofde(eeoefe:fddfddZzddexexexelddede#eeoefde(depfde(eoeeofde(eoeeofde(eoeeofde(eeoefe:fddfddZ{eldexexexdddde(eeoefe:fdeeoefde(depfde(eoeeofde(eoeeofde(eoeeofdepdepddfddZ|de(eeoefe:fdeeoeeoefffddZ}de:de e#eode&edffddZ~dexexexeld؜de(eeofde(depfde(eoeeofde(eoeeofde(eoeeofde(eeoefe:fddfddZeldfde(eoefdeeoefdepde:fddZeldfdeodeeoefdepfddZde eofddZdeode#eofddZ אdɐdeodeodepde#epfddZ אdɐdeodepde#epfddZd eode&eoeoffd d Zdeodeofd d Zdeode#eofddZdeodepfddZdeodeofddZdeode#eofddZdeodeodepfddZde(eoefdeeoeffddZde(eoefdeeoeffddZdeodepfddZdeodefd d!Zd"e<d#e<d$eepd?epde#eIfd@dAZdBdCZde(eoefde$fdDdEZdFee(eoe$fde$fdGdHZdFee(eoe$fde$fdIdJZdFee(eoe$fde$fdKdLZdMeeogefdeeogeffdNdOZdMeeogefdPeeoefdQeodefdRdSZdTeeoe ekfdeeoe ekffdUdVZdWeeoe ekfdXeodYeodeeoe ekffdZd[Z dːd\ed]ed^ed_e#ede&eeff d`daZdbedcde dcfdddeZdbedcde dcfdfdgZdhdiZdjeeoegeffdeeodefdkdlZdmedneeoeegeffdeeoddfdodpZdjeeoegeffdeeodeeoeffdqdrZdseeoefdneeoeegeffdeeodeeoeffdtduZde(eoefdveeoeegdffdeeodefdwdxZde(eoefdyeeoeegdffdeeodefdzd{Zdeod|e(eoefdefd}d~ZdeodeofddZdeodeofddZdeeodeode&e eoe epffddZde(eeoefe:fde:fddZdeeoefdeeoekffddZddd7eeoekfdepdeeoeffddZde:deofddZde:deodeddfddZgfdddeeoefde eodepdee&e eoeffddZdede eofddZelfde eeoe#efdeeoe#efdeeoe#effddZGdddZde>fddZddZdedepfddZdeodepfddZdeddddeodeeode dege"fde!eoefdedf ddZĐddZŐddZƐddZǐddZdefddZdeeoe eoffddZʐddZːd̐dedeodepfdĐdńZ̐d͐d]edeodepdefdǐdȄZdS(N) defaultdict)contextmanager)Path) ModuleType) TYPE_CHECKINGAnyCallableDict GeneratorIterableIteratorListMappingNoReturnOptionalPatternSetTupleTypeUnioncast)Registry RegistryError) Requirement)InvalidSpecifier SpecifierSet)InvalidVersionVersion)AdamConfigConfigValidationErrorModelNumpyOps Optimizerget_current_ops) compoundingdecayingfix_random_seed)about) CudaStreamcupyimportlib_metadata is_windows)OLD_MODEL_SHORTCUTSErrorsWarningsORTH)Language PipeCallable)DocSpan)Vocabi)csdadeelengrcidlbmkptrusrtath) paths variablessystemnlp componentscorporatraining pretraining initializeafaframamhararaazazebgbulbnbenbobodtibcacatr8cesczer9danr:deugerr;ellgrer<engesspaetesteueusbaqfafasperfifinfofaofrfrafregaglegdglagugujhehebiwhihinhrhrvscrhuhunhyhyer>indinisisliceititajajpnknkankokorkykirlalatr?ltzlglugltlitlvlavr@mkdmacmlmalmrmarmsmsamaynbnobnenepnlnlddutnnnnoplpolrAporro)ronrommomolrBrussasansisinskslksloslslvsqsqialbrCsrpsccsvswerDtamtetelrEthatitirtltgltntsntrturtttatukukrurdviwyorzhochimul)urviyozhxxspacyz)[%(asctime)s] [%(levelname)s] %(message)sc@seZdZdZdS)ENV_VARSSPACY_CONFIG_OVERRIDESN)__name__ __module__ __qualname__CONFIG_OVERRIDESrr>/home/ubuntu/.local/lib/python3.10/site-packages/spacy/util.pyrsrc @seZdZejddddZejddddZejddddZejddddZejddddZ ejdd ddZ ejdd ddZ ejdd ddZ ejdd ddZ ejdd ddZejddddZejddddZejddddZejddddZejddddZeddZejddddZejddddZed$ddZedeefddZedededefddZedededeeee ee!fffd d!Z"ededede#fd"d#Z$dS)%registryr languagesT) entry_points architectures tokenizers lemmatizerslookupsdisplacy_colorsmisc callbacksbatchersreaders augmentersloggersscorersvectors factoriesinternal_factoriesmodelsclireturnNcCs"ddlm}m}|s|dSdS)z?Ensure the registry is populated with all necessary components.r()REGISTRY_POPULATEDpopulate_registryN) registrationsrr)clsrrrrrensure_populateds zregistry.ensure_populatedcCsF|g}t|D]\}}|dst|tr||q t|S)zList all available registries._)r inspect getmembers startswith isinstancerappendsorted)r namesnamevaluerrrget_registry_namess zregistry.get_registry_names registry_name func_namec C|t||sd|pd}ttjj||dt||}z| |}W|Styg| drN| dd}z| |WYSt jyMYnwdt |pZd}ttjj|||ddw)z,Get a registered function from the registry., noner availablespacy. spacy-legacy.rreg_namerN)r hasattrjoinrrr/E892formatgetattrgetrreplace cataloguerget_allkeysE893)r rrrregfunc legacy_namerrrrr's2     z registry.getc Cr)auFind information about a registered function, including the module and path to the file it's defined in, the line number and the docstring, if available. registry_name (str): Name of the catalogue registry. func_name (str): Name of the registered function. RETURNS (Dict[str, Optional[Union[str, int]]]): The function info. rrrrrr N)r r"r#rrr/r$r%r&findrr(r)rr*r+r,)r rrrr- func_infor/rrrrr0s2     z registry.findcCsN|t||s dSt||}|dr#|dd}||vp"||vS||vS)z4Check whether a function is available in a registry.Frr)r r"r&rr()r rrr-r/rrrhas#s    z registry.has)rN)%rrrr)createrrrrrrrrrrrrrr_entry_point_factoriesrrr classmethodr r strrrr'r rrintr0boolr2rrrrrsF    #rcsLeZdZdZejddeddffddZdd Zdd d Z d d Z Z S)SimpleFrozenDictzSimplified implementation of a frozen dict, mainly used as default function or method argument (for arguments that should default to empty dictionary). Will raise an error if user or spaCy attempts to add to dict. errorr;rNcstj|i|||_dS)zInitialize the frozen dict. Can be initialized with pre-defined values. error (str): The error message when user tries to assign to dict. N)super__init__r;)selfr;argskwargs __class__rrr=6s zSimpleFrozenDict.__init__cC t|jNNotImplementedErrorr;)r>keyrrrr __setitem__? zSimpleFrozenDict.__setitem__cCrCrDrE)r>rGdefaultrrrpopBrIzSimpleFrozenDict.popcCrCrDrE)r>otherrrrupdateErIzSimpleFrozenDict.updaterD) rrr__doc__r/E095r6r=rHrKrM __classcell__rrrArr90s  r9csreZdZdZejddeddffddZdd Zd d Z d d Z ddZ ddZ ddZ ddZddZZS)SimpleFrozenListaWrapper class around a list that lets us raise custom errors if certain attributes/methods are accessed. Mostly used for properties like Language.pipeline that return an immutable list (and that we don't want to convert to a tuple to not break too much backwards compatibility). If a user accidentally calls nlp.pipeline.append(), we can raise a more helpful error. r:r;rNcs||_tj|dS)zpInitialize the frozen list. error (str): The error message when user tries to mutate the list. N)r;r<r=)r>r;r?rArrr=QszSimpleFrozenList.__init__cOrCrDrEr>r?r@rrrrYrIzSimpleFrozenList.appendcOrCrDrErRrrrclear\rIzSimpleFrozenList.clearcOrCrDrErRrrrextend_rIzSimpleFrozenList.extendcOrCrDrErRrrrinsertbrIzSimpleFrozenList.insertcOrCrDrErRrrrrKerIzSimpleFrozenList.popcOrCrDrErRrrrremovehrIzSimpleFrozenList.removecOrCrDrErRrrrreversekrIzSimpleFrozenList.reversecOrCrDrErRrrrsortnrIzSimpleFrozenList.sort)rrrrNr/E927r6r=rrSrTrUrKrVrWrXrPrrrArrQIsrQlangrcCs |tjvS)aCheck whether a Language class is already loaded. Language classes are loaded lazily, to avoid expensive setup code associated with the language data. lang (str): Two-letter language code, e.g. 'en'. RETURNS (bool): Whether a Language class has been loaded. )rr)rZrrrlang_class_is_loadedrs r[cCs.ddl}tD] \}}||vr|SqdS)a Given a two-letter ISO 639-1 or three-letter ISO 639-3 language code, find a supported spaCy language. Returns the language code if a matching language is available, or None if there is no matching language. >>> find_matching_language('fra') # ISO 639-3 code for French 'fr' >>> find_matching_language('fre') # ISO 639-2/B code for French 'fr' >>> find_matching_language('iw') # Obsolete ISO 639-1 code for Hebrew 'he' >>> find_matching_language('mo') # Deprecated code for Moldavian 'ro' >>> find_matching_language('scc') # Deprecated ISO 639-2/B code for Serbian 'sr' >>> find_matching_language('zxx') None rN) spacy.lang LANG_ALIASESitems)rZr lang_codealiasesrrrfind_matching_language}s rar3c Cs|tjvr tj|Sz td|d}Wn/tyE}z#t|}|r0|}td|d}n ttjj ||d|WYd}~nd}~wwt |t ||j dtj|S)zImport and load a Language class. lang (str): Two-letter ISO 639-1 or three-letter ISO 639-3 language code, such as 'en' and 'eng'. RETURNS (Language): Language class. z.lang.r)rZerrNr) rrr' importlib import_module ImportErrorrar/E048r%set_lang_classr&__all__)rZmodulerbmatchrrrget_lang_classs    rkrr cCstjj||ddS)zSet a custom Language class name that can be loaded via get_lang_class. name (str): Name of Language class. cls (Language): Language class. )r.N)rrregister)rr rrrrgrgpathcCst|tr t|S|S)zEnsure string is converted to a Path. path (Any): Anything. If string, it's converted to Path. RETURNS: Path or original argument. )rr6rrnrrr ensure_paths rpcCsNt|}|r t|S||jd}|rt|Sttj j |d)zLoad JSON language data using the given path as a base. If the provided path isn't present, will attempt to load a gzipped version before giving up. path (str / Path): The data to load. RETURNS: The loaded data. z.gzro) rpexistssrsly read_json with_suffixsuffixread_gzip_json ValueErrorr/E160r%rorrrload_language_datas  ryricCs@t|dsttjjt|dtttj t j |j j }|jS)zpGet the path of a Python module. module (ModuleType): The Python module. RETURNS (Path): The path. r)ri)r"rwr/E169r%reprrrosPathLikesysmodulesr__file__parent)ri file_pathrrrget_module_paths rTvocabdisableenableexcludeconfigrr7rrrrcCs|||||d}t|tr8|drt|ddSt|r't|fi|St|r7t t|fi|Sn t |drEt |fi|S|t vrUt t jj|t |dt t jj|d)aLoad a model from a package or data path. name (str): Package name or model path. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All others will be disabled. exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. RETURNS (Language): The loaded nlp object. rzblank:rq)rfullr)rr6rrkr( is_packageload_model_from_packagerrqload_model_from_pathr"r.IOErrorr/E941r%E050)rrrrrrr@rrr load_models&    rcCst|}|j|||||dS)aLoad a model from an installed package. name (str): The package name. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. RETURNS (Language): The loaded nlp object. r)rcrdload)rrrrrrr rrrrs r)metarrrrr model_pathrc Csj|s ttjj|d|st|}|d}t|dd}t||d}t||||||d} | j |||dS)aLoad a model from a data directory path. Creates Language class with pipeline from config.cfg and then calls from_disk() with path. model_path (Path): Model path. meta (Dict[str, Any]): Optional model meta. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. RETURNS (Language): The loaded nlp object. roz config.cfgT for_overrides) overrides)rrrrr)rr) rqrr/E052r%get_model_meta dict_to_dot load_configload_model_from_config from_disk) rrrrrrr config_pathrrIrrrr3s   rF)rrrrr auto_fillvalidaterrc Csrd|vr ttjj|d|d}d|vs|ddur$ttjj|dt|d} | j||||||||d} | S)a%Create an nlp object from a config. Expects the full config file including a section "nlp" containing the settings for the nlp object. name (str): Package name or model path. meta (Dict[str, Any]): Optional model meta. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. auto_fill (bool): Whether to auto-fill config with missing defaults. validate (bool): Whether to show config validation errors. RETURNS (Language): The loaded nlp object. rIrrZN)rrrrrrr)rwr/E985r%E993rk from_config) rrrrrrrr nlp_configlang_clsrIrrrras"  rcCsdd|diDS)zRETURNS (List[str]): All sourced components in the original config, e.g. {"source": "en_core_web_sm"}. If the config contains a key "factory", we assume it refers to a component factory. cSs&i|]\}}d|vrd|vr||qS)factorysourcer).0rcfgrrr s z*get_sourced_components..rJ)r'r^rrrrget_sourced_componentssr dot_names.c Csi}g}g}|D]X}|dur||q|dd}||vr=t||r2td||id}nt||}|||<z |t||Wqty`d|}||d|dYqw|rit||dt|S)a:Resolve one or more "dot notation" names, e.g. corpora.train. The paths could point anywhere into the config, so we don't know which top-level section we'll be looking within. We resolve the whole top-level section, although we could resolve less -- we could find the lowest part of the tree. N.rrznot a valid section reference: )locmsg)rerrors) rsplitr is_promiseresolve dot_to_objectKeyErrorr tuple) rrresolvedoutputrrsectionresultrrrrresolve_dot_namess*     r init_filec Csjt|j}t|}|dd|dd|d}||} |s*ttjj| dt| ||||||dS)aHelper function to use in the `load()` method of a model package's __init__.py. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. RETURNS (Language): The loaded nlp object. rZr r-versionro)rrrrrr) rrrrqrr/rr%r) rrrrrrrrdata_dir data_pathrrrload_model_from_init_pys  rr interpolatecCsdt|}ttd}t|dkr|jtj||dS|r |s*t t j j |dd|j |||dS)aLoad a config file. Takes care of path validation and section order. path (Union[str, Path]): Path to the config file or "-" to read from stdin. overrides: (Dict[str, Any]): Config overrides as nested dict or dict keyed by section values in dot notation. interpolate (bool): Whether to interpolate and resolve variables. RETURNS (Config): The loaded config.  section_orderrrrz config filernr)rprCONFIG_SECTION_ORDERr6from_strr~stdinreadis_filerr/E053r%r)rnrrrrrrrrs    rtextcCsttdj|||dS)zLoad a full config from a string. Wrapper around Thinc's Config.from_str. text (str): The string config to load. interpolate (bool): Whether to interpolate and resolve variables. RETURNS (Config): The loaded config. rr)rrr)rrrrrrload_config_from_str s rcCsttjS)z~List all model packages currently installed in the environment. RETURNS (List[str]): The string names of the models. )listrrr*r+rrrrget_installed_modelsrcCs$zt|WStjyYdSw)zGet the version of an installed package. Typically used to get model package versions. name (str): The name of the installed Python package. RETURNS (str / None): The version or None if package not installed. N)r,rPackageNotFoundErrorrrrrget_package_version!s  rr constraint prereleasesc CsR|dr d|}z t|}t|}Wn ttfy!YdSw||_||vS)a Check if a version (e.g. "2.0.0") is compatible given a version constraint (e.g. ">=1.9.0,<2.2.1"). If the constraint is a specific version, it's interpreted as =={version}. version (str): The version to check. constraint (str): The constraint string. prereleases (bool): Whether to allow prereleases. If set to False, prerelease versions will be considered incompatible. RETURNS (bool / None): Whether the version is compatible, or None if the version or constraint are invalid. r==N)isdigitrrrrr)rrrspecrrris_compatible_version.s   rcCs|drdSzt|}Wn tyYdSw||_dd|D}t|dkr2|djdvr2dStdd |Dr=dStd d |D}td d |D}|rU|rUdSdS) NrFcSsg|]}|qSrrrsprrr Ssz,is_unconstrained_version..r(>>=Tcs|]}|jdvVqdS)rNoperatorrrrr Xz+is_unconstrained_version..csr))=1.2.3 into ("spacy", ">=1.2.3").)rrr6 specifier)rreqrrrsplit_requirementcsrcCs.t|j}d|d|dd|dddS)z_Generate a version range like >=1.2.3,<1.3.0 based on a given version (e.g. of spaCy). rz,From a version range like >=1.2.3,<1.3.0 return the lower pin.)rrz~=N)rrr Exception)rspecsetrrrrget_model_lower_versionqs   rcC t|jS)zCheck whether a version is a prerelease version. version (str): The version, e.g. "3.0.0.dev1". RETURNS (bool): Whether the version is a prerelease version. )r is_prereleaserrrris_prerelease_version} rcCr)zGenerate the base version without any prerelease identifiers. version (str): The version, e.g. "3.0.0.dev1". RETURNS (str): The base version, e.g. "3.0.0". )r base_versionrrrrget_base_versionrrc Cs8zt|}Wn ttfyYdSw|jd|jS)zGet the major + minor version (without patch or prerelease identifiers). version (str): The version. RETURNS (str): The major + minor version or None if version is invalid. Nr)r TypeErrorrmajorminor)rvrrrget_minor_versions  r version_a version_bcCs(t|}t|}|duo|duo||kS)aNCompare two versions and check if they match in major and minor, without patch or prerelease identifiers. Used internally for compatibility checks that should be insensitive to patch releases. version_a (str): The first version version_b (str): The second version. RETURNS (bool): Whether the versions match. N)r)rrabrrris_minor_version_matchs rcCs^t|}|jsttjj|jd|r|s&ttjj|jddt |}dD]}||vs7||s@t tj j|dq-d|vrt tj|dst|d}t|}|dur`d|}n d |vrkd |d }nd }tjj|d d |d|d|tjd}t|t|drtjj|d d |d|d|dttjd}t||S)zLoad a model meta.json from a path and validate its contents. path (Union[str, Path]): Path to meta.json. RETURNS (Dict[str, Any]): The loaded meta. ro meta.jsonr)rZrr)setting spacy_versionNrspacy_git_versionz git commit zversion unknownrZr rr)model model_versionrcurrent)r rrexample)rprrqrr/rr%rrrrrsrwE054rr) __version__rrr0W095warningswarnrW094r)rnrr  lower_versionwarn_msgrrr load_metasF       rcCst|}t|dS)zGet model meta.json from a directory path and validate its contents. path (str / Path): Path to model directory. RETURNS (Dict[str, Any]): The model's meta data. r )rpr)rnrrrrrs rcCszt|WdSYdS)zCheck if string maps to a package installed via pip. name (str): Name of package. RETURNS (bool): True if installed package, False if not. TF)r, distributionrrrrrs  rcCs&t|}tttttjf|jj S)zxGet the path to an installed package. name (str): Package name. RETURNS (Path): Path to installed package. ) rcrdrrrr6r|r}rr)rpkgrrrget_package_paths rr target replacementcCsb|D]}||jvr||j|j|<q|D]}|jD]}|||ur-|||qqdS)zReplace a node within a model with a new one, updating refs. model (Model): The parent model. target (Model): The target node. replacement (Model): The node to replace the target with. N)walklayersindex ref_names maybe_get_refset_ref)r rrnoderef_namerrrreplace_model_nodes     r'commandcCstj|t dS)zSplit a string command using shlex. Handles platform compatibility. command (str) : The command to split RETURNS (List[str]): The split command. )posix)shlexrr-)r(rrr split_command sr+)rcapturerr,c Cst|tr t|}|}n|}d|}ztj|tj|dd|r#tj nd|r)tj ndd}Wnt yBt t j j||dddw|jdkrs|rsd|d }|d |j7}|jdurf|d 7}||j7}t|}||_||_||jdkr~t|j|S) aRun a command on the command line as a subprocess. If the subprocess returns a non-zero exit code, a system exit is performed. command (str / List[str]): The command. If provided as a string, the string will be split using shlex.split. stdin (Optional[Any]): stdin to read from or None. capture (bool): Whether to capture the output and errors. If False, the stdout and stderr will not be redirected, and if there's an error, sys.exit will be called with the return code. You should use capture=False when you want to turn over execution to the command, and capture=True when you want to run the command more like a function. RETURNS (Optional[CompletedProcess]): The process object.  utf8FN)envinputencodingcheckstdoutstderrr) str_commandtoolzError running command: z zSubprocess exited with status z$ Process log (stdout and stderr): )rr6r+r# subprocessrunr|environcopyPIPESTDOUTFileNotFoundErrorr/E970r% returncoder3SubprocessErrorretr(r~exit)r(rr,cmd_listcmd_strrAmessager;rrr run_commandsF           rFc csPt}t|}tt|z |VWtt|dStt|w)aUChange current working directory and returns to previous on exit. path (str / Path): The directory to navigate to. YIELDS (Path): The absolute path to the current working directory. This should be used if the block needs to perform actions within the working directory, to prevent mismatches with relative paths. N)rcwdrr|chdirr6)rnprev_cwdrrrr working_dirIs "rJc cstt}|Vdd}ztjdkr tjt||dWdStjt||dWdStyI}zt t j j ||dWYd}~dSd}~ww)zExecute a block in a temporary directory and remove the directory and its contents at the end of the with block. YIELDS (Path): The path of the temp directory. cSst|tj||dSrD)r|chmodstatS_IWRITE)rmfuncrnexrrr force_removees z"make_tempdir..force_remove) )onexc)onerror)dirrN)rtempfilemkdtempr~ version_infoshutilrmtreer6PermissionErrorrrr0W091r%)drPerrr make_tempdirZs  $r_cCsdztjjdkr WdStjjdkrWdSWn tyYnwzddl}WdSty1YdSw)zCheck if user is running spaCy from a Jupyter or Colab notebook by detecting the IPython kernel. Mainly used for the displaCy visualizer. RETURNS (bool): True if in Jupyter/Colab, False if not. ZMQInteractiveShellTzgoogle.colab._shellrNF) get_ipythonrBrr NameError google.colabre)googlerrr is_in_jupyterrs   recCsttdp ttdS)zCheck if user is running spaCy from an interactive Python shell. Will return True in Jupyter notebooks too. RETURNS (bool): True if in interactive mode, False if not. ps1ps2)r"r~rrrris_in_interactivermrhobjcCsPt|dr |jdur |jSt|dr|jSt|dr$t|jdr$|jjSt|S)zGet a human-readable name of a Python object, e.g. a pipeline component. obj (Any): The Python object, typically a function or class. RETURNS (str): A human-readable name. rNrrB)r"rrrBr{)rirrrget_object_names rjfunc1func2cCslt|rt|s dSt|drt|dsdS|j|jk}t|t|k}t|t|k}|o5|o5|S)aApproximately decide whether two functions are the same, even if their identity is different (e.g. after they have been live reloaded). Mostly used in the @Language.component and @Language.factory decorators to decide whether to raise if a factory already exists. Allows decorator to run multiple times with the same function. func1 (Callable): The first function. func2 (Callable): The second function. RETURNS (bool): Whether it's the same function (most likely). Fr)callabler"rr getfilegetsourcelines)rkrl same_name same_file same_coderrr is_same_funcs   rsrequire non_blockingcCs*t}tdur dSt|trdSt|dS)N)ru)r$r*rr")rtruopsrrrget_cuda_streams   rwcCs2tdur|Stj|jd|jd}|j||d|S)NC)orderdtype)stream)r+ndarrayshaperzset)r{ numpy_arrayarrayrrr get_asyncs rcCs`t|}|jdd}|d}Wdn1swYddd|D}t|S)Nr.)r1 |cSs"g|] }|rdt|qS^)stripreescaperpiecerrrrs"zread_regex..)rpopenrrr#rcompile)rnfile_entries expressionrrr read_regexs  rrcCddd|D}t|S)aCompile a sequence of prefix rules into a regex object. entries (Iterable[Union[str, Pattern]]): The prefix rules, e.g. spacy.lang.punctuation.TOKENIZER_PREFIXES. RETURNS (Pattern): The regex object. to be used for Tokenizer.prefix_search. rcSsg|] }|rd|qSrrrrrrrz(compile_prefix_regex..r#rrrrrrrcompile_prefix_regex rcCr)aCompile a sequence of suffix rules into a regex object. entries (Iterable[Union[str, Pattern]]): The suffix rules, e.g. spacy.lang.punctuation.TOKENIZER_SUFFIXES. RETURNS (Pattern): The regex object. to be used for Tokenizer.suffix_search. rcSsg|] }|r|dqS)$rrrrrrrz(compile_suffix_regex..rrrrrcompile_suffix_regexrrcCr)aCompile a sequence of infix rules into a regex object. entries (Iterable[Union[str, Pattern]]): The infix rules, e.g. spacy.lang.punctuation.TOKENIZER_INFIXES. RETURNS (regex object): The regex object. to be used for Tokenizer.infix_finditer. rcSsg|]}|r|qSrrrrrrrz'compile_infix_regex..rrrrrcompile_infix_regexrr default_funccGstt||S)aQExtend an attribute function with special cases. If a word is in the lookups, the value is returned. Otherwise the previous function is used. default_func (callable): The default function to execute. *lookups (dict): Lookup dictionary mapping string to attribute value. RETURNS (callable): Lexical attribute getter. ) functoolspartial_get_attr_unless_lookup)rrrrr add_lookupss rrstringcCs&|D] }||vr||Sq||SrDr)rrrlookuprrrrs  rbase_exceptionscGst|}|D];}|D]/\}}tdd|Ds#ttjj||dddd|D}||kr;ttjj||dq | |qt |dd}|S)a'Update and validate tokenizer exceptions. Will overwrite exceptions. base_exceptions (Dict[str, List[dict]]): Base exceptions. *addition_dicts (Dict[str, List[dict]]): Exceptions to add to the base dict, in order. RETURNS (Dict[str, List[dict]]): Combined tokenizer exceptions. css|] }t|ttVqdSrD)rr2r6rattrrrrrszupdate_exc..)rGorthsrcss|]}|tVqdSrDr1rrrrrs'u’) dictr^allrwr/E055r%r#E056rM expand_exc)raddition_dictsexc additionsorth token_attrsdescribed_orthrrr update_exc s   rexcssearchr(csXddt|}|D]\}}|vr)|}fdd|D}|||<q |S)abFind string in tokenizer exceptions, duplicate entry and replace string. For example, to add additional versions with typographic apostrophes. excs (Dict[str, List[dict]]): Tokenizer exceptions. search (str): String to find and replace. replace (str): Replacement. RETURNS (Dict[str, List[dict]]): Combined tokenizer exceptions. cSs t|}|t|||t<|SrD)rr2r()tokenrr(fixedrrr _fix_token/szexpand_exc.._fix_tokencsg|]}|qSrr)rtrr(rrrr8rzexpand_exc..)rr^r()rrr(new_excs token_stringtokensnew_key new_valuerrrr#s  rlengthstartstopstepcCs~|dus |dks ttj|durd}n|dkr||7}t|td|}|dur+|}n|dkr3||7}t|t||}||fS)Nr(r)rwr/E057minmax)rrrrrrrnormalize_slice=s rspansr6cCstdd}t||dd}g}t}|D]}|j|vr/|jd|vr/|||t|j|jqt|ddd}|S)aFilter a sequence of spans and remove duplicates or overlaps. Useful for creating named entities (where one token can only be part of one entity) or when merging spans with `Retokenizer.merge`. When spans overlap, the (first) longest span is preferred over shorter spans. spans (Iterable[Span]): The spans to filter. RETURNS (List[Span]): The filtered spans. cSs|j|j|j fSrD)endrspanrrrXszfilter_spans..T)rGrWr(cSs|jSrD)rrrrrras)rG)rr~rrrrMrange)r get_sort_key sorted_spansr seen_tokensrrrr filter_spansOs  rcGsttj|SrD)r itertoolschain)rrrrfilter_chain_spansesrcCstSrD)rrrrrmake_first_longest_spans_filterirgetterscCstt||SrD)rr msgpack_dumpsto_dict)rrrrrto_bytesmsr bytes_datasetterscCstt|||SrD) from_dictrr msgpack_loads)rrrrrr from_bytesqrrcCs6i}|D]\}}|dd|vr|||<q|SNrrr^r)rr serializedrGgetterrrrrys  rrcCs<|D]\}}|dd|vr||vr|||q|Srr)rrrrGsetterrrrrs  rwriterscCsLt|}|s ||D]\}}|dd|vr#|||q|Sr)rprqmkdirr^r)rnrrrGwriterrrrto_disks rrcCs<t|}|D]\}}|dd|vr|||q|Sr)rpr^r)rnrrrGreaderrrrrs  rrcCs.tj|t|}tj|}|j||S)zImport module from a file. Used to load models from a directory. name (str): Name of module to load. loc (str / Path): Path to the file. RETURNS: The loaded module. )rcutilspec_from_file_locationr6module_from_specloader exec_module)rrrrirrr import_files  rhtmlcCs|ddddS)zPerform a template-specific, rudimentary HTML minification for displaCy. Disclaimer: NOT a general-purpose solution, only removes indentation and newlines. html (str): Markup to minify. RETURNS (str): "Minified" HTML. z rr)rr()rrrr minify_htmlsrcCs4|dd}|dd}|dd}|dd}|S) zReplace <, >, &, " with their HTML encoded representation. Intended to prevent HTML errors in rendered displaCy markup. text (str): The original text. RETURNS (str): Equivalent text to be safely used within HTML. &z&rz<rz>"z")r()rrrr escape_htmls    rwordsc Cs>dd|d|krttjj||dg}g}d}dd|D}|D]\}z ||d|}WntyIttjj||ddw|dkrb||||||d||7}|||d|t|7}|t|kr||dkrd |d <|d 7}q*|t|kr|||d|d||fS) aGiven a list of words and a text, reconstruct the original tokens and return a list of words and spaces that can be used to create a Doc. This can help recover destructive tokenization that didn't preserve any whitespace information. words (Iterable[str]): The words. text (str): The original text. RETURNS (Tuple[List[str], List[bool]]): The words and spaces. r)rrrcSsg|]}|s|qSr)isspace)rwordrrrrrz(get_words_and_spaces..NFr-Tr() r#rrwr/E194r%r!rr)rr text_words text_spacestext_pos norm_wordsr word_startrrrget_words_and_spacess6"        rcCs2zt|WStyttjj|ddw)zDeep copy a Config. Will raise an error if the config contents are not JSON-serializable. config (Config): The config to copy. RETURNS (Config): The copied config. rN)rr:rwr/E961r%rrrr copy_configs  rvaluesc Csbi}|D](\}}|}|d}t|D]\}}|t|dk}|||r*|ni}qq|S)aConvert dot notation to a dict. For example: {"token.pos": True, "token._.xyz": True} becomes {"token": {"pos": True, "_": {"xyz": True }}}. values (Dict[str, Any]): The key/value pairs to convert. RETURNS (Dict[str, dict]): The converted values. rr()r^lowerr enumerater setdefault) rrrGrrnpartsiitemis_lastrrr dot_to_dictsr rrcCsddt||dDS)aeConvert dot notation to a dict. For example: {"token": {"pos": True, "_": {"xyz": True }}} becomes {"token.pos": True, "token._.xyz": True}. obj (Dict[str, dict]): The dict to convert. for_overrides (bool): Whether to enable special handling for registered functions in overrides. RETURNS (Dict[str, Any]): The key/value pairs. cSsi|] \}}d||qS)r)r#)rrGrrrrr"s zdict_to_dot..r) walk_dict)rirrrrrs rrc CsN|}|d}|D]}z||}Wq ttfy$ttjj|ddw|S)a`Convert dot notation of a "section" to a specific part of the Config. e.g. "training.optimizer" would return the Optimizer object. Throws an error if the section is not defined in this config. config (Config): The config. section (str): The dot notation of the section in the config. RETURNS: The object denoted by the section rrN)rrrr/E952r%)rr componentrrrrrr(s  rrc Csp|}|d}t|D]*\}}z|t|dkr|||<n||}Wq ttfy5ttjj|ddwdS)zUpdate a config at a given position from a dot notation. config (Config): The config. section (str): The dot notation of the section in the config. value (Any): The value to set in the config. rr(rN)rrrrrr/r r%)rrrr rrrrrrset_dot_to_object;s  rr%rccsd|D]*\}}g||}t|tr*|rtdd|Ds*t|||dEdHq||fVqdS)zWalk a dict and yield the path and values of the leaves. for_overrides (bool): Whether to treat registered functions that start with @ as final values rather than dicts to traverse. css|]}|dVqdS)@N)r)r value_keyrrrrZrzwalk_dict..rN)r^rrrr )r%rrrGr key_parentrrrr Ns   r r.cCs$t|}ttg|j|jS)zGet a list of all named arguments of a function (regular, keyword-only). func (Callable): The function RETURNS (List[str]): The argument names. )r getfullargspecrrfromkeysr? kwonlyargs)r.argspecrrr get_arg_namesas rweightscCsbdd|D}||tdd|D}|D]\}}|r.|dkr.t||d||<q|S)aCombine and normalize score weights defined by components, e.g. {"ents_r": 0.2, "ents_p": 0.3, "ents_f": 0.5} and {"some_other_score": 1.0}. weights (List[dict]): The weights defined by the components. overrides (Dict[str, Optional[Union[float, int]]]): Existing scores that should be preserved. RETURNS (Dict[str, float]): The combined and normalized weights. cSs$i|]}|D]\}}||qqSr)r^)rw_dictrGrrrrr{s  z)combine_score_weights..cSsg|]}|r|ndqS)gr)rrrrrrrz)combine_score_weights..r)rMsumrr^round)rrr weight_sumrGrrrrcombine_score_weightsls  rc@sjeZdZddZddZddZdeddfd d Zd ee e fdd fd dZ d ee e fddfddZ d S)DummyTokenizercCstrD)rF)r>rrrr__call__rzDummyTokenizer.__call__cks|D]}||VqdSrDr)r>textsr@rrrrpipes zDummyTokenizer.pipecKsdS)Nr)r>r@rrrrrzDummyTokenizer.to_bytesdatarcK|SrDr)r>r#r@rrrrrzDummyTokenizer.from_bytesrnNcKdSrDrr>rnr@rrrrrzDummyTokenizer.to_diskcKr$rDrr&rrrrrzDummyTokenizer.from_disk) rrrrr!rbytesrrr6rrrrrrrrsrcCstSrD)rrrrrcreate_default_optimizersr(ccs^t|tr t|}n|}t|} t|}tt|t|}t|dkr)dSt|Vq)zlIterate over batches of items. `size` may be an iterator, so that batch-size can vary on each step. TrN) rr7rrepeatiternextrislicer)r^sizesize_ batch_sizebatchrrr minibatchs    r1cCs`d}t||r dSt|dr.t|dr.|jtjvr.ttj|j|jdd}t||SdS)aSlightly hacky check for whether a callable is implemented in Cython. Can be used to implement slightly different behaviors, especially around inspecting and parameter annotations. Note that this will only return True for actual cdef functions and methods, not regular Python functions defined in Python modules. func (Callable): The callable to check. RETURNS (bool): Whether the callable is Cython (probably). __pyx_vtable__TrrrrF)r"rr~rvarsrr)r.rcls_funcrrris_cython_funcs    r5env_varcCs"tj|d}|dkr dSt|S)aConvert the value of an environment variable to a boolean. Add special check for "0" (falsy) and consider everything else truthy, except unset. env_var (str): The name of the environment variable to check. RETURNS (bool): Its boolean value. F0)r|r9r'r8)r6rrrrcheck_bool_env_varsr8docsr5procr4default_error_handlerr@c cst|dr|j|fi|EdHdSt|}|}t|dr#|}dD] }||vr0||q%|D])}z ||fi|}|VWq3ty\}z||||g|WYd}~q3d}~wwdS)Nr!get_error_handler)r/)r"r!rr<rKr) r9r:rr;r@ error_handlerargdocr^rrr_pipes(    r@cCs|rDr proc_namer:r9r^rrr raise_errorrrCcCr%rDrrArrr ignore_errorrrDcCsJtrddlm}|jdur!ddlm}|s#ttj dSdSdSdS)zqWarn about require_gpu if a jupyter notebook + cupy + mismatched contextvars vs. thread ops are detected r)CupyOpsN)contextvars_eq_thread_ops) rethinc.backends.cupy_opsrExpthinc.backendsrFrrr0W111)rErFrrrwarn_if_jupyter_cupys   rKcCsP|jdi}t|dkr$|jtvr&dt}ttj j ||ddSdSdS)N lexeme_normrr)r langs) r get_tablerrZLEXEME_NORM_LANGSr#loggerdebugr0W033r%)rcomponent_name lexeme_normsrMrrrcheck_lexeme_norms s  rUcCs@|durdS|dur dS|durdS|dkrdS|dkrdSdS)zConvert a value to the ternary 1/0/-1 int used for True/None/False in attributes such as SENT_START: True/1/1.0 is 1 (True), None/0/0.0 is 0 (None), any other values are -1 (False). Tr(NrFrr)valrrrto_ternary_intsrWcCsJtt}tD]}|dpdD] }|||jdqqt|S)zReturn a mapping of top-level packages to their distributions. We're inlining this helper from the importlib_metadata "backport" here, since it's not available in the builtin importlib.metadata. z top_level.txtrName) rrr, distributions read_textrrmetadatar) pkg_to_distdistrrrrpackages_distributions's  r^cCs t|}t|dot|d S)ztReturn True if all the elements are equal to each other (or if the input is an empty sequence), False otherwise.TF)rgroupbyr+)iterablegrrr all_equal3s rb localhostporthostcCs\ttjtj}z z|||fWW|dStjy(YW|dSw|w)zCheck if 'host:port' is in use. Return True if it is, False otherwise. port (int): the port to check host (str): the host to check (default "localhost") RETURNS (bool): Whether 'host:port' is in use. FT)socketAF_INET SOCK_STREAMbindcloser;)rdresrrr_is_port_in_use:s   rl auto_selectcCst||s|S|}|sttjj|dt||r*|dkr*|d7}t||r*|dks|dkrsd    L $           !"#$%&'()*+,-./0123456789:;<=>?@ABCDE N {)  "     -     1    2    (   ,          &&+  $   7$   $ $ $ ,     $$ 0     $   &)( 6"          ",