o i& @szddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z ddl mZddlmZmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'Z'ddl(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/dd l0m1Z1d d l2m3Z3m4Z4m5Z5d dl6m7Z7d dl8m9Z9m:Z:d dl;mZ>m?Z?m@Z@d dlAmBZBmCZCd dlDmEZEd dlFmGZGmHZHmIZId dlJmKZKmLZLmMZMmNZNmOZOd dlPmQZQd dlRmSZSd dlTmUZUd dlVmWZWd dlXmYZYmZZZd dl[m\Z\m]Z]d dl5m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfd dlgmhZhd dlimjZjmkZkeeUgeUfZleemjndZoe5peoZqeemjnd Zre#d!ZsGd"d#d#Ztd$ed%geSffd&d'Zud(d)ZvGd*d%d%Zwd+exd$exfd,d-ZyeGd.d/d/ZzGd0d1d1e{Z|d2eeYd$eeYfd3d4Z}d5ee$e~eUefesgeUfd6eed7eeUfd8e"exexexfd$dfd9d:ZGd;d<d<ZGd=d>d>ZeZdS)?N) ExitStackcontextmanager)deepcopy) dataclass)chaincycle)Path) default_timer)AnyCallableDictIterableIteratorListNoReturnOptionalPatternSequenceSetTupleTypeVarUnioncastoverload)Pool)ConfigCupyOps Optimizerget_current_ops)convert_recursive)abouttyutil)Literal)ErrorsWarnings) GIT_VERSION)TOKENIZER_INFIXESTOKENIZER_PREFIXESTOKENIZER_SUFFIXES)BASE_EXCEPTIONS URL_MATCH) load_lookups) analyze_pipesprint_pipe_analysisvalidate_attrs) ConfigSchemaConfigSchemaInitConfigSchemaNlpConfigSchemaPretrainvalidate_init_settings)Scorer Tokenizer)Doc) Underscore)Examplevalidate_examples) init_tok2vec init_vocab) _DEFAULT_EMPTY_PIPESCONFIG_SECTION_ORDERSimpleFrozenDictSimpleFrozenList_pipecombine_score_weights raise_errorregistrywarn_if_jupyter_cupy) BaseVectors)Vocab create_vocabzdefault_config.cfgzdefault_config_pretraining.cfg _AnyContextc@seZdZUdZeedZeed<eZ e e e e fed<eZeeee efed<eZeeee efed<eZeeee efed<dZeeed <eZeeed <iZe e efed <iZe eee geffed <e Z!e"e ed <ddddZ#dS) BaseDefaultszLanguage data defaults, available via Language.Defaults. Can be overwritten by language subclasses by defining their own subclasses of Language.Defaults.  section_orderconfigtokenizer_exceptionsprefixessuffixesinfixesN token_match url_matchsyntax_iteratorslex_attr_getters stop_wordsltrT) directionhas_case has_letters)$__name__ __module__ __qualname____doc__rr@rO__annotations__r+rPr strrdictr)rQrrrrr*rRr(rSrTr r,rUrVrWintr setrXrwriting_systemrgrgB/home/ubuntu/.local/lib/python3.10/site-packages/spacy/language.pyrLXs rLreturnLanguagecCsdddtfdd}|S)zRegistered function to create a tokenizer. Returns a factory that takes the nlp object and returns a Tokenizer instance using the language detaults. nlprjric Ssx|jj}|jj}|jj}|rt|jnd}|rt|jnd}|r(t|j nd}t |j |jj ||||jj |jjdS)N)rules prefix_search suffix_searchinfix_finditerrTrU)DefaultsrQrRrSr#compile_prefix_regexsearchcompile_suffix_regexcompile_infix_regexfinditerr8vocabrPrTrU)rkrQrRrSrmrnrorgrgrhtokenizer_factorypsz+create_tokenizer..tokenizer_factoryr7)rwrgrgrhcreate_tokenizerksrxcCstjd|t||d}|S)Nz+Loading lookups from spacy-lookups-data: %s)langtables)r#loggerdebugr-)ryrzlookupsrgrgrhload_lookups_datas r~cs eZdZUdZeZdZeee d<e Z e e jdZiZeedfe d< ddiddd d d eeefd ed eeefdeedgeegeffdeedgefdeddfddZfddZeddZedeeeffddZejdeeefddfddZede fddZ!e!jde ddfddZ!ede"efd d!Z#ede"efd"d#Z$ede"e%ee&ffd$d%Z'ede"efd&d'Z(ede"e%ee&ffd(d)Z)ede"efd*d+Z*edeeeffd,d-Z+edeee"effd.d/Z,e-d0edefd1d2Z.e-d0edefd3d4Z/e-d0eddfd5d6Z0e-d0eddddfd7d8Z1d0eddfd9d:Z2d0ede fd;d<Z3e-e e4e4d=e dd>d0ed?eeefd@e5edAe5edBedCeeee6fdDeedefdEdFZ7e-e4e4d=ddGd0ed@e5edAe5edBedDee&dedHeff dIdJZ8gdKd=dLdMe"edNedeeeeffdOdPZ9d0ede&fdQdRZ: de dddSdTed0eedUeeefdVee dWede&f dXdYZ;dZed[dd0ede%e&effd\d]Z< dddddde ddd^dTed0eed_eeeefd`eeeefdaeedbeed[eddUeeefdVee dWede&fdcddZ=    dd_eeeefd`eeeefdaeedbeedef dedfZ>d0edefdgdhZ?e ddid0edTedUeeefdWede&f djdkZ@dledmeddfdndoZAd0ede%ee&ffdpdqZBd0eddfdrdsZCd0eddfdtduZDe4ddvdweeefdxe5edyeeeeeeffdefdzd{ZEdd}d~ZFddddxeeee5efdeeee5efdd|fddZGdwedefddZHdeeeeIfdefddZJdeeeeIfdeKdefddZL ddddde4e4dde5eMdeede6deeNdeeee6fdyeeeeeeffde5ede5efddZOddde4dde5eMdeeNdeeee6fdyeeeeeeffde5edeee6ff ddZP ddddeege5eMfdeeNdeNfddZQ ddddeege5eMfdeeNdeNfddZRdddeeNdeNfddZSdeee&e"eeTgeUffddZVddddd=dde5eMdeedeeWdyeeeeeeffdeeeefdedeeeffddZXddZYeZdee[fddZ\e]dHdHdHdHdHdde5eeefde^d=deedxe5edyeeeeeeffdede_efddZ`e]dHdHdHdHdHdde5e%eeefeKfde^ddeedxe5edyeeeeeeffdede_e%eeKffddZ`d=de4ddddee5eeefe5e%eeefeKffdedeedxe5edyeeeeeeffdedee_ee_e%eeKfffddZ`dxe5efddZade5eeefde5edHe_efdedede_ef ddZbdddZce-ifdededede ddddUeeeefe fd eeefdxeee5efdeee5efdeee5efd eeefdedWeddfdd„Zededede5eddfddDŽZfeZddeegde_egfddʄZhe4d˜deeeifde5eddfdd΄Zjekdxeee5efdeee5efde5ede%edHffddфZle4e dҜdeeeifde5edeeefddfddՄZme4d˜de5edeIfddׄZne4d˜deIde5eddfddڄZoZpS)rjaA text-processing pipeline. Usually you'll load this once per process, and pass the instance around your application. Defaults (class): Settings, data and factory methods for creating the `nlp` object and processing pipeline. lang (str): Two-letter ISO 639-1 or three-letter ISO 639-3 language codes, such as 'en' and 'eng'. DOCS: https://spacy.io/api/language Nryerror FactoryMeta _factory_metaTi@Bi) max_lengthmetarxcreate_vectors batch_sizervrrrxrrIrric Kspddlm}|tjjt|j|_ t ||_ d|_ d|_ i|_i|_t|ts<|durFz%Language.disabled..rattrrrrBr%E926rrnamesrgrrhr>zLanguage.disabledcCst|j}t|S)z_Get names of all available factories. RETURNS (List[str]): The factory names. )r factoriesrrBrrgrgrh factory_namesIszLanguage.factory_namescCst|jtjjdddS)zoGet all (name, component) tuples in the pipeline, including the currently disabled components. rrr)rBrr%rrrrgrgrhrRszLanguage.componentscC&dd|jD}t|tjjdddS)zGet the names of the available pipeline components. Includes all active and inactive pipeline components. RETURNS (List[str]): List of component name strings, in order. cSg|]\}}|qSrgrgrrrrgrgrhrbz,Language.component_names..rrrrrrgrgrhr[szLanguage.component_namescr)zThe processing pipeline consisting of (name, component) tuples. The components are called on the Doc in order as it passes through the pipeline. RETURNS (List[Tuple[str, Callable[[Doc], Doc]]]): The pipeline. cs"g|] \}}|jvr||fqSrgr)rnprrgrhrms"z%Language.pipeline..rrrr)rpipesrgrrhrerzLanguage.pipelinecCr)zGet names of available active pipeline components. RETURNS (List[str]): List of component name strings, in order. cSrrgrgrrgrgrhrvrz'Language.pipe_names..rrr)rrBr%rrrrgrgrhrpszLanguage.pipe_namescCs,i}|jD] \}}||j||<qt|S)zGet the component factories for the available pipeline components. RETURNS (Dict[str, str]): Factory names, keyed by component names. )rrrrA)rrrpipergrgrhpipe_factoriesyszLanguage.pipe_factoriescCsJi}|jD]\}}t|dr|jdurqt|dr t|j||<qt|S)zGet the labels set by the pipeline components, if available (if the component exposes a labels property and the labels are not hidden). RETURNS (Dict[str, List[str]]): Labels keyed by component name. hide_labelsTr)rhasattrrrrrA)rrrrrgrgrhrs zLanguage.pipe_labelsrcCs||}|tjvp|tjvS)z=RETURNS (bool): Whether a factory of that name is registered.)get_factory_namerFrrr internal_namergrgrh has_factorys zLanguage.has_factorycCs|jdur|S|jd|S)zGet the internal factory name based on the language subclass. name (str): The factory name. RETURNS (str): The internal factory name. N.)ry)rrrgrgrhrs zLanguage.get_factory_namecCsF||}||jvr|j|S||jvr|j|Sttjjd|d)zGet the meta information for a given factory name. name (str): The component factory name. RETURNS (FactoryMeta): The meta for the given factory name. rrr)rrrr%E967rrrgrgrhget_factory_metas     zLanguage.get_factory_metacCs||j||<dS)zSet the meta information for a given factory name. name (str): The component factory name. value (FactoryMeta): The meta to set. N)rr)rrrrgrgrhset_factory_metaszLanguage.set_factory_metacCs(||jvrttjjd|d|j|S)zGet the meta information for a given component name. name (str): The component name. RETURNS (FactoryMeta): The meta for the given component name. componentr)rrr%rrrrrgrgrhrs  zLanguage.get_pipe_metacCs*||jvrttjj|d|j|}|S)zGet the config used to create a pipeline component. name (str): The component name. RETURNS (Config): The config used to create the pipeline component. r)rrr%E960r)rrrrgrgrhrs  zLanguage.get_pipe_configF)rassignsrequires retokenizesrfuncrrrrrr c sttsttjjdddvrttjjdtts/tjjdt d}t|dt dt ffd d } |d urH| |S| S) a~Register a new pipeline component factory. Can be used as a decorator on a function or classmethod, or called as a function with the factory provided as the func keyword argument. To create a component and add it to the pipeline, you can use nlp.add_pipe(name). name (str): The name of the component factory. default_config (Dict[str, Any]): Default configuration, describing the default values of the factory arguments. assigns (Iterable[str]): Doc/Token attributes assigned by this component, e.g. "token.ent_id". Used for pipeline analysis. requires (Iterable[str]): Doc/Token attributes required by this component, e.g. "token.ent_id". Used for pipeline analysis. retokenizes (bool): Whether the component changes the tokenization. Used for pipeline analysis. default_score_weights (Dict[str, Optional[float]]): The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to 1.0 per component and will be combined and normalized for the whole pipeline. If None, the score won't be shown in the logs or be weighted. func (Optional[Callable]): Factory function if not used as a decorator. DOCS: https://spacy.io/api/language#factory r decoratorrrzdefault configstylercfg_type factory_funcric s}|tjvr#tj|}t||s#tjj||d}t |t |}d|vs0d|vr9t tj jdtjj ||dt tttd}|ttjtjd_|S)N)rr new_funcrkrr)r )rrrrscoresrrr)rrFrrr# is_same_funcr%E004rr get_arg_namesE964registerrr0rrrrArE957)rr existing_funcerr arg_names factory_metarrrrrrrrgrh add_factorys4        z%Language.factory..add_factoryN) rrbrr%E963rE853rcE962rr ) rrrrrrrr rrrgrrhrs #  ")zLanguage.factoryrrrr .csdurttsttjjdddvrttjjddur%ntdt dt ffdd }durC|S|S) aRegister a new pipeline component. Can be used for stateless function components that don't require a separate factory. Can be used as a decorator on a function or classmethod, or called as a function with the factory provided as the func keyword argument. To create a component and add it to the pipeline, you can use nlp.add_pipe(name). name (str): The name of the component factory. assigns (Iterable[str]): Doc/Token attributes assigned by this component, e.g. "token.ent_id". Used for pipeline analysis. requires (Iterable[str]): Doc/Token attributes required by this component, e.g. "token.ent_id". Used for pipeline analysis. retokenizes (bool): Whether the component changes the tokenization. Used for pipeline analysis. func (Optional[Callable[[Doc], Doc]): Factory function if not used as a decorator. DOCS: https://spacy.io/api/language#component Nrr rrcomponent_funcricsttrttjjddtdtffdd }}|t j vrAt j |}|j }|r7dd|Ddnd}t |rA|}j|d S) NrrricsSrrgrkrr"rgrhrQz?Language.component..add_component..factory_funccSsg|]}|jqSrg) cell_contents)rcrgrgrhr_sz=Language.component..add_component..rr!)rrrr%E965rrb PipeCallablerrFrr __closure__r#rr)r"rrrclosurewrappedrrcomponent_namer rrrr$rh add_componentMs$     z)Language.component..add_component) rrbrr%rrrr#get_object_namer)r )rrrrrr r/rgr-rhr+s "zLanguage.component)rrrr)rprettyrr1cCs t||d}|rt||d|S)aAnalyze the current pipeline components, print a summary of what they assign or require and check that all requirements are met. keys (List[str]): The meta values to display in the table. Corresponds to values in FactoryMeta, defined by @Language.factory decorator. pretty (bool): Pretty-print the results. RETURNS (dict): The data. )r)r.r/)rrr1analysisrgrgrhr.ps  zLanguage.analyze_pipescCs6|jD] \}}||kr|Sqttjj||jd)zGet a pipeline component for a given component name. name (str): Name of pipeline component to get. RETURNS (callable): The pipeline component. DOCS: https://spacy.io/api/language#get_pipe ropts)rKeyErrorr%E001rr)rrrrrgrgrhget_pipes zLanguage.get_pipe)rO raw_configvalidate factory_namerOr8r9c Cs^|dur|n|}t|tstjjd|t|d}t|t|s*ttj j|d| |sFtj j|d |j dt||jd}t|||}|jrVt|j|}||}|tjvrb|}||d|d |i}||i} tj| |d } tjd | |i|d d } t| } || d <| d d| d d| dd|r| |} | |j|<| |S)aCreate a pipeline component. Mostly used internally. To create and add a component to the pipeline, you can use nlp.add_pipe. factory_name (str): Name of component factory. name (Optional[str]): Optional name to assign to component instance. Defaults to factory name if not set. config (Dict[str, Any]): Config parameters to use for this component. Will be merged with default config, if available. raw_config (Optional[Config]): Internals: the non-interpolated config. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#create_pipe NrOr r, create_pipe)rr4methodry lang_coder#z @factories)r9cfgrrkr)rrcr%r rrrrrrrE002joinrr#r0ryrrrrrrFrrfillpopr) rr:rrOr8r9rrrr?resolvedfilledrgrgrhr<sD            zLanguage.create_pipe source_namesourcecCst|tsttjj|t|d|jj|jjkr#t t j j|d||j vrBttjj||jdd|jdd|j d||}t|drO||_|j}t|d|}||j|<|jj|jjkrx|jjD] }|jj|qn||d fS) auCreate a pipeline component by copying it from an existing model. source_name (str): Name of the component in the source pipeline. source (Language): The source nlp object to copy from. name (str): Optional alternative name to use in current pipeline. RETURNS (Tuple[Callable[[Doc], Doc], str]): The component and its factory name. )rrGrryrrr;)rmodelr4rr)rrjrr%E945rrrvrwarningswarnr&W113rr5E944rrAr7rrrO interpolater# copy_configrstringsadd)rrFrGrr source_configrsrgrgrhcreate_pipe_from_sources,         z Language.create_pipe_from_source)beforeafterfirstlastrGrOr8r9rUrVrWrXcCst|tst|} tjj| |d} t| |dur|n|}||jvr-ttjj||jdd|vr>t t j j| dd|durM|j|||d\} }n |j|||| | d} |||||}|||j|<|j||| f|| S)aAdd a component to the processing pipeline. Valid components are callables that take a `Doc` object, modify it and return it. Only one of before/after/first/last can be set. Default behaviour is "last". factory_name (str): Name of the component factory. name (str): Name of pipeline component. Overwrites existing component.name attribute if available. If no name is set and the component exposes no name attribute, component.__name__ is used. An error is raised if a name already exists in the pipeline. before (Union[str, int]): Name or index of the component to insert new component directly before. after (Union[str, int]): Name or index of the component to insert new component directly after. first (bool): If True, insert component first in the pipeline. last (bool): If True, insert component last in the pipeline. source (Language): Optional loaded nlp object to copy the pipeline component from. config (Dict[str, Any]): Config parameters to use for this component. Will be merged with default config, if available. raw_config (Optional[Config]): Internals: the non-interpolated config. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#add_pipe rrNr3r)name_in_configr)rrOr8r9)rrbreprr%E966rrrE007rJrKr&W119rCrTr<_get_pipe_indexrrrinsert_link_components)rr:rrUrVrWrXrGrOr8r9bad_valrpipe_component pipe_indexrgrgrhadd_pipes2 (  zLanguage.add_pipecCs~||||d}tdd||||fDdkr!ttjj||jd|s/tdd|||fDs4t|jS|r8dSt |t rS||jvrMttj j||jd|j |St |t rp||jvrhttj j||jd|j |d St |tkr|t|jks|dkrtjjd ||jd }t||St |tkr|t|jks|dkrtjjd ||jd }t||d Sttjj||jd) aDetermine where to insert a pipeline component based on the before/ after/first/last values. before (str): Name or index of the component to insert directly before. after (str): Name or index of component to insert directly after. first (bool): If True, insert component first in the pipeline. last (bool): If True, insert component last in the pipeline. RETURNS (int): The index of the new pipeline component. )rUrVrWrXcs|]}|duVqdSrrg)rargrgrgrh Wz+Language._get_pipe_index..)argsr4csrfrrg)rrrgrgrhrh[rirr3r rU)diridxr4rV)sumrr%E006rranyrrrrbr6indexrrdE959)rrUrVrWrXall_argsrrgrgrhr_FsH        zLanguage._get_pipe_indexcCs ||jvS)a Check if a component name is present in the pipeline. Equivalent to `name in nlp.pipe_names`. name (str): Name of the component. RETURNS (bool): Whether a component of the name exists in the pipeline. DOCS: https://spacy.io/api/language#has_pipe )rrrgrgrhhas_pipe}s zLanguage.has_pipe)rOr9cCs||jvrttjj||jdt|dr#tjjt||d}t||j |}| |t |j r:|t |j krC|j ||||dS|j |||||dS)aBReplace a component in the pipeline. name (str): Name of the component to replace. factory_name (str): Factory name of replacement component. config (Optional[Dict[str, Any]]): Config parameters to use for this component. Will be merged with default config, if available. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. RETURNS (Callable[[Doc], Doc]): The new pipeline component. DOCS: https://spacy.io/api/language#replace_pipe r3__call__rY)rrOr9)rrUrOr9)rrr%r6rrrE968r[rq remove_piperrre)rrr:rOr9rrdrgrgrh replace_pipes$    zLanguage.replace_pipeold_namenew_namecCs||jvrttjj||jd||jvr ttjj||jd|j|}||j|df|j|<|j ||j|<|j ||j |<||j ddvr`|j dd |}||j dd|<| dS)zRename a pipeline component. old_name (str): Name of the component to rename. new_name (str): New name of the component. DOCS: https://spacy.io/api/language#rename_pipe r3r initializerN) rrr%r6rr]rqrrrCrrra)rryrziinit_cfgrgrgrh rename_pipes    zLanguage.rename_pipecCs||jvrttjj||jd|j|j|}|j||j ||j di|d||j ddvrD|j dd|||j vrO|j|||S)aRemove a component from the pipeline. name (str): Name of the component to remove. RETURNS (Tuple[str, Callable[[Doc], Doc]]): A `(name, component)` tuple of the removed component. DOCS: https://spacy.io/api/language#remove_pipe r3_sourced_vectors_hashesNr{r)rrr%r6rrrCrqrrrrrrrremovera)rrremovedrgrgrhrws     zLanguage.remove_pipecCs0||jvrttjj||jd|j|dS)aDisable a pipeline component. The component will still exist on the nlp object, but it won't be run as part of the pipeline. Does nothing if the component is already disabled. name (str): The name of the component to disable. r3N)rrr%r6rrrQrrgrgrh disable_pipes zLanguage.disable_pipecCs>||jvrttjj||jd||jvr|j|dSdS)zEnable a previously disabled pipeline component so it's run as part of the pipeline. Does nothing if the component is already enabled. name (str): The name of the component to enable. r3N)rrr%r6rrrrrrgrgrh enable_pipes  zLanguage.enable_pipe)disable component_cfgtextrrc Cs ||}|dur i}|jD]u\}}||vrqt|ds(ttjjt||d|j}t|dr4| }z||fi| |i}Wn/t yX}z ttj j|d|d}~wt yq}z||||g|WYd}~nd}~wwt|tsttjj|t|dq|S)aApply the pipeline to some text. The text can span multiple sentences, and can contain arbitrary whitespace. Alignment into the original string is preserved. text (Union[str, Doc]): If `str`, the text to be processed. If `Doc`, the doc will be passed directly to the pipeline, skipping `Language.make_doc`. disable (List[str]): Names of the pipeline components to disable. component_cfg (Dict[str, dict]): An optional dictionary with extra keyword arguments for specific components. RETURNS (Doc): A container for accessing the annotations. DOCS: https://spacy.io/api/language#call NrurYget_error_handlerr)r returned_type) _ensure_docrrrr%E003rrrrrr5E109 Exceptionrr9E005) rrrrdocrproc error_handlerergrgrhrus0    zLanguage.__call__ DisabledPipescGs@ttjtt|dkrt|dttfr|d}|j |dS)aVDisable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. Otherwise, a DisabledPipes object is returned, that has a `.restore()` method you can use to undo your changes. This method has been deprecated since 3.0 r r)r) rJrKr&W096DeprecationWarningrrrtuple select_pipesrrgrgrh disable_pipes's zLanguage.disable_pipes)renablercsdur |dur ttjt|tr|g}durAttr!gfddjD}|dur?||kr?ttjj|jd|}|dusGJfdd|D}t|S)aDisable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. Otherwise, a DisabledPipes object is returned, that has a `.restore()` method you can use to undo your changes. disable (str or iterable): The name(s) of the pipes to disable enable (str or iterable): The name(s) of the pipes to enable - all others will be disabled DOCS: https://spacy.io/api/language#select_pipes Ncg|]}|vr|qSrgrg)rrrrgrhrKz)Language.select_pipes..)rrrcsg|] }|jvr|qSrgr)rdrrgrhrWs) rr%E991rrbrE992rr)rrr to_disablerg)rrrhr4s$     zLanguage.select_pipescCs2t||jkrttjjt||jd||S)z{Turn a text into a Doc object. text (str): The text to process. RETURNS (Doc): The processed doc. )lengthr)rrrr%E088rr)rrrgrgrhmake_docZs  zLanguage.make_docdoc_likecCsRt|tr|St|tr||St|trt|j|Sttj j t |d)zCreate a Doc if need be, or raise an error if the input is not a Doc, string, or a byte array (generated by Doc.to_bytes()).)r) rr9rbrbytesrv from_bytesrr%E1041rr)rrrgrgrhrfs    zLanguage._ensure_doccontextcCs||}||_|S)z>Call _ensure_doc to generate a Doc and set its context object.)r_context)rrrrrgrgrh_ensure_doc_with_contextqs z!Language._ensure_doc_with_contextg)dropsgdlossesrexclude annotatesexamplesrrrrrrc Cs|dur ttj|duri}t|trt|dkr|St|dt|}|dur6|jdur3| |_|j}|dur.)rrrr)rr%E989rrrr<_copy_examplesrcreate_optimizer enumeraterrrrrrr"TrainableComponent is_trainablerH finish_updateziprCrr_replace_numpy_floats)rrrrrrrrr pipe_kwargsr|rrrrrgrgrhrysZ           r)rrrrc s|duri}t|trt|dkr|St|d|dur)|jdur&||_|j}t|j}t||dur9i}ifdd}|j |_ |j |_ |j |_ |D] \}} ||vs\t | ds]qOi| j |f||d||iqOD] \} \} } || | | qt|S)a|Make a "rehearsal" update to the models in the pipeline, to prevent forgetting. Rehearsal updates run an initial copy of the model over some data, and update the model so its current predictions are more like the initial ones. This is useful for keeping a pretrained model on-track, even if you're updating it with a smaller set of examples. examples (Iterable[Example]): A batch of `Example` objects. sgd (Optional[Optimizer]): An optimizer. component_cfg (Dict[str, Dict]): Config parameters for specific pipeline components, keyed by component name. exclude (Iterable[str]): Names of components that shouldn't be updated. RETURNS (dict): Results from the update. EXAMPLE: >>> raw_text_batches = minibatch(raw_texts) >>> for labelled_batch in minibatch(examples): >>> nlp.update(labelled_batch) >>> raw_batch = [Example.from_dict(nlp.make_doc(text), {}) for text in next(raw_text_batches)] >>> nlp.rehearse(raw_batch) DOCS: https://spacy.io/api/language#rehearse NrLanguage.rehearsecs||f|<||fSrrg)keyWdWgradsrgrh get_gradss z$Language.rehearse..get_gradsrehearser)rrrr<rrrrandomshuffle learn_rateb1b2rrritems) rrrrrrrrrrrrrrgrrhrs@        rr get_examplescCsttjt|j||dS)Nr)rJrKr&W089rr{)rrrrgrgrhbegin_trainingszLanguage.begin_trainingcs|durtjdt|jgddfdd}t|ds,tjjdt |d }t ||j }t j|d td }|d }|durF||zt||d |d|ddWntyhttjj|ddw|jjjddkr|t}|jj|t|jd rt|jj|dddd}|jj|fd|i||jD]&\} } t| tjr|d| i} t| j| d| d} | j|fd|i| q|d} | rt j| td } t || ||!||_"|dur||_"n |j"dur|#|_"|d}|dur|||j"S)aInitialize the pipe for training, using data examples if available. get_examples (Callable[[], Iterable[Example]]): Optional function that returns gold-standard Example objects. sgd (Optional[Optimizer]): An optimizer to use for updates. If not provided, will be created using the .create_optimizer() method. RETURNS (thinc.api.Optimizer): The optimizer. DOCS: https://spacy.io/api/language#initialize NzUNo 'get_examples' callback provided to 'Language.initialize', creating dummy examples)xyz)wordscstigSr)r; from_dictrgrrgrhr#sz)Language.initialize..get_examplesruLanguage.initialize)r=objr{)schema before_init vocab_datar}r)datar}r)rr r)sectionrrkr pretraining after_init)$r#r{r|r9rvrr%E930rr TypeErrorrOrNrFrr2r>IOErrorE884rshaperto_opsrr5r{rrr"InitializableComponentrr4r=rarr)rrrrrOIrops tok_settingsrr p_settings pretrain_cfgPrrgrrhr{ sp           rcCszt}|jjjddkr|jj||jD]\}}t|dr%t|j|_ q|dur0||_ |j S|j dur:| |_ |j S)a Continue training a pretrained model. Create and return an optimizer, and initialize "rehearsal" for any pipeline component that has a .rehearse() method. Rehearsal is used to prevent models from "forgetting" their initialized "knowledge". To perform rehearsal, collect samples of text you want the models to retain performance on, and call nlp.rehearse() with a batch of Example objects. RETURNS (Optimizer): The optimizer. DOCS: https://spacy.io/api/language#resume_training r _rehearsal_modelN) rrvrrrrrrrHrrr)rrrrrrgrgrhresume_trainingYs    zLanguage.resume_trainingrcCs.||_|jD]\}}t|dr||qdS)aSet an error handler object for all the components in the pipeline that implement a set_error_handler function. error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], NoReturn]): Function that deals with a failing batch of documents. This callable function should take in the component's name, the component itself, the offending batch of documents, and the exception that was thrown. DOCS: https://spacy.io/api/language#set_error_handler set_error_handlerN)rrrr)rrrrrgrgrhrrs    zLanguage.set_error_handler)rscorerr scorer_cfg per_componentrrrcCst|}t|dt|}|dur|j}|duri}|dur i}|dur5t|}|d|td i|}t}|D] } || j j q:|j dd|D||d} t || D]\} } | | _ qVt} |j||d} tdd|D}|| || d <t| S) aEvaluate a model's pipeline components. examples (Iterable[Example]): `Example` objects. batch_size (Optional[int]): Batch size to use. scorer (Optional[Scorer]): Scorer to use. If not passed in, a new one will be created. component_cfg (dict): An optional dictionary with extra keyword arguments for specific components. scorer_cfg (dict): An optional dictionary with extra keyword arguments for the scorer. per_component (bool): Whether to return the scores keyed by component name. Defaults to False. RETURNS (Scorer): The scorer containing the evaluation results. DOCS: https://spacy.io/api/language#evaluate Language.evaluateNrkcsrrrrrgrgrhrhrz$Language.evaluate..)rr)rcss|]}t|jVqdSr)rrrrgrgrhrhspeedrg)rr<rrrcrr6timerr referencerrrrscorernr)rrrrrrrr start_timerdocsrend_timeresultsn_wordsrgrgrhevaluates8   rcCs d|jddi}t|dS)zCCreate an optimizer, usually using the [training.optimizer] config. optimizerr)rOrFr)r subconfigrgrgrhrszLanguage.create_optimizerparamsc #s~sdVdSfdd|jD}|D]}zt|Wqty%YqwdV|D]}zt|Wq+ty<Yq+wdS)aReplace weights of models in the pipeline with those provided in the params dictionary. Can be used as a contextmanager, in which case, models go back to their original weights after the block. params (dict): A dictionary of parameters keyed by model ID. EXAMPLE: >>> with nlp.use_params(optimizer.averages): >>> nlp.to_disk("/tmp/checkpoint") DOCS: https://spacy.io/api/language#use_params Ncs.g|]\}}t|drt|dr|qS) use_paramsrH)rr)rrrrrgrhrsz'Language.use_params..)rnext StopIteration)rrcontextsrrgrrhrs(      zLanguage.use_params) as_tuplesrrr n_processtextsr r cCdSrrgrr r rrrr rgrgrhr z Language.pipecCrrrgrrgrgrhrrr c#sl|r7ttttttftf|}fdd|D}j|||||d}|D] } | j} d| _| | fVq'dStttttf|}|dkrJt }|durPi}|durWj }g} j D]%\} } | |vreq\| | i}|d|tjt| | |jd}| |q\|dkr|rttj|| ||}nfd d|D}| D]}||}q|D]} | VqdS) aXProcess texts as a stream, and yield `Doc` objects in order. texts (Iterable[Union[str, Doc]]): A sequence of texts or docs to process. as_tuples (bool): If set to True, inputs should be a sequence of (text, context) tuples. Output will then be a sequence of (doc, context) tuples. Defaults to False. batch_size (Optional[int]): The number of texts to buffer. disable (List[str]): Names of the pipeline components to disable. component_cfg (Dict[str, Dict]): An optional dictionary with extra keyword arguments for specific components. n_process (int): Number of processors to process texts. If -1, set `multiprocessing.cpu_count()`. YIELDS (Doc): Documents in the order of the original text. DOCS: https://spacy.io/api/language#pipe c3s |] \}}||VqdSr)r)rrrrrgrhrhs z Language.pipe..)rrr rNr)rrrrr c3s|]}|VqdSr)r)rrrrgrhrhQr)rr rrrbr9rKrrmp cpu_countrrrr functoolspartialrCrr_has_gpu_modelrJrKr&W114_multiprocessing_pipe)rr r rrrr docs_with_contextsrrrrrrrfrrgrrhrsd        cCs\|jD](\}}t|do|j}||vs|sqt|dr+t|jdr+t|jjtr+dSqdS)NrrHrTF)rrrrHrrr)rrrrrrgrgrhrWs $zLanguage._has_gpu_modelrc #sVdttttfdtttttftffdd}||}t|\}}ddt |D}t ddt |D\} } t ||} t | ||d} | | fd dt || D} | D]}|q_| D]}|qhtd d t| D}z{tt ||d D]>\}\}\}}}|durtj|}||_|Vn|durt|}dddttjj|d ||dkr| qW|D] }| t!|q| D]}|q| D]}|"qt#dd | Dst$%t&j'dSdS|D] }| t!|q| D]}|q| D]}|"qt#dd | Ds*t$%t&j'ww)Nr ricssB|D]}t|tr|tt|jfVq|ttdfVqdSr)rr9to_bytesrrKr)r rrgrgrh prepare_inputis  z5Language._multiprocessing_pipe..prepare_inputcSsg|]}tqSrg)rQueuerrrgrgrhryrz2Language._multiprocessing_pipe..cSsg|]}tdqS)F)rPiperrgrgrhr|s) chunk_sizec s.g|]\}}tjtj||tfdqS))targetrk)rProcess _apply_pipesrr: get_state)rrchschrrrgrhrs css|]}|VqdSr)recv)rr(rgrgrhrhs z1Language._multiprocessing_pipe..r rrcss|]}|jdkVqdS)rN)exitcode)rrrgrgrhrhr)(r rrbr9rrrK itertoolsteerangerr# minibatch_Sendersendstartcloser from_iterablerrrvrrr msgpack_loadsrrr%E871rstepput_WORK_DONE_SENTINELrAallrJrKr&W127)rr rr rrserialized_texts_with_ctx raw_textstexts_qbytedocs_recv_chbytedocs_send_ch batch_textssenderprocsrtx byte_tuplesr|rbyte_docr byte_errorrrqrrgr'rhrbs~                   zLanguage._multiprocessing_pipecCst|jD] \}}t|dr||_qt|jD]"\}\}}t|tjr7i|_|j|ddD] \}}||q-qdS)zmRegister 'listeners' within pipeline components, to allow them to effectively share weights. rr N) rrrrrr"ListenedToComponent listener_mapfind_listeners)rrrr|name1proc1name2proc2rgrgrhras   zLanguage._link_components)rvrrrr auto_fillr9rOc' sd|r t|jtd|}d|vrttjj|dd|dvr'ddi|dd<|dd} | durJ| |j krJttj j|dd|j t |d |j |dd<t |}|d i} |d d} i|d <|rqtj||td } n|} | | d <| |d <| dur| | d <| |d <tj| d|td } | d }| d}| d}| d}| d}|}|dur||}t|trt||r||urttjjt|dt|||||d}|dur||}t||sttjjdt|d| js| n| }|d i}i}i}d}|ddD] }||vr d|}ttjj||dt ||}t| d |}|vrd|vrJd|vrJtjj||d}t|d|vr`|d}|j |||||dqd|vsgJ|duru|j!j"ddgd}|d} | |vrt j#| |j!dgd|| <|d |}!d!}"d"|vr|| $|| j%D]\}#}$|!t&|$d#gvr|| '|#|!|d"d$}"qt()t(j*d%d&d'|j |!|| |d(Wdn 1swY| |vrt+|| j!j,j"dgd|| <d)|j-vri|j-d)<|| |j-d)|<|"r|| =q|dur|j!.|t|t/r(|g}t|t/r1|g}tt/r:gt0|t0t1krc|dd*g}%t2|%rct3|%4|sct(5t6j7j||%d+|8t9h||dd,g||dd}&t3fd-d.|&D|_:|dd/|_;|r| n||_<|dur||}t||sttjjd0t|d|S)1a5Create the nlp object from a loaded config. Will set up the tokenizer and language data, add pipeline components etc. If no config is provided, the default config of the given language is used. config (Dict[str, Any] / Config): The loaded config. vocab (Vocab): A Vocab object. If True, a vocab is created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. meta (Dict[str, Any]): Meta overrides for nlp.meta. auto_fill (bool): Automatically fill in missing values in config based on defaults and function argument annotations. validate (bool): Validate the component config and arguments against the types expected by the factory. RETURNS (Language): The initialized Language class. DOCS: https://spacy.io/api/language#from_config rMrkrrz@vectorszspacy.Vectors.v1ryN) bad_lang_coder>ryrr)r9rrbefore_creationafter_creationafter_pipeline_creation)r)rvrxrrcreation)rrrr;r3rrG)rrO)rrOr9r8r}rPr)rvrrFreplace_listenerslistening_componentsTignorez\[W113\])message)rGrrenabled)rrZrc3s|] }|vr|VqdSrrg)rrrUrgrhrhsz'Language.from_config..rpipeline_creation)=rrr@rrr%E985rrryE958r#r0rOrCrFrBr1rr3rr issubclassE943rGE942is_interpolatedrNrArE956E984rervr load_modelrargetattrrVrJcatch_warningsfilterwarningshashrrrrbidr?rreissubsetrKr&W123_resolve_component_statusrrrrO)'rrOrvrrrrrOr9 config_lang orig_pipelineorig_pretrainingrE resolved_nlprxrrQrRrSlang_clsrk interpolatedr source_nlpssource_nlp_vectors_hashesvocab_brr4pipe_cfgr8rrrHrFlisteners_replacedrrrZdisabled_pipesrgrUrh from_configs4#                                   zLanguage.from_config tok2vec_namer listenersc Cs||jvrtjj|||d|jd}t|||jvr.tjj|||d|jd}t|||}||}t|t j sJttj j|t |d|j }|j|g}||} |j|} |r tjd|tt|t|krtjj|||t|d}t||D]?} zt| | Wntytjj||| d}t|w|d} d|jvr|jd} | |d| dd } t| | | q|D]E}|}|jd }|d urtt|j}|d kr||}n|d kr||||}n ttj j|dt!| j |||"||qd Sd S)aFind listener layers (connecting to a token-to-vector embedding component) of a given pipeline component model and replace them with a standalone copy of the token-to-vector layer. This can be useful when training a pipeline with components sourced from an existing pipeline: if multiple components (e.g. tagger, parser, NER) listen to the same tok2vec component, but some of them are frozen and not updated, their performance may degrade significantly as the tok2vec component is updated with new data. To prevent this, listeners can be replaced with a standalone tok2vec layer that is owned by the component and doesn't change if the component isn't updated. tok2vec_name (str): Name of the token-to-vector component, typically "tok2vec" or "transformer". pipe_name (str): Name of pipeline component to replace listeners for. listeners (Iterable[str]): The paths to the listeners, relative to the component config, e.g. ["model.tok2vec"]. Typically, implementations will only connect to one tok2vec component, [model.tok2vec], but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a complete (!) list of the paths to all listener layers used by the model. DOCS: https://spacy.io/api/language#replace_listeners r;)tok2vecrunknownr4)rrz%Replacing listeners of component '%s')rr|paths n_listeners)rr|rrHreplace_listener_cfgr|replace_listenerNr ) num_params)#rr%E889rrArr7rrr"rHE888rrHrIrrr#r{r|rrE887 dot_to_objectr5E886attrsset_dot_to_objectcopyinspect signature parametersE1055replace_model_noderemove_listener)rrzrr{rr| tok2vec_cfg tok2vec_modelpipe_listenersrrv listener_path new_config replace_funclistener new_modelreplace_listener_funcrrgrgrhrVs                zLanguage.replace_listenersmemc cs|durt}t@}||j|g}t|jdr(|||j||jD]\}}t|dr?||||q+|VWddS1sNwYdS)aBegin a block where all resources allocated during the block will be freed at the end of it. If a resources was created within the memory zone block, accessing it outside the block is invalid. Behaviour of this invalid access is undefined. Memory zones should not be nested. The memory zone is helpful for services that need to process large volumes of text with a defined memory budget. Example ------- >>> with nlp.memory_zone(): ... for doc in nlp.pipe(texts): ... process_my_doc(doc) >>> # use_doc(doc) <-- Invalid: doc was allocated in the memory zone N memory_zone) rr enter_contextrvrrrrr)rrstackr rrrgrgrhr0s  "zLanguage.memory_zonerUrcst|}i}fdd|d<fdd|d<fdd|d<jD]\}}|vr+q"t|ds1q"|fd d||<q"fd d|d <t||d S) aaSave the current state to a directory. If a model is loaded, this will include the model. path (str / Path): Path to a directory, which will be created if it doesn't exist. exclude (Iterable[str]): Names of components or serialization fields to exclude. DOCS: https://spacy.io/api/language#to_disk cjj|dgdSNrvrU)rto_diskrrrgrh^z"Language.to_disk..rcst|tjSr)r write_jsonrrrrrgrhras  meta.jsoncs j|Sr)rOrrrrgrhrd config.cfgrcS|j|dgdSr)rrrrgrgrhrjcjj|dSNrU)rvrrrrrgrhrkrrvN)r# ensure_pathrrr)rrr serializersrrrgrrhrPs  zLanguage.to_diskrcsnt|tr|g}|}r3ttrghfdd|D|}tt|@r3ttjj|dt|S)afDerives whether (1) `disable` and `enable` values are consistent and (2) resolves those to a single set of disabled components. Raises an error in case of inconsistency. disable (Union[str, Iterable[str]]): Name(s) of component(s) or serialization fields to disable. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. pipe_names (Iterable[str]): Names of all pipeline components. RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t. specified includes and excludes. crrgrg)rrrrgrhrrz6Language._resolve_component_status..)rr) rrbrrerr%E1042rr)rrrrrgrrhrlns  z"Language._resolve_component_status)r overridesrc sdtddffdd }dtddffdd }t|}i}t|dr/fd d |d<||d <||d <fd d |d<jD]\}}|vrKqBt|dsQqB|fdd ||<qB|d skd vrktd gt|||_ S)aLoads state from a directory. Modifies the object in place and returns it. If the saved `Language` object contains a model, the model will be loaded. path (str / Path): A path to a directory. exclude (Iterable[str]): Names of components or serialization fields to exclude. RETURNS (Language): The modified `Language` object. DOCS: https://spacy.io/api/language#from_disk rriNcs>|rt|}j||didjj_dSdSNrr) existsr read_jsonrrrrvrr)rrrrgrhdeserialize_metas   z,Language.from_disk..deserialize_metacs |rjj|ddSdSr)rrv from_disk)rrrgrhdeserialize_vocabsz-Language.from_disk..deserialize_vocabrcsjj|ddS)NF)rNr)rOrr)rrrgrhrrz$Language.from_disk..rrvcrr)rrrrrgrhrrrrcSrr)rrrgrgrhr) rr#rrrrrrrra) rrrrrr deserializersrrrg)rrrrhrs*  zLanguage.from_diskcsi}fdd|d<fdd|d<fdd|d<fdd|d <jD]\}}|vr/q&t|d s5q&|fd d||<q&t|S) aSerialize the current state to a binary string. exclude (Iterable[str]): Names of components or serialization fields to exclude. RETURNS (bytes): The serialized form of the `Language` object. DOCS: https://spacy.io/api/language#to_bytes csjjdSr)rvrrgrrgrhrz#Language.to_bytes..rvcsjjdgdSr)rrrgrrgrhrrrcsttjSr)r json_dumpsrrrgrrgrhrsrcs jSr)rOrrgrrgrhrs rrcSs|jdgdSr)r)rrgrgrhrr)rrr#r)rrrrrrgrrhrs  zLanguage.to_bytes bytes_datacsfdd}i}fdd|d<||d<fdd|d<fd d|d <jD]\}}|vr1q(t|d s7q(|fd d||<q(t||S) a!Load state from a binary string. bytes_data (bytes): The data to load from. exclude (Iterable[str]): Names of components or serialization fields to exclude. RETURNS (Language): The `Language` object. DOCS: https://spacy.io/api/language#from_bytes cs2t|}j||didjj_dSr)r json_loadsrrrrvrr)brrrgrhrs  z-Language.from_bytes..deserialize_metacsjj|ddS)NF)rN)rOrrrrgrhrsz%Language.from_bytes..rrcrr)rvrrrrgrhrrrvcrr)rrrrrgrhrrrrcSrr)r)rrrgrgrhr r)rrr#rra)rrrrrrrrgrrhrs  zLanguage.from_bytes)Tr)NNNN)rirriN)qr]r^r_r`rLrpryrrbrarrrAr%rrrr rrIboolrdr r r9rHrrpropertyrrsetterrrOrrrrr)rrrrrr classmethodrrrrrrrBr floatrrr.r7r<rTrer_rtrxr~rwrrrurrrrrrKrr;rrrrr{rrrrr6rrrrcrrr$rrrrrar?ryrVrrrr staticmethodrlrrr __classcell__rgrgrrhrjs      H %"           Y  G   I  *     H 7  +   + &      O  B   L  ;#   &   W  h     h h     &   9 meta_dictcCstddddt|S)NcSs t|tjSr)rnumpyfloatingvrgrgrhr rz'_replace_numpy_floats..cSst|Sr)rrrgrgrhr s)rrc)rrgrgrhr src@seZdZUdZeed<dZeeee fed<e Z e eed<e Z e eed<dZeed<e Ze eed <dZeeeeefed <dS) raDataclass containing information about a component and its defaults provided by the @Language.component or @Language.factory decorator. It's created whenever a component is defined and stored on the Language class for each component instance and factory instance. rNrrrFrrr)r]r^r_r`rbrarrr r rrr rrrrrrrgrgrgrhr s   rc@sDeZdZdZdedeeddfddZdd Zd d Z dd d Z dS)rz)Manager for temporary pipeline disabling.rkrriNcCs>||_||_|jD]}|j|q t|||jdSr)rkrrrrextend)rrkrrrgrgrhr$ s   zDisabledPipes.__init__cCs|Srrgrrgrgrh __enter__, r%zDisabledPipes.__enter__cGs |dSr)restore)rrkrgrgrh__exit__/ s zDisabledPipes.__exit__cCsF|jD]}||jjvrttjj|d|j|qg|dd<dS)zARestore the pipeline to its state when DisabledPipes was created.rN)rrkrrr%E008rrrrgrgrhr2 s  zDisabledPipes.restorer) r]r^r_r`rjrrbrrrrrgrgrgrhr! s rrcCsdd|DS)zMake a copy of a batch of examples, copying the predicted Doc as well. This is used in contexts where we need to take ownership of the examples so that they can be mutated, for instance during Language.evaluate and Language.update. cSsg|] }t|j|jqSrg)r;rrrrrgrgrhrA rz"_copy_examples..rg)rrgrgrhr; sr ensure_docr.underscore_statec st| z>|}t|tr||WdSfdd|D}|D]}||}q&dd|D}dgt|t|} || } Wn!tyeddt t fg} dgt|d} | | } Ynwz| | Wnt y||YdSwq) a9Worker for Language.pipe ensure_doc (Callable[[Union[str, Doc]], Doc]): Function to create Doc from text or raise an error if the input is neither a Doc nor a string. pipes (Iterable[Pipe]): The components to apply. receiver (multiprocessing.Connection): Pipe to receive text. Usually created by `multiprocessing.Pipe()` sender (multiprocessing.Connection): Pipe to send doc. Usually created by `multiprocessing.Pipe()` underscore_state (Tuple[dict, dict, dict]): The data in the Underscore class of the parent. TNc3s|] \}}||VqdSrrg)rrrrrgrhrhb s z_apply_pipes..cSsg|] }||jdfqSr)rr)rrrgrgrhrh sz _apply_pipes..)NNNr )r: load_staterr_WorkDoneSentinelr1rrr msgpack_dumps traceback format_excr/BrokenPipeError) rrreceiverr@rtexts_with_ctxrr byte_docspaddingr error_msgrgrrhr#D s<       r#c@sHeZdZdZdeedeejde ddfddZ d d d Z d d d Z dS)r.zAUtil for sending data to multiprocessing workers in Language.piperqueuesr riNcCs(t||_tt||_||_d|_dS)Nr)iterrrrr count)rrrr rgrgrhr s  z_Sender.__init__cCs4tt|jt|j|jD] \}}||qdS)z1Send chunk_size items from self.data to channels.N)r*islicerrrrr r6)ritemrFrgrgrhr/ s  z _Sender.sendcCs0|jd7_|j|jkrd|_|dSdS)znTell sender that comsumed one item. Data is sent to the workers after every chunk_size calls. r rN)rr r/rrgrgrhr5 s   z _Sender.stepr) r]r^r_r`r r rrrrdrr/r5rgrgrgrhr.} s r.c@s eZdZdS)rN)r]r^r_rgrgrgrhr sr)rrr*multiprocessingrrrrJ contextlibrrrr dataclassesrrrpathlibrtimeitr rtypingr r r r rrrrrrrrrrrrrr cymem.cymemr thinc.apirrrr thinc.utilrrr!r"r#compatr$errorsr%r&git_infor'lang.punctuationr(r)r*lang.tokenizer_exceptionsr+r,r}r- pipe_analysisr.r/r0schemasr1r2r3r4r5rr6rr8tokensr9tokens.underscorer:rr;r<training.initializer=r>r?r@rArBrCrDrErFrGrrHrvrIrJr)__file__parentDEFAULT_CONFIG_PATH load_configrDEFAULT_CONFIG_PRETRAIN_PATHrKrLrxr~rjrcrrrrrrbrr#r.rr7rgrgrgrhs    H         ,     9