o i}^@slddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z mZddlZddlmZmZmZmZmZddlmZddlmZdd lmZdd lmZmZdd lm Z dd l!m"Z"dd l#m$Z$m%Z%ddl&m'Z'm(Z(m)Z)ddlm*Z*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3dZ4dZ5e6e5dZ7ddZ8ddZ9Gddde3Z:ddZ;dS) N)islice)Path)AnyCallableDictIterableListOptionalUnion)ConfigCosineDistanceModel Optimizerset_dropout_rate)Floats2d)util)Errors) Candidate KnowledgeBase)Language)Scorer)DocSpan)Examplevalidate_examplesvalidate_get_examples)SimpleFrozenListregistry)Vocab)EntityLinker_v1)deserialize_config) TrainablePipeTz [model] @architectures = "spacy.EntityLinker.v2" [model.tok2vec] @architectures = "spacy.HashEmbedCNN.v2" pretrained_vectors = null width = 96 depth = 2 embed_size = 2000 window_size = 1 maxout_pieces = 3 subword_features = true modelcKstj|fdtjgi|S)Nnegative_labels)r score_links EntityLinkerNIL)exampleskwargsr+P/home/ubuntu/.local/lib/python3.10/site-packages/spacy/pipeline/entity_linker.pyentity_linker_score-sr-cCstSN)r-r+r+r+r,make_entity_linker_scorer1r/c$@sNeZdZdZdZ dHeedddedede d e e d e d e d e d e de eege efde ee ege e efde ee gefde dee de de deeddf"ddZde ede efddZde egeffddZdIdd Zddd!d"e ge efd#eedee egeffd$d%Zd&d'Zd(ddd)de ed*ed+eed,eee efdee eff d-d.Zde ed/efd0d1Zd2e e de!e fd3d4Z"d2e e d5e!e ddfd6d7Z#e$d8d9d:Z%e$d8d;d<Z&e'd8d=e(e e)fd>e e ddfd?d@Z*e'd8d=e(e e)fd>e e ddfdAdBZ+dddCdDdEZ,dFdGZ-dS)Jr'z^Pipeline component for named entity linking. DOCS: https://spacy.io/api/entitylinker r( entity_linkerN) overwritescorer thresholdvocabr$namelabels_discardn_sents incl_prior incl_contextentity_vector_lengthget_candidatesget_candidates_batchgenerate_empty_kbr2r3 use_gold_entscandidates_batch_sizer4returnc s|durd|krdksnttjjdd|d|_|_|_t|_|_ |_ |_ | _ | _ d| i_tdd_| j|_|_|_|_|dkr[ttjdttffd d }|_dS) aUInitialize an entity linker. vocab (Vocab): The shared vocabulary. model (thinc.api.Model): The Thinc Model powering the pipeline component. name (str): The component instance name, used to add entries to the losses during training. labels_discard (Iterable[str]): NER labels that will automatically get a "NIL" prediction. n_sents (int): The number of neighbouring sentences to take into account. incl_prior (bool): Whether or not to include prior probabilities from the KB in the model. incl_context (bool): Whether or not to include the local context in the model. entity_vector_length (int): Size of encoding vectors in the KB. get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. get_candidates_batch ( Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], Iterable[Candidate]] ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions. generate_empty_kb (Callable[[Vocab, int], KnowledgeBase]): Callable returning empty KnowledgeBase. scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another component must provide entity annotations. candidates_batch_size (int): Size of batches for entity candidate generation. threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold, prediction is discarded. If None, predictions are not filtered by any threshold. DOCS: https://spacy.io/api/entitylinker#init Nrr ) range_start range_endvaluer2F) normalizer)csfsSjs|fi|S|}dd|D}t||D]\}}||_q#|fi|S)Ncss|]}|jVqdSr. predicted.0egr+r+r, szFEntityLinker.__init__.._score_with_ents_set..)r? _ensure_entspipeziprG)r)r*docsrJdocr3selfr+r,_score_with_ents_sets  z3EntityLinker.__init__.._score_with_ents_set) ValueErrorrE1043formatr5r$r6listr7r8r9r:r<r=cfgr distancekbr?r@r4E1044rrr3)rRr5r$r6r7r8r9r:r;r<r=r>r2r3r?r@r4rSr+rQr,__init__=s61     zEntityLinker.__init__r)cCsB|js|Sg}|D]}|\}}|}||j_||q |S)zLIf use_gold_ents is true, set the gold entities to (a copy of) eg.predicted.)r?get_aligned_ents_and_nercopyrGentsappend)rRr) new_examplesrJr__new_egr+r+r,rLs  zEntityLinker._ensure_ents kb_loadercCs.t|sttjjt|d||j|_dS)ziDefine the KB of this pipe by providing a function that will create it using this object's vocab.)arg_typeN)callablerTrE885rVtyper5rZ)rRrdr+r+r,set_kbszEntityLinker.set_kbcCsP|jdurttjj|jdt|jdr$|jr&ttjj|jddSdS)Nr6is_empty) rZrTrE1018rVr6hasattrrkE139rRr+r+r, validate_kbs zEntityLinker.validate_kb)nlprd get_examplesrqc Cst|d|dur||||jj}g}g}|t|d}|D]}|j} || ||j j |q%t |dksJJt jj|jdt |dksZJt jj|jdtdd|D} | sv|d} | dd} d | _| f| _|j j||j j j|d d d | sg| _dSdS) aInitialize the pipe for training, using a representative set of data examples. get_examples (Callable[[], Iterable[Example]]): Function that returns a representative sample of gold-standard Example objects. nlp (Language): The current nlp object the component is part of. kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance. Note that providing this argument will overwrite all data accumulated in the current KB. Use this only when loading a KB as-such from file. DOCS: https://spacy.io/api/entitylinker#initialize EntityLinker.initializeN rrjcSg|]}|jqSr+r_)rIrPr+r+r, z+EntityLinker.initialize..r XXXfloat32dtype)XY)rrirprZr;rLrxr`r$opsalloc1flenrE923rVr6anylabel_r_ initializeasarray) rRrrrqrdnO doc_sample vector_sampler)rJrPhas_annotationsentr+r+r,rs4       rscCs:|D]}|jjD]}t||j|}|rdSqqdS)zCheck if a batch contains a learnable example. If one isn't present, then the update step needs to be skipped. TF)rGr_rWr<rZ)rRr)rJr candidatesr+r+r,batch_has_learnable_examples z(EntityLinker.batch_has_learnable_example)dropsgdlossesrrrc Cs||dur i}||jd|s|S||}t|d||s&|St|j|dd|D}|j|\}}|j ||d\}} || |durQ| |||j|7<|S)a.Learn from a batch of documents and gold-standard information, updating the pipe's model. Delegates to predict and get_loss. examples (Iterable[Example]): A batch of Example objects. drop (float): The dropout rate. sgd (thinc.api.Optimizer): The optimizer. losses (Dict[str, float]): Optional record of the loss during training. Updated using the component name as the key. RETURNS (Dict[str, float]): The updated losses dictionary. DOCS: https://spacy.io/api/entitylinker#update NrEntityLinker.updatecSrur+rFrHr+r+r,rwrxz'EntityLinker.update..)sentence_encodingsr)) rp setdefaultr6rLrrrr$ begin_updateget_loss finish_update) rRr)rrrrOr bp_contextlossd_scoresr+r+r,updates*      rrcCst|dg}d}g}|D]+}|jddd}|D]}||j} | r3|j| } || |||d7}qq |jjj |dd}||} |sT|jjj |j } d| fS| j |j krft j jd d d } t| |j| |}|jjj |j } || |<|j| |}|t|}t|| fS) NEntityLinker.get_lossr ENT_KB_IDT) as_stringr rzr{rzgold entities do not match upmethodmsg)r get_alignedget_matching_entsstartrZ get_vectorr`r$r asarray2falloc2fshaperE147rV RuntimeErrorrYget_gradrrfloat)rRr)rentity_encodingseidx keep_entsrJkb_idsrkb_identity_encodingselected_encodingsouterr gradientsrr+r+r,r,s<          rrOc svd}g}jjj}|s|St|tr|g}t|D]\}}t|dkr)qdd|jD}t dt|j j D]h}|j ||j fddt tD} t j dkrj jfdd| Dn fdd| D} tD])\} } t| dsJt | j} || d|| d f}|d|dkrdksJJjrtd|dj}tt|d|dj}||j}||j}|||}j|gd}|j}|j|}|d7}| jjvr|jqyt | | }|s |jqyt|dkr j d ur ||dj!qyt"#||$d d|D}j%s=|$d d|D}|}jr|$d d|D}|jj|dd}t|t|krht&t'j(j)ddd|*||||}|j+|j+kr~t,t'j-||||}|j d us|j kr||./j!nt0jqyq;qt||kst'j(j)ddd}t&||S)apApply the pipeline's model to a batch of docs, without modifying them. Returns the KB IDs for each entity in each doc, including NIL if there is no prediction. docs (Iterable[Doc]): The documents to predict. RETURNS (List[str]): The models prediction for each document. DOCS: https://spacy.io/api/entitylinker#predict rcSsg|]}|qSr+r+)rIsr+r+r,rwiz(EntityLinker.predict..cs g|] }|jjvr|qSr+)rr7rIidx ent_batchrRr+r,rwps r csg|]}|qSr+r+r)rr+r,rwxscsg|] }j|qSr+)r<rZrrr+r,rw{ssentsNcSrur+) prior_probrIcr+r+r,rwrxcSsg|]}dqS)rr+)rIrbr+r+r,rwrcSrur+) entity_vectorrr+r+r,rwrx)axispredictzvectors not of equal lengthrz$result variables not of equal length)1rpr$rxp isinstancer enumeraterrranger_r@rWr=rZrmindexr:maxr8minrendas_docrTlinalgnormrr7r`r(r4entity_randomshufflerr9rrrrVdotrrTE161argmaxitemr')rRrO entity_count final_kb_idsrirP sentencesent_idx valid_ent_idxbatch_candidatesjrr sent_indicesstart_sentence end_sentence start_token end_tokensent_docsentence_encodingsentence_encoding_t sentence_normr prior_probsscoresr entity_normsimsrr+rr,rTs            $          [zEntityLinker.predictrc Cstdd|D}|t|krttjj|t|dd}|jd}|D]}|jD]}||}|d7}|D] } | jdks>|rA|| _q5q)q$dS)aModify a batch of documents, using pre-computed scores. docs (Iterable[Doc]): The documents to modify. kb_ids (List[str]): The IDs to set, produced by EntityLinker.predict. DOCS: https://spacy.io/api/entitylinker#set_annotations cSsg|] }|jD]}|qqSr+rv)rIrPrr+r+r,rwsz0EntityLinker.set_annotations..)r_idsrr2r N) rrTrE148rVrXr_ ent_kb_id ent_kb_id_) rRrOr count_entsrr2rPrrtokenr+r+r,set_annotationss    zEntityLinker.set_annotationsexcludecsfi}tdrjdurfdd|d<fdd|d<jj|d<jj|d<t|S) zSerialize the pipe to a bytestring. exclude (Iterable[str]): String names of serialization fields to exclude. RETURNS (bytes): The serialized object. DOCS: https://spacy.io/api/entitylinker#to_bytes rXNcs tjSr.)srsly json_dumpsrXr+ror+r, z'EntityLinker.to_bytes..csjjdSNr)r5to_bytesr+rrRr+r,rr5rZr$)_validate_serialization_attrsrmrXrZrr$r)rRr serializer+rr,rs   zEntityLinker.to_bytescsxfdd}i}tdrjdurfdd|d<fdd|d<fd d|d <||d <t||S) zLoad the pipe from a bytestring. exclude (Iterable[str]): String names of serialization fields to exclude. RETURNS (TrainablePipe): The loaded object. DOCS: https://spacy.io/api/entitylinker#from_bytes cs.z j|WdStyttjdwr.)r$ from_bytesAttributeErrorrTrE149bror+r, load_models   z+EntityLinker.from_bytes..load_modelrXNcsjt|Sr.)rXrr json_loadsrror+r,rrxz)EntityLinker.from_bytes..cjj|dSr)r5rrrr+r,rrr5c j|Sr.)rZrrror+r,rrrZr$)rrmrXrr)rR bytes_datarr deserializer+rr,rs zEntityLinker.from_bytespathrcsXi}fdd|d<fdd|d<fdd|d<fdd|d <t||d S) zSerialize the pipe to disk. path (str / Path): Path to a directory. exclude (Iterable[str]): String names of serialization fields to exclude. DOCS: https://spacy.io/api/entitylinker#to_disk crr)r5to_diskprr+r,rrz&EntityLinker.to_disk..r5cst|jSr.)r write_jsonrXrror+r,rrrXcr r.)rZr rror+r,rrrZcr r.)r$r rror+r,rrr$N)rr )rRr rrr+rr,r  s  zEntityLinker.to_diskcs\fdd}i}fdd|d<fdd|d<fdd|d <||d <t||S) aNLoad the pipe from disk. Modifies the object in place and returns it. path (str / Path): Path to a directory. exclude (Iterable[str]): String names of serialization fields to exclude. RETURNS (EntityLinker): The modified EntityLinker object. DOCS: https://spacy.io/api/entitylinker#from_disk cs`z"|d}j|WdWdS1swYWdSty/ttjdw)Nrb)openr$rreadrrTrr)rinfileror+r,r*s &  z*EntityLinker.from_disk..load_modelcsjt|Sr.)rXrr"rror+r,r2rz(EntityLinker.from_disk..rXcrr)r5 from_diskrrr+r,r3rr5cr r.)rZrrror+r,r4rrZr$)rr)rRr rrr r+rr,rs zEntityLinker.from_disk)rrcKtr.NotImplementedError)rRr)rrconfigr+r+r,rehearse9r0zEntityLinker.rehearsecCrr.r)rRlabelr+r+r, add_label<r0zEntityLinker.add_label)r1)rAN).__name__ __module__ __qualname____doc__r(BACKWARD_OVERWRITEr-rr strrintboolrrrrr rr\rrLrirprrrrrrrrrrrrtuplerrrr rr rrrr+r+r+r,r'5s     `   6  /(z     r'cCs,|dkr td}|jStdtd|)Nmake_entity_linkerzspacy.pipeline.factorieszmodule z has no attribute ) importlib import_moduler&rr)r6moduler+r+r, __getattr__As r*)s@  $