o i9@s<ddlZddlZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z mZddlZddlZddlmZmZmZmZddlmZmZddlmZdd lmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/dZ0dZ1e2e1dZ3Gddde/Z4ddZ5dS)N)Counter)islice)AnyCallableDictIterableListOptionalTuplecast)ConfigModelNumpyOpsSequenceCategoricalCrossentropy)Floats2dInts2d)util)Errors)Language)Doc)Examplevalidate_examplesvalidate_get_examples)Vocab) EditTrees)validate_edit_tree)lemmatizer_score) TrainablePipez [model] @architectures = "spacy.Tagger.v2" [model.tok2vec] @architectures = "spacy.HashEmbedCNN.v2" pretrained_vectors = null width = 96 depth = 4 embed_size = 2000 window_size = 1 maxout_pieces = 3 subword_features = true modelc@seZdZdZ d?ddddedded ed ed eed e d e de dee fddZ de edeedeeeeffddZde edeefddZddZddZddZde efd d!Zedee d"ffd#d$Zede fd%d&Zedefd'd(Zd)d)d*d+e ge efd,ee d-eefd.d/Z!e"d0d1d2Z#e"d0d3d4Z$e"fd5d6Z%e"fd7d8Z&d-efd9d:Z'd+e ge effd;d<Z(d@d=d>Z)d)S)AEditTreeLemmatizerzK Lemmatizer that lemmatizes each word using a predicted edit tree. trainable_lemmatizerorthFr)backoff min_tree_freq overwritetop_kscorervocabr!namer&r'r(r)r*c CsZ||_||_||_||_||_||_||_t|jj|_ i|_ dgi|_ ||_ t |_dS)a Construct an edit tree lemmatizer. backoff (Optional[str]): backoff to use when the predicted edit trees are not applicable. Must be an attribute of Token or None (leave the lemma unset). min_tree_freq (int): prune trees that are applied less than this frequency in the training data. overwrite (bool): overwrite existing lemma annotations. top_k (int): try to apply at most the k most probable edit trees. labelsN)r+r!r,r&r'r(r)rstringstrees tree2labelcfgr*r numpy_ops) selfr+r!r,r&r'r(r)r*r4W/home/ubuntu/.local/lib/python3.10/site-packages/spacy/pipeline/edit_tree_lemmatizer.py__init__1s  zEditTreeLemmatizer.__init__examplesscoresreturnc Cst|dtddd}g}|D]8}g}t|j|jdddD]#\}}|dus*|dkr-d} n|j|j|} |j | d } | | q| |q|||\} } |j j j | rattjj|jd t| | fS) NEditTreeLemmatizer.get_lossF) normalize missing_valueLEMMAT) as_stringrr,)rrzip predicted get_alignedr/addtextr0getappendr!opsxpisnan ValueErrorrE910formatr,float) r3r7r8 loss_functruthseg eg_truthsrC gold_lemmalabeltree_idd_scoreslossr4r4r5get_lossWs$      r:docscsjdkr j}n jtkrj}nj}tt|}tdd|Ds?tjdfdd|D}t||ks=J|Sj |}t||ksMJ|||}t||ksZJ|S)Nrcss|]}t|VqdSN)len).0docr4r4r5 sz-EditTreeLemmatizer.predict..r-csg|] }jjdqS)r)r!rIalloc2i)r]_n_labelsr3r4r5 sz.EditTreeLemmatizer.predict..) r)_scores2guesses_top_k_equals_1TOP_K_GUARDRAIL_scores2guesses_top_k_greater_1_scores2guesses_top_k_guardrailr\listanyr1r!predict)r3rZscores2guessesn_docsguessesr8r4rbr5rkrs     zEditTreeLemmatizer.predictc Csg}t||D]A\}}|jdd}|j|}g}t|D]"\}} |jd||} |j| | jdur:| | q| dq| t |q|S)Nr)axisr-r;) rBargmaxr2asarray enumerater1r/applyrFrHnparray) r3rZr8rnr^ doc_scores doc_guessesdoc_compat_guessesitokenrVr4r4r5res    z1EditTreeLemmatizer._scores2guesses_top_k_equals_1c Csg}t|jt|j}t||D]T\}}|j|}g}t|D];\}} t|D]-} t || } |j d| } |j | | jdurJ|| nttjj||| f<q(|dq |t|q|S)Nr-r;)minr)r\r-rBr2rqrrrangeintrpr1r/rsrFrHrtfinfofloat32ru) r3rZr8rnr)r^rvrxryrzra candidatecandidate_tree_idr4r4r5rgs"    z2EditTreeLemmatizer._scores2guesses_top_k_greater_1c Csg}t||D]O\}}t|dd|j ddf}|j|}g}t||D]%\}} d} | D]} |jd| } |j| |j durG| } nq0| | q(| t |q|S)N.rr;r-) rBrtargsortr)r2rqr1r/rsrFrHru) r3rZr8rnr^rvrwrxrz candidatesrVrrr4r4r5rhs    z2EditTreeLemmatizer._scores2guesses_top_k_guardrailc Cst|D]H\}}||}t|dr|}t|D]2\}}|js'||jdkrK|dkr<|jdur;t|||j||_q|j|||j }|||_ qqdS)NrGrr;) rrhasattrrGr(lemmar&getattrr/rsrFlemma_) r3rZbatch_tree_idsryr^ doc_tree_idsjrVrr4r4r5set_annotationss   z"EditTreeLemmatizer.set_annotations.cCst|jdS)z4Returns the labels currently added to the component.r-)tupler1r3r4r4r5r-szEditTreeLemmatizer.labelscCsdS)NTr4rr4r4r5 hide_labelsszEditTreeLemmatizer.hide_labelscCszg}tt|jD](}|j|}d|vr|jj|d|d<d|vr,|jj|d|d<||q t|t|jddS)Norigsubstr-)r/r-) r|r\r/r+r.rHdictrr1)r3r/rVtreer4r4r5 label_datas  zEditTreeLemmatizer.label_dataN)nlpr- get_examplesrr-c st|d|dur||n||g}g}t|dD]@}||jg}|jD]!}|jdkr5dn||j |j |fdd|j dDq+t t |}||jjj|ddq|t|dkssJtjj|jd t|dksJtjj|jd |jj||d dS) NEditTreeLemmatizer.initialize rcsg|] }|kr dndqS)g?gr4)r]rU gold_labelr4r5rdsz1EditTreeLemmatizer.initialize..r-r)dtyperA)XY)r_labels_from_data _add_labelsrrHx referencer _pair2labelrFrr1r rr!rIrq_require_labelsr\rE923rNr, initialize) r3rrr- doc_sample label_sampleexample gold_labelsrzr4rr5rs0          rexcludecsBfddfddfddfddd}t||S)Ncjt|Sr[)r1updatesrsly json_loadsbrr4r5z/EditTreeLemmatizer.from_bytes..c j|Sr[)r! from_bytesrrr4r5r cjj|dSNr)r+rrrr3r4r5rcrr[)r/rrrr4r5rrr1r!r+r/)rr)r3 bytes_datar deserializersr4rr5rs    zEditTreeLemmatizer.from_bytescs<fddfddfddfddd}t|S)Ncs tjSr[)r json_dumpsr1r4rr4r5r$rz-EditTreeLemmatizer.to_bytes..c jSr[)r!to_bytesr4rr4r5r% csjjdSr)r+rr4rr4r5r&crr[)r/rr4rr4r5r'rr)rr)r3r serializersr4rr5r"s     zEditTreeLemmatizer.to_bytescsLt|}fddfddfddfddd}t||dS)Ncst|jSr[)r write_jsonr1prr4r5r/rz,EditTreeLemmatizer.to_disk..crr[)r!to_diskrrr4r5r0rcrr)r+rrrr4r5r1rcrr[)r/rrrr4r5r2rr)r ensure_pathr)r3pathrrr4rr5r,s     zEditTreeLemmatizer.to_diskcsFfdd}fdd|fddfddd}t||S)Ncs`z"t|d}j|WdWdS1swYWdSty/ttjdw)Nrb)openr!rreadAttributeErrorrLrE149)rmfilerr4r5 load_model7s &  z0EditTreeLemmatizer.from_disk..load_modelcrr[)r1rr read_jsonrrr4r5r?rz.EditTreeLemmatizer.from_disk..crr)r+ from_diskrrr4r5rArcrr[)r/rrrr4r5rBrr)rr)r3rrrrr4rr5r6s    zEditTreeLemmatizer.from_diskcCsd|vr ttjjddd|vrttjjddt|d|jd<g}|dD];}t|}|r=ttjjd|dt |}d|vrP|j j |d|d<d|vr_|j j |d|d<| |q)|j|t|jD] \}}||j|<qpdS)Nr-rAr/ )errorsrr)rLrE857rNrir1rE1026joinrr+r.rErHr/ from_jsonrrr-r0)r3r-r/rrrUr4r4r5rHs(    zEditTreeLemmatizer._add_labelsc Cst}t|j}t}i}|D]%}|jD]}|jdkr4||j|j}||d7<|j|jf||<qq| D]\}} | |j krQ||\} } |j | | ddq:dS)NrrT) add_label) rrr.rrrrErFritemsr'r) r3rr+r/ tree_freqs repr_pairsrrzrVfreqformrr4r4r5rbs$      z$EditTreeLemmatizer._labels_from_datacCsN|j||}||jvr"|sdSt|jd|j|<|jd||j|S)z Look up the edit tree identifier for a form/label pair. If the edit tree is unknown and "add_label" is set, the edit tree will be added to the labels. Nr-)r/rEr0r\r1rH)r3rrrrVr4r4r5rws  zEditTreeLemmatizer._pair2label)r#)F)*__name__ __module__ __qualname____doc__rrr strr r}boolrr6rrrrr rOrYrrrkrergrhrpropertyr-rrrrrrrrrrrrrr4r4r4r5r",sx  &  *   r"cCs,|dkr td}|jStdtd|)Nmake_edit_tree_lemmatizerzspacy.pipeline.factorieszmodule z has no attribute ) importlib import_modulerrr)r,moduler4r4r5 __getattr__s r)6rsys collectionsr itertoolsrtypingrrrrrr r r numpyrtr thinc.apir r rr thinc.typesrrr@rrrlanguagertokensrtrainingrrrr+r_edit_tree_internals.edit_treesr_edit_tree_internals.schemasr lemmatizerrtrainable_piperrfdefault_model_configfrom_str"DEFAULT_EDIT_TREE_LEMMATIZER_MODELr"rr4r4r4r5s4  (          ^