o i? @s<ddlZddlmZddlmZddlmZddlmZddZ dd Z ej d d d gd dgdgddZ ej dd d gdggdgej d dgdgdgddZej dddZddZej d gdgdgd d!gd"d#Zej d gd$gd%d&gd'd(Zej d gd)gd*d+gd,d-Zej d gd.d/d0d1d2d3gd4d5Zej d gd6d7d8gd9gd:d;Zdd?gd9gd@dAZej d gd6gdBd9gdCdDZej d gd6d7d8gdEgdFdGgdHdIZej d gd6gdBdEgdFdGgdJdKZej d gd6dEgdGgd6dLd7idGgd6dLdMdNgidGgd6dLdMggidGgdOdPZej d gd6d7d8gd/d0dQd0d/dQdRdSgdTdUZej d gd6dVd/d0dWidXgdYdZZd[d\Zd]d^Z d_d`Z!dadbZ"ej #dcdddeZ$dS)fN)Doc)Example)to_ternary_int)VocabcCst}tt tddWdn1swYtttt|dgddWdn1s7wYtttdt|dgdWddS1sWwYdS)Nhiwords)rpytestraises TypeErrorrr)vocabr Y/home/ubuntu/.local/lib/python3.10/site-packages/spacy/tests/training/test_new_example.py&test_Example_init_requires_doc_objects s    "rcCsFtttddgddddgi}t|jtsJt|jts!JdS)Nhelloworldrr)r from_dictrr isinstancexyexampler r rtest_Example_from_dict_basics rannotsicecream somethingsuch)r weirdannotscCRt}t||dd}ttt||WddS1s"wYdSNrr)rrr r KeyErrorrrrr predictedr r rtest_Example_from_dict_invalids  "r$ pred_wordsicecream)icerNNrtagscCspt}t||d}t||}t|jD]\}}|j|d|ks#Jq|jddd}|dd|Dks6JdS)Nrr+TAGT as_stringcSsg|]}dqS)r)r ).0_r r r 0sz4test_Example_from_dict_with_tags..)rrrr enumerate referencetag_ get_aligned)r%rr r#rr'token aligned_tagsr r r test_Example_from_dict_with_tags%s  r8zignore::UserWarningc Csgd}gd}gd}||d}t}t||d}t||}|jddd}|gd ks/Jt||}|jddd} | gd ksFJdS) N)Applysome sunscreenunlessyoucannot)r9r:sunscreenr<r=cannot)VERBDETNOUNrESCONJPRONrCr*rr,Tr-)rCrDrErFrGrCrC)rrrrr5to_dict) r% gold_words gold_tagsrr r#example1 aligned_tags1example2 aligned_tags2r r rtest_aligned_tags3s   rOcCsbgd}gd}gd}||d}t}t||d}t||}|jddd}|gd ks/JdS) N) Applysomer;r<r=r>r?)r9somesunrAr<r=rB)rCrDrErFrGrCr*rr,Tr-)NNrFrGrCrC)rrrrr5)r%rIrJrr r#rr7r r rtest_aligned_tags_multiDs   rRIlikeLondonandBerlin.nsubjROOTdobjccconjpunctrbrbrcrb)rdepsheadscCsht}t||dd}t||}t|jD]\}}|j|d|ks%J|jj|d|ks1JqdS)Nrrrdre) rrrrr2r3dep_headr'rr r#rr'r6r r r!test_Example_from_dict_with_parsePs  ri)Sarahz'ssisterflew)zNounType=prop|Number=singzPoss=yesz Number=singzTense=past|VerbForm=fin)rmorphscCsTt}t||dd}t||}t|jD]\}}t|j|d|ks'JqdS)Nrrrm)rrrrr2r3strmorphrhr r r&test_Example_from_dict_with_morphologycs  rp)Thisisonesentencethisrranother)rbFrNTg)r sent_startscCst}t||dd}t||}tt|jjdksJt|jD]/\}}t |d|dkr8|j dus7Jq"t |d|dkrJ|j dusIJq"|j dusQJq"dS) NrrrcrxrbTrF) rrrrlenlistr3sentsr2r is_sent_startrhr r r&test_Example_from_dict_with_sent_startys  r})rqrrart??)cat1cat2cat3)rcatscCszt}t||dd}t||}tt|jjdksJ|jjddks'J|jjddks1J|jjdd ks;JdS) Nrrrrrrrr)rrrrryrzr3rrr r#rr r r test_Example_from_dict_with_catss  rrTrUNewYorkrWrXrYLOCrrentitiescst}t||dd}t||ttjjdksJfddtdDgdks.J dgd ks9Jjdj d ksCJjd j d ksMJjd j d ksWJdS) Nrrrccsg|]}j|jqSr )r3ent_iob_)r/r'rr rr1sz8test_Example_from_dict_with_entities..r)OrBrTrrrENT_IOB)rcrcrrbrcrrcrr) rrrrryrzr3entsranger5 ent_type_r"r rr$test_Example_from_dict_with_entitiess  "rcCsgdgd}t}t||dd}t||}|jds Jtt|jjdks,Jt dd|jDs8Jd|d <t||}|jdrJJ| d dt||}|jdr^JdS) Nrrrrrrcss|]}|jdkVqdS)rN)r)r/r6r r r sz=test_Example_from_dict_with_empty_entities..r) rrrrr3has_annotationryrzrallpoprr r r*test_Example_from_dict_with_empty_entitiess    r)rr)rcCsht}t||dd}ttt||}Wdn1s!wYtt|j j dks2JdS)Nrrr) rrr warns UserWarningrrryrzr3rrr r r,test_Example_from_dict_with_entities_invalids   r)r) rrrcCrr rrr r ValueErrorrrr"r r r0test_Example_from_dict_with_entities_overlappings  "r)rrbPERSONcitiespeoplerspanscCt}t||dd}t||}tt|jjdksJtt|jjddks+Jtt|jjddks9J|jjdD] }|j dksHJq?|jjdD] }|j d ksXJqOdS) Nrrrrrcrrbrr rrrrryrzr3rrlabel_rr r#rspanr r r!test_Example_from_dict_with_spans  rcCr) Nrrrrrrrbrrrrr r r-test_Example_from_dict_with_spans_overlappingrrrrrcCrr rr"r r r)test_Example_from_dict_with_spans_invalid)s  "r)Q60Q64))rr)rr)rrlinkscCst}t||dd}t||}|jdjdksJ|jdjdks%J|jdjdks/J|jdjdks9J|jd jdksCJ|jd jd ksMJ|jd jdksWJdS) Nrrrrbrcrrrrr)rrrrr3 ent_kb_id_rr r r!test_Example_from_dict_with_linksEs r)r)Q7381115Q2146908)rrcCrr rr"r r r)test_Example_from_dict_with_links_invalid_s   "rcCst}t|gdd}dgdi}t||}tt|jjdks#Jt|gdd}dgdi}t||}tt|jjdksCJdS) N)OnertrYrsmorerrx)rbrrrbrrc)rrtr?rsr)rbrwrrrrb)rrrrryrzr3r{)r r#rexr r r test_Example_from_dict_sentencesos    rc Cst}gd}gd}gd}||d}|||d}t||d}t||}dd|jDgd ks4Jt||}d d|jD|ksFJdS) NrSrZra)rrerrerdrcSg|]}|jjqSr rgr'r/tr r rr1z-test_Example_missing_deps..)rrbrcrrrcSrr rrr r rr1r)rrrrr3) r rrdreannots_head_onlyannots_head_depr# example_1 example_2r r rtest_Example_missing_depss     rc Cst}gd}gd}gd}|||d}t||d}t||}dd|jD}|d|dks3J|d |d ks=J|d |d ksGJ|d |d ksQJ|d |d ks[Jgd }dd|jD|kskJgd}||kswJdS)NrS)r[r\r]Nr_r`)rbrbrbNrcrbrrcSrr rrr r rr1rz.test_Example_missing_heads..rrbrcrr)TTTFTTcSsg|]}|qSr )has_headrr r rr1r)TFFFFF)rrrrr3get_aligned_sent_starts) r rrdrerr#r parsed_headsexpectedr r rtest_Example_missing_headss"   rcCsNgd}gd}t||d}t|||d}t||}|jddd|ks%JdS)N)r~ b)ASPACErrr*r,Tr-)rrr5)en_vocabrr+r#r3rr r rtest_Example_aligned_whitespaces   r11260cCsgddgdgdd}t}t||dd}t||}t|jjdd ks)Jt|jjd d ks5J|}d |d vsAJ|d d d|d dksQJ|d d d |d d ksaJt||}t|jjdt|jjdksyJt|jjd t|jjd ksJ|jjdD]}|jd ksJ|j dksJ|j dksJq|jjd D]}|jdksJ|j dksJ|j dksJqdS)N)rTrUrrrY)rrrr)rrbrrrrrrrrbrrdoc_annotationrzNew YorkrrrTr) rrrrryr3rrHrtext start_char)rr r#r output_dictoutput_examplerr r rtest_issue11260s<        r)%r spacy.tokensrspacy.training.exampler spacy.utilr spacy.vocabrrrmark parametrizer$r8filterwarningsrOrRrirpr}rrrrrrrrrrrrrrissuerr r r rs