o i.@sddlZddlZddlmZddlmZmZmZmZm Z m Z m Z ddl Z ddl mZddlmZmZddlmZmZddlmZd d lmZd d lmZerTdd lmZd Zejd d-de ede de!de!de ededgeeff ddZ"ejd d.de e e#efde!de!de!dedgeeff ddZ$ejddddede fd d!Z%ejd" d/de ede!de!dedgeeffd#d$Z&de e#efde efd%d&Z'Gd'd(d(Z(Gd)d*d*Z)Gd+d,d,Z*dS)0N)Path) TYPE_CHECKINGCallableIterableIteratorListOptionalUnion)util)ErrorsWarnings)DocDocBin)Vocab) dont_augment)Example)Languagez.spacyzspacy.Corpus.v1path gold_preproc max_lengthlimit augmenterreturnrcCs2|dur ttjtjd|t|||||dS)NzLoading corpus from path: %s)rrrr) ValueErrorr E913r loggerdebugCorpus)rrrrrr I/home/ubuntu/.local/lib/python3.10/site-packages/spacy/training/corpus.pycreate_docbin_readers r"zspacy.JsonlCorpus.v1 min_lengthcCst||||dS)N)r#rr) JsonlCorpus)rr#rrr r r!create_jsonl_reader*sr%zspacy.read_labels.v1F)requirer&cCs|s|sdSt|SN)existssrsly read_json)rr&r r r! read_labels4s  r+zspacy.PlainTextCorpus.v1cCs |dur ttjt|||dS)Iterate Example objects from a file or directory of plain text UTF-8 files with one line per doc. path (Path): The directory or filename to read from. min_length (int): Minimum document length (in tokens). Shorter documents will be skipped. Defaults to 0, which indicates no limit. max_length (int): Maximum document length (in tokens). Longer documents will be skipped. Defaults to 0, which indicates no limit. DOCS: https://spacy.io/api/corpus#plaintextcorpus Nr#r)rr rPlainTextCorpus)rr#rr r r!create_plain_text_reader=s r/cCst|}|s|jd|r|gS|}|g}g}t}|D]5}t||vr)q |t||jr<|jddrz(Corpus._make_example..cSg|]}t|jqSr bool whitespace_r]r r r!r`wordsspaces)has_unknown_spacesrrrNmake_docr\)rJrLrYrr r r! _make_examples    zCorpus._make_examplereference_docsccs|D]F}t|dkr q|jdkst||jkr!|||dVq|drI|jD]}t|dkr2q)|jdks>t||jkrH|||dVq)qdS)NrF SENT_START)r>rrlhas_annotationsentsas_doc)rJrLrmrYref_sentr r r!rSs    zCorpus.make_examplesccsZ|D]'}|drdd|jD}n|g}|D]}|||d}t|jr)|VqqdS)NrncSsg|]}|qSr )rq)r^sentr r r!r`sz5Corpus.make_examples_gold_preproc..T)rorprlr>x)rJrLrmrY ref_sentsrregr r r!rRs  z!Corpus.make_examples_gold_preprocrNrFccs|d}|D]6}t|}|jdtr;t|}||}|D]}t|r:|V|d7}|j dkr:||j kr:nq!qdS)z(Yield training examples as example dictsrr0rN) r r3r5r6rOr from_diskget_docsr>r)rJrNrFilocdoc_bindocsdocr r r!rMs   zCorpus.read_docbin)__name__ __module__ __qualname____doc__r r8rintrdrrrKrrrXrrlrrSrRrrMr r r r!rlsj      rc @s^eZdZdZdZdddddeeeefde de de d d f d d Z d dd e e fddZ d S)r$acIterate Example objects from a file or directory of jsonl formatted raw text files. path (Path): The directory or filename to read from. min_length (int): Minimum document length (in tokens). Shorter documents will be skipped. Defaults to 0, which indicates no limit. max_length (int): Maximum document length (in tokens). Longer documents will be skipped. Defaults to 0, which indicates no limit. limit (int): Limit corpus to a subset of examples, e.g. for debugging. Defaults to 0, which indicates no limit. DOCS: https://spacy.io/api/corpus#jsonlcorpus jsonlr)rr#rrrr#rrNcCs"t||_||_||_||_dSr')r r3rr#rr)rJrrr#rr r r!rKs  zJsonlCorpus.__init__rLrc cst|jdD]G}t|}|D]=}||d}|jdkr&t||jkr&q|jdkr3t||jkr3qdd|D}dd|D}t|t |j ||dVqqdS) zYield examples from the data. nlp (Language): The current nlp object. YIELDS (Example): The example objects. DOCS: https://spacy.io/api/corpus#jsonlcorpus-call z.jsonlr\rcSrZr r[r^wr r r!r`raz(JsonlCorpus.__call__..cSrbr rcrr r r!r`rfrgN) rHrr) read_jsonlrkr#r>rrrrN)rJrLrzrecordsrecordr}rhrir r r!rXs zJsonlCorpus.__call__r~rrrrCrr r8rrrKrrrXr r r r!r$s$  r$c @sXeZdZdZdZddddeeeefde de dd fd d Z d d de e fddZ d S)r.r,txtrr-rr#rrNcCst||_||_||_dSr')r r3rr#r)rJrr#rr r r!rK+s  zPlainTextCorpus.__init__rLrc cst|jdD]M}t|dd=}|D]2}|d}t|rD||}|jdkr/t||jkr/q|jdkrrkr#rrcopy)rJrLrzfr\r}r r r!rX6s"  zPlainTextCorpus.__call__rr r r r!r.s   r.)rrN)rrr)rr)+rQr?pathlibrtypingrrrrrrr r)r errorsr r tokensrrrNraugmentrexamplerlanguagerrOregistryreadersrdrr"r8r%r+r/rHrr$r.r r r r!s $          y7