o is# @sddlZddlmZmZmZmZmZmZmZddl m Z m Z ddl m Z mZdeedeefdd Zdeedeefd d Zdeedeefd d ZdeedeefddZd%de defddZde deefddZ d%de deeeeeeeffdedeefddZde deedeefddZde deedeeeeeeefffddZdeedeeeeeffddZd edeeeffd!d"Zd edefd#d$ZeZeZeZdS)&N)DictIterableIteratorListTupleUnioncast)ErrorsWarnings)DocSpantagsreturncCs4g}t|}|r|t||t||s|S)N)listextend _consume_os _consume_ent)routrL/home/ubuntu/.local/lib/python3.10/site-packages/spacy/training/iob_utils.py iob_to_biluosrcCsHg}|D]}|dur||q|dddddd}||q|S)NU-B-L-I-)appendreplace)rrtagrrr biluo_to_iobs  r ccs>|r|ddkr|dV|r|ddks dSdSdSdS)NrO)pop)rrrrrs  rcs|sgS|d}d|dd}d|dd}d}|r8|d||hvr8|d7}|d|r8|d||hvs%|dd|dkrVtdkrQttjj|ddgSd}d }fd d td|dD}|g||gS) NrIrLr rrrrcsg|]}dqS)rr.0_labelrr 3z _consume_ent..)r"len ValueErrorr E177formatrange)rr target_in target_lastlengthstartendmiddlerr)rr!s&     rr!docmissingcCst|dd|jD|dS)NcSg|] }|j|j|jfqSr start_charend_charlabel_)r'entrrrr+:z%doc_to_biluo_tags..r9)offsets_to_biluo_tagsents)r8r9rrrdoc_to_biluo_tags7s rDcCs4t|dd}t|D] \}}|jdkrd||<q |S)N-rAr r!)rD enumerateent_iob)r8rCitokenrrr_doc_to_biluo_tags_with_partial?s  rJentitiescCs(i}dd|D}dd|D}dd|D}|D]\}}} | s4|D]} | |kr2| |kr2d||| <q"qt||D]+} | |vr]ttjj|| d|| d|| d f||| fd ||| f|| <q9||} ||} | d ur| d ur| | krd | || <qd | || <t| d| D] }d| ||<qd| || <qt}|D]\}}} t||D]}||qq|D]}t|j |j t |D]}||vrnq|||j <qd|vr|dkrt |}t tjjt |jdkr|jd ddn|jt |dkr |d ddn|d|S)uEncode labelled spans into per-token tags, using the Begin/In/Last/Unit/Out scheme (BILUO). doc (Doc): The document that the entity offsets refer to. The output tags will refer to the token boundaries within the document. entities (iterable): A sequence of `(start, end, label)` triples. `start` and `end` should be character-offset integers denoting the slice into the original string. missing (str): The label used for missing values, e.g. if tokenization doesn’t align with the entity offsets. Defaults to "O". RETURNS (list): A list of unicode strings, describing the tags. Each tag string will be of the form either "", "O" or "{action}-{label}", where action is one of "B", "I", "L", "U". The missing label is used where the entity offsets don't align with the tokenization in the `Doc` object. The training algorithm will view these as missing values. "O" denotes a non-entity token. "B" denotes the beginning of a multi-token entity, "I" the inside of an entity of three or more tokens, and "L" the end of an entity of two or more tokens. "U" denotes a single-token entity. EXAMPLE: >>> text = 'I like London.' >>> entities = [(len('I like '), len('I like London'), 'LOC')] >>> doc = nlp.tokenizer(text) >>> tags = offsets_to_biluo_tags(doc, entities) >>> assert tags == ["O", "O", 'U-LOC', "O"] cSsi|]}|j|jqSr)idxrHr'rIrrr fr,z)offsets_to_biluo_tags..cSsi|] }|jt||jqSr)rLr-rHrMrrrrNgscSsg|]}dqS)rErr&rrrr+hsz)offsets_to_biluo_tags..r!rrr )span1span2NrrrrrE2z...)textrK)r1keysr.r E103r0getsetaddrLr-rHstrwarningswarnr W030rR)r8rKr9tokens_in_entsstartsendsbiluor<r=r*s token_index start_token end_tokenrH entity_charsrIent_strrrrrBGsl         $ rBcCs>t|}g}|D]\}}}t|||d|d}||q|S)aEncode per-token tags following the BILUO scheme into Span object, e.g. to overwrite the doc.ents. doc (Doc): The document that the BILUO tags refer to. tags (iterable): A sequence of BILUO tags with each tag describing one token. Each tag string will be of the form of either "", "O" or "{action}-{label}", where action is one of "B", "I", "L", "U". RETURNS (list): A sequence of Span objects. Each token with a missing IOB tag is returned as a Span with an empty label. rr))tags_to_entitiesr r)r8r token_offsetsspansr* start_idxend_idxspanrrrbiluo_tags_to_spanss   rlcCst||}dd|DS)aEncode per-token tags following the BILUO scheme into entity offsets. doc (Doc): The document that the BILUO tags refer to. tags (iterable): A sequence of BILUO tags with each tag describing one token. Each tags string will be of the form of either "", "O" or "{action}-{label}", where action is one of "B", "I", "L", "U". RETURNS (list): A sequence of `(start, end, label)` triples. `start` and `end` will be character-offset integers denoting the slice into the original string. cSr:rr;)r'rkrrrr+r@z)biluo_tags_to_offsets..)rl)r8rrhrrrbiluo_tags_to_offsetss rmcCs,g}d}t|D]\}}|dus|dr%|durd}q|d||fq|dr+q|drG|durFttjjdt|d|ddq|drY||d d||fq|d ra|}q|d r|dur|ttjjd t|d|dd||d d||fd}qttjj|d |S) zxNote that the end index returned by this function is inclusive. To use it for Span creation, increment the end by 1.NrEr!r#r)r5rUr Br$r%) rF startswithrr.r E067r0rE068)rrKr5rHrrrrrfs8     rfr*cCsttttf|ddSNrEr)rrrXsplitr)rrrsplit_bilu_labelsrvcCs|dddSrt)rur)rrrremove_bilu_prefixsrw)r!) rYtypingrrrrrrrerrorsr r tokensr r rXrr rrrDrJintrBrlrmrfrvrwoffsets_from_biluo_tagsspans_from_biluo_tagsbiluo_tags_from_offsetsrrrrsD$    W $#