o Ni@sdZddlmZddlmZddlmZddlZddlZddlZddlm m Z ddl m Zejr: r2cCr)z%s/dev/nq-dev-%02d.jsonl.gzrr r r r r>r)train validationc@sBeZdZdZejdZejdgZddZ ddZ dd Z d S) NaturalQuestionsrz0.0.2z0.0.1cCstjj|ttjtjtjtjtjtj tjtj ddtjtj tjdtj tjtj tj tj tj dtj tj tj tj tj tjdtjj ddgdd d dt td S) Ntokenis_htmltitleurlhtmltokenstextr start_token end_token start_byteend_byter r!r"r#rNOYES)namesid long_answer short_answers yes_no_answerr)documentquestion annotations)builder descriptionfeaturessupervised_keyshomepagecitation)tfdscore DatasetInfo _DESCRIPTIONr3 FeaturesDicttfstringTextSequenceboolint64 ClassLabel_URL _CITATION)selfr r r _infoKsN "zNaturalQuestions._infocCsB|t}tjjtjjd|didtjjtjjd|didgS)zReturns SplitGenerators. filepathsr)name gen_kwargsr)download_DOWNLOAD_URLSr7r8SplitGeneratorSplitTRAIN VALIDATION)rE dl_managerfilesr r r _split_generatorsvs   z"NaturalQuestions._split_generatorscs8tjjjfdd}||BjB|BS)zBuild PCollection of examples.cst|}|ddfddfddjjddt|d }|||d |d d d |dDd|d|ddfdd |dDdfS)z2Parse a single json line and emit an example dict. document_htmlutf-8cs\|d|d}|dd}|d}tddt|}|d|d |d|d|d S) z"Extract text of short answer.r"r#s  rTz <([^>]*)>r r!r$)replacedecoderesub html_unescape) short_ans ans_bytesr) html_bytesr r _parse_short_answers  zXNaturalQuestions._build_pcollection.._parse_example.._parse_short_answercsjt|d|dd|dd|dd|dddfdd |d D|d d kr/d dS|d dS)N annotation_idr*r r!r"r#rcg|]}|qSr r )r ansr_r r rzjNaturalQuestions._build_pcollection.._parse_example.._parse_annotation..r+r,NONEr()str)an_jsonrcr r _parse_annotations       zVNaturalQuestions._build_pcollection.._parse_example.._parse_annotationnqexamples example_iddocument_title document_urlcSsg|] }|d|ddqS)r html_tokenrr )r tr r r rszONaturalQuestions._build_pcollection.._parse_example..document_tokensr question_textquestion_tokensrcrar r )r rh)rir r rrdr0r-)jsonloadsencodemetricsMetricscounterincrg)lineex_jsonid_beam)rir_r^r _parse_examples,      z;NaturalQuestions._build_pcollection.._parse_example)r7r8 lazy_imports apache_beamCreateioReadAllFromTextMap)rEpipelinerGrr r~r _build_pcollections  @z#NaturalQuestions._build_pcollectionN) __name__ __module__ __qualname____doc__r7r8VersionVERSIONSUPPORTED_VERSIONSrFrRrr r r r rEs + r)r __future__rrrrtrYsixtensorflow.compat.v2compatv2r<tensorflow_datasets.public_api public_apir7PY2 HTMLParser html_parserunescaper[rrDr:rCrrangerKr8BeamBasedBuilderrr r r r s4