o
    Niw                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlm  m	Z
 ddlmZ dZdZdZd	Zd
ZdZdd Zdd ZG dd dejjZdS )z)Commonsense Explanations (CoS-E) Dataset.    )absolute_import)division)print_functionNa  
@inproceedings{rajani2019explain,
     title = "Explain Yourself! Leveraging Language models for Commonsense Reasoning",
    author = "Rajani, Nazneen Fatema  and
      McCann, Bryan  and
      Xiong, Caiming  and
      Socher, Richard",
      year="2019",
    booktitle = "Proceedings of the 2019 Conference of the Association for Computational Linguistics (ACL2019)",
    url ="https://arxiv.org/abs/1906.02361"
}
z
Common Sense Explanations (CoS-E) allows for training language models to
automatically generate explanations that can be used during training and
inference in a novel Commonsense Auto-Generated Explanation (CAGE) framework.
z?https://raw.githubusercontent.com/salesforce/cos-e/master/data/z=https://s3.amazonaws.com/commensenseqa/train_rand_split.jsonlz;https://s3.amazonaws.com/commensenseqa/dev_rand_split.jsonlzGhttps://s3.amazonaws.com/commensenseqa/test_rand_split_no_answers.jsonlc           
   	   C   s   |  tttd}ddg}g }|D ]0}tjj|| }t|D ]\}}t	
|}|| q!W d   n1 s:w   Y  i }	q|D ]}||	|d < qD|	S )zDDownloads CQA and returns it, indexed by id, for joining with Cos-E.)	cqa_traincqa_devcqa_testr   r   Nid)download_and_extract_CQA_URL_TRAIN_CQA_URL_DEV_CQA_URL_TESTtfiogfileGFile	enumeratejsonloadsappend)

dl_managerdownloaded_files
cqa_splitscqa_completesplitf_linedcqa_indexed r   R/home/ubuntu/.local/lib/python3.10/site-packages/tensorflow_datasets/text/cos_e.py_download_and_index_cqa8   s$   
r!   c                 C   sL   g }| d }d}| d d D ]}| |d  ||d kr!|d }q||fS )z2Returns choices and the answer from a cqa example.	answerKeyNquestionchoicestextlabel)r   )cqar$   
answer_keyanswerchoicer   r   r    _get_choices_and_answerQ   s   r+   c                   @   s4   e Zd ZdZejdZdd Zdd Z	dd Z
d	S )
CosEz(CoS-E: Common Sense Explanations corpus.z0.0.1c                 C   sZ   t jj| tt jt j t j t jt j t j t j t j dd dtdS )Nr   r#   r$   r)   abstractive_explanationextractive_explanationz#https://github.com/salesforce/cos-e)builderdescriptionfeaturessupervised_keyshomepagecitation)	tfdscoreDatasetInfo_DESCRIPTIONr2   FeaturesDictTextSequence	_CITATION)selfr   r   r    _infob   s   z
CosE._infoc                 C   sl   t |}|tjtdgtjtdgd}tjjtj	j
|d |ddtjjtj	j|d |ddgS )zReturns SplitGenerators.z(v1.11/dev/cose_dev_v1.11_processed.jsonlz,v1.11/train/cose_train_v1.11_processed.jsonl)devtrainr@   )filesr   )name
gen_kwargsrA   )r!   r	   ospathjoin
_COS_E_URLr6   r7   SplitGeneratorSplit
VALIDATIONTRAIN)r>   r   r   rB   r   r   r    _split_generatorss   s,   zCosE._split_generatorsc                 k   s    |d }|D ]K}t jj|:}|D ]/}t|}||d  }t|\}	}
|d |d |d d |	|
|d d |d d dfV  qW d	   n1 sMw   Y  qd	S )
zYields examples.r   r   r#   stemexplanationz
open-endedselectedr-   N)r   r   r   r   r   r   r+   )r>   rB   kwargsr   filepathr   r   cosr'   r$   r)   r   r   r    _generate_examples   s(   



zCosE._generate_examplesN)__name__
__module____qualname____doc__r6   r7   VersionVERSIONr?   rM   rT   r   r   r   r    r,   ]   s    r,   )rX   
__future__r   r   r   r   rE   tensorflow.compat.v2compatv2r   tensorflow_datasets.public_api
public_apir6   r=   r9   rH   r
   r   r   r!   r+   r7   GeneratorBasedBuilderr,   r   r   r   r    <module>   s"   