o }oi@sLddlZddlmZddlZddlmZGdddeZGdddeZ dS)N)Counter) corpus_bleuc@s<eZdZeddZeddZeddZeddZd S) DialogueGenerationMetricscCsg}tt|D]}|||||||dqt|ddd}|D] }|t|dq$WddS1s?sz4DialogueGenerationMetrics.get_f1..r)axis)r&r'rrmean)r3r4 total_p_r_f1 avg_p_r_f1rr2rget_f1<s  z DialogueGenerationMetrics.get_f1csVfddttD}fdd|Dfdd|Dtgdd}|jS)z Referenced from NMT evaluation Note 13a is the default tokenizer for English for WMT Known issue that it doesn't hand edge case of None or '' https://github.com/mjpost/sacrebleu/issues/161 cs g|] }|r|r|qSrrr0rrrrr5Os z6DialogueGenerationMetrics.get_bleu..cg|]}|qSrrr0)rrrr5Pcr<rrr0)rrrr5Qr=13a)tokenize)rrrscore)rr valid_indices sacre_bleurr;rget_bleuGs z"DialogueGenerationMetrics.get_bleuN)__name__ __module__ __qualname__ staticmethodr r/r:rCrrrrrs    rc@s2eZdZeddZed ddZeddZdS) DialogueClassificationMetricsc Csg}tt|D]} ||| || || || || || || dqt|ddd} |D] } | t| dq0WddS1sHwYdS)r)rrground_truth_slotsground_truth_labelsrgenerated_slotsgenerated_labelsr r r r Nr) rrLrKrJrIrrrrrrrrrrr Ws" "z.DialogueClassificationMetrics.save_predictionsFcCsg}g}|D]P}|rHdd|ddD}d}t|dkr"|\}}n t|dkr.|d}d}t|trDd|vr>|dd}|d }nd g}n|}g}||||q||fS) am Split target into label and slots when doing joint label (i.e. intent) classificaiton and slot filling For instance, split "reserve_restaurant slots: time_of_day(7pm), number_of_people(3)" into label = "reserve_restaurant" and slots = ["time_of_day(7pm)", "number_of_people(3)"] Args: fields: list of strings cSsg|]}|qSr)stripr0rrrr5r=zGDialogueClassificationMetrics.split_label_and_slots..zslots:noner!rzpossible intents:z, None)r#r isinstancestrr)fields with_slotslabels slots_listfieldcombolabelslotsrrrsplit_label_and_slotsys*        z3DialogueClassificationMetrics.split_label_and_slotscsg}g}g}tt|D]T}ttt||ttt||}fdd|D}tdkr9t|tnd}t|dkrIt|t|nd} t|k} |||| || q t|d} t|d} t|d} d| | | | d}| | || fS)z Args: generated_slots: list of list of strings. Each string is slot-name and slot-value pair e.g. location(Seattle) ground_truth_slots: list of list of strings csg|]}|vr|qSrr)r1rrrrr5szJDialogueClassificationMetrics.get_slot_filling_metrics..rr"r!g#B ;) rrsortedlistsetintrr&r7)rKrI all_recall all_precisionall_joint_goal_accuracyr predictedcorrectr-r,joint_goal_accuracyavg_joint_goal_accuracy avg_precision avg_recallavg_f1rr\rget_slot_filling_metricss$       z6DialogueClassificationMetrics.get_slot_filling_metricsN)F)rDrErFrGr r[rkrrrrrHVs ! #rH) r collectionsrnumpyr& sacrebleurobjectrrHrrrrs   ?