o }oi,4@sdZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZddlmZdgZdZdZdZd Zd ed efd d Z d eded efddZ!d e"fddZ#de"de"de"dede$de$f ddZ%dedededede$de$d e"fddZ&dS)z Evaluate predictions JSON file, w.r.t. ground truth file. This file contains code artifacts adapted from the original implementation: https://github.com/google-research/google-research/blob/master/schema_guided_dst/evaluate.py N)ACTIVE_INTENT_ACCURACYJOINT_CAT_ACCURACYJOINT_GOAL_ACCURACYJOINT_NONCAT_ACCURACYNAN_VALREQUESTED_SLOTS_F1REQUESTED_SLOTS_PRECISIONREQUESTED_SLOTS_RECALLSLOT_TAGGING_F1SLOT_TAGGING_PRECISIONSLOT_TAGGING_RECALLget_active_intent_accuracy#get_average_and_joint_goal_accuracyget_requested_slots_f1get_slot_tagging_f1)loggingget_in_domain_servicesz #ALL_SERVICESz#SEEN_SERVICESz#UNSEEN_SERVICESzdialogues_and_metrics.json schema_pathreturncCsbt}t|dd}t|}|D] }||dq|Wd|S1s*wY|S)z Get the set of all services present in a schema. Args: schema_path: schema file path Returns: service_set: set of services in file UTF-8encoding service_nameN)setopenjsonloadaddclose)r service_setfschemaservicer#c/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/nlp/data/dialogue/sgd/evaluate.pyget_service_set:s   r%rcCst||@}|S)zGet the set of common services between a schema and set of services. Args: schema_path: path to schema file service_set: set of services Returns: joint_services: joint services between schema path file and service set )r%)rrjoint_servicesr#r#r$rKs c Csi}t|tr |}ntt|}|D]G}t|vrqtd|t|dd,}t |}t|tr=|D]}|||d<q3n t|t rG| || Wdn1sUwYq|S)zRead the DSTC8/SGD json dialogue data as dictionary with dialog ID as keys. Args: file_path_patterns: list or directory of files Returns: dataset_dict: dataset dictionary with dialog ID as keys zLoading file: %srr dialogue_idN) isinstancelistsortedglobPER_FRAME_OUTPUT_FILENAMErdebugrrrdictupdater)file_path_patterns dataset_dictlist_fpfpr datadialr#r#r$get_dataset_as_dictWs(       r6 dataset_ref dataset_hypservice_schemasin_domain_servicesjoint_acc_across_turnuse_fuzzy_matchc$ Csrtdd}t|t|sJtdt|t|i}|D]Y\}} ||} t| dt| dkrCt d |t t t g} tt| d| dD],\} \} }tdd}| d|dkrpt d || dd krwqS| d |d krtd | d td |d t d |dd|dD}| dD]}|d}||vrt d ||||}||}t||}t||| d |}t||}t||||}t|t|jt|jt|ji}|dur|j|t<|j|t<|j|t<||d || |d}|||<||d<|d dd}t!|d|g}|d|vr*|"t#n|"t$|D]/}|D]'\}}|t%kr]|rT|| vrT||||9<q7|||"|q7q1q|r|D]}||D]\}}|||"|qoqgqSq'i} |D],\}}!i}"|!D]\}}#|#rt&t't()|#dd|"|<qt%|"|<q|"| |<q| |fS)a!Calculate the DSTC8/SGD metrics. Args: dataset_ref: The ground truth dataset represented as a dict mapping dialogue id to the corresponding dialogue. dataset_hyp: The predictions in the same format as `dataset_ref`. service_schemas: A dict mapping service name to the schema for the service. in_domain_services: The set of services which are present in the training set. joint_acc_across_turn: Whether to compute joint accuracy across turn instead of across service. Should be set to True when conducting multiwoz style evaluation. use_fuzzy_match: Whether to use fuzzy string matching when comparing non-categorical slot values. Should be set to False when conducting multiwoz style evaluation. Returns: all_metric_aggregate: A dict mapping a metric collection name to a dict containing the values for various metrics. Each metric collection aggregates the metrics across a specific set of frames in the dialogues. per_frame_metric: metrics aggregated for each frame cSs ttS)N) collections defaultdictr)r#r#r#r$s zget_metrics..z(len(dataset_hyp)=%d, len(dataset_ref)=%dservicesz[Set of services present in ground truth and predictions don't match for dialogue with id {}turnscSstddS)NcSsdS)Ng?r#r#r#r#r$r?sz/get_metrics....)r=r>r#r#r#r$r?sspeakerz+Speakers don't match in dialogue with id {}USER utterancez Ref utt: %sz Hyp utt: %sz.Utterances don't match for dialogue with id {}cSsi|]}|d|qS)r"r#).0framer#r#r$ szget_metrics..framesr"z5Frame for service {} not found in dialogue with id {}Nz{:s}-{:03d}-{:s}metrics_rgY@)*r=r>rkeysissubsetrr-lenitems ValueErrorformatrrr enumerateziperrorr rrrrrf1r precisionr recallr r r r/split ALL_SERVICESappend SEEN_SERVICESUNSEEN_SERVICESrroundfloatnpmean)$r7r8r9r:r;r<metric_collectionsper_frame_metricdial_iddial_hypdial_ref joint_metricsturn_idturn_refturn_hypmetric_collections_per_turnhyp_frames_by_service frame_refrr" frame_hypactive_intent_accslot_tagging_f1_scoresrequested_slots_f1_scoresgoal_accuracy_dict frame_metricframe_id domain_name domain_keys domain_key metric_key metric_valueall_metric_aggregatedomain_metric_valsdomain_metric_aggregate value_listr#r#r$ get_metricsrs $            H   r}prediction_dirdata_dir eval_datasetcCs~ttj||ddd}i}t|}|D]} | || d<q|Wdn1s-wYttj||d} ttj|d} t| | ||||\} } t | vret dt d t | t  t| vrzt dtd t | t t| vrt dtd t | t ttj|td dd}tj| |d dd|Wd| tS1swY| tS)aCalculate the DSTC8/SGD metrics for given data. Args: prediction_dir: prediction location data_dir: ground truth data location. eval_dataset: evaluation data split in_domain_services: The set of services which are present in the training set. joint_acc_across_turn: Whether to compute joint goal accuracy across turn instead of across service. Should be set to True when conducting multiwoz style evaluation. use_fuzzy_match: Whether to use fuzzy string matching when comparing non-categorical slot values. Should be set to False when conducting multiwoz style evaluation. Returns: A dict mapping a metric collection name to a dict containing the values for various metrics for all dialogues and all services z schema.jsonrrrNzdialogues_*.jsonz*.jsonzDialog metrics for z : : z : wrK),r)indent separators)rospathjoinrrrr6r}r[rinfor*rOr\rYr,dump)r~rrr:r;r<r eval_services list_servicesr"r7r8ryrJr#r#r$evaluates2   """  r)'__doc__r=r+rrnumpyr_(nemo.collections.nlp.metrics.sgd_metricsrrrrrrrr r r r r rrr nemo.utilsr__all__rYr[r\r,strrr%rr.r6boolr}rr#r#r#r$sZD