o %ݫi@sdZddlmZmZddlZddlmZmZddlm Z m Z m Z ddl m Z mZddlmZddlmZmZmZmZGd d d Zd$d d Zd%ddZGdddeZGdddeZGdddZGdddeZddZ d&ddZGdddeZGd d!d!Z d"d#Z!dS)'uThe ``metric_stats`` module provides an abstract class for storing statistics produced over the course of an experiment and summarizing them. Authors: * Peter Plantinga 2020 * Mirco Ravanelli 2020 * Gaëlle Laperrière 2021 * Sahar Ghannay 2021 )CallableOptionalN)Paralleldelayed)extract_concepts_values merge_char split_word)print_alignmentsprint_wer_summary) undo_padding) EDIT_SYMBOLS _str_equalswer_details_for_batch wer_summaryc@s>eZdZdZdddZddZdd Zdd d ZdddZd S) MetricStatsaA default class for storing and summarizing arbitrary metrics. More complex metrics can be created by sub-classing this class. Arguments --------- metric : function The function to use to compute the relevant metric. Should take at least two arguments (predictions and targets) and can optionally take the relative lengths of either or both arguments. Not usually used in sub-classes. n_jobs : int The number of jobs to use for computing the metric. If this is more than one, every sample is processed individually, otherwise the whole batch is passed at once. batch_eval : bool When True it feeds the evaluation metric with the batched input. When False and n_jobs=1, it performs metric evaluation one-by-one in a sequential way. When False and n_jobs>1, the evaluation runs in parallel over the different inputs using joblib. Example ------- >>> from speechbrain.nnet.losses import l1_loss >>> loss_stats = MetricStats(metric=l1_loss) >>> loss_stats.append( ... ids=["utterance1", "utterance2"], ... predictions=torch.tensor([[0.1, 0.2], [0.2, 0.3]]), ... targets=torch.tensor([[0.1, 0.2], [0.1, 0.2]]), ... reduction="batch", ... ) >>> stats = loss_stats.summarize() >>> stats['average'] 0.050... >>> stats['max_score'] 0.100... >>> stats['max_id'] 'utterance2' TcCs||_||_||_|dSN)metricn_jobs batch_evalclearselfrrrrR/home/ubuntu/.local/lib/python3.10/site-packages/speechbrain/utils/metric_stats.py__init__Hs zMetricStats.__init__cCsg|_g|_i|_dS)z=Creates empty container for storage, removing existing stats.N)scoresidssummaryrrrrrNs zMetricStats.clearcOs|j||jr|j|i|}n(d|vsd|vr td|jdkr0tdd|ji|}n td|j|jd|}|j |dS) 8Store a particular set of metric scores. Arguments --------- ids : list List of ids corresponding to utterances. *args : tuple Arguments to pass to the metric function. **kwargs : dict Arguments to pass to the metric function. predicttarget>Must pass 'predict' and 'target' as kwargs if batch_eval=FalserrrrNr) rextendrrdetach ValueErrorrsequence_evaluationmultiprocess_evaluationr)rrargskwargsrrrrappendTs  zMetricStats.appendNcCstt|j}tt|j}tt|jt|jt|j||j|t|j||j|d|_ |dur>|j |S|j S)aSummarize the metric scores, returning relevant stats. Arguments --------- field : str If provided, only returns selected statistic. If not, returns all computed statistics. Returns ------- float or dict Returns a float if ``field`` is provided, otherwise returns a dictionary containing all computed stats. )average min_scoremin_id max_scoremax_idN) torchargmintensorrargmaxfloatsumlenrr)rfield min_index max_indexrrr summarizevs   zMetricStats.summarizeFcCs|js|d|jdd}|d|jdd7}|d|jdd7}|d |jd d7}|d|jd d7}|||rIt|d Sd S) zWrite all relevant statistics to file. Arguments --------- filestream : file-like object A stream for the stats to be written to. verbose : bool Whether to also print the stats to stdout. zAverage score: r- z Min error: r. zid: r/z Max error: r0r1N)rr<writeprint)r filestreamverbosemessagerrr write_statss   zMetricStats.write_stats)rTr)F) __name__ __module__ __qualname____doc__rrr,r<rDrrrrrs ( "rc s|dur%||d}ddt||D}ddt||D} zt|ddfd d t||D}W|StyU}zt|td WYd}~nd}~wwq&) z6Runs metric evaluation if parallel over multiple jobs.NrcS g|] \}}|d|qSrcpu.0plengthrrr  z+multiprocess_evaluation..cSrJrrKrNtrPrrrrQrRT)rtimeoutc3s"|] \}}t||VqdSr)r)rNrOrTrrr s z*multiprocess_evaluation..z)Evaluation timeout...... (will try again))sizeroundintrLzipr Exceptionr@)rr!r"lengthsrrerrWrr)s"r)cCsz|dur%||d}ddt||D}ddt||D}g}t||D]\}}|||}||q,|S)z4Runs metric evaluation sequentially over the inputs.NrcSrJrrKrMrrrrQrRz'sequence_evaluation..cSrJrrKrSrrrrQrR)rYrZr[rLr\r,)rr!r"r^rrOrTscorerrrr(s  r(c@s^eZdZdZdddddddefdeeegeffddZ dd d Z dd d Z ddZ d S)ErrorRateStatsaA class for tracking error rates (e.g., WER, PER). Arguments --------- merge_tokens : bool Whether to merge the successive tokens (used for e.g., creating words out of character tokens). See ``speechbrain.dataio.dataio.merge_char``. split_tokens : bool Whether to split tokens (used for e.g. creating characters out of word tokens). See ``speechbrain.dataio.dataio.split_word``. space_token : str The character to use for boundaries. Used with ``merge_tokens`` this represents character to split on after merge. Used with ``split_tokens`` the sequence is joined with this token in between, and then the whole sequence is split. keep_values : bool Whether to keep the values of the concepts or not. extract_concepts_values : bool Process the predict and target to keep only concepts and values. tag_in : str Start of the concept ('<' for example). tag_out : str End of the concept ('>' for example). equality_comparator : Callable[[str, str], bool] The function used to check whether two words are equal. Example ------- >>> cer_stats = ErrorRateStats() >>> i2l = {0: 'a', 1: 'b'} >>> cer_stats.append( ... ids=['utterance1'], ... predict=torch.tensor([[0, 1, 1]]), ... target=torch.tensor([[0, 1, 0]]), ... target_len=torch.ones(1), ... ind2lab=lambda batch: [[i2l[int(x)] for x in seq] for seq in batch], ... ) >>> stats = cer_stats.summarize() >>> stats['WER'] 33.33... >>> stats['insertions'] 0 >>> stats['deletions'] 0 >>> stats['substitutions'] 1 F_Tequality_comparatorc Cs<|||_||_||_||_||_||_||_||_dSr) r merge_tokens split_tokens space_tokenr keep_valuestag_intag_outrd) rrerfrgrhrrirjrdrrrrs  zErrorRateStats.__init__NcCs|j||durt||}|durt||}|dur$||}||}|jr5t||jd}t||jd}|jrFt||jd}t||jd}|jrct||j |j |j |jd}t||j |j |j |jd}t |||d|j d}|j|dS)a^Add stats to the relevant containers. * See MetricStats.append() Arguments --------- ids : list List of ids corresponding to utterances. predict : torch.tensor A predicted output, for comparison with the target output target : torch.tensor The correct reference output, for comparison with the prediction. predict_len : torch.tensor The predictions relative lengths, used to undo padding if there is padding present in the predictions. target_len : torch.tensor The target outputs' relative lengths, used to undo padding if there is padding present in the target. ind2lab : callable Callable that maps from indices to labels, operating on batches, for writing alignments. N)spaceT)compute_alignmentsrd)rr%r rerrgrfrrrhrirjrrdr)rrr!r" predict_len target_lenind2labrrrrr,sJ   zErrorRateStats.appendcCs4t|j|_|jd|jd<|dur|j|S|jS)zhSummarize the error_rate and return relevant statistics. * See MetricStats.summarize() WER error_rateN)rrr)rr9rrrr<cs  zErrorRateStats.summarizecCs*|js|t|j|t|j|dS)zoWrite all relevant info (e.g., error rate alignments) to file. * See MetricStats.write_stats() N)rr<r r rrrArrrrDrs zErrorRateStats.write_stats)NNNr) rErFrGrHr rstrboolrr,r<rDrrrrras&4  M rac @sZeZdZdZ ddedeeeeeegefdefddZ dd Z dd d Z d dZ d S)WeightedErrorRateStatsaMetric that reweighs the WER from :class:`~ErrorRateStats` with any chosen method. This does not edit the sequence of found edits (insertion/deletion/substitution) but multiplies their impact on the metric by a value between 0 and 1 as returned by the cost function. Arguments --------- base_stats : ErrorRateStats The base WER calculator to use. cost_function : Callable[[str, Optional[str], Optional[str]], float] Cost function of signature `fn(edit_symbol, a, b) -> float`, where the returned value, between 0 and 1, is the weight that should be assigned to a particular edit in the weighted WER calculation. In the case of insertions and deletions, either of `a` or `b` may be `None`. In the case of substitutions, `a` and `b` will never be `None`. weight_name : str Prefix to be prepended to each metric name (e.g. `xxx_wer`) weighted base_stats cost_function weight_namecCs|||_||_||_dSr)rrwrxry)rrwrxryrrrrs zWeightedErrorRateStats.__init__cOstd)a}Append function, which should **NOT** be used for the weighted error rate stats. Please append to the specified `base_stats` instead. `WeightedErrorRateStats` reuses the scores from the base :class:`~ErrorRateStats` class. Arguments --------- *args : tuple Ignored. **kwargs : dict Ignored. z]Cannot append to a WeightedErrorRateStats. You should only append to the base ErrorRateStats.)r')rr*r+rrrr,szWeightedErrorRateStats.appendNc Csd}d}d}d}t|jjD]\}}d}d} d} d} |dD]N\} } }| dur-|d| nd}|dur9|d|nd}| tdkrh|| ||}| tdkrS||7}n| tdkr^| |7} n | td krh| |7} | d 7} q|| | }|| }|j|jj||d || | |d ||7}|| 7}|| 7}|| 7}q|||}||}|jd |d |jd||jd||jd||jd|i|_|dur|j|S|jS)aReturns a dict containing some detailed WER statistics after weighting every edit with a weight determined by `cost_function` (returning `0.0` for no error, `1.0` for the default error behavior, and anything in between). Does not require :meth:`~ErrorRateStats.summarize` to have been called. Full set of fields, **each of which are prepended with `_`**: - `wer`: Weighted WER (ratio `*100`) - `insertions`: Weighted insertions - `substitutions`: Weighted substitutions - `deletions`: Weighted deletions - `num_edits`: Sum of weighted insertions/substitutions/deletions Additionally, a `scores` list is populated by this function for each pair of sentences. Each entry of that list is a dict, with the fields: - `key`: the ID of the utterance. - `WER`, `insertions`, `substitutions`, `deletions`, `num_edits` with the same semantics as described above, but at sentence level rather than global. Arguments --------- field : str, optional The field to return, if you are only interested in one of them. If specified, a single `float` is returned, otherwise, a dict is. Returns ------- dict from str to float, if `field is None` A dictionary of the fields documented above. float, if `field is not None` The single field selected by `field`. alignmentN ref_tokens hyp_tokenseqinsdelsub?gY@)keyrp insertions substitutions deletions num_edits_wer _insertions_substitutions _deletions _num_edits) enumeraterwrr rxr,rryr)rr9weighted_insertionsweighted_substitutionsweighted_deletionstotali utteranceutt_weighted_insertionsutt_weighted_substitutionsutt_weighted_deletions utt_total edit_symbola_idxb_idxab pair_scoreutt_weighted_editsutt_weighted_wer_ratioweighted_editsweighted_wer_ratiorrrr<sv%                  z WeightedErrorRateStats.summarizecCsR|js|td|jd|d|jD]\}}t|d||dqdS)zWrite all relevant info to file; here, only the weighted info as returned by `summarize`. See :meth:`~ErrorRateStats.write_stats`. zWeighted WER metrics (z):file: N)rr<r@ryitems)rrAkvrrrrD s z"WeightedErrorRateStats.write_stats)rvr) rErFrGrHrarrsrr6rr,r<rDrrrrru}s   pruc@s\eZdZdZdeegeejfde de de fddZ ded eed eed e fd d Z dS)EmbeddingErrorRateSimilarityaImplements the similarity function from the EmbER metric as defined by https://www.isca-archive.org/interspeech_2022/roux22_interspeech.pdf This metric involves a dictionary to map a token to a single word embedding. Substitutions in the WER get weighted down when the embeddings are similar enough. The goal is to reduce the impact of substitution errors with small semantic impact. Only substitution errors get weighted. This is done by computing the cosine similarity between the two embeddings, then weighing the substitution with `low_similarity_weight` if `similarity >= threshold` or with `high_similarity_weight` otherwise (e.g. a substitution with high similarity could be weighted down to matter 10% as much as a substitution with low similarity). .. note :: The cited paper recommended `(1.0, 0.1, 0.4)` as defaults for fastTexst French embeddings, chosen empirically. When using different embeddings, you might want to test other values; thus we don't provide defaults. Arguments --------- embedding_function : Callable[[str], Optional[torch.Tensor]] Function that returns an embedding (as a :class:`torch.Tensor`) from a word. If no corresponding embedding could be found for the word, should return `None`. In that case, `low_similarity_weight` will be chosen. low_similarity_weight : float Weight applied to the substitution if `cosine_similarity < threshold`. high_similarity_weight : float Weight applied to the substitution if `cosine_similarity >= threshold`. threshold : float Cosine similarity threshold used to select by how much a substitution error should be weighed for this word. embedding_functionlow_similarity_weighthigh_similarity_weight thresholdcCs||_||_||_||_dSr)rrrr)rrrrrrrrrQs z%EmbeddingErrorRateSimilarity.__init__rrrreturncCs|tdtdfvr dS|tdkrW|dus|dkr|jS|dus%|dkr(|jS||}|dur4|jS||}|dur@|jStjjj||dd}||jkrT|j S|jSd S) a-Returns the weight that should be associated with a specific edit in the WER calculation. Compatible candidate for the cost function of :class:`~WeightedErrorRateStats` so an instance of this class can be passed as a `cost_function`. Arguments --------- edit_symbol: str Edit symbol as assigned by the WER functions, see `EDIT_SYMBOLS`. a: str, optional First word to compare (if present) b: str, optional Second word to compare (if present) Returns ------- float Weight to assign to the edit. For actual edits, either `low_similarity_weight` or `high_similarity_weight` depending on the embedding distance and threshold. rrrrNrcrdimrz) r rrr2nn functionalcosine_similarityitemrr)rrrra_embb_emb similarityrrr__call__]s,    z%EmbeddingErrorRateSimilarity.__call__N) rErFrGrHrrsrr2Tensorr6rrrrrrr.s("  rc@s6eZdZdZd ddZddZddZ dd d Zd S)BinaryMetricStatsz?Tracks binary metrics, such as precision, recall, F1, EER, etc.rcCs|||_dSr)rpositive_label)rrrrrrs zBinaryMetricStats.__init__cCsg|_g|_g|_i|_dS)zClears the stored metrics.N)rrlabelsrrrrrr zBinaryMetricStats.clearcCs0|j||j||j|dS)aAppends scores and labels to internal lists. Does not compute metrics until time of summary, since automatic thresholds (e.g., EER) need full set of scores. Arguments --------- ids : list The string ids for the samples. scores : list The scores corresponding to the ids. labels : list The labels corresponding to the ids. N)rr%rr&r)rrrrrrrr,s zBinaryMetricStats.appendN:0yE>cCst|jtrt|j|_t|j|_|dur|j|j|jkjdd}|j|j|jkjdd}|durzt||krXt |\}}|ddt dt|t t||D}t||krzt |\}}|ddt dt|t t||D}t ||\} }|j|k } |j} t | | } |jd<t d | d | } |jd <t | d | }|jd <t d | | }|jd <||| ||jd <|| |||jd<||| | ||jd<||jd<| | |||jd<| | |||jd<d |d| d |d| |d|||jd<| | ||| || || || ||d|jd<|durM|j|S|jS)aFCompute statistics using a full set of scores. Full set of fields: - TP - True Positive - TN - True Negative - FP - False Positive - FN - False Negative - FAR - False Acceptance Rate - FRR - False Rejection Rate - DER - Detection Error Rate (EER if no threshold passed) - threshold - threshold (EER threshold if no threshold passed) - precision - Precision (positive predictive value) - recall - Recall (sensitivity) - F-score - Balance of precision and recall (equal if beta=1) - MCC - Matthews Correlation Coefficient Arguments --------- field : str A key for selecting a single statistic. If not provided, a dict with all statistics is returned. threshold : float If no threshold is provided, equal error rate is used. max_samples: float How many samples to keep for positive/negative scores. If no max_samples is provided, all scores are kept. Only effective when threshold is None. beta : float How much to weight precision vs recall in F-score. Default of 1. is equal weight, while higher values weight recall higher, and lower values weight precision higher. eps : float A small value to avoid dividing by zero. Returns ------- summary if field is specified, only returns the score for that field. if field is None, returns the full set of fields. NT)as_tuplecSg|]}|qSrrrNrrrrrQz/BinaryMetricStats.summarize..rcSrrrrrrrrQrTPrTNFPFNFARFRRDERr precisionrecallg@zF-scoreg?MCC) isinstancerlistr2stackrrnonzeror8sortranger[EERr6mulr7r)rr9r max_samplesbetaepspositive_scoresnegative_scoresrbeerpredtruerrrrrrrr<sv +   $    "  zBinaryMetricStats.summarize)r)NNNrr)rErFrGrHrrr,r<rrrrrs rcCstt||g\}}t|}|dd|ddd}tt||g\}}d}d}d}t|D]B\}} || k} | d|jd} ~ || k} | d|jd} ~ | |  t||ksm|dkrw|}| }| }q5||d}t|t||fS)aqComputes the EER (and its threshold). Arguments --------- positive_scores : torch.tensor The scores from entries of the same class. negative_scores : torch.tensor The scores from entries of different classes. Returns ------- EER : float The EER score. threshold : float The corresponding threshold for the EER score. Example ------- >>> positive_scores = torch.tensor([0.6, 0.7, 0.8, 0.5]) >>> negative_scores = torch.tensor([0.4, 0.3, 0.2, 0.1]) >>> val_eer, threshold = EER(positive_scores, negative_scores) >>> val_eer 0.0 rrN) r2rcatuniquerr7r6shapeabsr)rr thresholdsrbintermediate_thresholdsr: final_FRR final_FARr cur_threshpos_scores_thresholdrneg_scores_thresholdrrrrrr)s* $ rr{Gz?cCs$tt||g\}}t|}|dd|ddd}tt||g\}}tt||dg}|dd|k}|d|j d} ~~tt||dg}|dd|k} | d|j d} ~~ || ||| d|} tj | dd\} }t| t||fS)aComputes the minDCF metric normally used to evaluate speaker verification systems. The min_DCF is the minimum of the following C_det function computed within the defined threshold range: C_det = c_miss * p_miss * p_target + c_fa * p_fa * (1 -p_target) where p_miss is the missing probability and p_fa is the probability of having a false alarm. Arguments --------- positive_scores : torch.tensor The scores from entries of the same class. negative_scores : torch.tensor The scores from entries of different classes. c_miss : float Cost assigned to a missing error (default 1.0). c_fa : float Cost assigned to a false alarm (default 1.0). p_target: float Prior probability of having a target (default 0.01). Returns ------- minDCF : float The minDCF score. threshold : float The corresponding threshold for the minDCF score. Example ------- >>> positive_scores = torch.tensor([0.6, 0.7, 0.8, 0.5]) >>> negative_scores = torch.tensor([0.4, 0.3, 0.2, 0.1]) >>> val_minDCF, threshold = minDCF(positive_scores, negative_scores) >>> val_minDCF 0.0 rrrNrr) r2rrrr8 unsqueeze transposer7r6rmin)rrc_missc_fap_targetrrbrrp_missrp_fac_detc_minr:rrrminDCFds*) rcseZdZdZfddZd%ddZd%ddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$ZZS)&ClassificationStatsaCComputes statistics pertaining to multi-label classification tasks, as well as tasks that can be loosely interpreted as such for the purpose of evaluations. Example ------- >>> import sys >>> from speechbrain.utils.metric_stats import ClassificationStats >>> cs = ClassificationStats() >>> cs.append( ... ids=["ITEM1", "ITEM2", "ITEM3", "ITEM4"], ... predictions=[ ... "M EY K AH", ... "T EY K", ... "B AE D", ... "M EY K", ... ], ... targets=[ ... "M EY K", ... "T EY K", ... "B AE D", ... "M EY K", ... ], ... categories=[ ... "make", ... "take", ... "bad", ... "make" ... ] ... ) >>> cs.write_stats(sys.stdout) Overall Accuracy: 75% Class-Wise Accuracy ------------------- bad -> B AE D : 1 / 1 (100.00%) make -> M EY K: 1 / 2 (50.00%) take -> T EY K: 1 / 1 (100.00%) Confusion --------- Target: bad -> B AE D -> B AE D : 1 / 1 (100.00%) Target: make -> M EY K -> M EY K : 1 / 2 (50.00%) -> M EY K AH: 1 / 2 (50.00%) Target: take -> T EY K -> T EY K : 1 / 1 (100.00%) >>> summary = cs.summarize() >>> summary['accuracy'] 0.75 >>> summary['classwise_stats'][('bad', 'B AE D')] {'total': 1.0, 'correct': 1.0, 'accuracy': 1.0} >>> summary['classwise_stats'][('make', 'M EY K')] {'total': 2.0, 'correct': 1.0, 'accuracy': 0.5} >>> summary['keys'] [('bad', 'B AE D'), ('make', 'M EY K'), ('take', 'T EY K')] >>> summary['predictions'] ['B AE D', 'M EY K', 'M EY K AH', 'T EY K'] >>> summary['classwise_total'] {('bad', 'B AE D'): 1.0, ('make', 'M EY K'): 2.0, ('take', 'T EY K'): 1.0} >>> summary['classwise_correct'] {('bad', 'B AE D'): 1.0, ('make', 'M EY K'): 1.0, ('take', 'T EY K'): 1.0} >>> summary['classwise_accuracy'] {('bad', 'B AE D'): 1.0, ('make', 'M EY K'): 0.5, ('take', 'T EY K'): 1.0} cst|d|_dSr)superrrr __class__rrrs zClassificationStats.__init__NcCs@|j||j||j||dur|j|dSdS)a/ Appends inputs, predictions and targets to internal lists Arguments --------- ids: list the string IDs for the samples predictions: list the model's predictions (human-interpretable, preferably strings) targets: list the ground truths (human-interpretable, preferably strings) categories: list an additional way to classify training samples. If available, the categories will be combined with targets N)rr% predictionstargets categories)rrrrrrrrr,s   zClassificationStats.appendcsz||}|||||j|jd|_dD]fdd|jdD|jd<q|dur:|j|S|jS)aSummarize the classification metric scores The following statistics are computed: accuracy: the overall accuracy (# correct / # total) confusion_matrix: a dictionary of type {(target, prediction): num_entries} representing the confusion matrix classwise_stats: computes the total number of samples, the number of correct classifications and accuracy for each class keys: all available class keys, which can be either target classes or (category, target) tuples predictions: all available predictions all predictions the model has made Arguments --------- field : str If provided, only returns selected statistic. If not, returns all computed statistics. Returns ------- float or dict Returns a float if ``field`` is provided, otherwise returns a dictionary containing all computed stats. )accuracyconfusion_matrixclasswise_statskeysrrcorrectrcsi|] \}}||qSrr)rNr key_statsstatrr 4sz1ClassificationStats.summarize..r classwise_N)_build_lookups_compute_confusion_matrix_compute_accuracy_compute_classwise_stats_available_keys_available_predictionsrr)rr9rrrrr< s   zClassificationStats.summarizecCs&tddt|j|jDt|jS)Ncss|] \}}||kVqdSrr)rN predictionr"rrrrX>s  z8ClassificationStats._compute_accuracy..)r7r\rrr8rrrrrr =s  z%ClassificationStats._compute_accuracycCsH||_tttdd|jD|_||j|_||j|_ dS)Ncss|]}|VqdSrrrNr rrrrXFsz5ClassificationStats._build_lookups..) _get_keysr rsortedsetrr  _index_lookup _keys_lookup_predictions_lookuprrrrrCs  z"ClassificationStats._build_lookupscCsVtt|jt|j}|D]\}}|j|}|j|}|||fd7<q|S)Nr)r2zerosr8r r _get_confusion_entriesrr)rrrr key_idxprediction_idxrrrrMs  z-ClassificationStats._compute_confusion_matrixcshjdd}js jnddjD}tfddt|D}||}ddtj|||DS)NrrcSsg|]\}}|qSrr)rNrbr"rrrrQaz@ClassificationStats._compute_classwise_stats..cs0g|]\}}|jvr|j|fndqS)r)r)rNidxr"rrrrrQds  cSs.i|]\}}}}||||dqS)r)r)rNr item_total item_correct item_accuracyrrrrns z@ClassificationStats._compute_classwise_stats..)r7rr r2r4rr\)rrr key_targetsrrrrrr Ws    z,ClassificationStats._compute_classwise_statscCs,|jr t|j|j}n|j}ttt|Sr)rr\rrrr)rrrrrryszClassificationStats._get_keyscCs>|jrddt|j|j|jD}nt|j|j}t|}|S)Ncss"|] \}}}||f|fVqdSrr)rNcategoryr"r rrrrXs  z=ClassificationStats._get_confusion_entries..)rr\rrr)rresultrrrrs z*ClassificationStats._get_confusion_entriescCsddt|DS)NcSsi|]\}}||qSrr)rNrrrrrrsz5ClassificationStats._index_lookup..)r)rrrrrrsz!ClassificationStats._index_lookupcCsg|_g|_g|_g|_dS)zClears the collected statisticsN)rrrrrrrrrrzClassificationStats.clearcCsX|jdur |td|jdd|dt|d||t|d||dS)zOutputs the stats to the specified filestream in a human-readable format Arguments --------- filestream: file a file-like object NzOverall Accuracy: rz.0%r)rr<r@_write_classwise_stats_write_confusionrrrrrrDs    zClassificationStats.write_statsc sjd|dfddjD}tdd|D}jD]/}jd|}||}t|dt|d d t|d d |d dd|dqdS)NzClass-Wise AccuracyrAcsi|]}||qSr)_format_key_label)rNrrrrrs z>ClassificationStats._write_classwise_stats..cs|]}t|VqdSrr8)rNlabelrrrrXsz=ClassificationStats._write_classwise_stats..rrr / r (r.2%)r) _write_headerr maxvaluesr_pad_to_lengthr%r@r[)rrA key_labelslongest_key_labelrstats padded_labelrrrr"s   0z*ClassificationStats._write_classwise_statsc Cs|jd|dtdd|jD}|jd}|jdd}t|j||D]K\}}}||}t d||d t |d k\} | }| D])} ||  } |j| } | | |} t d | d | d |d| |dd |d qFq%dS)N Confusionr$csr&rr'rrrrrXs z7ClassificationStats._write_confusion..rrrzTarget: rrz -> rr)r*r+r,)r-r.r rr[r7r\r r%r@r2whererr0)rrAlongest_predictionrtotalsrkey_predictionsr target_labelindexesindexcountr r4rrrr#s2    "z$ClassificationStats._write_confusioncCs$t||dtdt||ddS)Nr-)r@r8)rheaderrArrrr-s z!ClassificationStats._write_headercCstd|t|}|d|S)Nrr>)r.r8)rr(rPpaddingrrrr0s z"ClassificationStats._pad_to_lengthcCs(|jr|\}}|d|}|S|}|S)Nz -> )r)rrr r"r(rrrr%s z%ClassificationStats._format_key_labelr)rErFrGrHrr,r<r rrr rrrrrDr"r#r-r0r% __classcell__rrrrrs& B  0  " rc@s4eZdZdZd ddZddZdd Zdd d Zd S)MultiMetricStatsaT A wrapper that evaluates multiple metrics simultaneously Arguments --------- metric : function The function to use to compute the relevant metrics. Should take at least two arguments (predictions and targets) and can optionally take the relative lengths of either or both arguments. The function should return a dict or a namedtuple n_jobs : int The number of jobs to use for computing the metric. If this is more than one, every sample is processed individually, otherwise the whole batch is passed at once. batch_eval : bool When True it feeds the evaluation metric with the batched input. When False and n_jobs=1, it performs metric evaluation one-by-one in a sequential way. When False and n_jobs>1, the evaluation runs in parallel over the different inputs using joblib. Example ------- >>> def metric(a, b): ... return { ... "sum": a + b, ... "diff": a - b, ... "sum_sq": a**2 + b**2 ... } >>> multi_metric = MultiMetricStats(metric, batch_eval=True) >>> multi_metric.append([1, 2], a=torch.tensor([2.0, 1.0]), b=torch.tensor([1.0, 2.0])) >>> multi_metric.append([3, 4], a=torch.tensor([4.0, 5.0]), b=torch.tensor([0.0, 1.0])) >>> multi_metric.append([5, 6], a=torch.tensor([2.0, 4.0]), b=torch.tensor([4.0, 2.0])) >>> multi_metric.append([7, 8], a=torch.tensor([2.0, 4.0]), b=torch.tensor([4.0, 2.0])) >>> multi_metric.summarize() #doctest: +NORMALIZE_WHITESPACE {'sum': {'average': 5.0, 'min_score': 3.0, 'min_id': 1, 'max_score': 6.0, 'max_id': 4}, 'diff': {'average': 1.0, 'min_score': -2.0, 'min_id': 5, 'max_score': 4.0, 'max_id': 3}, 'sum_sq': {'average': 16.5, 'min_score': 5.0, 'min_id': 1, 'max_score': 26.0, 'max_id': 4}} >>> multi_metric.summarize(flat=True) #doctest: +NORMALIZE_WHITESPACE {'sum_average': 5.0, 'sum_min_score': 3.0, 'sum_min_id': 1, 'sum_max_score': 6.0, 'sum_max_id': 4, 'diff_average': 1.0, 'diff_min_score': -2.0, 'diff_min_id': 5, 'diff_max_score': 4.0, 'diff_max_id': 3, 'sum_sq_average': 16.5, 'sum_sq_min_score': 5.0, 'sum_sq_min_id': 1, 'sum_sq_max_score': 26.0, 'sum_sq_max_id': 4} rFcCs&t||_||_||_g|_i|_dSr)_dictifyrrrrmetricsrrrrr&s  zMultiMetricStats.__init__cs|j||jr|j|i|}n6d|vsd|vrtd|jdkr-t|jfi|n td|j|jd|d }fdd|D}| D]\}}||j vr`t d d d d |j |<|j | ||qLd S)r r!r"r#rr$rcs&i|]tfddDqS)csg|]}|qSrr)rNr`rrrrQOrz6MultiMetricStats.append...)r2r4)rN scores_rawrErrNsz+MultiMetricStats.append..cSs|Srr)xrrrUsz)MultiMetricStats.append..T)rNr)rr%r eval_simpler'rr(rr)rrrDrr,)rrr*r+rrr metric_scoresrrFrr,-s.     zMultiMetricStats.appendcOs"|j|i|}dd|DS)z3Evaluates the metric in a simple, sequential mannercSsi|] \}}||qSr)r&)rNrr`rrrr[sz0MultiMetricStats.eval_simple..)rr)rr*r+rrrrrJXszMultiMetricStats.eval_simpleNcs2fdd|jD}|rdd|D}|S)aSummarize the metric scores, returning relevant stats. Arguments --------- field : str If provided, only returns selected statistic. If not, returns all computed statistics. flat : bool whether to flatten the dictionary Returns ------- dict Returns a dictionary of all computed stats csi|] \}}||qSr)r<)rNrrr9rrrmsz.MultiMetricStats.summarize..cSs2i|]\}}|D] \}}|d||q qS)rb)r)rNrfieldsr9valuerrrrqs)rDr)rr9flatr!rrLrr<]s zMultiMetricStats.summarize)rF)NF)rErFrGrHrr,rJr<rrrrrBs  B+rBcsdfdd}|S)aA wrapper that converts functions returning namedtuples to functions returning dicts while leaving functions returning dicts intact Arguments --------- f : callable a function Returns ------- result : callable a wrapped function Ncs0|i|}durt|dr|S|S)zThe wrapper functionN_asdict)hasattrrP)r*r+r!f has_asdictrrwrappers z_dictify..wrapperr)rSrUrrRrrCysrC)NrIr)rrr)"rHtypingrrr2joblibrrspeechbrain.dataio.dataiorrrspeechbrain.dataio.werr r speechbrain.utils.data_utilsr speechbrain.utils.edit_distancer r rrrr)r(rarurrrrrrBrCrrrrs8    02i< H9