o .wi@sddlmZddlmZmZmZmZmZddlm Z ddl m Z m Z m Z mZmZmZddlZddlmZddlmZddlmZdd lmZdd lmZdd lmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$e slddgZ%de&de&de&fddZ'de&de&de&fddZ(GdddeZ)dS)) OrderedDict)HashableIterableIteratorMappingSequence)deepcopy)AnyClassVarDictListOptionalUnionN)Tensor) ModuleDict)Literal)Metric)rank_zero_warn)_flatten _flatten_dictallclose)_MATPLOTLIB_AVAILABLE)_AX_TYPE_PLOT_OUT_TYPEplot_single_or_multi_valMetricCollection.plotzMetricCollection.plot_allstringprefixreturncCs||r |t|dS|S)zPatch for older version with missing method `removeprefix`. >>> _remove_prefix("prefix_string", "prefix_") 'string' >>> _remove_prefix("not_prefix_string", "prefix_") 'not_prefix_string' N) startswithlen)rrr!U/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/torchmetrics/collections.py_remove_prefix#s r#suffixcCs ||r|dt| S|S)zPatch for older version with missing method `removesuffix`. >>> _remove_suffix("string_suffix", "_suffix") 'string' >>> _remove_suffix("string_suffix_missing", "_suffix") 'string_suffix_missing' N)endswithr )rr$r!r!r"_remove_suffix/s r&cs^eZdZUdZeeefed<dgZe e eed<ddddde ede e edfeee edfffd ed e ed e ed e ee e efd df fddZed eeeeefffddZejjdeded eeeffddZdeded dfddZdUddZededed efddZdVded dfd d!Zd eeeffd"d#Zd$ed%deded eeeffd&d'ZdUd(d)ZdWd e ed e ed dfd*d+Z dXd,ed dfd-d.Z!de ede e edfeee edfffd ed dfd/d0Z"dUd1d2Z#ed e$e%e&effd3d4Z'd5ed efd6d7Z(d e)eeffd8d9Z*d e+e,fd:d;Z-dVdZ/dYd>> from torch import tensor >>> from pprint import pprint >>> from torchmetrics import MetricCollection >>> from torchmetrics.regression import MeanSquaredError >>> from torchmetrics.classification import MulticlassAccuracy, MulticlassPrecision, MulticlassRecall >>> target = tensor([0, 2, 0, 2, 0, 1, 0, 2]) >>> preds = tensor([2, 1, 2, 0, 1, 2, 2, 2]) >>> metrics = MetricCollection([MulticlassAccuracy(num_classes=3, average='micro'), ... MulticlassPrecision(num_classes=3, average='macro'), ... MulticlassRecall(num_classes=3, average='macro')]) >>> metrics(preds, target) # doctest: +NORMALIZE_WHITESPACE {'MulticlassAccuracy': tensor(0.1250), 'MulticlassPrecision': tensor(0.0667), 'MulticlassRecall': tensor(0.1111)} Example:: Alternatively, metrics can be passed in as arguments. The keys of the output dict will be the same as the class name of the metric: >>> metrics = MetricCollection(MulticlassAccuracy(num_classes=3, average='micro'), ... MulticlassPrecision(num_classes=3, average='macro'), ... MulticlassRecall(num_classes=3, average='macro')) >>> metrics(preds, target) # doctest: +NORMALIZE_WHITESPACE {'MulticlassAccuracy': tensor(0.1250), 'MulticlassPrecision': tensor(0.0667), 'MulticlassRecall': tensor(0.1111)} Example:: If multiple of the same metric class (with different parameters) should be chained together, metrics can be passed in as a dict and the output dict will have the same keys as the input dict: >>> metrics = MetricCollection({'micro_recall': MulticlassRecall(num_classes=3, average='micro'), ... 'macro_recall': MulticlassRecall(num_classes=3, average='macro')}) >>> same_metric = metrics.clone() >>> pprint(metrics(preds, target)) {'macro_recall': tensor(0.1111), 'micro_recall': tensor(0.1250)} >>> pprint(same_metric(preds, target)) {'macro_recall': tensor(0.1111), 'micro_recall': tensor(0.1250)} Example:: Metric collections can also be nested up to a single time. The output of the collection will still be a single dict with the prefix and postfix arguments from the nested collection: >>> metrics = MetricCollection([ ... MetricCollection([ ... MulticlassAccuracy(num_classes=3, average='macro'), ... MulticlassPrecision(num_classes=3, average='macro') ... ], postfix='_macro'), ... MetricCollection([ ... MulticlassAccuracy(num_classes=3, average='micro'), ... MulticlassPrecision(num_classes=3, average='micro') ... ], postfix='_micro'), ... ], prefix='valmetrics/') >>> pprint(metrics(preds, target)) # doctest: +NORMALIZE_WHITESPACE {'valmetrics/MulticlassAccuracy_macro': tensor(0.1111), 'valmetrics/MulticlassAccuracy_micro': tensor(0.1250), 'valmetrics/MulticlassPrecision_macro': tensor(0.0667), 'valmetrics/MulticlassPrecision_micro': tensor(0.1250)} Example:: The `compute_groups` argument allow you to specify which metrics should share metric state. By default, this will automatically be derived but can also be set manually. >>> metrics = MetricCollection( ... MulticlassRecall(num_classes=3, average='macro'), ... MulticlassPrecision(num_classes=3, average='macro'), ... MeanSquaredError(), ... compute_groups=[['MulticlassRecall', 'MulticlassPrecision'], ['MeanSquaredError']] ... ) >>> metrics.update(preds, target) >>> pprint(metrics.compute()) {'MeanSquaredError': tensor(2.3750), 'MulticlassPrecision': tensor(0.0667), 'MulticlassRecall': tensor(0.1111)} >>> pprint(metrics.compute_groups) {0: ['MulticlassRecall', 'MulticlassPrecision'], 1: ['MeanSquaredError']} _modules metric_state__jit_unused_properties__NT)rpostfixcompute_groupsmetricsadditional_metricsrr+r,rcsTt||d|_||d|_||_d|_d|_i|_|j |g|RdS)Nrr+F) super__init__ _check_argrr+_enable_compute_groups_groups_checked_state_is_copy_groups add_metrics)selfr-rr+r,r. __class__r!r"r0s zMetricCollection.__init__cCsdd|jdddDS)z$Get the current state of the metric.cSsi|]\}}||jqSr!)r)).0kmr!r!r" sz1MetricCollection.metric_state..F keep_base copy_state)itemsr7r!r!r"r)szMetricCollection.metric_stateargskwargscOs|jdg|Ri|S)zCall forward for each metric sequentially. Positional arguments (args) will be passed to every metric in the collection, while keyword arguments (kwargs) will be filtered based on the signature of the individual metric. forward_compute_and_reduce)r7rCrDr!r!r"rEszMetricCollection.forwardc Os|jr<|jddD] }t|t|}d|_q |jD]}t||d}|j|i|jdi|qd|_ | dS|jddD]}|jdi|}|j|i|qB|j rh| d|_ | d|_dSdS)zCall update for each metric sequentially. Positional arguments (args) will be passed to every metric in the collection, while keyword arguments (kwargs) will be filtered based on the signature of the individual metric. Tr?NrFr@r!) r3keysgetattrstr _computedr5valuesupdate_filter_kwargsr4 _compute_groups_create_state_refr2_merge_compute_groups) r7rCrDr;micgm0r<m_kwargsr!r!r"rOs$  zMetricCollection.updatec Cst|j} t|jD]@\}}t|jD]+\}}||kr!qt||d}t||d}|||rC|j||j|nqt|j|krMnq t|j|krVnt|j}qt|j}i|_t| D] \} } | |j| <qjdS)aAIterate over the collection of metrics, checking if the state of each metric matches another. If so, their compute groups will be merged into one. The complexity of the method is approximately ``O(number_of_metrics_in_collection ** 2)``, as all metrics need to be compared to all other metrics. TrN) r r5rrArK_equal_metric_statesextendpop enumeraterN) r7 num_groupscg_idx1 cg_members1cg_idx2 cg_members2metric1metric2tempidxrNr!r!r"rR s0     z&MetricCollection._merge_compute_groupsr`racCst|jdkst|jdkrdS|j|jkrdS|jD]H}t||}t||}t|t|kr6dSt|trNt|trN|j|jkrKt||sNdSt|t rgt|t rgt ddt ||DsgdSqdS)z6Check if the metric state of two metrics are the same.rFcss*|]\}}|j|jkot||VqdSN)shaper)r:s1s2r!r!r" Ls(z8MetricCollection._equal_metric_states..T) r _defaultsrJrKtype isinstancerrerlistallzip)r`rakeystate1state2r!r!r"rW1s0   z%MetricCollection._equal_metric_statesFcopycCs|jsE|jD]<}t||d}tdt|D]+}t|||}|jD]}t||}t|||r4t|n|q$|r?t|j n|j |_ qq||_dS)zCreate reference between metrics in the same compute group. Args: copy: If `True` the metric state will between members will be copied instead of just passed by reference rN) r4r5rNrKranger risetattrr _update_count)r7rrrTrUirSstatem0_stater!r!r"rQRs   z1MetricCollection._compute_groups_create_state_refcCs |dS)z5Compute the result for each metric in the collection.computerFrBr!r!r"rzfs zMetricCollection.compute method_name)rzrEcs^i}jdddD]*\}}|dkr|}n|dkr(||i|jdi|}ntd||||<q t|\}} i} jdddD]`\}}||}t|tr|D]I\} } | ru|t|ddd} | t|d dd} | d | } t|d d r|j d ur|j | } t|d d r|j d ur| |j } | | | <qTqC|| |<qCfd d| DS)aCompute result from collection and reduce into a single dictionary. Args: method_name: The method to call on each metric in the collection. Should be either `compute` or `forward`. args: Positional arguments to pass to each metric (if method_name is `forward`) kwargs: Keyword arguments to pass to each metric (if method_name is `forward`) Raises: ValueError: If method_name is not `compute` or `forward`. TFr>rzrEz=method_name should be either 'compute' or 'forward', but got rr+__from_collectionNcsi|] \}}||qSr!) _set_name)r:r;vrBr!r"r=z8MetricCollection._compute_and_reduce..r!) rArzrP ValueErrorrrkdictreplacerKrr+)r7r{rCrDresultr;r<resr} duplicatesflattened_resultsror stripped_kr!rBr"rGjs4      z$MetricCollection._compute_and_reducecCs:|jddD]}|q|jr|jr|dSdSdS)z(Call reset for each metric sequentially.FrIN)rNresetr2r3rQ)r7r<r!r!r"rs    zMetricCollection.resetcCs0t|}|r ||d|_|r||d|_|S)zMake a copy of the metric collection. Args: prefix: a string to append in front of the metric keys postfix: a string to append after the keys of the output dict. rr+)rr1rr+)r7rr+mcr!r!r"clones zMetricCollection.clonemodecCs |jddD]}||qdS)zOChange if metric states should be saved to its state_dict after initialization.FrIN)rN persistent)r7rr<r!r!r"rs zMetricCollection.persistentc Gs&t|tr|g}t|tr1t|}g}|D]}t|tr|n|}||q|r0td|dn |r>td|d|dt|trt| D]A}||}t|tt fsatd|d|dt|trk|||<qI|j dd D]\}} |j | _ |j | _ d | _| ||d |<qqqInut|tr|D]D}t|tt fstd |d t|tr|jj}||vrtd||||<q|j dd D]\}} |j | _ |j | _ d | _| ||<qqn(t|t r|j dd D]\}}||vrtd|d|||<qntd|d|_|jr|dSi|_dS)z%Add new metrics to Metric Collection.z You have passes extra arguments z0 which are not `Metric` so they will be ignored.z7 which are not compatible with first passed dictionary z so they will be ignored.zValue z belonging to key zO is not an instance of `torchmetrics.Metric` or `torchmetrics.MetricCollection`FrHTr}Input zd to `MetricCollection` is not a instance of `torchmetrics.Metric` or `torchmetrics.MetricCollection`z#Encountered two metrics both named zMetric with name 'z#' already exists in the collection.zxUnknown input to MetricCollection. Expected, `Metric`, `MetricCollection` or `dict`/`sequence` of the previous, but got N)rkrrrlappendrrrsortedrJr'rAr+rr~r9__name__r3r2_init_compute_groupsr5) r7r-r.remainr<selnamemetricr;rr!r!r"r6s                zMetricCollection.add_metricsc Cst|jtrYtt|j|_|jD]}|D]}||vr/td|d|jd|jddqqt |j}t |j}|jddD]}||vrS|g|j|<|d7}qCd|_ d Sddt|jddD|_d S) zInitialize compute groups. If user provided a list, we check that all metrics in the list are also in the collection. If set to `True` we simply initialize each metric in the collection as its own group rz_ in `compute_groups` argument does not match a metric in the collection. Please make sure that z matches TrHrscSsi|] \}}|t|gqSr!)rL)r:rwr;r!r!r"r="rz9MetricCollection._init_compute_groups..N) rkr2rlrrZr5rNrrJrr r3)r7rralready_in_groupcounterr;r!r!r"r s.      z%MetricCollection._init_compute_groupscCs|jS)z@Return a dict with the current compute groups in the collection.)r5rBr!r!r"r,$szMetricCollection.compute_groupsbasecCs0|jdur|n|j|}|jdur|S||jS)z3Adjust name of metric with both prefix and postfix.N)rr+)r7rrr!r!r"r)szMetricCollection._set_namecCs<t|jtr tni}|jD] \}}||||<q|Srd)rkr(rrAr)r7 dict_modulesr;rr!r!r"_to_renamed_dict.sz!MetricCollection._to_renamed_dictcCs t|S)z3Return an iterator over the keys of the MetricDict.)iterrJrBr!r!r"__iter__5s zMetricCollection.__iter__r?cCs|r|jS|S)zReturn an iterable of the ModuleDict key. Args: keep_base: Whether to add prefix/postfix on the items collection. )r(rJr)r7r?r!r!r"rJ:s  zMetricCollection.keysr@cCs$|||r |jS|S)a&Return an iterable of the ModuleDict key/value pairs. Args: keep_base: Whether to add prefix/postfix on the collection. copy_state: If metric states should be copied between metrics in the same compute group or just passed by reference )rQr(rAr)r7r?r@r!r!r"rAEs   zMetricCollection.itemscCs|||jS)zReturn an iterable of the ModuleDict values. Args: copy_state: If metric states should be copied between metrics in the same compute group or just passed by reference )rQr(rN)r7r@r!r!r"rNSs  zMetricCollection.valuesrocCs8|||jrt||j}|jrt||j}|j|S)aRetrieve a single metric from the collection. Args: key: name of metric to retrieve copy_state: If metric states should be copied between metrics in the same compute group or just passed by reference )rQrr#r+r&r()r7ror@r!r!r" __getitem__^s    zMetricCollection.__getitem__argrcCs.|dus t|tr |Std|dt|)NzExpected input `z` to be a string, but got )rkrLrrj)rrr!r!r"r1nszMetricCollection._check_argcsbtdd}|jr|d|j|jrdnd7}|jr-||js$dndd|j7}|dS)z[Return the representation of the metric collection including all metrics in the collection.Nz , prefix=,r|z postfix=z ))r/__repr__rr+)r7repr_strr8r!r"rts zMetricCollection.__repr__dst_typecCs |jddD]}||q|S)zTransfer all metric state to specific dtype. Special version of standard `type` method. Arguments: dst_type: the desired type as ``torch.dtype`` or string. FrI)rN set_dtype)r7rr<r!r!r"r}s zMetricCollection.set_dtypevalaxtogetherc sRt|tstdt||durJ|r#t|ts#tdt|d|sJt|tr;tdd|Dr;t|t|ksJtdt|dt|d |pO|}|rXt ||d Sg}t |j d d d D]C\}\}t|t r|j ||dur{||n|d \}}nt|tr|j fd d|D|dur||n|d \}}|||fqc|S)a Plot a single or multiple values from the metric. The plot method has two modes of operation. If argument `together` is set to `False` (default), the `.plot` method of each metric will be called individually and the result will be list of figures. If `together` is set to `True`, the values of all metrics will instead be plotted in the same figure. Args: val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results. If no value is provided, will automatically call `metric.compute` and plot that result. ax: Either a single instance of matplotlib axis object or an sequence of matplotlib axis objects. If provided, will add the plots to the provided axis objects. If not provided, will create a new. If argument `together` is set to `True`, a single object is expected. If `together` is set to `False`, the number of axis objects needs to be the same length as the number of metrics in the collection. together: If `True`, will plot all metrics in the same axis. If `False`, will plot each metric in a separate Returns: Either install tuple of Figure and Axes object or an sequence of tuples with Figure and Axes object for each metric in the collection. Raises: ModuleNotFoundError: If `matplotlib` is not installed ValueError: If `together` is not an bool ValueError: If `ax` is not an instance of matplotlib axis object or a sequence of matplotlib axis objects .. plot:: :scale: 75 >>> # Example plotting a single value >>> import torch >>> from torchmetrics import MetricCollection >>> from torchmetrics.classification import BinaryAccuracy, BinaryPrecision, BinaryRecall >>> metrics = MetricCollection([BinaryAccuracy(), BinaryPrecision(), BinaryRecall()]) >>> metrics.update(torch.rand(10), torch.randint(2, (10,))) >>> fig_ax_ = metrics.plot() .. plot:: :scale: 75 >>> # Example plotting multiple values >>> import torch >>> from torchmetrics import MetricCollection >>> from torchmetrics.classification import BinaryAccuracy, BinaryPrecision, BinaryRecall >>> metrics = MetricCollection([BinaryAccuracy(), BinaryPrecision(), BinaryRecall()]) >>> values = [] >>> for _ in range(10): ... values.append(metrics(torch.rand(10), torch.randint(2, (10,)))) >>> fig_, ax_ = metrics.plot(values, together=True) z6Expected argument `together` to be a boolean, but got Nz?Expected argument `ax` to be a matplotlib axis object, but got z when `together=True`css|]}t|tVqdSrd)rkr)r:ar!r!r"rhsz(MetricCollection.plot..zExpected argument `ax` to be a sequence of matplotlib axis objects with the same length as the number of metrics in the collection, but got z with len z when `together=False`)rFr>csg|]}|qSr!r!)r:rr;r!r" sz)MetricCollection.plot..)rkboolrrjrrrmr rzrrZrArplotr) r7rrrfig_axsrwr<frr!rr"rs> :   ( 0r)rN)F)NN)T)FT)NNF)s$