Update Space (evaluate main: 7e21410f)

README.md

@@ -1,12 +1,112 @@
 ---
-title: Smape
-emoji: 👁
-colorFrom: purple
-colorTo: blue
+title: sMAPE
+emoji: 🤗 
+colorFrom: blue
+colorTo: red
 sdk: gradio
-sdk_version: 3.9
+sdk_version: 3.0.2
 app_file: app.py
 pinned: false
+tags:
+- evaluate
+- metric
+description: >-
+  Symmetric Mean Absolute Percentage Error (sMAPE) is the symmetric mean percentage error difference between the predicted and actual values defined by Chen and Yang (2004).
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Metric Card for sMAPE
+
+
+## Metric Description
+
+Symmetric Mean Absolute Error (sMAPE) is the symmetric mean of the percentage error of difference between the predicted $x_i$ and actual $y_i$ numeric values:
+
+![image](https://user-images.githubusercontent.com/8100/200009801-ae8be6c8-facf-401b-8df0-3f80a458b9f4.png)
+
+
+## How to Use
+
+At minimum, this metric requires predictions and references as inputs.
+
+```python
+>>> smape_metric = evaluate.load("smape")
+>>> predictions = [2.5, 0.0, 2, 8]
+>>> references = [3, -0.5, 2, 7]
+>>> results = smape_metric.compute(predictions=predictions, references=references)
+```
+
+### Inputs
+
+Mandatory inputs: 
+- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values.
+- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values.
+
+Optional arguments:
+- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`.
+- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`.
+  - `raw_values` returns a full set of errors in case of multioutput input.
+  - `uniform_average` means that the errors of all outputs are averaged with uniform weight. 
+  - the array-like value defines weights used to average errors.
+
+### Output Values
+This metric outputs a dictionary, containing the mean absolute error score, which is of type:
+- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned.
+- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. 
+
+Each sMAPE `float` value ranges from `0.0` to `2.0`, with the best value being 0.0.
+
+Output Example(s):
+```python
+{'smape': 0.5}
+```
+
+If `multioutput="raw_values"`:
+```python
+{'smape': array([0.5, 1.5 ])}
+```
+
+#### Values from Popular Papers
+
+
+### Examples
+
+Example with the `uniform_average` config:
+```python
+>>> smape_metric = evaluate.load("smape")
+>>> predictions = [2.5, 0.0, 2, 8]
+>>> references = [3, -0.5, 2, 7]
+>>> results = smape_metric.compute(predictions=predictions, references=references)
+>>> print(results)
+{'smape': 0.5787...}
+```
+
+Example with multi-dimensional lists, and the `raw_values` config:
+```python
+>>> smape_metric = evaluate.load("smape", "multilist")
+>>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
+>>> references = [[0.1, 2], [-1, 2], [8, -5]]
+>>> results = smape_metric.compute(predictions=predictions, references=references)
+>>> print(results)
+{'smape': 0.8874...}
+>>> results = smape_metric.compute(predictions=predictions, references=references, multioutput='raw_values')
+>>> print(results)
+{'smape': array([1.3749..., 0.4])}
+```
+
+## Limitations and Bias
+This metric is called a measure of "percentage error" even though there is no multiplier of 100. The range is between (0, 2) with it being two when the target and prediction are both zero. 
+
+## Citation(s)
+
+```bibtex
+@article{article,
+    author = {Chen, Zhuo and Yang, Yuhong},
+    year = {2004},
+    month = {04},
+    pages = {},
+    title = {Assessing forecast accuracy measures}
+}
+```
+
+## Further References
+- [Symmetric Mean absolute percentage error - Wikipedia](https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error)

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("smape")
+launch_gradio_widget(module)

requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/huggingface/evaluate@7e21410f9bcff651452f188b702cc80ecd3530e6
+sklearn

smape.py

@@ -0,0 +1,158 @@
+# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""sMAPE - Symmetric Mean Absolute Percentage Error Metric"""
+
+import datasets
+import numpy as np
+from sklearn.metrics._regression import _check_reg_targets
+from sklearn.utils.validation import check_consistent_length
+
+import evaluate
+
+
+_CITATION = """\
+@article{article,
+    author = {Chen, Zhuo and Yang, Yuhong},
+    year = {2004},
+    month = {04},
+    pages = {},
+    title = {Assessing forecast accuracy measures}
+}
+"""
+
+_DESCRIPTION = """\
+Symmetric Mean Absolute Percentage Error (sMAPE) is the symmetric mean percentage error
+difference between the predicted and actual values as defined by Chen and Yang (2004),
+based on the metric by Armstrong (1985) and Makridakis (1993).
+"""
+
+
+_KWARGS_DESCRIPTION = """
+Args:
+    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+    references: array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+    sample_weight: array-like of shape (n_samples,), default=None
+        Sample weights.
+    multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average"
+        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.
+
+                 "raw_values" : Returns a full set of errors in case of multioutput input.
+
+                 "uniform_average" : Errors of all outputs are averaged with uniform weight.
+
+Returns:
+    smape : symmetric mean absolute percentage error.
+        If multioutput is "raw_values", then symmetric mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned.
+        sMAPE output is non-negative floating point in the range (0, 2). The best value is 0.0.
+Examples:
+
+    >>> smape_metric = evaluate.load("smape")
+    >>> predictions = [2.5, 0.0, 2, 8]
+    >>> references = [3, -0.5, 2, 7]
+    >>> results = smape_metric.compute(predictions=predictions, references=references)
+    >>> print(results)
+    {'smape': 0.5787878787878785}
+
+    If you're using multi-dimensional lists, then set the config as follows :
+
+    >>> smape_metric = evaluate.load("smape", "multilist")
+    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
+    >>> references = [[0.1, 2], [-1, 2], [8, -5]]
+    >>> results = smape_metric.compute(predictions=predictions, references=references)
+    >>> print(results)
+    {'smape': 0.49696969558995985}
+    >>> results = smape_metric.compute(predictions=predictions, references=references, multioutput='raw_values')
+    >>> print(results)
+    {'smape': array([0.48888889, 0.50505051])}
+"""
+
+
+def symmetric_mean_absolute_percentage_error(y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"):
+    """Symmetric Mean absolute percentage error (sMAPE) metric using sklearn's api and helpers.
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Ground truth (correct) target values.
+    y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs)
+        Estimated target values.
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+    multioutput : {'raw_values', 'uniform_average'} or array-like
+        Defines aggregating of multiple output values.
+        Array-like value defines weights used to average errors.
+        If input is list then the shape must be (n_outputs,).
+        'raw_values' :
+            Returns a full set of errors in case of multioutput input.
+        'uniform_average' :
+            Errors of all outputs are averaged with uniform weight.
+    Returns
+    -------
+    loss : float or ndarray of floats
+        If multioutput is 'raw_values', then mean absolute percentage error
+        is returned for each output separately.
+        If multioutput is 'uniform_average' or an ndarray of weights, then the
+        weighted average of all output errors is returned.
+        sMAPE output is non-negative floating point. The best value is 0.0.
+    """
+    y_type, y_true, y_pred, multioutput = _check_reg_targets(y_true, y_pred, multioutput)
+    check_consistent_length(y_true, y_pred, sample_weight)
+    epsilon = np.finfo(np.float64).eps
+    smape = 2 * np.abs(y_pred - y_true) / (np.maximum(np.abs(y_true), epsilon) + np.maximum(np.abs(y_pred), epsilon))
+    output_errors = np.average(smape, weights=sample_weight, axis=0)
+    if isinstance(multioutput, str):
+        if multioutput == "raw_values":
+            return output_errors
+        elif multioutput == "uniform_average":
+            # pass None as weights to np.average: uniform mean
+            multioutput = None
+
+    return np.average(output_errors, weights=multioutput)
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class Smape(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(self._get_feature_types()),
+            reference_urls=["https://robjhyndman.com/hyndsight/smape/"],
+        )
+
+    def _get_feature_types(self):
+        if self.config_name == "multilist":
+            return {
+                "predictions": datasets.Sequence(datasets.Value("float")),
+                "references": datasets.Sequence(datasets.Value("float")),
+            }
+        else:
+            return {
+                "predictions": datasets.Value("float"),
+                "references": datasets.Value("float"),
+            }
+
+    def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"):
+
+        smape_score = symmetric_mean_absolute_percentage_error(
+            references,
+            predictions,
+            sample_weight=sample_weight,
+            multioutput=multioutput,
+        )
+
+        return {"smape": smape_score}