implement metric

.gitignore

@@ -0,0 +1,133 @@
+.vscode
+data/
+output/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

README.md

@@ -3,7 +3,7 @@ title: matching_series
 tags:
 - evaluate
 - metric
-description: 'TODO: add a description here'
+description: "Matching-based time-series generation metric"
 sdk: gradio
 sdk_version: 4.36.1
 app_file: app.py

__main__.py

@@ -0,0 +1,37 @@
+import json
+import logging
+from argparse import ArgumentParser
+
+import evaluate
+import numpy as np
+
+logger = logging.getLogger(__name__)
+parser = ArgumentParser(
+    description="Compute the matching series score between two time series freezed in a numpy array"
+)
+parser.add_argument("predictions", type=str, help="Path to the numpy array containing the predictions")
+parser.add_argument("references", type=str, help="Path to the numpy array containing the references")
+parser.add_argument("--output", type=str, help="Path to the output file")
+parser.add_argument("--batch_size", type=int, help="Batch size to use for the computation")
+args = parser.parse_args()
+
+if not args.predictions or not args.references:
+    raise ValueError("You must provide the path to the predictions and references numpy arrays")
+
+predictions = np.load(args.predictions)
+references = np.load(args.references)
+
+
+logger.info(f"predictions shape: {predictions.shape}")
+logger.info(f"references shape: {references.shape}")
+
+import matching_series
+
+metric = matching_series.matching_series()
+# metric = evaluate.load("matching_series.py")
+results = metric.compute(predictions=predictions, references=references, batch_size=args.batch_size)
+
+print(results)
+if args.output:
+    with open(args.output, "w") as f:
+        json.dump(results, f)

matching_series.py

@@ -13,9 +13,10 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
-import evaluate
 import datasets
-
+import evaluate
+import numpy as np
+import torch
 
 # TODO: Add BibTeX citation
 _CITATION = """\
@@ -36,26 +37,21 @@ This new module is designed to solve this great ML task and is crafted with a lo
 _KWARGS_DESCRIPTION = """
 Calculates how good are predictions given some references, using certain scores
 Args:
-    predictions: list of predictions to score. Each predictions
-        should be a string with tokens separated by spaces.
-    references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.
+    predictions: list of generated time series.
+        shape: (num_generation, num_timesteps, num_features)
+    references: list of reference
+        shape: (num_reference, num_timesteps, num_features)
 Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
 Examples:
     Examples should be written in doctest format, and should illustrate how
     to use the function.
 
-    >>> my_new_module = evaluate.load("my_new_module")
-    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
+    >>> my_new_module = evaluate.load("bowdbeg/matching_series")
+    >>> results = my_new_module.compute(references=[[[0.0, 1.0]]], predictions=[[[0.0, 1.0]]])
     >>> print(results)
-    {'accuracy': 1.0}
+    {'matchin': 1.0}
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class matching_series(evaluate.Metric):
@@ -70,26 +66,102 @@ class matching_series(evaluate.Metric):
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
             # This defines the format of each prediction and reference
-            features=datasets.Features({
-                'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
-            }),
+            features=datasets.Features(
+                {
+                    "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("float"))),
+                    "references": datasets.Sequence(datasets.Sequence(datasets.Value("float"))),
+                }
+            ),
             # Homepage of the module for documentation
             homepage="http://module.homepage",
             # Additional links to the codebase or references
             codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
-            reference_urls=["http://path.to.reference.url/new_module"]
+            reference_urls=["http://path.to.reference.url/new_module"],
         )
 
     def _download_and_prepare(self, dl_manager):
         """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
         pass
 
-    def _compute(self, predictions, references):
-        """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+    def _compute(self, predictions: list | np.ndarray, references: list | np.ndarray, batch_size: None | int = None):
+        """
+        Compute the scores of the module given the predictions and references
+        Args:
+            predictions: list of generated time series.
+                shape: (num_generation, num_timesteps, num_features)
+            references: list of reference
+                shape: (num_reference, num_timesteps, num_features)
+            batch_size: batch size to use for the computation. If None, the whole dataset is processed at once.
+        Returns:
+        """
+        predictions = np.array(predictions)
+        references = np.array(references)
+        if predictions.shape[1:] != references.shape[1:]:
+            raise ValueError(
+                "The number of features in the predictions and references should be the same. predictions: {}, references: {}".format(
+                    predictions.shape[1:], references.shape[1:]
+                )
+            )
+
+        # at first, convert the inputs to numpy arrays
+
+        # MSE between predictions and references for all example combinations for each features
+        # shape: (num_generation, num_reference, num_features)
+        if batch_size is not None:
+            mse = np.zeros((len(predictions), len(references), predictions.shape[-1]))
+            # iterate over the predictions and references in batches
+            for i in range(0, len(predictions) + batch_size, batch_size):
+                for j in range(0, len(references) + batch_size, batch_size):
+                    mse[i : i + batch_size, j : j + batch_size] = np.mean(
+                        (predictions[i : i + batch_size, None] - references[None, j : j + batch_size]) ** 2, axis=-2
+                    )
+        else:
+            mse = np.mean((predictions[:, None] - references) ** 2, axis=1)
+
+        index_mse = mse.diagonal(axis1=0, axis2=1).mean()
+
+        # matching scores
+        mse_mean = mse.mean(axis=-1)
+        # best match for each generated time series
+        # shape: (num_generation,)
+        best_match = np.argmin(mse_mean, axis=-1)
+
+        # matching mse
+        # shape: (num_generation,)
+        matching_mse = mse_mean[np.arange(len(best_match)), best_match].mean()
+
+        # best match for each reference time series
+        # shape: (num_reference,)
+        best_match_inv = np.argmin(mse_mean, axis=0)
+        covered_mse = mse_mean[best_match_inv, np.arange(len(best_match_inv))].mean()
+
+        harmonic_mean = 2 / (1 / matching_mse + 1 / covered_mse)
+
+        # take matching for each feature and compute metrics for them
+        matching_mse_features = []
+        covered_mse_features = []
+        harmonic_mean_features = []
+        index_mse_features = []
+        for f in range(predictions.shape[-1]):
+            mse_f = mse[:, :, f]
+            index_mse_f = mse_f.diagonal(axis1=0, axis2=1).mean()
+            best_match_f = np.argmin(mse_f, axis=-1)
+            matching_mse_f = mse_f[np.arange(len(best_match_f)), best_match_f].mean()
+            best_match_inv_f = np.argmin(mse_f, axis=0)
+            covered_mse_f = mse_f[best_match_inv_f, np.arange(len(best_match_inv_f))].mean()
+            harmonic_mean_f = 2 / (1 / matching_mse_f + 1 / covered_mse_f)
+            matching_mse_features.append(matching_mse_f)
+            covered_mse_features.append(covered_mse_f)
+            harmonic_mean_features.append(harmonic_mean_f)
+            index_mse_features.append(index_mse_f)
+
         return {
-            "accuracy": accuracy,
-        }
+            "matching_mse": matching_mse,
+            "harmonic_mean": harmonic_mean,
+            "covered_mse": covered_mse,
+            "index_mse": index_mse,
+            "matching_mse_features": matching_mse_features,
+            "harmonic_mean_features": harmonic_mean_features,
+            "covered_mse_features": covered_mse_features,
+            "index_mse_features": index_mse_features,
+        }

requirements.txt

@@ -1 +1,2 @@
-git+https://github.com/huggingface/evaluate@main
+git+https://github.com/huggingface/evaluate@main
+numpy