Initial commit

README.md

@@ -1,7 +1,5 @@
 ---
 title: Tokens per Byte
-datasets:
--  
 tags:
 - evaluate
 - measurement

tokens_per_byte.py

@@ -71,8 +71,7 @@ class TokensperByte(evaluate.Measurement):
             inputs_description=_KWARGS_DESCRIPTION,
             # This defines the format of each prediction and reference
             features=datasets.Features({
-                'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
+                'text':datasets.Value("string"),
             }),
             # Homepage of the module for documentation
             homepage="http://module.homepage",
@@ -86,10 +85,11 @@ class TokensperByte(evaluate.Measurement):
         # TODO: Download external resources if needed
         pass
 
-    def _compute(self, predictions, references):
+    def _compute(self, text, tokenizer):
         """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+        num_tokens = sum(tokenizer(text, return_length=True, return_attention_mask=False, add_special_tokens=False, return_token_type_ids=False)["length"])
+        num_bytes = sum([len(s.encode('utf-8')) for s in text])
+
         return {
-            "accuracy": accuracy,
-        }
+            "tokens_per_byte": num_tokens / num_bytes,
+        }