Upload dataset.py with huggingface_hub

dataset.py

@@ -1,3 +1,5 @@
+import os
+
 import datasets
 
 from .artifact import Artifact, UnitxtArtifactNotFoundError
@@ -27,7 +29,7 @@ from .processors import __file__ as _
 from .random_utils import __file__ as _
 from .recipe import __file__ as _
 from .register import __file__ as _
-from .register import register_all_artifacts
+from .register import _reset_env_local_catalogs, register_all_artifacts
 from .renderers import __file__ as _
 from .schema import __file__ as _
 from .split_utils import __file__ as _
@@ -43,6 +45,8 @@ from .validate import __file__ as _
 from .version import __file__ as _
 from .version import version
 
+__default_recipe__ = "common_recipe"
+
 
 def fetch(artifact_name):
     try:
@@ -77,6 +81,18 @@ def parse(query: str):
     return result
 
 
+def get_dataset_artifact(dataset_str):
+    _reset_env_local_catalogs()
+    register_all_artifacts()
+    recipe = fetch(dataset_str)
+    if recipe is None:
+        args = parse(dataset_str)
+        if "type" not in args:
+            args["type"] = os.environ.get("UNITXT_DEFAULT_RECIPE", __default_recipe__)
+        recipe = Artifact.from_dict(args)
+    return recipe
+
+
 class Dataset(datasets.GeneratorBasedBuilder):
     """TODO: Short description of my dataset."""
 
@@ -85,15 +101,25 @@ class Dataset(datasets.GeneratorBasedBuilder):
 
     @property
     def generators(self):
-        register_all_artifacts()
         if not hasattr(self, "_generators") or self._generators is None:
-            recipe = fetch(self.config.name)
-            if recipe is None:
-                args = parse(self.config.name)
-                if "type" not in args:
-                    args["type"] = "common_recipe"
-                recipe = Artifact.from_dict(args)
-            self._generators = recipe()
+            try:
+                from unitxt.dataset import (
+                    get_dataset_artifact as get_dataset_artifact_installed,
+                )
+
+                unitxt_installed = True
+            except ImportError:
+                unitxt_installed = False
+
+            if unitxt_installed:
+                print("Loading with installed unitxt library...")
+                dataset = get_dataset_artifact_installed(self.config.name)
+            else:
+                print("Loading with installed unitxt library...")
+                dataset = get_dataset_artifact(self.config.name)
+
+            self._generators = dataset()
+
         return self._generators
 
     def _info(self):