Upload artifact.py with huggingface_hub

artifact.py

@@ -6,10 +6,13 @@ import pkgutil
 from abc import abstractmethod
 from copy import deepcopy
 from functools import lru_cache
-from typing import Dict, List, Union, final
+from typing import Dict, List, Optional, Union, final
 
 from .dataclass import AbstractField, Dataclass, Field, InternalField, fields
 from .logging_utils import get_logger
+from .parsing_utils import (
+    separate_inside_and_outside_square_brackets,
+)
 from .settings_utils import get_settings
 from .text_utils import camel_to_snake_case, is_camel_case
 from .type_utils import issubtype
@@ -28,10 +31,19 @@ class Artifactories:
         return cls.instance
 
     def __iter__(self):
-        return iter(self.artifactories)
+        self._index = 0  # Initialize/reset the index for iteration
+        return self
 
     def __next__(self):
-        return next(self.artifactories)
+        while self._index < len(self.artifactories):
+            artifactory = self.artifactories[self._index]
+            self._index += 1
+            if (
+                settings.use_only_local_catalogs and not artifactory.is_local
+            ):  # Corrected typo from 'is_loacl' to 'is_local'
+                continue
+            return artifactory
+        raise StopIteration
 
     def register(self, artifactory):
         assert isinstance(
@@ -173,31 +185,33 @@ class Artifact(Dataclass):
         return clz in set(cls._class_register.values())
 
     @classmethod
-    def _recursive_load(cls, d):
-        if isinstance(d, dict):
+    def _recursive_load(cls, obj):
+        if isinstance(obj, dict):
             new_d = {}
-            for key, value in d.items():
+            for key, value in obj.items():
                 new_d[key] = cls._recursive_load(value)
-            d = new_d
-        elif isinstance(d, list):
-            d = [cls._recursive_load(value) for value in d]
+            obj = new_d
+        elif isinstance(obj, list):
+            obj = [cls._recursive_load(value) for value in obj]
         else:
             pass
-        if cls.is_artifact_dict(d):
-            cls.verify_artifact_dict(d)
-            return cls._class_register[d.pop("type")](**d)
+        if cls.is_artifact_dict(obj):
+            cls.verify_artifact_dict(obj)
+            return cls._class_register[obj.pop("type")](**obj)
 
-        return d
+        return obj
 
     @classmethod
-    def from_dict(cls, d):
+    def from_dict(cls, d, overwrite_args=None):
+        if overwrite_args is not None:
+            d = {**d, **overwrite_args}
         cls.verify_artifact_dict(d)
         return cls._recursive_load(d)
 
     @classmethod
-    def load(cls, path, artifact_identifier=None):
+    def load(cls, path, artifact_identifier=None, overwrite_args=None):
         d = load_json(path)
-        new_artifact = cls.from_dict(d)
+        new_artifact = cls.from_dict(d, overwrite_args=overwrite_args)
         new_artifact.artifact_identifier = artifact_identifier
         return new_artifact
 
@@ -267,6 +281,10 @@ class Artifactory(Artifact):
     def __getitem__(self, name) -> Artifact:
         pass
 
+    @abstractmethod
+    def get_with_overwrite(self, name, overwrite_args) -> Artifact:
+        pass
+
 
 class UnitxtArtifactNotFoundError(Exception):
     def __init__(self, name, artifactories):
@@ -285,14 +303,24 @@ def fetch_artifact(name):
     if Artifact.is_artifact_file(name):
         return Artifact.load(name), None
 
-    for artifactory in Artifactories():
-        if settings.use_only_local_catalogs:
-            if not artifactory.is_local:
-                continue
+    artifactory, name, args = get_artifactory_name_and_args(name=name)
+
+    return artifactory.get_with_overwrite(name, overwrite_args=args), artifactory
+
+
+def get_artifactory_name_and_args(
+    name: str, artifactories: Optional[List[Artifactory]] = None
+):
+    name, args = separate_inside_and_outside_square_brackets(name)
+
+    if artifactories is None:
+        artifactories = list(Artifactories())
+
+    for artifactory in artifactories:
         if name in artifactory:
-            return artifactory[name], artifactory
+            return artifactory, name, args
 
-    raise UnitxtArtifactNotFoundError(name, Artifactories().artifactories)
+    raise UnitxtArtifactNotFoundError(name, artifactories)
 
 
 @lru_cache(maxsize=None)