Upload artifact.py with huggingface_hub
Browse files- artifact.py +23 -13
artifact.py
CHANGED
|
@@ -2,19 +2,36 @@ import inspect
|
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import pkgutil
|
| 5 |
-
import re
|
| 6 |
from abc import ABC, abstractmethod
|
| 7 |
from dataclasses import asdict, dataclass, field, fields
|
| 8 |
from typing import final
|
| 9 |
|
|
|
|
|
|
|
| 10 |
|
| 11 |
class AbstractField:
|
| 12 |
pass
|
| 13 |
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
class BaseArtifact(ABC):
|
|
@@ -77,7 +94,7 @@ class BaseArtifact(ABC):
|
|
| 77 |
value = getattr(self, field.name)
|
| 78 |
if isinstance(value, str):
|
| 79 |
artifact, artifactory = fetch_artifact(value)
|
| 80 |
-
assert artifact is not None, f"Artifact {value} does not exist, in {
|
| 81 |
print(f"Artifact {value} is fetched from {artifactory}")
|
| 82 |
setattr(self, field.name, artifact)
|
| 83 |
|
|
@@ -162,18 +179,11 @@ def fetch_artifact(name):
|
|
| 162 |
if Artifact.is_artifact_file(name):
|
| 163 |
return Artifact.load(name), None
|
| 164 |
else:
|
| 165 |
-
for artifactory in
|
| 166 |
if name in artifactory:
|
| 167 |
return artifactory[name], artifactory
|
| 168 |
|
| 169 |
-
raise UnitxtArtifactNotFoundError(name, artifactories)
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
def register_atrifactory(artifactory):
|
| 173 |
-
assert isinstance(artifactory, Artifactory), "Artifactory must be an instance of Artifactory"
|
| 174 |
-
assert hasattr(artifactory, "__contains__"), "Artifactory must have __contains__ method"
|
| 175 |
-
assert hasattr(artifactory, "__getitem__"), "Artifactory must have __getitem__ method"
|
| 176 |
-
artifactories.append(artifactory)
|
| 177 |
|
| 178 |
|
| 179 |
def register_all_artifacts(path):
|
|
|
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import pkgutil
|
|
|
|
| 5 |
from abc import ABC, abstractmethod
|
| 6 |
from dataclasses import asdict, dataclass, field, fields
|
| 7 |
from typing import final
|
| 8 |
|
| 9 |
+
from .text_utils import camel_to_snake_case, is_camel_case
|
| 10 |
+
|
| 11 |
|
| 12 |
class AbstractField:
|
| 13 |
pass
|
| 14 |
|
| 15 |
|
| 16 |
+
class Artifactories(object):
|
| 17 |
+
def __new__(cls):
|
| 18 |
+
if not hasattr(cls, 'instance'):
|
| 19 |
+
cls.instance = super(Artifactories, cls).__new__(cls)
|
| 20 |
+
cls.instance.artifactories = []
|
| 21 |
+
|
| 22 |
+
return cls.instance
|
| 23 |
|
| 24 |
+
def __iter__(self):
|
| 25 |
+
return iter(self.artifactories)
|
| 26 |
+
|
| 27 |
+
def __next__(self):
|
| 28 |
+
return next(self.artifactories)
|
| 29 |
+
|
| 30 |
+
def register_atrifactory(self, artifactory):
|
| 31 |
+
assert isinstance(artifactory, Artifactory), "Artifactory must be an instance of Artifactory"
|
| 32 |
+
assert hasattr(artifactory, "__contains__"), "Artifactory must have __contains__ method"
|
| 33 |
+
assert hasattr(artifactory, "__getitem__"), "Artifactory must have __getitem__ method"
|
| 34 |
+
self.artifactories.append(artifactory)
|
| 35 |
|
| 36 |
|
| 37 |
class BaseArtifact(ABC):
|
|
|
|
| 94 |
value = getattr(self, field.name)
|
| 95 |
if isinstance(value, str):
|
| 96 |
artifact, artifactory = fetch_artifact(value)
|
| 97 |
+
assert artifact is not None, f"Artifact {value} does not exist, in {Artifactories()}"
|
| 98 |
print(f"Artifact {value} is fetched from {artifactory}")
|
| 99 |
setattr(self, field.name, artifact)
|
| 100 |
|
|
|
|
| 179 |
if Artifact.is_artifact_file(name):
|
| 180 |
return Artifact.load(name), None
|
| 181 |
else:
|
| 182 |
+
for artifactory in Artifactories():
|
| 183 |
if name in artifactory:
|
| 184 |
return artifactory[name], artifactory
|
| 185 |
|
| 186 |
+
raise UnitxtArtifactNotFoundError(name, Artifactories().artifactories)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
|
| 188 |
|
| 189 |
def register_all_artifacts(path):
|