Upload hf_utils.py with huggingface_hub
Browse files- hf_utils.py +44 -0
hf_utils.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datasets import (
|
| 2 |
+
disable_caching,
|
| 3 |
+
enable_caching,
|
| 4 |
+
is_caching_enabled,
|
| 5 |
+
)
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from .file_utils import get_all_files_in_dir
|
| 8 |
+
from datasets.utils.py_utils import get_imports
|
| 9 |
+
|
| 10 |
+
HF_CACHING_ENABLED = False
|
| 11 |
+
|
| 12 |
+
class HFCachingContextManager:
|
| 13 |
+
def __init__(self, should_cache):
|
| 14 |
+
self.should_cache = should_cache
|
| 15 |
+
self.was_caching_enabled = None
|
| 16 |
+
|
| 17 |
+
def __enter__(self):
|
| 18 |
+
self.was_caching_enabled = is_caching_enabled()
|
| 19 |
+
if self.should_cache:
|
| 20 |
+
if not self.was_caching_enabled:
|
| 21 |
+
enable_caching()
|
| 22 |
+
else:
|
| 23 |
+
if self.was_caching_enabled:
|
| 24 |
+
disable_caching()
|
| 25 |
+
|
| 26 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 27 |
+
if self.was_caching_enabled != is_caching_enabled():
|
| 28 |
+
if self.was_caching_enabled:
|
| 29 |
+
enable_caching()
|
| 30 |
+
else:
|
| 31 |
+
disable_caching()
|
| 32 |
+
|
| 33 |
+
def set_hf_caching(enabled):
|
| 34 |
+
return HFCachingContextManager(enabled)
|
| 35 |
+
|
| 36 |
+
def get_missing_imports(file, exclude=[]):
|
| 37 |
+
src_dir = Path(__file__).parent
|
| 38 |
+
python_files = get_all_files_in_dir(src_dir, file_extension='.py')
|
| 39 |
+
# get only the file without the path and extension
|
| 40 |
+
required_modules = [Path(p).stem for p in python_files]
|
| 41 |
+
imports = get_imports(file)
|
| 42 |
+
imported_modules = [i[1] for i in imports if i[0] == 'internal']
|
| 43 |
+
missing_imports = [i for i in required_modules if i not in imported_modules and i not in exclude]
|
| 44 |
+
return missing_imports
|