Update download_assets.py
Browse files- download_assets.py +16 -13
download_assets.py
CHANGED
|
@@ -3,13 +3,16 @@ import os
|
|
| 3 |
|
| 4 |
def download_assets():
|
| 5 |
"""Download necessary assets from Hugging Face Hub."""
|
|
|
|
| 6 |
os.makedirs("data", exist_ok=True)
|
| 7 |
os.makedirs("vectorstore/db_faiss", exist_ok=True)
|
| 8 |
|
| 9 |
repo_id = "MoizK/mindmedic-assets"
|
| 10 |
repo_type = "dataset"
|
| 11 |
-
|
|
|
|
| 12 |
|
|
|
|
| 13 |
pdf_files = [
|
| 14 |
"71763-gale-encyclopedia-of-medicine.-vol.-1.-2nd-ed.pdf",
|
| 15 |
"Depression-NIM-2024.pdf",
|
|
@@ -22,31 +25,31 @@ def download_assets():
|
|
| 22 |
|
| 23 |
for fname in pdf_files:
|
| 24 |
try:
|
| 25 |
-
hf_hub_download(
|
| 26 |
repo_id=repo_id,
|
| 27 |
repo_type=repo_type,
|
| 28 |
-
filename=
|
| 29 |
-
local_dir="
|
| 30 |
-
local_dir_use_symlinks=False,
|
| 31 |
token=token,
|
| 32 |
)
|
| 33 |
-
print(f"Downloaded {fname}")
|
| 34 |
except Exception as e:
|
| 35 |
print(f"⚠️ Failed to download {fname}: {e}")
|
| 36 |
|
| 37 |
-
|
|
|
|
|
|
|
| 38 |
try:
|
| 39 |
-
hf_hub_download(
|
| 40 |
repo_id=repo_id,
|
| 41 |
repo_type=repo_type,
|
| 42 |
-
filename=
|
| 43 |
-
local_dir="
|
| 44 |
-
local_dir_use_symlinks=False,
|
| 45 |
token=token,
|
| 46 |
)
|
| 47 |
-
print(f"Downloaded {
|
| 48 |
except Exception as e:
|
| 49 |
-
print(f"⚠️ Failed to download {
|
| 50 |
|
| 51 |
if __name__ == "__main__":
|
| 52 |
download_assets()
|
|
|
|
| 3 |
|
| 4 |
def download_assets():
|
| 5 |
"""Download necessary assets from Hugging Face Hub."""
|
| 6 |
+
# ensure local dirs exist
|
| 7 |
os.makedirs("data", exist_ok=True)
|
| 8 |
os.makedirs("vectorstore/db_faiss", exist_ok=True)
|
| 9 |
|
| 10 |
repo_id = "MoizK/mindmedic-assets"
|
| 11 |
repo_type = "dataset"
|
| 12 |
+
# allow either env var name
|
| 13 |
+
token = os.getenv("HUGGINGFACE_HUB_TOKEN") or os.getenv("HUGGINGFACE_API_TOKEN")
|
| 14 |
|
| 15 |
+
# list of PDF filenames *at the repo root*
|
| 16 |
pdf_files = [
|
| 17 |
"71763-gale-encyclopedia-of-medicine.-vol.-1.-2nd-ed.pdf",
|
| 18 |
"Depression-NIM-2024.pdf",
|
|
|
|
| 25 |
|
| 26 |
for fname in pdf_files:
|
| 27 |
try:
|
| 28 |
+
path = hf_hub_download(
|
| 29 |
repo_id=repo_id,
|
| 30 |
repo_type=repo_type,
|
| 31 |
+
filename=fname, # no "data/" prefix
|
| 32 |
+
local_dir="data", # drop into ./data/
|
|
|
|
| 33 |
token=token,
|
| 34 |
)
|
| 35 |
+
print(f"✅ Downloaded {fname} → {path}")
|
| 36 |
except Exception as e:
|
| 37 |
print(f"⚠️ Failed to download {fname}: {e}")
|
| 38 |
|
| 39 |
+
# FAISS index files also at the repo root
|
| 40 |
+
index_files = ["index.faiss", "index.pkl"]
|
| 41 |
+
for fname in index_files:
|
| 42 |
try:
|
| 43 |
+
path = hf_hub_download(
|
| 44 |
repo_id=repo_id,
|
| 45 |
repo_type=repo_type,
|
| 46 |
+
filename=fname, # root filename
|
| 47 |
+
local_dir="vectorstore/db_faiss", # into ./vectorstore/db_faiss/
|
|
|
|
| 48 |
token=token,
|
| 49 |
)
|
| 50 |
+
print(f"✅ Downloaded {fname} → {path}")
|
| 51 |
except Exception as e:
|
| 52 |
+
print(f"⚠️ Failed to download {fname}: {e}")
|
| 53 |
|
| 54 |
if __name__ == "__main__":
|
| 55 |
download_assets()
|