Add data download

Signed-off-by: Aivin V. Solatorio <[email protected]>

services.py

@@ -1,4 +1,5 @@
 import json
+import os
 import pandas as pd
 import torch
 import httpx
@@ -8,6 +9,7 @@ from typing import Optional, Any
 from sentence_transformers import SentenceTransformer
 
 from pydantic import BaseModel, Field
+from urllib.request import urlretrieve
 
 
 def get_best_torch_device():
@@ -26,9 +28,23 @@ device = get_best_torch_device()
 
 
 # Load the basic WDI metadata and vectors.
-wdi_data_vec_fpath = (
-    "./data/avsolatorio__GIST-small-Embedding-v0__005__indicator_embeddings.json"
+
+EMBEDDING_FNAME = "avsolatorio__GIST-small-Embedding-v0__005__indicator_embeddings.json"
+EMBEDDING_SOURCE = (
+    f"https://raw.githubusercontent.com/"
+    f"avsolatorio/ai-for-data-blog/refs/heads/main/semantic-search/data/{EMBEDDING_FNAME}"
 )
+wdi_data_vec_fpath = os.path.join("data", EMBEDDING_FNAME)
+
+os.makedirs(os.path.dirname(wdi_data_vec_fpath), exist_ok=True)
+
+if not os.path.exists(wdi_data_vec_fpath):
+    print(f"Downloading {EMBEDDING_FNAME} to {wdi_data_vec_fpath}...")
+    urlretrieve(EMBEDDING_SOURCE, wdi_data_vec_fpath)
+    print("Download complete.")
+else:
+    print(f"File already exists at {wdi_data_vec_fpath}.")
+
 df = pd.read_json(wdi_data_vec_fpath)
 
 # Make it easy to index based on the idno