Upload 5 files

add_model_explanations.py

@@ -9,7 +9,11 @@ from openai import OpenAI, APIError # Add back OpenAI imports
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-MODEL_DATA_DIR = "model_data_json"
+# Define the base persistent storage path (must match other scripts)
+PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
+
+# Point to the JSON data within persistent storage
+MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
 EXPLANATION_KEY = "model_explanation_gemini"
 DESCRIPTION_KEY = "description"
 MAX_RETRIES = 3 # Retries for API calls

app.py

@@ -22,6 +22,9 @@ except ImportError:
 
 app = Flask(__name__) # Create app object FIRST
 
+# Define the base persistent storage path (must match other scripts)
+PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
+
 # Configure Flask app logging (optional but recommended)
 # app.logger.setLevel(logging.INFO)
 
@@ -29,11 +32,12 @@ app = Flask(__name__) # Create app object FIRST
 CORS(app, origins=["http://127.0.0.1:3000", "http://localhost:3000", "https://rag-huggingface.vercel.app"], supports_credentials=True)
 
 # --- Configuration ---
-INDEX_FILE = "index.faiss"
-MAP_FILE = "index_to_metadata.pkl"
+# Point to index/map files in persistent storage
+INDEX_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index.faiss")
+MAP_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index_to_metadata.pkl")
 EMBEDDING_MODEL = 'all-mpnet-base-v2'
-# Corrected path joining for model_data_json - relative to app.py location
-MODEL_DATA_DIR = os.path.join(os.path.dirname(__file__), 'model_data_json')
+# Point to model data JSON in persistent storage
+MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
 # ---
 
 # --- Global variables for resources ---
@@ -72,7 +76,8 @@ def load_resources():
         print("Sentence transformer model loaded successfully.")
 
         # Load FAISS Index
-        index_path = os.path.join(os.path.dirname(__file__), INDEX_FILE)
+        # index_path = os.path.join(os.path.dirname(__file__), INDEX_FILE) # Old path
+        index_path = INDEX_FILE # Use configured path
         print(f"Loading FAISS index from: {index_path}")
         if not os.path.exists(index_path):
              raise FileNotFoundError(f"FAISS index file not found at {index_path}")
@@ -81,7 +86,8 @@ def load_resources():
         print("FAISS index loaded successfully.")
 
         # Load Index-to-Metadata Map
-        map_path = os.path.join(os.path.dirname(__file__), MAP_FILE)
+        # map_path = os.path.join(os.path.dirname(__file__), MAP_FILE) # Old path
+        map_path = MAP_FILE # Use configured path
         print(f"Loading index-to-Metadata map from: {map_path}")
         if not os.path.exists(map_path):
             raise FileNotFoundError(f"Metadata map file not found at {map_path}")
@@ -95,8 +101,8 @@ def load_resources():
 
     except FileNotFoundError as fnf_error:
         print(f"Error: {fnf_error}")
-        print(f"Please ensure {INDEX_FILE} and {MAP_FILE} exist in the 'backend' directory relative to app.py.")
-        print("You might need to run 'python build_index.py' first.")
+        print(f"Please ensure {os.path.basename(INDEX_FILE)} and {os.path.basename(MAP_FILE)} exist in the persistent storage directory ({PERSISTENT_STORAGE_PATH}).")
+        print("You might need to run the update process first or manually place initial files there.")
         RESOURCES_LOADED = False # Keep as False
     except ImportError as import_error:
          print(f"Import Error loading resources: {import_error}")
@@ -235,7 +241,7 @@ def search():
                 # --- Add description from model_data_json ---
                 model_id = metadata.get('model_id')
                 description = None
-                # Use the globally defined and corrected MODEL_DATA_DIR
+                # Use the globally defined MODEL_DATA_DIR pointing to persistent storage
                 if model_id and MODEL_DATA_DIR:
                     filename = model_id.replace('/', '_') + '.json'
                     filepath = os.path.join(MODEL_DATA_DIR, filename)

build_index.py

@@ -7,10 +7,15 @@ import pickle
 import json # Import json module
 from tqdm import tqdm
 
+# Define the base persistent storage path (must match other scripts)
+PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
+
 # --- Configuration ---
-MODEL_DATA_DIR = "model_data_json"  # Path to downloaded JSON data
-INDEX_FILE = "index.faiss"
-MAP_FILE = "index_to_metadata.pkl" # Changed filename to reflect content
+# Point to the JSON data within persistent storage
+MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
+# Save index and map to persistent storage
+INDEX_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index.faiss")
+MAP_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index_to_metadata.pkl")
 EMBEDDING_MODEL = 'all-mpnet-base-v2'  # Efficient and good quality model
 ENCODE_BATCH_SIZE = 32  # Process descriptions in smaller batches
 # Tags to exclude from indexing text

huggingface_model_descriptions.py

@@ -10,8 +10,11 @@ from requests.exceptions import RequestException
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import pickle # Add pickle for caching
 
-# Create a directory to store JSON data
-OUTPUT_DIR = "model_data_json"
+# Define the base persistent storage path
+PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
+
+# Create a directory to store JSON data within persistent storage
+OUTPUT_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 # Number of worker threads for parallel processing - REDUCED
@@ -41,7 +44,8 @@ def clean_readme_content(text):
     return text
 # ---
 
-MODELS_CACHE_FILE = "models_list_cache.pkl" # File to cache the raw model list
+# Use persistent storage for the cache file
+MODELS_CACHE_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "models_list_cache.pkl") # File to cache the raw model list
 
 def get_all_models_with_downloads(min_downloads=10000):
     """Fetch all models from Hugging Face with at least min_downloads, using a local cache for the list."""
@@ -154,9 +158,9 @@ def get_model_readme(model_id):
     return None
 
 def get_filename_for_model(model_id):
-    """Generate JSON filename for a model"""
+    """Generate JSON filename for a model (uses global OUTPUT_DIR)"""
     safe_id = model_id.replace("/", "_")
-    return os.path.join(OUTPUT_DIR, f"{safe_id}.json") # Change extension to .json
+    return os.path.join(OUTPUT_DIR, f"{safe_id}.json") # OUTPUT_DIR is already correct path
 
 def save_model_data(model_id, data):
     """Save model data (description, tags, downloads) to a JSON file."""