Refactor data directory creation in build_index.py to use absolute path and add permission checks

Dockerfile

@@ -19,8 +19,8 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Copy application code
 COPY app/ ./app/
 
-# Create data directory
-RUN mkdir -p app/data
+# Create data directory with proper permissions
+RUN mkdir -p app/data && chmod 777 app/data
 
 # Make start script executable
 RUN chmod +x app/start.py

app/build_index.py

@@ -293,9 +293,26 @@ def build_index(max_pages: int = 10, model: str = "text-embedding-3-small", use_
     tmdb_client = TMDBClient(settings.tmdb_api_key)
     openai_client = OpenAI(api_key=settings.openai_api_key)
     
-    # Create data directory
-    os.makedirs("app/data", exist_ok=True)
+    # Create data directory with absolute path
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    data_dir = os.path.join(script_dir, "data")
     
+    try:
+        os.makedirs(data_dir, exist_ok=True)
+        # Test write permissions
+        test_file = os.path.join(data_dir, ".write_test")
+        with open(test_file, 'w') as f:
+            f.write("test")
+        os.remove(test_file)
+        logger.info(f"Data directory ready: {data_dir}")
+    except PermissionError as e:
+        logger.error(f"❌ Permission denied when creating data directory: {e}")
+        logger.error("Make sure the data directory has write permissions")
+        return
+    except Exception as e:
+        logger.error(f"❌ Failed to create or write to data directory: {e}")
+        return
+
     # Check for existing movie data checkpoint
     movies_data = load_checkpoint(MOVIE_DATA_CHECKPOINT)
     
@@ -440,8 +457,13 @@ def build_index(max_pages: int = 10, model: str = "text-embedding-3-small", use_
     
     # Step 4: Save embeddings as numpy array
     embeddings_array = np.array(embeddings, dtype=np.float32)
-    np.save("app/data/movies.npy", embeddings_array)
-    logger.info(f"Saved embeddings matrix: {embeddings_array.shape}")
+    embeddings_path = os.path.join(data_dir, "movies.npy")
+    try:
+        np.save(embeddings_path, embeddings_array)
+        logger.info(f"Saved embeddings matrix: {embeddings_array.shape}")
+    except Exception as e:
+        logger.error(f"❌ Failed to save embeddings: {e}")
+        return
     
     # Step 5: Build and save FAISS index
     if use_faiss:
@@ -461,28 +483,41 @@ def build_index(max_pages: int = 10, model: str = "text-embedding-3-small", use_
             index.train(embeddings_array)
         
         index.add(embeddings_array)
-        faiss.write_index(index, "app/data/faiss.index")
-        logger.info(f"FAISS index saved (type: {type(index).__name__}, dimension: {dimension})")
+        index_path = os.path.join(data_dir, "faiss.index")
+        try:
+            faiss.write_index(index, index_path)
+            logger.info(f"FAISS index saved (type: {type(index).__name__}, dimension: {dimension})")
+        except Exception as e:
+            logger.error(f"❌ Failed to save FAISS index: {e}")
+            return
     
     # Step 6: Save metadata files
-    with open("app/data/id_map.json", "w") as f:
-        json.dump(id_map, f)
-    
-    with open("app/data/movie_metadata.json", "w") as f:
-        json.dump(movie_metadata, f)
-    
-    logger.info("✅ Index built successfully!")
-    logger.info(f"   - {len(embeddings)} movies indexed")
-    logger.info(f"   - Embedding model: {model}")
-    logger.info(f"   - Files saved in app/data/")
-    logger.info(f"     * movies.npy: embeddings matrix")
-    logger.info(f"     * id_map.json: TMDB ID to matrix position mapping")
-    logger.info(f"     * movie_metadata.json: movie metadata")
-    if use_faiss:
-        logger.info(f"     * faiss.index: FAISS search index")
+    id_map_path = os.path.join(data_dir, "id_map.json")
+    metadata_path = os.path.join(data_dir, "movie_metadata.json")
     
-    # Cleanup checkpoints
-    cleanup_checkpoints()
+    try:
+        with open(id_map_path, "w") as f:
+            json.dump(id_map, f)
+        
+        with open(metadata_path, "w") as f:
+            json.dump(movie_metadata, f)
+        
+        logger.info("✅ Index built successfully!")
+        logger.info(f"   - {len(embeddings)} movies indexed")
+        logger.info(f"   - Embedding model: {model}")
+        logger.info(f"   - Files saved in {data_dir}")
+        logger.info(f"     * movies.npy: embeddings matrix")
+        logger.info(f"     * id_map.json: TMDB ID to matrix position mapping")
+        logger.info(f"     * movie_metadata.json: movie metadata")
+        if use_faiss:
+            logger.info(f"     * faiss.index: FAISS search index")
+        
+        # Cleanup checkpoints
+        cleanup_checkpoints()
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to save metadata files: {e}")
+        return
 
 # Remove the old functions that are no longer needed
 # create_movie_embedding and load_movie_data are replaced by the new implementation