Speedup

Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y \
     libffi-dev \
     libgcc-s1 \
     libstdc++6 \
+    libopenblas-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Set environment variables for cache

app.py

@@ -1,8 +1,6 @@
 import json
 import numpy as np
 from sentence_transformers import SentenceTransformer
-import torch
-import torch.nn.functional as F
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
@@ -10,6 +8,7 @@ from llama_cpp import Llama
 from huggingface_hub import login, hf_hub_download
 import logging
 import os
+import faiss
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -25,13 +24,16 @@ if not hf_token:
 login(token=hf_token)
 
 try:
-    # Load precomputed CV embeddings
+    # Load precomputed CV embeddings and build FAISS index
     logger.info("Loading CV embeddings from cv_embeddings.json")
     with open("cv_embeddings.json", "r", encoding="utf-8") as f:
         cv_data = json.load(f)
         cv_chunks = [item["chunk"] for item in cv_data]
-        cv_embeddings = np.array([item["embedding"] for item in cv_data])
-    cv_embeddings_tensor = torch.tensor(cv_embeddings)
+        cv_embeddings = np.array([item["embedding"] for item in cv_data]).astype('float32')
+    faiss.normalize_L2(cv_embeddings)  # Normalize for cosine similarity
+    faiss_index = faiss.IndexFlatIP(cv_embeddings.shape[1])  # Inner Product for cosine similarity
+    faiss_index.add(cv_embeddings)
+    logger.info("FAISS index built successfully")
     logger.info("CV embeddings loaded successfully")
 
     # Load embedding model
@@ -39,11 +41,13 @@ try:
     embedder = SentenceTransformer("all-MiniLM-L6-v2", device="cpu")
     logger.info("SentenceTransformer model loaded")
 
-    # Load Qwen 2.5 1.5B model with llama_cpp
-    logger.info("Loading Qwen 2.5 1.5B model")
+    # Load Qwen 2.5 7B model
+    repo_id = "bartowski/Qwen2.5-7B-Instruct-GGUF"
+    filename = "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
+    logger.info(f"Loading {filename} model")
     model_path = hf_hub_download(
-        repo_id="bartowski/Qwen2.5-7B-Instruct-GGUF", #"Qwen/Qwen2.5-1.5B-Instruct-GGUF",
-        filename="Qwen2.5-7B-Instruct-Q6_K.gguf", #"qwen2.5-1.5b-instruct-q4_0.gguf",
+        repo_id=repo_id,
+        filename=filename,
         local_dir="/app/cache" if os.getenv("HF_HOME") else None,
         token=hf_token,
     )
@@ -52,8 +56,11 @@ try:
         model_path=model_path,
         n_ctx=2048,
         n_threads=4,
+        n_batch=512,  # Increase batch size for faster eval
+        n_gpu_layers=0,  # Explicitly set to 0 (no GPU in HF Spaces)
+        verbose=True,  # Keep for perf logging
     )
-    logger.info("Qwen 2.5 1.5B model loaded")
+    logger.info(f"{filename} model loaded")
 
 except Exception as e:
     logger.error(f"Startup error: {str(e)}", exc_info=True)
@@ -61,11 +68,12 @@ except Exception as e:
 
 def retrieve_context(query, top_k=3):
     try:
-        query_embedding = embedder.encode(query, convert_to_tensor=True).unsqueeze(0)
-        similarities = F.cosine_similarity(query_embedding, cv_embeddings_tensor, dim=1)
-        top_k = min(top_k, len(similarities))
-        top_indices = torch.topk(similarities, k=top_k).indices.cpu().numpy()
-        return "\n".join([cv_chunks[i] for i in top_indices])
+        # Encode query and normalize for FAISS
+        query_embedding = embedder.encode(query, convert_to_numpy=True).astype("float32")
+        query_embedding = query_embedding.reshape(1, -1)
+        faiss.normalize_L2(query_embedding)
+        distances, indices = faiss_index.search(query_embedding, top_k)
+        return "\n".join([cv_chunks[i] for i in indices[0]])
     except Exception as e:
         logger.error(f"Error in retrieve_context: {str(e)}")
         raise
@@ -80,12 +88,13 @@ def stream_response(query):
             f"<|im_start|>assistant\n"
         )
         
-        # Stream response with llama_cpp
         for chunk in generator(
             prompt,
             max_tokens=512,
             stream=True,
             stop=["<|im_end|>", "[DONE]"],
+            temperature=0.7,  # Slightly lower for consistency
+            top_p=0.9,  # Narrow sampling for faster generation
         ):
             yield f"data: {chunk['choices'][0]['text']}\n\n"
         yield "data: [DONE]\n\n"
@@ -106,4 +115,12 @@ async def predict(request: QueryRequest):
 
 @app.get("/health")
 async def health_check():
-    return {"status": "healthy"}
+    return {"status": "healthy"}
+
+@app.on_event("startup")
+async def warm_up_model():
+    logger.info("Warming up the model...")
+    dummy_query = "Hello, please warm up your model."
+    for _ in stream_response(dummy_query):
+        pass
+    logger.info("Model warm-up complete.")

requirements.txt

@@ -4,4 +4,5 @@ sentence-transformers==3.1.1
 torch==2.4.1
 numpy==1.26.4
 llama-cpp-python==0.3.1
-huggingface_hub==0.30.1
+huggingface_hub==0.30.1
+faiss-cpu==1.8.0