Improved model and RAG

app.py

@@ -25,8 +25,17 @@ login(token=hf_token)
 
 # Models
 sentence_transformer_model = "all-MiniLM-L6-v2"
-repo_id = "bartowski/Llama-3.2-1B-Instruct-GGUF"
-filename = "Llama-3.2-1B-Instruct-Q4_K_M.gguf"  # Q4_K for speed
+repo_id = "bartowski/Llama-3.2-3B-Instruct-GGUF"  # Switched to 3B; revert to "bartowski/Llama-3.2-1B-Instruct-GGUF" if too heavy
+filename = "Llama-3.2-3B-Instruct-Q4_K_M.gguf"    # Use "Llama-3.2-1B-Instruct-Q4_K_M.gguf" for 1B
+
+# Define FAQs
+faqs = [
+    {"question": "What is your name?", "answer": "My name is Tim Luka Horstmann."},
+    {"question": "Where do you live?", "answer": "I live in Paris, France."},
+    {"question": "What is your education?", "answer": "I am currently pursuing a MSc in Data and AI at Institut Polytechnique de Paris. I also hold an MPhil in Advanced Computer Science from the University of Cambridge and a BSc in Business Informatics from RheinMain University of Applied Sciences."},
+    {"question": "What are your skills?", "answer": "I am proficient in Python, Java, SQL, Cypher, SPARQL, VBA, JavaScript, HTML/CSS, and Ruby. I also use tools like PyTorch, Hugging Face, Scikit-Learn, NumPy, Pandas, Matplotlib, Jupyter, Git, Bash, IoT, Ansible, QuickSight, and Wordpress."},
+    # Add more from your CV
+]
 
 try:
     # Load CV embeddings and build FAISS index
@@ -45,6 +54,11 @@ try:
     embedder = SentenceTransformer(sentence_transformer_model, device="cpu")
     logger.info("SentenceTransformer model loaded")
 
+    # Compute FAQ embeddings
+    faq_questions = [faq["question"] for faq in faqs]
+    faq_embeddings = embedder.encode(faq_questions, convert_to_numpy=True).astype("float32")
+    faiss.normalize_L2(faq_embeddings)
+
     # Load Llama model
     logger.info(f"Loading {filename} model")
     model_path = hf_hub_download(
@@ -53,11 +67,10 @@ try:
         local_dir="/app/cache" if os.getenv("HF_HOME") else None,
         token=hf_token,
     )
-
     generator = Llama(
         model_path=model_path,
-        n_ctx=1024,  # Reduced for speed
-        n_threads=2,  # Match HF Spaces vCPUs
+        n_ctx=1024,
+        n_threads=2,
         n_batch=512,
         n_gpu_layers=0,
         verbose=True,
@@ -82,24 +95,35 @@ def retrieve_context(query, top_k=3):
 def stream_response(query):
     try:
         logger.info(f"Processing query: {query}")
-        context = retrieve_context(query)
-        prompt = (
-            f"<|im_start|>system\nI am Tim Luka Horstmann, a German Computer Scientist. This is my CV:\n{context}\n"
-            f"I will answer your questions about my CV as myself. Please ask me anything!\n<|im_end|>\n"
-            f"<|im_start|>user\n{query}\n<|im_end>\n"
-            f"<|im_start|>assistant\n"
-        )
-        
-        for chunk in generator(
-            prompt,
-            max_tokens=512,
-            stream=True,
-            stop=["<|im_end|>", "[DONE]"],
-            temperature=0.7,
-            top_p=0.9,
-        ):
-            yield f"data: {chunk['choices'][0]['text']}\n\n"
-        yield "data: [DONE]\n\n"
+        # Check FAQ cache
+        query_embedding = embedder.encode(query, convert_to_numpy=True).astype("float32")
+        query_embedding = query_embedding.reshape(1, -1)
+        faiss.normalize_L2(query_embedding)
+        similarities = np.dot(faq_embeddings, query_embedding.T).flatten()
+        max_sim = np.max(similarities)
+        if max_sim > 0.9:
+            idx = np.argmax(similarities)
+            yield f"data: {faqs[idx]['answer']}\n\n"
+            yield "data: [DONE]\n\n"
+        else:
+            context = retrieve_context(query)
+            prompt = (
+                f"<|im_start|>system\nYou are Tim Luka Horstmann, a Computer Scientist. Here is your CV:\n{context}\n"
+                f"A user is asking you a question about your CV. Respond as yourself, using the first person, and base your answer strictly on the information provided in the CV. Do not invent or assume any details not mentioned.\n<|im_end>\n"
+                f"<|im_start|>user\n{query}\n<|im_end>\n"
+                f"<|im_start|>assistant\n"
+            )
+            for chunk in generator(
+                prompt,
+                max_tokens=512,
+                stream=True,
+                stop=["<|im_end|>", "[DONE]"],
+                temperature=0.5,      # Lower for factual responses
+                top_p=0.9,
+                repeat_penalty=1.1,   # Reduce repetition/hallucination
+            ):
+                yield f"data: {chunk['choices'][0]['text']}\n\n"
+            yield "data: [DONE]\n\n"
     except Exception as e:
         logger.error(f"Error in stream_response: {str(e)}")
         yield f"data: Error: {str(e)}\n\n"
@@ -122,8 +146,8 @@ async def health_check():
 @app.get("/model_info")
 async def model_info():
     return {
-        "model_name": "Llama-3.2-1B-Instruct-GGUF",
-        "model_size": "1B",
+        "model_name": "Llama-3.2-3B-Instruct-GGUF",
+        "model_size": "3B",
         "embedding_model": sentence_transformer_model,
         "faiss_index_size": len(cv_chunks),
         "faiss_index_dim": cv_embeddings.shape[1],
@@ -132,7 +156,7 @@ async def model_info():
 @app.on_event("startup")
 async def warm_up_model():
     logger.info("Warming up the model...")
-    dummy_query = "Hi"  # Shorter prompt
+    dummy_query = "Hi"
     for _ in stream_response(dummy_query):
         pass
     logger.info("Model warm-up complete.")