Cogito model

app.py

@@ -25,14 +25,14 @@ login(token=hf_token)
 
 # Models
 sentence_transformer_model = "all-MiniLM-L6-v2"
-repo_id = "bartowski/Llama-3.2-3B-Instruct-GGUF"
-filename = "Llama-3.2-3B-Instruct-Q4_K_M.gguf"
+repo_id = "bartowski/deepcogito_cogito-v1-preview-llama-3B-GGUF"
+filename = "deepcogito_cogito-v1-preview-llama-3B-Q4_K_M.gguf"  # Updated to Cogito Q4_K_M
 
-# Define FAQs
+# Define FAQs (unchanged)
 faqs = [
     {"question": "What is your name?", "answer": "My name is Tim Luka Horstmann."},
     {"question": "Where do you live?", "answer": "I live in Paris, France."},
-    {"question": "What is your education?", "answer": "I am currenlty pursuing a MSc in Data and AI at Institut Polytechnique de Paris. I have an MPhil in Advanced Computer Science from the University of Cambridge, and a BSc in Business Informatics from RheinMain University of Applied Sciences."},
+    {"question": "What is your education?", "answer": "I am currently pursuing a MSc in Data and AI at Institut Polytechnique de Paris. I have an MPhil in Advanced Computer Science from the University of Cambridge, and a BSc in Business Informatics from RheinMain University of Applied Sciences."},
     {"question": "What are your skills?", "answer": "I am proficient in Python, Java, SQL, Cypher, SPARQL, VBA, JavaScript, HTML/CSS, and Ruby. I also use tools like PyTorch, Hugging Face, Scikit-Learn, NumPy, Pandas, Matplotlib, Jupyter, Git, Bash, IoT, Ansible, QuickSight, and Wordpress."},
     {"question": "How are you?", "answer": "I’m doing great, thanks for asking! I’m enjoying life in Paris and working on some exciting AI projects."},
     {"question": "What do you do?", "answer": "I’m a Computer Scientist and AI enthusiast, currently pursuing a MSc in Data and AI at Institut Polytechnique de Paris and interning as a Machine Learning Research Engineer at Hi! PARIS."},
@@ -40,7 +40,7 @@ faqs = [
 ]
 
 try:
-    # Load CV embeddings and build FAISS index
+    # Load CV embeddings and build FAISS index (unchanged)
     logger.info("Loading CV embeddings from cv_embeddings.json")
     with open("cv_embeddings.json", "r", encoding="utf-8") as f:
         cv_data = json.load(f)
@@ -51,17 +51,17 @@ try:
     faiss_index.add(cv_embeddings)
     logger.info("FAISS index built successfully")
 
-    # Load embedding model
+    # Load embedding model (unchanged)
     logger.info("Loading SentenceTransformer model")
     embedder = SentenceTransformer(sentence_transformer_model, device="cpu")
     logger.info("SentenceTransformer model loaded")
 
-    # Compute FAQ embeddings
+    # Compute FAQ embeddings (unchanged)
     faq_questions = [faq["question"] for faq in faqs]
     faq_embeddings = embedder.encode(faq_questions, convert_to_numpy=True).astype("float32")
     faiss.normalize_L2(faq_embeddings)
 
-    # Load Llama model
+    # Load Cogito model
     logger.info(f"Loading {filename} model")
     model_path = hf_hub_download(
         repo_id=repo_id,
@@ -71,10 +71,10 @@ try:
     )
     generator = Llama(
         model_path=model_path,
-        n_ctx=1024,
+        n_ctx=2048,  # Adjust if 128k is supported and memory allows; start with 1024
         n_threads=2,
         n_batch=512,
-        n_gpu_layers=0,
+        n_gpu_layers=0,  # No GPU on free tier
         verbose=True,
     )
     logger.info(f"{filename} model loaded")
@@ -97,7 +97,7 @@ def retrieve_context(query, top_k=3):
 def stream_response(query):
     try:
         logger.info(f"Processing query: {query}")
-        # Check FAQ cache
+        # Check FAQ cache (unchanged)
         query_embedding = embedder.encode(query, convert_to_numpy=True).astype("float32")
         query_embedding = query_embedding.reshape(1, -1)
         faiss.normalize_L2(query_embedding)
@@ -111,10 +111,12 @@ def stream_response(query):
 
         context = retrieve_context(query)
         prompt = (
-            f"<|im_start|>system\nYou are Tim Luka Horstmann, a Computer Scientist. Here is your CV:\n{context}\n"
-            f"A user is asking you a question. Respond as yourself, using the first person, in a friendly and concise manner. For questions about your CV, base your answer strictly on the provided CV information. For casual questions not covered by the CV, respond naturally but do not invent specific details beyond what’s generally true about you (e.g., your current location or field of work). Avoid meta-commentary or critiquing your own response.\n<|im_end>\n"
-            f"<|im_start|>user\n{query}\n<|im_end>\n"
-            f"<|im_start|>assistant\n"
+            f"<|begin_of_text|><|start_header_id|>system<|end_header_id>\n"
+            f"You are Tim Luka Horstmann, a Computer Scientist. Here is your CV:\n{context}\n"
+            f"A user is asking you a question. Respond as yourself, using the first person, in a friendly and concise manner. For questions about your CV, base your answer strictly on the provided CV information. For casual questions not covered by the CV, respond naturally but do not invent specific details beyond what’s generally true about you (e.g., your current location or field of work). Avoid meta-commentary or critiquing your own response.\n"
+            f"<|eot_id|><|start_header_id|>user<|end_header_id>\n"
+            f"{query}\n"
+            f"<|eot_id|><|start_header_id|>assistant<|end_header_id>\n"
         )
         
         response_text = ""
@@ -122,15 +124,15 @@ def stream_response(query):
             prompt,
             max_tokens=200,
             stream=True,
-            stop=["<|im_end|>", "[DONE]"],
-            temperature=0.5,     # Slightly higher for friendly tone
+            stop=["<|eot_id|>", "[DONE]"],  # Updated stop tokens
+            temperature=0.5,
             top_p=0.9,
-            repeat_penalty=1.2,  # Maintain control
+            repeat_penalty=1.2,
         ):
             text = chunk['choices'][0]['text']
             response_text += text
             yield f"data: {text}\n\n"
-            if "<|im_end>" in response_text or "[DONE]" in response_text:
+            if "<|eot_id>" in response_text or "[DONE]" in response_text:
                 break
         yield "data: [DONE]\n\n"
     except Exception as e:
@@ -155,8 +157,9 @@ async def health_check():
 @app.get("/model_info")
 async def model_info():
     return {
-        "model_name": "Llama-3.2-3B-Instruct-GGUF",
+        "model_name": "deepcogito_cogito-v1-preview-llama-3B-GGUF",
         "model_size": "3B",
+        "quantization": "Q4_K_M",
         "embedding_model": sentence_transformer_model,
         "faiss_index_size": len(cv_chunks),
         "faiss_index_dim": cv_embeddings.shape[1],