Update app.py

app.py

@@ -2,25 +2,22 @@ from fastapi import FastAPI, Request
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 import os
-
 import platform
 import psutil
 import multiprocessing
 import time
 
-
-
 app = FastAPI()
 
 # === Model Config ===
-REPO_ID = "TheBloke/Mistral-7B-Instruct-v0.1-GGUF"
-FILENAME = "mistral-7b-instruct-v0.1.Q4_K_M.gguf"
+REPO_ID = "TheBloke/Hermes-2-Pro-Mistral-7B-GGUF"
+FILENAME = "hermes-2-pro-mistral-7b.Q4_K_M.gguf"
 MODEL_DIR = "models"
 MODEL_PATH = os.path.join(MODEL_DIR, FILENAME)
 
-# === Download only if not already present ===
+# === Download if model not available ===
 if not os.path.exists(MODEL_PATH):
-    print(f"Downloading model {FILENAME} from Hugging Face...")
+    print(f"⬇️ Downloading {FILENAME} from Hugging Face...")
     model_path = hf_hub_download(
         repo_id=REPO_ID,
         filename=FILENAME,
@@ -29,101 +26,86 @@ if not os.path.exists(MODEL_PATH):
         local_dir_use_symlinks=False
     )
 else:
-    print(f"Model already exists at: {MODEL_PATH}")
+    print(f"✅ Model already available at: {MODEL_PATH}")
     model_path = MODEL_PATH
 
-# === Load LLM ===
+# === Optimal thread usage ===
+logical_cores = psutil.cpu_count(logical=True)
+physical_cores = psutil.cpu_count(logical=False)
+recommended_threads = min(physical_cores or 4, 8)
+
+# === Load the model ===
 llm = Llama(
     model_path=model_path,
-    n_ctx=1024,
-    n_threads=8,  # Adjust for your CPU
-    n_batch=32
+    n_ctx=8192,  # Can increase depending on memory
+    n_threads=recommended_threads,
+    n_batch=64,  # adjust depending on RAM
+    use_mlock=True,  # lock model in RAM for faster access
+    n_gpu_layers=0,  # CPU only, use >0 if GPU is present
+    chat_format="chatml",  # for Hermes 2
+    verbose=False
 )
 
 @app.get("/")
 def root():
-    return {"message": "Mistral API is live!"}
+    return {"message": "✅ Hermes 2 Mistral API is live and optimized!"}
 
 @app.get("/get_sys")
 def get_sys_specs():
-    cpu_info = {
-        "physical_cores": psutil.cpu_count(logical=False),
-        "logical_cores": psutil.cpu_count(logical=True),
-        "max_frequency_mhz": psutil.cpu_freq().max if psutil.cpu_freq() else None,
-        "cpu_usage_percent": psutil.cpu_percent(interval=1)
-    }
-
     memory = psutil.virtual_memory()
-    ram_info = {
-        "total_gb": round(memory.total / (1024 ** 3), 2),
-        "available_gb": round(memory.available / (1024 ** 3), 2),
-        "used_percent": memory.percent
-    }
-
-    system_info = {
-        "system": platform.system(),
-        "machine": platform.machine(),
-        "platform": platform.platform(),
-        "processor": platform.processor(),
-        "python_version": platform.python_version(),
-    }
-
     return {
-        "cpu": cpu_info,
-        "ram": ram_info,
-        "system": system_info,
-        "recommended_threads": min(psutil.cpu_count(logical=False) or 2, 8)
+        "CPU": {
+            "physical_cores": physical_cores,
+            "logical_cores": logical_cores,
+            "max_freq_mhz": psutil.cpu_freq().max,
+            "cpu_usage": psutil.cpu_percent(interval=1)
+        },
+        "RAM": {
+            "total_GB": round(memory.total / (1024 ** 3), 2),
+            "available_GB": round(memory.available / (1024 ** 3), 2),
+            "usage_percent": memory.percent
+        },
+        "System": {
+            "platform": platform.platform(),
+            "architecture": platform.machine(),
+            "python": platform.python_version()
+        }
     }
 
 @app.get("/process_list")
 def process_list():
-    p_l = []
-
-    # Warm up CPU usage stats
-    for proc in psutil.process_iter():
-        try:
-            proc.cpu_percent(interval=None)  # Prime the value
-        except (psutil.NoSuchProcess, psutil.AccessDenied):
-            pass
-
-    time.sleep(1)  # Let CPU usage accumulate over time
-
+    time.sleep(1)  # Let CPU settle
+    processes = []
     for proc in psutil.process_iter(['pid', 'name']):
         try:
-            usage = proc.cpu_percent(interval=None)
-            if usage > 10:
-                p_l.append({
+            cpu = proc.cpu_percent()
+            if cpu > 10:
+                processes.append({
                     "pid": proc.pid,
                     "name": proc.name(),
-                    "cpu_percent": usage
+                    "cpu_percent": cpu
                 })
         except (psutil.NoSuchProcess, psutil.AccessDenied):
             pass
-
-    return {
-        "process list": p_l
-    }
-
+    return {"heavy_processes": processes}
 
 @app.post("/generate")
 async def generate(request: Request):
     data = await request.json()
-    prompt = data.get("prompt", "")
-
-    print("🧾 Received prompt:", prompt)
+    prompt = data.get("prompt", "").strip()
+    print("🧾 Prompt received:", prompt)
 
     response = llm.create_chat_completion(
         messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "system", "content": "You are a helpful AI assistant."},
             {"role": "user", "content": prompt}
         ],
-        max_tokens=1024,
+        max_tokens=2048,
         temperature=0.7,
+        stop=["</s>"]
     )
 
-    print("📤 Raw model response:", response)
-
-    llm.reset()
+    llm.reset()  # Free memory after response
 
     return {
         "response": response["choices"][0]["message"]["content"].strip()