Update app.py

app.py

@@ -5,14 +5,20 @@ from typing import List, Optional
 from huggingface_hub import hf_hub_download
 from ctransformers import AutoModelForCausalLM
 
-REPO_ID  = "bartowski/Llama-3.2-3B-Instruct-GGUF"
-FILENAME = "Llama-3.2-3B-Instruct-Q4_K_M.gguf"   # <- safer quant
+# --- Model & cache config ---
+REPO_ID   = "bartowski/Llama-3.2-3B-Instruct-GGUF"
+FILENAME  = "Llama-3.2-3B-Instruct-Q4_K_M.gguf"  # safer quant than Q4_K_L
 MODEL_TYPE = "llama"
 
-
 CACHE_DIR = os.environ.get("HUGGINGFACE_HUB_CACHE", "/data/hf_cache")
 os.makedirs(CACHE_DIR, exist_ok=True)
 
+# Conservative defaults (override via env if needed)
+CTX_LEN     = int(os.environ.get("CTX_LEN", "1024"))
+BATCH_SIZE  = int(os.environ.get("BATCH", "16"))
+THREADS     = int(os.environ.get("THREADS", "4"))
+GPU_LAYERS  = int(os.environ.get("GPU_LAYERS", "0"))
+
 app = FastAPI(title="Llama 3.2 3B Instruct (ctransformers)")
 _model = None
 
@@ -32,12 +38,11 @@ def get_model():
         model_path_or_repo_id=os.path.dirname(local_path),
         model_file=os.path.basename(local_path),
         model_type=MODEL_TYPE,
-        context_length=int(os.environ.get("CTX_LEN", "4096")),
-        CTX_LEN  = int(os.environ.get("CTX_LEN",  "1024"))
-        BATCH    = int(os.environ.get("BATCH",    "16"))
-        THREADS  = int(os.environ.get("THREADS",  "4"))
-        GPU_LAY  = int(os.environ.get("GPU_LAYERS","0"))
-
+        context_length=CTX_LEN,
+        batch_size=BATCH_SIZE,
+        threads=THREADS,
+        gpu_layers=GPU_LAYERS,
+        f16_kv=True,
     )
     return _model
 
@@ -55,7 +60,17 @@ class GenerateOut(BaseModel):
 
 @app.get("/")
 def health():
-    return {"status": "ok", "cache_dir": CACHE_DIR}
+    return {
+        "status": "ok",
+        "cache_dir": CACHE_DIR,
+        "model": {"repo": REPO_ID, "file": FILENAME, "type": MODEL_TYPE},
+        "settings": {
+            "CTX_LEN": CTX_LEN,
+            "BATCH_SIZE": BATCH_SIZE,
+            "THREADS": THREADS,
+            "GPU_LAYERS": GPU_LAYERS,
+        },
+    }
 
 @app.post("/generate", response_model=GenerateOut)
 def generate(body: GenerateIn):
@@ -72,4 +87,7 @@ def generate(body: GenerateIn):
         )
         return GenerateOut(completion=text)
     except Exception as e:
+        # helpful for debugging in Space logs
+        import sys, traceback
+        traceback.print_exc(file=sys.stderr)
         raise HTTPException(status_code=500, detail=str(e))