swiched to 4-bit AWQ‑quantized version of OpenChat 3.5 7B as openchat 3.5 requires more then 16gn RAM

Dockerfile

@@ -1,34 +1,40 @@
+# Base image
 FROM python:3.10-slim
 
-# Create a non-root user with UID 1000
+# Create non-root user (required for Hugging Face Spaces)
 RUN useradd -m -u 1000 user
 
-# Switch to that user
-USER user
+# Set paths
 ENV HOME=/home/user
-WORKDIR /home/user/app
-
-# Set cache dirs inside user home
+ENV APP_HOME=$HOME/app
 ENV HF_HOME=$HOME/.hf_home
-ENV TRANSFORMERS_CACHE=$HOME/.cache/transformers
 
-# Create cache directories
-RUN mkdir -p $HF_HOME $TRANSFORMERS_CACHE
+# Use app directory
+WORKDIR $APP_HOME
 
-# Switch back to root to install dependencies
+# Switch to root for system setup
 USER root
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
 
-# Install Python deps under user home
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements.txt and install Python dependencies
 COPY --chown=user:user requirements.txt .
 RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
 
-# Copy app files
+# Copy app code
 COPY --chown=user:user . .
 
-# Expose port and switch user
-EXPOSE 7860
+# Ensure cache directories exist and are user-writable
+RUN mkdir -p $HF_HOME && chown -R user:user $HF_HOME
+
+# Set to non-root user (required for HF Spaces)
 USER user
 
+# Expose default port
+EXPOSE 7860
+
 # Entrypoint
 CMD ["python", "app.py"]

app.py

@@ -1,92 +1,79 @@
 """
-This is the main file for the OpenChat-3.5 LLM API.
--model_loader.py file to load the model and tokenizer.
+app.py FastAPI API for Quantized OpenChat 3.5 (GGUF) using ctransformers
 """
 
 import logging
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
-from model_loader import tokenizer, model
+from model_loader import model
 import uvicorn
-import torch
+from ctransformers import AutoTokenizer  # Add this at the top
 
+# Logger
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
-# Initialize FastAPI app
+tokenizer = model.tokenize  # Use model's built-in tokenizer if available
+
+# FastAPI app
 app = FastAPI(
     title="masx-openchat-llm",
-    description="MASX AI service exposing the OpenChat-3.5 LLM as an inference endpoint",
+    description="MASX AI service exposing a quantized OpenChat-3.5 model (GGUF)",
     version="1.0.0",
 )
 
 
-# Request ********schema*******
+# Request schema
 class PromptRequest(BaseModel):
     prompt: str
     max_tokens: int = 256
-    temperature: float = 0.0  # Deterministic by default
+    temperature: float = 0.0
 
 
-# Response ********schema*******
+# Response schema
 class ChatResponse(BaseModel):
     response: str
 
 
 @app.get("/status")
 async def status():
-    """Check model status and max supported tokens."""
     try:
-        max_context = getattr(model.config, "max_position_embeddings", "unknown")
         return {
             "status": "ok",
-            "model": model.name_or_path,
-            "max_context_tokens": max_context,
+            "model_path": getattr(model, "model_path", "unknown"),
+            "model_type": getattr(model, "model_type", "unknown"),
+            "context_length": getattr(model, "context_length", "unknown"),
+            "gpu_layers": getattr(model, "gpu_layers", 0),
         }
     except Exception as e:
-        logger.error("Status error: %s", str(e))
-        raise HTTPException(status_code=500, detail=str(e))
+        logger.error("Status check failed: %s", str(e), exc_info=True)
+        raise HTTPException(status_code=500, detail="Model status check failed")
 
 
 @app.post("/chat", response_model=ChatResponse)
 async def chat(req: PromptRequest):
-    """OpenChat-3.5 Run inference prompt"""
     try:
-        logger.info("Received prompt: %s", req.prompt)
-
-        # Dynamically choose device at request time
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        logger.info(f"Using device: {device}")
-
-        # Move model to device if not
-        if next(model.parameters()).device != device:
-            logger.info("Moving model to %s", device)
-            model.to(device)
-
-        # Tokenize input
-        inputs = tokenizer(req.prompt, return_tensors="pt").to(device)
-
-        # Generation parameters
-        gen_kwargs = {
-            "max_new_tokens": req.max_tokens,
-            "temperature": req.temperature,
-            "do_sample": req.temperature > 0,
-        }
-
-        # Generate output
-        outputs = model.generate(**inputs, **gen_kwargs)
-        generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-        # Trim echoed prompt if present
-        response_text = generated_text[len(req.prompt) :].strip()
-
-        logger.info("Generated response: %s", response_text)
-        return ChatResponse(response=response_text)
-
+        logger.info("Prompt: %s", req.prompt)
+
+        prompt_tokens = model.tokenize(req.prompt)
+        if len(prompt_tokens) > model.context_length:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Prompt too long ({len(prompt_tokens)} tokens). Max context: {model.context_length}",
+            )
+
+        response = model(
+            req.prompt,
+            max_new_tokens=req.max_tokens,
+            temperature=req.temperature,
+            stop=["</s>"],
+        )
+        logger.info("Response: %s", response)
+        return ChatResponse(response=response.strip())
     except Exception as e:
-        logger.error("Inference failed: %s", str(e), exc_info=True)
-        raise HTTPException(status_code=500, detail="Inference failure: " + str(e))
+        logger.error("Chat error: %s", str(e), exc_info=True)
+        raise HTTPException(status_code=500, detail="Inference failure")
 
 
 if __name__ == "__main__":
-    uvicorn.run("app:app", host="0.0.0.0", port=8080, log_level="info")
+    uvicorn.run("app:app", host="0.0.0.0", port=7860, log_level="info")

env.example

@@ -1 +1,4 @@
-MODEL_NAME = "openchat/openchat-3.5-1210"
+MODEL_NAME = "openchat/openchat-3.5-1210"
+MODEL_REPO=TheBloke/openchat_3.5-GGUF
+MODEL_FILE=openchat_3.5.Q4_K_M.gguf
+MODEL_TYPE=mistral

model_loader.py

@@ -1,20 +1,24 @@
-# model_loader.py
 import os
+from ctransformers import AutoModelForCausalLM
 
-# Safe fallback if ENV vars are not set (e.g., during local dev)
+# Optional: create a local cache dir for gguf model if needed
 os.environ.setdefault("HF_HOME", os.path.expanduser("~/.hf_home"))
-os.environ.setdefault("TRANSFORMERS_CACHE", os.path.expanduser("~/.cache/transformers"))
 
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch, os
+# Load environment variables if you plan to use .env (optional)
 from dotenv import load_dotenv
 
-# Load environment variables
 load_dotenv()
-MODEL_NAME = os.getenv("MODEL_NAME", "openchat/openchat-3.5-1210")
 
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+# Model path or name from environment, fallback to default OpenChat
+MODEL_REPO = os.getenv("MODEL_REPO", "TheBloke/openchat_3.5-GGUF")
+MODEL_FILE = os.getenv("MODEL_FILE", "openchat_3.5.Q4_K_M.gguf")
+MODEL_TYPE = os.getenv("MODEL_TYPE", "mistral")  # OpenChat 3.5 is Mistral-compatible
 
-# Load model initially on CPU
-model = AutoModelForCausalLM.from_pretrained(MODEL_NAME).to("cpu")
+# Load quantized GGUF model using ctransformers
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_REPO,
+    model_file=MODEL_FILE,
+    model_type=MODEL_TYPE,
+    gpu_layers=0,
+    local_files_only=False,
+)

requirements.txt

@@ -1,17 +1,10 @@
-# Core FastAPI dependencies
+# Core API
 fastapi>=0.104.0
 uvicorn[standard]>=0.24.0
 pydantic>=2.5.0
 
-# Machine Learning and Transformers
-torch>=2.1.0
-transformers>=4.36.0
-accelerate>=0.25.0
+# Quantized LLM support
+ctransformers>=0.2.27
 
-# Additional utilities
-numpy>=1.24.0
-requests>=2.31.0
-
-# Optional: For better performance and monitoring
-# tensorboard>=2.15.0  # Uncomment if you need training monitoring
-# wandb>=0.16.0        # Uncomment if you need experiment tracking
+# Optional: For local .env files
+python-dotenv>=1.0.0