Update app/main.py

app/main.py

@@ -287,8 +287,14 @@ from huggingface_hub import hf_hub_download
 from contextlib import asynccontextmanager
 
 # === CONFIGURATION === #
+from llama_cpp import Llama
+
+# REPO_ID = "FreedomIntelligence/Apollo-7B-GGUF"
+REPO_ID = "RichardErkhov/FreedomIntelligence_-_Apollo-2B-gguf"
+# MODEL_NAME = "FreedomIntelligence/Apollo-7B"
 MODEL_NAME = "FreedomIntelligence/Apollo-2B"
-MODEL_FILE = "Apollo-2B.Q4_K_S.gguf"
+# MODEL_FILE = "Apollo-7B.Q4_K_S.gguf"
+MODEL_FILE = "Apollo-2B.IQ4_XS.gguf"
 EMBEDDING_MODEL = "Omartificial-Intelligence-Space/GATE-AraBert-v1"
 COLLECTION_NAME = "arabic_rag_collection"
 QDRANT_URL = os.getenv("QDRANT_URL", "https://12efeef2-9f10-4402-9deb-f070977ddfc8.eu-central-1-0.aws.cloud.qdrant.io:6333")
@@ -328,7 +334,7 @@ async def lifespan(app: FastAPI):
 
         # Load LLM model
         model_path = hf_hub_download(
-            repo_id="FreedomIntelligence/Apollo-7B-GGUF",
+            repo_id=REPO_ID,
             filename=MODEL_FILE,
             local_dir="./models",
             local_dir_use_symlinks=False