Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
|
@@ -5,17 +5,17 @@ from sentence_transformers import SentenceTransformer
|
|
| 5 |
import json
|
| 6 |
from dspy import Example, MIPROv2, Evaluate, evaluate
|
| 7 |
|
| 8 |
-
#
|
| 9 |
dspy.settings.configure(lm=dspy.LM("mistralai/Mistral-7B-Instruct-v0.2"))
|
| 10 |
|
| 11 |
-
#
|
| 12 |
client = chromadb.PersistentClient(path="./chroma_db")
|
| 13 |
col = client.get_or_create_collection(name="arabic_docs")
|
| 14 |
|
| 15 |
-
#
|
| 16 |
embedder = SentenceTransformer("sentence-transformers/LaBSE")
|
| 17 |
|
| 18 |
-
#
|
| 19 |
def process_pdf(pdf_bytes):
|
| 20 |
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
| 21 |
texts = []
|
|
@@ -26,7 +26,7 @@ def process_pdf(pdf_bytes):
|
|
| 26 |
texts.append(chunk.strip())
|
| 27 |
return texts
|
| 28 |
|
| 29 |
-
#
|
| 30 |
def ingest(pdf_bytes):
|
| 31 |
texts = process_pdf(pdf_bytes)
|
| 32 |
embeddings = embedder.encode(texts, show_progress_bar=True)
|
|
@@ -34,16 +34,16 @@ def ingest(pdf_bytes):
|
|
| 34 |
col.add(ids=[f"chunk_{i}"], embeddings=[emb.tolist()], metadatas=[{"text": chunk}])
|
| 35 |
return f"✅ تمت إضافة {len(texts)} مقطعاً."
|
| 36 |
|
| 37 |
-
#
|
| 38 |
-
retriever = dspy.Retrieve(lambda q: [m["text"] for m in col.query(q, n_results=
|
| 39 |
|
| 40 |
-
#
|
| 41 |
class RagSig(dspy.Signature):
|
| 42 |
question: str
|
| 43 |
context: str
|
| 44 |
answer: str
|
| 45 |
|
| 46 |
-
# وحدة
|
| 47 |
class RagMod(dspy.Module):
|
| 48 |
def __init__(self):
|
| 49 |
super().__init__()
|
|
@@ -55,17 +55,17 @@ class RagMod(dspy.Module):
|
|
| 55 |
|
| 56 |
model = RagMod()
|
| 57 |
|
| 58 |
-
#
|
| 59 |
def answer(question):
|
| 60 |
out = model(question)
|
| 61 |
return out.answer
|
| 62 |
|
| 63 |
-
#
|
| 64 |
def load_dataset(path):
|
| 65 |
with open(path, "r", encoding="utf-8") as f:
|
| 66 |
return [Example(**json.loads(l)).with_inputs("question") for l in f]
|
| 67 |
|
| 68 |
-
#
|
| 69 |
def optimize(train_file, val_file):
|
| 70 |
global model
|
| 71 |
trainset = load_dataset(train_file.name)
|
|
@@ -75,7 +75,7 @@ def optimize(train_file, val_file):
|
|
| 75 |
model = optimized
|
| 76 |
return "✅ تم تحسين النموذج!"
|
| 77 |
|
| 78 |
-
#
|
| 79 |
with gr.Blocks() as demo:
|
| 80 |
gr.Markdown("## 🧠 نظام RAG عربي باستخدام DSPy + نموذج مفتوح المصدر")
|
| 81 |
|
|
|
|
| 5 |
import json
|
| 6 |
from dspy import Example, MIPROv2, Evaluate, evaluate
|
| 7 |
|
| 8 |
+
# إعداد نموذج مفتوح المصدر من Hugging Face
|
| 9 |
dspy.settings.configure(lm=dspy.LM("mistralai/Mistral-7B-Instruct-v0.2"))
|
| 10 |
|
| 11 |
+
# إعداد Chroma بالطريقة الجديدة
|
| 12 |
client = chromadb.PersistentClient(path="./chroma_db")
|
| 13 |
col = client.get_or_create_collection(name="arabic_docs")
|
| 14 |
|
| 15 |
+
# نموذج embeddings يدعم اللغة العربية
|
| 16 |
embedder = SentenceTransformer("sentence-transformers/LaBSE")
|
| 17 |
|
| 18 |
+
# استخراج النصوص من PDF
|
| 19 |
def process_pdf(pdf_bytes):
|
| 20 |
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
| 21 |
texts = []
|
|
|
|
| 26 |
texts.append(chunk.strip())
|
| 27 |
return texts
|
| 28 |
|
| 29 |
+
# إدخال البيانات في قاعدة Chroma
|
| 30 |
def ingest(pdf_bytes):
|
| 31 |
texts = process_pdf(pdf_bytes)
|
| 32 |
embeddings = embedder.encode(texts, show_progress_bar=True)
|
|
|
|
| 34 |
col.add(ids=[f"chunk_{i}"], embeddings=[emb.tolist()], metadatas=[{"text": chunk}])
|
| 35 |
return f"✅ تمت إضافة {len(texts)} مقطعاً."
|
| 36 |
|
| 37 |
+
# مكون الاسترجاع من Chroma (بلا k)
|
| 38 |
+
retriever = dspy.Retrieve(lambda q: [m["text"] for m in col.query(q, n_results=1)["metadatas"]])
|
| 39 |
|
| 40 |
+
# توقيع DSPy
|
| 41 |
class RagSig(dspy.Signature):
|
| 42 |
question: str
|
| 43 |
context: str
|
| 44 |
answer: str
|
| 45 |
|
| 46 |
+
# وحدة RAG
|
| 47 |
class RagMod(dspy.Module):
|
| 48 |
def __init__(self):
|
| 49 |
super().__init__()
|
|
|
|
| 55 |
|
| 56 |
model = RagMod()
|
| 57 |
|
| 58 |
+
# توليد الإجابة
|
| 59 |
def answer(question):
|
| 60 |
out = model(question)
|
| 61 |
return out.answer
|
| 62 |
|
| 63 |
+
# تحميل بيانات التدريب/التقييم
|
| 64 |
def load_dataset(path):
|
| 65 |
with open(path, "r", encoding="utf-8") as f:
|
| 66 |
return [Example(**json.loads(l)).with_inputs("question") for l in f]
|
| 67 |
|
| 68 |
+
# تحسين النموذج باستخدام MIPROv2
|
| 69 |
def optimize(train_file, val_file):
|
| 70 |
global model
|
| 71 |
trainset = load_dataset(train_file.name)
|
|
|
|
| 75 |
model = optimized
|
| 76 |
return "✅ تم تحسين النموذج!"
|
| 77 |
|
| 78 |
+
# واجهة Gradio
|
| 79 |
with gr.Blocks() as demo:
|
| 80 |
gr.Markdown("## 🧠 نظام RAG عربي باستخدام DSPy + نموذج مفتوح المصدر")
|
| 81 |
|