Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
@@ -5,17 +5,17 @@ from sentence_transformers import SentenceTransformer
|
|
5 |
import json
|
6 |
from dspy import Example, MIPROv2, Evaluate, evaluate
|
7 |
|
8 |
-
# إعداد نموذج مفتوح المصدر
|
9 |
dspy.settings.configure(lm=dspy.LM("mistralai/Mistral-7B-Instruct-v0.2"))
|
10 |
|
11 |
-
# إعداد Chroma
|
12 |
client = chromadb.PersistentClient(path="./chroma_db")
|
13 |
col = client.get_or_create_collection(name="arabic_docs")
|
14 |
|
15 |
-
# نموذج embeddings يدعم
|
16 |
embedder = SentenceTransformer("sentence-transformers/LaBSE")
|
17 |
|
18 |
-
#
|
19 |
def process_pdf(pdf_bytes):
|
20 |
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
21 |
texts = []
|
@@ -26,7 +26,7 @@ def process_pdf(pdf_bytes):
|
|
26 |
texts.append(chunk.strip())
|
27 |
return texts
|
28 |
|
29 |
-
# إدخال
|
30 |
def ingest(pdf_bytes):
|
31 |
texts = process_pdf(pdf_bytes)
|
32 |
embeddings = embedder.encode(texts, show_progress_bar=True)
|
@@ -34,16 +34,16 @@ def ingest(pdf_bytes):
|
|
34 |
col.add(ids=[f"chunk_{i}"], embeddings=[emb.tolist()], metadatas=[{"text": chunk}])
|
35 |
return f"✅ تمت إضافة {len(texts)} مقطعاً."
|
36 |
|
37 |
-
#
|
38 |
retriever = dspy.Retrieve(lambda q: [m["text"] for m in col.query(q, n_results=1)["metadatas"]])
|
39 |
|
40 |
-
#
|
41 |
class RagSig(dspy.Signature):
|
42 |
-
question:
|
43 |
-
context:
|
44 |
-
answer:
|
45 |
|
46 |
-
# وحدة
|
47 |
class RagMod(dspy.Module):
|
48 |
def __init__(self):
|
49 |
super().__init__()
|
@@ -55,12 +55,12 @@ class RagMod(dspy.Module):
|
|
55 |
|
56 |
model = RagMod()
|
57 |
|
58 |
-
# توليد
|
59 |
def answer(question):
|
60 |
out = model(question)
|
61 |
return out.answer
|
62 |
|
63 |
-
# تحميل بيانات
|
64 |
def load_dataset(path):
|
65 |
with open(path, "r", encoding="utf-8") as f:
|
66 |
return [Example(**json.loads(l)).with_inputs("question") for l in f]
|
|
|
5 |
import json
|
6 |
from dspy import Example, MIPROv2, Evaluate, evaluate
|
7 |
|
8 |
+
# إعداد نموذج مفتوح المصدر
|
9 |
dspy.settings.configure(lm=dspy.LM("mistralai/Mistral-7B-Instruct-v0.2"))
|
10 |
|
11 |
+
# إعداد Chroma بطريقة حديثة
|
12 |
client = chromadb.PersistentClient(path="./chroma_db")
|
13 |
col = client.get_or_create_collection(name="arabic_docs")
|
14 |
|
15 |
+
# نموذج توليد embeddings يدعم العربية
|
16 |
embedder = SentenceTransformer("sentence-transformers/LaBSE")
|
17 |
|
18 |
+
# تقطيع النصوص من PDF
|
19 |
def process_pdf(pdf_bytes):
|
20 |
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
21 |
texts = []
|
|
|
26 |
texts.append(chunk.strip())
|
27 |
return texts
|
28 |
|
29 |
+
# إدخال النصوص في قاعدة Chroma
|
30 |
def ingest(pdf_bytes):
|
31 |
texts = process_pdf(pdf_bytes)
|
32 |
embeddings = embedder.encode(texts, show_progress_bar=True)
|
|
|
34 |
col.add(ids=[f"chunk_{i}"], embeddings=[emb.tolist()], metadatas=[{"text": chunk}])
|
35 |
return f"✅ تمت إضافة {len(texts)} مقطعاً."
|
36 |
|
37 |
+
# استرجاع السياق من Chroma
|
38 |
retriever = dspy.Retrieve(lambda q: [m["text"] for m in col.query(q, n_results=1)["metadatas"]])
|
39 |
|
40 |
+
# ✅ تعريف التوقيع باستخدام InputField و OutputField
|
41 |
class RagSig(dspy.Signature):
|
42 |
+
question: dspy.InputField()
|
43 |
+
context: dspy.InputField()
|
44 |
+
answer: dspy.OutputField()
|
45 |
|
46 |
+
# وحدة DSPy
|
47 |
class RagMod(dspy.Module):
|
48 |
def __init__(self):
|
49 |
super().__init__()
|
|
|
55 |
|
56 |
model = RagMod()
|
57 |
|
58 |
+
# توليد إجابة
|
59 |
def answer(question):
|
60 |
out = model(question)
|
61 |
return out.answer
|
62 |
|
63 |
+
# تحميل بيانات تدريب/تقييم
|
64 |
def load_dataset(path):
|
65 |
with open(path, "r", encoding="utf-8") as f:
|
66 |
return [Example(**json.loads(l)).with_inputs("question") for l in f]
|