Update rag_pipeline.py

rag_pipeline.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 from langchain.chains import RetrievalQA
-from transformers import pipeline, AutoTokenizer
+from transformers import pipeline, T5Tokenizer
 from langchain_community.vectorstores import Chroma
 from langchain_community.document_loaders import PyMuPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -14,8 +14,10 @@ def load_documents(pdf_dir):
     return docs
 
 def load_rag_chain():
-    Path("data").mkdir(exist_ok=True)
-    raw_docs = load_documents("data")
+    pdf_dir = Path("data")
+    pdf_dir.mkdir(parents=True, exist_ok=True)
+
+    raw_docs = load_documents(pdf_dir)
     splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
     pages = splitter.split_documents(raw_docs)
 
@@ -24,13 +26,16 @@ def load_rag_chain():
         model_kwargs={"device": "cpu"},
     )
 
-    vectordb = Chroma.from_documents(pages, embeddings, persist_directory="chroma_db")
+    vectordb_dir = "chroma_db"
+    vectordb = Chroma.from_documents(pages, embeddings, persist_directory=vectordb_dir)
     retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": 5})
 
+    # ✅ Use slow tokenizer explicitly
+    tokenizer = T5Tokenizer.from_pretrained("ArabicNLP/mT5-base_ar", use_fast=False)
     hf_pipeline = pipeline(
         "text2text-generation",
         model="ArabicNLP/mT5-base_ar",
-        tokenizer=AutoTokenizer.from_pretrained("ArabicNLP/mT5-base_ar"),
+        tokenizer=tokenizer,
         max_new_tokens=512,
         temperature=0.3,
         device=-1,