Spaces:
Sleeping
Sleeping
File size: 1,613 Bytes
6750035 9f0b7c7 6750035 9f0b7c7 6750035 9f0b7c7 6750035 9f0b7c7 6750035 2d59ec1 9f0b7c7 6750035 2d59ec1 6750035 2d59ec1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
from pathlib import Path
from langchain.chains import RetrievalQA
from transformers import pipeline, T5Tokenizer
from langchain_community.vectorstores import Chroma
from langchain_community.document_loaders import PyMuPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline
def load_documents(pdf_dir):
docs = []
for pdf_file in Path(pdf_dir).glob("*.pdf"):
loader = PyMuPDFLoader(str(pdf_file))
docs.extend(loader.load())
return docs
def load_rag_chain():
pdf_dir = Path("data")
pdf_dir.mkdir(parents=True, exist_ok=True)
raw_docs = load_documents(pdf_dir)
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
pages = splitter.split_documents(raw_docs)
embeddings = HuggingFaceEmbeddings(
model_name="sentence-transformers/LaBSE",
model_kwargs={"device": "cpu"},
)
vectordb_dir = "chroma_db"
vectordb = Chroma.from_documents(pages, embeddings, persist_directory=vectordb_dir)
retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": 5})
# ✅ Use slow tokenizer explicitly
tokenizer = T5Tokenizer.from_pretrained("ArabicNLP/mT5-base_ar", use_fast=False)
hf_pipeline = pipeline(
"text2text-generation",
model="ArabicNLP/mT5-base_ar",
tokenizer=tokenizer,
max_new_tokens=512,
temperature=0.3,
device=-1,
)
llm = HuggingFacePipeline(pipeline=hf_pipeline)
return RetrievalQA.from_llm(llm=llm, retriever=retriever)
|