Spaces:
Sleeping
Sleeping
File size: 1,465 Bytes
6750035 2d59ec1 6750035 2d59ec1 6750035 2d59ec1 6750035 2d59ec1 6750035 2d59ec1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
from pathlib import Path
from langchain.chains import RetrievalQA
from transformers import pipeline, AutoTokenizer
from langchain_community.vectorstores import Chroma
from langchain_community.document_loaders import PyMuPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline
def load_documents(pdf_dir):
docs = []
for pdf_file in Path(pdf_dir).glob("*.pdf"):
loader = PyMuPDFLoader(str(pdf_file))
docs.extend(loader.load())
return docs
def load_rag_chain():
Path("data").mkdir(exist_ok=True)
raw_docs = load_documents("data")
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
pages = splitter.split_documents(raw_docs)
embeddings = HuggingFaceEmbeddings(
model_name="sentence-transformers/LaBSE",
model_kwargs={"device": "cpu"},
)
vectordb = Chroma.from_documents(pages, embeddings, persist_directory="chroma_db")
retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": 5})
hf_pipeline = pipeline(
"text2text-generation",
model="ArabicNLP/mT5-base_ar",
tokenizer=AutoTokenizer.from_pretrained("ArabicNLP/mT5-base_ar"),
max_new_tokens=512,
temperature=0.3,
device=-1,
)
llm = HuggingFacePipeline(pipeline=hf_pipeline)
return RetrievalQA.from_llm(llm=llm, retriever=retriever)
|