rag_pipeline.py

from pathlib import Path
from langchain.chains import RetrievalQA
from transformers import pipeline, AutoTokenizer
from langchain_community.vectorstores import Chroma
from langchain_community.document_loaders import PyMuPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline

def load_documents(pdf_dir):
    docs = []
    for pdf_file in Path(pdf_dir).glob("*.pdf"):
        loader = PyMuPDFLoader(str(pdf_file))
        docs.extend(loader.load())
    return docs

def load_rag_chain():
    Path("data").mkdir(exist_ok=True)
    raw_docs = load_documents("data")
    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
    pages = splitter.split_documents(raw_docs)

    embeddings = HuggingFaceEmbeddings(
        model_name="sentence-transformers/LaBSE",
        model_kwargs={"device": "cpu"},
    )

    vectordb = Chroma.from_documents(pages, embeddings, persist_directory="chroma_db")
    retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": 5})

    hf_pipeline = pipeline(
        "text2text-generation",
        model="ArabicNLP/mT5-base_ar",
        tokenizer=AutoTokenizer.from_pretrained("ArabicNLP/mT5-base_ar"),
        max_new_tokens=512,
        temperature=0.3,
        device=-1,
    )
    llm = HuggingFacePipeline(pipeline=hf_pipeline)

    return RetrievalQA.from_llm(llm=llm, retriever=retriever)