Delete rag_pipeline.py

rag_pipeline.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-from langchain.chains import RetrievalQA
-from transformers import pipeline, T5Tokenizer
-from langchain_community.vectorstores import Chroma
-from langchain_community.document_loaders import PyMuPDFLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_huggingface import HuggingFaceEmbeddings, HuggingFacePipeline
-
-def load_documents(pdf_dir):
-    docs = []
-    for pdf_file in Path(pdf_dir).glob("*.pdf"):
-        loader = PyMuPDFLoader(str(pdf_file))
-        docs.extend(loader.load())
-    return docs
-
-def load_rag_chain():
-    pdf_dir = Path("data")
-    pdf_dir.mkdir(parents=True, exist_ok=True)
-
-    raw_docs = load_documents(pdf_dir)
-    splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-    pages = splitter.split_documents(raw_docs)
-
-    embeddings = HuggingFaceEmbeddings(
-        model_name="sentence-transformers/LaBSE",
-        model_kwargs={"device": "cpu"},
-    )
-
-    vectordb_dir = "chroma_db"
-    vectordb = Chroma.from_documents(pages, embeddings, persist_directory=vectordb_dir)
-    retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": 5})
-
-    # ✅ Use slow tokenizer explicitly
-    tokenizer = T5Tokenizer.from_pretrained("ArabicNLP/mT5-base_ar", use_fast=False)
-    hf_pipeline = pipeline(
-        "text2text-generation",
-        model="ArabicNLP/mT5-base_ar",
-        tokenizer=tokenizer,
-        max_new_tokens=512,
-        temperature=0.3,
-        device=-1,
-    )
-    llm = HuggingFacePipeline(pipeline=hf_pipeline)
-
-    return RetrievalQA.from_llm(llm=llm, retriever=retriever)