initial commit

.gitignore

@@ -0,0 +1,2 @@
+.venv/
+.env

.python-version

@@ -0,0 +1 @@
+3.10

README.md

@@ -10,4 +10,44 @@ pinned: false
 license: cc-by-4.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# arXiv RAG System README
+
+## Key Stakeholder
+The primary stakeholder for this system is an agentic "System of Agents". This design choice emphasizes the need for modularity, flexibility, and the ability for the system to evolve and improve its own processes.
+
+## Architectural Vision
+- The system is designed with modularity in mind, using a microservices architecture to allow easy replacement of specific components (libraries, applications, LLMs).
+- A Hugging Face dataset is used to store metadata and interim results for retrieved documents, crucial for avoiding repetitive and costly processing.
+- The system captures and tracks the history of document reviews, summarizations, and evaluations performed by the agents.
+
+## Key Architectural Choices
+
+1. **Document Loading**: PyMuPDF for efficient PDF processing with image extraction.
+
+2. **Text Splitting**: RecursiveCharacterTextSplitter for content chunking.
+
+3. **Embedding Model**: OpenAI's text-embedding-3-small for generating embeddings.
+
+4. **Vector Database**: Qdrant for storing and retrieving embeddings.
+
+5. **Retrieval Mechanism**: Similarity search with cosine similarity threshold of 0.5 and k=5.
+
+6. **Language Model**: Llama3 70B via Groq API for response generation.
+
+7. **Orchestration**: LangChain/LCEL for RAG pipeline orchestration.
+
+8. **User Interface**: Gradio for demonstration purposes (note: primary interface is for the agentic system).
+
+9. **Logging and Monitoring**: LangSmith for comprehensive logging and LLM operations monitoring.
+
+10. **Metadata and Results Storage**: Hugging Face dataset for storing document metadata, interim results, and agent review history.
+
+## Sequence Diagrams
+
+### 1. Ingestion Flow
+
+[Mermaid Diagram - Ingestion Flow](ingestion-flow-diagram.mermaid)
+
+### 2. Retrieval Flow
+
+[Mermaid Diagram - Retrieval Flow](retrieval-flow-diagram.mermaid)

app.py

@@ -0,0 +1,17 @@
+# File: app.py
+import gradio as gr
+from retrieval import rag_query
+
+def gradio_interface(query: str) -> str:
+    return rag_query(query)
+
+iface = gr.Interface(
+    fn=gradio_interface,
+    inputs="text",
+    outputs="text",
+    title="arXiv RAG System",
+    description="Ask questions about arXiv papers"
+)
+
+if __name__ == "__main__":
+    iface.launch()

config.py

@@ -0,0 +1,15 @@
+# File: config.py
+import os
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
+QDRANT_API_URL = os.getenv("QDRANT_API_URL")
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+COLLECTION_NAME = "arxiv_papers"
+DATASET_NAME = "dwb2023/arxiv-papers-dataset"  # Replace with your actual dataset name
+
+LANGCHAIN_PROJECT="arxiv_papers"
+LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
+LANGCHAIN_TRACING_V2="true"
+LANGCHAIN_HUB_PROMPT="rlm/rag-prompt-llama3"
+LANGCHAIN_API_KEY=os.getenv("LANGCHAIN_API_KEY")

data_ingestion.py

@@ -0,0 +1,69 @@
+
+# File: data_ingestion.py
+import arxiv
+import io
+import requests
+from typing import List, Dict, Any
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_qdrant import Qdrant
+from datasets import load_dataset, Dataset
+from langchain_community.document_loaders import PyMuPDFLoader
+from config import *
+
+embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+
+def fetch_arxiv_metadata(query: str, max_results: int = 10) -> List[Dict[str, Any]]:
+    client = arxiv.Client(page_size=max_results, delay_seconds=3, num_retries=3)
+    search = arxiv.Search(query=query, max_results=max_results)
+    results = []
+    for result in client.results(search):
+        metadata = {
+            "title": result.title,
+            "authors": [author.name for author in result.authors],
+            "published": result.published.isoformat(),
+            "updated": result.updated.isoformat(),
+            "pdf_url": result.pdf_url,
+            "entry_id": result.entry_id,
+            "summary": result.summary
+        }
+        results.append(metadata)
+    return results
+
+def process_pdf(pdf_url: str) -> str:
+    loader = PyMuPDFLoader(pdf_url)
+    data = loader.load()
+    return "\n".join([page.page_content for page in data])
+
+def ingest_documents(metadata_list: List[Dict[str, Any]]):
+    qdrant = Qdrant.from_documents(
+        [],  # We'll add documents one by one
+        embeddings,
+        url=QDRANT_API_URL,
+        api_key=QDRANT_API_KEY,
+        collection_name=COLLECTION_NAME,
+    )
+
+    dataset = load_dataset(DATASET_NAME)
+    new_data = []
+
+    for metadata in metadata_list:
+        pdf_text = process_pdf(metadata["pdf_url"])
+        chunks = text_splitter.split_text(pdf_text)
+        
+        # Add to Qdrant
+        qdrant.add_texts(chunks, metadatas=[metadata] * len(chunks))
+        
+        # Prepare data for Hugging Face dataset
+        for chunk in chunks:
+            new_data.append({
+                "text": chunk,
+                "metadata": metadata,
+                "embedding": embeddings.embed_query(chunk)
+            })
+    
+    # Update Hugging Face dataset
+    new_dataset = Dataset.from_dict({k: [d[k] for d in new_data] for k in new_data[0]})
+    dataset = dataset.add_item(new_dataset)
+    dataset.push_to_hub(DATASET_NAME)

ingest_papers.py

@@ -0,0 +1,9 @@
+# File: ingest_papers.py
+from data_ingestion import fetch_arxiv_metadata, ingest_documents
+
+if __name__ == "__main__":
+    query = input("Enter a query to fetch arXiv papers: ")
+    max_results = int(input("Enter the maximum number of papers to fetch: "))
+    metadata_list = fetch_arxiv_metadata(query, max_results)
+    ingest_documents(metadata_list)
+    print(f"Ingested {len(metadata_list)} papers.")

ingestion-flow-diagram.mermaid

@@ -0,0 +1,15 @@
+sequenceDiagram
+    participant PDF as arXiv PDF Document
+    participant DL as Document Loader (PyMuPDF)
+    participant TS as Text Splitter (RecursiveCharacter)
+    participant EM as Embedding Model (OpenAI)
+    participant VDB as Vector Database (Qdrant)
+    participant DS as Dataset (Hugging Face)
+
+    PDF->>DL: Load document
+    Note over DL: extract_images=True
+    DL->>TS: Pass extracted text
+    TS->>EM: Send text chunks
+    EM->>VDB: Store embeddings
+    DL->>DS: Store metadata
+    DL->>DS: Store extracted text

requirements.txt

@@ -0,0 +1,51 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --no-annotate --strip-extras
+#
+--extra-index-url https://pypi.ngc.nvidia.com
+--trusted-host pypi.ngc.nvidia.com
+
+attrs==23.2.0
+build==1.2.1
+click==8.1.7
+colorama==0.4.6
+fastjsonschema==2.20.0
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter-core==5.7.2
+jupytext==1.16.2
+markdown-it-py==3.0.0
+mdit-py-plugins==0.4.1
+mdurl==0.1.2
+nbformat==5.10.4
+packaging==24.1
+platformdirs==4.2.2
+pyproject-hooks==1.1.0
+pyyaml==6.0.1
+referencing==0.35.1
+rpds-py==0.18.1
+tomli==2.0.1
+traitlets==5.14.3
+wheel==0.43.0
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
+
+langchain==0.2.5
+langchain-core==0.2.9
+langchain-groq==0.1.5
+langchain-openai==0.1.8
+langchain-qdrant==0.1.1
+langchain-text-splitters==0.2.1
+langchainhub==0.1.20
+langsmith==0.1.81
+openai==1.34.0
+qdrant-client==1.9.1
+arxiv
+pymupdf
+datasets
+gradio
+langchain-community

retrieval-flow-diagram.mermaid

@@ -0,0 +1,26 @@
+sequenceDiagram
+    Actor MOA as Agentic Reasoning System
+    Actor User
+    participant App as Application (Gradio)
+    participant Orch as Orchestration (LangChain/LCEL)
+    participant EM as Embedding Model (OpenAI)
+    participant VDB as Vector Database (Qdrant)
+    participant LLM as LLM (Llama3 70B via Groq)
+    participant HF as Hugging Face Dataset
+    participant Log as Logging (LangSmith)
+
+    MOA->>Orch: Submit query
+    User->>App: Submit query
+    App->>Orch: Process query
+    Orch->>EM: Generate query embedding
+    EM->>VDB: Retrieve similar vectors
+    Note over VDB: similarity threshold: 0.5, k=5
+    VDB->>Orch: Return relevant contexts
+    Orch->>HF: Retrieve additional metadata
+    Orch->>LLM: Send query + contexts + metadata
+    LLM->>Orch: Generate response
+    Orch->>MOA: Return response for analysis
+    MOA->>HF: Store analysis and evaluation
+    Orch->>App: Return response
+    App->>User: Display response
+    Orch->>Log: Log interaction

retrieval.py

@@ -0,0 +1,30 @@
+# File: retrieval.py
+from langchain_qdrant import Qdrant
+from langchain_groq import ChatGroq
+from langchain_openai import OpenAIEmbeddings
+from langchain.chains import RetrievalQA
+from config import *
+
+embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+llm = ChatGroq(model="llama3-70b-4096", temperature=0.3)
+
+def rag_query(query: str) -> str:
+    qdrant = Qdrant.from_existing_collection(
+        embedding=embeddings,
+        collection_name=COLLECTION_NAME,
+        url=QDRANT_API_URL,
+        api_key=QDRANT_API_KEY,
+        prefer_grpc=True,
+    )
+    
+    retriever = qdrant.as_retriever(search_kwargs={"k": 5})
+    
+    qa_chain = RetrievalQA.from_chain_type(
+        llm=llm,
+        chain_type="stuff",
+        retriever=retriever,
+        return_source_documents=True
+    )
+    
+    result = qa_chain({"query": query})
+    return result["result"]