simplify code

arxiv_fetcher.py

@@ -1,38 +0,0 @@
-# arxiv_fetcher.py
-
-import arxiv
-from typing import List, Dict, Any
-import logging
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-def fetch_arxiv_metadata(query: str, max_results: int = 10) -> List[Dict[str, Any]]:
-    logging.info(f"Fetching arXiv metadata for query: {query}")
-    if not query.strip():
-        logging.warning("Empty or whitespace-only query provided")
-        return []
-    
-    client = arxiv.Client(page_size=max_results, delay_seconds=3, num_retries=3)
-    search = arxiv.Search(query=query, max_results=max_results, sort_by=arxiv.SortCriterion.SubmittedDate)
-    
-    results = []
-    try:
-        for result in client.results(search):
-            metadata = {
-                "title": result.title,
-                "authors": [author.name for author in result.authors],
-                "published": result.published.isoformat(),
-                "updated": result.updated.isoformat(),
-                "pdf_url": result.pdf_url,
-                "entry_id": result.entry_id,
-                "summary": result.summary,
-                "categories": result.categories,
-                "primary_category": result.primary_category,
-                "html_url": f"http://arxiv.org/abs/{result.entry_id.split('/')[-1]}"
-            }
-            results.append(metadata)
-        logging.info(f"Fetched metadata for {len(results)} papers")
-    except Exception as e:
-        logging.error(f"Error fetching metadata: {str(e)}")
-    
-    return results

arxiv_metadata_service.py

@@ -1,58 +0,0 @@
-from arxiv_fetcher import fetch_arxiv_metadata
-from datasets import load_dataset, Dataset
-from huggingface_hub import HfApi
-from config import DATASET_NAME
-import logging
-from typing import List, Dict, Any
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-class ArxivMetadataService:
-    def __init__(self):
-        self.hf_api = HfApi()
-
-    def extract_and_update(self, query: str, max_results: int = 10) -> str:
-        metadata_list = fetch_arxiv_metadata(query, max_results)
-        if not metadata_list:
-            return "No metadata found for the given query."
-        return self.update_dataset(metadata_list)
-
-    def update_dataset(self, metadata_list: List[Dict[str, Any]]) -> str:
-        try:
-            # Load the existing dataset
-            try:
-                dataset = load_dataset(DATASET_NAME, split="train")
-                current_data = dataset.to_dict()
-            except Exception:
-                # If loading fails, start with an empty dictionary
-                current_data = {}
-
-            # If the dataset is empty, initialize it with the structure from metadata_list
-            if not current_data:
-                current_data = {key: [] for key in metadata_list[0].keys()}
-
-            updated = False
-            for paper in metadata_list:
-                entry_id = paper['entry_id'].split('/')[-1]
-                if 'entry_id' not in current_data or entry_id not in current_data['entry_id']:
-                    # Add new paper
-                    for key, value in paper.items():
-                        current_data.setdefault(key, []).append(value)
-                    updated = True
-                else:
-                    # Update existing paper
-                    index = current_data['entry_id'].index(entry_id)
-                    for key, value in paper.items():
-                        if current_data[key][index] != value:
-                            current_data[key][index] = value
-                            updated = True
-
-            if updated:
-                updated_dataset = Dataset.from_dict(current_data)
-                updated_dataset.push_to_hub(DATASET_NAME, split="train")
-                return f"Successfully updated dataset with {len(metadata_list)} papers"
-            else:
-                return "No new data to update."
-        except Exception as e:
-            logging.error(f"Failed to update dataset: {str(e)}")
-            return f"Failed to update dataset: {str(e)}"

config.py

@@ -1,15 +1,4 @@
 # File: config.py
 import os
 
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
-QDRANT_API_URL = os.getenv("QDRANT_API_URL")
-GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-COLLECTION_NAME = "arxiv_papers"
-DATASET_NAME = "dwb2023/arxiv-papers-dataset"
-
-LANGCHAIN_PROJECT="arxiv_papers"
-LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
-LANGCHAIN_TRACING_V2="true"
-LANGCHAIN_HUB_PROMPT="rlm/rag-prompt-llama3"
-LANGCHAIN_API_KEY=os.getenv("LANGCHAIN_API_KEY")
+DATASET_NAME = "dwb2023/arxiv-papers-dataset"