improvements

README.md

@@ -15,7 +15,13 @@ You can find an implementation of the Tutor at [DL4DS Tutor on Hugging Face](htt
    - Add URLs in the `urls.txt` file.
    - Add other PDF files in the `storage/data` directory.
 
-3. **Create the Vector Database**
+3. **To test Data Loading (Optional)**
+   ```bash
+   cd code
+   python -m modules.dataloader.data_loader
+   ```
+
+4. **Create the Vector Database**
    ```bash
    cd code
    python -m modules.vectorstore.store_manager
@@ -23,7 +29,7 @@ You can find an implementation of the Tutor at [DL4DS Tutor on Hugging Face](htt
    - Note: You need to run the above command when you add new data to the `storage/data` directory, or if the `storage/data/urls.txt` file is updated.
    - Alternatively, you can set `["vectorstore"]["embedd_files"]` to `True` in the `code/modules/config/config.yaml` file, which will embed files from the storage directory every time you run the below chainlit command.
 
-4. **Run the Chainlit App**
+5. **Run the Chainlit App**
    ```bash
    chainlit run main.py
    ```

code/main.py

@@ -10,27 +10,20 @@ import yaml
 import logging
 from dotenv import load_dotenv
 
-import os
-import sys
-
-# Add the 'code' directory to the Python path
-current_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(current_dir)
-
 from modules.chat.llm_tutor import LLMTutor
 from modules.config.constants import *
 from modules.chat.helpers import get_sources
 from modules.chat_processor.chat_processor import ChatProcessor
 
 global logger
+# Initialize logger
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
-logger.propagate = False
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 
 # Console Handler
 console_handler = logging.StreamHandler()
 console_handler.setLevel(logging.INFO)
-formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 console_handler.setFormatter(formatter)
 logger.addHandler(console_handler)
 

code/modules/__init__.py

@@ -1,2 +0,0 @@
-from . import vectorstore
-from . import dataloader

code/modules/chat/__init__.py

@@ -1,2 +0,0 @@
-from .llm_tutor import LLMTutor
-from .chat_model_loader import ChatModelLoader

code/modules/chat/llm_tutor.py

@@ -10,7 +10,7 @@ from modules.chat.helpers import get_prompt
 from modules.chat.chat_model_loader import ChatModelLoader
 from modules.vectorstore.store_manager import VectorStoreManager
 
-from modules.retriever import Retriever
+from modules.retriever.retriever import Retriever
 
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 from langchain_core.callbacks.manager import AsyncCallbackManagerForChainRun

code/modules/config/__init__.py

@@ -1 +0,0 @@
-from .constants import *

code/modules/config/config.yml

@@ -7,7 +7,7 @@ vectorstore:
   data_path: '../storage/data' # str
   url_file_path: '../storage/data/urls.txt' # str
   expand_urls: True # bool
-  db_option : 'RAGatouille' # str [FAISS, Chroma, RAGatouille, RAPTOR]
+  db_option : 'FAISS' # str [FAISS, Chroma, RAGatouille, RAPTOR]
   db_path : '../vectorstores' # str
   model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002']
   search_top_k : 3 # int

code/modules/dataloader/__init__.py

@@ -1,2 +0,0 @@
-from .webpage_crawler import WebpageCrawler
-from .data_loader import DataLoader

code/modules/dataloader/data_loader.py

@@ -345,7 +345,7 @@ if __name__ == "__main__":
     logger = logging.getLogger(__name__)
     logger.setLevel(logging.INFO)
 
-    with open("../code/config.yml", "r") as f:
+    with open("../code/modules/config/config.yml", "r") as f:
         config = yaml.safe_load(f)
 
     data_loader = DataLoader(config, logger=logger)
@@ -355,3 +355,6 @@ if __name__ == "__main__":
             ["https://dl4ds.github.io/sp2024/"],
         )
     )
+
+    print(document_names)
+    print(len(document_chunks))

code/modules/retriever/__init__.py

@@ -1,5 +0,0 @@
-from .faiss_retriever import FaissRetriever
-from .chroma_retriever import ChromaRetriever
-from .colbert_retriever import ColbertRetriever
-from .raptor_retriever import RaptorRetriever
-from .retriever import Retriever

code/modules/vectorstore/store_manager.py

@@ -16,37 +16,36 @@ class VectorStoreManager:
         self.document_names = None
 
         # Set up logging to both console and a file
-        if logger is None:
-            self.logger = logging.getLogger(__name__)
-            self.logger.setLevel(logging.INFO)
-            self.logger.propagate = False
+        self.logger = logger or self._setup_logging()
+        self.webpage_crawler = WebpageCrawler()
+        self.vector_db = VectorStore(self.config)
+
+        self.logger.info("VectorDB instance instantiated")
+
+    def _setup_logging(self):
+        logger = logging.getLogger(__name__)
+        if not logger.hasHandlers():
+            logger.setLevel(logging.INFO)
+            formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 
             # Console Handler
             console_handler = logging.StreamHandler()
             console_handler.setLevel(logging.INFO)
-            formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
             console_handler.setFormatter(formatter)
-            self.logger.addHandler(console_handler)
+            logger.addHandler(console_handler)
 
             # Ensure log directory exists
             log_directory = self.config["log_dir"]
-            if not os.path.exists(log_directory):
-                os.makedirs(log_directory)
+            os.makedirs(log_directory, exist_ok=True)
 
             # File Handler
-            log_file_path = f"{log_directory}/vector_db.log"  # Change this to your desired log file path
+            log_file_path = os.path.join(log_directory, "vector_db.log")
             file_handler = logging.FileHandler(log_file_path, mode="w")
             file_handler.setLevel(logging.INFO)
             file_handler.setFormatter(formatter)
-            self.logger.addHandler(file_handler)
-        else:
-            self.logger = logger
-
-        self.webpage_crawler = WebpageCrawler()
+            logger.addHandler(file_handler)
 
-        self.vector_db = VectorStore(self.config)
-
-        self.logger.info("VectorDB instance instantiated")
+        return logger
 
     def load_files(self):