Upload 6 files

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.linting.enabled": false
+}

app.py

@@ -0,0 +1,74 @@
+import streamlit as st
+import pandas as pd
+
+from knowledge_base import KnowledgeBase
+
+# Page setup
+st.set_page_config(page_title="Website to AI-Powered Knowledge Base", page_icon="🐍")
+st.title("AI-Powered Knowledge Base")
+
+# Remove whitespace from the top of the page and sidebar
+st.markdown(
+    """
+        <style>
+               .css-18e3th9 {
+                    padding-top: 0rem;
+                    padding-bottom: 10rem;
+                    padding-left: 5rem;
+                    padding-right: 5rem;
+                }
+               .css-1d391kg {
+                    padding-top: 3.5rem;
+                    padding-right: 1rem;
+                    padding-bottom: 3.5rem;
+                    padding-left: 1rem;
+                }
+        </style>
+""",
+    unsafe_allow_html=True,
+)
+
+st.markdown("## Config")
+
+col1, col2 = st.columns(2)
+
+with col1:
+    sitemap_url = st.text_input("URL to the website sitemap", value="")
+
+with col2:
+    pattern = st.text_input("URL filter pattern (optional)", value="")
+
+
+st.markdown("## Ask")
+
+
+@st.cache_resource
+def get_knowledge_base(url, pattern):
+    return KnowledgeBase(
+        sitemap_url=url,
+        pattern=pattern,
+        chunk_size=8000,
+        chunk_overlap=3000,
+    )
+
+
+@st.cache_resource
+def get_answer(url, pattern, query):
+    kb = get_knowledge_base(sitemap_url, pattern)
+    return kb.ask(query)
+
+
+if sitemap_url and pattern:
+    with st.spinner("Getting the knowledge base ready, this may take a bit ..."):
+        kb = get_knowledge_base(sitemap_url, pattern)
+
+    query = st.text_input("Question", value="")
+
+    if query:
+        with st.spinner("Getting the answer ..."):
+            result = get_answer(sitemap_url, pattern, query)
+
+        st.markdown("### Answer")
+        st.markdown(result["answer"])
+        st.markdown("### Sources")
+        st.markdown("\n ".join([f"- {x}" for x in result["sources"].split("\n")]))

knowledge_base.py

@@ -0,0 +1,100 @@
+from typing import Optional
+
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.document_loaders import UnstructuredURLLoader
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import RetrievalQAWithSourcesChain
+
+import requests
+import xml.etree.ElementTree as ET
+from dotenv import load_dotenv
+from loguru import logger
+
+load_dotenv()
+
+
+def extract_urls_from_sitemap(sitemap):
+    """
+    Extract all URLs from a sitemap XML string.
+
+    Args:
+        sitemap_string (str): The sitemap XML string.
+
+    Returns:
+        A list of URLs extracted from the sitemap.
+    """
+    # Parse the XML from the string
+    root = ET.fromstring(sitemap)
+
+    # Define the namespace for the sitemap XML
+    namespace = {"ns": "http://www.sitemaps.org/schemas/sitemap/0.9"}
+
+    # Find all <loc> elements under the <url> elements
+    urls = [
+        url.find("ns:loc", namespace).text for url in root.findall("ns:url", namespace)
+    ]
+
+    # Return the list of URLs
+    return urls
+
+
+class KnowledgeBase:
+    def __init__(
+        self,
+        sitemap_url: str,
+        chunk_size: int,
+        chunk_overlap: int,
+        pattern: Optional[str] = None,
+    ):
+        logger.info("Building the knowledge base ...")
+
+        logger.info("Loading sitemap from {sitemap_url} ...", sitemap_url=sitemap_url)
+        sitemap = requests.get(sitemap_url).text
+        urls = extract_urls_from_sitemap(sitemap)
+
+        if pattern:
+            logger.info("Filtering URLs with pattern {pattern} ...", pattern=pattern)
+            urls = [x for x in urls if pattern in x]
+        logger.info("{n} URLs extracted", n=len(urls))
+
+        logger.info("Loading URLs content ...")
+        loader = UnstructuredURLLoader(urls)
+        data = loader.load()
+
+        logger.info("Splitting documents in chunks ...")
+        doc_splitter = CharacterTextSplitter(
+            chunk_size=chunk_size, chunk_overlap=chunk_overlap
+        )
+        docs = doc_splitter.split_documents(data)
+        logger.info("{n} chunks created", n=len(docs))
+
+        logger.info("Building the vector database ...")
+        embeddings = OpenAIEmbeddings()
+        docsearch = Chroma.from_documents(docs, embeddings)
+
+        logger.info("Building the retrieval chain ...")
+        self.chain = RetrievalQAWithSourcesChain.from_chain_type(
+            ChatOpenAI(),
+            chain_type="map_reduce",
+            retriever=docsearch.as_retriever(),
+        )
+
+        logger.info("Knowledge base created!")
+
+    def ask(self, query: str):
+        return self.chain({"question": query}, return_only_outputs=True)
+
+
+if __name__ == "__main__":
+    # Build the knowledge base
+    kb = KnowledgeBase(
+        sitemap_url="https://nextjs.org/sitemap.xml",
+        pattern="docs/api-refe",
+        chunk_size=8000,
+        chunk_overlap=3000,
+    )
+
+    # Ask a question
+    res = kb.ask("How do I deploy my Next.js app?")

pyproject.toml

@@ -0,0 +1,27 @@
+[tool.poetry]
+name = "website-to-knowledge-base"
+version = "0.1.0"
+description = ""
+authors = ["JimZer <[email protected]>"]
+readme = "README.md"
+packages = [{include = "website_to_knowledge_base"}]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+openai = "^0.27.4"
+langchain = "^0.0.144"
+unstructured = "^0.5.13"
+chromadb = "^0.3.21"
+tiktoken = "^0.3.3"
+python-dotenv = "^1.0.0"
+loguru = "^0.7.0"
+streamlit = "^1.21.0"
+
+
+[tool.poetry.group.dev.dependencies]
+black = "^23.3.0"
+isort = "^5.12.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,2 @@
+streamlit
+poetry