main files

.env

@@ -0,0 +1,4 @@
+OPENAI_API_KEY=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
+VECTOR_DB_PATH_DB=D:\\rajesh\\python\\doge_hackathon\\vectordb\\openai_dbstore\\db
+LOG_PATH=D:\\rajesh\\python\\doge_hackathon\\logs\\
+GROK_API_KEY=xai-mrBds1WpANksRr9CA8k57BGUiWGF8spR0STmgKo9iWTAvmW62K0WulQ1CUKiP1sRMhOg0a6IVr7aOB8t

app.py

@@ -0,0 +1,130 @@
+import streamlit as st
+import os
+from dotenv import load_dotenv
+from file_handler import FileHandler
+from chat_handler import ChatHandler
+
+# Load environment variables
+load_dotenv()
+
+# Static credentials
+USERNAME = st.secrets["USERNAME"]
+PASSWORD = st.secrets["PASSWORD"]
+
+# Initialize Handlers
+VECTOR_DB_PATH = st.secrets["VECTOR_DB_PATH_DB"]
+OPENAI_API_KEY = st.secrets["OPENAI_API_KEY"]
+HUGGINGFACE_API_TOKEN = st.secrets["HUGGINGFACE_API_TOKEN"]
+GROQ_API_KEY_TOKEN = st.secrets["GROQ_API_KEY"]
+
+os.makedirs(VECTOR_DB_PATH, exist_ok=True)
+
+file_handler = FileHandler(VECTOR_DB_PATH, HUGGINGFACE_API_TOKEN)
+chat_handler = ChatHandler(VECTOR_DB_PATH, HUGGINGFACE_API_TOKEN, OPENAI_API_KEY,GROQ_API_KEY_TOKEN)
+
+# Streamlit UI
+st.set_page_config(layout="wide", page_title="AI Connect - Smarter Network Planning for the Future")
+
+# Session state to track login status
+if "logged_in" not in st.session_state:
+    st.session_state["logged_in"] = False
+
+# Login page
+# Refined Login Page
+if not st.session_state["logged_in"]:
+    # Customize page title
+    st.markdown(
+        """
+        <style>
+        .title {
+            font-size: 2.5rem;
+            color: #1f77b4;
+            font-weight: bold;
+            text-align: center;
+            margin-bottom: 10px;
+        }
+        .subtitle {
+            font-size: 1.2rem;
+            color: #555;
+            text-align: center;
+            margin-bottom: 20px;
+        }
+        .login-box {
+            margin: auto;
+            width: 50%;
+            padding: 20px;
+            background: #f9f9f9;
+            border: 1px solid #ddd;
+            border-radius: 10px;
+        }
+        .login-box input {
+            margin-bottom: 10px;
+        }
+        </style>
+        <div>
+            <div class="title">Welcome to AI Connect</div>
+            <div class="subtitle">Smarter Network Planning for the Future</div>
+        </div>
+        """,
+        unsafe_allow_html=True,
+    )
+
+    # Centered Login Box
+    # st.markdown('<div class="login-box">', unsafe_allow_html=True)
+    st.subheader("Login to Continue")
+    username = st.text_input("Username")
+    password = st.text_input("Password", type="password")
+    if st.button("Login"):
+        if username == USERNAME and password == PASSWORD:
+            st.session_state["logged_in"] = True
+            st.success("Login successful!")
+            st.rerun()
+        else:
+            st.error("Invalid username or password.")
+    st.markdown("</div>", unsafe_allow_html=True)
+else:
+    # Main app (Chat Interface)
+    st.title("Chatbot - Smarter Network Planning for the Future")
+    st.sidebar.header("Upload Documents")
+    uploaded_file = st.sidebar.file_uploader("Upload PDF, Excel, Docx, or Txt", type=["pdf", "xlsx", "docx", "txt", "csv"])
+    document_name = st.sidebar.text_input("Document Name", "")
+    document_description = st.sidebar.text_area("Document Description", "")
+
+    if st.sidebar.button("Process File"):
+        if uploaded_file:
+            with st.spinner("Processing your file..."):
+                response = file_handler.handle_file_upload(
+                    file=uploaded_file,
+                    document_name=document_name,
+                    document_description=document_description,
+                )
+                st.sidebar.success(f"File processed: {response['message']}")
+        else:
+            st.sidebar.warning("Please upload a file before processing.")
+
+    # Chat Interface
+    if "messages" not in st.session_state:
+        st.session_state["messages"] = []
+
+    # Display chat messages from history
+    for message in st.session_state["messages"]:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+    # Accept user input
+    if prompt := st.chat_input("Type your question here..."):
+        with st.chat_message("user"):
+            st.markdown(prompt)
+        st.session_state["messages"].append({"role": "user", "content": prompt})
+
+        with st.spinner("Processing your question..."):
+            response = chat_handler.answer_question(prompt)
+        with st.chat_message("assistant"):
+            st.markdown(response)
+        st.session_state["messages"].append({"role": "assistant", "content": response})
+
+# Logout button
+if st.session_state["logged_in"]:
+    if st.sidebar.button("Logout"):
+        st.session_state["logged_in"] = False
+        st.rerun()

chat_handler.py

@@ -0,0 +1,121 @@
+import os
+from langchain_community.vectorstores import FAISS
+from langchain_openai import ChatOpenAI
+from langchain_huggingface import HuggingFaceEmbeddings
+from groq import Groq
+import requests
+
+
+class ChatHandler:
+    def __init__(self, vector_db_path,api_token,open_api_key,grok_api_token):
+        self.vector_db_path = vector_db_path
+        self.groq_client = Groq(api_key=grok_api_token)
+        # Initialize the embedding model using Hugging Face
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name="sentence-transformers/all-MiniLM-L6-v2",
+            model_kwargs={"token": api_token},
+        )
+        self.llm = ChatOpenAI(
+            model_name="gpt-4",
+            api_key=open_api_key,
+            max_tokens=500,
+            temperature=0.2,
+        )
+    def _query_groq_model(self, prompt):
+        """
+        Query Groq's Llama model using the SDK.
+        """
+        try:
+            chat_completion = self.groq_client.chat.completions.create(
+                messages=[{"role": "user", "content": prompt}],
+                model="llama-3.1-8b-instant",  # Ensure the model name is correct
+            )
+            # Return the assistant's response
+            return chat_completion.choices[0].message.content
+        except Exception as e:
+            return f"Error querying Groq API: {e}"
+
+    def answer_question(self, question):
+        # Generate embedding for the question
+        responses = []
+        for root, dirs, files in os.walk(self.vector_db_path):
+            for dir in dirs:
+                index_path = os.path.join(root, dir, "index.faiss")
+                if os.path.exists(index_path):
+                    vector_store = FAISS.load_local(
+                        os.path.join(root, dir), self.embeddings, allow_dangerous_deserialization=True
+                    )
+                    response_with_scores = vector_store.similarity_search_with_relevance_scores(question, k=100)
+                    filtered_responses = [doc.page_content for doc, score in response_with_scores]
+                    responses.extend(filtered_responses)
+
+        if responses:
+            prompt = self._generate_prompt(question, responses)
+            # response = self.llm.invoke(prompt)
+            # if hasattr(response, "content"):
+            #     return response.content.strip()  # Ensure clean output
+            # else:
+            #     return "Error: 'content' attribute not found in the AI's response."
+            response = self._query_groq_model(prompt)
+            return response
+
+
+        return "No relevant documents found or context is insufficient to answer your question."
+
+    def _generate_prompt(self, question, documents):
+        """
+        Generate a structured prompt tailored to analyze government energy consumption data
+        and answer questions effectively using the provided documents.
+        """
+        context = "\n".join(
+            [f"Document {i + 1}:\n{doc.strip()}" for i, doc in enumerate(documents[:5])]
+        )
+
+        prompt = f"""
+            You are an advanced AI assistant with expertise in 5G network optimization, deployment strategies, 
+            and resource allocation. Your role is to analyze network datasets to identify inefficiencies, 
+            propose actionable deployment and optimization strategies, and quantify potential improvements.
+
+            ### Data Provided:
+            The following documents contain detailed information about 5G network deployment, resource utilization, 
+            and operational metrics:
+            {context}
+
+            ### Question:
+            {question}
+
+            ### Instructions:
+            1. **Highlight Areas of Network Inefficiencies**:
+               - Identify inefficiencies such as underutilized network nodes, high latency areas, or 
+                 imbalanced resource allocation.
+               - Use data points from the documents to back your observations.
+
+            2. **Suggest Strategies for Network Optimization**:
+               - Recommend actionable steps such as adjusting network configurations, deploying additional nodes, 
+                 or reallocating bandwidth.
+               - Ensure suggestions are feasible and aligned with the provided datasets.
+
+            3. **Quantify Cost-Saving and Performance Benefits**:
+               - Provide quantitative estimates of potential cost savings from the suggested strategies.
+               - Highlight the performance benefits, such as improved latency, higher throughput, or enhanced user experience.
+
+            4. **Present the Response Clearly**:
+               - Organize your findings in a step-by-step format.
+               - Use tables, bullet points, or concise paragraphs for clarity.
+
+            ### Example Output Format:
+            - **Network Inefficiencies Identified**:
+              1. ...
+              2. ...
+
+            - **Optimization Strategies**:
+              1. ...
+              2. ...
+
+            - **Cost-Saving and Performance Benefits**:
+              - Cost Savings: $...
+              - Performance Improvements: ...
+
+            Please ensure the response is data-driven, actionable, and easy to understand.
+        """
+        return prompt

file_handler.py

@@ -0,0 +1,123 @@
+import os
+import hashlib
+import io
+import json
+import pandas as pd
+from langchain_community.vectorstores import FAISS
+from PyPDF2 import PdfReader
+from docx import Document
+from langchain_huggingface import HuggingFaceEmbeddings
+
+class FileHandler:
+    def __init__(self, vector_db_path,api_token):
+        self.vector_db_path = vector_db_path
+        # Initialize the embedding model using Hugging Face
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name="sentence-transformers/all-MiniLM-L6-v2",
+            model_kwargs={"token": api_token},
+        )
+
+    def handle_file_upload(self, file, document_name, document_description):
+        try:
+            content = file.read()
+            file_hash = hashlib.md5(content).hexdigest()
+            file_key = f"{file.name}_{file_hash}"
+            vector_store_dir = os.path.join(self.vector_db_path, file_key)
+            os.makedirs(vector_store_dir, exist_ok=True)
+            vector_store_path = os.path.join(vector_store_dir, "index.faiss")
+
+            if os.path.exists(vector_store_path):
+                return {"message": "File already processed."}
+
+            # Process file based on type
+            if file.name.endswith(".pdf"):
+                texts, metadatas = self.load_and_split_pdf(file)
+            elif file.name.endswith(".docx"):
+                texts, metadatas = self.load_and_split_docx(file)
+            elif file.name.endswith(".txt"):
+                texts, metadatas = self.load_and_split_txt(content)
+            elif file.name.endswith(".xlsx"):
+                texts, metadatas = self.load_and_split_table(content)
+            elif file.name.endswith(".csv"):
+                texts, metadatas = self.load_and_split_csv(content)
+            else:
+                raise ValueError("Unsupported file format.")
+
+            if not texts:
+                return {"message": "No text extracted from the file. Check the file content."}
+
+            # Create FAISS vector store using LangChain's from_texts method
+            vector_store = FAISS.from_texts(texts, embedding=self.embeddings, metadatas=metadatas)
+            vector_store.save_local(vector_store_dir)
+
+            metadata = {
+                "filename": file.name,
+                "document_name": document_name,
+                "document_description": document_description,
+                "file_size": len(content),
+            }
+            metadata_path = os.path.join(vector_store_dir, "metadata.json")
+            with open(metadata_path, 'w') as md_file:
+                json.dump(metadata, md_file)
+
+            return {"message": "File processed successfully."}
+        except Exception as e:
+            return {"message": f"Error processing file: {str(e)}"}
+
+
+    def load_and_split_pdf(self, file):
+        reader = PdfReader(file)
+        texts = []
+        metadatas = []
+        for page_num, page in enumerate(reader.pages):
+            text = page.extract_text()
+            if text:
+                texts.append(text)
+                metadatas.append({"page_number": page_num + 1})
+        return texts, metadatas
+
+    def load_and_split_docx(self, file):
+        doc = Document(file)
+        texts = []
+        metadatas = []
+        for para_num, paragraph in enumerate(doc.paragraphs):
+            if paragraph.text:
+                texts.append(paragraph.text)
+                metadatas.append({"paragraph_number": para_num + 1})
+        return texts, metadatas
+
+    def load_and_split_txt(self, content):
+        text = content.decode("utf-8")
+        lines = text.split('\n')
+        texts = [line for line in lines if line.strip()]
+        metadatas = [{}] * len(texts)
+        return texts, metadatas
+
+    def load_and_split_table(self, content):
+        excel_data = pd.read_excel(io.BytesIO(content), sheet_name=None)
+        texts = []
+        metadatas = []
+        for sheet_name, df in excel_data.items():
+            df = df.dropna(how='all', axis=0).dropna(how='all', axis=1)
+            df = df.fillna('N/A')
+            for _, row in df.iterrows():
+                row_dict = row.to_dict()
+                # Combine key-value pairs into a string
+                row_text = ', '.join([f"{key}: {value}" for key, value in row_dict.items()])
+                texts.append(row_text)
+                metadatas.append({"sheet_name": sheet_name})
+        return texts, metadatas
+
+    def load_and_split_csv(self, content):
+        csv_data = pd.read_csv(io.StringIO(content.decode('utf-8')))
+        texts = []
+        metadatas = []
+        csv_data = csv_data.dropna(how='all', axis=0).dropna(how='all', axis=1)
+        csv_data = csv_data.fillna('N/A')
+        for _, row in csv_data.iterrows():
+            row_dict = row.to_dict()
+            row_text = ', '.join([f"{key}: {value}" for key, value in row_dict.items()])
+            texts.append(row_text)
+            metadatas.append({"row_index": _})
+        return texts, metadatas
+