import streamlit as st import os import zipfile import pandas as pd from langchain.document_loaders import DataFrameLoader #import tiktoken from langchain.vectorstores import Chroma from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.embeddings.sentence_transformer import SentenceTransformerEmbeddings from langchain_core.messages import HumanMessage, SystemMessage from langchain_openai import ChatOpenAI from transformers import AutoTokenizer, AutoModelForCausalLM from bs4 import BeautifulSoup import requests # Function to load vector database def load_vector_db(zip_file_path, extract_path): with st.spinner("Loading vector store..."): with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: zip_ref.extractall(extract_path) embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") vectordb = Chroma( persist_directory=extract_path, embedding_function=embedding_function ) st.success("Vector store loaded") return vectordb # # Function to augment prompt # def augment_prompt(query, vectordb): # results = vectordb.similarity_search(query, k=10) # source_knowledge = "\n".join([x.page_content for x in results]) # augmented_prompt = f""" # You are an AI assistant. Use the context provided below to answer the question as comprehensively as possible. # If the answer is not contained within the context, respond politely that you cannot provide that information. # Context: # {source_knowledge} # Question: {query} # """ # return augmented_prompt # Function to augment prompt def augment_prompt(query, vectordb, search_results): results = vectordb.similarity_search(query, k=5) source_knowledge = "\n".join([x.page_content for x in results]) augmented_prompt = f""" You are an AI assistant. Use the context provided below to answer the question as comprehensively as possible. If the answer is not contained within the context, respond with "I don't know". Context: {source_knowledge} Additional Web Search Results: {search_results} Question: {query} """ return augmented_prompt # Function to handle chat with OpenAI def chat_with_openai(query, vectordb, openai_api_key, search_results): chat = ChatOpenAI(model_name="gpt-3.5-turbo", openai_api_key=openai_api_key, timeout=30) # Increased timeout augmented_query = augment_prompt(query, vectordb, search_results) prompt = HumanMessage(content=augmented_query) messages = [ SystemMessage(content="You are a helpful assistant."), prompt ] res = chat(messages) return res.content # Function to perform web search def perform_web_search(query): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"} search_results = "" # Glassdoor search glassdoor_url = f"https://www.glassdoor.com/Search/results.htm?keyword={query}" response = requests.get(glassdoor_url, headers=headers) if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') glassdoor_results = soup.find_all('div', {'class': 'jobContainer'}) for result in glassdoor_results[:5]: # limiting to first 3 results title = result.find('a', {'class': 'jobInfoItem jobTitle'}).text.strip() if result.find('a', {'class': 'jobInfoItem jobTitle'}) else 'N/A' company = result.find('div', {'class': 'jobInfoItem jobEmpolyerName'}).text.strip() if result.find('div', {'class': 'jobInfoItem jobEmpolyerName'}) else 'N/A' location = result.find('span', {'class': 'subtle loc'}).text.strip() if result.find('span', {'class': 'subtle loc'}) else 'N/A' search_results += f"Glassdoor Result: {title} at {company}, {location}\n" # Indeed search indeed_url = f"https://www.indeed.com/jobs?q={query}&limit=10" response = requests.get(indeed_url, headers=headers) if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') indeed_results = soup.find_all('div', {'class': 'jobsearch-SerpJobCard'}) for result in indeed_results[:5]: # limiting to first 3 results title = result.find('h2', {'class': 'title'}).text.strip() if result.find('h2', {'class': 'title'}) else 'N/A' company = result.find('span', {'class': 'company'}).text.strip() if result.find('span', {'class': 'company'}) else 'N/A' location = result.find('span', {'class': 'location'}).text.strip() if result.find('span', {'class': 'location'}) else 'N/A' search_results += f"Indeed Result: {title} at {company}, {location}\n" return search_results # Streamlit UI st.title("Data Roles Company Finder Chatbot") st.write("This app helps users find companies hiring for data roles, providing information such as job title, salary estimate, job description, company rating, and more.") # Load vector database zip_file_path = "chroma_db_compressed_.zip" extract_path = "./chroma_db_extracted" vectordb = load_vector_db(zip_file_path, extract_path) # Initialize session state for chat history if "messages" not in st.session_state: st.session_state.messages = [] # Display chat history for message in st.session_state.messages: with st.chat_message(message["role"]): st.markdown(message["content"]) # User input if prompt := st.chat_input("Enter your query"): st.session_state.messages.append({"role": "user", "content": prompt}) with st.chat_message("user"): st.markdown(prompt) # Perform web search search_results = perform_web_search(prompt) # Chat with OpenAI openai_api_key = st.secrets["OPENAI_API_KEY"] response = chat_with_openai(prompt, vectordb, openai_api_key, search_results) # Display assistant response with st.chat_message("assistant"): st.markdown(response) st.session_state.messages.append({"role": "assistant", "content": response}) # # Function to handle chat with OpenAI # def chat_with_openai(query, vectordb, openai_api_key): # chat = ChatOpenAI(model_name="gpt-3.5-turbo", openai_api_key=openai_api_key) # augmented_query = augment_prompt(query, vectordb) # prompt = HumanMessage(content=augmented_query) # messages = [ # SystemMessage(content="You are a helpful assistant."), # prompt # ] # res = chat(messages) # return res.content # # Streamlit UI # st.title("Data Roles Company Finder Chatbot") # st.write("This app helps users find companies hiring for data roles, providing information such as job title, salary estimate, job description, company rating, and more.") # # Load vector database # zip_file_path = "chroma_db_compressed_.zip" # extract_path = "./chroma_db_extracted" # vectordb = load_vector_db(zip_file_path, extract_path) # # Initialize session state for chat history # if "messages" not in st.session_state: # st.session_state.messages = [] # # Display chat history # for message in st.session_state.messages: # with st.chat_message(message["role"]): # st.markdown(message["content"]) # # User input # if prompt := st.chat_input("Enter your query"): # st.session_state.messages.append({"role": "user", "content": prompt}) # with st.chat_message("user"): # st.markdown(prompt) # with st.chat_message("assistant"): # openai_api_key = st.secrets["OPENAI_API_KEY"] # response = chat_with_openai(prompt, vectordb, openai_api_key) # st.markdown(response) # st.session_state.messages.append({"role": "assistant", "content": response}) # # Query input # query = st.text_input("Enter your query", "") # if st.button("Send"): # if query: # # Add user query to chat history # st.session_state.messages.append({"role": "user", "content": query}) # with st.chat_message("user"): # st.markdown(query) # # Chat with OpenAI # openai_api_key = st.secrets["OPENAI_API_KEY"] # response = chat_with_openai(query, vectordb, openai_api_key) # # Add AI response to chat history # st.session_state.messages.append({"role": "assistant", "content": response}) # with st.chat_message("assistant"): # st.markdown(response) # # Streamlit UI # st.title("Document Processing and AI Chat with LangChain") # # Load vector database # zip_file_path = "chroma_db_compressed_.zip" # extract_path = "./chroma_db_extracted" # vectordb = load_vector_db(zip_file_path, extract_path) # # Query input # query = st.text_input("Enter your query", "List three companies where I can work as a business analyst with their location and salary") # if st.button("Get Answer"): # # Chat with OpenAI # openai_api_key = st.secrets["OPENAI_API_KEY"] # response = chat_with_openai(query, vectordb, openai_api_key) # st.write("Response from AI:") # st.write(response) # # Streamlit UI # st.title("Data Roles Company Finder Chatbot") # st.write("This app helps users find companies hiring for data roles, providing information such as job title, salary estimate, job description, company rating, and more.") # # Load vector database # zip_file_path = "chroma_db_compressed_.zip" # extract_path = "./chroma_db_extracted" # vectordb = load_vector_db(zip_file_path, extract_path) # # Initialize session state for chat history # if "messages" not in st.session_state: # st.session_state.messages = [ # SystemMessage(content="You are a helpful assistant.") # ] # # Display chat history # for message in st.session_state.messages: # if isinstance(message, HumanMessage): # st.write(f"You: {message.content}") # else: # st.write(f"AI: {message.content}") # # Query input # query = st.text_input("Enter your query", "List three companies where I can work as a business analyst with their location and salary") # if st.button("Send"): # if query: # # Add user query to chat history # st.session_state.messages.append(HumanMessage(content=query)) # # Chat with OpenAI # openai_api_key = st.secrets["OPENAI_API_KEY"] # response = chat_with_openai(query, vectordb, openai_api_key) # # Add AI response to chat history # st.session_state.messages.append(SystemMessage(content=response)) # # Display chat history # for message in st.session_state.messages: # if isinstance(message, HumanMessage): # st.write(f"You: {message.content}") # else: # st.write(f"AI: {message.content}")