tonic
commited on
Commit
·
2ad5eb7
1
Parent(s):
6448a30
Update app.py
Browse files- backend/app.py +55 -57
backend/app.py
CHANGED
@@ -1,9 +1,8 @@
|
|
1 |
import weaviate
|
2 |
import langchain
|
|
|
3 |
import gradio as gr
|
4 |
from langchain.embeddings import CohereEmbeddings
|
5 |
-
from langchain.memory import ConversationBufferMemory
|
6 |
-
from langchain.prompts.prompt import PromptTemplate
|
7 |
from langchain.document_loaders import UnstructuredFileLoader
|
8 |
from langchain.vectorstores import Weaviate
|
9 |
from langchain.llms import OpenAI
|
@@ -14,6 +13,8 @@ import ssl
|
|
14 |
import mimetypes
|
15 |
from dotenv import load_dotenv
|
16 |
import cohere
|
|
|
|
|
17 |
|
18 |
# Load environment variables
|
19 |
load_dotenv()
|
@@ -21,56 +22,61 @@ openai_api_key = os.getenv('OPENAI')
|
|
21 |
cohere_api_key = os.getenv('COHERE')
|
22 |
weaviate_api_key = os.getenv('WEAVIATE')
|
23 |
weaviate_url = os.getenv('WEAVIATE_URL')
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
"""
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
#
|
68 |
-
|
69 |
-
|
70 |
-
|
|
|
|
|
|
|
|
|
71 |
|
72 |
# Initialize vectorstore
|
73 |
vectorstore = Weaviate(client, index_name="HereChat", text_key="text")
|
|
|
74 |
vectorstore._query_attrs = ["text", "title", "url", "views", "lang", "_additional {distance}"]
|
75 |
vectorstore.embedding = CohereEmbeddings(model="embed-multilingual-v2.0", cohere_api_key=cohere_api_key)
|
76 |
|
@@ -102,15 +108,7 @@ def embed_pdf(file, collection_name):
|
|
102 |
os.remove(file_path)
|
103 |
return {"message": f"Documents embedded in Weaviate collection '{collection_name}'"}
|
104 |
|
105 |
-
def update_chat_history(user_message, ai_message):
|
106 |
-
chat_history.add_user_message(user_message)
|
107 |
-
chat_history.add_ai_message(ai_message)
|
108 |
-
# Update memory if needed
|
109 |
-
if len(chat_history) > memory.max_token:
|
110 |
-
memory.create_summary()
|
111 |
-
|
112 |
def retrieve_info(query):
|
113 |
-
update_chat_history(query, "")
|
114 |
llm = OpenAI(temperature=0, openai_api_key=openai_api_key)
|
115 |
qa = RetrievalQA.from_chain_type(llm, retriever=vectorstore.as_retriever())
|
116 |
|
|
|
1 |
import weaviate
|
2 |
import langchain
|
3 |
+
import apscheduler
|
4 |
import gradio as gr
|
5 |
from langchain.embeddings import CohereEmbeddings
|
|
|
|
|
6 |
from langchain.document_loaders import UnstructuredFileLoader
|
7 |
from langchain.vectorstores import Weaviate
|
8 |
from langchain.llms import OpenAI
|
|
|
13 |
import mimetypes
|
14 |
from dotenv import load_dotenv
|
15 |
import cohere
|
16 |
+
from apscheduler.schedulers.background import BackgroundScheduler
|
17 |
+
import time
|
18 |
|
19 |
# Load environment variables
|
20 |
load_dotenv()
|
|
|
22 |
cohere_api_key = os.getenv('COHERE')
|
23 |
weaviate_api_key = os.getenv('WEAVIATE')
|
24 |
weaviate_url = os.getenv('WEAVIATE_URL')
|
25 |
+
weaviate_username = os.getenv('WEAVIATE_USERNAME')
|
26 |
+
weaviate_password = os.getenv('WEAVIATE_PASSWORD')
|
27 |
+
|
28 |
+
|
29 |
+
# Function to refresh authentication
|
30 |
+
def refresh_authentication():
|
31 |
+
global my_credentials, client
|
32 |
+
my_credentials = weaviate.auth.AuthClientPassword(username=weaviate_username, password=weaviate_password)
|
33 |
+
client = weaviate.Client(weaviate_url, auth_client_secret=my_credentials)
|
34 |
+
|
35 |
+
# Initialize the scheduler for authentication refresh
|
36 |
+
scheduler = BackgroundScheduler()
|
37 |
+
scheduler.add_job(refresh_authentication, 'interval', minutes=30)
|
38 |
+
scheduler.start()
|
39 |
+
|
40 |
+
# Initial authentication
|
41 |
+
refresh_authentication()
|
42 |
+
|
43 |
+
Article = {
|
44 |
+
"class": "Article",
|
45 |
+
"description": "A class representing articles in the application",
|
46 |
+
"properties": [
|
47 |
+
{
|
48 |
+
"name": "title",
|
49 |
+
"description": "The title of the article",
|
50 |
+
"dataType": ["text"]
|
51 |
+
},
|
52 |
+
{
|
53 |
+
"name": "content",
|
54 |
+
"description": "The content of the article",
|
55 |
+
"dataType": ["text"]
|
56 |
+
},
|
57 |
+
{
|
58 |
+
"name": "author",
|
59 |
+
"description": "The author of the article",
|
60 |
+
"dataType": ["text"]
|
61 |
+
},
|
62 |
+
{
|
63 |
+
"name": "publishDate",
|
64 |
+
"description": "The date the article was published",
|
65 |
+
"dataType": ["date"]
|
66 |
+
}
|
67 |
+
],
|
68 |
+
# "vectorIndexType": "hnsw",
|
69 |
+
# "vectorizer": "text2vec-contextionary"
|
70 |
+
}
|
71 |
+
|
72 |
+
|
73 |
+
schema = {
|
74 |
+
"classes": [Article]
|
75 |
+
}
|
76 |
|
77 |
# Initialize vectorstore
|
78 |
vectorstore = Weaviate(client, index_name="HereChat", text_key="text")
|
79 |
+
client.schema.create(schema)
|
80 |
vectorstore._query_attrs = ["text", "title", "url", "views", "lang", "_additional {distance}"]
|
81 |
vectorstore.embedding = CohereEmbeddings(model="embed-multilingual-v2.0", cohere_api_key=cohere_api_key)
|
82 |
|
|
|
108 |
os.remove(file_path)
|
109 |
return {"message": f"Documents embedded in Weaviate collection '{collection_name}'"}
|
110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
111 |
def retrieve_info(query):
|
|
|
112 |
llm = OpenAI(temperature=0, openai_api_key=openai_api_key)
|
113 |
qa = RetrievalQA.from_chain_type(llm, retriever=vectorstore.as_retriever())
|
114 |
|