first deployment

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+final_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,126 @@
+from langchain.chains import LLMChain
+import streamlit as st
+from decouple import config
+from langchain.llms import OpenAI
+from langchain.document_loaders import PyPDFLoader
+from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.retrievers.document_compressors import LLMChainExtractor
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.chains import RetrievalQA
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.evaluation.qa import QAGenerateChain
+from langchain.chains import RetrievalQA
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import CSVLoader
+from langchain.indexes import VectorstoreIndexCreator
+from langchain.vectorstores import DocArrayInMemorySearch
+from langchain.prompts import ChatPromptTemplate
+from langchain.document_loaders.generic import GenericLoader
+from langchain.document_loaders.parsers import OpenAIWhisperParser
+from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
+from langchain.prompts import PromptTemplate
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+import time
+from htmlTemplates import css, bot_template, user_template
+
+from dotenv import load_dotenv, find_dotenv
+_ = load_dotenv(find_dotenv())  # read local .env file
+
+def timeit(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()  # Start time
+        result = func(*args, **kwargs)  # Function execution
+        end_time = time.time()  # End time
+        print(f"Function {func.__name__} took {end_time - start_time} seconds to execute.")
+        return result
+    return wrapper
+
+
+@timeit
+def get_llm():
+    return OpenAI(temperature=0.0)
+
+@timeit
+def get_memory():
+    return ConversationBufferMemory(
+        memory_key="chat_history",
+        return_messages=True
+    )
+
+@timeit
+def generate_response(question, vectordb, llm, memory,chat_history):
+
+    prompt = ChatPromptTemplate.from_template(
+        "You are a petroleum engineer specialist in hydralic fracture stimulation \
+    , please answer the question that surounded between the triple backtick \
+    ```{question}```"
+    )
+    
+    question_template = prompt.format_messages(question=question)
+    final_qa = question_template[0].content
+
+    qa = ConversationalRetrievalChain.from_llm(
+        llm=llm,
+        retriever=vectordb.as_retriever(search_type="mmr",k=5, fetch_k=10),
+        memory=memory,
+    )
+
+    handle_userinput((qa({"question": question, "chat_history": chat_history})))
+
+@timeit
+def create_embeding_function():
+    embedding_func_all_mpnet_base_v2 = SentenceTransformerEmbeddings(
+        model_name="all-mpnet-base-v2")
+    # embedding_func_all_MiniLM_L6_v2 = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+
+    return embedding_func_all_mpnet_base_v2
+
+@timeit
+def get_vector_db(embedding_function):
+    vector_db = Chroma(persist_directory="./final_db",
+                       embedding_function=embedding_function)
+    return vector_db
+
+def handle_userinput(user_question):
+    response = user_question
+    if chat_history not in st.session_state:
+        st.session_state.chat_history = []
+
+    st.session_state.chat_history = response['chat_history']
+
+    for i, message in enumerate(st.session_state.chat_history):
+        if i % 2 == 0:
+            st.write(user_template.replace(
+                "{{MSG}}", message.content), unsafe_allow_html=True)
+        else:
+            st.write(bot_template.replace(
+                "{{MSG}}", message.content), unsafe_allow_html=True)
+if __name__ == "__main__":
+
+    st.set_page_config(page_title = "Hydraulic Fracture Stimulation Chat",page_icon=":books:")
+    st.write(css, unsafe_allow_html=True)
+    st.title("Hydraulic Fracture Stimulation Chat")
+    st.write(
+        "This is a chatbot that can answer questions related to petroleum engineering specially in hydraulic fracture stimulation.")
+
+    # get embeding function   
+    embeding_function = create_embeding_function()
+    # get vector db
+    vector_db = get_vector_db(embeding_function)
+    # get llm
+    llm = get_llm()
+
+    # get memory
+    if 'memory' not in st.session_state:
+        st.session_state['memory'] = get_memory()
+    memory = st.session_state['memory']
+
+    # chat history
+    chat_history = []
+   
+    prompt_question = st.chat_input("Please ask a question:")
+    if prompt_question:
+        generate_response(question= prompt_question, vectordb=vector_db, llm=llm, memory=memory,chat_history=chat_history)

final_db/b14a837e-d1d1-45d9-9434-afbedcec46ba/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0b2dfa6fc6f8ed74e81120e268d444fa3033806c66a0440f1678c408b97218
+size 6424000

final_db/b14a837e-d1d1-45d9-9434-afbedcec46ba/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e972f51eb6bdd4796b659eb05da7af265a8628f46bc86a7992f0972445ab6a36
+size 100

final_db/b14a837e-d1d1-45d9-9434-afbedcec46ba/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f3544f58970ec3ab604362079f921d6b7e3bfeec39feb0368b5ca87c7a82993
+size 113989

final_db/b14a837e-d1d1-45d9-9434-afbedcec46ba/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf238bb4bf1c2a6b3669ca83fe1bf29bee19b7f8d1e743c7b4cb3e3cd196b093
+size 8000

final_db/b14a837e-d1d1-45d9-9434-afbedcec46ba/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61aaef00ec60f38a420980a9243be7a4579d4a3784cd4f0b31dbddc874d5355e
+size 16976

final_db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba67f4d0d95baa576e5fa3ef86c1cb9d5e8d56386b366020b91c3a07d65bb60e
+size 23465984

htmlTemplates.py

@@ -0,0 +1,44 @@
+css = '''
+<style>
+.chat-message {
+    padding: 1.5rem; border-radius: 0.5rem; margin-bottom: 1rem; display: flex
+}
+.chat-message.user {
+    background-color: #2b313e
+}
+.chat-message.bot {
+    background-color: #475063
+}
+.chat-message .avatar {
+  width: 20%;
+}
+.chat-message .avatar img {
+  max-width: 78px;
+  max-height: 78px;
+  border-radius: 50%;
+  object-fit: cover;
+}
+.chat-message .message {
+  width: 80%;
+  padding: 0 1.5rem;
+  color: #fff;
+}
+'''
+
+bot_template = '''
+<div class="chat-message bot">
+    <div class="avatar">
+        <img src="https://i.ibb.co/cN0nmSj/Screenshot-2023-05-28-at-02-37-21.png" style="max-height: 78px; max-width: 78px; border-radius: 50%; object-fit: cover;">
+    </div>
+    <div class="message">{{MSG}}</div>
+</div>
+'''
+
+user_template = '''
+<div class="chat-message user">
+    <div class="avatar">
+        <img src="https://th.bing.com/th/id/OIP.xXHQ5dk4qJH74WMGNezDjwHaHa?rs=1&pid=ImgDetMain">
+    </div>    
+    <div class="message">{{MSG}}</div>
+</div>
+'''

requirements.txt

@@ -0,0 +1,171 @@
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.1.0
+backoff==2.2.1
+bcrypt==4.0.1
+blinker==1.7.0
+Brotli==1.1.0
+cachetools==5.3.2
+certifi==2023.7.22
+cffi==1.16.0
+charset-normalizer==3.3.2
+Chroma==0.2.0
+chroma-hnswlib==0.7.3
+chromadb==0.4.17
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.2.0
+cryptography==41.0.5
+ctransformers==0.2.27
+dataclasses-json==0.6.2
+debugpy==1.8.0
+decorator==5.1.1
+Deprecated==1.2.14
+distro==1.8.0
+exceptiongroup==1.1.3
+executing==2.0.1
+fastapi==0.104.1
+filelock==3.13.1
+flatbuffers==23.5.26
+frozenlist==1.4.0
+fsspec==2023.10.0
+gitdb==4.0.11
+GitPython==3.1.40
+google-auth==2.23.4
+googleapis-common-protos==1.61.0
+greenlet==3.0.1
+grpcio==1.59.2
+h11==0.14.0
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.25.1
+huggingface-hub==0.19.3
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.1.1
+ipykernel==6.26.0
+ipython==8.17.2
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.2
+jsonschema-specifications==2023.11.1
+jupyter_client==8.6.0
+jupyter_core==5.5.0
+kubernetes==28.1.0
+langchain==0.0.336
+langsmith==0.0.64
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mutagen==1.47.0
+mypy-extensions==1.0.0
+nest-asyncio==1.5.8
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.2
+oauthlib==3.2.2
+onnxruntime==1.16.2
+openai==1.3.0
+opentelemetry-api==1.21.0
+opentelemetry-exporter-otlp-proto-common==1.21.0
+opentelemetry-exporter-otlp-proto-grpc==1.21.0
+opentelemetry-proto==1.21.0
+opentelemetry-sdk==1.21.0
+opentelemetry-semantic-conventions==0.42b0
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.3
+parso==0.8.3
+Pillow==10.1.0
+platformdirs==4.0.0
+posthog==3.0.2
+prompt-toolkit==3.0.41
+protobuf==4.25.1
+psutil==5.9.6
+pulsar-client==3.3.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==14.0.1
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycparser==2.21
+pycryptodomex==3.19.0
+pydantic==2.5.1
+pydantic_core==2.14.3
+pydeck==0.8.1b0
+pydub==0.25.1
+Pygments==2.16.1
+pypdf==3.17.1
+PyPika==0.48.9
+pyreadline3==3.4.1
+python-dateutil==2.8.2
+python-decouple==3.8
+python-dotenv==1.0.0
+pytz==2023.3.post1
+pywin32==306
+PyYAML==6.0.1
+pyzmq==25.1.1
+referencing==0.31.0
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.7.0
+rpds-py==0.12.0
+rsa==4.9
+safetensors==0.4.0
+scikit-learn==1.3.2
+scipy==1.11.3
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+SQLAlchemy==2.0.23
+stack-data==0.6.3
+starlette==0.27.0
+streamlit==1.28.2
+sympy==1.12
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tiktoken==0.5.1
+tokenizers==0.15.0
+toml==0.10.2
+toolz==0.12.0
+torch==2.1.1
+torchvision==0.16.1
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.13.0
+transformers==4.35.2
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tzdata==2023.3
+tzlocal==5.2
+urllib3==1.26.18
+uvicorn==0.24.0.post1
+validators==0.22.0
+watchdog==3.0.0
+watchfiles==0.21.0
+wcwidth==0.2.10
+websocket-client==1.6.4
+websockets==12.0
+wrapt==1.16.0
+yarl==1.9.2
+yt-dlp==2023.11.16
+zipp==3.17.0