Update app.py

app.py

@@ -10,6 +10,9 @@ from datasets import load_dataset
 # from transformers import AutoModelForCausalLM, AutoTokenizer
 import gradio as gr
 from mistral_inference.model import Transformer
+from gpt4all import GPT4All
+from pathlib import Path
+
 
 # Function to clear the cache
 def clear_cache(model_name):
@@ -84,9 +87,10 @@ vector_store.populate_vectors(dataset=None)
 # load model orca-mini general purpose model
 # tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3")
 # model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3")
-
-tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tokenizer.model.v3")
-model = Transformer.from_folder(mistral_models_path)
+model_name = 'Meta-Llama-3-8B-Instruct.Q4_0.gguf' # .gguf represents quantized model
+model_path = "gpt4all"
+# add path to download load the model locally, download once and load for subsequent inference
+model = GPT4All(model_name=model_name, model_path=model_path,device="cuda")
 
 # Define the chatbot response function
 def chatbot_response(user_input):