Use quantized model

app.py

@@ -7,7 +7,7 @@ import torch
 from conversation import get_default_conv_template
 import gradio as gr
 from pyllamacpp.model import Model
-from urllib import request, response, urlopen, error, parse, robotparser
+import wget
 
 """
 
@@ -25,17 +25,13 @@ while True:
         break
 """
 
-talkers = {
-    "m3b": {
-        "tokenizer": AutoTokenizer.from_pretrained("GeneZC/MiniChat-3B", use_fast=False),
-        "model": AutoModelForCausalLM.from_pretrained("GeneZC/MiniChat-3B", device_map="auto", low_cpu_mem_usage=True),
-        "conv": get_default_conv_template("minichat")
-    }
-}
+from huggingface_hub import hf_hub_download
 
-request.urlretrieve("https://huggingface.co/GGUF/MiniChat-3B/resolve/main/ggml-model-q8_0.bin", "minichat-3b-q8_0.gguf")
+model_path = "minichat-3b.q8_0.gguf"
 
-lcpp_model = Model(model_path="minichat-3b-q8_0.gguf")
+hf_hub_download(repo_id="afrideva/MiniChat-3B-GGUF", filename=model_path)
+
+lcpp_model = Model(model_path=model_path)
 
 def m3b_talk(text):
     resp = ""