Update app.py

app.py

@@ -25,6 +25,7 @@ try:
         n_gpu_layers=-1,  # change n_gpu_layers if you have more or less VRAM
         verbose=True
     )
+    
     print(f"START: AFTER LLAMA-CPP SETUP -- {time.time() - start_load_time}s")
 
 except Exception as e:
@@ -38,43 +39,31 @@ def generate_text(
     max_tokens,
     temperature,
     top_p,
-):
-    temp = ""
-    input_prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n "
-    for interaction in history:
-        input_prompt = (
-            input_prompt
-            + str(interaction[0])
-            + " [/INST] "
-            + str(interaction[1])
-            + " </s><s> [INST] "
-        )
+): 
+    messages = [{"role": "system", "content": system_message}]
 
-    input_prompt = input_prompt + str(message) + " [/INST] "
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
 
-    output = llm(
-        input_prompt,
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+    
+    for chunk in llm.create_chat_completion(
+        stream=True,
+        max_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
-        top_k=40,
-        repeat_penalty=1.1,
-        max_tokens=max_tokens,
-        stop=[
-            "<|prompter|>",
-            "<|endoftext|>",
-            "<|endoftext|> \n",
-            "ASSISTANT:",
-            "USER:",
-            "SYSTEM:",
-        ],
-        stream=True,
-    )
-    for out in output:
-        stream = copy.deepcopy(out)
-        temp += stream["choices"][0]["text"]
-        yield temp
-
-
+        messages=messages,
+    ):
+        part = chunk["choices"][0]["delta"].get("content", None)
+        if part:
+            response += part
+        yield response
+        
 demo = gr.ChatInterface(
     generate_text,
     title="llama-cpp-python on GPU",