Update app.py

app.py

@@ -208,6 +208,34 @@ def format_prompt_zephyr(message, history, system_message=system_message):
     print(prompt)
     return prompt
 
+def generate(
+    prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,
+):
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
+
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_tokens=max_new_tokens,
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        stop=LLM_STOP_WORDS
+    )
+    formatted_prompt = format_prompt(prompt, history)
+
+    del generate_kwargs["max_tokens"]
+    del generate_kwargs["stop"]
+    
+    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    output = ""
+
+    for response in stream:
+        output += response.token.text
+        yield output
+    return output
+    
 def generate_local(
     prompt,    
     history,