app.py

from huggingface_hub import InferenceClient
import gradio as gr
import random
import prompts
from pypipertts import PyPiper
pp=PyPiper()
pp.load_mod(instr="en_US-joe-medium")

client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")

def format_prompt(message, history):

  prompt = "<s>"
  for user_prompt, bot_response in history:
    print (bot_response)
    prompt += f"[INST] {user_prompt} [/INST]"
    prompt += f" {bot_response}</s> "
  prompt += f"[INST] {message} [/INST]"
  return prompt

def generate(prompt,history,max_new_tokens,seed):
    #seed = random.randint(1,9999999999999)
    print(seed)
    system_prompt = prompts.AI_REPORT_WRITER
    generate_kwargs = dict(
        temperature=0.9,
        max_new_tokens=max_new_tokens,
        top_p=0.95,
        repetition_penalty=1.0,
        do_sample=True,
        seed=seed,
    )

    formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
    output = ""

    for response in stream:
        if response.token.text == "\n":
            yield from pp.stream_tts(output)
            output=""
        output += response.token.text
        
        #yield output

with gr.Blocks() as iface:
    gr.HTML("""""")
    aud=gr.Audio(streaming=True,autoplay=True)
    #chatbot=gr.Chatbot(show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),
    chatbot=gr.Chatbot()
    msg = gr.Textbox()
    with gr.Row():
        submit_b = gr.Button()
        stop_b = gr.Button("Stop")
        clear = gr.ClearButton([msg, chatbot])
    sumbox=gr.Textbox("Summary", max_lines=100)
    with gr.Column():
        sum_out_box=gr.JSON(label="Summaries")
        hist_out_box=gr.JSON(label="History")
        
    #sub_b = submit_b.click(generate, [msg,chatbot],[msg,chatbot,sumbox,sum_out_box,hist_out_box])
    #sub_e = msg.submit(generate, [msg, chatbot], [msg, chatbot,sumbox,sum_out_box,hist_out_box])    
    sub_b = submit_b.click(generate, [msg,chatbot],aud)
    sub_e = msg.submit(generate, [msg, chatbot], aud)
    stop_b.click(None,None,None, cancels=[sub_b,sub_e])
iface.queue(default_concurrency_limit=10).launch()