File size: 1,131 Bytes
1be71e1 d7c49f6 1be71e1 d7c49f6 1be71e1 d7c49f6 1be71e1 d7c49f6 1be71e1 d7c49f6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
from ctransformers import AutoModelForCausalLM
import gradio as gr
def generate_prompt(history):
prompt = "<s> "
for chain in history[:-1]:
prompt += f"<human>: {chain[0]}\n<bot>: {chain[1]}{end_token}\n"
prompt += f"<human>: {history[-1][0]}\n<bot>: "
return prompt
def generate(history):
prompt = generate_prompt(history)
streamer = llm(prompt, temperature=0, stream=True)
return streamer
llm = AutoModelForCausalLM.from_pretrained("model/model_q4_0.bin", model_type='llama')
end_token = "</s>"
with gr.Blocks() as demo:
chatbot = gr.Chatbot()
msg = gr.Textbox()
clear = gr.Button("Clear")
def user(user_message, history):
return "", history + [[user_message, ""]]
def bot(history):
streamer = generate(history)
for token in streamer:
history[-1][1] += token
yield history
msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
bot, chatbot, chatbot
)
clear.click(lambda: None, None, chatbot, queue=False)
demo.queue()
if __name__ == "__main__":
demo.launch() |