|
|
|
|
|
from typing import Iterator |
|
|
|
|
|
from ctransformers import AutoModelForCausalLM |
|
|
|
|
|
|
|
|
llm = AutoModelForCausalLM.from_pretrained("OpenBuddy/openbuddy-gguf", model_file="openbuddy-mistral-7b-v13.1-Q3_K.gguf", model_type="mistral", gpu_layers=0) |
|
|
|
|
|
def run(message: str, |
|
|
chat_history: list[tuple[str, str]], |
|
|
system_prompt: str, |
|
|
max_new_tokens: int = 1024, |
|
|
temperature: float = 0.3, |
|
|
top_p: float = 0.85, |
|
|
top_k: int = 5) -> Iterator[str]: |
|
|
history = [] |
|
|
print(chat_history) |
|
|
result="" |
|
|
for i in chat_history: |
|
|
history.append({"role": "user", "content": i[0]}) |
|
|
history.append({"role": "assistant", "content": i[1]}) |
|
|
print(history) |
|
|
history.append({"role": "user", "content": message}) |
|
|
for response in llm.create_chat_completion(history,stop=["</s>"],stream=True,max_tokens=-1,temperature=temperature,top_k=top_k,top_p=top_p,repeat_penalty=1.1): |
|
|
if "content" in response["choices"][0]["delta"]: |
|
|
result = result + response["choices"][0]["delta"]["content"] |
|
|
yield result |
