File size: 1,191 Bytes
3cc267d 7eb86aa 3cc267d 7eb86aa 3cc267d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
from typing import Iterator
from ctransformers import AutoModelForCausalLM
# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.
llm = AutoModelForCausalLM.from_pretrained("OpenBuddy/openbuddy-gguf", model_file="openbuddy-mistral-7b-v13.1-Q3_K.gguf", model_type="mistral", gpu_layers=0)
def run(message: str,
chat_history: list[tuple[str, str]],
system_prompt: str,
max_new_tokens: int = 1024,
temperature: float = 0.3,
top_p: float = 0.85,
top_k: int = 5) -> Iterator[str]:
history = []
print(chat_history)
result=""
for i in chat_history:
history.append({"role": "user", "content": i[0]})
history.append({"role": "assistant", "content": i[1]})
print(history)
history.append({"role": "user", "content": message})
for response in llm.create_chat_completion(history,stop=["</s>"],stream=True,max_tokens=-1,temperature=temperature,top_k=top_k,top_p=top_p,repeat_penalty=1.1):
if "content" in response["choices"][0]["delta"]:
result = result + response["choices"][0]["delta"]["content"]
yield result |