app.py

import gradio as gr

import ctransformers

class Z(object):
    def __init__(self):
        self.llm = None

    def init(self):
        pass

    def greet(self, txt0):
        prompt0 = txt0

        prompt00 = f'''USER: {prompt0}
ASSISTANT:'''
        
        response0 = llm(prompt00, max_new_tokens=128, temperature=0.5) # 0.3
        
        return f'{response0}'

from ctransformers import AutoModelForCausalLM

# wizzard vicuna
# see https://github.com/melodysdreamj/WizardVicunaLM
llm = AutoModelForCausalLM.from_pretrained('TheBloke/Wizard-Vicuna-7B-Uncensored-GGML', model_file='Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_0.bin', model_type='llama')




z = Z()
z.llm = llm
z.init()

def greet(arg0):
    global z
    return z.greet(arg0)

iface = gr.Interface(fn=greet, inputs="text", outputs="text")
iface.launch()