app.py

from transformers import AutoModelForCausalLM, AutoTokenizer
import modin.pandas as pd
import gradio as gr

tokenizer = AutoTokenizer.from_pretrained("KoboldAI/OPT-2.7B-Nerybus-Mix")

model = AutoModelForCausalLM.from_pretrained("KoboldAI/OPT-2.7B-Nerybus-Mix")

def chat(Prompt):

    input_ids = tokenizer(Prompt, return_tensors="pt").input_ids
    generated_ids = model.generate(input_ids, max_length=256, repetition_penalty = 2.5, top_k = 75)
    bot = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
    return bot
 

gr.Interface(fn=chat, inputs='text', outputs='text').queue(max_queue=10).launch(max_threads=40, debug=True)