app.py

from huggingface_hub import InferenceClient
import gradio as gr

client = InferenceClient("grammarly/coedit-large")

def generate(prompt, temperature=0.9, max_new_tokens=256, top_p=0.95, top_k=50, repetition_penalty=1.0):
    temperature = float(temperature)
    if temperature < 1e-2: temperature = 1e-2
    top_p = float(top_p)
    top_k = int(top_k)  # Ensure top_k is an integer, as it was being treated like a float

    generate_kwargs = dict(temperature=temperature, max_new_tokens=max_new_tokens, top_p=top_p, top_k=top_k, repetition_penalty=repetition_penalty)  # seed=42,)

    formatted_prompt = "Fix grammatical errors in this sentence: " + prompt
    print("\nPROMPT: \n\t" + formatted_prompt)

    # Generate text from the HF inference
    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
    output = ""

    for response in stream:
        output += response.token.text
        yield output
    return output
    


additional_inputs=[
    gr.Slider( label="Temperature", value=0.9, minimum=0.0, maximum=1.0, step=0.05, interactive=True, info="Higher values produce more diverse outputs", ),
    gr.Slider( label="Max new tokens", value=256, minimum=0, maximum=1048, step=64, interactive=True, info="The maximum numbers of new tokens", ),
    gr.Slider( label="Top-p (nucleus sampling)", value=0.90, minimum=0.0, maximum=1, step=0.05, interactive=True, info="Higher values sample more low-probability tokens", ),
    gr.Slider(label="Top-k", value=50, minimum=0, maximum=100, step=1, interactive=True, info="Limits the number of top-k tokens considered at each step"),
]

gr.ChatInterface(
    fn=generate,
    chatbot=gr.Chatbot(show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),
    additional_inputs=additional_inputs,
    title="My Grammarly Space",
    concurrency_limit=20,
).launch(show_api=False)