try fast api

app.py

@@ -1,4 +1,8 @@
 from huggingface_hub import InferenceClient
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+
 import gradio as gr
 
 client = InferenceClient(
@@ -16,8 +20,7 @@ def format_prompt(message, history, system_message=None):
     return prompt
 
 def generate(
-    prompt, history, system_message=None, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,
-):
+    prompt, history, system_message=None, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0):
     temperature = float(temperature)
     if temperature < 1e-2:
         temperature = 1e-2
@@ -88,6 +91,21 @@ additional_inputs=[
     )
 ]
 
+class Item(BaseModel):
+    prompt: str
+    history: list
+    system_prompt: str
+    temperature: float = 0.0
+    max_new_tokens: int = 1048
+    top_p: float = 0.15
+    repetition_penalty: float = 1.0
+app = FastAPI()
+
+@app.post("/generate/")
+async def generate_text(item: Item):
+    return {"response": generate(item.prompt, item.history, item.system_message, item.temperature, item.max_new_tokens, item.top_p, item.repetition_penalty)}
+
+
 
 gr.ChatInterface(
     fn=generate,