Update app.py

app.py

@@ -1,12 +1,14 @@
-import gradio as gr, transformers
+import gradio as gr, transformers, torch
 
 pipe = transformers.pipeline("text-generation",
-                             model="marcelbinz/Llama-3.1-Minitaur-8B")
+                            model="marcelbinz/Llama-3.1-Minitaur-8B",
+                            device_map="auto",
+                            torch_dtype=torch.bfloat16)
 pipe.to('cuda')
 
 @spaces.GPU
 def infer(prompt, max_tokens):
-    return pipe(prompt, max_new_tokens=int(max_tokens))[0]["generated_text"]
+    return pipe(prompt, max_new_tokens=1, do_sample=True, temperature=1.0)[0]["generated_text"]
 
 demo = gr.Interface(
         fn=infer,
@@ -14,5 +16,5 @@ demo = gr.Interface(
         outputs="text",
         title="Minitaur",
         description="Just type and hit *Run*"
-)
+).queue()
 demo.launch()