jtordable commited on
Commit
53f3bb2
·
verified ·
1 Parent(s): d0c1a5d

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +46 -0
app.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gradio as gr
3
+ import spaces
4
+ import torch
5
+ from transformers import AutoTokenizer, AutoModelForCausalLM
6
+
7
+ model_id = "futurehouse/ether0"
8
+
9
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
10
+ model = AutoModelForCausalLM.from_pretrained(
11
+ model_id,
12
+ device_map="auto",
13
+ torch_dtype=torch.float32
14
+ )
15
+
16
+ @spaces.GPU
17
+ def chat_fn(prompt, max_tokens=512):
18
+ max_tokens = min(int(max_tokens), 32_000)
19
+ messages = [{"role": "user", "content": prompt}]
20
+ chat_prompt = tokenizer.apply_chat_template(
21
+ messages, tokenize=False, add_generation_prompt=True
22
+ )
23
+ inputs = tokenizer(chat_prompt, return_tensors="pt").to(model.device)
24
+
25
+ # Generate with proper parameters
26
+ outputs = model.generate(
27
+ **inputs,
28
+ max_new_tokens=max_tokens,
29
+ do_sample=True,
30
+ temperature=0.1,
31
+ pad_token_id=tokenizer.eos_token_id
32
+ )
33
+
34
+ # Decode only the new tokens (not the input)
35
+ generated_text = tokenizer.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
36
+ return generated_text
37
+
38
+ gr.Interface(
39
+ fn=chat_fn,
40
+ inputs=[
41
+ gr.Textbox(label="prompt"),
42
+ gr.Number(label="max_tokens", value=512, precision=0)
43
+ ],
44
+ outputs="text",
45
+ title="Ether0"
46
+ ).launch(ssr_mode=False)