Spaces:
Sleeping
Sleeping
File size: 2,090 Bytes
c311821 9b2c83e c311821 a447070 9b2c83e 890d27d 9b2c83e c311821 9b2c83e df78555 c311821 df78555 9b2c83e c311821 7665895 df78555 7665895 c311821 9b2c83e cb9cef0 4239ac9 cb9cef0 4239ac9 cb9cef0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import gradio as gr
model_name = "Qwen/Qwen2.5-Coder-14B-Instruct"
# Load model and tokenizer (outside the function for efficiency)
model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype="auto",
# device_map="auto",
trust_remote_code=True # Add this line for Qwen models
).to('cuda')
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) # Add this line for Qwen models
@spaces.GPU(required=True)
def generate_code(prompt):
messages = [
{"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
{"role": "user", "content": prompt}
]
text = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
with torch.no_grad():
generated_ids = model.generate(
**model_inputs,
max_new_tokens=768
)
generated_ids = [
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
]
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
return response
with gr.Blocks(title="Qwen 14b") as demo: # Updated title
with gr.Tab("Code Chat"):
run_button = gr.Button("Run", scale=0)
prompt = gr.Text(
label="Prompt",
show_label=False,
max_lines=1,
placeholder="Enter your prompt",
container=False,
)
result = gr.Text(
label="Result",
show_label=False,
max_lines=100,
container=False,
)
gr.on(
triggers=[
run_button.click,
],
# api_name="generate", # Add this line
fn=generate_code,
inputs=[
prompt,
],
outputs=[result],
)
demo.launch(share=False) |