Spaces:
Sleeping
Sleeping
File size: 975 Bytes
a846510 a634807 a846510 fb33d5d a846510 fb33d5d a846510 4891247 a846510 4891247 f4cd2da a846510 952fd8e a846510 952fd8e a846510 952fd8e 314161f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import subprocess
import threading
import os
import time
import spaces
def setup_mixinputs():
# Step 1: Run mixinputs setup
subprocess.run(["mixinputs", "setup"], check=True)
# @spaces.GPU(duration=240)
def launch_vllm_server(beta=1.0):
# Step 2: Set environment variables
env = os.environ.copy()
env["MIXINPUTS_BETA"] = str(beta)
env["VLLM_USE_V1"] = "1"
# Step 3: Launch vLLM with custom options
cmd = [
"vllm", "serve",
"Qwen/Qwen3-4B",
"--tensor-parallel-size", "1",
"--enforce-eager",
"--max-model-len", "2048",
"--max-seq-len-to-capture", "2048",
"--max-num-seqs", "1",
"--port", "8000",
"--disable-async-output-proc"
]
subprocess.run(cmd, env=env)
# Step 1: Setup
setup_mixinputs()
# Step 2: Launch vLLM server in background
threading.Thread(target=launch_vllm_server, daemon=True).start()
# Step 3: Give time for server to initialize
time.sleep(60)
|