Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -2,15 +2,11 @@
|
|
2 |
# Created by bilsimaging.com
|
3 |
|
4 |
import os
|
5 |
-
#
|
6 |
os.environ.setdefault("HF_PREFER_SAFETENSORS", "1")
|
7 |
|
8 |
import sys
|
9 |
-
import io
|
10 |
import json
|
11 |
-
import uuid
|
12 |
-
import time
|
13 |
-
import shutil
|
14 |
import base64
|
15 |
import random
|
16 |
import tempfile
|
@@ -37,10 +33,10 @@ OUTPUTS_DIR = Path(os.environ.get("OUTPUTS_DIR", str(ROOT / "outputs")))
|
|
37 |
OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
|
38 |
|
39 |
SPACE_TITLE = "π΅ ShortiFoley β HunyuanVideo-Foley"
|
40 |
-
SPACE_TAGLINE = "Text/Video β Audio Foley
|
41 |
WATERMARK_NOTE = "Made with β€οΈ by bilsimaging.com"
|
42 |
|
43 |
-
#
|
44 |
GPU_DURATION = int(os.environ.get("GPU_DURATION_SECS", "110"))
|
45 |
|
46 |
# Globals
|
@@ -63,10 +59,7 @@ def _setup_device(pref: str = "auto", gpu_id: int = 0) -> torch.device:
|
|
63 |
d = torch.device("cpu")
|
64 |
else:
|
65 |
d = torch.device(pref)
|
66 |
-
|
67 |
-
logger.info(f"Using CUDA {d}")
|
68 |
-
else:
|
69 |
-
logger.info(f"Using {d}")
|
70 |
return d
|
71 |
|
72 |
|
@@ -116,9 +109,9 @@ def auto_load_models() -> str:
|
|
116 |
global _model_dict, _cfg, _device
|
117 |
|
118 |
if _model_dict is not None and _cfg is not None:
|
119 |
-
return "Model already loaded."
|
120 |
|
121 |
-
#
|
122 |
os.environ["HF_PREFER_SAFETENSORS"] = "1"
|
123 |
|
124 |
sys.path.append(str(REPO_DIR))
|
@@ -133,9 +126,8 @@ def auto_load_models() -> str:
|
|
133 |
_model_dict, _cfg = load_model(str(WEIGHTS_DIR), str(CONFIG_PATH), _device)
|
134 |
return "β
Model loaded."
|
135 |
except OSError as e:
|
136 |
-
# If any OSError (often from trying to read pytorch_model.bin), retry after enforcing safetensors.
|
137 |
logger.error(str(e))
|
138 |
-
logger.info("Retrying
|
139 |
os.environ["HF_PREFER_SAFETENSORS"] = "1"
|
140 |
try:
|
141 |
_model_dict, _cfg = load_model(str(WEIGHTS_DIR), str(CONFIG_PATH), _device)
|
@@ -149,7 +141,7 @@ def auto_load_models() -> str:
|
|
149 |
|
150 |
|
151 |
def _merge_audio_video(audio_path: str, video_path: str, out_path: str) -> None:
|
152 |
-
"""
|
153 |
sys.path.append(str(REPO_DIR))
|
154 |
try:
|
155 |
from hunyuanvideo_foley.utils.media_utils import merge_audio_video
|
@@ -171,7 +163,7 @@ def _merge_audio_video(audio_path: str, video_path: str, out_path: str) -> None:
|
|
171 |
|
172 |
def _save_outputs(video_src: str, audio_tensor: torch.Tensor, sr: int, idx: int,
|
173 |
prompt: str) -> str:
|
174 |
-
"""Save WAV + MP4 in outputs/, add metadata
|
175 |
# torchaudio expects [C, N]
|
176 |
if audio_tensor.ndim == 1:
|
177 |
audio_tensor = audio_tensor.unsqueeze(0)
|
@@ -186,14 +178,14 @@ def _save_outputs(video_src: str, audio_tensor: torch.Tensor, sr: int, idx: int,
|
|
186 |
|
187 |
_merge_audio_video(str(wav_path), video_src, str(out_mp4))
|
188 |
|
189 |
-
#
|
190 |
meta = {
|
191 |
"id": base,
|
192 |
"created_utc": datetime.datetime.utcnow().isoformat() + "Z",
|
193 |
"source_video": Path(video_src).name,
|
194 |
"output_video": Path(out_mp4).name,
|
195 |
"prompt": prompt or "",
|
196 |
-
"
|
197 |
"tool": "ShortiFoley (HunyuanVideo-Foley)"
|
198 |
}
|
199 |
(OUTPUTS_DIR / f"{base}.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))
|
@@ -226,8 +218,11 @@ def infer_single_video(
|
|
226 |
Generate Foley audio for an uploaded video (1β6 variants).
|
227 |
Returns: (list of output video paths, status message)
|
228 |
"""
|
|
|
229 |
if _model_dict is None or _cfg is None:
|
230 |
-
|
|
|
|
|
231 |
|
232 |
if not video_file:
|
233 |
return [], "β Please provide a video."
|
@@ -269,84 +264,96 @@ def _about_html() -> str:
|
|
269 |
return f"""
|
270 |
<div style="line-height:1.6">
|
271 |
<h2>About ShortiFoley</h2>
|
272 |
-
<p><b>ShortiFoley</b>
|
273 |
-
Tencentβs HunyuanVideo-Foley
|
274 |
-
|
275 |
-
<p><b>Created by <a href="https://bilsimaging.com" target="_blank">bilsimaging.com</a></b></p>
|
276 |
|
277 |
-
<h3>
|
278 |
<ol>
|
279 |
-
<li>Upload a
|
280 |
-
<li>Optionally
|
281 |
-
<li>
|
282 |
-
<li>
|
283 |
</ol>
|
284 |
|
285 |
-
<h3>Tips</h3>
|
286 |
<ul>
|
287 |
-
<li>
|
288 |
-
<li>Include material
|
289 |
-
<li>
|
|
|
290 |
</ul>
|
291 |
|
292 |
-
<h3>MCP
|
293 |
-
<p>This
|
|
|
294 |
|
295 |
<h3>Watermark</h3>
|
296 |
-
<p>Each output
|
297 |
</div>
|
298 |
"""
|
299 |
|
300 |
|
301 |
def create_ui() -> gr.Blocks:
|
302 |
-
|
303 |
-
|
304 |
-
|
305 |
-
|
306 |
-
|
307 |
-
|
308 |
-
|
309 |
-
|
|
|
|
|
310 |
|
311 |
gr.HTML(f"<div class='main-header'><h1>{SPACE_TITLE}</h1><p>{SPACE_TAGLINE}</p></div>")
|
312 |
|
313 |
with gr.Tabs():
|
314 |
with gr.Tab("Run"):
|
315 |
with gr.Row():
|
|
|
316 |
with gr.Column(scale=1, elem_classes=["card"]):
|
317 |
gr.Markdown("### πΉ Input")
|
318 |
video_input = gr.Video(label="Upload Video", height=300)
|
319 |
text_input = gr.Textbox(
|
320 |
label="π― Audio Description (optional, English)",
|
321 |
-
placeholder="e.g., Rubber soles on wet tile
|
322 |
lines=3
|
323 |
)
|
324 |
with gr.Row():
|
325 |
-
guidance_scale = gr.Slider(1.0, 10.0, value=4.5, step=0.1, label="CFG
|
326 |
steps = gr.Slider(10, 100, value=50, step=5, label="Steps")
|
327 |
samples = gr.Slider(1, 6, value=1, step=1, label="Variants")
|
328 |
-
generate = gr.Button("π΅ Generate", variant="primary", elem_classes=["generate-btn"])
|
329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
330 |
with gr.Column(scale=1, elem_classes=["card"]):
|
331 |
gr.Markdown("### π₯ Result(s)")
|
332 |
v1 = gr.Video(label="Sample 1", height=260, visible=True)
|
333 |
-
|
334 |
-
|
335 |
-
|
336 |
-
|
|
|
|
|
337 |
v6 = gr.Video(label="Sample 6", height=160, visible=False)
|
338 |
-
|
339 |
|
340 |
# Generate handler
|
341 |
def _process_and_update(video_file, text_prompt, cfg, nsteps, nsamples):
|
342 |
outs, msg = infer_single_video(video_file, text_prompt, cfg, nsteps, nsamples)
|
343 |
-
|
344 |
for i in range(6):
|
345 |
if i < len(outs):
|
346 |
-
|
347 |
else:
|
348 |
-
|
349 |
-
return (*
|
350 |
|
351 |
generate.click(
|
352 |
fn=_process_and_update,
|
@@ -356,7 +363,15 @@ def create_ui() -> gr.Blocks:
|
|
356 |
api_description="Generate Foley audio for an uploaded video. Returns up to 6 video+audio files."
|
357 |
)
|
358 |
|
359 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
360 |
def _toggle_vis(n):
|
361 |
n = int(n)
|
362 |
return [
|
@@ -380,78 +395,110 @@ def create_ui() -> gr.Blocks:
|
|
380 |
refresh = gr.Button("π Refresh Gallery")
|
381 |
refresh.click(lambda: gr.update(value=_list_gallery()), outputs=[gallery])
|
382 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
383 |
with gr.Tab("βΉοΈ About"):
|
384 |
gr.HTML(_about_html())
|
385 |
|
386 |
-
|
387 |
-
|
388 |
-
|
389 |
-
|
390 |
-
|
391 |
-
|
392 |
-
|
393 |
-
|
394 |
-
|
395 |
-
|
396 |
-
|
397 |
-
|
398 |
-
|
399 |
-
|
400 |
-
|
401 |
-
|
402 |
-
|
403 |
-
|
404 |
-
|
405 |
-
|
406 |
-
b64 = data_url_or_b64
|
407 |
-
|
408 |
-
|
409 |
-
|
410 |
-
|
411 |
-
|
412 |
-
|
413 |
-
|
414 |
-
|
415 |
-
|
416 |
-
|
417 |
-
|
418 |
-
|
419 |
-
|
420 |
-
|
421 |
-
|
422 |
-
|
423 |
-
|
424 |
-
|
425 |
-
|
426 |
-
|
427 |
-
|
428 |
-
|
429 |
-
|
430 |
-
if
|
431 |
-
raise RuntimeError(
|
432 |
-
|
433 |
-
|
434 |
-
|
435 |
-
|
436 |
-
|
437 |
-
|
438 |
-
|
439 |
-
|
440 |
-
|
441 |
-
|
442 |
-
|
443 |
-
|
444 |
-
|
445 |
-
|
446 |
-
|
447 |
-
|
448 |
-
|
449 |
-
|
450 |
-
|
451 |
-
|
452 |
-
|
453 |
-
|
454 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
455 |
|
456 |
return demo
|
457 |
|
@@ -473,22 +520,15 @@ if __name__ == "__main__":
|
|
473 |
logger.info("===== Application Startup =====\n")
|
474 |
prepare_once()
|
475 |
|
476 |
-
#
|
477 |
sys.path.append(str(REPO_DIR))
|
478 |
try:
|
479 |
-
# Probe key modules early (better error surfacing)
|
480 |
from hunyuanvideo_foley.utils.model_utils import load_model, denoise_process # noqa: F401
|
481 |
from hunyuanvideo_foley.utils.feature_utils import feature_process # noqa: F401
|
482 |
from hunyuanvideo_foley.utils.media_utils import merge_audio_video # noqa: F401
|
483 |
except Exception as e:
|
484 |
logger.warning(f"Repo imports not ready yet: {e}")
|
485 |
|
486 |
-
msg = auto_load_models()
|
487 |
-
if not msg.startswith("β
"):
|
488 |
-
logger.error(f"[BOOT][ERROR] auto_load_models() failed:\n{msg}")
|
489 |
-
else:
|
490 |
-
logger.info(msg)
|
491 |
-
|
492 |
ui = create_ui()
|
493 |
|
494 |
# Enable MCP server so tools/resources/prompts are discoverable
|
@@ -496,5 +536,5 @@ if __name__ == "__main__":
|
|
496 |
server_name="0.0.0.0",
|
497 |
share=False,
|
498 |
show_error=True,
|
499 |
-
mcp_server=True, # MCP on
|
500 |
)
|
|
|
2 |
# Created by bilsimaging.com
|
3 |
|
4 |
import os
|
5 |
+
# Prefer safetensors globally (fixes CLAP .bin crash on ZeroGPU)
|
6 |
os.environ.setdefault("HF_PREFER_SAFETENSORS", "1")
|
7 |
|
8 |
import sys
|
|
|
9 |
import json
|
|
|
|
|
|
|
10 |
import base64
|
11 |
import random
|
12 |
import tempfile
|
|
|
33 |
OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
|
34 |
|
35 |
SPACE_TITLE = "π΅ ShortiFoley β HunyuanVideo-Foley"
|
36 |
+
SPACE_TAGLINE = "Text/Video β Audio Foley Β· Created by bilsimaging.com"
|
37 |
WATERMARK_NOTE = "Made with β€οΈ by bilsimaging.com"
|
38 |
|
39 |
+
# ZeroGPU limit (<=120)
|
40 |
GPU_DURATION = int(os.environ.get("GPU_DURATION_SECS", "110"))
|
41 |
|
42 |
# Globals
|
|
|
59 |
d = torch.device("cpu")
|
60 |
else:
|
61 |
d = torch.device(pref)
|
62 |
+
logger.info(f"Using {d}")
|
|
|
|
|
|
|
63 |
return d
|
64 |
|
65 |
|
|
|
109 |
global _model_dict, _cfg, _device
|
110 |
|
111 |
if _model_dict is not None and _cfg is not None:
|
112 |
+
return "β
Model already loaded."
|
113 |
|
114 |
+
# Make absolutely sure safetensors is preferred
|
115 |
os.environ["HF_PREFER_SAFETENSORS"] = "1"
|
116 |
|
117 |
sys.path.append(str(REPO_DIR))
|
|
|
126 |
_model_dict, _cfg = load_model(str(WEIGHTS_DIR), str(CONFIG_PATH), _device)
|
127 |
return "β
Model loaded."
|
128 |
except OSError as e:
|
|
|
129 |
logger.error(str(e))
|
130 |
+
logger.info("Retrying after enforcing safetensors preference...")
|
131 |
os.environ["HF_PREFER_SAFETENSORS"] = "1"
|
132 |
try:
|
133 |
_model_dict, _cfg = load_model(str(WEIGHTS_DIR), str(CONFIG_PATH), _device)
|
|
|
141 |
|
142 |
|
143 |
def _merge_audio_video(audio_path: str, video_path: str, out_path: str) -> None:
|
144 |
+
"""Preferred: project's util; fallback to ffmpeg."""
|
145 |
sys.path.append(str(REPO_DIR))
|
146 |
try:
|
147 |
from hunyuanvideo_foley.utils.media_utils import merge_audio_video
|
|
|
163 |
|
164 |
def _save_outputs(video_src: str, audio_tensor: torch.Tensor, sr: int, idx: int,
|
165 |
prompt: str) -> str:
|
166 |
+
"""Save WAV + MP4 in outputs/, add metadata with a soft watermark note."""
|
167 |
# torchaudio expects [C, N]
|
168 |
if audio_tensor.ndim == 1:
|
169 |
audio_tensor = audio_tensor.unsqueeze(0)
|
|
|
178 |
|
179 |
_merge_audio_video(str(wav_path), video_src, str(out_mp4))
|
180 |
|
181 |
+
# Sidecar JSON
|
182 |
meta = {
|
183 |
"id": base,
|
184 |
"created_utc": datetime.datetime.utcnow().isoformat() + "Z",
|
185 |
"source_video": Path(video_src).name,
|
186 |
"output_video": Path(out_mp4).name,
|
187 |
"prompt": prompt or "",
|
188 |
+
"watermark_note": WATERMARK_NOTE,
|
189 |
"tool": "ShortiFoley (HunyuanVideo-Foley)"
|
190 |
}
|
191 |
(OUTPUTS_DIR / f"{base}.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))
|
|
|
218 |
Generate Foley audio for an uploaded video (1β6 variants).
|
219 |
Returns: (list of output video paths, status message)
|
220 |
"""
|
221 |
+
# Lazy-load if needed
|
222 |
if _model_dict is None or _cfg is None:
|
223 |
+
msg = auto_load_models()
|
224 |
+
if not str(msg).startswith("β
"):
|
225 |
+
return [], f"β {msg}"
|
226 |
|
227 |
if not video_file:
|
228 |
return [], "β Please provide a video."
|
|
|
264 |
return f"""
|
265 |
<div style="line-height:1.6">
|
266 |
<h2>About ShortiFoley</h2>
|
267 |
+
<p><b>ShortiFoley</b> turns short videos (and an optional text hint) into realistic Foley sound.
|
268 |
+
Powered by Tencentβs HunyuanVideo-Foley (SigLIP2 + CLAP), with autosave and an MCP server for automation (e.g., n8n).</p>
|
269 |
+
<p><b>Created by <a href="https://bilsimaging.com" target="_blank" rel="noopener">bilsimaging.com</a></b></p>
|
|
|
270 |
|
271 |
+
<h3>Quick Steps</h3>
|
272 |
<ol>
|
273 |
+
<li>Upload a clip (ideally < 120s).</li>
|
274 |
+
<li>Optionally describe the sound (English).</li>
|
275 |
+
<li>Pick variants (1β6), adjust CFG and steps.</li>
|
276 |
+
<li>Hit <b>Generate</b>. Results show on the right and save into the Gallery.</li>
|
277 |
</ol>
|
278 |
|
279 |
+
<h3>Tips for Best Quality</h3>
|
280 |
<ul>
|
281 |
+
<li>Use tight clips (5β30s) around the action.</li>
|
282 |
+
<li>Include material & action cues: βmetal clangβ, βglass shatterβ, βrubber on wet tileβ.</li>
|
283 |
+
<li>Describe ambience: βroomyβ, βechoeyβ, βdistant crowdβ.</li>
|
284 |
+
<li>Generate 2β4 variants and pick the most natural.</li>
|
285 |
</ul>
|
286 |
|
287 |
+
<h3>MCP & API</h3>
|
288 |
+
<p>This Space exposes an <b>MCP server</b> and simple REST endpoints (see βAPI & MCPβ tab).
|
289 |
+
Perfect for pipelines and tools like <b>n8n</b>.</p>
|
290 |
|
291 |
<h3>Watermark</h3>
|
292 |
+
<p>Each output writes a JSON sidecar including: <i>{WATERMARK_NOTE}</i>. Ask if you want a visible overlay.</p>
|
293 |
</div>
|
294 |
"""
|
295 |
|
296 |
|
297 |
def create_ui() -> gr.Blocks:
|
298 |
+
css = """
|
299 |
+
.main-header{ text-align:center; padding:1.2rem; border-radius:18px; background:linear-gradient(135deg,#6366f1,#8b5cf6); color:white; box-shadow:0 12px 40px rgba(99,102,241,.35); margin-bottom:16px;}
|
300 |
+
.main-header h1{ margin:0; font-size:2.0rem; font-weight:800;}
|
301 |
+
.main-header p{ margin:.25rem 0 0; opacity:.95; font-weight:500;}
|
302 |
+
.card{ background:white; border:1px solid #e7e9ef; border-radius:16px; padding:14px; box-shadow:0 10px 28px rgba(0,0,0,.06);}
|
303 |
+
.generate-btn button{ font-weight:800; border-radius:12px; padding:10px 18px;}
|
304 |
+
.minor-btn button{ border-radius:10px;}
|
305 |
+
.muted{ color:#64748b; }
|
306 |
+
"""
|
307 |
+
with gr.Blocks(title="ShortiFoley β HunyuanVideo-Foley", css=css) as demo:
|
308 |
|
309 |
gr.HTML(f"<div class='main-header'><h1>{SPACE_TITLE}</h1><p>{SPACE_TAGLINE}</p></div>")
|
310 |
|
311 |
with gr.Tabs():
|
312 |
with gr.Tab("Run"):
|
313 |
with gr.Row():
|
314 |
+
# LEFT: input
|
315 |
with gr.Column(scale=1, elem_classes=["card"]):
|
316 |
gr.Markdown("### πΉ Input")
|
317 |
video_input = gr.Video(label="Upload Video", height=300)
|
318 |
text_input = gr.Textbox(
|
319 |
label="π― Audio Description (optional, English)",
|
320 |
+
placeholder="e.g., Rubber soles on wet tile; distant chatter; occasional splashes.",
|
321 |
lines=3
|
322 |
)
|
323 |
with gr.Row():
|
324 |
+
guidance_scale = gr.Slider(1.0, 10.0, value=4.5, step=0.1, label="CFG")
|
325 |
steps = gr.Slider(10, 100, value=50, step=5, label="Steps")
|
326 |
samples = gr.Slider(1, 6, value=1, step=1, label="Variants")
|
|
|
327 |
|
328 |
+
with gr.Row():
|
329 |
+
load_btn = gr.Button("βοΈ Load model", variant="secondary", elem_classes=["minor-btn"])
|
330 |
+
generate = gr.Button("π΅ Generate", variant="primary", elem_classes=["generate-btn"])
|
331 |
+
|
332 |
+
status = gr.Textbox(label="Status", interactive=False)
|
333 |
+
|
334 |
+
# RIGHT: results
|
335 |
with gr.Column(scale=1, elem_classes=["card"]):
|
336 |
gr.Markdown("### π₯ Result(s)")
|
337 |
v1 = gr.Video(label="Sample 1", height=260, visible=True)
|
338 |
+
with gr.Row():
|
339 |
+
v2 = gr.Video(label="Sample 2", height=160, visible=False)
|
340 |
+
v3 = gr.Video(label="Sample 3", height=160, visible=False)
|
341 |
+
with gr.Row():
|
342 |
+
v4 = gr.Video(label="Sample 4", height=160, visible=False)
|
343 |
+
v5 = gr.Video(label="Sample 5", height=160, visible=False)
|
344 |
v6 = gr.Video(label="Sample 6", height=160, visible=False)
|
345 |
+
gr.Markdown("<span class='muted'>Autosaved to the Gallery tab.</span>")
|
346 |
|
347 |
# Generate handler
|
348 |
def _process_and_update(video_file, text_prompt, cfg, nsteps, nsamples):
|
349 |
outs, msg = infer_single_video(video_file, text_prompt, cfg, nsteps, nsamples)
|
350 |
+
vis = []
|
351 |
for i in range(6):
|
352 |
if i < len(outs):
|
353 |
+
vis.append(gr.update(visible=True, value=outs[i]))
|
354 |
else:
|
355 |
+
vis.append(gr.update(visible=False, value=None))
|
356 |
+
return (*vis, msg)
|
357 |
|
358 |
generate.click(
|
359 |
fn=_process_and_update,
|
|
|
363 |
api_description="Generate Foley audio for an uploaded video. Returns up to 6 video+audio files."
|
364 |
)
|
365 |
|
366 |
+
load_btn.click(
|
367 |
+
fn=auto_load_models,
|
368 |
+
inputs=[],
|
369 |
+
outputs=[status],
|
370 |
+
api_name="/load_model",
|
371 |
+
api_description="Load/initialize the ShortiFoley model and encoders."
|
372 |
+
)
|
373 |
+
|
374 |
+
# Toggle visibility based on variants
|
375 |
def _toggle_vis(n):
|
376 |
n = int(n)
|
377 |
return [
|
|
|
395 |
refresh = gr.Button("π Refresh Gallery")
|
396 |
refresh.click(lambda: gr.update(value=_list_gallery()), outputs=[gallery])
|
397 |
|
398 |
+
with gr.Tab("API & MCP"):
|
399 |
+
gr.Markdown("""
|
400 |
+
### REST examples
|
401 |
+
|
402 |
+
**POST** `/api_generate_from_url`
|
403 |
+
```json
|
404 |
+
{
|
405 |
+
"video_url_or_b64": "https://yourhost/sample.mp4",
|
406 |
+
"text_prompt": "metallic clink; hollow room reverb",
|
407 |
+
"guidance_scale": 4.5,
|
408 |
+
"num_inference_steps": 50,
|
409 |
+
"sample_nums": 2
|
410 |
+
}
|
411 |
+
```
|
412 |
+
|
413 |
+
**POST** `/load_model_tool`
|
414 |
+
Loads the model proactively (useful before batch runs).
|
415 |
+
|
416 |
+
### MCP resources & prompt
|
417 |
+
- `shortifoley://status` β quick health info
|
418 |
+
- `foley_prompt` β reusable guidance for describing the sound
|
419 |
+
|
420 |
+
Works great with n8n: call `load_model_tool` once, then `api_generate_from_url` for each clip.
|
421 |
+
""")
|
422 |
+
|
423 |
with gr.Tab("βΉοΈ About"):
|
424 |
gr.HTML(_about_html())
|
425 |
|
426 |
+
# Keep gallery fresh after generation
|
427 |
+
generate.click(lambda: gr.update(value=_list_gallery()), outputs=[gallery])
|
428 |
+
|
429 |
+
# ---- REST + MCP endpoints (inside Blocks) ----
|
430 |
+
def _download_to_tmp(url: str) -> str:
|
431 |
+
try:
|
432 |
+
import requests
|
433 |
+
except Exception:
|
434 |
+
raise RuntimeError("Missing dependency 'requests'. Add it to requirements.txt to use URL inputs.")
|
435 |
+
r = requests.get(url, timeout=30)
|
436 |
+
r.raise_for_status()
|
437 |
+
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
|
438 |
+
tmp.write(r.content)
|
439 |
+
tmp.flush()
|
440 |
+
tmp.close()
|
441 |
+
return tmp.name
|
442 |
+
|
443 |
+
def _maybe_from_base64(data_url_or_b64: str) -> str:
|
444 |
+
b64 = data_url_or_b64
|
445 |
+
if data_url_or_b64.startswith("data:"):
|
446 |
+
b64 = data_url_or_b64.split(",", 1)[-1]
|
447 |
+
raw = base64.b64decode(b64)
|
448 |
+
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
|
449 |
+
tmp.write(raw)
|
450 |
+
tmp.flush()
|
451 |
+
tmp.close()
|
452 |
+
return tmp.name
|
453 |
+
|
454 |
+
def _normalize_video_input(video_url_or_b64: str) -> str:
|
455 |
+
v = (video_url_or_b64 or "").strip()
|
456 |
+
if v.startswith("http://") or v.startswith("https://"):
|
457 |
+
return _download_to_tmp(v)
|
458 |
+
return _maybe_from_base64(v)
|
459 |
+
|
460 |
+
@gr.api
|
461 |
+
def api_generate_from_url(
|
462 |
+
video_url_or_b64: str,
|
463 |
+
text_prompt: str = "",
|
464 |
+
guidance_scale: float = 4.5,
|
465 |
+
num_inference_steps: int = 50,
|
466 |
+
sample_nums: int = 1,
|
467 |
+
) -> Dict[str, List[str]]:
|
468 |
+
if _model_dict is None or _cfg is None:
|
469 |
+
msg = auto_load_models()
|
470 |
+
if not str(msg).startswith("β
"):
|
471 |
+
raise RuntimeError(msg)
|
472 |
+
local = _normalize_video_input(video_url_or_b64)
|
473 |
+
outs, msg = infer_single_video(local, text_prompt, guidance_scale, num_inference_steps, sample_nums)
|
474 |
+
return {"videos": outs, "message": msg}
|
475 |
+
|
476 |
+
@gr.api
|
477 |
+
def load_model_tool() -> str:
|
478 |
+
"""Ensure model is loaded on server (convenient for MCP/REST)."""
|
479 |
+
return auto_load_models()
|
480 |
+
|
481 |
+
@gr.mcp.resource("shortifoley://status")
|
482 |
+
def shortifoley_status() -> str:
|
483 |
+
"""Return a simple readiness string for MCP clients."""
|
484 |
+
ready = _model_dict is not None and _cfg is not None
|
485 |
+
dev = "cuda" if (_device and _device.type == "cuda") else ("mps" if (_device and _device.type == "mps") else "cpu")
|
486 |
+
return f"ShortiFoley status: {'ready' if ready else 'loading'} | device={dev} | outputs={OUTPUTS_DIR}"
|
487 |
+
|
488 |
+
@gr.mcp.prompt()
|
489 |
+
def foley_prompt(name: str = "default") -> str:
|
490 |
+
"""Reusable guidance for describing sound ambience."""
|
491 |
+
return (
|
492 |
+
"Describe the expected environmental sound precisely. Mention material, rhythm, intensity, and ambience.\n"
|
493 |
+
"Example: 'Soft leather footfalls on wet pavement with distant traffic hiss; occasional splashes.'"
|
494 |
+
)
|
495 |
+
|
496 |
+
# Auto-load model when UI first renders
|
497 |
+
demo.load(
|
498 |
+
fn=auto_load_models,
|
499 |
+
inputs=None,
|
500 |
+
outputs=[status]
|
501 |
+
)
|
502 |
|
503 |
return demo
|
504 |
|
|
|
520 |
logger.info("===== Application Startup =====\n")
|
521 |
prepare_once()
|
522 |
|
523 |
+
# Probe imports (early surfacing)
|
524 |
sys.path.append(str(REPO_DIR))
|
525 |
try:
|
|
|
526 |
from hunyuanvideo_foley.utils.model_utils import load_model, denoise_process # noqa: F401
|
527 |
from hunyuanvideo_foley.utils.feature_utils import feature_process # noqa: F401
|
528 |
from hunyuanvideo_foley.utils.media_utils import merge_audio_video # noqa: F401
|
529 |
except Exception as e:
|
530 |
logger.warning(f"Repo imports not ready yet: {e}")
|
531 |
|
|
|
|
|
|
|
|
|
|
|
|
|
532 |
ui = create_ui()
|
533 |
|
534 |
# Enable MCP server so tools/resources/prompts are discoverable
|
|
|
536 |
server_name="0.0.0.0",
|
537 |
share=False,
|
538 |
show_error=True,
|
539 |
+
mcp_server=True, # MCP on (great for n8n)
|
540 |
)
|