{SPACE_TITLE}
{SPACE_TAGLINE}
# app.py — ShortiFoley (Video -> Foley) # Created by bilsimaging.com import os import sys import io import json import uuid import time import shutil import base64 import random import tempfile import datetime from pathlib import Path from typing import List, Optional, Tuple, Dict import numpy as np import torch import torchaudio import gradio as gr from loguru import logger from huggingface_hub import snapshot_download import spaces # HF Spaces ZeroGPU & MCP integration # ------------------------- # Constants & configuration # ------------------------- ROOT = Path(__file__).parent.resolve() REPO_DIR = ROOT / "HunyuanVideo-Foley" WEIGHTS_DIR = Path(os.environ.get("HIFI_FOLEY_MODEL_PATH", str(ROOT / "weights"))) CONFIG_PATH = Path(os.environ.get("HIFI_FOLEY_CONFIG", str(REPO_DIR / "configs" / "hunyuanvideo-foley-xxl.yaml"))) OUTPUTS_DIR = Path(os.environ.get("OUTPUTS_DIR", str(ROOT / "outputs"))) OUTPUTS_DIR.mkdir(parents=True, exist_ok=True) SPACE_TITLE = "🎵 ShortiFoley — HunyuanVideo-Foley" SPACE_TAGLINE = "Text/Video → Audio Foley. Created by bilsimaging.com" WATERMARK_NOTE = "Made with ❤️ by bilsimaging.com" # Keep GPU <= 120s for ZeroGPU (default 110) GPU_DURATION = int(os.environ.get("GPU_DURATION_SECS", "110")) # Globals _model_dict = None _cfg = None _device: Optional[torch.device] = None # ------------ # Small helpers # ------------ def _setup_device(pref: str = "auto", gpu_id: int = 0) -> torch.device: """Pick CUDA if available, else MPS, else CPU.""" if pref == "auto": if torch.cuda.is_available(): d = torch.device(f"cuda:{gpu_id}") elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): d = torch.device("mps") else: d = torch.device("cpu") else: d = torch.device(pref) logger.info(f"Using CUDA {d}" if d.type == "cuda" else f"Using {d}") return d def _ensure_repo() -> None: """Shallow-clone Tencent repo with LFS smudge disabled (avoid LFS quota checkout).""" if REPO_DIR.exists(): return cmd = ( "GIT_LFS_SKIP_SMUDGE=1 " "git -c filter.lfs.smudge= -c filter.lfs.required=false " f"clone --depth 1 https://github.com/Tencent-Hunyuan/HunyuanVideo-Foley.git {REPO_DIR}" ) logger.info(f">> {cmd}") os.system(cmd) def _download_weights_if_needed() -> None: """Snapshot only needed files from HF weights/model hub.""" WEIGHTS_DIR.mkdir(parents=True, exist_ok=True) snapshot_download( repo_id="tencent/HunyuanVideo-Foley", local_dir=str(WEIGHTS_DIR), resume_download=True, allow_patterns=[ "hunyuanvideo_foley.pth", "synchformer_state_dict.pth", "vae_128d_48k.pth", "assets/*", "config.yaml", # harmless ], ) def prepare_once() -> None: _ensure_repo() _download_weights_if_needed() # ----------------------- # Model load & inference # ----------------------- def auto_load_models() -> str: """ Load HunyuanVideo-Foley + encoders on the chosen device. """ global _model_dict, _cfg, _device if _model_dict is not None and _cfg is not None: return "Model already loaded." sys.path.append(str(REPO_DIR)) from hunyuanvideo_foley.utils.model_utils import load_model _device = _setup_device("auto", 0) logger.info("Loading HunyuanVideo-Foley model...") logger.info(f"MODEL_PATH: {WEIGHTS_DIR}") logger.info(f"CONFIG_PATH: {CONFIG_PATH}") try: _model_dict, _cfg = load_model(str(WEIGHTS_DIR), str(CONFIG_PATH), _device) return "✅ Model loaded." except Exception as e: logger.error(e) return f"❌ Failed to load model: {e}" def _merge_audio_video(audio_path: str, video_path: str, out_path: str) -> None: """Use project's helper (preferred) with a fallback to ffmpeg via subprocess.""" sys.path.append(str(REPO_DIR)) try: from hunyuanvideo_foley.utils.media_utils import merge_audio_video merge_audio_video(audio_path, video_path, out_path) except Exception as e: # Fallback: plain ffmpeg merge (assumes same duration or lets ffmpeg handle) logger.warning(f"merge_audio_video failed, falling back to ffmpeg: {e}") import subprocess cmd = [ "ffmpeg", "-y", "-i", video_path, "-i", audio_path, "-c:v", "copy", "-c:a", "aac", "-shortest", out_path ] subprocess.run(cmd, check=True) def _save_outputs(video_src: str, audio_tensor: torch.Tensor, sr: int, idx: int, prompt: str) -> str: """Save WAV + MP4 in outputs/, add metadata and a small watermark note (metadata only).""" # torchaudio expects [C, N] if audio_tensor.ndim == 1: audio_tensor = audio_tensor.unsqueeze(0) tmpdir = Path(tempfile.mkdtemp()) wav_path = tmpdir / f"gen_{idx}.wav" torchaudio.save(str(wav_path), audio_tensor.cpu(), sr) ts = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f") base = f"shortifoley_{ts}_{idx}" out_mp4 = OUTPUTS_DIR / f"{base}.mp4" _merge_audio_video(str(wav_path), video_src, str(out_mp4)) # Save JSON sidecar meta = { "id": base, "created_utc": datetime.datetime.utcnow().isoformat() + "Z", "source_video": Path(video_src).name, "output_video": Path(out_mp4).name, "prompt": prompt or "", "watermark": WATERMARK_NOTE, "tool": "ShortiFoley (HunyuanVideo-Foley)" } (OUTPUTS_DIR / f"{base}.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2)) return str(out_mp4) def _list_gallery(limit: int = 100) -> List[str]: vids = [] for p in sorted(OUTPUTS_DIR.glob("*.mp4"), key=lambda x: x.stat().st_mtime, reverse=True): vids.append(str(p)) if len(vids) >= limit: break return vids # ================ # Inference kernel # ================ @spaces.GPU(duration=GPU_DURATION) @torch.inference_mode() def infer_single_video( video_file: str, text_prompt: str, guidance_scale: float = 4.5, num_inference_steps: int = 50, sample_nums: int = 1, ) -> Tuple[List[str], str]: """ Generate Foley audio for an uploaded video (1–6 variants). Returns: (list of output video paths, status message) """ if _model_dict is None or _cfg is None: return [], "❌ Load the model first (open the app once)." if not video_file: return [], "❌ Please provide a video." sys.path.append(str(REPO_DIR)) from hunyuanvideo_foley.utils.feature_utils import feature_process from hunyuanvideo_foley.utils.model_utils import denoise_process # preprocess visual_feats, text_feats, audio_len_s = feature_process( video_file, (text_prompt or "").strip(), _model_dict, _cfg ) # generate batch n = int(max(1, min(6, sample_nums))) audio, sr = denoise_process( visual_feats, text_feats, audio_len_s, _model_dict, _cfg, guidance_scale=float(guidance_scale), num_inference_steps=int(num_inference_steps), batch_size=n, ) # save results outs = [] for i in range(n): outs.append(_save_outputs(video_file, audio[i], sr, i + 1, text_prompt or "")) return outs, f"✅ Generated {len(outs)} result(s). Saved to {OUTPUTS_DIR}/" # --------------- # MCP-only APIs # --------------- def _download_to_tmp(url: str) -> str: """Download a remote file to temp.""" try: import requests except Exception: raise RuntimeError("Missing dependency 'requests'. Add it to requirements.txt to use URL inputs.") r = requests.get(url, timeout=30) r.raise_for_status() tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") tmp.write(r.content) tmp.flush() tmp.close() return tmp.name def _maybe_from_base64(data_url_or_b64: str) -> str: """Accept data: URLs or raw base64; returns temp file path.""" b64 = data_url_or_b64 if data_url_or_b64.startswith("data:"): b64 = data_url_or_b64.split(",", 1)[-1] raw = base64.b64decode(b64) tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") tmp.write(raw) tmp.flush() tmp.close() return tmp.name def _normalize_video_input(video_url_or_b64: str) -> str: v = (video_url_or_b64 or "").strip() if v.startswith("http://") or v.startswith("https://"): return _download_to_tmp(v) return _maybe_from_base64(v) with gr.Blocks() as mcp_only_endpoints: gr.Markdown("These endpoints are MCP/API only and have no visible UI.", show_label=False) @gr.api def api_generate_from_url( video_url_or_b64: str, text_prompt: str = "", guidance_scale: float = 4.5, num_inference_steps: int = 50, sample_nums: int = 1, ) -> Dict[str, List[str]]: """ Generate Foley from a remote video URL or base64-encoded video. Returns: {"videos": [paths], "message": str} """ if _model_dict is None or _cfg is None: raise RuntimeError("Model not loaded. Open the UI once or call /load_model tool.") local = _normalize_video_input(video_url_or_b64) outs, msg = infer_single_video(local, text_prompt, guidance_scale, num_inference_steps, sample_nums) return {"videos": outs, "message": msg} @gr.api def load_model_tool() -> str: """Ensure model is loaded on server (MCP convenience).""" return auto_load_models() @gr.mcp.resource("shortifoley://status") def shortifoley_status() -> str: """Return a simple readiness string for MCP clients.""" ready = _model_dict is not None and _cfg is not None dev = "cuda" if (_device and _device.type == "cuda") else ("mps" if (_device and _device.type == "mps") else "cpu") return f"ShortiFoley status: {'ready' if ready else 'loading'} | device={dev} | outputs={OUTPUTS_DIR}" @gr.mcp.prompt() def foley_prompt(name: str = "default") -> str: """Reusable guidance for describing sound ambience.""" return ( "Describe the expected environmental sound precisely. Mention material, rhythm, intensity, and ambience.\n" "Example: 'Soft leather footfalls on wet pavement with distant traffic hiss; occasional splashes.'" ) # ------------- # Gradio UI # ------------- def _about_html() -> str: return f"""
ShortiFoley automatically generates realistic Foley soundtracks for short videos using Tencent’s HunyuanVideo-Foley with CLAP & SigLIP2 encoders. It includes autosave and an MCP server so you can call it from agents or workflows (e.g., n8n).
Created by bilsimaging.com
This app runs as an MCP server. Open the footer “View API → MCP” to copy a ready config. You can also use the REST endpoints listed there. Perfect for n8n integrations.
Each output’s metadata includes: {WATERMARK_NOTE}. If you want a visible video overlay, I can add an ffmpeg overlay step on request.
{SPACE_TAGLINE}