Add demo

README.md

@@ -1,14 +1,25 @@
 ---
-title: Voxtream
-emoji: 🐠
+title: VoXtream
+emoji: ⚡
 colorFrom: green
 colorTo: yellow
 sdk: gradio
-sdk_version: 5.47.1
+python_version: 3.11
+sdk_version: 4.44.1
 app_file: app.py
-pinned: false
+pinned: true
 license: cc-by-4.0
 short_description: Generate speech from text and an audio prompt
+models:
+  - herimor/voxtream
+tags:
+  - text-to-speech
+  - speech-synthesis
+preload_from_hub:
+  - herimor/voxtream model.safetensors,config.json,phoneme_to_token.json
+  - kyutai/moshiko-pytorch-bf16 tokenizer-e351c8d8-checkpoint125.safetensors
+  - charsiu/en_w2v2_fc_10ms pytorch_model.bin,config.json
+  - charsiu/tokenizer_en_cmu special_tokens_map.json,tokenizer_config.json,vocab.json
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,126 @@
+import json
+from pathlib import Path
+
+import nltk
+import torch
+import spaces
+import gradio as gr
+import numpy as np
+
+from voxtream.generator import SpeechGenerator, SpeechGeneratorConfig
+
+with open("configs/generator.json") as f:
+    config = SpeechGeneratorConfig(**json.load(f))
+
+# Loading speaker encoder
+torch.hub.load(
+    config.spk_enc_repo,
+    config.spk_enc_model,
+    model_name=config.spk_enc_model_name,
+    train_type=config.spk_enc_train_type,
+    dataset=config.spk_enc_dataset,
+    trust_repo=True,
+    verbose=False,
+)
+# Loading NLTK packages
+nltk.download("averaged_perceptron_tagger_eng", quiet=True, raise_on_error=True)
+nltk.download("punkt", quiet=True, raise_on_error=True)
+
+# Initialize speech generator
+speech_generator = SpeechGenerator(config)
+
+CUSTOM_CSS = """
+/* overall width */
+.gradio-container {max-width: 1100px !important}
+/* stack labels tighter and even heights */
+#cols .wrap > .form {gap: 10px}
+#left-col, #right-col {gap: 14px}
+/* make submit centered + bigger */
+#submit {width: 260px; margin: 10px auto 0 auto;}
+/* make clear align left and look secondary */
+#clear {width: 120px;}
+/* give audio a little breathing room */
+audio {outline: none;}
+"""
+
+
+@spaces.GPU
+def synthesize_fn(prompt_audio_path, prompt_text, target_text):
+    if speech_generator.model.device == "cpu":
+        speech_generator.model.to("cuda")
+        speech_generator.mimi.to("cuda")
+        speech_generator.spk_enc.to("cuda")
+        speech_generator.aligner.to("cuda")
+
+    if not prompt_audio_path or not target_text:
+        return None
+    stream = speech_generator.generate_stream(
+        prompt_text=prompt_text,
+        prompt_audio_path=Path(prompt_audio_path),
+        text=target_text,
+    )
+    frames = [frame for frame, _ in stream]
+    if not frames:
+        return None
+    waveform = np.concatenate(frames).astype(np.float32)
+
+    # Fade out
+    fade_len_sec = 0.1
+    fade_out = np.linspace(1.0, 0.0, int(config.mimi_sr * fade_len_sec))
+    waveform[-int(config.mimi_sr * fade_len_sec) :] *= fade_out
+
+    return (config.mimi_sr, waveform)
+
+
+def main():
+    with gr.Blocks(css=CUSTOM_CSS, title="VoXtream") as demo:
+        gr.Markdown("# VoXtream TTS demo")
+
+        with gr.Row(equal_height=True, elem_id="cols"):
+            with gr.Column(scale=1, elem_id="left-col"):
+                prompt_audio = gr.Audio(
+                    sources=["microphone", "upload"],
+                    type="filepath",
+                    label="Prompt audio (3-5 sec of target voice)",
+                )
+                prompt_text = gr.Textbox(
+                    lines=3,
+                    label="Prompt transcript",
+                    placeholder="Text that matches the prompt audio (Required)",
+                )
+
+            with gr.Column(scale=1, elem_id="right-col"):
+                target_text = gr.Textbox(
+                    lines=3,
+                    label="Target text",
+                    placeholder="What you want the model to say",
+                )
+                output_audio = gr.Audio(
+                    type="numpy",
+                    label="Synthesized audio",
+                    interactive=False,
+                )
+
+        with gr.Row():
+            clear_btn = gr.Button("Clear", elem_id="clear", variant="secondary")
+            submit_btn = gr.Button("Submit", elem_id="submit", variant="primary")
+
+        # wire up actions
+        submit_btn.click(
+            fn=synthesize_fn,
+            inputs=[prompt_audio, prompt_text, target_text],
+            outputs=output_audio,
+        )
+
+        # reset everything
+        clear_btn.click(
+            fn=lambda: (None, "", "", None),
+            inputs=[],
+            outputs=[prompt_audio, prompt_text, target_text, output_audio],
+        )
+
+    demo.launch()
+
+
+if __name__ == "__main__":
+    main()

configs/generator.json

@@ -0,0 +1,49 @@
+{
+    "sil_token": 69,
+    "bos_token": 71,
+    "eos_token": 72,
+    "end_pad": 5,
+    "num_codebooks": 12,
+    "num_phones_per_frame": 2,
+    "audio_delay_frames": 1,
+    "temperature": 0.9,
+    "topk": 5,
+    "max_audio_length_ms": 60000,
+    "device": "cpu",
+    "model_repo": "herimor/voxtream",
+    "model_name": "model.safetensors",
+    "model_config_name": "config.json",
+    "mimi_sr": 24000,
+    "mimi_vocab_size": 2048,
+    "mimi_frame_ms": 80,
+    "mimi_repo": "kyutai/moshiko-pytorch-bf16",
+    "mimi_name": "tokenizer-e351c8d8-checkpoint125.safetensors",
+    "spk_enc_sr": 16000,
+    "spk_enc_repo": "IDRnD/ReDimNet",
+    "spk_enc_model": "ReDimNet",
+    "spk_enc_model_name": "M",
+    "spk_enc_train_type": "ft_mix",
+    "spk_enc_dataset": "vb2+vox2+cnc",
+    "phoneme_dict_name": "phoneme_to_token.json",
+    "nltk_resource": "taggers/averaged_perceptron_tagger_eng",
+    "aligner": "charsiu/en_w2v2_fc_10ms",
+    "cache_prompt": false,
+    "phoneme_index_map": {
+        "0": [
+            0,
+            1
+        ],
+        "1": [
+            0,
+            2
+        ],
+        "2": [
+            1,
+            1
+        ],
+        "3": [
+            1,
+            2
+        ]
+    }
+}

requirements.txt

@@ -0,0 +1,3 @@
+voxtream
+gradio_client==1.3.0
+pydantic==2.10.6