Update app.py

app.py

@@ -9,9 +9,9 @@ from transformers import (
     MusicgenForConditionalGeneration,
 )
 from scipy.io.wavfile import write
-from TTS.api import TTS
-import tempfile
+from pydub import AudioSegment
 from dotenv import load_dotenv
+import tempfile
 import spaces
 
 # Load environment variables
@@ -19,10 +19,10 @@ load_dotenv()
 hf_token = os.getenv("HF_TOKEN")
 
 # ---------------------------------------------------------------------
-# Load Llama 3 Pipeline with Zero GPU (Encapsulated)
+# Generate Script
 # ---------------------------------------------------------------------
 @spaces.GPU(duration=300)
-def generate_script(user_prompt: str, duration: int, model_id: str, token: str):
+def generate_script(user_prompt: str, model_id: str, token: str):
     try:
         tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=token)
         model = AutoModelForCausalLM.from_pretrained(
@@ -36,7 +36,7 @@ def generate_script(user_prompt: str, duration: int, model_id: str, token: str):
 
         system_prompt = (
             "You are an expert radio imaging producer specializing in sound design and music. "
-            f"Generate a concise, creative promo script for a {duration}-second ad, focusing on auditory elements and musical appeal."
+            "Take the user's concept and craft a concise, creative promo script with a strong focus on auditory elements and musical appeal."
         )
 
         combined_prompt = f"{system_prompt}\nUser concept: {user_prompt}\nRefined script:"
@@ -46,7 +46,7 @@ def generate_script(user_prompt: str, duration: int, model_id: str, token: str):
         return f"Error generating script: {e}"
 
 # ---------------------------------------------------------------------
-# Load MusicGen Model (Encapsulated)
+# Generate Music
 # ---------------------------------------------------------------------
 @spaces.GPU(duration=300)
 def generate_audio(prompt: str, audio_length: int):
@@ -62,128 +62,116 @@ def generate_audio(prompt: str, audio_length: int):
 
         audio_data = outputs[0, 0].cpu().numpy()
         normalized_audio = (audio_data / max(abs(audio_data)) * 32767).astype("int16")
-
-        output_path = f"{tempfile.gettempdir()}/generated_audio.wav"
+        output_path = f"{tempfile.gettempdir()}/generated_music.wav"
         write(output_path, musicgen_model.config.audio_encoder.sampling_rate, normalized_audio)
+
         return output_path
     except Exception as e:
         return f"Error generating audio: {e}"
 
 # ---------------------------------------------------------------------
-# Generate Voice-Over with Coqui XTTS-v2
+# Generate Voice-Over (TTS)
 # ---------------------------------------------------------------------
 @spaces.GPU(duration=300)
-def generate_voice(script: str, reference_audio: str, language: str):
+def generate_voice(script: str, language: str):
     try:
-        tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2", gpu=torch.cuda.is_available())
-        output_path = f"{tempfile.gettempdir()}/voice_over.wav"
-        tts.tts_to_file(
-            text=script,
-            file_path=output_path,
-            speaker_wav=reference_audio,
-            language=language,
-        )
-        return output_path
+        tts_model = pipeline("text-to-speech", model="coqui/XTTS-v2")
+        tts_output = tts_model(script, language=language)
+
+        voice_path = f"{tempfile.gettempdir()}/generated_voice.wav"
+        with open(voice_path, "wb") as f:
+            f.write(tts_output["audio"])
+
+        return voice_path
     except Exception as e:
         return f"Error generating voice-over: {e}"
 
 # ---------------------------------------------------------------------
-# Interface Functions
+# Mix Audio with Ducking Option
+# ---------------------------------------------------------------------
+def mix_audio(voice_file, music_file, output_file, ducking: bool):
+    try:
+        voice = AudioSegment.from_file(voice_file)
+        music = AudioSegment.from_file(music_file)
+
+        if ducking:
+            music = music - 10  # Lower the volume of the music
+        combined = music.overlay(voice, position=0)
+
+        combined.export(output_file, format="wav")
+        return output_file
+    except Exception as e:
+        return f"Error mixing audio: {e}"
+
+# ---------------------------------------------------------------------
+# Gradio Interface Functions
 # ---------------------------------------------------------------------
-def interface_generate_script(user_prompt, duration, llama_model_id):
-    return generate_script(user_prompt, duration, llama_model_id, hf_token)
+def interface_generate_script(user_prompt, llama_model_id):
+    return generate_script(user_prompt, llama_model_id, hf_token)
 
 def interface_generate_audio(script, audio_length):
     return generate_audio(script, audio_length)
 
-def interface_generate_voice(script, reference_audio, language):
-    return generate_voice(script, reference_audio, language)
+def interface_generate_voice(script, language):
+    return generate_voice(script, language)
+
+def interface_mix_audio(voice_file, music_file, ducking):
+    output_file = f"{tempfile.gettempdir()}/final_promo.wav"
+    return mix_audio(voice_file, music_file, output_file, ducking)
 
 # ---------------------------------------------------------------------
 # Interface
 # ---------------------------------------------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("""
-        # 🎧 All-in-One Radio Promo Studio 🚀  
-        ### Create professional scripts, soundscapes, and voice-overs in minutes!  
-        🔥 Powered by **Llama 3**, **MusicGen**, and **XTTS-v2**
-    """)
+    gr.Markdown(
+        """
+        # 🎙️ AI Radio Promo Maker 🚀  
+        ### Your one-stop solution for **scripts**, **voice-overs**, and **music**!  
+        🔥 **Zero GPU** integration powered by **Hugging Face** models.  
+        """
+    )
 
-    # Script Generation Section
+    # Step 1: Generate Script
     gr.Markdown("## ✍️ Step 1: Generate Your Promo Script")
     with gr.Row():
-        user_prompt = gr.Textbox(
-            label="🎤 Enter Promo Idea",
-            placeholder="E.g., A 15-second energetic jingle for a morning talk show.",
-            lines=2
-        )
-        duration = gr.Dropdown(
-            label="⏳ Duration",
-            choices=["15", "30", "60"],
-            value="15",
-            info="Choose the duration of the promo (in seconds)."
-        )
-        llama_model_id = gr.Textbox(
-            label="🎛️ Llama 3 Model ID",
-            value="meta-llama/Meta-Llama-3-8B-Instruct"
-        )
-    generate_script_button = gr.Button("Generate Script ✨")
-    script_output = gr.Textbox(label="🖌️ Generated Promo Script", lines=4, interactive=False)
+        user_prompt = gr.Textbox(label="Enter Promo Idea", placeholder="E.g., A 15-second energetic jingle.", lines=2)
+        llama_model_id = gr.Textbox(label="Llama 3 Model ID", value="meta-llama/Meta-Llama-3-8B-Instruct")
+    generate_script_button = gr.Button("Generate Script")
+    script_output = gr.Textbox(label="Generated Script", lines=4, interactive=False)
 
-    # Audio Generation Section
-    gr.Markdown("## 🎵 Step 2: Generate Background Music")
+    # Step 2: Generate Voice-Over
+    gr.Markdown("## 🎤 Step 2: Generate Voice-Over")
     with gr.Row():
-        audio_length = gr.Slider(
-            label="🎶 Audio Length (tokens)",
-            minimum=128,
-            maximum=1024,
-            step=64,
-            value=512
-        )
-    generate_audio_button = gr.Button("Generate Audio 🎶")
-    audio_output = gr.Audio(label="🎵 Generated Audio", type="filepath")
+        language = gr.Dropdown(label="Select Language", choices=["en", "es", "fr", "de"], value="en")
+    generate_voice_button = gr.Button("Generate Voice")
+    voice_output = gr.Audio(label="Generated Voice", type="filepath", interactive=False)
 
-    # Voice-Over Section
-    gr.Markdown("## 🎙️ Step 3: Generate Voice-Over")
+    # Step 3: Generate Music
+    gr.Markdown("## 🎵 Step 3: Generate Background Music")
     with gr.Row():
-        reference_audio = gr.Audio(
-            label="🎤 Upload Reference Voice (6 seconds)",
-            type="filepath"
-        )
-        language = gr.Dropdown(
-            label="🌍 Language",
-            choices=["en", "es", "fr", "de", "it"],
-            value="en"
-        )
-    generate_voice_button = gr.Button("Generate Voice-Over 🎤")
-    voice_output = gr.Audio(label="🔊 Generated Voice-Over", type="filepath")
+        audio_length = gr.Slider(label="Audio Length (tokens)", minimum=128, maximum=1024, step=64, value=512)
+    generate_audio_button = gr.Button("Generate Music")
+    audio_output = gr.Audio(label="Generated Music", type="filepath", interactive=False)
 
-    # Footer
-    gr.Markdown("""
-        <br><hr>
-        <p style="text-align: center; font-size: 0.9em;">
-            Created with ❤️ by <a href="https://bilsimaging.com" target="_blank">bilsimaging.com</a>
-        </p>
-    """)
+    # Step 4: Mix Audio
+    gr.Markdown("## 🎶 Step 4: Mix Audio")
+    with gr.Row():
+        ducking = gr.Checkbox(label="Enable Ducking (lower background music volume)", value=True)
+    mix_audio_button = gr.Button("Mix Audio")
+    final_output = gr.Audio(label="Final Promo Audio", type="filepath", interactive=False)
 
     # Button Actions
-    generate_script_button.click(
-        fn=interface_generate_script,
-        inputs=[user_prompt, duration, llama_model_id],
-        outputs=script_output
-    )
-    generate_audio_button.click(
-        fn=interface_generate_audio,
-        inputs=[script_output, audio_length],
-        outputs=audio_output
-    )
-    generate_voice_button.click(
-        fn=interface_generate_voice,
-        inputs=[script_output, reference_audio, language],
-        outputs=voice_output
+    generate_script_button.click(interface_generate_script, inputs=[user_prompt, llama_model_id], outputs=script_output)
+    generate_voice_button.click(interface_generate_voice, inputs=[script_output, language], outputs=voice_output)
+    generate_audio_button.click(interface_generate_audio, inputs=[script_output, audio_length], outputs=audio_output)
+    mix_audio_button.click(interface_mix_audio, inputs=[voice_output, audio_output, ducking], outputs=final_output)
+
+    gr.Markdown(
+        """
+        <hr>
+        <p style="text-align: center;">Created with ❤️ by <a href="https://bilsimaging.com" target="_blank">bilsimaging.com</a></p>
+        """
     )
 
-# ---------------------------------------------------------------------
 # Launch App
-# ---------------------------------------------------------------------
 demo.launch(debug=True)