app.py

import os
import shutil
import tempfile
import subprocess
from pathlib import Path
import numpy as np
import soundfile as sf
from pydub import AudioSegment
from faster_whisper import WhisperModel
from openai import OpenAI
import httpx
import asyncio
import gradio as gr
import requests

# --- Demucs-based vocal separation ---
def separate_vocals(input_path, progress=gr.Progress()):
    """Use Demucs to separate vocals and background music"""
    progress(0.1, desc="Separating vocals and music (Demucs)")
    temp_dir = tempfile.mkdtemp()
    try:
        output_dir = os.path.join(temp_dir, "separated")
        os.makedirs(output_dir, exist_ok=True)
        from demucs.separate import main as demucs_main
        import sys
        original_argv = sys.argv
        sys.argv = [
            "demucs",
            "--two-stems", "vocals",
            "-o", output_dir,
            input_path
        ]
        try:
            demucs_main()
        finally:
            sys.argv = original_argv
        base_name = Path(input_path).stem
        vocals_path = os.path.join(output_dir, "htdemucs", base_name, "vocals.wav")
        noise_path = os.path.join(output_dir, "htdemucs", base_name, "no_vocals.wav")
        if not os.path.exists(vocals_path) or not os.path.exists(noise_path):
            raise FileNotFoundError("Demucs output missing")
        progress(0.3, desc="Vocals separated")
        return vocals_path, noise_path, temp_dir
    except Exception as e:
        print(f"Demucs error: {e}")
        shutil.rmtree(temp_dir, ignore_errors=True)
        return None, None, None

# --- AudioProcessor class ---
class AudioProcessor:
    def __init__(self, device="cpu"):
        self.whisper_model = WhisperModel("small", device=device)
        self.openrouter_api_key = "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"
        self.client = OpenAI(
            base_url="https://openrouter.ai/api/v1",
            api_key=self.openrouter_api_key,
            http_client=httpx.Client(headers={
                "Authorization": f"Bearer {self.openrouter_api_key}",
                "HTTP-Referer": "https://github.com",
                "X-Title": "Audio Translation App"
            })
        )
    def transcribe_audio_with_pauses(self, audio_path, progress):
        progress(0.35, desc="Transcribing audio (Whisper)")
        segments, _ = self.whisper_model.transcribe(audio_path, word_timestamps=True)
        previous_end = 0.0
        results = []
        for segment in segments:
            if segment.start > previous_end + 0.5:
                results.append((previous_end, segment.start, None))
            results.append((segment.start, segment.end, segment.text.strip()))
            previous_end = segment.end
        audio_duration = get_audio_duration(audio_path)
        if audio_duration and audio_duration > previous_end + 0.5:
            results.append((previous_end, audio_duration, None))
        progress(0.5, desc="Transcription complete")
        return results

    def translate_segments_batch(self, segments, target_language, progress):
        """Translate all text segments in a single batch request"""
        progress(0.55, desc="Translating segments")
        try:
            # Filter out None segments (pauses)
            text_segments = [seg for seg in segments if seg is not None]
            if not text_segments:
                return segments  # Return original if no text to translate
            print(f"Translating {len(text_segments)} segments in batch...")
            # Prepare the prompt with clear formatting instructions
            prompt = f"""Translate the following text segments to {target_language} while maintaining EXACTLY the same format and order:
            {chr(10).join(text_segments)}
            IMPORTANT INSTRUCTIONS:
            1. Maintain the EXACT same order and number of segments
            2. Each line must be a separate translation
            3. Use natural conversational {target_language}
            4. Preserve meaning/context
            5. Leave proper nouns unchanged
            6.Make sure the translated sentence is meaningful also
            7. Match original word count where possible
            8. Output ONLY the translations, one per line, no numbers or bullet points
            9. Do not add any additional text or explanations
            Example Input:
            Hello world
            How are you?
            Example Output:
            नमस्ते दुनिया
            आप कैसे हैं?
            """
            completion = self.client.chat.completions.create(
                model="gpt-3.5-turbo",
                messages=[
                    {
                        "role": "system",
                        "content": f"You are a professional translator from English to {target_language}. Translate exactly as requested."
                    },
                    {
                        "role": "user",
                        "content": prompt
                    }
                ],
                temperature=0.1,  # Lower temperature for more consistent results
                max_tokens=2000
            )
            translated_text = completion.choices[0].message.content.strip()
            translations = translated_text.split('\n')
            # Reconstruct the segments with translations
            translated_segments = []
            translation_idx = 0
            for seg in segments:
                if seg is None:
                    translated_segments.append(None)
                else:
                    if translation_idx < len(translations):
                        translated_segments.append(translations[translation_idx])
                        translation_idx += 1
                    else:
                        translated_segments.append(seg)  # Fallback to original if missing translation
            progress(0.7, desc="Translation complete")
            return translated_segments
        except Exception as e:
            print(f"Batch translation error: {e}")
            return segments  # Return original segments if translation fails

# --- Helper functions ---
def get_audio_duration(audio_path):
    try:
        with sf.SoundFile(audio_path) as f:
            return len(f) / f.samplerate
    except Exception as e:
        print(f"Duration error: {e}")
        return None

async def synthesize_tts_to_wav(text, voice, target_language):
    import edge_tts
    temp_mp3 = "temp_tts.mp3"
    communicate = edge_tts.Communicate(text, voice)
    await communicate.save(temp_mp3)
    audio = AudioSegment.from_file(temp_mp3)
    audio = audio.set_channels(1).set_frame_rate(22050)
    output_wav = "temp_tts.wav"
    audio.export(output_wav, format="wav")
    os.remove(temp_mp3)
    return output_wav

def stretch_audio(input_wav, target_duration, api_url="https://sox-api.onrender.com/stretch"):
    # Read the input audio file
    with open(input_wav, "rb") as f:
        files = {"file": f}
        data = {"target_duration": str(target_duration)}
        response = requests.post(api_url, files=files, data=data)
    # Check if the request was successful
    if response.status_code != 200:
        raise RuntimeError(f"API error: {response.status_code} - {response.text}")
    # Save the response content to a temporary file
    output_wav = tempfile.mkstemp(suffix=".wav")[1]
    with open(output_wav, "wb") as out:
        out.write(response.content)
    return output_wav

def generate_silence_wav(duration_s, output_path, sample_rate=22050):
    samples = np.zeros(int(duration_s * sample_rate), dtype=np.float32)
    sf.write(output_path, samples, sample_rate)

def cleanup_files(file_list):
    for file in file_list:
        if os.path.exists(file):
            os.remove(file)

# --- Main Process Function ---
async def process_audio_chunks(input_audio_path, voice, target_language, progress):
    audio_processor = AudioProcessor()
    print("🔎 Separating vocals and music using Demucs...")
    vocals_path, background_path, temp_dir = separate_vocals(input_audio_path, progress)
    if not vocals_path:
        return None, None

    print("🔎 Transcribing vocals...")
    segments = audio_processor.transcribe_audio_with_pauses(vocals_path, progress)
    print(f"Transcribed {len(segments)} segments.")

    # Extract text segments for batch processing
    segment_texts = [seg[2] if seg[2] is not None else None for seg in segments]

    # Batch translate all segments at once
    translated_texts = audio_processor.translate_segments_batch(segment_texts, target_language, progress)

    chunk_files = []
    chunk_idx = 0
    total_segments = len(segments)
    for (start, end, _), translated in zip(segments, translated_texts):
        duration = end - start
        chunk_idx += 1
        progress(0.7 + (chunk_idx / total_segments) * 0.15, desc=f"Processing chunk {chunk_idx}/{total_segments}")
        if translated is None:
            filename = f"chunk_{chunk_idx:03d}_pause.wav"
            generate_silence_wav(duration, filename)
            chunk_files.append(filename)
        else:
            print(f"🔤 {chunk_idx}: Translated: {translated}")
            # Synthesize TTS audio
            raw_tts = await synthesize_tts_to_wav(translated, voice, target_language)
            # Stretch the audio to match the target duration
            stretched = stretch_audio(raw_tts, duration)
            chunk_files.append(stretched)
            os.remove(raw_tts)

    combined_tts = AudioSegment.empty()
    for f in chunk_files:
        combined_tts += AudioSegment.from_wav(f)

    print("🎼 Adding original background music...")
    background_music = AudioSegment.from_wav(background_path)
    background_music = background_music[:len(combined_tts)]
    final_mix = combined_tts.overlay(background_music)
    output_path = "final_translated_with_music.wav"
    final_mix.export(output_path, format="wav")
    print(f"✅ Output saved as: {output_path}")

    final_audio_path = output_path
    final_background_path = background_path # Keep this for cleanup if needed

    cleanup_files(chunk_files)
    shutil.rmtree(temp_dir, ignore_errors=True)
    progress(0.9, desc="Audio processing complete")
    return final_audio_path, final_background_path

# --- Gradio Interface ---
def gradio_interface(video_file, voice, target_language, progress=gr.Progress()):
    try:
        progress(0.05, desc="Starting video dubbing process")
        # Create temporary directory for processing
        temp_dir = Path(tempfile.mkdtemp())
        input_video_path = temp_dir / "input_video.mp4"
        # Check if file is a video
        if not os.path.splitext(video_file.name)[1].lower() in ['.mp4', '.mov', '.avi', '.mkv']:
            raise ValueError("Invalid file type. Please upload a video file.")
        # Save the uploaded file to the temporary directory
        shutil.copyfile(video_file.name, input_video_path)

        # Extract audio from video
        progress(0.1, desc="Extracting audio from video")
        audio_path, audio_temp_dir = extract_audio_from_video(str(input_video_path))
        if not audio_path:
            return None

        # Process audio chunks
        audio_output_path, background_path = asyncio.run(process_audio_chunks(audio_path, voice, target_language, progress))
        if audio_output_path is None or background_path is None:
            return None

        # Combine with original video
        progress(0.95, desc="Combining video and new audio")
        output_video_path = temp_dir / "translated_video.mp4"
        success = combine_video_audio(str(input_video_path), audio_output_path, str(output_video_path))
        if success:
            progress(1.0, desc="Dubbing complete!")
            # Return the path to the output video
            return str(output_video_path)
        else:
            return None
    except Exception as e:
        print(f"Error processing video: {e}")
        return None
    finally:
        # Cleanup temporary files
        # Commented out for debugging purposes
        # shutil.rmtree(temp_dir, ignore_errors=True)
        pass

def extract_audio_from_video(video_path):
    """Extract audio from video file using ffmpeg"""
    temp_dir = tempfile.mkdtemp()
    audio_path = os.path.join(temp_dir, "extracted_audio.wav")
    try:
        subprocess.run([
            "ffmpeg", "-y", "-i", video_path,
            "-vn", "-acodec", "pcm_s16le", "-ar", "44100", "-ac", "2",
            audio_path
        ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if not os.path.exists(audio_path):
            raise FileNotFoundError("Audio extraction failed")
        return audio_path, temp_dir
    except Exception as e:
        print(f"Audio extraction error: {e}")
        shutil.rmtree(temp_dir, ignore_errors=True)
        return None, None

def combine_video_audio(video_path, audio_path, output_path):
    """Combine original video with new audio track"""
    try:
        subprocess.run([
            "ffmpeg", "-y", "-i", video_path,
            "-i", audio_path,
            "-c:v", "copy", "-map", "0:v:0", "-map", "1:a:0",
            "-shortest", output_path
        ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        return True
    except Exception as e:
        print(f"Video combining error: {e}")
        return False

# Voice options for each language
voice_options = {
    "Hindi": [
        "hi-IN-MadhurNeural",   # Male
        "hi-IN-SwaraNeural"     # Female
    ],
    "English": [
        "en-US-GuyNeural",    # Male
        "en-US-ChristopherNeural",  # Male
        "en-US-AriaNeural",     # Female
        "en-US-JessaNeural",    # Female
        "en-US-JennyNeural"     # Female
    ],
    "Spanish": [
        "es-ES-AlvaroNeural",   # Male
        "es-MX-JorgeNeural",    # Male
        "es-US-AlonsoNeural",   # Female
        "es-MX-DaliaNeural",    # Female
        "es-US-PalomaNeural"    # Female
    ],
    "French": [
        "fr-FR-HenriNeural",    # Male
        "fr-FR-RemyMultilingualNeural", # Male
        "fr-CA-AntoineNeural",  # Male
        "fr-FR-DeniseNeural",
        "fr-FR-VivienneMultilingualNeural"  # Female
    ],
    "Japanese": [
        "ja-JP-KeitaNeural", 
        "ja-JP-NanamiNeural"
    ],
    "Korean": [
        "ko-KR-InJoonNeural",   # Male
         "ko-KR-SunHiNeural" # Female
    ]}

custom_css = """
/* Overall Body Background - Deep & Vibrant Gradient */
body {
    background: linear-gradient(135deg, #1A202C, #2D3748, #4A5568) !important; /* Dark blue-grey gradient */
    font-family: 'Inter', sans-serif; /* Modern font, ensure it's available or use fallback */
    color: #E2E8F0; /* Light text color for contrast */
    overflow-x: hidden;
}
/* --- Core Gradio Block Blending --- */
/* Make Gradio's main container transparent to show body background */
.gradio-container {
    background: transparent !important;
    box-shadow: none !important;
    border: none !important;
    padding: 0 !important;
}
/* Specific Gradio block elements - subtle transparency */
.block {
    background-color: hsla(210, 20%, 25%, 0.5) !important; /* Semi-transparent dark blue-grey */
    backdrop-filter: blur(8px); /* Frosted glass effect */
    border: 1px solid hsla(210, 20%, 35%, 0.6) !important; /* Subtle border */
    border-radius: 20px !important; /* Rounded corners for the block */
    box-shadow: 0 8px 30px hsla(0, 0%, 0%, 0.3) !important; /* Stronger shadow for depth */
    margin-bottom: 25px !important;
    padding: 25px !important; /* Add internal padding to blocks */
}
/* Remove default Gradio layout wrappers' backgrounds */
.main-wrapper, .panel-container {
    background: transparent !important;
    box-shadow: none !important;
    border: none !important;
}
/* --- Application Title and Description --- */
.gradio-header h1 {
    color: #8D5BFC !important; /* Vibrant Purple for main title */
    font-size: 3em !important;
    text-shadow: 0 0 15px hsla(260, 90%, 70%, 0.5); /* Glowing effect */
    margin-bottom: 10px !important;
    font-weight: 700 !important;
    text-align: center;
}
.gradio-markdown p {
    color: #CBD5E0 !important; /* Lighter text for description */
    font-size: 1.25em !important;
    text-align: center;
    margin-bottom: 40px !important;
    font-weight: 300;
}
/* --- Input Components (File, Dropdowns) --- */
.gradio-file, .gradio-dropdown {
    background-color: hsla(210, 20%, 18%, 0.7) !important; /* Darker, slightly transparent */
    border: 1px solid hsla(240, 60%, 70%, 0.4) !important; /* Subtle blue border */
    border-radius: 15px !important;
    padding: 12px 18px !important;
    color: #E2E8F0 !important; /* Light text for input */
    font-size: 1.1em !important;
    transition: all 0.3s ease;
    box-shadow: 0 4px 15px hsla(0, 0%, 0%, 0.2);
}
.gradio-file input[type="file"] {
    color: #E2E8F0 !important;
}
.gradio-file:hover, .gradio-dropdown:hover {
    border-color: #A78BFA !important; /* Lighter purple on hover */
    box-shadow: 0 6px 20px hsla(0, 0%, 0%, 0.3);
}
/* Focus state for inputs */
.gradio-dropdown.gr-text-input:focus,
.gradio-file input:focus {
    border-color: #8D5BFC !important; /* Vibrant purple on focus */
    box-shadow: 0 0 20px hsla(260, 90%, 70%, 0.5);
    background-color: hsla(210, 20%, 20%, 0.9) !important; /* Slightly less transparent */
}
/* Labels for inputs */
.gradio-label {
    color: #A78BFA !important; /* Soft purple for labels */
    font-weight: 600 !important;
    font-size: 1.15em !important;
    margin-bottom: 8px !important;
    text-align: left;
    width: 100%;
}
/* --- Submit Button --- */
.gradio-button {
    background: linear-gradient(90deg, #FF6B8B, #FF8E53) !important; /* Vibrant Pink to Orange gradient */
    color: white !important;
    border: none !important;
    border-radius: 30px !important;
    padding: 15px 35px !important;
    font-size: 1.3em !important;
    font-weight: bold !important;
    cursor: pointer !important;
    transition: all 0.3s ease !important;
    box-shadow: 0 8px 25px hsla(0, 0%, 0%, 0.4) !important;
    margin-top: 35px !important;
    min-width: 220px;
    align-self: center;
    text-transform: uppercase; /* Make button text uppercase */
    letter-spacing: 1px;
}
.gradio-button:hover {
    background: linear-gradient(90deg, #FF4B7B, #FF7E43) !important;
    box-shadow: 0 10px 30px hsla(0, 0%, 0%, 0.5) !important;
    transform: translateY(-3px) !important;
}
/* --- Output Video Player --- */
.gradio-video {
    background-color: hsla(210, 20%, 15%, 0.8) !important; /* Darker, more opaque background for video */
    border: 2px solid #8D5BFC !important; /* Vibrant purple border for the video player */
    border-radius: 20px !important;
    padding: 15px !important;
    box-shadow: 0 10px 40px hsla(0, 0%, 0%, 0.5) !important; /* Stronger shadow */
    margin-top: 40px !important;
}
/* --- Translated Text Output --- */
.gradio-markdown-output, .gradio-textbox {
    background-color: hsla(210, 20%, 18%, 0.7) !important;
    border: 1px solid hsla(240, 60%, 70%, 0.4) !important;
    border-radius: 15px !important;
    padding: 20px !important;
    color: #E2E8F0 !important;
    font-size: 1.0em !important;
    min-height: 200px; /* Give it some height */
    overflow-y: auto; /* Enable scrolling for long text */
    white-space: pre-wrap; /* Preserve line breaks */
    box-shadow: 0 4px 15px hsla(0, 0%, 0%, 0.2);
}
/* Flexbox for the Row to control spacing and alignment */
.gradio-row {
    display: flex;
    justify-content: space-around; /* Distribute items with space around */
    align-items: flex-start; /* Align items to the start of the cross-axis */
    gap: 20px; /* Space between items in the row */
    flex-wrap: wrap; /* Allow items to wrap on smaller screens */
}
/* Ensure individual components in a row take up appropriate space */
.gradio-row > .gradio-component {
    flex: 1; /* Allow components to grow and shrink */
    min-width: 250px; /* Minimum width for components in a row */
}
/* Adjust padding for gr.Blocks content */
.gr-box {
    padding: 0 !important; /* Remove internal padding if present to let elements breathe */
    background: transparent !important;
    box-shadow: none !important;
}
"""
# Create Gradio interface with radio buttons for both language and voice selection
with gr.Blocks(css=custom_css, theme=gr.themes.Soft(
    primary_hue=gr.themes.Color(
        c50='#e6e9ff', c100='#c2c9ff', c200='#9faaff', c300='#7c8bff', c400='#5a6bff',
        c500='#384aff', c600='#2c38cc', c700='#202b99', c800='#141d66', c900='#080e33',
        c950='#04071a'
    ),
    secondary_hue=gr.themes.Color(
        c50='#fff0e6', c100='#ffe0cc', c200='#ffb380', c300='#ff8533', c400='#ff5700',
        c500='#cc4600', c600='#993400', c700='#662200', c800='#331100', c900='#1a0900',
        c950='#0d0500'
    ),
    neutral_hue=gr.themes.Color(
        c50='#f8f8fa', c100='#f1f5f9', c200='#e2e8f0', c300='#cbd5e1', c400='#94a3b8',
        c500='#64748b', c600='#475569', c700='#334155', c800='#1e293b', c900='#0f172a',
        c950='#020617'
    )
)) as demo:
    gr.Markdown("# DeepDub : A Video Dubbing Application")
    gr.Markdown("Upload a video and get a dubbed version with translated audio")
    

    with gr.Row():
        video_input = gr.File(label="Upload Video", file_types=[".mp4", ".mov", ".avi", ".mkv"])
        
        # Use Radio buttons for language selection
        language_radio = gr.Radio(
            list(voice_options.keys()),
            label="Target Language",
            value="Hindi",
            interactive=True
        )
        
        # Use Radio buttons for voice selection
        voice_radio = gr.Radio(
            voice_options["Hindi"],
            label="Select Voice",
            value=voice_options["Hindi"][0],
            interactive=True
        )
    gr.Markdown("Note : If you see Queue that means someone is using and please wait")
    output_video = gr.Video(label="Dubbed Video")
    submit_btn = gr.Button("Start Dubbing")

    def update_voice_options(language):
        # Update voice radio buttons based on selected language
        return gr.update(choices=voice_options[language], value=voice_options[language][0])

    # Update voice options when language changes
    language_radio.change(
        update_voice_options,
        inputs=[language_radio],
        outputs=[voice_radio]
    )

    submit_btn.click(
        gradio_interface,
        inputs=[video_input, voice_radio, language_radio],
        outputs=output_video,
        api_name="dub_video"
    )

demo.queue().launch(server_name="0.0.0.0", debug=True, share=True)