Update app.py

app.py

@@ -1,6 +1,7 @@
 import numpy as np
 import cvxpy as cp
 import re
+import copy
 import concurrent.futures
 import gradio as gr
 from datetime import datetime
@@ -556,7 +557,7 @@ def process_entry(entry, i, tts_model, video_width, video_height, process_mode,
         try:
             segment_audio_path = f"segment_{i}_voiceover.wav"
             desired_duration = entry["end"] - entry["start"]
-            desired_speed = calibrated_speed(entry['translated'], desired_duration)
+            desired_speed = entry['speed'] #calibrated_speed(entry['translated'], desired_duration)
 
             speaker = entry.get("speaker", "default")
             speaker_wav_path = f"speaker_{speaker}_sample.wav"
@@ -608,7 +609,7 @@ def add_transcript_voiceover(video_path, translated_json, output_path, process_m
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = [executor.submit(process_entry, entry, i, tts_model, video.w, video.h, process_mode, target_language, font_path, speaker_sample_paths)
-                   for i, entry in enumerate(translated_json)]
+                   for i, entry in enumerate(translated_json_withspeed)]
 
         results = []
         for future in concurrent.futures.as_completed(futures):
@@ -715,6 +716,84 @@ def generate_voiceover_clone(full_text, tts_model, desired_speed, target_languag
         logger.error(traceback.format_exc())
         return None, err_msg, err_msg
 
+def apply_adaptive_speed(translated_json_raw, source_language, target_language, k=3.0, default_prior_speed=5.0):
+    """
+    Adds a `speed` (chars/sec) and `target_duration` (sec) field to each segment
+    using shrinkage-based estimation and language stretch ratios.
+    Optionally modulates based on tone or style tags (e.g., "dramatic", "urgent").
+    """
+    translated_json = copy.deepcopy(translated_json_raw)
+
+    # Prior average speech speeds by (category, target language)
+    priors = {
+        ("drama", "en"): 5.0,
+        ("drama", "zh"): 4.5,
+        ("tutorial", "en"): 5.2,
+        ("tutorial", "zh"): 4.8,
+        ("shortplay", "en"): 5.1,
+        ("shortplay", "zh"): 4.7,
+    }
+
+    # Adjustment ratio based on language pair (source → target)
+    lang_ratio = {
+        ("zh", "en"): 0.85,
+        ("en", "zh"): 1.15,
+        ("zh", "jp"): 1.05,
+        ("en", "ja"): 0.9,
+    }
+
+    # Optional style modulation factor
+    style_modifiers = {
+        "dramatic": 0.9,   # slower
+        "urgent": 1.1,     # faster
+        "neutral": 1.0
+    }
+
+    for idx, entry in enumerate(translated_json):
+        start, end = float(entry.get("start", 0)), float(entry.get("end", 0))
+        duration = max(0.1, end - start)
+
+        original_text = entry.get("original", "")
+        translated_text = entry.get("translated", "")
+        category = entry.get("category", "drama")
+        source_lang = source_language
+        target_lang = target_language
+        style = entry.get("style", "neutral")  # Optional field like "dramatic"
+
+        # Observed speed from original
+        base_text = original_text or translated_text
+        obs_speed = len(base_text) / duration
+
+        # Prior speed from category + language
+        prior_speed = priors.get((category, target_lang), default_prior_speed)
+
+        # Shrinkage estimate
+        shrink_speed = (duration * obs_speed + k * prior_speed) / (duration + k)
+
+        # Adjust for language-specific pacing
+        ratio = lang_ratio.get((source_lang, target_lang), 1.0)
+        adjusted_speed = shrink_speed * ratio
+
+        # Optional tone/style modulation (if available)
+        mod = style_modifiers.get(style.lower(), 1.0)
+        adjusted_speed *= mod
+
+        # Final estimated duration for synthesized segment
+        target_chars = len(translated_text)
+        target_duration = round(target_chars / adjusted_speed, 2)
+
+        # Logging for debugging
+        logger.info(
+            f"Segment {idx}: dur={duration:.2f}s, obs={obs_speed:.2f}, "
+            f"prior={prior_speed:.2f}, shrink={shrink_speed:.2f}, "
+            f"final_speed={adjusted_speed:.2f}, target_dur={target_duration:.2f}s"
+        )
+
+        entry["speed"] = round(adjusted_speed, 3)
+        entry["target_duration"] = target_duration
+
+    return translated_json
+
 def calibrated_speed(text, desired_duration):
     """
     Compute a speed factor to help TTS fit audio into desired duration,
@@ -757,14 +836,15 @@ def upload_and_manage(file, target_language, process_mode):
 
         # Step 2: Translate the transcription
         logger.info(f"Translating transcription from {source_language} to {target_language}...")
-        translated_json = translate_text(transcription_json, source_language, target_language)
+        translated_json_raw = translate_text(transcription_json, source_language, target_language)
         logger.info(f"Translation completed. Number of translated segments: {len(translated_json)}")
 
         # translated_json = post_edit_translated_segments(translated_json, file.name)
-
+        translated_json = apply_adaptive_speed(translated_json_raw, source_language, target_language)
+        
         # Step 3: Add transcript to video based on timestamps
         logger.info("Adding translated transcript to video...")
-        add_transcript_voiceover(file.name, translated_json, output_video_path, process_mode, target_language)
+        add_transcript_voiceover(file.name, translated_json_speedcontrol, output_video_path, process_mode, target_language)
         logger.info(f"Transcript added to video. Output video saved at {output_video_path}")
 
         # Convert translated JSON into a format for the editable table