import gradio as gr import torch from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler from transformers import pipeline import numpy as np from PIL import Image, ImageDraw, ImageFont import io import base64 import json import re import time import os from typing import List, Dict, Tuple, Optional import warnings warnings.filterwarnings("ignore") # Global variables for models sd_pipe = None tts_pipe = None def initialize_models(): """Initialize AI models on first use""" global sd_pipe, tts_pipe if sd_pipe is None: print("Loading Stable Diffusion model...") sd_pipe = StableDiffusionPipeline.from_pretrained( "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float32, # Use float32 for CPU safety_checker=None, requires_safety_checker=False ) sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) sd_pipe = sd_pipe.to("cpu") sd_pipe.enable_attention_slicing() if tts_pipe is None: try: print("Loading TTS model...") tts_pipe = pipeline("text-to-speech", model="microsoft/speecht5_tts", device=-1) except: print("TTS model not available, continuing without audio...") tts_pipe = None class StorySegmenter: """Handles story segmentation and prompt enhancement""" @staticmethod def split_story(story: str, max_segments: int = 15) -> List[str]: """Split story into segments suitable for 10-second videos""" # Split by sentences first sentences = re.split(r'[.!?]+', story) sentences = [s.strip() for s in sentences if s.strip()] segments = [] current_segment = "" for sentence in sentences: # If adding this sentence would make segment too long, start new segment if len(current_segment + " " + sentence) > 200 or len(segments) >= max_segments: if current_segment: segments.append(current_segment.strip()) current_segment = sentence else: segments.append(sentence) else: current_segment += (" " + sentence if current_segment else sentence) # Add final segment if current_segment: segments.append(current_segment.strip()) return segments[:max_segments] @staticmethod def enhance_prompt(segment: str, character_name: str, character_traits: str, style: str, scene_context: str = "") -> str: """Enhance segment prompt with character and style information""" enhanced = f"{segment}. " enhanced += f"Character: {character_name} ({character_traits}). " enhanced += f"Style: {style}, high quality, detailed. " if scene_context: enhanced += f"Scene context: {scene_context}. " # Add negative prompts enhanced += "NEGATIVE: blurry, low quality, distorted, bad anatomy" return enhanced class ConsistencyManager: """Manages visual consistency across segments""" def __init__(self, base_seed: int = 42): self.base_seed = base_seed self.character_prompt = "" self.scene_context = "" self.last_scene_elements = [] def get_segment_seed(self, segment_index: int) -> int: """Get consistent seed for segment""" return self.base_seed + segment_index def update_context(self, segment: str): """Update scene context based on current segment""" # Simple context extraction - in production would use NLP if any(word in segment.lower() for word in ['house', 'home', 'room', 'kitchen']): self.scene_context = "indoor domestic setting" elif any(word in segment.lower() for word in ['forest', 'tree', 'nature', 'outdoor']): self.scene_context = "outdoor natural setting" elif any(word in segment.lower() for word in ['city', 'street', 'building']): self.scene_context = "urban setting" class VideoGenerator: """Handles video/image generation for each segment""" def __init__(self): self.consistency_manager = ConsistencyManager() def generate_segment_image(self, enhanced_prompt: str, seed: int, width: int = 512, height: int = 512) -> Image.Image: """Generate image for a story segment""" initialize_models() if sd_pipe is None: # Fallback: create a placeholder image return self.create_placeholder_image(enhanced_prompt, width, height) try: generator = torch.Generator().manual_seed(seed) # Generate image with torch.no_grad(): result = sd_pipe( prompt=enhanced_prompt, negative_prompt="blurry, low quality, distorted, bad anatomy, ugly", num_inference_steps=20, # Reduced for faster generation guidance_scale=7.5, width=width, height=height, generator=generator ) return result.images[0] except Exception as e: print(f"Error generating image: {e}") return self.create_placeholder_image(enhanced_prompt, width, height) def create_placeholder_image(self, prompt: str, width: int, height: int) -> Image.Image: """Create a placeholder image when generation fails""" img = Image.new('RGB', (width, height), color='lightblue') draw = ImageDraw.Draw(img) # Try to load a font, fallback to default if not available try: font = ImageFont.truetype("arial.ttf", 20) except: font = ImageFont.load_default() # Wrap text words = prompt[:100].split() lines = [] current_line = "" for word in words: if len(current_line + word) < 40: current_line += word + " " else: lines.append(current_line) current_line = word + " " if current_line: lines.append(current_line) # Draw text y_offset = height // 2 - (len(lines) * 25) // 2 for line in lines: bbox = draw.textbbox((0, 0), line, font=font) text_width = bbox[2] - bbox[0] x_offset = (width - text_width) // 2 draw.text((x_offset, y_offset), line, fill='black', font=font) y_offset += 25 return img class AudioGenerator: """Handles audio generation for segments""" @staticmethod def generate_segment_audio(text: str, speaker_id: int = 0) -> Optional[bytes]: """Generate audio for a text segment""" initialize_models() if tts_pipe is None: return None try: # Generate audio audio_data = tts_pipe(text) # Convert to bytes (simplified - in production would handle proper audio format) if 'audio' in audio_data: # Convert audio array to bytes representation audio_array = np.array(audio_data['audio']) return audio_array.tobytes() except Exception as e: print(f"Error generating audio: {e}") return None def process_story(story_text: str, character_name: str, character_traits: str, style: str, enable_voiceover: bool, reference_image: Optional[Image.Image] = None, progress=gr.Progress()) -> Tuple[List[Image.Image], List[str], str]: """Main processing function""" if not story_text.strip(): return [], [], "Please enter a story to generate." if not character_name.strip(): character_name = "Main Character" if not character_traits.strip(): character_traits = "detailed character design" # Initialize components segmenter = StorySegmenter() video_gen = VideoGenerator() audio_gen = AudioGenerator() # Step 1: Split story into segments progress(0.1, "Splitting story into segments...") segments = segmenter.split_story(story_text) if not segments: return [], [], "Could not create segments from the story." progress(0.2, f"Created {len(segments)} segments") # Step 2: Generate content for each segment generated_images = [] generated_audio_info = [] for i, segment in enumerate(segments): progress((0.2 + 0.7 * (i / len(segments))), f"Generating segment {i+1}/{len(segments)}") # Update consistency context video_gen.consistency_manager.update_context(segment) # Enhance prompt enhanced_prompt = segmenter.enhance_prompt( segment, character_name, character_traits, style, video_gen.consistency_manager.scene_context ) # Generate image seed = video_gen.consistency_manager.get_segment_seed(i) image = video_gen.generate_segment_image(enhanced_prompt, seed) generated_images.append(image) # Generate audio if enabled if enable_voiceover: audio_bytes = audio_gen.generate_segment_audio(segment) audio_info = f"Audio generated for segment {i+1}" if audio_bytes else f"Audio generation failed for segment {i+1}" else: audio_info = f"No audio (voiceover disabled)" generated_audio_info.append(audio_info) # Small delay to prevent overwhelming the system time.sleep(0.1) progress(1.0, "Generation complete!") # Create summary summary = f""" ## Generation Summary **Story Segments**: {len(segments)} **Character**: {character_name} ({character_traits}) **Style**: {style} **Voiceover**: {'Enabled' if enable_voiceover else 'Disabled'} ### Segments Generated: """ for i, segment in enumerate(segments): summary += f"\n**Segment {i+1}**: {segment[:100]}{'...' if len(segment) > 100 else ''}" return generated_images, generated_audio_info, summary def create_interface(): """Create the Gradio interface""" with gr.Blocks(title="AI Video Story Generator", theme=gr.themes.Soft()) as interface: gr.Markdown(""" # 🎬 AI Video Story Generator Generate animated story segments with consistent characters and scenes. Running on free CPU tier - generates 10-15 segments of ~10 seconds each. """) with gr.Row(): with gr.Column(scale=2): story_input = gr.Textbox( label="📖 Story Text", placeholder="Enter your story here (up to 50,000 characters)...", lines=10, max_lines=20 ) with gr.Row(): character_name = gr.Textbox( label="👤 Character Name", placeholder="e.g., Alice, Hero, The Detective", value="Main Character" ) character_traits = gr.Textbox( label="✨ Character Traits", placeholder="e.g., young woman, brown hair, blue dress", value="detailed character design" ) with gr.Row(): style = gr.Dropdown( label="🎨 Visual Style", choices=[ "anime style", "realistic", "cartoon style", "fantasy art", "sci-fi concept art", "children's book illustration", "comic book style" ], value="anime style" ) enable_voiceover = gr.Checkbox( label="🔊 Enable Voiceover", value=False ) reference_image = gr.Image( label="🖼️ Reference Image (Optional)", type="pil" ) with gr.Column(scale=1): generate_btn = gr.Button( "🎬 Generate Story Video", variant="primary", size="lg" ) gr.Markdown(""" ### 📋 Instructions: 1. Enter your story text 2. Define your main character 3. Choose visual style 4. Enable voiceover if desired 5. Click Generate! ### ⚡ Free Tier Limits: - Max 15 segments (~3 minutes) - CPU processing (slower) - Sequential generation """) # Results section with gr.Row(): summary_output = gr.Markdown(label="📊 Generation Summary") # Generated content gallery with gr.Row(): generated_gallery = gr.Gallery( label="🖼️ Generated Segments", show_label=True, elem_id="gallery", columns=3, rows=2, height="auto" ) with gr.Row(): audio_info = gr.Textbox( label="🔊 Audio Information", lines=5, interactive=False ) # Download section gr.Markdown(""" ### 📥 Download Instructions: 1. Right-click on any image to save individual segments 2. Use external tools to combine segments into final video 3. Audio files (if generated) can be downloaded separately ### 🔧 Recommended Tools for Combining: - **Free**: OpenShot, DaVinci Resolve, Blender - **Online**: Kapwing, Canva Video Editor - **Command Line**: FFmpeg """) # Event handlers generate_btn.click( fn=process_story, inputs=[ story_input, character_name, character_traits, style, enable_voiceover, reference_image ], outputs=[ generated_gallery, audio_info, summary_output ], show_progress=True ) # Example stories gr.Markdown(""" ### 📚 Example Stories to Try: **Fantasy Adventure**: "A brave knight discovers a magical forest where the trees whisper ancient secrets. She meets a wise dragon who offers to teach her the old magic. Together they must stop an evil sorcerer from destroying the realm." **Sci-Fi Mystery**: "On a space station orbiting Mars, Detective Chen investigates strange disappearances. The security cameras show nothing, but she notices the artificial gravity fluctuating. Her investigation leads to a discovery that changes everything." **Children's Tale**: "Little Bear couldn't sleep because of the thunder. He decided to visit his friend Owl, who lived in the big oak tree. Owl taught him that storms bring rain for flowers, and showed him how lightning dances across the sky." """) return interface # Create and launch the interface if __name__ == "__main__": interface = create_interface() interface.launch( share=True, server_name="0.0.0.0", server_port=7860, show_error=True )