app.py

import gradio as gr
import requests
from PIL import Image
from transformers import BlipProcessor, BlipForConditionalGeneration
import torch
from diffusers import AnimateDiffPipeline, LCMScheduler, MotionAdapter
from moviepy.editor import VideoFileClip, AudioFileClip, concatenate_videoclips
from transformers import AutoProcessor, MusicgenForConditionalGeneration
import scipy.io.wavfile
import re
from io import BytesIO

# 定义图像到文本函数
def img2text(image):
    processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
    model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
    inputs = processor(image, return_tensors="pt")
    out = model.generate(**inputs)
    caption = processor.decode(out[0], skip_special_tokens=True)
    return caption

# 定义文本生成函数
def text2text(user_input):
    api_key = "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"
    base_url = "https://openrouter.ai/api/v1"
    
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }
    
    data = {
        "model": "openai/gpt-3.5-turbo",
        "messages": [
            {
                "role": "system", 
                "content": (
                    "You are an expert who is very good at writing stories. Please expand it into a continuous story based on the input, and logically cut the story into sentences. Each sentence is a scene (as many sentences and scenes as possible, and at least 10 sentences). Each sentence is required The content of the sentence description should be detailed and do not use rhetorical techniques, and no ambiguous words such as pronouns should appear in the sentence. Be as detailed as possible to accurately describe who is doing what, and the scene descriptions before and after should have a certain correlation. In addition, I require your answer to follow a certain format. Let me give you an example. For example, I enter: a dolphin jumping out of the water at sunset. "
                    "Your answer format: "
                    """
                    [1] The sun nears the horizon, illuminating the calm sea surface with a warm glow. 
                    [2] A dolphin swims swiftly below the calm sea surface, moving closer to the top. 
                    [3] The dolphin uses its powerful tail fin to prepare for a leap out of the water. 
                    [4] The dolphin's body starts to emerge from the water, exposing itself above the surface. 
                    [5] The dolphin leaps completely out of the water, creating an arch with its body in the air. 
                    [6] The dolphin rotates its body in the air before it begins its descent back to the water. 
                    [7] The dolphin's head and back make the first contact with the water, creating a splash. 
                    [8] The dolphin fully submerges under the water, causing the splashes around it to slowly disperse. 
                    [9] The dolphin moves forward underwater, gradually disappearing into the dimming light of the sunset. 
                    """
                    "My input is as follows, please answer me in the format without adding any other words."
                )
            },
            { "role": "user", "content": user_input }
        ]
    }

    response = requests.post(f"{base_url}/chat/completions", headers=headers, json=data)
    response.raise_for_status()
    completion = response.json()
    return completion['choices'][0]['message']['content']

# 定义文本到视频函数
def text2vid(input_text):
    sentences = re.findall(r'\[\d+\] (.+?)(?:\n|\Z)', input_text)
    adapter = MotionAdapter.from_pretrained("your-motion-adapter")
    pipe = AnimateDiffPipeline.from_pretrained("your-diffusion-model", motion_adapter=adapter)
    video_clips = []
    for sentence in sentences:
        frames = pipe(sentence, num_inference_steps=50, guidance_scale=7.5)
        video_clip = frames_to_video_clip(frames)  # Assume this function converts frames to a video clip
        video_clips.append(video_clip)
    final_clip = concatenate_videoclips(video_clips, method="compose")
    return final_clip

def text2text_A(user_input):
    # 设置API密钥和基础URL
    api_key = "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"
    base_url = "https://openrouter.ai/api/v1"
    
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }
    
    data = {
        "model": "openai/gpt-3.5-turbo",
        "messages": [
            {
                "role": "system", 
                "content": (
                    "You are an expert in music criticism, please match this story with a suitable musical style based on my input and describe it, please make sure you follow my format output and do not add any other statements e.g. Input: in a small tavern everyone danced, the bartender poured drinks for everyone, everyone had a good time and was very happy and sang and danced. Output: 80s pop track with bassy drums and synth."
                    "Again, please make sure you follow the format of the output, here is my input:"

                )
            },
            { "role": "user", "content": user_input }
        ]
    }

    response = requests.post(f"{base_url}/chat/completions", headers=headers, json=data)
    response.raise_for_status()  # 确保请求成功

    completion = response.json()
    print(completion['choices'][0]['message']['content'])
    return completion['choices'][0]['message']['content']
    
# 定义文本到音频函数
def text2audio(text_input, duration_seconds):
    processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
    model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
    inputs = processor(text=[text_input], padding=True, return_tensors="pt")
    max_new_tokens = int((duration_seconds / 5) * 256)
    audio_values = model.generate(**inputs, max_new_tokens=max_new_tokens)
    audio_clip = numpy_array_to_audio_clip(audio_values.numpy(), rate=model.config.audio_encoder.sampling_rate)  # Assume this function converts numpy array to audio clip
    return audio_clip

# 定义生成结果视频的函数
def result_generate(video_clip, audio_clip):
    video = video_clip.set_audio(audio_clip)
    video_buffer = BytesIO()
    video.write_videofile(video_buffer, codec="libx264", audio_codec="aac")
    video_buffer.seek(0)
    return video_buffer

# 整合所有步骤到主函数
def generate_video(image):
    text = img2text(image)
    sentences = text2text(text)
    final_video_clip = text2vid(sentences)
    video = VideoFileClip(final_video_clip)  # Assumes final_video_clip is a path or BytesIO object
    duration = video.duration
    audio_text = text2text_A(text)
    audio_clip = text2audio(audio_text, duration)
    result_video = result_generate(final_video_clip, audio_clip)
    return result_video

# 定义 Gradio 接口
interface = gr.Interface(
    fn=lambda img: generate_video(img),
    inputs=gr.Image(type="pil"),
    outputs=gr.Video(),
    title="InspiroV Video Generation",
    description="Upload an image to generate a video",
    theme="soft"
)

# 启动 Gradio 应用
interface.launch()