Update app.py

app.py

@@ -4,13 +4,10 @@ from PIL import Image
 from transformers import BlipProcessor, BlipForConditionalGeneration
 import torch
 from diffusers import AnimateDiffPipeline, LCMScheduler, MotionAdapter
-from diffusers.utils import export_to_gif
 from moviepy.editor import VideoFileClip, AudioFileClip, concatenate_videoclips
 from transformers import AutoProcessor, MusicgenForConditionalGeneration
 import scipy.io.wavfile
 import re
-import glob
-import os
 from io import BytesIO
 
 # 定义图像到文本函数
@@ -63,83 +60,82 @@ def text2text(user_input):
     completion = response.json()
     return completion['choices'][0]['message']['content']
 
+# 定义文本到视频函数
 def text2vid(input_text):
     sentences = re.findall(r'\[\d+\] (.+?)(?:\n|\Z)', input_text)
-    adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM", config_file="wangfuyun/AnimateLCM/config.json", torch_dtype=torch.float16)
-    pipe = AnimateDiffPipeline.from_pretrained("emilianJR/epiCRealism", motion_adapter=adapter, torch_dtype=torch.float16)
-    pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config, beta_schedule="linear")
-    pipe.load_lora_weights("wangfuyun/AnimateLCM", weight_name="AnimateLCM_sd15_t2v_lora.safetensors", adapter_name="lcm-lora")
-    try:
-        pipe.set_adapters(["lcm-lora"], [0.8])
-    except ValueError as e:
-        print("Ignoring the error:", str(e))
-    pipe.enable_vae_slicing()
-    pipe.enable_model_cpu_offload()
+    adapter = MotionAdapter.from_pretrained("your-motion-adapter")
+    pipe = AnimateDiffPipeline.from_pretrained("your-diffusion-model", motion_adapter=adapter)
     video_clips = []
     for sentence in sentences:
-        output = pipe(
-            prompt=sentence + ", 4k, high resolution",
-            negative_prompt="bad quality, worse quality, low resolution",
-            num_frames=24,
-            guidance_scale=2.0,
-            num_inference_steps=6,
-            generator=torch.Generator("cpu").manual_seed(0)
-        )
-        frames = output.frames[0]
-        video_clip = frames_to_video_clip(frames)
+        frames = pipe(sentence, num_inference_steps=50, guidance_scale=7.5)
+        video_clip = frames_to_video_clip(frames)  # Assume this function converts frames to a video clip
         video_clips.append(video_clip)
     final_clip = concatenate_videoclips(video_clips, method="compose")
     return final_clip
 
-# 定义生成最终视频的函数
-def video_generate():
-    frame_rate = 24
-    gif_files = sorted(glob.glob('./*.gif'))
-    clips = [VideoFileClip(gif) for gif in gif_files]
-    final_clip = concatenate_videoclips(clips, method="compose")
-    final_clip.write_videofile('output_video.mp4', codec='libx264')
+def text2text_A(user_input):
+    # 设置API密钥和基础URL
+    api_key = "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"
+    base_url = "https://openrouter.ai/api/v1"
+    
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+    
+    data = {
+        "model": "openai/gpt-3.5-turbo",
+        "messages": [
+            {
+                "role": "system", 
+                "content": (
+                    "You are an expert in music criticism, please match this story with a suitable musical style based on my input and describe it, please make sure you follow my format output and do not add any other statements e.g. Input: in a small tavern everyone danced, the bartender poured drinks for everyone, everyone had a good time and was very happy and sang and danced. Output: 80s pop track with bassy drums and synth."
+                    "Again, please make sure you follow the format of the output, here is my input:"
 
-# 修改音频生成函数
+                )
+            },
+            { "role": "user", "content": user_input }
+        ]
+    }
+
+    response = requests.post(f"{base_url}/chat/completions", headers=headers, json=data)
+    response.raise_for_status()  # 确保请求成功
+
+    completion = response.json()
+    print(completion['choices'][0]['message']['content'])
+    return completion['choices'][0]['message']['content']
+    
+# 定义文本到音频函数
 def text2audio(text_input, duration_seconds):
     processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
     model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
     inputs = processor(text=[text_input], padding=True, return_tensors="pt")
     max_new_tokens = int((duration_seconds / 5) * 256)
     audio_values = model.generate(**inputs, max_new_tokens=max_new_tokens)
-    audio_array = audio_values[0, 0].numpy()
-    audio_clip = numpy_array_to_audio_clip(audio_array, rate=model.config.audio_encoder.sampling_rate)
+    audio_clip = numpy_array_to_audio_clip(audio_values.numpy(), rate=model.config.audio_encoder.sampling_rate)  # Assume this function converts numpy array to audio clip
     return audio_clip
 
-# 修改最终视频生成函数
+# 定义生成结果视频的函数
 def result_generate(video_clip, audio_clip):
     video = video_clip.set_audio(audio_clip)
-    video_bytes = video_clip_to_bytes(video)
-    return video_bytes
-
+    video_buffer = BytesIO()
+    video.write_videofile(video_buffer, codec="libx264", audio_codec="aac")
+    video_buffer.seek(0)
+    return video_buffer
 
-# 主函数，结合上述修改
+# 整合所有步骤到主函数
 def generate_video(image):
     text = img2text(image)
     sentences = text2text(text)
     final_video_clip = text2vid(sentences)
-    video = VideoFileClip(final_video_clip)
+    video = VideoFileClip(final_video_clip)  # Assumes final_video_clip is a path or BytesIO object
     duration = video.duration
-    audio_text = text2text(text)
+    audio_text = text2text_A(text)
     audio_clip = text2audio(audio_text, duration)
     result_video = result_generate(final_video_clip, audio_clip)
     return result_video
 
 # 定义 Gradio 接口
-# interface = gr.Interface(
-#     fn=generate_video,
-#     inputs=gr.Image(type="pil"),
-#     outputs=gr.Video(),
-#     title="InspiroV Video Generation",
-#     description="Upload an image to generate a video using a custom model",
-#     theme="soft"
-# )
-
-
 interface = gr.Interface(
     fn=lambda img: generate_video(img),
     inputs=gr.Image(type="pil"),
@@ -149,6 +145,5 @@ interface = gr.Interface(
     theme="soft"
 )
 
-
 # 启动 Gradio 应用
 interface.launch()