Update app.py

app.py

@@ -7,106 +7,144 @@ from transformers import (
     AutoTokenizer, 
     AutoModelForCausalLM
 )
-from modelscope.pipelines import pipeline as ms_pipeline
 from PIL import Image
+import os
+import logging
 
-def load_models():
-    # Chargement des modèles
-    blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
-    blip_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
-    
-    # Modèle de transcription audio
-    audio_transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-small")
-    
-    # Modèle de génération de texte (version gratuite GPT-2)
-    text_generator = pipeline("text-generation", model="gpt2")
-    
-    return blip_processor, blip_model, audio_transcriber, text_generator
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-def analyze_image(image, blip_processor, blip_model):
-    # Questions pour l'analyse d'image
-    questions = [
-        "What is in the picture?",
-        "What are the main colors?",
-        "What is the setting or background?",
-        "What is happening in the image?",
-    ]
-    
-    responses = {}
-    for question in questions:
-        inputs = blip_processor(images=image, text=question, return_tensors="pt")
-        outputs = blip_model.generate(**inputs)
-        answer = blip_processor.decode(outputs[0], skip_special_tokens=True)
-        responses[question] = answer
-    
-    description = f"This image shows {responses['What is in the picture?']}. "
-    description += f"The main colors are {responses['What are the main colors?']}. "
-    description += f"The setting is {responses['What is the setting or background?']}. "
-    description += f"In the scene, {responses['What is happening in the image?']}."
-    
-    return description
+class MultimodalProcessor:
+    def __init__(self):
+        self.load_models()
+    
+    def load_models(self):
+        """Charge les modèles avec gestion d'erreurs"""
+        try:
+            logger.info("Chargement des modèles...")
+            self.blip_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+            self.blip_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
+            self.audio_transcriber = pipeline("automatic-speech-recognition", 
+                                           model="openai/whisper-small")
+            self.text_generator = pipeline("text-generation", 
+                                        model="gpt2")
+            logger.info("Modèles chargés avec succès")
+        except Exception as e:
+            logger.error(f"Erreur lors du chargement des modèles: {str(e)}")
+            raise
 
-def process_inputs(image, audio, text, models):
-    blip_processor, blip_model, audio_transcriber, text_generator = models
-    
-    final_prompt = ""
-    
-    # Analyse de l'image si présente
-    if image is not None:
-        image_description = analyze_image(image, blip_processor, blip_model)
-        final_prompt += f"Visual description: {image_description}\n"
-    
-    # Transcription audio si présent
-    if audio is not None:
-        audio_text = audio_transcriber(audio)["text"]
-        final_prompt += f"Audio content: {audio_text}\n"
-    
-    # Ajout du texte si présent
-    if text:
-        final_prompt += f"Additional context: {text}\n"
-    
-    # Génération du prompt optimisé avec GPT-2
-    prompt_enhancement = text_generator(
-        final_prompt,
-        max_length=200,
-        num_return_sequences=1
-    )[0]["generated_text"]
-    
-    # Création de la vidéo avec ModelScope
-    video_pipeline = ms_pipeline(
-        'text-to-video-synthesis',
-        model='damo/text-to-video-synthesis'
-    )
-    
-    result = video_pipeline({
-        'text': prompt_enhancement,
-        'output_video_path': 'output.mp4'
-    })
-    
-    return 'output.mp4', prompt_enhancement
+    def analyze_image(self, image):
+        """Analyse une image et retourne une description"""
+        try:
+            if image is None:
+                return ""
+            
+            questions = [
+                "What is in the picture?",
+                "What are the main colors?",
+                "What is the setting or background?",
+                "What is happening in the image?",
+            ]
+            
+            responses = {}
+            for question in questions:
+                inputs = self.blip_processor(images=image, text=question, return_tensors="pt")
+                outputs = self.blip_model.generate(**inputs)
+                answer = self.blip_processor.decode(outputs[0], skip_special_tokens=True)
+                responses[question] = answer
+            
+            description = (
+                f"This image shows {responses['What is in the picture?']}. "
+                f"The main colors are {responses['What are the main colors?']}. "
+                f"The setting is {responses['What is the setting or background?']}. "
+                f"In the scene, {responses['What is happening in the image?']}"
+            )
+            
+            return description
+        except Exception as e:
+            logger.error(f"Erreur lors de l'analyse de l'image: {str(e)}")
+            return "Erreur lors de l'analyse de l'image."
+
+    def transcribe_audio(self, audio_path):
+        """Transcrit un fichier audio"""
+        try:
+            if audio_path is None:
+                return ""
+            return self.audio_transcriber(audio_path)["text"]
+        except Exception as e:
+            logger.error(f"Erreur lors de la transcription audio: {str(e)}")
+            return "Erreur lors de la transcription audio."
+
+    def generate_text(self, prompt):
+        """Génère du texte à partir d'un prompt"""
+        try:
+            if not prompt:
+                return ""
+            response = self.text_generator(prompt, 
+                                        max_length=200,
+                                        num_return_sequences=1)[0]["generated_text"]
+            return response
+        except Exception as e:
+            logger.error(f"Erreur lors de la génération de texte: {str(e)}")
+            return "Erreur lors de la génération de texte."
+
+    def process_inputs(self, image, audio, text):
+        """Traite les entrées multimodales"""
+        try:
+            # Analyse de l'image
+            image_description = self.analyze_image(image) if image is not None else ""
+            
+            # Transcription audio
+            audio_text = self.transcribe_audio(audio) if audio is not None else ""
+            
+            # Combinaison des entrées
+            combined_input = ""
+            if image_description:
+                combined_input += f"Visual description: {image_description}\n"
+            if audio_text:
+                combined_input += f"Audio content: {audio_text}\n"
+            if text:
+                combined_input += f"Additional context: {text}\n"
+            
+            # Génération du prompt final
+            if combined_input:
+                final_prompt = self.generate_text(combined_input)
+            else:
+                final_prompt = "Aucune entrée fournie."
+            
+            return final_prompt
+            
+        except Exception as e:
+            logger.error(f"Erreur lors du traitement des entrées: {str(e)}")
+            return "Une erreur est survenue lors du traitement des entrées."
 
-# Interface Gradio
 def create_interface():
-    models = load_models()
+    """Crée l'interface Gradio"""
+    processor = MultimodalProcessor()
     
     interface = gr.Interface(
-        fn=lambda img, audio, txt: process_inputs(img, audio, txt, models),
+        fn=processor.process_inputs,
         inputs=[
-            gr.Image(type="pil", label="Upload Image"),
-            gr.Audio(type="filepath", label="Upload Audio"),
-            gr.Textbox(label="Enter Additional Text")
+            gr.Image(type="pil", label="Télécharger une image"),
+            gr.Audio(type="filepath", label="Télécharger un fichier audio"),
+            gr.Textbox(label="Entrez du texte additionnel")
         ],
         outputs=[
-            gr.Video(label="Generated Video"),
-            gr.Textbox(label="Generated Prompt")
+            gr.Textbox(label="Description générée")
         ],
-        title="Multimodal Content to Video Generator",
-        description="Upload an image, audio, or text (or any combination) to generate a video."
+        title="Analyseur de Contenu Multimodal",
+        description="""
+        Cette application analyse vos contenus multimodaux :
+        - Images : génère une description détaillée
+        - Audio : transcrit le contenu
+        - Texte : enrichit la description
+        
+        La sortie combine toutes ces informations en une description cohérente.
+        """
     )
     
     return interface
 
-# Lancement de l'application
 if __name__ == "__main__":
     interface = create_interface()
     interface.launch()