Clean up

app.py

@@ -13,6 +13,7 @@ import librosa
 import soundfile as sf
 from midi2audio import FluidSynth
 import spaces
+from tqdm import tqdm
 
 # Remove CPU forcing since we'll use ZeroGPU
 # os.environ["CUDA_VISIBLE_DEVICES"] = ""
@@ -21,7 +22,12 @@ import spaces
 from aria.image_encoder import ImageEncoder
 from aria.aria import ARIA
 
-print("Checking model files...")
+print("=" * 60)
+print("ARIA - Art to Music Generator")
+print("=" * 60)
+print("Initializing model downloads...")
+sys.stdout.flush()
+
 # Pre-download all model files at startup
 MODEL_FILES = {
     "image_encoder": "image_encoder.pt",
@@ -33,34 +39,71 @@ MODEL_FILES = {
 # Create cache directory
 CACHE_DIR = os.path.join(os.path.dirname(__file__), "model_cache")
 os.makedirs(CACHE_DIR, exist_ok=True)
+print(f"Cache directory: {CACHE_DIR}")
+sys.stdout.flush()
 
 # Download and cache all files
 cached_files = {}
+total_files = sum(len(files) if isinstance(files, list) else 1 for files in MODEL_FILES.values())
+current_file = 0
+
 for model_type, files in MODEL_FILES.items():
+    print(f"\nProcessing {model_type} model files...")
+    sys.stdout.flush()
+    
     if isinstance(files, str):
         files = [files]
     
     cached_files[model_type] = []
     for file in files:
+        current_file += 1
+        print(f"[{current_file}/{total_files}] {file}")
+        sys.stdout.flush()
+        
         try:
             # Check if file already exists in cache
             repo_id = "vincentamato/aria"
             cached_path = os.path.join(CACHE_DIR, repo_id, file)
+            
             if os.path.exists(cached_path):
-                print(f"Using cached file: {file}")
+                file_size = os.path.getsize(cached_path) / (1024 * 1024)  # MB
+                print(f"   Found cached file ({file_size:.1f} MB)")
                 cached_files[model_type].append(cached_path)
             else:
-                print(f"Downloading file: {file}")
+                print(f"   Downloading from HuggingFace Hub...")
+                print(f"   Repository: {repo_id}")
+                sys.stdout.flush()
+                
+                # Download with progress
                 cached_path = hf_hub_download(
                     repo_id=repo_id,
                     filename=file,
-                    cache_dir=CACHE_DIR
+                    cache_dir=CACHE_DIR,
+                    # resume_download=True  # Enable resume if connection drops
                 )
-                cached_files[model_type].append(cached_path)
+                
+                if os.path.exists(cached_path):
+                    file_size = os.path.getsize(cached_path) / (1024 * 1024)  # MB
+                    print(f"Download complete ({file_size:.1f} MB)")
+                    cached_files[model_type].append(cached_path)
+                else:
+                    print(f"Download failed - file not found")
+                    
         except Exception as e:
-            print(f"Error with file {file}: {str(e)}")
+            print(f"   Error with file {file}: {str(e)}")
+            sys.stdout.flush()
+
+print("\n" + "=" * 60)
+print("Model file preparation complete!")
+print("=" * 60)
+sys.stdout.flush()
 
-print("Model files ready.")
+# Check what we actually got
+for model_type, paths in cached_files.items():
+    print(f"{model_type}: {len(paths)} files ready")
+    
+print(f"\nStarting Gradio application...")
+sys.stdout.flush()
 
 # Global model cache
 models = {}
@@ -151,30 +194,64 @@ def convert_midi_to_wav(midi_path):
         return wav_path
         
     try:
-        # Check common soundfont locations
-        soundfont_paths = [
-            '/usr/share/sounds/sf2/FluidR3_GM.sf2',  # Linux
-            '/usr/share/soundfonts/default.sf2',      # Linux alternative
-            '/usr/local/share/fluidsynth/generaluser.sf2',  # macOS
-            'C:\\soundfonts\\generaluser.sf2'         # Windows
+        # Search common soundfont directories for any .sf2 or .sf3 files
+        import glob
+        soundfont_search_dirs = [
+            'C:\\soundfonts\\',                          # Windows user soundfonts
+            'C:\\Program Files\\FluidSynth\\sf2\\',      # Windows FluidSynth installation
+            '/usr/share/sounds/sf2/',                    # Linux system soundfonts
+            '/usr/share/soundfonts/',                    # Linux alternative
+            '/usr/local/share/fluidsynth/',              # macOS homebrew
+            '/System/Library/Audio/Sounds/Banks/',       # macOS system
         ]
         
         soundfont = None
-        for sf_path in soundfont_paths:
-            if os.path.exists(sf_path):
-                soundfont = sf_path
-                break
+        for search_dir in soundfont_search_dirs:
+            if os.path.exists(search_dir):
+                # Look for .sf2 and .sf3 files in this directory
+                for extension in ['*.sf2', '*.sf3']:
+                    matches = glob.glob(os.path.join(search_dir, extension))
+                    if matches:
+                        soundfont = matches[0]  # Use first soundfont found
+                        break
+                if soundfont:
+                    break
                 
         if soundfont is None:
-            raise RuntimeError("No SoundFont file found. Please install fluid-soundfont-gm package.")
+            print(f"No SoundFont found. Audio playback not available.")
+            print(f"MIDI file saved: {midi_path}")
+            print(f"To enable audio: Install FluidSynth and place a .sf2 file in C:\\soundfonts\\")
+            return None
             
-        # Convert MIDI to WAV using FluidSynth with explicit soundfont
-        fs = FluidSynth(sound_font=soundfont)
-        fs.midi_to_audio(midi_path, wav_path)
+        # Convert MIDI to WAV using direct FluidSynth command
+        print(f"Converting MIDI to WAV using SoundFont: {soundfont}")
+        
+        # Use subprocess to call fluidsynth directly with proper arguments
+        import subprocess
+        cmd = [
+            'fluidsynth',
+            '-ni',          # No interactive mode
+            '-g', '0.5',    # Gain
+            '-r', '44100',  # Sample rate  
+            '-F', wav_path, # Output WAV file
+            soundfont,      # SoundFont file
+            midi_path       # Input MIDI file
+        ]
+        
+        print(f"FluidSynth command: {' '.join(cmd)}")
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+        
+        if result.returncode == 0 and os.path.exists(wav_path):
+            print(f"WAV file created: {wav_path}")
+            return wav_path
+        else:
+            print(f"FluidSynth failed with return code: {result.returncode}")
+            print(f"Error output: {result.stderr}")
+            return None
         
-        return wav_path
     except Exception as e:
         print(f"Error converting MIDI to WAV: {str(e)}")
+        print(f"MIDI file still available: {midi_path}")
         return None
 
 @spaces.GPU(duration=120)
@@ -186,7 +263,7 @@ def generate_music(image, conditioning_type, gen_len, temperature, top_p, min_in
         return (
             None,  # For emotion_chart
             None,  # For midi_output
-            f"⚠️ Error: Failed to initialize {conditioning_type} model. Please check the logs."
+            f"Error: Failed to initialize {conditioning_type} model. Please check the logs."
         )
     
     try:
@@ -205,19 +282,41 @@ def generate_music(image, conditioning_type, gen_len, temperature, top_p, min_in
             min_instruments=int(min_instruments)
         )
         
+        # Create emotion plot first (needed for both success and failure cases)
+        plot_path = create_emotion_plot(valence, arousal)
+        
         # Convert MIDI to WAV
         wav_path = convert_midi_to_wav(midi_path)
         if wav_path is None:
-            return (
-                None,
-                None,
-                "⚠️ Error: Failed to convert MIDI to WAV for playback"
-            )
-        
-        # Create emotion plot
-        plot_path = create_emotion_plot(valence, arousal)
+            # WAV conversion failed, but we still have MIDI
+            result_text = f"""
+**Model Type:** {conditioning_type}
+
+**Predicted Emotions:**
+- Valence: {valence:.3f} (negative → positive)
+- Arousal: {arousal:.3f} (calm → excited)
+
+**Generation Parameters:**
+- Temperature: {temperature}
+- Top-p: {top_p}
+- Min Instruments: {min_instruments}
+
+**Audio Playback Unavailable**
+Your music has been generated as a MIDI file, but audio conversion failed.
+
+**MIDI File:** `{os.path.basename(midi_path)}`
+
+**To Enable Audio Playback:**
+1. Install FluidSynth: `choco install fluidsynth` (or download from GitHub)
+2. Download a SoundFont file (e.g., GeneralUser GS)
+3. Place it at: `C:\\soundfonts\\generaluser.sf2`
+
+You can still download and play the MIDI file in any MIDI player!
+"""
+            # Return MIDI file for download instead of WAV
+            return (plot_path, midi_path, result_text)
         
-        # Build a nice Markdown result string
+        # Build a nice Markdown result string for successful WAV conversion
         result_text = f"""
 **Model Type:** {conditioning_type}
 
@@ -240,7 +339,7 @@ Your music has been generated! Click the play button above to listen.
         return (
             None,
             None,
-            f"⚠️ Error generating music: {str(e)}"
+            f"Error generating music: {str(e)}"
         )
 
 def generate_music_wrapper(image, conditioning_type, gen_len, note_temp, rest_temp, top_p, min_instruments):
@@ -261,16 +360,31 @@ with gr.Blocks(title="ARIA - Art to Music Generator", theme=gr.themes.Soft(
         font=[gr.themes.GoogleFont("Inter"), "system-ui", "sans-serif"]
     )) as demo:
     gr.Markdown("""
-    # 🎨 ARIA: Artistic Rendering of Images into Audio
+    # ARIA: Artistic Rendering of Images into Audio
     
     Upload an image and ARIA will analyze its emotional content to generate matching music!
     
-    ### How it works:
+    ## How it works:
     1. ARIA first analyzes the emotional content of your image along two dimensions:
        - **Valence**: How positive or negative the emotion is (-1 to 1)
        - **Arousal**: How calm or excited the emotion is (-1 to 1)
     2. These emotions are then used to generate music that matches the mood
     """)
+
+    # Subtle gradient background for a more modern look
+    gr.HTML(
+        """
+        <style>
+            body {
+                background: radial-gradient(circle at top left, #0d1117 0%, #06080d 100%);
+            }
+            /* Elevate accordion header visibility */
+            .gr-accordion-summary {
+                font-weight: 600;
+            }
+        </style>
+        """
+    )
     
     with gr.Row():
         with gr.Column(scale=3):
@@ -278,8 +392,17 @@ with gr.Blocks(title="ARIA - Art to Music Generator", theme=gr.themes.Soft(
                 type="filepath",
                 label="Upload Image"
             )
-            
-            with gr.Group():
+
+            # Quick-start guidance so first-time users immediately know what to do
+            gr.Markdown(
+                "## Quick Start\n"
+                "1. **Click an example artwork below** *or* **upload your own image** above.\n"
+                "2. (Optional) Open **Advanced Settings** to fine-tune the generation.\n"
+                "3. Hit **Generate Music** to inference the model!"
+            )
+
+            # Advanced controls are tucked away inside a collapsible panel to keep the UI clean
+            with gr.Accordion("Advanced Settings", open=False):
                 gr.Markdown("### Generation Settings")
                 
                 with gr.Row():
@@ -340,16 +463,18 @@ with gr.Blocks(title="ARIA - Art to Music Generator", theme=gr.themes.Soft(
                             info="Minimum number of instruments in the generated music"
                         )
             
-            generate_btn = gr.Button("🎵 Generate Music", variant="primary", size="lg")
+            generate_btn = gr.Button("Generate Music", variant="primary", size="lg")
             
             # Add examples
+            # Dynamic path resolution for local vs HF Spaces deployment
+            examples_dir = "examples" if os.path.exists("examples") else "ARIA/examples"
             gr.Examples(
                 examples=[
-                    ["examples/happy.jpg", "continuous_concat", 1024, 1.2, 1.2, 0.6, 2],
-                    ["examples/sad.jpeg", "continuous_concat", 1024, 1.2, 1.2, 0.6, 2],
+                    [f"{examples_dir}/happy.jpg", "continuous_concat", 1024, 1.2, 1.2, 0.6, 2],
+                    [f"{examples_dir}/sad.jpeg", "continuous_concat", 1024, 1.2, 1.2, 0.6, 2],
                 ],
                 inputs=[image_input, conditioning_type, gen_len, note_temperature, rest_temperature, top_p, min_instruments],
-                label="Try these examples"
+                label="Example Artworks (click to load)"
             )
         
         with gr.Column(scale=2):
@@ -367,7 +492,7 @@ with gr.Blocks(title="ARIA - Art to Music Generator", theme=gr.themes.Soft(
     ### About ARIA
     
     ARIA is a deep learning system that generates music from artwork by:
-    1. Using a image emotion model to extract emotional content from images
+    1. Using a image-emotion model to extract emotional content from images
     2. Generating matching music using an emotion-conditioned music generation model
     
     The emotion-conditioned MIDI generation model is based on the work by Serkan Sulun et al. in their paper 
@@ -375,9 +500,15 @@ with gr.Blocks(title="ARIA - Art to Music Generator", theme=gr.themes.Soft(
     Original implementation: [github.com/serkansulun/midi-emotion](https://github.com/serkansulun/midi-emotion)
     
     ### Conditioning Types
-    - **continuous_concat**: Emotions are concatenated with music features (recommended)
-    - **continuous_token**: Emotions are added as special tokens
-    - **discrete_token**: Emotions are discretized into tokens
+    
+    **continuous_concat (Recommended)**  
+    Creates a single vector from valence and arousal values, repeats it across the sequence, and concatenates it with every music token embedding. This approach gives the emotion information *global influence* throughout the entire generation process, allowing the transformer to access emotional context at every timestep. Research shows this method achieves the best performance in both note prediction accuracy and emotional coherence.
+    
+    **continuous_token**  
+    Converts each emotion value (valence and arousal) into separate condition vectors with the same length as music token embeddings, then concatenates them in the sequence dimension. The emotion vectors are inserted at the beginning of the input sequence during generation. This treats emotions similarly to music tokens but can lose influence as the sequence grows longer.
+    
+    **discrete_token**  
+    Quantizes continuous emotion values into 5 discrete bins (very low, low, moderate, high, very high) and converts them into control tokens. These tokens are placed before the music tokens in the sequence. While this represents the current state-of-the-art approach in conditional text generation, it suffers from information loss due to binning and can lose emotional context during longer generations when tokens are truncated.
     """)
 
     def generate_music_wrapper(image, conditioning_type, gen_len, note_temp, rest_temp, top_p, min_instruments):

aria/aria.py

@@ -31,15 +31,39 @@ class ARIA:
             conditioning: Type of conditioning to use (continuous_concat, continuous_token, discrete_token)
             device: Device to run on (default: auto-detect)
         """
-        # Initialize device
-        self.device = torch.device("cuda")  # Always use CUDA with ZeroGPU
+        # Initialize device - use CPU if CUDA not available
+        if device is not None:
+            self.device = torch.device(device)
+        elif torch.cuda.is_available():
+            self.device = torch.device("cuda")
+        else:
+            self.device = torch.device("cpu")
+            
         print(f"Using device: {self.device}")
         self.conditioning = conditioning
         
         # Load image emotion model
         self.image_model = ImageEncoder()
-        checkpoint = torch.load(image_model_checkpoint, map_location=self.device, weights_only=True)
-        self.image_model.load_state_dict(checkpoint["model_state_dict"])
+        try:
+            checkpoint = torch.load(image_model_checkpoint, map_location=self.device, weights_only=True)
+            # Extract only the custom heads from the checkpoint (ignore CLIP model weights)
+            state_dict = {}
+            for key, value in checkpoint["model_state_dict"].items():
+                if key.startswith(('valence_head.', 'arousal_head.')):
+                    state_dict[key] = value
+            
+            # Initialize the model first so the heads exist
+            self.image_model._ensure_initialized()
+            
+            # Load only the custom head weights
+            self.image_model.load_state_dict(state_dict, strict=False)
+            print("ImageEncoder custom heads loaded successfully")
+        except Exception as e:
+            print(f"Warning: Could not load ImageEncoder checkpoint: {e}")
+            print("Using randomly initialized heads")
+            # Initialize anyway with random weights
+            self.image_model._ensure_initialized()
+            
         self.image_model = self.image_model.to(self.device)
         self.image_model.eval()
         
@@ -53,8 +77,8 @@ class ARIA:
         mappings_fp = os.path.join(midi_model_dir, 'mappings.pt')
         config_fp = os.path.join(midi_model_dir, 'model_config.pt')
         
-        self.maps = torch.load(mappings_fp, weights_only=True)
-        config = torch.load(config_fp, weights_only=True)
+        self.maps = torch.load(mappings_fp, map_location=self.device, weights_only=True)
+        config = torch.load(config_fp, map_location=self.device, weights_only=True)
         self.midi_model, _ = build_model(None, load_config_dict=config)
         self.midi_model = self.midi_model.to(self.device)
         self.midi_model.load_state_dict(torch.load(model_fp, map_location=self.device, weights_only=True))

aria/image_encoder.py

@@ -13,9 +13,28 @@ class ImageEncoder(nn.Module):
         """
         super().__init__()
         
+        # Store model name for lazy loading
+        self.clip_model_name = clip_model_name
+        self.clip_model = None
+        self.processor = None
+        self.valence_head = None
+        self.arousal_head = None
+        self.device = None
+        self._initialized = False
+    
+    def _ensure_initialized(self):
+        """Lazy initialization of the model components."""
+        if self._initialized:
+            return
+            
+        print(f"Initializing ImageEncoder with {self.clip_model_name}...")
+        print("Downloading CLIP model (this may take a moment)...")
+        
         # Load CLIP model and processor
-        self.clip_model = CLIPModel.from_pretrained(clip_model_name)
-        self.processor = CLIPProcessor.from_pretrained(clip_model_name)
+        self.clip_model = CLIPModel.from_pretrained(self.clip_model_name)
+        self.processor = CLIPProcessor.from_pretrained(self.clip_model_name)
+        
+        print("CLIP model loaded successfully")
         
         # Freeze CLIP parameters
         for param in self.clip_model.parameters():
@@ -50,6 +69,9 @@ class ImageEncoder(nn.Module):
         # Move model to GPU if available
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.to(self.device)
+        
+        print(f"Model moved to device: {self.device}")
+        self._initialized = True
 
     def forward(self, images: Union[Image.Image, torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]:
         """Forward pass to get valence and arousal predictions.
@@ -60,6 +82,9 @@ class ImageEncoder(nn.Module):
         Returns:
             Tuple of predicted valence and arousal scores
         """
+        # Ensure model is initialized
+        self._ensure_initialized()
+        
         # Process images if they're PIL images
         if isinstance(images, Image.Image):
             inputs = self.processor(images=images, return_tensors="pt")
@@ -85,6 +110,9 @@ class ImageEncoder(nn.Module):
         Returns:
             Image embedding tensor
         """
+        # Ensure model is initialized
+        self._ensure_initialized()
+        
         inputs = self.processor(images=image, return_tensors="pt")
         with torch.no_grad():
             image_features = self.clip_model.get_image_features(inputs.pixel_values.to(self.device))

requirements.txt

@@ -6,8 +6,10 @@ gradio>=4.0.0
 matplotlib>=3.7.0
 huggingface_hub>=0.19.0
 pretty-midi>=0.2.9
-librosa>=0.10.0
+librosa>=0.10.1
 soundfile>=0.12.0
 midi2audio>=0.1.1
 transformers>=4.35.0
-spaces>=0.32.0
+spaces>=0.32.0
+numba>=0.60.0
+llvmlite>=0.43.0