import streamlit as st import streamlit.components.v1 as components import mido from queue import Queue import threading # We'll store inbound hardware MIDI in a queue and dispatch them to the browser. if 'incoming_midi' not in st.session_state: st.session_state.incoming_midi = Queue() # A callback for Mido hardware input: def midi_in_callback(msg): """ Called in background when hardware MIDI messages arrive. We'll just forward note_on / note_off to the session's queue. """ if msg.type in ['note_on', 'note_off']: event_type = 'noteOn' if msg.type == 'note_on' else 'noteOff' st.session_state.incoming_midi.put({ 'type': event_type, 'note': msg.note, 'velocity': msg.velocity }) def open_midi_input(port_name: str): if port_name == "None": return None return mido.open_input(port_name, callback=midi_in_callback) def open_midi_output(port_name: str): if port_name == "None": return None return mido.open_output(port_name) def main(): st.title("Browser-based Synth with Tone.js (5-Octave + 16 Pads)") st.write(""" This demo uses **Tone.js** in the browser to produce audio, so no ALSA or JACK is required on the server side. Hardware MIDI input is captured by Python (Mido) and forwarded to the browser so Tone.js can play it. """) # 1. Let user pick hardware MIDI in/out in_ports = ["None"] + mido.get_input_names() out_ports = ["None"] + mido.get_output_names() in_choice = st.selectbox("MIDI Input Port", in_ports) out_choice = st.selectbox("MIDI Output Port", out_ports) # Keep references to the open ports in session state if 'midi_in' not in st.session_state: st.session_state.midi_in = None if 'midi_out' not in st.session_state: st.session_state.midi_out = None # If changed, close old and open new if st.session_state.midi_in and st.session_state.midi_in.name != in_choice: st.session_state.midi_in.close() st.session_state.midi_in = open_midi_input(in_choice) elif (not st.session_state.midi_in) and in_choice != "None": st.session_state.midi_in = open_midi_input(in_choice) if st.session_state.midi_out and st.session_state.midi_out.name != out_choice: st.session_state.midi_out.close() st.session_state.midi_out = open_midi_output(out_choice) elif (not st.session_state.midi_out) and out_choice != "None": st.session_state.midi_out = open_midi_output(out_choice) # 2. We'll embed a large HTML with Tone.js for the UI & sound. # It will: # - Create a poly synth for the 5-octave keys # - Create a sampler or player for 16 drum pads # - Provide an optional arpeggiator toggling # - Post "noteOn"/"noteOff"/"drum" messages to Python # - Listen for messages from Python to trigger notes # For brevity, let's place it in a separate variable or file. tone_html = get_tone_html() # 3. Insert the custom HTML component # We'll give it enough vertical space for the layout, say 600px components.html(tone_html, height=600, scrolling=True) # 4. We'll poll for inbound hardware MIDI messages from st.session_state.incoming_midi # and send them to the browser using st.session_state.js_events if 'js_events' not in st.session_state: # We'll store "events to the browser" in a queue or list st.session_state.js_events = Queue() # We also define a function to dispatch note events to the hardware out def send_to_midi_out(evt): if st.session_state.midi_out is not None: from mido import Message if evt['type'] == 'noteOn': msg = Message('note_on', note=evt['note'], velocity=evt['velocity']) st.session_state.midi_out.send(msg) elif evt['type'] == 'noteOff': msg = Message('note_off', note=evt['note'], velocity=0) st.session_state.midi_out.send(msg) # if drum events => you can decide how you want to handle them # 5. We'll add a small "polling" approach to handle new hardware MIDI events def poll_hardware_midi(): while not st.session_state.incoming_midi.empty(): evt = st.session_state.incoming_midi.get_nowait() # Forward to browser st.session_state.js_events.put(evt) poll_hardware_midi() # 6. We'll also add a placeholder to show the events from the browser debug_placeholder = st.empty() # 7. Hidden HTML snippet to route browser postMessage -> streamlit # This snippet listens for "streamlit:message" from the iframe and puts them in python queues # We'll handle them in real-time in the main loop components.html(""" """, height=0) # 8. We'll create a function to handle inbound "toneEvent" from the browser # (these are user clicks on the on-screen keys/pads). # We can forward them to hardware MIDI out if desired. if 'incoming_browser' not in st.session_state: from collections import deque st.session_state.incoming_browser = deque() def handle_browser_event(e): # e = {eventType, note, velocity, padIndex} debug_placeholder.write(f"Browser event: {e}") # If it's a noteOn / noteOff, optionally forward to MIDI out if e['eventType'] in ['noteOn', 'noteOff']: send_to_midi_out({ 'type': e['eventType'], 'note': e['note'], 'velocity': e['velocity'] }) elif e['eventType'] == 'drum': # Could also forward to a hardware sampler via MIDI out (e.g., note_on) pass # 9. We'll do a second poll to read from st.session_state.incoming_browser # and handle them. while st.session_state.incoming_browser: ev = st.session_state.incoming_browser.popleft() handle_browser_event(ev) # 10. We'll also do a final step: # * If we have events in st.session_state.js_events (from hardware), # we embed them in a \n" components.html(script_code, height=0) st.write("Press keys on the on-screen 5-octave keyboard or 16 pads in the browser UI. " "If a MIDI Input is selected, hardware note_on/off will also trigger Tone.js. " "If a MIDI Output is selected, on-screen or inbound hardware events can be echoed back out.") # 11. On session end, close ports def cleanup(): if st.session_state.midi_in: st.session_state.midi_in.close() if st.session_state.midi_out: st.session_state.midi_out.close() st.on_session_end(cleanup) def get_tone_html(): """ Returns an HTML/JS string that: - loads Tone.js - creates a 5-octave key + 16 drum pads UI - sets up an arpeggiator if desired - listens for hardwareMidi messages from Python - uses postMessage to send 'toneEvent' to Python For brevity, we’ll put everything inline. In production, put it in a separate .html file. """ return r"""
All audio is generated client-side. Python just handles hardware MIDI and forwards it here.