neural-os

Runtime error

File size: 48,534 Bytes

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>NeuralOS Demo</title>
    <!-- Bootstrap CSS -->
    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
    <style>
        body {
            padding: 20px;
            background-color: #f8f9fa;
        }
        
        .title-container {
            text-align: center;
            margin-bottom: 30px;
        }
        
        .canvas-container {
            display: flex;
            flex-direction: column;
            align-items: center;
            margin-bottom: 20px;
        }
        
        .canvas-wrapper {
            position: relative;
            border: 3px solid #007bff;
            border-radius: 8px;
            background-color: #f8f9fa;
            margin-bottom: 15px;
        }
        
        .instruction-box {
            position: absolute;
            top: -40px;
            left: 50%;
            transform: translateX(-50%);
            background-color: #007bff;
            color: white;
            padding: 5px 15px;
            border-radius: 5px;
            font-weight: bold;
            white-space: nowrap;
        }
        
        #displayCanvas {
            cursor: none;
            display: block;
        }
        
        .controls {
            margin-top: 20px;
            display: flex;
            gap: 15px;
            align-items: center;
            justify-content: center;
            flex-wrap: wrap;
        }
        
        .control-button {
            cursor: pointer;
        }
        
        .step-control {
            display: flex;
            align-items: center;
            gap: 10px;
        }
        
        #samplingSteps {
            width: 70px;
        }
        
        .footer {
            margin-top: 30px;
            text-align: center;
        }
    </style>
</head>
<body>
    <div class="container">
        <div class="title-container">
            <h1 class="mb-2">NeuralOS: Towards Simulating Operating Systems<br>via Neural Generative Models</h1>
            <p class="mb-3">
                <a href="https://anonymous.4open.science/r/neural-os" target="_blank" class="text-decoration-none">
                    Project Code: anonymous.4open.science/r/neural-os
                </a>
            </p>
        </div>
        
        <div class="canvas-container">
            <div id="timeoutWarning" class="alert alert-warning" style="display: none; margin-bottom: 15px;">
                <strong>Connection Timeout Warning:</strong> 
                <span id="timeoutMessage">No user activity detected. Connection will be dropped in <span id="timeoutCountdown">10</span> seconds and page will refresh automatically.</span>
                <button type="button" class="btn btn-sm btn-primary ms-2" onclick="resetTimeout()">Stay Connected</button>
            </div>
            
            <div class="canvas-wrapper">
                <div class="instruction-box">Move your mouse inside to interact</div>
                <canvas id="displayCanvas" width="512" height="384"></canvas>
            </div>
            
            <div class="controls">
                <button id="resetButton" class="btn btn-primary control-button">Reset Simulation</button>
                
                <div class="step-control">
                    <label for="samplingSteps" class="form-label mb-0">Sampling Steps:</label>
                    <input type="number" id="samplingSteps" class="form-control" min="1" max="100" value="32">
                    <button id="updateStepsButton" class="btn btn-secondary control-button">Update</button>
                </div>
                
                <div class="form-check form-switch">
                    <input class="form-check-input" type="checkbox" role="switch" id="useRnnToggle">
                    <label class="form-check-label" for="useRnnToggle">Use RNN</label>
                </div>
                
                <div class="form-check form-switch">
                    <input class="form-check-input" type="checkbox" role="switch" id="autoInputToggle" checked>
                    <label class="form-check-label" for="autoInputToggle" id="autoInputLabel">Auto Input</label>
                </div>
            </div>
        </div>
        
        <div class="row justify-content-center">
            <div class="col-md-8">
                <div class="card">
                    <div class="card-body">
                        <h5 class="card-title">Instructions:</h5>
                        <ul class="mb-0">
                            <li>Move your mouse inside the blue box to interact with NeuralOS</li>
                            <li>Click (left-click or right-click) to perform click actions</li>
                            <li>Use your keyboard to type within the simulated environment</li>
                            <li>Adjust sampling steps to control quality/speed tradeoff</li>
                            <li>Toggle "Use RNN" to switch between RNN and diffusion mode</li>
                            <li>Toggle "Auto Input" to enable automatic frame generation (starts after 2s idle once you move mouse inside canvas, then runs every 0.5s)</li>
                        </ul>
                    </div>
                </div>
            </div>
        </div>
    </div>

    <script>
        const canvas = document.getElementById('displayCanvas');
        const ctx = canvas.getContext('2d');
        
        let socket;
        let isConnected = false;
        let reconnectAttempts = 0;
        const MAX_RECONNECT_DELAY = 30000; // Maximum delay between reconnection attempts (30 seconds)
        let connectionAttempted = false; // Flag to prevent multiple connection attempts

        let isProcessing = false;
        
        // Add flag to control trace visibility, default to false (hidden)
        let showTrace = false;

        // Function to draw the initial canvas state
        function drawInitialCanvas() {
            ctx.fillStyle = "#ffffff";
            ctx.fillRect(0, 0, canvas.width, canvas.height);
            ctx.font = "18px Arial";
            ctx.fillStyle = "#666666";
            ctx.textAlign = "center";
            ctx.fillText("Move your mouse here to interact with NeuralOS", canvas.width/2, canvas.height/2 - 10);
            ctx.fillText("The neural model will render in this area", canvas.width/2, canvas.height/2 + 20);
        }

        // Function to show connection status
        function showConnectionStatus(message, subtitle = null) {
            ctx.fillStyle = "#ffffff";
            ctx.fillRect(0, 0, canvas.width, canvas.height);
            ctx.font = "18px Arial";
            ctx.fillStyle = "#666666";
            ctx.textAlign = "center";
            //ctx.fillText(message, canvas.width/2, canvas.height/2 - 10);
            ctx.fillText(message, canvas.width/2, canvas.height/2);
            
            if (subtitle) {
                ctx.font = "14px Arial";
                ctx.fillStyle = "#888888";
                ctx.fillText(subtitle, canvas.width/2, canvas.height/2 + 15);
            }
        }

        function connect() {
            console.log("connect() function called - creating new WebSocket connection");
            //const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
            //socket = new WebSocket(`${protocol}//${window.location.host}/ws`);
            const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';

        // Fix for HF Spaces - construct correct domain from path
        let host = window.location.host;
        if (host === 'huggingface.co' && window.location.pathname.startsWith('/spaces/')) {
            // Extract username and spacename from path like /spaces/username/spacename
            const pathParts = window.location.pathname.split('/');
            if (pathParts.length >= 4) {
                const username = pathParts[2];
                const spacename = pathParts[3];
                host = `${username}-${spacename}.hf.space`;
            }
        }

        socket = new WebSocket(`${protocol}//${host}/ws`);

            socket.onopen = function(event) {
                console.log("WebSocket connection established");
                isConnected = true;
                //connectionAttempted = false; // Reset flag for potential reconnections
                //reconnectAttempts = 0;
                
                // Show brief success message
                //showConnectionStatus("Connected! Processing input...");
                //setTimeout(() => {
                //    // Clear the message after 0.5 second
                //    if (isConnected) {
                //        ctx.clearRect(0, 0, canvas.width, canvas.height);
                //    }
                //}, 500);
                
                // Start auto-input mechanism
                startAutoInput();
                
                // Request current settings from server to sync UI
                socket.send(JSON.stringify({
                    type: "get_settings"
                }));
                
                //startHeartbeat();
            };

            socket.onclose = function(event) {
                console.log("WebSocket connection closed. Attempting to reconnect...");
                console.log("Close event code:", event.code, "reason:", event.reason);
                
                // Check if this was a timeout closure
                if (event.code === 1000 && event.reason === "User inactivity timeout") {
                    console.log("Connection closed due to timeout - refreshing page");
                    stopAutoInput(); // Stop auto-input when connection is lost
                    stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
                    //clearInterval(heartbeatInterval);
                    // Refresh the page to reconnect
                    window.location.reload();
                    return;
                }
                
                isConnected = false;
                //connectionAttempted = false; // Reset flag to allow reconnection attempts
                stopAutoInput(); // Stop auto-input when connection is lost
                stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
                //clearInterval(heartbeatInterval);
                //scheduleReconnection();
            };

            socket.onerror = function(error) {
                console.error("WebSocket error:", error);
            };

            socket.onmessage = function (event) {
                const data = JSON.parse(event.data);
                if (data.type === "heartbeat_response") {
                    console.log("Heartbeat response received");
                } else if (data.image) {
                    let img = new Image();
                    img.onload = function() {
                        ctx.clearRect(0, 0, canvas.width, canvas.height);
                        ctx.drawImage(img, 0, 0);
                        //isProcessing = false; // Reset the processing flag when we get a response
                    };
                    img.src = 'data:image/png;base64,' + data.image;
                } else if (data.type === "reset_confirmed") {
                    console.log("Simulation reset confirmed by server");
                    // Clear the canvas and reset to initial state
                    drawInitialCanvas();
                    // Reset user interaction state
                    userHasInteracted = false;
                    lastSentPosition = null;
                    console.log("Reset user interaction state - waiting for user to move mouse inside canvas");
                } else if (data.type === "rnn_updated") {
                    console.log(`USE_RNN setting updated to: ${data.use_rnn}`);
                    // Update the toggle to match the server state
                    document.getElementById('useRnnToggle').checked = data.use_rnn;
                } else if (data.type === "settings") {
                    // Update UI elements to match server settings
                    console.log(`Received settings from server: SAMPLING_STEPS=${data.sampling_steps}, USE_RNN=${data.use_rnn}`);
                    document.getElementById('samplingSteps').value = data.sampling_steps;
                    document.getElementById('useRnnToggle').checked = data.use_rnn;
                } else if (data.type === "timeout_warning") {
                    console.log(`Received timeout warning: ${data.timeout_in} seconds remaining`);
                    setTimeoutMessage(`No user activity detected. Connection will be dropped in <span id="timeoutCountdown">${data.timeout_in}</span> seconds and page will refresh automatically.`);
                    startTimeoutCountdown(data.timeout_in, false, 'idle'); // false = show stay connected button
                } else if (data.type === "activity_reset") {
                    console.log("Server detected user activity, resetting timeout");
                    stopTimeoutCountdown('idle'); // Only clear idle timeout
                    
                    // If there's still a session time limit active, update it
                    if (data.session_time_remaining && data.session_time_remaining > 0) {
                        console.log(`Updating session countdown: ${data.session_time_remaining} seconds remaining`);
                        setTimeoutMessage(`⏰ Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.session_time_remaining)}</span> seconds.`);
                        startTimeoutCountdown(Math.ceil(data.session_time_remaining), true, 'session'); // true = hide stay connected button
                    }
                } else if (data.type === "queue_update") {
                    console.log(`Queue update: Position ${data.position}/${data.total_waiting}, wait: ${data.maximum_wait_seconds.toFixed(1)} seconds`);
                    const waitSeconds = Math.ceil(data.maximum_wait_seconds);
                    
                    if (waitSeconds === 0) {
                        showConnectionStatus("Starting soon...");
                        stopQueueCountdown();
                    } else {
                        // Only restart countdown if not already running or if time changed significantly
                        if (!queueCountdownActive) {
                            console.log(`Starting queue countdown: ${waitSeconds} seconds`);
                            startQueueCountdown(waitSeconds);
                        } else if (waitSeconds < queueWaitTime - 10 || waitSeconds > queueWaitTime + 15) {
                            // Be responsive to decreases (users dropping out) and significant increases
                            console.log(`Updating queue countdown: ${queueWaitTime} -> ${waitSeconds} (significant change)`);
                            startQueueCountdown(waitSeconds);
                        } else if (Math.abs(queueWaitTime - waitSeconds) <= 3) {
                            // Small differences (1-3 seconds) - keep local countdown running
                            console.log(`Ignoring small change: ${queueWaitTime} vs ${waitSeconds} (keeping local countdown)`);
                        } else {
                            console.log(`Moderate change ignored: ${queueWaitTime} -> ${waitSeconds} (${Math.abs(queueWaitTime - waitSeconds)}s difference)`);
                        }
                    }
                } else if (data.type === "session_start") {
                    console.log("Session started, clearing queue display");
                    // Stop queue countdown and clear the display
                    stopQueueCountdown();
                    //ctx.clearRect(0, 0, canvas.width, canvas.height);
                } else if (data.type === "session_warning") {
                    console.log(`Session time warning: ${data.time_remaining} seconds remaining`);
                    setTimeoutMessage(`Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
                    startTimeoutCountdown(Math.ceil(data.time_remaining), true, 'session'); // true = hide stay connected button
                } else if (data.type === "idle_warning") {
                    console.log(`Idle warning: ${data.time_remaining} seconds until timeout`);
                    setTimeoutMessage(`No user activity detected. Connection will be dropped in <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds and page will refresh automatically.`);
                    startTimeoutCountdown(Math.ceil(data.time_remaining), false, 'idle'); // false = show stay connected button
                } else if (data.type === "grace_period") {
                    console.log(`Grace period: ${data.time_remaining} seconds remaining`);
                    setTimeoutMessage(`Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
                    startTimeoutCountdown(Math.ceil(data.time_remaining), true, 'grace'); // true = hide stay connected button
                } else if (data.type === "time_limit_removed") {
                    console.log("Time limit removed - queue became empty");
                    stopTimeoutCountdown('session'); // Only clear session timeout
                    stopTimeoutCountdown('grace'); // Also clear grace period if active
                } else if (data.type === "queue_limit_applied") {
                    console.log(`Queue limit applied, ${data.time_remaining} seconds remaining`);
                    setTimeoutMessage(`⏰ Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
                    startTimeoutCountdown(Math.ceil(data.time_remaining), true, 'session'); // true = hide stay connected button
                } else if (data.type === "worker_failure") {
                    console.log("Worker failure detected, reconnecting...");
                    showConnectionStatus("🔄 GPU worker failed. Reconnecting to healthy worker...");
                    // Clear the canvas to show we're reconnecting
                    ctx.clearRect(0, 0, canvas.width, canvas.height);
                    ctx.fillStyle = '#f0f0f0';
                    ctx.fillRect(0, 0, canvas.width, canvas.height);
                    ctx.fillStyle = '#666';
                    ctx.font = '20px Arial';
                    ctx.textAlign = 'center';
                    ctx.fillText('🔄 Reconnecting to healthy GPU...', canvas.width/2, canvas.height/2);
                }
            };
        }

        function scheduleReconnection() {
            const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), MAX_RECONNECT_DELAY);
            console.log(`Scheduling reconnection in ${delay}ms`);
            setTimeout(connect, delay);
            reconnectAttempts++;
        }

        let heartbeatInterval;
        function startHeartbeat() {
            heartbeatInterval = setInterval(() => {
                if (isConnected) {
                    try {
                        socket.send(JSON.stringify({ type: "heartbeat" }));
                        console.error("finished sending heartbeat");

                    } catch (error) {
                        console.error("Error sending heartbeat:", error);
                    }
                }
            }, 1000); // Send heartbeat every 15 seconds
        }

        // Don't connect immediately - wait for user interaction
        // connect();

        // Draw initial state on the canvas
        drawInitialCanvas();

        let lastSentPosition = null;
        let lastSentTime = 0;
        const SEND_INTERVAL = 10; // Send updates every 50ms
        
        // Auto-input mechanism
        let lastUserInputTime = 0;
        let lastAutoInputTime = 0;
        let autoInputInterval = null;
        let autoInputActive = false; // Track if auto-input mode is active
        const INITIAL_AUTO_INPUT_DELAY = 2000; // Start auto-input after 2 seconds of no user input
        const AUTO_INPUT_INTERVAL = 500; // Send auto-input every 0.5 second once active
        let autoInputEnabled = true; // Default to enabled
        let userHasInteracted = false; // Track if user has moved mouse inside canvas
        
        // Timeout countdown mechanism - support concurrent timeouts
        let timeoutCountdownInterval = null;
        let timeoutCountdown = 10;
        let timeoutWarningActive = false;
        
        // Track concurrent timeout systems
        let currentTimeouts = {
            idle: null,        // { remaining: 10, hideButton: false }
            session: null,     // { remaining: 60, hideButton: true }
            grace: null        // { remaining: 5, hideButton: true }
        };
        
        // Queue waiting countdown mechanism
        let queueCountdownInterval = null;
        let queueWaitTime = 0;
        let queueCountdownActive = false;
        
        function startAutoInput() {
            if (autoInputInterval) {
                clearInterval(autoInputInterval);
            }
            
            autoInputInterval = setInterval(() => {
                const currentTime = Date.now();
                
                if (!autoInputEnabled || !lastSentPosition || !isConnected || socket.readyState !== WebSocket.OPEN || !userHasInteracted) {
                    return;
                }
                
                // Check if we should start auto-input mode
                if (!autoInputActive && currentTime - lastUserInputTime >= INITIAL_AUTO_INPUT_DELAY) {
                    console.log("Starting auto-input mode (no user activity for 2 seconds)");
                    autoInputActive = true;
                    lastAutoInputTime = currentTime;
                    // Update UI to show auto-input is active
                    try {
                        const label = document.getElementById('autoInputLabel');
                        if (label) {
                            label.textContent = "Auto Input (Active)";
                            label.style.color = "#28a745";
                        }
                    } catch (error) {
                        console.error("Error updating auto-input UI:", error);
                    }
                }
                
                // Send auto-input if mode is active and enough time has passed
                if (autoInputActive && currentTime - lastAutoInputTime >= AUTO_INPUT_INTERVAL) {
                    console.log("Sending auto-input (auto-input mode active)");
                    try {
                        socket.send(JSON.stringify({
                            "x": lastSentPosition.x,
                            "y": lastSentPosition.y,
                            "is_left_click": false,
                            "is_right_click": false,
                            "keys_down": [],
                            "keys_up": [],
                            "wheel_delta_x": 0,
                            "wheel_delta_y": 0,
                            "is_auto_input": true  // Flag to identify auto-generated inputs
                        }));
                        lastAutoInputTime = currentTime;
                    } catch (error) {
                        console.error("Error sending auto-input:", error);
                        // Stop auto-input if there's an error (connection likely closed)
                        stopAutoInput();
                    }
                }
            }, 100); // Check every 100ms
        }

        function stopAutoInput() {
            if (autoInputInterval) {
                clearInterval(autoInputInterval);
                autoInputInterval = null;
            }
            autoInputActive = false; // Reset auto-input mode
        }

        function updateLastUserInputTime() {
            lastUserInputTime = Date.now();
            // Reset auto-input mode when user provides input
            if (autoInputActive) {
                console.log("User activity detected, stopping auto-input mode");
                autoInputActive = false;
                // Reset UI indicator
                try {
                    const label = document.getElementById('autoInputLabel');
                    if (label) {
                        label.textContent = "Auto Input";
                        label.style.color = "";
                    }
                } catch (error) {
                    console.error("Error updating auto-input UI:", error);
                }
            }
        }

        function startTimeoutCountdown(initialTime = 10, hideStayConnectedButton = false, timeoutType = 'idle') {
            // Update the specific timeout type
            currentTimeouts[timeoutType] = {
                remaining: initialTime,
                hideButton: hideStayConnectedButton,
                type: timeoutType
            };
            
            // Find the timeout that will expire first
            let earliestTimeout = null;
            let earliestTime = Infinity;
            
            for (const [type, timeout] of Object.entries(currentTimeouts)) {
                if (timeout && timeout.remaining < earliestTime) {
                    earliestTime = timeout.remaining;
                    earliestTimeout = timeout;
                }
            }
            
            if (!earliestTimeout) {
                stopTimeoutCountdown();
                return;
            }
            
            // Stop existing countdown
            if (timeoutCountdownInterval) {
                clearInterval(timeoutCountdownInterval);
            }
            
            timeoutCountdown = earliestTimeout.remaining;
            timeoutWarningActive = true;
            
            // Show warning
            const warning = document.getElementById('timeoutWarning');
            if (warning) {
                warning.style.display = 'block';
            }
            
            // Show/hide Stay Connected button based on the earliest timeout
            const stayConnectedButton = warning.querySelector('button');
            if (stayConnectedButton) {
                stayConnectedButton.style.display = earliestTimeout.hideButton ? 'none' : 'inline-block';
            }
            
            // Update initial display
                const countdownElement = document.getElementById('timeoutCountdown');
                if (countdownElement) {
                    countdownElement.textContent = timeoutCountdown;
                }
                
            console.log(`Starting ${earliestTimeout.type} timeout countdown: ${timeoutCountdown} seconds`);
            
            // Start countdown
            timeoutCountdownInterval = setInterval(() => {
                // Update all active timeouts
                for (const [type, timeout] of Object.entries(currentTimeouts)) {
                    if (timeout) {
                        timeout.remaining--;
                    }
                }
                
                // Check if ANY timeout has reached 0 (expired)
                let expiredTimeout = null;
                for (const [type, timeout] of Object.entries(currentTimeouts)) {
                    if (timeout && timeout.remaining <= 0) {
                        expiredTimeout = { type, ...timeout };
                        break;
                    }
                }
                
                // If any timeout expired, trigger the appropriate action
                if (expiredTimeout) {
                    console.log(`${expiredTimeout.type} timeout expired - refreshing page`);
                    // Remove the expired timeout from tracking
                    currentTimeouts[expiredTimeout.type] = null;
                    stopAutoInput();
                    stopTimeoutCountdown();
                    // All timeout types result in page refresh for now
                    window.location.reload();
                    return;
                }
                
                // Find the timeout that will expire first among remaining timeouts
                let newEarliestTimeout = null;
                let newEarliestTime = Infinity;
                
                for (const [type, timeout] of Object.entries(currentTimeouts)) {
                    if (timeout && timeout.remaining > 0 && timeout.remaining < newEarliestTime) {
                        newEarliestTime = timeout.remaining;
                        newEarliestTimeout = timeout;
                    }
                }
                
                // If no active timeouts remain, hide the warning
                if (!newEarliestTimeout) {
                    stopTimeoutCountdown();
                    return;
                }
                
                // Update display with the earliest timeout
                timeoutCountdown = newEarliestTimeout.remaining;
                const countdownElement = document.getElementById('timeoutCountdown');
                if (countdownElement) {
                    countdownElement.textContent = timeoutCountdown;
                }
                
                // Update Stay Connected button visibility if the timeout type changed
                if (newEarliestTimeout.type !== earliestTimeout.type) {
                    console.log(`Timeout type changed from ${earliestTimeout.type} to ${newEarliestTimeout.type}`);
                    const stayConnectedButton = warning.querySelector('button');
                    if (stayConnectedButton) {
                        stayConnectedButton.style.display = newEarliestTimeout.hideButton ? 'none' : 'inline-block';
                    }
                    
                    // Update timeout message based on new timeout type
                    if (newEarliestTimeout.type === 'idle') {
                        setTimeoutMessage(`No user activity detected. Connection will be dropped in <span id="timeoutCountdown">${newEarliestTimeout.remaining}</span> seconds and page will refresh automatically.`);
                    } else if (newEarliestTimeout.type === 'session' || newEarliestTimeout.type === 'grace') {
                        setTimeoutMessage(`⏰ Other users waiting. Time remaining: <span id="timeoutCountdown">${newEarliestTimeout.remaining}</span> seconds.`);
                    }
                    
                    earliestTimeout = newEarliestTimeout;
                }
            }, 1000);
        }

        function stopTimeoutCountdown(timeoutType = null) {
            if (timeoutType) {
                // Clear specific timeout type
                currentTimeouts[timeoutType] = null;
                console.log(`Cleared ${timeoutType} timeout`);
                
                // Check if any other timeouts are still active
                let hasActiveTimeouts = false;
                for (const [type, timeout] of Object.entries(currentTimeouts)) {
                    if (timeout && timeout.remaining > 0) {
                        hasActiveTimeouts = true;
                        break;
                    }
                }
                
                if (hasActiveTimeouts) {
                    // Restart countdown with remaining timeouts
                    // Find the earliest remaining timeout and restart with it
                    let earliestTimeout = null;
                    let earliestTime = Infinity;
                    
                    for (const [type, timeout] of Object.entries(currentTimeouts)) {
                        if (timeout && timeout.remaining < earliestTime) {
                            earliestTime = timeout.remaining;
                            earliestTimeout = timeout;
                        }
                    }
                    
                    if (earliestTimeout) {
                        startTimeoutCountdown(earliestTimeout.remaining, earliestTimeout.hideButton, earliestTimeout.type);
                    }
                    return;
                }
            } else {
                // Clear all timeouts
                currentTimeouts = {
                    idle: null,
                    session: null,
                    grace: null
                };
                console.log("Cleared all timeouts");
            }
            
            if (timeoutCountdownInterval) {
                clearInterval(timeoutCountdownInterval);
                timeoutCountdownInterval = null;
            }
            
            timeoutWarningActive = false;
            timeoutCountdown = 10;
            
            // Reset the countdown display to 10 seconds
            const countdownElement = document.getElementById('timeoutCountdown');
            if (countdownElement) {
                countdownElement.textContent = timeoutCountdown;
            }
            
            // Hide warning
            const warning = document.getElementById('timeoutWarning');
            if (warning) {
                warning.style.display = 'none';
            }
        }

        function startQueueCountdown(initialWaitTime) {
            if (queueCountdownInterval) {
                clearInterval(queueCountdownInterval);
            }
            
            queueWaitTime = initialWaitTime;
            queueCountdownActive = true;
            
            // Show initial countdown
            updateQueueCountdownDisplay();
            
            // Start countdown
            queueCountdownInterval = setInterval(() => {
                queueWaitTime--;
                updateQueueCountdownDisplay();
                
                if (queueWaitTime <= 0) {
                    stopQueueCountdown();
                    showConnectionStatus("Starting soon...");
                }
            }, 1000);
        }

        function stopQueueCountdown() {
            if (queueCountdownInterval) {
                clearInterval(queueCountdownInterval);
                queueCountdownInterval = null;
            }
            queueCountdownActive = false;
            queueWaitTime = 0;
        }

        function updateQueueCountdownDisplay() {
            if (queueWaitTime <= 0) {
                showConnectionStatus("Starting soon...");
            } else if (queueWaitTime === 1) {
                showConnectionStatus("Max wait: 1 second");
            } else {
                showConnectionStatus(`Max wait: ${queueWaitTime} seconds`);
            }
        }

        function setTimeoutMessage(message) {
            const messageElement = document.getElementById('timeoutMessage');
            if (messageElement) {
                messageElement.innerHTML = message;
            }
        }

        function resetTimeout() {
            // Send a stay-connected ping to reset the server's timeout
            if (socket && socket.readyState === WebSocket.OPEN && lastSentPosition) {
                try {
                    socket.send(JSON.stringify({
                        "x": lastSentPosition.x,
                        "y": lastSentPosition.y,
                        "is_left_click": false,
                        "is_right_click": false,
                        "keys_down": [],
                        "keys_up": [],
                        "wheel_delta_x": 0,
                        "wheel_delta_y": 0,
                        "is_stay_connected": true  // Mark this as a stay-connected ping
                    }));
                    updateLastUserInputTime(); // Update for auto-input mechanism
                    console.log("Stay connected ping sent to server");
                } catch (error) {
                    console.error("Error sending stay connected ping:", error);
                }
            }
            stopTimeoutCountdown('idle'); // Only clear idle timeout, not session timeout
        }

        function sendInputState(x, y, isLeftClick = false, isRightClick = false, keysDownArr = [], keysUpArr = [], wheelDeltaX = 0, wheelDeltaY = 0) {
            const currentTime = Date.now();
            if (isConnected && socket.readyState === WebSocket.OPEN && (isLeftClick || isRightClick || keysDownArr.length > 0 || keysUpArr.length > 0 || wheelDeltaX !== 0 || wheelDeltaY !== 0 || !lastSentPosition || currentTime - lastSentTime >= SEND_INTERVAL)) {
                try {
                    socket.send(JSON.stringify({
                        "x": x,
                        "y": y,
                        "is_left_click": isLeftClick,
                        "is_right_click": isRightClick,
                        "keys_down": keysDownArr,
                        "keys_up": keysUpArr,
                        "wheel_delta_x": wheelDeltaX,
                        "wheel_delta_y": wheelDeltaY,
                    }));
                    lastSentPosition = { x, y };
                    lastSentTime = currentTime;
                    
                    // Mark that user has interacted with the canvas
                    if (!userHasInteracted) {
                        userHasInteracted = true;
                        console.log("User has interacted with canvas for the first time");
                    }
                    
                    // Update last user input time for auto-input mechanism
                    updateLastUserInputTime();
                    
                    //if (isLeftClick || isRightClick) {
                    //    isProcessing = true; // Block further inputs until response
                    //}
                } catch (error) {
                    console.error("Error sending input state:", error);
                }
            }
        }

        // Capture mouse movements and clicks
        canvas.addEventListener("mousemove", async function (event) {
            // Establish connection on first mouse movement if not already connected
            if (!isConnected && !connectionAttempted) {
                console.log("First mouse movement detected - establishing WebSocket connection");
                connectionAttempted = true;
                showConnectionStatus("Connecting to NeuralOS...");
                connect();
                
                // Wait for connection to be established
                let attempts = 0;
                const maxAttempts = 50; // 5 seconds max wait
                while (!isConnected && attempts < maxAttempts) {
                    await new Promise(resolve => setTimeout(resolve, 100)); // Wait 100ms
                    attempts++;
                }
                
                if (!isConnected) {
                    console.error("Failed to establish connection after 5 seconds");
                    connectionAttempted = false; // Reset flag to allow retry
                    showConnectionStatus("Connection failed. Move mouse to retry.");
                    return;
                }
                
                console.log("Connection established, processing first mouse movement");
            }
            
            if (!isConnected || isProcessing) return;
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;
            
            // Only draw the trace if showTrace is true
            if (showTrace && lastSentPosition) {
                ctx.beginPath();
                ctx.moveTo(lastSentPosition.x, lastSentPosition.y);
                ctx.lineTo(x, y);
                ctx.stroke();
            }

            sendInputState(x, y);
        });

        canvas.addEventListener("click", function (event) {
            if (!isConnected || isProcessing) return;
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;

            sendInputState(x, y, true, false);
        });

        // Handle right clicks
        canvas.addEventListener("contextmenu", function (event) {
            event.preventDefault(); // Prevent default context menu
            if (!isConnected || isProcessing) return;
            
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;

            sendInputState(x, y, false, true);
        });

        // Handle mouse wheel events
        canvas.addEventListener("wheel", function (event) {
            event.preventDefault(); // Prevent page scrolling
            if (!isConnected || isProcessing) return;
            
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;
            
            // Normalize wheel delta values (different browsers handle this differently)
            let deltaX = event.deltaX;
            let deltaY = event.deltaY;
            
            // Clamp values to reasonable range
            //deltaX = Math.max(-10, Math.min(10, deltaX));
            //deltaY = Math.max(-10, Math.min(10, deltaY));
            
            console.log(`Wheel event: deltaX=${deltaX}, deltaY=${deltaY} at (${x}, ${y})`);
            sendInputState(x, y, false, false, [], [], deltaX, deltaY);
        });

        // Track keyboard events
        const TROUBLESOME = new Set([
            "Tab",          // focus change
            " ",            // Space - scrolls page
            "Escape",       // Esc
            "ArrowUp",      // scroll up
            "ArrowDown",    // scroll down
            "ArrowLeft",    // horiz. scroll or browser Back with Alt/⌥
            "ArrowRight",   // horiz. scroll or browser Forward with Alt/⌥
            "Backspace",    // navigate back in some browsers
            "PageUp",       // scroll
            "PageDown",     // scroll
            "Home",         // scroll to top
            "End",           // scroll to bottom
            "Enter",
            "Delete",
            "/"
        ]);

        function isForbiddenCombo(e) {
            // Reload
            if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "r") return true;
            // Close tab
            if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "w") return true;
            return false;
        }

        document.addEventListener("keydown", function (event) {
            // Only prevent default if user has interacted with the canvas
            if (userHasInteracted && (TROUBLESOME.has(event.key) || isForbiddenCombo(event))) {
                event.preventDefault();
                event.stopPropagation();
            }
            if (!isConnected || isProcessing || !userHasInteracted) return;
            
            // Get the current mouse position
            let rect = canvas.getBoundingClientRect();
            let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
            let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
            
            sendInputState(x, y, false, false, [event.key], []);
        });

        document.addEventListener("keyup", function (event) {
            // Only prevent default if user has interacted with the canvas
            if (userHasInteracted && TROUBLESOME.has(event.key)) {
                event.preventDefault();
                event.stopPropagation();
            }
            if (!isConnected || socket.readyState !== WebSocket.OPEN || !userHasInteracted) return;
            
            // Get the current mouse position
            let rect = canvas.getBoundingClientRect();
            let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
            let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
            
            // For key up events, we send the key in the keys_up array
            sendInputState(x, y, false, false, [], [event.key]);
        });

        // Graceful disconnection
        window.addEventListener('beforeunload', function (e) {
            stopAutoInput(); // Clean up auto-input interval
            stopTimeoutCountdown(); // Clean up timeout countdown
            stopQueueCountdown(); // Clean up queue countdown
            if (isConnected) {
                try {
                    //socket.send(JSON.stringify({ type: "disconnect" }));
                    //socket.close();
                } catch (error) {
                    console.error("Error during disconnection:", error);
                }
            }
        });

        // Add event listener for the reset button
        document.getElementById('resetButton').addEventListener('click', function() {
            if (socket && socket.readyState === WebSocket.OPEN) {
                console.log("Sending reset command to server");
                socket.send(JSON.stringify({
                    type: "reset"
                }));
            } else {
                console.error("WebSocket not connected, cannot reset");
            }
        });
        
        // Add event listener for updating sampling steps
        document.getElementById('updateStepsButton').addEventListener('click', function() {
            const stepsInput = document.getElementById('samplingSteps');
            const newSteps = parseInt(stepsInput.value, 10);
            
            if (isNaN(newSteps) || newSteps < 1) {
                alert("Please enter a valid number of steps (minimum 1)");
                return;
            }
            
            if (socket && socket.readyState === WebSocket.OPEN) {
                console.log(`Sending update to set sampling steps to ${newSteps}`);
                socket.send(JSON.stringify({
                    type: "update_sampling_steps",
                    steps: newSteps
                }));
            } else {
                console.error("WebSocket not connected, cannot update steps");
            }
        });
        
        // Add event listener for the USE_RNN toggle
        document.getElementById('useRnnToggle').addEventListener('change', function() {
            const useRnn = this.checked;
            
            if (socket && socket.readyState === WebSocket.OPEN) {
                console.log(`Sending update to set USE_RNN to ${useRnn}`);
                socket.send(JSON.stringify({
                    type: "update_use_rnn",
                    use_rnn: useRnn
                }));
            } else {
                console.error("WebSocket not connected, cannot update USE_RNN");
            }
        });
        
        // Add event listener for the auto-input toggle
        document.getElementById('autoInputToggle').addEventListener('change', function() {
            autoInputEnabled = this.checked;
            console.log(`Auto-input ${autoInputEnabled ? 'enabled' : 'disabled'}`);
            
            if (autoInputEnabled) {
                // Reset the timers when enabling to start fresh
                updateLastUserInputTime();
                lastAutoInputTime = 0;
                autoInputActive = false;
                // Reset UI indicator
                try {
                    const label = document.getElementById('autoInputLabel');
                    if (label) {
                        label.textContent = "Auto Input";
                        label.style.color = "";
                    }
                } catch (error) {
                    console.error("Error updating auto-input UI:", error);
                }
            } else {
                // Stop auto-input mode when disabling
                autoInputActive = false;
                // Reset UI indicator
                try {
                    const label = document.getElementById('autoInputLabel');
                    if (label) {
                        label.textContent = "Auto Input";
                        label.style.color = "";
                    }
                } catch (error) {
                    console.error("Error updating auto-input UI:", error);
                }
            }
        });
    </script>
    
    <!-- Bootstrap JS (optional) -->
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
</body>
</html>