neural-os

Runtime error

File size: 31,115 Bytes

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>NeuralOS Demo</title>
    <!-- Bootstrap CSS -->
    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
    <style>
        body {
            padding: 20px;
            background-color: #f8f9fa;
        }
        
        .title-container {
            text-align: center;
            margin-bottom: 30px;
        }
        
        .canvas-container {
            display: flex;
            flex-direction: column;
            align-items: center;
            margin-bottom: 20px;
        }
        
        .canvas-wrapper {
            position: relative;
            border: 3px solid #007bff;
            border-radius: 8px;
            background-color: #f8f9fa;
            margin-bottom: 15px;
        }
        
        .instruction-box {
            position: absolute;
            top: -40px;
            left: 50%;
            transform: translateX(-50%);
            background-color: #007bff;
            color: white;
            padding: 5px 15px;
            border-radius: 5px;
            font-weight: bold;
            white-space: nowrap;
        }
        
        #displayCanvas {
            cursor: none;
            display: block;
        }
        
        .controls {
            margin-top: 20px;
            display: flex;
            gap: 15px;
            align-items: center;
            justify-content: center;
            flex-wrap: wrap;
        }
        
        .control-button {
            cursor: pointer;
        }
        
        .step-control {
            display: flex;
            align-items: center;
            gap: 10px;
        }
        
        #samplingSteps {
            width: 70px;
        }
        
        .footer {
            margin-top: 30px;
            text-align: center;
        }
    </style>
</head>
<body>
    <div class="container">
        <div class="title-container">
            <h1 class="mb-2">NeuralOS: Towards Simulating Operating Systems<br>via Neural Generative Models</h1>
            <p class="mb-3">
                <a href="https://anonymous.4open.science/r/neural-os" target="_blank" class="text-decoration-none">
                    Project Code: anonymous.4open.science/r/neural-os
                </a>
            </p>
        </div>
        
        <div class="canvas-container">
            <div id="timeoutWarning" class="alert alert-warning" style="display: none; margin-bottom: 15px;">
                <strong>Connection Timeout Warning:</strong> 
                No user activity detected. Connection will be dropped in <span id="timeoutCountdown">10</span> seconds and page will refresh automatically.
                <button type="button" class="btn btn-sm btn-primary ms-2" onclick="resetTimeout()">Stay Connected</button>
            </div>
            
            <div class="canvas-wrapper">
                <div class="instruction-box">Move your mouse inside to interact</div>
                <canvas id="displayCanvas" width="512" height="384"></canvas>
            </div>
            
            <div class="controls">
                <button id="resetButton" class="btn btn-primary control-button">Reset Simulation</button>
                
                <div class="step-control">
                    <label for="samplingSteps" class="form-label mb-0">Sampling Steps:</label>
                    <input type="number" id="samplingSteps" class="form-control" min="1" max="100" value="32">
                    <button id="updateStepsButton" class="btn btn-secondary control-button">Update</button>
                </div>
                
                <div class="form-check form-switch">
                    <input class="form-check-input" type="checkbox" role="switch" id="useRnnToggle">
                    <label class="form-check-label" for="useRnnToggle">Use RNN</label>
                </div>
                
                <div class="form-check form-switch">
                    <input class="form-check-input" type="checkbox" role="switch" id="autoInputToggle" checked>
                    <label class="form-check-label" for="autoInputToggle" id="autoInputLabel">Auto Input</label>
                </div>
            </div>
        </div>
        
        <div class="row justify-content-center">
            <div class="col-md-8">
                <div class="card">
                    <div class="card-body">
                        <h5 class="card-title">Instructions:</h5>
                        <ul class="mb-0">
                            <li>Move your mouse inside the blue box to interact with NeuralOS</li>
                            <li>Click (left-click or right-click) to perform click actions</li>
                            <li>Use your keyboard to type within the simulated environment</li>
                            <li>Adjust sampling steps to control quality/speed tradeoff</li>
                            <li>Toggle "Use RNN" to switch between RNN and diffusion mode</li>
                            <li>Toggle "Auto Input" to enable automatic frame generation (starts after 2s idle once you move mouse inside canvas, then runs every 0.5s)</li>
                        </ul>
                    </div>
                </div>
            </div>
        </div>
    </div>

    <script>
        const canvas = document.getElementById('displayCanvas');
        const ctx = canvas.getContext('2d');
        
        let socket;
        let isConnected = false;
        let reconnectAttempts = 0;
        const MAX_RECONNECT_DELAY = 30000; // Maximum delay between reconnection attempts (30 seconds)
        let connectionAttempted = false; // Flag to prevent multiple connection attempts

        let isProcessing = false;
        
        // Add flag to control trace visibility, default to false (hidden)
        let showTrace = false;

        // Function to draw the initial canvas state
        function drawInitialCanvas() {
            ctx.fillStyle = "#ffffff";
            ctx.fillRect(0, 0, canvas.width, canvas.height);
            ctx.font = "18px Arial";
            ctx.fillStyle = "#666666";
            ctx.textAlign = "center";
            ctx.fillText("Move your mouse here to interact with NeuralOS", canvas.width/2, canvas.height/2 - 10);
            ctx.fillText("The neural model will render in this area", canvas.width/2, canvas.height/2 + 20);
        }

        // Function to show connection status
        function showConnectionStatus(message) {
            ctx.fillStyle = "#ffffff";
            ctx.fillRect(0, 0, canvas.width, canvas.height);
            ctx.font = "18px Arial";
            ctx.fillStyle = "#666666";
            ctx.textAlign = "center";
            ctx.fillText(message, canvas.width/2, canvas.height/2);
        }

        function connect() {
            console.log("connect() function called - creating new WebSocket connection");
            const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
            socket = new WebSocket(`${protocol}//${window.location.host}/ws`);

            socket.onopen = function(event) {
                console.log("WebSocket connection established");
                isConnected = true;
                //connectionAttempted = false; // Reset flag for potential reconnections
                //reconnectAttempts = 0;
                
                // Show brief success message
                //showConnectionStatus("Connected! Processing input...");
                //setTimeout(() => {
                //    // Clear the message after 0.5 second
                //    if (isConnected) {
                //        ctx.clearRect(0, 0, canvas.width, canvas.height);
                //    }
                //}, 500);
                
                // Start auto-input mechanism
                startAutoInput();
                
                // Request current settings from server to sync UI
                socket.send(JSON.stringify({
                    type: "get_settings"
                }));
                
                //startHeartbeat();
            };

            socket.onclose = function(event) {
                console.log("WebSocket connection closed. Attempting to reconnect...");
                console.log("Close event code:", event.code, "reason:", event.reason);
                
                // Check if this was a timeout closure
                if (event.code === 1000 && event.reason === "User inactivity timeout") {
                    console.log("Connection closed due to timeout - refreshing page");
                    stopAutoInput(); // Stop auto-input when connection is lost
                    stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
                    //clearInterval(heartbeatInterval);
                    // Refresh the page to reconnect
                    window.location.reload();
                    return;
                }
                
                isConnected = false;
                //connectionAttempted = false; // Reset flag to allow reconnection attempts
                stopAutoInput(); // Stop auto-input when connection is lost
                stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
                //clearInterval(heartbeatInterval);
                //scheduleReconnection();
            };

            socket.onerror = function(error) {
                console.error("WebSocket error:", error);
            };

            socket.onmessage = function (event) {
                const data = JSON.parse(event.data);
                if (data.type === "heartbeat_response") {
                    console.log("Heartbeat response received");
                } else if (data.image) {
                    let img = new Image();
                    img.onload = function() {
                        ctx.clearRect(0, 0, canvas.width, canvas.height);
                        ctx.drawImage(img, 0, 0);
                        //isProcessing = false; // Reset the processing flag when we get a response
                    };
                    img.src = 'data:image/png;base64,' + data.image;
                } else if (data.type === "reset_confirmed") {
                    console.log("Simulation reset confirmed by server");
                    // Clear the canvas and reset to initial state
                    drawInitialCanvas();
                    // Reset user interaction state
                    userHasInteracted = false;
                    lastSentPosition = null;
                    console.log("Reset user interaction state - waiting for user to move mouse inside canvas");
                } else if (data.type === "rnn_updated") {
                    console.log(`USE_RNN setting updated to: ${data.use_rnn}`);
                    // Update the toggle to match the server state
                    document.getElementById('useRnnToggle').checked = data.use_rnn;
                } else if (data.type === "settings") {
                    // Update UI elements to match server settings
                    console.log(`Received settings from server: SAMPLING_STEPS=${data.sampling_steps}, USE_RNN=${data.use_rnn}`);
                    document.getElementById('samplingSteps').value = data.sampling_steps;
                    document.getElementById('useRnnToggle').checked = data.use_rnn;
                } else if (data.type === "timeout_warning") {
                    console.log(`Received timeout warning: ${data.timeout_in} seconds remaining`);
                    startTimeoutCountdown();
                } else if (data.type === "activity_reset") {
                    console.log("Server detected user activity, resetting timeout");
                    stopTimeoutCountdown();
                }
            };
        }

        function scheduleReconnection() {
            const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), MAX_RECONNECT_DELAY);
            console.log(`Scheduling reconnection in ${delay}ms`);
            setTimeout(connect, delay);
            reconnectAttempts++;
        }

        let heartbeatInterval;
        function startHeartbeat() {
            heartbeatInterval = setInterval(() => {
                if (isConnected) {
                    try {
                        socket.send(JSON.stringify({ type: "heartbeat" }));
                        console.error("finished sending heartbeat");

                    } catch (error) {
                        console.error("Error sending heartbeat:", error);
                    }
                }
            }, 1000); // Send heartbeat every 15 seconds
        }

        // Don't connect immediately - wait for user interaction
        // connect();

        // Draw initial state on the canvas
        drawInitialCanvas();

        let lastSentPosition = null;
        let lastSentTime = 0;
        const SEND_INTERVAL = 10; // Send updates every 50ms
        
        // Auto-input mechanism
        let lastUserInputTime = 0;
        let lastAutoInputTime = 0;
        let autoInputInterval = null;
        let autoInputActive = false; // Track if auto-input mode is active
        const INITIAL_AUTO_INPUT_DELAY = 2000; // Start auto-input after 2 seconds of no user input
        const AUTO_INPUT_INTERVAL = 500; // Send auto-input every 0.5 second once active
        let autoInputEnabled = true; // Default to enabled
        let userHasInteracted = false; // Track if user has moved mouse inside canvas
        
        // Timeout countdown mechanism
        let timeoutCountdownInterval = null;
        let timeoutCountdown = 10;
        let timeoutWarningActive = false;
        
        function startAutoInput() {
            if (autoInputInterval) {
                clearInterval(autoInputInterval);
            }
            
            autoInputInterval = setInterval(() => {
                const currentTime = Date.now();
                
                if (!autoInputEnabled || !lastSentPosition || !isConnected || socket.readyState !== WebSocket.OPEN || !userHasInteracted) {
                    return;
                }
                
                // Check if we should start auto-input mode
                if (!autoInputActive && currentTime - lastUserInputTime >= INITIAL_AUTO_INPUT_DELAY) {
                    console.log("Starting auto-input mode (no user activity for 2 seconds)");
                    autoInputActive = true;
                    lastAutoInputTime = currentTime;
                    // Update UI to show auto-input is active
                    try {
                        const label = document.getElementById('autoInputLabel');
                        if (label) {
                            label.textContent = "Auto Input (Active)";
                            label.style.color = "#28a745";
                        }
                    } catch (error) {
                        console.error("Error updating auto-input UI:", error);
                    }
                }
                
                // Send auto-input if mode is active and enough time has passed
                if (autoInputActive && currentTime - lastAutoInputTime >= AUTO_INPUT_INTERVAL) {
                    console.log("Sending auto-input (auto-input mode active)");
                    try {
                        socket.send(JSON.stringify({
                            "x": lastSentPosition.x,
                            "y": lastSentPosition.y,
                            "is_left_click": false,
                            "is_right_click": false,
                            "keys_down": [],
                            "keys_up": [],
                            "is_auto_input": true  // Flag to identify auto-generated inputs
                        }));
                        lastAutoInputTime = currentTime;
                    } catch (error) {
                        console.error("Error sending auto-input:", error);
                        // Stop auto-input if there's an error (connection likely closed)
                        stopAutoInput();
                    }
                }
            }, 100); // Check every 100ms
        }

        function stopAutoInput() {
            if (autoInputInterval) {
                clearInterval(autoInputInterval);
                autoInputInterval = null;
            }
            autoInputActive = false; // Reset auto-input mode
        }

        function updateLastUserInputTime() {
            lastUserInputTime = Date.now();
            // Reset auto-input mode when user provides input
            if (autoInputActive) {
                console.log("User activity detected, stopping auto-input mode");
                autoInputActive = false;
                // Reset UI indicator
                try {
                    const label = document.getElementById('autoInputLabel');
                    if (label) {
                        label.textContent = "Auto Input";
                        label.style.color = "";
                    }
                } catch (error) {
                    console.error("Error updating auto-input UI:", error);
                }
            }
        }

        function startTimeoutCountdown() {
            if (timeoutCountdownInterval) {
                clearInterval(timeoutCountdownInterval);
            }
            
            timeoutCountdown = 10;
            timeoutWarningActive = true;
            
            // Show warning
            const warning = document.getElementById('timeoutWarning');
            if (warning) {
                warning.style.display = 'block';
            }
            
            // Start countdown
            timeoutCountdownInterval = setInterval(() => {
                timeoutCountdown--;
                const countdownElement = document.getElementById('timeoutCountdown');
                if (countdownElement) {
                    countdownElement.textContent = timeoutCountdown;
                }
                
                if (timeoutCountdown <= 0) {
                    stopAutoInput();
                    stopTimeoutCountdown();
                    //clearInterval(heartbeatInterval);
                    console.log("Connection timeout countdown finished - refreshing page");
                    // Refresh the page to reconnect
                    window.location.reload();
                }
            }, 1000);
        }

        function stopTimeoutCountdown() {
            if (timeoutCountdownInterval) {
                clearInterval(timeoutCountdownInterval);
                timeoutCountdownInterval = null;
            }
            
            timeoutWarningActive = false;
            timeoutCountdown = 10;
            
            // Reset the countdown display to 10 seconds
            const countdownElement = document.getElementById('timeoutCountdown');
            if (countdownElement) {
                countdownElement.textContent = timeoutCountdown;
            }
            
            // Hide warning
            const warning = document.getElementById('timeoutWarning');
            if (warning) {
                warning.style.display = 'none';
            }
        }

        function resetTimeout() {
            // Send a regular input to reset the server's timeout
            if (socket && socket.readyState === WebSocket.OPEN && lastSentPosition) {
                try {
                    socket.send(JSON.stringify({
                        "x": lastSentPosition.x,
                        "y": lastSentPosition.y,
                        "is_left_click": false,
                        "is_right_click": false,
                        "keys_down": [],
                        "keys_up": []
                    }));
                    updateLastUserInputTime(); // Update for auto-input mechanism
                } catch (error) {
                    console.error("Error sending timeout reset input:", error);
                }
            }
            stopTimeoutCountdown();
        }

        function sendInputState(x, y, isLeftClick = false, isRightClick = false, keysDownArr = [], keysUpArr = []) {
            const currentTime = Date.now();
            if (isConnected && socket.readyState === WebSocket.OPEN && (isLeftClick || isRightClick || keysDownArr.length > 0 || keysUpArr.length > 0 || !lastSentPosition || currentTime - lastSentTime >= SEND_INTERVAL)) {
                try {
                    socket.send(JSON.stringify({
                        "x": x,
                        "y": y,
                        "is_left_click": isLeftClick,
                        "is_right_click": isRightClick,
                        "keys_down": keysDownArr,
                        "keys_up": keysUpArr,
                    }));
                    lastSentPosition = { x, y };
                    lastSentTime = currentTime;
                    
                    // Mark that user has interacted with the canvas
                    if (!userHasInteracted) {
                        userHasInteracted = true;
                        console.log("User has interacted with canvas for the first time");
                    }
                    
                    // Update last user input time for auto-input mechanism
                    updateLastUserInputTime();
                    
                    //if (isLeftClick || isRightClick) {
                    //    isProcessing = true; // Block further inputs until response
                    //}
                } catch (error) {
                    console.error("Error sending input state:", error);
                }
            }
        }

        // Capture mouse movements and clicks
        canvas.addEventListener("mousemove", async function (event) {
            // Establish connection on first mouse movement if not already connected
            if (!isConnected && !connectionAttempted) {
                console.log("First mouse movement detected - establishing WebSocket connection");
                connectionAttempted = true;
                showConnectionStatus("Connecting to NeuralOS...");
                connect();
                
                // Wait for connection to be established
                let attempts = 0;
                const maxAttempts = 50; // 5 seconds max wait
                while (!isConnected && attempts < maxAttempts) {
                    await new Promise(resolve => setTimeout(resolve, 100)); // Wait 100ms
                    attempts++;
                }
                
                if (!isConnected) {
                    console.error("Failed to establish connection after 5 seconds");
                    connectionAttempted = false; // Reset flag to allow retry
                    showConnectionStatus("Connection failed. Move mouse to retry.");
                    return;
                }
                
                console.log("Connection established, processing first mouse movement");
            }
            
            if (!isConnected || isProcessing) return;
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;
            
            // Only draw the trace if showTrace is true
            if (showTrace && lastSentPosition) {
                ctx.beginPath();
                ctx.moveTo(lastSentPosition.x, lastSentPosition.y);
                ctx.lineTo(x, y);
                ctx.stroke();
            }

            sendInputState(x, y);
        });

        canvas.addEventListener("click", function (event) {
            if (!isConnected || isProcessing) return;
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;

            sendInputState(x, y, true, false);
        });

        // Handle right clicks
        canvas.addEventListener("contextmenu", function (event) {
            event.preventDefault(); // Prevent default context menu
            if (!isConnected || isProcessing) return;
            
            let rect = canvas.getBoundingClientRect();
            let x = event.clientX - rect.left;
            let y = event.clientY - rect.top;

            sendInputState(x, y, false, true);
        });

        // Track keyboard events
        const TROUBLESOME = new Set([
            "Tab",          // focus change
            " ",            // Space - scrolls page
            "Escape",       // Esc
            "ArrowUp",      // scroll up
            "ArrowDown",    // scroll down
            "ArrowLeft",    // horiz. scroll or browser Back with Alt/⌥
            "ArrowRight",   // horiz. scroll or browser Forward with Alt/⌥
            "Backspace",    // navigate back in some browsers
            "PageUp",       // scroll
            "PageDown",     // scroll
            "Home",         // scroll to top
            "End",           // scroll to bottom
            "Enter",
            "Delete"
        ]);

        function isForbiddenCombo(e) {
            // Reload
            if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "r") return true;
            // Close tab
            if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "w") return true;
            return false;
        }

        document.addEventListener("keydown", function (event) {
            if (TROUBLESOME.has(event.key) || isForbiddenCombo(event)) {
                event.preventDefault();
                event.stopPropagation();
            }
            if (!isConnected || isProcessing) return;
            
            // Get the current mouse position
            let rect = canvas.getBoundingClientRect();
            let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
            let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
            
            sendInputState(x, y, false, false, [event.key], []);
        });

        document.addEventListener("keyup", function (event) {
            if (TROUBLESOME.has(event.key)) {
                event.preventDefault();
                event.stopPropagation();
            }
            if (!isConnected || socket.readyState !== WebSocket.OPEN) return;
            
            // Get the current mouse position
            let rect = canvas.getBoundingClientRect();
            let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
            let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
            
            // For key up events, we send the key in the keys_up array
            sendInputState(x, y, false, false, [], [event.key]);
        });

        // Graceful disconnection
        window.addEventListener('beforeunload', function (e) {
            stopAutoInput(); // Clean up auto-input interval
            stopTimeoutCountdown(); // Clean up timeout countdown
            if (isConnected) {
                try {
                    //socket.send(JSON.stringify({ type: "disconnect" }));
                    //socket.close();
                } catch (error) {
                    console.error("Error during disconnection:", error);
                }
            }
        });

        // Add event listener for the reset button
        document.getElementById('resetButton').addEventListener('click', function() {
            if (socket && socket.readyState === WebSocket.OPEN) {
                console.log("Sending reset command to server");
                socket.send(JSON.stringify({
                    type: "reset"
                }));
            } else {
                console.error("WebSocket not connected, cannot reset");
            }
        });
        
        // Add event listener for updating sampling steps
        document.getElementById('updateStepsButton').addEventListener('click', function() {
            const stepsInput = document.getElementById('samplingSteps');
            const newSteps = parseInt(stepsInput.value, 10);
            
            if (isNaN(newSteps) || newSteps < 1) {
                alert("Please enter a valid number of steps (minimum 1)");
                return;
            }
            
            if (socket && socket.readyState === WebSocket.OPEN) {
                console.log(`Sending update to set sampling steps to ${newSteps}`);
                socket.send(JSON.stringify({
                    type: "update_sampling_steps",
                    steps: newSteps
                }));
            } else {
                console.error("WebSocket not connected, cannot update steps");
            }
        });
        
        // Add event listener for the USE_RNN toggle
        document.getElementById('useRnnToggle').addEventListener('change', function() {
            const useRnn = this.checked;
            
            if (socket && socket.readyState === WebSocket.OPEN) {
                console.log(`Sending update to set USE_RNN to ${useRnn}`);
                socket.send(JSON.stringify({
                    type: "update_use_rnn",
                    use_rnn: useRnn
                }));
            } else {
                console.error("WebSocket not connected, cannot update USE_RNN");
            }
        });
        
        // Add event listener for the auto-input toggle
        document.getElementById('autoInputToggle').addEventListener('change', function() {
            autoInputEnabled = this.checked;
            console.log(`Auto-input ${autoInputEnabled ? 'enabled' : 'disabled'}`);
            
            if (autoInputEnabled) {
                // Reset the timers when enabling to start fresh
                updateLastUserInputTime();
                lastAutoInputTime = 0;
                autoInputActive = false;
                // Reset UI indicator
                try {
                    const label = document.getElementById('autoInputLabel');
                    if (label) {
                        label.textContent = "Auto Input";
                        label.style.color = "";
                    }
                } catch (error) {
                    console.error("Error updating auto-input UI:", error);
                }
            } else {
                // Stop auto-input mode when disabling
                autoInputActive = false;
                // Reset UI indicator
                try {
                    const label = document.getElementById('autoInputLabel');
                    if (label) {
                        label.textContent = "Auto Input";
                        label.style.color = "";
                    }
                } catch (error) {
                    console.error("Error updating auto-input UI:", error);
                }
            }
        });
    </script>
    
    <!-- Bootstrap JS (optional) -->
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
</body>
</html>