neural-os / static /index.html
da03
.
2d37f4f
raw
history blame
30.5 kB
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NeuralOS Demo</title>
<!-- Bootstrap CSS -->
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
<style>
body {
padding: 20px;
background-color: #f8f9fa;
}
.title-container {
text-align: center;
margin-bottom: 30px;
}
.canvas-container {
display: flex;
flex-direction: column;
align-items: center;
margin-bottom: 20px;
}
.canvas-wrapper {
position: relative;
border: 3px solid #007bff;
border-radius: 8px;
background-color: #f8f9fa;
margin-bottom: 15px;
}
.instruction-box {
position: absolute;
top: -40px;
left: 50%;
transform: translateX(-50%);
background-color: #007bff;
color: white;
padding: 5px 15px;
border-radius: 5px;
font-weight: bold;
white-space: nowrap;
}
#displayCanvas {
cursor: none;
display: block;
}
.controls {
margin-top: 20px;
display: flex;
gap: 15px;
align-items: center;
justify-content: center;
flex-wrap: wrap;
}
.control-button {
cursor: pointer;
}
.step-control {
display: flex;
align-items: center;
gap: 10px;
}
#samplingSteps {
width: 70px;
}
.footer {
margin-top: 30px;
text-align: center;
}
</style>
</head>
<body>
<div class="container">
<div class="title-container">
<h1 class="mb-2">NeuralOS: Towards Simulating Operating Systems<br>via Neural Generative Models</h1>
<p class="mb-3">
<a href="https://anonymous.4open.science/r/neural-os" target="_blank" class="text-decoration-none">
Project Code: anonymous.4open.science/r/neural-os
</a>
</p>
</div>
<div class="canvas-container">
<div id="timeoutWarning" class="alert alert-warning" style="display: none; margin-bottom: 15px;">
<strong>Connection Timeout Warning:</strong>
No user activity detected. Connection will be dropped in <span id="timeoutCountdown">10</span> seconds and page will refresh automatically.
<button type="button" class="btn btn-sm btn-primary ms-2" onclick="resetTimeout()">Stay Connected</button>
</div>
<div class="canvas-wrapper">
<div class="instruction-box">Move your mouse inside to interact</div>
<canvas id="displayCanvas" width="512" height="384"></canvas>
</div>
<div class="controls">
<button id="resetButton" class="btn btn-primary control-button">Reset Simulation</button>
<div class="step-control">
<label for="samplingSteps" class="form-label mb-0">Sampling Steps:</label>
<input type="number" id="samplingSteps" class="form-control" min="1" max="100" value="32">
<button id="updateStepsButton" class="btn btn-secondary control-button">Update</button>
</div>
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="useRnnToggle">
<label class="form-check-label" for="useRnnToggle">Use RNN</label>
</div>
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="autoInputToggle" checked>
<label class="form-check-label" for="autoInputToggle" id="autoInputLabel">Auto Input</label>
</div>
</div>
</div>
<div class="row justify-content-center">
<div class="col-md-8">
<div class="card">
<div class="card-body">
<h5 class="card-title">Instructions:</h5>
<ul class="mb-0">
<li>Move your mouse inside the blue box to interact with NeuralOS</li>
<li>Click (left-click or right-click) to perform click actions</li>
<li>Use your keyboard to type within the simulated environment</li>
<li>Adjust sampling steps to control quality/speed tradeoff</li>
<li>Toggle "Use RNN" to switch between RNN and diffusion mode</li>
<li>Toggle "Auto Input" to enable automatic frame generation (starts after 2s idle once you move mouse inside canvas, then runs every 0.5s)</li>
</ul>
</div>
</div>
</div>
</div>
</div>
<script>
const canvas = document.getElementById('displayCanvas');
const ctx = canvas.getContext('2d');
let socket;
let isConnected = false;
let reconnectAttempts = 0;
const MAX_RECONNECT_DELAY = 30000; // Maximum delay between reconnection attempts (30 seconds)
let connectionAttempted = false; // Flag to prevent multiple connection attempts
let isProcessing = false;
// Add flag to control trace visibility, default to false (hidden)
let showTrace = false;
// Function to draw the initial canvas state
function drawInitialCanvas() {
ctx.fillStyle = "#ffffff";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.font = "18px Arial";
ctx.fillStyle = "#666666";
ctx.textAlign = "center";
ctx.fillText("Move your mouse here to interact with NeuralOS", canvas.width/2, canvas.height/2 - 10);
ctx.fillText("The neural model will render in this area", canvas.width/2, canvas.height/2 + 20);
}
// Function to show connection status
function showConnectionStatus(message) {
ctx.fillStyle = "#ffffff";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.font = "18px Arial";
ctx.fillStyle = "#666666";
ctx.textAlign = "center";
ctx.fillText(message, canvas.width/2, canvas.height/2);
}
function connect() {
console.log("connect() function called - creating new WebSocket connection");
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
socket = new WebSocket(`${protocol}//${window.location.host}/ws`);
socket.onopen = function(event) {
console.log("WebSocket connection established");
isConnected = true;
//connectionAttempted = false; // Reset flag for potential reconnections
//reconnectAttempts = 0;
// Show brief success message
//showConnectionStatus("Connected! Processing input...");
//setTimeout(() => {
// // Clear the message after 0.5 second
// if (isConnected) {
// ctx.clearRect(0, 0, canvas.width, canvas.height);
// }
//}, 500);
// Start auto-input mechanism
startAutoInput();
// Request current settings from server to sync UI
socket.send(JSON.stringify({
type: "get_settings"
}));
//startHeartbeat();
};
socket.onclose = function(event) {
console.log("WebSocket connection closed. Attempting to reconnect...");
console.log("Close event code:", event.code, "reason:", event.reason);
// Check if this was a timeout closure
if (event.code === 1000 && event.reason === "User inactivity timeout") {
console.log("Connection closed due to timeout - refreshing page");
stopAutoInput(); // Stop auto-input when connection is lost
stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
//clearInterval(heartbeatInterval);
// Refresh the page to reconnect
window.location.reload();
return;
}
isConnected = false;
//connectionAttempted = false; // Reset flag to allow reconnection attempts
stopAutoInput(); // Stop auto-input when connection is lost
stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
//clearInterval(heartbeatInterval);
//scheduleReconnection();
};
socket.onerror = function(error) {
console.error("WebSocket error:", error);
};
socket.onmessage = function (event) {
const data = JSON.parse(event.data);
if (data.type === "heartbeat_response") {
console.log("Heartbeat response received");
} else if (data.image) {
let img = new Image();
img.onload = function() {
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
//isProcessing = false; // Reset the processing flag when we get a response
};
img.src = 'data:image/png;base64,' + data.image;
} else if (data.type === "reset_confirmed") {
console.log("Simulation reset confirmed by server");
// Clear the canvas and reset to initial state
drawInitialCanvas();
// Reset user interaction state
userHasInteracted = false;
lastSentPosition = null;
console.log("Reset user interaction state - waiting for user to move mouse inside canvas");
} else if (data.type === "rnn_updated") {
console.log(`USE_RNN setting updated to: ${data.use_rnn}`);
// Update the toggle to match the server state
document.getElementById('useRnnToggle').checked = data.use_rnn;
} else if (data.type === "settings") {
// Update UI elements to match server settings
console.log(`Received settings from server: SAMPLING_STEPS=${data.sampling_steps}, USE_RNN=${data.use_rnn}`);
document.getElementById('samplingSteps').value = data.sampling_steps;
document.getElementById('useRnnToggle').checked = data.use_rnn;
} else if (data.type === "timeout_warning") {
console.log(`Received timeout warning: ${data.timeout_in} seconds remaining`);
startTimeoutCountdown();
} else if (data.type === "activity_reset") {
console.log("Server detected user activity, resetting timeout");
stopTimeoutCountdown();
}
};
}
function scheduleReconnection() {
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), MAX_RECONNECT_DELAY);
console.log(`Scheduling reconnection in ${delay}ms`);
setTimeout(connect, delay);
reconnectAttempts++;
}
let heartbeatInterval;
function startHeartbeat() {
heartbeatInterval = setInterval(() => {
if (isConnected) {
try {
socket.send(JSON.stringify({ type: "heartbeat" }));
console.error("finished sending heartbeat");
} catch (error) {
console.error("Error sending heartbeat:", error);
}
}
}, 1000); // Send heartbeat every 15 seconds
}
// Don't connect immediately - wait for user interaction
// connect();
// Draw initial state on the canvas
drawInitialCanvas();
let lastSentPosition = null;
let lastSentTime = 0;
const SEND_INTERVAL = 10; // Send updates every 50ms
// Auto-input mechanism
let lastUserInputTime = 0;
let lastAutoInputTime = 0;
let autoInputInterval = null;
let autoInputActive = false; // Track if auto-input mode is active
const INITIAL_AUTO_INPUT_DELAY = 2000; // Start auto-input after 2 seconds of no user input
const AUTO_INPUT_INTERVAL = 500; // Send auto-input every 0.5 second once active
let autoInputEnabled = true; // Default to enabled
let userHasInteracted = false; // Track if user has moved mouse inside canvas
// Timeout countdown mechanism
let timeoutCountdownInterval = null;
let timeoutCountdown = 10;
let timeoutWarningActive = false;
// Track currently pressed keys
const pressedKeys = new Set();
function startAutoInput() {
if (autoInputInterval) {
clearInterval(autoInputInterval);
}
autoInputInterval = setInterval(() => {
const currentTime = Date.now();
if (!autoInputEnabled || !lastSentPosition || !isConnected || socket.readyState !== WebSocket.OPEN || !userHasInteracted) {
return;
}
// Check if we should start auto-input mode
if (!autoInputActive && currentTime - lastUserInputTime >= INITIAL_AUTO_INPUT_DELAY) {
console.log("Starting auto-input mode (no user activity for 2 seconds)");
autoInputActive = true;
lastAutoInputTime = currentTime;
// Update UI to show auto-input is active
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input (Active)";
label.style.color = "#28a745";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
}
// Send auto-input if mode is active and enough time has passed
if (autoInputActive && currentTime - lastAutoInputTime >= AUTO_INPUT_INTERVAL) {
console.log("Sending auto-input (auto-input mode active)");
try {
socket.send(JSON.stringify({
"x": lastSentPosition.x,
"y": lastSentPosition.y,
"is_left_click": false,
"is_right_click": false,
"keys_down": [],
"keys_up": [],
"is_auto_input": true // Flag to identify auto-generated inputs
}));
lastAutoInputTime = currentTime;
} catch (error) {
console.error("Error sending auto-input:", error);
// Stop auto-input if there's an error (connection likely closed)
stopAutoInput();
}
}
}, 100); // Check every 100ms
}
function stopAutoInput() {
if (autoInputInterval) {
clearInterval(autoInputInterval);
autoInputInterval = null;
}
autoInputActive = false; // Reset auto-input mode
}
function updateLastUserInputTime() {
lastUserInputTime = Date.now();
// Reset auto-input mode when user provides input
if (autoInputActive) {
console.log("User activity detected, stopping auto-input mode");
autoInputActive = false;
// Reset UI indicator
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input";
label.style.color = "";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
}
}
function startTimeoutCountdown() {
if (timeoutCountdownInterval) {
clearInterval(timeoutCountdownInterval);
}
timeoutCountdown = 10;
timeoutWarningActive = true;
// Show warning
const warning = document.getElementById('timeoutWarning');
if (warning) {
warning.style.display = 'block';
}
// Start countdown
timeoutCountdownInterval = setInterval(() => {
timeoutCountdown--;
const countdownElement = document.getElementById('timeoutCountdown');
if (countdownElement) {
countdownElement.textContent = timeoutCountdown;
}
if (timeoutCountdown <= 0) {
stopAutoInput();
stopTimeoutCountdown();
//clearInterval(heartbeatInterval);
console.log("Connection timeout countdown finished - refreshing page");
// Refresh the page to reconnect
window.location.reload();
}
}, 1000);
}
function stopTimeoutCountdown() {
if (timeoutCountdownInterval) {
clearInterval(timeoutCountdownInterval);
timeoutCountdownInterval = null;
}
timeoutWarningActive = false;
timeoutCountdown = 10;
// Reset the countdown display to 10 seconds
const countdownElement = document.getElementById('timeoutCountdown');
if (countdownElement) {
countdownElement.textContent = timeoutCountdown;
}
// Hide warning
const warning = document.getElementById('timeoutWarning');
if (warning) {
warning.style.display = 'none';
}
}
function resetTimeout() {
// Send a regular input to reset the server's timeout
if (socket && socket.readyState === WebSocket.OPEN && lastSentPosition) {
try {
socket.send(JSON.stringify({
"x": lastSentPosition.x,
"y": lastSentPosition.y,
"is_left_click": false,
"is_right_click": false,
"keys_down": [],
"keys_up": []
}));
updateLastUserInputTime(); // Update for auto-input mechanism
} catch (error) {
console.error("Error sending timeout reset input:", error);
}
}
stopTimeoutCountdown();
}
function sendInputState(x, y, isLeftClick = false, isRightClick = false) {
const currentTime = Date.now();
if (isConnected && socket.readyState === WebSocket.OPEN && (isLeftClick || isRightClick || !lastSentPosition || currentTime - lastSentTime >= SEND_INTERVAL)) {
try {
socket.send(JSON.stringify({
"x": x,
"y": y,
"is_left_click": isLeftClick,
"is_right_click": isRightClick,
"keys_down": Array.from(pressedKeys),
"keys_up": [],
}));
lastSentPosition = { x, y };
lastSentTime = currentTime;
// Mark that user has interacted with the canvas
if (!userHasInteracted) {
userHasInteracted = true;
console.log("User has interacted with canvas for the first time");
}
// Update last user input time for auto-input mechanism
updateLastUserInputTime();
//if (isLeftClick || isRightClick) {
// isProcessing = true; // Block further inputs until response
//}
} catch (error) {
console.error("Error sending input state:", error);
}
}
}
// Capture mouse movements and clicks
canvas.addEventListener("mousemove", async function (event) {
// Establish connection on first mouse movement if not already connected
if (!isConnected && !connectionAttempted) {
console.log("First mouse movement detected - establishing WebSocket connection");
connectionAttempted = true;
showConnectionStatus("Connecting to NeuralOS...");
connect();
// Wait for connection to be established
let attempts = 0;
const maxAttempts = 50; // 5 seconds max wait
while (!isConnected && attempts < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, 100)); // Wait 100ms
attempts++;
}
if (!isConnected) {
console.error("Failed to establish connection after 5 seconds");
connectionAttempted = false; // Reset flag to allow retry
showConnectionStatus("Connection failed. Move mouse to retry.");
return;
}
console.log("Connection established, processing first mouse movement");
}
if (!isConnected || isProcessing) return;
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
// Only draw the trace if showTrace is true
if (showTrace && lastSentPosition) {
ctx.beginPath();
ctx.moveTo(lastSentPosition.x, lastSentPosition.y);
ctx.lineTo(x, y);
ctx.stroke();
}
sendInputState(x, y);
});
canvas.addEventListener("click", function (event) {
if (!isConnected || isProcessing) return;
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
sendInputState(x, y, true, false);
});
// Handle right clicks
canvas.addEventListener("contextmenu", function (event) {
event.preventDefault(); // Prevent default context menu
if (!isConnected || isProcessing) return;
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
sendInputState(x, y, false, true);
});
// Track keyboard events
document.addEventListener("keydown", function (event) {
if (!isConnected || isProcessing) return;
// Add the key to our set of pressed keys
pressedKeys.add(event.key);
// Get the current mouse position
let rect = canvas.getBoundingClientRect();
let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
sendInputState(x, y);
});
document.addEventListener("keyup", function (event) {
if (!isConnected || socket.readyState !== WebSocket.OPEN) return;
// Remove the key from our set of pressed keys
pressedKeys.delete(event.key);
// Get the current mouse position
let rect = canvas.getBoundingClientRect();
let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
// For key up events, we send the key in the keys_up array
try {
socket.send(JSON.stringify({
"x": x,
"y": y,
"is_left_click": false,
"is_right_click": false,
"keys_down": Array.from(pressedKeys),
"keys_up": [event.key],
}));
updateLastUserInputTime(); // Update for auto-input mechanism
} catch (error) {
console.error("Error sending key up event:", error);
}
});
// Graceful disconnection
window.addEventListener('beforeunload', function (e) {
stopAutoInput(); // Clean up auto-input interval
stopTimeoutCountdown(); // Clean up timeout countdown
if (isConnected) {
try {
//socket.send(JSON.stringify({ type: "disconnect" }));
//socket.close();
} catch (error) {
console.error("Error during disconnection:", error);
}
}
});
// Add event listener for the reset button
document.getElementById('resetButton').addEventListener('click', function() {
if (socket && socket.readyState === WebSocket.OPEN) {
console.log("Sending reset command to server");
socket.send(JSON.stringify({
type: "reset"
}));
} else {
console.error("WebSocket not connected, cannot reset");
}
});
// Add event listener for updating sampling steps
document.getElementById('updateStepsButton').addEventListener('click', function() {
const stepsInput = document.getElementById('samplingSteps');
const newSteps = parseInt(stepsInput.value, 10);
if (isNaN(newSteps) || newSteps < 1) {
alert("Please enter a valid number of steps (minimum 1)");
return;
}
if (socket && socket.readyState === WebSocket.OPEN) {
console.log(`Sending update to set sampling steps to ${newSteps}`);
socket.send(JSON.stringify({
type: "update_sampling_steps",
steps: newSteps
}));
} else {
console.error("WebSocket not connected, cannot update steps");
}
});
// Add event listener for the USE_RNN toggle
document.getElementById('useRnnToggle').addEventListener('change', function() {
const useRnn = this.checked;
if (socket && socket.readyState === WebSocket.OPEN) {
console.log(`Sending update to set USE_RNN to ${useRnn}`);
socket.send(JSON.stringify({
type: "update_use_rnn",
use_rnn: useRnn
}));
} else {
console.error("WebSocket not connected, cannot update USE_RNN");
}
});
// Add event listener for the auto-input toggle
document.getElementById('autoInputToggle').addEventListener('change', function() {
autoInputEnabled = this.checked;
console.log(`Auto-input ${autoInputEnabled ? 'enabled' : 'disabled'}`);
if (autoInputEnabled) {
// Reset the timers when enabling to start fresh
updateLastUserInputTime();
lastAutoInputTime = 0;
autoInputActive = false;
// Reset UI indicator
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input";
label.style.color = "";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
} else {
// Stop auto-input mode when disabling
autoInputActive = false;
// Reset UI indicator
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input";
label.style.color = "";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
}
});
</script>
<!-- Bootstrap JS (optional) -->
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
</body>
</html>