Hugging Face's logo

Spaces:
yuntian-group
/
neural-os
Runtime error

App Files Community
neural-os / static /index.html
da03
.
e733937
raw
history blame
50.9 kB
 <!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NeuralOS Demo</title>
<!-- Bootstrap CSS -->
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
<style>
body {
padding: 20px;
background-color: #f8f9fa;
}
.title-container {
text-align: center;
margin-bottom: 30px;
}
.canvas-container {
display: flex;
flex-direction: column;
align-items: center;
margin-bottom: 20px;
}
.canvas-wrapper {
position: relative;
border: 3px solid #007bff;
border-radius: 8px;
background-color: #f8f9fa;
margin-bottom: 15px;
}
.instruction-box {
position: absolute;
top: -40px;
left: 50%;
transform: translateX(-50%);
background-color: #007bff;
color: white;
padding: 5px 15px;
border-radius: 5px;
font-weight: bold;
white-space: nowrap;
}
#displayCanvas {
cursor: none;
display: block;
}
.controls {
margin-top: 20px;
display: flex;
gap: 15px;
align-items: center;
justify-content: center;
flex-wrap: wrap;
}
.control-button {
cursor: pointer;
}
.step-control {
display: flex;
align-items: center;
gap: 10px;
}
#samplingSteps {
width: 70px;
}
.footer {
margin-top: 30px;
text-align: center;
}
</style>
</head>
<body>
<div class="container">
<div class="title-container">
<h1 class="mb-2">NeuralOS: Towards Simulating Operating Systems<br>via Neural Generative Models</h1>
<p class="mb-3">
<a href="https://anonymous.4open.science/r/neural-os" target="_blank" class="text-decoration-none">
Project Code: anonymous.4open.science/r/neural-os
</a>
</p>
</div>
<div class="canvas-container">
<div id="timeoutWarning" class="alert alert-warning" style="display: none; margin-bottom: 15px;">
<strong>Connection Timeout Warning:</strong>
<span id="timeoutMessage">No user activity detected. Connection will be dropped in <span id="timeoutCountdown">10</span> seconds and page will refresh automatically.</span>
<button type="button" class="btn btn-sm btn-primary ms-2" onclick="resetTimeout()">Stay Connected</button>
</div>
<div class="canvas-wrapper">
<div class="instruction-box">Move your mouse inside to interact</div>
<canvas id="displayCanvas" width="512" height="384"></canvas>
</div>
<div class="controls">
<button id="resetButton" class="btn btn-primary control-button">Reset Simulation</button>
<div class="step-control">
<label for="samplingSteps" class="form-label mb-0">Sampling Steps:</label>
<input type="number" id="samplingSteps" class="form-control" min="1" max="100" value="32">
<button id="updateStepsButton" class="btn btn-secondary control-button">Update</button>
</div>
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="useRnnToggle">
<label class="form-check-label" for="useRnnToggle">Use RNN</label>
</div>
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="autoInputToggle" checked>
<label class="form-check-label" for="autoInputToggle" id="autoInputLabel">Auto Input</label>
</div>
</div>
</div>
<div class="row justify-content-center">
<div class="col-md-8">
<div class="card">
<div class="card-body">
<h5 class="card-title">Instructions:</h5>
<ul class="mb-0">
<li>Move your mouse inside the blue box to interact with NeuralOS</li>
<li>Click (left-click or right-click) to perform click actions</li>
<li>Use your keyboard to type within the simulated environment</li>
<li>Adjust sampling steps to control quality/speed tradeoff</li>
<li>Toggle "Use RNN" to switch between RNN and diffusion mode</li>
<li>Toggle "Auto Input" to enable automatic frame generation (starts after 2s idle once you move mouse inside canvas, then runs every 0.5s)</li>
</ul>
</div>
</div>
</div>
</div>
</div>
<script>
const canvas = document.getElementById('displayCanvas');
const ctx = canvas.getContext('2d');
let socket;
let isConnected = false;
let reconnectAttempts = 0;
const MAX_RECONNECT_DELAY = 30000; // Maximum delay between reconnection attempts (30 seconds)
let connectionAttempted = false; // Flag to prevent multiple connection attempts
let isProcessing = false;
// Add flag to control trace visibility, default to false (hidden)
let showTrace = false;
// Function to draw the initial canvas state
function drawInitialCanvas() {
ctx.fillStyle = "#ffffff";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.font = "18px Arial";
ctx.fillStyle = "#666666";
ctx.textAlign = "center";
ctx.fillText("Move your mouse here to interact with NeuralOS", canvas.width/2, canvas.height/2 - 10);
ctx.fillText("The neural model will render in this area", canvas.width/2, canvas.height/2 + 20);
}
// Function to show connection status
function showConnectionStatus(message, subtitle = null) {
ctx.fillStyle = "#ffffff";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.font = "18px Arial";
ctx.fillStyle = "#666666";
ctx.textAlign = "center";
//ctx.fillText(message, canvas.width/2, canvas.height/2 - 10);
ctx.fillText(message, canvas.width/2, canvas.height/2);
if (subtitle) {
ctx.font = "14px Arial";
ctx.fillStyle = "#888888";
ctx.fillText(subtitle, canvas.width/2, canvas.height/2 + 15);
}
}
function connect() {
console.log("connect() function called - creating new WebSocket connection");
//const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const protocol = 'ws:';
socket = new WebSocket(`${protocol}//${window.location.host}/ws`);
socket.onopen = function(event) {
console.log("WebSocket connection established");
isConnected = true;
//connectionAttempted = false; // Reset flag for potential reconnections
//reconnectAttempts = 0;
// Show brief success message
//showConnectionStatus("Connected! Processing input...");
//setTimeout(() => {
// // Clear the message after 0.5 second
// if (isConnected) {
// ctx.clearRect(0, 0, canvas.width, canvas.height);
// }
//}, 500);
// Start auto-input mechanism
startAutoInput();
// Request current settings from server to sync UI
socket.send(JSON.stringify({
type: "get_settings"
}));
//startHeartbeat();
};
socket.onclose = function(event) {
console.log("WebSocket connection closed. Attempting to reconnect...");
console.log("Close event code:", event.code, "reason:", event.reason);
// Check if this was a timeout closure
if (event.code === 1000 && event.reason === "User inactivity timeout") {
console.log("Connection closed due to timeout - refreshing page");
stopAutoInput(); // Stop auto-input when connection is lost
stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
//clearInterval(heartbeatInterval);
// Refresh the page to reconnect
window.location.reload();
return;
}
isConnected = false;
//connectionAttempted = false; // Reset flag to allow reconnection attempts
stopAutoInput(); // Stop auto-input when connection is lost
stopTimeoutCountdown(); // Stop timeout countdown when connection is lost
//clearInterval(heartbeatInterval);
//scheduleReconnection();
};
socket.onerror = function(error) {
console.error("WebSocket error:", error);
};
socket.onmessage = function (event) {
const data = JSON.parse(event.data);
if (data.type === "heartbeat_response") {
console.log("Heartbeat response received");
} else if (data.image) {
let img = new Image();
img.onload = function() {
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
//isProcessing = false; // Reset the processing flag when we get a response
};
img.src = 'data:image/png;base64,' + data.image;
} else if (data.type === "reset_confirmed") {
console.log("Simulation reset confirmed by server");
// Clear the canvas and reset to initial state
drawInitialCanvas();
// Reset user interaction state
userHasInteracted = false;
lastSentPosition = null;
console.log("Reset user interaction state - waiting for user to move mouse inside canvas");
} else if (data.type === "rnn_updated") {
console.log(`USE_RNN setting updated to: ${data.use_rnn}`);
// Update the toggle to match the server state
document.getElementById('useRnnToggle').checked = data.use_rnn;
} else if (data.type === "settings") {
// Update UI elements to match server settings
console.log(`Received settings from server: SAMPLING_STEPS=${data.sampling_steps}, USE_RNN=${data.use_rnn}`);
document.getElementById('samplingSteps').value = data.sampling_steps;
document.getElementById('useRnnToggle').checked = data.use_rnn;
} else if (data.type === "timeout_warning") {
console.log(`Received timeout warning: ${data.timeout_in} seconds remaining`);
setTimeoutMessage(`No user activity detected. Connection will be dropped in <span id="timeoutCountdown">${data.timeout_in}</span> seconds and page will refresh automatically.`);
startTimeoutCountdown(data.timeout_in, false, 'idle'); // false = show stay connected button
} else if (data.type === "activity_reset") {
console.log("Server detected user activity, resetting timeout");
stopTimeoutCountdown('idle'); // Only clear idle timeout
// If there's still a session time limit active, update it
if (data.session_time_remaining && data.session_time_remaining > 0) {
console.log(`Updating session countdown: ${data.session_time_remaining} seconds remaining`);
setTimeoutMessage(`⏰ Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.session_time_remaining)}</span> seconds.`);
startTimeoutCountdown(Math.ceil(data.session_time_remaining), true, 'session'); // true = hide stay connected button
}
} else if (data.type === "queue_update") {
console.log(`Queue update: Position ${data.position}/${data.total_waiting}, wait: ${data.maximum_wait_seconds.toFixed(1)} seconds`);
const waitSeconds = Math.ceil(data.maximum_wait_seconds);
// Disable canvas interaction while in queue
disableCanvasInteraction();
if (waitSeconds === 0) {
showConnectionStatus("Starting soon...");
stopQueueCountdown();
} else {
// Only restart countdown if not already running or if time changed significantly
if (!queueCountdownActive) {
console.log(`Starting queue countdown: ${waitSeconds} seconds`);
startQueueCountdown(waitSeconds);
} else if (waitSeconds < queueWaitTime - 10 || waitSeconds > queueWaitTime + 15) {
// Be responsive to decreases (users dropping out) and significant increases
console.log(`Updating queue countdown: ${queueWaitTime} -> ${waitSeconds} (significant change)`);
startQueueCountdown(waitSeconds);
} else if (Math.abs(queueWaitTime - waitSeconds) <= 3) {
// Small differences (1-3 seconds) - keep local countdown running
console.log(`Ignoring small change: ${queueWaitTime} vs ${waitSeconds} (keeping local countdown)`);
} else {
console.log(`Moderate change ignored: ${queueWaitTime} -> ${waitSeconds} (${Math.abs(queueWaitTime - waitSeconds)}s difference)`);
}
}
} else if (data.type === "session_start") {
console.log("Session started, clearing queue display");
// Stop queue countdown and clear the display
stopQueueCountdown();
// Enable canvas interaction when session starts
enableCanvasInteraction();
//ctx.clearRect(0, 0, canvas.width, canvas.height);
} else if (data.type === "session_warning") {
console.log(`Session time warning: ${data.time_remaining} seconds remaining`);
setTimeoutMessage(`Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
startTimeoutCountdown(Math.ceil(data.time_remaining), true, 'session'); // true = hide stay connected button
} else if (data.type === "idle_warning") {
console.log(`Idle warning: ${data.time_remaining} seconds until timeout`);
setTimeoutMessage(`No user activity detected. Connection will be dropped in <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds and page will refresh automatically.`);
startTimeoutCountdown(Math.ceil(data.time_remaining), false, 'idle'); // false = show stay connected button
} else if (data.type === "grace_period") {
console.log(`Grace period: ${data.time_remaining} seconds remaining`);
setTimeoutMessage(`Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
startTimeoutCountdown(Math.ceil(data.time_remaining), true, 'grace'); // true = hide stay connected button
} else if (data.type === "time_limit_removed") {
console.log("Time limit removed - queue became empty");
stopTimeoutCountdown('session'); // Only clear session timeout
stopTimeoutCountdown('grace'); // Also clear grace period if active
} else if (data.type === "queue_limit_applied") {
console.log(`Queue limit applied, ${data.time_remaining} seconds remaining`);
setTimeoutMessage(`⏰ Other users waiting. Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
startTimeoutCountdown(Math.ceil(data.time_remaining), true, 'session'); // true = hide stay connected button
} else if (data.type === "worker_failure") {
console.log("Worker failure detected, reconnecting...");
showConnectionStatus("🔄 GPU worker failed. Reconnecting to healthy worker...");
// Clear the canvas to show we're reconnecting
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = '#f0f0f0';
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = '#666';
ctx.font = '20px Arial';
ctx.textAlign = 'center';
ctx.fillText('🔄 Reconnecting to healthy GPU...', canvas.width/2, canvas.height/2);
}
};
}
function scheduleReconnection() {
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), MAX_RECONNECT_DELAY);
console.log(`Scheduling reconnection in ${delay}ms`);
setTimeout(connect, delay);
reconnectAttempts++;
}
let heartbeatInterval;
function startHeartbeat() {
heartbeatInterval = setInterval(() => {
if (isConnected) {
try {
socket.send(JSON.stringify({ type: "heartbeat" }));
console.error("finished sending heartbeat");
} catch (error) {
console.error("Error sending heartbeat:", error);
}
}
}, 1000); // Send heartbeat every 15 seconds
}
// Don't connect immediately - wait for user interaction
// connect();
// Draw initial state on the canvas
drawInitialCanvas();
let lastSentPosition = null;
let lastSentTime = 0;
const SEND_INTERVAL = 10; // Send updates every 50ms
// Auto-input mechanism
let lastUserInputTime = 0;
let lastAutoInputTime = 0;
let autoInputInterval = null;
let autoInputActive = false; // Track if auto-input mode is active
const INITIAL_AUTO_INPUT_DELAY = 2000; // Start auto-input after 2 seconds of no user input
const AUTO_INPUT_INTERVAL = 500; // Send auto-input every 0.5 second once active
let autoInputEnabled = true; // Default to enabled
let userHasInteracted = false; // Track if user has moved mouse inside canvas
// Session state tracking
let sessionState = 'queued'; // 'queued', 'active', 'disconnected'
let canvasInteractionEnabled = false;
// Timeout countdown mechanism - support concurrent timeouts
let timeoutCountdownInterval = null;
let timeoutCountdown = 10;
let timeoutWarningActive = false;
// Track concurrent timeout systems
let currentTimeouts = {
idle: null, // { remaining: 10, hideButton: false }
session: null, // { remaining: 60, hideButton: true }
grace: null // { remaining: 5, hideButton: true }
};
// Queue waiting countdown mechanism
let queueCountdownInterval = null;
let queueWaitTime = 0;
let queueCountdownActive = false;
function startAutoInput() {
if (autoInputInterval) {
clearInterval(autoInputInterval);
}
autoInputInterval = setInterval(() => {
const currentTime = Date.now();
if (!autoInputEnabled || !lastSentPosition || !isConnected || socket.readyState !== WebSocket.OPEN || !userHasInteracted) {
return;
}
// Check if we should start auto-input mode
if (!autoInputActive && currentTime - lastUserInputTime >= INITIAL_AUTO_INPUT_DELAY) {
console.log("Starting auto-input mode (no user activity for 2 seconds)");
autoInputActive = true;
lastAutoInputTime = currentTime;
// Update UI to show auto-input is active
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input (Active)";
label.style.color = "#28a745";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
}
// Send auto-input if mode is active and enough time has passed
if (autoInputActive && currentTime - lastAutoInputTime >= AUTO_INPUT_INTERVAL) {
console.log("Sending auto-input (auto-input mode active)");
try {
socket.send(JSON.stringify({
"x": lastSentPosition.x,
"y": lastSentPosition.y,
"is_left_click": false,
"is_right_click": false,
"keys_down": [],
"keys_up": [],
"wheel_delta_x": 0,
"wheel_delta_y": 0,
"is_auto_input": true // Flag to identify auto-generated inputs
}));
lastAutoInputTime = currentTime;
} catch (error) {
console.error("Error sending auto-input:", error);
// Stop auto-input if there's an error (connection likely closed)
stopAutoInput();
}
}
}, 100); // Check every 100ms
}
function stopAutoInput() {
if (autoInputInterval) {
clearInterval(autoInputInterval);
autoInputInterval = null;
}
autoInputActive = false; // Reset auto-input mode
}
function updateLastUserInputTime() {
lastUserInputTime = Date.now();
// Reset auto-input mode when user provides input
if (autoInputActive) {
console.log("User activity detected, stopping auto-input mode");
autoInputActive = false;
// Reset UI indicator
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input";
label.style.color = "";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
}
}
function startTimeoutCountdown(initialTime = 10, hideStayConnectedButton = false, timeoutType = 'idle') {
// Update the specific timeout type
currentTimeouts[timeoutType] = {
remaining: initialTime,
hideButton: hideStayConnectedButton,
type: timeoutType
};
// Find the timeout that will expire first
let earliestTimeout = null;
let earliestTime = Infinity;
for (const [type, timeout] of Object.entries(currentTimeouts)) {
if (timeout && timeout.remaining < earliestTime) {
earliestTime = timeout.remaining;
earliestTimeout = timeout;
}
}
if (!earliestTimeout) {
stopTimeoutCountdown();
return;
}
// Stop existing countdown
if (timeoutCountdownInterval) {
clearInterval(timeoutCountdownInterval);
}
timeoutCountdown = earliestTimeout.remaining;
timeoutWarningActive = true;
// Show warning
const warning = document.getElementById('timeoutWarning');
if (warning) {
warning.style.display = 'block';
}
// Show/hide Stay Connected button based on the earliest timeout
const stayConnectedButton = warning.querySelector('button');
if (stayConnectedButton) {
stayConnectedButton.style.display = earliestTimeout.hideButton ? 'none' : 'inline-block';
}
// Update initial display
const countdownElement = document.getElementById('timeoutCountdown');
if (countdownElement) {
countdownElement.textContent = timeoutCountdown;
}
console.log(`Starting ${earliestTimeout.type} timeout countdown: ${timeoutCountdown} seconds`);
// Start countdown
timeoutCountdownInterval = setInterval(() => {
// Update all active timeouts
for (const [type, timeout] of Object.entries(currentTimeouts)) {
if (timeout) {
timeout.remaining--;
}
}
// Check if ANY timeout has reached 0 (expired)
let expiredTimeout = null;
for (const [type, timeout] of Object.entries(currentTimeouts)) {
if (timeout && timeout.remaining <= 0) {
expiredTimeout = { type, ...timeout };
break;
}
}
// If any timeout expired, trigger the appropriate action
if (expiredTimeout) {
console.log(`${expiredTimeout.type} timeout expired - refreshing page`);
// Remove the expired timeout from tracking
currentTimeouts[expiredTimeout.type] = null;
stopAutoInput();
stopTimeoutCountdown();
// All timeout types result in page refresh for now
window.location.reload();
return;
}
// Find the timeout that will expire first among remaining timeouts
let newEarliestTimeout = null;
let newEarliestTime = Infinity;
for (const [type, timeout] of Object.entries(currentTimeouts)) {
if (timeout && timeout.remaining > 0 && timeout.remaining < newEarliestTime) {
newEarliestTime = timeout.remaining;
newEarliestTimeout = timeout;
}
}
// If no active timeouts remain, hide the warning
if (!newEarliestTimeout) {
stopTimeoutCountdown();
return;
}
// Update display with the earliest timeout
timeoutCountdown = newEarliestTimeout.remaining;
const countdownElement = document.getElementById('timeoutCountdown');
if (countdownElement) {
countdownElement.textContent = timeoutCountdown;
}
// Update Stay Connected button visibility if the timeout type changed
if (newEarliestTimeout.type !== earliestTimeout.type) {
console.log(`Timeout type changed from ${earliestTimeout.type} to ${newEarliestTimeout.type}`);
const stayConnectedButton = warning.querySelector('button');
if (stayConnectedButton) {
stayConnectedButton.style.display = newEarliestTimeout.hideButton ? 'none' : 'inline-block';
}
// Update timeout message based on new timeout type
if (newEarliestTimeout.type === 'idle') {
setTimeoutMessage(`No user activity detected. Connection will be dropped in <span id="timeoutCountdown">${newEarliestTimeout.remaining}</span> seconds and page will refresh automatically.`);
} else if (newEarliestTimeout.type === 'session' || newEarliestTimeout.type === 'grace') {
setTimeoutMessage(`⏰ Other users waiting. Time remaining: <span id="timeoutCountdown">${newEarliestTimeout.remaining}</span> seconds.`);
}
earliestTimeout = newEarliestTimeout;
}
}, 1000);
}
function stopTimeoutCountdown(timeoutType = null) {
if (timeoutType) {
// Clear specific timeout type
currentTimeouts[timeoutType] = null;
console.log(`Cleared ${timeoutType} timeout`);
// Check if any other timeouts are still active
let hasActiveTimeouts = false;
for (const [type, timeout] of Object.entries(currentTimeouts)) {
if (timeout && timeout.remaining > 0) {
hasActiveTimeouts = true;
break;
}
}
if (hasActiveTimeouts) {
// Restart countdown with remaining timeouts
// Find the earliest remaining timeout and restart with it
let earliestTimeout = null;
let earliestTime = Infinity;
for (const [type, timeout] of Object.entries(currentTimeouts)) {
if (timeout && timeout.remaining < earliestTime) {
earliestTime = timeout.remaining;
earliestTimeout = timeout;
}
}
if (earliestTimeout) {
startTimeoutCountdown(earliestTimeout.remaining, earliestTimeout.hideButton, earliestTimeout.type);
}
return;
}
} else {
// Clear all timeouts
currentTimeouts = {
idle: null,
session: null,
grace: null
};
console.log("Cleared all timeouts");
}
if (timeoutCountdownInterval) {
clearInterval(timeoutCountdownInterval);
timeoutCountdownInterval = null;
}
timeoutWarningActive = false;
timeoutCountdown = 10;
// Reset the countdown display to 10 seconds
const countdownElement = document.getElementById('timeoutCountdown');
if (countdownElement) {
countdownElement.textContent = timeoutCountdown;
}
// Hide warning
const warning = document.getElementById('timeoutWarning');
if (warning) {
warning.style.display = 'none';
}
}
function startQueueCountdown(initialWaitTime) {
if (queueCountdownInterval) {
clearInterval(queueCountdownInterval);
}
queueWaitTime = initialWaitTime;
queueCountdownActive = true;
// Show initial countdown
updateQueueCountdownDisplay();
// Start countdown
queueCountdownInterval = setInterval(() => {
queueWaitTime--;
updateQueueCountdownDisplay();
if (queueWaitTime <= 0) {
stopQueueCountdown();
showConnectionStatus("Starting soon...");
}
}, 1000);
}
function stopQueueCountdown() {
if (queueCountdownInterval) {
clearInterval(queueCountdownInterval);
queueCountdownInterval = null;
}
queueCountdownActive = false;
queueWaitTime = 0;
}
function enableCanvasInteraction() {
canvasInteractionEnabled = true;
sessionState = 'active';
// Remove visual queue indicator
if (canvas) {
canvas.style.opacity = '1';
canvas.style.cursor = 'crosshair';
canvas.style.pointerEvents = 'auto';
}
// Update status
const statusElement = document.getElementById('connectionStatus');
if (statusElement) {
statusElement.textContent = 'Active';
statusElement.className = 'connected';
}
}
function disableCanvasInteraction() {
canvasInteractionEnabled = false;
sessionState = 'queued';
// Add visual queue indicator
if (canvas) {
canvas.style.opacity = '0.5';
canvas.style.cursor = 'not-allowed';
canvas.style.pointerEvents = 'none';
}
// Update status
const statusElement = document.getElementById('connectionStatus');
if (statusElement) {
statusElement.textContent = 'Queued';
statusElement.className = 'connecting';
}
}
function updateQueueCountdownDisplay() {
if (queueWaitTime <= 0) {
showConnectionStatus("Starting soon...");
} else if (queueWaitTime === 1) {
showConnectionStatus("Max wait: 1 second");
} else {
showConnectionStatus(`Max wait: ${queueWaitTime} seconds`);
}
}
function setTimeoutMessage(message) {
const messageElement = document.getElementById('timeoutMessage');
if (messageElement) {
messageElement.innerHTML = message;
}
}
function resetTimeout() {
// Send a stay-connected ping to reset the server's timeout
if (socket && socket.readyState === WebSocket.OPEN && lastSentPosition) {
try {
socket.send(JSON.stringify({
"x": lastSentPosition.x,
"y": lastSentPosition.y,
"is_left_click": false,
"is_right_click": false,
"keys_down": [],
"keys_up": [],
"wheel_delta_x": 0,
"wheel_delta_y": 0,
"is_stay_connected": true // Mark this as a stay-connected ping
}));
updateLastUserInputTime(); // Update for auto-input mechanism
console.log("Stay connected ping sent to server");
} catch (error) {
console.error("Error sending stay connected ping:", error);
}
}
stopTimeoutCountdown('idle'); // Only clear idle timeout, not session timeout
}
function sendInputState(x, y, isLeftClick = false, isRightClick = false, keysDownArr = [], keysUpArr = [], wheelDeltaX = 0, wheelDeltaY = 0) {
const currentTime = Date.now();
if (isConnected && socket.readyState === WebSocket.OPEN && (isLeftClick || isRightClick || keysDownArr.length > 0 || keysUpArr.length > 0 || wheelDeltaX !== 0 || wheelDeltaY !== 0 || !lastSentPosition || currentTime - lastSentTime >= SEND_INTERVAL)) {
try {
socket.send(JSON.stringify({
"x": x,
"y": y,
"is_left_click": isLeftClick,
"is_right_click": isRightClick,
"keys_down": keysDownArr,
"keys_up": keysUpArr,
"wheel_delta_x": wheelDeltaX,
"wheel_delta_y": wheelDeltaY,
}));
lastSentPosition = { x, y };
lastSentTime = currentTime;
// Mark that user has interacted with the canvas
if (!userHasInteracted) {
userHasInteracted = true;
console.log("User has interacted with canvas for the first time");
}
// Update last user input time for auto-input mechanism
updateLastUserInputTime();
//if (isLeftClick || isRightClick) {
// isProcessing = true; // Block further inputs until response
//}
} catch (error) {
console.error("Error sending input state:", error);
}
}
}
// Capture mouse movements and clicks
canvas.addEventListener("mousemove", async function (event) {
// Establish connection on first mouse movement if not already connected
if (!isConnected && !connectionAttempted) {
console.log("First mouse movement detected - establishing WebSocket connection");
connectionAttempted = true;
showConnectionStatus("Connecting to NeuralOS...");
connect();
// Wait for connection to be established
let attempts = 0;
const maxAttempts = 50; // 5 seconds max wait
while (!isConnected && attempts < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, 100)); // Wait 100ms
attempts++;
}
if (!isConnected) {
console.error("Failed to establish connection after 5 seconds");
connectionAttempted = false; // Reset flag to allow retry
showConnectionStatus("Connection failed. Move mouse to retry.");
return;
}
console.log("Connection established, processing first mouse movement");
}
if (!isConnected || isProcessing) return;
// Check if canvas interaction is enabled (not queued)
if (!canvasInteractionEnabled) {
console.log("Canvas interaction disabled - user is queued");
return;
}
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
// Only draw the trace if showTrace is true
if (showTrace && lastSentPosition) {
ctx.beginPath();
ctx.moveTo(lastSentPosition.x, lastSentPosition.y);
ctx.lineTo(x, y);
ctx.stroke();
}
sendInputState(x, y);
});
canvas.addEventListener("click", function (event) {
if (!isConnected || isProcessing) return;
// Check if canvas interaction is enabled (not queued)
if (!canvasInteractionEnabled) {
console.log("Canvas interaction disabled - user is queued");
return;
}
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
sendInputState(x, y, true, false);
});
// Handle right clicks
canvas.addEventListener("contextmenu", function (event) {
event.preventDefault(); // Prevent default context menu
if (!isConnected || isProcessing) return;
// Check if canvas interaction is enabled (not queued)
if (!canvasInteractionEnabled) {
console.log("Canvas interaction disabled - user is queued");
return;
}
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
sendInputState(x, y, false, true);
});
// Handle mouse wheel events
canvas.addEventListener("wheel", function (event) {
event.preventDefault(); // Prevent page scrolling
if (!isConnected || isProcessing) return;
let rect = canvas.getBoundingClientRect();
let x = event.clientX - rect.left;
let y = event.clientY - rect.top;
// Normalize wheel delta values (different browsers handle this differently)
let deltaX = event.deltaX;
let deltaY = event.deltaY;
// Clamp values to reasonable range
//deltaX = Math.max(-10, Math.min(10, deltaX));
//deltaY = Math.max(-10, Math.min(10, deltaY));
console.log(`Wheel event: deltaX=${deltaX}, deltaY=${deltaY} at (${x}, ${y})`);
sendInputState(x, y, false, false, [], [], deltaX, deltaY);
});
// Track keyboard events
const TROUBLESOME = new Set([
"Tab", // focus change
" ", // Space - scrolls page
"Escape", // Esc
"ArrowUp", // scroll up
"ArrowDown", // scroll down
"ArrowLeft", // horiz. scroll or browser Back with Alt/⌥
"ArrowRight", // horiz. scroll or browser Forward with Alt/⌥
"Backspace", // navigate back in some browsers
"PageUp", // scroll
"PageDown", // scroll
"Home", // scroll to top
"End", // scroll to bottom
"Enter",
"Delete",
"/"
]);
function isForbiddenCombo(e) {
// Reload
if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "r") return true;
// Close tab
if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "w") return true;
return false;
}
document.addEventListener("keydown", function (event) {
// Only prevent default if user has interacted with the canvas
if (userHasInteracted && (TROUBLESOME.has(event.key) || isForbiddenCombo(event))) {
event.preventDefault();
event.stopPropagation();
}
if (!isConnected || isProcessing || !userHasInteracted) return;
// Check if canvas interaction is enabled (not queued)
if (!canvasInteractionEnabled) {
console.log("Canvas interaction disabled - user is queued");
return;
}
// Get the current mouse position
let rect = canvas.getBoundingClientRect();
let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
sendInputState(x, y, false, false, [event.key], []);
});
document.addEventListener("keyup", function (event) {
// Only prevent default if user has interacted with the canvas
if (userHasInteracted && TROUBLESOME.has(event.key)) {
event.preventDefault();
event.stopPropagation();
}
if (!isConnected || socket.readyState !== WebSocket.OPEN || !userHasInteracted) return;
// Check if canvas interaction is enabled (not queued)
if (!canvasInteractionEnabled) {
console.log("Canvas interaction disabled - user is queued");
return;
}
// Get the current mouse position
let rect = canvas.getBoundingClientRect();
let x = lastSentPosition ? lastSentPosition.x : canvas.width / 2;
let y = lastSentPosition ? lastSentPosition.y : canvas.height / 2;
// For key up events, we send the key in the keys_up array
sendInputState(x, y, false, false, [], [event.key]);
});
// Graceful disconnection
window.addEventListener('beforeunload', function (e) {
stopAutoInput(); // Clean up auto-input interval
stopTimeoutCountdown(); // Clean up timeout countdown
stopQueueCountdown(); // Clean up queue countdown
if (isConnected) {
try {
//socket.send(JSON.stringify({ type: "disconnect" }));
//socket.close();
} catch (error) {
console.error("Error during disconnection:", error);
}
}
});
// Add event listener for the reset button
document.getElementById('resetButton').addEventListener('click', function() {
if (socket && socket.readyState === WebSocket.OPEN) {
console.log("Sending reset command to server");
socket.send(JSON.stringify({
type: "reset"
}));
} else {
console.error("WebSocket not connected, cannot reset");
}
});
// Add event listener for updating sampling steps
document.getElementById('updateStepsButton').addEventListener('click', function() {
const stepsInput = document.getElementById('samplingSteps');
const newSteps = parseInt(stepsInput.value, 10);
if (isNaN(newSteps) || newSteps < 1) {
alert("Please enter a valid number of steps (minimum 1)");
return;
}
if (socket && socket.readyState === WebSocket.OPEN) {
console.log(`Sending update to set sampling steps to ${newSteps}`);
socket.send(JSON.stringify({
type: "update_sampling_steps",
steps: newSteps
}));
} else {
console.error("WebSocket not connected, cannot update steps");
}
});
// Add event listener for the USE_RNN toggle
document.getElementById('useRnnToggle').addEventListener('change', function() {
const useRnn = this.checked;
if (socket && socket.readyState === WebSocket.OPEN) {
console.log(`Sending update to set USE_RNN to ${useRnn}`);
socket.send(JSON.stringify({
type: "update_use_rnn",
use_rnn: useRnn
}));
} else {
console.error("WebSocket not connected, cannot update USE_RNN");
}
});
// Add event listener for the auto-input toggle
document.getElementById('autoInputToggle').addEventListener('change', function() {
autoInputEnabled = this.checked;
console.log(`Auto-input ${autoInputEnabled ? 'enabled' : 'disabled'}`);
if (autoInputEnabled) {
// Reset the timers when enabling to start fresh
updateLastUserInputTime();
lastAutoInputTime = 0;
autoInputActive = false;
// Reset UI indicator
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input";
label.style.color = "";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
} else {
// Stop auto-input mode when disabling
autoInputActive = false;
// Reset UI indicator
try {
const label = document.getElementById('autoInputLabel');
if (label) {
label.textContent = "Auto Input";
label.style.color = "";
}
} catch (error) {
console.error("Error updating auto-input UI:", error);
}
}
});
// Initialize canvas in disabled state (user starts queued)
disableCanvasInteraction();
</script>
<!-- Bootstrap JS (optional) -->
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
</body>
</html>