Spaces:
Runtime error
Runtime error
da03
commited on
Commit
·
b9e6b75
1
Parent(s):
fa82766
- analyze_analytics.py +19 -3
- dispatcher.py +81 -58
- static/index.html +14 -6
analyze_analytics.py
CHANGED
@@ -129,6 +129,7 @@ class AnalyticsAnalyzer:
|
|
129 |
bypasses = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_bypass']
|
130 |
waits = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_wait']
|
131 |
statuses = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_status']
|
|
|
132 |
|
133 |
total_users = len(bypasses) + len(waits)
|
134 |
if total_users == 0:
|
@@ -151,14 +152,29 @@ class AnalyticsAnalyzer:
|
|
151 |
|
152 |
if statuses:
|
153 |
queue_sizes = [r['queue_size'] for r in statuses]
|
154 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
155 |
|
156 |
print(f"\nQueue size statistics:")
|
157 |
print(f" Average queue size: {statistics.mean(queue_sizes):.1f}")
|
158 |
print(f" Max queue size: {max(queue_sizes)}")
|
159 |
|
160 |
-
if
|
161 |
-
print(f" Average
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
162 |
print()
|
163 |
|
164 |
def analyze_ip_usage(self):
|
|
|
129 |
bypasses = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_bypass']
|
130 |
waits = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_wait']
|
131 |
statuses = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_status']
|
132 |
+
limit_applications = [r for r in self.data['queue_metrics'] if r.get('type') == 'queue_limits_applied']
|
133 |
|
134 |
total_users = len(bypasses) + len(waits)
|
135 |
if total_users == 0:
|
|
|
152 |
|
153 |
if statuses:
|
154 |
queue_sizes = [r['queue_size'] for r in statuses]
|
155 |
+
# Handle both old 'estimated_wait' and new 'maximum_wait' fields for backwards compatibility
|
156 |
+
maximum_waits = []
|
157 |
+
for r in statuses:
|
158 |
+
if r['queue_size'] > 0:
|
159 |
+
if 'maximum_wait' in r:
|
160 |
+
maximum_waits.append(r['maximum_wait'])
|
161 |
+
elif 'estimated_wait' in r:
|
162 |
+
maximum_waits.append(r['estimated_wait'])
|
163 |
|
164 |
print(f"\nQueue size statistics:")
|
165 |
print(f" Average queue size: {statistics.mean(queue_sizes):.1f}")
|
166 |
print(f" Max queue size: {max(queue_sizes)}")
|
167 |
|
168 |
+
if maximum_waits:
|
169 |
+
print(f" Average maximum wait: {statistics.mean(maximum_waits):.1f}s")
|
170 |
+
print(f" Peak maximum wait: {max(maximum_waits):.1f}s")
|
171 |
+
|
172 |
+
if limit_applications:
|
173 |
+
total_affected = sum(r['affected_sessions'] for r in limit_applications)
|
174 |
+
print(f"\nQueue limit applications:")
|
175 |
+
print(f" Times limits applied to existing sessions: {len(limit_applications)}")
|
176 |
+
print(f" Total sessions affected: {total_affected}")
|
177 |
+
print(f" Average sessions affected per application: {total_affected/len(limit_applications):.1f}")
|
178 |
print()
|
179 |
|
180 |
def analyze_ip_usage(self):
|
dispatcher.py
CHANGED
@@ -240,7 +240,24 @@ class SystemAnalytics:
|
|
240 |
self._write_log(f"⚠️ CRITICAL: No GPU workers available! {queue_size} users waiting")
|
241 |
self._write_log(" Please check worker processes and GPU availability")
|
242 |
|
243 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
244 |
"""Log queue status"""
|
245 |
self.queue_size_samples.append(queue_size)
|
246 |
|
@@ -252,13 +269,13 @@ class SystemAnalytics:
|
|
252 |
"type": "queue_status",
|
253 |
"timestamp": timestamp,
|
254 |
"queue_size": queue_size,
|
255 |
-
"
|
256 |
"avg_queue_size": avg_queue_size
|
257 |
})
|
258 |
|
259 |
# Only log to human-readable if there's a queue
|
260 |
if queue_size > 0:
|
261 |
-
self._write_log(f"📝 QUEUE STATUS: {queue_size} users waiting |
|
262 |
self._write_log(f" 📊 Avg queue size: {avg_queue_size:.1f}")
|
263 |
|
264 |
def log_periodic_summary(self):
|
@@ -371,11 +388,51 @@ class SessionManager:
|
|
371 |
|
372 |
async def add_session_to_queue(self, session: UserSession):
|
373 |
"""Add a session to the queue"""
|
|
|
|
|
|
|
374 |
self.sessions[session.session_id] = session
|
375 |
self.session_queue.append(session.session_id)
|
376 |
session.status = SessionStatus.QUEUED
|
377 |
session.queue_start_time = time.time()
|
378 |
logger.info(f"Added session {session.session_id} to queue. Queue size: {len(self.session_queue)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
379 |
|
380 |
async def process_queue(self):
|
381 |
"""Process the session queue"""
|
@@ -571,14 +628,14 @@ class SessionManager:
|
|
571 |
session = self.sessions.get(session_id)
|
572 |
if session and session.status == SessionStatus.QUEUED:
|
573 |
try:
|
574 |
-
# Calculate
|
575 |
-
|
576 |
|
577 |
await session.websocket.send_json({
|
578 |
"type": "queue_update",
|
579 |
"position": i + 1,
|
580 |
"total_waiting": len(self.session_queue),
|
581 |
-
"
|
582 |
"active_sessions": len(self.active_sessions),
|
583 |
"available_workers": len([w for w in self.workers.values() if w.is_available])
|
584 |
})
|
@@ -587,69 +644,35 @@ class SessionManager:
|
|
587 |
|
588 |
# Log queue status if there's a queue
|
589 |
if self.session_queue:
|
590 |
-
|
591 |
-
analytics.log_queue_status(len(self.session_queue),
|
592 |
|
593 |
-
def
|
594 |
-
"""Calculate
|
595 |
-
current_time = time.time()
|
596 |
available_workers = len([w for w in self.workers.values() if w.is_available])
|
597 |
|
598 |
# If there are available workers, no wait time
|
599 |
if available_workers > 0:
|
600 |
return 0
|
601 |
|
602 |
-
# Calculate
|
603 |
-
min_remaining_time = float('inf')
|
604 |
-
active_session_times = []
|
605 |
-
|
606 |
-
for session_id in self.active_sessions:
|
607 |
-
session = self.sessions.get(session_id)
|
608 |
-
if session and session.last_activity:
|
609 |
-
if session.max_session_time:
|
610 |
-
# Session has time limit (queue exists)
|
611 |
-
elapsed = current_time - session.last_activity
|
612 |
-
remaining = session.max_session_time - elapsed
|
613 |
-
remaining = max(0, remaining) # Don't go negative
|
614 |
-
else:
|
615 |
-
# No time limit, estimate based on average usage
|
616 |
-
elapsed = current_time - session.last_activity
|
617 |
-
# Assume sessions without time limits will run for average of 45 seconds more
|
618 |
-
remaining = max(45 - elapsed, 10) # Minimum 10 seconds
|
619 |
-
|
620 |
-
active_session_times.append(remaining)
|
621 |
-
min_remaining_time = min(min_remaining_time, remaining)
|
622 |
-
|
623 |
-
# If no active sessions found, use default
|
624 |
-
if not active_session_times:
|
625 |
-
min_remaining_time = 30.0
|
626 |
-
|
627 |
-
# Calculate estimated wait time based on position
|
628 |
num_workers = len(self.workers)
|
629 |
if num_workers == 0:
|
630 |
return 999 # No workers available
|
631 |
|
632 |
-
|
633 |
-
|
634 |
-
|
635 |
-
|
636 |
-
|
637 |
-
|
638 |
-
|
639 |
-
|
640 |
-
|
641 |
-
|
642 |
-
|
643 |
-
|
644 |
-
|
645 |
-
if remaining_in_current_cycle <= len(active_session_times):
|
646 |
-
# Sort session times to get when the Nth worker will be free
|
647 |
-
sorted_times = sorted(active_session_times)
|
648 |
-
current_cycle_time = sorted_times[remaining_in_current_cycle - 1]
|
649 |
-
else:
|
650 |
-
current_cycle_time = min_remaining_time
|
651 |
-
|
652 |
-
return full_cycles_time + current_cycle_time
|
653 |
|
654 |
async def handle_user_activity(self, session_id: str):
|
655 |
"""Update user activity timestamp and reset warning flags"""
|
|
|
240 |
self._write_log(f"⚠️ CRITICAL: No GPU workers available! {queue_size} users waiting")
|
241 |
self._write_log(" Please check worker processes and GPU availability")
|
242 |
|
243 |
+
def log_queue_limits_applied(self, affected_sessions: int, queue_size: int):
|
244 |
+
"""Log when time limits are applied to existing sessions due to queue formation"""
|
245 |
+
timestamp = time.time()
|
246 |
+
|
247 |
+
# Human-readable log
|
248 |
+
self._write_log(f"🕐 QUEUE LIMITS APPLIED: {affected_sessions} existing sessions now have 60s limits")
|
249 |
+
self._write_log(f" 📊 Reason: Queue formed with {queue_size} waiting users")
|
250 |
+
|
251 |
+
# Structured data log
|
252 |
+
self._write_json_log(self.queue_metrics_file, {
|
253 |
+
"type": "queue_limits_applied",
|
254 |
+
"timestamp": timestamp,
|
255 |
+
"affected_sessions": affected_sessions,
|
256 |
+
"queue_size": queue_size,
|
257 |
+
"time_limit_applied": 60.0
|
258 |
+
})
|
259 |
+
|
260 |
+
def log_queue_status(self, queue_size: int, maximum_wait: float):
|
261 |
"""Log queue status"""
|
262 |
self.queue_size_samples.append(queue_size)
|
263 |
|
|
|
269 |
"type": "queue_status",
|
270 |
"timestamp": timestamp,
|
271 |
"queue_size": queue_size,
|
272 |
+
"maximum_wait": maximum_wait,
|
273 |
"avg_queue_size": avg_queue_size
|
274 |
})
|
275 |
|
276 |
# Only log to human-readable if there's a queue
|
277 |
if queue_size > 0:
|
278 |
+
self._write_log(f"📝 QUEUE STATUS: {queue_size} users waiting | Max wait: {maximum_wait:.1f}s")
|
279 |
self._write_log(f" 📊 Avg queue size: {avg_queue_size:.1f}")
|
280 |
|
281 |
def log_periodic_summary(self):
|
|
|
388 |
|
389 |
async def add_session_to_queue(self, session: UserSession):
|
390 |
"""Add a session to the queue"""
|
391 |
+
# Check if queue was empty before adding this session
|
392 |
+
was_queue_empty = len(self.session_queue) == 0
|
393 |
+
|
394 |
self.sessions[session.session_id] = session
|
395 |
self.session_queue.append(session.session_id)
|
396 |
session.status = SessionStatus.QUEUED
|
397 |
session.queue_start_time = time.time()
|
398 |
logger.info(f"Added session {session.session_id} to queue. Queue size: {len(self.session_queue)}")
|
399 |
+
|
400 |
+
# If queue was empty and now has users, apply time limits to existing active sessions
|
401 |
+
if was_queue_empty and len(self.session_queue) > 0:
|
402 |
+
await self.apply_queue_limits_to_existing_sessions()
|
403 |
+
|
404 |
+
async def apply_queue_limits_to_existing_sessions(self):
|
405 |
+
"""Apply 60-second time limits to existing unlimited sessions when queue forms"""
|
406 |
+
current_time = time.time()
|
407 |
+
affected_sessions = 0
|
408 |
+
|
409 |
+
for session_id in list(self.active_sessions.keys()):
|
410 |
+
session = self.sessions.get(session_id)
|
411 |
+
if session and session.max_session_time is None: # Currently unlimited
|
412 |
+
# Give them 60 seconds from now
|
413 |
+
session.max_session_time = 60.0
|
414 |
+
session.last_activity = current_time # Reset activity timer to start 60s countdown
|
415 |
+
session.session_warning_sent = False # Reset warning flag
|
416 |
+
affected_sessions += 1
|
417 |
+
|
418 |
+
# Notify the user about the new time limit
|
419 |
+
try:
|
420 |
+
queue_size = len(self.session_queue)
|
421 |
+
user_text = "user" if queue_size == 1 else "users"
|
422 |
+
message = f"{queue_size} other {user_text} waiting. You have 60 seconds to finish."
|
423 |
+
|
424 |
+
await session.websocket.send_json({
|
425 |
+
"type": "queue_limit_applied",
|
426 |
+
"message": message,
|
427 |
+
"time_remaining": 60.0,
|
428 |
+
"queue_size": queue_size
|
429 |
+
})
|
430 |
+
logger.info(f"Applied 60s time limit to existing session {session_id} due to queue formation ({queue_size} users waiting)")
|
431 |
+
except Exception as e:
|
432 |
+
logger.error(f"Failed to notify session {session_id} about queue limit: {e}")
|
433 |
+
|
434 |
+
if affected_sessions > 0:
|
435 |
+
analytics.log_queue_limits_applied(affected_sessions, len(self.session_queue))
|
436 |
|
437 |
async def process_queue(self):
|
438 |
"""Process the session queue"""
|
|
|
628 |
session = self.sessions.get(session_id)
|
629 |
if session and session.status == SessionStatus.QUEUED:
|
630 |
try:
|
631 |
+
# Calculate maximum possible wait time
|
632 |
+
maximum_wait = self._calculate_maximum_wait_time(i + 1)
|
633 |
|
634 |
await session.websocket.send_json({
|
635 |
"type": "queue_update",
|
636 |
"position": i + 1,
|
637 |
"total_waiting": len(self.session_queue),
|
638 |
+
"maximum_wait_seconds": maximum_wait,
|
639 |
"active_sessions": len(self.active_sessions),
|
640 |
"available_workers": len([w for w in self.workers.values() if w.is_available])
|
641 |
})
|
|
|
644 |
|
645 |
# Log queue status if there's a queue
|
646 |
if self.session_queue:
|
647 |
+
maximum_wait = self._calculate_maximum_wait_time(1)
|
648 |
+
analytics.log_queue_status(len(self.session_queue), maximum_wait)
|
649 |
|
650 |
+
def _calculate_maximum_wait_time(self, position_in_queue: int) -> float:
|
651 |
+
"""Calculate maximum possible wait time (worst case scenario)"""
|
|
|
652 |
available_workers = len([w for w in self.workers.values() if w.is_available])
|
653 |
|
654 |
# If there are available workers, no wait time
|
655 |
if available_workers > 0:
|
656 |
return 0
|
657 |
|
658 |
+
# Calculate maximum wait time based on position and worker count
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
659 |
num_workers = len(self.workers)
|
660 |
if num_workers == 0:
|
661 |
return 999 # No workers available
|
662 |
|
663 |
+
# When queue exists, each session is limited to 60 seconds maximum
|
664 |
+
# Calculate how many "waves" of users need to complete before this user gets GPU
|
665 |
+
waves_to_wait = (position_in_queue - 1) // num_workers
|
666 |
+
position_in_final_wave = (position_in_queue - 1) % num_workers + 1
|
667 |
+
|
668 |
+
# Maximum time per wave is 60 seconds (session time limit when queue exists)
|
669 |
+
max_session_time = self.MAX_SESSION_TIME_WITH_QUEUE
|
670 |
+
|
671 |
+
# Total maximum wait = (complete waves * 60s) + 60s for final wave
|
672 |
+
# The +1 wave accounts for current active sessions finishing
|
673 |
+
maximum_wait = (waves_to_wait + 1) * max_session_time
|
674 |
+
|
675 |
+
return maximum_wait
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
676 |
|
677 |
async def handle_user_activity(self, session_id: str):
|
678 |
"""Update user activity timestamp and reset warning flags"""
|
static/index.html
CHANGED
@@ -280,20 +280,24 @@
|
|
280 |
console.log("Server detected user activity, resetting timeout");
|
281 |
stopTimeoutCountdown();
|
282 |
} else if (data.type === "queue_update") {
|
283 |
-
console.log(`Queue update: Position ${data.position}/${data.total_waiting},
|
284 |
-
const waitSeconds = Math.ceil(data.
|
285 |
|
286 |
let waitText;
|
287 |
if (waitSeconds === 0) {
|
288 |
waitText = "Starting soon...";
|
289 |
} else if (waitSeconds === 1) {
|
290 |
-
waitText = "1 second";
|
291 |
} else if (waitSeconds < 60) {
|
292 |
-
waitText = `${waitSeconds} seconds`;
|
293 |
} else {
|
294 |
const minutes = Math.floor(waitSeconds / 60);
|
295 |
const seconds = waitSeconds % 60;
|
296 |
-
|
|
|
|
|
|
|
|
|
297 |
}
|
298 |
|
299 |
const statusText = data.available_workers > 0 ?
|
@@ -302,7 +306,7 @@
|
|
302 |
|
303 |
showConnectionStatus(
|
304 |
statusText,
|
305 |
-
`
|
306 |
);
|
307 |
} else if (data.type === "session_start") {
|
308 |
console.log("Session started, clearing queue display");
|
@@ -323,6 +327,10 @@
|
|
323 |
} else if (data.type === "time_limit_removed") {
|
324 |
console.log("Time limit removed - queue became empty");
|
325 |
stopTimeoutCountdown();
|
|
|
|
|
|
|
|
|
326 |
}
|
327 |
};
|
328 |
}
|
|
|
280 |
console.log("Server detected user activity, resetting timeout");
|
281 |
stopTimeoutCountdown();
|
282 |
} else if (data.type === "queue_update") {
|
283 |
+
console.log(`Queue update: Position ${data.position}/${data.total_waiting}, maximum wait: ${data.maximum_wait_seconds.toFixed(1)} seconds`);
|
284 |
+
const waitSeconds = Math.ceil(data.maximum_wait_seconds);
|
285 |
|
286 |
let waitText;
|
287 |
if (waitSeconds === 0) {
|
288 |
waitText = "Starting soon...";
|
289 |
} else if (waitSeconds === 1) {
|
290 |
+
waitText = "1 second max";
|
291 |
} else if (waitSeconds < 60) {
|
292 |
+
waitText = `${waitSeconds} seconds max`;
|
293 |
} else {
|
294 |
const minutes = Math.floor(waitSeconds / 60);
|
295 |
const seconds = waitSeconds % 60;
|
296 |
+
if (seconds === 0) {
|
297 |
+
waitText = `${minutes} minutes max`;
|
298 |
+
} else {
|
299 |
+
waitText = `${minutes}m ${seconds}s max`;
|
300 |
+
}
|
301 |
}
|
302 |
|
303 |
const statusText = data.available_workers > 0 ?
|
|
|
306 |
|
307 |
showConnectionStatus(
|
308 |
statusText,
|
309 |
+
`Maximum wait: ${waitText} (${data.active_sessions} active sessions)`
|
310 |
);
|
311 |
} else if (data.type === "session_start") {
|
312 |
console.log("Session started, clearing queue display");
|
|
|
327 |
} else if (data.type === "time_limit_removed") {
|
328 |
console.log("Time limit removed - queue became empty");
|
329 |
stopTimeoutCountdown();
|
330 |
+
} else if (data.type === "queue_limit_applied") {
|
331 |
+
console.log(`Queue limit applied: ${data.message}, ${data.time_remaining} seconds remaining`);
|
332 |
+
setTimeoutMessage(`⏰ ${data.message} Time remaining: <span id="timeoutCountdown">${Math.ceil(data.time_remaining)}</span> seconds.`);
|
333 |
+
startTimeoutCountdown(Math.ceil(data.time_remaining));
|
334 |
}
|
335 |
};
|
336 |
}
|