Spaces:
Running
Running
File size: 13,445 Bytes
28e1dba |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 |
import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';
import githubService from './githubService.js';
import huggingfaceStorageService from './huggingfaceStorageService.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
/**
* 备份调度服务
* 管理用户数据的定时备份到GitHub和服务器重启时的数据恢复
*/
class BackupSchedulerService {
constructor() {
this.backupQueue = new Map(); // userId -> { timer, scheduledTime }
this.isRestoring = false;
this.backupInProgress = new Set(); // 正在备份的用户ID
this.initialized = false;
// 备份配置
this.BACKUP_DELAY = 8 * 60 * 60 * 1000; // 8小时
this.MAX_BACKUP_RETRIES = 3;
this.BACKUP_RETRY_DELAY = 5 * 60 * 1000; // 5分钟
}
/**
* 初始化备份调度服务
*/
async initialize() {
if (this.initialized) return;
try {
// 服务器启动时检查是否需要从GitHub恢复数据
await this.checkAndRestoreFromGitHub();
this.initialized = true;
console.log('✅ BackupSchedulerService initialized successfully');
} catch (error) {
console.error('❌ Failed to initialize BackupSchedulerService:', error);
throw error;
}
}
/**
* 调度用户数据备份
* @param {string} userId - 用户ID
* @param {number} delay - 延迟时间(毫秒),默认8小时
*/
scheduleUserBackup(userId, delay = this.BACKUP_DELAY) {
if (!this.initialized) {
console.warn('BackupSchedulerService not initialized, skipping backup schedule');
return;
}
// 如果已经有调度,先取消
this.cancelUserBackup(userId);
const scheduledTime = Date.now() + delay;
const timer = setTimeout(async () => {
try {
await this.backupUserData(userId);
} catch (error) {
console.error(`❌ Scheduled backup failed for user ${userId}:`, error);
// 重试机制
this.retryBackup(userId, 1);
} finally {
this.backupQueue.delete(userId);
}
}, delay);
this.backupQueue.set(userId, { timer, scheduledTime });
console.log(`⏰ Backup scheduled for user ${userId} in ${Math.round(delay / 1000 / 60)} minutes`);
}
/**
* 取消用户备份调度
* @param {string} userId - 用户ID
*/
cancelUserBackup(userId) {
const backup = this.backupQueue.get(userId);
if (backup) {
clearTimeout(backup.timer);
this.backupQueue.delete(userId);
console.log(`❌ Backup cancelled for user ${userId}`);
}
}
/**
* 立即备份用户数据
* @param {string} userId - 用户ID
* @returns {Promise<boolean>} 备份结果
*/
async backupUserData(userId) {
if (this.backupInProgress.has(userId)) {
console.log(`⏳ Backup already in progress for user ${userId}`);
return false;
}
this.backupInProgress.add(userId);
try {
console.log(`🔄 Starting backup for user ${userId}`);
// 获取用户的所有数据
const userData = await this.collectUserData(userId);
if (!userData || Object.keys(userData).length === 0) {
console.log(`📭 No data to backup for user ${userId}`);
return true;
}
// 备份到GitHub
const backupResult = await this.backupToGitHub(userId, userData);
if (backupResult.success) {
console.log(`✅ Backup completed for user ${userId}`);
return true;
} else {
throw new Error(backupResult.error || 'Backup failed');
}
} catch (error) {
console.error(`❌ Backup failed for user ${userId}:`, error);
throw error;
} finally {
this.backupInProgress.delete(userId);
}
}
/**
* 重试备份
* @param {string} userId - 用户ID
* @param {number} attempt - 重试次数
*/
retryBackup(userId, attempt) {
if (attempt > this.MAX_BACKUP_RETRIES) {
console.error(`❌ Max backup retries exceeded for user ${userId}`);
return;
}
console.log(`🔄 Retrying backup for user ${userId} (attempt ${attempt}/${this.MAX_BACKUP_RETRIES})`);
setTimeout(async () => {
try {
await this.backupUserData(userId);
} catch (error) {
this.retryBackup(userId, attempt + 1);
}
}, this.BACKUP_RETRY_DELAY * attempt); // 递增延迟
}
/**
* 收集用户数据
* @param {string} userId - 用户ID
* @returns {Promise<Object>} 用户数据
*/
async collectUserData(userId) {
try {
const userData = {
userId,
backupTime: new Date().toISOString(),
ppts: {},
images: []
};
// 获取用户的PPT数据
const pptList = await githubService.getUserPPTList(userId);
for (const ppt of pptList) {
try {
const pptData = await githubService.getPPT(userId, ppt.id);
if (pptData && pptData.content) {
userData.ppts[ppt.id] = {
id: ppt.id,
title: ppt.title,
content: pptData.content,
updatedAt: ppt.updatedAt,
createdAt: ppt.createdAt
};
}
} catch (error) {
console.warn(`⚠️ Failed to get PPT ${ppt.id} for backup:`, error.message);
}
}
// 获取用户的图片数据
const userImages = await huggingfaceStorageService.getUserImages(userId);
userData.images = userImages;
return userData;
} catch (error) {
console.error(`❌ Failed to collect user data for ${userId}:`, error);
throw error;
}
}
/**
* 备份数据到GitHub
* @param {string} userId - 用户ID
* @param {Object} userData - 用户数据
* @returns {Promise<Object>} 备份结果
*/
async backupToGitHub(userId, userData) {
try {
const backupFileName = `backup-${userId}-${Date.now()}.json`;
const backupPath = `backups/${userId}/${backupFileName}`;
// 压缩数据
const compressedData = JSON.stringify(userData);
// 上传到GitHub
const result = await githubService.uploadFile(
backupPath,
compressedData,
`Backup user data for ${userId}`,
0 // 使用第一个仓库进行备份
);
if (result.success) {
// 更新备份索引
await this.updateBackupIndex(userId, {
fileName: backupFileName,
path: backupPath,
timestamp: Date.now(),
size: compressedData.length,
pptCount: Object.keys(userData.ppts).length,
imageCount: userData.images.length
});
return { success: true, path: backupPath };
} else {
return { success: false, error: result.error };
}
} catch (error) {
return { success: false, error: error.message };
}
}
/**
* 更新备份索引
* @param {string} userId - 用户ID
* @param {Object} backupInfo - 备份信息
*/
async updateBackupIndex(userId, backupInfo) {
try {
const indexPath = `backups/${userId}/index.json`;
// 获取现有索引
let index = { backups: [] };
try {
const existingIndex = await githubService.getFile(indexPath, 0);
if (existingIndex.success) {
index = JSON.parse(existingIndex.content);
}
} catch (error) {
// 索引不存在,使用默认值
}
// 添加新备份记录
index.backups.unshift(backupInfo);
// 保留最近10个备份记录
index.backups = index.backups.slice(0, 10);
// 更新索引文件
await githubService.uploadFile(
indexPath,
JSON.stringify(index, null, 2),
`Update backup index for ${userId}`,
0
);
} catch (error) {
console.warn(`⚠️ Failed to update backup index for ${userId}:`, error.message);
}
}
/**
* 检查并从GitHub恢复数据
*/
async checkAndRestoreFromGitHub() {
if (this.isRestoring) return;
this.isRestoring = true;
try {
console.log('🔍 Checking if data restoration is needed...');
// 检查Huggingface存储是否为空或需要恢复
const storageStats = await huggingfaceStorageService.getStorageStats();
if (storageStats.totalImages === 0) {
console.log('📦 No existing data found, attempting to restore from GitHub...');
await this.restoreAllDataFromGitHub();
} else {
console.log(`📊 Found ${storageStats.totalImages} existing images, skipping restoration`);
}
} catch (error) {
console.error('❌ Failed to check/restore data from GitHub:', error);
} finally {
this.isRestoring = false;
}
}
/**
* 从GitHub恢复所有数据
*/
async restoreAllDataFromGitHub() {
try {
console.log('🔄 Starting data restoration from GitHub...');
// 获取所有备份用户
const backupUsers = await this.getBackupUsers();
let restoredCount = 0;
for (const userId of backupUsers) {
try {
const restored = await this.restoreUserDataFromGitHub(userId);
if (restored) {
restoredCount++;
}
} catch (error) {
console.warn(`⚠️ Failed to restore data for user ${userId}:`, error.message);
}
}
console.log(`✅ Data restoration completed. Restored ${restoredCount} users.`);
} catch (error) {
console.error('❌ Failed to restore all data from GitHub:', error);
throw error;
}
}
/**
* 获取所有有备份的用户
* @returns {Promise<Array>} 用户ID数组
*/
async getBackupUsers() {
try {
// 尝试获取备份目录列表
const backupDirs = await githubService.listDirectory('backups', 0);
if (backupDirs.success && backupDirs.items) {
return backupDirs.items
.filter(item => item.type === 'dir')
.map(item => item.name);
}
return [];
} catch (error) {
console.warn('⚠️ Failed to get backup users:', error.message);
return [];
}
}
/**
* 从GitHub恢复用户数据
* @param {string} userId - 用户ID
* @returns {Promise<boolean>} 恢复结果
*/
async restoreUserDataFromGitHub(userId) {
try {
console.log(`🔄 Restoring data for user ${userId}...`);
// 获取用户的备份索引
const indexPath = `backups/${userId}/index.json`;
const indexResult = await githubService.getFile(indexPath, 0);
if (!indexResult.success) {
console.log(`📭 No backup index found for user ${userId}`);
return false;
}
const index = JSON.parse(indexResult.content);
if (!index.backups || index.backups.length === 0) {
console.log(`📭 No backups found for user ${userId}`);
return false;
}
// 获取最新的备份
const latestBackup = index.backups[0];
const backupResult = await githubService.getFile(latestBackup.path, 0);
if (!backupResult.success) {
console.error(`❌ Failed to get backup file for user ${userId}`);
return false;
}
const userData = JSON.parse(backupResult.content);
// 恢复PPT数据
let restoredPPTs = 0;
for (const [pptId, pptData] of Object.entries(userData.ppts || {})) {
try {
await githubService.savePPT(userId, pptId, pptData.content);
restoredPPTs++;
} catch (error) {
console.warn(`⚠️ Failed to restore PPT ${pptId}:`, error.message);
}
}
console.log(`✅ Restored ${restoredPPTs} PPTs for user ${userId}`);
// 注意:图片数据需要重新生成,因为备份中只有元数据
console.log(`ℹ️ Image data for user ${userId} will be regenerated on demand`);
return true;
} catch (error) {
console.error(`❌ Failed to restore user data for ${userId}:`, error);
return false;
}
}
/**
* 获取备份状态
* @returns {Object} 备份状态信息
*/
getBackupStatus() {
const scheduledBackups = [];
for (const [userId, backup] of this.backupQueue.entries()) {
scheduledBackups.push({
userId,
scheduledTime: new Date(backup.scheduledTime).toISOString(),
remainingTime: Math.max(0, backup.scheduledTime - Date.now())
});
}
return {
initialized: this.initialized,
isRestoring: this.isRestoring,
scheduledBackups: scheduledBackups.length,
backupsInProgress: this.backupInProgress.size,
scheduledBackupDetails: scheduledBackups
};
}
/**
* 清理资源
*/
cleanup() {
// 取消所有调度的备份
for (const [userId] of this.backupQueue.entries()) {
this.cancelUserBackup(userId);
}
console.log('✅ BackupSchedulerService cleaned up');
}
}
// 创建单例实例
const backupSchedulerService = new BackupSchedulerService();
// 优雅关闭处理
process.on('SIGTERM', () => {
backupSchedulerService.cleanup();
});
process.on('SIGINT', () => {
backupSchedulerService.cleanup();
});
export default backupSchedulerService; |