import { fileURLToPath } from 'url'; import { v4 as uuidv4 } from 'uuid'; import crypto from 'crypto'; import path from 'path'; import fs from 'fs/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); /** * Huggingface存储服务 * 管理PPT图片文件的内存存储、链接生成和版本控制 * 利用Huggingface Space的16G内存进行临时存储 */ class HuggingfaceStorageService { constructor() { // 文件系统路径配置 this.usersDir = path.join(__dirname, '../../data/users'); // 内存存储配置 this.memoryStorage = new Map(); // 存储图片Buffer数据 this.linkMap = new Map(); // 图片链接映射 this.metadataMap = new Map(); // 图片元数据映射 this.userDataMap = new Map(); // 用户数据映射 // 内存管理配置 this.maxMemoryUsage = 14 * 1024 * 1024 * 1024; // 14GB 最大内存使用量 this.currentMemoryUsage = 0; this.cleanupThreshold = 0.8; // 80%时开始清理 // 缓存策略配置 this.maxImageAge = 24 * 60 * 60 * 1000; // 24小时过期 this.maxImagesPerUser = 100; // 每用户最大图片数 this.initialized = false; // 定期清理内存 this.startMemoryCleanup(); } /** * 初始化内存存储服务 */ async initialize() { if (this.initialized) return; try { // 初始化内存存储 this.memoryStorage.clear(); this.linkMap.clear(); this.metadataMap.clear(); this.userDataMap.clear(); this.currentMemoryUsage = 0; this.initialized = true; console.log('✅ HuggingfaceStorageService (Memory Mode) initialized successfully'); console.log(`💾 Max memory usage: ${(this.maxMemoryUsage / 1024 / 1024 / 1024).toFixed(1)}GB`); console.log(`🧹 Cleanup threshold: ${(this.cleanupThreshold * 100)}%`); console.log(`⏰ Image expiry: ${this.maxImageAge / 1000 / 60 / 60}h`); } catch (error) { console.error('❌ Failed to initialize HuggingfaceStorageService:', error); throw error; } } /** * 启动内存清理定时器 */ startMemoryCleanup() { // 每5分钟检查一次内存使用情况 setInterval(() => { this.performMemoryCleanup(); }, 5 * 60 * 1000); console.log('🧹 Memory cleanup scheduler started (every 5 minutes)'); } /** * 执行内存清理 */ performMemoryCleanup() { const memoryUsageRatio = this.currentMemoryUsage / this.maxMemoryUsage; if (memoryUsageRatio > this.cleanupThreshold) { console.log(`🧹 Memory cleanup triggered (${(memoryUsageRatio * 100).toFixed(1)}% usage)`); // 清理过期图片 this.cleanupExpiredImages(); // 如果内存使用仍然过高,清理最旧的图片 if (this.currentMemoryUsage / this.maxMemoryUsage > this.cleanupThreshold) { this.cleanupOldestImages(); } } } /** * 清理过期图片 */ cleanupExpiredImages() { const now = Date.now(); let cleanedCount = 0; let freedMemory = 0; for (const [imageId, metadata] of this.metadataMap.entries()) { const imageAge = now - new Date(metadata.createdAt).getTime(); if (imageAge > this.maxImageAge) { const imageData = this.memoryStorage.get(imageId); if (imageData) { freedMemory += imageData.length; this.memoryStorage.delete(imageId); this.metadataMap.delete(imageId); this.linkMap.delete(imageId); cleanedCount++; } } } this.currentMemoryUsage -= freedMemory; if (cleanedCount > 0) { console.log(`🧹 Cleaned ${cleanedCount} expired images, freed ${(freedMemory / 1024 / 1024).toFixed(1)}MB`); } } /** * 清理最旧的图片 */ cleanupOldestImages() { const images = Array.from(this.metadataMap.entries()) .sort((a, b) => new Date(a[1].createdAt) - new Date(b[1].createdAt)); const targetCleanup = Math.floor(images.length * 0.2); // 清理20%最旧的图片 let cleanedCount = 0; let freedMemory = 0; for (let i = 0; i < targetCleanup && i < images.length; i++) { const [imageId, metadata] = images[i]; const imageData = this.memoryStorage.get(imageId); if (imageData) { freedMemory += imageData.length; this.memoryStorage.delete(imageId); this.metadataMap.delete(imageId); this.linkMap.delete(imageId); cleanedCount++; } } this.currentMemoryUsage -= freedMemory; if (cleanedCount > 0) { console.log(`🧹 Cleaned ${cleanedCount} oldest images, freed ${(freedMemory / 1024 / 1024).toFixed(1)}MB`); } } /** * 生成图片ID * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @param {number} pageIndex - 页面索引 * @param {string} version - 版本号 * @returns {string} 图片ID */ generateImageId(userId, pptId, pageIndex, version = 'latest') { const data = `${userId}-${pptId}-${pageIndex}-${version}`; return crypto.createHash('sha256').update(data).digest('hex').substring(0, 16); } /** * 生成版本号 * @returns {string} 版本号 */ generateVersion() { return Date.now().toString(); } /** * 存储图片到内存 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @param {number} pageIndex - 页面索引 * @param {Buffer} imageBuffer - 图片数据 * @param {Object} options - 存储选项 * @returns {Promise} 存储结果 */ async storeImage(userId, pptId, pageIndex, imageBuffer, options = {}) { if (!this.initialized) { await this.initialize(); } const { format = 'png', quality = 0.9, updateExisting = true } = options; try { // 检查内存使用情况 if (this.currentMemoryUsage + imageBuffer.length > this.maxMemoryUsage) { console.log('⚠️ Memory limit approaching, performing cleanup...'); this.performMemoryCleanup(); // 如果清理后仍然超出限制,拒绝存储 if (this.currentMemoryUsage + imageBuffer.length > this.maxMemoryUsage) { throw new Error('Memory limit exceeded, cannot store image'); } } // 检查用户图片数量限制 const userImages = Array.from(this.metadataMap.values()) .filter(meta => meta.userId === userId); if (userImages.length >= this.maxImagesPerUser) { // 删除用户最旧的图片 const oldestImage = userImages .sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt))[0]; if (oldestImage) { this.deleteImageFromMemory(oldestImage.imageId); } } // 生成版本号 const version = this.generateVersion(); // 生成图片ID const imageId = this.generateImageId(userId, pptId, pageIndex, 'latest'); const versionedImageId = this.generateImageId(userId, pptId, pageIndex, version); // 如果已存在相同的图片ID,先删除旧的 if (this.memoryStorage.has(imageId)) { this.deleteImageFromMemory(imageId); } // 存储图片到内存 this.memoryStorage.set(imageId, imageBuffer); this.memoryStorage.set(versionedImageId, imageBuffer); // 更新内存使用量 this.currentMemoryUsage += imageBuffer.length * 2; // 存储了两份(latest和versioned) // 创建元数据 const metadata = { imageId, versionedImageId, userId, pptId, pageIndex, version, format, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), size: imageBuffer.length, memoryStored: true }; // 存储元数据 this.metadataMap.set(imageId, metadata); this.metadataMap.set(versionedImageId, { ...metadata, imageId: versionedImageId }); // 更新链接映射 this.linkMap.set(imageId, metadata); this.linkMap.set(versionedImageId, { ...metadata, imageId: versionedImageId }); // 更新用户数据统计 if (!this.userDataMap.has(userId)) { this.userDataMap.set(userId, { imageCount: 0, totalSize: 0 }); } const userData = this.userDataMap.get(userId); userData.imageCount++; userData.totalSize += imageBuffer.length; console.log(`✅ Image stored in memory: ${imageId} (${imageBuffer.length} bytes)`); console.log(`💾 Memory usage: ${(this.currentMemoryUsage / 1024 / 1024).toFixed(1)}MB / ${(this.maxMemoryUsage / 1024 / 1024 / 1024).toFixed(1)}GB`); return { success: true, imageId, versionedImageId, version, url: `/api/images/${imageId}`, versionedUrl: `/api/images/${versionedImageId}`, memoryStored: true, size: imageBuffer.length, memoryUsage: { current: this.currentMemoryUsage, max: this.maxMemoryUsage, percentage: (this.currentMemoryUsage / this.maxMemoryUsage * 100).toFixed(1) } }; } catch (error) { console.error('❌ Failed to store image in memory:', error); throw error; } } /** * 从内存中删除图片 * @param {string} imageId - 图片ID */ deleteImageFromMemory(imageId) { const imageData = this.memoryStorage.get(imageId); if (imageData) { this.currentMemoryUsage -= imageData.length; this.memoryStorage.delete(imageId); this.metadataMap.delete(imageId); this.linkMap.delete(imageId); console.log(`🗑️ Deleted image from memory: ${imageId} (freed ${imageData.length} bytes)`); } } /** * 从内存中获取图片 * @param {string} imageId - 图片ID * @returns {Promise} 图片数据和元信息 */ async getImage(imageId) { if (!this.initialized) { await this.initialize(); } // 从内存中获取图片数据 const imageBuffer = this.memoryStorage.get(imageId); if (!imageBuffer) { throw new Error(`Image not found in memory: ${imageId}`); } // 获取元数据 const metadata = this.metadataMap.get(imageId); if (!metadata) { throw new Error(`Image metadata not found: ${imageId}`); } try { return { success: true, data: imageBuffer, metadata: { imageId, format: metadata.format, size: metadata.size, createdAt: metadata.createdAt, updatedAt: metadata.updatedAt, version: metadata.version, memoryStored: true } }; } catch (error) { console.error(`❌ Failed to get image ${imageId}:`, error); throw error; } } /** * 从内存中删除图片 * @param {string} imageId - 图片ID * @returns {Promise} 删除结果 */ async deleteImage(imageId) { if (!this.initialized) { await this.initialize(); } const metadata = this.metadataMap.get(imageId); if (!metadata) { return false; } try { // 从内存中删除图片数据 this.deleteImageFromMemory(imageId); // 如果有版本化的图片ID,也删除它 if (metadata.versionedImageId && metadata.versionedImageId !== imageId) { this.deleteImageFromMemory(metadata.versionedImageId); } // 更新用户数据统计 const userData = this.userDataMap.get(metadata.userId); if (userData) { userData.imageCount = Math.max(0, userData.imageCount - 1); userData.totalSize = Math.max(0, userData.totalSize - metadata.size); } console.log(`✅ Image deleted from memory: ${imageId}`); return true; } catch (error) { console.error(`❌ Failed to delete image from memory ${imageId}:`, error); return false; } } /** * 批量存储PPT所有页面图片 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @param {Array} imageBuffers - 图片Buffer数组 * @param {Object} options - 存储选项 * @returns {Promise} 存储结果数组 */ async storeAllImages(userId, pptId, imageBuffers, options = {}) { const results = []; for (let i = 0; i < imageBuffers.length; i++) { try { if (imageBuffers[i].success) { const result = await this.storeImage( userId, pptId, i, imageBuffers[i].data, options ); results.push({ pageIndex: i, success: true, ...result }); } else { results.push({ pageIndex: i, success: false, error: imageBuffers[i].error }); } } catch (error) { results.push({ pageIndex: i, success: false, error: error.message }); } } return results; } /** * 存储PPT数据到Huggingface硬盘 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @param {Object} pptData - PPT数据 * @returns {Promise} 存储结果 */ async storePPTData(userId, pptId, pptData) { if (!this.initialized) { await this.initialize(); } try { const userDir = path.join(this.usersDir, userId); const pptDir = path.join(userDir, pptId); // 确保目录存在 await fs.mkdir(pptDir, { recursive: true }); // 存储PPT数据 const pptDataFile = path.join(pptDir, 'data.json'); await fs.writeFile(pptDataFile, JSON.stringify(pptData, null, 2)); // 存储元数据 const metadata = { pptId, userId, title: pptData.title || '未命名演示文稿', slidesCount: pptData.slides?.length || 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), storageType: 'huggingface', size: JSON.stringify(pptData).length }; const metaFile = path.join(pptDir, 'meta.json'); await fs.writeFile(metaFile, JSON.stringify(metadata, null, 2)); console.log(`✅ PPT data stored to Huggingface: ${pptId} for user ${userId}`); return { success: true, pptId, metadata }; } catch (error) { console.error(`❌ Failed to store PPT data ${pptId}:`, error); throw error; } } /** * 从Huggingface硬盘获取PPT数据 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @returns {Promise} PPT数据 */ async getPPTData(userId, pptId) { if (!this.initialized) { await this.initialize(); } try { const pptDataFile = path.join(this.usersDir, userId, pptId, 'data.json'); const data = await fs.readFile(pptDataFile, 'utf-8'); const pptData = JSON.parse(data); console.log(`✅ PPT data loaded from Huggingface: ${pptId} for user ${userId}`); return pptData; } catch (error) { console.error(`❌ Failed to get PPT data ${pptId}:`, error); throw error; } } /** * 获取用户的PPT列表 * @param {string} userId - 用户ID * @returns {Promise} PPT列表 */ async getUserPPTList(userId) { if (!this.initialized) { await this.initialize(); } try { const userDir = path.join(this.usersDir, userId); // 检查用户目录是否存在 try { await fs.access(userDir); } catch { console.log(`📁 No PPTs found for user ${userId} in Huggingface storage`); return []; } const pptFolders = await fs.readdir(userDir, { withFileTypes: true }); const pptList = []; for (const folder of pptFolders) { if (folder.isDirectory()) { try { const metaFile = path.join(userDir, folder.name, 'meta.json'); const metaData = await fs.readFile(metaFile, 'utf-8'); const metadata = JSON.parse(metaData); pptList.push({ name: folder.name, title: metadata.title || '未命名演示文稿', updatedAt: metadata.updatedAt || new Date().toISOString(), slidesCount: metadata.slidesCount || 0, storageType: 'huggingface', size: metadata.size || 0, repoUrl: 'Huggingface Storage' }); } catch (error) { console.warn(`⚠️ Skipping invalid PPT folder ${folder.name}:`, error.message); } } } console.log(`📁 Found ${pptList.length} PPTs in Huggingface storage for user ${userId}`); return pptList.sort((a, b) => new Date(b.updatedAt) - new Date(a.updatedAt)); } catch (error) { console.error(`❌ Failed to get PPT list for user ${userId}:`, error); throw error; } } /** * 删除PPT数据 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @returns {Promise} 删除结果 */ async deletePPTData(userId, pptId) { if (!this.initialized) { await this.initialize(); } try { const pptDir = path.join(this.usersDir, userId, pptId); // 递归删除PPT目录 await fs.rm(pptDir, { recursive: true, force: true }); console.log(`✅ PPT data deleted from Huggingface: ${pptId} for user ${userId}`); return true; } catch (error) { console.error(`❌ Failed to delete PPT data ${pptId}:`, error); return false; } } /** * 获取用户的所有图片 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID (可选) * @returns {Promise} 图片列表 */ async getUserImages(userId, pptId = null) { if (!this.initialized) { await this.initialize(); } const userImages = []; for (const [imageId, linkInfo] of this.linkMap.entries()) { if (linkInfo.userId === userId) { if (!pptId || linkInfo.pptId === pptId) { userImages.push({ imageId, pptId: linkInfo.pptId, pageIndex: linkInfo.pageIndex, version: linkInfo.version, format: linkInfo.format, size: linkInfo.size, url: `/api/images/${imageId}`, createdAt: linkInfo.createdAt, updatedAt: linkInfo.updatedAt }); } } } return userImages.sort((a, b) => { if (a.pptId !== b.pptId) { return a.pptId.localeCompare(b.pptId); } return a.pageIndex - b.pageIndex; }); } /** * 清理过期图片 * @param {number} maxAge - 最大保留时间(毫秒) * @returns {Promise} 清理的图片数量 */ async cleanupExpiredImages(maxAge = 30 * 24 * 60 * 60 * 1000) { // 默认30天 if (!this.initialized) { await this.initialize(); } const now = Date.now(); let cleanedCount = 0; for (const [imageId, linkInfo] of this.linkMap.entries()) { const createdAt = new Date(linkInfo.createdAt).getTime(); if (now - createdAt > maxAge) { await this.deleteImage(imageId); cleanedCount++; } } console.log(`🧹 Cleaned up ${cleanedCount} expired images`); return cleanedCount; } /** * 获取存储统计信息 * @returns {Promise} 统计信息 */ async getStorageStats() { if (!this.initialized) { await this.initialize(); } let totalSize = 0; let totalImages = 0; const userStats = {}; // 从内存存储中统计 for (const [imageId, metadata] of this.metadataMap.entries()) { totalSize += metadata.size || 0; totalImages++; if (!userStats[metadata.userId]) { userStats[metadata.userId] = { imageCount: 0, totalSize: 0, ppts: new Set() }; } userStats[metadata.userId].imageCount++; userStats[metadata.userId].totalSize += metadata.size || 0; userStats[metadata.userId].ppts.add(metadata.pptId); } // 转换Set为数组 for (const userId in userStats) { userStats[userId].pptCount = userStats[userId].ppts.size; delete userStats[userId].ppts; } return { totalImages, totalSize, totalSizeMB: Math.round(totalSize / 1024 / 1024 * 100) / 100, userCount: Object.keys(userStats).length, userStats, memoryUsage: { current: this.currentMemoryUsage, max: this.maxMemoryUsage, currentMB: Math.round(this.currentMemoryUsage / 1024 / 1024 * 100) / 100, maxGB: Math.round(this.maxMemoryUsage / 1024 / 1024 / 1024 * 100) / 100, percentage: Math.round(this.currentMemoryUsage / this.maxMemoryUsage * 100 * 100) / 100, imagesInMemory: this.memoryStorage.size } }; } /** * 获取内存使用情况 * @returns {Object} 内存使用统计 */ getMemoryUsage() { return { current: this.currentMemoryUsage, max: this.maxMemoryUsage, currentMB: Math.round(this.currentMemoryUsage / 1024 / 1024 * 100) / 100, maxGB: Math.round(this.maxMemoryUsage / 1024 / 1024 / 1024 * 100) / 100, percentage: Math.round(this.currentMemoryUsage / this.maxMemoryUsage * 100 * 100) / 100, imagesInMemory: this.memoryStorage.size, metadataCount: this.metadataMap.size, userCount: this.userDataMap.size }; } /** * 清空所有内存数据 */ clearAllMemory() { this.memoryStorage.clear(); this.metadataMap.clear(); this.linkMap.clear(); this.userDataMap.clear(); this.currentMemoryUsage = 0; console.log('🧹 All memory data cleared'); } /** * 健康检查 */ async healthCheck() { try { if (!this.initialized) { return { status: 'unhealthy', message: 'Service not initialized', details: { initialized: false, memoryUsage: 0, imagesCount: 0 } }; } // 检查内存使用情况 const memoryUsage = this.getMemoryUsage(); const isMemoryHealthy = memoryUsage.percentage < 90; // 90%以下认为健康 // 检查数据目录是否可访问 let dirAccessible = true; try { await fs.access(this.usersDir); } catch (error) { dirAccessible = false; } const isHealthy = isMemoryHealthy && dirAccessible; return { status: isHealthy ? 'healthy' : 'unhealthy', message: isHealthy ? 'Huggingface Storage Service is running normally' : 'Service has issues', details: { initialized: this.initialized, memoryUsage: memoryUsage, dirAccessible: dirAccessible, usersDir: this.usersDir, isMemoryHealthy: isMemoryHealthy, maxMemoryUsageGB: Math.round(this.maxMemoryUsage / 1024 / 1024 / 1024 * 100) / 100, cleanupThreshold: this.cleanupThreshold } }; } catch (error) { return { status: 'unhealthy', message: 'Health check failed', details: { error: error.message, stack: error.stack } }; } } /** * 获取用户的图片列表 * @param {string} userId - 用户ID * @returns {Array} 用户的图片列表 */ getUserImageList(userId) { const userImages = []; for (const [imageId, metadata] of this.metadataMap.entries()) { if (metadata.userId === userId) { userImages.push({ imageId, pptId: metadata.pptId, pageIndex: metadata.pageIndex, format: metadata.format, size: metadata.size, createdAt: metadata.createdAt, version: metadata.version }); } } return userImages; } /** * 获取PPT的所有图片 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @returns {Array} PPT的图片列表 */ getPPTImages(userId, pptId) { const pptImages = []; for (const [imageId, metadata] of this.metadataMap.entries()) { if (metadata.userId === userId && metadata.pptId === pptId) { pptImages.push({ imageId, pageIndex: metadata.pageIndex, format: metadata.format, size: metadata.size, createdAt: metadata.createdAt, version: metadata.version, url: `/api/images/${imageId}` }); } } return pptImages.sort((a, b) => a.pageIndex - b.pageIndex); } /** * 公开访问:根据用户ID、PPT ID和页面索引获取图片 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @param {number} pageIndex - 页面索引 * @returns {Promise} 图片数据和元信息 */ async getPublicImage(userId, pptId, pageIndex) { if (!this.initialized) { await this.initialize(); } // 生成图片ID const imageId = this.generateImageId(userId, pptId, pageIndex, 'latest'); // 从内存中获取图片数据 const imageBuffer = this.memoryStorage.get(imageId); if (!imageBuffer) { throw new Error(`Public image not found: ${userId}/${pptId}/${pageIndex}`); } // 获取元数据 const metadata = this.metadataMap.get(imageId); if (!metadata) { throw new Error(`Public image metadata not found: ${userId}/${pptId}/${pageIndex}`); } console.log(`✅ Public image accessed: ${userId}/${pptId}/${pageIndex} (${imageBuffer.length} bytes)`); return { success: true, data: imageBuffer, metadata: { imageId, userId, pptId, pageIndex, format: metadata.format, size: metadata.size, createdAt: metadata.createdAt, updatedAt: metadata.updatedAt, version: metadata.version, memoryStored: true } }; } /** * 公开访问:检查图片是否存在 * @param {string} userId - 用户ID * @param {string} pptId - PPT ID * @param {number} pageIndex - 页面索引 * @returns {boolean} 图片是否存在 */ hasPublicImage(userId, pptId, pageIndex) { if (!this.initialized) { return false; } const imageId = this.generateImageId(userId, pptId, pageIndex, 'latest'); return this.memoryStorage.has(imageId) && this.metadataMap.has(imageId); } /** * 公开访问:获取用户的PPT列表(仅包含有图片的PPT) * @param {string} userId - 用户ID * @returns {Array} PPT列表 */ getPublicPPTList(userId) { if (!this.initialized) { return []; } const pptMap = new Map(); for (const [imageId, metadata] of this.metadataMap.entries()) { if (metadata.userId === userId) { if (!pptMap.has(metadata.pptId)) { pptMap.set(metadata.pptId, { pptId: metadata.pptId, userId: metadata.userId, pageCount: 0, totalSize: 0, createdAt: metadata.createdAt, updatedAt: metadata.updatedAt, pages: [] }); } const pptInfo = pptMap.get(metadata.pptId); pptInfo.pageCount++; pptInfo.totalSize += metadata.size; pptInfo.pages.push({ pageIndex: metadata.pageIndex, format: metadata.format, size: metadata.size, url: `/api/public/image/${userId}/${metadata.pptId}/${metadata.pageIndex}` }); // 更新最新时间 if (new Date(metadata.updatedAt) > new Date(pptInfo.updatedAt)) { pptInfo.updatedAt = metadata.updatedAt; } } } // 排序页面并返回结果 const result = Array.from(pptMap.values()); result.forEach(ppt => { ppt.pages.sort((a, b) => a.pageIndex - b.pageIndex); }); return result.sort((a, b) => new Date(b.updatedAt) - new Date(a.updatedAt)); } } // 创建单例实例 const huggingfaceStorageService = new HuggingfaceStorageService(); export default huggingfaceStorageService;