import { useState, useEffect, useRef } from 'react'; import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision'; import { drawLandmarks, analyzeHandGesture } from '../utils/handUtils'; const useHandDetection = (videoRef, canvasRef, isMobile) => { const [handLandmarker, setHandLandmarker] = useState(null); const [handDetected, setHandDetected] = useState(false); const [cardsDetected, setCardsDetected] = useState(false); const [detectedCards, setDetectedCards] = useState([]); const [isMouthOpen, setIsMouthOpen] = useState(false); const [isLeftHand, setIsLeftHand] = useState(true); const [thumbPosition, setThumbPosition] = useState({ x: 0, y: 0 }); const [isFirstLoad, setIsFirstLoad] = useState(true); const [detectionMode, setDetectionMode] = useState('both'); // 'cards', 'hand', 'both' const requestRef = useRef(null); const lastDetectionTimeRef = useRef(0); const lastCardDetectionTimeRef = useRef(0); const isComponentMounted = useRef(true); // Card detection using canvas analysis or Gemini Vision API const detectPlayingCards = async (canvas) => { try { // Canvas를 이미지로 변환 const imageData = canvas.toDataURL('image/jpeg', 0.8); // Gemini Vision API를 사용하여 카드 감지 const response = await fetch('/api/gemini', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ image: imageData, prompt: 'Detect any playing cards in this image. List the cards you see (suit and rank). If no cards are visible, respond with "NO_CARDS".', mode: 'card_detection' }), }); if (response.ok) { const data = await response.json(); if (data.result && data.result !== 'NO_CARDS') { return { detected: true, cards: data.result }; } } return { detected: false, cards: [] }; } catch (error) { console.error('Error detecting cards:', error); return { detected: false, cards: [] }; } }; // Initialize the HandLandmarker useEffect(() => { isComponentMounted.current = true; const initializeHandLandmarker = async () => { try { const vision = await FilesetResolver.forVisionTasks( "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" ); if (!isComponentMounted.current) return; const landmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task", delegate: "GPU" }, runningMode: "VIDEO", numHands: 1, minHandDetectionConfidence: 0.5, minHandPresenceConfidence: 0.5, minTrackingConfidence: 0.5 }); if (!isComponentMounted.current) return; setHandLandmarker(landmarker); console.log("Hand landmarker initialized successfully"); // Set first load to false after initialization setTimeout(() => { if (isComponentMounted.current) { setIsFirstLoad(false); } }, 3000); } catch (error) { console.error("Error initializing hand landmarker:", error); } }; initializeHandLandmarker(); return () => { isComponentMounted.current = false; if (requestRef.current) { cancelAnimationFrame(requestRef.current); requestRef.current = null; } }; }, []); // Process video frames and detect hand gestures and cards useEffect(() => { if (!handLandmarker || !videoRef.current || !canvasRef.current) return; const video = videoRef.current; const canvas = canvasRef.current; const ctx = canvas.getContext('2d'); const detectFrame = async (now) => { if (!isComponentMounted.current) return; if (video.readyState < 2) { requestRef.current = requestAnimationFrame(detectFrame); return; } // Clear the canvas ctx.clearRect(0, 0, canvas.width, canvas.height); // Draw the video frame on the canvas const videoWidth = video.videoWidth; const videoHeight = video.videoHeight; let drawWidth = canvas.width; let drawHeight = canvas.height; let offsetX = 0; let offsetY = 0; ctx.drawImage(video, offsetX, offsetY, drawWidth, drawHeight); // PRIORITY: Detect playing cards first (every 500ms) if (now - lastCardDetectionTimeRef.current > 500) { lastCardDetectionTimeRef.current = now; const cardResult = await detectPlayingCards(canvas); setCardsDetected(cardResult.detected); setDetectedCards(cardResult.cards); // If cards are detected, we can skip hand detection or make it secondary if (cardResult.detected) { setDetectionMode('cards'); // Draw card detection overlay ctx.strokeStyle = '#00ff00'; ctx.lineWidth = 3; ctx.font = '20px Arial'; ctx.fillStyle = '#00ff00'; ctx.fillText('🎴 Playing Cards Detected!', 10, 30); } else { setDetectionMode('hand'); } } // Only detect hands if no cards are detected or in 'both' mode if (!cardsDetected || detectionMode === 'both') { // Hand detection (every 100ms) if (now - lastDetectionTimeRef.current > 100) { lastDetectionTimeRef.current = now; const results = handLandmarker.detectForVideo(video, now); // Check if hands are detected if (results.landmarks && results.landmarks.length > 0) { const landmarks = results.landmarks[0]; setHandDetected(true); // Draw hand landmarks drawLandmarks(ctx, landmarks, canvas, isMobile); // Analyze hand gesture const { isOpen, isLeftHand: isLeft, thumbPosition: thumbPos } = analyzeHandGesture(landmarks); // Update state with hand information setIsLeftHand(isLeft); setThumbPosition({ x: thumbPos.x * canvas.width, y: thumbPos.y * canvas.height }); // Update UI based on hand state if (isOpen !== isMouthOpen) { setIsMouthOpen(isOpen); } } else { // No hands detected setHandDetected(false); if (isMouthOpen) { setIsMouthOpen(false); } } } } // Draw detection status ctx.font = '14px Arial'; ctx.fillStyle = '#ffffff'; ctx.strokeStyle = '#000000'; ctx.lineWidth = 3; const statusText = cardsDetected ? '🎴 Cards Priority Mode' : '✋ Hand Detection Mode'; ctx.strokeText(statusText, 10, canvas.height - 20); ctx.fillText(statusText, 10, canvas.height - 20); requestRef.current = requestAnimationFrame(detectFrame); }; requestRef.current = requestAnimationFrame(detectFrame); return () => { if (requestRef.current) { cancelAnimationFrame(requestRef.current); requestRef.current = null; } }; }, [handLandmarker, isMouthOpen, isMobile, videoRef, canvasRef, cardsDetected, detectionMode]); return { handDetected, cardsDetected, detectedCards, isMouthOpen, isLeftHand, thumbPosition, isFirstLoad, isComponentMounted, detectionMode, // 우선순위 기반 감지 상태 반환 primaryDetection: cardsDetected ? 'cards' : (handDetected ? 'hand' : 'none') }; }; export default useHandDetection;