import { useState, useRef, useEffect } from "react"; import { useVLMContext } from "../context/useVLMContext"; import { extractJsonFromMarkdown, drawBoundingBoxesOnCanvas } from "./BoxAnnotator"; const MODES = ["Webcam", "URL", "File"] as const; type Mode = typeof MODES[number]; const EXAMPLE_VIDEO_URL = "/videos/1.mp4"; const EXAMPLE_PROMPT = "Detect all people in the image. For each person, output a JSON array of objects with fields: 'label' (string) and 'bbox_2d' ([x1, y1, x2, y2]) where coordinates are in pixel values. Example: [{\"label\": \"person\", \"bbox_2d\": [100, 50, 200, 300]}]"; function parseFlatBoxArray(arr: any[]): { label: string, bbox_2d: number[] }[] { if (typeof arr[0] === "string" && Array.isArray(arr[1])) { const label = arr[0]; return arr.slice(1).map(bbox => ({ label, bbox_2d: bbox })); } return []; } function normalizeBoxes(raw: any): { label: string, bbox_2d: number[] }[] { let boxes = []; if (Array.isArray(raw)) { boxes = raw; } else if (typeof raw === "object" && raw !== null) { boxes = [raw]; } // Normalize bbox_2d to flat [x1, y1, x2, y2] return boxes.map(obj => { let bbox = obj.bbox_2d; if (Array.isArray(bbox) && Array.isArray(bbox[0])) { // Convert [[x1, y1], [x2, y2]] to [x1, y1, x2, y2] bbox = [bbox[0][0], bbox[0][1], bbox[1][0], bbox[1][1]]; } return { ...obj, bbox_2d: bbox }; }); } function isImageFile(file: File) { return file.type.startsWith("image/"); } function isVideoFile(file: File) { return file.type.startsWith("video/"); } export default function MultiSourceCaptioningView() { const [mode, setMode] = useState("File"); const [videoUrl, setVideoUrl] = useState(EXAMPLE_VIDEO_URL); const [inputUrl, setInputUrl] = useState(EXAMPLE_VIDEO_URL); const [prompt, setPrompt] = useState(EXAMPLE_PROMPT); const [processing, setProcessing] = useState(false); const [error, setError] = useState(null); const [webcamActive, setWebcamActive] = useState(false); const [uploadedFile, setUploadedFile] = useState(null); const [uploadedUrl, setUploadedUrl] = useState(""); const [videoProcessing, setVideoProcessing] = useState(false); const [imageProcessed, setImageProcessed] = useState(false); const [exampleProcessing, setExampleProcessing] = useState(false); const [urlProcessing, setUrlProcessing] = useState(false); const [debugOutput, setDebugOutput] = useState(""); const [canvasDims, setCanvasDims] = useState<{w:number,h:number}|null>(null); const [videoDims, setVideoDims] = useState<{w:number,h:number}|null>(null); const [inferenceStatus, setInferenceStatus] = useState(""); const videoRef = useRef(null); const canvasRef = useRef(null); const imageRef = useRef(null); const webcamStreamRef = useRef(null); const { isLoaded, isLoading, error: modelError, runInference } = useVLMContext(); const processVideoFrame = async () => { if (!videoRef.current || !canvasRef.current) return; const video = videoRef.current; const canvas = canvasRef.current; if (video.paused || video.ended || video.videoWidth === 0) return; canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); await runInference(video, prompt, (output: string) => { setDebugOutput(output); // <-- Ensure Raw Model Output is updated let boxes = extractJsonFromMarkdown(output) || []; if (boxes.length === 0 && Array.isArray(output)) { boxes = parseFlatBoxArray(output); } boxes = normalizeBoxes(boxes); if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); }); }; const handleFileChange = (e: React.ChangeEvent) => { const file = e.target.files?.[0] || null; setUploadedFile(file); setUploadedUrl(file ? URL.createObjectURL(file) : ""); setError(null); setImageProcessed(false); setVideoProcessing(false); setExampleProcessing(false); }; // Webcam setup and teardown (unchanged) useEffect(() => { if (mode !== "Webcam") { if (webcamStreamRef.current) { webcamStreamRef.current.getTracks().forEach((track: MediaStreamTrack) => track.stop()); webcamStreamRef.current = null; } setWebcamActive(false); return; } const setupWebcam = async () => { try { setError(null); const stream = await navigator.mediaDevices.getUserMedia({ video: true }); webcamStreamRef.current = stream; if (videoRef.current) { videoRef.current.srcObject = stream; setWebcamActive(true); } } catch (e) { setError("Could not access webcam: " + (e instanceof Error ? e.message : String(e))); setWebcamActive(false); } }; setupWebcam(); return () => { if (webcamStreamRef.current) { webcamStreamRef.current.getTracks().forEach((track: MediaStreamTrack) => track.stop()); webcamStreamRef.current = null; } setWebcamActive(false); }; }, [mode]); // Webcam mode: process frames with setInterval useEffect(() => { if (mode !== "Webcam" || !isLoaded || !webcamActive) return; let interval: ReturnType | null = null; interval = setInterval(() => { processVideoFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, webcamActive]); // URL mode: process frames with setInterval useEffect(() => { if (mode !== "URL" || !isLoaded || !urlProcessing) return; let interval: ReturnType | null = null; interval = setInterval(() => { processVideoFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, urlProcessing]); // File video mode: process frames with setInterval useEffect(() => { if (mode !== "File" || !isLoaded || !uploadedFile || !isVideoFile(uploadedFile) || !videoProcessing) return; let interval: ReturnType | null = null; interval = setInterval(() => { processVideoFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, uploadedFile, videoProcessing]); // Example video mode: process frames with setInterval useEffect(() => { if (mode !== "File" || uploadedFile || !isLoaded || !exampleProcessing) return; let interval: ReturnType | null = null; interval = setInterval(() => { processVideoFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, uploadedFile, exampleProcessing]); // File mode: process uploaded image (only on button click) const handleProcessImage = async () => { if (!isLoaded || !uploadedFile || !isImageFile(uploadedFile) || !imageRef.current || !canvasRef.current) return; const img = imageRef.current; const canvas = canvasRef.current; canvas.width = img.naturalWidth; canvas.height = img.naturalHeight; setCanvasDims({w:canvas.width,h:canvas.height}); setVideoDims({w:img.naturalWidth,h:img.naturalHeight}); const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(img, 0, 0, canvas.width, canvas.height); setProcessing(true); setError(null); setInferenceStatus("Running inference..."); await runInference(img, prompt, (output: string) => { setDebugOutput(output); setInferenceStatus("Inference complete."); ctx.drawImage(img, 0, 0, canvas.width, canvas.height); let boxes = extractJsonFromMarkdown(output) || []; if (boxes.length === 0 && Array.isArray(output)) { boxes = parseFlatBoxArray(output); } boxes = normalizeBoxes(boxes); if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); setImageProcessed(true); }); setProcessing(false); }; // File mode: process uploaded video frames (start/stop) const handleToggleVideoProcessing = () => { setVideoProcessing((prev) => !prev); }; // Handle start/stop for example video processing const handleToggleExampleProcessing = () => { setExampleProcessing((prev) => !prev); }; // Handle start/stop for URL video processing const handleToggleUrlProcessing = () => { setUrlProcessing((prev) => !prev); }; // Test draw box function const handleTestDrawBox = () => { if (!canvasRef.current) return; const canvas = canvasRef.current; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.strokeStyle = "#FF00FF"; ctx.lineWidth = 4; ctx.strokeRect(40, 40, Math.max(40,canvas.width/4), Math.max(40,canvas.height/4)); ctx.font = "20px Arial"; ctx.fillStyle = "#FF00FF"; ctx.fillText("Test Box", 50, 35); }; return (
{isLoading ? "Loading model..." : isLoaded ? "Model loaded" : modelError ? `Model error: ${modelError}` : "Model not loaded"}
{inferenceStatus}
{/* Mode Selector */}
{MODES.map((m) => ( ))}
{/* Mode Content */}
{mode === "Webcam" && (