import { useState, useRef, useEffect } from "react"; import { useVLMContext } from "../context/useVLMContext"; import { extractJsonFromMarkdown, drawBoundingBoxesOnCanvas } from "./BoxAnnotator"; const MODES = ["Webcam", "URL", "File"] as const; type Mode = typeof MODES[number]; const EXAMPLE_VIDEO_URL = "https://dm0qx8t0i9gc9.cloudfront.net/watermarks/video/47Fj2US_gijjhliil/large-group-of-people-walking-at-city_rpem-bqvu__f51e7e41cf28b832502c9709c8eb2fd8__P360.mp4"; const EXAMPLE_PROMPT = "Find as many objects in the video and box them."; function isImageFile(file: File) { return file.type.startsWith("image/"); } function isVideoFile(file: File) { return file.type.startsWith("video/"); } export default function MultiSourceCaptioningView() { const [mode, setMode] = useState("File"); const [videoUrl, setVideoUrl] = useState(EXAMPLE_VIDEO_URL); const [inputUrl, setInputUrl] = useState(EXAMPLE_VIDEO_URL); const [prompt, setPrompt] = useState(EXAMPLE_PROMPT); const [processing, setProcessing] = useState(false); const [error, setError] = useState(null); const [webcamActive, setWebcamActive] = useState(false); const [uploadedFile, setUploadedFile] = useState(null); const [uploadedUrl, setUploadedUrl] = useState(""); const [videoProcessing, setVideoProcessing] = useState(false); const [imageProcessed, setImageProcessed] = useState(false); const [exampleProcessing, setExampleProcessing] = useState(false); const [urlProcessing, setUrlProcessing] = useState(false); const [debugOutput, setDebugOutput] = useState(""); const [canvasDims, setCanvasDims] = useState<{w:number,h:number}|null>(null); const [videoDims, setVideoDims] = useState<{w:number,h:number}|null>(null); const [inferenceStatus, setInferenceStatus] = useState(""); const videoRef = useRef(null); const canvasRef = useRef(null); const imageRef = useRef(null); const webcamStreamRef = useRef(null); const { isLoaded, isLoading, error: modelError, runInference } = useVLMContext(); // Webcam setup and teardown (unchanged) useEffect(() => { if (mode !== "Webcam") { if (webcamStreamRef.current) { webcamStreamRef.current.getTracks().forEach((track: MediaStreamTrack) => track.stop()); webcamStreamRef.current = null; } setWebcamActive(false); return; } const setupWebcam = async () => { try { setError(null); const stream = await navigator.mediaDevices.getUserMedia({ video: true }); webcamStreamRef.current = stream; if (videoRef.current) { videoRef.current.srcObject = stream; setWebcamActive(true); } } catch (e) { setError("Could not access webcam: " + (e instanceof Error ? e.message : String(e))); setWebcamActive(false); } }; setupWebcam(); return () => { if (webcamStreamRef.current) { webcamStreamRef.current.getTracks().forEach((track: MediaStreamTrack) => track.stop()); webcamStreamRef.current = null; } setWebcamActive(false); }; }, [mode]); // Process webcam frames (unchanged) useEffect(() => { if (mode !== "Webcam" || !isLoaded || !webcamActive) return; let interval: ReturnType | null = null; const processFrame = async () => { if (!videoRef.current || !canvasRef.current) return; const video = videoRef.current; const canvas = canvasRef.current; if (video.videoWidth === 0) return; canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); try { setProcessing(true); setError(null); setInferenceStatus("Running inference..."); const fakeVideo = { videoWidth: canvas.width, videoHeight: canvas.height, getContext: () => ctx, } as unknown as HTMLVideoElement; const result = await runInference(fakeVideo, prompt); setDebugOutput(result); setInferenceStatus("Inference complete."); ctx.drawImage(video, 0, 0, canvas.width, canvas.height); const boxes = extractJsonFromMarkdown(result) || []; if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); } catch (e) { setError(e instanceof Error ? e.message : String(e)); setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e))); } finally { setProcessing(false); } }; interval = setInterval(() => { processFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, webcamActive]); // URL mode: process video frames only when urlProcessing is true useEffect(() => { if (mode !== "URL" || !isLoaded || !urlProcessing) return; let interval: ReturnType | null = null; const processFrame = async () => { if (!videoRef.current || !canvasRef.current) return; const video = videoRef.current; const canvas = canvasRef.current; if (video.paused || video.ended || video.videoWidth === 0) return; canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); try { setProcessing(true); setError(null); setInferenceStatus("Running inference..."); const fakeVideo = { videoWidth: canvas.width, videoHeight: canvas.height, getContext: () => ctx, } as unknown as HTMLVideoElement; const result = await runInference(fakeVideo, prompt); setDebugOutput(result); setInferenceStatus("Inference complete."); ctx.drawImage(video, 0, 0, canvas.width, canvas.height); const boxes = extractJsonFromMarkdown(result) || []; if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); } catch (e) { setError(e instanceof Error ? e.message : String(e)); setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e))); } finally { setProcessing(false); } }; interval = setInterval(() => { processFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, urlProcessing]); // File mode: process uploaded image (only on button click) const handleProcessImage = async () => { if (!isLoaded || !uploadedFile || !isImageFile(uploadedFile) || !imageRef.current || !canvasRef.current) return; const img = imageRef.current; const canvas = canvasRef.current; canvas.width = img.naturalWidth; canvas.height = img.naturalHeight; setCanvasDims({w:canvas.width,h:canvas.height}); setVideoDims({w:img.naturalWidth,h:img.naturalHeight}); const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(img, 0, 0, canvas.width, canvas.height); try { setProcessing(true); setError(null); setInferenceStatus("Running inference..."); const fakeVideo = { videoWidth: canvas.width, videoHeight: canvas.height, getContext: () => ctx, } as unknown as HTMLVideoElement; const result = await runInference(fakeVideo, prompt); setDebugOutput(result); setInferenceStatus("Inference complete."); ctx.drawImage(img, 0, 0, canvas.width, canvas.height); const boxes = extractJsonFromMarkdown(result) || []; if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); setImageProcessed(true); } catch (e) { setError(e instanceof Error ? e.message : String(e)); setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e))); } finally { setProcessing(false); } }; // File mode: process uploaded video frames (start/stop) useEffect(() => { if (mode !== "File" || !isLoaded || !uploadedFile || !isVideoFile(uploadedFile) || !videoProcessing) return; let interval: ReturnType | null = null; const processFrame = async () => { if (!videoRef.current || !canvasRef.current) return; const video = videoRef.current; const canvas = canvasRef.current; if (video.paused || video.ended || video.videoWidth === 0) return; canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); try { setProcessing(true); setError(null); setInferenceStatus("Running inference..."); const fakeVideo = { videoWidth: canvas.width, videoHeight: canvas.height, getContext: () => ctx, } as unknown as HTMLVideoElement; const result = await runInference(fakeVideo, prompt); setDebugOutput(result); setInferenceStatus("Inference complete."); ctx.drawImage(video, 0, 0, canvas.width, canvas.height); const boxes = extractJsonFromMarkdown(result) || []; if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); } catch (e) { setError(e instanceof Error ? e.message : String(e)); setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e))); } finally { setProcessing(false); } }; interval = setInterval(() => { processFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, uploadedFile, videoProcessing]); // File mode: process example video frames (start/stop) useEffect(() => { if (mode !== "File" || uploadedFile || !isLoaded || !exampleProcessing) return; let interval: ReturnType | null = null; const processFrame = async () => { if (!videoRef.current || !canvasRef.current) return; const video = videoRef.current; const canvas = canvasRef.current; if (video.paused || video.ended || video.videoWidth === 0) return; canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); try { setProcessing(true); setError(null); setInferenceStatus("Running inference..."); const fakeVideo = { videoWidth: canvas.width, videoHeight: canvas.height, getContext: () => ctx, } as unknown as HTMLVideoElement; const result = await runInference(fakeVideo, prompt); setDebugOutput(result); setInferenceStatus("Inference complete."); ctx.drawImage(video, 0, 0, canvas.width, canvas.height); const boxes = extractJsonFromMarkdown(result) || []; if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid."); drawBoundingBoxesOnCanvas(ctx, boxes); } catch (e) { setError(e instanceof Error ? e.message : String(e)); setInferenceStatus("Inference error: " + (e instanceof Error ? e.message : String(e))); } finally { setProcessing(false); } }; interval = setInterval(() => { processFrame(); }, 1000); return () => { if (interval) clearInterval(interval); }; }, [mode, isLoaded, prompt, runInference, uploadedFile, exampleProcessing]); // Handle file upload const handleFileChange = (e: any) => { const file = e.target.files?.[0] || null; setUploadedFile(file); setUploadedUrl(file ? URL.createObjectURL(file) : ""); setError(null); setImageProcessed(false); setVideoProcessing(false); setExampleProcessing(false); }; // Handle start/stop for video processing const handleToggleVideoProcessing = () => { setVideoProcessing((prev) => !prev); }; // Handle start/stop for example video processing const handleToggleExampleProcessing = () => { setExampleProcessing((prev) => !prev); }; // Handle start/stop for URL video processing const handleToggleUrlProcessing = () => { setUrlProcessing((prev) => !prev); }; // Test draw box function const handleTestDrawBox = () => { if (!canvasRef.current) return; const canvas = canvasRef.current; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.strokeStyle = "#FF00FF"; ctx.lineWidth = 4; ctx.strokeRect(40, 40, Math.max(40,canvas.width/4), Math.max(40,canvas.height/4)); ctx.font = "20px Arial"; ctx.fillStyle = "#FF00FF"; ctx.fillText("Test Box", 50, 35); }; return (
{isLoading ? "Loading model..." : isLoaded ? "Model loaded" : modelError ? `Model error: ${modelError}` : "Model not loaded"}
{inferenceStatus}
{/* Mode Selector */}
{MODES.map((m) => ( ))}
{/* Mode Content */}
{mode === "Webcam" && (