Upload 51 files

src/components/MultiSourceCaptioningView.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useRef, useEffect } from "react";
+import { useState, useRef, useEffect } from "react";
 import { useVLMContext } from "../context/useVLMContext";
 import { extractJsonFromMarkdown, drawBoundingBoxesOnCanvas } from "./BoxAnnotator";
 
@@ -82,55 +82,12 @@ export default function MultiSourceCaptioningView() {
   const [inferenceStatus, setInferenceStatus] = useState<string>("");
 
   const videoRef = useRef<HTMLVideoElement | null>(null);
-  const overlayVideoRef = useRef<HTMLVideoElement | null>(null); // NEW: overlay video
+  const overlayVideoRef = useRef<HTMLVideoElement | null>(null);
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const imageRef = useRef<HTMLImageElement | null>(null);
   const webcamStreamRef = useRef<MediaStream | null>(null);
   const { isLoaded, isLoading, error: modelError, runInference } = useVLMContext();
 
-  // Persistent boxes state: {boxes: [...], timestamp: number}
-  const [persistentBoxes, setPersistentBoxes] = useState<{boxes: {label: string, bbox_2d: number[]}[], timestamp: number}[]>([]);
-  const BOX_PERSIST_MS = 2000; // 2 seconds
-
-  // Helper: Add new boxes with timestamp
-  const addBoxesWithTimestamp = (boxes: {label: string, bbox_2d: number[]}[]) => {
-    if (!boxes || boxes.length === 0) return;
-    setPersistentBoxes((prev: {boxes: {label: string, bbox_2d: number[]}[], timestamp: number}[]) => [
-      ...prev.filter((entry: {boxes: {label: string, bbox_2d: number[]}[], timestamp: number}) => Date.now() - entry.timestamp < BOX_PERSIST_MS),
-      { boxes, timestamp: Date.now() }
-    ]);
-  };
-
-  // Helper: Get all boxes from last 2 seconds
-  const getCurrentBoxes = () => {
-    const now = Date.now();
-    return persistentBoxes
-      .filter((entry: {boxes: {label: string, bbox_2d: number[]}[], timestamp: number}) => now - entry.timestamp < BOX_PERSIST_MS)
-      .flatMap((entry: {boxes: {label: string, bbox_2d: number[]}[], timestamp: number}) => entry.boxes);
-  };
-
-  // Synchronize overlay video with main video
-  useEffect(() => {
-    const main = videoRef.current;
-    const overlay = overlayVideoRef.current;
-    if (!main || !overlay) return;
-    // Sync play/pause
-    main.addEventListener('play', () => overlay.play());
-    main.addEventListener('pause', () => overlay.pause());
-    // Sync seeking
-    const syncTime = () => { if (Math.abs(main.currentTime - overlay.currentTime) > 0.05) overlay.currentTime = main.currentTime; };
-    main.addEventListener('seeked', syncTime);
-    main.addEventListener('timeupdate', syncTime);
-    // Clean up
-    return () => {
-      main.removeEventListener('play', () => overlay.play());
-      main.removeEventListener('pause', () => overlay.pause());
-      main.removeEventListener('seeked', syncTime);
-      main.removeEventListener('timeupdate', syncTime);
-    };
-  }, [videoRef, overlayVideoRef, uploadedUrl, videoUrl, mode]);
-
-  // Update: processVideoFrame now adds boxes to persistentBoxes
   const processVideoFrame = async () => {
     if (!videoRef.current || !canvasRef.current) return;
     const video = videoRef.current;
@@ -142,13 +99,12 @@ export default function MultiSourceCaptioningView() {
     if (!ctx) return;
     ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
     await runInference(video, prompt, (output: string) => {
-      setDebugOutput(output);
+      setDebugOutput(output); // <-- Ensure Raw Model Output is updated
       let boxes = extractJsonFromMarkdown(output) || [];
       if (boxes.length === 0 && Array.isArray(output)) {
         boxes = parseFlatBoxArray(output);
       }
       boxes = normalizeBoxes(boxes);
-      // Restore debug logging
       console.log("Model output:", output);
       console.log("Boxes after normalization:", boxes);
       console.log("Canvas size:", canvas.width, canvas.height);
@@ -158,35 +114,13 @@ export default function MultiSourceCaptioningView() {
       }
       if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
       if (Array.isArray(boxes) && boxes.length > 0) {
-        addBoxesWithTimestamp(boxes); // <-- Add to persistent state
-      }
-    });
-  };
-
-  // Draw persistent boxes on every frame
-  useEffect(() => {
-    const draw = () => {
-      // Use overlay video for dimensions if available, else fallback to main video
-      const video = overlayVideoRef.current || videoRef.current;
-      if (!video || !canvasRef.current) return;
-      if (video.videoWidth === 0) return;
-      const canvas = canvasRef.current;
-      canvas.width = video.videoWidth;
-      canvas.height = video.videoHeight;
-      const ctx = canvas.getContext("2d");
-      if (!ctx) return;
-      ctx.clearRect(0, 0, canvas.width, canvas.height);
-      const boxes = getCurrentBoxes();
-      if (boxes.length > 0) {
         const scaleX = canvas.width / video.videoWidth;
         const scaleY = canvas.height / video.videoHeight;
-        drawBoundingBoxesOnCanvas(ctx, boxes, { color: "#FF00FF", lineWidth: 4, font: "20px Arial", scaleX, scaleY });
+        ctx.clearRect(0, 0, canvas.width, canvas.height); // Clear canvas before drawing boxes
+        drawBoundingBoxesOnCanvas(ctx, boxes, { color: "#FF00FF", lineWidth: 4, font: "20px Arial", scaleX, scaleY }); // Use visible color and thick line
       }
-    };
-    draw();
-    const interval = setInterval(draw, 100);
-    return () => clearInterval(interval);
-  }, [persistentBoxes, videoRef, overlayVideoRef, canvasRef]);
+    });
+  };
 
   const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
     const file = e.target.files?.[0] || null;
@@ -442,7 +376,7 @@ export default function MultiSourceCaptioningView() {
                   controls
                   autoPlay
                   loop
-                  className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 z-0"
+                  className="w-full rounded-lg shadow-lg mb-2"
                   style={{ background: "#222" }}
                 />
                 <video
@@ -452,8 +386,8 @@ export default function MultiSourceCaptioningView() {
                   autoPlay
                   loop
                   muted
-                  className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 z-10 opacity-60 pointer-events-none"
-                  style={{ background: "#222" }}
+                  className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 opacity-60 pointer-events-none"
+                  style={{ background: "#222", zIndex: 10 }}
                 />
                 <canvas
                   ref={canvasRef}
@@ -461,7 +395,7 @@ export default function MultiSourceCaptioningView() {
                   style={{ zIndex: 20, pointerEvents: "none" }}
                 />
                 <button
-                  className="mt-4 px-6 py-2 rounded-lg bg-blue-600 text-white font-semibold relative z-30"
+                  className="mt-4 px-6 py-2 rounded-lg bg-blue-600 text-white font-semibold"
                   onClick={handleToggleUrlProcessing}
                 >
                   {urlProcessing ? "Stop Processing" : "Start Processing"}
@@ -475,6 +409,11 @@ export default function MultiSourceCaptioningView() {
               >
                 Test Draw Box
               </button>
+              <div className="mt-2 p-2 bg-gray-800 rounded text-xs">
+                <div>Canvas: {canvasDims ? `${canvasDims.w}x${canvasDims.h}` : "-"} | Video: {videoDims ? `${videoDims.w}x${videoDims.h}` : "-"}</div>
+                <div>Raw Model Output:</div>
+                <pre className="overflow-x-auto max-h-32 whitespace-pre-wrap">{debugOutput}</pre>
+              </div>
             </div>
           )}
           {mode === "File" && (
@@ -529,7 +468,7 @@ export default function MultiSourceCaptioningView() {
                     controls
                     autoPlay
                     loop
-                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 z-0"
+                    className="w-full rounded-lg shadow-lg mb-2"
                     style={{ background: "#222" }}
                   />
                   <video
@@ -539,8 +478,8 @@ export default function MultiSourceCaptioningView() {
                     autoPlay
                     loop
                     muted
-                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 z-10 opacity-60 pointer-events-none"
-                    style={{ background: "#222" }}
+                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 opacity-60 pointer-events-none"
+                    style={{ background: "#222", zIndex: 10 }}
                   />
                   <canvas
                     ref={canvasRef}
@@ -548,7 +487,7 @@ export default function MultiSourceCaptioningView() {
                     style={{ zIndex: 20, pointerEvents: "none" }}
                   />
                   <button
-                    className="mt-4 px-6 py-2 rounded-lg bg-blue-600 text-white font-semibold relative z-30"
+                    className="mt-4 px-6 py-2 rounded-lg bg-blue-600 text-white font-semibold"
                     onClick={handleToggleVideoProcessing}
                   >
                     {videoProcessing ? "Stop Processing" : "Start Processing"}
@@ -564,7 +503,7 @@ export default function MultiSourceCaptioningView() {
                     controls
                     autoPlay
                     loop
-                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 z-0"
+                    className="w-full rounded-lg shadow-lg mb-2"
                     style={{ background: "#222" }}
                   />
                   <video
@@ -574,8 +513,8 @@ export default function MultiSourceCaptioningView() {
                     autoPlay
                     loop
                     muted
-                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 z-10 opacity-60 pointer-events-none"
-                    style={{ background: "#222" }}
+                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 opacity-60 pointer-events-none"
+                    style={{ background: "#222", zIndex: 10 }}
                   />
                   <canvas
                     ref={canvasRef}
@@ -583,7 +522,7 @@ export default function MultiSourceCaptioningView() {
                     style={{ zIndex: 20, pointerEvents: "none" }}
                   />
                   <button
-                    className="mt-4 px-6 py-2 rounded-lg bg-blue-600 text-white font-semibold relative z-30"
+                    className="mt-4 px-6 py-2 rounded-lg bg-blue-600 text-white font-semibold"
                     onClick={handleToggleExampleProcessing}
                   >
                     {exampleProcessing ? "Stop Processing" : "Start Processing"}
@@ -598,14 +537,13 @@ export default function MultiSourceCaptioningView() {
               >
                 Test Draw Box
               </button>
+              <div className="mt-2 p-2 bg-gray-800 rounded text-xs">
+                <div>Canvas: {canvasDims ? `${canvasDims.w}x${canvasDims.h}` : "-"} | Video: {videoDims ? `${videoDims.w}x${videoDims.h}` : "-"}</div>
+                <div>Raw Model Output:</div>
+                <pre className="overflow-x-auto max-h-32 whitespace-pre-wrap">{debugOutput}</pre>
+              </div>
             </div>
           )}
-          {/* Always show Raw Model Output at the bottom */}
-          <div className="mt-2 p-2 bg-gray-800 rounded text-xs w-full max-w-xl">
-            <div>Canvas: {canvasDims ? `${canvasDims.w}x${canvasDims.h}` : "-"} | Video: {videoDims ? `${videoDims.w}x${videoDims.h}` : "-"}</div>
-            <div>Raw Model Output:</div>
-            <pre className="overflow-x-auto max-h-32 whitespace-pre-wrap">{debugOutput}</pre>
-          </div>
         </div>
       </div>
     </div>