Upload 51 files

src/components/MultiSourceCaptioningView.tsx

@@ -83,6 +83,7 @@ export default function MultiSourceCaptioningView() {
 
   const videoRef = useRef<HTMLVideoElement | null>(null);
   const overlayVideoRef = useRef<HTMLVideoElement | null>(null);
+  const processingVideoRef = useRef<HTMLVideoElement | null>(null);
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const imageRef = useRef<HTMLImageElement | null>(null);
   const webcamStreamRef = useRef<MediaStream | null>(null);
@@ -116,8 +117,8 @@ export default function MultiSourceCaptioningView() {
   }, [videoRef, overlayVideoRef, uploadedUrl, videoUrl, mode]);
 
   const processVideoFrame = async () => {
-    if (!videoRef.current || !canvasRef.current) return;
-    const video = videoRef.current;
+    if (!processingVideoRef.current || !canvasRef.current) return;
+    const video = processingVideoRef.current;
     const canvas = canvasRef.current;
     if (video.paused || video.ended || video.videoWidth === 0) return;
     canvas.width = video.videoWidth;
@@ -132,19 +133,20 @@ export default function MultiSourceCaptioningView() {
         boxes = parseFlatBoxArray(output);
       }
       boxes = normalizeBoxes(boxes);
-      console.log("Model output:", output);
-      console.log("Boxes after normalization:", boxes);
-      console.log("Canvas size:", canvas.width, canvas.height);
-      if (boxes.length > 0) {
-        const [x1, y1, x2, y2] = boxes[0].bbox_2d;
-        console.log("First box coords:", x1, y1, x2, y2);
-      }
-      if (boxes.length === 0) setInferenceStatus("No boxes detected or model output invalid.");
+      // Box persistence logic (2 seconds)
+      const now = Date.now();
       if (Array.isArray(boxes) && boxes.length > 0) {
+        if (!window._boxHistory) window._boxHistory = [];
+        window._boxHistory = window._boxHistory.filter((b) => now - b.timestamp < 2000);
+        window._boxHistory.push(...boxes.map(box => ({ ...box, timestamp: now })));
+      }
+      // Draw all boxes from last 2 seconds
+      const boxHistory = (window._boxHistory || []).filter((b) => now - b.timestamp < 2000);
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      if (boxHistory.length > 0) {
         const scaleX = canvas.width / video.videoWidth;
         const scaleY = canvas.height / video.videoHeight;
-        ctx.clearRect(0, 0, canvas.width, canvas.height); // Clear canvas before drawing boxes
-        drawBoundingBoxesOnCanvas(ctx, boxes, { color: "#FF00FF", lineWidth: 4, font: "20px Arial", scaleX, scaleY }); // Use visible color and thick line
+        drawBoundingBoxesOnCanvas(ctx, boxHistory, { color: "#FF00FF", lineWidth: 4, font: "20px Arial", scaleX, scaleY });
       }
     });
   };
@@ -397,24 +399,27 @@ export default function MultiSourceCaptioningView() {
                 />
               </div>
               <div className="relative w-full max-w-xl">
+                {/* Visible overlay video for user */}
                 <video
-                  ref={videoRef}
+                  ref={overlayVideoRef}
                   src={videoUrl}
                   controls
                   autoPlay
                   loop
+                  muted
+                  playsInline
                   className="w-full rounded-lg shadow-lg mb-2"
                   style={{ background: "#222" }}
                 />
+                {/* Hidden processing video for FastVLM/canvas */}
                 <video
-                  ref={overlayVideoRef}
+                  ref={processingVideoRef}
                   src={videoUrl}
-                  controls={false}
                   autoPlay
                   loop
                   muted
-                  className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 opacity-60 pointer-events-none"
-                  style={{ background: "#222", zIndex: 10 }}
+                  playsInline
+                  style={{ display: "none" }}
                 />
                 <canvas
                   ref={canvasRef}
@@ -489,24 +494,27 @@ export default function MultiSourceCaptioningView() {
               {/* Show uploaded video */}
               {uploadedFile && isVideoFile(uploadedFile) && (
                 <div className="relative w-full max-w-xl">
+                  {/* Visible overlay video for user */}
                   <video
-                    ref={videoRef}
+                    ref={overlayVideoRef}
                     src={uploadedUrl}
                     controls
                     autoPlay
                     loop
+                    muted
+                    playsInline
                     className="w-full rounded-lg shadow-lg mb-2"
                     style={{ background: "#222" }}
                   />
+                  {/* Hidden processing video for FastVLM/canvas */}
                   <video
-                    ref={overlayVideoRef}
+                    ref={processingVideoRef}
                     src={uploadedUrl}
-                    controls={false}
                     autoPlay
                     loop
                     muted
-                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 opacity-60 pointer-events-none"
-                    style={{ background: "#222", zIndex: 10 }}
+                    playsInline
+                    style={{ display: "none" }}
                   />
                   <canvas
                     ref={canvasRef}
@@ -524,24 +532,27 @@ export default function MultiSourceCaptioningView() {
               {/* Show example video if no file uploaded */}
               {!uploadedFile && (
                 <div className="relative w-full max-w-xl">
+                  {/* Visible overlay video for user */}
                   <video
-                    ref={videoRef}
+                    ref={overlayVideoRef}
                     src={EXAMPLE_VIDEO_URL}
                     controls
                     autoPlay
                     loop
+                    muted
+                    playsInline
                     className="w-full rounded-lg shadow-lg mb-2"
                     style={{ background: "#222" }}
                   />
+                  {/* Hidden processing video for FastVLM/canvas */}
                   <video
-                    ref={overlayVideoRef}
+                    ref={processingVideoRef}
                     src={EXAMPLE_VIDEO_URL}
-                    controls={false}
                     autoPlay
                     loop
                     muted
-                    className="w-full rounded-lg shadow-lg mb-2 absolute top-0 left-0 opacity-60 pointer-events-none"
-                    style={{ background: "#222", zIndex: 10 }}
+                    playsInline
+                    style={{ display: "none" }}
                   />
                   <canvas
                     ref={canvasRef}