Spaces:
Running
Running
File size: 4,487 Bytes
1a9c884 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
import { useState, useRef, useEffect, useCallback } from "react";
import WebcamCapture from "./WebcamCapture";
import DraggableContainer from "./DraggableContainer";
import PromptInput from "./PromptInput";
import LiveCaption from "./LiveCaption";
import { useVLMContext } from "../context/useVLMContext";
import { PROMPTS, TIMING } from "../constants";
interface CaptioningViewProps {
videoRef: React.RefObject<HTMLVideoElement | null>;
}
function useCaptioningLoop(
videoRef: React.RefObject<HTMLVideoElement | null>,
isRunning: boolean,
promptRef: React.RefObject<string>,
onCaptionUpdate: (caption: string) => void,
onError: (error: string) => void,
) {
const { isLoaded, runInference } = useVLMContext();
const abortControllerRef = useRef<AbortController | null>(null);
const onCaptionUpdateRef = useRef(onCaptionUpdate);
const onErrorRef = useRef(onError);
useEffect(() => {
onCaptionUpdateRef.current = onCaptionUpdate;
}, [onCaptionUpdate]);
useEffect(() => {
onErrorRef.current = onError;
}, [onError]);
useEffect(() => {
abortControllerRef.current?.abort();
if (!isRunning || !isLoaded) return;
abortControllerRef.current = new AbortController();
const signal = abortControllerRef.current.signal;
const video = videoRef.current;
const captureLoop = async () => {
while (!signal.aborted) {
if (video && video.readyState >= 2 && !video.paused && video.videoWidth > 0) {
try {
const currentPrompt = promptRef.current || "";
const result = await runInference(video, currentPrompt, onCaptionUpdateRef.current);
if (result && !signal.aborted) onCaptionUpdateRef.current(result);
} catch (error) {
if (!signal.aborted) {
const message = error instanceof Error ? error.message : String(error);
onErrorRef.current(message);
console.error("Error processing frame:", error);
}
}
}
if (signal.aborted) break;
await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY));
}
};
// NB: Wrap with a setTimeout to ensure abort controller can run before starting the loop
// This is necessary for React's strict mode which calls effects twice in development.
setTimeout(captureLoop, 0);
return () => {
abortControllerRef.current?.abort();
};
}, [isRunning, isLoaded, runInference, promptRef, videoRef]);
}
export default function CaptioningView({ videoRef }: CaptioningViewProps) {
const [caption, setCaption] = useState<string>("");
const [isLoopRunning, setIsLoopRunning] = useState<boolean>(true);
const [currentPrompt, setCurrentPrompt] = useState<string>(PROMPTS.default);
const [error, setError] = useState<string | null>(null);
// Use ref to store current prompt to avoid loop restarts
const promptRef = useRef<string>(currentPrompt);
// Update prompt ref when state changes
useEffect(() => {
promptRef.current = currentPrompt;
}, [currentPrompt]);
const handleCaptionUpdate = useCallback((newCaption: string) => {
setCaption(newCaption);
setError(null);
}, []);
const handleError = useCallback((errorMessage: string) => {
setError(errorMessage);
setCaption(`Error: ${errorMessage}`);
}, []);
useCaptioningLoop(videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError);
const handlePromptChange = useCallback((prompt: string) => {
setCurrentPrompt(prompt);
setError(null);
}, []);
const handleToggleLoop = useCallback(() => {
setIsLoopRunning((prev) => !prev);
if (error) setError(null);
}, [error]);
return (
<div className="absolute inset-0 text-white">
<div className="relative w-full h-full">
<WebcamCapture isRunning={isLoopRunning} onToggleRunning={handleToggleLoop} error={error} />
{/* Draggable Prompt Input - Bottom Left */}
<DraggableContainer initialPosition="bottom-left">
<PromptInput onPromptChange={handlePromptChange} />
</DraggableContainer>
{/* Draggable Live Caption - Bottom Right */}
<DraggableContainer initialPosition="bottom-right">
<LiveCaption caption={caption} isRunning={isLoopRunning} error={error} />
</DraggableContainer>
</div>
</div>
);
}
|