import { useState, useEffect, useRef, useCallback } from "react"; import { Button } from "./ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "./ui/card"; import { Badge } from "./ui/badge"; import { Alert, AlertDescription } from "./ui/alert"; import { teleoperate, type TeleoperationProcess, type TeleoperationState, type TeleoperateConfig, } from "@lerobot/web"; import { getUnifiedRobotData } from "../lib/unified-storage"; import type { RobotConnection } from "@lerobot/web"; import { SO100_KEYBOARD_CONTROLS } from "@lerobot/web"; interface TeleoperationPanelProps { robot: RobotConnection; onClose: () => void; } export function TeleoperationPanel({ robot, onClose, }: TeleoperationPanelProps) { const [teleoperationState, setTeleoperationState] = useState({ isActive: false, motorConfigs: [], lastUpdate: 0, keyStates: {}, }); const [error, setError] = useState(null); const [, setIsInitialized] = useState(false); // Separate refs for keyboard and direct teleoperators const keyboardProcessRef = useRef(null); const directProcessRef = useRef(null); // Initialize both teleoperation processes useEffect(() => { const initializeTeleoperation = async () => { if (!robot || !robot.robotType) { setError("No robot configuration available"); return; } try { // Load calibration data from demo storage (app concern) let calibrationData; if (robot.serialNumber) { const data = getUnifiedRobotData(robot.serialNumber); calibrationData = data?.calibration; if (calibrationData) { console.log("โœ… Loaded calibration data for", robot.serialNumber); } } // Create keyboard teleoperation process const keyboardConfig: TeleoperateConfig = { robot: robot, teleop: { type: "keyboard", }, calibrationData, onStateUpdate: (state: TeleoperationState) => { setTeleoperationState(state); }, }; const keyboardProcess = await teleoperate(keyboardConfig); // Create direct teleoperation process const directConfig: TeleoperateConfig = { robot: robot, teleop: { type: "direct", }, calibrationData, }; const directProcess = await teleoperate(directConfig); keyboardProcessRef.current = keyboardProcess; directProcessRef.current = directProcess; setTeleoperationState(keyboardProcess.getState()); setIsInitialized(true); setError(null); console.log("โœ… Initialized both keyboard and direct teleoperators"); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to initialize teleoperation"; setError(errorMessage); console.error("โŒ Failed to initialize teleoperation:", error); } }; initializeTeleoperation(); return () => { // Cleanup on unmount const cleanup = async () => { try { if (keyboardProcessRef.current) { await keyboardProcessRef.current.disconnect(); keyboardProcessRef.current = null; } if (directProcessRef.current) { await directProcessRef.current.disconnect(); directProcessRef.current = null; } console.log("๐Ÿงน Teleoperation cleanup completed"); } catch (error) { console.warn("Error during teleoperation cleanup:", error); } }; cleanup(); }; }, [robot]); // Keyboard event handlers const handleKeyDown = useCallback( (event: KeyboardEvent) => { if (!teleoperationState.isActive || !keyboardProcessRef.current) return; const key = event.key; event.preventDefault(); keyboardProcessRef.current.updateKeyState(key, true); }, [teleoperationState.isActive] ); const handleKeyUp = useCallback( (event: KeyboardEvent) => { if (!teleoperationState.isActive || !keyboardProcessRef.current) return; const key = event.key; event.preventDefault(); keyboardProcessRef.current.updateKeyState(key, false); }, [teleoperationState.isActive] ); // Register keyboard events useEffect(() => { if (teleoperationState.isActive) { window.addEventListener("keydown", handleKeyDown); window.addEventListener("keyup", handleKeyUp); return () => { window.removeEventListener("keydown", handleKeyDown); window.removeEventListener("keyup", handleKeyUp); }; } }, [teleoperationState.isActive, handleKeyDown, handleKeyUp]); const handleStart = () => { if (!keyboardProcessRef.current || !directProcessRef.current) { setError("Teleoperation not initialized"); return; } try { keyboardProcessRef.current.start(); directProcessRef.current.start(); console.log("๐ŸŽฎ Both keyboard and direct teleoperation started"); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to start teleoperation"; setError(errorMessage); } }; const handleStop = async () => { try { if (keyboardProcessRef.current) { keyboardProcessRef.current.stop(); } if (directProcessRef.current) { directProcessRef.current.stop(); } console.log("๐Ÿ›‘ Both keyboard and direct teleoperation stopped"); } catch (error) { console.warn("Error during teleoperation stop:", error); } }; const handleClose = async () => { try { if (keyboardProcessRef.current) { keyboardProcessRef.current.stop(); await keyboardProcessRef.current.disconnect(); } if (directProcessRef.current) { directProcessRef.current.stop(); await directProcessRef.current.disconnect(); } console.log("๐Ÿ”Œ Properly disconnected from robot"); } catch (error) { console.warn("Error during teleoperation cleanup:", error); } onClose(); }; const simulateKeyPress = (key: string) => { if (!keyboardProcessRef.current) return; keyboardProcessRef.current.updateKeyState(key, true); }; const simulateKeyRelease = (key: string) => { if (!keyboardProcessRef.current) return; keyboardProcessRef.current.updateKeyState(key, false); }; // Unified motor control: Both sliders AND keyboard use the same teleoperator // This ensures the UI always shows the correct motor positions const moveMotorToPosition = async (motorIndex: number, position: number) => { if (!keyboardProcessRef.current) return; try { const motorName = teleoperationState.motorConfigs[motorIndex]?.name; if (motorName) { const keyboardTeleoperator = keyboardProcessRef.current .teleoperator as any; await keyboardTeleoperator.moveMotor(motorName, position); } } catch (error) { console.warn( `Failed to move motor ${motorIndex + 1} to position ${position}:`, error ); } }; const isConnected = robot?.isConnected || false; const isActive = teleoperationState.isActive; const motorConfigs = teleoperationState.motorConfigs; const keyStates = teleoperationState.keyStates; // Virtual keyboard component const VirtualKeyboard = () => { const isKeyPressed = (key: string) => { return keyStates?.[key]?.pressed || false; }; const KeyButton = ({ keyCode, children, className = "", size = "default" as "default" | "sm" | "lg" | "icon", }: { keyCode: string; children: React.ReactNode; className?: string; size?: "default" | "sm" | "lg" | "icon"; }) => { const control = SO100_KEYBOARD_CONTROLS[ keyCode as keyof typeof SO100_KEYBOARD_CONTROLS ]; const pressed = isKeyPressed(keyCode); const handleMouseDown = (e: React.MouseEvent) => { e.preventDefault(); if (!isActive) return; simulateKeyPress(keyCode); }; const handleMouseUp = (e: React.MouseEvent) => { e.preventDefault(); if (!isActive) return; simulateKeyRelease(keyCode); }; return ( ); }; return (
{/* Arrow Keys */}

Shoulder

โ†‘
โ† โ†“ โ†’
{/* WASD Keys */}

Elbow/Wrist

W
A S D
{/* Q/E and Gripper */}

Roll

Q E

Gripper

O C
{/* Emergency Stop */}
ESC
); }; return (
{/* Header */}

๐ŸŽฎ Robot Teleoperation

{robot.robotId || robot.name} - {robot.serialNumber}

{/* Error Alert */} {error && ( {error} )}
{/* Status Panel */} Status {isConnected ? "Connected" : "Disconnected"}
Teleoperation {isActive ? "Active" : "Stopped"}
Active Keys { Object.values(keyStates || {}).filter( (state) => state.pressed ).length }
{isActive ? ( ) : ( )}
{/* Virtual Keyboard */} Virtual Keyboard {/* Motor Status */} Motor Positions {motorConfigs.map((motor, index) => { return (
{motor.name.replace("_", " ")} {motor.currentPosition}
{ if (!isActive) return; const newPosition = parseInt(e.target.value); try { await moveMotorToPosition(index, newPosition); } catch (error) { console.warn( "Failed to move motor via slider:", error ); } }} />
{motor.minPosition} {motor.maxPosition}
); })}
{/* Help Card */} Control Instructions

Arrow Keys

  • โ†‘ โ†“ Shoulder lift
  • โ† โ†’ Shoulder pan

WASD Keys

  • W S Elbow flex
  • A D Wrist flex

Other Keys

  • Q E Wrist roll
  • O Open gripper
  • C Close gripper

Emergency

  • ESC Emergency stop

๐Ÿ’ก Pro tip: Use your physical keyboard for faster control, or click the virtual keys below. Hold keys down for continuous movement.

); }