deploy sam3-zerogpu

README.md

@@ -38,6 +38,14 @@ Sapiens pose" section below).
   stats (mean / p95 / max magnitude px, dominant direction deg, direction
   consistency)
 
+**Bundled Video Depth Anything (Phase 4 — same cap reasoning):**
+
+- `api_video_depth` — video URL + anchor timestamps + optional per-frame
+  bboxes; returns RELATIVE depth in [0, 1] (never metric), per-bbox
+  p10/p50/p90 percentiles, and a near→far ordering of the bboxes per
+  frame. Set `is_relative: true` flag is always present in the response
+  so downstream code never promises metric distances.
+
 The backend client lives in
 `cadayn-backend/app/services/training/sam3p1_client.py`. Both halves are
 versioned independently; the contract below documents what they must agree
@@ -100,9 +108,10 @@ it is not a substitute for ordering.
 - `SAPIENS_HF_SPACE_URL` — `api_pose_image` + `api_pose_video_frames`
   (point at this same Space — the bundled endpoints share the container)
 - `NEUFLOW_HF_SPACE_URL` — `api_optical_flow` (same Space; Phase 2)
+- `VDA_HF_SPACE_URL` — `api_video_depth` (same Space; Phase 4)
 
-All four env vars can point at the same Space URL — the bundled Space
-registers all six Gradio routes.
+All five env vars can point at the same Space URL — the bundled Space
+registers all seven Gradio routes.
 
 ## Bundled Sapiens pose
 

app.py

@@ -119,12 +119,10 @@ def _get_video_predictor() -> tuple[str, Any]:
     print("[GPU worker] Falling back to HF Transformers Sam3VideoModel...")
     from transformers import Sam3VideoModel, Sam3VideoProcessor
 
-    _video_hf_processor = Sam3VideoProcessor.from_pretrained(
-        "facebook/sam3", token=HF_TOKEN
+    _video_hf_processor = Sam3VideoProcessor.from_pretrained("facebook/sam3", token=HF_TOKEN)
+    _video_hf_model = Sam3VideoModel.from_pretrained("facebook/sam3", token=HF_TOKEN, torch_dtype=torch.bfloat16).to(
+        "cuda"
     )
-    _video_hf_model = Sam3VideoModel.from_pretrained(
-        "facebook/sam3", token=HF_TOKEN, torch_dtype=torch.bfloat16
-    ).to("cuda")
     print("[GPU worker] HF Sam3VideoModel loaded")
     return ("hf_sam3", (_video_hf_model, _video_hf_processor))
 
@@ -147,9 +145,7 @@ def _get_image_model_and_processor() -> tuple[Any, Any]:
         _image_model = model
         _image_processor = Sam3Processor(_image_model)
         dtype_sample = next(model.parameters()).dtype
-        print(
-            f"[GPU worker] SAM 3 image model ready (first-param dtype={dtype_sample})"
-        )
+        print(f"[GPU worker] SAM 3 image model ready (first-param dtype={dtype_sample})")
     return _image_model, _image_processor
 
 
@@ -331,28 +327,16 @@ def _run_tracking_gpu(video_path: str, query: str) -> dict[str, Any]:
             video_storage_device="cpu",
             dtype=torch.bfloat16,
         )
-        inference_session = processor.add_text_prompt(
-            inference_session=inference_session, text=query
-        )
+        inference_session = processor.add_text_prompt(inference_session=inference_session, text=query)
         outputs_per_frame: dict[int, dict[str, Any]] = {}
-        for model_outputs in model.propagate_in_video_iterator(
-            inference_session=inference_session
-        ):
+        for model_outputs in model.propagate_in_video_iterator(inference_session=inference_session):
             processed = processor.postprocess_outputs(inference_session, model_outputs)
             outputs_per_frame[model_outputs.frame_idx] = processed
 
-        visible_indices = [
-            fi
-            for fi, payload in outputs_per_frame.items()
-            if (payload.get("object_ids") or [])
-        ]
+        visible_indices = [fi for fi, payload in outputs_per_frame.items() if (payload.get("object_ids") or [])]
         visible_indices.sort()
 
-        fps_est = (
-            len(frames) / _video_duration_s(video_path)
-            if _video_duration_s(video_path) > 0
-            else SAMPLE_FPS
-        )
+        fps_est = len(frames) / _video_duration_s(video_path) if _video_duration_s(video_path) > 0 else SAMPLE_FPS
         tracked_segments: list[dict[str, float]] = []
         if visible_indices:
             current_start = visible_indices[0]
@@ -445,9 +429,7 @@ def _download_image_bytes(url: str, *, timeout_s: float = 30.0) -> bytes:
         for chunk in response.iter_bytes(chunk_size=256 * 1024):
             buf.write(chunk)
             if buf.tell() > MAX_IMAGE_BYTES:
-                raise ValueError(
-                    f"Image exceeds {MAX_IMAGE_BYTES // (1024 * 1024)} MB limit"
-                )
+                raise ValueError(f"Image exceeds {MAX_IMAGE_BYTES // (1024 * 1024)} MB limit")
     return buf.getvalue()
 
 
@@ -525,9 +507,7 @@ def _masks_to_rle(output: Any) -> list[dict[str, Any] | None]:
                 masks = candidate
                 break
         if masks is None:
-            print(
-                f"[GPU worker] mask extraction: no mask-like key in output (got {sorted(output.keys())})"
-            )
+            print(f"[GPU worker] mask extraction: no mask-like key in output (got {sorted(output.keys())})")
     if masks is None:
         return []
 
@@ -585,9 +565,7 @@ def _binary_mask_to_rle(mask: np.ndarray) -> dict[str, Any]:
         prev = 1 - prev
     return {
         "size": [int(h), int(w)],
-        "counts": base64.b64encode(
-            ",".join(str(r) for r in runs).encode("ascii")
-        ).decode("ascii"),
+        "counts": base64.b64encode(",".join(str(r) for r in runs).encode("ascii")).decode("ascii"),
         "format": "uncompressed_rle_b64",
     }
 
@@ -769,9 +747,7 @@ def _bbox_to_rle(
 
     return {
         "size": [int(img_h), int(img_w)],
-        "counts": base64.b64encode(
-            ",".join(str(r) for r in runs).encode("ascii")
-        ).decode("ascii"),
+        "counts": base64.b64encode(",".join(str(r) for r in runs).encode("ascii")).decode("ascii"),
         "format": "bbox_rle_b64",
     }
 
@@ -946,9 +922,7 @@ SAPIENS_MAX_PERSONS_PER_FRAME = 12
 SAPIENS_INPUT_HEIGHT = 1024
 SAPIENS_INPUT_WIDTH = 768
 
-SAPIENS_KEYPOINT_NAMES_PATH = os.path.join(
-    os.path.dirname(__file__), "goliath_keypoints.txt"
-)
+SAPIENS_KEYPOINT_NAMES_PATH = os.path.join(os.path.dirname(__file__), "goliath_keypoints.txt")
 
 # Worker-local model cache — separate from the SAM caches above so the
 # two model families don't compete for the same global slot.
@@ -963,9 +937,7 @@ def _sapiens_load_keypoint_names() -> list[str]:
         return _sapiens_worker_keypoint_names
     if os.path.exists(SAPIENS_KEYPOINT_NAMES_PATH):
         with open(SAPIENS_KEYPOINT_NAMES_PATH) as f:
-            names = [
-                line.strip() for line in f if line.strip() and not line.startswith("#")
-            ]
+            names = [line.strip() for line in f if line.strip() and not line.startswith("#")]
             if names:
                 _sapiens_worker_keypoint_names = names
                 return _sapiens_worker_keypoint_names
@@ -1073,19 +1045,12 @@ def _sapiens_preprocess(image: np.ndarray) -> tuple[Any, tuple[int, int]]:
     from PIL import Image
 
     orig_h, orig_w = image.shape[:2]
-    pil = Image.fromarray(image).resize(
-        (SAPIENS_INPUT_WIDTH, SAPIENS_INPUT_HEIGHT), Image.BILINEAR
-    )
+    pil = Image.fromarray(image).resize((SAPIENS_INPUT_WIDTH, SAPIENS_INPUT_HEIGHT), Image.BILINEAR)
     arr = np.array(pil).astype(np.float32) / 255.0
     mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
     std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
     arr = (arr - mean) / std
-    tensor = (
-        torch.from_numpy(arr)
-        .permute(2, 0, 1)
-        .unsqueeze(0)
-        .to("cuda", dtype=torch.float32)
-    )
+    tensor = torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0).to("cuda", dtype=torch.float32)
     return tensor, (orig_h, orig_w)
 
 
@@ -1215,9 +1180,7 @@ def _sapiens_extract_video_frames(
                 subprocess.run(cmd, check=True, capture_output=True, timeout=30)
                 frame_bgr = cv2.imread(out_path)
                 if frame_bgr is not None:
-                    extracted_ff.append(
-                        (ts, cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB))
-                    )
+                    extracted_ff.append((ts, cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)))
             except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
                 print(f"[sapiens] ffmpeg seek failed for ts={ts}: {exc}")
                 continue
@@ -1264,10 +1227,7 @@ def _sapiens_track_persons_across_frames(
         # ByteTrack drops rejected detections from its output, so output
         # length ≤ input length, and order isn't preserved. Per-input-row
         # bbox-equality matching is the only safe way to assign IDs.
-        if (
-            getattr(tracked, "tracker_id", None) is not None
-            and getattr(tracked, "xyxy", None) is not None
-        ):
+        if getattr(tracked, "tracker_id", None) is not None and getattr(tracked, "xyxy", None) is not None:
             for out_idx in range(len(tracked.tracker_id)):
                 tid = tracked.tracker_id[out_idx]
                 if tid is None:
@@ -1276,10 +1236,7 @@ def _sapiens_track_persons_across_frames(
                 for p in frame_persons:
                     if "id" in p:
                         continue  # already matched to a tracker_id
-                    if (
-                        abs(p["bbox"]["x1"] - float(out_x1)) < 1.0
-                        and abs(p["bbox"]["y1"] - float(out_y1)) < 1.0
-                    ):
+                    if abs(p["bbox"]["x1"] - float(out_x1)) < 1.0 and abs(p["bbox"]["y1"] - float(out_y1)) < 1.0:
                         p["id"] = f"p_{int(tid)}"
                         break
 
@@ -1312,9 +1269,7 @@ def api_pose_image(
         image_array = _sapiens_decode_image_np(raw)
         h, w = image_array.shape[:2]
 
-        person_bboxes = _sapiens_detect_persons(
-            image_array, detector_conf=detector_conf
-        )
+        person_bboxes = _sapiens_detect_persons(image_array, detector_conf=detector_conf)
         if not person_bboxes:
             return {
                 "ok": True,
@@ -1328,9 +1283,7 @@ def api_pose_image(
             }
 
         pose_model = _sapiens_load_pose_model()
-        persons = _sapiens_infer_pose_for_persons(
-            pose_model, image_array, person_bboxes, confidence_threshold
-        )
+        persons = _sapiens_infer_pose_for_persons(pose_model, image_array, person_bboxes, confidence_threshold)
         for j, p in enumerate(persons):
             p["id"] = f"p_{j}"
 
@@ -1410,14 +1363,10 @@ def api_pose_video_frames(
 
         for ts, image_array in frames:
             h, w = image_array.shape[:2]
-            person_bboxes = _sapiens_detect_persons(
-                image_array, detector_conf=detector_conf
-            )
+            person_bboxes = _sapiens_detect_persons(image_array, detector_conf=detector_conf)
             if len(person_bboxes) >= SAPIENS_MAX_PERSONS_PER_FRAME:
                 any_capped = True
-            persons = _sapiens_infer_pose_for_persons(
-                pose_model, image_array, person_bboxes, confidence_threshold
-            )
+            persons = _sapiens_infer_pose_for_persons(pose_model, image_array, person_bboxes, confidence_threshold)
             per_frame_persons.append(persons)
             per_frame_meta.append({"timestamp_s": ts, "width": w, "height": h})
 
@@ -1536,9 +1485,7 @@ def _neuflow_preprocess(image: np.ndarray, max_dim: int) -> tuple[Any, tuple[int
 
     # cv2 default is BGR — match the reference infer_hf.py exactly (no
     # RGB conversion, no /255 scaling, no mean/std).
-    tensor = (
-        torch.from_numpy(image_small).permute(2, 0, 1).unsqueeze(0).half().to("cuda")
-    )
+    tensor = torch.from_numpy(image_small).permute(2, 0, 1).unsqueeze(0).half().to("cuda")
     return tensor, (new_h, new_w)
 
 
@@ -1583,9 +1530,7 @@ def _neuflow_aggregate_flow(
     # Dominant direction: angle of the mean flow vector
     mean_fx = float(np.mean(fx_orig))
     mean_fy = float(np.mean(fy_orig))
-    dominant_dir_deg = float(
-        (np.degrees(np.arctan2(-mean_fy, mean_fx)) + 360.0) % 360.0
-    )
+    dominant_dir_deg = float((np.degrees(np.arctan2(-mean_fy, mean_fx)) + 360.0) % 360.0)
 
     # Direction consistency: |mean(unit_vectors)|
     eps = 1e-6
@@ -1723,6 +1668,227 @@ def api_optical_flow(
 # ═════════════════════════════════════════════════════════════════════════════
 
 
+# ═════════════════════════════════════════════════════════════════════════════
+# ███████████████████████████████████████████████████████████████████████████
+#  BUNDLED — VIDEO DEPTH ANYTHING SMALL (Phase 4 spatial-tools)
+#
+#  VDA Small (CVPR 2025 Highlight) — temporally-consistent monocular
+#  depth on long video. Bundled into this Space alongside SAM, Sapiens,
+#  and NeuFlow because of the 10 ZeroGPU per-user cap. Returns RELATIVE
+#  depth in [0, 1]; never claim metric distances downstream.
+#
+#  Endpoint added:
+#    * api_video_depth(video_url, frame_timestamps_s, bboxes_per_frame,
+#                      downsample_to) — for each anchor timestamp,
+#      returns depth_range + foreground_threshold + (optional) per-bbox
+#      depth p10/p50/p90 + relative_ordering of bboxes near→far.
+# ███████████████████████████████████████████████████████████████████████████
+# ═════════════════════════════════════════════════════════════════════════════
+
+VDA_REPO = "depth-anything/Video-Depth-Anything-Small"
+VDA_GPU_DURATION_S = 300
+VDA_MAX_TIMESTAMPS = 32
+VDA_DEFAULT_DOWNSAMPLE = 518  # VDA's native 518×518 inference resolution
+VDA_FOREGROUND_PERCENTILE = 40  # depth at this percentile = "foreground threshold"
+
+_vda_worker_model: Any = None
+
+
+def _vda_load_model() -> Any:
+    """Lazy-load Video Depth Anything Small from the HF hub.
+
+    The depth-anything package isn't on PyPI as a single install but the
+    HF hub copy uses transformers AutoModelForDepthEstimation under the
+    hood. We try the transformers path first (zero extra install on top
+    of the existing requirements) and fall back to the depth_anything
+    package if it's vendored in.
+    """
+    global _vda_worker_model
+    if _vda_worker_model is not None:
+        return _vda_worker_model
+
+    print(f"[vda] Loading {VDA_REPO} into GPU memory...")
+    try:
+        from transformers import AutoImageProcessor, AutoModelForDepthEstimation
+
+        processor = AutoImageProcessor.from_pretrained(VDA_REPO, token=HF_TOKEN)
+        model = AutoModelForDepthEstimation.from_pretrained(VDA_REPO, token=HF_TOKEN).to("cuda")
+        model.eval()
+        _vda_worker_model = (model, processor)
+        print("[vda] Model ready on CUDA (transformers path)")
+        return _vda_worker_model
+    except Exception as exc:
+        print(f"[vda] transformers path failed ({exc}); cannot fall back without vendored package")
+        raise
+
+
+def _vda_infer_frame(model_pair: Any, image: np.ndarray, downsample_to: int) -> np.ndarray:
+    """Run VDA on a single frame; return depth map in original-image dims."""
+    model, processor = model_pair
+
+    h, w = image.shape[:2]
+    scale = downsample_to / max(h, w)
+    if scale < 1.0:
+        new_h, new_w = int(h * scale), int(w * scale)
+        image_small = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA)
+    else:
+        image_small = image
+
+    inputs = processor(images=image_small, return_tensors="pt").to("cuda")
+    with torch.no_grad():
+        outputs = model(**inputs)
+    depth = outputs.predicted_depth  # (B, H', W')
+    depth_cpu = depth.detach().float().cpu().numpy()
+    if depth_cpu.ndim == 3:
+        depth_cpu = depth_cpu[0]
+
+    # Resize back to original resolution
+    depth_full = cv2.resize(depth_cpu, (w, h), interpolation=cv2.INTER_LINEAR)
+    # Normalise to [0, 1] within this frame (RELATIVE depth)
+    d_min = float(np.min(depth_full))
+    d_max = float(np.max(depth_full))
+    if d_max - d_min > 1e-6:
+        depth_full = (depth_full - d_min) / (d_max - d_min)
+    return depth_full
+
+
+def _vda_aggregate_frame(
+    depth_map: np.ndarray,
+    timestamp_s: float,
+    bboxes_for_frame: list[dict[str, Any]],
+) -> dict[str, Any]:
+    """Reduce a depth map + per-frame bboxes to the schema the backend expects."""
+    h, w = depth_map.shape
+
+    objects: list[dict[str, Any]] = []
+    for entry in bboxes_for_frame:
+        bbox = entry.get("bbox") or {}
+        if not isinstance(bbox, dict):
+            continue
+        x1 = max(0, min(w, int(bbox.get("x1", 0))))
+        y1 = max(0, min(h, int(bbox.get("y1", 0))))
+        x2 = max(0, min(w, int(bbox.get("x2", 0))))
+        y2 = max(0, min(h, int(bbox.get("y2", 0))))
+        if x2 <= x1 or y2 <= y1:
+            continue
+        crop = depth_map[y1:y2, x1:x2]
+        if crop.size == 0:
+            continue
+        p10 = float(np.percentile(crop, 10))
+        p50 = float(np.percentile(crop, 50))
+        p90 = float(np.percentile(crop, 90))
+        objects.append(
+            {
+                "bbox_id": str(entry.get("bbox_id", f"bbox_{len(objects)}")),
+                "p10": p10,
+                "p50": p50,
+                "p90": p90,
+                "spans_foreground_and_background": (p90 - p10) > 0.4,
+            }
+        )
+
+    # Order objects near→far by p50
+    relative_ordering = [o["bbox_id"] for o in sorted(objects, key=lambda o: o["p50"])]
+
+    return {
+        "timestamp_s": timestamp_s,
+        "depth_range": {"min": float(np.min(depth_map)), "max": float(np.max(depth_map))},
+        "foreground_threshold": float(np.percentile(depth_map, VDA_FOREGROUND_PERCENTILE)),
+        "objects": objects,
+        "relative_ordering": relative_ordering,
+    }
+
+
+@spaces.GPU(duration=VDA_GPU_DURATION_S)
+def api_video_depth(
+    video_url: str,
+    frame_timestamps_s: list[float],
+    bboxes_per_frame: list[dict[str, Any]] | None = None,
+    downsample_to: int = VDA_DEFAULT_DOWNSAMPLE,
+) -> dict[str, Any]:
+    """Estimate per-frame relative depth + optional per-bbox depth percentiles.
+
+    Bundled into sam3-zerogpu — see banner above.
+    """
+    started = time.monotonic()
+    if not video_url:
+        return {"ok": False, "error": "video_url is required", "elapsed_s": 0.0}
+    if not isinstance(frame_timestamps_s, list) or not frame_timestamps_s:
+        return {
+            "ok": False,
+            "error": "frame_timestamps_s must be a non-empty list",
+            "elapsed_s": 0.0,
+        }
+    if len(frame_timestamps_s) > VDA_MAX_TIMESTAMPS:
+        return {
+            "ok": False,
+            "error": f"timestamp batch exceeds {VDA_MAX_TIMESTAMPS}",
+            "elapsed_s": 0.0,
+        }
+
+    try:
+        video_path = _download_video(video_url, timeout=60.0)
+        try:
+            frames = _sapiens_extract_video_frames(video_path, frame_timestamps_s)
+        finally:
+            try:
+                os.unlink(video_path)
+            except OSError:
+                pass
+
+        if not frames:
+            return {
+                "ok": False,
+                "frames": [],
+                "error": "frame extraction returned no frames",
+                "elapsed_s": round(time.monotonic() - started, 3),
+            }
+
+        # Group bboxes by their nominal timestamp_s; use a small tolerance
+        # match because float timestamps may not exact-equal what came back
+        # from ffmpeg.
+        bboxes_by_ts: dict[float, list[dict[str, Any]]] = {}
+        for entry in bboxes_per_frame or []:
+            if not isinstance(entry, dict):
+                continue
+            ts = float(entry.get("timestamp_s", 0.0))
+            bboxes_by_ts.setdefault(round(ts, 3), []).append(entry)
+
+        model_pair = _vda_load_model()
+        per_frame_out: list[dict[str, Any]] = []
+        for ts, image in frames:
+            depth_map = _vda_infer_frame(model_pair, image, downsample_to)
+            bboxes_for_frame = bboxes_by_ts.get(round(ts, 3), [])
+            per_frame_out.append(_vda_aggregate_frame(depth_map, ts, bboxes_for_frame))
+
+        return {
+            "ok": True,
+            "frames": per_frame_out,
+            "model": VDA_REPO,
+            "is_relative": True,
+            "camera_motion_hint": None,  # TODO: derive from depth-flow between frames
+            "error": None,
+            "elapsed_s": round(time.monotonic() - started, 3),
+        }
+    except Exception as exc:
+        traceback.print_exc()
+        return {
+            "ok": False,
+            "frames": [],
+            "error": f"{type(exc).__name__}: {exc}",
+            "elapsed_s": round(time.monotonic() - started, 3),
+        }
+    finally:
+        gc.collect()
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# ███████████████████████████████████████████████████████████████████████████
+#  END OF BUNDLED VDA BLOCK
+# ███████████████████████████████████████████████████████████████████████████
+# ═════════════════════════════════════════════════════════════════════════════
+
+
 # ═════════════════════════════════════════════════════════════════════════════
 #  Gradio UI
 # ═════════════════════════════════════════════════════════════════════════════
@@ -1853,6 +2019,26 @@ with gr.Blocks(title="SAM 3 / SAM 3.1 for Cadayn") as demo:
         api_name="api_optical_flow",
     )
 
+    # ── BUNDLED VDA depth endpoint (see banner mid-file) ──────────────────
+    with gr.Row(visible=False):
+        api_depth_video_url = gr.Textbox()
+        api_depth_timestamps = gr.JSON(value=[])
+        api_depth_bboxes = gr.JSON(value=[])
+        api_depth_downsample = gr.Number(value=VDA_DEFAULT_DOWNSAMPLE)
+        api_depth_output = gr.JSON()
+
+    api_depth_video_url.change(
+        fn=api_video_depth,
+        inputs=[
+            api_depth_video_url,
+            api_depth_timestamps,
+            api_depth_bboxes,
+            api_depth_downsample,
+        ],
+        outputs=api_depth_output,
+        api_name="api_video_depth",
+    )
+
 
 if __name__ == "__main__":
     demo.launch(server_name="0.0.0.0", server_port=7860)