feat(medsam): prompt-only segmentation (bboxes/points JSON); skip if none; polygons by default; optional raw masks

app.py

@@ -6,6 +6,7 @@ import torch
 import numpy as np
 import cv2
 import time
+import json
 import traceback
 
 # Simple timestamped logger
@@ -643,7 +644,7 @@ def analyze(image):
         return {"error": "Internal error in analyze"}
 
 
-def analyze_with_medsam(base_result, image, include_raw_masks=False):
+def analyze_with_medsam(base_result, image, include_raw_masks=False, bboxes_json="", points_json=""):
     try:
         log("analyze_with_medsam: start")
         if not isinstance(base_result, dict):
@@ -664,20 +665,36 @@ def analyze_with_medsam(base_result, image, include_raw_masks=False):
             img_path = tmp_path
         _medsam.load_image(img_path)
 
+        # Parse prompts
+        parsed_bboxes = []
+        parsed_points = []
+        try:
+            if bboxes_json:
+                parsed_bboxes = json.loads(bboxes_json)
+            if points_json:
+                parsed_points = json.loads(points_json)
+        except Exception:
+            log("analyze_with_medsam: failed to parse prompts JSON")
+
+        # If no prompts provided, skip (follow original behavior)
+        if not parsed_bboxes and not parsed_points:
+            log("analyze_with_medsam: no prompts provided; skipping segmentation")
+            return base_result, None
+
         segmentations = []
         masks_for_overlay = []
 
-        # MedSAM over candidate boxes (original behavior)
-        cand_bboxes = _find_topk_foreground_bboxes(pil_img, max_regions=20, min_area=100)
-        log(f"analyze_with_medsam: candidate boxes={len(cand_bboxes)}")
-        for bbox in cand_bboxes:
+        # Run MedSAM for provided boxes
+        for bbox in parsed_bboxes:
+            if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
+                continue
             m = _medsam.segment_with_box(bbox)
             if m is None or not isinstance(m.get('mask'), np.ndarray):
                 continue
             mask_np = m['mask'].astype(np.uint8)
             seg_entry = {
                 "confidence": float(m.get('confidence', 1.0)),
-                "method": m.get("method", "medsam_box_auto"),
+                "method": m.get("method", "medsam_box"),
                 "polygons": _mask_to_polygons(mask_np)
             }
             if include_raw_masks:
@@ -685,6 +702,33 @@ def analyze_with_medsam(base_result, image, include_raw_masks=False):
             segmentations.append(seg_entry)
             masks_for_overlay.append(m)
 
+        # Run MedSAM for provided points by converting to bbox
+        for item in parsed_points:
+            try:
+                # Expect item like {"points": [[x,y],...]} or [ [x,y], ... ]
+                pts = item.get("points") if isinstance(item, dict) else item
+                pts_np = np.array(pts)
+                x_min, y_min = pts_np.min(axis=0)
+                x_max, y_max = pts_np.max(axis=0)
+                pad = 20
+                H, W = _medsam.current_image.shape[:2]
+                bbox = [max(0, x_min - pad), max(0, y_min - pad), min(W - 1, x_max + pad), min(H - 1, y_max + pad)]
+                m = _medsam.segment_with_box(bbox)
+                if m is None or not isinstance(m.get('mask'), np.ndarray):
+                    continue
+                mask_np = m['mask'].astype(np.uint8)
+                seg_entry = {
+                    "confidence": float(m.get('confidence', 1.0)),
+                    "method": m.get("method", "medsam_points_box"),
+                    "polygons": _mask_to_polygons(mask_np)
+                }
+                if include_raw_masks:
+                    seg_entry["mask"] = mask_np.tolist()
+                segmentations.append(seg_entry)
+                masks_for_overlay.append(m)
+            except Exception:
+                continue
+
         W, H = pil_img.size
         base_result["medsam"] = {
             "available": True,
@@ -745,6 +789,8 @@ with gr.Blocks(
                 elem_id="image-input"
             )
             include_raw_masks_cb = gr.Checkbox(value=False, visible=False, elem_id="include-raw-masks")
+            bboxes_tb = gr.Textbox(value="", visible=False, elem_id="bboxes-json")
+            points_tb = gr.Textbox(value="", visible=False, elem_id="points-json")
 
             # Analyze button (single)
             analyze_btn = gr.Button(
@@ -776,10 +822,10 @@ with gr.Blocks(
         api_name="/predict"  # ✅ Standard API name that gradio_client expects
     )
 
-    # Automatic overlay generation step for medical images
+    # MedSAM step (prompt-only). If no prompts, it will skip
     analyze_event.then(
         fn=analyze_with_medsam,
-        inputs=[result_output, image_input, include_raw_masks_cb],
+        inputs=[result_output, image_input, include_raw_masks_cb, bboxes_tb, points_tb],
         outputs=[result_output, overlay_output],
     )