import gradio as gr import numpy as np import cv2 import torch import pathlib import sys import json from PIL import Image from PIL.ExifTags import TAGS import matplotlib.pyplot as plt import matplotlib.patches as patches from typing import Dict, List, Tuple, Optional import warnings warnings.filterwarnings('ignore') # Add the agent module to path ROOT = pathlib.Path(__file__).resolve().parent sys.path.insert(0, str(ROOT / "goal2" / "src")) from agent import models, geometry, io # Device configuration DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Camera presets for common devices CAMERA_PRESETS = { "iPhone 12/13/14 (Main Camera)": {"fx": 1840, "fy": 1840, "description": "26mm equivalent, f/1.6"}, "iPhone 12/13/14 (Ultra Wide)": {"fx": 920, "fy": 920, "description": "13mm equivalent, f/2.4"}, "Samsung Galaxy S21/S22": {"fx": 1950, "fy": 1950, "description": "26mm equivalent"}, "Google Pixel 6/7": {"fx": 1800, "fy": 1800, "description": "27mm equivalent"}, "Generic Smartphone": {"fx": 1500, "fy": 1500, "description": "Typical smartphone camera"}, "Custom": {"fx": 1500, "fy": 1500, "description": "Enter your own focal length values"} } class SizeEstimatorApp: def __init__(self): self.depth_net = None self.mask_gen = None self.current_image = None self.current_depth = None self.current_masks = None self.reference_object = None def detect_camera_from_exif(self, image_pil: Image.Image) -> Tuple[str, Dict]: """Try to detect camera type from EXIF data""" try: exif = image_pil._getexif() if not exif: return "Unknown", {} # Extract relevant EXIF data exif_data = {} for tag_id, value in exif.items(): tag = TAGS.get(tag_id, tag_id) exif_data[tag] = value # Try to identify camera make/model make = exif_data.get('Make', '').lower() model = exif_data.get('Model', '').lower() # Match against known camera presets if 'apple' in make or 'iphone' in model: if any(x in model for x in ['12', '13', '14']): return "iPhone 12/13/14 (Main Camera)", exif_data else: return "Generic Smartphone", exif_data elif 'samsung' in make: return "Samsung Galaxy S21/S22", exif_data elif 'google' in make or 'pixel' in model: return "Google Pixel 6/7", exif_data else: return "Generic Smartphone", exif_data except Exception as e: print(f"EXIF detection failed: {e}") return "Unknown", {} def load_models(self): """Load the depth and segmentation models""" if self.depth_net is None: print("Loading Depth Anything V2...") self.depth_net = models.load_depth(DEVICE) if self.mask_gen is None: print("Loading SAM...") self.mask_gen = models.load_sam(DEVICE) return "✅ Models loaded successfully!" def process_image(self, image: np.ndarray, camera_preset: str, fx_custom: float, fy_custom: float) -> Tuple[np.ndarray, str]: """Process uploaded image and generate depth + segmentation""" try: # Input validation if image is None: return None, "❌ No image provided. Please upload an image." if len(image.shape) != 3 or image.shape[2] != 3: return None, "❌ Invalid image format. Please upload a color image (RGB)." # Check image size constraints h, w = image.shape[:2] if h < 100 or w < 100: return None, "❌ Image too small. Please upload an image at least 100x100 pixels." if h > 4000 or w > 4000: status_msg = "⚠️ Large image detected. Resizing for processing...\n" # Resize very large images max_size = 2000 scale = min(max_size/w, max_size/h) if scale < 1: new_w, new_h = int(w * scale), int(h * scale) image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA) status_msg += f"📏 Resized from {w}×{h} to {new_w}×{new_h}\n" else: status_msg = "" # Ensure models are loaded if self.depth_net is None or self.mask_gen is None: self.load_models() # Store the original image self.current_image = image.copy() # Validate camera parameters if camera_preset == "Custom": if fx_custom <= 0 or fy_custom <= 0: return None, "❌ Invalid focal length values. Must be greater than 0." if fx_custom < 100 or fy_custom < 100 or fx_custom > 5000 or fy_custom > 5000: return None, "❌ Focal length values seem unrealistic. Typical range: 100-5000 pixels." fx, fy = fx_custom, fy_custom else: preset = CAMERA_PRESETS[camera_preset] fx, fy = preset["fx"], preset["fy"] # Generate depth and masks using the robust approach depth, masks, processed_img = models.predict_depth_and_masks( self.depth_net, self.mask_gen, image, DEVICE, approach="aligned" ) # Validate results if depth is None or len(depth.shape) != 2: return None, "❌ Failed to generate depth map. Please try a different image." if not masks or len(masks) == 0: return None, "❌ No objects detected in the image. Try an image with clearer objects." # Filter out very small masks (likely noise) min_area = (image.shape[0] * image.shape[1]) * 0.001 # 0.1% of image area filtered_masks = [m for m in masks if m['area'] > min_area] if len(filtered_masks) == 0: return None, "❌ No significant objects detected. Try an image with larger, clearer objects." self.current_depth = depth self.current_masks = filtered_masks # Create visualization vis_image = self.create_mask_visualization(processed_img, filtered_masks) status = status_msg + f"✅ Processed successfully! Found {len(filtered_masks)} objects.\n" status += f"📷 Camera: {camera_preset} (fx={fx:.0f}, fy={fy:.0f})\n" status += f"🖼️ Image size: {image.shape[1]}×{image.shape[0]}\n" if len(masks) > len(filtered_masks): status += f"🔍 Filtered out {len(masks) - len(filtered_masks)} small objects\n" status += f"📏 Ready for size estimation - select object number and known size below" return vis_image, status except Exception as e: import traceback error_details = traceback.format_exc() print("Full error:", error_details) # For debugging return None, f"❌ Error processing image: {str(e)}\nPlease try a different image." def create_mask_visualization(self, image: np.ndarray, masks: List[Dict]) -> np.ndarray: """Create visualization with colored masks and labels""" vis_img = image.copy() # Sort masks by area (largest first) sorted_masks = sorted(masks, key=lambda x: x['area'], reverse=True) # Color each mask with different colors colors = plt.cm.Set3(np.linspace(0, 1, len(sorted_masks))) for i, mask_data in enumerate(sorted_masks): mask = mask_data['segmentation'] color = colors[i][:3] # RGB values # Apply colored overlay colored_mask = np.zeros_like(vis_img) colored_mask[mask] = [int(c * 255) for c in color] vis_img = cv2.addWeighted(vis_img, 0.7, colored_mask, 0.3, 0) # Add number label y, x = np.where(mask) if len(x) > 0 and len(y) > 0: center_x, center_y = int(np.mean(x)), int(np.mean(y)) cv2.putText(vis_img, str(i+1), (center_x-10, center_y+5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) cv2.putText(vis_img, str(i+1), (center_x-10, center_y+5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 1) return vis_img def select_reference_object(self, mask_number: int, reference_size_cm: float, dimension: str) -> str: """Select a mask as reference object and specify its known size""" try: if self.current_masks is None: return "❌ No image processed yet. Please upload and process an image first." if mask_number < 1 or mask_number > len(self.current_masks): return f"❌ Invalid mask number. Choose between 1 and {len(self.current_masks)}" if reference_size_cm <= 0: return "❌ Reference size must be greater than 0" # Get the selected mask (convert to 0-based index) sorted_masks = sorted(self.current_masks, key=lambda x: x['area'], reverse=True) selected_mask = sorted_masks[mask_number - 1] # Store reference object info self.reference_object = { 'mask_data': selected_mask, 'known_size_cm': reference_size_cm, 'dimension': dimension # 'width' or 'height' } return f"✅ Reference object #{mask_number} selected!\n📏 Known {dimension}: {reference_size_cm} cm" except Exception as e: return f"❌ Error selecting reference: {str(e)}" def calculate_all_sizes(self, camera_preset: str, fx_custom: float, fy_custom: float) -> str: """Calculate sizes of all objects using the reference object for scale""" try: if self.current_masks is None: return "❌ No image processed yet." if self.reference_object is None: return "❌ No reference object selected. Please select a reference object first." # Get camera parameters if camera_preset == "Custom": fx, fy = fx_custom, fy_custom else: preset = CAMERA_PRESETS[camera_preset] fx, fy = preset["fx"], preset["fy"] # Calculate reference object's pixel dimensions first ref_mask = self.reference_object['mask_data']['segmentation'] ref_stats = geometry.pixel_to_metric(ref_mask, self.current_depth, fx, fy) # Get the reference object's measured dimension in pixels if self.reference_object['dimension'] == 'width': ref_pixel_size = ref_stats['width_m'] * 100 # Convert to cm else: # height ref_pixel_size = ref_stats['height_m'] * 100 # Convert to cm # Calculate scale factor: known_size / measured_size scale_factor = self.reference_object['known_size_cm'] / ref_pixel_size # Calculate sizes for all objects results = [] sorted_masks = sorted(self.current_masks, key=lambda x: x['area'], reverse=True) for i, mask_data in enumerate(sorted_masks): mask = mask_data['segmentation'] stats = geometry.pixel_to_metric(mask, self.current_depth, fx, fy) # Apply scale correction corrected_width = stats['width_m'] * 100 * scale_factor # cm corrected_height = stats['height_m'] * 100 * scale_factor # cm corrected_distance = stats['distance_m'] * scale_factor # meters # Check if this is the reference object by comparing mask data is_reference = np.array_equal(mask_data['segmentation'], self.reference_object['mask_data']['segmentation']) ref_marker = " (REFERENCE)" if is_reference else "" results.append(f"Object #{i+1}{ref_marker}:") results.append(f" 📏 Width: {corrected_width:.1f} cm") results.append(f" 📏 Height: {corrected_height:.1f} cm") results.append(f" 📍 Distance: {corrected_distance:.2f} m") results.append(f" 📐 Area: {mask_data['area']} pixels") results.append("") # Find reference object number for display ref_object_num = None for i, mask_data in enumerate(sorted_masks): if np.array_equal(mask_data['segmentation'], self.reference_object['mask_data']['segmentation']): ref_object_num = i + 1 break # Add calibration info results.append("=" * 40) results.append("📊 Calibration Info:") results.append(f"📷 Camera: {camera_preset}") results.append(f"🔍 Scale factor: {scale_factor:.3f}") results.append(f"📏 Reference: Object #{ref_object_num if ref_object_num else 'Unknown'}") results.append(f"📐 Known {self.reference_object['dimension']}: {self.reference_object['known_size_cm']} cm") return "\n".join(results) except Exception as e: return f"❌ Error calculating sizes: {str(e)}" # Initialize the app app = SizeEstimatorApp() # Gradio interface def create_interface(): with gr.Blocks(title="📏 Smart Object Size Estimator", theme=gr.themes.Soft()) as demo: gr.Markdown(""" # 📏 Smart Object Size Estimator Upload an image and get real-world size measurements of objects using AI-powered depth estimation and segmentation. ## How to use: 1. **Upload an image** and select your camera type 2. **Click Process** to detect objects 3. **Select a reference object** by clicking its number and entering its known size 4. **Calculate sizes** to get measurements of all objects """) with gr.Row(): with gr.Column(scale=1): # Input section gr.Markdown("### 📤 Input") image_input = gr.Image(type="numpy", label="Upload Image") # Camera settings gr.Markdown("### 📷 Camera Settings") camera_preset = gr.Dropdown( choices=list(CAMERA_PRESETS.keys()), value="iPhone 12/13/14 (Main Camera)", label="Camera Type", info="Select your camera or choose 'Custom' for manual input" ) with gr.Row(): fx_custom = gr.Number(value=1500, label="Focal Length X (pixels)", visible=False) fy_custom = gr.Number(value=1500, label="Focal Length Y (pixels)", visible=False) process_btn = gr.Button("🔄 Process Image", variant="primary", size="lg") # Reference object selection gr.Markdown("### 📏 Reference Object") with gr.Row(): mask_number = gr.Number(value=1, label="Object Number", precision=0, minimum=1) reference_size = gr.Number(value=10.0, label="Known Size (cm)", minimum=0.1) dimension_choice = gr.Radio( choices=["width", "height"], value="width", label="Which dimension is the known size?" ) select_ref_btn = gr.Button("📌 Set as Reference", variant="secondary") calculate_btn = gr.Button("📊 Calculate All Sizes", variant="primary", size="lg") with gr.Column(scale=2): # Output section gr.Markdown("### 🖼️ Results") image_output = gr.Image(label="Detected Objects") status_output = gr.Textbox(label="Status", lines=4, max_lines=10) results_output = gr.Textbox(label="Size Measurements", lines=15, max_lines=25) # Event handlers def toggle_custom_focal(preset): if preset == "Custom": return gr.update(visible=True), gr.update(visible=True) else: return gr.update(visible=False), gr.update(visible=False) camera_preset.change( toggle_custom_focal, inputs=[camera_preset], outputs=[fx_custom, fy_custom] ) # Load models on startup demo.load(app.load_models, outputs=[status_output]) process_btn.click( app.process_image, inputs=[image_input, camera_preset, fx_custom, fy_custom], outputs=[image_output, status_output] ) select_ref_btn.click( app.select_reference_object, inputs=[mask_number, reference_size, dimension_choice], outputs=[status_output] ) calculate_btn.click( app.calculate_all_sizes, inputs=[camera_preset, fx_custom, fy_custom], outputs=[results_output] ) # Additional controls and info with gr.Row(): with gr.Column(): gr.Markdown("### 🎯 Quick Actions") clear_btn = gr.Button("🗑️ Clear All", variant="secondary") with gr.Column(): gr.Markdown("### 📊 Session Info") session_info = gr.Textbox(label="Current Session", value="No image processed", interactive=False) # Event handlers for additional features def clear_session(): app.current_image = None app.current_depth = None app.current_masks = None app.reference_object = None return ( None, # image_output "🗑️ Session cleared. Upload a new image to start.", # status_output "", # results_output "No image processed" # session_info ) def update_session_info(camera_preset, fx_custom, fy_custom): if app.current_masks is None: return "No image processed" if camera_preset == "Custom": cam_info = f"Custom (fx={fx_custom:.0f}, fy={fy_custom:.0f})" else: cam_info = camera_preset ref_info = "None selected" if app.reference_object: ref_info = f"Object with {app.reference_object['known_size_cm']} cm {app.reference_object['dimension']}" return f"📷 Camera: {cam_info}\n📏 Reference: {ref_info}\n🎯 Objects: {len(app.current_masks)}" clear_btn.click( clear_session, outputs=[image_output, status_output, results_output, session_info] ) # Update session info when things change for component in [camera_preset, fx_custom, fy_custom]: component.change( update_session_info, inputs=[camera_preset, fx_custom, fy_custom], outputs=[session_info] ) gr.Markdown(""" ### 💡 Tips for best results: - Use good lighting and avoid shadows - Ensure objects are clearly visible and separated - Choose a reference object you know the exact size of - For phones, try the camera-specific presets first - Custom focal lengths can be calibrated using camera calibration tools """) return demo if __name__ == "__main__": demo = create_interface() demo.launch(share=True, debug=True)