Merge pull request #2 from yichuan520030910320/qwen

app.py

@@ -3,38 +3,31 @@ import json
 import os
 import time
 import re
-from io import BytesIO
-from PIL import Image
 from pathlib import Path
-import pyperclip
 
-from geo_bot import GeoBot, AGENT_PROMPT_TEMPLATE
+from geo_bot import GeoBot
 from benchmark import MapGuesserBenchmark
-from config import MODELS_CONFIG, get_data_paths, SUCCESS_THRESHOLD_KM, DEFAULT_MODEL, DEFAULT_TEMPERATURE
-from langchain_openai import ChatOpenAI
-from langchain_anthropic import ChatAnthropic
-from langchain_google_genai import ChatGoogleGenerativeAI
-from hf_chat import HuggingFaceChat
-
-# Simple API key setup
-if "OPENAI_API_KEY" in st.secrets:
-    os.environ["OPENAI_API_KEY"] = st.secrets["OPENAI_API_KEY"]
-if "ANTHROPIC_API_KEY" in st.secrets:
-    os.environ["ANTHROPIC_API_KEY"] = st.secrets["ANTHROPIC_API_KEY"]
-if "GOOGLE_API_KEY" in st.secrets:
-    os.environ["GOOGLE_API_KEY"] = st.secrets["GOOGLE_API_KEY"]
-if "HF_TOKEN" in st.secrets:
-    os.environ["HF_TOKEN"] = st.secrets["HF_TOKEN"]
+from config import (
+    MODELS_CONFIG,
+    get_data_paths,
+    SUCCESS_THRESHOLD_KM,
+    get_model_class,
+    DEFAULT_MODEL,
+    DEFAULT_TEMPERATURE,
+    setup_environment_variables,
+)
+
+setup_environment_variables(st.secrets)
 
 
 def convert_google_to_mapcrunch_url(google_url):
     """Convert Google Maps URL to MapCrunch URL format."""
     try:
         # Extract coordinates using regex
-        match = re.search(r'@(-?\d+\.\d+),(-?\d+\.\d+)', google_url)
+        match = re.search(r"@(-?\d+\.\d+),(-?\d+\.\d+)", google_url)
         if not match:
             return None
-        
+
         lat, lon = match.groups()
         # MapCrunch format: lat_lon_heading_pitch_zoom
         # Using default values for heading (317.72), pitch (0.86), and zoom (0)
@@ -58,21 +51,10 @@ def get_available_datasets():
     return datasets if datasets else ["default"]
 
 
-def get_model_class(class_name):
-    if class_name == "ChatOpenAI":
-        return ChatOpenAI
-    elif class_name == "ChatAnthropic":
-        return ChatAnthropic
-    elif class_name == "ChatGoogleGenerativeAI":
-        return ChatGoogleGenerativeAI
-    elif class_name == "HuggingFaceChat":
-        return HuggingFaceChat
-    else:
-        raise ValueError(f"Unknown model class: {class_name}")
-
-
 # UI Setup
-st.set_page_config(page_title="🧠 Omniscient - Multiturn Geographic Intelligence", layout="wide")
+st.set_page_config(
+    page_title="🧠 Omniscient - Multiturn Geographic Intelligence", layout="wide"
+)
 st.title("🧠 Omniscient")
 st.markdown("""
 ### *An all-seeing AI agent for geographic analysis and deduction*
@@ -86,14 +68,18 @@ with st.sidebar:
 
     # Mode selection
     mode = st.radio("Mode", ["Dataset Mode", "Online Mode"], index=0)
-    
+
     if mode == "Dataset Mode":
         # Get available datasets and ensure we have a valid default
         available_datasets = get_available_datasets()
         default_dataset = available_datasets[0] if available_datasets else "default"
-        
+
         dataset_choice = st.selectbox("Dataset", available_datasets, index=0)
-        model_choice = st.selectbox("Model", list(MODELS_CONFIG.keys()), index=list(MODELS_CONFIG.keys()).index(DEFAULT_MODEL))
+        model_choice = st.selectbox(
+            "Model",
+            list(MODELS_CONFIG.keys()),
+            index=list(MODELS_CONFIG.keys()).index(DEFAULT_MODEL),
+        )
         steps_per_sample = st.slider("Max Steps", 1, 20, 10)
         temperature = st.slider(
             "Temperature",
@@ -109,14 +95,16 @@ with st.sidebar:
         try:
             with open(data_paths["golden_labels"], "r") as f:
                 golden_labels = json.load(f).get("samples", [])
-            
+
             st.info(f"Dataset '{dataset_choice}' has {len(golden_labels)} samples")
             if len(golden_labels) == 0:
                 st.error(f"Dataset '{dataset_choice}' contains no samples!")
                 st.stop()
-                
+
         except FileNotFoundError:
-            st.error(f"❌ Dataset '{dataset_choice}' not found at {data_paths['golden_labels']}")
+            st.error(
+                f"❌ Dataset '{dataset_choice}' not found at {data_paths['golden_labels']}"
+            )
             st.info("💡 Available datasets: " + ", ".join(available_datasets))
             st.stop()
         except Exception as e:
@@ -128,19 +116,21 @@ with st.sidebar:
         )
     else:  # Online Mode
         st.info("Enter a URL to analyze a specific location")
-        
+
         # Add example URLs
         example_google_url = "https://www.google.com/maps/@37.8728123,-122.2445339,3a,75y,3.36h,90t/data=!3m7!1e1!3m5!1s4DTABKOpCL6hdNRgnAHTgw!2e0!6shttps:%2F%2Fstreetviewpixels-pa.googleapis.com%2Fv1%2Fthumbnail%3Fcb_client%3Dmaps_sv.tactile%26w%3D900%26h%3D600%26pitch%3D0%26panoid%3D4DTABKOpCL6hdNRgnAHTgw%26yaw%3D3.3576431!7i13312!8i6656?entry=ttu"
-        example_mapcrunch_url = "http://www.mapcrunch.com/p/37.882284_-122.269626_293.91_-6.63_0"
-        
+        example_mapcrunch_url = (
+            "http://www.mapcrunch.com/p/37.882284_-122.269626_293.91_-6.63_0"
+        )
+
         # Create tabs for different URL types
         input_tab1, input_tab2 = st.tabs(["Google Maps URL", "MapCrunch URL"])
-        
+
         google_url = ""
         mapcrunch_url = ""
         golden_labels = None
         num_samples = None
-        
+
         with input_tab1:
             url_col1, url_col2 = st.columns([3, 1])
             with url_col1:
@@ -149,56 +139,68 @@ with st.sidebar:
                     placeholder="https://www.google.com/maps/@37.5851338,-122.1519467,9z?entry=ttu",
                     key="google_maps_url",
                 )
-            st.markdown(f"💡 **Example Location:** [View in Google Maps]({example_google_url})")
+            st.markdown(
+                f"💡 **Example Location:** [View in Google Maps]({example_google_url})"
+            )
             if google_url:
                 mapcrunch_url_converted = convert_google_to_mapcrunch_url(google_url)
                 if mapcrunch_url_converted:
                     st.success(f"Converted to MapCrunch URL: {mapcrunch_url_converted}")
                     try:
-                        golden_labels = [{
-                            "id": "online",
-                            "lat": float(re.search(r'@(-?\d+\.\d+),(-?\d+\.\d+)', google_url).group(1)),
-                            "lng": float(re.search(r'@(-?\d+\.\d+),(-?\d+\.\d+)', google_url).group(2)),
-                            "url": mapcrunch_url_converted
-                        }]
+                        match = re.search(r"@(-?\d+\.\d+),(-?\d+\.\d+)", google_url)
+                        if not match:
+                            st.error("Invalid Google Maps URL format")
+                            st.stop()
+
+                        lat, lon = match.groups()
+
+                        golden_labels = [
+                            {
+                                "id": "online",
+                                "lat": float(lat),
+                                "lng": float(lon),
+                                "url": mapcrunch_url_converted,
+                            }
+                        ]
                         num_samples = 1
                     except Exception as e:
                         st.error(f"Invalid Google Maps URL format: {str(e)}")
                 else:
                     st.error("Invalid Google Maps URL format")
-        
+
         with input_tab2:
             st.markdown("💡 **Example Location:**")
             st.markdown(f"[View in MapCrunch]({example_mapcrunch_url})")
             st.code(example_mapcrunch_url, language="text")
             mapcrunch_url = st.text_input(
-                "MapCrunch URL",
-                placeholder=example_mapcrunch_url,
-                key="mapcrunch_url"
+                "MapCrunch URL", placeholder=example_mapcrunch_url, key="mapcrunch_url"
             )
             if mapcrunch_url:
                 try:
-                    coords = mapcrunch_url.split('/')[-1].split('_')
+                    coords = mapcrunch_url.split("/")[-1].split("_")
                     lat, lon = float(coords[0]), float(coords[1])
-                    golden_labels = [{
-                        "id": "online",
-                        "lat": lat,
-                        "lng": lon,
-                        "url": mapcrunch_url
-                    }]
+                    golden_labels = [
+                        {"id": "online", "lat": lat, "lng": lon, "url": mapcrunch_url}
+                    ]
                     num_samples = 1
                 except Exception as e:
                     st.error(f"Invalid MapCrunch URL format: {str(e)}")
-        
+
         # Only stop if neither input is provided
         if not google_url and not mapcrunch_url:
-            st.warning("Please enter a Google Maps URL or MapCrunch URL, or use the example above.")
+            st.warning(
+                "Please enter a Google Maps URL or MapCrunch URL, or use the example above."
+            )
             st.stop()
         if golden_labels is None or num_samples is None:
             st.warning("Please enter a valid URL.")
             st.stop()
-        
-        model_choice = st.selectbox("Model", list(MODELS_CONFIG.keys()), index=list(MODELS_CONFIG.keys()).index(DEFAULT_MODEL))
+
+        model_choice = st.selectbox(
+            "Model",
+            list(MODELS_CONFIG.keys()),
+            index=list(MODELS_CONFIG.keys()).index(DEFAULT_MODEL),
+        )
         steps_per_sample = st.slider("Max Steps", 1, 20, 10)
         temperature = st.slider(
             "Temperature",
@@ -217,7 +219,9 @@ if start_button:
     config = MODELS_CONFIG[model_choice]
     model_class = get_model_class(config["class"])
 
-    benchmark_helper = MapGuesserBenchmark(dataset_name=dataset_choice if mode == "Dataset Mode" else "online")
+    benchmark_helper = MapGuesserBenchmark(
+        dataset_name=dataset_choice if mode == "Dataset Mode" else "online"
+    )
     all_results = []
 
     progress_bar = st.progress(0)
@@ -238,7 +242,7 @@ if start_button:
             else:
                 # Load from dataset as before
                 bot.controller.load_location_from_data(sample)
-            
+
             bot.controller.setup_clean_environment()
 
             # Create containers for UI updates
@@ -421,7 +425,10 @@ if start_button:
             mime="application/json",
         )
 
+
 def handle_tab_completion():
     """Handle tab completion for the Google Maps URL input."""
     if st.session_state.google_maps_url == "":
-        st.session_state.google_maps_url = "https://www.google.com/maps/@37.5851338,-122.1519467,9z?entry=ttu"
+        st.session_state.google_maps_url = (
+            "https://www.google.com/maps/@37.5851338,-122.1519467,9z?entry=ttu"
+        )

benchmark.py

@@ -9,7 +9,7 @@ from pathlib import Path
 import math
 
 from geo_bot import GeoBot
-from config import get_data_paths, MODELS_CONFIG, SUCCESS_THRESHOLD_KM
+from config import get_data_paths, MODELS_CONFIG, SUCCESS_THRESHOLD_KM, get_model_class
 
 
 class MapGuesserBenchmark:
@@ -29,25 +29,6 @@ class MapGuesserBenchmark:
         except Exception:
             return []
 
-    def get_model_class(self, model_name: str):
-        config = MODELS_CONFIG.get(model_name)
-        if not config:
-            raise ValueError(f"Unknown model: {model_name}")
-        class_name, model_class_name = config["class"], config["model_name"]
-        if class_name == "ChatOpenAI":
-            from langchain_openai import ChatOpenAI
-
-            return ChatOpenAI, model_class_name
-        if class_name == "ChatAnthropic":
-            from langchain_anthropic import ChatAnthropic
-
-            return ChatAnthropic, model_class_name
-        if class_name == "ChatGoogleGenerativeAI":
-            from langchain_google_genai import ChatGoogleGenerativeAI
-
-            return ChatGoogleGenerativeAI, model_class_name
-        raise ValueError(f"Unknown model class: {class_name}")
-
     def calculate_distance(
         self, true_coords: Dict, predicted_coords: Optional[Tuple[float, float]]
     ) -> Optional[float]:
@@ -99,7 +80,9 @@ class MapGuesserBenchmark:
         all_results = []
         for model_name in models_to_test:
             print(f"\n🤖 Testing model: {model_name}")
-            model_class, model_class_name = self.get_model_class(model_name)
+            model_config = MODELS_CONFIG[model_name]
+            model_class = get_model_class(model_config["class"])
+            model_class_name = model_config["model_name"]
 
             try:
                 with GeoBot(

config.py

@@ -1,5 +1,10 @@
 # Configuration file for MapCrunch benchmark
 
+from pydantic import SecretStr, Field
+from typing import Optional
+import os
+
+
 SUCCESS_THRESHOLD_KM = 100
 
 # MapCrunch settings
@@ -42,10 +47,15 @@ MODELS_CONFIG = {
         "model_name": "gpt-4o-mini",
         "description": "OpenAI GPT-4o Mini",
     },
-    "claude-3.5-sonnet": {
+    "claude-3-7-sonnet": {
+        "class": "ChatAnthropic",
+        "model_name": "claude-3-7-sonnet-20250219",
+        "description": "Anthropic Claude 3.7 Sonnet",
+    },
+    "claude-4-sonnet": {
         "class": "ChatAnthropic",
-        "model_name": "claude-3-5-sonnet-20240620",
-        "description": "Anthropic Claude 3.5 Sonnet",
+        "model_name": "claude-4-sonnet-20250514",
+        "description": "Anthropic Claude 4 Sonnet",
     },
     "gemini-1.5-pro": {
         "class": "ChatGoogleGenerativeAI",
@@ -62,18 +72,93 @@ MODELS_CONFIG = {
         "model_name": "gemini-2.5-pro-preview-06-05",
         "description": "Google Gemini 2.5 Pro",
     },
-    "qwen2-vl-7b": {
-        "class": "HuggingFaceChat",
-        "model_name": "Qwen/Qwen2-VL-7B-Instruct",
-        "description": "Qwen2-VL 7B (older but API supported)",
+    "qwen-vl-max": {
+        "class": "OpenRouter",
+        "model_name": "qwen/qwen-vl-max",
+        "description": "Qwen VL Max - OpenRouter (Best Performance)",
+    },
+    "qwen2.5-vl-32b-free": {
+        "class": "OpenRouter",
+        "model_name": "qwen/qwen2.5-vl-32b-instruct:free",
+        "description": "Qwen2.5 VL 32B - OpenRouter (FREE!)",
+    },
+    "qwen2.5-vl-7b": {
+        "class": "OpenRouter",
+        "model_name": "qwen/qwen2.5-vl-7b-instruct",
+        "description": "Qwen2.5 VL 7B - OpenRouter",
     },
-    "qwen2-vl-2b": {
-        "class": "HuggingFaceChat",
-        "model_name": "Qwen/Qwen2-VL-2B-Instruct",
-        "description": "Qwen2-VL 2B (faster, API supported)",
+    "qwen2.5-vl-3b": {
+        "class": "OpenRouter",
+        "model_name": "qwen/qwen2.5-vl-3b-instruct",
+        "description": "Qwen2.5 VL 3B - OpenRouter (Fastest)",
     },
 }
 
+POSSIBLE_API_KEYS = [
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "GOOGLE_API_KEY",
+    "HF_TOKEN",
+    "OPENROUTER_API_KEY",
+]
+
+
+def setup_environment_variables(st_secrets=None):
+    for key in POSSIBLE_API_KEYS:
+        # Try Streamlit secrets first if provided
+        if st_secrets and key in st_secrets:
+            os.environ[key] = st_secrets[key]
+        elif key in os.environ:
+            continue
+
+
+def get_model_class(class_name):
+    """Get actual model class from string name"""
+    if class_name == "ChatOpenAI":
+        from langchain_openai import ChatOpenAI
+
+        return ChatOpenAI
+    elif class_name == "ChatAnthropic":
+        from langchain_anthropic import ChatAnthropic
+
+        return ChatAnthropic
+    elif class_name == "ChatGoogleGenerativeAI":
+        from langchain_google_genai import ChatGoogleGenerativeAI
+
+        return ChatGoogleGenerativeAI
+    elif class_name == "HuggingFaceChat":
+        from hf_chat import HuggingFaceChat
+
+        return HuggingFaceChat
+    elif class_name == "OpenRouter":
+        from langchain_openai import ChatOpenAI
+        from langchain_core.utils.utils import secret_from_env
+
+        # LangChain does not support OpenRouter directly, so we need to create a custom class
+        # See https://github.com/langchain-ai/langchain/discussions/27964.
+        class ChatOpenRouter(ChatOpenAI):
+            openai_api_key: Optional[SecretStr] = Field(
+                alias="api_key",
+                default_factory=secret_from_env("OPENROUTER_API_KEY", default=None),
+            )
+
+            @property
+            def lc_secrets(self) -> dict[str, str]:
+                return {"openai_api_key": "OPENROUTER_API_KEY"}
+
+            def __init__(self, openai_api_key: Optional[str] = None, **kwargs):
+                openai_api_key = openai_api_key or os.environ.get("OPENROUTER_API_KEY")
+                super().__init__(
+                    base_url="https://openrouter.ai/api/v1",
+                    api_key=SecretStr(openai_api_key) if openai_api_key else None,
+                    **kwargs,
+                )
+
+        return ChatOpenRouter
+    else:
+        raise ValueError(f"Unknown model class: {class_name}")
+
+
 # Data paths - now supports named datasets
 def get_data_paths(dataset_name: str = "default"):
     """Get data paths for a specific dataset"""
@@ -83,5 +168,6 @@ def get_data_paths(dataset_name: str = "default"):
         "results": f"results/{dataset_name}/",
     }
 
+
 # Backward compatibility - default paths
 DATA_PATHS = get_data_paths("default")

geo_bot.py

@@ -6,13 +6,7 @@ from typing import Tuple, List, Optional, Dict, Any, Type
 
 from PIL import Image
 from langchain_core.messages import HumanMessage, BaseMessage
-from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_openai import ChatOpenAI
-from langchain_anthropic import ChatAnthropic
-from langchain_google_genai import ChatGoogleGenerativeAI
-
 from hf_chat import HuggingFaceChat
-
 from mapcrunch_controller import MapCrunchController
 
 # The "Golden" Prompt (v7): add more descprtions in context and task

main.py

@@ -1,16 +1,10 @@
 import argparse
 import json
-import random
-from typing import Dict, Optional, List
-
-from langchain_openai import ChatOpenAI
-from langchain_anthropic import ChatAnthropic
-from langchain_google_genai import ChatGoogleGenerativeAI
 
 from geo_bot import GeoBot
 from benchmark import MapGuesserBenchmark
 from data_collector import DataCollector
-from config import MODELS_CONFIG, get_data_paths, SUCCESS_THRESHOLD_KM
+from config import MODELS_CONFIG, get_data_paths, SUCCESS_THRESHOLD_KM, get_model_class
 
 
 def agent_mode(
@@ -48,7 +42,7 @@ def agent_mode(
     print(f"Will run on {len(test_samples)} samples from dataset '{dataset_name}'.")
 
     config = MODELS_CONFIG.get(model_name)
-    model_class = globals()[config["class"]]
+    model_class = get_model_class(config["class"])
     model_instance_name = config["model_name"]
 
     benchmark_helper = MapGuesserBenchmark(dataset_name=dataset_name, headless=True)