app.py

import gradio as gr
import json
import requests
import os
import pandas as pd
import folium
from folium.plugins import MeasureControl, Fullscreen, MarkerCluster, Search
from geopy.geocoders import Nominatim
from geopy.exc import GeocoderTimedOut, GeocoderServiceError
import time
import random
from typing import List, Tuple, Optional
import io
import tempfile
import warnings

warnings.filterwarnings("ignore")

# Map Tile Providers with reliable sources
MAP_TILES = {
    "Satellite": {
        "url": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}",
        "attr": "Esri",
        "fallback": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Topo_Map/MapServer/tile/{z}/{y}/{x}"
    },
    "Topographic": {
        "url": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Topo_Map/MapServer/tile/{z}/{y}/{x}",
        "attr": "Esri",
        "fallback": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
    },
    "OpenStreetMap": {
        "url": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",
        "attr": "OpenStreetMap",
        "fallback": None
    },
    "Terrain": {
        "url": "https://server.arcgisonline.com/ArcGIS/rest/services/World_Terrain_Base/MapServer/tile/{z}/{y}/{x}",
        "attr": "Esri",
        "fallback": None
    },
    "Toner": {
        "url": "https://tiles.stadiamaps.com/tiles/stamen_toner/{z}/{x}/{y}.png",
        "attr": "Stadia Maps",
        "fallback": None
    }
}

# NuExtract API configuration
API_URL = "https://api-inference.huggingface.co/models/numind/NuExtract-1.5"
headers = {"Authorization": f"Bearer {os.environ.get('HF_TOKEN', '')}"}

# Geocoding Service
class GeocodingService:
    def __init__(self, user_agent: str = None, timeout: int = 10, rate_limit: float = 1.1):
        if user_agent is None:
            user_agent = f"python_geocoding_script_{random.randint(1000, 9999)}"

        self.geolocator = Nominatim(
            user_agent=user_agent,
            timeout=timeout
        )
        self.rate_limit = rate_limit
        self.last_request = 0
        self.cache = {}  # Simple in-memory cache

    def _rate_limit_wait(self):
        current_time = time.time()
        time_since_last = current_time - self.last_request
        if time_since_last < self.rate_limit:
            time.sleep(self.rate_limit - time_since_last)
        self.last_request = time.time()

    def geocode_location(self, location: str, max_retries: int = 3) -> Optional[Tuple[float, float]]:
        # Check cache first
        if location in self.cache:
            return self.cache[location]
            
        for attempt in range(max_retries):
            try:
                self._rate_limit_wait()
                location_data = self.geolocator.geocode(location)
                if location_data:
                    # Store in cache and return
                    self.cache[location] = (location_data.latitude, location_data.longitude)
                    return self.cache[location]
                # Cache None results too
                self.cache[location] = None
                return None
            except (GeocoderTimedOut, GeocoderServiceError) as e:
                if attempt == max_retries - 1:
                    print(f"Failed to geocode '{location}' after {max_retries} attempts: {e}")
                    self.cache[location] = None
                    return None
                time.sleep(2 ** attempt)  # Exponential backoff
            except Exception as e:
                print(f"Error geocoding '{location}': {e}")
                self.cache[location] = None
                return None
        return None

    def process_locations(self, locations: str) -> List[Optional[Tuple[float, float]]]:
        if pd.isna(locations) or not locations:
            return []

        try:
            # First try to intelligently parse
            import re
            pattern = r"([^,]+(?:,\s*[A-Za-z]+)?)"
            matches = re.findall(pattern, locations)
            location_list = [match.strip() for match in matches if match.strip()]
            
            # If regex finds nothing, fall back to simple comma splitting
            if not location_list:
                location_list = [loc.strip() for loc in locations.split(',') if loc.strip()]
                
            # For debugging
            print(f"Parsed '{locations}' into: {location_list}")
                
            return [self.geocode_location(loc) for loc in location_list]
        except Exception as e:
            print(f"Error parsing locations '{locations}': {e}")
            # Fall back to simple method
            location_list = [loc.strip() for loc in locations.split(',') if loc.strip()]
            return [self.geocode_location(loc) for loc in location_list]

def create_reliable_map(df, location_col):
    """Create a map with multiple layer options and better error handling"""
    
    # Set default tile
    default_tile_name = "Toner"
    
    # Initialize map
    m = folium.Map(location=[20, 0], zoom_start=2, control_scale=True)
    
    # Add all tile layers with the appropriate one active, but no layer control
    for name, config in MAP_TILES.items():
        folium.TileLayer(
            tiles=config["url"],
            attr=f"{config['attr']} ({name})",
            name=name,
            overlay=False,
            control=False,  # Disable tile layer in controls
            show=(name == default_tile_name)  # Only show the default layer initially
        ).add_to(m)
    
    # Add plugins for better user experience
    Fullscreen().add_to(m)
    MeasureControl(position='topright', primary_length_unit='kilometers').add_to(m)
    
    # Add markers
    geocoder = SafeGeocoder()
    coords = []
    
    # Create marker cluster for better performance with many points
    marker_cluster = MarkerCluster(name="Locations").add_to(m)
    
    # Process each location
    processed_count = 0
    for idx, row in df.iterrows():
        if pd.isna(row[location_col]):
            continue
            
        location = str(row[location_col]).strip()
        
        # Get additional info if available
        additional_info = ""
        for col in df.columns:
            if col != location_col and not pd.isna(row[col]):
                additional_info += f"<br><b>{col}:</b> {row[col]}"
        
        # Parse multiple locations if comma-separated
        try:
            locations = [loc.strip() for loc in location.split(',') if loc.strip()]
            if not locations:
                locations = [location]
        except:
            locations = [location]
            
        # Process each location
        for loc in locations:
            # Geocode location
            point = geocoder.get_coords(loc)
            if point:
                # Create popup content
                popup_content = f"""
                <div style="min-width: 200px; max-width: 300px">
                    <h4 style="font-family: 'Source Sans Pro', sans-serif; margin-bottom: 5px;">{loc}</h4>
                    <div style="font-family: 'Source Sans Pro', sans-serif; font-size: 14px;">
                        {additional_info}
                    </div>
                </div>
                """
                
                # Add marker
                folium.Marker(
                    location=point,
                    popup=folium.Popup(popup_content, max_width=300),
                    tooltip=loc,
                    icon=folium.Icon(color="blue", icon="info-sign")
                ).add_to(marker_cluster)
                
                coords.append(point)
                processed_count += 1
    
    # Layer control - removed as requested
    # folium.LayerControl(collapsed=False).add_to(m)
    
    # Set bounds if we have coordinates
    if coords:
        m.fit_bounds(coords)
    
    # Add better tile error handling with JavaScript
    m.get_root().html.add_child(folium.Element("""
    <script>
    // Wait for the map to be fully loaded
    document.addEventListener('DOMContentLoaded', function() {
        setTimeout(function() {
            // Get the map instance
            var maps = document.querySelectorAll('.leaflet-container');
            if (maps.length > 0) {
                var map = maps[0];
                
                // Add error handler for tiles
                var layers = map.querySelectorAll('.leaflet-tile-pane .leaflet-layer');
                for (var i = 0; i < layers.length; i++) {
                    var layer = layers[i];
                    var tiles = layer.querySelectorAll('.leaflet-tile');
                    
                    // Check if layer has no loaded tiles
                    var loadedTiles = layer.querySelectorAll('.leaflet-tile-loaded');
                    if (tiles.length > 0 && loadedTiles.length === 0) {
                        // Force switch to OpenStreetMap if current layer failed
                        var osmButton = document.querySelector('.leaflet-control-layers-list input[type="radio"]:nth-child(3)');
                        if (osmButton) {
                            osmButton.click();
                        }
                        console.log("Switched to fallback tile layer due to loading issues");
                    }
                }
            }
        }, 3000); // Wait 3 seconds for tiles to load
    });
    </script>
    
    <style>
    .leaflet-popup-content {
        font-family: 'Source Sans Pro', sans-serif;
    }
    .leaflet-popup-content h4 {
        font-weight: 600;
        margin-bottom: 8px;
    }
    .leaflet-control-layers {
        font-family: 'Source Sans Pro', sans-serif;
    }
    </style>
    """))
    
    # Add custom CSS for better fonts
    custom_css = """
    <style>
    @import url('https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600&display=swap');
    .leaflet-container {
        font-family: 'Source Sans Pro', sans-serif;
    }
    </style>
    """
    m.get_root().header.add_child(folium.Element(custom_css))
    
    return m._repr_html_(), processed_count

# SafeGeocoder with better error handling
class SafeGeocoder:
    def __init__(self):
        user_agent = f"location_mapper_v1_{random.randint(1000, 9999)}"
        self.geolocator = Nominatim(user_agent=user_agent, timeout=10)
        self.cache = {}  # Simple cache to avoid repeated requests
        self.last_request = 0
    
    def _respect_rate_limit(self):
        # Ensure at least 1 second between requests
        current_time = time.time()
        elapsed = current_time - self.last_request
        if elapsed < 1.0:
            time.sleep(1.0 - elapsed)
        self.last_request = time.time()
    
    def get_coords(self, location: str):
        if not location or pd.isna(location):
            return None
            
        # Convert to string if needed
        location = str(location).strip()
        
        # Check cache first
        if location in self.cache:
            return self.cache[location]
        
        try:
            self._respect_rate_limit()
            result = self.geolocator.geocode(location)
            if result:
                coords = (result.latitude, result.longitude)
                self.cache[location] = coords
                return coords
            self.cache[location] = None
            return None
        except Exception as e:
            print(f"Geocoding error for '{location}': {e}")
            self.cache[location] = None
            return None

def process_excel(file, places_column):
    # Check if file is None
    if file is None:
        return None, "No file uploaded", None
    
    try:
        # Handle various file object types that Gradio might provide
        if hasattr(file, 'name'):
            # Gradio file object
            df = pd.read_excel(file.name)
        elif isinstance(file, bytes):
            # Raw bytes
            df = pd.read_excel(io.BytesIO(file))
        else:
            # Assume it's a filepath string
            df = pd.read_excel(file)
        
        # Print column names for debugging
        print(f"Columns in Excel file: {list(df.columns)}")
        
        if places_column not in df.columns:
            return None, f"Column '{places_column}' not found in the Excel file. Available columns: {', '.join(df.columns)}", None
        
        # Create map
        map_html, processed_count = create_reliable_map(df, places_column)
        
        # Save processed data
        with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp:
            processed_path = tmp.name
            df.to_excel(processed_path, index=False)
        
        # Generate stats
        total_locations = df[places_column].count()
        success_rate = (processed_count / total_locations * 100) if total_locations > 0 else 0
        
        stats = f"Found {processed_count} of {total_locations} locations ({success_rate:.1f}%)"
        
        return map_html, stats, processed_path
    except Exception as e:
        import traceback
        trace = traceback.format_exc()
        print(f"Error processing file: {e}\n{trace}")
        return None, f"Error processing file: {str(e)}", None

def process_and_map(file, column):
    if file is None:
        return None, "Please upload an Excel file", None
    
    try:
        map_html, stats, processed_path = process_excel(file, column)
        
        if map_html and processed_path:
            # Create responsive container for the map
            responsive_html = f"""
            <div style="width:100%; height:70vh; margin:0; padding:0; border:1px solid #e0e0e0; border-radius:8px; overflow:hidden;">
                {map_html}
            </div>
            """
            return responsive_html, stats, processed_path
        else:
            return None, stats, None
    except Exception as e:
        import traceback
        trace = traceback.format_exc()
        print(f"Error in process_and_map: {e}\n{trace}")
        return None, f"Error: {str(e)}", None

# NuExtract Functions
def extract_info(template, text):
    try:
        # Format prompt according to NuExtract-1.5 requirements
        prompt = f"<|input|>\n### Template:\n{template}\n### Text:\n{text}\n\n<|output|>"
        
        # Call API
        payload = {
            "inputs": prompt,
            "parameters": {
                "max_new_tokens": 1000,
                "do_sample": False
            }
        }
        
        response = requests.post(API_URL, headers=headers, json=payload)
        
        # If the model is loading, inform the user
        if response.status_code == 503:
            response_json = response.json()
            if "error" in response_json and "loading" in response_json["error"]:
                estimated_time = response_json.get("estimated_time", "unknown")
                return f"⏳ Model is loading (ETA: {int(float(estimated_time)) if isinstance(estimated_time, (int, float, str)) else 'unknown'} seconds)", "Please try again in a few minutes"
        
        if response.status_code != 200:
            return f"❌ API Error: {response.status_code}", response.text
        
        # Process result
        result = response.json()
        
        # Handle different response formats
        try:
            if isinstance(result, list):
                if len(result) > 0:
                    result_text = result[0].get("generated_text", "")
                else:
                    return "❌ Empty result list", "{}"
            else:
                result_text = str(result)
            
            # Split at output marker if present
            if "<|output|>" in result_text:
                parts = result_text.split("<|output|>")
                if len(parts) > 1:
                    json_text = parts[1].strip()
                else:
                    json_text = result_text
            else:
                json_text = result_text
            
            # Try to parse as JSON
            try:
                extracted = json.loads(json_text)
                formatted = json.dumps(extracted, indent=2)
            except json.JSONDecodeError:
                return "❌ JSON parsing error", json_text
                
            return "✅ Success", formatted
        except Exception as inner_e:
            return f"❌ Error processing result: {str(inner_e)}", "{}"
    except Exception as e:
        return f"❌ Error: {str(e)}", "{}"

# Custom CSS for improved styling
custom_css = """
<style>
@import url('https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@300;400;600;700&display=swap');

:root {
    --primary-color: #2c6bb3;
    --secondary-color: #4e8fd1;
    --background-color: #f7f9fc;
    --text-color: #333333;
    --border-color: #e0e0e0;
}

body, .gradio-container {
    font-family: 'Source Sans Pro', sans-serif !important;
    background-color: var(--background-color);
    color: var(--text-color);
}

h1 {
    font-weight: 700 !important;
    color: var(--primary-color) !important;
    font-size: 2.5rem !important;
    margin-bottom: 1rem !important;
}

h2 {
    font-weight: 600 !important;
    color: var(--secondary-color) !important;
    font-size: 1.5rem !important;
    margin-top: 1rem !important;
    margin-bottom: 0.75rem !important;
}

.gradio-button.primary {
    background-color: var(--primary-color) !important;
}

.gradio-button.primary:hover {
    background-color: var(--secondary-color) !important;
}

.gradio-tab-nav button {
    font-family: 'Source Sans Pro', sans-serif !important;
    font-weight: 600 !important;
}

.gradio-tab-nav button.selected {
    color: var(--primary-color) !important;
    border-color: var(--primary-color) !important;
}

.info-box {
    background-color: #e8f4fd;
    border-left: 4px solid var(--primary-color);
    padding: 15px;
    margin: 15px 0;
    border-radius: 4px;
}

.stats-box {
    background-color: white;
    border: 1px solid var(--border-color);
    border-radius: 8px;
    padding: 15px;
    font-size: 1rem;
    line-height: 1.5;
}

.subtle-text {
    font-size: 0.9rem;
    color: #666;
    font-style: italic;
}

.file-upload-box {
    border: 2px dashed var(--border-color);
    border-radius: 8px;
    padding: 20px;
    text-align: center;
    transition: all 0.3s ease;
}

.file-upload-box:hover {
    border-color: var(--primary-color);
}

</style>
"""

# Create the Gradio interface
with gr.Blocks(css=custom_css) as demo:
    gr.HTML("""
    <div style="text-align: center; margin-bottom: 1rem">
        <h1>Historical Data Analysis Tools</h1>
        <p style="font-size: 1.1rem; margin-top: -10px;">Extract, visualize, and analyze historical data with ease</p>
    </div>
    """)
    
    with gr.Tabs():
        with gr.TabItem("🔍 Text Extraction"):
            gr.HTML("""
            <div class="info-box">
                <h3 style="margin-top: 0;">Extract Structured Data from Text</h3>
                <p>Use NuExtract-1.5 to automatically extract structured information from historical texts. Define the JSON template for the data you want to extract.</p>
            </div>
            """)
            
            with gr.Row():
                with gr.Column():
                    template = gr.Textbox(
                        label="JSON Template", 
                        value='{"earthquake location": "", "dateline location": ""}',
                        lines=5,
                        placeholder="Define the fields you want to extract as a JSON template"
                    )
                    text = gr.Textbox(
                        label="Text to Extract From",
                        value="Neues Erdbeben in Japan. Aus Tokio wird berichtet, daß in Yokohama bei einem Erdbeben sechs Personen getötet und 22 verwundet, in Tokio vier getötet und 22 verwundet wurden. In Yokohama seien 6VV Häuser zerstört worden. Die telephonische und telegraphische Verbindung zwischen Tokio und Osaka ist unterbrochen worden. Der Trambahnverkehr in Tokio liegt still. Auch der Eisenbahnverkehr zwischen Tokio und Yokohama ist unterbrochen. In Sngamo, einer Vorstadt von Tokio sind Brände ausgebrochen. Ein Eisenbahnzug stürzte in den Vajugawafluß zwischen Gotemba und Tokio. Sechs Züge wurden umgeworfen. Mit dem letzten japanischen Erdbeben sind seit eineinhalb Jahrtausenden bis heute in Japan 229 größere Erdbeben zu verzeichnen gewesen.",
                        lines=8,
                        placeholder="Enter the text you want to extract information from"
                    )
                    extract_btn = gr.Button("Extract Information", variant="primary", size="lg")
                
                with gr.Column():
                    status = gr.Textbox(
                        label="Status",
                        elem_classes="stats-box"
                    )
                    output = gr.Textbox(
                        label="Extracted Data",
                        elem_classes="stats-box",
                        lines=10
                    )
            
            extract_btn.click(
                fn=extract_info,
                inputs=[template, text],
                outputs=[status, output]
            )
        
        with gr.TabItem("📍 Location Mapping"):
            gr.HTML("""
            <div class="info-box">
                <h3 style="margin-top: 0;">Map Your Historical Locations</h3>
                <p>Upload an Excel file containing location data to create an interactive map visualization. The tool will geocode your locations and display them on a customizable map.</p>
            </div>
            """)
            
            with gr.Row():
                with gr.Column():
                    template = gr.Textbox(
                        label="JSON Template", 
                        value='{"earthquake location": "", "dateline location": ""}',
                        lines=5,
                        placeholder="Define the fields you want to extract as a JSON template"
                    )
                    text = gr.Textbox(
                        label="Text to Extract From",
                        value="Neues Erdbeben in Japan. Aus Tokio wird berichtet, daß in Yokohama bei einem Erdbeben sechs Personen getötet und 22 verwundet, in Tokio vier getötet und 22 verwundet wurden. In Yokohama seien 6VV Häuser zerstört worden. Die telephonische und telegraphische Verbindung zwischen Tokio und Osaka ist unterbrochen worden. Der Trambahnverkehr in Tokio liegt still. Auch der Eisenbahnverkehr zwischen Tokio und Yokohama ist unterbrochen. In Sngamo, einer Vorstadt von Tokio sind Brände ausgebrochen. Ein Eisenbahnzug stürzte in den Vajugawafluß zwischen Gotemba und Tokio. Sechs Züge wurden umgeworfen. Mit dem letzten japanischen Erdbeben sind seit eineinhalb Jahrtausenden bis heute in Japan 229 größere Erdbeben zu verzeichnen gewesen.",
                        lines=8,
                        placeholder="Enter the text you want to extract information from"
                    )
                    extract_btn = gr.Button("Extract Information", variant="primary", size="lg")
                
                with gr.Column():
                    status = gr.Textbox(
                        label="Status",
                        elem_classes="stats-box"
                    )
                    output = gr.JSON(
                        label="Extracted Data",
                        elem_classes="stats-box"
                    )
            
            extract_btn.click(
                fn=extract_info,
                inputs=[template, text],
                outputs=[status, output]
            )
    
    gr.HTML("""
    <div style="text-align: center; margin-top: 2rem; padding-top: 1rem; border-top: 1px solid #eee; font-size: 0.9rem; color: #666;">
        <p>Made with <span style="color: #e25555;">❤</span> for historical data research</p>
    </div>
    """)

if __name__ == "__main__":
    demo.launch()