Spaces:

DebasishDhal99
/

map-wikireader

Runtime error

DebasishDhal commited on Jul 14

Commit

21ac8b9

unverified ·

Merge pull request #9 from DebasishDhal/feat/wiki-geosearch

- Wikipedia api can handle search area of radius <= 1 km
- Used multiple hexagonally arranged 10km circles to fill the larger circle, fetch wiki responses for all of them and use in frontend
- Made Search Radius max limit (both slider and keyboard dynamic)
- Enabled nearby mode map exploration possible with exploration sidebar closed

Render the html content below to see hexagonal circular packing of a 50km circle centered at Bangalore, with 10km circles.

Disadvantage - Assumes spherical earth. And it misses out some points in the gaps.
```
<!DOCTYPE html>
<html>
<head>

<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/leaflet.js"></script>
<script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/Leaflet.awesome-markers/2.0.2/leaflet.awesome-markers.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/leaflet.css"/>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"/>
<link rel="stylesheet" href="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap-glyphicons.css"/>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/[email protected]/css/all.min.css"/>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/Leaflet.awesome-markers/2.0.2/leaflet.awesome-markers.css"/>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/python-visualization/folium/folium/templates/leaflet.awesome.rotate.min.css"/>

<meta name="viewport" content="width=device-width,
initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<style>
#map_0d8b86cbc8c3d9e67b61e4080646ece2 {
position: relative;
width: 100.0%;
height: 100.0%;
left: 0.0%;
top: 0.0%;
}
.leaflet-container { font-size: 1rem; }
</style>

<style>html, body {
width: 100%;
height: 100%;
margin: 0;
padding: 0;
}
</style>

<style>#map {
position:absolute;
top:0;
bottom:0;
right:0;
left:0;
}
</style>

<script>
L_NO_TOUCH = false;
L_DISABLE_3D = false;
</script>

</head>
<body>

<div class="folium-map" id="map_0d8b86cbc8c3d9e67b61e4080646ece2" ></div>

</body>
<script>

var map_0d8b86cbc8c3d9e67b61e4080646ece2 = L.map(
"map_0d8b86cbc8c3d9e67b61e4080646ece2",
{
center: [12.9716, 77.5946],
crs: L.CRS.EPSG3857,
...{
"zoom": 10,
"zoomControl": true,
"preferCanvas": false,
}

}
);

var tile_layer_5539c953595e8644a534907b01c9b9dd = L.tileLayer(
"https://tile.openstreetmap.org/{z}/{x}/{y}.png",
{
"minZoom": 0,
"maxZoom": 19,
"maxNativeZoom": 19,
"noWrap": false,
"attribution": "\u0026copy; \u003ca href=\"https://www.openstreetmap.org/copyright\"\u003eOpenStreetMap\u003c/a\u003e contributors",
"subdomains": "abc",
"detectRetina": false,
"tms": false,
"opacity": 1,
}

);

tile_layer_5539c953595e8644a534907b01c9b9dd.addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_f1c8f9ae3fe1a67cf46e8338270315f5 = L.circle(
[12.9716, 77.5946],
{"bubblingMouseEvents": true, "color": "blue", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "blue", "fillOpacity": 0.1, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 50000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var popup_52aa2bb8a1b914eaf507be92fd68695e = L.popup({
"maxWidth": "100%",
});

var html_7447c4f769cea93e6ae52ce15cb34a8b = $(`<div id="html_7447c4f769cea93e6ae52ce15cb34a8b" style="width: 100.0%; height: 100.0%;">Target Area</div>`)[0];
popup_52aa2bb8a1b914eaf507be92fd68695e.setContent(html_7447c4f769cea93e6ae52ce15cb34a8b);

circle_f1c8f9ae3fe1a67cf46e8338270315f5.bindPopup(popup_52aa2bb8a1b914eaf507be92fd68695e)
;

var circle_52a6246c4981f1366d92201def7f9cbe = L.circle(
[12.504298785861298, 77.31773841119686],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_e78eb9cfb4635c517d3e43cf7bd7233a = L.circle(
[12.504298785861298, 77.50231280373228],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_e4ec44b83736eca64c05417624a37ed8 = L.circle(
[12.504298785861298, 77.68688719626772],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_caf16f07b56c6f6fd3dc92a0957efc0d = L.circle(
[12.504298785861298, 77.87146158880314],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_4b75e906a045a46964343af78f5d0829 = L.circle(
[12.660065857240864, 77.22545121492914],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_dc1305c877ce55a877e1577d4eb6c1ec = L.circle(
[12.660065857240864, 77.41002560746458],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_c6273fc19f0d6c7c53e11797678a26d7 = L.circle(
[12.660065857240864, 77.5946],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_72c14238321f382b75d7e7252e099771 = L.circle(
[12.660065857240864, 77.77917439253542],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_87d8d6d7da9e3dadc2f449b41d2c3283 = L.circle(
[12.660065857240864, 77.96374878507086],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_73b1316540dab26935b6628bc29e8373 = L.circle(
[12.815832928620432, 77.13316401866143],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_abbf0ccd94dc3585f29e6097cf31816c = L.circle(
[12.815832928620432, 77.31773841119686],
{"bubblingMouseEvents": true, "color": "red", "dashArray": null, "dashOffset": null, "fill": true, "fillColor": "red", "fillOpacity": 0.2, "fillRule": "evenodd", "lineCap": "round", "lineJoin": "round", "opacity": 1.0, "radius": 10000, "stroke": true, "weight": 3}
).addTo(map_0d8b86cbc8c3d9e67b61e4080646ece2);

var circle_123feae420dca7db7f16836de

Files changed (3) hide show

backend/utils.py +43 -0
frontend/src/components/Map.js +17 -12
main.py +82 -29

backend/utils.py ADDED Viewed

	@@ -0,0 +1,43 @@

+import math
+def generate_circle_centers(center_lat, center_lon, radius_km, small_radius_km=10):
+    """
+        Generate a list of centers of small circles (radius=10km) needed to cover a larger circle.
+        Circles are arranged in hexagonal pattern to minimize # of small circles.
+        No overlapping among small circles, but some small circles may be outside the larger circle.
+        Input:
+        - center_lat: Latitude of the center of the larger circle
+        - center_lon: Longitude of the center of the larger circle
+        - radius_km: Radius of the larger circle in kilometers
+        - small_radius_km: Radius of the smaller circles in kilometers (default 15km)
+        Output:
+        - A list of tuples, each containing the latitude and longitude of a small circle's center. [(lat1, lon1), (lat2, lon2),...]
+    """
+    R = 6371  # Earth radius
+    dx = 2 * small_radius_km
+    dy = math.sqrt(3) * small_radius_km
+    max_dist = radius_km + small_radius_km
+    results = []
+    lat_rad = math.radians(center_lat)
+    n_y = int(max_dist // dy) + 2
+    for row in range(-n_y, n_y + 1):
+        y = row * dy
+        offset = 0 if row % 2 == 0 else dx / 2
+        n_x = int((max_dist + dx) // dx) + 2
+        for col in range(-n_x, n_x + 1):
+            x = col * dx + offset
+            distance = math.sqrt(x ** 2 + y ** 2)
+            if distance <= max_dist:
+                delta_lat = (y / R) * (180 / math.pi)
+                delta_lon = (x / (R * math.cos(lat_rad))) * (180 / math.pi)
+                lat = center_lat + delta_lat
+                lon = center_lon + delta_lon
+                results.append((lat, lon))
+    return results

frontend/src/components/Map.js CHANGED Viewed

@@ -26,6 +26,8 @@ L.Icon.Default.mergeOptions({
 // Add scale
 // L.control.scale().addTo(window.Map);
 const ClickHandler = ({ onClick }) => {
     useMapEvents({
       click(e) {
@@ -83,7 +85,7 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
     const [countryBorders, setCountryBorders] = useState(null);
     const [explorationMode, setExplorationMode] = useState(false);
-    const [explorationRadius, setExplorationRadius] = useState(10000);
     const [explorationLimit, setExplorationLimit] = useState(10);
     const [explorationMarkers, setExplorationMarkers] = useState([]);
     const [explorationSidebarOpen, setExplorationSidebarOpen] = useState(false);
@@ -238,14 +240,16 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
                     body: JSON.stringify({
                         lat: lat,
                         lon: lon,
-                        radius: explorationRadius,
                         limit: explorationLimit
                     }),
                 });
                 if (res.ok) {
                     const data = await res.json();
-                    const markers = data.pages.map(page => ({
                         position: [page.lat, page.lon],
                         title: page.title,
                         distance: page.dist
@@ -963,8 +967,9 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
                             <button
                                 onClick={() => {
                                     setExplorationSidebarOpen(false);
-                                    setExplorationMode(false);
-                                    setExplorationMarkers([]);
                                 }}
                                 style={{
                                     background: 'none',
@@ -979,12 +984,12 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
                         <div>
                             <label style={{ fontWeight: 500, marginBottom: 8, display: 'block' }}>
-                                Search Radius (meters):
                             </label>
                             <input
                                 type="range"
-                                min="10"
-                                max="10000"
                                 value={explorationRadius}
                                 onChange={(e) => setExplorationRadius(parseInt(e.target.value))}
                                 style={{ width: '100%' }}
@@ -998,12 +1003,12 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
                             }}>
                                 <input
                                     type="number"
-                                    min="10"
-                                    max="10000"
                                     value={explorationRadius}
                                     onChange={(e) => {
                                         const value = parseInt(e.target.value);
-                                        if (value >= 10 && value <= 10000) {
                                             setExplorationRadius(value);
                                         }
                                     }}
@@ -1015,7 +1020,7 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
                                         textAlign: 'center'
                                     }}
                                 />
-                                <span>m</span>
                             </div>
                         </div>

 // Add scale
 // L.control.scale().addTo(window.Map);
+const maxExplorationLimit = 50; // kilometers, the maximum amount user can select to explore.
 const ClickHandler = ({ onClick }) => {
     useMapEvents({
       click(e) {
     const [countryBorders, setCountryBorders] = useState(null);
     const [explorationMode, setExplorationMode] = useState(false);
+    const [explorationRadius, setExplorationRadius] = useState(10);
     const [explorationLimit, setExplorationLimit] = useState(10);
     const [explorationMarkers, setExplorationMarkers] = useState([]);
     const [explorationSidebarOpen, setExplorationSidebarOpen] = useState(false);
                     body: JSON.stringify({
                         lat: lat,
                         lon: lon,
+                        radius: explorationRadius*1000,
                         limit: explorationLimit
                     }),
                 });
                 if (res.ok) {
                     const data = await res.json();
+                    const markers = data.pages.filter(
+                                page => typeof page.dist === "number" && page.dist <= explorationRadius * 1000
+                            ).map(page => ({
                         position: [page.lat, page.lon],
                         title: page.title,
                         distance: page.dist
                             <button
                                 onClick={() => {
                                     setExplorationSidebarOpen(false);
+                                    // setExplorationMode(false); // even with exp. sidebar closed, you can do exploration normally.
+                                    // setExplorationMarkers([]);
+                                    setShouldZoom(false); // If this line is removed, it map automatically zooms after re-opening/closing the exp. sidebar
                                 }}
                                 style={{
                                     background: 'none',
                         <div>
                             <label style={{ fontWeight: 500, marginBottom: 8, display: 'block' }}>
+                                Search Radius (km):
                             </label>
                             <input
                                 type="range"
+                                min="1"
+                                max={maxExplorationLimit}
                                 value={explorationRadius}
                                 onChange={(e) => setExplorationRadius(parseInt(e.target.value))}
                                 style={{ width: '100%' }}
                             }}>
                                 <input
                                     type="number"
+                                    min="1"
+                                    max={maxExplorationLimit}
                                     value={explorationRadius}
                                     onChange={(e) => {
                                         const value = parseInt(e.target.value);
+                                        if (value >= 1 && value <= maxExplorationLimit) {
                                             setExplorationRadius(value);
                                         }
                                     }}
                                         textAlign: 'center'
                                     }}
                                 />
+                                <span>km</span>
                             </div>
                         </div>

main.py CHANGED Viewed

@@ -2,12 +2,13 @@ from fastapi import FastAPI, BackgroundTasks
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
-import requests
 from geopy.geocoders import Nominatim
 import geopy.distance
 from cachetools import TTLCache
 import os
 from dotenv import load_dotenv
 load_dotenv()
@@ -25,7 +26,7 @@ class Geodistance(BaseModel):
 class NearbyWikiPage(BaseModel):
     lat: float = Field(default=54.163337, ge=-90, le=90)
     lon: float = Field(default=37.561109, ge=-180, le=180)
-    radius: int = Field(default=1000, ge=10, le=10000,description="Distance in meters from the reference point")
     limit: int = Field(10, ge=1, description="Number of pages to return")
 app.add_middleware(
@@ -173,6 +174,18 @@ def get_geodistance(payload: Geodistance):
         status_code=200
     )
 @app.post("/wiki/nearby")
 async def get_nearby_wiki_pages(payload: NearbyWikiPage):
     """
@@ -198,42 +211,82 @@ async def get_nearby_wiki_pages(payload: NearbyWikiPage):
             ],
             "count": 10 #Total no. of such pages
         }
     """
-    lat, lon = payload.lat, payload.lon
     radius = payload.radius
     limit = payload.limit
-    url = ("https://en.wikipedia.org/w/api.php"+"?action=query"
-            "&list=geosearch"
-            f"&gscoord={lat}|{lon}"
-            f"&gsradius={radius}"
-            f"&gslimit={limit}"
-            "&format=json")
-    print(url)
-    try:
-        response = requests.get(url, timeout=10)
-        if response.status_code != 200:
             return JSONResponse(
-                content={"error": "Failed to fetch nearby pages"},
                 status_code=500
             )
-        data = response.json()
-        pages = data.get("query", {}).get("geosearch", [])
-        return JSONResponse(
-            content={
-                "pages": pages,
-                "count": len(pages)
-            },
-            status_code=200
-        )
-    except Exception as e:
-        return JSONResponse(
-            content={"error": str(e)},
-            status_code=500
-        )

 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
+import requests, httpx, asyncio
 from geopy.geocoders import Nominatim
 import geopy.distance
 from cachetools import TTLCache
 import os
 from dotenv import load_dotenv
+from backend.utils import generate_circle_centers
 load_dotenv()
 class NearbyWikiPage(BaseModel):
     lat: float = Field(default=54.163337, ge=-90, le=90)
     lon: float = Field(default=37.561109, ge=-180, le=180)
+    radius: int = Field(default=1000, ge=10, le=100_000,description="Distance in meters from the reference point")
     limit: int = Field(10, ge=1, description="Number of pages to return")
 app.add_middleware(
         status_code=200
     )
+async def fetch_url(client: httpx.AsyncClient, url: str):
+    try:
+        response = await client.get(url, timeout=10.0)
+        return {
+            "url": url,
+            "status": response.status_code,
+            "data": response.json() if response.status_code == 200 else None,
+        }
+    except Exception as e:
+        return {"url": url, "error": str(e)}
 @app.post("/wiki/nearby")
 async def get_nearby_wiki_pages(payload: NearbyWikiPage):
     """
             ],
             "count": 10 #Total no. of such pages
         }
+    Example raw respone from Wikipedia API: https://en.wikipedia.org/w/api.php?action=query&list=geosearch&gscoord=40.7128%7C-74.0060&gsradius=10000&gslimit=1&format=json
     """
+    lat_center, lon_center = payload.lat, payload.lon
     radius = payload.radius
     limit = payload.limit
+    if radius <= 10000:
+        url = ("https://en.wikipedia.org/w/api.php"+"?action=query"
+                "&list=geosearch"
+                f"&gscoord={lat_center}|{lon_center}"
+                f"&gsradius={radius}"
+                f"&gslimit={limit}"
+                "&format=json")
+        # print(url)
+        try:
+            response = requests.get(url, timeout=10)
+            if response.status_code != 200:
+                return JSONResponse(
+                    content={"error": "Failed to fetch nearby pages"},
+                    status_code=500
+                )
+            data = response.json()
+            pages = data.get("query", {}).get("geosearch", [])
+            return JSONResponse(
+                content={
+                    "pages": pages,
+                    "count": len(pages)
+                },
+                status_code=200
+            )
+        except Exception as e:
             return JSONResponse(
+                content={"error": str(e)},
                 status_code=500
             )
+    elif radius > 10000:
+        small_circle_centers = generate_circle_centers(lat_center, lon_center, radius / 1000, small_radius_km=10)
+        all_pages = []
+        base_url = "https://en.wikipedia.org/w/api.php?action=query&list=geosearch&gscoord={lat}|{lon}&gsradius={small_radius_km}&gslimit={page_limit}&format=json"
+        urls = [base_url.format(lat=center[0], lon=center[1], small_radius_km=10*1000, page_limit=100) for center in small_circle_centers]
+        print("URL Counts:", len(urls))
+        try:
+            async with httpx.AsyncClient() as client:
+                tasks = [fetch_url(client, url) for url in urls]
+                results = await asyncio.gather(*tasks)
+            # print(results)
+            for result in results:
+                for unit in result.get("data", {}).get("query", {}).get("geosearch", []):
+                    lat, lon = unit.get("lat"), unit.get("lon")
+                    if lat is not None and lon is not None:
+                        dist = int(geopy.distance.distance(
+                                (lat_center, lon_center), (lat, lon)
+                            ).m)
+                        print(dist)
+                    else:
+                        dist = None
+                    unit_with_dist = {**unit, "dist": dist}
+                    all_pages.append(unit_with_dist)
+            return JSONResponse(
+                content={
+                    "pages": all_pages,
+                    "count": len(all_pages)
+                }
+            )
+        except Exception as e:
+            return JSONResponse(
+                content={"error": str(e)},
+                status_code=500
+        )