use URL hashes

app.R

@@ -13,15 +13,16 @@ library(overture)
 source("utils.R")
 source("inat-ranges.R")
 
+duckdb_config(threads = 24) # I/O limited so overclock threads
 
 ui <- page_sidebar(
   shinybusy::add_busy_spinner(),
 
   title = "iNaturalist Species Ranges",
   sidebar = sidebar(
-    textInput("location", "Location", "California"),
-    varSelectInput("rank", NULL, taxa, selected = "scientificName"),
-    textInput("taxon", NULL, "Canis lupus"),
+    textInput("location", "Location", "United States"),
+    varSelectInput("rank", NULL, taxa, selected = "class"),
+    textInput("taxon", NULL, "Aves"),
     actionButton("button", "Go")
   ),
   card(
@@ -32,24 +33,16 @@ ui <- page_sidebar(
 
 server <- function(input, output, session) {
   output$map <- renderMaplibre({
-    if (file.exists(cache)) {
-      meta <- jsonlite::read_json(cache)
-      print(meta$url)
-    } else {
-      aoi <- get_division(input$location)
-      meta <- richness(inat, aoi, rank = input$rank, taxon = input$taxon)
-    }
-    m <- richness_map(meta)
+    aoi <- get_division(input$location)
+    url <- richness(inat, aoi, rank = input$rank, taxon = input$taxon)
+    m <- richness_map(url, aoi)
     m
   })
 
   observeEvent(input$button, {
     print(input$location)
     aoi <- get_division(input$location)
-    meta <- richness(inat, aoi, rank = input$rank, taxon = input$taxon)
-
-    jsonlite::write_json(meta, cache, auto_unbox = TRUE)
-    message(paste("rendering", meta$url))
+    richness(inat, aoi, rank = input$rank, taxon = input$taxon)
 
     session$reload()
   })

inat-ranges.R

@@ -23,8 +23,9 @@ taxa <- duckdbfs::open_dataset(
   recursive = FALSE
 )
 
-cache <- tempfile(fileext = ".json")
-
+get_hash <- function(aoi, rank, taxon) {
+  digest::digest(list(aoi, rank, taxon))
+}
 
 # Also requires get_h3_aoi() from utils.R
 richness <- function(inat, aoi, rank = NULL, taxon = NULL, zoom = 3) {
@@ -32,12 +33,25 @@ richness <- function(inat, aoi, rank = NULL, taxon = NULL, zoom = 3) {
     "AWS_PUBLIC_ENDPOINT",
     Sys.getenv("AWS_S3_ENDPOINT")
   )
-  hash <- digest::digest(list(aoi, rank, taxon))
+  hash <- get_hash(aoi, rank, taxon)
   s3 <- paste0("s3://public-data/cache/inat/", hash, ".h3j")
+  url <- gsub("s3://", glue("https://{public_endpoint}/"), s3)
 
   # check if hash exists
+  cache_hit <- tryCatch(
+    {
+      duckdbfs::open_dataset(s3)
+      TRUE
+    },
+    error = function(e) FALSE,
+    finally = FALSE
+  )
+
+  if (cache_hit) {
+    return(url)
+  }
 
-  # filter
+  # Subset by taxon, if requested
   if (!is.null(rank) && !is.null(taxon)) {
     taxa <- open_dataset(
       glue("s3://public-inat/taxonomy/taxa.parquet"),
@@ -50,41 +64,33 @@ richness <- function(inat, aoi, rank = NULL, taxon = NULL, zoom = 3) {
       inner_join(inat, by = "taxon_id")
   }
 
-  h3_aoi <- get_h3_aoi(aoi, precision = 4) |> select(h3id)
+  inat <- inat |> rename(h3id = h4)
 
-  clock <- bench::bench_time({
-    inat |>
-      rename(h3id = h4) |>
-      inner_join(h3_aoi, by = "h3id") |>
-      distinct(taxon_id, h3id) |>
-      group_by(h3id) |>
-      summarise(n = n()) |>
-      mutate(height = n / max(n)) |>
-      duckdbfs::to_h3j(s3)
-    #  write_dataset("s3://public-data/inat-tmp-ranges.parquet")
-  })
+  # Subset by area, if requested
+  if (nrow(aoi) > 0) {
+    inat <-
+      get_h3_aoi(aoi, precision = 4) |>
+      select(h3id) |>
+      inner_join(inat, by = "h3id")
+  }
 
-  center <- c(st_coordinates(st_centroid(st_as_sfc(st_bbox(aoi)))))
-  bounds <- as.vector(st_bbox(aoi))
+  # Aggregate to h4 hex
+  inat |>
+    distinct(taxon_id, h3id) |>
+    group_by(h3id) |>
+    summarise(n = n()) |>
+    mutate(height = n / max(n)) |>
+    duckdbfs::to_h3j(s3)
 
-  url <- gsub("s3://", glue("https://{public_endpoint}/"), s3)
-
-  meta <- list(
-    X = center[1],
-    Y = center[2],
-    zoom = zoom,
-    url = url,
-    time = clock[[2]],
-    bounds = bounds
-  )
-  return(meta)
+  return(url)
 }
 
-richness_map <- function(meta) {
+richness_map <- function(url, gdf) {
+  bounds <- as.vector(sf::st_bbox(gdf))
   m <-
     maplibre() |>
     add_draw_control() |>
-    add_h3j_source("h3j_source", url = meta$url) |>
+    add_h3j_source("h3j_source", url = url) |>
     add_fill_extrusion_layer(
       id = "h3j_layer",
       source = "h3j_source",
@@ -101,7 +107,7 @@ richness_map <- function(meta) {
       ),
       fill_extrusion_opacity = 0.7
     ) |>
-    fit_bounds(meta$bounds, animate = TRUE)
+    fit_bounds(bounds, animate = TRUE)
 
   return(m)
 }

test.R

@@ -1,4 +1,3 @@
-
 source("utils.R")
 source("inat-ranges.R")
 
@@ -9,26 +8,25 @@ url <- "https://minio.carlboettiger.info/public-data/cache/inat/f0108ef86feababf
 
 #x <- jsonlite::read_json(url)
 
- m = maplibre(center = c(-110,37), zoom=3) |>
-    add_draw_control() |>
-    add_h3j_source("h3j_source",
-                  url = url
-    ) |>
-    add_fill_extrusion_layer(
-      id = "h3j_layer",
-      source = "h3j_source",
-      tooltip = "n",
-      fill_extrusion_color = viridis_pal("height"),
-      fill_extrusion_height = list(
-        "interpolate",
-        list("linear"),
-        list("zoom"),
-        0,
-        0, 1,
-        list("*", 100000, list("get", "height"))
-      ),
-      fill_extrusion_opacity = 0.7
-    )
+m = maplibre(center = c(-110, 37), zoom = 3) |>
+  add_draw_control() |>
+  add_h3j_source("h3j_source", url = url) |>
+  add_fill_extrusion_layer(
+    id = "h3j_layer",
+    source = "h3j_source",
+    tooltip = "n",
+    fill_extrusion_color = viridis_pal("height"),
+    fill_extrusion_height = list(
+      "interpolate",
+      list("linear"),
+      list("zoom"),
+      0,
+      0,
+      1,
+      list("*", 100000, list("get", "height"))
+    ),
+    fill_extrusion_opacity = 0.7
+  )
 
 
 htmlwidgets::saveWidget(m, "test2.html")
@@ -81,54 +79,44 @@ htmlwidgets::saveWidget(m, "mammals-richness.html")
 
 # mutate(h3 = h3_cell_to_parent(h4, 3L))
 
+m <- maplibre(center = c(-110.5, 34.8), zoom = 4) |> add_draw_control()
+richness_map(
+  m,
+  "https://minio.carlboettiger.info/public-data/inat-tmp-ranges.h3j"
+)
 
 
-m <- maplibre(center = c(-110.5, 34.8), zoom = 4) |> add_draw_control() 
-richness_map(m, "https://minio.carlboettiger.info/public-data/inat-tmp-ranges.h3j")
-
-
-
-
-
-
-
-
-
+install.packages(
+  'spDataLarge',
+  repos = 'https://nowosad.github.io/drat/',
+  type = 'source'
+)
 
 
 library(htmlwidgets)
 htmlwidgets::saveWidget(m, "example.html")
 
 
+amphib = open_dataset(
+  "s3://public-inat/polygon/Amphibia.parquet",
+  recursive = FALSE
+)
 
-
-
-
-
-
-
-
-
-
-amphib = open_dataset("s3://public-inat/polygon/Amphibia.parquet", recursive = FALSE)
-
-gdf <- amphib |> 
-filter(name == "Ambystoma californiense") |>
- to_sf(crs=4326)
+gdf <- amphib |>
+  filter(name == "Ambystoma californiense") |>
+  to_sf(crs = 4326)
 
 maplibre(center = c(-122.5, 37.8), zoom = 4) |>
-      add_source(id = "gdf", gdf) |>
-      add_layer("gdf-layer",
-                type = "fill",
-                source = "gdf",
-                paint = list(
-                  "fill-color" =  "darkgreen",
-                  "fill-opacity" = .9
-                )
-        )
-      
-
-
+  add_source(id = "gdf", gdf) |>
+  add_layer(
+    "gdf-layer",
+    type = "fill",
+    source = "gdf",
+    paint = list(
+      "fill-color" = "darkgreen",
+      "fill-opacity" = .9
+    )
+  )
 
 
 # Access SVI
@@ -144,30 +132,29 @@ ca <- tracts |>
   mutate(h4 = tolower(as.character(h4)))
 
 out <- ca |>
- inner_join(inat, by = "h4") |>
- count(STATE, COUNTY, FIPS, h5)
+  inner_join(inat, by = "h4") |>
+  count(STATE, COUNTY, FIPS, h5)
 
 
 #  mutate(height = n / max(n)) |>
 
 url = "https://minio.carlboettiger.info/public-data/cache/inat/cec4b3087f0b6c41ecc384da2521f97c.h3j"
- maplibre() |>
-    add_draw_control() |>
-    add_h3j_source("h3j_source",
-                  url = url
-    ) |>
-    add_fill_extrusion_layer(
-      id = "h3j_layer",
-      source = "h3j_source",
-      tooltip = "n",
-      fill_extrusion_color = viridis_pal("height"),
-      fill_extrusion_height = list(
-        "interpolate",
-        list("linear"),
-        list("zoom"),
-        0,
-        0, 1,
-        list("*", 100000, list("get", "height"))
-      ),
-      fill_extrusion_opacity = 0.7
-    )
+maplibre() |>
+  add_draw_control() |>
+  add_h3j_source("h3j_source", url = url) |>
+  add_fill_extrusion_layer(
+    id = "h3j_layer",
+    source = "h3j_source",
+    tooltip = "n",
+    fill_extrusion_color = viridis_pal("height"),
+    fill_extrusion_height = list(
+      "interpolate",
+      list("linear"),
+      list("zoom"),
+      0,
+      0,
+      1,
+      list("*", 100000, list("get", "height"))
+    ),
+    fill_extrusion_opacity = 0.7
+  )

utils.R

@@ -1,4 +1,3 @@
-
 library(dplyr)
 library(duckdbfs)
 library(sf)
@@ -6,7 +5,9 @@ library(sf)
 as_dataset.sf <- function(sf, ...) {
   # cludgy way to get polygon into duckdb as spatial data
   tmp <- tempfile(fileext = ".fgb")
-  sf |> sf::st_transform(4326) |> sf::write_sf(tmp, append = FALSE, quiet = TRUE)
+  sf |>
+    sf::st_transform(4326) |>
+    sf::write_sf(tmp, append = FALSE, quiet = TRUE)
   aoi <- duckdbfs::open_dataset(tmp, ...)
 
   aoi
@@ -14,23 +15,20 @@ as_dataset.sf <- function(sf, ...) {
 
 # promote bbox to polygon
 as_poly <- function(aoi) {
-
   crs <- st_crs(aoi)
-  if (crs$input != "EPSG:4326" ) {
+  if (crs$input != "EPSG:4326") {
     aoi <- aoi |> st_transform(4326)
-
-  } 
+  }
   if (inherits(aoi, "bbox")) {
-    aoi <- aoi |> 
-      st_as_sfc() |> 
-      st_as_sf() |> 
+    aoi <- aoi |>
+      st_as_sfc() |>
+      st_as_sf() |>
       rename(geom = x)
   }
-aoi
+  aoi
 }
 
 
-
 get_h3_aoi <- function(aoi, zoom = 0L, precision = 6L, upper = FALSE) {
   duckdbfs::load_h3()
 
@@ -39,23 +37,30 @@ get_h3_aoi <- function(aoi, zoom = 0L, precision = 6L, upper = FALSE) {
   precision <- as.integer(precision)
   res <- paste0("h", precision)
 
-  if(inherits(aoi, "sf") || inherits(aoi, "bbox")) {
+  if (inherits(aoi, "sf") || inherits(aoi, "bbox")) {
     aoi <- as_poly(aoi)
     aoi <- as_dataset.sf(aoi)
   }
 
   # multipolygon dump may not be needed for draw tools.
   h3_aoi <- aoi |>
-    dplyr::mutate(poly = array_extract(unnest(st_dump(geom)),"geom"),
-          h3id = h3_polygon_wkt_to_cells(poly,{precision}),
-          h3id = unnest(h3id)
-          ) |>
-    dplyr::mutate(h0 = h3_h3_to_string( h3_cell_to_parent(h3id, {zoom})),
-           h3id = h3_h3_to_string (h3id) )
-
-  if(upper) {
-    h3_aoi <- h3_aoi |> 
-    dplyr::mutate(h0 = toupper(h0), h3id = toupper(h3id))
+    dplyr::mutate(
+      poly = array_extract(unnest(st_dump(geom)), "geom"),
+      h3id = h3_polygon_wkt_to_cells(poly, {
+        precision
+      }),
+      h3id = unnest(h3id)
+    ) |>
+    dplyr::mutate(
+      h0 = h3_h3_to_string(h3_cell_to_parent(h3id, {
+        zoom
+      })),
+      h3id = h3_h3_to_string(h3id)
+    )
+
+  if (upper) {
+    h3_aoi <- h3_aoi |>
+      dplyr::mutate(h0 = toupper(h0), h3id = toupper(h3id))
   }
 
   h3_aoi |>
@@ -64,68 +69,77 @@ get_h3_aoi <- function(aoi, zoom = 0L, precision = 6L, upper = FALSE) {
 }
 
 hex_res <- function(x) {
-    x |>
+  x |>
     utils::head(1) |>
     dplyr::mutate(res = h3_get_resolution(h3id)) |>
     dplyr::pull(res)
 }
 
-hex_join <- function(x,y) {
+hex_join <- function(x, y) {
   res_x <- hex_res(x)
   res_y <- hex_res(y)
 
   if (res_x > res_y) {
-    y <- y |> 
-      dplyr::mutate(h3id = unnest(h3_cell_to_children(h3id, {res_x})),
-                    h3id = toupper(h3id))
+    y <- y |>
+      dplyr::mutate(
+        h3id = unnest(h3_cell_to_children(h3id, {
+          res_x
+        })),
+        h3id = toupper(h3id)
+      )
   }
-    if (res_x < res_y) {
-    y <- y |> 
-      dplyr::mutate(h3id = h3_cell_to_parent(h3id, {res_x}))
+  if (res_x < res_y) {
+    y <- y |>
+      dplyr::mutate(
+        h3id = h3_cell_to_parent(h3id, {
+          res_x
+        })
+      )
   }
 
   dplyr::inner_join(x, y)
 }
 
-
 antimeridian_hexes <- function(zoom = 4, con = duckdbfs::cached_connection()) {
   duckdbfs::load_h3(con)
   duckdbfs::load_spatial(con)
-  DBI::dbExecute(con, 
-  "
+  DBI::dbExecute(
+    con,
+    "
   CREATE OR REPLACE TABLE antimeridian AS (
     SELECT ST_GeomFromText(
     'POLYGON ((170 85, 190 85, 190 -85, 170 -85, 170 85))'
     ) AS geometry
   )
-  ")
+  "
+  )
   zoom <- as.integer(zoom)
-  
-  am <- 
+
+  am <-
     dplyr::tbl(con, "antimeridian") |>
-    dplyr::mutate(h3id = unnest(
-      h3_polygon_wkt_to_cells_string(geometry, {zoom})
-      )) |>
+    dplyr::mutate(
+      h3id = unnest(
+        h3_polygon_wkt_to_cells_string(geometry, {
+          zoom
+        })
+      )
+    ) |>
     dplyr::select(h3id)
 
-am
+  am
 }
 
 
-viridis_pal <- 
-  function(column = "height",
-           n = 61, 
-           min_v = 0, 
-           max_v = 1) {
-  pal <- viridisLite::viridis(n)
-  fill_color = mapgl::step_expr(
+viridis_pal <-
+  function(column = "height", n = 61, min_v = 0, max_v = 1) {
+    pal <- viridisLite::viridis(n)
+    fill_color = mapgl::step_expr(
       column = column,
       base = pal[1],
       stops = pal[2:n],
-      values = seq(min_v, max_v, length.out = n-1),
+      values = seq(min_v, max_v, length.out = n - 1),
       na_color = "white"
     )
-  
-  fill_color
-}
 
+    fill_color
+  }