Merge pull request #24 from NewGraphEnvironment/7-stac-s3

Add STAC catalog generation and S3 push (#7, #8)

Author: NewGraphEnvironment

NAMESPACE

@@ -14,6 +14,8 @@ export(cd_derive)
 export(cd_extract)
 export(cd_fetch)
 export(cd_periods)
+export(cd_s3_push)
+export(cd_stac_catalog)
 export(cd_summary)
 export(cd_trend)
 export(cd_variables)

R/cd_catalog.R

@@ -51,5 +51,5 @@ cd_catalog <- function(catalog = cd_catalog_default()) {
 #'
 #' @export
 cd_catalog_default <- function() {
-  getOption("cd.catalog_url", default = "https://nge-bc-ce.s3.us-west-2.amazonaws.com/cd/catalog.json")
+  getOption("cd.catalog_url", default = "https://stac-era5-land.s3.us-west-2.amazonaws.com/catalog.json")
 }

R/cd_s3_push.R

@@ -0,0 +1,58 @@
+#' Push files to S3
+#'
+#' Syncs a local directory to an S3 bucket using `aws s3 sync`.
+#' Only uploads new or changed files (`--size-only`). Requires
+#' the AWS CLI to be installed and configured.
+#'
+#' @param local_dir Character. Local directory to sync.
+#' @param bucket Character. S3 bucket name. Default `"stac-era5-land"`.
+#' @param prefix Character. S3 key prefix (subdirectory in bucket).
+#'   Default `""` (bucket root).
+#' @param dry_run Logical. If `TRUE`, shows what would be uploaded
+#'   without actually uploading. Default `FALSE`.
+#'
+#' @return The exit code from `aws s3 sync` (invisibly). Zero on success.
+#'
+#' @examples
+#' \dontrun{
+#' # Preview what would be uploaded
+#' cd_s3_push("data/cogs", dry_run = TRUE)
+#'
+#' # Upload for real
+#' cd_s3_push("data/cogs")
+#'
+#' # Upload to a subdirectory in the bucket
+#' cd_s3_push("data/cogs", prefix = "v1")
+#' }
+#'
+#' @export
+cd_s3_push <- function(local_dir,
+                       bucket = "stac-era5-land",
+                       prefix = "",
+                       dry_run = FALSE) {
+  if (!dir.exists(local_dir)) {
+    rlang::abort(paste("Directory not found:", local_dir))
+  }
+
+  s3_target <- if (nchar(prefix) > 0) {
+    paste0("s3://", bucket, "/", prefix)
+  } else {
+    paste0("s3://", bucket)
+  }
+
+  cmd <- sprintf(
+    "aws s3 sync %s %s --exclude '.*' --size-only%s",
+    shQuote(local_dir),
+    shQuote(s3_target),
+    if (dry_run) " --dryrun" else ""
+  )
+
+  message("Running: ", cmd)
+  exit_code <- system(cmd)
+
+  if (exit_code != 0 && !dry_run) {
+    rlang::abort(paste("S3 sync failed with exit code", exit_code))
+  }
+
+  invisible(exit_code)
+}

R/cd_stac_catalog.R

@@ -0,0 +1,123 @@
+#' Generate a static STAC catalog from COGs
+#'
+#' Scans a directory for Cloud-Optimized GeoTIFFs and builds a
+#' STAC catalog JSON file. Each COG becomes one item with
+#' `cd:variable` and `cd:period` properties parsed from the filename.
+#' The resulting catalog is compatible with [cd_catalog()].
+#'
+#' @param cog_dir Character. Directory containing COG files (.tif).
+#' @param output_path Character. Path to write the catalog JSON.
+#'   Default `"catalog.json"`.
+#' @param catalog_id Character. STAC catalog ID.
+#'   Default `"era5-land"`.
+#' @param title Character. Human-readable catalog title.
+#' @param description Character. Optional catalog description.
+#' @param base_url Character. Base URL where COGs will be served.
+#'   Asset hrefs are built as `{base_url}/{filename}`.
+#'
+#' @return The output path (invisibly).
+#'
+#' @examples
+#' \dontrun{
+#' cd_stac_catalog(
+#'   "data/cogs",
+#'   output_path = "data/catalog.json",
+#'   base_url = "https://stac-era5-land.s3.us-west-2.amazonaws.com"
+#' )
+#' }
+#'
+#' @export
+cd_stac_catalog <- function(cog_dir,
+                            output_path = "catalog.json",
+                            catalog_id = "era5-land",
+                            title = "ERA5-Land Climate Data",
+                            description = NULL,
+                            base_url = "https://stac-era5-land.s3.us-west-2.amazonaws.com") {
+
+  tif_files <- list.files(cog_dir, pattern = "\\.tif$", full.names = TRUE)
+  if (length(tif_files) == 0) {
+    rlang::abort(paste("No .tif files found in", cog_dir))
+  }
+
+  items <- lapply(tif_files, function(f) {
+    cd_stac_item(f, base_url)
+  })
+
+  catalog <- list(
+    type = "Catalog",
+    id = catalog_id,
+    stac_version = "1.0.0",
+    description = description %||% paste(title, "- static STAC catalog"),
+    links = list(
+      list(rel = "root", href = paste0("./", basename(output_path)),
+           type = "application/json")
+    ),
+    items = items
+  )
+
+  dir.create(dirname(output_path), recursive = TRUE, showWarnings = FALSE)
+  jsonlite::write_json(catalog, output_path, pretty = TRUE, auto_unbox = TRUE)
+  message("Wrote STAC catalog: ", output_path, " (", length(items), " items)")
+  invisible(output_path)
+}
+
+#' Build a STAC item from a COG file
+#'
+#' Parses variable and period from the filename, extracts spatial
+#' metadata from the raster.
+#'
+#' @param cog_path Path to a COG file.
+#' @param base_url Base URL for asset hrefs.
+#' @return A list representing a STAC Feature item.
+#' @noRd
+cd_stac_item <- function(cog_path, base_url) {
+  fname <- basename(cog_path)
+  name_parts <- tools::file_path_sans_ext(fname)
+
+  # Parse variable and period from filename
+  # Expected patterns: "tmean_annual.tif", "vpd_2024.tif",
+  # "example_climate.tif", etc.
+  known_vars <- cd_variables()$variable
+  known_periods <- cd_periods(include_monthly = TRUE)
+
+  var_match <- known_vars[vapply(known_vars, function(v) grepl(v, name_parts), logical(1))]
+  period_match <- known_periods[vapply(known_periods, function(p) grepl(p, name_parts), logical(1))]
+
+  variable <- if (length(var_match) > 0) var_match[1] else name_parts
+  period <- if (length(period_match) > 0) period_match[1] else "unknown"
+
+  # Extract spatial metadata
+  r <- terra::rast(cog_path)
+  e <- as.vector(terra::ext(r))
+  n_bands <- terra::nlyr(r)
+  band_names <- names(r)
+
+  # Parse years from band names if numeric
+  years <- suppressWarnings(as.integer(band_names))
+  years <- years[!is.na(years)]
+
+  item_id <- paste(variable, period, sep = "-")
+
+  list(
+    type = "Feature",
+    stac_version = "1.0.0",
+    id = item_id,
+    geometry = NA,
+    bbox = e[c(1, 3, 2, 4)],
+    properties = list(
+      `cd:variable` = variable,
+      `cd:period` = period,
+      datetime = NA,
+      start_datetime = if (length(years) > 0) paste0(min(years), "-01-01T00:00:00Z") else NULL,
+      end_datetime = if (length(years) > 0) paste0(max(years), "-12-31T23:59:59Z") else NULL
+    ),
+    links = list(),
+    assets = list(
+      data = list(
+        href = paste0(base_url, "/", fname),
+        type = "image/tiff; application=geotiff; profile=cloud-optimized",
+        title = paste(variable, period)
+      )
+    )
+  )
+}

inst/extdata/example_catalog.json

@@ -20,14 +20,18 @@
       "type": "Feature",
       "stac_version": "1.0.0",
       "id": "tmean-annual",
-      "geometry": {},
-      "bbox": [-126.75, 54.1, -125.75, 54.7],
+      "bbox": [
+        -126.75,
+        54.1,
+        -125.75,
+        54.7
+      ],
       "properties": {
         "cd:variable": "tmean",
         "cd:period": "annual",
-        "datetime": {},
         "start_datetime": "1951-01-01T00:00:00Z",
-        "end_datetime": "1960-12-31T23:59:59Z"
+        "end_datetime": "1960-12-31T23:59:59Z",
+        "datetime": null
       },
       "links": [],
       "assets": {
@@ -36,7 +40,8 @@
           "type": "image/tiff; application=geotiff; profile=cloud-optimized",
           "title": "Mean temperature annual values"
         }
-      }
+      },
+      "geometry": null
     }
   ]
 }

man/cd_s3_push.Rd

@@ -0,0 +1,23 @@
+# Progress
+
+## Status: Complete
+
+### cd_stac_catalog() — Issue #7
+- [x] Implementation
+- [x] Tests
+- [x] Docs
+- [x] `/code-check` — fixed geometry/datetime null serialization (NA for JSON null)
+- [x] Committed
+
+### cd_s3_push() — Issue #8
+- [x] Implementation
+- [x] Tests
+- [x] Docs
+- [x] `/code-check` — fixed command injection via shQuote on s3_target
+- [x] Committed
+
+### Final
+- [x] `devtools::test()` all pass (142/142)
+- [x] `lintr::lint_package()` clean (false positives only)
+- [ ] End-to-end with real S3 (after merge)
+- [ ] PR created

man/cd_stac_catalog.Rd

@@ -0,0 +1,18 @@
+# STAC Catalog + S3 Push
+
+## Context
+Final producer functions. Generate static STAC catalog JSON from COGs, push to `stac-era5-land` S3 bucket. Completes the producer pipeline: fetch → derive → cog_write → stac_catalog → s3_push.
+
+## Issues
+- #7 `cd_stac_catalog()` — STAC JSON generation
+- #8 `cd_s3_push()` — S3 upload
+
+## Tasks
+- [ ] Fix `cd_catalog_default()` URL to real bucket
+- [ ] Implement `cd_stac_catalog()` in `R/cd_stac_catalog.R`
+- [ ] Implement `cd_s3_push()` in `R/cd_s3_push.R`
+- [ ] Write tests for each function
+- [ ] End-to-end: generate catalog → read with cd_catalog()
+- [ ] `devtools::test()` — all pass
+- [ ] `lintr::lint_package()` — clean
+- [ ] `/code-check` before each commit

planning/active/progress.md

@@ -0,0 +1,16 @@
+test_that("cd_s3_push is a function", {
+  expect_true(is.function(cd_s3_push))
+})
+
+test_that("cd_s3_push errors on missing directory", {
+  expect_error(
+    cd_s3_push("/nonexistent/path/abc123"),
+    "Directory not found"
+  )
+})
+
+test_that("cd_s3_push default bucket is stac-era5-land", {
+  # Verify the default by inspecting the function formals
+  defaults <- formals(cd_s3_push)
+  expect_equal(defaults$bucket, "stac-era5-land")
+})