feat: tune GDAL COG settings, add STAC 1.0.0 compliance, WebP/JPEG drivers, and bilinear resampling

Author: mwkorver

README.md

@@ -480,7 +480,9 @@ Set in the generated mapfile (`mapfile_generator.py` reads env vars):
 CONFIG "GDAL_CACHEMAX"                      "128"          # MB block cache
 CONFIG "VSI_CACHE"                          "FALSE"        # RAM VSI cache disabled by default (nginx does the heavy lifting)
 CONFIG "VSI_CACHE_SIZE"                     "33554432"     # 32 MB per worker (ignored unless VSI_CACHE is TRUE)
-CONFIG "GDAL_DISABLE_READDIR_ON_OPEN"       "TRUE"         # Avoid costly ListBucket operations on S3
+CONFIG "GDAL_DISABLE_READDIR_ON_OPEN"       "EMPTY_DIR"    # Treat virtual directory as empty to avoid ListBucket on S3
+CONFIG "GDAL_INGESTED_BYTES_AT_OPEN"       "32768"        # 32 KB header fetch on open to read full IFD tree in 1 GET
+CONFIG "CPL_VSIL_CURL_CHUNK_SIZE"           "65536"        # 64 KB chunk size for vsicurl range reads
 CONFIG "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES" "YES"          # Merge consecutive byte reads
 ```
 

cdk/lambda/db_init/handler.py

@@ -46,14 +46,27 @@
     native_epsg   INT  NOT NULL,
     geom          GEOMETRY(Polygon, 3857) NOT NULL,
     geom_native   GEOMETRY NOT NULL,
+    stac_item     JSONB,
     uploaded_at   TIMESTAMPTZ DEFAULT now(),
     UNIQUE (collection_id, location)
 );
 
+ALTER TABLE cog_index ADD COLUMN IF NOT EXISTS stac_item JSONB;
+
 CREATE INDEX IF NOT EXISTS cog_index_collection_idx  ON cog_index(collection_id);
 CREATE INDEX IF NOT EXISTS cog_index_geom_idx        ON cog_index USING GIST(geom);
 CREATE INDEX IF NOT EXISTS cog_index_geom_native_idx ON cog_index USING GIST(geom_native);
 CREATE INDEX IF NOT EXISTS cog_index_file_name_trgm  ON cog_index USING GIN(file_name gin_trgm_ops);
+CREATE INDEX IF NOT EXISTS cog_index_stac_item_gin   ON cog_index USING GIN(stac_item);
+
+CREATE OR REPLACE VIEW stac_items AS
+SELECT
+    id,
+    collection_id AS collection,
+    stac_item,
+    geom AS geometry,
+    location
+FROM cog_index;
 """
 
 # Retry parameters: total wait budget is ~90 s so we stay well inside the

etc/mapfile_generator.py

@@ -26,6 +26,9 @@
 PUBLIC_HOST = os.environ.get("PUBLIC_HOST", "localhost")
 DEBUG_LEVEL = os.environ.get("MS_DEBUGLEVEL", "0")
 GDAL_CACHEMAX = os.environ.get("GDAL_CACHEMAX", "128")
+GDAL_INGESTED_BYTES_AT_OPEN = os.environ.get("GDAL_INGESTED_BYTES_AT_OPEN", "32768")
+CPL_VSIL_CURL_CHUNK_SIZE = os.environ.get("CPL_VSIL_CURL_CHUNK_SIZE", "65536")
+GDAL_DISABLE_READDIR_ON_OPEN = os.environ.get("GDAL_DISABLE_READDIR_ON_OPEN", "EMPTY_DIR")
 # Per-worker /vsicurl/ byte-range cache.  Disabled by default because the
 # in-container nginx proxy_cache (4 GB on disk, shared across all FastCGI
 # workers) already serves repeat byte-range reads — a second per-worker
@@ -128,6 +131,25 @@ def header(extent, srs_set):
         '  END',
         '',
         '  OUTPUTFORMAT',
+        '    NAME "webp"',
+        '    DRIVER "AGG/WEBP"',
+        '    MIMETYPE "image/webp"',
+        '    IMAGEMODE RGBA',
+        '    TRANSPARENT ON',
+        '    EXTENSION "webp"',
+        '    FORMATOPTION "QUALITY=80"',
+        '  END',
+        '',
+        '  OUTPUTFORMAT',
+        '    NAME "jpeg"',
+        '    DRIVER "AGG/JPEG"',
+        '    MIMETYPE "image/jpeg"',
+        '    IMAGEMODE RGB',
+        '    EXTENSION "jpg"',
+        '    FORMATOPTION "QUALITY=80"',
+        '  END',
+        '',
+        '  OUTPUTFORMAT',
         '    NAME "geojson"',
         '    DRIVER "OGR/GEOJSON"',
         '    MIMETYPE "application/json; subtype=geojson"',
@@ -165,7 +187,9 @@ def header(extent, srs_set):
         f'  CONFIG "GDAL_CACHEMAX" "{GDAL_CACHEMAX}"',
         f'  CONFIG "VSI_CACHE" "{VSI_CACHE}"',
         f'  CONFIG "VSI_CACHE_SIZE" "{VSI_CACHE_SIZE}"',
-        '  CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "TRUE"',
+        f'  CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "{GDAL_DISABLE_READDIR_ON_OPEN}"',
+        f'  CONFIG "GDAL_INGESTED_BYTES_AT_OPEN" "{GDAL_INGESTED_BYTES_AT_OPEN}"',
+        f'  CONFIG "CPL_VSIL_CURL_CHUNK_SIZE" "{CPL_VSIL_CURL_CHUNK_SIZE}"',
         '  CONFIG "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES" "YES"',
         '  CONFIG "CPL_VSIL_CURL_ALLOWED_EXTENSIONS" ".tif,.tiff,.parquet"',
     ]
@@ -297,7 +321,7 @@ def tileindex_layer_for_group(c, db_conn, group):
 def raster_layer_for_group(c, group):
     processing = c.get("raster_processing") or [
         "BANDS=1,2,3",
-        "RESAMPLE=AVERAGE",
+        "RESAMPLE=BILINEAR",
     ]
     processing_lines = [f'    PROCESSING "{item}"' for item in processing]
     layer_name = group.get("layer_name") or c.get("layer_name") or c["id"]

mapfiles/mapfile.map

@@ -22,6 +22,25 @@ MAP
     EXTENSION "png"
   END
 
+  OUTPUTFORMAT
+    NAME "webp"
+    DRIVER "AGG/WEBP"
+    MIMETYPE "image/webp"
+    IMAGEMODE RGBA
+    TRANSPARENT ON
+    EXTENSION "webp"
+    FORMATOPTION "QUALITY=80"
+  END
+
+  OUTPUTFORMAT
+    NAME "jpeg"
+    DRIVER "AGG/JPEG"
+    MIMETYPE "image/jpeg"
+    IMAGEMODE RGB
+    EXTENSION "jpg"
+    FORMATOPTION "QUALITY=80"
+  END
+
   OUTPUTFORMAT
     NAME "geojson"
     DRIVER "OGR/GEOJSON"
@@ -47,9 +66,9 @@ MAP
   CONFIG "GDAL_CACHEMAX" "128"
   CONFIG "VSI_CACHE" "TRUE"
   CONFIG "VSI_CACHE_SIZE" "33554432"
-  CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "TRUE"
-  CONFIG "GDAL_HTTP_MULTIPLEX" "YES"
-  CONFIG "GDAL_HTTP_VERSION" "2"
+  CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "EMPTY_DIR"
+  CONFIG "GDAL_INGESTED_BYTES_AT_OPEN" "32768"
+  CONFIG "CPL_VSIL_CURL_CHUNK_SIZE" "65536"
   CONFIG "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES" "YES"
   CONFIG "CPL_VSIL_CURL_ALLOWED_EXTENSIONS" ".tif,.tiff"
 
@@ -101,7 +120,7 @@ MAP
     PROCESSING "BANDS=1,2,3"
     PROCESSING "USE_MASK_BAND=NO"
     PROCESSING "SCALE=0,65535"
-    PROCESSING "RESAMPLE=AVERAGE"
+    PROCESSING "RESAMPLE=BILINEAR"
     PROJECTION
       "init=epsg:6527"
     END

scripts/scan_cog_collection.py

@@ -518,13 +518,38 @@ def load_into_postgis(collection_id, native_features, web_features, source_epsg,
             continue
         native_wkt = _ring_to_wkt(nf["geometry"]["coordinates"][0])
         web_wkt = _ring_to_wkt(wf["geometry"]["coordinates"][0])
+        coords = wf["geometry"]["coordinates"][0]
+        min_x = min(p[0] for p in coords)
+        min_y = min(p[1] for p in coords)
+        max_x = max(p[0] for p in coords)
+        max_y = max(p[1] for p in coords)
+        stac_item = {
+            "type": "Feature",
+            "stac_version": "1.0.0",
+            "id": nf["properties"]["file_name"],
+            "collection": collection_id,
+            "geometry": wf["geometry"],
+            "bbox": [min_x, min_y, max_x, max_y],
+            "properties": {
+                "datetime": dt.datetime.now(dt.timezone.utc).isoformat(),
+                "proj:epsg": int(source_epsg),
+            },
+            "assets": {
+                "data": {
+                    "href": nf["properties"]["location"],
+                    "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+                    "roles": ["data"],
+                }
+            },
+        }
         rows.append((
             collection_id,
             nf["properties"]["location"],
             nf["properties"]["file_name"],
             int(source_epsg),
             web_wkt,
             native_wkt,
+            json.dumps(stac_item),
         ))
 
     conn = psycopg2.connect(
@@ -547,14 +572,15 @@ def load_into_postgis(collection_id, native_features, web_features, source_epsg,
                     cur,
                     """
                     INSERT INTO cog_index
-                        (collection_id, location, file_name, native_epsg, geom, geom_native)
+                        (collection_id, location, file_name, native_epsg, geom, geom_native, stac_item)
                     VALUES %s
                     """,
                     rows,
                     template=(
                         "(%s, %s, %s, %s, "
                         "ST_GeomFromText(%s, 3857), "
-                        "ST_SetSRID(ST_GeomFromText(%s), " + str(int(source_epsg)) + "))"
+                        "ST_SetSRID(ST_GeomFromText(%s), " + str(int(source_epsg)) + "), "
+                        "%s::jsonb)"
                     ),
                     page_size=500,
                 )

tests/unit/test_mapfile_generator.py

@@ -136,7 +136,7 @@ def test_default_processing_bands_and_resample(self):
         lines = mg.raster_layer_for_group(c, group)
         joined = "\n".join(lines)
         assert 'PROCESSING "BANDS=1,2,3"' in joined
-        assert 'PROCESSING "RESAMPLE=AVERAGE"' in joined
+        assert 'PROCESSING "RESAMPLE=BILINEAR"' in joined
 
     def test_custom_raster_processing_override(self):
         c = {
@@ -281,3 +281,37 @@ def test_fgb_uses_stem_without_parsing(self, tmp_path):
 
     def test_geojson_missing_file_uses_stem(self):
         assert mg.ogr_layer_name("/nonexistent/path/layer.geojson") == "layer"
+
+
+# ---------------------------------------------------------------------------
+# header GDAL config
+# ---------------------------------------------------------------------------
+
+class TestHeaderGdalConfig:
+    def test_header_contains_tuned_gdal_options(self):
+        extent = [-100, -50, 100, 50]
+        srs_set = {3857, 4326}
+        lines = mg.header(extent, srs_set)
+        joined = "\n".join(lines)
+        assert 'CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "EMPTY_DIR"' in joined
+        assert 'CONFIG "GDAL_INGESTED_BYTES_AT_OPEN" "32768"' in joined
+        assert 'CONFIG "CPL_VSIL_CURL_CHUNK_SIZE" "65536"' in joined
+        assert 'NAME "webp"' in joined
+        assert 'NAME "jpeg"' in joined
+
+    def test_header_respects_env_overrides(self, monkeypatch):
+        monkeypatch.setenv("GDAL_INGESTED_BYTES_AT_OPEN", "65536")
+        monkeypatch.setenv("CPL_VSIL_CURL_CHUNK_SIZE", "131072")
+        monkeypatch.setenv("GDAL_DISABLE_READDIR_ON_OPEN", "YES")
+        
+        # Re-import module-level env reads if needed or check dynamic reads
+        import importlib
+        importlib.reload(mg)
+        
+        extent = [-100, -50, 100, 50]
+        lines = mg.header(extent, {3857})
+        joined = "\n".join(lines)
+        assert 'CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "YES"' in joined
+        assert 'CONFIG "GDAL_INGESTED_BYTES_AT_OPEN" "65536"' in joined
+        assert 'CONFIG "CPL_VSIL_CURL_CHUNK_SIZE" "131072"' in joined
+

tests/unit/test_scanner.py

@@ -305,6 +305,34 @@ def capture_execute_values(cur, sql, rows, template=None, page_size=500):
         assert "26918" not in captured_templates[0]
         assert "26917" not in captured_templates[1]
 
+    def test_rows_contain_valid_stac_item_json(self, monkeypatch):
+        """Verify load_into_postgis formats valid STAC 1.0.0 Item JSONB objects."""
+        import json
+        mock_pg, _, _ = self._mock_psycopg2()
+        captured_rows = []
+
+        def capture_execute_values(cur, sql, rows, template=None, page_size=500):
+            captured_rows.extend(rows)
+
+        monkeypatch.setattr(scanner, "psycopg2", mock_pg)
+        monkeypatch.setattr(scanner, "execute_values", capture_execute_values, raising=False)
+        monkeypatch.setenv("DB_HOST", "localhost")
+        monkeypatch.setenv("DB_USER", "mapserver")
+
+        f = make_feature("test.tif", 3857)
+        scanner.load_into_postgis("my-col", [f], [f], 3857, delete_first=False)
+
+        assert len(captured_rows) == 1
+        stac_raw = captured_rows[0][6]
+        item = json.loads(stac_raw)
+        assert item["type"] == "Feature"
+        assert item["stac_version"] == "1.0.0"
+        assert item["id"] == "test.tif"
+        assert item["collection"] == "my-col"
+        assert item["properties"]["proj:epsg"] == 3857
+        assert item["assets"]["data"]["href"] == f["properties"]["location"]
+
+
 
 # ---------------------------------------------------------------------------
 # write_tileindex_fgb — requires real GDAL