From 8b436f16e2b82f17f8e6c08378d9752f581334f4 Mon Sep 17 00:00:00 2001 From: Quintin Date: Thu, 16 Jul 2026 11:26:49 -0700 Subject: [PATCH 1/4] Add Peace to data import script --- docker/data-import/import_data.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docker/data-import/import_data.sh b/docker/data-import/import_data.sh index 8c43fd7..7784e27 100755 --- a/docker/data-import/import_data.sh +++ b/docker/data-import/import_data.sh @@ -39,6 +39,19 @@ ogr2ogr \ -append \ -addfields +echo "Importing Peace rivers" +ogr2ogr \ + -f "PostgreSQL" \ + "PG:$DB_DSN" \ + /data/Peace_3005_rivers.gpkg \ + -nlt MULTILINESTRING \ + -nln rivers \ + -lco GEOMETRY_NAME=geom \ + -lco FID=fid \ + -a_srs EPSG:3005 \ + -append \ + -addfields + echo "Rivers data imported. Importing lakes data..." echo "Importing Fraser lakes" ogr2ogr \ @@ -66,6 +79,19 @@ ogr2ogr \ -append \ -addfields +echo "Importing Peace lakes" +ogr2ogr \ + -f "PostgreSQL" \ + "PG:$DB_DSN" \ + /data/Peace_3005_lakes.gpkg \ + -nlt MULTIPOLYGON \ + -nln lakes \ + -lco GEOMETRY_NAME=geom \ + -lco FID=fid \ + -a_srs EPSG:3005 \ + -append \ + -addfields + echo "Updating tables and adding indices..." psql "$DB_DSN" <<-EOSQL DO \$\$ @@ -149,4 +175,4 @@ CREATE MATERIALIZED VIEW downstreams AS VACUUM ANALYZE downstreams; EOSQL -echo "Data import complete." \ No newline at end of file +echo "Data import complete." From 2403f5fcf2eedf972b0f8a6ea1a742d13698fdf9 Mon Sep 17 00:00:00 2001 From: Quintin Date: Thu, 16 Jul 2026 11:29:14 -0700 Subject: [PATCH 2/4] DRY import data script --- docker/data-import/import_data.sh | 92 +++++++++---------------------- 1 file changed, 26 insertions(+), 66 deletions(-) diff --git a/docker/data-import/import_data.sh b/docker/data-import/import_data.sh index 7784e27..2677444 100755 --- a/docker/data-import/import_data.sh +++ b/docker/data-import/import_data.sh @@ -13,84 +13,44 @@ psql "$DB_DSN" <<-EOSQL DROP SEQUENCE IF EXISTS shared_uid_seq CASCADE; EOSQL +import_dataset() { + file=$1 + geometry_type=$2 + table=$3 + shift 3 + + ogr2ogr \ + -f "PostgreSQL" \ + "PG:$DB_DSN" \ + "/data/$file" \ + -nlt "$geometry_type" \ + -nln "$table" \ + -lco GEOMETRY_NAME=geom \ + -lco FID=fid \ + -a_srs EPSG:3005 \ + "$@" \ + -addfields +} + echo "PostGIS is ready. Importing rivers data..." echo "Importing Fraser rivers" -ogr2ogr \ - -f "PostgreSQL" \ - "PG:$DB_DSN" \ - /data/Fraser_3005_rivers.gpkg \ - -nlt MULTILINESTRING \ - -nln rivers \ - -lco GEOMETRY_NAME=geom \ - -lco FID=fid \ - -a_srs EPSG:3005 \ - -addfields - +import_dataset Fraser_3005_rivers.gpkg MULTILINESTRING rivers + echo "Importing BC Coast rivers" -ogr2ogr \ - -f "PostgreSQL" \ - "PG:$DB_DSN" \ - /data/BC_Coast_3005_rivers.gpkg \ - -nlt MULTILINESTRING \ - -nln rivers \ - -lco GEOMETRY_NAME=geom \ - -lco FID=fid \ - -a_srs EPSG:3005 \ - -append \ - -addfields +import_dataset BC_Coast_3005_rivers.gpkg MULTILINESTRING rivers -append echo "Importing Peace rivers" -ogr2ogr \ - -f "PostgreSQL" \ - "PG:$DB_DSN" \ - /data/Peace_3005_rivers.gpkg \ - -nlt MULTILINESTRING \ - -nln rivers \ - -lco GEOMETRY_NAME=geom \ - -lco FID=fid \ - -a_srs EPSG:3005 \ - -append \ - -addfields +import_dataset Peace_3005_rivers.gpkg MULTILINESTRING rivers -append echo "Rivers data imported. Importing lakes data..." echo "Importing Fraser lakes" -ogr2ogr \ - -f "PostgreSQL" \ - "PG:$DB_DSN" \ - /data/Fraser_3005_lakes.gpkg \ - -nlt MULTIPOLYGON \ - -nln lakes \ - -lco GEOMETRY_NAME=geom \ - -lco FID=fid \ - -a_srs EPSG:3005 \ - -addfields - +import_dataset Fraser_3005_lakes.gpkg MULTIPOLYGON lakes echo "Importing BC Coast lakes" -ogr2ogr \ - -f "PostgreSQL" \ - "PG:$DB_DSN" \ - /data/BC_Coast_3005_lakes.gpkg \ - -nlt MULTIPOLYGON \ - -nln lakes \ - -lco GEOMETRY_NAME=geom \ - -lco FID=fid \ - -a_srs EPSG:3005 \ - -append \ - -addfields +import_dataset BC_Coast_3005_lakes.gpkg MULTIPOLYGON lakes -append echo "Importing Peace lakes" -ogr2ogr \ - -f "PostgreSQL" \ - "PG:$DB_DSN" \ - /data/Peace_3005_lakes.gpkg \ - -nlt MULTIPOLYGON \ - -nln lakes \ - -lco GEOMETRY_NAME=geom \ - -lco FID=fid \ - -a_srs EPSG:3005 \ - -append \ - -addfields +import_dataset Peace_3005_lakes.gpkg MULTIPOLYGON lakes -append echo "Updating tables and adding indices..." psql "$DB_DSN" <<-EOSQL From a081ad93b031d3af0463d70201db21d3acd164d2 Mon Sep 17 00:00:00 2001 From: Quintin Date: Fri, 17 Jul 2026 08:04:37 -0700 Subject: [PATCH 3/4] Add Columbia to data import script --- docker/data-import/import_data.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/data-import/import_data.sh b/docker/data-import/import_data.sh index 2677444..6f25db5 100755 --- a/docker/data-import/import_data.sh +++ b/docker/data-import/import_data.sh @@ -42,6 +42,9 @@ import_dataset BC_Coast_3005_rivers.gpkg MULTILINESTRING rivers -append echo "Importing Peace rivers" import_dataset Peace_3005_rivers.gpkg MULTILINESTRING rivers -append +echo "Importing Columbia rivers" +import_dataset Columbia_3005_rivers.gpkg MULTILINESTRING rivers -append + echo "Rivers data imported. Importing lakes data..." echo "Importing Fraser lakes" import_dataset Fraser_3005_lakes.gpkg MULTIPOLYGON lakes @@ -52,6 +55,9 @@ import_dataset BC_Coast_3005_lakes.gpkg MULTIPOLYGON lakes -append echo "Importing Peace lakes" import_dataset Peace_3005_lakes.gpkg MULTIPOLYGON lakes -append +echo "Importing Columbia lakes" +import_dataset Columbia_3005_lakes.gpkg MULTIPOLYGON lakes -append + echo "Updating tables and adding indices..." psql "$DB_DSN" <<-EOSQL DO \$\$ From 87909b280ad6f08338ae71faef0ab65e7beed610 Mon Sep 17 00:00:00 2001 From: QSparks <105678101+QSparks@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:39:56 -0700 Subject: [PATCH 4/4] I33 lighthouse (#35), I34 controlled startup (#36), I37 bulk download (#39) --- .github/workflows/docker-publish.yml | 10 + README.md | 11 +- app/index.html | 7 +- app/src/App.jsx | 6 +- app/src/baseMapConfig.js | 53 ++++ app/src/components/data/DataSelection.css | 26 +- .../components/data/DataSelectionTable.jsx | 115 +++++++- app/src/components/info/HelpGuide.jsx | 9 +- app/src/components/map/FwaNameSearch.jsx | 2 + app/src/components/map/InteractionLayer.css | 28 ++ app/src/components/map/InteractionLayer.jsx | 207 ++++++++++++-- app/src/components/map/MapComponent.jsx | 5 +- app/src/components/map/NetworkLegend.css | 86 ++++++ app/src/components/map/NetworkLegend.jsx | 100 +++++++ app/src/components/map/PointPlotter.jsx | 7 +- .../map/vectorGridCanvasRenderer.js | 67 +++++ app/src/main.jsx | 3 +- app/src/services/geoJsonApi.js | 82 ++++++ app/src/services/streamNetApi.js | 24 +- app/src/services/timeseriesApi.js | 49 +++- app/src/styles.js | 8 +- app/src/utils/downloadFile.js | 14 + app/vite.config.js | 15 +- config/bbox.template.toml | 99 ++++++- docker/BBOX/Dockerfile | 6 +- docker/BBOX/start-bbox-server.sh | 52 +++- docker/app/server.js | 237 ++++++++++++++-- docker/data-import/import_data.sh | 254 ++++++++++++++---- docker/varnish/Dockerfile | 15 ++ docker/varnish/start-varnish.sh | 23 ++ docs/swarm-readiness.md | 72 +++++ 31 files changed, 1531 insertions(+), 161 deletions(-) create mode 100644 app/src/baseMapConfig.js create mode 100644 app/src/components/map/InteractionLayer.css create mode 100644 app/src/components/map/NetworkLegend.css create mode 100644 app/src/components/map/NetworkLegend.jsx create mode 100644 app/src/components/map/vectorGridCanvasRenderer.js create mode 100644 app/src/services/geoJsonApi.js create mode 100644 app/src/utils/downloadFile.js create mode 100644 docker/varnish/Dockerfile create mode 100644 docker/varnish/start-varnish.sh create mode 100644 docs/swarm-readiness.md diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 81c4390..ef0d661 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -57,3 +57,13 @@ jobs: tags: | pcic/chyp-server:${{ github.ref_name }} ${{ github.ref_name == 'main' && 'pcic/chyp-server:latest' || '' }} + + - name: Build and Publish Varnish + uses: docker/build-push-action@v4 + with: + context: . + file: docker/varnish/Dockerfile + push: true + tags: | + pcic/chyp-varnish:${{ github.ref_name }} + ${{ github.ref_name == 'main' && 'pcic/chyp-varnish:latest' || '' }} diff --git a/README.md b/README.md index cc81fbd..d103756 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Commands should be run from within the `app` folder. - Vector tiles used to display over 100k lakes and river segments across British Columbia - Lakes and rivers shown in blue, highlighted in red when selected. - Hover to dynamically highlight features. -- Selecting a feature open the **Data Download Panel** and provides a link to download the GeoJSON for the feature. +- Selecting a feature opens the **Data Download Panel** and provides links to download GeoJSON for the selected feature or its complete upstream/downstream network. ### BC Freshwater Atlas Lake & River Search @@ -80,6 +80,9 @@ A BBox instance running at the same host is assumed by InteractionLayer.jsx and The project includes GitHub Actions workflows for Docker image publishing. Docker images for the app, data import, and BBOX server are automatically built and published upon branch pushes and tagged releases. +Docker Swarm deployments must use the repository's explicit readiness gates; +See [Swarm startup readiness](docs/swarm-readiness.md) for the required stack configuration and rollout procedure. + ## API Integration The app connects to the PCIC Hydromosaic API for: @@ -95,7 +98,10 @@ BC Geographic Warehouse (BCGW) [Public Map Server](https://delivery.maps.gov.bc. ## Environment Variables -- `REACT_APP_BC_BASE_MAP_TILES_URL`: Base map tile URL (defaults to PCIC swarm server) +- `REACT_APP_BC_BASE_MAP_TILES_URL`: Base map tile URL (defaults to the + production PNG tiles on `services.pacificclimate.org`). The app container + reads this variable when it starts, so non-production deployments can use a + different tile server and format without rebuilding the image. ## Docker Images @@ -105,6 +111,7 @@ The following images are published to Docker Hub: - `pcic/chyp-data-import`: Data import utilities - `pcic/chyp-postgis`: PostGIS database - `pcic/chyp-server`: BBOX tile server +- `pcic/chyp-varnish`: BBOX tile cache with an upstream readiness check ## Requirements diff --git a/app/index.html b/app/index.html index e7c06a7..c16818a 100644 --- a/app/index.html +++ b/app/index.html @@ -3,11 +3,14 @@ - + - + Channel-Scale Hydrologic Model Output Portal diff --git a/app/src/App.jsx b/app/src/App.jsx index 98b628f..62b7616 100644 --- a/app/src/App.jsx +++ b/app/src/App.jsx @@ -1,5 +1,9 @@ import MapComponent from "./components/map/MapComponent.jsx"; -const App = () => ; +const App = () => ( +
+ +
+); export default App; diff --git a/app/src/baseMapConfig.js b/app/src/baseMapConfig.js new file mode 100644 index 0000000..43d6009 --- /dev/null +++ b/app/src/baseMapConfig.js @@ -0,0 +1,53 @@ +import { BCBaseMap } from "pcic-react-leaflet-components"; + +const runtimeTileUrl = + globalThis.__CHYP_CONFIG__?.REACT_APP_BC_BASE_MAP_TILES_URL?.trim(); + +export const baseMapTileUrl = runtimeTileUrl || BCBaseMap.tileset.url; + +const upsertResourceHint = ({ id, rel, href, as, fetchPriority }) => { + const link = document.getElementById(id) || document.createElement("link"); + + link.id = id; + link.rel = rel; + link.href = href; + if (as) { + link.as = as; + } + if (fetchPriority) { + link.fetchPriority = fetchPriority; + } + + if (!link.isConnected) { + document.head.appendChild(link); + } +}; + +export const configureBaseMapResourceHints = () => { + if (!baseMapTileUrl) { + return; + } + + try { + upsertResourceHint({ + id: "base-map-preconnect", + rel: "preconnect", + href: new URL(baseMapTileUrl).origin, + }); + } catch { + // Leaflet will report an invalid tile URL when it tries to use it. + } + + upsertResourceHint({ + id: "base-map-preload", + rel: "preload", + as: "image", + fetchPriority: "high", + href: baseMapTileUrl + .replace("{z}", "6") + .replace("{x}", "33") + .replace("{y}", "29"), + }); +}; + +export { BCBaseMap }; diff --git a/app/src/components/data/DataSelection.css b/app/src/components/data/DataSelection.css index 0aff587..2665a0a 100644 --- a/app/src/components/data/DataSelection.css +++ b/app/src/components/data/DataSelection.css @@ -7,7 +7,10 @@ border-radius: 12px; box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); z-index: 1000; + width: 380px; + max-width: calc(100vw - 32px); min-width: 300px; + box-sizing: border-box; font-family: Arial, sans-serif; } @@ -18,6 +21,9 @@ } .data-selection select { + width: 100%; + min-width: 0; + box-sizing: border-box; padding: 8px 12px; border: 1px solid #ccc; border-radius: 8px; @@ -44,6 +50,24 @@ background-color: #0056b3; } +.data-selection button:disabled { + background-color: #7b8a99; + cursor: not-allowed; +} + +.data-selection .network-downloads { + display: flex; + flex-direction: column; + gap: 8px; + padding-top: 4px; + border-top: 1px solid #bbb; +} + +.data-selection .network-download-note { + color: #444; + text-align: center; +} + .data-selection .header { display: flex; justify-content: space-between; @@ -95,4 +119,4 @@ .data-selection.shake { animation: shake 0.4s ease-in-out; -} \ No newline at end of file +} diff --git a/app/src/components/data/DataSelectionTable.jsx b/app/src/components/data/DataSelectionTable.jsx index 346eb1f..63e83ff 100644 --- a/app/src/components/data/DataSelectionTable.jsx +++ b/app/src/components/data/DataSelectionTable.jsx @@ -1,9 +1,11 @@ import { useState, useEffect, useCallback } from "react"; import PropTypes from "prop-types"; import { + fetchBulkTimeseries, getAvailableOptions, - downloadTimeseries, + getTimeseriesDownload, } from "../../services/timeseriesApi.js"; +import { downloadBlob, downloadUrl } from "../../utils/downloadFile.js"; import "./DataSelection.css"; const HISTORICAL_SCENARIO = "historical"; @@ -15,7 +17,22 @@ const getModelLabel = (model) => const getScenarioLabel = (scenario) => scenario === HISTORICAL_SCENARIO ? `${scenario} (PNWNAmet only)` : scenario; -const DataSelectionTable = ({ featureId, onClose }) => { +const getVariableLabel = (variable) => + variable.startsWith( + "mass concentration of maximum amount of oxygen that will dissolve" + ) + ? variable.replace( + /^mass concentration of maximum amount of oxygen that will dissolve in water at given temperature and pressure/, + "Dissolved oxygen saturation" + ) + : variable; + +const DataSelectionTable = ({ + featureId, + upstreamSubids, + downstreamSubids, + onClose, +}) => { const outletId = `${featureId}`; const [options, setOptions] = useState({ models: [], @@ -28,7 +45,7 @@ const DataSelectionTable = ({ featureId, onClose }) => { variable: "", }); const [isFetching, setIsFetching] = useState(true); - const [isLoading, setIsLoading] = useState(false); + const [activeDownload, setActiveDownload] = useState(null); const [showValidation, setShowValidation] = useState(false); const [shake, setShake] = useState(false); @@ -75,17 +92,53 @@ const DataSelectionTable = ({ featureId, onClose }) => { return; } - setIsLoading(true); + setActiveDownload("selected"); try { - await downloadTimeseries(outletId, selections); + const { url, filename } = await getTimeseriesDownload( + outletId, + selections + ); + downloadUrl(url, filename); } catch (error) { console.error("Download error:", error); alert("Failed to download data"); } finally { - setIsLoading(false); + setActiveDownload(null); } }, [outletId, selections]); + const handleBulkDownload = useCallback( + async (direction) => { + if (Object.values(selections).some((v) => !v)) { + setShowValidation(true); + setShake(true); + setTimeout(() => setShake(false), 650); + return; + } + + const subids = + direction === "upstream" ? upstreamSubids : downstreamSubids; + if (!subids || subids.length <= 1) return; + + setActiveDownload(direction); + try { + const { blob, filename } = await fetchBulkTimeseries( + outletId, + direction, + subids, + selections + ); + downloadBlob(blob, filename); + } catch (error) { + console.error("Bulk download error:", error); + alert(`Failed to download ${direction} data`); + } finally { + setActiveDownload(null); + } + }, + [downstreamSubids, outletId, selections, upstreamSubids] + ); + return (
e.preventDefault()}> @@ -147,14 +200,56 @@ const DataSelectionTable = ({ featureId, onClose }) => { {options.variables.map((variable) => ( ))} - + +
+ + +
+ + Network downloads include the selected segment. +
); @@ -162,6 +257,8 @@ const DataSelectionTable = ({ featureId, onClose }) => { DataSelectionTable.propTypes = { featureId: PropTypes.string.isRequired, + upstreamSubids: PropTypes.arrayOf(PropTypes.string), + downstreamSubids: PropTypes.arrayOf(PropTypes.string), onClose: PropTypes.func.isRequired, }; diff --git a/app/src/components/info/HelpGuide.jsx b/app/src/components/info/HelpGuide.jsx index e6a989c..4662a1f 100644 --- a/app/src/components/info/HelpGuide.jsx +++ b/app/src/components/info/HelpGuide.jsx @@ -107,8 +107,8 @@ const HelpGuide = () => {
  • Enter X/Y or Longitude/Latitude coordinates
    • -
  • Click "Plot Point" to add marker
    • -
  • Use "Clear Marker" to remove point
    • +
  • Click "Plot Point" to add marker
    • +
  • Use "Clear Marker" to remove point
    • @@ -129,7 +129,10 @@ const HelpGuide = () => {
  • Choose model, scenario, and variable from dropdown menus
    • -
  • Click "Download CSV" to get timeseries data
    • +
  • + Download the selected segment as CSV, or its complete + upstream/downstream network as a multi-outlet NetCDF +
    • diff --git a/app/src/components/map/FwaNameSearch.jsx b/app/src/components/map/FwaNameSearch.jsx index 9889828..0067dbf 100644 --- a/app/src/components/map/FwaNameSearch.jsx +++ b/app/src/components/map/FwaNameSearch.jsx @@ -361,6 +361,8 @@ export default function FwaNameSearch({ onPickedFeature, fwaStyles }) {
    { diff --git a/app/src/components/map/InteractionLayer.css b/app/src/components/map/InteractionLayer.css new file mode 100644 index 0000000..62dab69 --- /dev/null +++ b/app/src/components/map/InteractionLayer.css @@ -0,0 +1,28 @@ +.custom-popup .leaflet-popup-content-wrapper, +.custom-popup .leaflet-popup-tip { + background: rgba(255, 255, 255, 0.7); +} + +.custom-popup .leaflet-popup-content { + margin: 10px 14px; +} + +.geojson-download-popup { + line-height: 1.35; +} + +.geojson-download-popup__heading { + display: block; + margin-bottom: 2px; + font-size: 14px; +} + +.geojson-download-popup__subid { + margin-bottom: 6px; + color: #444; + font-size: 12px; +} + +.geojson-download-popup a { + white-space: nowrap; +} diff --git a/app/src/components/map/InteractionLayer.jsx b/app/src/components/map/InteractionLayer.jsx index a7ee613..e718ea5 100644 --- a/app/src/components/map/InteractionLayer.jsx +++ b/app/src/components/map/InteractionLayer.jsx @@ -3,10 +3,83 @@ import { useState, useRef, useCallback, useEffect, memo, lazy, Suspense } from " import PropTypes from "prop-types"; import L from "leaflet"; import "leaflet.vectorgrid"; -import { fetchDownstreams, fetchUpstreams } from "../../services/streamNetApi.js"; +import { + fetchDownstreamNetwork, + fetchUpstreamNetwork, +} from "../../services/streamNetApi.js"; +import { fetchNetworkGeoJson } from "../../services/geoJsonApi.js"; +import { downloadBlob } from "../../utils/downloadFile.js"; +import { interactiveCanvasTile } from "./vectorGridCanvasRenderer.js"; +import "./InteractionLayer.css"; const DataSelectionTable = lazy(() => import("../data/DataSelectionTable.jsx")); +const makePopupLink = (label) => { + const link = document.createElement("a"); + link.href = "#"; + link.textContent = label; + link.style.display = "block"; + link.style.color = "blue"; + link.style.textDecoration = "underline"; + link.style.marginTop = "4px"; + return link; +}; + +const setPopupLinkUnavailable = (link, label) => { + link.textContent = label; + link.setAttribute("aria-disabled", "true"); + link.style.color = "#666"; + link.style.pointerEvents = "none"; +}; + +const enableNetworkGeoJsonLink = ({ + link, + selectedSubid, + direction, + subids, + controllers, +}) => { + if (subids.length <= 1) { + setPopupLinkUnavailable(link, `No ${direction} outlets`); + return; + } + + link.textContent = `${direction[0].toUpperCase()}${direction.slice(1)} GeoJSON (${subids.length})`; + link.removeAttribute("aria-disabled"); + link.style.color = "blue"; + link.style.pointerEvents = "auto"; + + link.addEventListener("click", async (event) => { + event.preventDefault(); + if (link.dataset.downloading === "true") return; + + link.dataset.downloading = "true"; + link.textContent = `Preparing ${direction} GeoJSON...`; + link.style.pointerEvents = "none"; + const controller = new AbortController(); + controllers.add(controller); + + try { + const { blob, filename } = await fetchNetworkGeoJson({ + selectedSubid, + direction, + signal: controller.signal, + }); + downloadBlob(blob, filename); + link.textContent = `${direction[0].toUpperCase()}${direction.slice(1)} GeoJSON (${subids.length})`; + } catch (error) { + if (error.name !== "AbortError") { + console.error(`Failed to download ${direction} GeoJSON:`, error); + link.textContent = `Failed. Retry ${direction} GeoJSON`; + } + } finally { + controllers.delete(controller); + delete link.dataset.downloading; + link.style.pointerEvents = "auto"; + } + }); +}; + const InteractionLayer = ({ baseStyles, interactionStyles }) => { const stateRef = useRef({ hoverHighlight: null, @@ -19,10 +92,21 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { }); const vectorTileLayerRef = useRef(null); const mapRef = useRef(null); - const popup = useRef(L.popup({ className: "custom-popup", autoPan: false })); + const popup = useRef( + L.popup({ + className: "custom-popup", + autoPan: false, + minWidth: 160, + maxWidth: 230, + }) + ); const [showDataTable, setShowDataTable] = useState(false); const [selectedSubId, setSelectedSubId] = useState(null); + const [networkSubids, setNetworkSubids] = useState({ + upstream: null, + downstream: null, + }); const updateCursor = (() => { let lastCursor = null; @@ -79,9 +163,12 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { updateCursor("grab"); }, dragstart: () => { + stateRef.current.isDragging = true; + clearHoverHighlight(); updateCursor("grabbing"); }, dragend: () => { + stateRef.current.isDragging = false; updateCursor("grab"); }, }); @@ -102,14 +189,16 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { const vectorTileLayer = L.vectorGrid.protobuf( `${window.location.origin}/bbox-server/xyz/water_tiles/{z}/{x}/{y}.mvt`, { + rendererFactory: interactiveCanvasTile, vectorTileLayerStyles: baseStyles, maxNativeZoom: 13, interactive: true, + // Increase the Canvas hit area without making stream lines thicker. + tolerance: 5, getFeatureId: (feature) => feature.properties.uid, updateWhenIdle: true, updateWhenZooming: false, keepBuffer: 2, - preferCanvas: true, pane: "interactive", // Use the dedicated pane zIndex: 1, } @@ -156,6 +245,7 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { const { uid, properties, layerType } = getFeatureInfo(event); setSelectedSubId(properties.subid); + setNetworkSubids({ upstream: null, downstream: null }); setShowDataTable(true); // Reset previous clicked feature if exists if (stateRef.current.clickedFeature && vectorTileLayerRef.current) { @@ -175,6 +265,8 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { mapRef.current.closePopup(stateRef.current.currentPopup); } + let networkGeoJsonLinks = null; + const geoJsonDownloadControllers = new Set(); try { const collection = layerType === "lakes" ? "lakes" : "rivers"; const response = await fetch( @@ -191,20 +283,51 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { }); const url = URL.createObjectURL(blob); + const popupContent = document.createElement("div"); + popupContent.className = "geojson-download-popup"; + + const heading = document.createElement("strong"); + heading.className = "geojson-download-popup__heading"; + heading.textContent = "Download GeoJSON"; + popupContent.appendChild(heading); + + const subidLine = document.createElement("div"); + subidLine.className = "geojson-download-popup__subid"; + const subidLabel = document.createElement("span"); + subidLabel.textContent = "SubId:"; + subidLine.append(subidLabel, ` ${properties.subid}`); + popupContent.appendChild(subidLine); + + const selectedLink = makePopupLink("Selected GeoJSON"); + selectedLink.href = url; + selectedLink.download = `${properties.subid}.geojson`; + popupContent.appendChild(selectedLink); + + const upstreamLink = makePopupLink("Loading upstream network..."); + const downstreamLink = makePopupLink("Loading downstream network..."); + setPopupLinkUnavailable(upstreamLink, "Loading upstream network..."); + setPopupLinkUnavailable(downstreamLink, "Loading downstream network..."); + popupContent.append(upstreamLink, downstreamLink); + networkGeoJsonLinks = { + upstream: upstreamLink, + downstream: downstreamLink, + }; + popup.current .setLatLng(event.latlng) - .setContent( - ` -
    - SubId: ${properties.subid}
    - Download GeoJSON -
    - ` - ) + .setContent(popupContent) .openOn(mapRef.current); + stateRef.current.currentPopup = popup.current; - popup.current.on("remove", () => { + popup.current.once("remove", () => { URL.revokeObjectURL(url); + for (const controller of geoJsonDownloadControllers) { + controller.abort(); + } + geoJsonDownloadControllers.clear(); + if (stateRef.current.currentPopup === popup.current) { + stateRef.current.currentPopup = null; + } }); } catch (error) { console.error("Error fetching GeoJSON:", error); @@ -222,8 +345,13 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { // Accordingly, both sets of features are cleared and highlighted in tandem. try { // fetch upstream and downstream features - const downstreamList = await fetchDownstreams(properties.subid, properties.uid); - const upstreamList = await fetchUpstreams(properties.subid, properties.uid); + const [downstreamNetwork, upstreamNetwork] = await Promise.all([ + fetchDownstreamNetwork(properties.subid, properties.uid), + fetchUpstreamNetwork(properties.subid, properties.uid), + ]); + + // A second feature may have been clicked while these requests ran. + if (stateRef.current.clickedFeature !== uid) return; // clear old highlighted upstream and downstream features if (stateRef.current.downstreamFeatures.length > 0 && vectorTileLayerRef.current) { @@ -242,34 +370,55 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { } // highlight new upstream and downstream features - stateRef.current.downstreamFeatures = downstreamList; + stateRef.current.downstreamFeatures = downstreamNetwork.uids; for (const uid of stateRef.current.downstreamFeatures) { vectorTileLayer.setFeatureStyle(uid, interactionStyles.highlight["downstream"]); } - stateRef.current.upstreamFeatures = upstreamList; + stateRef.current.upstreamFeatures = upstreamNetwork.uids; for (const uid of stateRef.current.upstreamFeatures) { vectorTileLayer.setFeatureStyle(uid, interactionStyles.highlight["upstream"]); } + setNetworkSubids({ + upstream: upstreamNetwork.subids, + downstream: downstreamNetwork.subids, + }); + if (networkGeoJsonLinks) { + enableNetworkGeoJsonLink({ + link: networkGeoJsonLinks.upstream, + selectedSubid: properties.subid, + direction: "upstream", + subids: upstreamNetwork.subids, + controllers: geoJsonDownloadControllers, + }); + enableNetworkGeoJsonLink({ + link: networkGeoJsonLinks.downstream, + selectedSubid: properties.subid, + direction: "downstream", + subids: downstreamNetwork.subids, + controllers: geoJsonDownloadControllers, + }); + } } catch (error) { console.error("Error fetching upstream and downstream features:", error); + if (stateRef.current.clickedFeature === uid) { + setNetworkSubids({ upstream: [], downstream: [] }); + if (networkGeoJsonLinks) { + setPopupLinkUnavailable( + networkGeoJsonLinks.upstream, + "Upstream GeoJSON unavailable" + ); + setPopupLinkUnavailable( + networkGeoJsonLinks.downstream, + "Downstream GeoJSON unavailable" + ); + } + } } }; - const handleMouseDown = () => { - stateRef.current.isDragging = true; - updateCursor("grabbing"); - }; - - const handleMouseUp = () => { - stateRef.current.isDragging = false; - updateCursor(stateRef.current.hoverHighlight ? "pointer" : "grab"); - }; - vectorTileLayer.on("mouseover", handleMouseOver); vectorTileLayer.on("mouseout", handleMouseOut); vectorTileLayer.on("click", handleClick); - vectorTileLayer.on("mousedown", handleMouseDown); - vectorTileLayer.on("mouseup", handleMouseUp); vectorTileLayer.addTo(mapRef.current); @@ -304,6 +453,8 @@ const InteractionLayer = ({ baseStyles, interactionStyles }) => { diff --git a/app/src/components/map/MapComponent.jsx b/app/src/components/map/MapComponent.jsx index ff74076..2b4c36f 100644 --- a/app/src/components/map/MapComponent.jsx +++ b/app/src/components/map/MapComponent.jsx @@ -1,8 +1,9 @@ -import { BCBaseMap } from "pcic-react-leaflet-components"; +import { BCBaseMap, baseMapTileUrl } from "../../baseMapConfig.js"; import { useRef, useEffect, lazy, Suspense } from "react"; import InteractionLayer from "./InteractionLayer.jsx"; import LogoBox from "../info/LogoBox.jsx"; import FwaNameSearch from "./FwaNameSearch.jsx"; +import NetworkLegend from "./NetworkLegend.jsx"; import { baseStyles, interactionStyles, fwaStyles } from "../../styles.js"; const PointPlotter = lazy(() => import("./PointPlotter.jsx")); @@ -35,6 +36,7 @@ const MapComponent = () => { return ( { + ); }; diff --git a/app/src/components/map/NetworkLegend.css b/app/src/components/map/NetworkLegend.css new file mode 100644 index 0000000..99268be --- /dev/null +++ b/app/src/components/map/NetworkLegend.css @@ -0,0 +1,86 @@ +.network-legend { + position: absolute; + right: 10px; + bottom: 72px; + z-index: 1000; + min-width: 148px; + padding: 10px 12px; + border: 1px solid rgba(0, 0, 0, 0.12); + border-radius: 10px; + background: rgba(255, 255, 255, 0.7); + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.14); + color: #222; + font-family: Arial, sans-serif; + font-size: 13px; +} + +.network-legend-toggle { + display: flex; + align-items: center; + justify-content: center; + width: 40px; + height: 40px; + min-width: 0; + padding: 0; + border: 0; + border-radius: 50%; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + color: #222; + cursor: pointer; +} + +.network-legend-toggle:hover { + background: rgba(255, 255, 255, 0.85); +} + +.network-legend-toggle:focus-visible { + outline: 2px solid #007bff; + outline-offset: 2px; +} + +.network-legend-icon { + width: 24px; + height: 24px; +} + +.network-legend-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.network-legend-title { + font-size: 13px; + font-weight: 700; +} + +.network-legend-close { + margin: -6px -6px -2px 0; + padding: 2px 5px; + border: 0; + background: transparent; + color: #555; + font-size: 18px; + line-height: 1; + cursor: pointer; +} + +.network-legend-header + .network-legend-item { + margin-top: 7px; +} + +.network-legend-item { + display: flex; + align-items: center; + gap: 8px; + line-height: 1.5; +} + +.network-legend-swatch { + display: inline-block; + width: 25px; + height: 5px; + border: 1px solid rgba(0, 0, 0, 0.18); + border-radius: 3px; +} diff --git a/app/src/components/map/NetworkLegend.jsx b/app/src/components/map/NetworkLegend.jsx new file mode 100644 index 0000000..6f4c842 --- /dev/null +++ b/app/src/components/map/NetworkLegend.jsx @@ -0,0 +1,100 @@ +import { useState } from "react"; +import PropTypes from "prop-types"; +import "./NetworkLegend.css"; + +const NetworkLegend = ({ highlightStyles }) => { + const [isOpen, setIsOpen] = useState(false); + const items = [ + { label: "Selected segment", color: highlightStyles.rivers.color }, + { label: "Upstream", color: highlightStyles.upstream.color }, + { label: "Downstream", color: highlightStyles.downstream.color }, + ]; + + if (!isOpen) { + return ( + + ); + } + + return ( + + ); +}; + +NetworkLegend.propTypes = { + highlightStyles: PropTypes.shape({ + rivers: PropTypes.shape({ color: PropTypes.string.isRequired }).isRequired, + upstream: PropTypes.shape({ color: PropTypes.string.isRequired }).isRequired, + downstream: PropTypes.shape({ color: PropTypes.string.isRequired }).isRequired, + }).isRequired, +}; + +export default NetworkLegend; diff --git a/app/src/components/map/PointPlotter.jsx b/app/src/components/map/PointPlotter.jsx index fae5045..394d99a 100644 --- a/app/src/components/map/PointPlotter.jsx +++ b/app/src/components/map/PointPlotter.jsx @@ -1,6 +1,5 @@ import { useState, useEffect, useRef, memo } from "react"; import { circleMarker } from "leaflet"; -import L from "leaflet"; import proj4 from "proj4"; import { useMap } from "react-leaflet"; import "./PointPlotter.css"; @@ -165,7 +164,11 @@ const PointPlotter = () => { setCoords((prev) => ({ ...prev, y: e.target.value })) } /> - setProjType(e.target.value)} + > diff --git a/app/src/components/map/vectorGridCanvasRenderer.js b/app/src/components/map/vectorGridCanvasRenderer.js new file mode 100644 index 0000000..6637ad9 --- /dev/null +++ b/app/src/components/map/vectorGridCanvasRenderer.js @@ -0,0 +1,67 @@ +import L from "leaflet"; +import "leaflet.vectorgrid"; + +// VectorGrid 1.3 calculates Canvas hit-test coordinates from Leaflet's pixel +// origin. That becomes inaccurate when GridLayer scales its tile container for +// a fractional zoom. Measure from the transformed canvas instead so the point +// remains in the tile's 256px coordinate space after zooming and panning. +const InteractiveCanvasTile = L.Canvas.Tile.extend({ + _eventToTilePoint(event) { + const bounds = this._container.getBoundingClientRect(); + + if (!bounds.width || !bounds.height) { + return null; + } + + return L.point( + ((event.clientX - bounds.left) * this._size.x) / bounds.width, + ((event.clientY - bounds.top) * this._size.y) / bounds.height + ); + }, + + _onClick(event) { + if (!this._map) return; + + const point = this._eventToTilePoint(event); + if (!point) return; + + let clickedLayer; + + for (let order = this._drawFirst; order; order = order.next) { + const layer = order.layer; + const isClickAfterDrag = + (event.type === "click" || event.type === "preclick") && + this._map._draggableMoved(layer); + + if ( + layer.options.interactive && + layer._containsPoint(point) && + !isClickAfterDrag + ) { + clickedLayer = layer; + } + } + + // Match Leaflet 1.9's Canvas event flow. VectorGrid 1.3 calls the removed + // L.DomEvent.fakeStop here, which prevents clicks from reaching the layer. + this._fireEvent(clickedLayer ? [clickedLayer] : false, event); + }, + + _onMouseMove(event) { + if ( + !this._map || + this._map.dragging.moving() || + this._map._animatingZoom + ) { + return; + } + + const point = this._eventToTilePoint(event); + if (point) { + this._handleMouseHover(event, point); + } + }, +}); + +export const interactiveCanvasTile = (tileCoord, tileSize, options) => + new InteractiveCanvasTile(tileCoord, tileSize, options); diff --git a/app/src/main.jsx b/app/src/main.jsx index cbf497a..29efb07 100644 --- a/app/src/main.jsx +++ b/app/src/main.jsx @@ -1,6 +1,7 @@ -import React from 'react' import { createRoot } from 'react-dom/client' import './index.css' import App from './App.jsx' +import { configureBaseMapResourceHints } from './baseMapConfig.js' +configureBaseMapResourceHints() createRoot(document.getElementById('root')).render() diff --git a/app/src/services/geoJsonApi.js b/app/src/services/geoJsonApi.js new file mode 100644 index 0000000..ab6bfec --- /dev/null +++ b/app/src/services/geoJsonApi.js @@ -0,0 +1,82 @@ +const NETWORK_PAGE_SIZE = 500; + +const fetchNetworkPage = async ({ + collection, + selectedSubid, + offset, + signal, +}) => { + const url = new URL( + `${window.location.origin}/bbox-server/collections/${collection}/items.json` + ); + url.searchParams.set("network_subid", selectedSubid); + url.searchParams.set("limit", NETWORK_PAGE_SIZE); + url.searchParams.set("offset", offset); + + const response = await fetch(url, { signal }); + if (!response.ok) { + throw new Error(`Failed to fetch network GeoJSON (${response.status})`); + } + + const page = await response.json(); + if (page.type !== "FeatureCollection" || !Array.isArray(page.features)) { + throw new Error("Invalid network FeatureCollection response"); + } + return page; +}; + +export const fetchNetworkGeoJson = async ({ + selectedSubid, + direction, + signal, +}) => { + const collection = `${direction}_feature_collections`; + const blobParts = ['{"type":"FeatureCollection","features":[']; + let offset = 0; + let numberMatched = null; + let hasFeatures = false; + let featureCount = 0; + + while (numberMatched === null || offset < numberMatched) { + const page = await fetchNetworkPage({ + collection, + selectedSubid, + offset, + signal, + }); + const serializedFeatures = page.features.map(JSON.stringify).join(","); + if (serializedFeatures) { + if (hasFeatures) blobParts.push(","); + blobParts.push(serializedFeatures); + hasFeatures = true; + } + + featureCount += page.features.length; + offset += page.features.length; + const reportedTotal = Number(page.numberMatched); + numberMatched = Number.isFinite(reportedTotal) + ? reportedTotal + : page.features.length < NETWORK_PAGE_SIZE + ? offset + : Number.POSITIVE_INFINITY; + + if (page.features.length === 0 || page.features.length < NETWORK_PAGE_SIZE) { + break; + } + } + + if (featureCount <= 1) { + throw new Error(`No ${direction} outlets found`); + } + + blobParts.push("]}"); + const blob = new Blob(blobParts, { + type: "application/geo+json", + }); + const filename = `${selectedSubid}_${direction}.geojson`.replace( + /[^A-Za-z0-9_.-]+/g, + "_" + ); + + return { blob, filename }; +}; diff --git a/app/src/services/streamNetApi.js b/app/src/services/streamNetApi.js index 9464524..fbb08a9 100644 --- a/app/src/services/streamNetApi.js +++ b/app/src/services/streamNetApi.js @@ -10,14 +10,30 @@ const fetchStreamNetwork = async (subid, selectedUid, direction) => { throw new Error(`Failed to fetch ${direction}: ${response.statusText}`); } const json = await response.json(); - const network = json.properties[`${direction.substring(0, direction.length - 1)}_uids`]; - return network.filter(uid => uid != selectedUid); + const propertyPrefix = direction.substring(0, direction.length - 1); + const uids = json.properties[`${propertyPrefix}_uids`] ?? []; + const subids = json.properties[`${propertyPrefix}_subids`] ?? []; + + return { + // The selected feature has its own style, so it is not highlighted as + // part of the surrounding network. It remains in subids for downloads. + uids: uids.filter(uid => uid != selectedUid), + subids: [...new Set(subids.map(String))], + }; }; export const fetchUpstreams = async (subid, uid) => { - return fetchStreamNetwork(subid, uid, "upstreams"); + return (await fetchStreamNetwork(subid, uid, "upstreams")).uids; } export const fetchDownstreams = async (subid, uid) => { + return (await fetchStreamNetwork(subid, uid, "downstreams")).uids; +} + +export const fetchUpstreamNetwork = async (subid, uid) => { + return fetchStreamNetwork(subid, uid, "upstreams"); +}; + +export const fetchDownstreamNetwork = async (subid, uid) => { return fetchStreamNetwork(subid, uid, "downstreams"); -} \ No newline at end of file +}; diff --git a/app/src/services/timeseriesApi.js b/app/src/services/timeseriesApi.js index 84c93f5..ee77275 100644 --- a/app/src/services/timeseriesApi.js +++ b/app/src/services/timeseriesApi.js @@ -77,11 +77,11 @@ export const getAvailableOptions = async (outletId) => { export const getApiVariable = async (displayName) => { const variableInfo = await fetchVariableInfo(); return Object.entries(variableInfo).find( - ([key, info]) => displayName === getDisplayName(info.name, info.units) + ([, info]) => displayName === getDisplayName(info.name, info.units) )?.[0]; }; -export const downloadTimeseries = async (outletId, selections) => { +export const getTimeseriesDownload = async (outletId, selections) => { const [allTimeseries, apiVariable] = await Promise.all([ fetchTimeseriesForOutlet(outletId), getApiVariable(selections.variable), @@ -96,11 +96,42 @@ export const downloadTimeseries = async (outletId, selections) => { if (!timeseriesId) throw new Error("No matching timeseries found"); - const url = `${BASE_URL}/outlets/${outletId}/timeseries/${timeseriesId}/data`; - const a = document.createElement("a"); - a.href = url; - a.download = `${outletId}_${selections.model}_${selections.scenario}_${apiVariable}.csv`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); + return { + url: `${BASE_URL}/outlets/${outletId}/timeseries/${timeseriesId}/data`, + filename: `${outletId}_${selections.model}_${selections.scenario}_${apiVariable}.csv`, + }; +}; + +export const fetchBulkTimeseries = async ( + outletId, + direction, + subids, + selections +) => { + if (subids.length <= 1) throw new Error(`No ${direction} outlets found`); + + const apiVariable = await getApiVariable(selections.variable); + if (!apiVariable) throw new Error("No matching variable found"); + + const response = await fetch(`${BASE_URL}/bulk-downloads`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + subids, + model: selections.model, + scenario: selections.scenario, + variable: apiVariable, + format: "netcdf", + }), + }); + + if (!response.ok) { + const details = await response.text(); + throw new Error(details || `Bulk download failed (${response.status})`); + } + + const filename = `${outletId}_${direction}_${selections.model}_${selections.scenario}_${apiVariable}.nc` + .replace(/[^A-Za-z0-9_.-]+/g, "_"); + + return { blob: await response.blob(), filename }; }; diff --git a/app/src/styles.js b/app/src/styles.js index 65938d2..2d7c322 100644 --- a/app/src/styles.js +++ b/app/src/styles.js @@ -40,15 +40,15 @@ export const interactionStyles = { }, downstream: { weight: 4, - color: "#88419d", - fillColor: "#88419d", + color: "#4500cf", + fillColor: "#4500cf", fill: true, opacity: .8, }, upstream: { weight: 4, - color: "#8cc0c6", - fillColor: "#8cc0c6", + color: "#06d9cb", + fillColor: "#06d9cb", fill: true, opacity: .8, }, diff --git a/app/src/utils/downloadFile.js b/app/src/utils/downloadFile.js new file mode 100644 index 0000000..c4c6f60 --- /dev/null +++ b/app/src/utils/downloadFile.js @@ -0,0 +1,14 @@ +export const downloadUrl = (url, filename) => { + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = filename; + document.body.appendChild(anchor); + anchor.click(); + document.body.removeChild(anchor); +}; + +export const downloadBlob = (blob, filename) => { + const url = URL.createObjectURL(blob); + downloadUrl(url, filename); + setTimeout(() => URL.revokeObjectURL(url), 0); +}; diff --git a/app/vite.config.js b/app/vite.config.js index 4c32e6e..710b3ec 100644 --- a/app/vite.config.js +++ b/app/vite.config.js @@ -9,10 +9,13 @@ export default defineConfig({ EnvironmentPlugin({ REACT_APP_BC_BASE_MAP_TILES_URL: - "https://swarm.pacificclimate.org/tiles/bc-albers-lite/{z}/{x}/{y}.png", + "https://services.pacificclimate.org/tiles/bc-albers-lite/{z}/{x}/{y}.png", }), ], base: "/chyp", + build: { + sourcemap: true, + }, // Local Dev only server: { warmup: { @@ -25,5 +28,15 @@ export default defineConfig({ }, port: 3000, host: true, + proxy: { + "/bbox-server": { + target: "https://beehive.pacificclimate.org", + changeOrigin: true, + }, + "/hydromosaic": { + target: "https://beehive.pacificclimate.org", + changeOrigin: true, + }, + }, }, }); diff --git a/config/bbox.template.toml b/config/bbox.template.toml index 0bc69ac..335eb72 100644 --- a/config/bbox.template.toml +++ b/config/bbox.template.toml @@ -25,7 +25,7 @@ FROM lakes """ fid_field = "subid" geometry_field = "geojson" -queryable_fields = ["Uid","SubId","DowSubId"] +queryable_fields = ["uid","subid","dowsubid"] [[collection]] name = "rivers" @@ -44,7 +44,7 @@ FROM rivers """ fid_field = "subid" geometry_field = "geojson" -queryable_fields = ["Uid","SubId","DowSubId"] +queryable_fields = ["uid","subid","dowsubid"] [[collection]] name = "upstreams" @@ -62,7 +62,7 @@ FROM upstreams """ fid_field = "subid" geometry_field = "origin" -queryable_fields = ["SubId"] +queryable_fields = ["subid"] [[collection]] name="downstreams" @@ -80,7 +80,97 @@ FROM downstreams """ fid_field = "subid" geometry_field = "mouth" -queryable_fields = ["SubId"] +queryable_fields = ["subid"] + +[[collection]] +name = "upstream_feature_collections" +title = "Upstream Network Features" +description = "Paginated river and lake features belonging to upstream networks" +[collection.postgis] +datasource = "postgis_db" +sql = """ +SELECT + concat(network.subid, ':', member.position, ':', water.source) AS network_member_id, + network.subid AS network_subid, + member.position AS network_position, + water.subid, + water.uid, + water.dowsubid, + water.islake, + water.source, + ST_SetSRID(water.geom, 3005) AS geom +FROM upstreams network +CROSS JOIN LATERAL unnest(network.upstream_subids) + WITH ORDINALITY AS member(subid, position) +JOIN LATERAL ( + SELECT subid, uid, dowsubid, islake, 'river' AS source, geom + FROM rivers + WHERE subid = member.subid + UNION ALL + SELECT subid, uid, dowsubid, islake, 'lake' AS source, geom + FROM lakes + WHERE subid = member.subid +) water ON TRUE +ORDER BY network.subid, member.position, water.source +""" +fid_field = "network_member_id" +geometry_field = "geom" +queryable_fields = ["network_subid"] + +[[collection]] +name = "downstream_feature_collections" +title = "Downstream Network Features" +description = "Paginated river and lake features belonging to downstream networks" +[collection.postgis] +datasource = "postgis_db" +sql = """ +SELECT + concat(network.subid, ':', member.position, ':', water.source) AS network_member_id, + network.subid AS network_subid, + member.position AS network_position, + water.subid, + water.uid, + water.dowsubid, + water.islake, + water.source, + ST_SetSRID(water.geom, 3005) AS geom +FROM downstreams network +CROSS JOIN LATERAL unnest(network.downstream_subids) + WITH ORDINALITY AS member(subid, position) +JOIN LATERAL ( + SELECT subid, uid, dowsubid, islake, 'river' AS source, geom + FROM rivers + WHERE subid = member.subid + UNION ALL + SELECT subid, uid, dowsubid, islake, 'lake' AS source, geom + FROM lakes + WHERE subid = member.subid +) water ON TRUE +ORDER BY network.subid, member.position, water.source +""" +fid_field = "network_member_id" +geometry_field = "geom" +queryable_fields = ["network_subid"] + +[[collection]] +name = "import_status" +title = "CHYP import status" +description = "Completed CHYP data imports" +[collection.postgis] +datasource = "postgis_db" +sql = """ +SELECT + import_id, + status, + river_count, + lake_count, + ST_AsGeoJSON(ST_SetSRID(ST_Point(52.628, -118.430), 3005)) AS geojson +FROM chyp_import_status +WHERE status = 'ready' +""" +fid_field = "import_id" +geometry_field = "geojson" +queryable_fields = ["import_id", "status"] [[grid]] json = "assets/BCAlbersCustomGrid.json" @@ -123,4 +213,3 @@ sql = """ FROM lakes WHERE geom && ST_MakeEnvelope($1, $2, $3, $4, 3005) """ - diff --git a/docker/BBOX/Dockerfile b/docker/BBOX/Dockerfile index b405bc9..089dcdc 100644 --- a/docker/BBOX/Dockerfile +++ b/docker/BBOX/Dockerfile @@ -3,9 +3,10 @@ FROM sourcepole/bbox-server-qgis # Switch to root to install packages USER root -# Install psql client and envsubst +# Install psql client, config templating, and healthcheck client. RUN apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ + curl \ gettext-base \ postgresql-client && \ rm -rf /var/lib/apt/lists/* @@ -16,4 +17,3 @@ RUN chmod +x /usr/local/bin/start-bbox-server.sh RUN mkdir -p /tmp/tilecache && chmod 777 /tmp/tilecache ENTRYPOINT ["/usr/local/bin/start-bbox-server.sh"] - diff --git a/docker/BBOX/start-bbox-server.sh b/docker/BBOX/start-bbox-server.sh index b8b7ef5..2725866 100644 --- a/docker/BBOX/start-bbox-server.sh +++ b/docker/BBOX/start-bbox-server.sh @@ -3,14 +3,56 @@ set -e echo "Starting bbox-server entrypoint..." -echo "Waiting for data import to finish..." -until psql "$DB_DSN" -c "SELECT 1 FROM rivers LIMIT 1;" 2>/dev/null; do - echo "Data not loaded yet, sleeping..." - sleep 2 +if [ -z "${CHYP_IMPORT_ID:-}" ]; then + echo "ERROR: CHYP_IMPORT_ID must be set." + exit 1 +fi + +case "$CHYP_IMPORT_ID" in + *[!A-Za-z0-9._-]*) + echo "ERROR: CHYP_IMPORT_ID may contain only letters, numbers, dots, underscores, and hyphens." + exit 1 + ;; +esac + +read_import_status() { + psql "$DB_DSN" \ + -v ON_ERROR_STOP=1 \ + -v import_id="$CHYP_IMPORT_ID" \ + -tA 2>/dev/null <<-'EOSQL' + SELECT status + FROM chyp_import_status + WHERE import_id = :'import_id'; +EOSQL +} + +echo "Waiting for data import ${CHYP_IMPORT_ID} to finish..." +while true; do + if IMPORT_STATUS=$(read_import_status); then + case "$IMPORT_STATUS" in + ready) + echo "Data import ${CHYP_IMPORT_ID} is ready." + break + ;; + failed) + echo "Data import ${CHYP_IMPORT_ID} failed; waiting for its retry." + ;; + running) + echo "Data import ${CHYP_IMPORT_ID} is still running." + ;; + *) + echo "Data import ${CHYP_IMPORT_ID} has not started yet." + ;; + esac + else + echo "PostGIS or the import status table is not ready yet." + fi + + sleep 5 done echo "Reading DB password from secret..." -POSTGRES_PASSWORD=`cat /run/secrets/bbox-postgis-SU` +POSTGRES_PASSWORD=$(cat /run/secrets/bbox-postgis-SU) echo "Parsing DB host from DB_DSN..." DB_HOST=$(echo "$DB_DSN" | sed -E 's#^postgresql://[^@]+@([^:/?]+).*#\1#') diff --git a/docker/app/server.js b/docker/app/server.js index 56c6a0a..95b315d 100644 --- a/docker/app/server.js +++ b/docker/app/server.js @@ -1,10 +1,29 @@ const http = require("node:http"); const fs = require("node:fs"); const path = require("node:path"); +const zlib = require("node:zlib"); const PORT = Number(process.env.PORT || 8080); const DIST_DIR = path.join(__dirname, "dist"); const BASE_PATH = "/chyp"; +const STARTUP_RETRY_MS = Number(process.env.CHYP_STARTUP_RETRY_MS || 5000); +const STARTUP_TIMEOUT_MS = Number(process.env.CHYP_STARTUP_TIMEOUT_MS || 5000); +const STARTUP_URLS = (process.env.CHYP_STARTUP_URLS || "") + .split(",") + .map((value) => value.trim()) + .filter(Boolean) + .map((value) => new URL(value).toString()); +const RUNTIME_CONFIG_PLACEHOLDER = "window.__CHYP_CONFIG__ = {};"; +const COMPRESSIBLE_EXTENSIONS = new Set([ + ".css", + ".html", + ".js", + ".json", + ".map", + ".svg", + ".txt", +]); +const compressedResponseCache = new Map(); const MIME_TYPES = { ".css": "text/css; charset=utf-8", @@ -19,7 +38,8 @@ const MIME_TYPES = { ".webp": "image/webp", }; -const IMMUTABLE_ASSET_RE = /\/assets\/.+-[A-Za-z0-9_-]{8,}\.[^.]+$/; +const IMMUTABLE_ASSET_RE = + /\/assets\/.+-[A-Za-z0-9_-]{8,}(?:\.[^.]+)+$/; const getContentType = (filepath) => MIME_TYPES[path.extname(filepath).toLowerCase()] || @@ -37,6 +57,66 @@ const getCacheControl = (requestPath) => { return "public, max-age=3600"; }; +const getContentEncoding = (acceptEncoding = "") => { + const accepted = new Map( + acceptEncoding.split(",").map((entry) => { + const [encoding, ...parameters] = entry.trim().toLowerCase().split(";"); + const qualityParameter = parameters.find((value) => + value.trim().startsWith("q=") + ); + const quality = qualityParameter + ? Number(qualityParameter.trim().slice(2)) + : 1; + return [encoding, Number.isFinite(quality) ? quality : 0]; + }) + ); + const qualityFor = (encoding) => + accepted.has(encoding) + ? accepted.get(encoding) + : accepted.get("*") || 0; + const brotliQuality = qualityFor("br"); + const gzipQuality = qualityFor("gzip"); + + if (brotliQuality > 0 && brotliQuality >= gzipQuality) { + return "br"; + } + if (gzipQuality > 0) { + return "gzip"; + } + return null; +}; + +const compressResponse = (body, encoding, cacheKey, callback) => { + const cached = compressedResponseCache.get(cacheKey); + if (cached) { + callback(null, cached); + return; + } + + const done = (err, compressedBody) => { + if (!err) { + compressedResponseCache.set(cacheKey, compressedBody); + } + callback(err, compressedBody); + }; + + if (encoding === "br") { + zlib.brotliCompress( + body, + { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: 5, + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: body.length, + }, + }, + done + ); + return; + } + + zlib.gzip(body, { level: 6 }, done); +}; + const normalizePath = (requestPath) => { if (requestPath === BASE_PATH || requestPath === `${BASE_PATH}/`) { return "/"; @@ -49,7 +129,50 @@ const normalizePath = (requestPath) => { return requestPath; }; -const sendFile = (res, filepath, requestPath) => { +const delay = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +const waitForStartupUrl = async (url) => { + let lastError; + + while (true) { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(STARTUP_TIMEOUT_MS), + }); + + if (response.ok) { + await response.body?.cancel(); + console.log(`Startup dependency is ready: ${url}`); + return; + } + + await response.body?.cancel(); + throw new Error(`HTTP ${response.status}`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message !== lastError) { + console.log(`Waiting for startup dependency ${url}: ${message}`); + lastError = message; + } + await delay(STARTUP_RETRY_MS); + } + } +}; + +const getRuntimeConfigAssignment = () => { + const tileUrl = ( + process.env.REACT_APP_BC_BASE_MAP_TILES_URL || "" + ).trim(); + const config = tileUrl + ? { REACT_APP_BC_BASE_MAP_TILES_URL: tileUrl } + : {}; + + const serializedConfig = JSON.stringify(config).replaceAll("<", "\\u003c"); + return `window.__CHYP_CONFIG__ = ${serializedConfig};`; +}; + +const sendFile = (req, res, filepath, requestPath) => { fs.readFile(filepath, (err, data) => { if (err) { res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" }); @@ -57,40 +180,102 @@ const sendFile = (res, filepath, requestPath) => { return; } - res.writeHead(200, { + const responseBody = filepath.endsWith(".html") + ? data + .toString() + .replace(RUNTIME_CONFIG_PLACEHOLDER, getRuntimeConfigAssignment()) + : data; + + const headers = { "Content-Type": getContentType(filepath), "Cache-Control": getCacheControl(requestPath), - }); - res.end(data); - }); -}; + }; + const body = Buffer.isBuffer(responseBody) + ? responseBody + : Buffer.from(responseBody); + const canCompress = + body.length >= 1024 && + COMPRESSIBLE_EXTENSIONS.has(path.extname(filepath).toLowerCase()); + const encoding = canCompress + ? getContentEncoding(req.headers["accept-encoding"]) + : null; -http - .createServer((req, res) => { - if (!req.url) { - res.writeHead(400, { "Content-Type": "text/plain; charset=utf-8" }); - res.end("Bad Request"); - return; + if (canCompress) { + headers.Vary = "Accept-Encoding"; } - const url = new URL(req.url, `http://${req.headers.host || "localhost"}`); - const requestPath = normalizePath(decodeURIComponent(url.pathname)); - const safePath = path.normalize(requestPath).replace(/^(\.\.[/\\])+/, ""); - - let filepath = path.join(DIST_DIR, safePath); - if (requestPath === "/" || requestPath === "") { - filepath = path.join(DIST_DIR, "index.html"); + if (!encoding) { + res.writeHead(200, headers); + res.end(body); + return; } - fs.stat(filepath, (err, stat) => { - if (!err && stat.isFile()) { - sendFile(res, filepath, requestPath); + compressResponse(body, encoding, `${filepath}:${encoding}`, (error, compressedBody) => { + if (error) { + res.writeHead(200, headers); + res.end(body); return; } - sendFile(res, path.join(DIST_DIR, "index.html"), "/"); + res.writeHead(200, { + ...headers, + "Content-Encoding": encoding, + }); + res.end(compressedBody); + }); + }); +}; + +const server = http.createServer((req, res) => { + if (!req.url) { + res.writeHead(400, { "Content-Type": "text/plain; charset=utf-8" }); + res.end("Bad Request"); + return; + } + + const url = new URL(req.url, `http://${req.headers.host || "localhost"}`); + if (url.pathname === "/healthz") { + res.writeHead(200, { + "Content-Type": "text/plain; charset=utf-8", + "Cache-Control": "no-store", }); - }) - .listen(PORT, () => { + res.end("ok"); + return; + } + + const requestPath = normalizePath(decodeURIComponent(url.pathname)); + + const safePath = path.normalize(requestPath).replace(/^(\.\.[/\\])+/, ""); + + let filepath = path.join(DIST_DIR, safePath); + if (requestPath === "/" || requestPath === "") { + filepath = path.join(DIST_DIR, "index.html"); + } + + fs.stat(filepath, (err, stat) => { + if (!err && stat.isFile()) { + sendFile(req, res, filepath, requestPath); + return; + } + + sendFile(req, res, path.join(DIST_DIR, "index.html"), "/"); + }); +}); + +const startServer = async () => { + if (STARTUP_URLS.length) { + console.log( + `Waiting for ${STARTUP_URLS.length} startup dependency URL(s)...` + ); + await Promise.all(STARTUP_URLS.map(waitForStartupUrl)); + } + + server.listen(PORT, () => { console.log(`Serving dist on port ${PORT}`); }); +}; + +startServer().catch((error) => { + console.error("Unable to start the application server:", error); + process.exitCode = 1; +}); diff --git a/docker/data-import/import_data.sh b/docker/data-import/import_data.sh index 6f25db5..1881c35 100755 --- a/docker/data-import/import_data.sh +++ b/docker/data-import/import_data.sh @@ -1,16 +1,115 @@ #!/bin/sh set -e -echo "Waiting for PostGIS..." -until pg_isready -d "$DB_DSN"; do - sleep 1 +if [ -z "${CHYP_IMPORT_ID:-}" ]; then + echo "ERROR: CHYP_IMPORT_ID must be set." + exit 1 +fi + +case "$CHYP_IMPORT_ID" in + *[!A-Za-z0-9._-]*) + echo "ERROR: CHYP_IMPORT_ID may contain only letters, numbers, dots, underscores, and hyphens." + exit 1 + ;; +esac + +echo "Waiting for the initialized PostGIS database..." +until psql "$DB_DSN" -v ON_ERROR_STOP=1 -tAc "SELECT 1" >/dev/null 2>&1; do + sleep 2 done -echo "Dropping existing tables if they exist..." -psql "$DB_DSN" <<-EOSQL - DROP TABLE IF EXISTS rivers CASCADE; - DROP TABLE IF EXISTS lakes CASCADE; - DROP SEQUENCE IF EXISTS shared_uid_seq CASCADE; +psql "$DB_DSN" -v ON_ERROR_STOP=1 <<-'EOSQL' + CREATE TABLE IF NOT EXISTS chyp_import_status ( + import_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + completed_at TIMESTAMPTZ, + river_count BIGINT, + lake_count BIGINT + ); +EOSQL + +IMPORT_IS_READY=$( + psql "$DB_DSN" \ + -v ON_ERROR_STOP=1 \ + -v import_id="$CHYP_IMPORT_ID" \ + -tA <<-'EOSQL' + SELECT ( + EXISTS ( + SELECT 1 + FROM chyp_import_status + WHERE import_id = :'import_id' + AND status = 'ready' + AND river_count IS NOT NULL + AND lake_count IS NOT NULL + ) + AND to_regclass('public.rivers') IS NOT NULL + AND to_regclass('public.lakes') IS NOT NULL + AND to_regclass('public.upstreams') IS NOT NULL + AND to_regclass('public.downstreams') IS NOT NULL + ); +EOSQL +) + +if [ "$IMPORT_IS_READY" = "t" ] && [ "${CHYP_FORCE_IMPORT:-false}" != "true" ]; then + echo "Import ${CHYP_IMPORT_ID} is already ready; nothing to do." + exit 0 +fi + +echo "Recording import ${CHYP_IMPORT_ID} as running..." +psql "$DB_DSN" \ + -v ON_ERROR_STOP=1 \ + -v import_id="$CHYP_IMPORT_ID" <<-'EOSQL' + INSERT INTO chyp_import_status ( + import_id, + status, + started_at, + completed_at, + river_count, + lake_count + ) + VALUES ( + :'import_id', + 'running', + clock_timestamp(), + NULL, + NULL, + NULL + ) + ON CONFLICT (import_id) DO UPDATE + SET status = EXCLUDED.status, + started_at = EXCLUDED.started_at, + completed_at = NULL, + river_count = NULL, + lake_count = NULL; +EOSQL + +mark_import_failed() { + exit_code=$1 + + if [ "$exit_code" -eq 0 ]; then + return + fi + + echo "Import ${CHYP_IMPORT_ID} failed with exit code ${exit_code}." + psql "$DB_DSN" \ + -v ON_ERROR_STOP=1 \ + -v import_id="$CHYP_IMPORT_ID" <<-'EOSQL' || true + UPDATE chyp_import_status + SET status = 'failed', + completed_at = clock_timestamp() + WHERE import_id = :'import_id'; +EOSQL +} + +trap 'mark_import_failed "$?"' EXIT + +echo "Preparing staging tables..." +psql "$DB_DSN" -v ON_ERROR_STOP=1 <<-EOSQL + DROP MATERIALIZED VIEW IF EXISTS upstreams_next; + DROP MATERIALIZED VIEW IF EXISTS downstreams_next; + DROP TABLE IF EXISTS rivers_next CASCADE; + DROP TABLE IF EXISTS lakes_next CASCADE; EOSQL import_dataset() { @@ -27,6 +126,7 @@ import_dataset() { -nln "$table" \ -lco GEOMETRY_NAME=geom \ -lco FID=fid \ + -lco SPATIAL_INDEX=NONE \ -a_srs EPSG:3005 \ "$@" \ -addfields @@ -34,66 +134,58 @@ import_dataset() { echo "PostGIS is ready. Importing rivers data..." echo "Importing Fraser rivers" -import_dataset Fraser_3005_rivers.gpkg MULTILINESTRING rivers +import_dataset Fraser_3005_rivers.gpkg MULTILINESTRING rivers_next echo "Importing BC Coast rivers" -import_dataset BC_Coast_3005_rivers.gpkg MULTILINESTRING rivers -append +import_dataset BC_Coast_3005_rivers.gpkg MULTILINESTRING rivers_next -append echo "Importing Peace rivers" -import_dataset Peace_3005_rivers.gpkg MULTILINESTRING rivers -append +import_dataset Peace_3005_rivers.gpkg MULTILINESTRING rivers_next -append echo "Importing Columbia rivers" -import_dataset Columbia_3005_rivers.gpkg MULTILINESTRING rivers -append +import_dataset Columbia_3005_rivers.gpkg MULTILINESTRING rivers_next -append echo "Rivers data imported. Importing lakes data..." echo "Importing Fraser lakes" -import_dataset Fraser_3005_lakes.gpkg MULTIPOLYGON lakes +import_dataset Fraser_3005_lakes.gpkg MULTIPOLYGON lakes_next echo "Importing BC Coast lakes" -import_dataset BC_Coast_3005_lakes.gpkg MULTIPOLYGON lakes -append +import_dataset BC_Coast_3005_lakes.gpkg MULTIPOLYGON lakes_next -append echo "Importing Peace lakes" -import_dataset Peace_3005_lakes.gpkg MULTIPOLYGON lakes -append +import_dataset Peace_3005_lakes.gpkg MULTIPOLYGON lakes_next -append echo "Importing Columbia lakes" -import_dataset Columbia_3005_lakes.gpkg MULTIPOLYGON lakes -append +import_dataset Columbia_3005_lakes.gpkg MULTIPOLYGON lakes_next -append echo "Updating tables and adding indices..." -psql "$DB_DSN" <<-EOSQL - DO \$\$ - BEGIN - -- Create a shared sequence for unique IDs - IF NOT EXISTS (SELECT 1 FROM pg_sequences WHERE schemaname = 'public' AND sequencename = 'shared_uid_seq') THEN - CREATE SEQUENCE shared_uid_seq START 1; - END IF; +psql "$DB_DSN" -v ON_ERROR_STOP=1 <<-EOSQL + CREATE TEMP SEQUENCE chyp_uid_seq START 1; + + ALTER TABLE rivers_next ADD COLUMN Uid INT; + ALTER TABLE rivers_next ADD COLUMN IsLake BOOLEAN DEFAULT FALSE; + UPDATE rivers_next SET uid = nextval('chyp_uid_seq'); - -- Update rivers table - ALTER TABLE rivers ADD COLUMN Uid INT; - ALTER TABLE rivers ADD COLUMN IsLake BOOLEAN DEFAULT FALSE; - UPDATE rivers SET uid = nextval('shared_uid_seq'); - - -- Update lakes table - ALTER TABLE lakes ADD COLUMN Uid INT; - ALTER TABLE lakes ADD COLUMN IsLake BOOLEAN DEFAULT TRUE; - UPDATE lakes SET uid = nextval('shared_uid_seq'); - END \$\$; - - CREATE INDEX rivers_geom_idx ON rivers USING GIST(geom); - CREATE INDEX lakes_geom_idx ON lakes USING GIST(geom); - VACUUM ANALYZE rivers; - VACUUM ANALYZE lakes; + ALTER TABLE lakes_next ADD COLUMN Uid INT; + ALTER TABLE lakes_next ADD COLUMN IsLake BOOLEAN DEFAULT TRUE; + UPDATE lakes_next SET uid = nextval('chyp_uid_seq'); + + CREATE INDEX rivers_next_geom_idx ON rivers_next USING GIST(geom); + CREATE INDEX lakes_next_geom_idx ON lakes_next USING GIST(geom); + CREATE INDEX rivers_next_subid_idx ON rivers_next(subid); + CREATE INDEX lakes_next_subid_idx ON lakes_next(subid); + VACUUM ANALYZE rivers_next; + VACUUM ANALYZE lakes_next; EOSQL echo "Creating upstreams and downstreams tables..." -psql "$DB_DSN" <<-EOSQL - DROP MATERIALIZED VIEW IF EXISTS upstreams; - - CREATE MATERIALIZED VIEW upstreams AS +psql "$DB_DSN" -v ON_ERROR_STOP=1 <<-EOSQL + CREATE MATERIALIZED VIEW upstreams_next AS WITH RECURSIVE drainage(subid, downsubid, uid, mouth) AS ( WITH segments(subid, dowsubid, uid) AS ( - SELECT subid, dowsubid, uid FROM lakes + SELECT subid, dowsubid, uid FROM lakes_next UNION - SELECT subid, dowsubid, uid FROM rivers + SELECT subid, dowsubid, uid FROM rivers_next ) SELECT subid, dowsubid, uid, subid AS mouth @@ -111,14 +203,12 @@ psql "$DB_DSN" <<-EOSQL ST_AsGeoJSON(ST_SetSRID(ST_Point(52.628, -118.430, 4326), 3005)) AS origin FROM drainage GROUP BY mouth; -DROP MATERIALIZED VIEW IF EXISTS downstreams; - -CREATE MATERIALIZED VIEW downstreams AS +CREATE MATERIALIZED VIEW downstreams_next AS WITH RECURSIVE course(subid, downsubid, uid, origin) AS ( WITH segments(subid, dowsubid, uid) AS ( - SELECT subid, dowsubid, uid FROM lakes + SELECT subid, dowsubid, uid FROM lakes_next UNION - SELECT subid, dowsubid, uid FROM rivers + SELECT subid, dowsubid, uid FROM rivers_next ) SELECT subid, dowsubid, uid, subid AS origin @@ -135,10 +225,66 @@ CREATE MATERIALIZED VIEW downstreams AS ST_AsGeoJSON(ST_SetSRID(ST_Point(49.1778, -123.241, 4326), 3005)) AS mouth FROM course GROUP BY origin; - CREATE INDEX upstream_uid_idx ON upstreams(subid); - CREATE INDEX downstream_uid_idx ON downstreams(subid); - VACUUM ANALYZE upstreams; - VACUUM ANALYZE downstreams; + CREATE INDEX upstreams_next_uid_idx ON upstreams_next(subid); + CREATE INDEX downstreams_next_uid_idx ON downstreams_next(subid); + VACUUM ANALYZE upstreams_next; + VACUUM ANALYZE downstreams_next; +EOSQL + +echo "Atomically publishing import ${CHYP_IMPORT_ID}..." +psql "$DB_DSN" \ + -v ON_ERROR_STOP=1 \ + -v import_id="$CHYP_IMPORT_ID" <<-'EOSQL' + BEGIN; + + DROP MATERIALIZED VIEW IF EXISTS upstreams_old; + DROP MATERIALIZED VIEW IF EXISTS downstreams_old; + DROP TABLE IF EXISTS rivers_old CASCADE; + DROP TABLE IF EXISTS lakes_old CASCADE; + + DO $$ + BEGIN + IF to_regclass('public.rivers') IS NOT NULL THEN + EXECUTE 'ALTER TABLE public.rivers RENAME TO rivers_old'; + END IF; + IF to_regclass('public.lakes') IS NOT NULL THEN + EXECUTE 'ALTER TABLE public.lakes RENAME TO lakes_old'; + END IF; + IF to_regclass('public.upstreams') IS NOT NULL THEN + EXECUTE 'ALTER MATERIALIZED VIEW public.upstreams RENAME TO upstreams_old'; + END IF; + IF to_regclass('public.downstreams') IS NOT NULL THEN + EXECUTE 'ALTER MATERIALIZED VIEW public.downstreams RENAME TO downstreams_old'; + END IF; + END $$; + + ALTER TABLE rivers_next RENAME TO rivers; + ALTER TABLE lakes_next RENAME TO lakes; + ALTER MATERIALIZED VIEW upstreams_next RENAME TO upstreams; + ALTER MATERIALIZED VIEW downstreams_next RENAME TO downstreams; + + DROP MATERIALIZED VIEW IF EXISTS upstreams_old; + DROP MATERIALIZED VIEW IF EXISTS downstreams_old; + DROP TABLE IF EXISTS rivers_old CASCADE; + DROP TABLE IF EXISTS lakes_old CASCADE; + DROP SEQUENCE IF EXISTS shared_uid_seq; + + ALTER INDEX rivers_next_geom_idx RENAME TO rivers_geom_idx; + ALTER INDEX lakes_next_geom_idx RENAME TO lakes_geom_idx; + ALTER INDEX rivers_next_subid_idx RENAME TO rivers_subid_idx; + ALTER INDEX lakes_next_subid_idx RENAME TO lakes_subid_idx; + ALTER INDEX upstreams_next_uid_idx RENAME TO upstream_uid_idx; + ALTER INDEX downstreams_next_uid_idx RENAME TO downstream_uid_idx; + + UPDATE chyp_import_status + SET status = 'ready', + completed_at = clock_timestamp(), + river_count = (SELECT count(*) FROM rivers), + lake_count = (SELECT count(*) FROM lakes) + WHERE import_id = :'import_id'; + + COMMIT; EOSQL -echo "Data import complete." +trap - EXIT +echo "Data import ${CHYP_IMPORT_ID} complete." diff --git a/docker/varnish/Dockerfile b/docker/varnish/Dockerfile new file mode 100644 index 0000000..91eff50 --- /dev/null +++ b/docker/varnish/Dockerfile @@ -0,0 +1,15 @@ +FROM varnish:7.5.0 + +USER root + +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + rm -rf /var/lib/apt/lists/* + +COPY docker/varnish/start-varnish.sh /usr/local/bin/start-chyp-varnish.sh +RUN chmod +x /usr/local/bin/start-chyp-varnish.sh + +USER varnish + +ENTRYPOINT ["/usr/local/bin/start-chyp-varnish.sh"] +CMD ["-F", "-f", "/etc/varnish/default.vcl", "-a", ":80", "-s", "file,/var/lib/varnish/cache.bin,4G"] diff --git a/docker/varnish/start-varnish.sh b/docker/varnish/start-varnish.sh new file mode 100644 index 0000000..0c2b54d --- /dev/null +++ b/docker/varnish/start-varnish.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +: "${CHYP_IMPORT_ID:?CHYP_IMPORT_ID must be set}" + +case "$CHYP_IMPORT_ID" in + *[!A-Za-z0-9._-]*) + echo "ERROR: CHYP_IMPORT_ID may contain only letters, numbers, dots, underscores, and hyphens." + exit 1 + ;; +esac + +READY_URL="${CHYP_SERVER_READY_URL:-http://chyp-server:8080/collections/import_status/items/${CHYP_IMPORT_ID}.json}" +RETRY_SECONDS="${CHYP_STARTUP_RETRY_SECONDS:-5}" + +echo "Waiting for BBOX server at ${READY_URL}..." +until curl --fail --silent --show-error --max-time 5 "$READY_URL" >/dev/null; do + sleep "$RETRY_SECONDS" +done + +echo "BBOX server is ready. Starting Varnish." +rm -f /var/lib/varnish/cache.bin +exec varnishd "$@" diff --git a/docs/swarm-readiness.md b/docs/swarm-readiness.md new file mode 100644 index 0000000..1e29ec0 --- /dev/null +++ b/docs/swarm-readiness.md @@ -0,0 +1,72 @@ +# Swarm readiness + +## Import identifier + +Set a stable identifier tied to the dataset directly in the `environment` +section of `data-import`, `chyp-server`, `chyp-server-varnish`, and `chyp-app`: + +```yaml +- CHYP_IMPORT_ID=fraser-coast-peace-columbia-2026-07 +``` + +The app's readiness URL must contain the same identifier: + +```yaml +- CHYP_STARTUP_URLS=http://chyp-server:8080/collections/import_status/items/fraser-coast-peace-columbia-2026-07.json,http://hydromosaic:8000/variables +``` + +Compose has no top-level environment-variable section, so keep these values +identical when editing an environment-specific stack. There is deliberately no +application default: a missing value must fail rather than accidentally reuse +a development dataset identifier in production. + +Keep the value unchanged for an image or configuration-only redeploy. Change +it when deploying new source data. Use a dataset version rather than an image +build time: every new value requests a complete import. The importer builds +staging relations and publishes them in one database transaction, so existing +readers continue to see the old dataset until the new dataset is complete. + +## Readiness and health checks + +Swarm does not use `depends_on` to sequence service readiness. The stack uses +this chain instead: + +```text +PostGIS -> authenticated import -> matching import-status endpoint -> BBOX -> app and Varnish +``` + +- The importer retries its authenticated database query, records + `running`, `ready`, or `failed`, and skips an already-ready import ID. +- BBOX waits for the matching database marker before starting. +- The app waits for both BBOX's matching import endpoint and Hydromosaic. +- Varnish waits for the same BBOX endpoint and clears its tile cache only when + the new import is ready. +- BBOX, the app, and Varnish have long health-check start periods so a slow + import does not consume restart attempts. A successful check still marks a + task healthy immediately. +- `start-first` updates retain the healthy old reader while its replacement + waits. PostGIS and the one-shot importer use `stop-first`. + +Normal Portainer **Pull and redeploy** is therefore the intended workflow; no +manual service scaling or full stack shutdown is required. Restarting a single +reader with the same `CHYP_IMPORT_ID` is also safe. + +PostGIS must use durable storage that is visible at the same path on every node +eligible to run it. Bind a dedicated directory from the shared storage mount: + +```yaml +postgis: + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - type: bind + source: /storage/.../chyp/postgres + target: /var/lib/postgresql/data + deploy: + ... + placement: + constraints: + - node.labels.pcic-dev == true + - node.labels.pcic-storage == true + ... +``` \ No newline at end of file