From a5079d6f066a6cb10faa3a3f808c6e23da3845bf Mon Sep 17 00:00:00 2001 From: Lorin Gaertner Date: Thu, 5 Mar 2026 10:23:58 -0800 Subject: [PATCH 1/4] addd a preferred format in rsdynamics layer defs --- pipelines/rsdynamics_to_athena/layer_definitions.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/rsdynamics_to_athena/layer_definitions.json b/pipelines/rsdynamics_to_athena/layer_definitions.json index 695ab93..341e872 100644 --- a/pipelines/rsdynamics_to_athena/layer_definitions.json +++ b/pipelines/rsdynamics_to_athena/layer_definitions.json @@ -24,7 +24,8 @@ "friendly_name": "Segment Distance", "dtype": "INTEGER", "data_unit": "m", - "description": "Approximate distance from the upstream headwater of the level path" + "description": "Approximate distance from the upstream headwater of the level path", + "preferred_format": "{:.0f}" }, { "name": "fcode", From 30685907d24012d64b62b1f8ce35ddadb6b68734 Mon Sep 17 00:00:00 2001 From: Lorin Gaertner Date: Mon, 16 Mar 2026 17:01:13 -0700 Subject: [PATCH 2/4] enhance metadata for rme report view --- scripts/rpt_rme/layer_definitions.json | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/rpt_rme/layer_definitions.json b/scripts/rpt_rme/layer_definitions.json index 2eff985..c21b2b8 100644 --- a/scripts/rpt_rme/layer_definitions.json +++ b/scripts/rpt_rme/layer_definitions.json @@ -6,7 +6,7 @@ { "layer_id": "rpt_rme", "layer_name": "rpt_rme_pq", - "description": "report view", + "description": "report view of RME data with additional lookup table columns", "layer_type": "DataTable", "columns": [ { @@ -17,7 +17,7 @@ "description": "", "is_key": false, "is_required": false, - "theme": "", + "theme": "Descriptive", "preferred_bin_definition": "", "default_value": null }, @@ -26,10 +26,22 @@ "dtype": "STRING", "friendly_name": "Ownership", "data_unit": "", - "description": "", + "description": "Dominant ownership within the DGO (full name)", + "is_key": false, + "is_required": false, + "theme": "Descriptive", + "preferred_bin_definition": "", + "default_value": null + }, + { + "name": "rme_project_name", + "dtype": "STRING", + "friendly_name": "RME Project Name", + "data_unit": "", + "description": "Project Name for the source metric engine project on Riverscapes Data Exchange, data.riverscapes.net", "is_key": false, "is_required": false, - "theme": "", + "theme": "meta", "preferred_bin_definition": "", "default_value": null } From a20c1bac42ca3e14103fff56e81335748388a827 Mon Sep 17 00:00:00 2001 From: Lorin Gaertner Date: Mon, 16 Mar 2026 17:03:07 -0700 Subject: [PATCH 3/4] formatting & docstring improvements --- .../rme_to_athena/rme_to_athena_parquet.py | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/pipelines/rme_to_athena/rme_to_athena_parquet.py b/pipelines/rme_to_athena/rme_to_athena_parquet.py index dc7b779..cafc63c 100644 --- a/pipelines/rme_to_athena/rme_to_athena_parquet.py +++ b/pipelines/rme_to_athena/rme_to_athena_parquet.py @@ -1,30 +1,34 @@ """ Searches Data Exchange for RME projects matching criteria Generates parquet files from metrics in the GeoPackages +Adds simplified geometry Uploads to s3 Lorin Gaertner Sept 2025 Enhances Philip's June 2025 rme_to_athena.py +POSSIBLE ENHANCEMENTS: +* Move read of layer_definitions out of the loop +* Use this to validate the input/output (i.e. warn on fields that are not defined) """ + import argparse import json import logging import os -from pathlib import Path import re import shutil import warnings +from pathlib import Path import apsw import boto3 import geopandas as gpd import pandas as pd -from shapely import wkb -from semver import Version - +from rsxml import Logger, ProgressBar, dotenv from rsxml.util import safe_makedirs -from rsxml import dotenv, Logger, ProgressBar +from semver import Version +from shapely import wkb from pydex import RiverscapesAPI, RiverscapesProject from pydex.lib.athena import query_to_dataframe @@ -118,11 +122,7 @@ def get_matching_file(parent_dir: str, regex_str: str) -> str | None: return None -def download_rme_geopackage( - rs_api: RiverscapesAPI, - project: RiverscapesProject, - huc_dir: str | Path -) -> str: +def download_rme_geopackage(rs_api: RiverscapesAPI, project: RiverscapesProject, huc_dir: str | Path) -> str: """ Download the RME GeoPackage for a project and return its file path. """ @@ -219,10 +219,7 @@ def extract_metrics_to_geodataframe(gpkg_path: str, spatialite_path: str) -> gpd bbox_df = gdf.geometry.bounds.rename(columns={'minx': 'xmin', 'miny': 'ymin', 'maxx': 'xmax', 'maxy': 'ymax'}) # Combine into a struct-like dict for each row - gdf['dgo_geom_bbox'] = bbox_df.apply( - lambda row: {'xmin': float(row.xmin), 'ymin': float(row.ymin), 'xmax': float(row.xmax), 'ymax': float(row.ymax)}, - axis=1 - ) + gdf['dgo_geom_bbox'] = bbox_df.apply(lambda row: {'xmin': float(row.xmin), 'ymin': float(row.ymin), 'xmax': float(row.xmax), 'ymax': float(row.ymax)}, axis=1) return gdf @@ -239,11 +236,7 @@ def delete_folder(dirpath: Path) -> None: log.error(f'Error deleting download directory {dirpath}: {e}') -def upload_to_s3( - file_path: str | Path, - s3_bucket: str, - s3_key: str -) -> None: +def upload_to_s3(file_path: str | Path, s3_bucket: str, s3_key: str) -> None: """upload a file to s3 Args: @@ -340,7 +333,7 @@ def main(): parser.add_argument('stage', help='Environment: staging or production', type=str) parser.add_argument('spatialite_path', help='Path to the mod_spatialite library', type=str) parser.add_argument('working_folder', help='top level folder for downloads and output', type=str) - parser.add_argument('--delete', help='Whether or not to delete downloaded GeoPackages', action='store_true', default=False) + parser.add_argument('--delete', help='Whether or not to delete downloaded GeoPackages', action='store_true', default=False) args = dotenv.parse_args_env(parser) # Set up some reasonable folders to store things @@ -361,13 +354,7 @@ def main(): log.info(f"Data bucket: {DATA_BUCKET} (env {DATA_BUCKET_ENV_VAR}); Athena output bucket: {ATHENA_OUTPUT_BUCKET} (env {OUTPUT_BUCKET_ENV_VAR})") with RiverscapesAPI(stage=args.stage) as api: - scrape_rme( - api, - args.spatialite_path, - download_folder, - DATA_BUCKET, - args.delete - ) + scrape_rme(api, args.spatialite_path, download_folder, DATA_BUCKET, args.delete) log.info('Process complete') From 54ae2c9461b134c4bb7b0daa295b67b215587f78 Mon Sep 17 00:00:00 2001 From: Lorin Gaertner Date: Mon, 16 Mar 2026 17:03:17 -0700 Subject: [PATCH 4/4] python venv --- .vscode/settings.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..1c49803 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "python-envs.pythonProjects": [ + { + "path": ".", + "envManager": "ms-python.python:venv", + "packageManager": "ms-python.python:pip" + } + ] +} \ No newline at end of file