From 704fc24b283dd4a6105489df5b82455efe4b7406 Mon Sep 17 00:00:00 2001 From: Lorin Date: Fri, 27 Mar 2026 10:43:23 -0700 Subject: [PATCH 1/3] tweaks to generated sql formatting to match our athena repo convention --- packages/vector_prep/vector_prep/orchestrate.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/vector_prep/vector_prep/orchestrate.py b/packages/vector_prep/vector_prep/orchestrate.py index bd7c17e1..5ac7c517 100644 --- a/packages/vector_prep/vector_prep/orchestrate.py +++ b/packages/vector_prep/vector_prep/orchestrate.py @@ -20,6 +20,7 @@ @dataclass class RunInputs: + """Stuff user needs to supply to establish what is being prepared""" input_vector_path: Path | str input_layer_name: str | None source_category: str @@ -214,7 +215,7 @@ def _map_layer_dtype_to_athena(dtype: str, col_name: str) -> str: def build_athena_ddl(cfg: RunInputs, bucket: str = "riverscapes-athena", database: str = "rs_raw") -> Path: - """Build Athena CREATE EXTERNAL TABLE DDL from cfg and layer_definitions. + """Build Athena CREATE EXTERNAL TABLE DDL from cfg and `layer_definitions.json` Returns the path to the generated .sql file. """ @@ -263,7 +264,7 @@ def build_athena_ddl(cfg: RunInputs, bucket: str = "riverscapes-athena", databas # Capture/read parquet compression from pipeline output metadata so this is not hard-coded. ddl = ( f"CREATE EXTERNAL TABLE `{database}`.`{table_name}`(\n" - + ",\n".join(col_lines) + + ", \n".join(col_lines) + "\n)\n" + "ROW FORMAT SERDE \n" + " 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' \n" @@ -274,11 +275,11 @@ def build_athena_ddl(cfg: RunInputs, bucket: str = "riverscapes-athena", databas + "LOCATION\n" + f" '{location}'\n" + "TBLPROPERTIES (\n" - + f" 'comment'='{_sql_escape(table_comment)}',\n" + " 'classification'='parquet', \n" + + f" 'comment'='{_sql_escape(table_comment)}',\n" + " 'compressionType'='snappy', \n" + " 'typeOfData'='file'\n" - + ");\n" + + ")" ) repo_root = next(p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists()) From 945d753947c92a3e5467b82db6438b31a39816fa Mon Sep 17 00:00:00 2001 From: Lorin Date: Sat, 11 Apr 2026 11:42:47 -0700 Subject: [PATCH 2/3] adding 2 more layers - nm bootheel pastures and USACE NID --- .../vector_prep/fetch_arcgis_metadata.py | 82 ++- .../vector_prep/layer_definitions.json | 103 ++++ .../orchestrate-pastures-bootheel.py | 406 ++++++++++++ .../vector_prep/orchestrate-pastures.py | 326 ++++++++++ .../runlog-pastures-NMbootheel-20260407.md | 103 ++++ .../vector_prep/usace_nid/inputs.json | 11 + .../usace_nid/layer_definitions.json | 576 ++++++++++++++++++ .../vector_prep/usace_nid/runlog-usace-nid.md | 15 + pyproject.toml | 7 +- uv.lock | 7 + 10 files changed, 1610 insertions(+), 26 deletions(-) create mode 100644 packages/vector_prep/vector_prep/orchestrate-pastures-bootheel.py create mode 100644 packages/vector_prep/vector_prep/orchestrate-pastures.py create mode 100644 packages/vector_prep/vector_prep/runlog-pastures-NMbootheel-20260407.md create mode 100644 packages/vector_prep/vector_prep/usace_nid/inputs.json create mode 100644 packages/vector_prep/vector_prep/usace_nid/layer_definitions.json create mode 100644 packages/vector_prep/vector_prep/usace_nid/runlog-usace-nid.md diff --git a/packages/vector_prep/vector_prep/fetch_arcgis_metadata.py b/packages/vector_prep/vector_prep/fetch_arcgis_metadata.py index 24a1d767..abc45776 100644 --- a/packages/vector_prep/vector_prep/fetch_arcgis_metadata.py +++ b/packages/vector_prep/vector_prep/fetch_arcgis_metadata.py @@ -156,7 +156,7 @@ def _resolve_hub_slug(slug: str) -> tuple[str, str, int]: # --------------------------------------------------------------------------- # Feature Service field fetch # --------------------------------------------------------------------------- -def _fetch_service_fields(service_url: str) -> list[dict]: +def fetch_service_fields(service_url: str) -> list[dict]: """GET the feature service layer JSON and return the ``fields`` array.""" resp = requests.get(service_url, params={"f": "json"}, timeout=30) resp.raise_for_status() @@ -201,27 +201,27 @@ def _fetch_fgdc_descriptions(item_id: str) -> dict[str, str]: return descs -# --------------------------------------------------------------------------- -# Core: merge into columns list -# --------------------------------------------------------------------------- -def fetch_columns_from_hub_url(hub_url: str, include_system_fields: bool = False) -> list[dict]: - """Fetch column definitions from an ArcGIS Hub dataset URL. +def build_columns_from_service_fields( + service_fields: list[dict], + field_descriptions: dict[str, str] | None = None, + friendly_name_overrides: dict[str, str] | None = None, + include_system_fields: bool = False, + use_service_alias_as_friendly_name: bool = True, +) -> list[dict]: + """Build layer_definitions ``columns`` from ArcGIS feature-service fields. Args: - hub_url: A Hub dataset page URL, e.g. - ``https://gbp-blm-egis.hub.arcgis.com/datasets/BLM-EGIS::blm-natl-grazing-allotment-polygons/about`` - include_system_fields: If True, include Esri system fields such as - ``OBJECTID``, ``GlobalID``, ``Shape__Area``, etc. - - Returns: - A list of column dicts ready to insert into a layer_definitions.json - ``columns`` array. + service_fields: Raw ``fields`` array from a feature service layer JSON. + field_descriptions: Optional field-name keyed descriptions. + friendly_name_overrides: Optional field-name keyed friendly-name mapping. + If present for a field, override values take precedence over service + aliases. + include_system_fields: If True, include Esri-generated system fields. + use_service_alias_as_friendly_name: If True, use field ``alias`` when a + friendly-name override is not provided. """ - slug = _extract_slug(hub_url) - item_id, service_url, _layer_index = _resolve_hub_slug(slug) - - service_fields = _fetch_service_fields(service_url) - fgdc_descs = _fetch_fgdc_descriptions(item_id) + field_descriptions = field_descriptions or {} + friendly_name_overrides = friendly_name_overrides or {} columns: list[dict] = [] for field in service_fields: @@ -238,15 +238,17 @@ def fetch_columns_from_hub_url(hub_url: str, include_system_fields: bool = False col: dict = {"name": name} - # friendly_name from service alias (only if it differs from the raw name) - alias = field.get("alias", "") - if alias and alias != name: - col["friendly_name"] = alias + # friendly_name from overrides or service alias. + friendly_name = friendly_name_overrides.get(name, "") + if not friendly_name and use_service_alias_as_friendly_name: + friendly_name = field.get("alias", "") + if friendly_name and friendly_name != name: + col["friendly_name"] = friendly_name col["dtype"] = dtype - # description from FGDC metadata - desc = fgdc_descs.get(name, "") + # description from external metadata map. + desc = field_descriptions.get(name, "") if desc: col["description"] = desc @@ -255,6 +257,36 @@ def fetch_columns_from_hub_url(hub_url: str, include_system_fields: bool = False return columns +# --------------------------------------------------------------------------- +# Core: merge into columns list +# --------------------------------------------------------------------------- +def fetch_columns_from_hub_url(hub_url: str, include_system_fields: bool = False) -> list[dict]: + """Fetch column definitions from an ArcGIS Hub dataset URL. + + Args: + hub_url: A Hub dataset page URL, e.g. + ``https://gbp-blm-egis.hub.arcgis.com/datasets/BLM-EGIS::blm-natl-grazing-allotment-polygons/about`` + include_system_fields: If True, include Esri system fields such as + ``OBJECTID``, ``GlobalID``, ``Shape__Area``, etc. + + Returns: + A list of column dicts ready to insert into a layer_definitions.json + ``columns`` array. + """ + slug = _extract_slug(hub_url) + item_id, service_url, _layer_index = _resolve_hub_slug(slug) + + service_fields = fetch_service_fields(service_url) + fgdc_descs = _fetch_fgdc_descriptions(item_id) + + return build_columns_from_service_fields( + service_fields, + field_descriptions=fgdc_descs, + include_system_fields=include_system_fields, + use_service_alias_as_friendly_name=True, + ) + + # --------------------------------------------------------------------------- # Optional: patch an existing layer_definitions.json # --------------------------------------------------------------------------- diff --git a/packages/vector_prep/vector_prep/layer_definitions.json b/packages/vector_prep/vector_prep/layer_definitions.json index 77d243b4..51b1fd71 100644 --- a/packages/vector_prep/vector_prep/layer_definitions.json +++ b/packages/vector_prep/vector_prep/layer_definitions.json @@ -154,6 +154,109 @@ "description": "Used for improved spatial query performance. Added in Riverscapes processing." } ] + }, + { + "layer_id": "blm-natl-grazing-pasture-polygons-nm-bootheel", + "layer_name": "BLM Grazing Pastures for NM Bootheel", + "source_title": "US DOI Bureau of Land Management National Operations Center Staff", + "columns": [ + { + "name": "OBJECTID", + "friendly_name": "OBJECTID", + "dtype": "INTEGER", + "description": "" + }, + { + "name": "Allot_Number", + "friendly_name": "Allot_Number", + "dtype": "STRING", + "description": "" + }, + { + "name": "Allot_Name", + "friendly_name": "Allot_Name", + "dtype": "STRING", + "description": "" + }, + { + "name": "Pasture_Number", + "friendly_name": "Pasture_Number", + "dtype": "STRING", + "description": "" + }, + { + "name": "Pasture_Name", + "friendly_name": "Pasture_Name", + "dtype": "STRING", + "description": "" + }, + { + "name": "Pasture_Acres", + "friendly_name": "Pasture_Acres", + "dtype": "FLOAT", + "description": "" + }, + { + "name": "Admin_State", + "friendly_name": "Admin_State", + "dtype": "STRING", + "description": "" + }, + { + "name": "Admin_Agency", + "friendly_name": "Admin_Agency", + "dtype": "STRING", + "description": "" + }, + { + "name": "Admin_Office", + "friendly_name": "Admin_Office", + "dtype": "STRING", + "description": "" + }, + { + "name": "Pasture_Latitude", + "friendly_name": "Pasture_Latitude", + "dtype": "FLOAT", + "description": "" + }, + { + "name": "Pasture_Longitude", + "friendly_name": "Pasture_Longitude", + "dtype": "FLOAT", + "description": "" + }, + { + "name": "Shape_Length", + "friendly_name": "Shape_Length", + "dtype": "FLOAT", + "description": "" + }, + { + "name": "Shape_Area", + "friendly_name": "Shape_Area", + "dtype": "FLOAT", + "description": "" + }, + { + "name": "Pasture_ID", + "friendly_name": "Pasture ID", + "dtype": "STRING", + "description": "Unique ID for pasture" + }, + { + "name": "geometry", + "friendly_name": "Geometry (binary)", + "dtype": "GEOMETRY", + "description": "Pasture Multi-Polygon geometry" + }, + { + "name": "geometry_bbox", + "friendly_name": "Geometry Bounding Box", + "dtype": "STRUCTURED", + "description": "Used for improved spatial query performance. Added in Riverscapes processing." + } + ] } ] } diff --git a/packages/vector_prep/vector_prep/orchestrate-pastures-bootheel.py b/packages/vector_prep/vector_prep/orchestrate-pastures-bootheel.py new file mode 100644 index 00000000..77c0cbaa --- /dev/null +++ b/packages/vector_prep/vector_prep/orchestrate-pastures-bootheel.py @@ -0,0 +1,406 @@ +"""Run a sequence of vector preparation steps +e.g. (get data, check, prepare, document, output, upload) +Based on orchestrate-pastures (the original), this one is modified for the bootheel subset sent as GDB +May require user input / view or manual steps ... we may add some questionary prompts +Or run in ipynb notebook? (easier to add interactive visualizations as needed + self documents) +See runlog-pastures-NMbootheel-202600407.md +- Lorin March / April 2026 +""" +import io +import json +import uuid +from dataclasses import dataclass +from pathlib import Path + +import geopandas as gpd +from rsxml import Logger + +from fetch_arcgis_metadata import fetch_columns_from_hub_url, update_layer_definitions +from vector_prep import output_gdf, vector_prep + +# Fixed namespace for RS_ROW_ID derivation — do not change once data is published +_RS_ROW_ID_NAMESPACE = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') # uuid.NAMESPACE_OID + +@dataclass +class RunInputs: + """Stuff user needs to supply to establish what is being prepared""" + input_vector_path: Path | str + input_layer_name: str | None + source_category: str + source_title: str + source_url: str | None + layer_id: str + snapshot_id: str + tolerance: float = 0.0 + epsg: int = 5070 + special_notes: str = "" + data_prep_operator: str = "" + + @classmethod + def from_json(cls, path: str) -> "RunInputs": + with open(path, encoding="utf-8") as f: + return cls(**json.load(f)) + + +def step_1(cfg: RunInputs, dist_dir): + """vector prep (error checks) and output to 4326""" + prepped_gdf = vector_prep(cfg.input_vector_path, None, cfg.tolerance, cfg.epsg) + source_category_stub = 'usgov_sources' if cfg.source_category == 'usgov' else f'raw_{cfg.source_category}' + + output_dir = dist_dir / source_category_stub / cfg.layer_id / cfg.snapshot_id + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / f"{cfg.layer_id}.gpkg" + output_gdf(prepped_gdf, output_file, cfg.input_layer_name) + return prepped_gdf, output_file + +def step_2(gdf, cfg: RunInputs, output_file: Path): + """Add ST_ALLOT_PAST_NAME, ST_ALLOT_PAST_MULTI, and deterministic RS_ROW_ID from GlobalID.""" + log = Logger("Step2") + # Uniqueness check on GlobalID + n_dupes = gdf['GlobalID'].duplicated().sum() + if n_dupes > 0: + raise ValueError(f"GlobalID is not unique: {n_dupes} duplicate value(s) found. Cannot derive deterministic RS_ROW_ID.") + null_count = gdf['GlobalID'].isna().sum() + if null_count > 0: + raise ValueError(f"GlobalID has {null_count} null value(s). Cannot derive deterministic RS_ROW_ID.") + + # for idempotence, # Drop derived columns if re-running on already-enriched data + for col in ('ST_ALLOT_PAST_NAME', 'ST_ALLOT_PAST_MULTI', 'RS_ROW_ID'): + if col in gdf.columns: + gdf = gdf.drop(columns=[col]) + + # min name combo per ST_ALLOT_PAST entity + name_min = ( + gdf.assign(_nc=gdf['ADMIN_ST'].fillna('') + '_' + gdf['ALLOT_NAME'].fillna('') + '_' + gdf['PAST_NAME'].fillna('')) + .groupby('ST_ALLOT_PAST')['_nc'] + .min() + .rename('ST_ALLOT_PAST_NAME') + ) + multi_flag = ( + gdf.groupby('ST_ALLOT_PAST') + .size() + .gt(1) + # Use pandas nullable Int64 so NaN rows (null ST_ALLOT_PAST) don't upcast the whole column to float64 + .astype("Int64") + .rename('ST_ALLOT_PAST_MULTI') + ) + gdf = gdf.join(name_min, on='ST_ALLOT_PAST').join(multi_flag, on='ST_ALLOT_PAST') + + # Warn if any rows got NaN in derived columns — indicates null ST_ALLOT_PAST values + for derived_col in ('ST_ALLOT_PAST_NAME', 'ST_ALLOT_PAST_MULTI'): + n_null = gdf[derived_col].isna().sum() + if n_null > 0: + log.warning(f"{derived_col}: {n_null} row(s) have NaN — likely null ST_ALLOT_PAST values. These rows were excluded from the groupby.") + + # Deterministic UUID5 derived from GlobalID + gdf['RS_ROW_ID'] = gdf['GlobalID'].apply( + lambda gid: str(uuid.uuid5(_RS_ROW_ID_NAMESPACE, gid)) + ) + + output_gdf(gdf, output_file, cfg.input_layer_name) + log.info(f"step_2 complete: added ST_ALLOT_PAST_NAME, ST_ALLOT_PAST_MULTI, RS_ROW_ID. Shape: {gdf.shape}") + return gdf + + +def export_to_geoparquet(gdf: gpd.GeoDataFrame, output_path: Path | str) -> Path: + """Export a GeoDataFrame to Snappy-compressed GeoParquet in EPSG:4326. + + Produces a valid GeoParquet file (readable by QGIS and other tools) by + delegating to geopandas for geometry encoding and the required *geo* Parquet + file metadata. A *geometry_bbox* column typed as + struct is then + appended for spatial predicate push-down (e.g. Athena partition pruning). + + Returns the resolved output path. + """ + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as exc: + raise ImportError( + "GeoParquet export requires optional dependency 'pyarrow'. " + "Install with: uv sync --extra geoparquet" + ) from exc + + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + + if gdf.crs is None or gdf.crs.to_epsg() != 4326: + gdf = gdf.to_crs(epsg=4326) + + # Idempotency: drop bbox column if already present + if "geometry_bbox" in gdf.columns: + gdf = gdf.drop(columns=["geometry_bbox"]) + + bounds = gdf.geometry.bounds # DataFrame: minx, miny, maxx, maxy + + # Let geopandas write a proper GeoParquet file (including `geo` metadata) + # into a buffer, then read it back as a PyArrow table so the metadata + # is preserved when we append the bbox column. + buf = io.BytesIO() + gdf.to_parquet(buf, compression="snappy", index=False) + buf.seek(0) + table = pq.read_table(buf) + + bbox_type = pa.struct([ + pa.field("xmin", pa.float32()), + pa.field("ymin", pa.float32()), + pa.field("xmax", pa.float32()), + pa.field("ymax", pa.float32()), + ]) + bbox_col = pa.StructArray.from_arrays( + [ + pa.array(bounds["minx"].to_numpy(dtype="float32"), type=pa.float32()), + pa.array(bounds["miny"].to_numpy(dtype="float32"), type=pa.float32()), + pa.array(bounds["maxx"].to_numpy(dtype="float32"), type=pa.float32()), + pa.array(bounds["maxy"].to_numpy(dtype="float32"), type=pa.float32()), + ], + fields=list(bbox_type), + ) + # append_column preserves schema.metadata, keeping the `geo` key intact + table = table.append_column(pa.field("geometry_bbox", bbox_type), bbox_col) + + pq.write_table(table, str(output_path), compression="snappy") + return output_path + + +# Derived columns added by step_2 — hard-coded since they are always the same for this dataset. +_STEP2_COLUMNS: list[dict] = [ + { + "name": "Pasture_ID", + "friendly_name": "Pasture ID", + "dtype": "STRING", + "description": ( + "Unique ID for pasture" + ), + }, +] + +_GEO_COLUMNS: list[dict] = [ + { + "name": "geometry", + "friendly_name": "Geometry (binary)", + "dtype": "GEOMETRY", + "description": ( + "Pasture Multi-Polygon geometry" + ), + }, + { + "name": "geometry_bbox", + "friendly_name": "Geometry Bounding Box", + "dtype": "STRUCTURED", + "description": ( + "Used for improved spatial query performance. Added in Riverscapes processing." + ), + }, +] + +def _infer_dtype(series) -> str: + """Map a pandas/geopandas Series dtype to a layer_definitions dtype string.""" + name = getattr(series.dtype, "name", "").lower() + if name == "geometry": + return "GEOMETRY" + if "int" in name: + return "INTEGER" + if "float" in name: + return "FLOAT" + if "datetime" in name: + return "DATETIME" + return "STRING" + + +def build_layer_defs(cfg: RunInputs, gdf: gpd.GeoDataFrame) -> None: + """Add a new layer entry to layer_definitions.json for cfg.layer_id. + + Fetches column definitions from the ArcGIS Hub URL in cfg.source_url when + available; otherwise (or for any columns not returned by the remote source) + falls back to inspecting *gdf* directly and inferring dtypes. The derived + columns from step_2 and the geometry columns are appended last. + + Skips silently if cfg.layer_id is already present in the file. + """ + log = Logger("build_layer_defs") + layer_defs_path = Path(__file__).parent / "layer_definitions.json" + + with open(layer_defs_path, "r", encoding="utf-8") as fh: + doc = json.load(fh) + + if any(layer.get("layer_id") == cfg.layer_id for layer in doc.get("layers", [])): + log.warning(f"Layer '{cfg.layer_id}' already exists in layer_definitions.json — skipping.") + return + + # Add skeleton (columns filled in below via update_layer_definitions) + new_layer: dict = { + "layer_id": cfg.layer_id, + "layer_name": cfg.input_layer_name or cfg.layer_id, + "source_title": cfg.source_title, + **({"source_url": cfg.source_url} if cfg.source_url else {}), + "columns": [], + } + doc["layers"].append(new_layer) + with open(layer_defs_path, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=4, ensure_ascii=False) + fh.write("\n") + + columns: list[dict] = [] + if cfg.source_url: + log.info(f"Fetching column metadata from {cfg.source_url} ...") + columns = fetch_columns_from_hub_url(cfg.source_url) + + # Add any GDF columns not already covered by the remote fetch or the + # hard-coded step_2 / geometry entries + reserved = {col["name"] for col in _STEP2_COLUMNS + _GEO_COLUMNS} + covered = {col["name"] for col in columns} + for col_name in gdf.columns: + if col_name not in covered and col_name not in reserved: + columns.append({ + "name": col_name, + "friendly_name": col_name, + "dtype": _infer_dtype(gdf[col_name]), + "description": "", + }) + + columns.extend(_STEP2_COLUMNS) + columns.extend(_GEO_COLUMNS) + update_layer_definitions(str(layer_defs_path), cfg.layer_id, columns) + + log.info(f"Added layer '{cfg.layer_id}' with {len(columns)} columns to {layer_defs_path}") + + +def _sql_escape(value: str) -> str: + """Escape single quotes for SQL string literals/comments.""" + return value.replace("'", "''") + + +def _athena_table_name(layer_id: str, snapshot_id: str) -> str: + """Build snapshot table name for rs_raw from layer and snapshot ids.""" + snapshot_stub = snapshot_id.replace("-", "") + return f"{layer_id.replace('-', '_')}_snapshot_{snapshot_stub}" + + +def _map_layer_dtype_to_athena(dtype: str, col_name: str) -> str: + """Map layer_definitions dtype to Athena/Hive type.""" + mapping = { + "STRING": "string", + "INTEGER": "bigint", + "FLOAT": "double", + "DATETIME": "timestamp", + "GEOMETRY": "binary", + } + if dtype == "STRUCTURED" and col_name.lower() == "geometry_bbox": + return "struct" + return mapping.get(dtype, "string") + + +def build_athena_ddl(cfg: RunInputs, bucket: str = "riverscapes-athena", database: str = "rs_raw") -> Path: + """Build Athena CREATE EXTERNAL TABLE DDL from cfg and `layer_definitions.json` + + Returns the path to the generated .sql file. + TODO (ENHANCEMENT): Add additional metadata with custom table properties using `rs.` namespace) + """ + log = Logger("build_athena_ddl") + layer_defs_path = Path(__file__).parent / "layer_definitions.json" + with open(layer_defs_path, "r", encoding="utf-8") as fh: + layer_defs = json.load(fh) + + layer = next((lyr for lyr in layer_defs.get("layers", []) if lyr.get("layer_id") == cfg.layer_id), None) + if layer is None: + raise LookupError(f"layer_id '{cfg.layer_id}' not found in {layer_defs_path}") + + columns = layer.get("columns", []) + if not columns: + raise ValueError(f"layer_id '{cfg.layer_id}' has no columns in {layer_defs_path}") + + source_category_stub = "usgov_sources" if cfg.source_category == "usgov" else f"raw_{cfg.source_category}" + location = f"s3://{bucket}/{source_category_stub}/{cfg.layer_id}/{cfg.snapshot_id}/" + table_name = _athena_table_name(cfg.layer_id, cfg.snapshot_id) + + layer_name = layer.get("layer_name") or cfg.input_layer_name or cfg.layer_id + table_comment = ( + f"{layer_name}. Source: {cfg.source_title}. URL: {cfg.source_url}. " + f"Snapshot: {cfg.snapshot_id}." + ) + + col_lines: list[str] = [] + for col in columns: + name = col.get("name") + if not name: + continue + athena_name = name.lower() # Athena normalises identifiers to lower-case + dtype = _map_layer_dtype_to_athena(col.get("dtype", "STRING"), name) + desc = (col.get("description") or "").strip() + friendly = (col.get("friendly_name") or "").strip() + comment = desc or friendly + if desc and friendly and friendly not in desc: + comment = f"{friendly}. {desc}" + + if comment: + col_lines.append(f" `{athena_name}` {dtype} COMMENT '{_sql_escape(comment)}'") + else: + col_lines.append(f" `{athena_name}` {dtype}") + + # TODO: Compression is currently assumed from the manual QGIS export. + # Capture/read parquet compression from pipeline output metadata so this is not hard-coded. + ddl = ( + f"CREATE EXTERNAL TABLE `{database}`.`{table_name}`(\n" + + ", \n".join(col_lines) + + "\n)\n" + + "ROW FORMAT SERDE \n" + + " 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' \n" + + "STORED AS INPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' \n" + + "OUTPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'\n" + + "LOCATION\n" + + f" '{location}'\n" + + "TBLPROPERTIES (\n" + + " 'classification'='parquet', \n" + + f" 'comment'='{_sql_escape(table_comment)}',\n" + + " 'compressionType'='snappy', \n" + + " 'typeOfData'='file'\n" + + ")" + ) + + repo_root = next(p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists()) + dist_dir = repo_root / "dist" / source_category_stub / cfg.layer_id / cfg.snapshot_id + dist_dir.mkdir(parents=True, exist_ok=True) + ddl_path = dist_dir / f"{table_name}.sql" + ddl_path.write_text(ddl, encoding="utf-8") + log.info(f"Wrote Athena DDL to {ddl_path}") + return ddl_path + +def main(): + run_inputs = "/home/narlorin/udata/blm/pasture_polygons_bootheel_2026-04-02/inputs.json" + repo_root = next(p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists()) + logs_dir = repo_root / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + dist_dir = repo_root / "dist" + dist_dir.mkdir(parents=True, exist_ok=True) + + log = Logger("Vector Prep Orchestrate") + cfg = RunInputs.from_json(run_inputs) + log_file = logs_dir / f"vector_prep_orchestrate_{cfg.layer_id}_{cfg.snapshot_id}.log" + log.setup(log_path=str(log_file), verbose=True) + + # gdf, output_file = step_1(cfg, dist_dir) + # log.info(f"Prepped dataframe with shape {gdf.shape} and outputed to {output_file}") + + # duplicates logic in step1 + source_category_stub = 'usgov_sources' if cfg.source_category == 'usgov' else f'raw_{cfg.source_category}' + output_file = dist_dir / source_category_stub / cfg.layer_id / cfg.snapshot_id / f"{cfg.layer_id}.gpkg" + + log.info(f"Loading step_1 output from {output_file}") + gdf = gpd.read_file(output_file) + log.info(f"Loaded {len(gdf)} features") + + # enriched_gdf = step_2(gdf, cfg, output_file) + # log.info(f"Enriched dataframe with shape {enriched_gdf.shape} written to {output_file}") + + build_layer_defs(cfg, gdf) + export_to_geoparquet(gdf, output_file.with_suffix(".parquet")) + ddl_path = build_athena_ddl(cfg) + log.info(f"Athena DDL generated at {ddl_path}") + + +if __name__ == '__main__': + main() diff --git a/packages/vector_prep/vector_prep/orchestrate-pastures.py b/packages/vector_prep/vector_prep/orchestrate-pastures.py new file mode 100644 index 00000000..13256b04 --- /dev/null +++ b/packages/vector_prep/vector_prep/orchestrate-pastures.py @@ -0,0 +1,326 @@ +"""Run a sequence of vector preparation steps +e.g. (get data, check, prepare, document, output, upload) +May require user input / view or manual steps ... we may add some questionary prompts +Or run in ipynb notebook? (easier to add interactive visualizations as needed + self documents) +See runlog-pastures-20260324.md and plan-vectorPrepIngestionFoundationRefined.prompt.md +- Lorin March 2026 +""" +import json +import uuid +from dataclasses import dataclass +from pathlib import Path + +import geopandas as gpd +from rsxml import Logger +from vector_prep import vector_prep, output_gdf +from fetch_arcgis_metadata import fetch_columns_from_hub_url, update_layer_definitions + +# Fixed namespace for RS_ROW_ID derivation — do not change once data is published +_RS_ROW_ID_NAMESPACE = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') # uuid.NAMESPACE_OID + +@dataclass +class RunInputs: + """Stuff user needs to supply to establish what is being prepared""" + input_vector_path: Path | str + input_layer_name: str | None + source_category: str + source_title: str + source_url: str + layer_id: str + snapshot_id: str + tolerance: float = 0.0 + epsg: int = 5070 + special_notes: str = "" + data_prep_operator: str = "" + + @classmethod + def from_json(cls, path: str) -> "RunInputs": + with open(path, encoding="utf-8") as f: + return cls(**json.load(f)) + + +def step_1(cfg: RunInputs, dist_dir): + """vector prep (error checks) and output to 4326""" + prepped_gdf = vector_prep(cfg.input_vector_path, None, cfg.tolerance, cfg.epsg) + source_category_stub = 'usgov_sources' if cfg.source_category == 'usgov' else f'raw_{cfg.source_category}' + + output_dir = dist_dir / source_category_stub / cfg.layer_id / cfg.snapshot_id + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / f"{cfg.layer_id}.gpkg" + output_gdf(prepped_gdf, output_file, cfg.input_layer_name) + return prepped_gdf, output_file + +def step_2(gdf, cfg: RunInputs, output_file: Path): + """Add ST_ALLOT_PAST_NAME, ST_ALLOT_PAST_MULTI, and deterministic RS_ROW_ID from GlobalID.""" + log = Logger("Step2") + # Uniqueness check on GlobalID + n_dupes = gdf['GlobalID'].duplicated().sum() + if n_dupes > 0: + raise ValueError(f"GlobalID is not unique: {n_dupes} duplicate value(s) found. Cannot derive deterministic RS_ROW_ID.") + null_count = gdf['GlobalID'].isna().sum() + if null_count > 0: + raise ValueError(f"GlobalID has {null_count} null value(s). Cannot derive deterministic RS_ROW_ID.") + + # for idempotence, # Drop derived columns if re-running on already-enriched data + for col in ('ST_ALLOT_PAST_NAME', 'ST_ALLOT_PAST_MULTI', 'RS_ROW_ID'): + if col in gdf.columns: + gdf = gdf.drop(columns=[col]) + + # min name combo per ST_ALLOT_PAST entity + name_min = ( + gdf.assign(_nc=gdf['ADMIN_ST'].fillna('') + '_' + gdf['ALLOT_NAME'].fillna('') + '_' + gdf['PAST_NAME'].fillna('')) + .groupby('ST_ALLOT_PAST')['_nc'] + .min() + .rename('ST_ALLOT_PAST_NAME') + ) + multi_flag = ( + gdf.groupby('ST_ALLOT_PAST') + .size() + .gt(1) + # Use pandas nullable Int64 so NaN rows (null ST_ALLOT_PAST) don't upcast the whole column to float64 + .astype("Int64") + .rename('ST_ALLOT_PAST_MULTI') + ) + gdf = gdf.join(name_min, on='ST_ALLOT_PAST').join(multi_flag, on='ST_ALLOT_PAST') + + # Warn if any rows got NaN in derived columns — indicates null ST_ALLOT_PAST values + for derived_col in ('ST_ALLOT_PAST_NAME', 'ST_ALLOT_PAST_MULTI'): + n_null = gdf[derived_col].isna().sum() + if n_null > 0: + log.warning(f"{derived_col}: {n_null} row(s) have NaN — likely null ST_ALLOT_PAST values. These rows were excluded from the groupby.") + + # Deterministic UUID5 derived from GlobalID + gdf['RS_ROW_ID'] = gdf['GlobalID'].apply( + lambda gid: str(uuid.uuid5(_RS_ROW_ID_NAMESPACE, gid)) + ) + + output_gdf(gdf, output_file, cfg.input_layer_name) + log.info(f"step_2 complete: added ST_ALLOT_PAST_NAME, ST_ALLOT_PAST_MULTI, RS_ROW_ID. Shape: {gdf.shape}") + return gdf + +# Derived columns added by step_2 — hard-coded since they are always the same for this dataset. +_STEP2_COLUMNS: list[dict] = [ + { + "name": "ST_ALLOT_PAST_NAME", + "friendly_name": "State Allotment Pasture Name", + "dtype": "STRING", + "description": ( + "Minimum concatenation of ADMIN_ST, ALLOT_NAME, and PAST_NAME across all rows " + "sharing the same ST_ALLOT_PAST value. Added in Riverscapes processing." + ), + }, + { + "name": "ST_ALLOT_PAST_MULTI", + "friendly_name": "State Allotment Pasture Multi-Row Flag", + "dtype": "INTEGER", + "description": ( + "1 if the ST_ALLOT_PAST entity appears on more than one row, 0 otherwise. " + "Added in Riverscapes processing." + ), + }, + { + "name": "RS_ROW_ID", + "friendly_name": "Riverscapes Row ID", + "dtype": "STRING", + "description": ( + "Deterministic UUID5 derived from GlobalID using the Riverscapes OID namespace " + "(_RS_ROW_ID_NAMESPACE). Unique per row. Added in Riverscapes processing." + ), + }, +] + +_GEO_COLUMNS: list[dict] = [ + { + "name": "geometry", + "friendly_name": "Geometry (binary)", + "dtype": "GEOMETRY", + "description": ( + "Pasture Polygon geometry" + ), + }, + { + "name": "geometry_bbox", + "friendly_name": "Geometry Bounding Box", + "dtype": "STRUCTURED", + "description": ( + "Used for improved spatial query performance. Added in Riverscapes processing." + ), + }, +] + +def build_layer_defs(cfg: RunInputs) -> None: + """Add a new layer entry to layer_definitions.json for cfg.layer_id. + + Fetches column definitions from the ArcGIS Hub URL in cfg.source_url, appends + the three derived columns produced by step_2, then writes the result into + layer_definitions.json (located next to this file). + + Skips silently if cfg.layer_id is already present in the file. + """ + log = Logger("build_layer_defs") + layer_defs_path = Path(__file__).parent / "layer_definitions.json" + + with open(layer_defs_path, "r", encoding="utf-8") as fh: + doc = json.load(fh) + + if any(layer.get("layer_id") == cfg.layer_id for layer in doc.get("layers", [])): + log.warning(f"Layer '{cfg.layer_id}' already exists in layer_definitions.json — skipping.") + return + + # Add skeleton (columns filled in below via update_layer_definitions) + new_layer: dict = { + "layer_id": cfg.layer_id, + "layer_name": cfg.input_layer_name or cfg.layer_id, + "source_url": cfg.source_url, + "source_title": cfg.source_title, + "columns": [], + } + doc["layers"].append(new_layer) + with open(layer_defs_path, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=4, ensure_ascii=False) + fh.write("\n") + + log.info(f"Fetching column metadata from {cfg.source_url} ...") + columns = fetch_columns_from_hub_url(cfg.source_url) + columns.extend(_STEP2_COLUMNS) + columns.extend(_GEO_COLUMNS) + update_layer_definitions(str(layer_defs_path), cfg.layer_id, columns) + + log.info(f"Added layer '{cfg.layer_id}' with {len(columns)} columns to {layer_defs_path}") + + +def _sql_escape(value: str) -> str: + """Escape single quotes for SQL string literals/comments.""" + return value.replace("'", "''") + + +def _athena_table_name(layer_id: str, snapshot_id: str) -> str: + """Build snapshot table name for rs_raw from layer and snapshot ids.""" + snapshot_stub = snapshot_id.replace("-", "") + return f"{layer_id.replace('-', '_')}_snapshot_{snapshot_stub}" + + +def _map_layer_dtype_to_athena(dtype: str, col_name: str) -> str: + """Map layer_definitions dtype to Athena/Hive type.""" + mapping = { + "STRING": "string", + "INTEGER": "bigint", + "FLOAT": "double", + "DATETIME": "timestamp", + "GEOMETRY": "binary", + } + if dtype == "STRUCTURED" and col_name.lower() == "geometry_bbox": + return "struct" + return mapping.get(dtype, "string") + + +def build_athena_ddl(cfg: RunInputs, bucket: str = "riverscapes-athena", database: str = "rs_raw") -> Path: + """Build Athena CREATE EXTERNAL TABLE DDL from cfg and `layer_definitions.json` + + Returns the path to the generated .sql file. + """ + log = Logger("build_athena_ddl") + layer_defs_path = Path(__file__).parent / "layer_definitions.json" + with open(layer_defs_path, "r", encoding="utf-8") as fh: + layer_defs = json.load(fh) + + layer = next((lyr for lyr in layer_defs.get("layers", []) if lyr.get("layer_id") == cfg.layer_id), None) + if layer is None: + raise LookupError(f"layer_id '{cfg.layer_id}' not found in {layer_defs_path}") + + columns = layer.get("columns", []) + if not columns: + raise ValueError(f"layer_id '{cfg.layer_id}' has no columns in {layer_defs_path}") + + source_category_stub = "usgov_sources" if cfg.source_category == "usgov" else f"raw_{cfg.source_category}" + location = f"s3://{bucket}/{source_category_stub}/{cfg.layer_id}/{cfg.snapshot_id}/" + table_name = _athena_table_name(cfg.layer_id, cfg.snapshot_id) + + layer_name = layer.get("layer_name") or cfg.input_layer_name or cfg.layer_id + table_comment = ( + f"{layer_name}. Source: {cfg.source_title}. URL: {cfg.source_url}. " + f"Snapshot: {cfg.snapshot_id}." + ) + + col_lines: list[str] = [] + for col in columns: + name = col.get("name") + if not name: + continue + athena_name = name.lower() # Athena normalises identifiers to lower-case + dtype = _map_layer_dtype_to_athena(col.get("dtype", "STRING"), name) + desc = (col.get("description") or "").strip() + friendly = (col.get("friendly_name") or "").strip() + comment = desc or friendly + if desc and friendly and friendly not in desc: + comment = f"{friendly}. {desc}" + + if comment: + col_lines.append(f" `{athena_name}` {dtype} COMMENT '{_sql_escape(comment)}'") + else: + col_lines.append(f" `{athena_name}` {dtype}") + + # TODO: Compression is currently assumed from the manual QGIS export. + # Capture/read parquet compression from pipeline output metadata so this is not hard-coded. + ddl = ( + f"CREATE EXTERNAL TABLE `{database}`.`{table_name}`(\n" + + ", \n".join(col_lines) + + "\n)\n" + + "ROW FORMAT SERDE \n" + + " 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' \n" + + "STORED AS INPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' \n" + + "OUTPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'\n" + + "LOCATION\n" + + f" '{location}'\n" + + "TBLPROPERTIES (\n" + + " 'classification'='parquet', \n" + + f" 'comment'='{_sql_escape(table_comment)}',\n" + + " 'compressionType'='snappy', \n" + + " 'typeOfData'='file'\n" + + ")" + ) + + repo_root = next(p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists()) + dist_dir = repo_root / "dist" / source_category_stub / cfg.layer_id / cfg.snapshot_id + dist_dir.mkdir(parents=True, exist_ok=True) + ddl_path = dist_dir / f"{table_name}.sql" + ddl_path.write_text(ddl, encoding="utf-8") + log.info(f"Wrote Athena DDL to {ddl_path}") + return ddl_path + +def main(): + run_inputs = "/home/narlorin/udata/blm/pasture_polygons_2026-03-24/inputs.json" + repo_root = next(p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists()) + logs_dir = repo_root / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + dist_dir = repo_root / "dist" + dist_dir.mkdir(parents=True, exist_ok=True) + + log = Logger("Vector Prep Orchestrate") + cfg = RunInputs.from_json(run_inputs) + log_file = logs_dir / f"vector_prep_orchestrate_{cfg.layer_id}_{cfg.snapshot_id}.log" + log.setup(log_path=str(log_file), verbose=True) + + # prepped_gdf, outputpath = step_1(cfg, dist_dir) + # log.info(f"Prepped dataframe with shape {gdf.shape} and outputed to {output_file}") + + # duplicates logic in step1 + source_category_stub = 'usgov_sources' if cfg.source_category == 'usgov' else f'raw_{cfg.source_category}' + output_file = dist_dir / source_category_stub / cfg.layer_id / cfg.snapshot_id / f"{cfg.layer_id}.gpkg" + + log.info(f"Loading step_1 output from {output_file}") + gdf = gpd.read_file(output_file) + log.info(f"Loaded {len(gdf)} features") + + enriched_gdf = step_2(gdf, cfg, output_file) + log.info(f"Enriched dataframe with shape {enriched_gdf.shape} written to {output_file}") + + build_layer_defs(cfg) + ddl_path = build_athena_ddl(cfg) + log.info(f"Athena DDL generated at {ddl_path}") + + +if __name__ == '__main__': + main() diff --git a/packages/vector_prep/vector_prep/runlog-pastures-NMbootheel-20260407.md b/packages/vector_prep/vector_prep/runlog-pastures-NMbootheel-20260407.md new file mode 100644 index 00000000..9b34a052 --- /dev/null +++ b/packages/vector_prep/vector_prep/runlog-pastures-NMbootheel-20260407.md @@ -0,0 +1,103 @@ +# Vector Prep Run Log + +## Summary + +Updated version of Pasture polygons for New Mexico Bootheel from Meg McLachlan at BLM, to be used for the pilot project April 2026. + +I believe they merged the polygons so that now there is one multi-part per ST-allotment-pasture ID. + +This entire file is hand-written, eventually automate or partially automate. + +Inputs in: `/home/narlorin/udata/blm/pasture_polygons_bootheel_2026-04-02/inputs.json` + +## Notes/Description of Inputs & Parameters + +(standard definitions of inputs and parameters should go somewhere else someday like a json.schema assuming the inputs are stored in json) + +* input-vector-path, help="Input vector (shapefile, gpkg, etc.) Path. Supplied to vector_prep as input (first param)" +* output-vector-path, help="Output vector path". Required by vector_prep, but should be built from /dist + layer_id, snapshot_id +* input-layer-name: help="Layer name (for geopackage). If not provided and input is geopackage, first layer is used.", default=None vector_prep --layer parameter +* "--tolerance", type=float, help="Simplify tolerance in METRES (0 to skip).", default=0.0 +* "--epsg", type=int, help="Cartesian CRS EPSG code to reproject to **before** processing (optional). Default is 5070 (NAD83 / Conus Albers).", default=5070 +* source_title. this is the layer provenance friendly name +* source_url. this is the layer provenance url +* source_category ("usgov") - determines where in s3://riverscapes-athena/raw/ it goes +* layer_id. This should be unique within our layer_source_category. should be lower case, will be used for athena table names, identifier in layer_definitions. there the namespace is usually repo+tool. perhaps for this "tool" is vectorprep+source_category. In reality we are inconsistent in namespacing athena vs s3 vs layer_definitions hierarchy, but should move towards consistency +* snapshot_id - usually date we downloaded/received the data. default today's date + +## Overview of planned steps + +1. Run vector_prep.py on the gpkg — (geometry cleaning) + transform to EPSG:4326. +2. document/address any errors +3. Identify business entity definition (natural key) and entity mode (ONE_ROW_PER_ENTITY vs MANY_ROWS_PER_ENTITY) +4. Add rs_row_id using UUID4 (later consider deterministic based on natural key) +5. populate layer_defs metadata with inputs / scraped from source +6. export to 4326 geoparquet including bbox - either with qgis or same as pipelines/rme_to_athena +7. upload data to s3 (raw/) +8. upload layer_definitions metadata to s3 (use existing gh action/script) +9. create `rs_raw` athena table - ideally use metadata to add COMMENTs +10. create spatial intersection tables with existing layers +11. create view/materialized tables in rs_rpt + +## 1,2 Vector Prep Standard Error Checking + +* run `orchestrate.py` / `step_1` +* output copied it below. We'll need to write it somewhere. The log is not designed for long-term, and indeed currently is overwritten whenever run the process (and for expediency when we run into issues we don't always restart the code at step 1 each time, we comment out that part and go to step 2 or whatever) + +```text +[INFO] [Vector Prep] Reading input dataset with GeoPandas: /home/narlorin/udata/blm/pasture_polygons_bootheel_2026-04-02/boot_pastures_blm20260402.gpkg +[DEBUG] [Vector Prep] No layer name specified, geopandas will choose default/first layer +[INFO] [Vector Prep] Loaded 159 features. CRS: EPSG:5070 +[INFO] [Vector Prep] Geometry type of input: {'MultiPolygon': 159} +[INFO] [Vector Prep] Reprojecting to EPSG:5070 for processing... +[INFO] [Vector Prep] Reprojection complete. New CRS: EPSG:5070 +[INFO] [Vector Prep] Cleaning geometries (tolerance=0.0)... +[INFO] [Vector Prep] Input features: 159 +[INFO] [Vector Prep] Null/empty geometries found: 0 +[INFO] [Vector Prep] Invalid geometries fixed: 0 +[INFO] [Vector Prep] Invalid geometries unfixed (dropped): 0 +[INFO] [Vector Prep] Features simplified: 0 +[INFO] [Vector Prep] Dropped features after cleaning: 0 +[INFO] [Vector Prep] Remaining features to write: 159 +[INFO] [Output GDF] Reprojecting to EPSG 4326 for output +[INFO] [Output GDF] Writing cleaned layer to /home/narlorin/ucode/riverscapes-tools/dist/usgov_sources/blm-natl-grazing-pasture-polygons-nm-bootheel/2026-04-02/blm-natl-grazing-pasture-polygons-nm-bootheel.gpkg (driver=GPKG)... +[INFO] [Output GDF] Write complete. +[INFO] [Vector Prep Orchestrate] Prepped dataframe with shape (159, 16) and outputed to /home/narlorin/ucode/riverscapes-tools/dist/usgov_sources/blm-natl-grazing-pasture-polygons-nm-bootheel/2026-04-02/blm-natl-grazing-pasture-polygons-nm-bootheel.gpkg + +``` + +* TLDR: no errors found + +### 3 Business Entity Definition - Name & ID Field + +This step in datagrip - SQL is most convenient. Copy file out of WSL filesystem, bit of pain but works. +`Pasture_ID` is field added by Megan McLachlan's team at National Operations Center. It is unique (159 records, 159 values) + +#### Name field? + +* 158 distinct `Pasture_Name`. `Flying W Mountain8` occurs twice. + +Skip adding any extra fields - we will use what was provided only. + +### build Metadata (layer_definitions) + +* fn build_layer_defs in orchestrate (adapted to omit source_url since there is none) +* updated this to include all fields in source, even if there is no metadata (infers data types) + +### Export to parquet, with bounding box + +* added function to do this +* saved to `"C:\nardata\work\reference_data_prep\blm_pastures\blm-natl-grazing-pasture-polygons-2026-03-24.parquet"` + +* upload to s3 manually: +`s3://riverscapes-athena/usgov_sources/blm-natl-grazing-pasture-polygons-nm-bootheel/2026-04-02/` + +### Build Athena table + +* ran `build_athena_ddl` +* [x] executed in Athena + +* [ ] upload new layer_definitions (merge with main) + +### Spatial intersection + diff --git a/packages/vector_prep/vector_prep/usace_nid/inputs.json b/packages/vector_prep/vector_prep/usace_nid/inputs.json new file mode 100644 index 00000000..4af5b3d2 --- /dev/null +++ b/packages/vector_prep/vector_prep/usace_nid/inputs.json @@ -0,0 +1,11 @@ +{ + "input_vector_path": "nation.gpkg", + "input_layer_name": "National Inventory of Dams", + "source_category": "usgov", + "source_title": "National Inventory of Dams", + "source_url": "https://nid.sec.usace.army.mil/nid/#/downloads", + "layer_id": "usace-nid", + "snapshot_id": "2026-04-10", + "special_notes": "We have used the ESRI feature server at https://geospatial.sec.usace.army.mil/dls/rest/services/NID/National_Inventory_of_Dams_Public_Service/FeatureServer/0. GPKG is slightly different. Used the CSV and the [NID Data Dictionary](https://nid.sec.usace.army.mil/nid/#/documents) for friendly names and descriptions.", + "data_prep_operator": "Lorin Gaertner" +} diff --git a/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json b/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json new file mode 100644 index 00000000..cf1db315 --- /dev/null +++ b/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json @@ -0,0 +1,576 @@ +{ + "$schema": "https://xml.riverscapes.net/riverscapes_metadata/schema/layer_definitions.schema.json", + "tool_schema_name": "vector-prep-usace-nid", + "tool_schema_version": "0.0.1", + "layers": [ + { + "layer_id": "usace-nid", + "layer_name": "National Inventory of Dams", + "description": "The National Inventory of Dams (NID) is a congressionally authorized database that documents more than 91,000 dams across the U.S. and its territories. It is maintained and published by the U.S. Army Corps of Engineers, in cooperation with the Association of State Dam Safety Officials, the states, territories, and federal agencies.", + "layer_type": "Vector", + "columns": [ + { + "name": "NAME", + "dtype": "STRING", + "friendly_name": "Dam Name", + "description": "The official name of the dam. For dams that do not have an official name, the popular name is used.", + "theme": "Description" + }, + { + "name": "OTHER_NAMES", + "dtype": "STRING", + "friendly_name": "Other Names", + "theme": "Description" + }, + { + "name": "FORMER_NAMES", + "dtype": "STRING", + "friendly_name": "Former Names", + "theme": "Description" + }, + { + "name": "NIDID", + "dtype": "STRING", + "friendly_name": "NID ID", + "description": "The official NID identification number for the dam, known formerly as the National ID.", + "theme": "Description" + }, + { + "name": "OTHER_STRUCTURE_ID", + "dtype": "STRING", + "friendly_name": "Other Structure ID", + "description": "The identification number (S001, S002, etc.) of a separate structure, such as a saddle dam or dike, associated with the dam project. This field applies only to saddle dams, dikes or other separate structures associated with a primary dam. This field is blank for all other dams.", + "theme": "Description" + }, + { + "name": "FEDERAL_ID", + "dtype": "STRING", + "friendly_name": "Federal ID", + "description": "The unique identifier for each dam record. For saddle dams, dikes or other separate structures associated with the dam project, it is a concatenation of the primary dam\u2019s NID ID and the Other Structure ID. For all other dams, it is the NID ID.", + "theme": "Description" + }, + { + "name": "X_OWNER_NAMES", + "dtype": "STRING", + "friendly_name": "Owner Names", + "description": "Name(s) of the dam owner. If multiple owners, different owners are separated by a semicolon.", + "theme": "Description" + }, + { + "name": "OWNER_TYPES", + "dtype": "STRING", + "friendly_name": "Owner Types", + "description": "Category describing the dam owner(s).", + "theme": "Description" + }, + { + "name": "PRIMARY_OWNER_TYPE", + "dtype": "STRING", + "friendly_name": "Primary Owner Type", + "theme": "Description" + }, + { + "name": "NUMBER_ASSOCIATED_STRUCTURES", + "dtype": "INTEGER", + "friendly_name": "Number of Associated Structures", + "theme": "Description" + }, + { + "name": "IS_ASSOCIATED_STRUCTURE", + "dtype": "STRING", + "friendly_name": "Is Associated Structure?", + "theme": "Description" + }, + { + "name": "DESIGNER_NAMES", + "dtype": "STRING", + "friendly_name": "Designer Names", + "theme": "Description" + }, + { + "name": "NON_FED_ON_FED", + "dtype": "STRING", + "friendly_name": "Non-Federal Dam on Federal Property", + "theme": "Description" + }, + { + "name": "PRIMARY_PURPOSE", + "dtype": "STRING", + "friendly_name": "Primary Purpose", + "description": "Category describing the main purpose for which the reservoir is used. If more than one purpose, the most important is used.", + "theme": "Description" + }, + { + "name": "PURPOSES", + "dtype": "STRING", + "friendly_name": "Purposes", + "description": "Category describing the current purpose(s) for which the reservoir is used.", + "theme": "Description" + }, + { + "name": "PRIMARY_SOURCE_AGENCY", + "dtype": "STRING", + "friendly_name": "Source Agency", + "theme": "Description" + }, + { + "name": "STATE_FED_AGENCY", + "dtype": "STRING", + "friendly_name": "State or Federal Agency ID", + "theme": "Description" + }, + { + "name": "LATITUDE", + "dtype": "FLOAT", + "friendly_name": "Latitude", + "description": "Latitude at dam centerline as a single value in decimal degrees, NAD83.", + "theme": "Description" + }, + { + "name": "LONGITUDE", + "dtype": "FLOAT", + "friendly_name": "Longitude", + "description": "Longitude at dam centerline as a single value in decimal degrees, NAD83.", + "theme": "Description" + }, + { + "name": "STATE", + "dtype": "STRING", + "friendly_name": "State", + "theme": "Description" + }, + { + "name": "COUNTYSTATE", + "dtype": "STRING", + "friendly_name": "County", + "theme": "Description" + }, + { + "name": "CITY", + "dtype": "STRING", + "friendly_name": "City", + "theme": "Description" + }, + { + "name": "DISTANCE", + "dtype": "FLOAT", + "friendly_name": "Distance to Nearest City", + "data_unit": "Miles", + "theme": "Description" + }, + { + "name": "RIVER_OR_STREAM", + "dtype": "STRING", + "friendly_name": "River or Stream Name", + "description": "River or Stream Standard Entry: The official name of the river or stream on which the dam is built. If the stream is unnamed, identify it as a tributary to a named river.", + "theme": "Description" + }, + { + "name": "CONGDIST", + "dtype": "STRING", + "friendly_name": "Congressional District", + "theme": "Description" + }, + { + "name": "AIANNH", + "dtype": "STRING", + "friendly_name": "American Indian/Alaska Native/Native Hawaiian", + "theme": "Description" + }, + { + "name": "STATE_REGULATED", + "dtype": "STRING", + "friendly_name": "State Regulated Dam", + "description": "Calculated field based on State Permitting Authority, State Inspection Authority, State Enforcement Authority, and State Jurisdictional Dam. If Yes to all four authority criteria, then the dam is considered state regulated and listed as Yes. If the state regulatory organization does not have all four authorities for this dam, then it is considered not state regulated and listed as No.", + "theme": "Description" + }, + { + "name": "STATE_JURISDICTION", + "dtype": "STRING", + "friendly_name": "State Jurisdictional Dam", + "theme": "Description" + }, + { + "name": "STATE_REGULATORY_AGENCY", + "dtype": "STRING", + "friendly_name": "State Regulatory Agency", + "theme": "Description" + }, + { + "name": "STATE_PERMITTING", + "dtype": "STRING", + "friendly_name": "State Permitting Authority", + "theme": "Description" + }, + { + "name": "STATE_INSPECTION", + "dtype": "STRING", + "friendly_name": "State Inspection Authority", + "theme": "Description" + }, + { + "name": "STATE_ENFORCEMENT", + "dtype": "STRING", + "friendly_name": "State Enforcement Authority", + "theme": "Description" + }, + { + "name": "FEDERALLY_REGULATED_DAM", + "dtype": "STRING", + "friendly_name": "Federally Regulated Dam", + "description": "Calculated field based on the data field \u201cFederal Agency Involvement Regulatory\u201d. If a Federal Agency is listed as being involved in the regulatory aspects of the dam, then the dam is listed as Yes, federally regulated.", + "theme": "Description" + }, + { + "name": "FED_AGENCY_OWNERS", + "dtype": "STRING", + "friendly_name": "Federal Agency Owners", + "theme": "Description" + }, + { + "name": "FED_AGENCY_FUNDINGS", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Funding", + "theme": "Description" + }, + { + "name": "FED_AGENCY_DESIGNERS", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Design", + "theme": "Description" + }, + { + "name": "FED_AGENCY_CONSTRUCTIONS", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Construction", + "theme": "Description" + }, + { + "name": "FED_AGENCY_REGULATORIES", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Regulatory", + "theme": "Description" + }, + { + "name": "FED_AGENCY_INSPECTIONS", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Inspection", + "theme": "Description" + }, + { + "name": "FED_AGENCY_OPERATIONS", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Operation", + "theme": "Description" + }, + { + "name": "FED_AGENCY_OTHERS", + "dtype": "STRING", + "friendly_name": "Federal Agency Involvement Other", + "theme": "Description" + }, + { + "name": "BUILT_UNDER_SEC_OF_AG_AUTH", + "dtype": "STRING", + "friendly_name": "Built Under the Authority of the Secretary of Agriculture", + "theme": "Description" + }, + { + "name": "NRCS_WATERSHED_DAM_AUTH", + "dtype": "STRING", + "friendly_name": "NRCS Watershed Dam Authorization", + "theme": "Description" + }, + { + "name": "PRIMARY_DAM_TYPE", + "dtype": "STRING", + "friendly_name": "Primary Dam Type", + "description": "Category describing the main type of dam. If more than one type, the most dominant is used.", + "theme": "Structure" + }, + { + "name": "DAM_TYPES", + "dtype": "STRING", + "friendly_name": "Dam Types", + "description": "Category describing the type of dam.", + "theme": "Structure" + }, + { + "name": "CORE_TYPES", + "dtype": "STRING", + "friendly_name": "Core Types", + "theme": "Structure" + }, + { + "name": "FOUNDATIONS", + "dtype": "STRING", + "friendly_name": "Foundation", + "theme": "Structure" + }, + { + "name": "DAM_HEIGHT", + "dtype": "FLOAT", + "friendly_name": "Dam Height", + "data_unit": "ft", + "description": "Height of the dam, which is defined as the vertical distance between the lowest point on the crest of the dam and the lowest point in the original streambed.", + "theme": "Structure" + }, + { + "name": "HYDRAULIC_HEIGHT", + "dtype": "FLOAT", + "friendly_name": "Hydraulic Height", + "data_unit": "ft", + "description": "Hydraulic height of the dam, which is defined as the vertical difference between the maximum design water level and the lowest point in the original streambed.", + "theme": "Structure" + }, + { + "name": "STRUCTURAL_HEIGHT", + "dtype": "FLOAT", + "friendly_name": "Structural Height", + "data_unit": "ft", + "description": "Structural height of the dam, which is defined as the vertical distance from the lowest point of the excavated foundation to the top of the dam. Top of dam refers to the parapet wall and not the crest.", + "theme": "Structure" + }, + { + "name": "NID_HEIGHT", + "dtype": "FLOAT", + "friendly_name": "NID Height", + "data_unit": "ft", + "description": "Calculated field: Maximum value of dam height, structural height, and hydraulic height. Accepted as the general height of the dam.", + "theme": "Structure" + }, + { + "name": "X_NID_HEIGHT_CATEGORY", + "dtype": "STRING", + "friendly_name": "NID Height Category", + "description": "Calculated field: Based on the NID Height, grouped into categories: less than 25 feet, 25\u201349 feet, 50\u2013100 feet, and greater than 100 feet.", + "theme": "Structure" + }, + { + "name": "DAM_LENGTH", + "dtype": "FLOAT", + "friendly_name": "Dam Length", + "data_unit": "ft", + "description": "Length of the dam, which is defined as the length along the top of the dam. This also includes the spillway, powerplant, navigation lock, fish pass, etc., where these form part of the length of the dam. If detached from the dam, these structures should not be included.", + "theme": "Structure" + }, + { + "name": "DAM_VOLUME", + "dtype": "FLOAT", + "friendly_name": "Volume", + "data_unit": "cubic yards", + "description": "Total volume occupied by the materials used in the dam structure. Portions of powerhouse, locks, and spillways are included only if they are an integral part of the dam and required for structural stability.", + "theme": "Structure" + }, + { + "name": "YEAR_COMPLETED", + "dtype": "INTEGER", + "friendly_name": "Year Completed", + "description": "Year (four digits) when the original main dam structure was completed. If unknown, and reasonable estimate is unavailable, the value will be blank.", + "theme": "Structure" + }, + { + "name": "X_YEAR_COMPLETED_CATEGORY", + "dtype": "STRING", + "friendly_name": "Year Completed Category", + "description": "Calculated field: Based on the Year Completed Date, grouped into categories: before 1900, 1900\u20131909, 1910\u20131919, 1920\u20131929, 1930\u20131939, 1940\u20131949, 1950\u20131959, 1960\u20131969, 1970\u20131979, 1980\u20131989, 1900\u20131999, Since 2000, and Undetermined.", + "theme": "Structure" + }, + { + "name": "YEARS_MODIFIED", + "friendly_name": "Years Modified", + "theme": "Structure" + }, + { + "name": "NID_STORAGE", + "dtype": "FLOAT", + "friendly_name": "NID Storage", + "data_unit": "acre-ft", + "description": "Calculated field: Maximum value of normal storage and maximum storage. Accepted as the general storage of the dam.", + "theme": "Structure" + }, + { + "name": "MAX_STORAGE", + "dtype": "FLOAT", + "friendly_name": "Max Storage", + "data_unit": "acre-ft", + "description": "Maximum storage, which is defined as the total storage space in a reservoir below the maximum attainable water surface elevation, including any surcharge storage.", + "theme": "Structure" + }, + { + "name": "NORMAL_STORAGE", + "dtype": "FLOAT", + "friendly_name": "Normal Storage", + "data_unit": "acre-ft", + "description": "Normal storage, which is defined as the total storage space in a reservoir below the normal retention level, including dead and inactive storage and excluding any flood control or surcharge storage. For normally dry dams, the normal storage will be a zero value. If unknown, the value will be blank and not zero.", + "theme": "Structure" + }, + { + "name": "SURFACE_AREA", + "dtype": "FLOAT", + "friendly_name": "Surface Area", + "data_unit": "acres", + "description": "Surface area of the impoundment at its normal retention level.", + "theme": "Structure" + }, + { + "name": "DRAINAGE_AREA", + "dtype": "FLOAT", + "friendly_name": "Drainage Area", + "data_unit": "square miles", + "description": "Drainage area of the dam, which is defined as the area that drains to a particular point (in this case, the dam) on a river or stream.", + "theme": "Structure" + }, + { + "name": "MAX_DISCHARGE", + "dtype": "FLOAT", + "friendly_name": "Max Discharge", + "data_unit": "ft**3/s", + "description": "Volume per time which the spillway is capable of discharging when the reservoir is at its maximum designed water surface elevation", + "theme": "Structure" + }, + { + "name": "SPILLWAY_TYPE", + "dtype": "STRING", + "friendly_name": "Spillway Type", + "description": "Category describing the type of spillway. (Controlled, Uncontrolled, None)", + "theme": "Structure" + }, + { + "name": "SPILLWAY_WIDTH", + "dtype": "FLOAT", + "friendly_name": "Spillway Width", + "data_unit": "ft", + "description": "The width of the spillway, available for discharge when the reservoir is at its maximum designed water surface elevation. Typically for an open channel spillway, this is the bottom width. For pipe spillways or drop inlets that have diameters, use the diameter of the pipe.", + "theme": "Structure" + }, + { + "name": "NUMBER_OF_LOCKS", + "friendly_name": "Number of Locks", + "data_unit": "count", + "theme": "Structure" + }, + { + "name": "LENGTH_OF_LOCKS", + "dtype": "FLOAT", + "friendly_name": "Length of Locks", + "data_unit": "ft", + "theme": "Structure" + }, + { + "name": "WIDTH_OF_LOCKS", + "dtype": "FLOAT", + "friendly_name": "Lock Width", + "data_unit": "ft", + "theme": "Structure" + }, + { + "name": "X_LENGTH_SECONDARY_LOCK", + "friendly_name": "Length of Secondary Lock", + "data_unit": "ft", + "theme": "Structure" + }, + { + "name": "X_SECONDARY_LOCK_WIDTH", + "friendly_name": "Secondary Lock Width", + "data_unit": "ft", + "theme": "Structure" + }, + { + "name": "OUTLET_GATES", + "dtype": "STRING", + "friendly_name": "Outlet Gate Type", + "theme": "Structure" + }, + { + "name": "DATA_UPDATED", + "friendly_name": "Data Last Updated", + "theme": "Inspection and Evaluation" + }, + { + "name": "LAST_INSPECTION_DATE", + "friendly_name": "Last Inspection Date", + "theme": "Inspection and Evaluation" + }, + { + "name": "INSPECTION_FREQUENCY", + "friendly_name": "Inspection Frequency", + "theme": "Inspection and Evaluation" + }, + { + "name": "HAZARD_POTENTIAL", + "dtype": "STRING", + "friendly_name": "Hazard Potential Classification", + "description": "Category to indicate the potential hazard to the downstream area resulting from failure or mis-operation of the dam or facilities. It reflects probable loss of human life and impacts on economic, environmental, and lifeline interests. The hazard potential does not speak to the condition of the dam or the risk of the dam failing. Low / Significant / High / Undetermined", + "theme": "Inspection and Evaluation" + }, + { + "name": "CONDITION_ASSESSMENT", + "dtype": "STRING", + "friendly_name": "Condition Assessment", + "description": "Assessment that best describes the condition of the dam based on available information. Satisfactory / Fair / Poor / Unsatisfactory / Not Rated / Not Available", + "theme": "Inspection and Evaluation" + }, + { + "name": "CONDITION_ASSESS_DATE", + "friendly_name": "Condition Assessment Date", + "theme": "Inspection and Evaluation" + }, + { + "name": "OPERATIONAL_STATUS", + "dtype": "STRING", + "friendly_name": "Operational Status", + "description": "Category that best describes the operational or remediation activities of the dam based on available information. (Normal Operations / Under Investigation, Planning, Permitting, or Design for Remediation / Under Remediation / Enforcement Pending/Ongoing / Not Applicable)", + "theme": "Inspection and Evaluation" + }, + { + "name": "OPERATIONAL_STATUS_DATE", + "friendly_name": "Operational Status Date", + "theme": "Inspection and Evaluation" + }, + { + "name": "EAP_PREPARED", + "friendly_name": "EAP Prepared", + "theme": "Response Preparedness" + }, + { + "name": "EAP_LAST_REV_DATE", + "friendly_name": "EAP Last Revision Date", + "theme": "Response Preparedness" + }, + { + "name": "X_INUNDATION_MAPS_ADDED_TO_NID", + "friendly_name": "Inundation Maps Added to NID?", + "theme": "Response Preparedness" + }, + { + "name": "WEBSITE_URL", + "dtype": "STRING", + "friendly_name": "Website URL", + "description": "Web Site for more information on specific dam or regulatory agency.", + "theme": "Other" + }, + { + "name": "OBJECTID", + "description": "(not present in CSV or Data Dictionary)", + "theme": "Other" + }, + { + "name": "CONG_REPRESNTATIVE", + "description": "(not present in CSV or Data Dictionary)", + "theme": "Description" + }, + { + "name": "ASSOCIATED_STRUCTURES", + "description": "(not present in CSV or Data Dictionary)", + "theme": "Description" + }, + { + "name": "SE_ANNO_CAD_DATA", + "description": "(not present in CSV or Data Dictionary)", + "theme": "Other" + } + ] + } + ] +} \ No newline at end of file diff --git a/packages/vector_prep/vector_prep/usace_nid/runlog-usace-nid.md b/packages/vector_prep/vector_prep/usace_nid/runlog-usace-nid.md new file mode 100644 index 00000000..6c6535c3 --- /dev/null +++ b/packages/vector_prep/vector_prep/usace_nid/runlog-usace-nid.md @@ -0,0 +1,15 @@ +# 2026-04-11 + +There are 3 separate formats for the data: + +* CSV download +* GPGK download +* ESRI feature service + +They all have slightly different set of columns and the way they name them is different. +I noticed the GPKG columns seem to be different in a version download Jan vs April of 2026. +The feature service allows selection of a sub-area - so we used that as an API call in rs-reports-gen to fetch latest version of the data, and only what's needed. + +For column names, descriptions, see the PDF data dictionary. The file name when you download it, says "June 2025" but the contents say August 2024. + +I assembled this somewhat manually in google sheet, export to CSV, use the rsxml script to convert to layer_definitions.json. diff --git a/pyproject.toml b/pyproject.toml index 94c2bf07..082d25ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,11 @@ dependencies = [ "geopandas>=1.1.1", ] +[project.optional-dependencies] +geoparquet = [ + "pyarrow>=23.0.1", +] + [dependency-groups] dev = [ "autopep8>=2.3", @@ -119,4 +124,4 @@ rcat = "packages/rcat/rcat" "rcat.lib.accessibility" = "packages/rcat/rcat/lib/accessibility" [tool.setuptools.package-data] -"rcat.lib.accessibility" = ["*.pyx"] \ No newline at end of file +"rcat.lib.accessibility" = ["*.pyx"] diff --git a/uv.lock b/uv.lock index dbd126c4..27bf07dc 100644 --- a/uv.lock +++ b/uv.lock @@ -837,6 +837,11 @@ dependencies = [ { name = "urllib3" }, ] +[package.optional-dependencies] +geoparquet = [ + { name = "pyarrow" }, +] + [package.dev-dependencies] catalog = [ { name = "riverscapes-metadata" }, @@ -859,6 +864,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.26" }, { name = "postgis", specifier = ">=1.0.4" }, { name = "psycopg2-binary", specifier = ">=2.9.9" }, + { name = "pyarrow", marker = "extra == 'geoparquet'", specifier = ">=23.0.1" }, { name = "python-dateutil", specifier = ">=2.9.0.post0" }, { name = "questionary", specifier = ">=2.0.1" }, { name = "rasterio", specifier = ">=1.3.9" }, @@ -874,6 +880,7 @@ requires-dist = [ { name = "termcolor", specifier = ">=2.4" }, { name = "urllib3", specifier = ">=2.2" }, ] +provides-extras = ["geoparquet"] [package.metadata.requires-dev] catalog = [{ name = "riverscapes-metadata", git = "https://github.com/Riverscapes/RiverscapesXML.git?subdirectory=riverscapes_metadata&rev=master" }] From 241f6209f81cab3cb60138971bec514c93ae3111 Mon Sep 17 00:00:00 2001 From: Lorin Date: Sat, 11 Apr 2026 11:53:13 -0700 Subject: [PATCH 3/3] fix units --- .../vector_prep/usace_nid/layer_definitions.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json b/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json index cf1db315..733aba0c 100644 --- a/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json +++ b/packages/vector_prep/vector_prep/usace_nid/layer_definitions.json @@ -155,7 +155,7 @@ "name": "DISTANCE", "dtype": "FLOAT", "friendly_name": "Distance to Nearest City", - "data_unit": "Miles", + "data_unit": "mile", "theme": "Description" }, { @@ -385,7 +385,7 @@ "name": "NID_STORAGE", "dtype": "FLOAT", "friendly_name": "NID Storage", - "data_unit": "acre-ft", + "data_unit": "acre * ft", "description": "Calculated field: Maximum value of normal storage and maximum storage. Accepted as the general storage of the dam.", "theme": "Structure" }, @@ -393,7 +393,7 @@ "name": "MAX_STORAGE", "dtype": "FLOAT", "friendly_name": "Max Storage", - "data_unit": "acre-ft", + "data_unit": "acre * ft", "description": "Maximum storage, which is defined as the total storage space in a reservoir below the maximum attainable water surface elevation, including any surcharge storage.", "theme": "Structure" }, @@ -401,7 +401,7 @@ "name": "NORMAL_STORAGE", "dtype": "FLOAT", "friendly_name": "Normal Storage", - "data_unit": "acre-ft", + "data_unit": "acre * ft", "description": "Normal storage, which is defined as the total storage space in a reservoir below the normal retention level, including dead and inactive storage and excluding any flood control or surcharge storage. For normally dry dams, the normal storage will be a zero value. If unknown, the value will be blank and not zero.", "theme": "Structure" },