From c4e1279eeeb8cb7eff79ac5407eb32dbf7ad5c3c Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Tue, 7 Jul 2026 16:18:00 +0200 Subject: [PATCH 01/19] Bug fixes for Windows - Changed order of renaming in PLD in case users specify "lake_id" as their id as well - Open config with utf-8 encoding to avoid bugs when editing .yaml files with Windows - Path normalization (function in "general" + fixes to project.py) to avoid path issues when opening nc files with hdf5/netcdf4 on Windows (breaks subsetting of sentinel files) - fixed bug in geometry with coords.pop function set as index --- .gitignore | 3 ++- HydroEO/flows.py | 7 +++---- HydroEO/project.py | 15 ++++++++------- HydroEO/utils/general.py | 10 ++++++++++ HydroEO/utils/geometry.py | 2 +- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index c8166c0..74d81d3 100644 --- a/.gitignore +++ b/.gitignore @@ -158,4 +158,5 @@ config_raster.yaml config_pixel_cloud.yaml notebooks/SWOT_Download.ipynb notebooks/swot_examples/* -notebooks/swot_pixel_examples/* \ No newline at end of file +notebooks/swot_pixel_examples/* +test_notebook.ipynb diff --git a/HydroEO/flows.py b/HydroEO/flows.py index 5e18052..fc60c9a 100644 --- a/HydroEO/flows.py +++ b/HydroEO/flows.py @@ -102,6 +102,9 @@ def _assign_pld_id(prj: "Project") -> None: """Spatial join reservoirs with PLD to assign prior_lake_id.""" pld = gpd.read_file(prj.dirs["pld"]) + pld = pld.rename( + columns={"lake_id": "prior_lake_id", "res_id": "prior_res_id"} + ) joined_gdf = gpd.sjoin_nearest( prj.reservoirs.gdf.to_crs(prj.local_crs), pld.to_crs(prj.local_crs), @@ -113,10 +116,6 @@ def _assign_pld_id(prj: "Project") -> None: ) joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) - joined_gdf = joined_gdf.rename( - columns={"lake_id": "prior_lake_id", "res_id": "prior_res_id"} - ) - joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 prj.reservoirs.gdf = joined_gdf diff --git a/HydroEO/project.py b/HydroEO/project.py index ba6c9ec..5639849 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -52,7 +52,7 @@ def __post_init__(self): self.enddates = dict() ### Load in the config file and extract parameters - with open(self.config, "rt") as f: + with open(self.config, "rt", encoding="utf-8") as f: self.config = yaml.safe_load(f.read()) if self.config is None: @@ -63,7 +63,7 @@ def __post_init__(self): ### Define the project directory for saving outputs, etc if "project" in self.config.keys(): - self.dirs["main"] = self.config["project"]["main_dir"] + self.dirs["main"] = general.normalize_path(self.config["project"]["main_dir"]) general.ifnotmakedirs(self.dirs["main"]) else: raise Warning("Project directory must be defined within configuration file") @@ -81,7 +81,7 @@ def __post_init__(self): self.dirs["main"], "aux", "PLD", "PLD_subset.gpkg" ) if "raw_pld_path" in hydroweb_cfg: - self.dirs["pld_raw"] = hydroweb_cfg["raw_pld_path"] + self.dirs["pld_raw"] = general.normalize_path(hydroweb_cfg["raw_pld_path"]) self.keep_raw_pld = hydroweb_cfg.get("keep_raw_pld", False) # Set SWORD database paths and configuration @@ -91,9 +91,9 @@ def __post_init__(self): self.dirs["main"], "aux", "SWORD", "SWORD_subset.gpkg" ) if "sword_subset_path" in sword_db_cfg: - self.dirs["sword_subset"] = sword_db_cfg["sword_subset_path"] + self.dirs["sword_subset"] = general.normalize_path(sword_db_cfg["sword_subset_path"]) if "raw_sword_path" in sword_db_cfg: - self.dirs["sword_raw"] = sword_db_cfg["raw_sword_path"] + self.dirs["sword_raw"] = general.normalize_path(sword_db_cfg["raw_sword_path"]) # Safety: if raw_sword_path is outside main_dir, force keep_raw_sword=True raw_path = os.path.abspath(sword_db_cfg["raw_sword_path"]) main_path = os.path.abspath(self.dirs["main"]) @@ -119,7 +119,7 @@ def __post_init__(self): or os.environ.get("EARTHDATA_PASSWORD") or os.environ.get("EDL_PASSWORD") ) - + if self.earthdata_user: os.environ["EARTHDATA_USERNAME"] = self.earthdata_user os.environ["EDL_USERNAME"] = self.earthdata_user @@ -321,7 +321,8 @@ def __sat_init(self, name: str): self.to_process.append(name) if "download_dir" in self.config[name].keys(): - self.dirs[name] = self.config[name]["download_dir"] + self.dirs[name] = general.normalize_path(self.config[name]["download_dir"]) + #self.dirs[name] = Path(self.config[name]["download_dir"]) else: self.dirs[name] = os.path.join(self.dirs["main"], "raw", name) diff --git a/HydroEO/utils/general.py b/HydroEO/utils/general.py index 5c5f34a..3ea7b8c 100644 --- a/HydroEO/utils/general.py +++ b/HydroEO/utils/general.py @@ -1,12 +1,22 @@ """General Utilities to aid in other modules""" import os +import re import shutil import zipfile from typing import Union from tqdm import tqdm +def normalize_path(path_str: str) -> str: + """Make a path from a config file robust to mixed / and \ separators, + regardless of what OS it was written on or is being read on.""" + path_str = path_str.strip().strip('"').strip("'") # strip stray quotes too + unified = re.sub(r'[\\/]+', '/', path_str) # collapse any run of slashes to a single / + return os.path.normpath(unified) # let os.path convert to native separators + + + def ifnotmakedirs(dir: str): if not os.path.exists(dir): os.makedirs(dir) diff --git a/HydroEO/utils/geometry.py b/HydroEO/utils/geometry.py index 358ac79..9025312 100644 --- a/HydroEO/utils/geometry.py +++ b/HydroEO/utils/geometry.py @@ -84,7 +84,7 @@ def format_coord_list(coords: list): # unpack the 1d list into coordinate pairs new_list = list() while len(coords) > 0: - new_list.append((coords.pop[0], coords.pop(0))) + new_list.append((coords.pop(0), coords.pop(0))) coords = new_list return coords From be12261908f24758ed37fa6bc02a04dae123c9b2 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Mon, 13 Jul 2026 16:29:28 +0200 Subject: [PATCH 02/19] Updating workflows for reservoirs and rivers - Kalman epoch grouping + closed-form update - svr_radial_max_iter / svr_linear_max_iter separation: linear can be capped to 5000, radial needs longer run - Extraction skip-if-exists + Kalman grouping performance: allow overwrite = False to avoid re-reading all individual files, Kalman speed up - Bias correction tuning (relative_orbit, bias_group_by="platform_orbit") = Group by platform and by relative orbit Spatial correction tools (distance_penalty, spatial_correction_model) = Bias correction tools for tracks far a part - off by default Config wiring (merging_options layering, mission_options whitelist) Rivers: - SWORD extraction pipeline (core) - Sentinel-6 EarthData download (for HR data) - Plotting fixes and Misc bug fixes (stale warning, rivers.yaml) for river processing - Possibility to exclude tracks - Use SWORD river width to buffer along river line when extracting other missions - Handle multipolygon extraction - Clean up of processing files when re-running - Reach slope correction for river VS. --- HydroEO/flows.py | 2014 ++++++++++++++++++++- HydroEO/plotting.py | 250 ++- HydroEO/project.py | 227 ++- HydroEO/satellites/icesat2/preprocess.py | 18 + HydroEO/satellites/sentinel/__init__.py | 36 +- HydroEO/satellites/sentinel/download.py | 145 +- HydroEO/satellites/sentinel/preprocess.py | 258 +-- HydroEO/utils/filters/basic_filters.py | 835 ++++++++- HydroEO/utils/timeseries.py | 530 +++++- 9 files changed, 3946 insertions(+), 367 deletions(-) diff --git a/HydroEO/flows.py b/HydroEO/flows.py index fc60c9a..071b7ec 100644 --- a/HydroEO/flows.py +++ b/HydroEO/flows.py @@ -8,6 +8,8 @@ import logging import os import datetime +import json +import yaml from io import StringIO from typing import TYPE_CHECKING @@ -18,6 +20,7 @@ from HydroEO.satellites import swot, icesat2, sentinel from HydroEO.utils import general, timeseries +from HydroEO.utils.filters import basic_filters from HydroEO.downloaders import hydroweb from HydroEO import plotting @@ -413,9 +416,7 @@ def _download_reservoirs_icesat2(prj: "Project") -> None: id = prj.reservoirs.download_gdf.loc[i, prj.reservoirs.id_key] logger.info("Downloading data for id %s", id) - geom = prj.reservoirs.download_gdf.loc[i, "geometry"] - if hasattr(geom, "geoms"): - geom = geom.geoms[0] + geom = _simplify_to_one_polygon(prj.reservoirs.download_gdf.loc[i, "geometry"]) coords = list(geom.exterior.coords) parquet_dir = os.path.join(prj.dirs["icesat2_processed"], rf"{id}") @@ -446,15 +447,121 @@ def _download_reservoirs_icesat2(prj: "Project") -> None: logger.warning("ICESat-2 download skipped for %s: %s", id, exc) +def _sentinel6_use_earthdata(prj: "Project") -> bool: + """ + Whether Sentinel-6 should be downloaded from PO.DAAC/EarthData (HR + product, 20Hz Ku-band) rather than CREODIAS (LR product only, see + query()'s productType="P4_2__LR_____"). Set via + mission_options["sentinel6"]["source"] = "earthdata" in config. + """ + return ( + prj.mission_options.get("sentinel6", {}) + .get("source", "creodias") + .lower() + == "earthdata" + ) + + +def _download_sentinel_for_target( + prj: "Project", mission: str, product: str, coords, download_dir, + startdate, enddate, sentinel_creds, session_token, session_start_time, +) -> tuple: + """ + Download + subset Sentinel-3/6 data for one target's AOI (a + reservoir polygon, or a river waterbody corridor's envelope) -- + shared by both _download_reservoirs_sentinel and + _download_rivers_sentinel so the CREODIAS/EarthData branching logic + only needs to exist in one place. + + For Sentinel-6, if _sentinel6_use_earthdata(prj) is True, uses + PO.DAAC/EarthData (see sentinel.query_earthdata/download_earthdata) + to get the HR product instead of CREODIAS's LR-only product. + EarthData files arrive flat (no SAFE-zip directory), so the unzip + step is skipped for that path -- subset() already handles both flat + and zipped-folder inputs (see sentinel/preprocess.py's file + discovery, extended for this). + + Returns (session_token, session_start_time) -- unchanged from what + was passed in when using the EarthData path, since that mechanism + (CREODIAS session reuse) doesn't apply to it. + """ + dir_key = mission + use_earthdata = mission == "sentinel6" and _sentinel6_use_earthdata(prj) + + logger.info( + "Searching for Sentinel-%s (%s) from %s to %s", + product, + "PO.DAAC/EarthData HR" if use_earthdata else "CREODIAS", + startdate, + enddate, + ) + + if use_earthdata: + s6_opts = prj.mission_options.get("sentinel6", {}) + results = sentinel.query_earthdata( + aoi=coords, + startdate=startdate, + enddate=enddate, + latency=s6_opts.get("latency", "NTC"), + short_name=s6_opts.get("short_name"), + ) + sentinel.download_earthdata(results, download_directory=download_dir) + # EarthData granules arrive flat already -- no SAFE-zip to unzip. + else: + ids = sentinel.query( + aoi=coords, + startdate=startdate, + enddate=enddate, + product=product, + creodias_credentials=sentinel_creds, + ) + + session_token, session_start_time = sentinel.download( + ids, + download_directory=download_dir, + creodias_credentials=sentinel_creds, + token=session_token, + session_start_time=session_start_time, + threads=prj.mission_options.get(dir_key, {}).get("download_threads", 1), + ) + + general.unzip_dir_files_with_ext( + download_dir, download_dir, ".nc", show_progress=True + ) + + sentinel.subset( + aoi=coords, + download_dir=download_dir, + dest_dir=download_dir, + file_id=prj.mission_options.get(dir_key, {}).get( + "subset_file_id", "enhanced_measurement.nc" + ), + product=product, + show_progress=True, + ) + + return session_token, session_start_time + + def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: """Download Sentinel-3 or Sentinel-6 data for reservoirs.""" product = "S3" if mission == "sentinel3" else "S6" - dir_key = mission session_token = None session_start_time = None - sentinel_creds = prj._require_creodias_credentials() + # EarthData (Sentinel-6 HR) needs no CREODIAS credentials at all -- + # only require them if we're actually going to use CREODIAS. But it + # does need its OWN upfront check -- without it, earthaccess.login() + # silently falls through to interactive prompting when nothing else + # is configured, which hangs in a non-interactive run instead of + # failing clearly (see Project._require_earthdata_credentials). + sentinel_creds = None + use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) + if use_earthdata_s6: + prj._require_earthdata_credentials() + else: + sentinel_creds = prj._require_creodias_credentials() for i in prj.reservoirs.download_gdf.index: id = prj.reservoirs.download_gdf.loc[i, prj.reservoirs.id_key] @@ -467,7 +574,7 @@ def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: ].envelope.exterior.coords ] - download_dir = os.path.join(prj.dirs[dir_key], rf"{id}") + download_dir = os.path.join(prj.dirs[mission], rf"{id}") general.ifnotmakedirs(download_dir) startdate = prj.startdates[mission] @@ -478,42 +585,9 @@ def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: if isinstance(enddate, list): enddate = datetime.date(*enddate) - logger.info( - "Searching for Sentinel-%s for aoi from %s to %s", - product, - startdate, - enddate, - ) - ids = sentinel.query( - aoi=coords, - startdate=startdate, - enddate=enddate, - product=product, - creodias_credentials=sentinel_creds, - ) - - session_token, session_start_time = sentinel.download( - ids, - download_directory=download_dir, - creodias_credentials=sentinel_creds, - token=session_token, - session_start_time=session_start_time, - threads=prj.mission_options.get(dir_key, {}).get("download_threads", 1), - ) - - general.unzip_dir_files_with_ext( - download_dir, download_dir, ".nc", show_progress=True - ) - - sentinel.subset( - aoi=coords, - download_dir=download_dir, - dest_dir=download_dir, - file_id=prj.mission_options.get(dir_key, {}).get( - "subset_file_id", "enhanced_measurement.nc" - ), - product=product, - show_progress=True, + session_token, session_start_time = _download_sentinel_for_target( + prj, mission, product, coords, download_dir, + startdate, enddate, sentinel_creds, session_token, session_start_time, ) @@ -523,7 +597,14 @@ def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: def download_rivers(prj: "Project") -> None: - """Download SWOT Hydrocron data for rivers. + """Download altimetry data for all configured missions (rivers mode). + + SWOT uses the Hydrocron timeseries API directly (per node/reach, no + clustering needed -- see _download_swot_hydrocron_timeseries). + ICESat-2/Sentinel-3/6 download raw observations over a buffered + corridor around each waterbody's SWORD targets (see + _river_target_corridor); associating individual points with a + specific target happens later, during extraction. Parameters ---------- @@ -533,18 +614,38 @@ def download_rivers(prj: "Project") -> None: if not hasattr(prj, "rivers"): return - if "swot" not in prj.to_download: + if "swot" not in prj.to_download and not any( + m in prj.to_download for m in ("icesat2", "sentinel3", "sentinel6") + ): + logger.warning( + "Rivers are configured but no mission is enabled for download. " + "Add a top-level mission section (e.g. 'swot:', 'icesat2:', " + "'sentinel3:'/'sentinel6:') with 'download: true' to actually " + "download river observations. SWORD itself will still have " + "been prepared by initialize(), which is why you may see " + "SWORD files but no timeseries data." + ) return - startdate = prj.startdates.get("swot") - enddate = prj.enddates.get("swot") + if "swot" in prj.to_download: + startdate = prj.startdates.get("swot") + enddate = prj.enddates.get("swot") - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + _download_swot_hydrocron_timeseries(prj, startdate, enddate) + + if "icesat2" in prj.to_download: + _download_rivers_icesat2(prj) - _download_swot_hydrocron_timeseries(prj, startdate, enddate) + if "sentinel3" in prj.to_download: + _download_rivers_sentinel(prj, "sentinel3") + + if "sentinel6" in prj.to_download: + _download_rivers_sentinel(prj, "sentinel6") def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> None: @@ -741,6 +842,277 @@ def _group_river_targets_by_waterbody(prj: "Project") -> dict: ) +def _river_target_corridor( + prj: "Project", target_ids, buffer_meters=None, width_buffer_factor=1.05, +): + """ + Build one buffered, dissolved corridor polygon covering the given + river targets (nodes or reaches), for use as the spatial AOI when + downloading/extracting ICESat-2 and Sentinel-3/6 observations. + + This is deliberately a SEPARATE buffer distance from + prj.rivers.buffer_meters (used earlier to decide which SWORD + targets intersect the user's AOI at all) -- that question ("is this + target in scope") and this one ("how far from the centerline could + real river water still be, for a raw altimetry point to plausibly + belong to this target") are different, and conflating them risks + the same "one parameter doing two jobs badly" issue found elsewhere + in this pipeline. + + Parameters + ---------- + buffer_meters : float or None, optional + Explicit, uniform buffer distance (meters), applied to every + target regardless of its actual width. If None (default), uses + each target's own SWORD "width" attribute instead: buffer + distance = (width / 2) * width_buffer_factor. This is HALF the + width, not the full width -- buffering a line expands it + symmetrically by the given distance on EACH side, so a buffer + of width/2 gives a corridor whose TOTAL span is approximately + width * width_buffer_factor, matching the river's actual extent + plus a margin, rather than doubling it. Falls back to + _river_extraction_buffer_meters() (a flat scalar) if no usable + "width" column is found -- e.g. if your SWORD data names it + differently than assumed here, this degrades gracefully with a + log message rather than failing. + width_buffer_factor : float, optional + Margin applied on top of each target's own width when using the + width-based default. Default 1.05 -- 5% wider than the river's + actual channel width. Only used when buffer_meters is None. + + NOTE: "width" is the expected SWORD column name per the standard + SWORD data dictionary -- this has NOT been verified against a real + downloaded SWORD file in this session (no sample data was + available), unlike most other assumptions in this codebase. Check + your actual target_features columns if width-based buffering + doesn't seem to be kicking in. + + Returns a single-row GeoDataFrame in prj.global_crs (matching what + icesat2.extract_observations/sentinel.extract_observations expect + for their `features` argument, same as reservoirs), or None if no + matching SWORD geometry is found for target_ids. Note the returned + geometry may be a MultiPolygon if targets form disconnected pieces + (e.g. separate reaches far enough apart that their buffers never + touch) -- see _iter_geometry_pieces for how downloads handle this. + """ + features = prj.rivers.target_features + subset = features.loc[features[prj.rivers.target_id_col].isin(target_ids)] + if subset.empty: + return None + + local = subset.to_crs(prj.local_crs) + + if buffer_meters is not None: + distances = buffer_meters + elif "width" in local.columns and local["width"].notna().any(): + fallback_width = local["width"].median() + distances = (local["width"].fillna(fallback_width) / 2) * width_buffer_factor + else: + logger.info( + "No 'width' column found in SWORD target_features for this " + "waterbody -- falling back to a flat extraction buffer " + "instead of width-based sizing. Check your SWORD data's " + "actual column names if this is unexpected." + ) + distances = _river_extraction_buffer_meters(prj) + + buffered = local.buffer(distances) + corridor = buffered.unary_union + corridor_gdf = gpd.GeoDataFrame( + geometry=[corridor], crs=prj.local_crs + ).to_crs(prj.global_crs) + return corridor_gdf + + +def _iter_geometry_pieces(geom): + """ + Yield each individual polygon from a geometry: every part of a + MultiPolygon, or the geometry itself for a plain Polygon. Used for + river downloads, where disconnected corridor pieces should each get + their own query rather than only querying the first piece (silently + dropping coverage of the rest) or merging them into one shape that + would also cover the (possibly large, irrelevant) gap between them. + """ + if hasattr(geom, "geoms"): + return list(geom.geoms) + return [geom] + + +def _simplify_to_one_polygon(geom): + """ + Collapse a MultiPolygon into a single encompassing polygon via + convex hull. Used for reservoirs: unlike rivers, a reservoir is + treated as one target regardless of how many disconnected parts its + input polygon has, so a single combined query is preferred over + splitting into several separate ones. Convex hull guarantees full + coverage of every part, at the cost of also covering some in-between + area that may not be real water -- an accepted tradeoff for treating + one reservoir as one query rather than several. + """ + if hasattr(geom, "geoms"): + return geom.convex_hull + return geom + + +def _river_extraction_buffer_meters(prj: "Project") -> float: + """ + Resolve the extraction-corridor buffer distance: prefer an explicit + prj.rivers.extraction_buffer_meters if set, else fall back to the + SWORD-intersection prj.rivers.buffer_meters, else a conservative + default. Kept as its own small function since this fallback chain + is used by both download and extraction. + """ + explicit = getattr(prj.rivers, "extraction_buffer_meters", None) + if explicit: + return explicit + if prj.rivers.buffer_meters: + return prj.rivers.buffer_meters + return 500.0 + + +def _download_rivers_icesat2(prj: "Project") -> None: + """Download ICESat-2 ATL13 data for river waterbody groups. + + Mirrors _download_reservoirs_icesat2, but queries over a buffered + corridor around each waterbody's SWORD targets (see + _river_target_corridor) rather than a single reservoir polygon. If + a waterbody's corridor comes out as disconnected pieces (a + MultiPolygon), queries each piece separately (see + _iter_geometry_pieces) rather than only the first -- unlike + reservoirs, a river waterbody's targets can legitimately be + disjoint (e.g. separate reaches far apart), so collapsing to one + query would either miss coverage or require an artificially large + combined shape. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + + startdate = prj.startdates["icesat2"] + enddate = prj.enddates["icesat2"] + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + for wb_id, target_ids in waterbody_groups.items(): + logger.info("Downloading ICESat-2 data for waterbody %s", wb_id) + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + logger.warning( + "No SWORD geometry found for waterbody %s; skipping " + "ICESat-2 download.", wb_id, + ) + continue + + pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) + parquet_dir = os.path.join(prj.dirs["icesat2_processed"], f"{wb_id}") + general.ifnotmakedirs(parquet_dir) + + for piece_idx, geom in enumerate(pieces): + coords = list(geom.exterior.coords) + + logger.info( + "Searching for Icesat2 ATL13 for waterbody %s (piece %d/%d) " + "from %s to %s", wb_id, piece_idx + 1, len(pieces), startdate, enddate, + ) + try: + _ = icesat2.query( + aoi=coords, + startdate=startdate, + enddate=enddate, + download_directory=parquet_dir, + atl13_options=prj.mission_options.get("icesat2", {}).get("atl13", {}), + atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields") + or None, + ) + except Exception as exc: + logger.warning( + "ICESat-2 download skipped for waterbody %s (piece %d/%d): %s", + wb_id, piece_idx + 1, len(pieces), exc, + ) + + +def _download_rivers_sentinel(prj: "Project", mission: str) -> None: + """Download Sentinel-3 or Sentinel-6 data for river waterbody groups. + + Mirrors _download_reservoirs_sentinel (both now share + _download_sentinel_for_target, including the CREODIAS/EarthData + branching for Sentinel-6). NOTE: sentinel.query/query_earthdata take + a bounding box (envelope), not the exact corridor polygon -- for a + long or winding river corridor this can query/download a + meaningfully larger area than the actual buffered corridor. This is + an existing limitation inherited from the reservoir path (where it + matters far less, since a reservoir's envelope is close to its + actual extent), not something new introduced here -- worth + revisiting if it turns out to matter in practice for a large or + winding waterbody. + """ + product = "S3" if mission == "sentinel3" else "S6" + + sentinel_creds = None + use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) + if use_earthdata_s6: + prj._require_earthdata_credentials() + else: + sentinel_creds = prj._require_creodias_credentials() + + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + + startdate = prj.startdates[mission] + enddate = prj.enddates[mission] + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + session_token = None + session_start_time = None + + for wb_id, target_ids in waterbody_groups.items(): + logger.info("Downloading data for waterbody %s", wb_id) + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + logger.warning( + "No SWORD geometry found for waterbody %s; skipping " + "Sentinel-%s download.", wb_id, product, + ) + continue + + # Envelope each disconnected piece separately rather than the + # whole (possibly MultiPolygon) corridor at once -- sentinel's + # API only accepts a bounding box, so one envelope covering + # widely separated pieces could be far larger than any of them + # individually. + pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) + + download_dir = os.path.join(prj.dirs[mission], f"{wb_id}") + general.ifnotmakedirs(download_dir) + + for piece_idx, geom in enumerate(pieces): + coords = [(x, y) for x, y in geom.envelope.exterior.coords] + + logger.info( + "Searching for Sentinel-%s for waterbody %s (piece %d/%d) " + "from %s to %s", product, wb_id, piece_idx + 1, len(pieces), + startdate, enddate, + ) + session_token, session_start_time = _download_sentinel_for_target( + prj, mission, product, coords, download_dir, + startdate, enddate, sentinel_creds, session_token, session_start_time, + ) + + def _get_latest_hydrocron_obs_date(output_path) -> datetime.date: """Get latest observation date from existing Hydrocron output.""" if not os.path.exists(output_path): @@ -766,6 +1138,386 @@ def _get_latest_hydrocron_obs_date(output_path) -> datetime.date: return datetime.date(latest_obs.year, latest_obs.month, latest_obs.day) +# ============================================================================ +# RIVERS: Timeseries Processing (extraction) +# ============================================================================ + + +def _assign_points_to_river_targets( + points, targets, target_id_col, max_distance_meters, local_crs +): + """ + Assign each point in `points` to its nearest feature in `targets` + (SWORD node or reach geometries, whichever prj.rivers.target_id_col + is configured for), dropping points farther than max_distance_meters + from any target. + + Uses gpd.sjoin_nearest rather than a custom NearestNeighbors/DBSCAN + approach -- it handles point-to-line matching natively (needed for + reaches, not just nodes), and max_distance is expressed directly in + real distance units once both inputs are reprojected to local_crs. + + Returns points (unprojected, original CRS) with target_id_col and a + _dist_to_target_m column added; rows with no target within range are + dropped entirely. + """ + points_local = points.to_crs(local_crs) + targets_local = targets[[target_id_col, "geometry"]].to_crs(local_crs) + + joined = gpd.sjoin_nearest( + points_local, + targets_local, + how="inner", + max_distance=max_distance_meters, + distance_col="_dist_to_target_m", + ) + + result = points.loc[joined.index].copy() + result[target_id_col] = joined[target_id_col].values + result["_dist_to_target_m"] = joined["_dist_to_target_m"].values + return result + + +def _extract_rivers_timeseries(prj: "Project", overwrite: bool = False) -> None: + """Extract timeseries observations from raw downloaded files, for rivers. + + SWOT still needs a (lightweight) extraction step here: Hydrocron + already returns a per-node/reach timeseries directly, but grouped + per WATERBODY (one CSV covering every target in that waterbody) -- + see _extract_rivers_swot_observations for splitting that into the + same per-target file structure ICESat-2/Sentinel-3/6 use, so the + shared clean/merge pipeline can treat every mission identically. + + Parameters + ---------- + overwrite : bool, optional + Same semantics as _extract_reservoirs_timeseries: if False + (default), any target whose output .gpkg already exists is + skipped rather than re-extracted. + """ + if "icesat2" in prj.to_process: + _extract_rivers_icesat2_observations(prj, overwrite=overwrite) + + if "sentinel3" in prj.to_process: + _extract_rivers_sentinel_observations( + prj, "sentinel3", "S3", overwrite=overwrite + ) + + if "sentinel6" in prj.to_process: + _extract_rivers_sentinel_observations( + prj, "sentinel6", "S6", overwrite=overwrite + ) + + if "swot" in prj.to_process: + _extract_rivers_swot_observations(prj, overwrite=overwrite) + + +def _extract_rivers_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: + """Extract ICESat-2 ATL13 observations for each river target. + + Unlike reservoirs (one polygon = one target), a river waterbody's + raw download covers many targets at once. This extracts once per + waterbody -- reusing icesat2.extract_observations exactly as + reservoirs use it, with the buffered corridor (see + _river_target_corridor) as the spatial filter instead of a single + reservoir polygon -- then assigns each surviving point to its + nearest target via sjoin_nearest, and splits the result into the + same per-target {output}/{target_id}/raw_observations/icesat2.gpkg + structure reservoirs already use, so everything downstream + (clean/merge) can treat a river target exactly like a reservoir. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + max_assign_dist = ( + getattr(prj.rivers, "max_node_assignment_meters", None) + or _river_extraction_buffer_meters(prj) + ) + + tmp_dir = os.path.join(prj.dirs["output"], "_tmp_river_extraction") + + for wb_id, target_ids in waterbody_groups.items(): + parquet_dir = os.path.join(prj.dirs["icesat2_processed"], f"{wb_id}") + if not os.path.exists(os.path.join(parquet_dir, "atl13.parquet")): + continue + + if not overwrite: + remaining = [ + t + for t in target_ids + if not os.path.exists( + os.path.join( + prj.dirs["output"], f"{t}", "raw_observations", "icesat2.gpkg" + ) + ) + ] + if not remaining: + continue + target_ids = remaining + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + continue + + general.ifnotmakedirs(tmp_dir) + tmp_dst = os.path.join(tmp_dir, f"{wb_id}_icesat2.gpkg") + + try: + icesat2.extract_observations( + src_dir=parquet_dir, + dst_path=tmp_dst, + features=corridor_gdf, + atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields"), + track_keys=prj.mission_options.get("icesat2", {}).get("track_keys"), + ) + except Exception as exc: + logger.warning( + "Failed to extract ICESat-2 for waterbody %s: %s", wb_id, exc + ) + continue + + if not os.path.exists(tmp_dst): + continue + + points = gpd.read_file(tmp_dst) + os.remove(tmp_dst) + if points.empty: + continue + + targets = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col].isin(target_ids) + ] + assigned = _assign_points_to_river_targets( + points, targets, prj.rivers.target_id_col, max_assign_dist, prj.local_crs + ) + if assigned.empty: + logger.warning( + "No ICESat-2 points within %sm of any target for waterbody %s", + max_assign_dist, wb_id, + ) + continue + + for target_id, group in assigned.groupby(prj.rivers.target_id_col): + dst_dir = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations" + ) + general.ifnotmakedirs(dst_dir) + dst_path = os.path.join(dst_dir, "icesat2.gpkg") + group.drop( + columns=["index_right", "_dist_to_target_m"], errors="ignore" + ).to_file(dst_path, driver="GPKG") + + +def _extract_rivers_sentinel_observations( + prj: "Project", mission_key: str, product: str, overwrite: bool = False +) -> None: + """Extract Sentinel-3 or Sentinel-6 observations for each river target. + + Same per-waterbody-then-split approach as + _extract_rivers_icesat2_observations, plus a water-only filter: a + buffered river corridor is much looser than a reservoir polygon (it + genuinely includes riverbank, fields, vegetation alongside the + channel), and unlike ICESat-2, Sentinel-3/6 have no built-in water + classification. sigma0_min filters this out as a self-contained + post-processing step here (rather than modifying + sentinel.extract_observations itself, whose internals haven't been + verified) -- water gives a strong, consistent specular radar + return; land gives a weaker, noisier one. Needs empirical tuning + against real river data, same as every other threshold in this + pipeline -- the default here (0.0, i.e. no-op) is a safe starting + point, not a verified value; set mission_options[mission_key] + ['sigma0_min'] once you have real data to check it against. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + max_assign_dist = ( + getattr(prj.rivers, "max_node_assignment_meters", None) + or _river_extraction_buffer_meters(prj) + ) + sigma0_min = prj.mission_options.get(mission_key, {}).get("sigma0_min", 0.0) + + tmp_dir = os.path.join(prj.dirs["output"], "_tmp_river_extraction") + + for wb_id, target_ids in waterbody_groups.items(): + download_dir = os.path.join(prj.dirs[mission_key], f"{wb_id}") + if not os.path.exists(download_dir): + continue + + if not overwrite: + remaining = [ + t + for t in target_ids + if not os.path.exists( + os.path.join( + prj.dirs["output"], + f"{t}", + "raw_observations", + f"{mission_key}.gpkg", + ) + ) + ] + if not remaining: + continue + target_ids = remaining + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + continue + + general.ifnotmakedirs(tmp_dir) + tmp_dst = os.path.join(tmp_dir, f"{wb_id}_{mission_key}.gpkg") + + try: + sentinel.extract_observations( + src_dir=download_dir, + dst_path=tmp_dst, + features=corridor_gdf, + sigma0_max=prj.mission_options.get(mission_key, {}).get( + "sigma0_max", 1e5 + ), + ) + except Exception as exc: + logger.warning( + "Failed to extract %s for waterbody %s: %s", mission_key, wb_id, exc + ) + continue + + if not os.path.exists(tmp_dst): + continue + + points = gpd.read_file(tmp_dst) + os.remove(tmp_dst) + if points.empty: + continue + + if "sigma0" in points.columns and sigma0_min: + before = len(points) + points = points.loc[points["sigma0"] >= sigma0_min].reset_index(drop=True) + logger.info( + "%s waterbody %s: sigma0_min=%s kept %d/%d points", + mission_key, wb_id, sigma0_min, len(points), before, + ) + if points.empty: + continue + + targets = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col].isin(target_ids) + ] + assigned = _assign_points_to_river_targets( + points, targets, prj.rivers.target_id_col, max_assign_dist, prj.local_crs + ) + if assigned.empty: + logger.warning( + "No %s points within %sm of any target for waterbody %s", + mission_key, max_assign_dist, wb_id, + ) + continue + + for target_id, group in assigned.groupby(prj.rivers.target_id_col): + dst_dir = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations" + ) + general.ifnotmakedirs(dst_dir) + dst_path = os.path.join(dst_dir, f"{mission_key}.gpkg") + group.drop( + columns=["index_right", "_dist_to_target_m"], errors="ignore" + ).to_file(dst_path, driver="GPKG") + + +def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) -> None: + """ + Split Hydrocron's per-waterbody timeseries CSV into the same + per-target {output}/{target_id}/raw_observations/swot.gpkg structure + every other mission uses, so the shared clean/merge pipeline can + treat SWOT identically to ICESat-2/Sentinel-3/6 for rivers. + + Unlike LakeSP for reservoirs, Hydrocron's own CSV doesn't include + per-observation coordinates in the default field lists (see + rivers.yaml) -- but nothing downstream actually needs per-observation + lat/lon for SWOT (see PRODUCT_TIMESERIES_KEYS: no lat_key/lon_key for + "swot"), so this attaches the target's own SWORD geometry as a + constant placeholder purely so the file can be saved/read as .gpkg + like every other mission's output -- the geometry's actual value is + never used downstream, only the height/date/platform/orbit columns. + + Quality filtering (max_q) is already applied at download time (see + _download_swot_hydrocron_timeseries), so it isn't repeated here. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" + + for wb_id, target_ids in waterbody_groups.items(): + src_path = os.path.join( + prj.dirs["swot"], str(wb_id), f"{id_label}_timeseries.csv" + ) + if not os.path.exists(src_path): + continue + + if not overwrite: + remaining = [ + t + for t in target_ids + if not os.path.exists( + os.path.join( + prj.dirs["output"], f"{t}", "raw_observations", "swot.gpkg" + ) + ) + ] + if not remaining: + continue + target_ids = remaining + + try: + df = pd.read_csv(src_path) + except Exception as exc: + logger.warning( + "Failed to read Hydrocron CSV for waterbody %s: %s", wb_id, exc + ) + continue + + if df.empty or prj.rivers.target_id_col not in df.columns: + continue + + df = df.loc[df[prj.rivers.target_id_col].isin(target_ids)].copy() + if df.empty: + continue + + df["height"] = df["wse"] + df["date"] = pd.to_datetime(df["time_str"]) + df["platform"] = "swot" + df["product"] = f"SWOT_Hydrocron_{id_label}" + # Matches the reservoir SWOT convention (orbit = lake_id, constant + # per target) -- there's no meaningful "which persistent track" + # concept distinct from the target itself for Hydrocron data. + df["orbit"] = df[prj.rivers.target_id_col] + + for target_id, group in df.groupby(prj.rivers.target_id_col): + target_row = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col] == target_id + ] + if target_row.empty: + continue + + group = group.copy() + group["geometry"] = target_row.geometry.iloc[0] + gdf = gpd.GeoDataFrame( + group, geometry="geometry", crs=prj.rivers.target_features.crs + ) + + dst_dir = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations" + ) + general.ifnotmakedirs(dst_dir) + gdf.to_file(os.path.join(dst_dir, "swot.gpkg"), driver="GPKG") + + # ============================================================================ # RESERVOIRS: Timeseries Processing # ============================================================================ @@ -782,8 +1534,13 @@ def create_reservoirs_timeseries(prj: "Project") -> None: if not hasattr(prj, "reservoirs"): return - # Extract raw observations from downloaded files - _extract_reservoirs_timeseries(prj) + # Extract raw observations from downloaded files (skips reservoirs whose + # gpkg already exists, unless prj.reservoirs.overwrite_extraction=True -- + # confirmed this re-read/re-extraction was the dominant real-world cost, + # far more than anything in clean()/merge()) + _extract_reservoirs_timeseries( + prj, overwrite=getattr(prj.reservoirs, "overwrite_extraction", False) + ) # Clean observations with filters _clean_reservoirs_timeseries(prj) @@ -796,22 +1553,59 @@ def create_reservoirs_timeseries(prj: "Project") -> None: _merge_reservoirs_timeseries(prj) -def _extract_reservoirs_timeseries(prj: "Project") -> None: - """Extract timeseries observations from raw downloaded files.""" +def create_rivers_timeseries(prj: "Project") -> None: + """Extract, clean, and merge timeseries for river targets (nodes/reaches). + + Mirrors create_reservoirs_timeseries. Not yet done: an export_to_dfs0 + equivalent for rivers, since _export_cleaned_to_dfs0 currently + iterates prj.reservoirs.download_gdf specifically -- left out here + rather than silently generalizing something not explicitly asked + for yet. + + Parameters + ---------- + prj : Project + Project instance with rivers configuration + """ + if not hasattr(prj, "rivers"): + return + + _extract_rivers_timeseries( + prj, overwrite=getattr(prj.rivers, "overwrite_extraction", False) + ) + + _clean_rivers_timeseries(prj) + + _merge_rivers_timeseries(prj) + + +def _extract_reservoirs_timeseries(prj: "Project", overwrite: bool = False) -> None: + """Extract timeseries observations from raw downloaded files. + + Parameters + ---------- + overwrite : bool, optional + If False (default), any reservoir/mission whose output .gpkg + already exists is skipped entirely rather than re-read and + re-extracted. Confirmed on real data that this re-extraction -- + not clean()/merge() -- was the dominant cost in real end-to-end + runs (orders of magnitude larger than the merge pipeline itself). + Set True to force re-extraction (e.g. new raw downloads arrived). + """ if "icesat2" in prj.to_process: - _extract_icesat2_observations(prj) + _extract_icesat2_observations(prj, overwrite=overwrite) if "sentinel3" in prj.to_process: - _extract_sentinel_observations(prj, "sentinel3", "S3") + _extract_sentinel_observations(prj, "sentinel3", "S3", overwrite=overwrite) if "sentinel6" in prj.to_process: - _extract_sentinel_observations(prj, "sentinel6", "S6") + _extract_sentinel_observations(prj, "sentinel6", "S6", overwrite=overwrite) if "swot" in prj.to_process: - _extract_swot_observations(prj) + _extract_swot_observations(prj, overwrite=overwrite) -def _extract_icesat2_observations(prj: "Project") -> None: +def _extract_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: """Extract ICESat-2 ATL13 observations for each reservoir.""" available_ids = [ id @@ -824,6 +1618,27 @@ def _extract_icesat2_observations(prj: "Project") -> None: logger.warning("No ICESat-2 downloads found; skipping timeseries extraction.") return + if not overwrite: + skip_count = 0 + remaining_ids = [] + for id in available_ids: + dst_path = os.path.join( + prj.dirs["output"], f"{id}", "raw_observations", "icesat2.gpkg" + ) + if os.path.exists(dst_path): + skip_count += 1 + else: + remaining_ids.append(id) + if skip_count: + logger.info( + "ICESat-2 extraction: skipping %d reservoir(s) with an existing " + "icesat2.gpkg (pass overwrite=True to force re-extraction).", + skip_count, + ) + available_ids = remaining_ids + if not available_ids: + return + empty_ids = [] for id in tqdm(available_ids, desc="Extracting ICESat-2 ATL13 product"): sub_gdf = prj.reservoirs.download_gdf.loc[ @@ -856,7 +1671,7 @@ def _extract_icesat2_observations(prj: "Project") -> None: def _extract_sentinel_observations( - prj: "Project", mission_key: str, product: str + prj: "Project", mission_key: str, product: str, overwrite: bool = False ) -> None: """Extract Sentinel-3 or Sentinel-6 observations for each reservoir.""" available_ids = [ @@ -870,6 +1685,27 @@ def _extract_sentinel_observations( ) return + if not overwrite: + skip_count = 0 + remaining_ids = [] + for id in available_ids: + dst_path = os.path.join( + prj.dirs["output"], f"{id}", "raw_observations", f"{mission_key}.gpkg" + ) + if os.path.exists(dst_path): + skip_count += 1 + else: + remaining_ids.append(id) + if skip_count: + logger.info( + "%s extraction: skipping %d reservoir(s) with an existing " + "%s.gpkg (pass overwrite=True to force re-extraction).", + mission_key, skip_count, mission_key, + ) + available_ids = remaining_ids + if not available_ids: + return + empty_ids = [] for id in tqdm(available_ids, desc=f"Extracting Sentinel-{product} product"): sub_gdf = prj.reservoirs.download_gdf.loc[ @@ -903,19 +1739,40 @@ def _extract_sentinel_observations( ) -def _extract_swot_observations(prj: "Project") -> None: +def _extract_swot_observations(prj: "Project", overwrite: bool = False) -> None: """Extract SWOT Lake SP observations for all reservoirs.""" download_dir = prj.dirs["swot"] if not os.path.exists(download_dir): logger.warning("No SWOT downloads found; skipping timeseries extraction.") return + features = prj.reservoirs.download_gdf + id_key = prj.reservoirs.id_key + + if not overwrite: + def _has_output(id): + return os.path.exists( + os.path.join(prj.dirs["output"], f"{id}", "raw_observations", "swot.gpkg") + ) + all_ids = features[id_key].tolist() + remaining_ids = [i for i in all_ids if not _has_output(i)] + skip_count = len(all_ids) - len(remaining_ids) + if skip_count: + logger.info( + "SWOT extraction: skipping %d reservoir(s) with an existing " + "swot.gpkg (pass overwrite=True to force re-extraction).", + skip_count, + ) + if not remaining_ids: + return + features = features.loc[features[id_key].isin(remaining_ids)] + empty_ids = swot.extract_observations( src_dir=download_dir, dst_dir=prj.dirs["output"], dst_file_name="swot.gpkg", - features=prj.reservoirs.download_gdf, - id_key=prj.reservoirs.id_key, + features=features, + id_key=id_key, exclude_obs_id_values=prj.mission_options.get("swot", {}).get( "exclude_obs_id_values", ["no_data"] ), @@ -927,20 +1784,169 @@ def _extract_swot_observations(prj: "Project") -> None: ) -def _clean_reservoirs_timeseries(prj: "Project") -> None: - """Apply quality filters to extracted timeseries.""" +# Per-product mapping from generic Timeseries key attributes to the actual +# column names each mission's extractor writes. Sentinel-3 shares +# Sentinel-6's extractor/schema (same sentinel.extract_observations +# function, see _extract_sentinel_observations). +# +# **************************************************************************** +# TERMINOLOGY TRAP -- read before touching orbit_key/pass_key for Sentinel: +# The raw Sentinel-3/6 data has TWO similarly-named but opposite-meaning +# columns: +# - "orbit": the absolute revolution counter. Unique on every single +# crossing, never repeats. USELESS as orbit_key (bias_correct needs a +# persistent identifier to accumulate overlap against -- grouping by +# something that's different every time means every "source" has +# exactly 1 observation and nothing can ever be calibrated: this is +# exactly the bug that caused every S3A/S3B track to be dropped as +# unanchored in practice). +# - "pass": the satellite-engineering term for the STABLE, REPEATING +# ground track number (same value every ~27-day repeat cycle for +# S3A/S3B). This is what orbit_key actually needs. +# Confusingly, our own framework's `pass_key` means the OPPOSITE thing (one +# specific, one-time crossing -- e.g. file_name) from what "pass" means in +# the satellite data itself (the repeating track). Do not be tempted to +# point pass_key at the raw "pass" column -- file_name is correct there. +# **************************************************************************** +# +# ICESat-2's orbit_key is "beam" (the persistent ground track/virtual +# station) -- cycle_number only matters as an ingredient of the compound +# "pass" column built at extraction time (see +# HydroEO.satellites.icesat2.preprocess.extract_observations). SWOT's +# LakeSP product is already one integrated WSE per crossing with its own +# formal uncertainty (wse_u), so it needs neither lat/lon nor pass_key -- +# see preset_error_key, and daily_mad_error's handling of it. +PRODUCT_TIMESERIES_KEYS = { + "sentinel3": dict( + lat_key="lat", lon_key="lon", pass_key="file_name", + platform_key="platform", orbit_key="relative_orbit", + ), + # NOTE: sentinel6 still uses "pass" as orbit_key -- NOT verified to be + # unstable the way it was for sentinel3 (confirmed empirically: on + # real data, "pass" was unique-per-crossing for every S3A/S3B visit, + # i.e. not stable at all, while "relative_orbit" genuinely repeated + # across multiple visits -- e.g. S3B crossed via 2 distinct stable + # configurations, with real biases of -0.14m and +0.22m that a + # platform-only grouping was averaging into one misleading +0.04m). + # Sentinel-6 may have the same "pass" instability and may also have + # its own "relative_orbit"-equivalent column, but this hasn't been + # checked against real S6 data -- don't assume the same fix applies + # without verifying first. + "sentinel6": dict( + lat_key="lat", lon_key="lon", pass_key="file_name", + platform_key="platform", orbit_key="pass", + ), + "icesat2": dict( + lat_key="lat", lon_key="lon", pass_key="pass", + platform_key="platform", orbit_key="beam", + ), + "swot": dict( + platform_key="platform", orbit_key="orbit", preset_error_key="wse_u", + ), +} + +# Default .merge() tuning for reservoirs, mirroring the shape of +# processing_options (a project-level dict of pipeline parameters) but +# applied once per reservoir rather than per-product, since .merge() runs +# on the already-combined multi-product timeseries. Override via +# prj.merging_options in project config; falls back to these reservoir- +# appropriate defaults if that attribute isn't set. +DEFAULT_RESERVOIR_MERGING_OPTIONS = { + "window_km": 1.5, + "svr_linear_err": 0.1, + "svr_linear_epsilon": 0.1, + # Both updated from DAHITI's lake-tuned defaults (err=0.1, gamma= + # 0.0000438) based on real reservoir data validated this session -- + # the lake-tuned gamma implied a ~151-day smoothing lengthscale, far + # too coarse for a reservoir with real multi-week transitions (see + # the svr_radial oversmoothing discussion). err=1.0 (river-like, + # rather than the stricter lake value) and gamma x50 (~21-day + # lengthscale instead of ~151 days) let the trend actually track + # real fast changes instead of rejecting them as if they were noise. + "svr_radial_err": 1.0, + "svr_radial_rbf_c": 10000, + "svr_radial_gamma": 0.0000438 * 50, + "svr_radial_epsilon": 0.1, + # Confirmed on real data across two reservoirs: revisit sparsity varies a + # lot (e.g. one reservoir had icesat2/S3A/S3B visiting only 7/14/13 + # distinct days all year). At "10D"/3, sparse sources can fail to ever + # find 3 overlapping bins and get dropped as unanchored ENTIRELY (not + # just trimmed) -- confirmed: this silently dropped 2 of 3 missions + # (icesat2, S3B) for one real reservoir. "20D"/1 recovered all of it. + # Widening is monotonically safe against data loss (a wider window can + # only find equal-or-more overlapping bins, never fewer) -- the + # tradeoff is a very wide bin could blur real water-level change within + # the window into the bias estimate; 20D is a modest widening, not an + # extreme one. + "bias_time_bin": "20D", + "bias_min_overlap": 1, + # Confirmed empirically on real data: "platform_orbit" (using + # orbit_key -- now sentinel3's verified-stable "relative_orbit" + # column, see PRODUCT_TIMESERIES_KEYS) reveals genuine within-platform + # bias heterogeneity that "platform" alone was masking. One real + # reservoir's S3B crosses via two distinct, independently stable + # configurations (5 days on one, 8 on the other) with biases of + # -0.14m and +0.22m respectively -- "platform" grouping averaged + # these into one misleading +0.04m. Same pattern for ICESat-2's + # beams (orbit_key="beam"): per-beam biases ranged 0.06-0.18m under + # "platform_orbit", collapsed to one number under "platform". Total + # kept-row count was IDENTICAL either way on the reservoir tested + # (3182/4901) -- this is a precision gain, not a data-loss risk, at + # least for sentinel3/icesat2. NOTE: sentinel6 still uses "pass" as + # orbit_key (unverified whether it's stable or has a + # relative_orbit-equivalent -- see PRODUCT_TIMESERIES_KEYS) -- if + # it's actually unstable like sentinel3's old "pass" mapping was, + # "platform_orbit" could fragment sentinel6 into single-crossing + # sources. Recheck against real sentinel6 data before trusting this + # default for a project relying heavily on sentinel6. + "bias_group_by": "platform_orbit", + # Not a spatial correction -- just flags (and records in + # ts.bias_correct_diagnostics) when a source's observations are + # centered far from the anchor's, since for a large/elongated + # reservoir some of the estimated bias could be real spatial signal. + # Worth a closer look per-reservoir if this fires, not an error. + "bias_centroid_warn_km": 5.0, + # Off by default -- inflates Kalman input error by distance from the + # reservoir polygon's own centroid, addressing crossings that may be + # hydraulically unrepresentative (e.g. far upstream, subject to real + # slope bias) even when ADM alone reports them as highly precise. Set + # to a real value (m of extra error per km of distance) to enable -- + # the right scale depends on the true magnitude of upstream slope bias + # for your reservoirs, which needs empirical tuning, not a guessed + # default. + "distance_penalty_scale_per_km": None, + # Off by default -- a genuine height correction (not just error + # inflation) using a spatial deviation model fit once from a dense + # source (default ICESat-2) and persisted to disk per reservoir (see + # _get_or_fit_spatial_correction_model) so past corrections don't + # shift retroactively as new data arrives. Turn on once you've + # confirmed (as we did empirically) that the target reservoir shows a + # real, day-to-day-consistent spatial deviation pattern -- fitting + # requires several qualifying dense-source days (see + # fit_spatial_correction_model's min_days), and silently does nothing + # if there isn't enough dense-source data yet. + "use_spatial_correction": False, + "spatial_correction_dense_source": "icesat2", +} + + +def _clean_timeseries(prj: "Project", target_type: str) -> None: + """Apply quality filters to extracted timeseries, for either + reservoirs or river targets (nodes/reaches).""" + target_ids = _get_target_ids(prj, target_type) ids_with_raw = [ id - for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] + for id in target_ids if os.path.exists(os.path.join(prj.dirs["output"], f"{id}", "raw_observations")) ] if not ids_with_raw: logger.warning( - "No raw observations found for any reservoir; skipping timeseries cleaning." + "No raw observations found for any %s; skipping timeseries cleaning.", + target_type, ) return - for id in tqdm(ids_with_raw, desc="Cleaning product timeseries"): + for id in tqdm(ids_with_raw, desc=f"Cleaning product timeseries ({target_type})"): for product in prj.to_process: df = _load_product_timeseries( os.path.join(prj.dirs["output"], f"{id}", "raw_observations"), @@ -959,7 +1965,10 @@ def _clean_reservoirs_timeseries(prj: "Project") -> None: }, ) - ts = timeseries.Timeseries(df, date_key="date", height_key="height") + ts = timeseries.Timeseries( + df, date_key="date", height_key="height", + **PRODUCT_TIMESERIES_KEYS.get(product, {}), + ) ts.clean( product_options.get("processing_filters", ["elevation", "MAD"]), @@ -979,6 +1988,16 @@ def _clean_reservoirs_timeseries(prj: "Project") -> None: ts.export_csv(os.path.join(export_dir, f"{product}.csv")) +def _clean_reservoirs_timeseries(prj: "Project") -> None: + """Apply quality filters to extracted reservoir timeseries.""" + _clean_timeseries(prj, "reservoirs") + + +def _clean_rivers_timeseries(prj: "Project") -> None: + """Apply quality filters to extracted river timeseries.""" + _clean_timeseries(prj, "rivers") + + def _load_product_timeseries(data_dir, ext, products, reader_fn): """Load files of given extension from directory, optionally filtered to products.""" if not os.path.exists(data_dir): @@ -999,22 +2018,683 @@ def _load_product_timeseries(data_dir, ext, products, reader_fn): return pd.concat(df_list) if df_list else None -def _merge_reservoirs_timeseries(prj: "Project") -> None: - """Merge multi-mission timeseries into combined datasets.""" +def _get_target_ids(prj: "Project", target_type: str): + """Return the list of target IDs to process, for either 'reservoirs' or 'rivers'.""" + if target_type == "reservoirs": + return list(prj.reservoirs.download_gdf[prj.reservoirs.id_key]) + if target_type == "rivers": + return list(prj.rivers.target_ids) + raise ValueError(f"Unknown target_type: {target_type!r}") + + +def _target_centroid(prj: "Project", target_type: str, id): + """ + Return (lat, lon) of a target's own geometry centroid -- the + reservoir polygon for target_type="reservoirs", or the SWORD + node/reach geometry for target_type="rivers" -- computed in a + projected (local) CRS for accuracy, then converted back to lat/lon. + Used as the reference location for apply_distance_penalty/ + apply_spatial_correction. Returns (None, None) if the target's + geometry can't be found, so callers can treat that as "skip" rather + than fail. + """ + try: + if target_type == "reservoirs": + gdf = prj.reservoirs.gdf + id_key = prj.reservoirs.id_key + elif target_type == "rivers": + gdf = prj.rivers.target_features + id_key = prj.rivers.target_id_col + else: + raise ValueError(f"Unknown target_type: {target_type!r}") + + row = gdf.loc[gdf[id_key] == id] + if len(row) == 0 or row.geometry.isna().all(): + return None, None + centroid = row.to_crs(prj.local_crs).geometry.centroid.to_crs(prj.global_crs) + pt = centroid.iloc[0] + return pt.y, pt.x # lat, lon + except Exception as exc: + logger.warning( + "Could not compute %s centroid for %s: %s", target_type, id, exc + ) + return None, None + + +def _reservoir_centroid(prj: "Project", id): + """Backward-compatible wrapper -- see _target_centroid.""" + return _target_centroid(prj, "reservoirs", id) + + +def _get_or_fit_spatial_correction_model( + prj: "Project", target_type: str, id, dense_source_platform="icesat2", + recalibrate=False, **fit_kwargs, +): + """ + Load a persisted spatial correction model for this target if one + exists, or fit a fresh one and persist it. Works identically for + reservoirs and river targets -- see _target_centroid. + + This is deliberately NOT re-fit automatically every run: doing so + would make past corrections shift retroactively every time new + dense-source data arrives, since the fitted slope would change. + Pass recalibrate=True to explicitly force a re-fit (e.g. as a + deliberate, occasional recalibration step) -- not something that + should happen as a silent side effect of routine reprocessing. + + Returns None if no model exists yet and there isn't enough dense + source data to fit one (see fit_spatial_correction_model) -- callers + should treat this the same as "no correction available." + """ + model_path = os.path.join( + prj.dirs["output"], f"{id}", "spatial_correction_model.json" + ) + + if os.path.exists(model_path) and not recalibrate: + with open(model_path, "r") as f: + return json.load(f) + + cleaned_path = os.path.join( + prj.dirs["output"], f"{id}", "all_cleaned_timeseries.csv" + ) + if not os.path.exists(cleaned_path): + logger.info( + "No cleaned observations yet for %s; cannot fit spatial " + "correction model.", id, + ) + return None + + df = pd.read_csv(cleaned_path) + if "platform" not in df.columns or dense_source_platform not in df["platform"].values: + logger.info( + "No %s data available for %s; cannot fit spatial correction " + "model from it.", dense_source_platform, id, + ) + return None + + df["date"] = pd.to_datetime(df["date"]) + dense_df = df.loc[df["platform"] == dense_source_platform] + + ref_lat, ref_lon = _target_centroid(prj, target_type, id) + if ref_lat is None: + logger.warning( + "Could not determine %s centroid for %s; cannot fit spatial " + "correction model.", target_type, id, + ) + return None + + model = basic_filters.fit_spatial_correction_model( + dense_df, lat_key="lat", lon_key="lon", height_key="height", + date_key="date", ref_lat=ref_lat, ref_lon=ref_lon, **fit_kwargs, + ) + + if model is not None: + general.ifnotmakedirs(os.path.dirname(model_path)) + with open(model_path, "w") as f: + json.dump(model, f, indent=2) + + return model + + +# ============================================================================ +# Per-target run config: exclusions + per-target merging option overrides +# ============================================================================ +# +# One YAML file per target ({output}/{id}/run_config.yaml) that is +# simultaneously: (a) a human-readable log of decisions made about this +# target, (b) the actual source of truth _merge_timeseries reads to apply +# those decisions, and (c) something a user can hand-edit directly for a +# fully config-driven workflow. Interactive functions below +# (exclude_from_target, set_merging_option, ...) read-modify-write this +# same file, so a decision made once in a notebook session is exactly the +# same artifact you'd edit by hand or check into version control -- there +# is no separate "notebook state" to keep in sync with "the config". + + +def _run_config_path(prj: "Project", id) -> str: + return os.path.join(prj.dirs["output"], f"{id}", "run_config.yaml") + + +def _default_run_config(id) -> dict: + return { + "target_id": id, + "last_updated": None, + "merging_option_overrides": {}, + "exclusions": [], + } + + +def _load_run_config(prj: "Project", id) -> dict: + """Load a target's run_config.yaml, or a fresh default if none exists yet.""" + path = _run_config_path(prj, id) + if os.path.exists(path): + with open(path, "r") as f: + loaded = yaml.safe_load(f) + if loaded: + # tolerate a hand-edited file missing a key or two + defaults = _default_run_config(id) + defaults.update(loaded) + return defaults + return _default_run_config(id) + + +def _save_run_config(prj: "Project", id, config: dict) -> None: + config["last_updated"] = datetime.datetime.now().isoformat() + path = _run_config_path(prj, id) + general.ifnotmakedirs(os.path.dirname(path)) + with open(path, "w") as f: + yaml.safe_dump(config, f, sort_keys=False) + + +def _invalidate_spatial_correction_cache(prj: "Project", id) -> None: + """ + Delete any cached spatial correction model for this target, forcing + a fresh fit next time use_spatial_correction is used. Called whenever + exclusions or spatial-correction-relevant options change -- the + model may have been fit using observations that are no longer + included, and this is exactly the kind of deliberate, explicit + trigger (not routine reprocessing) that recalibration is meant for -- + see _get_or_fit_spatial_correction_model. + """ + model_path = os.path.join(prj.dirs["output"], f"{id}", "spatial_correction_model.json") + if os.path.exists(model_path): + os.remove(model_path) + logger.info( + "Invalidated cached spatial correction model for %s " + "(exclusions or related options changed).", id, + ) + + +def _invalidate_reach_slope_correction_cache(prj: "Project", id) -> None: + """ + Delete any cached reach slope correction model for this target, + forcing a fresh fit next time use_reach_slope_correction is used. + Called whenever exclusions change -- an exclusion could target SWOT + observations specifically, which is exactly what this model is fit + from (see _fit_reach_slope_correction), so a cached model could + otherwise silently keep reflecting now-excluded SWOT slope values. + """ + model_path = os.path.join( + prj.dirs["output"], f"{id}", "reach_slope_correction_model.json" + ) + if os.path.exists(model_path): + os.remove(model_path) + logger.info( + "Invalidated cached reach slope correction model for %s " + "(exclusions or related options changed).", id, + ) + + +def _fit_reach_slope_correction(prj: "Project", target_id, recalibrate: bool = False): + """ + Fit (or load a persisted) reach-level slope correction from SWOT's + own directly-measured "slope" field (RiverSP reach product), used to + reference-correct OTHER missions' (ICESat-2/Sentinel-3/6) crossings + to what they'd read at the reach's geometric midpoint. + + ONLY meaningful for reaches (prj.rivers.target_id_col == "reach_id") + -- a node is a single ~200m-spaced point, not a ~10km segment with + its own along-reach slope in the same sense. Callers must not invoke + this for node-mode projects. + + Rationale: SWOT's reach-level WSE is an aggregate over the reach's + ~50 constituent, roughly-evenly-spaced nodes, not a value evaluated + at one specific point -- for an evenly-sampled linear profile, the + mean equals the value at the mean position, so this is treated as + approximately midpoint-referenced. This is an evidence-based + inference from the RiverSP processing chain, NOT a fact directly + confirmed in SWOT's product documentation (which does not explicitly + state a reference point) -- validate against real Hydrocron + node-vs-reach output for a known reach before trusting this deeply. + + Uses the MEDIAN of all available SWOT slope observations for this + reach as a single, persistent correction -- not a per-date-specific + one -- consistent with this pipeline's existing preference (see + fit_spatial_correction_model) for a stable, once-fit value over a + per-observation one, and avoiding the complexity/fragility of + matching a specific SWOT overpass date to each individual non-SWOT + observation's date. + + Persisted to {output}/{target_id}/reach_slope_correction_model.json + -- fit once, loaded thereafter, only refit on explicit + recalibrate=True -- so past corrections don't shift retroactively + as new SWOT data arrives, same reasoning as the spatial correction + model's caching. + + Returns None if no model exists yet and there's no usable SWOT slope + data to fit one from -- callers should treat this as "no correction + available," not an error. + """ + model_path = os.path.join( + prj.dirs["output"], f"{target_id}", "reach_slope_correction_model.json" + ) + if os.path.exists(model_path) and not recalibrate: + with open(model_path, "r") as f: + return json.load(f) + + swot_path = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations", "swot.gpkg" + ) + if not os.path.exists(swot_path): + logger.info( + "No raw SWOT observations for %s; cannot fit reach slope " + "correction.", target_id, + ) + return None + + swot_gdf = gpd.read_file(swot_path) + if "slope" not in swot_gdf.columns: + logger.warning( + "No 'slope' field found in SWOT observations for %s -- was " + "it requested in mission_options['swot']['hydrocron_fields']" + "['reaches']? Cannot fit reach slope correction.", target_id, + ) + return None + + valid_slopes = pd.to_numeric(swot_gdf["slope"], errors="coerce").dropna() + if valid_slopes.empty: + logger.info( + "No valid (non-null, numeric) SWOT slope observations for " + "%s; cannot fit reach slope correction.", target_id, + ) + return None + + model = { + "target_id": str(target_id), + "median_slope": float(valid_slopes.median()), + "n_observations": int(len(valid_slopes)), + "fitted_at": datetime.datetime.now().isoformat(), + } + + general.ifnotmakedirs(os.path.dirname(model_path)) + with open(model_path, "w") as f: + json.dump(model, f, indent=2) + + return model + + +def _apply_reach_slope_correction( + ts_df: pd.DataFrame, prj: "Project", target_id, slope_model: dict, +) -> pd.DataFrame: + """ + Apply a fitted reach slope correction (see _fit_reach_slope_correction) + to non-SWOT rows in ts_df -- adjusts "height" to what each row would + read at the reach's geometric midpoint, using its along-reach + projected position and the reach's persistent median slope. SWOT's + own rows are left untouched (already assumed midpoint-referenced -- + see _fit_reach_slope_correction's docstring for the reasoning and + its caveats). + + NOTE: the sign convention for "correction = slope x distance" here + has NOT been empirically verified against real data in this + session -- confirm it actually reduces cross-mission scatter for a + real reach (not increases it) before trusting this in production; + flip the sign if it doesn't. + + Rows without usable lat/lon (or if the target's geometry can't be + found) are left uncorrected rather than dropped. + """ + if "platform" not in ts_df.columns or "lat" not in ts_df.columns or "lon" not in ts_df.columns: + return ts_df + + target_row = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col] == target_id + ] + if target_row.empty: + logger.warning( + "Could not find reach geometry for %s; skipping reach slope " + "correction.", target_id, + ) + return ts_df + + reach_geom_global = target_row.geometry.iloc[0] + local_crs = prj.local_crs + reach_geom_local = ( + gpd.GeoSeries([reach_geom_global], crs=target_row.crs).to_crs(local_crs).iloc[0] + ) + reach_midpoint_dist = reach_geom_local.length / 2.0 + median_slope = slope_model["median_slope"] + + mask = (ts_df["platform"] != "swot") & ts_df["lat"].notna() & ts_df["lon"].notna() + if not mask.any(): + return ts_df + + points_local = gpd.GeoSeries( + gpd.points_from_xy(ts_df.loc[mask, "lon"], ts_df.loc[mask, "lat"]), + crs=prj.global_crs, + ).to_crs(local_crs) + + along_reach_dist = points_local.apply(reach_geom_local.project) + distance_from_midpoint = along_reach_dist.values - reach_midpoint_dist + + ts_df = ts_df.copy() + ts_df.loc[mask, "height"] = ( + ts_df.loc[mask, "height"] - median_slope * distance_from_midpoint + ) + return ts_df + + +def list_target_observations(prj: "Project", target_type: str, id) -> pd.DataFrame: + """ + Summarize what observations exist for a target, at (platform, orbit) + granularity -- the "what could I exclude" view. Meant to be read + alongside plot_merging's platform-colored progress plots (which show + WHERE a problem shows up in the actual data), not a replacement for + looking at the data itself. + """ + cleaned_path = os.path.join(prj.dirs["output"], f"{id}", "all_cleaned_timeseries.csv") + if not os.path.exists(cleaned_path): + logger.warning( + "No cleaned observations yet for %s -- has create_%s_timeseries() " + "been run?", id, target_type, + ) + return pd.DataFrame(columns=["platform", "orbit", "n_points", "date_min", "date_max"]) + + df = pd.read_csv(cleaned_path) + df["date"] = pd.to_datetime(df["date"]) + group_cols = [c for c in ["platform", "orbit"] if c in df.columns] + summary = ( + df.groupby(group_cols) + .agg(n_points=("date", "size"), date_min=("date", "min"), date_max=("date", "max")) + .reset_index() + .sort_values(group_cols) + .reset_index(drop=True) + ) + return summary + + +def list_exclusions(prj: "Project", target_type: str, id) -> list: + """Current exclusion rules for a target, from its run_config.yaml.""" + return _load_run_config(prj, id)["exclusions"] + + +def exclude_from_target( + prj: "Project", target_type: str, id, + platform=None, orbit=None, date=None, reason=None, +) -> None: + """ + Exclude observations from a target's merge, at whatever granularity + is given -- a whole platform, a specific orbit/pass value, a specific + date, or any combination (all given fields must match for a row to + be excluded). Persisted to {output}/{id}/run_config.yaml. + + Applied at the start of _merge_timeseries, before any processing -- + an excluded pass never reaches bias_correct/Kalman/svr_radial at all, + rather than being fought against downstream. + + Invalidates any cached spatial correction model for this target, + since it may have been fit using data that's now excluded. + + Examples + -------- + exclude_from_target(prj, "reservoirs", my_id, platform="S3B") + exclude_from_target(prj, "reservoirs", my_id, platform="S3B", orbit=1517) + exclude_from_target(prj, "rivers", my_id, date="2024-03-19") + """ + if platform is None and orbit is None and date is None: + raise ValueError( + "Specify at least one of platform, orbit, or date to exclude." + ) + + config = _load_run_config(prj, id) + config["exclusions"].append({ + "platform": platform, + "orbit": orbit, + "date": str(date) if date is not None else None, + "reason": reason, + "added": datetime.datetime.now().isoformat(), + }) + _save_run_config(prj, id, config) + _invalidate_spatial_correction_cache(prj, id) + _invalidate_reach_slope_correction_cache(prj, id) + logger.info( + "Added exclusion for %s: platform=%s orbit=%s date=%s (%s)", + id, platform, orbit, date, reason or "no reason given", + ) + + +def _exclusion_value_matches(a, b) -> bool: + """ + Robust equality check for exclusion matching -- tries numeric + comparison first, so int/float/numeric-string representations of + the same value all compare correctly (e.g. 1517 == 1517.0 == "1517" + -- same int/float representation issue confirmed for + _apply_exclusions' dataframe matching, applied here too for + consistency), falling back to exact equality for non-numeric values + (e.g. a string-based orbit identifier) or when either side is None. + """ + if a is None or b is None: + return a == b + try: + return float(a) == float(b) + except (TypeError, ValueError): + return a == b + + +def remove_exclusion( + prj: "Project", target_type: str, id, + index: int = None, platform=None, orbit=None, date=None, +) -> list: + """ + Remove one or more exclusion rules, either by position (index, from + list_exclusions()) or by matching criteria -- the same + platform/orbit/date fields used to add one via exclude_from_target. + Matching by criteria is usually more convenient than looking up an + index first: e.g. remove_exclusion(prj, "reservoirs", my_id, + platform="S3B", orbit=1517) removes exactly the rule that excluded + that mission+orbit combination (or platform+beam, for ICESat-2 -- + beam values ARE the "orbit" field once concatenated with other + missions, see PRODUCT_TIMESERIES_KEYS -- there's no separate "beam" + parameter needed). + + Specify either index, or at least one of platform/orbit/date, not + both. Criteria matching removes EVERY exclusion rule whose given + fields match (fields not specified are ignored, not required to be + None on the stored rule). + + Returns the list of removed rule(s), for confirmation/logging. + """ + config = _load_run_config(prj, id) + exclusions = config["exclusions"] + criteria_given = platform is not None or orbit is not None or date is not None + + if index is not None and criteria_given: + raise ValueError( + "Specify either index OR platform/orbit/date criteria, not both." + ) + + if index is not None: + if index < 0 or index >= len(exclusions): + raise IndexError( + f"No exclusion at index {index} for {id}; there are " + f"{len(exclusions)}. See list_exclusions()." + ) + removed = [exclusions.pop(index)] + elif criteria_given: + date_str = str(date) if date is not None else None + to_remove = [ + rule for rule in exclusions + if (platform is None or _exclusion_value_matches(rule.get("platform"), platform)) + and (orbit is None or _exclusion_value_matches(rule.get("orbit"), orbit)) + and (date is None or rule.get("date") == date_str) + ] + if not to_remove: + raise ValueError( + f"No exclusion found matching platform={platform!r} " + f"orbit={orbit!r} date={date!r} for {id}. See list_exclusions()." + ) + for rule in to_remove: + exclusions.remove(rule) + removed = to_remove + else: + raise ValueError( + "Specify index, or at least one of platform/orbit/date, to " + "identify which exclusion(s) to remove." + ) + + _save_run_config(prj, id, config) + _invalidate_spatial_correction_cache(prj, id) + _invalidate_reach_slope_correction_cache(prj, id) + logger.info("Removed %d exclusion(s) for %s: %s", len(removed), id, removed) + return removed + + +def set_merging_option(prj: "Project", target_type: str, id, **kwargs) -> None: + """ + Override one or more merging_options for just this one target, + persisted the same way as exclusions (highest-priority layer: these + override prj.reservoirs/rivers.merging_options, which override the + DEFAULT_*_MERGING_OPTIONS defaults). + + Example: set_merging_option(prj, "reservoirs", my_id, svr_radial_err=0.5) + """ + config = _load_run_config(prj, id) + config["merging_option_overrides"].update(kwargs) + _save_run_config(prj, id, config) + if "use_spatial_correction" in kwargs or "spatial_correction_dense_source" in kwargs: + _invalidate_spatial_correction_cache(prj, id) + if "use_reach_slope_correction" in kwargs: + _invalidate_reach_slope_correction_cache(prj, id) + logger.info("Updated merging options for %s: %s", id, kwargs) + + +def _apply_exclusions(df: pd.DataFrame, exclusions: list) -> pd.DataFrame: + """ + Filter out rows matching any exclusion rule. Within one rule, every + specified field (platform/orbit/date) must match for a row to be + excluded by it; a row is dropped if it matches ANY rule. + """ + if not exclusions: + return df + + keep_mask = pd.Series(True, index=df.index) + for rule in exclusions: + rule_mask = pd.Series(True, index=df.index) + if rule.get("platform") is not None: + rule_mask &= df["platform"] == rule["platform"] + if rule.get("orbit") is not None: + if "orbit" in df.columns: + # Compare numerically when possible, not as strings -- + # a real orbit column commonly gets upcast to float64 by + # pandas the moment ANY value in it is missing (very + # common in real satellite data), so a genuine orbit + # value of 1517 reads as 1517.0 in the dataframe while a + # YAML-loaded exclusion rule reads it as the plain int + # 1517. Comparing as strings ("1517.0" vs "1517") then + # silently matches nothing -- confirmed as a real, + # reproducible bug, not a hypothetical one. Falls back + # to string comparison only if the orbit value genuinely + # isn't numeric (e.g. a string-based identifier). + try: + target_orbit = float(rule["orbit"]) + rule_mask &= ( + pd.to_numeric(df["orbit"], errors="coerce") == target_orbit + ) + except (TypeError, ValueError): + rule_mask &= df["orbit"].astype(str) == str(rule["orbit"]) + else: + rule_mask &= False + if rule.get("date") is not None: + rule_mask &= df["date"].dt.floor("D").astype(str) == str(rule["date"]) + keep_mask &= ~rule_mask + + return df.loc[keep_mask].reset_index(drop=True) + + +DEFAULT_RIVER_MERGING_OPTIONS = { + # Mostly a starting point copied from the reservoir defaults and NOT + # independently validated against real river data the way the + # reservoir defaults were validated this session -- river dynamics + # differ genuinely (e.g. a real, expected along-reach gradient), so + # do not assume the rest of these are correct without checking. + # svr_radial_err/gamma below ARE an explicit exception (set directly, + # not copied): gamma x100 (~15-day lengthscale, vs DAHITI's ~151-day + # lake value) and err=1.0, matching the same oversmoothing reasoning + # as the reservoir defaults, just with a shorter lengthscale given + # rivers can change faster still. + "window_km": 1.5, + "svr_linear_err": 0.1, + "svr_linear_epsilon": 0.1, + "svr_radial_err": 1.0, + "svr_radial_rbf_c": 10000, + "svr_radial_gamma": 0.0000438 * 100, + "svr_radial_epsilon": 0.1, + "bias_time_bin": "20D", + "bias_min_overlap": 1, + # Same reasoning/evidence as the reservoir default (see + # DEFAULT_RESERVOIR_MERGING_OPTIONS) for switching from "platform" to + # "platform_orbit" -- but this is carried over, not independently + # verified against real river data. A single river target (node/reach) + # is a much smaller footprint than a reservoir, so it's genuinely + # unclear whether the same within-platform configuration split + # (e.g. S3B's two distinct crossing geometries) would even occur at + # this scale -- check real per-target bias diagnostics once river + # data exists before trusting this. + "bias_group_by": "platform_orbit", + "bias_centroid_warn_km": 5.0, + # Off by default, same reasoning as reservoirs. NOTE: an earlier + # version of this comment claimed a river target's footprint is + # "much smaller than a reservoir" -- that's wrong for reaches + # specifically (confirmed ~10km typical length, comparable to or + # larger than many reservoirs), so distance_penalty/spatial + # correction may matter just as much for reaches as for reservoirs. + # It remains true that these tools address spread WITHIN one + # target's own crossing footprint, never the natural gradient + # BETWEEN different targets, which should never be "corrected away". + "distance_penalty_scale_per_km": None, + "use_spatial_correction": False, + "spatial_correction_dense_source": "icesat2", + # Off by default. ONLY meaningful when + # prj.rivers.target_id_col == "reach_id" -- reference-corrects + # non-SWOT crossings (ICESat-2/Sentinel-3/6) to what they'd read at + # the reach's geometric midpoint, using SWOT's own directly-measured + # "slope" field (see _fit_reach_slope_correction/ + # _apply_reach_slope_correction). Requires "slope" to be present in + # mission_options["swot"]["hydrocron_fields"]["reaches"]. The + # midpoint-referenced assumption for SWOT's own reach WSE is an + # evidence-based inference from the RiverSP processing chain, not a + # fact directly confirmed in SWOT's documentation -- and the sign of + # the correction has not been empirically verified against real + # data in this session. Validate both before trusting this in + # production. + "use_reach_slope_correction": False, +} + + +def _merge_timeseries(prj: "Project", target_type: str) -> None: + """Merge multi-mission timeseries into combined datasets, for either + reservoirs or river targets (nodes/reaches).""" + target_ids = _get_target_ids(prj, target_type) ids_with_cleaned = [ id - for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] + for id in target_ids if os.path.exists( os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") ) ] if not ids_with_cleaned: logger.warning( - "No cleaned observations found for any reservoir; skipping timeseries merging." + "No cleaned observations found for any %s; skipping timeseries merging.", + target_type, ) return - for id in tqdm(ids_with_cleaned, desc="Merging product timeseries"): + default_options = ( + DEFAULT_RESERVOIR_MERGING_OPTIONS + if target_type == "reservoirs" + else DEFAULT_RIVER_MERGING_OPTIONS + ) + # prj.reservoirs.merging_options / prj.rivers.merging_options are the + # intended per-target-type override locations (set from the + # respective YAML config sections); fall back to the older shared + # prj.merging_options for backward compatibility if the per-type one + # isn't set yet. + target_owner = prj.reservoirs if target_type == "reservoirs" else prj.rivers + overrides = getattr(target_owner, "merging_options", None) + if overrides is None: + overrides = getattr(prj, "merging_options", None) + + for id in tqdm(ids_with_cleaned, desc=f"Merging product timeseries ({target_type})"): ts_list = [] for product in prj.to_process: df = _load_product_timeseries( @@ -1029,7 +2709,10 @@ def _merge_reservoirs_timeseries(prj: "Project") -> None: ).dt.tz_convert(None) df = df.sort_values(by="date") ts_list.append( - timeseries.Timeseries(df, date_key="date", height_key="height") + timeseries.Timeseries( + df, date_key="date", height_key="height", + **PRODUCT_TIMESERIES_KEYS.get(product, {}), + ) ) if len(ts_list) > 0: @@ -1037,15 +2720,107 @@ def _merge_reservoirs_timeseries(prj: "Project") -> None: data_dir = os.path.join(prj.dirs["output"], f"{id}") general.ifnotmakedirs(data_dir) + + run_config = _load_run_config(prj, id) + + merging_options = dict(default_options) + merging_options.update(overrides or {}) + # Per-target overrides (from notebook calls to + # set_merging_option, or hand-edited in run_config.yaml) take + # priority over project-wide settings for just this target. + merging_options.update(run_config.get("merging_option_overrides", {})) + distance_penalty_scale = merging_options.pop( + "distance_penalty_scale_per_km", None + ) + use_spatial_correction = merging_options.pop( + "use_spatial_correction", False + ) + spatial_correction_dense_source = merging_options.pop( + "spatial_correction_dense_source", "icesat2" + ) + # Off by default. Only meaningful for reach-mode river + # projects (a node is a single ~200m point, not a ~10km + # segment with its own along-reach slope) -- see + # _fit_reach_slope_correction for the full reasoning and the + # sign-convention caveat that should be checked against real + # data before trusting this in production. + use_reach_slope_correction = merging_options.pop( + "use_reach_slope_correction", False + ) + + # Apply exclusions BEFORE exporting all_cleaned_timeseries.csv + # (not just before merge processing) -- this file is meant to + # reflect what's actually being worked with, and writing it + # before exclusions were applied meant it always showed + # excluded data regardless of how many times you re-ran, + # which looked exactly like a stale file from an old run but + # was actually happening on every single run. The full, + # pre-exclusion record is still available per-mission in + # cleaned_observations/{product}.csv (written earlier, in + # _clean_timeseries, before any exclusion is applied) -- so + # nothing is lost by making this file reflect exclusions. + exclusions = run_config.get("exclusions", []) + if exclusions: + before = len(ts.df) + ts.df = _apply_exclusions(ts.df, exclusions) + logger.info( + "%s: %d exclusion rule(s) applied, %d/%d observations kept.", + id, len(exclusions), len(ts.df), before, + ) + + if ( + use_reach_slope_correction + and target_type == "rivers" + and getattr(prj.rivers, "target_id_col", None) == "reach_id" + ): + slope_model = _fit_reach_slope_correction(prj, id) + if slope_model is not None: + ts.df = _apply_reach_slope_correction(ts.df, prj, id, slope_model) + logger.info( + "%s: applied reach slope correction (median_slope=%.6g " + "from %d SWOT observations).", + id, slope_model["median_slope"], slope_model["n_observations"], + ) + else: + logger.info( + "%s: use_reach_slope_correction is enabled but no " + "usable SWOT slope data was found; skipping " + "correction for this target.", id, + ) + ts.export_csv(os.path.join(data_dir, "all_cleaned_timeseries.csv")) + ref_lat, ref_lon = _target_centroid(prj, target_type, id) + + spatial_correction_model = None + if use_spatial_correction: + spatial_correction_model = _get_or_fit_spatial_correction_model( + prj, target_type, id, + dense_source_platform=spatial_correction_dense_source, + ) + ts = ts.merge( save_progress=True, dir=os.path.join(data_dir, "merged_progress"), + ref_lat=ref_lat, + ref_lon=ref_lon, + distance_penalty_scale_per_km=distance_penalty_scale, + spatial_correction_model=spatial_correction_model, + **merging_options, ) ts.export_csv(os.path.join(data_dir, "merged_timeseries.csv")) +def _merge_reservoirs_timeseries(prj: "Project") -> None: + """Merge multi-mission timeseries into combined datasets, for reservoirs.""" + _merge_timeseries(prj, "reservoirs") + + +def _merge_rivers_timeseries(prj: "Project") -> None: + """Merge multi-mission timeseries into combined datasets, for river targets.""" + _merge_timeseries(prj, "rivers") + + # ============================================================================ # RESERVOIRS: Summaries & Visualization # ============================================================================ @@ -1145,6 +2920,44 @@ def _load_merged_timeseries(prj, id): # ============================================================================ +def _project_num_months(prj: "Project") -> int: + """ + Approximate number of months spanned by the project's configured + date range -- used as a minimum-observation-count threshold for + plotting (see _has_enough_observations_to_plot). Falls back to 1 if + the project-level dates aren't resolvable for some reason. + """ + project_cfg = prj.config.get("project", {}) + start = project_cfg.get("startdate") + end = project_cfg.get("enddate") + if not start or not end: + return 1 + + start_date = datetime.date(*start) if isinstance(start, list) else start + end_date = datetime.date(*end) if isinstance(end, list) else end + months = ( + (end_date.year - start_date.year) * 12 + + (end_date.month - start_date.month) + + 1 + ) + return max(months, 1) + + +def _has_enough_observations_to_plot(prj: "Project", target_id, min_months: int) -> bool: + """ + Whether a target has enough merged observations to be worth + plotting -- more than min_months (the project's date range in + months) or more than 2, whichever is larger. A reach/reservoir with + only 1-2 points produces a plot that adds noise without telling you + anything. + """ + df = _load_merged_timeseries(prj, target_id) + if df is None: + return False + threshold = max(min_months, 2) + return len(df) > threshold + + def generate_rivers_summaries( prj: "Project", show: bool = False, save: bool = True ) -> None: @@ -1163,16 +2976,59 @@ def generate_rivers_summaries( return waterbody_groups = _group_river_targets_by_waterbody(prj) + min_months = _project_num_months(prj) for wb_id, target_ids in waterbody_groups.items(): + # Only plot targets with enough observations to be worth looking + # at -- applies to all three plot types (map, time series, merge + # progress) so a target excluded from one isn't confusingly still + # shown in another. + plottable_ids = [ + t for t in target_ids + if _has_enough_observations_to_plot(prj, t, min_months) + ] + if not plottable_ids: + logger.info( + "Skipping plots for waterbody %s -- no targets with more " + "than %d observations.", wb_id, max(min_months, 2), + ) + continue + + # Compute the actual extraction corridor (same buffer resolution + # used for real extraction, see _river_target_corridor) so the + # shaded area shown is exactly what extraction uses, not an + # approximation -- lets you visually judge whether width-based + # buffering produced a reasonable corridor for this waterbody + # (e.g. a lake-flagged reach whose SWORD width reflects a much + # wider lake extent) without needing external knowledge of the + # real river geometry. + corridor_gdf = _river_target_corridor( + prj, plottable_ids, + buffer_meters=getattr(prj.rivers, "extraction_buffer_meters", None), + width_buffer_factor=getattr(prj.rivers, "width_buffer_factor", 1.05), + ) + corridor_geometry = corridor_gdf.geometry.iloc[0] if corridor_gdf is not None else None + plotting.plot_river_crossings( - prj, wb_id, target_ids, prj.dirs["output"], show=show, save=save + prj, wb_id, plottable_ids, prj.dirs["output"], show=show, save=save, + corridor_geometry=corridor_geometry, ) plotting.plot_river_data( - prj, wb_id, target_ids, prj.dirs["output"], show=show, save=save + prj, wb_id, plottable_ids, prj.dirs["output"], + get_merged_fn=lambda id: _load_merged_timeseries(prj, id), + show=show, save=save, ) + for target_id in plottable_ids: + plotting.plot_merging( + reservoir_id=target_id, + output_dir=prj.dirs["output"], + reservoir_type="river", + show=show, + save=save, + ) + # ============================================================================ # MIKEIO @@ -1255,4 +3111,4 @@ def _export_cleaned_to_dfs0(prj: "Project") -> None: product, id, exc, - ) + ) \ No newline at end of file diff --git a/HydroEO/plotting.py b/HydroEO/plotting.py index f84deb8..1847381 100644 --- a/HydroEO/plotting.py +++ b/HydroEO/plotting.py @@ -6,6 +6,7 @@ from matplotlib.patches import Patch import matplotlib.dates as mdates import matplotlib.pyplot as plt +import numpy as np from cmcrameri import cm import pandas as pd import geopandas as gpd @@ -19,6 +20,39 @@ logger = logging.getLogger(__name__) +# Shared platform color scheme, used by both plot_crossings (reservoir map) +# and plot_merging (progress plot) so a given platform's color is +# consistent across every plot type, not just within one function. +PLATFORM_COLORS_CMAP = cm.batlow.resampled(10) + +# Mission-level colors: used by plot_crossings, which colors by +# raw_observations FILENAME (one file per mission, e.g. "sentinel3.gpkg" +# -- which can contain BOTH S3A and S3B rows together, since the file is +# named by mission, not by satellite instance). +PLATFORM_COLORS = { + "icesat2": PLATFORM_COLORS_CMAP(0), + "sentinel3": PLATFORM_COLORS_CMAP(3), + "sentinel6": PLATFORM_COLORS_CMAP(6), + "swot": PLATFORM_COLORS_CMAP(9), +} + +# Satellite-instance colors: used by plot_merging/plot_river_data, which +# color by the actual "platform" COLUMN value in the data -- this is the +# specific satellite (e.g. "S3A"/"S3B", confirmed from +# sentinel.preprocess.extract_observations: platform = file_split[1] from +# the raw filename), not the generic mission name. S3A/S3B (and S6A/S6B, +# once Sentinel-6B exists) get distinct shades near their parent mission's +# PLATFORM_COLORS entry, so the two dicts stay visually related without +# being identical. +SATELLITE_COLORS = { + "icesat2": PLATFORM_COLORS["icesat2"], + "S3A": PLATFORM_COLORS_CMAP(2), + "S3B": PLATFORM_COLORS_CMAP(4), + "S6A": PLATFORM_COLORS_CMAP(5), + "S6B": PLATFORM_COLORS_CMAP(7), + "swot": PLATFORM_COLORS["swot"], +} + plt.rcParams["font.family"] = "serif" plt.rcParams.update({"font.size": 10}) @@ -51,13 +85,7 @@ def plot_crossings( save : bool Whether to save the plot to PNG. """ - cmap = cm.batlow.resampled(5) - colors = { - "icesat2": cmap(0), - "sentinel3": cmap(2), - "sentinel6": cmap(3), - "swot": cmap(4), - } + colors = PLATFORM_COLORS # start figure fig, ax = plt.subplots() @@ -329,16 +357,22 @@ def plot_merging( show=True, save=False, ): - """Plot merging progression for a reservoir (intermediate processing steps). + """Plot merging progression for a target (intermediate processing steps). + + Works identically for reservoirs and river targets (nodes/reaches) -- + reservoir_id is just used to build the output path + ({output_dir}/{reservoir_id}/merged_progress), and reservoir_type is + only a display label. For rivers, pass the node/reach id as + reservoir_id and e.g. "river" as reservoir_type. Parameters ---------- reservoir_id : str/int - The specific reservoir ID to plot. + The specific reservoir or river target (node/reach) ID to plot. output_dir : str Base output directory containing merged_progress subdirectory. reservoir_type : str - Type of reservoir (e.g., 'reservoirs'). + Display label (e.g. 'reservoirs' or 'river'). show : bool Whether to display the plot. save : bool @@ -348,30 +382,64 @@ def plot_merging( if os.path.exists(merged_dir): file_names = os.listdir(merged_dir) - num_files = len(file_names) - fig, main_ax = plt.subplots(num_files, 1, figsize=(10, 10)) + # The actual, current save_progress filenames from + # Timeseries.merge(), in pipeline order. Previously this + # referenced "mad_filter.csv", which the pipeline has never + # actually produced (bias_correct.csv is the real step there), + # so bias_correct silently never appeared; spatial_correction.csv + # and distance_penalty.csv (both optional, newer steps) were + # missing entirely too. + pipeline_steps = [ + "svr_linear.csv", + "spatial_correction.csv", + "bias_correct.csv", + "daily_mad_error.csv", + "distance_penalty.csv", + "kalman.csv", + "svr_radial.csv", + ] + present_steps = [f for f in pipeline_steps if f in file_names] + + if not present_steps: + logger.warning( + "No recognized merge-progress files found in %s -- nothing to plot.", + merged_dir, + ) + return + + # Base subplot count on recognized-and-present steps, not the raw + # directory listing -- otherwise a mismatch between allocated + # subplots and actually-plotted steps leaves blank axes. + fig, main_ax = plt.subplots(len(present_steps), 1, figsize=(10, 10)) fig.suptitle(f"{reservoir_type}: {reservoir_id}") - def _plot_file(i, fig, main_ax, file_name, file_names, dir): - if file_name in file_names: - i = i + 1 - ax = main_ax.flat[i] - df = pd.read_csv(os.path.join(dir, file_name)) - df["date"] = pd.to_datetime(df.date) - df.plot(ax=ax, x="date", y="height", c="k", kind="scatter") - ax.set_title(file_name) - ax.tick_params(direction="in", width=1.5) - for spine in ax.spines.values(): - spine.set_linewidth(1.5) - return i - - i = -1 - i = _plot_file(i, fig, main_ax, "svr_linear.csv", file_names, merged_dir) - i = _plot_file(i, fig, main_ax, "mad_filter.csv", file_names, merged_dir) - i = _plot_file(i, fig, main_ax, "daily_mad_error.csv", file_names, merged_dir) - i = _plot_file(i, fig, main_ax, "kalman.csv", file_names, merged_dir) - i = _plot_file(i, fig, main_ax, "svr_radial.csv", file_names, merged_dir) + # plt.subplots returns a bare Axes (not an array) when there's + # only one subplot -- normalize so the loop below works either way. + axes = np.atleast_1d(main_ax) + + for ax, file_name in zip(axes, present_steps): + df = pd.read_csv(os.path.join(merged_dir, file_name)) + df["date"] = pd.to_datetime(df.date) + + if "platform" in df.columns: + # Color by platform where the data still has it -- lost + # after Kalman collapses to one value per date (kalman.csv, + # svr_radial.csv), which fall through to the neutral case. + for platform, group in df.groupby("platform"): + color = SATELLITE_COLORS.get(platform, "gray") + ax.scatter( + group["date"], group["height"], + color=color, label=platform, s=12, + ) + ax.legend(fontsize=8, loc="best", frameon=False) + else: + ax.scatter(df["date"], df["height"], color="k", s=12) + + ax.set_title(file_name) + ax.tick_params(direction="in", width=1.5) + for spine in ax.spines.values(): + spine.set_linewidth(1.5) fig.tight_layout() if save: @@ -392,6 +460,8 @@ def plot_river_crossings( output_dir: str, save: bool = False, show: bool = False, + zoom="auto", + corridor_geometry=None, ) -> None: """Plot river target locations (nodes or reaches) on a basemap. @@ -409,6 +479,32 @@ def plot_river_crossings( Whether to save the plot to PNG show : bool Whether to display the plot interactively + zoom : int or "auto", optional + Basemap tile zoom level. Default "auto" lets contextily pick a + zoom appropriate to the actual plotted extent. A fixed zoom + (e.g. 15, street-level detail) was previously hardcoded here -- + fine for a single reservoir's small extent (which is why the + reservoir equivalent, plot_crossings, never needed this at all), + but for a river AOI spanning a whole network, a fixed high zoom + can mean fetching thousands of tiles over the network regardless + of how small the local files are -- confirmed as the likely + cause of a multi-minute stall on a real run. Pass an explicit + int only if you specifically want more/less detail than the + auto-selected level for a particular AOI size. + corridor_geometry : shapely geometry, optional + The actual buffered extraction corridor for this waterbody (see + flows._river_target_corridor), drawn as a translucent shaded + fill behind the reach/node lines. Whether width-based buffering + (see _river_target_corridor) produces a reasonable corridor size + -- e.g. for a lake-flagged reach whose SWORD "width" reflects a + much wider lake extent rather than a narrow channel -- isn't + something that can be judged from the numbers alone; seeing the + actual shaded area against the real river geometry is what lets + you decide whether an adjustment (an explicit + extraction_buffer_meters, or a max_extraction_buffer_meters + ceiling) is actually needed. None (default) skips the shading + entirely -- purely additive, doesn't change any existing + behavior if not provided. """ subset_path = prj.dirs.get("sword_subset") if not subset_path or not os.path.exists(subset_path): @@ -424,19 +520,60 @@ def plot_river_crossings( xmin, ymin, xmax, ymax = features["geometry"].total_bounds + # If the corridor extends beyond the reaches' own bounds (it always + # does, by definition -- it's a buffer around them), widen the axis + # limits to show the full shaded area, not just the reach lines. + if corridor_geometry is not None: + cxmin, cymin, cxmax, cymax = corridor_geometry.bounds + xmin, ymin = min(xmin, cxmin), min(ymin, cymin) + xmax, ymax = max(xmax, cxmax), max(ymax, cymax) + ax.set_xlim([xmin - 0.05, xmax + 0.05]) ax.set_ylim([ymin - 0.05, ymax + 0.05]) - if id_label == "nodes": - features.plot(ax=ax, color="black", markersize=5, edgecolor="none") - else: - features.plot(ax=ax, color="black") + if corridor_geometry is not None: + gpd.GeoDataFrame(geometry=[corridor_geometry], crs=features.crs).plot( + ax=ax, facecolor="gray", edgecolor="gray", alpha=0.25, + linewidth=0.5, zorder=1, + ) + + # Color each target individually rather than plotting everything in a + # single black color -- otherwise a multi-reach map is nearly useless + # for telling targets apart. Uses the same colormap/index scheme as + # plot_river_data, so a given target's color is consistent between + # the map and its time series line. + cmap = cm.hawaii + n = len(target_ids) + for idx, target_id in enumerate(target_ids): + target_features = features[features[prj.rivers.target_id_col] == target_id] + if target_features.empty: + continue + color = cmap(idx / max(n - 1, 1)) + if id_label == "nodes": + target_features.plot( + ax=ax, color=color, markersize=5, edgecolor="none", zorder=3, + ) + else: + target_features.plot(ax=ax, color=color, zorder=3) + # Reaches are lines -- unlike nodes, there's nothing marking + # an individual reach's location distinctly. Add a marker at + # each reach's midpoint (guaranteed to sit ON the line, unlike + # .centroid which can fall off it for a curved geometry), in + # the same color as its line. + for geom in target_features.geometry: + midpoint = geom.interpolate(0.5, normalized=True) + ax.plot( + midpoint.x, midpoint.y, + marker="o", markersize=5, color=color, linestyle="none", + zorder=4, + ) ctx.add_basemap( ax, crs=features.crs, - zoom=15, + zoom=zoom, source=ctx.providers.CartoDB.Positron, # ctx.providers.OpenStreetMap.Mapnik + zorder=0, ) ax.set_xlabel("lon") @@ -461,11 +598,21 @@ def plot_river_data( wb_id: str, target_ids: list, output_dir: str, + get_merged_fn, show: bool = False, save: bool = False, ) -> None: """Plot water surface elevation (WSE) timeseries for river targets. + Reads each target's actual merged/cleaned output (via get_merged_fn, + e.g. flows._load_merged_timeseries) -- NOT the raw Hydrocron CSV. + The raw per-waterbody CSV is single-mission (SWOT only) and + unfiltered; the merged output reflects whatever missions were + actually configured (ICESat-2/Sentinel-3/6 too, if enabled) and has + already been through quality filtering, bias correction, and Kalman + smoothing -- it's the thing actually worth looking at, and the two + can otherwise look completely disconnected from each other. + Parameters ---------- prj : Project @@ -476,13 +623,15 @@ def plot_river_data( List of target node or reach IDs to plot output_dir : str Base output directory where plots will be saved + get_merged_fn : callable + Function taking a target_id and returning its merged timeseries + DataFrame (with "date"/"height" columns) or None if unavailable. show : bool Whether to display the plot interactively save : bool Whether to save the plot to PNG """ id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" - csv_path = os.path.join(prj.dirs["swot"], str(wb_id), f"{id_label}_timeseries.csv") fig, ax = plt.subplots() fig.suptitle(f"{wb_id}: {id_label}") @@ -490,23 +639,34 @@ def plot_river_data( cmap = cm.hawaii n = len(target_ids) - df = pd.read_csv(csv_path) - df["date"] = pd.to_datetime(df.time_str) + any_data = False for idx, target_id in enumerate(target_ids): - feature_df = df[df[prj.rivers.target_id_col] == target_id] - feature_df = feature_df.sort_values(by="date", ascending=True) - feature_df.plot( + df = get_merged_fn(target_id) + if df is None or df.empty: + continue + any_data = True + df = df.sort_values(by="date", ascending=True) + df.plot( ax=ax, x="date", - y="wse", + y="height", c=cmap(idx / max(n - 1, 1)), linewidth="0.5", linestyle="-.", + marker="o", + markersize=3, legend=False, ) + if not any_data: + logger.warning( + "No merged data available to plot for waterbody %s (%s) -- " + "has create_rivers_timeseries() been run for these targets?", + wb_id, id_label, + ) + ax.set_xlabel("date") - ax.set_ylabel("wse [m]") + ax.set_ylabel("height [m]") ax.tick_params(direction="in", width=1.5) for spine in ax.spines.values(): spine.set_linewidth(1.5) @@ -521,4 +681,4 @@ def plot_river_data( if show: plt.show() - return + return \ No newline at end of file diff --git a/HydroEO/project.py b/HydroEO/project.py index 5639849..915805b 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -46,6 +46,7 @@ def __post_init__(self): self.to_process = list() self.mission_options = dict() self.processing_options = dict() + self.merging_options = dict() self.dirs = dict() self.startdates = dict() @@ -186,6 +187,35 @@ def __post_init__(self): self.reservoirs.export_to_dfs0 = self.config["reservoirs"].get( "export_to_dfs0", False ) + # NOTE: this was previously read via getattr(prj.reservoirs, + # "overwrite_extraction", False) in flows.py but never + # actually wired to config -- meaning it silently always + # defaulted to False regardless of what a user might have + # tried to set. Fixed here. + self.reservoirs.overwrite_extraction = self.config["reservoirs"].get( + "overwrite_extraction", False + ) + + # User-configurable overrides for the merge()/Kalman/svr_radial + # pipeline (see flows.DEFAULT_RESERVOIR_MERGING_OPTIONS for + # every available key and its default). Only the keys the user + # actually sets here are used to override the defaults -- any + # keys not mentioned keep their default value, so a user only + # needs to specify what they want to change. + # NOTE: DEFAULT_RESERVOIR_MERGING_OPTIONS' svr_radial_err/gamma + # trace back to DAHITI's own published values -- and DAHITI's + # calibration is for LAKES specifically. A reservoir with + # managed/operational water level changes (fill/drawdown + # cycles) can have real dynamics on a much faster timescale + # than a natural lake, making the lake-tuned defaults too + # strict (rejecting genuine fast changes as if they were + # noise). If your reservoirs behave more like this, override + # svr_radial_err and svr_radial_gamma here rather than relying + # on the lake-tuned defaults. + self.merging_options = self.config["reservoirs"].get( + "merging_options", {} + ) + self.reservoirs.merging_options = self.merging_options if "rivers" in self.config.keys() and self.config["rivers"].get( "enabled", True @@ -248,6 +278,51 @@ def __post_init__(self): self.rivers.target_id_col = target_id_col self.rivers.target_ids = target_ids + # Corridor buffer for ICESat-2/Sentinel-3/6 extraction (see + # flows._river_target_corridor). Deliberately separate from + # buffer_meters above, which only decides which SWORD + # targets count as "in the AOI" at all. If left unset + # (None, the default), each target's own SWORD "width" + # attribute is used instead of one flat value for every + # target -- see width_buffer_factor below. Only falls back + # to a flat value (prj.rivers.buffer_meters, then 500m) if + # no usable "width" column is found. + self.rivers.extraction_buffer_meters = rivers_cfg.get( + "extraction_buffer_meters" + ) + # Margin applied on top of each target's own SWORD width + # when using the width-based default above (ignored if + # extraction_buffer_meters is set explicitly). Default 1.05 + # = 5% wider than the target's actual channel width. + self.rivers.width_buffer_factor = rivers_cfg.get( + "width_buffer_factor", 1.05 + ) + # Max distance (m) for assigning a raw altimetry point to its + # nearest SWORD target (see flows._assign_points_to_river_targets). + # Falls back to the extraction buffer if not set. + self.rivers.max_node_assignment_meters = rivers_cfg.get( + "max_node_assignment_meters" + ) + self.rivers.overwrite_extraction = rivers_cfg.get( + "overwrite_extraction", False + ) + + # User-configurable overrides for the merge()/Kalman/svr_radial + # pipeline (see flows.DEFAULT_RIVER_MERGING_OPTIONS for every + # available key and its default). Set directly on + # prj.rivers rather than routed through the shared + # project-level self.merging_options reservoirs use, so a + # project with both reservoirs and rivers configured doesn't + # have them silently collide -- see flows._merge_timeseries's + # per-target-type override lookup. + # NOTE: DEFAULT_RIVER_MERGING_OPTIONS is currently a direct + # copy of the reservoir defaults and has NOT been + # independently validated against real river data -- unlike + # the reservoir defaults, which were tuned this way against + # real reservoirs. Treat it as a starting point to check + # kept/rejected counts against, not a verified value. + self.rivers.merging_options = rivers_cfg.get("merging_options", {}) + if "swot_raster" in self.config.keys() and self.config["swot_raster"].get( "enabled", True ): @@ -294,8 +369,13 @@ def __post_init__(self): "Must provide a local crs or a river or reservoir shapefile to determine local crs" ) - ### Warn when lake/reservoir-only satellites are configured for rivers or raster modes - if not hasattr(self, "reservoirs"): + ### Warn when lake/reservoir-only satellites are configured for neither + # reservoirs nor rivers mode. Previously this only checked for + # 'reservoirs' since icesat2/sentinel3/sentinel6 had no river support + # at all -- now that they work for rivers too (see + # flows._download_rivers_icesat2/_download_rivers_sentinel), the + # warning should only fire if NEITHER mode is configured. + if not hasattr(self, "reservoirs") and not hasattr(self, "rivers"): incompatible = [ m for m in ["icesat2", "sentinel3", "sentinel6"] @@ -303,10 +383,11 @@ def __post_init__(self): ] if incompatible: warnings.warn( - f"Satellite(s) {incompatible} are configured but have no effect without a " - "'reservoirs' section. ICESat-2, Sentinel-3, and Sentinel-6 require reservoir " - "waterbody polygons for spatial filtering. Remove these sections or add a " - "'reservoirs' section to silence this warning.", + f"Satellite(s) {incompatible} are configured but have no effect " + "without a 'reservoirs' or 'rivers' section. ICESat-2, Sentinel-3, " + "and Sentinel-6 require reservoir waterbody polygons or river " + "SWORD targets for spatial filtering. Remove these sections or add " + "a 'reservoirs'/'rivers' section to silence this warning.", UserWarning, stacklevel=2, ) @@ -351,6 +432,10 @@ def __sat_init(self, name: str): "track_keys", "subset_file_id", "sigma0_max", + "sigma0_min", + "source", + "latency", + "short_name", "download_threads", "exclude_obs_id_values", "pld_match_max_distance_m", @@ -396,6 +481,43 @@ def _require_creodias_credentials(self): ) return (self.creodias_user, self.creodias_pass) + def _require_earthdata_credentials(self): + """ + Check upfront that EarthData credentials are available in some + form earthaccess recognizes, before calling earthaccess.login(). + Without this check, earthaccess.login() -- called with no + explicit strategy, same as this codebase's existing SWOT path -- + falls through to environment variables, then a .netrc file, then + INTERACTIVE PROMPTING if neither is found (confirmed against + earthaccess's own documentation). In a non-interactive run (a + scheduled job, a CLI invocation) that prompt either hangs waiting + for input that will never come, or raises a confusing low-level + error deep inside earthaccess -- rather than a clear, immediate + one here. + """ + has_env = bool( + os.environ.get("EARTHDATA_USERNAME") and os.environ.get("EARTHDATA_PASSWORD") + ) + has_token = bool(os.environ.get("EARTHDATA_TOKEN")) + netrc_path = os.environ.get( + "NETRC", + os.path.join(os.path.expanduser("~"), "_netrc" if os.name == "nt" else ".netrc"), + ) + has_netrc = os.path.exists(netrc_path) + + if not (has_env or has_token or has_netrc): + raise ValueError( + "Missing EarthData credentials for the Sentinel-6 EarthData " + "source (mission_options['sentinel6']['source'] = 'earthdata'). " + "Set EARTHDATA_USERNAME and EARTHDATA_PASSWORD in the " + "environment, or EARTHDATA_TOKEN, or create a .netrc file with " + "your Earthdata Login credentials (register free at " + "https://urs.earthdata.nasa.gov). Without one of these, " + "earthaccess.login() falls through to an interactive prompt, " + "which will hang in a non-interactive run rather than fail " + "clearly." + ) + def validate_config(self): """Validate loaded config and report all discovered issues at once. @@ -473,9 +595,7 @@ def create_timeseries(self): if hasattr(self, "reservoirs"): flows.create_reservoirs_timeseries(self) if hasattr(self, "rivers"): - logger.warning( - "Rivers preprocessing is not implemented yet; skipping create_timeseries for rivers." - ) + flows.create_rivers_timeseries(self) def generate_summaries(self, show=False, save=True): warnings.filterwarnings("ignore", module="pandas\\..*") @@ -484,3 +604,92 @@ def generate_summaries(self, show=False, save=True): flows.generate_reservoirs_summaries(self, show=show, save=save) if hasattr(self, "rivers"): flows.generate_rivers_summaries(self, show=show, save=save) + + def _infer_target_type(self, target_type=None): + """ + Resolve which target type (reservoirs/rivers) a per-target call + applies to. If target_type is given explicitly, use it. Otherwise, + infer it automatically when the project only has one of the two + configured -- the common case -- and require an explicit choice + only when both are configured, since there's no way to guess + correctly between them. + """ + if target_type is not None: + return target_type + has_reservoirs = hasattr(self, "reservoirs") + has_rivers = hasattr(self, "rivers") + if has_reservoirs and not has_rivers: + return "reservoirs" + if has_rivers and not has_reservoirs: + return "rivers" + if has_reservoirs and has_rivers: + raise ValueError( + "Both reservoirs and rivers are configured for this " + "project -- specify target_type='reservoirs' or " + "target_type='rivers' explicitly." + ) + raise ValueError("Neither reservoirs nor rivers is configured for this project.") + + def list_target_observations(self, id, target_type=None): + """ + Summarize what observations exist for a target (reservoir or + river node/reach) at (platform, orbit) granularity -- the "what + could I exclude" view. See generate_summaries()'s plots + (platform-colored merge progress) for where a problem actually + shows up in the data. + """ + target_type = self._infer_target_type(target_type) + return flows.list_target_observations(self, target_type, id) + + def exclude_observations( + self, id, platform=None, orbit=None, date=None, reason=None, target_type=None, + ): + """ + Exclude observations from a target's merge, at whatever + granularity is given (a whole platform, a specific orbit/pass + value, a specific date, or any combination). Persisted to + {output}/{id}/run_config.yaml -- survives across runs, and can be + hand-edited or version-controlled directly. Re-run + create_timeseries() (or just the merge step) afterward to apply it. + + Examples + -------- + project.exclude_observations(my_id, platform="S3B", reason="bad calibration pass") + project.exclude_observations(my_id, platform="S3B", orbit=1517) + project.exclude_observations(my_id, date="2024-03-19") + """ + target_type = self._infer_target_type(target_type) + return flows.exclude_from_target( + self, target_type, id, platform=platform, orbit=orbit, date=date, reason=reason, + ) + + def list_exclusions(self, id, target_type=None): + """Current exclusion rules for a target, from its run_config.yaml.""" + target_type = self._infer_target_type(target_type) + return flows.list_exclusions(self, target_type, id) + + def remove_exclusion( + self, id, index=None, platform=None, orbit=None, date=None, target_type=None, + ): + """ + Remove one or more exclusion rules, either by position (index, + from list_exclusions()) or by matching criteria -- the same + platform/orbit/date fields used with exclude_observations(). + + Examples + -------- + project.remove_exclusion(my_id, platform="S3B", orbit=1517) + project.remove_exclusion(my_id, index=0) + """ + target_type = self._infer_target_type(target_type) + return flows.remove_exclusion( + self, target_type, id, index=index, platform=platform, orbit=orbit, date=date, + ) + + def set_merging_option(self, id, target_type=None, **kwargs): + """ + Override one or more merging_options for just this one target. + Example: project.set_merging_option(my_id, svr_radial_err=0.5) + """ + target_type = self._infer_target_type(target_type) + return flows.set_merging_option(self, target_type, id, **kwargs) \ No newline at end of file diff --git a/HydroEO/satellites/icesat2/preprocess.py b/HydroEO/satellites/icesat2/preprocess.py index f2a8c9c..e96327b 100644 --- a/HydroEO/satellites/icesat2/preprocess.py +++ b/HydroEO/satellites/icesat2/preprocess.py @@ -1,11 +1,14 @@ from __future__ import annotations import datetime +import logging import os import geopandas as gpd import pandas as pd +logger = logging.getLogger(__name__) + def extract_observations( src_dir, @@ -31,6 +34,21 @@ def extract_observations( gdf["platform"] = "icesat2" gdf["product"] = "ATL13" + # A single beam repeats across cycles (~91 days), so beam alone is not a + # unique pass identifier -- combine it with cycle_number, which together + # uniquely identify one physical crossing. This becomes the pass_key + # used by Timeseries for along-track grouping (windowed ADM, svr_linear). + if "cycle_number" in gdf.columns and "beam" in gdf.columns: + gdf["pass"] = ( + gdf["cycle_number"].astype(str) + "_" + gdf["beam"].astype(str) + ) + else: + logger.warning( + "cycle_number/beam columns not found in ATL13 extraction; " + "'pass' identifier not created for ICESat-2 (falls back to " + "date-based grouping downstream)." + ) + # Ensure CRS matches features; convert if necessary. if gdf.crs is None: gdf = gdf.set_crs(features.crs) diff --git a/HydroEO/satellites/sentinel/__init__.py b/HydroEO/satellites/sentinel/__init__.py index 9bebfb0..59c4061 100644 --- a/HydroEO/satellites/sentinel/__init__.py +++ b/HydroEO/satellites/sentinel/__init__.py @@ -47,6 +47,37 @@ def download( ) +S6_HR_SHORT_NAMES = _download.S6_HR_SHORT_NAMES + + +def query_earthdata( + aoi: list, + startdate: datetime.date, + enddate: datetime.date, + latency: str = "NTC", + short_name: str = None, +) -> list: + """ + Query PO.DAAC/EarthData for the Sentinel-6 HR product (not available + via CREODIAS). Reuses earthaccess, same as SWOT. + """ + return _download.query_earthdata( + aoi=aoi, + startdate=startdate, + enddate=enddate, + format_coord_list=geometry.format_coord_list, + latency=latency, + short_name=short_name, + ) + + +def download_earthdata(results: list, download_directory: str): + return _download.download_earthdata( + results=results, + download_directory=download_directory, + ) + + def subset( aoi: list, download_dir: str, @@ -81,7 +112,10 @@ def get_latest_obs_date(data_dir, product): __all__ = [ "query", "download", + "query_earthdata", + "download_earthdata", + "S6_HR_SHORT_NAMES", "subset", "extract_observations", "get_latest_obs_date", -] +] \ No newline at end of file diff --git a/HydroEO/satellites/sentinel/download.py b/HydroEO/satellites/sentinel/download.py index a3bfaf9..0c4faee 100644 --- a/HydroEO/satellites/sentinel/download.py +++ b/HydroEO/satellites/sentinel/download.py @@ -1,11 +1,154 @@ +import contextlib import datetime import logging import os +import warnings +import earthaccess import shapely logger = logging.getLogger(__name__) +# Sentinel-6 MF HR product short names on PO.DAAC, by latency tier. NTC +# (non-time-critical, ~60 day latency) is the final, most complete +# reprocessed product -- the right default for a research/monitoring +# pipeline that favors completeness over speed. STC (~36hr latency) +# trades some completeness for lower latency if that's ever needed. +# G01 is the current reprocessing baseline as of this writing (Dec 2025) -- +# confirmed directly against PO.DAAC's own dataset pages, not guessed. +# NRT is not included here: no NRT short name for the HR product was +# directly confirmed, so it's deliberately left out rather than guessed -- +# add it once verified, if needed. +S6_HR_SHORT_NAMES = { + "NTC": "JASON_CS_S6A_L2_ALT_HR_STD_OST_NTC_G01", + "STC": "JASON_CS_S6A_L2_ALT_HR_STD_OST_STC_F", +} + + +@contextlib.contextmanager +def _suppress_granule_size_warning(): + """Suppress known earthaccess DataGranule.size deprecation warning.""" + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"As of version 1\.0, `DataGranule\.size` will be accessed as an attribute", + category=FutureWarning, + module=r"earthaccess\.(results|store)", + ) + yield + + +def _earthdata_login(): + earthaccess.login() + + +def query_earthdata( + aoi: list, + startdate: datetime.date, + enddate: datetime.date, + format_coord_list, + latency: str = "NTC", + short_name: str = None, +) -> list: + """ + Query PO.DAAC/EarthData for Sentinel-6 HR granules via earthaccess -- + High Rate (HR) product (20Hz Ku-band data, not just 1Hz) is only + available via PO.DAAC/EarthData, not CDSE. + + Parameters + ---------- + aoi : list + Corner coordinates of the area of interest. + startdate, enddate : datetime.date + format_coord_list : callable + Same coordinate-normalizing function used by query() (CREODIAS + path) -- passed in rather than imported directly to avoid a new + import-time dependency here. + latency : {"NTC", "STC"}, optional + Which latency tier's short name to use if short_name isn't given + explicitly. Default "NTC" (final, ~60-day-latency reprocessed + product). See S6_HR_SHORT_NAMES. + short_name : str, optional + Explicit PO.DAAC collection short name, overriding `latency` -- + use this to pin an exact reprocessing baseline (e.g. an older + F08 short name) rather than relying on whatever S6_HR_SHORT_NAMES + currently points to. + + Returns + ------- + list + earthaccess granule result objects + """ + if short_name is None: + if latency not in S6_HR_SHORT_NAMES: + raise ValueError( + f"Unknown latency tier {latency!r}; expected one of " + f"{list(S6_HR_SHORT_NAMES)} or an explicit short_name." + ) + short_name = S6_HR_SHORT_NAMES[latency] + + aoi = format_coord_list(aoi) + + _earthdata_login() + + params = { + "short_name": short_name, + "temporal": (startdate, enddate), + "bounding_box": shapely.Polygon(aoi).bounds, + } + + try: + with _suppress_granule_size_warning(): + results = earthaccess.search_data(**params) + return results + except Exception as exc: + logger.error("Error searching EarthData for Sentinel-6 HR: %s", exc) + return [] + + +def download_earthdata(results: list, download_directory: str) -> list: + """ + Download Sentinel-6 HR granules found via query_earthdata. Mirrors + SWOT's download() (see HydroEO.satellites.swot._download) exactly, + including the same downloaded.log progress tracking to avoid + re-downloading files across runs. + """ + log_path = os.path.join(download_directory, "downloaded.log") + if not os.path.exists(log_path): + with open(log_path, "w") as log: + pass + + with open(log_path, "r") as log: + downloaded_ids = [line.rstrip() for line in log] + + to_download = [] + for result in results: + file_name = result.data_links()[0].split("/")[-1].split(".")[0] + if file_name not in downloaded_ids: + to_download.append(result) + + logger.info( + "%s files shown as downloaded in log", len(results) - len(to_download) + ) + logger.info("%s files will be downloaded", len(to_download)) + + if not to_download: + return [] + + try: + with _suppress_granule_size_warning(): + files = earthaccess.download(to_download, download_directory) + except Exception as exc: + logger.error("Error downloading Sentinel-6 HR files: %s", exc) + return [] + + with open(log_path, "a") as log: + for file in files or []: + file_name = str(file).replace("\\", "/").split("/")[-1] + log.write(file_name.split(".nc")[0] + "\n") + + return files or [] + def query( aoi: list, @@ -83,4 +226,4 @@ def download( outdir=download_directory, log_file=log_path, threads=threads, - ) + ) \ No newline at end of file diff --git a/HydroEO/satellites/sentinel/preprocess.py b/HydroEO/satellites/sentinel/preprocess.py index 39c5a66..3a0d2ad 100644 --- a/HydroEO/satellites/sentinel/preprocess.py +++ b/HydroEO/satellites/sentinel/preprocess.py @@ -591,19 +591,35 @@ def _defer_warning(message, *args): else: deferred_warnings.append(message) - # loop through the downloads folder and subset any products with the right extention + # loop through the downloads folder and subset any products with the right extension + # (DSE-style: .SEN3/.SEN6 directories) OR a flat, already-.nc file + # (EarthData/PO.DAAC-style download) pbar = tqdm( total=int(len(os.listdir(download_dir)) / 2), desc=f"Subsetting data in {os.path.basename(download_dir)}", unit="file", disable=not show_progress, ) - for folder in os.listdir(download_dir): - if folder.endswith(EXTENSION[product]): - pbar.update(1) - folder_path = os.path.join(download_dir, folder) + for item in os.listdir(download_dir): + item_path = os.path.join(download_dir, item) + + is_creodias_folder = item.endswith(EXTENSION[product]) + is_flat_earthdata_file = ( + product == "S6" + and os.path.isfile(item_path) + and item.endswith(".nc") + and "STD" in item.split("_") + ) - try: + if not (is_creodias_folder or is_flat_earthdata_file): + continue + + pbar.update(1) + folder = item + + try: + if is_creodias_folder: + folder_path = item_path # within the sentinel file, select the correct type of mearurements with the file_id key if product == "S3": file = _find_subset_file(folder_path, product, file_id) @@ -635,57 +651,81 @@ def _defer_warning(message, *args): ) file = candidates[0] - if not os.path.isfile(file): - _defer_warning( - "Skipping subset for %s: source file does not exist: %s", - folder, - file, - ) - continue - # we will save a new file with the same name as the sentinel folder but only keeping the nc file we choose - nc_cropped = os.path.join( - dest_dir, - "sub_" - + os.path.split(folder)[-1].split(EXTENSION[product])[0] - + ".nc", + base_name = os.path.split(folder)[-1].split(EXTENSION[product])[0] + else: + # Flat EarthData/PO.DAAC file -- it IS the file to crop + # directly, nothing to search for inside it. + file = item_path + base_name = os.path.splitext(item)[0] + + if not os.path.isfile(file): + _defer_warning( + "Skipping subset for %s: source file does not exist: %s", + folder, + file, ) + continue - with netCDF4.Dataset(file) as nc: - _validate_subset_source(nc, product, file) - - # SciHub contains 1Hz and 20Hz variables - # Get index of values within AOI at both frequencies - # The way that variabels are accesed/called within sentinel 3 and 6 are different, so we extract the indices as needed - if product == "S3": - freq_20 = "_20_ku" - freq_01 = "_01" - - lat20 = nc["lat" + freq_20][:] - lon20 = nc["lon" + freq_20][:] + nc_cropped = os.path.join(dest_dir, "sub_" + base_name + ".nc") - lat01 = nc["lat" + freq_01][:] - lon01 = nc["lon" + freq_01][:] + with netCDF4.Dataset(file) as nc: + _validate_subset_source(nc, product, file) - elif product == "S6": - freq_20 = "data_20" - freq_01 = "data_01" + # SciHub contains 1Hz and 20Hz variables + # Get index of values within AOI at both frequencies + # The way that variabels are accesed/called within sentinel 3 and 6 are different, so we extract the indices as needed + if product == "S3": + freq_20 = "_20_ku" + freq_01 = "_01" - lat = "latitude" - lon = "longitude" + lat20 = nc["lat" + freq_20][:] + lon20 = nc["lon" + freq_20][:] - lat20 = nc[freq_20]["ku"][lat][:] - lon20 = nc[freq_20]["ku"][lon][:] + lat01 = nc["lat" + freq_01][:] + lon01 = nc["lon" + freq_01][:] - lat01 = nc[freq_01][lat][:] - lon01 = nc[freq_01][lon][:] + elif product == "S6": + freq_20 = "data_20" + freq_01 = "data_01" + + lat = "latitude" + lon = "longitude" + + lat20 = nc[freq_20]["ku"][lat][:] + lon20 = nc[freq_20]["ku"][lon][:] + + lat01 = nc[freq_01][lat][:] + lon01 = nc[freq_01][lon][:] + + # Adjust the longitude + lon20 = center_longitude(lon20) + lon01 = center_longitude(lon01) + + logger.debug( + "Subset candidate %s bounds: AOI lon=[%.6f, %.6f] lat=[%.6f, %.6f], 20Hz lon=%s lat=%s, 1Hz lon=%s lat=%s", + file, + ulx, + lrx, + lry, + uly, + _format_array_range(lon20), + _format_array_range(lat20), + _format_array_range(lon01), + _format_array_range(lat01), + ) - # Adjust the longitude - lon20 = center_longitude(lon20) - lon01 = center_longitude(lon01) + # Get the indices within the bounds for 20Hz + selected = np.where( + (lon20 <= lrx) + & (lon20 >= ulx) + & (lat20 >= lry) + & (lat20 <= uly) + )[0] - logger.debug( - "Subset candidate %s bounds: AOI lon=[%.6f, %.6f] lat=[%.6f, %.6f], 20Hz lon=%s lat=%s, 1Hz lon=%s lat=%s", + if len(selected) == 0: + _defer_warning( + "Skipping subset for %s: no 20Hz points fall inside AOI. AOI lon=[%.6f, %.6f] lat=[%.6f, %.6f]; file lon=%s lat=%s.", file, ulx, lrx, @@ -693,85 +733,63 @@ def _defer_warning(message, *args): uly, _format_array_range(lon20), _format_array_range(lat20), - _format_array_range(lon01), - _format_array_range(lat01), ) + continue - # Get the indices within the bounds for 20Hz - selected = np.where( - (lon20 <= lrx) - & (lon20 >= ulx) - & (lat20 >= lry) - & (lat20 <= uly) - )[0] - - if len(selected) == 0: - _defer_warning( - "Skipping subset for %s: no 20Hz points fall inside AOI. AOI lon=[%.6f, %.6f] lat=[%.6f, %.6f]; file lon=%s lat=%s.", - file, - ulx, - lrx, - lry, - uly, - _format_array_range(lon20), - _format_array_range(lat20), - ) - continue + # Get the indices within the bounds for 1Hz + selected01 = np.where( + (lon01 <= lrx) + & (lon01 >= ulx) + & (lat01 >= lry) + & (lat01 <= uly) + )[0] - # Get the indices within the bounds for 1Hz - selected01 = np.where( - (lon01 <= lrx) - & (lon01 >= ulx) - & (lat01 >= lry) - & (lat01 <= uly) - )[0] + if len(selected01) == 0: + _defer_warning( + "Skipping subset for %s: no 1Hz points fall inside AOI. AOI lon=[%.6f, %.6f] lat=[%.6f, %.6f]; file lon=%s lat=%s.", + file, + ulx, + lrx, + lry, + uly, + _format_array_range(lon01), + _format_array_range(lat01), + ) + continue - if len(selected01) == 0: - _defer_warning( - "Skipping subset for %s: no 1Hz points fall inside AOI. AOI lon=[%.6f, %.6f] lat=[%.6f, %.6f]; file lon=%s lat=%s.", - file, - ulx, - lrx, - lry, - uly, - _format_array_range(lon01), - _format_array_range(lat01), - ) - continue + min_index20, max_index20 = selected[0], selected[-1] + min_index01, max_index01 = selected01[0], selected01[-1] - min_index20, max_index20 = selected[0], selected[-1] - min_index01, max_index01 = selected01[0], selected01[-1] - - with ( - netCDF4.Dataset(file) as src, - netCDF4.Dataset(nc_cropped, "w") as dst, - ): - if product == "S3": - _crop_s3( - src, - dst, - (min_index20, max_index20), - (min_index01, max_index01), - ) + with ( + netCDF4.Dataset(file) as src, + netCDF4.Dataset(nc_cropped, "w") as dst, + ): + if product == "S3": + _crop_s3( + src, + dst, + (min_index20, max_index20), + (min_index01, max_index01), + ) - elif product == "S6": - _crop_s6( - src, - dst, - (min_index20, max_index20), - (min_index01, max_index01), - ) + elif product == "S6": + _crop_s6( + src, + dst, + (min_index20, max_index20), + (min_index01, max_index01), + ) - with netCDF4.Dataset(nc_cropped, "r") as dst: - _validate_subset_output(dst, product, nc_cropped) + with netCDF4.Dataset(nc_cropped, "r") as dst: + _validate_subset_output(dst, product, nc_cropped) - except Exception: - logger.exception( - "Subset failed for folder %s using source file %s. Continuing with next file.", - folder, - locals().get("file", "unresolved"), - ) - continue + except Exception: + logger.exception( + "Subset failed for folder %s using source file %s. Continuing with next file.", + folder, + locals().get("file", "unresolved"), + ) + continue pbar.close() @@ -855,8 +873,10 @@ def get_latest_obs_date(data_dir, product): shp_path = os.path.join(data_dir, "sentinel3.gpkg") elif product.upper() == "S6": shp_path = os.path.join(data_dir, "sentinel6.gpkg") + else: + raise ValueError(f"Unsupported product '{product}'. Expected 'S3' or 'S6'.") if os.path.exists(shp_path): gdf = gpd.read_file(shp_path) last_obs_date = max(gdf.date.values).astype(datetime.date) - return last_obs_date + return last_obs_date \ No newline at end of file diff --git a/HydroEO/utils/filters/basic_filters.py b/HydroEO/utils/filters/basic_filters.py index 375af9d..8086d49 100644 --- a/HydroEO/utils/filters/basic_filters.py +++ b/HydroEO/utils/filters/basic_filters.py @@ -1,11 +1,104 @@ """simple filters that can be applied to sat timeseries objects""" +import logging + import numpy as np import pandas as pd +from scipy.stats import theilslopes from sklearn.svm import SVR from datetime import datetime +logger = logging.getLogger(__name__) + +_EARTH_RADIUS_KM = 6371.0088 + + +def _haversine_km(lat1, lon1, lat2, lon2): + """Great-circle distance (km) between two points given in degrees.""" + lat1, lon1, lat2, lon2 = map(np.radians, (lat1, lon1, lat2, lon2)) + dlat = lat2 - lat1 + dlon = lon2 - lon1 + a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2 + return 2 * _EARTH_RADIUS_KM * np.arcsin(np.sqrt(a)) + + +def _along_track_distance_km(lats, lons): + """ + Cumulative along-track distance (km) for points already ordered along + the ground track (see _order_along_track for how that order is chosen). + """ + lats = np.asarray(lats, dtype=float) + lons = np.asarray(lons, dtype=float) + if len(lats) < 2: + return np.zeros_like(lats) + step = _haversine_km(lats[:-1], lons[:-1], lats[1:], lons[1:]) + return np.concatenate([[0.0], np.cumsum(step)]) + + +def _order_along_track(lats, lons): + """ + Return the index order that best approximates along-track order for a + single pass, given only lat/lon (no reliable sub-daily timestamp to sort + by). Sorts by whichever of latitude/longitude spans the larger range + within this group, since a single pass is normally close to monotonic in + that coordinate (avoids failing near-equatorial, near-east-west passes + where latitude barely changes). + """ + lat_range = np.ptp(lats) if len(lats) else 0 + lon_range = np.ptp(lons) if len(lons) else 0 + sort_vals = lats if lat_range >= lon_range else lons + return np.argsort(sort_vals) + + +def _resolve_pass_groups(df, date_key, pass_key=None, platform_key=None, orbit_key=None): + """ + Resolve a grouping key identifying individual physical passes/crossings, + generic across missions (this function knows nothing about which real + column names any given mission uses -- that mapping happens when the + Timeseries object is constructed). + + Resolved per ROW, not per dataframe -- this matters as soon as sources + from different missions are concatenated together (see + Timeseries.concat), since one mission's rows may have a valid pass_key + while another's are NaN. Each row falls back independently through: + 1. `pass_key`, if non-null for that row. + 2. (date_key, platform_key, orbit_key), if platform_key and orbit_key + are both non-null for that row. + 3. date_key alone. + + Returns a pandas Series of group labels (strings), same index as df. + """ + group = pd.Series(pd.NA, index=df.index, dtype=object) + + if pass_key and pass_key in df.columns: + has_pass = df[pass_key].notna() + group.loc[has_pass] = "pass:" + df.loc[has_pass, pass_key].astype(str) + + remaining = group.isna() + if ( + remaining.any() + and platform_key + and orbit_key + and platform_key in df.columns + and orbit_key in df.columns + ): + has_po = remaining & df[platform_key].notna() & df[orbit_key].notna() + group.loc[has_po] = ( + "po:" + + df.loc[has_po, date_key].astype(str) + + "_" + + df.loc[has_po, platform_key].astype(str) + + "_" + + df.loc[has_po, orbit_key].astype(str) + ) + + remaining = group.isna() + if remaining.any(): + group.loc[remaining] = "date:" + df.loc[remaining, date_key].astype(str) + + return group + def elevation_filter(timeseries, height_range): min_height, max_height = height_range @@ -13,7 +106,7 @@ def elevation_filter(timeseries, height_range): timeseries.df = timeseries.df.loc[timeseries.df[timeseries.height_key] < max_height] timeseries.df = timeseries.df.reset_index(drop=True) - timeseries.df.sort_values(by=timeseries.date_key) + timeseries.df = timeseries.df.sort_values(by=timeseries.date_key) def mad_filter(timeseries, threshold=5): @@ -31,43 +124,189 @@ def mad_filter(timeseries, threshold=5): return timeseries -def daily_mad_error(timeseries, reg_weight=0.1, reg_default=0.5, error_key="ADM"): +def daily_mad_error( + timeseries, window_km=None, reg_weight=0.1, reg_default=0.5, + min_window_points=4, max_theilsen_points=60, +): + """ + Assign each observation an error estimate for use as the Kalman filter's + observation uncertainty. + + Rows with a pre-supplied, mission-native formal uncertainty (see + timeseries.preset_error_key, e.g. SWOT's wse_u) skip computed ADM + entirely and use that value directly -- SWOT's LakeSP WSE is already + one integrated value per crossing, not raw along-track points, so + there is nothing to compute local dispersion or a local trend over. + All other rows fall through to one of two modes: + + window_km=None (default): original behavior. Groups observations by + calendar day and uses the absolute deviation from the daily median + height as the error, regularized by day-level observation count. + + window_km=: along-track, distance-based sliding window per + physical pass (see _resolve_pass_groups for how a pass is + identified). Within +/- window_km of each point, fits a robust + (Theil-Sen) local linear trend against along-track distance and uses + the deviation from that local trend as the error -- rather than the + local median -- so a real, physical along-track slope (e.g. a river + gradient) isn't mistaken for noise. Falls back to a local median + within the window if fewer than min_window_points fall inside it, + and further to reg_default alone if the pass has only 1 point. + Requires timeseries.lat_key/lon_key columns; if they are missing or + empty, this silently falls back to the window_km=None behavior with + a warning, rather than failing the whole pipeline. + + Parameters + ---------- + timeseries : Timeseries + window_km : float, optional + Along-track half-width (km) of the sliding window. None (default) + keeps the original day-grouped-median behavior. + reg_weight : float, optional + Regularization numerator, divided by the local point count. + reg_default : float, optional + Regularization used when a group/window has only 1 point. + min_window_points : int, optional + Minimum points required inside a window to fit a local Theil-Sen + trend rather than falling back to a local median. Default 4. + max_theilsen_points : int, optional + Cap on points fed to a single Theil-Sen fit. Theil-Sen is O(k^2) in + window size (pairwise slopes), so dense along-track data (e.g. + ATL13) can make an uncapped fit the dominant cost. Windows larger + than this are evenly subsampled down to this many points rather + than truncated, to keep coverage across the whole window. Default + 60 (60^2 = 3600 pairs per fit, vs. e.g. 800^2 = 640000 uncapped). + """ # sort inplace the timeseries object timeseries.df = timeseries.df.sort_values(by=timeseries.date_key).reset_index( drop=True ) - # exctract for calculations - df = timeseries.df.copy() + full_df = timeseries.df date_key = timeseries.date_key height_key = timeseries.height_key + error_key = timeseries.error_key + lat_key = getattr(timeseries, "lat_key", None) + lon_key = getattr(timeseries, "lon_key", None) + preset_error_key = getattr(timeseries, "preset_error_key", None) + + # Split off rows that already carry a formal, mission-supplied + # uncertainty -- they bypass everything below. + has_preset = pd.Series(False, index=full_df.index) + if preset_error_key and preset_error_key in full_df.columns: + has_preset = full_df[preset_error_key].notna() + + error_full = pd.Series(np.nan, index=full_df.index, dtype=float) + if has_preset.any(): + error_full.loc[has_preset] = ( + full_df.loc[has_preset, preset_error_key].astype(float).clip(lower=1e-6) + ) - # Assign and use a consistent day key for grouping and mapping. - # Mixing full timestamps and date-only keys can produce NaN mapped values. - day_key = df[date_key].dt.floor("D") + df = full_df.loc[~has_preset].copy() - # Group by day and get median/count. - medval = df.groupby(day_key).median(numeric_only=True)[height_key] - day_grp = df.groupby(day_key).count()[height_key] + if df.empty: + timeseries.df[error_key] = error_full.values + return timeseries - # Add regularization factor to avoid giving advantage to median: - reg = reg_weight / day_grp - reg[day_grp == 1] = reg_default + have_coords = ( + window_km is not None + and lat_key + and lon_key + and lat_key in df.columns + and lon_key in df.columns + and df[lat_key].notna().any() + and df[lon_key].notna().any() + ) - # map the median val and regularization to the dates they belong to - med_map = medval.to_dict() - reg_map = reg.to_dict() - df["med"] = day_key.map(med_map) - df["reg"] = day_key.map(reg_map) + if window_km is not None and not have_coords: + logger.warning( + "daily_mad_error: window_km given but lat_key/lon_key ('%s'/'%s') " + "not found or empty -- falling back to day-grouped-median ADM.", + lat_key, + lon_key, + ) + + if not have_coords: + # ----- original day-grouped-median behavior ----- + day_key = df[date_key].dt.floor("D") + + medval = df.groupby(day_key).median(numeric_only=True)[height_key] + day_grp = df.groupby(day_key).count()[height_key] + + reg = reg_weight / day_grp + reg[day_grp == 1] = reg_default + + med_map = medval.to_dict() + reg_map = reg.to_dict() + df["med"] = day_key.map(med_map) + df["reg"] = day_key.map(reg_map) + + error = np.abs(df[height_key] - df["med"]) + df["reg"] + error = ( + error.replace([np.inf, -np.inf], np.nan) + .fillna(reg_default) + .clip(lower=1e-6) + ) - # calculate error and enforce a positive finite lower bound for Kalman stability - error = np.abs(df[height_key] - df["med"]) + df["reg"] - error = ( - error.replace([np.inf, -np.inf], np.nan).fillna(reg_default).clip(lower=1e-6) + error_full.loc[df.index] = error.values + timeseries.df[error_key] = error_full.values + return timeseries + + # ----- windowed, slope-aware path ----- + groups = _resolve_pass_groups( + df, + date_key, + getattr(timeseries, "pass_key", None), + getattr(timeseries, "platform_key", None), + getattr(timeseries, "orbit_key", None), ) - # add error to timeseries - timeseries.df[error_key] = error.values + error = pd.Series(np.nan, index=df.index, dtype=float) + + for _, idx in df.groupby(groups).groups.items(): + idx = np.asarray(idx) + sub = df.loc[idx] + n = len(idx) + + if n == 1: + error.loc[idx] = reg_default + continue + + order = _order_along_track(sub[lat_key].values, sub[lon_key].values) + idx_sorted = idx[order] + lat = sub[lat_key].values[order] + lon = sub[lon_key].values[order] + heights = sub[height_key].values[order] + dist = _along_track_distance_km(lat, lon) + + for i in range(n): + window_mask = np.abs(dist - dist[i]) <= window_km + n_window = int(window_mask.sum()) + + if n_window >= min_window_points: + win_h = heights[window_mask] + win_d = dist[window_mask] + if n_window > max_theilsen_points: + # Theil-Sen is O(k^2) in window size (pairwise slopes); + # dense along-track data (e.g. ATL13) can otherwise make + # this the dominant cost. Evenly subsample rather than + # truncate, to keep coverage across the whole window. + sel = np.linspace( + 0, n_window - 1, max_theilsen_points + ).round().astype(int) + win_h = win_h[sel] + win_d = win_d[sel] + slope, intercept, _, _ = theilslopes(win_h, win_d) + local_ref = intercept + slope * dist[i] + else: + local_ref = np.median(heights[window_mask]) + + reg = reg_weight / n_window if n_window > 1 else reg_default + error.loc[idx_sorted[i]] = abs(heights[i] - local_ref) + reg + + error = error.replace([np.inf, -np.inf], np.nan).fillna(reg_default).clip(lower=1e-6) + error_full.loc[df.index] = error.values + timeseries.df[error_key] = error_full.values return timeseries @@ -100,7 +339,7 @@ def daily_mean_merge(timeseries): timeseries.df[timeseries.date_key] = pd.to_datetime(timeseries.df.index) timeseries.df = timeseries.df.reset_index(drop=True) - timeseries.df.sort_values(by=timeseries.date_key) + timeseries.df = timeseries.df.sort_values(by=timeseries.date_key) return timeseries @@ -134,73 +373,132 @@ def rolling_median(timeseries, window=7): return timeseries -def _run_svr_linear(heights, err=0.1, epsilon=0.1): +def _run_svr_linear(heights, err=0.1, epsilon=0.1, max_iter=5000): """ - Linear Support Vector Regression - Fit 0-slope line through heights along-track to remove outliers. - This method allows non-0 slope + Linear Support Vector Regression outlier filter. + + Fits a free-slope linear SVR through heights along-track and flags + points that deviate from that fitted line by more than `err` as + outliers. The fitted slope is used only to decide which points are + outliers here -- it is not removed from the retained heights (that is + a separate, later step). Parameters ---------- heights : array - Heights to be fit. + Heights to be fit, already ordered along-track. err : Float, optional - Allowed deviation from linear regression. The default is .01. + Allowed deviation from the linear fit. The default is 0.1. epsilon : Float, optional "Epsilon in the epsilon-SVR model. It specifies the epsilon-tube within which no penalty is associated in the training loss function with points predicted within a distance epsilon from the actual value." (from https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVR.html) The default is .1. + max_iter : int, optional + Hard cap on the underlying solver's iterations. sklearn's SVR + defaults to max_iter=-1 (no cap at all) -- on a pathological or + unexpectedly large/ill-conditioned group, that can make a single + fit call run for a very long time with no visible progress, + indistinguishable from a genuine hang. Default 5000; if this is + hit, a warning is logged and the (not fully converged, but usually + still reasonable) fit is used rather than blocking indefinitely. Returns ------- array - Outlier filtered heights. - + Integer positions (0-based, within `heights`) of the points that + survive the filter. """ + heights = np.asarray(heights, dtype=float) + # sequential x axis along track x = np.arange(0, len(heights)) x = np.vstack( [x, np.ones(len(x))] ).T # Extend x data to contain another row vector of 1s - y = heights.values # TODO: investigate First remove slope if any + y = heights # make SVR kernel and fit with confidence bounds - svr_rbf = SVR(kernel="linear", epsilon=epsilon) + svr_rbf = SVR(kernel="linear", epsilon=epsilon, max_iter=max_iter) rbf = svr_rbf.fit(x, y) + if getattr(rbf, "n_iter_", 0) is not None and np.any( + np.asarray(rbf.n_iter_) >= max_iter + ): + logger.warning( + "_run_svr_linear: SVR hit max_iter=%d without full convergence " + "on a group of %d points -- result used anyway, but check " + "whether this group is unexpectedly large (pass-grouping " + "issue) or has pathological/duplicate values.", + max_iter, len(heights), + ) uconf = rbf.predict(x) + err lconf = rbf.predict(x) - err - # only keep the filtered valeus - filtered = np.where((y >= lconf) & (y <= uconf))[0] + # return the positions of the retained points (not the values), so the + # caller can keep every column for those rows, not just height + return np.where((y >= lconf) & (y <= uconf))[0] - return np.array(heights)[filtered] +def svr_linear(timeseries, err=0.1, epsilon=0.1, max_iter=5000, warn_group_size=500): + """ + Along-track linear-SVR outlier filter, grouped by physical pass (see + _resolve_pass_groups) rather than calendar day, and preserving every + column of the surviving rows (not just date/height). -def svr_linear( - timeseries, -): # TODO: this should maybe be processed on the individual product timeseries? - df = timeseries.df + Parameters + ---------- + timeseries : Timeseries + err : float, optional + Allowed deviation from the local linear fit (m). Default 0.1. + epsilon : float, optional + SVR epsilon-tube. Default 0.1. + max_iter : int, optional + Passed through to the underlying SVR fit; see _run_svr_linear. + warn_group_size : int, optional + Log a warning up front for any single pass/group larger than this, + before attempting to fit it -- SVR's cost scales badly with group + size, so an unexpectedly large group (e.g. a pass-grouping fallback + issue lumping many points together) is worth surfacing immediately + rather than only being discovered via a slow or capped fit. + """ + df = timeseries.df.copy() date_key = timeseries.date_key height_key = timeseries.height_key - # get the remaining heights after the filter - lin_filt = df.groupby(date_key)[height_key].apply(_run_svr_linear).reset_index() - - # reassign the values to their date (essentially, "ungrouby") - r = pd.DataFrame( - { - col: np.repeat(lin_filt[col].values, lin_filt[height_key].str.len()) - for col in lin_filt.columns.drop(height_key) - } + groups = _resolve_pass_groups( + df, + date_key, + getattr(timeseries, "pass_key", None), + getattr(timeseries, "platform_key", None), + getattr(timeseries, "orbit_key", None), ) - r = r.assign(**{height_key: np.concatenate(lin_filt[height_key].values)})[ - lin_filt.columns - ] - # Reset the time series to the filtered values - timeseries.df = r + group_items = df.groupby(groups).groups.items() + group_sizes = {g: len(idx) for g, idx in group_items} + oversized = {g: n for g, n in group_sizes.items() if n > warn_group_size} + if oversized: + logger.warning( + "svr_linear: %d group(s) exceed warn_group_size=%d (largest: " + "%s with %d points) -- SVR fit cost scales badly with group " + "size; if this is unexpected, check pass_key/platform_key/" + "orbit_key resolution for this timeseries.", + len(oversized), warn_group_size, + max(oversized, key=oversized.get), max(oversized.values()), + ) + + keep_idx = [] + for _, idx in group_items: + idx = np.asarray(idx) + if len(idx) < 2: + # nothing to compare a single point against; keep it as-is + keep_idx.extend(idx) + continue + heights = df.loc[idx, height_key].values + local_keep = _run_svr_linear(heights, err=err, epsilon=epsilon, max_iter=max_iter) + keep_idx.extend(idx[local_keep]) + + timeseries.df = df.loc[np.sort(np.asarray(keep_idx))].reset_index(drop=True) return timeseries @@ -245,6 +543,24 @@ def _update(obs, xk, cov_xx, height="height", error="ADM", n=1): obs_list = obs[height].values m = len(obs_list) + + if n == 1: + # Closed-form scalar update: combining m independent observations + # (variances slk) with a Gaussian prior (xk, cov_xx) has an exact + # closed form (precision-weighted average) -- mathematically + # identical to the general matrix path below for n=1 (verified via + # fuzz testing against it), but O(m) instead of O(m^3) from + # np.linalg.pinv on an m x m matrix. n=1 is the only case used + # anywhere in this codebase today (n>1 grid support is not yet + # implemented -- see docstring), so this is the common path. + slk = np.maximum((obs[error].values) ** 2, 1e-12) + xk_s = float(np.asarray(xk).reshape(())) + cov_s = float(np.asarray(cov_xx).reshape(())) + prec = 1.0 / cov_s + np.sum(1.0 / slk) + xk_plus = (xk_s / cov_s + np.sum(obs_list / slk)) / prec + cov_xx_plus = 1.0 / prec + return np.array(xk).reshape(1, 1) * 0 + xk_plus, np.array(cov_xx).reshape(1, 1) * 0 + cov_xx_plus + lk = np.array(obs_list).reshape(m, n) Ak = np.ones((m, n)) @@ -266,7 +582,7 @@ def _update(obs, xk, cov_xx, height="height", error="ADM", n=1): return xk_plus, cov_xx_plus -def _pred(xk, cov_xx, n=1, system_noise=0, system_noise_unc=0.05): +def _pred(xk, cov_xx, n=1, system_noise=0, system_noise_unc=0.0005): """ Prediction function for Kalman filter @@ -281,7 +597,8 @@ def _pred(xk, cov_xx, n=1, system_noise=0, system_noise_unc=0.05): system_noise : float, optional System noise. The default is 0. system_noise_unc : float, optional - Uncertainty of the system noise. The default is 0.05. + Uncertainty (variance, m^2) of the system noise. The default is + 0.0005 (5 cm^2), matching DAHITI (Schwatke et al., 2015). Returns ------- @@ -306,21 +623,295 @@ def _pred(xk, cov_xx, n=1, system_noise=0, system_noise_unc=0.05): return xk_next, cov_xx_next -def kalman(timeseries, error_key="ADM", n=1): +def fit_spatial_correction_model( + df, lat_key, lon_key, height_key, date_key, ref_lat, ref_lon, + min_points_per_day=20, min_days=3, +): + """ + Fit a 1D spatial deviation model along a reservoir's most stable + spatial axis, from a dense, wide-spanning source (e.g. ICESat-2). + Unlike apply_distance_penalty (which only inflates uncertainty), this + is a genuine height CORRECTION: it estimates how much a crossing's + value tends to differ from the reservoir centroid's value as a + function of position, so that correction can be applied to ANY + mission's points -- letting a sparse source (e.g. Sentinel-3/6, with + only 1-2 points per crossing and no ability to fit its own local + trend) benefit from a dense source's spatial mapping of the reservoir. + + Method: for each day with at least min_points_per_day points, remove + that day's own median (strips out real level change over time, + leaving just the spatial deviation pattern for that day), then fit a + separate 1D linear slope of residual vs. position along each of two + orthogonal local axes (east-west, north-south). The axis whose slope + is most CONSISTENT across independent days (highest weighted-mean / + weighted-std ratio -- like a t-statistic) is taken as the real, + physically meaningful axis; the other is treated as noise and + discarded. This is deliberately more conservative than trusting a + single pooled regression across all days pooled together, which can + show a misleadingly non-zero coefficient on the noisy axis (confirmed + empirically: a pooled fit suggested a coefficient on the noisy axis + of similar magnitude to the real one, while per-day fits showed it + flipping sign inconsistently -- pooling does not reliably distinguish + a real, stable effect from incidental within-day sampling structure). + + IMPORTANT for update stability: this function has no notion of when + it's called or what's been fit before -- it fits fresh from whatever + df is passed in every time. If you re-fit every run as new data + arrives, past corrections WILL shift retroactively. Persist the + returned model (e.g. to JSON) and reuse it across runs rather than + refitting automatically; only refit on an explicit recalibration + trigger. See flows.py for the persistence wrapper. + + Parameters + ---------- + df : DataFrame + Points from the dense source only (e.g. filter to platform=="icesat2" + before calling), with valid lat_key/lon_key/height_key/date_key. + ref_lat, ref_lon : float + Reference location (e.g. the reservoir polygon's own centroid -- + see flows._reservoir_centroid) that corrected heights are + expressed relative to. + min_points_per_day : int, optional + Minimum points on a given day to attempt a local fit for that day. + Default 20. + min_days : int, optional + Minimum number of qualifying days required to fit a model at all. + Default 3 -- a single day's pattern could be a real but transient + anomaly (e.g. a wind event), not a persistent geometric feature; + requiring several independent days is what lets us distinguish + "persistent, worth correcting for" from "one-off." + + Returns + ------- + dict or None + {"axis": "x" or "y", "slope_m_per_km": float, "ref_lat": float, + "ref_lon": float, "n_days_used": int, "diagnostics": {...}} + None if there isn't enough dense data to fit anything -- callers + should treat this as "no correction available," not an error. + """ + df = df.copy() + day_key = df[date_key].dt.floor("D") + day_median = df.groupby(day_key)[height_key].transform("median") + residual = df[height_key] - day_median + + x_km = (df[lon_key] - ref_lon) * 111.0 * np.cos(np.radians(ref_lat)) + y_km = (df[lat_key] - ref_lat) * 111.0 + + records = [] + for day, idx in df.groupby(day_key).groups.items(): + idx = np.asarray(idx) + if len(idx) < min_points_per_day: + continue + r = residual.loc[idx].values + for axis_name, coord in (("x", x_km.loc[idx].values), ("y", y_km.loc[idx].values)): + if np.std(coord) < 1e-6: + continue + slope, intercept = np.polyfit(coord, r, 1) + pred = slope * coord + intercept + ss_res = np.sum((r - pred) ** 2) + ss_tot = np.sum((r - r.mean()) ** 2) + r2 = float(1 - ss_res / ss_tot) if ss_tot > 0 else float("nan") + records.append(dict(day=day, axis=axis_name, n=len(idx), slope=slope, r2=r2)) + + if not records: + logger.warning( + "fit_spatial_correction_model: no day had >= %d points -- " + "not enough dense data to fit a model.", min_points_per_day, + ) + return None + + fits = pd.DataFrame(records) + n_days = fits["day"].nunique() + if n_days < min_days: + logger.warning( + "fit_spatial_correction_model: only %d qualifying day(s) " + "available (need >= %d) -- not fitting a model.", n_days, min_days, + ) + return None + + best_axis, best_slope, best_score = None, None, -np.inf + diagnostics = {} + for axis_name in ("x", "y"): + sub = fits[fits.axis == axis_name] + if sub.empty: + continue + w = sub.n.values + mean_slope = float(np.average(sub.slope, weights=w)) + weighted_std = float(np.sqrt(np.average((sub.slope - mean_slope) ** 2, weights=w))) + score = abs(mean_slope) / (weighted_std + 1e-9) + diagnostics[axis_name] = dict( + mean_slope_m_per_km=mean_slope, weighted_std_across_days=weighted_std, + stability_score=float(score), n_days=int(sub["day"].nunique()), + ) + if score > best_score: + best_axis, best_slope, best_score = axis_name, mean_slope, score + + logger.info( + "fit_spatial_correction_model: chose axis '%s' (score=%.2f) over " + "the other (see diagnostics for both): %s", best_axis, best_score, diagnostics, + ) + + return { + "axis": best_axis, + "slope_m_per_km": best_slope, + "ref_lat": float(ref_lat), + "ref_lon": float(ref_lon), + "n_days_used": int(n_days), + "diagnostics": diagnostics, + } + + +def apply_spatial_correction(timeseries, model): + """ + Apply a fitted spatial correction model (see fit_spatial_correction_model) + to every point (any mission) with valid lat_key/lon_key, referencing + every crossing to the model's reference location along its dominant + axis. Unlike apply_distance_penalty, this adjusts the height itself, + not just its error/uncertainty. + + A no-op (with a log note, not an error) if model is None (nothing was + fit -- e.g. not enough dense-source data yet) or if lat_key/lon_key + aren't available on this timeseries (e.g. SWOT). + """ + if model is None: + logger.info("apply_spatial_correction: no model given -- skipping (no-op).") + return timeseries + + lat_key = getattr(timeseries, "lat_key", None) + lon_key = getattr(timeseries, "lon_key", None) + have_coords = ( + lat_key + and lon_key + and lat_key in timeseries.df.columns + and lon_key in timeseries.df.columns + and timeseries.df[lat_key].notna().any() + ) + if not have_coords: + logger.info( + "apply_spatial_correction: lat_key/lon_key not available -- " + "skipping (no-op). Expected e.g. for SWOT." + ) + return timeseries + + ref_lat, ref_lon = model["ref_lat"], model["ref_lon"] + axis, slope = model["axis"], model["slope_m_per_km"] + + mask = timeseries.df[lat_key].notna() & timeseries.df[lon_key].notna() + correction = pd.Series(0.0, index=timeseries.df.index) + + if axis == "x": + coord = (timeseries.df.loc[mask, lon_key] - ref_lon) * 111.0 * np.cos(np.radians(ref_lat)) + else: + coord = (timeseries.df.loc[mask, lat_key] - ref_lat) * 111.0 + + correction.loc[mask] = slope * coord + timeseries.df[timeseries.height_key] = timeseries.df[timeseries.height_key] - correction + + return timeseries + + +def apply_distance_penalty(timeseries, ref_lat, ref_lon, scale_per_km=0.05): + """ + Inflate each observation's error (timeseries.error_key) by an amount + proportional to its distance from a reference location -- a smooth, + additive down-weighting for the Kalman filter. + + Motivation: local ADM only measures how much a crossing's points scatter + around each other -- it says nothing about whether that crossing's + average value is actually representative of the main reservoir body. A + well-sampled crossing far upstream (subject to real slope effects, e.g. + a riverine arm during drawdown) can have very LOW internal scatter (all + its points agreeing closely with each other) while still being + systematically biased relative to the main body -- ADM alone would + treat that as a highly trustworthy observation, the opposite of what's + wanted. This function adds a second, independent uncertainty term based + purely on location, so Kalman's existing precision-weighting naturally + discounts far-from-reference crossings without needing to guess at + "representativeness" from point count or any other proxy. + + Call this AFTER daily_mad_error (requires timeseries.error_key to + already be populated) and BEFORE kalman(). + + Parameters + ---------- + timeseries : Timeseries + ref_lat, ref_lon : float + Reference location, e.g. the reservoir polygon's own centroid (not + a centroid of wherever satellite data happened to sample -- the + true reservoir geometry is a more principled, sampling-independent + reference), or a dam/outlet location if available. + scale_per_km : float, optional + Additional error (m) added per km of distance from the reference. + Default 0.05 m/km -- e.g. a crossing 20 km upstream gets +1.0 m of + inflated error. This is a real physical scale that should be tuned + against reservoirs where the true magnitude of upstream slope bias + is known, not trusted blindly at this default. + + Requires timeseries.lat_key/lon_key. If unavailable (e.g. SWOT, which + has no per-observation coordinates -- see preset_error_key), this is a + no-op with a log note, not an error: there's nothing to compute a + distance from, and SWOT's own wse_u already supersedes ADM entirely for + those rows regardless. + """ + lat_key = getattr(timeseries, "lat_key", None) + lon_key = getattr(timeseries, "lon_key", None) + error_key = timeseries.error_key + + have_coords = ( + lat_key + and lon_key + and lat_key in timeseries.df.columns + and lon_key in timeseries.df.columns + and timeseries.df[lat_key].notna().any() + ) + if not have_coords: + logger.info( + "apply_distance_penalty: lat_key/lon_key not available -- " + "skipping (no-op). Expected e.g. for SWOT, which has no " + "per-observation coordinates." + ) + return timeseries + + if error_key not in timeseries.df.columns: + logger.warning( + "apply_distance_penalty: error_key '%s' not found -- call this " + "after daily_mad_error, not before. Skipping (no-op).", + error_key, + ) + return timeseries + + mask = timeseries.df[lat_key].notna() & timeseries.df[lon_key].notna() + dist_km = pd.Series(0.0, index=timeseries.df.index) + if mask.any(): + dist_km.loc[mask] = _haversine_km( + timeseries.df.loc[mask, lat_key].values, + timeseries.df.loc[mask, lon_key].values, + ref_lat, + ref_lon, + ) + + timeseries.df[error_key] = timeseries.df[error_key] + scale_per_km * dist_km + return timeseries + + +def kalman(timeseries, n=1, system_noise_unc=0.0005): """ Run Kalman filter Parameters ---------- - time_series : DataFrame - Outlier filtered dataframe to be used as input for Kalman filter. - Must contain height and error columns - height : string, optional - Name of height column. The default is 'height_OCOG'. - error : string, optional - Name of error column. The default is 'ADM'. + timeseries : Timeseries + Outlier-filtered timeseries to be used as input for the Kalman filter. + Reads height_key, error_key, and date_key from the timeseries object + itself (must contain those columns already, e.g. after daily_mad_error). n : Int, optional Grid size - only relevant for large lakes e.g. Not yet implemented. The default is 1. + system_noise_unc : float, optional + Variance (m^2) of the system noise injected at each prediction step, + i.e. how much the state is allowed to drift between updates. + The default is 0.0005 (5 cm^2), matching DAHITI (Schwatke et al., 2015). + Increasing the value allows the filter to track + raw observations much more closely (less smoothing) than intended. Returns ------- @@ -334,13 +925,19 @@ def kalman(timeseries, error_key="ADM", n=1): df = timeseries.df.copy() date_key = timeseries.date_key height_key = timeseries.height_key + error_key = timeseries.error_key + + # Group by calendar day, not exact timestamp. + day_key = df[date_key].dt.floor("D") + dates = sorted(day_key.unique()) + xks = np.ones((n, len(dates))) * np.nan + cov_xxs = np.ones((n, n, len(dates))) * np.nan - dates = sorted(df[date_key].unique()) - xks = np.ones((n, len(df[date_key].unique()))) * np.nan - cov_xxs = np.ones((n, n, len(df[date_key].unique()))) * np.nan + # Group once up front + grouped = {k: v for k, v in df.groupby(day_key)} # Initialize prediction and uncertainty matrices - obs = df.loc[df[date_key] == dates[0]] + obs = grouped[dates[0]] lk = obs[height_key].values slk = (obs[error_key].values) ** 2 @@ -351,14 +948,15 @@ def kalman(timeseries, error_key="ADM", n=1): # Observation model for i, d in enumerate(dates): # Update - obs = df.loc[df[date_key] == d] + obs = grouped[d] xk_plus, cov_xx_plus = _update( obs, xks[:, i], cov_xxs[:, :, i], height=height_key, error=error_key, n=n ) # Predict xk1, cov_xx1 = _pred( - xk_plus, cov_xx_plus, n=n, system_noise=0, system_noise_unc=0.05 + xk_plus, cov_xx_plus, n=n, system_noise=0, + system_noise_unc=system_noise_unc ) xks[:, i] = xk_plus cov_xxs[:, :, i] = cov_xx_plus @@ -375,7 +973,8 @@ def kalman(timeseries, error_key="ADM", n=1): return df_kalman -def _run_svr_rbf(dates, heights, err=1, rbf_c=10000, gamma=0.0000438, epsilon=0.1): +def _run_svr_rbf(dates, heights, err=1.0, rbf_c=10000, gamma=0.0000438, epsilon=0.1, + max_iter=-1, max_fit_points=None): """ Radial Base Function outlier filtering post-Kalman filter. This is run at virtual station level @@ -401,6 +1000,27 @@ def _run_svr_rbf(dates, heights, err=1, rbf_c=10000, gamma=0.0000438, epsilon=0. function with points predicted within a distance epsilon from the actual value." (from https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVR.html) The default is .1. + max_iter : int, optional + Passed to the underlying SVR fit. UNLIKE _run_svr_linear, this + defaults to -1 (unbounded) -- confirmed empirically that capping + this specific configuration (RBF kernel + rbf_c=10000, very little + regularization) produces non-monotonic, unreliable output quality: + e.g. max_iter=5000 kept 10/5760 points, 50000 kept 827/5760, and + 100000 kept only 30/5760 -- only fully unbounded (23s on that real + test case) gave the correct 2154/5760. Do not cap this without + checking output row counts, not just wall-clock time. Use + max_fit_points instead for a speedup that doesn't have this risk. + max_fit_points : int, optional + If set and there are more than this many points, the SVR is fit on + an evenly-subsampled subset of this size (not truncated -- spread + across the whole series), then applied via .predict() to ALL + original points for the outlier decision. This reduces the cost + driver (problem size going into the O(n^2)-ish solver) directly, + rather than truncating iterations on the full problem -- much + safer than capping max_iter for this kernel/C combination, since + the underlying trend being fit is smooth and a representative + subsample captures it about as well as the full set. None (default) + disables this and fits on every point, as before. Returns ------- @@ -420,8 +1040,21 @@ def _run_svr_rbf(dates, heights, err=1, rbf_c=10000, gamma=0.0000438, epsilon=0. y = heights x = np.vstack([x, np.ones(len(x))]).T - svr_rbf = SVR(kernel="rbf", C=rbf_c, gamma=gamma, epsilon=epsilon) - rbf = svr_rbf.fit(x, y) + if max_fit_points and len(x) > max_fit_points: + fit_sel = np.linspace(0, len(x) - 1, max_fit_points).round().astype(int) + fit_x, fit_y = x[fit_sel], y[fit_sel] + else: + fit_x, fit_y = x, y + + svr_rbf = SVR(kernel="rbf", C=rbf_c, gamma=gamma, epsilon=epsilon, max_iter=max_iter) + rbf = svr_rbf.fit(fit_x, fit_y) + if getattr(rbf, "n_iter_", 0) is not None and np.any( + np.asarray(rbf.n_iter_) >= max_iter > 0 + ): + logger.warning( + "_run_svr_rbf: SVR hit max_iter=%d without full convergence on " + "%d fit points -- result used anyway.", max_iter, len(fit_x), + ) corr = rbf.predict(x) uconf = corr + err @@ -438,7 +1071,43 @@ def _year_fraction(dt): return dt.year + float(dt.toordinal() - start) / year_length -def svr_radial(timeseries): +def svr_radial(timeseries, err=1.0, rbf_c=10000, gamma=0.0000438, epsilon=0.1, + max_iter=-1, max_fit_points=None): + """ + Radial-basis post-Kalman outlier filter, run at virtual station level. + + Parameters + ---------- + timeseries : Timeseries + Kalman-filtered timeseries to be filtered. + err : float, optional + Observation uncertainty / half-width of the confidence band (m). + DAHITI uses about 1 m for rivers and 0.1 m for lakes -- since + flow.py runs separate river vs. reservoir flows, pass the + appropriate value explicitly from there rather than relying on + this default. The default here (1.0) is the more conservative + (river-like) choice. + rbf_c : float, optional + Regularization parameter C of the RBF SVR. The default is 10000, + matching DAHITI (lower values make the fit more regularized + / less able to follow the data than intended). + gamma : float, optional + RBF kernel coefficient. The default is 0.0000438, as in DAHITI. + epsilon : float, optional + Epsilon-tube of the SVR (see sklearn.svm.SVR). The default is 0.1. + max_iter : int, optional + Passed through to the underlying SVR fit; see _run_svr_rbf for why + this defaults to -1 (unbounded) rather than being capped. + max_fit_points : int, optional + Passed through to _run_svr_rbf -- the safe speedup for this stage + (fit on an evenly-subsampled subset, apply to all points), rather + than capping max_iter. + + Returns + ------- + vs : DataFrame + Filtered (date, height) virtual station timeseries. + """ df = timeseries.df.copy() date_key = timeseries.date_key height_key = timeseries.height_key @@ -446,11 +1115,13 @@ def svr_radial(timeseries): rbf_filter = _run_svr_rbf( df[date_key].values, df[height_key].values, - err=1, # TODO: consider 0.1 for lakes after bias correction? - rbf_c=1000, - gamma=0.0000438, - epsilon=0.1, - ) # these are the default values as in DAHITI with error changed from 1 to 0.1m for lakes + err=err, + rbf_c=rbf_c, + gamma=gamma, + epsilon=epsilon, + max_iter=max_iter, + max_fit_points=max_fit_points, + ) nb_obs = df.groupby(date_key).count().reset_index() @@ -464,4 +1135,4 @@ def svr_radial(timeseries): } ) - return vs + return vs \ No newline at end of file diff --git a/HydroEO/utils/timeseries.py b/HydroEO/utils/timeseries.py index e1b6979..8396c84 100644 --- a/HydroEO/utils/timeseries.py +++ b/HydroEO/utils/timeseries.py @@ -10,29 +10,47 @@ logger = logging.getLogger(__name__) +def _parse_bin_days(time_bin): + """Parse a simple 'D' style string into an integer number of days.""" + s = str(time_bin).strip().upper() + if s.endswith("D"): + return int(s[:-1]) if s[:-1] else 1 + raise ValueError("time_bin must be a string like '1D' or '5D'") + + @dataclass class Timeseries: df: pd.DataFrame date_key: str = "date" height_key: str = "height" - error_key: str = "error" + error_key: str = "ADM" + lat_key: str = "lat" + lon_key: str = "lon" + pass_key: str = "pass" + platform_key: str = "platform" + orbit_key: str = "orbit" + preset_error_key: str = None - def __post__init__(self): - if ( - self.error_key not in self.df.columns - or self.height_key not in self.df.columns - ): - logger.warning("Height or error columns missing") - return + def __post_init__(self): + if self.height_key not in self.df.columns: + logger.warning( + "Height column '%s' missing from dataframe", self.height_key + ) + # error_key is intentionally not checked here: it's populated + # later in the pipeline (by daily_mad_error), not expected to + # exist at construction time. Functions that actually need it + # (daily_mad_error, kalman) will raise a clear error at the + # point they read it if it's genuinely still missing then. if self.date_key not in self.df.columns: if isinstance(self.df.index, pd.DatetimeIndex): self.df[self.date_key] = self.df.index else: logger.warning( - "date key is not in dataframe but index has been set as date column" + "date_key '%s' is not in dataframe and index is not a " + "DatetimeIndex, so it could not be auto-populated", + self.date_key, ) - return def clean(self, filters: list, filter_params: dict = None): filter_params = filter_params or {} @@ -76,32 +94,434 @@ def clean(self, filters: list, filter_params: dict = None): if "rolling_median" in filters: fltrs.rolling_median(self) - def bias_correct(self, orbit_key="orbit", product_key="platform"): - raise NotImplementedError("bias_correct is not yet implemented") + def bias_correct(self, platform_key=None, orbit_key=None, group_by="platform_orbit", + time_bin="1D", min_overlap=3, priority=None, + centroid_distance_warn_km=5.0): + """ + Cross-calibrate multiple persistent sources within this timeseries + onto a common datum, then remove each source's estimated constant + offset from its raw observations (multi-mission/multi-track bias + correction; ported from the standalone harmonize_and_merge work, + adapted to operate on a single Timeseries.df in place rather than a + dict of external per-source series). + + A "source" is one persistent virtual station. By default + (group_by="platform_orbit") that's identified by (platform_key, + orbit_key) TOGETHER -- e.g. (platform="icesat2", orbit="30") for + one ICESat-2 beam, which genuinely is stable across every revisit. + This is deliberately NOT the same grouping as pass_key: pass_key + identifies one specific crossing in time (used for along-track + slope/ADM work in basic_filters.py), while (platform_key, orbit_key) + is meant to identify the same physical ground track revisited many + times -- the thing that can carry a stable geoid/retracker bias + worth estimating and removing before Kalman filtering treats every + source as directly comparable. + + group_by="platform" ignores orbit_key entirely and groups by + platform_key alone. Use this when there is no field that's + genuinely constant across revisits of the same ground track -- + e.g. for Sentinel-3/6, where the available "orbit"-mapped field + turned out to increment roughly once per repeat cycle (confirmed + empirically: ~23.6 days/value for a 27-day S3 cycle, ~9.6 days/ + value for a ~9.9-day S6 cycle) rather than staying constant, so + grouping by it fragments each platform into many near-single- + observation "sources" that can't be calibrated against anything, + and the real, visible inter-platform bias never gets corrected. + platform-level grouping is coarser (loses any genuine within- + platform inter-track bias) but is what's actually achievable + without a reliable persistent-track field, and is what fixes an + inter-mission bias that's otherwise passing straight through. + + If fewer than 2 distinct sources are present (e.g. a river target + fed only by SWOT Hydrocron today), this is a no-op. + + Sources that never overlap in time with anything else cannot be + calibrated; their observations are dropped from the corrected + output, with a warning naming them so you can decide whether to + anchor one manually (e.g. against an in-situ gauge) instead. + + Parameters + ---------- + platform_key, orbit_key : str, optional + Defaults to self.platform_key / self.orbit_key. orbit_key is + ignored entirely when group_by="platform". + group_by : {"platform_orbit", "platform"}, optional + See above. Default "platform_orbit" (unchanged default + behavior); pass "platform" for Sentinel-3/6 given the finding + above. + time_bin : str, optional + "D"-style window used to line up observations in time across + sources before comparing them. Default "1D". + min_overlap : int, optional + Minimum overlapping time bins required to estimate a source's + bias against the running combined reference. Default 3. + priority : list[str], optional + Preferred anchor source id(s), formatted "{platform}_{orbit}" + (or just "{platform}" when group_by="platform"), if you trust + one source's calibration more than the others. Defaults to + whichever source has the most observations. + centroid_distance_warn_km : float, optional + If lat_key/lon_key are available, each source's centroid + (mean lat/lon of its own observations) is computed and compared + to the anchor's centroid -- purely as transparency metadata, + NOT a spatial correction (this method still only compares + sources temporally). A warning is logged if a source's centroid + is more than this many km from the anchor's, since the + estimated bias for that source may be partly real spatial + signal rather than pure calibration offset. Default 5.0. + + After calling, self.bias_correct_diagnostics holds a dict with + "anchor", "biases", "centroids", "distance_km_from_anchor", and + "unanchored", for inspection/export by the caller. + """ + platform_key = platform_key or self.platform_key + orbit_key = orbit_key or self.orbit_key + + df = self.df.copy() + + if group_by not in ("platform_orbit", "platform"): + raise ValueError("group_by must be 'platform_orbit' or 'platform'") + + if platform_key not in df.columns or df[platform_key].isna().all(): + logger.warning( + "bias_correct: platform_key ('%s') not usable on this " + "timeseries; skipping (no-op).", platform_key, + ) + return self + + if group_by == "platform_orbit": + if orbit_key not in df.columns or df[orbit_key].isna().all(): + logger.warning( + "bias_correct: orbit_key ('%s') not usable on this " + "timeseries; skipping (no-op). Pass group_by='platform' " + "if you want platform-level grouping instead.", + orbit_key, + ) + return self + source_id = df[platform_key].astype(str) + "_" + df[orbit_key].astype(str) + else: + source_id = df[platform_key].astype(str) + + if source_id.nunique() < 2: + logger.info( + "bias_correct: only one source present (%s); nothing to " + "harmonize.", source_id.iloc[0] if len(source_id) else "n/a" + ) + return self + + df = df.assign(_source=source_id) + dates = pd.to_datetime(df[self.date_key]) + + # Centroid tracking: NOT a spatial correction -- bias_correct still + # only compares sources temporally, exactly as before. This purely + # records where each source's observations are actually located, so + # a caller can tell whether an estimated "bias" might be partly real + # spatial signal (e.g. wind setup, inflow gradient) rather than pure + # calibration offset, for a large/elongated reservoir where + # different missions cross at genuinely different locations. + lat_key = getattr(self, "lat_key", None) + lon_key = getattr(self, "lon_key", None) + have_coords = bool( + lat_key and lon_key and lat_key in df.columns and lon_key in df.columns + ) + centroids = {} + if have_coords: + for src, g in df.groupby("_source"): + valid = g[[lat_key, lon_key]].dropna() + if len(valid): + centroids[src] = ( + float(valid[lat_key].mean()), float(valid[lon_key].mean()) + ) + + n_days = _parse_bin_days(time_bin) + epoch = pd.Timestamp("1970-01-01") + bin_id = ((dates - epoch).dt.days // n_days) * n_days + df = df.assign(_bin=epoch + pd.to_timedelta(bin_id, unit="D")) + + frames = {} + for src, g in df.groupby("_source"): + binned = g.groupby("_bin")[self.height_key].median().to_frame("height") + binned["n_points"] = g.groupby("_bin")[self.height_key].count() + frames[src] = binned - def merge(self, save_progress=False, dir=".\\merged_progress"): + # Anchor = source with the most non-empty time bins, a close proxy + # for "most individual observation dates" given date-based binning. + anchor = next((s for s in (priority or []) if s in frames), None) + if anchor is None: + anchor = max(frames, key=lambda s: len(frames[s])) + + harmonized = {anchor: frames[anchor]} + biases = {anchor: 0.0} + remaining = {s: f for s, f in frames.items() if s != anchor} + + def combined_reference(): + allh = pd.concat( + {s: f["height"] for s, f in harmonized.items()}, axis=1, sort=False + ) + w = pd.concat( + {s: f["n_points"] for s, f in harmonized.items()}, axis=1, sort=False + ) + w = w.where(allh.notna()) + return (allh * w).sum(axis=1) / w.sum(axis=1) + + changed = True + while remaining and changed: + changed = False + ref = combined_reference() + for src in list(remaining): + b = remaining[src] + common = ref.index.intersection(b.index) + common = common[ref.loc[common].notna() & b.loc[common, "height"].notna()] + if len(common) >= min_overlap: + bias = float((b.loc[common, "height"] - ref.loc[common]).median()) + biases[src] = bias + harmonized[src] = b + del remaining[src] + changed = True + + if remaining: + logger.warning( + "bias_correct: sources %s never overlapped in time with " + "anything else and could not be calibrated; their " + "observations are dropped from the corrected output.", + list(remaining.keys()), + ) + + keep_mask = df["_source"].isin(biases.keys()) + bias_values = df["_source"].map(biases) + + corrected = self.df.loc[keep_mask.values].copy() + corrected[self.height_key] = ( + corrected[self.height_key].values - bias_values[keep_mask].values + ) + + distance_km_from_anchor = {} + if have_coords and anchor in centroids: + anchor_lat, anchor_lon = centroids[anchor] + for src in biases: + if src in centroids: + src_lat, src_lon = centroids[src] + distance_km_from_anchor[src] = float( + fltrs._haversine_km(anchor_lat, anchor_lon, src_lat, src_lon) + ) + far = {s: d for s, d in distance_km_from_anchor.items() + if s != anchor and d > centroid_distance_warn_km} + if far: + logger.warning( + "bias_correct: source(s) %s are >%.0f km from the anchor " + "source's centroid (anchor='%s'). The estimated bias for " + "these may be partly real spatial signal (e.g. wind " + "setup, inflow gradient) rather than pure calibration " + "offset -- treat with caution for large/elongated " + "targets. Distances (km): %s", + list(far.keys()), centroid_distance_warn_km, anchor, far, + ) + + logger.info("bias_correct: estimated biases (m): %s", biases) + if centroids: + logger.info("bias_correct: source centroids (lat, lon): %s", centroids) + + self.bias_correct_diagnostics = { + "anchor": anchor, + "biases": biases, + "centroids": centroids, + "distance_km_from_anchor": distance_km_from_anchor, + "unanchored": list(remaining.keys()), + } + + self.df = corrected.reset_index(drop=True) + return self + + def merge( + self, + save_progress=False, + dir=".\\merged_progress", + window_km=None, + max_theilsen_points=60, + spatial_correction_model=None, + bias_time_bin="1D", + bias_min_overlap=3, + bias_priority=None, + bias_group_by="platform_orbit", + bias_centroid_warn_km=5.0, + ref_lat=None, + ref_lon=None, + distance_penalty_scale_per_km=None, + svr_linear_err=0.1, + svr_linear_epsilon=0.1, + svr_linear_max_iter=5000, + svr_radial_max_iter=-1, + svr_radial_err=1.0, + svr_radial_rbf_c=10000, + svr_radial_gamma=0.0000438, + svr_radial_epsilon=0.1, + ): + """ + Run the full cleaning/merging pipeline: along-track outlier + rejection, multi-source bias correction, ADM error estimation, + Kalman filtering, and a final RBF outlier pass. + + Parameters worth tuning per target type (e.g. river vs. reservoir, + typically passed in from flow-level code): + window_km : along-track half-width (km) for the windowed ADM. + None keeps the original day-grouped-median ADM. Rivers with + a real slope benefit from a real window (e.g. ~0.5-1.5 km); + small/round lakes may not need one at all. + svr_radial_err : final RBF confidence band (m). DAHITI-style + defaults are ~1.0 for rivers, ~0.1 for lakes/reservoirs. + bias_time_bin : widen this (e.g. "10D"-"15D") if your sources + have sparse, non-coincident revisit patterns (e.g. ICESat-2 + vs. a 27-day Sentinel repeat) -- at "1D" they may never be + seen as overlapping and get dropped by bias_correct. + bias_group_by : "platform_orbit" groups sources by (platform, + orbit) -- only meaningful if orbit_key is a genuinely + stable, repeating ground-track identifier. If it isn't + (e.g. confirmed for Sentinel-3/6: the available field + increments roughly once per repeat cycle rather than + staying constant), use "platform" instead, which groups by + platform alone and is what actually resolves an inter- + mission bias rather than fragmenting each platform into + many single-crossing "sources" that can't be calibrated. + bias_centroid_warn_km : logs a warning (and records distances + in self.bias_correct_diagnostics) when a source's + observations are centered more than this far from the + anchor source's -- NOT a spatial correction, just a flag + that the estimated bias for that source may be partly real + spatial signal rather than pure calibration offset, worth a + closer look for large/elongated targets. Default 5.0 km. + ref_lat, ref_lon, distance_penalty_scale_per_km : if all three + are given, applies apply_distance_penalty after + daily_mad_error -- inflates each observation's error based + on distance from (ref_lat, ref_lon), e.g. the reservoir + polygon's own centroid, so Kalman naturally down-weights + crossings far from the main reservoir body (e.g. far + upstream, subject to real slope bias that local ADM alone + cannot detect since it only measures local scatter, not + representativeness). None (default) skips this step + entirely -- opt-in, not a silent behavior change. + spatial_correction_model : if given (see + basic_filters.fit_spatial_correction_model), applies + apply_spatial_correction right after svr_linear -- a + genuine height correction (not just error inflation) using + a spatial deviation model fit from a dense source (e.g. + ICESat-2), letting sparse missions benefit from it too. + Applied early, before bias_correct, so bias_correct + estimates pure platform calibration offset rather than + having spatial sampling differences between missions + contaminate the bias estimate. None (default) skips this + step entirely. See flows.py for the fit-once/persist-to- + disk wrapper -- refitting this model every run would make + past corrections shift retroactively as new data arrives. + See basic_filters.svr_linear/daily_mad_error/svr_radial for the + rest of these parameters. + + IMPORTANT: svr_linear_max_iter (for svr_linear) and svr_radial_max_iter + (for svr_radial) are deliberately SEPARATE parameters, not shared. + Confirmed empirically that capping svr_radial's RBF+high-C solver + produces non-monotonic, unreliable output quality (e.g. max_iter= + 5000 kept 10/5760 points on real data; only unbounded gave the + correct 2154/5760) -- svr_radial_max_iter must stay -1 (unbounded) + by default. Do not consolidate these into one shared parameter. + """ # make a folder for saving steps of the timeseries cleaning process general.ifnotmakedirs(dir) + # spatial_correction.csv/distance_penalty.csv are only written + # below if that feature is actually active for THIS run (both + # are opt-in, off by default) -- but ifnotmakedirs only creates + # the directory if it's missing, it never clears existing + # content. Without this, a run where the feature is OFF would + # simply skip writing a new file, leaving a stale one from + # whenever it was last ON sitting here looking like current + # output -- confirmed as a real, reported point of confusion. + for _stale_candidate in ("spatial_correction.csv", "distance_penalty.csv"): + _stale_path = os.path.join(dir, _stale_candidate) + if os.path.exists(_stale_path): + os.remove(_stale_path) + # run the SVR linear outlier filtering - self = fltrs.svr_linear(self) + self = fltrs.svr_linear(self, err=svr_linear_err, epsilon=svr_linear_epsilon, max_iter=svr_linear_max_iter) if save_progress: self.export_csv(os.path.join(dir, "svr_linear.csv")) + # optional: correct heights using a spatial deviation model (e.g. + # fit from dense ICESat-2 coverage) BEFORE bias_correct, so the + # bias estimate reflects pure platform calibration offset rather + # than being contaminated by different missions sampling + # spatially different parts of the reservoir + if spatial_correction_model is not None: + self = fltrs.apply_spatial_correction(self, spatial_correction_model) + if save_progress: + self.export_csv(os.path.join(dir, "spatial_correction.csv")) + + # cross-calibrate multiple sources (missions/tracks) onto a common + # datum, if more than one is present; a no-op otherwise (e.g. a + # river target fed only by SWOT Hydrocron today) + self = self.bias_correct( + time_bin=bias_time_bin, min_overlap=bias_min_overlap, + priority=bias_priority, group_by=bias_group_by, + centroid_distance_warn_km=bias_centroid_warn_km, + ) + if save_progress: + self.export_csv(os.path.join(dir, "bias_correct.csv")) + # run the ADM running filter - self = fltrs.daily_mad_error(self) + self = fltrs.daily_mad_error( + self, window_km=window_km, max_theilsen_points=max_theilsen_points + ) if save_progress: self.export_csv(os.path.join(dir, "daily_mad_error.csv")) + # optional: inflate error by distance from a reference location + # (e.g. reservoir centroid), so Kalman down-weights crossings that + # may be hydraulically unrepresentative (e.g. far upstream) even if + # ADM alone would treat them as highly precise + if ref_lat is not None and ref_lon is not None and distance_penalty_scale_per_km is not None: + self = fltrs.apply_distance_penalty( + self, ref_lat=ref_lat, ref_lon=ref_lon, + scale_per_km=distance_penalty_scale_per_km, + ) + if save_progress: + self.export_csv(os.path.join(dir, "distance_penalty.csv")) + # run Kalman filter df_kalman = fltrs.kalman(self) - self = Timeseries(df_kalman, date_key=self.date_key, height_key=self.height_key) + self = Timeseries( + df_kalman, + date_key=self.date_key, + height_key=self.height_key, + error_key=self.error_key, + lat_key=self.lat_key, + lon_key=self.lon_key, + pass_key=self.pass_key, + platform_key=self.platform_key, + orbit_key=self.orbit_key, + preset_error_key=self.preset_error_key, + ) if save_progress: self.export_csv(os.path.join(dir, "kalman.csv")) # a radial base svr to get the final timeseries - df_rbf = fltrs.svr_radial(self) - self = Timeseries(df_rbf, date_key=self.date_key, height_key=self.height_key) + df_rbf = fltrs.svr_radial( + self, + err=svr_radial_err, + rbf_c=svr_radial_rbf_c, + gamma=svr_radial_gamma, + epsilon=svr_radial_epsilon, + max_iter=svr_radial_max_iter, + ) + self = Timeseries( + df_rbf, + date_key=self.date_key, + height_key=self.height_key, + error_key=self.error_key, + lat_key=self.lat_key, + lon_key=self.lon_key, + pass_key=self.pass_key, + platform_key=self.platform_key, + orbit_key=self.orbit_key, + preset_error_key=self.preset_error_key, + ) if save_progress: self.export_csv(os.path.join(dir, "svr_radial.csv")) @@ -111,29 +531,77 @@ def export_csv(self, path): self.df.to_csv(path) -def concat(timeseries: list = [], main_date_key="date", main_height_key="height"): +def concat( + timeseries: list = [], + main_date_key="date", + main_height_key="height", + main_lat_key="lat", + main_lon_key="lon", + main_pass_key="pass", + main_platform_key="platform", + main_orbit_key="orbit", + main_preset_error_key="preset_error", +): + """ + Concatenate several Timeseries objects (typically one per mission/product) + into a single Timeseries on a common set of column names. + + Different missions may use different underlying column names for the same + conceptual field (e.g. Sentinel-6's "file_name" vs. ICESat-2's lack of an + equivalent pass identifier) -- each source's key attributes (ts.pass_key, + ts.platform_key, etc.) are read and renamed onto the common main_*_key + names here, rather than assuming a fixed literal column name. A column is + only carried through for sources that actually have it; sources missing + a given key simply get NaN for that column after concatenation. + + This includes preset_error_key: a mission that already supplies its own + formal per-observation uncertainty (e.g. SWOT's wse_u) needs that column + preserved through concatenation, or it silently disappears the moment + it's combined with missions that don't have one -- exactly the same + class of bug as lat/lon/pass being dropped without this handling. + """ + optional_keys = { + main_lat_key: "lat_key", + main_lon_key: "lon_key", + main_pass_key: "pass_key", + main_platform_key: "platform_key", + main_orbit_key: "orbit_key", + main_preset_error_key: "preset_error_key", + } + df_list = [] # create a single timeseries object from the multiple timeseries for ts in timeseries: + rename_map = {ts.date_key: main_date_key, ts.height_key: main_height_key} keep = [ts.date_key, ts.height_key] - if "orbit" in ts.df.columns: - keep = keep + ["orbit"] - if "platform" in ts.df.columns: - keep = keep + ["platform"] - - df = ts.df[keep] - df = df.rename( - columns={ts.date_key: main_date_key, ts.height_key: main_height_key} - ) + for main_name, attr in optional_keys.items(): + src_col = getattr(ts, attr, None) + if src_col and src_col in ts.df.columns: + keep.append(src_col) + rename_map[src_col] = main_name + + df = ts.df[keep].rename(columns=rename_map) df_list.append(df) # concatenate dfs - df = pd.concat(df_list) + df = pd.concat(df_list, ignore_index=True) # turn this combined df into a timeseries object and clean - ts = Timeseries(df, date_key=main_date_key, height_key=main_height_key) + ts = Timeseries( + df, + date_key=main_date_key, + height_key=main_height_key, + lat_key=main_lat_key, + lon_key=main_lon_key, + pass_key=main_pass_key, + platform_key=main_platform_key, + orbit_key=main_orbit_key, + preset_error_key=( + main_preset_error_key if main_preset_error_key in df.columns else None + ), + ) # return the merged timeseries - return ts + return ts \ No newline at end of file From 9bb8c2c9f1f283edcea9fcbf32de92eb6b71fe9f Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Tue, 14 Jul 2026 10:46:48 +0200 Subject: [PATCH 03/19] Update rivers.yaml Update to new functionalities for rivers --- configs/rivers.yaml | 78 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/configs/rivers.yaml b/configs/rivers.yaml index 2d93cc2..f3cbbb1 100644 --- a/configs/rivers.yaml +++ b/configs/rivers.yaml @@ -16,9 +16,9 @@ # ─── PROJECT ────────────────────────────────────────────────────────────────── project: - main_dir: "/path/to/your/project" - startdate: [2024, 1, 1] - enddate: [2024, 12, 31] + main_dir: "" + startdate: [2025, 1, 1] + enddate: [2025, 12, 31] gis: global_crs: 'EPSG:4326' @@ -29,7 +29,7 @@ gis: sword_db: # raw_sword_path: "/path/to/SWORD_v17b_gpkg.zip" # reuse existing zip or folder; skip download # sword_subset_path: "/path/to/SWORD_subset.gpkg" # pre-made subset; skip download + subsetting - keep_raw_sword: false # true = keep raw SWORD zip after extraction; false = delete + keep_raw_sword: true # true = keep raw SWORD zip after extraction; false = delete # ─── RIVERS ─────────────────────────────────────────────────────────────────── @@ -37,10 +37,10 @@ rivers: enabled: true # Option A — AOI file (recommended): subsets SWORD river database to your area. - aoi_path: "/path/to/your/river_aoi.gpkg" # .shp or .gpkg - continent_key: 'eu' # SWORD continent code: af | as | eu | na | oc | sa + aoi_path: "" # .shp or .gpkg + continent_key: 'af' # SWORD continent code: af | as | eu | na | oc | sa feature_type: 'reaches' # nodes | reaches - id_key: 'river_id' # AOI column used to name per-river output folders + id_key: 'id' # AOI column used to name per-river output folders # buffer_meters: 1000.0 # optional AOI buffer (metres) before SWORD subsetting # Option B — explicit IDs (comment out aoi_path + continent_key + id_key above). @@ -49,6 +49,61 @@ rivers: # feature_type: 'reaches' # id: 'my_river' # used as the output folder name + # ── ICESat-2/Sentinel-3/6 extraction (only relevant if those missions are + # enabled below) ───────────────────────────────────────────────────────── + # extraction_buffer_meters: 750.0 # corridor width around SWORD targets for + # # raw altimetry extraction -- SEPARATE + # # from buffer_meters above (that one only + # # decides which targets are "in scope"; + # # this one decides how far from the + # # centerline a raw point could plausibly + # # be real river water). Falls back to + # # buffer_meters, then 500m, if omitted. + # max_node_assignment_meters: 300.0 # max distance (m) for assigning a raw + # # point to its nearest node/reach. Falls + # # back to extraction_buffer_meters if + # # omitted. + # overwrite_extraction: false # true forces re-extraction even if a + # # target's output .gpkg already exists + + # merging_options: + # # See flows.DEFAULT_RIVER_MERGING_OPTIONS for every available key. + # # NOTE: these are currently a direct copy of the reservoir-tuned + # # defaults and have NOT been independently validated against real + # # river data -- check kept/rejected observation counts on your own + # # rivers before trusting them, the same way the reservoir defaults + # # were validated this session. + # svr_radial_err: 1.0 + # svr_radial_gamma: 0.00438 + + +# ─── SWOT (REQUIRED — see quick-start note above) ──────────────────────────── +# Rivers require SWOT Hydrocron for at least node/reach WSE. This section +# must be enabled (download: true, process: true) or nothing will download, +# even with a valid aoi_path/SWORD setup above. +swot: + download: true + process: true + + +# ─── ICESat-2 / Sentinel-3 / Sentinel-6 (OPTIONAL) ──────────────────────────── +# Additional missions, clustered onto the same SWORD targets as SWOT (see +# extraction_buffer_meters/max_node_assignment_meters above). Uncomment and +# enable any of these to add them; SWOT alone is sufficient on its own. +# ────────────────────────────────────────────────────────────────────────────── +icesat2: + download: false + process: false + +sentinel3: + download: false + process: false + +sentinel6: + download: true + process: true + source: "earthdata" + # ─── ADVANCED OPTIONS ───────────────────────────────────────────────────────── # Hydrocron field lists and quality filters for SWOT river downloads. @@ -67,3 +122,12 @@ rivers: # quality_filters: # nodes: {max_q: 2} # reaches: {max_q: 2} +# sentinel3: +# sigma0_min: 0.0 # water-only filter for river extraction -- a buffered +# # river corridor is much looser than a reservoir +# # polygon (genuinely includes riverbank/vegetation), +# # and unlike ICESat-2, Sentinel has no built-in water +# # classification. 0.0 is a safe no-op default -- tune +# # against real data before trusting a nonzero value. +# sentinel6: +# sigma0_min: 0.0 From cf3788d42b4751d136b3a33c659ef3fca3a187e9 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Tue, 14 Jul 2026 10:51:22 +0200 Subject: [PATCH 04/19] Update rivers.yaml Update rivers config file --- configs/rivers.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/configs/rivers.yaml b/configs/rivers.yaml index f3cbbb1..11bad2e 100644 --- a/configs/rivers.yaml +++ b/configs/rivers.yaml @@ -24,6 +24,29 @@ gis: global_crs: 'EPSG:4326' +# ─── CREDENTIALS ────────────────────────────────────────────────────────────── +# Tip: set credentials as environment variables to avoid storing secrets in files. +# +# earthaccess : EARTHDATA_USERNAME / EARTHDATA_PASSWORD +# hydroweb : EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY +# creodias : CREODIAS_USERNAME / CREODIAS_PASSWORD + +earthaccess: # NASA Earthdata — required for SWOT download + username: '' # or set EARTHDATA_USERNAME + password: '' # or set EARTHDATA_PASSWORD + token: '' + +hydroweb: # HydroWeb API — required for Prior Lake Database (PLD) matching + api_key: "" # or set EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY + raw_pld_path: "" # optional: reuse an existing PLD zip or folder + keep_raw_pld: true # true = keep raw PLD zip after subset; false = delete + +creodias: # CREODIAS/CDSE — required for Sentinel-3 and Sentinel-6 + username: "" # or set CREODIAS_USERNAME + password: "" # or set CREODIAS_PASSWORD + + + # ─── SWORD DATABASE ─────────────────────────────────────────────────────────── # Optional shortcuts to skip download or subsetting steps. sword_db: From bc0eded644fb66c3d3100454712cff9442b2c5f0 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Tue, 14 Jul 2026 11:04:32 +0200 Subject: [PATCH 05/19] Update test suite and split flows.py from 3000+lines of code to individual files --- HydroEO/flows.py | 3114 --------------------- HydroEO/flows/__init__.py | 134 + HydroEO/flows/_clean_engine.py | 86 + HydroEO/flows/_constants.py | 181 ++ HydroEO/flows/_merge_engine.py | 191 ++ HydroEO/flows/_reservoir_download.py | 183 ++ HydroEO/flows/_reservoir_init.py | 140 + HydroEO/flows/_reservoir_pipeline.py | 395 +++ HydroEO/flows/_river_common.py | 118 + HydroEO/flows/_river_download.py | 426 +++ HydroEO/flows/_river_init.py | 214 ++ HydroEO/flows/_river_pipeline.py | 421 +++ HydroEO/flows/_run_config.py | 616 ++++ HydroEO/flows/_sentinel_shared.py | 118 + HydroEO/flows/_summaries.py | 341 +++ HydroEO/project.py | 131 +- HydroEO/satellites/sentinel/__init__.py | 6 +- HydroEO/satellites/sentinel/preprocess.py | 50 +- HydroEO/satellites/swot/__init__.py | 4 + HydroEO/satellites/swot/preprocess.py | 78 +- HydroEO/utils/filters/basic_filters.py | 1 + HydroEO/utils/general.py | 92 + tests/unit/test_flows.py | 186 +- tests/unit/test_project_config.py | 53 +- tests/unit/test_timeseries.py | 438 ++- 25 files changed, 4501 insertions(+), 3216 deletions(-) delete mode 100644 HydroEO/flows.py create mode 100644 HydroEO/flows/__init__.py create mode 100644 HydroEO/flows/_clean_engine.py create mode 100644 HydroEO/flows/_constants.py create mode 100644 HydroEO/flows/_merge_engine.py create mode 100644 HydroEO/flows/_reservoir_download.py create mode 100644 HydroEO/flows/_reservoir_init.py create mode 100644 HydroEO/flows/_reservoir_pipeline.py create mode 100644 HydroEO/flows/_river_common.py create mode 100644 HydroEO/flows/_river_download.py create mode 100644 HydroEO/flows/_river_init.py create mode 100644 HydroEO/flows/_river_pipeline.py create mode 100644 HydroEO/flows/_run_config.py create mode 100644 HydroEO/flows/_sentinel_shared.py create mode 100644 HydroEO/flows/_summaries.py diff --git a/HydroEO/flows.py b/HydroEO/flows.py deleted file mode 100644 index 071b7ec..0000000 --- a/HydroEO/flows.py +++ /dev/null @@ -1,3114 +0,0 @@ -"""Standalone flow functions for HydroEO pipelines. - -These functions implement the core download, processing, and visualization logic -previously embedded in Reservoirs and Rivers classes. They operate on Project state -and external data, with no direct method dependencies. -""" - -import logging -import os -import datetime -import json -import yaml -from io import StringIO -from typing import TYPE_CHECKING - -import geopandas as gpd -import mikeio -import pandas as pd -from tqdm import tqdm - -from HydroEO.satellites import swot, icesat2, sentinel -from HydroEO.utils import general, timeseries -from HydroEO.utils.filters import basic_filters -from HydroEO.downloaders import hydroweb -from HydroEO import plotting - -if TYPE_CHECKING: - from HydroEO.project import Project - -logger = logging.getLogger(__name__) - - -# ============================================================================ -# RESERVOIRS: Initialization -# ============================================================================ - - -def initialize_reservoirs(prj: "Project") -> None: - """Initialize PLD matching for reservoirs mode. - - Downloads PLD database, matches it to input reservoirs, and stores - prior_lake_id values on prj.reservoirs.gdf. - - Parameters - ---------- - prj : Project - Project instance with reservoirs config/state populated - """ - if not hasattr(prj, "reservoirs"): - return - - if "swot" not in prj.to_download and "swot" not in prj.to_process: - return - - # Download PLD if needed - _download_pld(prj) - - # Match reservoirs to PLD - _assign_pld_id(prj) - - # Export flags for missing priors - _flag_missing_priors(prj) - - # Set download geometry (for reservoirs, same as input boundaries) - prj.reservoirs.download_gdf = prj.reservoirs.gdf - - -def _download_pld(prj: "Project") -> None: - """Download PLD database to project directory.""" - pld_path = prj.dirs["pld"] - - if os.path.exists(pld_path): - logger.info("PLD located") - return - - logger.info("Downloading PLD") - download_dir = os.path.dirname(pld_path) - bounds = list(prj.reservoirs.gdf.unary_union.bounds) - raw_pld_path = prj.dirs.get("pld_raw") - - # Determine if raw_pld_path is inside project main_dir - keep_raw = getattr(prj, "keep_raw_pld", False) - effective_keep_raw = keep_raw - if raw_pld_path is not None and os.path.exists(raw_pld_path): - if not os.path.abspath(raw_pld_path).startswith( - os.path.abspath(prj.dirs["main"]) - ): - logger.warning( - "raw_pld_path '%s' is outside project folder '%s'. " - "Skipping deletion of raw PLD files to preserve external data.", - raw_pld_path, - prj.dirs["main"], - ) - effective_keep_raw = True - - hydroweb.download_PLD( - download_dir=download_dir, - bounds=bounds, - raw_pld_path=raw_pld_path, - keep_raw=effective_keep_raw, - ) - - -def _assign_pld_id(prj: "Project") -> None: - """Spatial join reservoirs with PLD to assign prior_lake_id.""" - pld = gpd.read_file(prj.dirs["pld"]) - - pld = pld.rename( - columns={"lake_id": "prior_lake_id", "res_id": "prior_res_id"} - ) - joined_gdf = gpd.sjoin_nearest( - prj.reservoirs.gdf.to_crs(prj.local_crs), - pld.to_crs(prj.local_crs), - how="left", - max_distance=prj.mission_options.get("swot", {}).get( - "pld_match_max_distance_m", 100 - ), - distance_col="dist_to_pld", - ) - joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) - - joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 - - prj.reservoirs.gdf = joined_gdf - - -def _flag_missing_priors(prj: "Project") -> None: - """Export geopackages of reservoirs present/missing in PLD to aux/PLD folder.""" - gdf = prj.reservoirs.gdf - present = gdf.loc[gdf.prior_lake_id > 0].reset_index(drop=True) - missing = gdf.loc[gdf.prior_lake_id < 0].reset_index(drop=True) - - # Output to aux/PLD/ folder - pld_dir = os.path.dirname(prj.dirs["pld"]) - present_path = os.path.join(pld_dir, "present_in_pld.gpkg") - missing_path = os.path.join(pld_dir, "missing_in_pld.gpkg") - - present.to_file(present_path, driver="GPKG") - missing.to_file(missing_path, driver="GPKG") - - logger.info( - "Out of the %s reservoirs, %s are present and %s are missing from the PLD.", - len(gdf), - len(present), - len(missing), - ) - - -# ============================================================================ -# RIVERS: Initialization -# ============================================================================ - - -def initialize_rivers(prj: "Project") -> None: - """Initialize SWORD target IDs for rivers mode. - - Parameters - ---------- - prj : Project - Project instance with rivers config/state populated - """ - if not hasattr(prj, "rivers"): - return - - if prj.rivers.input_mode == "aoi_path": - _prepare_rivers_from_sword(prj) - - id_label = "node" if prj.rivers.target_id_col == "node_id" else "reach" - logger.info( - "Found river %s %s ids", - len(prj.rivers.target_ids), - id_label, - ) - logger.debug( - "Found river %s ids: %s", - id_label, - ", ".join(str(target_id) for target_id in prj.rivers.target_ids), - ) - - -def _prepare_rivers_from_sword(prj: "Project") -> None: - """Prepare SWORD target features by spatial intersection with AOI or from saved subset. - - If SWORD_subset.gpkg exists, reads from it directly (skips download and spatial operations). - Otherwise, ensures SWORD database, performs spatial intersection with AOI, saves subset. - """ - subset_path = prj.dirs.get("sword_subset") - - # Gate 1: Check if subset already exists - if subset_path and os.path.exists(subset_path): - logger.info("SWORD subset located at %s", subset_path) - subset = gpd.read_file(subset_path) - else: - # Gate 2: Ensure SWORD database and perform spatial intersection - _ensure_sword_database(prj) - - gpkg_name = ( - f"{prj.rivers.continent_key}_sword_{prj.rivers.feature_type}_v17b.gpkg" - ) - gpkg_path = os.path.join(prj.dirs["sword"], gpkg_name) - - if not os.path.exists(gpkg_path): - raise FileNotFoundError(f"Expected SWORD file not found: {gpkg_path}") - - sword_gdf = gpd.read_file(gpkg_path) - - # Buffer AOI if requested - aoi_local = prj.rivers.aoi_gdf.to_crs(prj.local_crs).copy() - if prj.rivers.buffer_meters and prj.rivers.buffer_meters > 0: - aoi_local["geometry"] = aoi_local.geometry.buffer(prj.rivers.buffer_meters) - - # Intersect with SWORD - sword_local = sword_gdf.to_crs(prj.local_crs) - subset = sword_local.loc[sword_local.intersects(aoi_local.unary_union)].copy() - - if prj.rivers.id_key not in prj.rivers.aoi_gdf.columns: - raise KeyError( - f"Expected AOI column '{prj.rivers.id_key}' missing from river input file" - ) - - aoi_join = aoi_local[[prj.rivers.id_key, "geometry"]].copy() - subset = gpd.sjoin( - subset, - aoi_join, - how="inner", - predicate="intersects", - ).drop(columns=["index_right"], errors="ignore") - subset = subset.drop_duplicates().to_crs(prj.rivers.aoi_gdf.crs) - - # Save subset to disk - if subset_path: - general.ifnotmakedirs(os.path.dirname(subset_path)) - subset.to_file(subset_path, driver="GPKG") - logger.info("SWORD subset saved to %s", subset_path) - - # Cleanup: delete gpkg folder if keep_raw_sword is False - if not prj.keep_raw_sword: - try: - import shutil - - sword_dir = prj.dirs.get("sword") - if sword_dir and os.path.isdir(sword_dir): - shutil.rmtree(sword_dir) - logger.info("Deleted SWORD gpkg folder (keep_raw_sword=False)") - except Exception as e: - logger.warning("Failed to delete SWORD gpkg folder: %s", e) - else: - logger.info("Kept SWORD gpkg folder (keep_raw_sword=True)") - - # Extract target IDs from subset - source_id_col = "node_id" if prj.rivers.feature_type == "nodes" else "reach_id" - if source_id_col not in subset.columns: - raise KeyError(f"Expected SWORD column '{source_id_col}' missing from subset") - - prj.rivers.target_features = subset - prj.rivers.target_id_col = source_id_col - prj.rivers.target_ids = [int(value) for value in subset[source_id_col]] - - -def _ensure_sword_database(prj: "Project") -> None: - """Ensure SWORD database is available locally. - - Checks if full SWORD database (GPKGs) already exists in prj.dirs["sword"]. - If not, handles three scenarios: - 1. User-provided zip: extract to {main_dir}/aux/SWORD/ - 2. User-provided directory: use it directly - 3. Auto-download from Zenodo: download and extract to {main_dir}/aux/SWORD/ - - Respects keep_raw_sword config to optionally delete downloaded zip. - """ - from urllib import request as url_request - import zipfile - - SWORD_V17B_ZIP_URL = ( - "https://zenodo.org/records/15299138/files/SWORD_v17b_gpkg.zip?download=1" - ) - - sword_dir = prj.dirs["sword"] - - # Check if SWORD database already exists - if os.path.isdir(sword_dir): - # Check if directory contains any SWORD GPKGs - gpkg_files = [f for f in os.listdir(sword_dir) if f.endswith("_v17b.gpkg")] - if gpkg_files: - logger.info("SWORD database located at %s", sword_dir) - return - - logger.info("SWORD database not found. Preparing it now.") - - # User-provided raw_sword_path - if "sword_raw" in prj.dirs: - raw_path = prj.dirs["sword_raw"] - - # Case 1: User provided a zip file - if raw_path.lower().endswith(".zip") and os.path.isfile(raw_path): - logger.info("Using user-provided SWORD zip: %s", raw_path) - general.ifnotmakedirs(os.path.dirname(sword_dir)) - with zipfile.ZipFile(raw_path, "r") as zip_ref: - zip_ref.extractall(os.path.dirname(sword_dir)) - logger.info("SWORD extracted to %s", sword_dir) - return - - # Case 2: User provided a directory - elif os.path.isdir(raw_path): - logger.info("Using user-provided SWORD directory: %s", raw_path) - # Check if GPKGs are in raw_path/gpkg/ or directly in raw_path - gpkg_subdir = os.path.join(raw_path, "gpkg") - if os.path.isdir(gpkg_subdir): - prj.dirs["sword"] = gpkg_subdir - logger.info("SWORD database found in %s", gpkg_subdir) - else: - prj.dirs["sword"] = raw_path - logger.info("SWORD database found in %s", raw_path) - return - - # Case 3: Auto-download from Zenodo - logger.info("Downloading SWORD v17b from Zenodo...") - general.ifnotmakedirs(os.path.dirname(sword_dir)) - - zip_path = os.path.join(os.path.dirname(sword_dir), "SWORD_v17b_gpkg.zip") - url_request.urlretrieve(SWORD_V17B_ZIP_URL, zip_path) - logger.info("Downloaded SWORD v17b to %s", zip_path) - - logger.info("Extracting SWORD v17b...") - with zipfile.ZipFile(zip_path, "r") as zip_ref: - zip_ref.extractall(os.path.dirname(sword_dir)) - - logger.info("SWORD extracted to %s", sword_dir) - - # Cleanup: delete zip if keep_raw_sword is False - if not prj.keep_raw_sword: - try: - os.remove(zip_path) - logger.info("Deleted raw SWORD zip file (keep_raw_sword=False)") - except Exception as e: - logger.warning("Failed to delete SWORD zip %s: %s", zip_path, e) - else: - logger.info("Kept raw SWORD zip file at %s (keep_raw_sword=True)", zip_path) - - -# ============================================================================ -# RESERVOIRS: Download -# ============================================================================ - - -def download_reservoirs(prj: "Project") -> None: - """Download altimetry data for all configured missions (reservoirs mode). - - Parameters - ---------- - prj : Project - Project instance with download configuration - """ - for mission in prj.to_download: - if mission == "swot": - _download_reservoirs_swot(prj) - elif mission == "icesat2": - _download_reservoirs_icesat2(prj) - elif mission in ["sentinel3", "sentinel6"]: - _download_reservoirs_sentinel(prj, mission) - else: - logger.warning("Skipping unsupported mission in download: %s", mission) - - -def _download_reservoirs_swot(prj: "Project") -> None: - """Download SWOT Lake SP data for reservoirs.""" - download_dir = prj.dirs["swot"] - general.ifnotmakedirs(download_dir) - - startdate = prj.startdates["swot"] - enddate = prj.enddates["swot"] - - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) - - coords = [ - (x, y) - for x, y in prj.reservoirs.download_gdf.unary_union.envelope.exterior.coords - ] - - logger.info( - "Searching for %s for aoi from %s to %s", - swot.SWOT_LAKE_SHORT_NAME, - startdate, - enddate, - ) - results = swot.query(aoi=coords, startdate=startdate, enddate=enddate) - - # Filter to prior-lake granules - logger.info("%s products returned from query", len(results)) - to_download = [] - for result in results: - link = result.data_links()[0] - filename = link.split("/")[-1].lower() - if "_prior_" in filename or "prior" in filename.split("_"): - to_download.append(result) - logger.info("%s prior-lake granules selected for download", len(to_download)) - - _ = swot.download(to_download, download_directory=download_dir) - - files_in_dir = [ - os.path.join(download_dir, f) - for f in os.listdir(download_dir) - if f.endswith(".zip") - ] - swot.subset_by_id( - files_in_dir, prj.reservoirs.download_gdf["prior_lake_id"].astype(int).values - ) - - -def _download_reservoirs_icesat2(prj: "Project") -> None: - """Download ICESat-2 ATL13 data for reservoirs.""" - for i in prj.reservoirs.download_gdf.index: - id = prj.reservoirs.download_gdf.loc[i, prj.reservoirs.id_key] - logger.info("Downloading data for id %s", id) - - geom = _simplify_to_one_polygon(prj.reservoirs.download_gdf.loc[i, "geometry"]) - coords = list(geom.exterior.coords) - - parquet_dir = os.path.join(prj.dirs["icesat2_processed"], rf"{id}") - general.ifnotmakedirs(parquet_dir) - - startdate = prj.startdates["icesat2"] - enddate = prj.enddates["icesat2"] - - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) - - logger.info( - "Searching for Icesat2 ATL13 for aoi from %s to %s", startdate, enddate - ) - try: - _ = icesat2.query( - aoi=coords, - startdate=startdate, - enddate=enddate, - download_directory=parquet_dir, - atl13_options=prj.mission_options.get("icesat2", {}).get("atl13", {}), - atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields") - or None, - ) - except Exception as exc: - logger.warning("ICESat-2 download skipped for %s: %s", id, exc) - - -def _sentinel6_use_earthdata(prj: "Project") -> bool: - """ - Whether Sentinel-6 should be downloaded from PO.DAAC/EarthData (HR - product, 20Hz Ku-band) rather than CREODIAS (LR product only, see - query()'s productType="P4_2__LR_____"). Set via - mission_options["sentinel6"]["source"] = "earthdata" in config. - """ - return ( - prj.mission_options.get("sentinel6", {}) - .get("source", "creodias") - .lower() - == "earthdata" - ) - - -def _download_sentinel_for_target( - prj: "Project", mission: str, product: str, coords, download_dir, - startdate, enddate, sentinel_creds, session_token, session_start_time, -) -> tuple: - """ - Download + subset Sentinel-3/6 data for one target's AOI (a - reservoir polygon, or a river waterbody corridor's envelope) -- - shared by both _download_reservoirs_sentinel and - _download_rivers_sentinel so the CREODIAS/EarthData branching logic - only needs to exist in one place. - - For Sentinel-6, if _sentinel6_use_earthdata(prj) is True, uses - PO.DAAC/EarthData (see sentinel.query_earthdata/download_earthdata) - to get the HR product instead of CREODIAS's LR-only product. - EarthData files arrive flat (no SAFE-zip directory), so the unzip - step is skipped for that path -- subset() already handles both flat - and zipped-folder inputs (see sentinel/preprocess.py's file - discovery, extended for this). - - Returns (session_token, session_start_time) -- unchanged from what - was passed in when using the EarthData path, since that mechanism - (CREODIAS session reuse) doesn't apply to it. - """ - dir_key = mission - use_earthdata = mission == "sentinel6" and _sentinel6_use_earthdata(prj) - - logger.info( - "Searching for Sentinel-%s (%s) from %s to %s", - product, - "PO.DAAC/EarthData HR" if use_earthdata else "CREODIAS", - startdate, - enddate, - ) - - if use_earthdata: - s6_opts = prj.mission_options.get("sentinel6", {}) - results = sentinel.query_earthdata( - aoi=coords, - startdate=startdate, - enddate=enddate, - latency=s6_opts.get("latency", "NTC"), - short_name=s6_opts.get("short_name"), - ) - sentinel.download_earthdata(results, download_directory=download_dir) - # EarthData granules arrive flat already -- no SAFE-zip to unzip. - else: - ids = sentinel.query( - aoi=coords, - startdate=startdate, - enddate=enddate, - product=product, - creodias_credentials=sentinel_creds, - ) - - session_token, session_start_time = sentinel.download( - ids, - download_directory=download_dir, - creodias_credentials=sentinel_creds, - token=session_token, - session_start_time=session_start_time, - threads=prj.mission_options.get(dir_key, {}).get("download_threads", 1), - ) - - general.unzip_dir_files_with_ext( - download_dir, download_dir, ".nc", show_progress=True - ) - - sentinel.subset( - aoi=coords, - download_dir=download_dir, - dest_dir=download_dir, - file_id=prj.mission_options.get(dir_key, {}).get( - "subset_file_id", "enhanced_measurement.nc" - ), - product=product, - show_progress=True, - ) - - return session_token, session_start_time - - -def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: - """Download Sentinel-3 or Sentinel-6 data for reservoirs.""" - product = "S3" if mission == "sentinel3" else "S6" - - session_token = None - session_start_time = None - - # EarthData (Sentinel-6 HR) needs no CREODIAS credentials at all -- - # only require them if we're actually going to use CREODIAS. But it - # does need its OWN upfront check -- without it, earthaccess.login() - # silently falls through to interactive prompting when nothing else - # is configured, which hangs in a non-interactive run instead of - # failing clearly (see Project._require_earthdata_credentials). - sentinel_creds = None - use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) - if use_earthdata_s6: - prj._require_earthdata_credentials() - else: - sentinel_creds = prj._require_creodias_credentials() - - for i in prj.reservoirs.download_gdf.index: - id = prj.reservoirs.download_gdf.loc[i, prj.reservoirs.id_key] - logger.info("Downloading data for id %s", id) - - coords = [ - (x, y) - for x, y in prj.reservoirs.download_gdf.loc[ - i, "geometry" - ].envelope.exterior.coords - ] - - download_dir = os.path.join(prj.dirs[mission], rf"{id}") - general.ifnotmakedirs(download_dir) - - startdate = prj.startdates[mission] - enddate = prj.enddates[mission] - - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) - - session_token, session_start_time = _download_sentinel_for_target( - prj, mission, product, coords, download_dir, - startdate, enddate, sentinel_creds, session_token, session_start_time, - ) - - -# ============================================================================ -# RIVERS: Download -# ============================================================================ - - -def download_rivers(prj: "Project") -> None: - """Download altimetry data for all configured missions (rivers mode). - - SWOT uses the Hydrocron timeseries API directly (per node/reach, no - clustering needed -- see _download_swot_hydrocron_timeseries). - ICESat-2/Sentinel-3/6 download raw observations over a buffered - corridor around each waterbody's SWORD targets (see - _river_target_corridor); associating individual points with a - specific target happens later, during extraction. - - Parameters - ---------- - prj : Project - Project instance with rivers configuration - """ - if not hasattr(prj, "rivers"): - return - - if "swot" not in prj.to_download and not any( - m in prj.to_download for m in ("icesat2", "sentinel3", "sentinel6") - ): - logger.warning( - "Rivers are configured but no mission is enabled for download. " - "Add a top-level mission section (e.g. 'swot:', 'icesat2:', " - "'sentinel3:'/'sentinel6:') with 'download: true' to actually " - "download river observations. SWORD itself will still have " - "been prepared by initialize(), which is why you may see " - "SWORD files but no timeseries data." - ) - return - - if "swot" in prj.to_download: - startdate = prj.startdates.get("swot") - enddate = prj.enddates.get("swot") - - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) - - _download_swot_hydrocron_timeseries(prj, startdate, enddate) - - if "icesat2" in prj.to_download: - _download_rivers_icesat2(prj) - - if "sentinel3" in prj.to_download: - _download_rivers_sentinel(prj, "sentinel3") - - if "sentinel6" in prj.to_download: - _download_rivers_sentinel(prj, "sentinel6") - - -def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> None: - """Download SWOT Hydrocron timeseries for river targets.""" - from urllib import parse, request as url_request - import json - - HYDROCRON_TIMESERIES_URL = ( - "https://soto.podaac.earthdatacloud.nasa.gov/hydrocron/v1/timeseries" - ) - - # Determine feature config - if prj.rivers.target_id_col == "node_id": - feature = "Node" - quality_column = "node_q" - fields = ( - prj.mission_options.get("swot", {}) - .get("hydrocron_fields", {}) - .get("nodes", []) - ) - max_q = ( - prj.mission_options.get("swot", {}) - .get("quality_filters", {}) - .get("nodes", {}) - .get("max_q", 2) - ) - else: - feature = "Reach" - quality_column = "reach_q" - fields = ( - prj.mission_options.get("swot", {}) - .get("hydrocron_fields", {}) - .get("reaches", []) - ) - max_q = ( - prj.mission_options.get("swot", {}) - .get("quality_filters", {}) - .get("reaches", {}) - .get("max_q", 2) - ) - - # Group targets by waterbody - waterbody_groups = _group_river_targets_by_waterbody(prj) - id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" - - summary = { - "requested": 0, - "successful": 0, - "failed": 0, - "empty_after_filter": 0, - } - - for wb_id, target_ids in waterbody_groups.items(): - summary["requested"] = summary["requested"] + len(target_ids) - output_path = os.path.join( - prj.dirs["swot"], str(wb_id), f"{id_label}_timeseries.csv" - ) - general.ifnotmakedirs(os.path.dirname(output_path)) - - wb_startdate = startdate - latest_obs = _get_latest_hydrocron_obs_date(output_path) - if latest_obs is not None: - wb_startdate = latest_obs - - deferred_warnings = [] - - def _defer_warning(message, *args): - if args: - deferred_warnings.append(message % args) - else: - deferred_warnings.append(message) - - frames = [] - for target_id in tqdm(target_ids, desc="Downloading hydrocron data"): - try: - query_params = { - "feature": feature, - "feature_id": str(target_id), - "start_time": wb_startdate.strftime("%Y-%m-%dT%H:%M:%SZ"), - "end_time": enddate.strftime("%Y-%m-%dT%H:%M:%SZ"), - "output": "csv", - "fields": ",".join(fields), - } - request_url = ( - f"{HYDROCRON_TIMESERIES_URL}?{parse.urlencode(query_params)}" - ) - - with url_request.urlopen(request_url) as response: - status_code = getattr(response, "status", response.getcode()) - payload = json.loads(response.read().decode("utf-8")) - except Exception as exc: - _defer_warning( - "Hydrocron request failed for %s %s: %s", - prj.rivers.target_id_col, - target_id, - exc, - ) - summary["failed"] += 1 - continue - - csv_payload = ( - payload.get("results", {}).get("csv") - if isinstance(payload, dict) - else None - ) - if status_code != 200 or not csv_payload: - _defer_warning( - "Hydrocron returned status %s for %s %s", - status_code, - prj.rivers.target_id_col, - target_id, - ) - summary["failed"] += 1 - continue - - try: - df = pd.read_csv(StringIO(csv_payload)) - except Exception as exc: - _defer_warning( - "Failed to parse CSV for %s %s: %s", - prj.rivers.target_id_col, - target_id, - exc, - ) - summary["failed"] += 1 - continue - - if df.empty or quality_column not in df.columns: - if df.empty: - _defer_warning( - "Hydrocron returned no data for %s %s", - prj.rivers.target_id_col, - target_id, - ) - else: - _defer_warning( - "Quality column %s not in Hydrocron response for %s %s", - quality_column, - prj.rivers.target_id_col, - target_id, - ) - continue - - df = df[df[quality_column] <= max_q] - if df.empty: - _defer_warning( - "All Hydrocron observations filtered for %s %s (quality > %s)", - prj.rivers.target_id_col, - target_id, - max_q, - ) - summary["empty_after_filter"] += 1 - continue - - frames.append(df) - summary["successful"] += 1 - - if frames: - combined = pd.concat(frames, ignore_index=True) - combined.to_csv(output_path, index=False) - - for warning in deferred_warnings: - logger.debug(warning) - - logger.info( - "Hydrocron download complete: %s requested, %s successful, %s failed, %s empty after filtering. See file logs for more info.", - summary["requested"], - summary["successful"], - summary["failed"], - summary["empty_after_filter"], - ) - - -def _group_river_targets_by_waterbody(prj: "Project") -> dict: - """Return {waterbody_id: [target_id, ...]} grouping.""" - if prj.rivers.target_features is not None and len(prj.rivers.target_features) > 0: - groups: dict = {} - seen_target_ids: set = set() - for _, row in prj.rivers.target_features.iterrows(): - target_id = int(row[prj.rivers.target_id_col]) - if target_id in seen_target_ids: - continue - seen_target_ids.add(target_id) - wb_id = str(row[prj.rivers.id_key]) - groups.setdefault(wb_id, []).append(target_id) - return groups - - if prj.rivers.configured_id: - return {str(prj.rivers.configured_id): list(prj.rivers.target_ids)} - - raise ValueError( - "Unable to group river targets by waterbody. " - "Configure rivers.id or provide rivers.aoi_path with rivers.id_key." - ) - - -def _river_target_corridor( - prj: "Project", target_ids, buffer_meters=None, width_buffer_factor=1.05, -): - """ - Build one buffered, dissolved corridor polygon covering the given - river targets (nodes or reaches), for use as the spatial AOI when - downloading/extracting ICESat-2 and Sentinel-3/6 observations. - - This is deliberately a SEPARATE buffer distance from - prj.rivers.buffer_meters (used earlier to decide which SWORD - targets intersect the user's AOI at all) -- that question ("is this - target in scope") and this one ("how far from the centerline could - real river water still be, for a raw altimetry point to plausibly - belong to this target") are different, and conflating them risks - the same "one parameter doing two jobs badly" issue found elsewhere - in this pipeline. - - Parameters - ---------- - buffer_meters : float or None, optional - Explicit, uniform buffer distance (meters), applied to every - target regardless of its actual width. If None (default), uses - each target's own SWORD "width" attribute instead: buffer - distance = (width / 2) * width_buffer_factor. This is HALF the - width, not the full width -- buffering a line expands it - symmetrically by the given distance on EACH side, so a buffer - of width/2 gives a corridor whose TOTAL span is approximately - width * width_buffer_factor, matching the river's actual extent - plus a margin, rather than doubling it. Falls back to - _river_extraction_buffer_meters() (a flat scalar) if no usable - "width" column is found -- e.g. if your SWORD data names it - differently than assumed here, this degrades gracefully with a - log message rather than failing. - width_buffer_factor : float, optional - Margin applied on top of each target's own width when using the - width-based default. Default 1.05 -- 5% wider than the river's - actual channel width. Only used when buffer_meters is None. - - NOTE: "width" is the expected SWORD column name per the standard - SWORD data dictionary -- this has NOT been verified against a real - downloaded SWORD file in this session (no sample data was - available), unlike most other assumptions in this codebase. Check - your actual target_features columns if width-based buffering - doesn't seem to be kicking in. - - Returns a single-row GeoDataFrame in prj.global_crs (matching what - icesat2.extract_observations/sentinel.extract_observations expect - for their `features` argument, same as reservoirs), or None if no - matching SWORD geometry is found for target_ids. Note the returned - geometry may be a MultiPolygon if targets form disconnected pieces - (e.g. separate reaches far enough apart that their buffers never - touch) -- see _iter_geometry_pieces for how downloads handle this. - """ - features = prj.rivers.target_features - subset = features.loc[features[prj.rivers.target_id_col].isin(target_ids)] - if subset.empty: - return None - - local = subset.to_crs(prj.local_crs) - - if buffer_meters is not None: - distances = buffer_meters - elif "width" in local.columns and local["width"].notna().any(): - fallback_width = local["width"].median() - distances = (local["width"].fillna(fallback_width) / 2) * width_buffer_factor - else: - logger.info( - "No 'width' column found in SWORD target_features for this " - "waterbody -- falling back to a flat extraction buffer " - "instead of width-based sizing. Check your SWORD data's " - "actual column names if this is unexpected." - ) - distances = _river_extraction_buffer_meters(prj) - - buffered = local.buffer(distances) - corridor = buffered.unary_union - corridor_gdf = gpd.GeoDataFrame( - geometry=[corridor], crs=prj.local_crs - ).to_crs(prj.global_crs) - return corridor_gdf - - -def _iter_geometry_pieces(geom): - """ - Yield each individual polygon from a geometry: every part of a - MultiPolygon, or the geometry itself for a plain Polygon. Used for - river downloads, where disconnected corridor pieces should each get - their own query rather than only querying the first piece (silently - dropping coverage of the rest) or merging them into one shape that - would also cover the (possibly large, irrelevant) gap between them. - """ - if hasattr(geom, "geoms"): - return list(geom.geoms) - return [geom] - - -def _simplify_to_one_polygon(geom): - """ - Collapse a MultiPolygon into a single encompassing polygon via - convex hull. Used for reservoirs: unlike rivers, a reservoir is - treated as one target regardless of how many disconnected parts its - input polygon has, so a single combined query is preferred over - splitting into several separate ones. Convex hull guarantees full - coverage of every part, at the cost of also covering some in-between - area that may not be real water -- an accepted tradeoff for treating - one reservoir as one query rather than several. - """ - if hasattr(geom, "geoms"): - return geom.convex_hull - return geom - - -def _river_extraction_buffer_meters(prj: "Project") -> float: - """ - Resolve the extraction-corridor buffer distance: prefer an explicit - prj.rivers.extraction_buffer_meters if set, else fall back to the - SWORD-intersection prj.rivers.buffer_meters, else a conservative - default. Kept as its own small function since this fallback chain - is used by both download and extraction. - """ - explicit = getattr(prj.rivers, "extraction_buffer_meters", None) - if explicit: - return explicit - if prj.rivers.buffer_meters: - return prj.rivers.buffer_meters - return 500.0 - - -def _download_rivers_icesat2(prj: "Project") -> None: - """Download ICESat-2 ATL13 data for river waterbody groups. - - Mirrors _download_reservoirs_icesat2, but queries over a buffered - corridor around each waterbody's SWORD targets (see - _river_target_corridor) rather than a single reservoir polygon. If - a waterbody's corridor comes out as disconnected pieces (a - MultiPolygon), queries each piece separately (see - _iter_geometry_pieces) rather than only the first -- unlike - reservoirs, a river waterbody's targets can legitimately be - disjoint (e.g. separate reaches far apart), so collapsing to one - query would either miss coverage or require an artificially large - combined shape. - """ - waterbody_groups = _group_river_targets_by_waterbody(prj) - explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) - width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) - - startdate = prj.startdates["icesat2"] - enddate = prj.enddates["icesat2"] - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) - - for wb_id, target_ids in waterbody_groups.items(): - logger.info("Downloading ICESat-2 data for waterbody %s", wb_id) - - corridor_gdf = _river_target_corridor( - prj, target_ids, buffer_meters=explicit_buffer, - width_buffer_factor=width_buffer_factor, - ) - if corridor_gdf is None: - logger.warning( - "No SWORD geometry found for waterbody %s; skipping " - "ICESat-2 download.", wb_id, - ) - continue - - pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) - parquet_dir = os.path.join(prj.dirs["icesat2_processed"], f"{wb_id}") - general.ifnotmakedirs(parquet_dir) - - for piece_idx, geom in enumerate(pieces): - coords = list(geom.exterior.coords) - - logger.info( - "Searching for Icesat2 ATL13 for waterbody %s (piece %d/%d) " - "from %s to %s", wb_id, piece_idx + 1, len(pieces), startdate, enddate, - ) - try: - _ = icesat2.query( - aoi=coords, - startdate=startdate, - enddate=enddate, - download_directory=parquet_dir, - atl13_options=prj.mission_options.get("icesat2", {}).get("atl13", {}), - atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields") - or None, - ) - except Exception as exc: - logger.warning( - "ICESat-2 download skipped for waterbody %s (piece %d/%d): %s", - wb_id, piece_idx + 1, len(pieces), exc, - ) - - -def _download_rivers_sentinel(prj: "Project", mission: str) -> None: - """Download Sentinel-3 or Sentinel-6 data for river waterbody groups. - - Mirrors _download_reservoirs_sentinel (both now share - _download_sentinel_for_target, including the CREODIAS/EarthData - branching for Sentinel-6). NOTE: sentinel.query/query_earthdata take - a bounding box (envelope), not the exact corridor polygon -- for a - long or winding river corridor this can query/download a - meaningfully larger area than the actual buffered corridor. This is - an existing limitation inherited from the reservoir path (where it - matters far less, since a reservoir's envelope is close to its - actual extent), not something new introduced here -- worth - revisiting if it turns out to matter in practice for a large or - winding waterbody. - """ - product = "S3" if mission == "sentinel3" else "S6" - - sentinel_creds = None - use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) - if use_earthdata_s6: - prj._require_earthdata_credentials() - else: - sentinel_creds = prj._require_creodias_credentials() - - waterbody_groups = _group_river_targets_by_waterbody(prj) - explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) - width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) - - startdate = prj.startdates[mission] - enddate = prj.enddates[mission] - if isinstance(startdate, list): - startdate = datetime.date(*startdate) - if isinstance(enddate, list): - enddate = datetime.date(*enddate) - - session_token = None - session_start_time = None - - for wb_id, target_ids in waterbody_groups.items(): - logger.info("Downloading data for waterbody %s", wb_id) - - corridor_gdf = _river_target_corridor( - prj, target_ids, buffer_meters=explicit_buffer, - width_buffer_factor=width_buffer_factor, - ) - if corridor_gdf is None: - logger.warning( - "No SWORD geometry found for waterbody %s; skipping " - "Sentinel-%s download.", wb_id, product, - ) - continue - - # Envelope each disconnected piece separately rather than the - # whole (possibly MultiPolygon) corridor at once -- sentinel's - # API only accepts a bounding box, so one envelope covering - # widely separated pieces could be far larger than any of them - # individually. - pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) - - download_dir = os.path.join(prj.dirs[mission], f"{wb_id}") - general.ifnotmakedirs(download_dir) - - for piece_idx, geom in enumerate(pieces): - coords = [(x, y) for x, y in geom.envelope.exterior.coords] - - logger.info( - "Searching for Sentinel-%s for waterbody %s (piece %d/%d) " - "from %s to %s", product, wb_id, piece_idx + 1, len(pieces), - startdate, enddate, - ) - session_token, session_start_time = _download_sentinel_for_target( - prj, mission, product, coords, download_dir, - startdate, enddate, sentinel_creds, session_token, session_start_time, - ) - - -def _get_latest_hydrocron_obs_date(output_path) -> datetime.date: - """Get latest observation date from existing Hydrocron output.""" - if not os.path.exists(output_path): - return None - - try: - existing = pd.read_csv(output_path) - except Exception as exc: - logger.warning( - "Unable to read existing Hydrocron output %s: %s", output_path, exc - ) - return None - - if "time_str" not in existing.columns or existing.empty: - return None - - timestamps = pd.to_datetime(existing["time_str"], errors="coerce", utc=True) - timestamps = timestamps.dropna() - if timestamps.empty: - return None - - latest_obs = timestamps.max().to_pydatetime() - return datetime.date(latest_obs.year, latest_obs.month, latest_obs.day) - - -# ============================================================================ -# RIVERS: Timeseries Processing (extraction) -# ============================================================================ - - -def _assign_points_to_river_targets( - points, targets, target_id_col, max_distance_meters, local_crs -): - """ - Assign each point in `points` to its nearest feature in `targets` - (SWORD node or reach geometries, whichever prj.rivers.target_id_col - is configured for), dropping points farther than max_distance_meters - from any target. - - Uses gpd.sjoin_nearest rather than a custom NearestNeighbors/DBSCAN - approach -- it handles point-to-line matching natively (needed for - reaches, not just nodes), and max_distance is expressed directly in - real distance units once both inputs are reprojected to local_crs. - - Returns points (unprojected, original CRS) with target_id_col and a - _dist_to_target_m column added; rows with no target within range are - dropped entirely. - """ - points_local = points.to_crs(local_crs) - targets_local = targets[[target_id_col, "geometry"]].to_crs(local_crs) - - joined = gpd.sjoin_nearest( - points_local, - targets_local, - how="inner", - max_distance=max_distance_meters, - distance_col="_dist_to_target_m", - ) - - result = points.loc[joined.index].copy() - result[target_id_col] = joined[target_id_col].values - result["_dist_to_target_m"] = joined["_dist_to_target_m"].values - return result - - -def _extract_rivers_timeseries(prj: "Project", overwrite: bool = False) -> None: - """Extract timeseries observations from raw downloaded files, for rivers. - - SWOT still needs a (lightweight) extraction step here: Hydrocron - already returns a per-node/reach timeseries directly, but grouped - per WATERBODY (one CSV covering every target in that waterbody) -- - see _extract_rivers_swot_observations for splitting that into the - same per-target file structure ICESat-2/Sentinel-3/6 use, so the - shared clean/merge pipeline can treat every mission identically. - - Parameters - ---------- - overwrite : bool, optional - Same semantics as _extract_reservoirs_timeseries: if False - (default), any target whose output .gpkg already exists is - skipped rather than re-extracted. - """ - if "icesat2" in prj.to_process: - _extract_rivers_icesat2_observations(prj, overwrite=overwrite) - - if "sentinel3" in prj.to_process: - _extract_rivers_sentinel_observations( - prj, "sentinel3", "S3", overwrite=overwrite - ) - - if "sentinel6" in prj.to_process: - _extract_rivers_sentinel_observations( - prj, "sentinel6", "S6", overwrite=overwrite - ) - - if "swot" in prj.to_process: - _extract_rivers_swot_observations(prj, overwrite=overwrite) - - -def _extract_rivers_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: - """Extract ICESat-2 ATL13 observations for each river target. - - Unlike reservoirs (one polygon = one target), a river waterbody's - raw download covers many targets at once. This extracts once per - waterbody -- reusing icesat2.extract_observations exactly as - reservoirs use it, with the buffered corridor (see - _river_target_corridor) as the spatial filter instead of a single - reservoir polygon -- then assigns each surviving point to its - nearest target via sjoin_nearest, and splits the result into the - same per-target {output}/{target_id}/raw_observations/icesat2.gpkg - structure reservoirs already use, so everything downstream - (clean/merge) can treat a river target exactly like a reservoir. - """ - waterbody_groups = _group_river_targets_by_waterbody(prj) - explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) - width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) - max_assign_dist = ( - getattr(prj.rivers, "max_node_assignment_meters", None) - or _river_extraction_buffer_meters(prj) - ) - - tmp_dir = os.path.join(prj.dirs["output"], "_tmp_river_extraction") - - for wb_id, target_ids in waterbody_groups.items(): - parquet_dir = os.path.join(prj.dirs["icesat2_processed"], f"{wb_id}") - if not os.path.exists(os.path.join(parquet_dir, "atl13.parquet")): - continue - - if not overwrite: - remaining = [ - t - for t in target_ids - if not os.path.exists( - os.path.join( - prj.dirs["output"], f"{t}", "raw_observations", "icesat2.gpkg" - ) - ) - ] - if not remaining: - continue - target_ids = remaining - - corridor_gdf = _river_target_corridor( - prj, target_ids, buffer_meters=explicit_buffer, - width_buffer_factor=width_buffer_factor, - ) - if corridor_gdf is None: - continue - - general.ifnotmakedirs(tmp_dir) - tmp_dst = os.path.join(tmp_dir, f"{wb_id}_icesat2.gpkg") - - try: - icesat2.extract_observations( - src_dir=parquet_dir, - dst_path=tmp_dst, - features=corridor_gdf, - atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields"), - track_keys=prj.mission_options.get("icesat2", {}).get("track_keys"), - ) - except Exception as exc: - logger.warning( - "Failed to extract ICESat-2 for waterbody %s: %s", wb_id, exc - ) - continue - - if not os.path.exists(tmp_dst): - continue - - points = gpd.read_file(tmp_dst) - os.remove(tmp_dst) - if points.empty: - continue - - targets = prj.rivers.target_features.loc[ - prj.rivers.target_features[prj.rivers.target_id_col].isin(target_ids) - ] - assigned = _assign_points_to_river_targets( - points, targets, prj.rivers.target_id_col, max_assign_dist, prj.local_crs - ) - if assigned.empty: - logger.warning( - "No ICESat-2 points within %sm of any target for waterbody %s", - max_assign_dist, wb_id, - ) - continue - - for target_id, group in assigned.groupby(prj.rivers.target_id_col): - dst_dir = os.path.join( - prj.dirs["output"], f"{target_id}", "raw_observations" - ) - general.ifnotmakedirs(dst_dir) - dst_path = os.path.join(dst_dir, "icesat2.gpkg") - group.drop( - columns=["index_right", "_dist_to_target_m"], errors="ignore" - ).to_file(dst_path, driver="GPKG") - - -def _extract_rivers_sentinel_observations( - prj: "Project", mission_key: str, product: str, overwrite: bool = False -) -> None: - """Extract Sentinel-3 or Sentinel-6 observations for each river target. - - Same per-waterbody-then-split approach as - _extract_rivers_icesat2_observations, plus a water-only filter: a - buffered river corridor is much looser than a reservoir polygon (it - genuinely includes riverbank, fields, vegetation alongside the - channel), and unlike ICESat-2, Sentinel-3/6 have no built-in water - classification. sigma0_min filters this out as a self-contained - post-processing step here (rather than modifying - sentinel.extract_observations itself, whose internals haven't been - verified) -- water gives a strong, consistent specular radar - return; land gives a weaker, noisier one. Needs empirical tuning - against real river data, same as every other threshold in this - pipeline -- the default here (0.0, i.e. no-op) is a safe starting - point, not a verified value; set mission_options[mission_key] - ['sigma0_min'] once you have real data to check it against. - """ - waterbody_groups = _group_river_targets_by_waterbody(prj) - explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) - width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) - max_assign_dist = ( - getattr(prj.rivers, "max_node_assignment_meters", None) - or _river_extraction_buffer_meters(prj) - ) - sigma0_min = prj.mission_options.get(mission_key, {}).get("sigma0_min", 0.0) - - tmp_dir = os.path.join(prj.dirs["output"], "_tmp_river_extraction") - - for wb_id, target_ids in waterbody_groups.items(): - download_dir = os.path.join(prj.dirs[mission_key], f"{wb_id}") - if not os.path.exists(download_dir): - continue - - if not overwrite: - remaining = [ - t - for t in target_ids - if not os.path.exists( - os.path.join( - prj.dirs["output"], - f"{t}", - "raw_observations", - f"{mission_key}.gpkg", - ) - ) - ] - if not remaining: - continue - target_ids = remaining - - corridor_gdf = _river_target_corridor( - prj, target_ids, buffer_meters=explicit_buffer, - width_buffer_factor=width_buffer_factor, - ) - if corridor_gdf is None: - continue - - general.ifnotmakedirs(tmp_dir) - tmp_dst = os.path.join(tmp_dir, f"{wb_id}_{mission_key}.gpkg") - - try: - sentinel.extract_observations( - src_dir=download_dir, - dst_path=tmp_dst, - features=corridor_gdf, - sigma0_max=prj.mission_options.get(mission_key, {}).get( - "sigma0_max", 1e5 - ), - ) - except Exception as exc: - logger.warning( - "Failed to extract %s for waterbody %s: %s", mission_key, wb_id, exc - ) - continue - - if not os.path.exists(tmp_dst): - continue - - points = gpd.read_file(tmp_dst) - os.remove(tmp_dst) - if points.empty: - continue - - if "sigma0" in points.columns and sigma0_min: - before = len(points) - points = points.loc[points["sigma0"] >= sigma0_min].reset_index(drop=True) - logger.info( - "%s waterbody %s: sigma0_min=%s kept %d/%d points", - mission_key, wb_id, sigma0_min, len(points), before, - ) - if points.empty: - continue - - targets = prj.rivers.target_features.loc[ - prj.rivers.target_features[prj.rivers.target_id_col].isin(target_ids) - ] - assigned = _assign_points_to_river_targets( - points, targets, prj.rivers.target_id_col, max_assign_dist, prj.local_crs - ) - if assigned.empty: - logger.warning( - "No %s points within %sm of any target for waterbody %s", - mission_key, max_assign_dist, wb_id, - ) - continue - - for target_id, group in assigned.groupby(prj.rivers.target_id_col): - dst_dir = os.path.join( - prj.dirs["output"], f"{target_id}", "raw_observations" - ) - general.ifnotmakedirs(dst_dir) - dst_path = os.path.join(dst_dir, f"{mission_key}.gpkg") - group.drop( - columns=["index_right", "_dist_to_target_m"], errors="ignore" - ).to_file(dst_path, driver="GPKG") - - -def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) -> None: - """ - Split Hydrocron's per-waterbody timeseries CSV into the same - per-target {output}/{target_id}/raw_observations/swot.gpkg structure - every other mission uses, so the shared clean/merge pipeline can - treat SWOT identically to ICESat-2/Sentinel-3/6 for rivers. - - Unlike LakeSP for reservoirs, Hydrocron's own CSV doesn't include - per-observation coordinates in the default field lists (see - rivers.yaml) -- but nothing downstream actually needs per-observation - lat/lon for SWOT (see PRODUCT_TIMESERIES_KEYS: no lat_key/lon_key for - "swot"), so this attaches the target's own SWORD geometry as a - constant placeholder purely so the file can be saved/read as .gpkg - like every other mission's output -- the geometry's actual value is - never used downstream, only the height/date/platform/orbit columns. - - Quality filtering (max_q) is already applied at download time (see - _download_swot_hydrocron_timeseries), so it isn't repeated here. - """ - waterbody_groups = _group_river_targets_by_waterbody(prj) - id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" - - for wb_id, target_ids in waterbody_groups.items(): - src_path = os.path.join( - prj.dirs["swot"], str(wb_id), f"{id_label}_timeseries.csv" - ) - if not os.path.exists(src_path): - continue - - if not overwrite: - remaining = [ - t - for t in target_ids - if not os.path.exists( - os.path.join( - prj.dirs["output"], f"{t}", "raw_observations", "swot.gpkg" - ) - ) - ] - if not remaining: - continue - target_ids = remaining - - try: - df = pd.read_csv(src_path) - except Exception as exc: - logger.warning( - "Failed to read Hydrocron CSV for waterbody %s: %s", wb_id, exc - ) - continue - - if df.empty or prj.rivers.target_id_col not in df.columns: - continue - - df = df.loc[df[prj.rivers.target_id_col].isin(target_ids)].copy() - if df.empty: - continue - - df["height"] = df["wse"] - df["date"] = pd.to_datetime(df["time_str"]) - df["platform"] = "swot" - df["product"] = f"SWOT_Hydrocron_{id_label}" - # Matches the reservoir SWOT convention (orbit = lake_id, constant - # per target) -- there's no meaningful "which persistent track" - # concept distinct from the target itself for Hydrocron data. - df["orbit"] = df[prj.rivers.target_id_col] - - for target_id, group in df.groupby(prj.rivers.target_id_col): - target_row = prj.rivers.target_features.loc[ - prj.rivers.target_features[prj.rivers.target_id_col] == target_id - ] - if target_row.empty: - continue - - group = group.copy() - group["geometry"] = target_row.geometry.iloc[0] - gdf = gpd.GeoDataFrame( - group, geometry="geometry", crs=prj.rivers.target_features.crs - ) - - dst_dir = os.path.join( - prj.dirs["output"], f"{target_id}", "raw_observations" - ) - general.ifnotmakedirs(dst_dir) - gdf.to_file(os.path.join(dst_dir, "swot.gpkg"), driver="GPKG") - - -# ============================================================================ -# RESERVOIRS: Timeseries Processing -# ============================================================================ - - -def create_reservoirs_timeseries(prj: "Project") -> None: - """Extract, clean, and merge timeseries for reservoirs. - - Parameters - ---------- - prj : Project - Project instance with reservoirs configuration - """ - if not hasattr(prj, "reservoirs"): - return - - # Extract raw observations from downloaded files (skips reservoirs whose - # gpkg already exists, unless prj.reservoirs.overwrite_extraction=True -- - # confirmed this re-read/re-extraction was the dominant real-world cost, - # far more than anything in clean()/merge()) - _extract_reservoirs_timeseries( - prj, overwrite=getattr(prj.reservoirs, "overwrite_extraction", False) - ) - - # Clean observations with filters - _clean_reservoirs_timeseries(prj) - - # Export to dfs0 if enabled - if getattr(prj.reservoirs, "export_to_dfs0", False): - _export_cleaned_to_dfs0(prj) - - # Merge multi-mission timeseries - _merge_reservoirs_timeseries(prj) - - -def create_rivers_timeseries(prj: "Project") -> None: - """Extract, clean, and merge timeseries for river targets (nodes/reaches). - - Mirrors create_reservoirs_timeseries. Not yet done: an export_to_dfs0 - equivalent for rivers, since _export_cleaned_to_dfs0 currently - iterates prj.reservoirs.download_gdf specifically -- left out here - rather than silently generalizing something not explicitly asked - for yet. - - Parameters - ---------- - prj : Project - Project instance with rivers configuration - """ - if not hasattr(prj, "rivers"): - return - - _extract_rivers_timeseries( - prj, overwrite=getattr(prj.rivers, "overwrite_extraction", False) - ) - - _clean_rivers_timeseries(prj) - - _merge_rivers_timeseries(prj) - - -def _extract_reservoirs_timeseries(prj: "Project", overwrite: bool = False) -> None: - """Extract timeseries observations from raw downloaded files. - - Parameters - ---------- - overwrite : bool, optional - If False (default), any reservoir/mission whose output .gpkg - already exists is skipped entirely rather than re-read and - re-extracted. Confirmed on real data that this re-extraction -- - not clean()/merge() -- was the dominant cost in real end-to-end - runs (orders of magnitude larger than the merge pipeline itself). - Set True to force re-extraction (e.g. new raw downloads arrived). - """ - if "icesat2" in prj.to_process: - _extract_icesat2_observations(prj, overwrite=overwrite) - - if "sentinel3" in prj.to_process: - _extract_sentinel_observations(prj, "sentinel3", "S3", overwrite=overwrite) - - if "sentinel6" in prj.to_process: - _extract_sentinel_observations(prj, "sentinel6", "S6", overwrite=overwrite) - - if "swot" in prj.to_process: - _extract_swot_observations(prj, overwrite=overwrite) - - -def _extract_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: - """Extract ICESat-2 ATL13 observations for each reservoir.""" - available_ids = [ - id - for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] - if os.path.exists( - os.path.join(prj.dirs["icesat2_processed"], f"{id}", "atl13.parquet") - ) - ] - if not available_ids: - logger.warning("No ICESat-2 downloads found; skipping timeseries extraction.") - return - - if not overwrite: - skip_count = 0 - remaining_ids = [] - for id in available_ids: - dst_path = os.path.join( - prj.dirs["output"], f"{id}", "raw_observations", "icesat2.gpkg" - ) - if os.path.exists(dst_path): - skip_count += 1 - else: - remaining_ids.append(id) - if skip_count: - logger.info( - "ICESat-2 extraction: skipping %d reservoir(s) with an existing " - "icesat2.gpkg (pass overwrite=True to force re-extraction).", - skip_count, - ) - available_ids = remaining_ids - if not available_ids: - return - - empty_ids = [] - for id in tqdm(available_ids, desc="Extracting ICESat-2 ATL13 product"): - sub_gdf = prj.reservoirs.download_gdf.loc[ - prj.reservoirs.download_gdf[prj.reservoirs.id_key] == id - ] - download_dir = os.path.join(prj.dirs["icesat2_processed"], f"{id}") - dst_dir = os.path.join(prj.dirs["output"], f"{id}", "raw_observations") - general.ifnotmakedirs(dst_dir) - dst_path = os.path.join(dst_dir, "icesat2.gpkg") - - try: - icesat2.extract_observations( - src_dir=download_dir, - dst_path=dst_path, - features=sub_gdf, - atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields"), - track_keys=prj.mission_options.get("icesat2", {}).get("track_keys"), - ) - except Exception as exc: - logger.warning("Failed to extract ICESat-2 for %s: %s", id, exc) - - if not os.path.exists(dst_path): - empty_ids.append(id) - - if empty_ids: - logger.warning( - "ICESat-2 timeseries empty for: %s (no observations passed the spatial filter or the download returned no data)", - ", ".join(str(i) for i in empty_ids), - ) - - -def _extract_sentinel_observations( - prj: "Project", mission_key: str, product: str, overwrite: bool = False -) -> None: - """Extract Sentinel-3 or Sentinel-6 observations for each reservoir.""" - available_ids = [ - id - for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] - if os.path.exists(os.path.join(prj.dirs[mission_key], f"{id}")) - ] - if not available_ids: - logger.warning( - "No %s downloads found; skipping timeseries extraction.", mission_key - ) - return - - if not overwrite: - skip_count = 0 - remaining_ids = [] - for id in available_ids: - dst_path = os.path.join( - prj.dirs["output"], f"{id}", "raw_observations", f"{mission_key}.gpkg" - ) - if os.path.exists(dst_path): - skip_count += 1 - else: - remaining_ids.append(id) - if skip_count: - logger.info( - "%s extraction: skipping %d reservoir(s) with an existing " - "%s.gpkg (pass overwrite=True to force re-extraction).", - mission_key, skip_count, mission_key, - ) - available_ids = remaining_ids - if not available_ids: - return - - empty_ids = [] - for id in tqdm(available_ids, desc=f"Extracting Sentinel-{product} product"): - sub_gdf = prj.reservoirs.download_gdf.loc[ - prj.reservoirs.download_gdf[prj.reservoirs.id_key] == id - ] - download_dir = os.path.join(prj.dirs[mission_key], f"{id}") - dst_dir = os.path.join(prj.dirs["output"], f"{id}", "raw_observations") - general.ifnotmakedirs(dst_dir) - dst_path = os.path.join(dst_dir, f"{mission_key}.gpkg") - - try: - sentinel.extract_observations( - src_dir=download_dir, - dst_path=dst_path, - features=sub_gdf, - sigma0_max=prj.mission_options.get(mission_key, {}).get( - "sigma0_max", 1e5 - ), - ) - except Exception as exc: - logger.warning("Failed to extract %s for %s: %s", mission_key, id, exc) - - if not os.path.exists(dst_path): - empty_ids.append(id) - - if empty_ids: - logger.warning( - "Sentinel-%s timeseries empty for: %s (no observations passed the spatial filter or the download returned no data)", - product, - ", ".join(str(i) for i in empty_ids), - ) - - -def _extract_swot_observations(prj: "Project", overwrite: bool = False) -> None: - """Extract SWOT Lake SP observations for all reservoirs.""" - download_dir = prj.dirs["swot"] - if not os.path.exists(download_dir): - logger.warning("No SWOT downloads found; skipping timeseries extraction.") - return - - features = prj.reservoirs.download_gdf - id_key = prj.reservoirs.id_key - - if not overwrite: - def _has_output(id): - return os.path.exists( - os.path.join(prj.dirs["output"], f"{id}", "raw_observations", "swot.gpkg") - ) - all_ids = features[id_key].tolist() - remaining_ids = [i for i in all_ids if not _has_output(i)] - skip_count = len(all_ids) - len(remaining_ids) - if skip_count: - logger.info( - "SWOT extraction: skipping %d reservoir(s) with an existing " - "swot.gpkg (pass overwrite=True to force re-extraction).", - skip_count, - ) - if not remaining_ids: - return - features = features.loc[features[id_key].isin(remaining_ids)] - - empty_ids = swot.extract_observations( - src_dir=download_dir, - dst_dir=prj.dirs["output"], - dst_file_name="swot.gpkg", - features=features, - id_key=id_key, - exclude_obs_id_values=prj.mission_options.get("swot", {}).get( - "exclude_obs_id_values", ["no_data"] - ), - ) - if empty_ids: - logger.warning( - "SWOT timeseries empty for: %s (no observations matched the prior lake ID or all were excluded)", - ", ".join(str(i) for i in empty_ids), - ) - - -# Per-product mapping from generic Timeseries key attributes to the actual -# column names each mission's extractor writes. Sentinel-3 shares -# Sentinel-6's extractor/schema (same sentinel.extract_observations -# function, see _extract_sentinel_observations). -# -# **************************************************************************** -# TERMINOLOGY TRAP -- read before touching orbit_key/pass_key for Sentinel: -# The raw Sentinel-3/6 data has TWO similarly-named but opposite-meaning -# columns: -# - "orbit": the absolute revolution counter. Unique on every single -# crossing, never repeats. USELESS as orbit_key (bias_correct needs a -# persistent identifier to accumulate overlap against -- grouping by -# something that's different every time means every "source" has -# exactly 1 observation and nothing can ever be calibrated: this is -# exactly the bug that caused every S3A/S3B track to be dropped as -# unanchored in practice). -# - "pass": the satellite-engineering term for the STABLE, REPEATING -# ground track number (same value every ~27-day repeat cycle for -# S3A/S3B). This is what orbit_key actually needs. -# Confusingly, our own framework's `pass_key` means the OPPOSITE thing (one -# specific, one-time crossing -- e.g. file_name) from what "pass" means in -# the satellite data itself (the repeating track). Do not be tempted to -# point pass_key at the raw "pass" column -- file_name is correct there. -# **************************************************************************** -# -# ICESat-2's orbit_key is "beam" (the persistent ground track/virtual -# station) -- cycle_number only matters as an ingredient of the compound -# "pass" column built at extraction time (see -# HydroEO.satellites.icesat2.preprocess.extract_observations). SWOT's -# LakeSP product is already one integrated WSE per crossing with its own -# formal uncertainty (wse_u), so it needs neither lat/lon nor pass_key -- -# see preset_error_key, and daily_mad_error's handling of it. -PRODUCT_TIMESERIES_KEYS = { - "sentinel3": dict( - lat_key="lat", lon_key="lon", pass_key="file_name", - platform_key="platform", orbit_key="relative_orbit", - ), - # NOTE: sentinel6 still uses "pass" as orbit_key -- NOT verified to be - # unstable the way it was for sentinel3 (confirmed empirically: on - # real data, "pass" was unique-per-crossing for every S3A/S3B visit, - # i.e. not stable at all, while "relative_orbit" genuinely repeated - # across multiple visits -- e.g. S3B crossed via 2 distinct stable - # configurations, with real biases of -0.14m and +0.22m that a - # platform-only grouping was averaging into one misleading +0.04m). - # Sentinel-6 may have the same "pass" instability and may also have - # its own "relative_orbit"-equivalent column, but this hasn't been - # checked against real S6 data -- don't assume the same fix applies - # without verifying first. - "sentinel6": dict( - lat_key="lat", lon_key="lon", pass_key="file_name", - platform_key="platform", orbit_key="pass", - ), - "icesat2": dict( - lat_key="lat", lon_key="lon", pass_key="pass", - platform_key="platform", orbit_key="beam", - ), - "swot": dict( - platform_key="platform", orbit_key="orbit", preset_error_key="wse_u", - ), -} - -# Default .merge() tuning for reservoirs, mirroring the shape of -# processing_options (a project-level dict of pipeline parameters) but -# applied once per reservoir rather than per-product, since .merge() runs -# on the already-combined multi-product timeseries. Override via -# prj.merging_options in project config; falls back to these reservoir- -# appropriate defaults if that attribute isn't set. -DEFAULT_RESERVOIR_MERGING_OPTIONS = { - "window_km": 1.5, - "svr_linear_err": 0.1, - "svr_linear_epsilon": 0.1, - # Both updated from DAHITI's lake-tuned defaults (err=0.1, gamma= - # 0.0000438) based on real reservoir data validated this session -- - # the lake-tuned gamma implied a ~151-day smoothing lengthscale, far - # too coarse for a reservoir with real multi-week transitions (see - # the svr_radial oversmoothing discussion). err=1.0 (river-like, - # rather than the stricter lake value) and gamma x50 (~21-day - # lengthscale instead of ~151 days) let the trend actually track - # real fast changes instead of rejecting them as if they were noise. - "svr_radial_err": 1.0, - "svr_radial_rbf_c": 10000, - "svr_radial_gamma": 0.0000438 * 50, - "svr_radial_epsilon": 0.1, - # Confirmed on real data across two reservoirs: revisit sparsity varies a - # lot (e.g. one reservoir had icesat2/S3A/S3B visiting only 7/14/13 - # distinct days all year). At "10D"/3, sparse sources can fail to ever - # find 3 overlapping bins and get dropped as unanchored ENTIRELY (not - # just trimmed) -- confirmed: this silently dropped 2 of 3 missions - # (icesat2, S3B) for one real reservoir. "20D"/1 recovered all of it. - # Widening is monotonically safe against data loss (a wider window can - # only find equal-or-more overlapping bins, never fewer) -- the - # tradeoff is a very wide bin could blur real water-level change within - # the window into the bias estimate; 20D is a modest widening, not an - # extreme one. - "bias_time_bin": "20D", - "bias_min_overlap": 1, - # Confirmed empirically on real data: "platform_orbit" (using - # orbit_key -- now sentinel3's verified-stable "relative_orbit" - # column, see PRODUCT_TIMESERIES_KEYS) reveals genuine within-platform - # bias heterogeneity that "platform" alone was masking. One real - # reservoir's S3B crosses via two distinct, independently stable - # configurations (5 days on one, 8 on the other) with biases of - # -0.14m and +0.22m respectively -- "platform" grouping averaged - # these into one misleading +0.04m. Same pattern for ICESat-2's - # beams (orbit_key="beam"): per-beam biases ranged 0.06-0.18m under - # "platform_orbit", collapsed to one number under "platform". Total - # kept-row count was IDENTICAL either way on the reservoir tested - # (3182/4901) -- this is a precision gain, not a data-loss risk, at - # least for sentinel3/icesat2. NOTE: sentinel6 still uses "pass" as - # orbit_key (unverified whether it's stable or has a - # relative_orbit-equivalent -- see PRODUCT_TIMESERIES_KEYS) -- if - # it's actually unstable like sentinel3's old "pass" mapping was, - # "platform_orbit" could fragment sentinel6 into single-crossing - # sources. Recheck against real sentinel6 data before trusting this - # default for a project relying heavily on sentinel6. - "bias_group_by": "platform_orbit", - # Not a spatial correction -- just flags (and records in - # ts.bias_correct_diagnostics) when a source's observations are - # centered far from the anchor's, since for a large/elongated - # reservoir some of the estimated bias could be real spatial signal. - # Worth a closer look per-reservoir if this fires, not an error. - "bias_centroid_warn_km": 5.0, - # Off by default -- inflates Kalman input error by distance from the - # reservoir polygon's own centroid, addressing crossings that may be - # hydraulically unrepresentative (e.g. far upstream, subject to real - # slope bias) even when ADM alone reports them as highly precise. Set - # to a real value (m of extra error per km of distance) to enable -- - # the right scale depends on the true magnitude of upstream slope bias - # for your reservoirs, which needs empirical tuning, not a guessed - # default. - "distance_penalty_scale_per_km": None, - # Off by default -- a genuine height correction (not just error - # inflation) using a spatial deviation model fit once from a dense - # source (default ICESat-2) and persisted to disk per reservoir (see - # _get_or_fit_spatial_correction_model) so past corrections don't - # shift retroactively as new data arrives. Turn on once you've - # confirmed (as we did empirically) that the target reservoir shows a - # real, day-to-day-consistent spatial deviation pattern -- fitting - # requires several qualifying dense-source days (see - # fit_spatial_correction_model's min_days), and silently does nothing - # if there isn't enough dense-source data yet. - "use_spatial_correction": False, - "spatial_correction_dense_source": "icesat2", -} - - -def _clean_timeseries(prj: "Project", target_type: str) -> None: - """Apply quality filters to extracted timeseries, for either - reservoirs or river targets (nodes/reaches).""" - target_ids = _get_target_ids(prj, target_type) - ids_with_raw = [ - id - for id in target_ids - if os.path.exists(os.path.join(prj.dirs["output"], f"{id}", "raw_observations")) - ] - if not ids_with_raw: - logger.warning( - "No raw observations found for any %s; skipping timeseries cleaning.", - target_type, - ) - return - - for id in tqdm(ids_with_raw, desc=f"Cleaning product timeseries ({target_type})"): - for product in prj.to_process: - df = _load_product_timeseries( - os.path.join(prj.dirs["output"], f"{id}", "raw_observations"), - ".gpkg", - [product], - lambda path: gpd.read_file(path).drop(columns=["geometry"]), - ) - if df is not None: - product_options = prj.processing_options.get( - product, - { - "processing_filters": ["elevation", "MAD"], - "elevation_min_m": 0.0, - "elevation_max_m": 8000.0, - "mad_threshold": 5.0, - }, - ) - - ts = timeseries.Timeseries( - df, date_key="date", height_key="height", - **PRODUCT_TIMESERIES_KEYS.get(product, {}), - ) - - ts.clean( - product_options.get("processing_filters", ["elevation", "MAD"]), - filter_params={ - "elevation_min_m": product_options.get("elevation_min_m", 0.0), - "elevation_max_m": product_options.get( - "elevation_max_m", 8000.0 - ), - "mad_threshold": product_options.get("mad_threshold", 5.0), - }, - ) - - export_dir = os.path.join( - prj.dirs["output"], f"{id}", "cleaned_observations" - ) - general.ifnotmakedirs(export_dir) - ts.export_csv(os.path.join(export_dir, f"{product}.csv")) - - -def _clean_reservoirs_timeseries(prj: "Project") -> None: - """Apply quality filters to extracted reservoir timeseries.""" - _clean_timeseries(prj, "reservoirs") - - -def _clean_rivers_timeseries(prj: "Project") -> None: - """Apply quality filters to extracted river timeseries.""" - _clean_timeseries(prj, "rivers") - - -def _load_product_timeseries(data_dir, ext, products, reader_fn): - """Load files of given extension from directory, optionally filtered to products.""" - if not os.path.exists(data_dir): - return None - df_list = [] - for file in os.listdir(data_dir): - if file.endswith(ext): - if not products or file.split(".")[0] in products: - try: - df_list.append(reader_fn(os.path.join(data_dir, file))) - except Exception as exc: - logger.warning( - "Failed to load %s from %s: %s", - file, - data_dir, - exc, - ) - return pd.concat(df_list) if df_list else None - - -def _get_target_ids(prj: "Project", target_type: str): - """Return the list of target IDs to process, for either 'reservoirs' or 'rivers'.""" - if target_type == "reservoirs": - return list(prj.reservoirs.download_gdf[prj.reservoirs.id_key]) - if target_type == "rivers": - return list(prj.rivers.target_ids) - raise ValueError(f"Unknown target_type: {target_type!r}") - - -def _target_centroid(prj: "Project", target_type: str, id): - """ - Return (lat, lon) of a target's own geometry centroid -- the - reservoir polygon for target_type="reservoirs", or the SWORD - node/reach geometry for target_type="rivers" -- computed in a - projected (local) CRS for accuracy, then converted back to lat/lon. - Used as the reference location for apply_distance_penalty/ - apply_spatial_correction. Returns (None, None) if the target's - geometry can't be found, so callers can treat that as "skip" rather - than fail. - """ - try: - if target_type == "reservoirs": - gdf = prj.reservoirs.gdf - id_key = prj.reservoirs.id_key - elif target_type == "rivers": - gdf = prj.rivers.target_features - id_key = prj.rivers.target_id_col - else: - raise ValueError(f"Unknown target_type: {target_type!r}") - - row = gdf.loc[gdf[id_key] == id] - if len(row) == 0 or row.geometry.isna().all(): - return None, None - centroid = row.to_crs(prj.local_crs).geometry.centroid.to_crs(prj.global_crs) - pt = centroid.iloc[0] - return pt.y, pt.x # lat, lon - except Exception as exc: - logger.warning( - "Could not compute %s centroid for %s: %s", target_type, id, exc - ) - return None, None - - -def _reservoir_centroid(prj: "Project", id): - """Backward-compatible wrapper -- see _target_centroid.""" - return _target_centroid(prj, "reservoirs", id) - - -def _get_or_fit_spatial_correction_model( - prj: "Project", target_type: str, id, dense_source_platform="icesat2", - recalibrate=False, **fit_kwargs, -): - """ - Load a persisted spatial correction model for this target if one - exists, or fit a fresh one and persist it. Works identically for - reservoirs and river targets -- see _target_centroid. - - This is deliberately NOT re-fit automatically every run: doing so - would make past corrections shift retroactively every time new - dense-source data arrives, since the fitted slope would change. - Pass recalibrate=True to explicitly force a re-fit (e.g. as a - deliberate, occasional recalibration step) -- not something that - should happen as a silent side effect of routine reprocessing. - - Returns None if no model exists yet and there isn't enough dense - source data to fit one (see fit_spatial_correction_model) -- callers - should treat this the same as "no correction available." - """ - model_path = os.path.join( - prj.dirs["output"], f"{id}", "spatial_correction_model.json" - ) - - if os.path.exists(model_path) and not recalibrate: - with open(model_path, "r") as f: - return json.load(f) - - cleaned_path = os.path.join( - prj.dirs["output"], f"{id}", "all_cleaned_timeseries.csv" - ) - if not os.path.exists(cleaned_path): - logger.info( - "No cleaned observations yet for %s; cannot fit spatial " - "correction model.", id, - ) - return None - - df = pd.read_csv(cleaned_path) - if "platform" not in df.columns or dense_source_platform not in df["platform"].values: - logger.info( - "No %s data available for %s; cannot fit spatial correction " - "model from it.", dense_source_platform, id, - ) - return None - - df["date"] = pd.to_datetime(df["date"]) - dense_df = df.loc[df["platform"] == dense_source_platform] - - ref_lat, ref_lon = _target_centroid(prj, target_type, id) - if ref_lat is None: - logger.warning( - "Could not determine %s centroid for %s; cannot fit spatial " - "correction model.", target_type, id, - ) - return None - - model = basic_filters.fit_spatial_correction_model( - dense_df, lat_key="lat", lon_key="lon", height_key="height", - date_key="date", ref_lat=ref_lat, ref_lon=ref_lon, **fit_kwargs, - ) - - if model is not None: - general.ifnotmakedirs(os.path.dirname(model_path)) - with open(model_path, "w") as f: - json.dump(model, f, indent=2) - - return model - - -# ============================================================================ -# Per-target run config: exclusions + per-target merging option overrides -# ============================================================================ -# -# One YAML file per target ({output}/{id}/run_config.yaml) that is -# simultaneously: (a) a human-readable log of decisions made about this -# target, (b) the actual source of truth _merge_timeseries reads to apply -# those decisions, and (c) something a user can hand-edit directly for a -# fully config-driven workflow. Interactive functions below -# (exclude_from_target, set_merging_option, ...) read-modify-write this -# same file, so a decision made once in a notebook session is exactly the -# same artifact you'd edit by hand or check into version control -- there -# is no separate "notebook state" to keep in sync with "the config". - - -def _run_config_path(prj: "Project", id) -> str: - return os.path.join(prj.dirs["output"], f"{id}", "run_config.yaml") - - -def _default_run_config(id) -> dict: - return { - "target_id": id, - "last_updated": None, - "merging_option_overrides": {}, - "exclusions": [], - } - - -def _load_run_config(prj: "Project", id) -> dict: - """Load a target's run_config.yaml, or a fresh default if none exists yet.""" - path = _run_config_path(prj, id) - if os.path.exists(path): - with open(path, "r") as f: - loaded = yaml.safe_load(f) - if loaded: - # tolerate a hand-edited file missing a key or two - defaults = _default_run_config(id) - defaults.update(loaded) - return defaults - return _default_run_config(id) - - -def _save_run_config(prj: "Project", id, config: dict) -> None: - config["last_updated"] = datetime.datetime.now().isoformat() - path = _run_config_path(prj, id) - general.ifnotmakedirs(os.path.dirname(path)) - with open(path, "w") as f: - yaml.safe_dump(config, f, sort_keys=False) - - -def _invalidate_spatial_correction_cache(prj: "Project", id) -> None: - """ - Delete any cached spatial correction model for this target, forcing - a fresh fit next time use_spatial_correction is used. Called whenever - exclusions or spatial-correction-relevant options change -- the - model may have been fit using observations that are no longer - included, and this is exactly the kind of deliberate, explicit - trigger (not routine reprocessing) that recalibration is meant for -- - see _get_or_fit_spatial_correction_model. - """ - model_path = os.path.join(prj.dirs["output"], f"{id}", "spatial_correction_model.json") - if os.path.exists(model_path): - os.remove(model_path) - logger.info( - "Invalidated cached spatial correction model for %s " - "(exclusions or related options changed).", id, - ) - - -def _invalidate_reach_slope_correction_cache(prj: "Project", id) -> None: - """ - Delete any cached reach slope correction model for this target, - forcing a fresh fit next time use_reach_slope_correction is used. - Called whenever exclusions change -- an exclusion could target SWOT - observations specifically, which is exactly what this model is fit - from (see _fit_reach_slope_correction), so a cached model could - otherwise silently keep reflecting now-excluded SWOT slope values. - """ - model_path = os.path.join( - prj.dirs["output"], f"{id}", "reach_slope_correction_model.json" - ) - if os.path.exists(model_path): - os.remove(model_path) - logger.info( - "Invalidated cached reach slope correction model for %s " - "(exclusions or related options changed).", id, - ) - - -def _fit_reach_slope_correction(prj: "Project", target_id, recalibrate: bool = False): - """ - Fit (or load a persisted) reach-level slope correction from SWOT's - own directly-measured "slope" field (RiverSP reach product), used to - reference-correct OTHER missions' (ICESat-2/Sentinel-3/6) crossings - to what they'd read at the reach's geometric midpoint. - - ONLY meaningful for reaches (prj.rivers.target_id_col == "reach_id") - -- a node is a single ~200m-spaced point, not a ~10km segment with - its own along-reach slope in the same sense. Callers must not invoke - this for node-mode projects. - - Rationale: SWOT's reach-level WSE is an aggregate over the reach's - ~50 constituent, roughly-evenly-spaced nodes, not a value evaluated - at one specific point -- for an evenly-sampled linear profile, the - mean equals the value at the mean position, so this is treated as - approximately midpoint-referenced. This is an evidence-based - inference from the RiverSP processing chain, NOT a fact directly - confirmed in SWOT's product documentation (which does not explicitly - state a reference point) -- validate against real Hydrocron - node-vs-reach output for a known reach before trusting this deeply. - - Uses the MEDIAN of all available SWOT slope observations for this - reach as a single, persistent correction -- not a per-date-specific - one -- consistent with this pipeline's existing preference (see - fit_spatial_correction_model) for a stable, once-fit value over a - per-observation one, and avoiding the complexity/fragility of - matching a specific SWOT overpass date to each individual non-SWOT - observation's date. - - Persisted to {output}/{target_id}/reach_slope_correction_model.json - -- fit once, loaded thereafter, only refit on explicit - recalibrate=True -- so past corrections don't shift retroactively - as new SWOT data arrives, same reasoning as the spatial correction - model's caching. - - Returns None if no model exists yet and there's no usable SWOT slope - data to fit one from -- callers should treat this as "no correction - available," not an error. - """ - model_path = os.path.join( - prj.dirs["output"], f"{target_id}", "reach_slope_correction_model.json" - ) - if os.path.exists(model_path) and not recalibrate: - with open(model_path, "r") as f: - return json.load(f) - - swot_path = os.path.join( - prj.dirs["output"], f"{target_id}", "raw_observations", "swot.gpkg" - ) - if not os.path.exists(swot_path): - logger.info( - "No raw SWOT observations for %s; cannot fit reach slope " - "correction.", target_id, - ) - return None - - swot_gdf = gpd.read_file(swot_path) - if "slope" not in swot_gdf.columns: - logger.warning( - "No 'slope' field found in SWOT observations for %s -- was " - "it requested in mission_options['swot']['hydrocron_fields']" - "['reaches']? Cannot fit reach slope correction.", target_id, - ) - return None - - valid_slopes = pd.to_numeric(swot_gdf["slope"], errors="coerce").dropna() - if valid_slopes.empty: - logger.info( - "No valid (non-null, numeric) SWOT slope observations for " - "%s; cannot fit reach slope correction.", target_id, - ) - return None - - model = { - "target_id": str(target_id), - "median_slope": float(valid_slopes.median()), - "n_observations": int(len(valid_slopes)), - "fitted_at": datetime.datetime.now().isoformat(), - } - - general.ifnotmakedirs(os.path.dirname(model_path)) - with open(model_path, "w") as f: - json.dump(model, f, indent=2) - - return model - - -def _apply_reach_slope_correction( - ts_df: pd.DataFrame, prj: "Project", target_id, slope_model: dict, -) -> pd.DataFrame: - """ - Apply a fitted reach slope correction (see _fit_reach_slope_correction) - to non-SWOT rows in ts_df -- adjusts "height" to what each row would - read at the reach's geometric midpoint, using its along-reach - projected position and the reach's persistent median slope. SWOT's - own rows are left untouched (already assumed midpoint-referenced -- - see _fit_reach_slope_correction's docstring for the reasoning and - its caveats). - - NOTE: the sign convention for "correction = slope x distance" here - has NOT been empirically verified against real data in this - session -- confirm it actually reduces cross-mission scatter for a - real reach (not increases it) before trusting this in production; - flip the sign if it doesn't. - - Rows without usable lat/lon (or if the target's geometry can't be - found) are left uncorrected rather than dropped. - """ - if "platform" not in ts_df.columns or "lat" not in ts_df.columns or "lon" not in ts_df.columns: - return ts_df - - target_row = prj.rivers.target_features.loc[ - prj.rivers.target_features[prj.rivers.target_id_col] == target_id - ] - if target_row.empty: - logger.warning( - "Could not find reach geometry for %s; skipping reach slope " - "correction.", target_id, - ) - return ts_df - - reach_geom_global = target_row.geometry.iloc[0] - local_crs = prj.local_crs - reach_geom_local = ( - gpd.GeoSeries([reach_geom_global], crs=target_row.crs).to_crs(local_crs).iloc[0] - ) - reach_midpoint_dist = reach_geom_local.length / 2.0 - median_slope = slope_model["median_slope"] - - mask = (ts_df["platform"] != "swot") & ts_df["lat"].notna() & ts_df["lon"].notna() - if not mask.any(): - return ts_df - - points_local = gpd.GeoSeries( - gpd.points_from_xy(ts_df.loc[mask, "lon"], ts_df.loc[mask, "lat"]), - crs=prj.global_crs, - ).to_crs(local_crs) - - along_reach_dist = points_local.apply(reach_geom_local.project) - distance_from_midpoint = along_reach_dist.values - reach_midpoint_dist - - ts_df = ts_df.copy() - ts_df.loc[mask, "height"] = ( - ts_df.loc[mask, "height"] - median_slope * distance_from_midpoint - ) - return ts_df - - -def list_target_observations(prj: "Project", target_type: str, id) -> pd.DataFrame: - """ - Summarize what observations exist for a target, at (platform, orbit) - granularity -- the "what could I exclude" view. Meant to be read - alongside plot_merging's platform-colored progress plots (which show - WHERE a problem shows up in the actual data), not a replacement for - looking at the data itself. - """ - cleaned_path = os.path.join(prj.dirs["output"], f"{id}", "all_cleaned_timeseries.csv") - if not os.path.exists(cleaned_path): - logger.warning( - "No cleaned observations yet for %s -- has create_%s_timeseries() " - "been run?", id, target_type, - ) - return pd.DataFrame(columns=["platform", "orbit", "n_points", "date_min", "date_max"]) - - df = pd.read_csv(cleaned_path) - df["date"] = pd.to_datetime(df["date"]) - group_cols = [c for c in ["platform", "orbit"] if c in df.columns] - summary = ( - df.groupby(group_cols) - .agg(n_points=("date", "size"), date_min=("date", "min"), date_max=("date", "max")) - .reset_index() - .sort_values(group_cols) - .reset_index(drop=True) - ) - return summary - - -def list_exclusions(prj: "Project", target_type: str, id) -> list: - """Current exclusion rules for a target, from its run_config.yaml.""" - return _load_run_config(prj, id)["exclusions"] - - -def exclude_from_target( - prj: "Project", target_type: str, id, - platform=None, orbit=None, date=None, reason=None, -) -> None: - """ - Exclude observations from a target's merge, at whatever granularity - is given -- a whole platform, a specific orbit/pass value, a specific - date, or any combination (all given fields must match for a row to - be excluded). Persisted to {output}/{id}/run_config.yaml. - - Applied at the start of _merge_timeseries, before any processing -- - an excluded pass never reaches bias_correct/Kalman/svr_radial at all, - rather than being fought against downstream. - - Invalidates any cached spatial correction model for this target, - since it may have been fit using data that's now excluded. - - Examples - -------- - exclude_from_target(prj, "reservoirs", my_id, platform="S3B") - exclude_from_target(prj, "reservoirs", my_id, platform="S3B", orbit=1517) - exclude_from_target(prj, "rivers", my_id, date="2024-03-19") - """ - if platform is None and orbit is None and date is None: - raise ValueError( - "Specify at least one of platform, orbit, or date to exclude." - ) - - config = _load_run_config(prj, id) - config["exclusions"].append({ - "platform": platform, - "orbit": orbit, - "date": str(date) if date is not None else None, - "reason": reason, - "added": datetime.datetime.now().isoformat(), - }) - _save_run_config(prj, id, config) - _invalidate_spatial_correction_cache(prj, id) - _invalidate_reach_slope_correction_cache(prj, id) - logger.info( - "Added exclusion for %s: platform=%s orbit=%s date=%s (%s)", - id, platform, orbit, date, reason or "no reason given", - ) - - -def _exclusion_value_matches(a, b) -> bool: - """ - Robust equality check for exclusion matching -- tries numeric - comparison first, so int/float/numeric-string representations of - the same value all compare correctly (e.g. 1517 == 1517.0 == "1517" - -- same int/float representation issue confirmed for - _apply_exclusions' dataframe matching, applied here too for - consistency), falling back to exact equality for non-numeric values - (e.g. a string-based orbit identifier) or when either side is None. - """ - if a is None or b is None: - return a == b - try: - return float(a) == float(b) - except (TypeError, ValueError): - return a == b - - -def remove_exclusion( - prj: "Project", target_type: str, id, - index: int = None, platform=None, orbit=None, date=None, -) -> list: - """ - Remove one or more exclusion rules, either by position (index, from - list_exclusions()) or by matching criteria -- the same - platform/orbit/date fields used to add one via exclude_from_target. - Matching by criteria is usually more convenient than looking up an - index first: e.g. remove_exclusion(prj, "reservoirs", my_id, - platform="S3B", orbit=1517) removes exactly the rule that excluded - that mission+orbit combination (or platform+beam, for ICESat-2 -- - beam values ARE the "orbit" field once concatenated with other - missions, see PRODUCT_TIMESERIES_KEYS -- there's no separate "beam" - parameter needed). - - Specify either index, or at least one of platform/orbit/date, not - both. Criteria matching removes EVERY exclusion rule whose given - fields match (fields not specified are ignored, not required to be - None on the stored rule). - - Returns the list of removed rule(s), for confirmation/logging. - """ - config = _load_run_config(prj, id) - exclusions = config["exclusions"] - criteria_given = platform is not None or orbit is not None or date is not None - - if index is not None and criteria_given: - raise ValueError( - "Specify either index OR platform/orbit/date criteria, not both." - ) - - if index is not None: - if index < 0 or index >= len(exclusions): - raise IndexError( - f"No exclusion at index {index} for {id}; there are " - f"{len(exclusions)}. See list_exclusions()." - ) - removed = [exclusions.pop(index)] - elif criteria_given: - date_str = str(date) if date is not None else None - to_remove = [ - rule for rule in exclusions - if (platform is None or _exclusion_value_matches(rule.get("platform"), platform)) - and (orbit is None or _exclusion_value_matches(rule.get("orbit"), orbit)) - and (date is None or rule.get("date") == date_str) - ] - if not to_remove: - raise ValueError( - f"No exclusion found matching platform={platform!r} " - f"orbit={orbit!r} date={date!r} for {id}. See list_exclusions()." - ) - for rule in to_remove: - exclusions.remove(rule) - removed = to_remove - else: - raise ValueError( - "Specify index, or at least one of platform/orbit/date, to " - "identify which exclusion(s) to remove." - ) - - _save_run_config(prj, id, config) - _invalidate_spatial_correction_cache(prj, id) - _invalidate_reach_slope_correction_cache(prj, id) - logger.info("Removed %d exclusion(s) for %s: %s", len(removed), id, removed) - return removed - - -def set_merging_option(prj: "Project", target_type: str, id, **kwargs) -> None: - """ - Override one or more merging_options for just this one target, - persisted the same way as exclusions (highest-priority layer: these - override prj.reservoirs/rivers.merging_options, which override the - DEFAULT_*_MERGING_OPTIONS defaults). - - Example: set_merging_option(prj, "reservoirs", my_id, svr_radial_err=0.5) - """ - config = _load_run_config(prj, id) - config["merging_option_overrides"].update(kwargs) - _save_run_config(prj, id, config) - if "use_spatial_correction" in kwargs or "spatial_correction_dense_source" in kwargs: - _invalidate_spatial_correction_cache(prj, id) - if "use_reach_slope_correction" in kwargs: - _invalidate_reach_slope_correction_cache(prj, id) - logger.info("Updated merging options for %s: %s", id, kwargs) - - -def _apply_exclusions(df: pd.DataFrame, exclusions: list) -> pd.DataFrame: - """ - Filter out rows matching any exclusion rule. Within one rule, every - specified field (platform/orbit/date) must match for a row to be - excluded by it; a row is dropped if it matches ANY rule. - """ - if not exclusions: - return df - - keep_mask = pd.Series(True, index=df.index) - for rule in exclusions: - rule_mask = pd.Series(True, index=df.index) - if rule.get("platform") is not None: - rule_mask &= df["platform"] == rule["platform"] - if rule.get("orbit") is not None: - if "orbit" in df.columns: - # Compare numerically when possible, not as strings -- - # a real orbit column commonly gets upcast to float64 by - # pandas the moment ANY value in it is missing (very - # common in real satellite data), so a genuine orbit - # value of 1517 reads as 1517.0 in the dataframe while a - # YAML-loaded exclusion rule reads it as the plain int - # 1517. Comparing as strings ("1517.0" vs "1517") then - # silently matches nothing -- confirmed as a real, - # reproducible bug, not a hypothetical one. Falls back - # to string comparison only if the orbit value genuinely - # isn't numeric (e.g. a string-based identifier). - try: - target_orbit = float(rule["orbit"]) - rule_mask &= ( - pd.to_numeric(df["orbit"], errors="coerce") == target_orbit - ) - except (TypeError, ValueError): - rule_mask &= df["orbit"].astype(str) == str(rule["orbit"]) - else: - rule_mask &= False - if rule.get("date") is not None: - rule_mask &= df["date"].dt.floor("D").astype(str) == str(rule["date"]) - keep_mask &= ~rule_mask - - return df.loc[keep_mask].reset_index(drop=True) - - -DEFAULT_RIVER_MERGING_OPTIONS = { - # Mostly a starting point copied from the reservoir defaults and NOT - # independently validated against real river data the way the - # reservoir defaults were validated this session -- river dynamics - # differ genuinely (e.g. a real, expected along-reach gradient), so - # do not assume the rest of these are correct without checking. - # svr_radial_err/gamma below ARE an explicit exception (set directly, - # not copied): gamma x100 (~15-day lengthscale, vs DAHITI's ~151-day - # lake value) and err=1.0, matching the same oversmoothing reasoning - # as the reservoir defaults, just with a shorter lengthscale given - # rivers can change faster still. - "window_km": 1.5, - "svr_linear_err": 0.1, - "svr_linear_epsilon": 0.1, - "svr_radial_err": 1.0, - "svr_radial_rbf_c": 10000, - "svr_radial_gamma": 0.0000438 * 100, - "svr_radial_epsilon": 0.1, - "bias_time_bin": "20D", - "bias_min_overlap": 1, - # Same reasoning/evidence as the reservoir default (see - # DEFAULT_RESERVOIR_MERGING_OPTIONS) for switching from "platform" to - # "platform_orbit" -- but this is carried over, not independently - # verified against real river data. A single river target (node/reach) - # is a much smaller footprint than a reservoir, so it's genuinely - # unclear whether the same within-platform configuration split - # (e.g. S3B's two distinct crossing geometries) would even occur at - # this scale -- check real per-target bias diagnostics once river - # data exists before trusting this. - "bias_group_by": "platform_orbit", - "bias_centroid_warn_km": 5.0, - # Off by default, same reasoning as reservoirs. NOTE: an earlier - # version of this comment claimed a river target's footprint is - # "much smaller than a reservoir" -- that's wrong for reaches - # specifically (confirmed ~10km typical length, comparable to or - # larger than many reservoirs), so distance_penalty/spatial - # correction may matter just as much for reaches as for reservoirs. - # It remains true that these tools address spread WITHIN one - # target's own crossing footprint, never the natural gradient - # BETWEEN different targets, which should never be "corrected away". - "distance_penalty_scale_per_km": None, - "use_spatial_correction": False, - "spatial_correction_dense_source": "icesat2", - # Off by default. ONLY meaningful when - # prj.rivers.target_id_col == "reach_id" -- reference-corrects - # non-SWOT crossings (ICESat-2/Sentinel-3/6) to what they'd read at - # the reach's geometric midpoint, using SWOT's own directly-measured - # "slope" field (see _fit_reach_slope_correction/ - # _apply_reach_slope_correction). Requires "slope" to be present in - # mission_options["swot"]["hydrocron_fields"]["reaches"]. The - # midpoint-referenced assumption for SWOT's own reach WSE is an - # evidence-based inference from the RiverSP processing chain, not a - # fact directly confirmed in SWOT's documentation -- and the sign of - # the correction has not been empirically verified against real - # data in this session. Validate both before trusting this in - # production. - "use_reach_slope_correction": False, -} - - -def _merge_timeseries(prj: "Project", target_type: str) -> None: - """Merge multi-mission timeseries into combined datasets, for either - reservoirs or river targets (nodes/reaches).""" - target_ids = _get_target_ids(prj, target_type) - ids_with_cleaned = [ - id - for id in target_ids - if os.path.exists( - os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") - ) - ] - if not ids_with_cleaned: - logger.warning( - "No cleaned observations found for any %s; skipping timeseries merging.", - target_type, - ) - return - - default_options = ( - DEFAULT_RESERVOIR_MERGING_OPTIONS - if target_type == "reservoirs" - else DEFAULT_RIVER_MERGING_OPTIONS - ) - # prj.reservoirs.merging_options / prj.rivers.merging_options are the - # intended per-target-type override locations (set from the - # respective YAML config sections); fall back to the older shared - # prj.merging_options for backward compatibility if the per-type one - # isn't set yet. - target_owner = prj.reservoirs if target_type == "reservoirs" else prj.rivers - overrides = getattr(target_owner, "merging_options", None) - if overrides is None: - overrides = getattr(prj, "merging_options", None) - - for id in tqdm(ids_with_cleaned, desc=f"Merging product timeseries ({target_type})"): - ts_list = [] - for product in prj.to_process: - df = _load_product_timeseries( - os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations"), - ".csv", - [product], - pd.read_csv, - ) - if df is not None: - df["date"] = pd.to_datetime( - df.date, format="mixed", utc=True - ).dt.tz_convert(None) - df = df.sort_values(by="date") - ts_list.append( - timeseries.Timeseries( - df, date_key="date", height_key="height", - **PRODUCT_TIMESERIES_KEYS.get(product, {}), - ) - ) - - if len(ts_list) > 0: - ts = timeseries.concat(ts_list) - - data_dir = os.path.join(prj.dirs["output"], f"{id}") - general.ifnotmakedirs(data_dir) - - run_config = _load_run_config(prj, id) - - merging_options = dict(default_options) - merging_options.update(overrides or {}) - # Per-target overrides (from notebook calls to - # set_merging_option, or hand-edited in run_config.yaml) take - # priority over project-wide settings for just this target. - merging_options.update(run_config.get("merging_option_overrides", {})) - distance_penalty_scale = merging_options.pop( - "distance_penalty_scale_per_km", None - ) - use_spatial_correction = merging_options.pop( - "use_spatial_correction", False - ) - spatial_correction_dense_source = merging_options.pop( - "spatial_correction_dense_source", "icesat2" - ) - # Off by default. Only meaningful for reach-mode river - # projects (a node is a single ~200m point, not a ~10km - # segment with its own along-reach slope) -- see - # _fit_reach_slope_correction for the full reasoning and the - # sign-convention caveat that should be checked against real - # data before trusting this in production. - use_reach_slope_correction = merging_options.pop( - "use_reach_slope_correction", False - ) - - # Apply exclusions BEFORE exporting all_cleaned_timeseries.csv - # (not just before merge processing) -- this file is meant to - # reflect what's actually being worked with, and writing it - # before exclusions were applied meant it always showed - # excluded data regardless of how many times you re-ran, - # which looked exactly like a stale file from an old run but - # was actually happening on every single run. The full, - # pre-exclusion record is still available per-mission in - # cleaned_observations/{product}.csv (written earlier, in - # _clean_timeseries, before any exclusion is applied) -- so - # nothing is lost by making this file reflect exclusions. - exclusions = run_config.get("exclusions", []) - if exclusions: - before = len(ts.df) - ts.df = _apply_exclusions(ts.df, exclusions) - logger.info( - "%s: %d exclusion rule(s) applied, %d/%d observations kept.", - id, len(exclusions), len(ts.df), before, - ) - - if ( - use_reach_slope_correction - and target_type == "rivers" - and getattr(prj.rivers, "target_id_col", None) == "reach_id" - ): - slope_model = _fit_reach_slope_correction(prj, id) - if slope_model is not None: - ts.df = _apply_reach_slope_correction(ts.df, prj, id, slope_model) - logger.info( - "%s: applied reach slope correction (median_slope=%.6g " - "from %d SWOT observations).", - id, slope_model["median_slope"], slope_model["n_observations"], - ) - else: - logger.info( - "%s: use_reach_slope_correction is enabled but no " - "usable SWOT slope data was found; skipping " - "correction for this target.", id, - ) - - ts.export_csv(os.path.join(data_dir, "all_cleaned_timeseries.csv")) - - ref_lat, ref_lon = _target_centroid(prj, target_type, id) - - spatial_correction_model = None - if use_spatial_correction: - spatial_correction_model = _get_or_fit_spatial_correction_model( - prj, target_type, id, - dense_source_platform=spatial_correction_dense_source, - ) - - ts = ts.merge( - save_progress=True, - dir=os.path.join(data_dir, "merged_progress"), - ref_lat=ref_lat, - ref_lon=ref_lon, - distance_penalty_scale_per_km=distance_penalty_scale, - spatial_correction_model=spatial_correction_model, - **merging_options, - ) - ts.export_csv(os.path.join(data_dir, "merged_timeseries.csv")) - - -def _merge_reservoirs_timeseries(prj: "Project") -> None: - """Merge multi-mission timeseries into combined datasets, for reservoirs.""" - _merge_timeseries(prj, "reservoirs") - - -def _merge_rivers_timeseries(prj: "Project") -> None: - """Merge multi-mission timeseries into combined datasets, for river targets.""" - _merge_timeseries(prj, "rivers") - - -# ============================================================================ -# RESERVOIRS: Summaries & Visualization -# ============================================================================ - - -def generate_reservoirs_summaries( - prj: "Project", show: bool = False, save: bool = True -) -> None: - """Generate per-reservoir plotting summaries. - - Parameters - ---------- - prj : Project - Project instance with reservoirs configuration - show : bool - Whether to display plots interactively - save : bool - Whether to save plots to disk - """ - if not hasattr(prj, "reservoirs"): - return - - for reservoir_id in prj.reservoirs.download_gdf[prj.reservoirs.id_key]: - plotting.plot_crossings( - gdf=prj.reservoirs.gdf, - id_key=prj.reservoirs.id_key, - reservoir_id=reservoir_id, - output_dir=prj.dirs["output"], - reservoir_type=prj.reservoirs.type, - show=show, - save=save, - ) - - plotting.plot_cleaning( - reservoir_id=reservoir_id, - output_dir=prj.dirs["output"], - get_unfiltered_fn=lambda id, products: _load_product_timeseries( - os.path.join(prj.dirs["output"], f"{id}", "raw_observations"), - ".gpkg", - products, - lambda path: gpd.read_file(path).drop(columns=["geometry"]), - ), - get_cleaned_fn=lambda id, products: _load_and_parse_cleaned_timeseries( - prj, id, products - ), - get_merged_fn=lambda id: _load_merged_timeseries(prj, id), - reservoir_type=prj.reservoirs.type, - show=show, - save=save, - products=getattr(prj, "to_process", None), - ) - - plotting.plot_merging( - reservoir_id=reservoir_id, - output_dir=prj.dirs["output"], - reservoir_type=prj.reservoirs.type, - show=show, - save=save, - ) - - -def _load_and_parse_cleaned_timeseries(prj, id, products): - """Load cleaned observations and parse dates.""" - df = _load_product_timeseries( - os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations"), - ".csv", - products, - pd.read_csv, - ) - if df is not None: - df["date"] = pd.to_datetime(df.date, format="mixed", utc=True).dt.tz_convert( - None - ) - df = df.sort_values(by="date") - return df - - -def _load_merged_timeseries(prj, id): - """Load merged timeseries if it exists.""" - data_path = os.path.join(prj.dirs["output"], f"{id}", "merged_timeseries.csv") - - if os.path.exists(data_path): - df = pd.read_csv(data_path) - df["date"] = pd.to_datetime(df.date) - df = df.sort_values(by="date") - return df - else: - logger.warning( - "%s does not exist, be sure to merge product timeseries first!", - data_path, - ) - return None - - -# ============================================================================ -# RIVERS: Summaries & Visualization -# ============================================================================ - - -def _project_num_months(prj: "Project") -> int: - """ - Approximate number of months spanned by the project's configured - date range -- used as a minimum-observation-count threshold for - plotting (see _has_enough_observations_to_plot). Falls back to 1 if - the project-level dates aren't resolvable for some reason. - """ - project_cfg = prj.config.get("project", {}) - start = project_cfg.get("startdate") - end = project_cfg.get("enddate") - if not start or not end: - return 1 - - start_date = datetime.date(*start) if isinstance(start, list) else start - end_date = datetime.date(*end) if isinstance(end, list) else end - months = ( - (end_date.year - start_date.year) * 12 - + (end_date.month - start_date.month) - + 1 - ) - return max(months, 1) - - -def _has_enough_observations_to_plot(prj: "Project", target_id, min_months: int) -> bool: - """ - Whether a target has enough merged observations to be worth - plotting -- more than min_months (the project's date range in - months) or more than 2, whichever is larger. A reach/reservoir with - only 1-2 points produces a plot that adds noise without telling you - anything. - """ - df = _load_merged_timeseries(prj, target_id) - if df is None: - return False - threshold = max(min_months, 2) - return len(df) > threshold - - -def generate_rivers_summaries( - prj: "Project", show: bool = False, save: bool = True -) -> None: - """Generate per-river plotting summaries. - - Parameters - ---------- - prj : Project - Project instance with rivers configuration - show : bool - Whether to display plots interactively - save : bool - Whether to save plots to disk - """ - if not hasattr(prj, "rivers"): - return - - waterbody_groups = _group_river_targets_by_waterbody(prj) - min_months = _project_num_months(prj) - - for wb_id, target_ids in waterbody_groups.items(): - # Only plot targets with enough observations to be worth looking - # at -- applies to all three plot types (map, time series, merge - # progress) so a target excluded from one isn't confusingly still - # shown in another. - plottable_ids = [ - t for t in target_ids - if _has_enough_observations_to_plot(prj, t, min_months) - ] - if not plottable_ids: - logger.info( - "Skipping plots for waterbody %s -- no targets with more " - "than %d observations.", wb_id, max(min_months, 2), - ) - continue - - # Compute the actual extraction corridor (same buffer resolution - # used for real extraction, see _river_target_corridor) so the - # shaded area shown is exactly what extraction uses, not an - # approximation -- lets you visually judge whether width-based - # buffering produced a reasonable corridor for this waterbody - # (e.g. a lake-flagged reach whose SWORD width reflects a much - # wider lake extent) without needing external knowledge of the - # real river geometry. - corridor_gdf = _river_target_corridor( - prj, plottable_ids, - buffer_meters=getattr(prj.rivers, "extraction_buffer_meters", None), - width_buffer_factor=getattr(prj.rivers, "width_buffer_factor", 1.05), - ) - corridor_geometry = corridor_gdf.geometry.iloc[0] if corridor_gdf is not None else None - - plotting.plot_river_crossings( - prj, wb_id, plottable_ids, prj.dirs["output"], show=show, save=save, - corridor_geometry=corridor_geometry, - ) - - plotting.plot_river_data( - prj, wb_id, plottable_ids, prj.dirs["output"], - get_merged_fn=lambda id: _load_merged_timeseries(prj, id), - show=show, save=save, - ) - - for target_id in plottable_ids: - plotting.plot_merging( - reservoir_id=target_id, - output_dir=prj.dirs["output"], - reservoir_type="river", - show=show, - save=save, - ) - - -# ============================================================================ -# MIKEIO -# ============================================================================ - - -def _export_cleaned_to_dfs0(prj: "Project") -> None: - """Export cleaned timeseries observations to dfs0 format. - - Parameters - ---------- - prj : Project - Project instance with reservoirs configuration - - Notes - ----- - Exports height data from cleaned CSV observations to dfs0 format - for each product in each reservoir's cleaned_observations folder. - """ - ids_with_cleaned = [ - id - for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] - if os.path.exists( - os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") - ) - ] - - if not ids_with_cleaned: - logger.warning( - "No cleaned observations found for any reservoir; skipping dfs0 export." - ) - return - - for id in ids_with_cleaned: - cleaned_dir = os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") - - for product in tqdm( - prj.to_process, desc="Exporting cleaned observations to dfs0" - ): - csv_path = os.path.join(cleaned_dir, f"{product}.csv") - - if not os.path.exists(csv_path): - continue - - try: - # Load and prepare data - df = pd.read_csv(csv_path) - - # Parse and set datetime index - df["date"] = pd.to_datetime( - df.date, format="mixed", utc=True - ).dt.tz_convert(None) - df = df.set_index("date") - df = df.sort_index() - - # Remove rows with NaN heights - df = df.dropna(subset=["height"]) - - if len(df) == 0: - logger.warning( - "No valid height data for %s in %s after cleaning", - product, - id, - ) - continue - - # Create Dataset and export to dfs0 - items = {"height": mikeio.ItemInfo(mikeio.EUMType.Water_Level)} - ds = mikeio.from_pandas(df[["height"]], items=items) - - # Write dfs0 file - dfs0_path = os.path.join(cleaned_dir, f"{product}.dfs0") - ds.to_dfs(dfs0_path) - - logger.debug("Exported dfs0: %s", dfs0_path) - - except Exception as exc: - logger.error( - "Failed to export %s to dfs0 for %s: %s", - product, - id, - exc, - ) \ No newline at end of file diff --git a/HydroEO/flows/__init__.py b/HydroEO/flows/__init__.py new file mode 100644 index 0000000..f01079d --- /dev/null +++ b/HydroEO/flows/__init__.py @@ -0,0 +1,134 @@ +"""Standalone flow functions for HydroEO pipelines. + +These functions implement the core download, processing, and visualization +logic previously embedded in Reservoirs and Rivers classes. They operate on +Project state and external data, with no direct method dependencies. + +This package replaces what used to be a single ~3,100-line flows.py file, +split by concern (init/download/extract-clean-merge/run-config/summaries) +-- see each submodule's docstring for why its particular grouping was +chosen. Every name below (public and private) is re-exported here so that +`from HydroEO import flows; flows.` keeps working exactly as it did +against the old single-file module, including for tests that patch +private helpers via `patch.object(flows, "_name")`. +""" + +import mikeio # noqa: F401 -- re-exported so `flows.mikeio` resolves (see _reservoir_pipeline.py) + +from HydroEO import plotting # noqa: F401 -- re-exported so `flows.plotting`/`HydroEO.flows.plotting` resolves + +from ._reservoir_init import ( + _assign_pld_id, + _download_pld, + _flag_missing_priors, + initialize_reservoirs, +) +from ._river_init import ( + _ensure_sword_database, + _prepare_rivers_from_sword, + initialize_rivers, +) +from ._sentinel_shared import ( + _download_sentinel_for_target, + _sentinel6_use_earthdata, +) +from ._reservoir_download import ( + _download_reservoirs_icesat2, + _download_reservoirs_sentinel, + _download_reservoirs_swot, + download_reservoirs, +) +from ._river_download import ( + _download_rivers_icesat2, + _download_rivers_sentinel, + _download_swot_hydrocron_timeseries, + _get_latest_hydrocron_obs_date, + download_rivers, +) +from ._river_common import ( + _assign_points_to_river_targets, + _group_river_targets_by_waterbody, + _iter_geometry_pieces, + _river_extraction_buffer_meters, + _simplify_to_one_polygon, +) +from ._river_pipeline import ( + _clean_rivers_timeseries, + _extract_rivers_icesat2_observations, + _extract_rivers_sentinel_observations, + _extract_rivers_swot_observations, + _extract_rivers_timeseries, + _merge_rivers_timeseries, + create_rivers_timeseries, +) +from ._reservoir_pipeline import ( + _clean_reservoirs_timeseries, + _export_cleaned_to_dfs0, + _extract_icesat2_observations, + _extract_reservoirs_timeseries, + _extract_sentinel_observations, + _extract_swot_observations, + _merge_reservoirs_timeseries, + create_reservoirs_timeseries, +) +from ._constants import ( + DEFAULT_RESERVOIR_MERGING_OPTIONS, + DEFAULT_RIVER_MERGING_OPTIONS, + PRODUCT_TIMESERIES_KEYS, +) +from ._clean_engine import ( + _clean_timeseries, +) +from ._run_config import ( + _apply_exclusions, + _apply_reach_slope_correction, + _default_run_config, + _exclusion_value_matches, + _fit_reach_slope_correction, + _get_or_fit_spatial_correction_model, + _get_target_ids, + _invalidate_reach_slope_correction_cache, + _invalidate_spatial_correction_cache, + _load_run_config, + _reservoir_centroid, + _run_config_path, + _save_run_config, + _target_centroid, + exclude_from_target, + list_exclusions, + list_target_observations, + remove_exclusion, + set_merging_option, +) +from ._merge_engine import ( + _merge_timeseries, +) +from ._summaries import ( + _has_enough_observations_to_plot, + _load_and_parse_cleaned_timeseries, + _load_merged_timeseries, + _load_product_timeseries, + _project_num_months, + _river_target_corridor, + generate_reservoirs_summaries, + generate_rivers_summaries, +) + +__all__ = [ + # Reservoirs + "initialize_reservoirs", + "download_reservoirs", + "create_reservoirs_timeseries", + "generate_reservoirs_summaries", + # Rivers + "initialize_rivers", + "download_rivers", + "create_rivers_timeseries", + "generate_rivers_summaries", + # Per-target exclusion/merging-option API (used by Project) + "list_target_observations", + "exclude_from_target", + "list_exclusions", + "remove_exclusion", + "set_merging_option", +] diff --git a/HydroEO/flows/_clean_engine.py b/HydroEO/flows/_clean_engine.py new file mode 100644 index 0000000..df8470c --- /dev/null +++ b/HydroEO/flows/_clean_engine.py @@ -0,0 +1,86 @@ +"""Shared cleaning engine used by both reservoirs and rivers. + +_clean_timeseries applies per-mission processing filters generically for +either target_type -- called by _clean_reservoirs_timeseries (in +_reservoir_pipeline.py) and _clean_rivers_timeseries (in +_river_pipeline.py). Not itself patched as a sibling of either wrapper in +the test suite, so it's free to live in its own module. +""" + +import logging +import os + +import geopandas as gpd +from tqdm import tqdm + +from HydroEO.utils import general, timeseries +from ._constants import PRODUCT_TIMESERIES_KEYS +from ._run_config import _get_target_ids +from ._summaries import _load_product_timeseries + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def _clean_timeseries(prj: "Project", target_type: str) -> None: + """Apply quality filters to extracted timeseries, for either + reservoirs or river targets (nodes/reaches).""" + target_ids = _get_target_ids(prj, target_type) + ids_with_raw = [ + id + for id in target_ids + if os.path.exists(os.path.join(prj.dirs["output"], f"{id}", "raw_observations")) + ] + if not ids_with_raw: + logger.warning( + "No raw observations found for any %s; skipping timeseries cleaning.", + target_type, + ) + return + + for id in tqdm(ids_with_raw, desc=f"Cleaning product timeseries ({target_type})"): + for product in prj.to_process: + df = _load_product_timeseries( + os.path.join(prj.dirs["output"], f"{id}", "raw_observations"), + ".gpkg", + [product], + lambda path: gpd.read_file(path).drop(columns=["geometry"]), + ) + if df is not None: + product_options = prj.processing_options.get( + product, + { + "processing_filters": ["elevation", "MAD"], + "elevation_min_m": 0.0, + "elevation_max_m": 8000.0, + "mad_threshold": 5.0, + }, + ) + + ts = timeseries.Timeseries( + df, date_key="date", height_key="height", + **PRODUCT_TIMESERIES_KEYS.get(product, {}), + ) + + ts.clean( + product_options.get("processing_filters", ["elevation", "MAD"]), + filter_params={ + "elevation_min_m": product_options.get("elevation_min_m", 0.0), + "elevation_max_m": product_options.get( + "elevation_max_m", 8000.0 + ), + "mad_threshold": product_options.get("mad_threshold", 5.0), + }, + ) + + export_dir = os.path.join( + prj.dirs["output"], f"{id}", "cleaned_observations" + ) + general.ifnotmakedirs(export_dir) + ts.export_csv(os.path.join(export_dir, f"{product}.csv")) + + diff --git a/HydroEO/flows/_constants.py b/HydroEO/flows/_constants.py new file mode 100644 index 0000000..276827c --- /dev/null +++ b/HydroEO/flows/_constants.py @@ -0,0 +1,181 @@ +"""Shared constants for the merge/clean pipeline (both reservoirs and rivers). + +Split out of the original flows.py so both _clean_engine.py and +_merge_engine.py (and their reservoir/river wrapper modules) can import +these without needing to import each other. +""" + +PRODUCT_TIMESERIES_KEYS = { + "sentinel3": dict( + lat_key="lat", lon_key="lon", pass_key="file_name", + platform_key="platform", orbit_key="relative_orbit", + ), + # NOTE: sentinel6 still uses "pass" as orbit_key -- NOT verified to be + # unstable the way it was for sentinel3 (confirmed empirically: on + # real data, "pass" was unique-per-crossing for every S3A/S3B visit, + # i.e. not stable at all, while "relative_orbit" genuinely repeated + # across multiple visits -- e.g. S3B crossed via 2 distinct stable + # configurations, with real biases of -0.14m and +0.22m that a + # platform-only grouping was averaging into one misleading +0.04m). + # Sentinel-6 may have the same "pass" instability and may also have + # its own "relative_orbit"-equivalent column, but this hasn't been + # checked against real S6 data -- don't assume the same fix applies + # without verifying first. + "sentinel6": dict( + lat_key="lat", lon_key="lon", pass_key="file_name", + platform_key="platform", orbit_key="pass", + ), + "icesat2": dict( + lat_key="lat", lon_key="lon", pass_key="pass", + platform_key="platform", orbit_key="beam", + ), + "swot": dict( + platform_key="platform", orbit_key="orbit", preset_error_key="wse_u", + ), +} + +# Default .merge() tuning for reservoirs, mirroring the shape of +# processing_options (a project-level dict of pipeline parameters) but +# applied once per reservoir rather than per-product, since .merge() runs +# on the already-combined multi-product timeseries. Override via +# prj.merging_options in project config; falls back to these reservoir- +# appropriate defaults if that attribute isn't set. +DEFAULT_RESERVOIR_MERGING_OPTIONS = { + "window_km": 1.5, + "svr_linear_err": 0.1, + "svr_linear_epsilon": 0.1, + # Both updated from DAHITI's lake-tuned defaults (err=0.1, gamma= + # 0.0000438) based on real reservoir data validated this session -- + # the lake-tuned gamma implied a ~151-day smoothing lengthscale, far + # too coarse for a reservoir with real multi-week transitions (see + # the svr_radial oversmoothing discussion). err=1.0 (river-like, + # rather than the stricter lake value) and gamma x50 (~21-day + # lengthscale instead of ~151 days) let the trend actually track + # real fast changes instead of rejecting them as if they were noise. + "svr_radial_err": 1.0, + "svr_radial_rbf_c": 10000, + "svr_radial_gamma": 0.0000438 * 50, + "svr_radial_epsilon": 0.1, + # Confirmed on real data across two reservoirs: revisit sparsity varies a + # lot (e.g. one reservoir had icesat2/S3A/S3B visiting only 7/14/13 + # distinct days all year). At "10D"/3, sparse sources can fail to ever + # find 3 overlapping bins and get dropped as unanchored ENTIRELY (not + # just trimmed) -- confirmed: this silently dropped 2 of 3 missions + # (icesat2, S3B) for one real reservoir. "20D"/1 recovered all of it. + # Widening is monotonically safe against data loss (a wider window can + # only find equal-or-more overlapping bins, never fewer) -- the + # tradeoff is a very wide bin could blur real water-level change within + # the window into the bias estimate; 20D is a modest widening, not an + # extreme one. + "bias_time_bin": "20D", + "bias_min_overlap": 1, + # Confirmed empirically on real data: "platform_orbit" (using + # orbit_key -- now sentinel3's verified-stable "relative_orbit" + # column, see PRODUCT_TIMESERIES_KEYS) reveals genuine within-platform + # bias heterogeneity that "platform" alone was masking. One real + # reservoir's S3B crosses via two distinct, independently stable + # configurations (5 days on one, 8 on the other) with biases of + # -0.14m and +0.22m respectively -- "platform" grouping averaged + # these into one misleading +0.04m. Same pattern for ICESat-2's + # beams (orbit_key="beam"): per-beam biases ranged 0.06-0.18m under + # "platform_orbit", collapsed to one number under "platform". Total + # kept-row count was IDENTICAL either way on the reservoir tested + # (3182/4901) -- this is a precision gain, not a data-loss risk, at + # least for sentinel3/icesat2. NOTE: sentinel6 still uses "pass" as + # orbit_key (unverified whether it's stable or has a + # relative_orbit-equivalent -- see PRODUCT_TIMESERIES_KEYS) -- if + # it's actually unstable like sentinel3's old "pass" mapping was, + # "platform_orbit" could fragment sentinel6 into single-crossing + # sources. Recheck against real sentinel6 data before trusting this + # default for a project relying heavily on sentinel6. + "bias_group_by": "platform_orbit", + # Not a spatial correction -- just flags (and records in + # ts.bias_correct_diagnostics) when a source's observations are + # centered far from the anchor's, since for a large/elongated + # reservoir some of the estimated bias could be real spatial signal. + # Worth a closer look per-reservoir if this fires, not an error. + "bias_centroid_warn_km": 5.0, + # Off by default -- inflates Kalman input error by distance from the + # reservoir polygon's own centroid, addressing crossings that may be + # hydraulically unrepresentative (e.g. far upstream, subject to real + # slope bias) even when ADM alone reports them as highly precise. Set + # to a real value (m of extra error per km of distance) to enable -- + # the right scale depends on the true magnitude of upstream slope bias + # for your reservoirs, which needs empirical tuning, not a guessed + # default. + "distance_penalty_scale_per_km": None, + # Off by default -- a genuine height correction (not just error + # inflation) using a spatial deviation model fit once from a dense + # source (default ICESat-2) and persisted to disk per reservoir (see + # _get_or_fit_spatial_correction_model) so past corrections don't + # shift retroactively as new data arrives. Turn on once you've + # confirmed (as we did empirically) that the target reservoir shows a + # real, day-to-day-consistent spatial deviation pattern -- fitting + # requires several qualifying dense-source days (see + # fit_spatial_correction_model's min_days), and silently does nothing + # if there isn't enough dense-source data yet. + "use_spatial_correction": False, + "spatial_correction_dense_source": "icesat2", +} + + +DEFAULT_RIVER_MERGING_OPTIONS = { + # Mostly a starting point copied from the reservoir defaults and NOT + # independently validated against real river data the way the + # reservoir defaults were validated this session -- river dynamics + # differ genuinely (e.g. a real, expected along-reach gradient), so + # do not assume the rest of these are correct without checking. + # svr_radial_err/gamma below ARE an explicit exception (set directly, + # not copied): gamma x100 (~15-day lengthscale, vs DAHITI's ~151-day + # lake value) and err=1.0, matching the same oversmoothing reasoning + # as the reservoir defaults, just with a shorter lengthscale given + # rivers can change faster still. + "window_km": 1.5, + "svr_linear_err": 0.1, + "svr_linear_epsilon": 0.1, + "svr_radial_err": 1.0, + "svr_radial_rbf_c": 10000, + "svr_radial_gamma": 0.0000438 * 100, + "svr_radial_epsilon": 0.1, + "bias_time_bin": "20D", + "bias_min_overlap": 1, + # Same reasoning/evidence as the reservoir default (see + # DEFAULT_RESERVOIR_MERGING_OPTIONS) for switching from "platform" to + # "platform_orbit" -- but this is carried over, not independently + # verified against real river data. A single river target (node/reach) + # is a much smaller footprint than a reservoir, so it's genuinely + # unclear whether the same within-platform configuration split + # (e.g. S3B's two distinct crossing geometries) would even occur at + # this scale -- check real per-target bias diagnostics once river + # data exists before trusting this. + "bias_group_by": "platform_orbit", + "bias_centroid_warn_km": 5.0, + # Off by default, same reasoning as reservoirs. NOTE: an earlier + # version of this comment claimed a river target's footprint is + # "much smaller than a reservoir" -- that's wrong for reaches + # specifically (confirmed ~10km typical length, comparable to or + # larger than many reservoirs), so distance_penalty/spatial + # correction may matter just as much for reaches as for reservoirs. + # It remains true that these tools address spread WITHIN one + # target's own crossing footprint, never the natural gradient + # BETWEEN different targets, which should never be "corrected away". + "distance_penalty_scale_per_km": None, + "use_spatial_correction": False, + "spatial_correction_dense_source": "icesat2", + # Off by default. ONLY meaningful when + # prj.rivers.target_id_col == "reach_id" -- reference-corrects + # non-SWOT crossings (ICESat-2/Sentinel-3/6) to what they'd read at + # the reach's geometric midpoint, using SWOT's own directly-measured + # "slope" field (see _fit_reach_slope_correction/ + # _apply_reach_slope_correction). Requires "slope" to be present in + # mission_options["swot"]["hydrocron_fields"]["reaches"]. The + # midpoint-referenced assumption for SWOT's own reach WSE is an + # evidence-based inference from the RiverSP processing chain, not a + # fact directly confirmed in SWOT's documentation -- and the sign of + # the correction has not been empirically verified against real + # data in this session. Validate both before trusting this in + # production. + "use_reach_slope_correction": False, +} + + diff --git a/HydroEO/flows/_merge_engine.py b/HydroEO/flows/_merge_engine.py new file mode 100644 index 0000000..a8331a2 --- /dev/null +++ b/HydroEO/flows/_merge_engine.py @@ -0,0 +1,191 @@ +"""Shared merge engine used by both reservoirs and rivers. + +_merge_timeseries applies the merge()/Kalman/svr_radial pipeline +generically for either target_type -- called by +_merge_reservoirs_timeseries (in _reservoir_pipeline.py) and +_merge_rivers_timeseries (in _river_pipeline.py). Not itself patched as a +sibling of either wrapper in the test suite, so it's free to live in its +own module. +""" + +import logging +import os + +import pandas as pd +from tqdm import tqdm + +from HydroEO.utils import general, timeseries +from ._constants import ( + PRODUCT_TIMESERIES_KEYS, + DEFAULT_RESERVOIR_MERGING_OPTIONS, + DEFAULT_RIVER_MERGING_OPTIONS, +) +from ._run_config import ( + _get_target_ids, + _target_centroid, + _load_run_config, + _apply_exclusions, + _get_or_fit_spatial_correction_model, + _fit_reach_slope_correction, + _apply_reach_slope_correction, +) +from ._summaries import _load_product_timeseries + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def _merge_timeseries(prj: "Project", target_type: str) -> None: + """Merge multi-mission timeseries into combined datasets, for either + reservoirs or river targets (nodes/reaches).""" + target_ids = _get_target_ids(prj, target_type) + ids_with_cleaned = [ + id + for id in target_ids + if os.path.exists( + os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") + ) + ] + if not ids_with_cleaned: + logger.warning( + "No cleaned observations found for any %s; skipping timeseries merging.", + target_type, + ) + return + + default_options = ( + DEFAULT_RESERVOIR_MERGING_OPTIONS + if target_type == "reservoirs" + else DEFAULT_RIVER_MERGING_OPTIONS + ) + # prj.reservoirs.merging_options / prj.rivers.merging_options are the + # intended per-target-type override locations (set from the + # respective YAML config sections); fall back to the older shared + # prj.merging_options for backward compatibility if the per-type one + # isn't set yet. + target_owner = prj.reservoirs if target_type == "reservoirs" else prj.rivers + overrides = getattr(target_owner, "merging_options", None) + if overrides is None: + overrides = getattr(prj, "merging_options", None) + + for id in tqdm(ids_with_cleaned, desc=f"Merging product timeseries ({target_type})"): + ts_list = [] + for product in prj.to_process: + df = _load_product_timeseries( + os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations"), + ".csv", + [product], + pd.read_csv, + ) + if df is not None: + df["date"] = pd.to_datetime( + df.date, format="mixed", utc=True + ).dt.tz_convert(None) + df = df.sort_values(by="date") + ts_list.append( + timeseries.Timeseries( + df, date_key="date", height_key="height", + **PRODUCT_TIMESERIES_KEYS.get(product, {}), + ) + ) + + if len(ts_list) > 0: + ts = timeseries.concat(ts_list) + + data_dir = os.path.join(prj.dirs["output"], f"{id}") + general.ifnotmakedirs(data_dir) + + run_config = _load_run_config(prj, id) + + merging_options = dict(default_options) + merging_options.update(overrides or {}) + # Per-target overrides (from notebook calls to + # set_merging_option, or hand-edited in run_config.yaml) take + # priority over project-wide settings for just this target. + merging_options.update(run_config.get("merging_option_overrides", {})) + distance_penalty_scale = merging_options.pop( + "distance_penalty_scale_per_km", None + ) + use_spatial_correction = merging_options.pop( + "use_spatial_correction", False + ) + spatial_correction_dense_source = merging_options.pop( + "spatial_correction_dense_source", "icesat2" + ) + # Off by default. Only meaningful for reach-mode river + # projects (a node is a single ~200m point, not a ~10km + # segment with its own along-reach slope) -- see + # _fit_reach_slope_correction for the full reasoning and the + # sign-convention caveat that should be checked against real + # data before trusting this in production. + use_reach_slope_correction = merging_options.pop( + "use_reach_slope_correction", False + ) + + # Apply exclusions BEFORE exporting all_cleaned_timeseries.csv + # (not just before merge processing) -- this file is meant to + # reflect what's actually being worked with, and writing it + # before exclusions were applied meant it always showed + # excluded data regardless of how many times you re-ran, + # which looked exactly like a stale file from an old run but + # was actually happening on every single run. The full, + # pre-exclusion record is still available per-mission in + # cleaned_observations/{product}.csv (written earlier, in + # _clean_timeseries, before any exclusion is applied) -- so + # nothing is lost by making this file reflect exclusions. + exclusions = run_config.get("exclusions", []) + if exclusions: + before = len(ts.df) + ts.df = _apply_exclusions(ts.df, exclusions) + logger.info( + "%s: %d exclusion rule(s) applied, %d/%d observations kept.", + id, len(exclusions), len(ts.df), before, + ) + + if ( + use_reach_slope_correction + and target_type == "rivers" + and getattr(prj.rivers, "target_id_col", None) == "reach_id" + ): + slope_model = _fit_reach_slope_correction(prj, id) + if slope_model is not None: + ts.df = _apply_reach_slope_correction(ts.df, prj, id, slope_model) + logger.info( + "%s: applied reach slope correction (median_slope=%.6g " + "from %d SWOT observations).", + id, slope_model["median_slope"], slope_model["n_observations"], + ) + else: + logger.info( + "%s: use_reach_slope_correction is enabled but no " + "usable SWOT slope data was found; skipping " + "correction for this target.", id, + ) + + ts.export_csv(os.path.join(data_dir, "all_cleaned_timeseries.csv")) + + ref_lat, ref_lon = _target_centroid(prj, target_type, id) + + spatial_correction_model = None + if use_spatial_correction: + spatial_correction_model = _get_or_fit_spatial_correction_model( + prj, target_type, id, + dense_source_platform=spatial_correction_dense_source, + ) + + ts = ts.merge( + save_progress=True, + dir=os.path.join(data_dir, "merged_progress"), + ref_lat=ref_lat, + ref_lon=ref_lon, + distance_penalty_scale_per_km=distance_penalty_scale, + spatial_correction_model=spatial_correction_model, + **merging_options, + ) + ts.export_csv(os.path.join(data_dir, "merged_timeseries.csv")) + + diff --git a/HydroEO/flows/_reservoir_download.py b/HydroEO/flows/_reservoir_download.py new file mode 100644 index 0000000..fd77df8 --- /dev/null +++ b/HydroEO/flows/_reservoir_download.py @@ -0,0 +1,183 @@ +"""Reservoirs: multi-mission download orchestration. + +download_reservoirs and its three per-mission workers +(_download_reservoirs_swot, _download_reservoirs_icesat2, +_download_reservoirs_sentinel) are tested together via +patch.object(flows, "_name") in tests/unit/test_flows.py -- keep them in +this one module. +""" + +import logging +import os +import datetime + +from HydroEO.satellites import swot, icesat2 +from HydroEO.utils import general +from ._river_common import _simplify_to_one_polygon +from ._sentinel_shared import _sentinel6_use_earthdata, _download_sentinel_for_target + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def download_reservoirs(prj: "Project") -> None: + """Download altimetry data for all configured missions (reservoirs mode). + + Parameters + ---------- + prj : Project + Project instance with download configuration + """ + for mission in prj.to_download: + if mission == "swot": + _download_reservoirs_swot(prj) + elif mission == "icesat2": + _download_reservoirs_icesat2(prj) + elif mission in ["sentinel3", "sentinel6"]: + _download_reservoirs_sentinel(prj, mission) + else: + logger.warning("Skipping unsupported mission in download: %s", mission) + + +def _download_reservoirs_swot(prj: "Project") -> None: + """Download SWOT Lake SP data for reservoirs.""" + download_dir = prj.dirs["swot"] + general.ifnotmakedirs(download_dir) + + startdate = prj.startdates["swot"] + enddate = prj.enddates["swot"] + + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + coords = [ + (x, y) + for x, y in prj.reservoirs.download_gdf.unary_union.envelope.exterior.coords + ] + + logger.info( + "Searching for %s for aoi from %s to %s", + swot.SWOT_LAKE_SHORT_NAME, + startdate, + enddate, + ) + results = swot.query(aoi=coords, startdate=startdate, enddate=enddate) + + # Filter to prior-lake granules + logger.info("%s products returned from query", len(results)) + to_download = [] + for result in results: + link = result.data_links()[0] + filename = link.split("/")[-1].lower() + if "_prior_" in filename or "prior" in filename.split("_"): + to_download.append(result) + logger.info("%s prior-lake granules selected for download", len(to_download)) + + _ = swot.download(to_download, download_directory=download_dir) + + files_in_dir = [ + os.path.join(download_dir, f) + for f in os.listdir(download_dir) + if f.endswith(".zip") + ] + swot.subset_by_id( + files_in_dir, prj.reservoirs.download_gdf["prior_lake_id"].astype(int).values + ) + + +def _download_reservoirs_icesat2(prj: "Project") -> None: + """Download ICESat-2 ATL13 data for reservoirs.""" + for i in prj.reservoirs.download_gdf.index: + id = prj.reservoirs.download_gdf.loc[i, prj.reservoirs.id_key] + logger.info("Downloading data for id %s", id) + + geom = _simplify_to_one_polygon(prj.reservoirs.download_gdf.loc[i, "geometry"]) + coords = list(geom.exterior.coords) + + parquet_dir = os.path.join(prj.dirs["icesat2_processed"], rf"{id}") + general.ifnotmakedirs(parquet_dir) + + startdate = prj.startdates["icesat2"] + enddate = prj.enddates["icesat2"] + + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + logger.info( + "Searching for Icesat2 ATL13 for aoi from %s to %s", startdate, enddate + ) + try: + _ = icesat2.query( + aoi=coords, + startdate=startdate, + enddate=enddate, + download_directory=parquet_dir, + atl13_options=prj.mission_options.get("icesat2", {}).get("atl13", {}), + atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields") + or None, + ) + except Exception as exc: + logger.warning("ICESat-2 download skipped for %s: %s", id, exc) + + +def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: + """Download Sentinel-3 or Sentinel-6 data for reservoirs.""" + product = "S3" if mission == "sentinel3" else "S6" + + session_token = None + session_start_time = None + + # EarthData (Sentinel-6 HR) needs no CREODIAS credentials at all -- + # only require them if we're actually going to use CREODIAS. But it + # does need its OWN upfront check -- without it, earthaccess.login() + # silently falls through to interactive prompting when nothing else + # is configured, which hangs in a non-interactive run instead of + # failing clearly (see Project._require_earthdata_credentials). + sentinel_creds = None + use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) + if use_earthdata_s6: + prj._require_earthdata_credentials() + else: + sentinel_creds = prj._require_creodias_credentials() + + for i in prj.reservoirs.download_gdf.index: + id = prj.reservoirs.download_gdf.loc[i, prj.reservoirs.id_key] + logger.info("Downloading data for id %s", id) + + coords = [ + (x, y) + for x, y in prj.reservoirs.download_gdf.loc[ + i, "geometry" + ].envelope.exterior.coords + ] + + download_dir = os.path.join(prj.dirs[mission], rf"{id}") + general.ifnotmakedirs(download_dir) + + startdate = prj.startdates[mission] + enddate = prj.enddates[mission] + + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + session_token, session_start_time = _download_sentinel_for_target( + prj, mission, product, coords, download_dir, + startdate, enddate, sentinel_creds, session_token, session_start_time, + ) + + +# ============================================================================ +# RIVERS: Download +# ============================================================================ + + diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py new file mode 100644 index 0000000..db7fd99 --- /dev/null +++ b/HydroEO/flows/_reservoir_init.py @@ -0,0 +1,140 @@ +"""Reservoirs: PLD (Prior Lake Database) initialization. + +initialize_reservoirs and the three helpers it calls (_download_pld, +_assign_pld_id, _flag_missing_priors) are tested together via +patch.object(flows, "_name") in tests/unit/test_flows.py -- keep them in +this one module so those patches keep intercepting the right calls. +""" + +import logging +import os + +import geopandas as gpd + +from HydroEO.downloaders import hydroweb +from HydroEO.utils import general + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def initialize_reservoirs(prj: "Project") -> None: + """Initialize PLD matching for reservoirs mode. + + Downloads PLD database, matches it to input reservoirs, and stores + prior_lake_id values on prj.reservoirs.gdf. + + Parameters + ---------- + prj : Project + Project instance with reservoirs config/state populated + """ + if not hasattr(prj, "reservoirs"): + return + + if "swot" not in prj.to_download and "swot" not in prj.to_process: + return + + # Download PLD if needed + _download_pld(prj) + + # Match reservoirs to PLD + _assign_pld_id(prj) + + # Export flags for missing priors + _flag_missing_priors(prj) + + # Set download geometry (for reservoirs, same as input boundaries) + prj.reservoirs.download_gdf = prj.reservoirs.gdf + + +def _download_pld(prj: "Project") -> None: + """Download PLD database to project directory.""" + pld_path = prj.dirs["pld"] + + if os.path.exists(pld_path): + logger.info("PLD located") + return + + logger.info("Downloading PLD") + download_dir = os.path.dirname(pld_path) + bounds = list(prj.reservoirs.gdf.unary_union.bounds) + raw_pld_path = prj.dirs.get("pld_raw") + + # Determine if raw_pld_path is inside project main_dir + keep_raw = getattr(prj, "keep_raw_pld", False) + effective_keep_raw = keep_raw + if raw_pld_path is not None and os.path.exists(raw_pld_path): + if not os.path.abspath(raw_pld_path).startswith( + os.path.abspath(prj.dirs["main"]) + ): + logger.warning( + "raw_pld_path '%s' is outside project folder '%s'. " + "Skipping deletion of raw PLD files to preserve external data.", + raw_pld_path, + prj.dirs["main"], + ) + effective_keep_raw = True + + hydroweb.download_PLD( + download_dir=download_dir, + bounds=bounds, + raw_pld_path=raw_pld_path, + keep_raw=effective_keep_raw, + ) + + +def _assign_pld_id(prj: "Project") -> None: + """Spatial join reservoirs with PLD to assign prior_lake_id.""" + pld = gpd.read_file(prj.dirs["pld"]) + + pld = pld.rename( + columns={"lake_id": "prior_lake_id", "res_id": "prior_res_id"} + ) + joined_gdf = gpd.sjoin_nearest( + prj.reservoirs.gdf.to_crs(prj.local_crs), + pld.to_crs(prj.local_crs), + how="left", + max_distance=prj.mission_options.get("swot", {}).get( + "pld_match_max_distance_m", 100 + ), + distance_col="dist_to_pld", + ) + joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) + + joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 + + prj.reservoirs.gdf = joined_gdf + + +def _flag_missing_priors(prj: "Project") -> None: + """Export geopackages of reservoirs present/missing in PLD to aux/PLD folder.""" + gdf = prj.reservoirs.gdf + present = gdf.loc[gdf.prior_lake_id > 0].reset_index(drop=True) + missing = gdf.loc[gdf.prior_lake_id < 0].reset_index(drop=True) + + # Output to aux/PLD/ folder + pld_dir = os.path.dirname(prj.dirs["pld"]) + present_path = os.path.join(pld_dir, "present_in_pld.gpkg") + missing_path = os.path.join(pld_dir, "missing_in_pld.gpkg") + + present.to_file(present_path, driver="GPKG") + missing.to_file(missing_path, driver="GPKG") + + logger.info( + "Out of the %s reservoirs, %s are present and %s are missing from the PLD.", + len(gdf), + len(present), + len(missing), + ) + + +# ============================================================================ +# RIVERS: Initialization +# ============================================================================ + + diff --git a/HydroEO/flows/_reservoir_pipeline.py b/HydroEO/flows/_reservoir_pipeline.py new file mode 100644 index 0000000..81818dd --- /dev/null +++ b/HydroEO/flows/_reservoir_pipeline.py @@ -0,0 +1,395 @@ +"""Reservoirs: extraction + clean + merge + dfs0-export orchestration. + +create_reservoirs_timeseries and everything it (transitively) calls -- +_extract_reservoirs_timeseries, its three per-mission workers, +_clean_reservoirs_timeseries, _merge_reservoirs_timeseries, and +_export_cleaned_to_dfs0 -- are tested together via +patch.object(flows, "_name") in tests/unit/test_flows.py and +tests/unit/test_timeseries.py, so they all live in this one module. +`mikeio` is imported here (rather than at the package level only) because +_export_cleaned_to_dfs0 is the only place it's actually called, and tests +patch it as `flows.mikeio` -- see flows/__init__.py's re-export. +""" + +import logging +import os + +import mikeio +import pandas as pd +from tqdm import tqdm + +from HydroEO.satellites import swot, icesat2, sentinel +from HydroEO.utils import general +from ._clean_engine import _clean_timeseries +from ._merge_engine import _merge_timeseries + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def create_reservoirs_timeseries(prj: "Project") -> None: + """Extract, clean, and merge timeseries for reservoirs. + + Parameters + ---------- + prj : Project + Project instance with reservoirs configuration + """ + if not hasattr(prj, "reservoirs"): + return + + # Extract raw observations from downloaded files (skips reservoirs whose + # gpkg already exists, unless prj.reservoirs.overwrite_extraction=True -- + # confirmed this re-read/re-extraction was the dominant real-world cost, + # far more than anything in clean()/merge()) + _extract_reservoirs_timeseries( + prj, overwrite=getattr(prj.reservoirs, "overwrite_extraction", False) + ) + + # Clean observations with filters + _clean_reservoirs_timeseries(prj) + + # Export to dfs0 if enabled + if getattr(prj.reservoirs, "export_to_dfs0", False): + _export_cleaned_to_dfs0(prj) + + # Merge multi-mission timeseries + _merge_reservoirs_timeseries(prj) + + +def _extract_reservoirs_timeseries(prj: "Project", overwrite: bool = False) -> None: + """Extract timeseries observations from raw downloaded files. + + Parameters + ---------- + overwrite : bool, optional + If False (default), any reservoir/mission whose output .gpkg + already exists is skipped entirely rather than re-read and + re-extracted. Confirmed on real data that this re-extraction -- + not clean()/merge() -- was the dominant cost in real end-to-end + runs (orders of magnitude larger than the merge pipeline itself). + Set True to force re-extraction (e.g. new raw downloads arrived). + """ + if "icesat2" in prj.to_process: + _extract_icesat2_observations(prj, overwrite=overwrite) + + if "sentinel3" in prj.to_process: + _extract_sentinel_observations(prj, "sentinel3", "S3", overwrite=overwrite) + + if "sentinel6" in prj.to_process: + _extract_sentinel_observations(prj, "sentinel6", "S6", overwrite=overwrite) + + if "swot" in prj.to_process: + _extract_swot_observations(prj, overwrite=overwrite) + + +def _extract_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: + """Extract ICESat-2 ATL13 observations for each reservoir. + + ICESat-2's raw data is a single atl13.parquet per reservoir that is + rewritten in full (the entire configured date range) on every + download -- unlike SWOT/Sentinel, there's no per-granule file list + to diff against a "processed" log. Re-extraction is instead gated + on whether the parquet is newer than the last extraction (e.g. + after project.update() downloaded through a later date), rather + than simply "does icesat2.gpkg already exist" -- otherwise data + added by update() would never reach the merged timeseries without + overwrite=True forcing a full reprocess of every reservoir. + """ + available_ids = [ + id + for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] + if os.path.exists( + os.path.join(prj.dirs["icesat2_processed"], f"{id}", "atl13.parquet") + ) + ] + if not available_ids: + logger.warning("No ICESat-2 downloads found; skipping timeseries extraction.") + return + + ids_to_process = [] + if overwrite: + ids_to_process = available_ids + else: + skip_count = 0 + for id in available_ids: + parquet_path = os.path.join( + prj.dirs["icesat2_processed"], f"{id}", "atl13.parquet" + ) + dst_path = os.path.join( + prj.dirs["output"], f"{id}", "raw_observations", "icesat2.gpkg" + ) + if not os.path.exists(dst_path): + ids_to_process.append(id) + elif os.path.getmtime(parquet_path) > os.path.getmtime(dst_path): + ids_to_process.append(id) + else: + skip_count += 1 + if skip_count: + logger.info( + "ICESat-2 extraction: skipping %d reservoir(s) with an existing " + "icesat2.gpkg no older than its source parquet (pass " + "overwrite=True to force re-extraction).", + skip_count, + ) + if not ids_to_process: + return + + empty_ids = [] + for id in tqdm(ids_to_process, desc="Extracting ICESat-2 ATL13 product"): + sub_gdf = prj.reservoirs.download_gdf.loc[ + prj.reservoirs.download_gdf[prj.reservoirs.id_key] == id + ] + download_dir = os.path.join(prj.dirs["icesat2_processed"], f"{id}") + dst_dir = os.path.join(prj.dirs["output"], f"{id}", "raw_observations") + general.ifnotmakedirs(dst_dir) + dst_path = os.path.join(dst_dir, "icesat2.gpkg") + + try: + icesat2.extract_observations( + src_dir=download_dir, + dst_path=dst_path, + features=sub_gdf, + atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields"), + track_keys=prj.mission_options.get("icesat2", {}).get("track_keys"), + ) + except Exception as exc: + logger.warning("Failed to extract ICESat-2 for %s: %s", id, exc) + + if not os.path.exists(dst_path): + empty_ids.append(id) + + if empty_ids: + logger.warning( + "ICESat-2 timeseries empty for: %s (no observations passed the spatial filter or the download returned no data)", + ", ".join(str(i) for i in empty_ids), + ) + + +def _extract_sentinel_observations( + prj: "Project", mission_key: str, product: str, overwrite: bool = False +) -> None: + """Extract Sentinel-3 or Sentinel-6 observations for each reservoir. + + Uses a per-reservoir "already extracted" file log (see + HydroEO.satellites.sentinel.preprocess.extract_observations's + processed_log_path) rather than a per-reservoir skip based on + whether {mission_key}.gpkg already exists -- so new subset files + downloaded after the first extraction (e.g. via project.update()) + are picked up and appended on the next run instead of being + silently ignored until overwrite=True forces a full reprocess of + every reservoir. + """ + available_ids = [ + id + for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] + if os.path.exists(os.path.join(prj.dirs[mission_key], f"{id}")) + ] + if not available_ids: + logger.warning( + "No %s downloads found; skipping timeseries extraction.", mission_key + ) + return + + empty_ids = [] + for id in tqdm(available_ids, desc=f"Extracting Sentinel-{product} product"): + sub_gdf = prj.reservoirs.download_gdf.loc[ + prj.reservoirs.download_gdf[prj.reservoirs.id_key] == id + ] + download_dir = os.path.join(prj.dirs[mission_key], f"{id}") + dst_dir = os.path.join(prj.dirs["output"], f"{id}", "raw_observations") + general.ifnotmakedirs(dst_dir) + dst_path = os.path.join(dst_dir, f"{mission_key}.gpkg") + processed_log_path = os.path.join(download_dir, "extracted.log") + + try: + sentinel.extract_observations( + src_dir=download_dir, + dst_path=dst_path, + features=sub_gdf, + sigma0_max=prj.mission_options.get(mission_key, {}).get( + "sigma0_max", 1e5 + ), + processed_log_path=processed_log_path, + overwrite=overwrite, + ) + except Exception as exc: + logger.warning("Failed to extract %s for %s: %s", mission_key, id, exc) + + if not os.path.exists(dst_path): + empty_ids.append(id) + + if empty_ids: + logger.warning( + "Sentinel-%s timeseries empty for: %s (no observations passed the spatial filter or the download returned no data)", + product, + ", ".join(str(i) for i in empty_ids), + ) + + +def _extract_swot_observations(prj: "Project", overwrite: bool = False) -> None: + """Extract SWOT Lake SP observations for all reservoirs. + + Uses a project-wide "already extracted" file log (see + HydroEO.satellites.swot.preprocess.extract_observations's + processed_log_path) rather than a per-reservoir skip based on + whether swot.gpkg already exists. SWOT granule shapefiles + accumulate forever in one shared download directory across ALL + reservoirs (see satellites.swot.preprocess.subset_by_id), so + whether a granule is "new" can only be decided at the file level, + not per reservoir -- and doing so lets new downloads (e.g. via + project.update()) get picked up and appended on the next run + instead of requiring overwrite=True to reprocess every reservoir + from the full granule history. + """ + download_dir = prj.dirs["swot"] + if not os.path.exists(download_dir): + logger.warning("No SWOT downloads found; skipping timeseries extraction.") + return + + features = prj.reservoirs.download_gdf + id_key = prj.reservoirs.id_key + processed_log_path = os.path.join(download_dir, "extracted.log") + + empty_ids = swot.extract_observations( + src_dir=download_dir, + dst_dir=prj.dirs["output"], + dst_file_name="swot.gpkg", + features=features, + id_key=id_key, + exclude_obs_id_values=prj.mission_options.get("swot", {}).get( + "exclude_obs_id_values", ["no_data"] + ), + processed_log_path=processed_log_path, + overwrite=overwrite, + ) + if empty_ids: + logger.warning( + "SWOT timeseries empty for: %s (no observations matched the prior lake ID or all were excluded)", + ", ".join(str(i) for i in empty_ids), + ) + + +# Per-product mapping from generic Timeseries key attributes to the actual +# column names each mission's extractor writes. Sentinel-3 shares +# Sentinel-6's extractor/schema (same sentinel.extract_observations +# function, see _extract_sentinel_observations). +# +# **************************************************************************** +# TERMINOLOGY TRAP -- read before touching orbit_key/pass_key for Sentinel: +# The raw Sentinel-3/6 data has TWO similarly-named but opposite-meaning +# columns: +# - "orbit": the absolute revolution counter. Unique on every single +# crossing, never repeats. USELESS as orbit_key (bias_correct needs a +# persistent identifier to accumulate overlap against -- grouping by +# something that's different every time means every "source" has +# exactly 1 observation and nothing can ever be calibrated: this is +# exactly the bug that caused every S3A/S3B track to be dropped as +# unanchored in practice). +# - "pass": the satellite-engineering term for the STABLE, REPEATING +# ground track number (same value every ~27-day repeat cycle for +# S3A/S3B). This is what orbit_key actually needs. +# Confusingly, our own framework's `pass_key` means the OPPOSITE thing (one +# specific, one-time crossing -- e.g. file_name) from what "pass" means in +# the satellite data itself (the repeating track). Do not be tempted to +# point pass_key at the raw "pass" column -- file_name is correct there. +# **************************************************************************** +# +# ICESat-2's orbit_key is "beam" (the persistent ground track/virtual +# station) -- cycle_number only matters as an ingredient of the compound +# "pass" column built at extraction time (see +# HydroEO.satellites.icesat2.preprocess.extract_observations). SWOT's +# LakeSP product is already one integrated WSE per crossing with its own +# formal uncertainty (wse_u), so it needs neither lat/lon nor pass_key -- +# see preset_error_key, and daily_mad_error's handling of it. +def _clean_reservoirs_timeseries(prj: "Project") -> None: + """Apply quality filters to extracted reservoir timeseries.""" + _clean_timeseries(prj, "reservoirs") + + +def _merge_reservoirs_timeseries(prj: "Project") -> None: + """Merge multi-mission timeseries into combined datasets, for reservoirs.""" + _merge_timeseries(prj, "reservoirs") + + +def _export_cleaned_to_dfs0(prj: "Project") -> None: + """Export cleaned timeseries observations to dfs0 format. + + Parameters + ---------- + prj : Project + Project instance with reservoirs configuration + + Notes + ----- + Exports height data from cleaned CSV observations to dfs0 format + for each product in each reservoir's cleaned_observations folder. + """ + ids_with_cleaned = [ + id + for id in prj.reservoirs.download_gdf[prj.reservoirs.id_key] + if os.path.exists( + os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") + ) + ] + + if not ids_with_cleaned: + logger.warning( + "No cleaned observations found for any reservoir; skipping dfs0 export." + ) + return + + for id in ids_with_cleaned: + cleaned_dir = os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations") + + for product in tqdm( + prj.to_process, desc="Exporting cleaned observations to dfs0" + ): + csv_path = os.path.join(cleaned_dir, f"{product}.csv") + + if not os.path.exists(csv_path): + continue + + try: + # Load and prepare data + df = pd.read_csv(csv_path) + + # Parse and set datetime index + df["date"] = pd.to_datetime( + df.date, format="mixed", utc=True + ).dt.tz_convert(None) + df = df.set_index("date") + df = df.sort_index() + + # Remove rows with NaN heights + df = df.dropna(subset=["height"]) + + if len(df) == 0: + logger.warning( + "No valid height data for %s in %s after cleaning", + product, + id, + ) + continue + + # Create Dataset and export to dfs0 + items = {"height": mikeio.ItemInfo(mikeio.EUMType.Water_Level)} + ds = mikeio.from_pandas(df[["height"]], items=items) + + # Write dfs0 file + dfs0_path = os.path.join(cleaned_dir, f"{product}.dfs0") + ds.to_dfs(dfs0_path) + + logger.debug("Exported dfs0: %s", dfs0_path) + + except Exception as exc: + logger.error( + "Failed to export %s to dfs0 for %s: %s", + product, + id, + exc, + ) \ No newline at end of file diff --git a/HydroEO/flows/_river_common.py b/HydroEO/flows/_river_common.py new file mode 100644 index 0000000..294e511 --- /dev/null +++ b/HydroEO/flows/_river_common.py @@ -0,0 +1,118 @@ +"""River geometry helpers shared by download, extraction, and summaries. + +None of these are themselves the target of a sibling patch in the test +suite (only _river_target_corridor is, which is why that one function +lives in _summaries.py instead -- see generate_rivers_summaries's test). +""" + +import logging + +import geopandas as gpd + +logger = logging.getLogger(__name__) + + +def _group_river_targets_by_waterbody(prj: "Project") -> dict: + """Return {waterbody_id: [target_id, ...]} grouping.""" + if prj.rivers.target_features is not None and len(prj.rivers.target_features) > 0: + groups: dict = {} + seen_target_ids: set = set() + for _, row in prj.rivers.target_features.iterrows(): + target_id = int(row[prj.rivers.target_id_col]) + if target_id in seen_target_ids: + continue + seen_target_ids.add(target_id) + wb_id = str(row[prj.rivers.id_key]) + groups.setdefault(wb_id, []).append(target_id) + return groups + + if prj.rivers.configured_id: + return {str(prj.rivers.configured_id): list(prj.rivers.target_ids)} + + raise ValueError( + "Unable to group river targets by waterbody. " + "Configure rivers.id or provide rivers.aoi_path with rivers.id_key." + ) + + +def _iter_geometry_pieces(geom): + """ + Yield each individual polygon from a geometry: every part of a + MultiPolygon, or the geometry itself for a plain Polygon. Used for + river downloads, where disconnected corridor pieces should each get + their own query rather than only querying the first piece (silently + dropping coverage of the rest) or merging them into one shape that + would also cover the (possibly large, irrelevant) gap between them. + """ + if hasattr(geom, "geoms"): + return list(geom.geoms) + return [geom] + + +def _simplify_to_one_polygon(geom): + """ + Collapse a MultiPolygon into a single encompassing polygon via + convex hull. Used for reservoirs: unlike rivers, a reservoir is + treated as one target regardless of how many disconnected parts its + input polygon has, so a single combined query is preferred over + splitting into several separate ones. Convex hull guarantees full + coverage of every part, at the cost of also covering some in-between + area that may not be real water -- an accepted tradeoff for treating + one reservoir as one query rather than several. + """ + if hasattr(geom, "geoms"): + return geom.convex_hull + return geom + + +def _river_extraction_buffer_meters(prj: "Project") -> float: + """ + Resolve the extraction-corridor buffer distance: prefer an explicit + prj.rivers.extraction_buffer_meters if set, else fall back to the + SWORD-intersection prj.rivers.buffer_meters, else a conservative + default. Kept as its own small function since this fallback chain + is used by both download and extraction. + """ + explicit = getattr(prj.rivers, "extraction_buffer_meters", None) + if explicit: + return explicit + if prj.rivers.buffer_meters: + return prj.rivers.buffer_meters + return 500.0 + + +def _assign_points_to_river_targets( + points, targets, target_id_col, max_distance_meters, local_crs +): + """ + Assign each point in `points` to its nearest feature in `targets` + (SWORD node or reach geometries, whichever prj.rivers.target_id_col + is configured for), dropping points farther than max_distance_meters + from any target. + + Uses gpd.sjoin_nearest rather than a custom NearestNeighbors/DBSCAN + approach -- it handles point-to-line matching natively (needed for + reaches, not just nodes), and max_distance is expressed directly in + real distance units once both inputs are reprojected to local_crs. + + Returns points (unprojected, original CRS) with target_id_col and a + _dist_to_target_m column added; rows with no target within range are + dropped entirely. + """ + points_local = points.to_crs(local_crs) + targets_local = targets[[target_id_col, "geometry"]].to_crs(local_crs) + + joined = gpd.sjoin_nearest( + points_local, + targets_local, + how="inner", + max_distance=max_distance_meters, + distance_col="_dist_to_target_m", + ) + + result = points.loc[joined.index].copy() + result[target_id_col] = joined[target_id_col].values + result["_dist_to_target_m"] = joined["_dist_to_target_m"].values + return result + + diff --git a/HydroEO/flows/_river_download.py b/HydroEO/flows/_river_download.py new file mode 100644 index 0000000..2da49a4 --- /dev/null +++ b/HydroEO/flows/_river_download.py @@ -0,0 +1,426 @@ +"""Rivers: multi-mission download orchestration. + +download_rivers and _download_swot_hydrocron_timeseries are tested +together via patch.object(flows, "_name") in tests/unit/test_flows.py -- +keep them in this one module. _download_rivers_icesat2/_sentinel and +_get_latest_hydrocron_obs_date are not themselves patched as siblings of +anything, so they're free to live here too (this mirrors +_reservoir_download.py's structure). +""" + +import logging +import os +import datetime + +import pandas as pd + +from HydroEO.satellites import icesat2, sentinel +from HydroEO.utils import general +from ._river_common import _group_river_targets_by_waterbody, _iter_geometry_pieces +from ._sentinel_shared import _sentinel6_use_earthdata, _download_sentinel_for_target +from ._summaries import _river_target_corridor + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def download_rivers(prj: "Project") -> None: + """Download altimetry data for all configured missions (rivers mode). + + SWOT uses the Hydrocron timeseries API directly (per node/reach, no + clustering needed -- see _download_swot_hydrocron_timeseries). + ICESat-2/Sentinel-3/6 download raw observations over a buffered + corridor around each waterbody's SWORD targets (see + _river_target_corridor); associating individual points with a + specific target happens later, during extraction. + + Parameters + ---------- + prj : Project + Project instance with rivers configuration + """ + if not hasattr(prj, "rivers"): + return + + if "swot" not in prj.to_download and not any( + m in prj.to_download for m in ("icesat2", "sentinel3", "sentinel6") + ): + logger.warning( + "Rivers are configured but no mission is enabled for download. " + "Add a top-level mission section (e.g. 'swot:', 'icesat2:', " + "'sentinel3:'/'sentinel6:') with 'download: true' to actually " + "download river observations. SWORD itself will still have " + "been prepared by initialize(), which is why you may see " + "SWORD files but no timeseries data." + ) + return + + if "swot" in prj.to_download: + startdate = prj.startdates.get("swot") + enddate = prj.enddates.get("swot") + + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + _download_swot_hydrocron_timeseries(prj, startdate, enddate) + + if "icesat2" in prj.to_download: + _download_rivers_icesat2(prj) + + if "sentinel3" in prj.to_download: + _download_rivers_sentinel(prj, "sentinel3") + + if "sentinel6" in prj.to_download: + _download_rivers_sentinel(prj, "sentinel6") + + +def _download_swot_hydrocron_timeseries(prj: "Project", startdate, enddate) -> None: + """Download SWOT Hydrocron timeseries for river targets.""" + from urllib import parse, request as url_request + import json + + HYDROCRON_TIMESERIES_URL = ( + "https://soto.podaac.earthdatacloud.nasa.gov/hydrocron/v1/timeseries" + ) + + # Determine feature config + if prj.rivers.target_id_col == "node_id": + feature = "Node" + quality_column = "node_q" + fields = ( + prj.mission_options.get("swot", {}) + .get("hydrocron_fields", {}) + .get("nodes", []) + ) + max_q = ( + prj.mission_options.get("swot", {}) + .get("quality_filters", {}) + .get("nodes", {}) + .get("max_q", 2) + ) + else: + feature = "Reach" + quality_column = "reach_q" + fields = ( + prj.mission_options.get("swot", {}) + .get("hydrocron_fields", {}) + .get("reaches", []) + ) + max_q = ( + prj.mission_options.get("swot", {}) + .get("quality_filters", {}) + .get("reaches", {}) + .get("max_q", 2) + ) + + # Group targets by waterbody + waterbody_groups = _group_river_targets_by_waterbody(prj) + id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" + + summary = { + "requested": 0, + "successful": 0, + "failed": 0, + "empty_after_filter": 0, + } + + for wb_id, target_ids in waterbody_groups.items(): + summary["requested"] = summary["requested"] + len(target_ids) + output_path = os.path.join( + prj.dirs["swot"], str(wb_id), f"{id_label}_timeseries.csv" + ) + general.ifnotmakedirs(os.path.dirname(output_path)) + + wb_startdate = startdate + latest_obs = _get_latest_hydrocron_obs_date(output_path) + if latest_obs is not None: + wb_startdate = latest_obs + + deferred_warnings = [] + + def _defer_warning(message, *args): + if args: + deferred_warnings.append(message % args) + else: + deferred_warnings.append(message) + + frames = [] + for target_id in tqdm(target_ids, desc="Downloading hydrocron data"): + try: + query_params = { + "feature": feature, + "feature_id": str(target_id), + "start_time": wb_startdate.strftime("%Y-%m-%dT%H:%M:%SZ"), + "end_time": enddate.strftime("%Y-%m-%dT%H:%M:%SZ"), + "output": "csv", + "fields": ",".join(fields), + } + request_url = ( + f"{HYDROCRON_TIMESERIES_URL}?{parse.urlencode(query_params)}" + ) + + with url_request.urlopen(request_url) as response: + status_code = getattr(response, "status", response.getcode()) + payload = json.loads(response.read().decode("utf-8")) + except Exception as exc: + _defer_warning( + "Hydrocron request failed for %s %s: %s", + prj.rivers.target_id_col, + target_id, + exc, + ) + summary["failed"] += 1 + continue + + csv_payload = ( + payload.get("results", {}).get("csv") + if isinstance(payload, dict) + else None + ) + if status_code != 200 or not csv_payload: + _defer_warning( + "Hydrocron returned status %s for %s %s", + status_code, + prj.rivers.target_id_col, + target_id, + ) + summary["failed"] += 1 + continue + + try: + df = pd.read_csv(StringIO(csv_payload)) + except Exception as exc: + _defer_warning( + "Failed to parse CSV for %s %s: %s", + prj.rivers.target_id_col, + target_id, + exc, + ) + summary["failed"] += 1 + continue + + if df.empty or quality_column not in df.columns: + if df.empty: + _defer_warning( + "Hydrocron returned no data for %s %s", + prj.rivers.target_id_col, + target_id, + ) + else: + _defer_warning( + "Quality column %s not in Hydrocron response for %s %s", + quality_column, + prj.rivers.target_id_col, + target_id, + ) + continue + + df = df[df[quality_column] <= max_q] + if df.empty: + _defer_warning( + "All Hydrocron observations filtered for %s %s (quality > %s)", + prj.rivers.target_id_col, + target_id, + max_q, + ) + summary["empty_after_filter"] += 1 + continue + + frames.append(df) + summary["successful"] += 1 + + if frames: + combined = pd.concat(frames, ignore_index=True) + combined.to_csv(output_path, index=False) + + for warning in deferred_warnings: + logger.debug(warning) + + logger.info( + "Hydrocron download complete: %s requested, %s successful, %s failed, %s empty after filtering. See file logs for more info.", + summary["requested"], + summary["successful"], + summary["failed"], + summary["empty_after_filter"], + ) + + +def _download_rivers_icesat2(prj: "Project") -> None: + """Download ICESat-2 ATL13 data for river waterbody groups. + + Mirrors _download_reservoirs_icesat2, but queries over a buffered + corridor around each waterbody's SWORD targets (see + _river_target_corridor) rather than a single reservoir polygon. If + a waterbody's corridor comes out as disconnected pieces (a + MultiPolygon), queries each piece separately (see + _iter_geometry_pieces) rather than only the first -- unlike + reservoirs, a river waterbody's targets can legitimately be + disjoint (e.g. separate reaches far apart), so collapsing to one + query would either miss coverage or require an artificially large + combined shape. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + + startdate = prj.startdates["icesat2"] + enddate = prj.enddates["icesat2"] + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + for wb_id, target_ids in waterbody_groups.items(): + logger.info("Downloading ICESat-2 data for waterbody %s", wb_id) + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + logger.warning( + "No SWORD geometry found for waterbody %s; skipping " + "ICESat-2 download.", wb_id, + ) + continue + + pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) + parquet_dir = os.path.join(prj.dirs["icesat2_processed"], f"{wb_id}") + general.ifnotmakedirs(parquet_dir) + + for piece_idx, geom in enumerate(pieces): + coords = list(geom.exterior.coords) + + logger.info( + "Searching for Icesat2 ATL13 for waterbody %s (piece %d/%d) " + "from %s to %s", wb_id, piece_idx + 1, len(pieces), startdate, enddate, + ) + try: + _ = icesat2.query( + aoi=coords, + startdate=startdate, + enddate=enddate, + download_directory=parquet_dir, + atl13_options=prj.mission_options.get("icesat2", {}).get("atl13", {}), + atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields") + or None, + ) + except Exception as exc: + logger.warning( + "ICESat-2 download skipped for waterbody %s (piece %d/%d): %s", + wb_id, piece_idx + 1, len(pieces), exc, + ) + + +def _download_rivers_sentinel(prj: "Project", mission: str) -> None: + """Download Sentinel-3 or Sentinel-6 data for river waterbody groups. + + Mirrors _download_reservoirs_sentinel (both now share + _download_sentinel_for_target, including the CREODIAS/EarthData + branching for Sentinel-6). NOTE: sentinel.query/query_earthdata take + a bounding box (envelope), not the exact corridor polygon -- for a + long or winding river corridor this can query/download a + meaningfully larger area than the actual buffered corridor. This is + an existing limitation inherited from the reservoir path (where it + matters far less, since a reservoir's envelope is close to its + actual extent), not something new introduced here -- worth + revisiting if it turns out to matter in practice for a large or + winding waterbody. + """ + product = "S3" if mission == "sentinel3" else "S6" + + sentinel_creds = None + use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) + if use_earthdata_s6: + prj._require_earthdata_credentials() + else: + sentinel_creds = prj._require_creodias_credentials() + + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + + startdate = prj.startdates[mission] + enddate = prj.enddates[mission] + if isinstance(startdate, list): + startdate = datetime.date(*startdate) + if isinstance(enddate, list): + enddate = datetime.date(*enddate) + + session_token = None + session_start_time = None + + for wb_id, target_ids in waterbody_groups.items(): + logger.info("Downloading data for waterbody %s", wb_id) + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + logger.warning( + "No SWORD geometry found for waterbody %s; skipping " + "Sentinel-%s download.", wb_id, product, + ) + continue + + # Envelope each disconnected piece separately rather than the + # whole (possibly MultiPolygon) corridor at once -- sentinel's + # API only accepts a bounding box, so one envelope covering + # widely separated pieces could be far larger than any of them + # individually. + pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) + + download_dir = os.path.join(prj.dirs[mission], f"{wb_id}") + general.ifnotmakedirs(download_dir) + + for piece_idx, geom in enumerate(pieces): + coords = [(x, y) for x, y in geom.envelope.exterior.coords] + + logger.info( + "Searching for Sentinel-%s for waterbody %s (piece %d/%d) " + "from %s to %s", product, wb_id, piece_idx + 1, len(pieces), + startdate, enddate, + ) + session_token, session_start_time = _download_sentinel_for_target( + prj, mission, product, coords, download_dir, + startdate, enddate, sentinel_creds, session_token, session_start_time, + ) + + +def _get_latest_hydrocron_obs_date(output_path) -> datetime.date: + """Get latest observation date from existing Hydrocron output.""" + if not os.path.exists(output_path): + return None + + try: + existing = pd.read_csv(output_path) + except Exception as exc: + logger.warning( + "Unable to read existing Hydrocron output %s: %s", output_path, exc + ) + return None + + if "time_str" not in existing.columns or existing.empty: + return None + + timestamps = pd.to_datetime(existing["time_str"], errors="coerce", utc=True) + timestamps = timestamps.dropna() + if timestamps.empty: + return None + + latest_obs = timestamps.max().to_pydatetime() + return datetime.date(latest_obs.year, latest_obs.month, latest_obs.day) + + +# ============================================================================ +# RIVERS: Timeseries Processing (extraction) +# ============================================================================ + + diff --git a/HydroEO/flows/_river_init.py b/HydroEO/flows/_river_init.py new file mode 100644 index 0000000..bf8dee3 --- /dev/null +++ b/HydroEO/flows/_river_init.py @@ -0,0 +1,214 @@ +"""Rivers: SWORD database initialization/subsetting. + +initialize_rivers -> _prepare_rivers_from_sword -> _ensure_sword_database +are tested together via patch.object(flows, "_name") in +tests/unit/test_flows.py -- keep them in this one module. +""" + +import logging +import os + +import geopandas as gpd + +from HydroEO.utils import general + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def initialize_rivers(prj: "Project") -> None: + """Initialize SWORD target IDs for rivers mode. + + Parameters + ---------- + prj : Project + Project instance with rivers config/state populated + """ + if not hasattr(prj, "rivers"): + return + + if prj.rivers.input_mode == "aoi_path": + _prepare_rivers_from_sword(prj) + + id_label = "node" if prj.rivers.target_id_col == "node_id" else "reach" + logger.info( + "Found river %s %s ids", + len(prj.rivers.target_ids), + id_label, + ) + logger.debug( + "Found river %s ids: %s", + id_label, + ", ".join(str(target_id) for target_id in prj.rivers.target_ids), + ) + + +def _prepare_rivers_from_sword(prj: "Project") -> None: + """Prepare SWORD target features by spatial intersection with AOI or from saved subset. + + If SWORD_subset.gpkg exists, reads from it directly (skips download and spatial operations). + Otherwise, ensures SWORD database, performs spatial intersection with AOI, saves subset. + """ + subset_path = prj.dirs.get("sword_subset") + + # Gate 1: Check if subset already exists + if subset_path and os.path.exists(subset_path): + logger.info("SWORD subset located at %s", subset_path) + subset = gpd.read_file(subset_path) + else: + # Gate 2: Ensure SWORD database and perform spatial intersection + _ensure_sword_database(prj) + + gpkg_name = ( + f"{prj.rivers.continent_key}_sword_{prj.rivers.feature_type}_v17b.gpkg" + ) + gpkg_path = os.path.join(prj.dirs["sword"], gpkg_name) + + if not os.path.exists(gpkg_path): + raise FileNotFoundError(f"Expected SWORD file not found: {gpkg_path}") + + sword_gdf = gpd.read_file(gpkg_path) + + # Buffer AOI if requested + aoi_local = prj.rivers.aoi_gdf.to_crs(prj.local_crs).copy() + if prj.rivers.buffer_meters and prj.rivers.buffer_meters > 0: + aoi_local["geometry"] = aoi_local.geometry.buffer(prj.rivers.buffer_meters) + + # Intersect with SWORD + sword_local = sword_gdf.to_crs(prj.local_crs) + subset = sword_local.loc[sword_local.intersects(aoi_local.unary_union)].copy() + + if prj.rivers.id_key not in prj.rivers.aoi_gdf.columns: + raise KeyError( + f"Expected AOI column '{prj.rivers.id_key}' missing from river input file" + ) + + aoi_join = aoi_local[[prj.rivers.id_key, "geometry"]].copy() + subset = gpd.sjoin( + subset, + aoi_join, + how="inner", + predicate="intersects", + ).drop(columns=["index_right"], errors="ignore") + subset = subset.drop_duplicates().to_crs(prj.rivers.aoi_gdf.crs) + + # Save subset to disk + if subset_path: + general.ifnotmakedirs(os.path.dirname(subset_path)) + subset.to_file(subset_path, driver="GPKG") + logger.info("SWORD subset saved to %s", subset_path) + + # Cleanup: delete gpkg folder if keep_raw_sword is False + if not prj.keep_raw_sword: + try: + import shutil + + sword_dir = prj.dirs.get("sword") + if sword_dir and os.path.isdir(sword_dir): + shutil.rmtree(sword_dir) + logger.info("Deleted SWORD gpkg folder (keep_raw_sword=False)") + except Exception as e: + logger.warning("Failed to delete SWORD gpkg folder: %s", e) + else: + logger.info("Kept SWORD gpkg folder (keep_raw_sword=True)") + + # Extract target IDs from subset + source_id_col = "node_id" if prj.rivers.feature_type == "nodes" else "reach_id" + if source_id_col not in subset.columns: + raise KeyError(f"Expected SWORD column '{source_id_col}' missing from subset") + + prj.rivers.target_features = subset + prj.rivers.target_id_col = source_id_col + prj.rivers.target_ids = [int(value) for value in subset[source_id_col]] + + +def _ensure_sword_database(prj: "Project") -> None: + """Ensure SWORD database is available locally. + + Checks if full SWORD database (GPKGs) already exists in prj.dirs["sword"]. + If not, handles three scenarios: + 1. User-provided zip: extract to {main_dir}/aux/SWORD/ + 2. User-provided directory: use it directly + 3. Auto-download from Zenodo: download and extract to {main_dir}/aux/SWORD/ + + Respects keep_raw_sword config to optionally delete downloaded zip. + """ + from urllib import request as url_request + import zipfile + + SWORD_V17B_ZIP_URL = ( + "https://zenodo.org/records/15299138/files/SWORD_v17b_gpkg.zip?download=1" + ) + + sword_dir = prj.dirs["sword"] + + # Check if SWORD database already exists + if os.path.isdir(sword_dir): + # Check if directory contains any SWORD GPKGs + gpkg_files = [f for f in os.listdir(sword_dir) if f.endswith("_v17b.gpkg")] + if gpkg_files: + logger.info("SWORD database located at %s", sword_dir) + return + + logger.info("SWORD database not found. Preparing it now.") + + # User-provided raw_sword_path + if "sword_raw" in prj.dirs: + raw_path = prj.dirs["sword_raw"] + + # Case 1: User provided a zip file + if raw_path.lower().endswith(".zip") and os.path.isfile(raw_path): + logger.info("Using user-provided SWORD zip: %s", raw_path) + general.ifnotmakedirs(os.path.dirname(sword_dir)) + with zipfile.ZipFile(raw_path, "r") as zip_ref: + zip_ref.extractall(os.path.dirname(sword_dir)) + logger.info("SWORD extracted to %s", sword_dir) + return + + # Case 2: User provided a directory + elif os.path.isdir(raw_path): + logger.info("Using user-provided SWORD directory: %s", raw_path) + # Check if GPKGs are in raw_path/gpkg/ or directly in raw_path + gpkg_subdir = os.path.join(raw_path, "gpkg") + if os.path.isdir(gpkg_subdir): + prj.dirs["sword"] = gpkg_subdir + logger.info("SWORD database found in %s", gpkg_subdir) + else: + prj.dirs["sword"] = raw_path + logger.info("SWORD database found in %s", raw_path) + return + + # Case 3: Auto-download from Zenodo + logger.info("Downloading SWORD v17b from Zenodo...") + general.ifnotmakedirs(os.path.dirname(sword_dir)) + + zip_path = os.path.join(os.path.dirname(sword_dir), "SWORD_v17b_gpkg.zip") + url_request.urlretrieve(SWORD_V17B_ZIP_URL, zip_path) + logger.info("Downloaded SWORD v17b to %s", zip_path) + + logger.info("Extracting SWORD v17b...") + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(os.path.dirname(sword_dir)) + + logger.info("SWORD extracted to %s", sword_dir) + + # Cleanup: delete zip if keep_raw_sword is False + if not prj.keep_raw_sword: + try: + os.remove(zip_path) + logger.info("Deleted raw SWORD zip file (keep_raw_sword=False)") + except Exception as e: + logger.warning("Failed to delete SWORD zip %s: %s", zip_path, e) + else: + logger.info("Kept raw SWORD zip file at %s (keep_raw_sword=True)", zip_path) + + +# ============================================================================ +# RESERVOIRS: Download +# ============================================================================ + + diff --git a/HydroEO/flows/_river_pipeline.py b/HydroEO/flows/_river_pipeline.py new file mode 100644 index 0000000..4fab9d8 --- /dev/null +++ b/HydroEO/flows/_river_pipeline.py @@ -0,0 +1,421 @@ +"""Rivers: extraction + clean + merge orchestration. + +create_rivers_timeseries and everything it (transitively) calls -- +_extract_rivers_timeseries, its three per-mission workers, and the +_clean_rivers_timeseries/_merge_rivers_timeseries wrappers -- are tested +together via patch.object(flows, "_name") in tests/unit/test_flows.py and +tests/unit/test_timeseries.py, so they all live in this one module (this +mirrors _reservoir_pipeline.py's equivalent constraint). +""" + +import logging +import os + +import geopandas as gpd +import pandas as pd + +from HydroEO.satellites import swot, icesat2, sentinel +from HydroEO.utils import general +from ._river_common import ( + _group_river_targets_by_waterbody, + _river_extraction_buffer_meters, + _assign_points_to_river_targets, +) +from ._summaries import _river_target_corridor +from ._clean_engine import _clean_timeseries +from ._merge_engine import _merge_timeseries + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def _extract_rivers_timeseries(prj: "Project", overwrite: bool = False) -> None: + """Extract timeseries observations from raw downloaded files, for rivers. + + SWOT still needs a (lightweight) extraction step here: Hydrocron + already returns a per-node/reach timeseries directly, but grouped + per WATERBODY (one CSV covering every target in that waterbody) -- + see _extract_rivers_swot_observations for splitting that into the + same per-target file structure ICESat-2/Sentinel-3/6 use, so the + shared clean/merge pipeline can treat every mission identically. + + Parameters + ---------- + overwrite : bool, optional + Same semantics as _extract_reservoirs_timeseries: if False + (default), any target whose output .gpkg already exists is + skipped rather than re-extracted. + """ + if "icesat2" in prj.to_process: + _extract_rivers_icesat2_observations(prj, overwrite=overwrite) + + if "sentinel3" in prj.to_process: + _extract_rivers_sentinel_observations( + prj, "sentinel3", "S3", overwrite=overwrite + ) + + if "sentinel6" in prj.to_process: + _extract_rivers_sentinel_observations( + prj, "sentinel6", "S6", overwrite=overwrite + ) + + if "swot" in prj.to_process: + _extract_rivers_swot_observations(prj, overwrite=overwrite) + + +def _extract_rivers_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: + """Extract ICESat-2 ATL13 observations for each river target. + + Unlike reservoirs (one polygon = one target), a river waterbody's + raw download covers many targets at once. This extracts once per + waterbody -- reusing icesat2.extract_observations exactly as + reservoirs use it, with the buffered corridor (see + _river_target_corridor) as the spatial filter instead of a single + reservoir polygon -- then assigns each surviving point to its + nearest target via sjoin_nearest, and splits the result into the + same per-target {output}/{target_id}/raw_observations/icesat2.gpkg + structure reservoirs already use, so everything downstream + (clean/merge) can treat a river target exactly like a reservoir. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + max_assign_dist = ( + getattr(prj.rivers, "max_node_assignment_meters", None) + or _river_extraction_buffer_meters(prj) + ) + + tmp_dir = os.path.join(prj.dirs["output"], "_tmp_river_extraction") + + for wb_id, target_ids in waterbody_groups.items(): + parquet_dir = os.path.join(prj.dirs["icesat2_processed"], f"{wb_id}") + if not os.path.exists(os.path.join(parquet_dir, "atl13.parquet")): + continue + + if not overwrite: + remaining = [ + t + for t in target_ids + if not os.path.exists( + os.path.join( + prj.dirs["output"], f"{t}", "raw_observations", "icesat2.gpkg" + ) + ) + ] + if not remaining: + continue + target_ids = remaining + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + continue + + general.ifnotmakedirs(tmp_dir) + tmp_dst = os.path.join(tmp_dir, f"{wb_id}_icesat2.gpkg") + + try: + icesat2.extract_observations( + src_dir=parquet_dir, + dst_path=tmp_dst, + features=corridor_gdf, + atl13_fields=prj.mission_options.get("icesat2", {}).get("atl13_fields"), + track_keys=prj.mission_options.get("icesat2", {}).get("track_keys"), + ) + except Exception as exc: + logger.warning( + "Failed to extract ICESat-2 for waterbody %s: %s", wb_id, exc + ) + continue + + if not os.path.exists(tmp_dst): + continue + + points = gpd.read_file(tmp_dst) + os.remove(tmp_dst) + if points.empty: + continue + + targets = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col].isin(target_ids) + ] + assigned = _assign_points_to_river_targets( + points, targets, prj.rivers.target_id_col, max_assign_dist, prj.local_crs + ) + if assigned.empty: + logger.warning( + "No ICESat-2 points within %sm of any target for waterbody %s", + max_assign_dist, wb_id, + ) + continue + + for target_id, group in assigned.groupby(prj.rivers.target_id_col): + dst_dir = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations" + ) + general.ifnotmakedirs(dst_dir) + dst_path = os.path.join(dst_dir, "icesat2.gpkg") + group.drop( + columns=["index_right", "_dist_to_target_m"], errors="ignore" + ).to_file(dst_path, driver="GPKG") + + +def _extract_rivers_sentinel_observations( + prj: "Project", mission_key: str, product: str, overwrite: bool = False +) -> None: + """Extract Sentinel-3 or Sentinel-6 observations for each river target. + + Same per-waterbody-then-split approach as + _extract_rivers_icesat2_observations, plus a water-only filter: a + buffered river corridor is much looser than a reservoir polygon (it + genuinely includes riverbank, fields, vegetation alongside the + channel), and unlike ICESat-2, Sentinel-3/6 have no built-in water + classification. sigma0_min filters this out as a self-contained + post-processing step here (rather than modifying + sentinel.extract_observations itself, whose internals haven't been + verified) -- water gives a strong, consistent specular radar + return; land gives a weaker, noisier one. Needs empirical tuning + against real river data, same as every other threshold in this + pipeline -- the default here (0.0, i.e. no-op) is a safe starting + point, not a verified value; set mission_options[mission_key] + ['sigma0_min'] once you have real data to check it against. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) + width_buffer_factor = getattr(prj.rivers, "width_buffer_factor", 1.05) + max_assign_dist = ( + getattr(prj.rivers, "max_node_assignment_meters", None) + or _river_extraction_buffer_meters(prj) + ) + sigma0_min = prj.mission_options.get(mission_key, {}).get("sigma0_min", 0.0) + + tmp_dir = os.path.join(prj.dirs["output"], "_tmp_river_extraction") + + for wb_id, target_ids in waterbody_groups.items(): + download_dir = os.path.join(prj.dirs[mission_key], f"{wb_id}") + if not os.path.exists(download_dir): + continue + + if not overwrite: + remaining = [ + t + for t in target_ids + if not os.path.exists( + os.path.join( + prj.dirs["output"], + f"{t}", + "raw_observations", + f"{mission_key}.gpkg", + ) + ) + ] + if not remaining: + continue + target_ids = remaining + + corridor_gdf = _river_target_corridor( + prj, target_ids, buffer_meters=explicit_buffer, + width_buffer_factor=width_buffer_factor, + ) + if corridor_gdf is None: + continue + + general.ifnotmakedirs(tmp_dir) + tmp_dst = os.path.join(tmp_dir, f"{wb_id}_{mission_key}.gpkg") + + try: + sentinel.extract_observations( + src_dir=download_dir, + dst_path=tmp_dst, + features=corridor_gdf, + sigma0_max=prj.mission_options.get(mission_key, {}).get( + "sigma0_max", 1e5 + ), + ) + except Exception as exc: + logger.warning( + "Failed to extract %s for waterbody %s: %s", mission_key, wb_id, exc + ) + continue + + if not os.path.exists(tmp_dst): + continue + + points = gpd.read_file(tmp_dst) + os.remove(tmp_dst) + if points.empty: + continue + + if "sigma0" in points.columns and sigma0_min: + before = len(points) + points = points.loc[points["sigma0"] >= sigma0_min].reset_index(drop=True) + logger.info( + "%s waterbody %s: sigma0_min=%s kept %d/%d points", + mission_key, wb_id, sigma0_min, len(points), before, + ) + if points.empty: + continue + + targets = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col].isin(target_ids) + ] + assigned = _assign_points_to_river_targets( + points, targets, prj.rivers.target_id_col, max_assign_dist, prj.local_crs + ) + if assigned.empty: + logger.warning( + "No %s points within %sm of any target for waterbody %s", + mission_key, max_assign_dist, wb_id, + ) + continue + + for target_id, group in assigned.groupby(prj.rivers.target_id_col): + dst_dir = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations" + ) + general.ifnotmakedirs(dst_dir) + dst_path = os.path.join(dst_dir, f"{mission_key}.gpkg") + group.drop( + columns=["index_right", "_dist_to_target_m"], errors="ignore" + ).to_file(dst_path, driver="GPKG") + + +def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) -> None: + """ + Split Hydrocron's per-waterbody timeseries CSV into the same + per-target {output}/{target_id}/raw_observations/swot.gpkg structure + every other mission uses, so the shared clean/merge pipeline can + treat SWOT identically to ICESat-2/Sentinel-3/6 for rivers. + + Unlike LakeSP for reservoirs, Hydrocron's own CSV doesn't include + per-observation coordinates in the default field lists (see + rivers.yaml) -- but nothing downstream actually needs per-observation + lat/lon for SWOT (see PRODUCT_TIMESERIES_KEYS: no lat_key/lon_key for + "swot"), so this attaches the target's own SWORD geometry as a + constant placeholder purely so the file can be saved/read as .gpkg + like every other mission's output -- the geometry's actual value is + never used downstream, only the height/date/platform/orbit columns. + + Quality filtering (max_q) is already applied at download time (see + _download_swot_hydrocron_timeseries), so it isn't repeated here. + """ + waterbody_groups = _group_river_targets_by_waterbody(prj) + id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" + + for wb_id, target_ids in waterbody_groups.items(): + src_path = os.path.join( + prj.dirs["swot"], str(wb_id), f"{id_label}_timeseries.csv" + ) + if not os.path.exists(src_path): + continue + + if not overwrite: + remaining = [ + t + for t in target_ids + if not os.path.exists( + os.path.join( + prj.dirs["output"], f"{t}", "raw_observations", "swot.gpkg" + ) + ) + ] + if not remaining: + continue + target_ids = remaining + + try: + df = pd.read_csv(src_path) + except Exception as exc: + logger.warning( + "Failed to read Hydrocron CSV for waterbody %s: %s", wb_id, exc + ) + continue + + if df.empty or prj.rivers.target_id_col not in df.columns: + continue + + df = df.loc[df[prj.rivers.target_id_col].isin(target_ids)].copy() + if df.empty: + continue + + df["height"] = df["wse"] + df["date"] = pd.to_datetime(df["time_str"]) + df["platform"] = "swot" + df["product"] = f"SWOT_Hydrocron_{id_label}" + # Matches the reservoir SWOT convention (orbit = lake_id, constant + # per target) -- there's no meaningful "which persistent track" + # concept distinct from the target itself for Hydrocron data. + df["orbit"] = df[prj.rivers.target_id_col] + + for target_id, group in df.groupby(prj.rivers.target_id_col): + target_row = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col] == target_id + ] + if target_row.empty: + continue + + group = group.copy() + group["geometry"] = target_row.geometry.iloc[0] + gdf = gpd.GeoDataFrame( + group, geometry="geometry", crs=prj.rivers.target_features.crs + ) + + dst_dir = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations" + ) + general.ifnotmakedirs(dst_dir) + gdf.to_file(os.path.join(dst_dir, "swot.gpkg"), driver="GPKG") + + +# ============================================================================ +# RESERVOIRS: Timeseries Processing +# ============================================================================ + + +def create_rivers_timeseries(prj: "Project") -> None: + """Extract, clean, and merge timeseries for river targets (nodes/reaches). + + Mirrors create_reservoirs_timeseries. Not yet done: an export_to_dfs0 + equivalent for rivers, since _export_cleaned_to_dfs0 currently + iterates prj.reservoirs.download_gdf specifically -- left out here + rather than silently generalizing something not explicitly asked + for yet. + + Parameters + ---------- + prj : Project + Project instance with rivers configuration + """ + if not hasattr(prj, "rivers"): + return + + _extract_rivers_timeseries( + prj, overwrite=getattr(prj.rivers, "overwrite_extraction", False) + ) + + _clean_rivers_timeseries(prj) + + _merge_rivers_timeseries(prj) + + +def _clean_rivers_timeseries(prj: "Project") -> None: + """Apply quality filters to extracted river timeseries.""" + _clean_timeseries(prj, "rivers") + + +def _merge_rivers_timeseries(prj: "Project") -> None: + """Merge multi-mission timeseries into combined datasets, for river targets.""" + _merge_timeseries(prj, "rivers") + + +# ============================================================================ +# RESERVOIRS: Summaries & Visualization +# ============================================================================ + + diff --git a/HydroEO/flows/_run_config.py b/HydroEO/flows/_run_config.py new file mode 100644 index 0000000..6eacb73 --- /dev/null +++ b/HydroEO/flows/_run_config.py @@ -0,0 +1,616 @@ +"""Per-target run_config persistence, exclusions, and spatial/reach-slope +correction caching -- shared by both reservoirs and rivers. + +One YAML file per target ({output}/{id}/run_config.yaml) that is +simultaneously: (a) a human-readable log of decisions made about this +target, (b) the actual source of truth _merge_engine._merge_timeseries +reads to apply those decisions, and (c) something a user can hand-edit +directly for a fully config-driven workflow. + +None of this is itself patched as a sibling of any single caller in the +test suite, so it's free to live in its own module. +""" + +import logging +import os +import datetime +import json + +import geopandas as gpd +import pandas as pd +import yaml + +from HydroEO.utils import general +from HydroEO.utils.filters import basic_filters + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def _get_target_ids(prj: "Project", target_type: str): + """Return the list of target IDs to process, for either 'reservoirs' or 'rivers'.""" + if target_type == "reservoirs": + return list(prj.reservoirs.download_gdf[prj.reservoirs.id_key]) + if target_type == "rivers": + return list(prj.rivers.target_ids) + raise ValueError(f"Unknown target_type: {target_type!r}") + + +def _target_centroid(prj: "Project", target_type: str, id): + """ + Return (lat, lon) of a target's own geometry centroid -- the + reservoir polygon for target_type="reservoirs", or the SWORD + node/reach geometry for target_type="rivers" -- computed in a + projected (local) CRS for accuracy, then converted back to lat/lon. + Used as the reference location for apply_distance_penalty/ + apply_spatial_correction. Returns (None, None) if the target's + geometry can't be found, so callers can treat that as "skip" rather + than fail. + """ + try: + if target_type == "reservoirs": + gdf = prj.reservoirs.gdf + id_key = prj.reservoirs.id_key + elif target_type == "rivers": + gdf = prj.rivers.target_features + id_key = prj.rivers.target_id_col + else: + raise ValueError(f"Unknown target_type: {target_type!r}") + + row = gdf.loc[gdf[id_key] == id] + if len(row) == 0 or row.geometry.isna().all(): + return None, None + centroid = row.to_crs(prj.local_crs).geometry.centroid.to_crs(prj.global_crs) + pt = centroid.iloc[0] + return pt.y, pt.x # lat, lon + except Exception as exc: + logger.warning( + "Could not compute %s centroid for %s: %s", target_type, id, exc + ) + return None, None + + +def _reservoir_centroid(prj: "Project", id): + """Backward-compatible wrapper -- see _target_centroid.""" + return _target_centroid(prj, "reservoirs", id) + + +def _get_or_fit_spatial_correction_model( + prj: "Project", target_type: str, id, dense_source_platform="icesat2", + recalibrate=False, **fit_kwargs, +): + """ + Load a persisted spatial correction model for this target if one + exists, or fit a fresh one and persist it. Works identically for + reservoirs and river targets -- see _target_centroid. + + This is deliberately NOT re-fit automatically every run: doing so + would make past corrections shift retroactively every time new + dense-source data arrives, since the fitted slope would change. + Pass recalibrate=True to explicitly force a re-fit (e.g. as a + deliberate, occasional recalibration step) -- not something that + should happen as a silent side effect of routine reprocessing. + + Returns None if no model exists yet and there isn't enough dense + source data to fit one (see fit_spatial_correction_model) -- callers + should treat this the same as "no correction available." + """ + model_path = os.path.join( + prj.dirs["output"], f"{id}", "spatial_correction_model.json" + ) + + if os.path.exists(model_path) and not recalibrate: + with open(model_path, "r") as f: + return json.load(f) + + cleaned_path = os.path.join( + prj.dirs["output"], f"{id}", "all_cleaned_timeseries.csv" + ) + if not os.path.exists(cleaned_path): + logger.info( + "No cleaned observations yet for %s; cannot fit spatial " + "correction model.", id, + ) + return None + + df = pd.read_csv(cleaned_path) + if "platform" not in df.columns or dense_source_platform not in df["platform"].values: + logger.info( + "No %s data available for %s; cannot fit spatial correction " + "model from it.", dense_source_platform, id, + ) + return None + + df["date"] = pd.to_datetime(df["date"]) + dense_df = df.loc[df["platform"] == dense_source_platform] + + ref_lat, ref_lon = _target_centroid(prj, target_type, id) + if ref_lat is None: + logger.warning( + "Could not determine %s centroid for %s; cannot fit spatial " + "correction model.", target_type, id, + ) + return None + + model = basic_filters.fit_spatial_correction_model( + dense_df, lat_key="lat", lon_key="lon", height_key="height", + date_key="date", ref_lat=ref_lat, ref_lon=ref_lon, **fit_kwargs, + ) + + if model is not None: + general.ifnotmakedirs(os.path.dirname(model_path)) + with open(model_path, "w") as f: + json.dump(model, f, indent=2) + + return model + + +# ============================================================================ +# Per-target run config: exclusions + per-target merging option overrides +# ============================================================================ +# +# One YAML file per target ({output}/{id}/run_config.yaml) that is +# simultaneously: (a) a human-readable log of decisions made about this +# target, (b) the actual source of truth _merge_timeseries reads to apply +# those decisions, and (c) something a user can hand-edit directly for a +# fully config-driven workflow. Interactive functions below +# (exclude_from_target, set_merging_option, ...) read-modify-write this +# same file, so a decision made once in a notebook session is exactly the +# same artifact you'd edit by hand or check into version control -- there +# is no separate "notebook state" to keep in sync with "the config". + + +def _run_config_path(prj: "Project", id) -> str: + return os.path.join(prj.dirs["output"], f"{id}", "run_config.yaml") + + +def _default_run_config(id) -> dict: + return { + "target_id": id, + "last_updated": None, + "merging_option_overrides": {}, + "exclusions": [], + } + + +def _load_run_config(prj: "Project", id) -> dict: + """Load a target's run_config.yaml, or a fresh default if none exists yet.""" + path = _run_config_path(prj, id) + if os.path.exists(path): + with open(path, "r") as f: + loaded = yaml.safe_load(f) + if loaded: + # tolerate a hand-edited file missing a key or two + defaults = _default_run_config(id) + defaults.update(loaded) + return defaults + return _default_run_config(id) + + +def _save_run_config(prj: "Project", id, config: dict) -> None: + config["last_updated"] = datetime.datetime.now().isoformat() + path = _run_config_path(prj, id) + general.ifnotmakedirs(os.path.dirname(path)) + with open(path, "w") as f: + yaml.safe_dump(config, f, sort_keys=False) + + +def _invalidate_spatial_correction_cache(prj: "Project", id) -> None: + """ + Delete any cached spatial correction model for this target, forcing + a fresh fit next time use_spatial_correction is used. Called whenever + exclusions or spatial-correction-relevant options change -- the + model may have been fit using observations that are no longer + included, and this is exactly the kind of deliberate, explicit + trigger (not routine reprocessing) that recalibration is meant for -- + see _get_or_fit_spatial_correction_model. + """ + model_path = os.path.join(prj.dirs["output"], f"{id}", "spatial_correction_model.json") + if os.path.exists(model_path): + os.remove(model_path) + logger.info( + "Invalidated cached spatial correction model for %s " + "(exclusions or related options changed).", id, + ) + + +def _invalidate_reach_slope_correction_cache(prj: "Project", id) -> None: + """ + Delete any cached reach slope correction model for this target, + forcing a fresh fit next time use_reach_slope_correction is used. + Called whenever exclusions change -- an exclusion could target SWOT + observations specifically, which is exactly what this model is fit + from (see _fit_reach_slope_correction), so a cached model could + otherwise silently keep reflecting now-excluded SWOT slope values. + """ + model_path = os.path.join( + prj.dirs["output"], f"{id}", "reach_slope_correction_model.json" + ) + if os.path.exists(model_path): + os.remove(model_path) + logger.info( + "Invalidated cached reach slope correction model for %s " + "(exclusions or related options changed).", id, + ) + + +def _fit_reach_slope_correction(prj: "Project", target_id, recalibrate: bool = False): + """ + Fit (or load a persisted) reach-level slope correction from SWOT's + own directly-measured "slope" field (RiverSP reach product), used to + reference-correct OTHER missions' (ICESat-2/Sentinel-3/6) crossings + to what they'd read at the reach's geometric midpoint. + + ONLY meaningful for reaches (prj.rivers.target_id_col == "reach_id") + -- a node is a single ~200m-spaced point, not a ~10km segment with + its own along-reach slope in the same sense. Callers must not invoke + this for node-mode projects. + + Rationale: SWOT's reach-level WSE is an aggregate over the reach's + ~50 constituent, roughly-evenly-spaced nodes, not a value evaluated + at one specific point -- for an evenly-sampled linear profile, the + mean equals the value at the mean position, so this is treated as + approximately midpoint-referenced. This is an evidence-based + inference from the RiverSP processing chain, NOT a fact directly + confirmed in SWOT's product documentation (which does not explicitly + state a reference point) -- validate against real Hydrocron + node-vs-reach output for a known reach before trusting this deeply. + + Uses the MEDIAN of all available SWOT slope observations for this + reach as a single, persistent correction -- not a per-date-specific + one -- consistent with this pipeline's existing preference (see + fit_spatial_correction_model) for a stable, once-fit value over a + per-observation one, and avoiding the complexity/fragility of + matching a specific SWOT overpass date to each individual non-SWOT + observation's date. + + Persisted to {output}/{target_id}/reach_slope_correction_model.json + -- fit once, loaded thereafter, only refit on explicit + recalibrate=True -- so past corrections don't shift retroactively + as new SWOT data arrives, same reasoning as the spatial correction + model's caching. + + Returns None if no model exists yet and there's no usable SWOT slope + data to fit one from -- callers should treat this as "no correction + available," not an error. + """ + model_path = os.path.join( + prj.dirs["output"], f"{target_id}", "reach_slope_correction_model.json" + ) + if os.path.exists(model_path) and not recalibrate: + with open(model_path, "r") as f: + return json.load(f) + + swot_path = os.path.join( + prj.dirs["output"], f"{target_id}", "raw_observations", "swot.gpkg" + ) + if not os.path.exists(swot_path): + logger.info( + "No raw SWOT observations for %s; cannot fit reach slope " + "correction.", target_id, + ) + return None + + swot_gdf = gpd.read_file(swot_path) + if "slope" not in swot_gdf.columns: + logger.warning( + "No 'slope' field found in SWOT observations for %s -- was " + "it requested in mission_options['swot']['hydrocron_fields']" + "['reaches']? Cannot fit reach slope correction.", target_id, + ) + return None + + valid_slopes = pd.to_numeric(swot_gdf["slope"], errors="coerce").dropna() + if valid_slopes.empty: + logger.info( + "No valid (non-null, numeric) SWOT slope observations for " + "%s; cannot fit reach slope correction.", target_id, + ) + return None + + model = { + "target_id": str(target_id), + "median_slope": float(valid_slopes.median()), + "n_observations": int(len(valid_slopes)), + "fitted_at": datetime.datetime.now().isoformat(), + } + + general.ifnotmakedirs(os.path.dirname(model_path)) + with open(model_path, "w") as f: + json.dump(model, f, indent=2) + + return model + + +def _apply_reach_slope_correction( + ts_df: pd.DataFrame, prj: "Project", target_id, slope_model: dict, +) -> pd.DataFrame: + """ + Apply a fitted reach slope correction (see _fit_reach_slope_correction) + to non-SWOT rows in ts_df -- adjusts "height" to what each row would + read at the reach's geometric midpoint, using its along-reach + projected position and the reach's persistent median slope. SWOT's + own rows are left untouched (already assumed midpoint-referenced -- + see _fit_reach_slope_correction's docstring for the reasoning and + its caveats). + + NOTE: the sign convention for "correction = slope x distance" here + has NOT been empirically verified against real data in this + session -- confirm it actually reduces cross-mission scatter for a + real reach (not increases it) before trusting this in production; + flip the sign if it doesn't. + + Rows without usable lat/lon (or if the target's geometry can't be + found) are left uncorrected rather than dropped. + """ + if "platform" not in ts_df.columns or "lat" not in ts_df.columns or "lon" not in ts_df.columns: + return ts_df + + target_row = prj.rivers.target_features.loc[ + prj.rivers.target_features[prj.rivers.target_id_col] == target_id + ] + if target_row.empty: + logger.warning( + "Could not find reach geometry for %s; skipping reach slope " + "correction.", target_id, + ) + return ts_df + + reach_geom_global = target_row.geometry.iloc[0] + local_crs = prj.local_crs + reach_geom_local = ( + gpd.GeoSeries([reach_geom_global], crs=target_row.crs).to_crs(local_crs).iloc[0] + ) + reach_midpoint_dist = reach_geom_local.length / 2.0 + median_slope = slope_model["median_slope"] + + mask = (ts_df["platform"] != "swot") & ts_df["lat"].notna() & ts_df["lon"].notna() + if not mask.any(): + return ts_df + + points_local = gpd.GeoSeries( + gpd.points_from_xy(ts_df.loc[mask, "lon"], ts_df.loc[mask, "lat"]), + crs=prj.global_crs, + ).to_crs(local_crs) + + along_reach_dist = points_local.apply(reach_geom_local.project) + distance_from_midpoint = along_reach_dist.values - reach_midpoint_dist + + ts_df = ts_df.copy() + ts_df.loc[mask, "height"] = ( + ts_df.loc[mask, "height"] - median_slope * distance_from_midpoint + ) + return ts_df + + +def list_target_observations(prj: "Project", target_type: str, id) -> pd.DataFrame: + """ + Summarize what observations exist for a target, at (platform, orbit) + granularity -- the "what could I exclude" view. Meant to be read + alongside plot_merging's platform-colored progress plots (which show + WHERE a problem shows up in the actual data), not a replacement for + looking at the data itself. + """ + cleaned_path = os.path.join(prj.dirs["output"], f"{id}", "all_cleaned_timeseries.csv") + if not os.path.exists(cleaned_path): + logger.warning( + "No cleaned observations yet for %s -- has create_%s_timeseries() " + "been run?", id, target_type, + ) + return pd.DataFrame(columns=["platform", "orbit", "n_points", "date_min", "date_max"]) + + df = pd.read_csv(cleaned_path) + df["date"] = pd.to_datetime(df["date"]) + group_cols = [c for c in ["platform", "orbit"] if c in df.columns] + summary = ( + df.groupby(group_cols) + .agg(n_points=("date", "size"), date_min=("date", "min"), date_max=("date", "max")) + .reset_index() + .sort_values(group_cols) + .reset_index(drop=True) + ) + return summary + + +def list_exclusions(prj: "Project", target_type: str, id) -> list: + """Current exclusion rules for a target, from its run_config.yaml.""" + return _load_run_config(prj, id)["exclusions"] + + +def exclude_from_target( + prj: "Project", target_type: str, id, + platform=None, orbit=None, date=None, reason=None, +) -> None: + """ + Exclude observations from a target's merge, at whatever granularity + is given -- a whole platform, a specific orbit/pass value, a specific + date, or any combination (all given fields must match for a row to + be excluded). Persisted to {output}/{id}/run_config.yaml. + + Applied at the start of _merge_timeseries, before any processing -- + an excluded pass never reaches bias_correct/Kalman/svr_radial at all, + rather than being fought against downstream. + + Invalidates any cached spatial correction model for this target, + since it may have been fit using data that's now excluded. + + Examples + -------- + exclude_from_target(prj, "reservoirs", my_id, platform="S3B") + exclude_from_target(prj, "reservoirs", my_id, platform="S3B", orbit=1517) + exclude_from_target(prj, "rivers", my_id, date="2024-03-19") + """ + if platform is None and orbit is None and date is None: + raise ValueError( + "Specify at least one of platform, orbit, or date to exclude." + ) + + config = _load_run_config(prj, id) + config["exclusions"].append({ + "platform": platform, + "orbit": orbit, + "date": str(date) if date is not None else None, + "reason": reason, + "added": datetime.datetime.now().isoformat(), + }) + _save_run_config(prj, id, config) + _invalidate_spatial_correction_cache(prj, id) + _invalidate_reach_slope_correction_cache(prj, id) + logger.info( + "Added exclusion for %s: platform=%s orbit=%s date=%s (%s)", + id, platform, orbit, date, reason or "no reason given", + ) + + +def _exclusion_value_matches(a, b) -> bool: + """ + Robust equality check for exclusion matching -- tries numeric + comparison first, so int/float/numeric-string representations of + the same value all compare correctly (e.g. 1517 == 1517.0 == "1517" + -- same int/float representation issue confirmed for + _apply_exclusions' dataframe matching, applied here too for + consistency), falling back to exact equality for non-numeric values + (e.g. a string-based orbit identifier) or when either side is None. + """ + if a is None or b is None: + return a == b + try: + return float(a) == float(b) + except (TypeError, ValueError): + return a == b + + +def remove_exclusion( + prj: "Project", target_type: str, id, + index: int = None, platform=None, orbit=None, date=None, +) -> list: + """ + Remove one or more exclusion rules, either by position (index, from + list_exclusions()) or by matching criteria -- the same + platform/orbit/date fields used to add one via exclude_from_target. + Matching by criteria is usually more convenient than looking up an + index first: e.g. remove_exclusion(prj, "reservoirs", my_id, + platform="S3B", orbit=1517) removes exactly the rule that excluded + that mission+orbit combination (or platform+beam, for ICESat-2 -- + beam values ARE the "orbit" field once concatenated with other + missions, see PRODUCT_TIMESERIES_KEYS -- there's no separate "beam" + parameter needed). + + Specify either index, or at least one of platform/orbit/date, not + both. Criteria matching removes EVERY exclusion rule whose given + fields match (fields not specified are ignored, not required to be + None on the stored rule). + + Returns the list of removed rule(s), for confirmation/logging. + """ + config = _load_run_config(prj, id) + exclusions = config["exclusions"] + criteria_given = platform is not None or orbit is not None or date is not None + + if index is not None and criteria_given: + raise ValueError( + "Specify either index OR platform/orbit/date criteria, not both." + ) + + if index is not None: + if index < 0 or index >= len(exclusions): + raise IndexError( + f"No exclusion at index {index} for {id}; there are " + f"{len(exclusions)}. See list_exclusions()." + ) + removed = [exclusions.pop(index)] + elif criteria_given: + date_str = str(date) if date is not None else None + to_remove = [ + rule for rule in exclusions + if (platform is None or _exclusion_value_matches(rule.get("platform"), platform)) + and (orbit is None or _exclusion_value_matches(rule.get("orbit"), orbit)) + and (date is None or rule.get("date") == date_str) + ] + if not to_remove: + raise ValueError( + f"No exclusion found matching platform={platform!r} " + f"orbit={orbit!r} date={date!r} for {id}. See list_exclusions()." + ) + for rule in to_remove: + exclusions.remove(rule) + removed = to_remove + else: + raise ValueError( + "Specify index, or at least one of platform/orbit/date, to " + "identify which exclusion(s) to remove." + ) + + _save_run_config(prj, id, config) + _invalidate_spatial_correction_cache(prj, id) + _invalidate_reach_slope_correction_cache(prj, id) + logger.info("Removed %d exclusion(s) for %s: %s", len(removed), id, removed) + return removed + + +def set_merging_option(prj: "Project", target_type: str, id, **kwargs) -> None: + """ + Override one or more merging_options for just this one target, + persisted the same way as exclusions (highest-priority layer: these + override prj.reservoirs/rivers.merging_options, which override the + DEFAULT_*_MERGING_OPTIONS defaults). + + Example: set_merging_option(prj, "reservoirs", my_id, svr_radial_err=0.5) + """ + config = _load_run_config(prj, id) + config["merging_option_overrides"].update(kwargs) + _save_run_config(prj, id, config) + if "use_spatial_correction" in kwargs or "spatial_correction_dense_source" in kwargs: + _invalidate_spatial_correction_cache(prj, id) + if "use_reach_slope_correction" in kwargs: + _invalidate_reach_slope_correction_cache(prj, id) + logger.info("Updated merging options for %s: %s", id, kwargs) + + +def _apply_exclusions(df: pd.DataFrame, exclusions: list) -> pd.DataFrame: + """ + Filter out rows matching any exclusion rule. Within one rule, every + specified field (platform/orbit/date) must match for a row to be + excluded by it; a row is dropped if it matches ANY rule. + """ + if not exclusions: + return df + + keep_mask = pd.Series(True, index=df.index) + for rule in exclusions: + rule_mask = pd.Series(True, index=df.index) + if rule.get("platform") is not None: + rule_mask &= df["platform"] == rule["platform"] + if rule.get("orbit") is not None: + if "orbit" in df.columns: + # Compare numerically when possible, not as strings -- + # a real orbit column commonly gets upcast to float64 by + # pandas the moment ANY value in it is missing (very + # common in real satellite data), so a genuine orbit + # value of 1517 reads as 1517.0 in the dataframe while a + # YAML-loaded exclusion rule reads it as the plain int + # 1517. Comparing as strings ("1517.0" vs "1517") then + # silently matches nothing -- confirmed as a real, + # reproducible bug, not a hypothetical one. Falls back + # to string comparison only if the orbit value genuinely + # isn't numeric (e.g. a string-based identifier). + try: + target_orbit = float(rule["orbit"]) + rule_mask &= ( + pd.to_numeric(df["orbit"], errors="coerce") == target_orbit + ) + except (TypeError, ValueError): + rule_mask &= df["orbit"].astype(str) == str(rule["orbit"]) + else: + rule_mask &= False + if rule.get("date") is not None: + rule_mask &= df["date"].dt.floor("D").astype(str) == str(rule["date"]) + keep_mask &= ~rule_mask + + return df.loc[keep_mask].reset_index(drop=True) + + diff --git a/HydroEO/flows/_sentinel_shared.py b/HydroEO/flows/_sentinel_shared.py new file mode 100644 index 0000000..b041b65 --- /dev/null +++ b/HydroEO/flows/_sentinel_shared.py @@ -0,0 +1,118 @@ +"""Sentinel-3/6 download logic shared by both reservoirs and rivers. + +_sentinel6_use_earthdata and _download_sentinel_for_target are used by +both _reservoir_download.py and _river_download.py (the CREODIAS/EarthData +branching only needs to exist in one place); neither is itself the target +of a sibling patch in the test suite, so this module has no test-imposed +co-location constraint of its own. +""" + +import logging + +from HydroEO.satellites import sentinel +from HydroEO.utils import general + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def _sentinel6_use_earthdata(prj: "Project") -> bool: + """ + Whether Sentinel-6 should be downloaded from PO.DAAC/EarthData (HR + product, 20Hz Ku-band) rather than CREODIAS (LR product only, see + query()'s productType="P4_2__LR_____"). Set via + mission_options["sentinel6"]["source"] = "earthdata" in config. + """ + return ( + prj.mission_options.get("sentinel6", {}) + .get("source", "creodias") + .lower() + == "earthdata" + ) + + +def _download_sentinel_for_target( + prj: "Project", mission: str, product: str, coords, download_dir, + startdate, enddate, sentinel_creds, session_token, session_start_time, +) -> tuple: + """ + Download + subset Sentinel-3/6 data for one target's AOI (a + reservoir polygon, or a river waterbody corridor's envelope) -- + shared by both _download_reservoirs_sentinel and + _download_rivers_sentinel so the CREODIAS/EarthData branching logic + only needs to exist in one place. + + For Sentinel-6, if _sentinel6_use_earthdata(prj) is True, uses + PO.DAAC/EarthData (see sentinel.query_earthdata/download_earthdata) + to get the HR product instead of CREODIAS's LR-only product. + EarthData files arrive flat (no SAFE-zip directory), so the unzip + step is skipped for that path -- subset() already handles both flat + and zipped-folder inputs (see sentinel/preprocess.py's file + discovery, extended for this). + + Returns (session_token, session_start_time) -- unchanged from what + was passed in when using the EarthData path, since that mechanism + (CREODIAS session reuse) doesn't apply to it. + """ + dir_key = mission + use_earthdata = mission == "sentinel6" and _sentinel6_use_earthdata(prj) + + logger.info( + "Searching for Sentinel-%s (%s) from %s to %s", + product, + "PO.DAAC/EarthData HR" if use_earthdata else "CREODIAS", + startdate, + enddate, + ) + + if use_earthdata: + s6_opts = prj.mission_options.get("sentinel6", {}) + results = sentinel.query_earthdata( + aoi=coords, + startdate=startdate, + enddate=enddate, + latency=s6_opts.get("latency", "NTC"), + short_name=s6_opts.get("short_name"), + ) + sentinel.download_earthdata(results, download_directory=download_dir) + # EarthData granules arrive flat already -- no SAFE-zip to unzip. + else: + ids = sentinel.query( + aoi=coords, + startdate=startdate, + enddate=enddate, + product=product, + creodias_credentials=sentinel_creds, + ) + + session_token, session_start_time = sentinel.download( + ids, + download_directory=download_dir, + creodias_credentials=sentinel_creds, + token=session_token, + session_start_time=session_start_time, + threads=prj.mission_options.get(dir_key, {}).get("download_threads", 1), + ) + + general.unzip_dir_files_with_ext( + download_dir, download_dir, ".nc", show_progress=True + ) + + sentinel.subset( + aoi=coords, + download_dir=download_dir, + dest_dir=download_dir, + file_id=prj.mission_options.get(dir_key, {}).get( + "subset_file_id", "enhanced_measurement.nc" + ), + product=product, + show_progress=True, + ) + + return session_token, session_start_time + + diff --git a/HydroEO/flows/_summaries.py b/HydroEO/flows/_summaries.py new file mode 100644 index 0000000..93e5b36 --- /dev/null +++ b/HydroEO/flows/_summaries.py @@ -0,0 +1,341 @@ +"""Diagnostic plots for reservoirs and rivers, plus shared observation +loading helpers. + +generate_reservoirs_summaries is tested with _load_product_timeseries +patched as a sibling; generate_rivers_summaries is tested with +_has_enough_observations_to_plot/_project_num_months/_river_target_corridor/ +_load_merged_timeseries patched as siblings (tests/unit/test_flows.py) -- +so all of these live in one module. _river_target_corridor is also called +directly (unpatched) by the download/extraction modules, which import it +from here. +""" + +import logging +import os +import datetime + +import geopandas as gpd +import pandas as pd + +from HydroEO import plotting +from ._river_common import _group_river_targets_by_waterbody, _river_extraction_buffer_meters + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + +logger = logging.getLogger(__name__) + + +def _river_target_corridor( + prj: "Project", target_ids, buffer_meters=None, width_buffer_factor=1.05, +): + """ + Build one buffered, dissolved corridor polygon covering the given + river targets (nodes or reaches), for use as the spatial AOI when + downloading/extracting ICESat-2 and Sentinel-3/6 observations. + + This is deliberately a SEPARATE buffer distance from + prj.rivers.buffer_meters (used earlier to decide which SWORD + targets intersect the user's AOI at all) -- that question ("is this + target in scope") and this one ("how far from the centerline could + real river water still be, for a raw altimetry point to plausibly + belong to this target") are different, and conflating them risks + the same "one parameter doing two jobs badly" issue found elsewhere + in this pipeline. + + Parameters + ---------- + buffer_meters : float or None, optional + Explicit, uniform buffer distance (meters), applied to every + target regardless of its actual width. If None (default), uses + each target's own SWORD "width" attribute instead: buffer + distance = (width / 2) * width_buffer_factor. This is HALF the + width, not the full width -- buffering a line expands it + symmetrically by the given distance on EACH side, so a buffer + of width/2 gives a corridor whose TOTAL span is approximately + width * width_buffer_factor, matching the river's actual extent + plus a margin, rather than doubling it. Falls back to + _river_extraction_buffer_meters() (a flat scalar) if no usable + "width" column is found -- e.g. if your SWORD data names it + differently than assumed here, this degrades gracefully with a + log message rather than failing. + width_buffer_factor : float, optional + Margin applied on top of each target's own width when using the + width-based default. Default 1.05 -- 5% wider than the river's + actual channel width. Only used when buffer_meters is None. + + NOTE: "width" is the expected SWORD column name per the standard + SWORD data dictionary -- this has NOT been verified against a real + downloaded SWORD file in this session (no sample data was + available), unlike most other assumptions in this codebase. Check + your actual target_features columns if width-based buffering + doesn't seem to be kicking in. + + Returns a single-row GeoDataFrame in prj.global_crs (matching what + icesat2.extract_observations/sentinel.extract_observations expect + for their `features` argument, same as reservoirs), or None if no + matching SWORD geometry is found for target_ids. Note the returned + geometry may be a MultiPolygon if targets form disconnected pieces + (e.g. separate reaches far enough apart that their buffers never + touch) -- see _iter_geometry_pieces for how downloads handle this. + """ + features = prj.rivers.target_features + subset = features.loc[features[prj.rivers.target_id_col].isin(target_ids)] + if subset.empty: + return None + + local = subset.to_crs(prj.local_crs) + + if buffer_meters is not None: + distances = buffer_meters + elif "width" in local.columns and local["width"].notna().any(): + fallback_width = local["width"].median() + distances = (local["width"].fillna(fallback_width) / 2) * width_buffer_factor + else: + logger.info( + "No 'width' column found in SWORD target_features for this " + "waterbody -- falling back to a flat extraction buffer " + "instead of width-based sizing. Check your SWORD data's " + "actual column names if this is unexpected." + ) + distances = _river_extraction_buffer_meters(prj) + + buffered = local.buffer(distances) + corridor = buffered.unary_union + corridor_gdf = gpd.GeoDataFrame( + geometry=[corridor], crs=prj.local_crs + ).to_crs(prj.global_crs) + return corridor_gdf + + +def _load_product_timeseries(data_dir, ext, products, reader_fn): + """Load files of given extension from directory, optionally filtered to products.""" + if not os.path.exists(data_dir): + return None + df_list = [] + for file in os.listdir(data_dir): + if file.endswith(ext): + if not products or file.split(".")[0] in products: + try: + df_list.append(reader_fn(os.path.join(data_dir, file))) + except Exception as exc: + logger.warning( + "Failed to load %s from %s: %s", + file, + data_dir, + exc, + ) + return pd.concat(df_list) if df_list else None + + +def generate_reservoirs_summaries( + prj: "Project", show: bool = False, save: bool = True +) -> None: + """Generate per-reservoir plotting summaries. + + Parameters + ---------- + prj : Project + Project instance with reservoirs configuration + show : bool + Whether to display plots interactively + save : bool + Whether to save plots to disk + """ + if not hasattr(prj, "reservoirs"): + return + + for reservoir_id in prj.reservoirs.download_gdf[prj.reservoirs.id_key]: + plotting.plot_crossings( + gdf=prj.reservoirs.gdf, + id_key=prj.reservoirs.id_key, + reservoir_id=reservoir_id, + output_dir=prj.dirs["output"], + reservoir_type=prj.reservoirs.type, + show=show, + save=save, + ) + + plotting.plot_cleaning( + reservoir_id=reservoir_id, + output_dir=prj.dirs["output"], + get_unfiltered_fn=lambda id, products: _load_product_timeseries( + os.path.join(prj.dirs["output"], f"{id}", "raw_observations"), + ".gpkg", + products, + lambda path: gpd.read_file(path).drop(columns=["geometry"]), + ), + get_cleaned_fn=lambda id, products: _load_and_parse_cleaned_timeseries( + prj, id, products + ), + get_merged_fn=lambda id: _load_merged_timeseries(prj, id), + reservoir_type=prj.reservoirs.type, + show=show, + save=save, + products=getattr(prj, "to_process", None), + ) + + plotting.plot_merging( + reservoir_id=reservoir_id, + output_dir=prj.dirs["output"], + reservoir_type=prj.reservoirs.type, + show=show, + save=save, + ) + + +def _load_and_parse_cleaned_timeseries(prj, id, products): + """Load cleaned observations and parse dates.""" + df = _load_product_timeseries( + os.path.join(prj.dirs["output"], f"{id}", "cleaned_observations"), + ".csv", + products, + pd.read_csv, + ) + if df is not None: + df["date"] = pd.to_datetime(df.date, format="mixed", utc=True).dt.tz_convert( + None + ) + df = df.sort_values(by="date") + return df + + +def _load_merged_timeseries(prj, id): + """Load merged timeseries if it exists.""" + data_path = os.path.join(prj.dirs["output"], f"{id}", "merged_timeseries.csv") + + if os.path.exists(data_path): + df = pd.read_csv(data_path) + df["date"] = pd.to_datetime(df.date) + df = df.sort_values(by="date") + return df + else: + logger.warning( + "%s does not exist, be sure to merge product timeseries first!", + data_path, + ) + return None + + +# ============================================================================ +# RIVERS: Summaries & Visualization +# ============================================================================ + + +def _project_num_months(prj: "Project") -> int: + """ + Approximate number of months spanned by the project's configured + date range -- used as a minimum-observation-count threshold for + plotting (see _has_enough_observations_to_plot). Falls back to 1 if + the project-level dates aren't resolvable for some reason. + """ + project_cfg = prj.config.get("project", {}) + start = project_cfg.get("startdate") + end = project_cfg.get("enddate") + if not start or not end: + return 1 + + start_date = datetime.date(*start) if isinstance(start, list) else start + end_date = datetime.date(*end) if isinstance(end, list) else end + months = ( + (end_date.year - start_date.year) * 12 + + (end_date.month - start_date.month) + + 1 + ) + return max(months, 1) + + +def _has_enough_observations_to_plot(prj: "Project", target_id, min_months: int) -> bool: + """ + Whether a target has enough merged observations to be worth + plotting -- more than min_months (the project's date range in + months) or more than 2, whichever is larger. A reach/reservoir with + only 1-2 points produces a plot that adds noise without telling you + anything. + """ + df = _load_merged_timeseries(prj, target_id) + if df is None: + return False + threshold = max(min_months, 2) + return len(df) > threshold + + +def generate_rivers_summaries( + prj: "Project", show: bool = False, save: bool = True +) -> None: + """Generate per-river plotting summaries. + + Parameters + ---------- + prj : Project + Project instance with rivers configuration + show : bool + Whether to display plots interactively + save : bool + Whether to save plots to disk + """ + if not hasattr(prj, "rivers"): + return + + waterbody_groups = _group_river_targets_by_waterbody(prj) + min_months = _project_num_months(prj) + + for wb_id, target_ids in waterbody_groups.items(): + # Only plot targets with enough observations to be worth looking + # at -- applies to all three plot types (map, time series, merge + # progress) so a target excluded from one isn't confusingly still + # shown in another. + plottable_ids = [ + t for t in target_ids + if _has_enough_observations_to_plot(prj, t, min_months) + ] + if not plottable_ids: + logger.info( + "Skipping plots for waterbody %s -- no targets with more " + "than %d observations.", wb_id, max(min_months, 2), + ) + continue + + # Compute the actual extraction corridor (same buffer resolution + # used for real extraction, see _river_target_corridor) so the + # shaded area shown is exactly what extraction uses, not an + # approximation -- lets you visually judge whether width-based + # buffering produced a reasonable corridor for this waterbody + # (e.g. a lake-flagged reach whose SWORD width reflects a much + # wider lake extent) without needing external knowledge of the + # real river geometry. + corridor_gdf = _river_target_corridor( + prj, plottable_ids, + buffer_meters=getattr(prj.rivers, "extraction_buffer_meters", None), + width_buffer_factor=getattr(prj.rivers, "width_buffer_factor", 1.05), + ) + corridor_geometry = corridor_gdf.geometry.iloc[0] if corridor_gdf is not None else None + + plotting.plot_river_crossings( + prj, wb_id, plottable_ids, prj.dirs["output"], show=show, save=save, + corridor_geometry=corridor_geometry, + ) + + plotting.plot_river_data( + prj, wb_id, plottable_ids, prj.dirs["output"], + get_merged_fn=lambda id: _load_merged_timeseries(prj, id), + show=show, save=save, + ) + + for target_id in plottable_ids: + plotting.plot_merging( + reservoir_id=target_id, + output_dir=prj.dirs["output"], + reservoir_type="river", + show=show, + save=save, + ) + + +# ============================================================================ +# MIKEIO +# ============================================================================ + + diff --git a/HydroEO/project.py b/HydroEO/project.py index 915805b..a62cc24 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -310,10 +310,9 @@ def __post_init__(self): # User-configurable overrides for the merge()/Kalman/svr_radial # pipeline (see flows.DEFAULT_RIVER_MERGING_OPTIONS for every # available key and its default). Set directly on - # prj.rivers rather than routed through the shared - # project-level self.merging_options reservoirs use, so a - # project with both reservoirs and rivers configured doesn't - # have them silently collide -- see flows._merge_timeseries's + # prj.rivers, mirroring how prj.reservoirs.merging_options + # works, rather than routed through the shared project-level + # self.merging_options -- see flows._merge_timeseries's # per-target-type override lookup. # NOTE: DEFAULT_RIVER_MERGING_OPTIONS is currently a direct # copy of the reservoir defaults and has NOT been @@ -552,43 +551,74 @@ def download(self): ) def update(self): - # get the current date of the system + """Extend existing downloads through today. + + Re-runs download() for every configured mission with that + mission's enddate temporarily replaced by today's date + (startdate is left as configured). This relies on each + download function's own de-duplication rather than + reconstructing "resume from latest observation" logic here: + + - SWOT (satellites.swot._download.download), Sentinel-3/6 via + CREODIAS (satellites.sentinel.download), and Sentinel-6 via + EarthData (satellites.sentinel.download_earthdata) all track + already-downloaded granules in a `downloaded.log` file per + directory and only fetch what's new -- safe and cheap to + re-run the full configured range. + - ICESat-2 (satellites.icesat2.download.query) has no such + de-duplication: it always re-submits the full + [startdate, enddate] request to SlideRule and overwrites + atl13.parquet from scratch each call. Still correct (the + file always reflects the complete range afterward), but not + incremental -- update() costs roughly the same as a fresh + download() for ICESat-2 specifically. + + NOTE: this intentionally does not use the per-mission + get_latest_obs_date() helpers (satellites/{swot,icesat2, + sentinel}) to also advance startdate and skip already-covered + history. satellites.swot.preprocess.get_latest_obs_date() + currently returns the MAX observation date across all + reservoirs rather than the min, which would silently skip the + gap for any reservoir with sparser data if used as a shared + resume point -- fix that first if startdate-advancing is + wanted here. + + This previously called self.reservoirs.download(...) / + self.rivers.download(...), methods that do not exist on + Reservoirs/Rivers (waterbody.py defines only report()) with a + keyword signature (update_existing, enddate_overrides) that + flows.download_reservoirs/download_rivers never implemented -- + every call raised AttributeError. + """ + if not hasattr(self, "reservoirs") and not hasattr(self, "rivers"): + logger.warning( + "update() has no effect: neither 'reservoirs' nor 'rivers' is " + "configured for this project. (swot_raster/swot_pixc are " + "one-off extraction pipelines, not incremental archives, and " + "are not affected by update().)" + ) + return + current_date = datetime.date.today() logger.info("Updating download archives up to %s", current_date) - current_date = [current_date.year, current_date.month, current_date.day] + current_date_list = [current_date.year, current_date.month, current_date.day] - if hasattr(self, "reservoirs"): - if "swot" in self.to_download: - logger.info("Updating SWOT Lake SP product") - if "icesat2" in self.to_download: - logger.info("Updating Icesat-2 ATL13 product") - if "sentinel3" in self.to_download: - logger.info("Updating Sentinel-3 Hydro product") - if "sentinel6" in self.to_download: - logger.info("Updating Sentinel-6 Hydro product") - - self.reservoirs.download( - to_download=self.to_download, - startdates=self.startdates, - enddates=self.enddates, - earthdata_credentials=(self.earthdata_user, self.earthdata_pass), - creodias_credentials_provider=self._require_creodias_credentials, - update_existing=True, - enddate_overrides={ - mission: current_date for mission in self.to_download - }, - ) + mission_labels = { + "swot": "SWOT Lake SP / Hydrocron product", + "icesat2": "ICESat-2 ATL13 product", + "sentinel3": "Sentinel-3 Hydro product", + "sentinel6": "Sentinel-6 Hydro product", + } - if hasattr(self, "rivers"): - self.rivers.download( - to_download=self.to_download, - startdates=self.startdates, - enddates=self.enddates, - update_existing=True, - enddate_overrides={ - mission: current_date for mission in self.to_download - }, - ) + original_enddates = dict(self.enddates) + try: + for mission in self.to_download: + self.enddates[mission] = current_date_list + logger.info("Updating %s", mission_labels.get(mission, mission)) + + self.download() + finally: + self.enddates = original_enddates def create_timeseries(self): warnings.filterwarnings("ignore", module="pyogrio\\..*") @@ -608,26 +638,23 @@ def generate_summaries(self, show=False, save=True): def _infer_target_type(self, target_type=None): """ Resolve which target type (reservoirs/rivers) a per-target call - applies to. If target_type is given explicitly, use it. Otherwise, - infer it automatically when the project only has one of the two - configured -- the common case -- and require an explicit choice - only when both are configured, since there's no way to guess - correctly between them. + applies to. If target_type is given explicitly, use it (mainly + useful for tests or direct flows.* calls on a Project built + without going through normal config validation). Otherwise, + infer it from whichever of prj.reservoirs/prj.rivers is present. + + validate_config() only allows one of 'reservoirs'/'rivers'/ + 'swot_raster'/'swot_pixc' to be active per config, so a validly + constructed Project never has both prj.reservoirs and + prj.rivers set -- there is no "both configured" case to + disambiguate here. """ if target_type is not None: return target_type - has_reservoirs = hasattr(self, "reservoirs") - has_rivers = hasattr(self, "rivers") - if has_reservoirs and not has_rivers: + if hasattr(self, "reservoirs"): return "reservoirs" - if has_rivers and not has_reservoirs: + if hasattr(self, "rivers"): return "rivers" - if has_reservoirs and has_rivers: - raise ValueError( - "Both reservoirs and rivers are configured for this " - "project -- specify target_type='reservoirs' or " - "target_type='rivers' explicitly." - ) raise ValueError("Neither reservoirs nor rivers is configured for this project.") def list_target_observations(self, id, target_type=None): diff --git a/HydroEO/satellites/sentinel/__init__.py b/HydroEO/satellites/sentinel/__init__.py index 59c4061..6f8e768 100644 --- a/HydroEO/satellites/sentinel/__init__.py +++ b/HydroEO/satellites/sentinel/__init__.py @@ -96,12 +96,16 @@ def subset( ) -def extract_observations(src_dir, dst_path, features, sigma0_max=1e5): +def extract_observations( + src_dir, dst_path, features, sigma0_max=1e5, processed_log_path=None, overwrite=False +): return _preprocess.extract_observations( src_dir=src_dir, dst_path=dst_path, features=features, sigma0_max=sigma0_max, + processed_log_path=processed_log_path, + overwrite=overwrite, ) diff --git a/HydroEO/satellites/sentinel/preprocess.py b/HydroEO/satellites/sentinel/preprocess.py index 3a0d2ad..28609ab 100644 --- a/HydroEO/satellites/sentinel/preprocess.py +++ b/HydroEO/satellites/sentinel/preprocess.py @@ -11,6 +11,7 @@ import pandas as pd import geopandas as gpd +from HydroEO.utils import general from HydroEO.utils.general import center_longitude @@ -805,13 +806,41 @@ def __format_coord_bounds(aoi): return shapely.Polygon(geometry.format_coord_list(aoi)).bounds -def extract_observations(src_dir, dst_path, features, sigma0_max=1e5): - """Extract Sentinel observations to shapefile within feature geometries.""" +def extract_observations( + src_dir, dst_path, features, sigma0_max=1e5, processed_log_path=None, overwrite=False +): + """Extract Sentinel observations to shapefile within feature geometries. + + Parameters + ---------- + processed_log_path : str, optional + Path to a newline-delimited log of subset file names already + incorporated into dst_path (see HydroEO.utils.general. + read_id_log/append_id_log/write_id_log). If given, only files + NOT yet in the log are read, and the resulting new observations + are appended to (and de-duplicated against) the existing + dst_path rather than overwriting it. If None (default), + behaves exactly as before: reads every file in src_dir and + overwrites dst_path from scratch. + overwrite : bool, optional + If True, ignores processed_log_path's existing contents, + re-reads every file in src_dir, overwrites dst_path from + scratch, and replaces processed_log_path wholesale with + exactly the files just read. + """ + incremental = processed_log_path is not None and not overwrite + already_processed = ( + general.read_id_log(processed_log_path) if incremental else set() + ) + # read data for each availble option in directory gdf_list = list() + files_read = list() files = list(os.listdir(src_dir)) for file in files: + if incremental and file in already_processed: + continue try: file_split = file.split("_") if file_split[0] == "sub": # assume that we have subsetted the data already @@ -857,14 +886,29 @@ def extract_observations(src_dir, dst_path, features, sigma0_max=1e5): # if we have data for the reservoir add it to the reservoir specific dataframe gdf_list.append(data_gdf) + + files_read.append(file) except Exception: logger.exception("Unable to open sentinel file: %s", file) + # Not added to files_read: a file that raised here (e.g. a + # partially-written/corrupt subset) is retried on the next + # extraction run rather than being marked processed and + # silently skipped forever. # once all tracks are processed combine them and save in the destination dir if len(gdf_list) > 0: observations = pd.concat(gdf_list).reset_index(drop=True) observations = observations.set_crs(features.crs) - observations.to_file(dst_path, driver="GPKG") + if incremental: + general.append_and_dedupe_gpkg(dst_path, observations) + else: + observations.to_file(dst_path, driver="GPKG") + + if processed_log_path: + if overwrite: + general.write_id_log(processed_log_path, files_read) + elif files_read: + general.append_id_log(processed_log_path, files_read) def get_latest_obs_date(data_dir, product): diff --git a/HydroEO/satellites/swot/__init__.py b/HydroEO/satellites/swot/__init__.py index 8275083..bb30f37 100644 --- a/HydroEO/satellites/swot/__init__.py +++ b/HydroEO/satellites/swot/__init__.py @@ -44,6 +44,8 @@ def extract_observations( features, id_key, exclude_obs_id_values=None, + processed_log_path=None, + overwrite=False, ): return _preprocess.extract_observations( src_dir=src_dir, @@ -53,6 +55,8 @@ def extract_observations( id_key=id_key, exclude_obs_id_values=exclude_obs_id_values, product_name=SWOT_LAKE_SHORT_NAME, + processed_log_path=processed_log_path, + overwrite=overwrite, ) diff --git a/HydroEO/satellites/swot/preprocess.py b/HydroEO/satellites/swot/preprocess.py index c1d8abf..3a86a61 100644 --- a/HydroEO/satellites/swot/preprocess.py +++ b/HydroEO/satellites/swot/preprocess.py @@ -67,6 +67,30 @@ def merge_shps(dir): return gdf +def _merge_new_shps(dir, exclude=None): + """Like merge_shps, but skips file names in `exclude` and also + returns the list of file names actually read, so a caller can build + an "already extracted" log from it (see extract_observations's + processed_log_path). Kept separate from the public merge_shps() + rather than changing its signature/return type, since merge_shps is + exported as public API (HydroEO.satellites.swot.merge_shps) and + external callers expect a single GeoDataFrame back. + """ + exclude = exclude or set() + gdf_list = list() + read_files = list() + for file in os.listdir(dir): + if file.endswith(".shp") and file not in exclude: + gdf_list.append(gpd.read_file(os.path.join(dir, file))) + read_files.append(file) + + if not gdf_list: + return None, read_files + + gdf = pd.concat(gdf_list).reset_index(drop=True) + return gdf, read_files + + def extract_observations( src_dir, dst_dir, @@ -75,9 +99,44 @@ def extract_observations( id_key, exclude_obs_id_values=None, product_name="SWOT_L2_HR_LakeSP_D", + processed_log_path=None, + overwrite=False, ): - # load in combined observations from individual files in download directory - data_gdf = merge_shps(src_dir) + """Extract per-reservoir SWOT Lake SP observations from downloaded granules. + + Parameters + ---------- + processed_log_path : str, optional + Path to a newline-delimited log (see HydroEO.utils.general. + read_id_log/append_id_log/write_id_log) of granule shapefile + names already incorporated into reservoirs' raw_observations/ + swot.gpkg files. If given, only shapefiles NOT yet in the log + are read, and the resulting new observations are appended to + (and de-duplicated against, on "obs_id") each reservoir's + existing output rather than overwriting it -- this avoids + re-reading every SWOT granule shapefile ever downloaded on + every extraction run (raw granule shapefiles accumulate + forever in src_dir across all reservoirs -- see + HydroEO.satellites.swot.preprocess.subset_by_id -- so a full + re-merge scales with total history downloaded to date, not + with what's actually new). If None (default), behaves exactly + as before: reads every shapefile in src_dir and overwrites each + reservoir's output from scratch. + overwrite : bool, optional + If True, ignores processed_log_path's existing contents (reads + every shapefile in src_dir, as if nothing had been processed + yet), overwrites each reservoir's output from scratch rather + than appending, and replaces processed_log_path wholesale with + exactly the files just read. Use when raw data or extraction + logic changed in a way that could make previously-extracted + rows stale. + """ + incremental = processed_log_path is not None and not overwrite + already_processed = ( + general.read_id_log(processed_log_path) if incremental else set() + ) + + data_gdf, files_read = _merge_new_shps(src_dir, exclude=already_processed) if data_gdf is None: return [] excluded_obs_ids = set(exclude_obs_id_values or ["no_data"]) @@ -117,10 +176,23 @@ def extract_observations( general.ifnotmakedirs(dst_sub_dir) dst_path = os.path.join(dst_sub_dir, dst_file_name) - observations.to_file(dst_path) + if incremental: + general.append_and_dedupe_gpkg( + dst_path, + observations, + subset=["obs_id"] if "obs_id" in observations.columns else None, + ) + else: + observations.to_file(dst_path) else: empty_ids.append(dl_id) + if processed_log_path: + if overwrite: + general.write_id_log(processed_log_path, files_read) + elif files_read: + general.append_id_log(processed_log_path, files_read) + return empty_ids diff --git a/HydroEO/utils/filters/basic_filters.py b/HydroEO/utils/filters/basic_filters.py index 8086d49..f249a75 100644 --- a/HydroEO/utils/filters/basic_filters.py +++ b/HydroEO/utils/filters/basic_filters.py @@ -69,6 +69,7 @@ def _resolve_pass_groups(df, date_key, pass_key=None, platform_key=None, orbit_k Returns a pandas Series of group labels (strings), same index as df. """ + n = len(df) group = pd.Series(pd.NA, index=df.index, dtype=object) if pass_key and pass_key in df.columns: diff --git a/HydroEO/utils/general.py b/HydroEO/utils/general.py index 3ea7b8c..d578240 100644 --- a/HydroEO/utils/general.py +++ b/HydroEO/utils/general.py @@ -86,6 +86,98 @@ def remove_non_exts(dir: str, ext: Union[str, list]): os.remove(item) +def read_id_log(log_path: str) -> set: + """Read a newline-delimited log of identifiers into a set. + + Matches the `downloaded.log` convention already used by + HydroEO.satellites.swot._download.download and + HydroEO.satellites.sentinel.download to track which granules have + already been fetched. Used the same way here to track which raw + files have already been read during extraction (see + HydroEO.satellites.swot.preprocess.extract_observations and + HydroEO.satellites.sentinel.preprocess.extract_observations). + + Returns an empty set if the log doesn't exist yet. + """ + if not os.path.exists(log_path): + return set() + with open(log_path, "r") as f: + return {line.rstrip() for line in f if line.strip()} + + +def append_id_log(log_path: str, ids) -> None: + """Append identifiers to a newline-delimited log, creating it if needed.""" + ids = list(ids) + if not ids: + return + log_dir = os.path.dirname(log_path) + if log_dir: + ifnotmakedirs(log_dir) + with open(log_path, "a") as f: + for i in ids: + f.write(f"{i}\n") + + +def write_id_log(log_path: str, ids) -> None: + """Replace a newline-delimited log wholesale with the given identifiers. + + Used after a forced full re-extraction (overwrite=True), so the log + reflects exactly the files that were just (re)read, rather than + keeping stale entries from before the rebuild or leaving the log + partially out of sync with what's actually on disk. + """ + log_dir = os.path.dirname(log_path) + if log_dir: + ifnotmakedirs(log_dir) + with open(log_path, "w") as f: + for i in ids: + f.write(f"{i}\n") + + +def append_and_dedupe_gpkg(dst_path: str, new_gdf, subset=None): + """Merge newly-extracted observations into an existing GeoPackage. + + If `dst_path` doesn't exist yet, `new_gdf` is written as-is. If it + does exist, the existing contents are read, concatenated with + `new_gdf`, de-duplicated, and written back -- so repeated + incremental extraction runs (see satellites.swot.preprocess and + satellites.sentinel.preprocess extract_observations) accumulate + history instead of losing whatever was previously extracted. + + Parameters + ---------- + subset : list[str], optional + Non-geometry columns to de-duplicate on (e.g. ["obs_id"] for a + stable per-observation identifier). Defaults to every + non-geometry column when no such identifier is available -- + rows are only considered duplicates if they match everywhere + except geometry, which is a safe (if slightly conservative) + fallback since genuinely new observations essentially never + collide on every other field by chance. + + Returns + ------- + GeoDataFrame + The combined, de-duplicated result that was written to disk. + """ + import geopandas as gpd + import pandas as pd + + if os.path.exists(dst_path): + existing = gpd.read_file(dst_path) + combined = pd.concat([existing, new_gdf], ignore_index=True) + dedup_cols = subset or [c for c in combined.columns if c != "geometry"] + combined = combined.drop_duplicates(subset=dedup_cols, keep="last").reset_index( + drop=True + ) + combined = gpd.GeoDataFrame(combined, geometry="geometry", crs=new_gdf.crs) + else: + combined = new_gdf + + combined.to_file(dst_path, driver="GPKG") + return combined + + def center_longitude(lon_org): # This assumes the longitude is provided in degrees east of the greenwhich meridian diff --git a/tests/unit/test_flows.py b/tests/unit/test_flows.py index 7e6de96..45a54d9 100644 --- a/tests/unit/test_flows.py +++ b/tests/unit/test_flows.py @@ -112,9 +112,9 @@ def test_initialize_reservoirs_with_pld_download(mock_project_reservoirs): mock_project_reservoirs.to_process = [] with ( - patch.object(flows, "_download_pld") as mock_download, - patch.object(flows, "_assign_pld_id") as mock_assign, - patch.object(flows, "_flag_missing_priors") as mock_flag, + patch.object(flows._reservoir_init, "_download_pld") as mock_download, + patch.object(flows._reservoir_init, "_assign_pld_id") as mock_assign, + patch.object(flows._reservoir_init, "_flag_missing_priors") as mock_flag, ): flows.initialize_reservoirs(mock_project_reservoirs) @@ -136,7 +136,7 @@ def test_initialize_rivers_aoi_branch(mock_project_rivers): mock_project_rivers.rivers.feature_type = "nodes" mock_project_rivers.rivers.buffer_meters = 500 - with patch.object(flows, "_prepare_rivers_from_sword") as mock_prepare: + with patch.object(flows._river_init, "_prepare_rivers_from_sword") as mock_prepare: flows.initialize_rivers(mock_project_rivers) mock_prepare.assert_called_once_with(mock_project_rivers) @@ -146,7 +146,7 @@ def test_initialize_rivers_configured_id_branch(mock_project_rivers): """initialize_rivers skips SWORD for configured_id mode.""" mock_project_rivers.rivers.input_mode = "configured_id" - with patch.object(flows, "_prepare_rivers_from_sword") as mock_prepare: + with patch.object(flows._river_init, "_prepare_rivers_from_sword") as mock_prepare: flows.initialize_rivers(mock_project_rivers) mock_prepare.assert_not_called() @@ -183,7 +183,7 @@ def test_prepare_sword_skips_when_subset_exists(mock_project_rivers, tmp_path): mock_project_rivers.rivers.buffer_meters = 0 mock_project_rivers.rivers.id_key = "river_id" - with patch.object(flows, "_ensure_sword_database") as mock_ensure: + with patch.object(flows._river_init, "_ensure_sword_database") as mock_ensure: flows._prepare_rivers_from_sword(mock_project_rivers) # Should NOT call _ensure_sword_database mock_ensure.assert_not_called() @@ -222,7 +222,7 @@ def test_prepare_sword_saves_subset(mock_project_rivers, tmp_path): mock_project_rivers.rivers.id_key = "river_id" mock_project_rivers.local_crs = "EPSG:3857" - with patch.object(flows, "_ensure_sword_database"): + with patch.object(flows._river_init, "_ensure_sword_database"): flows._prepare_rivers_from_sword(mock_project_rivers) # Verify subset was saved @@ -391,9 +391,9 @@ def test_download_reservoirs_dispatches_swot(mock_project_reservoirs): mock_project_reservoirs.to_download = ["swot"] with ( - patch.object(flows, "_download_reservoirs_swot") as mock_swot, - patch.object(flows, "_download_reservoirs_icesat2") as mock_ice, - patch.object(flows, "_download_reservoirs_sentinel") as mock_sent, + patch.object(flows._reservoir_download, "_download_reservoirs_swot") as mock_swot, + patch.object(flows._reservoir_download, "_download_reservoirs_icesat2") as mock_ice, + patch.object(flows._reservoir_download, "_download_reservoirs_sentinel") as mock_sent, ): flows.download_reservoirs(mock_project_reservoirs) @@ -413,9 +413,9 @@ def test_download_reservoirs_dispatches_all_missions(mock_project_reservoirs): ] with ( - patch.object(flows, "_download_reservoirs_swot") as mock_swot, - patch.object(flows, "_download_reservoirs_icesat2") as mock_ice, - patch.object(flows, "_download_reservoirs_sentinel") as mock_sent, + patch.object(flows._reservoir_download, "_download_reservoirs_swot") as mock_swot, + patch.object(flows._reservoir_download, "_download_reservoirs_icesat2") as mock_ice, + patch.object(flows._reservoir_download, "_download_reservoirs_sentinel") as mock_sent, ): flows.download_reservoirs(mock_project_reservoirs) @@ -430,8 +430,8 @@ def test_download_reservoirs_skips_disabled_missions(mock_project_reservoirs): mock_project_reservoirs.to_download = ["swot"] with ( - patch.object(flows, "_download_reservoirs_swot") as mock_swot, - patch.object(flows, "_download_reservoirs_icesat2") as mock_ice, + patch.object(flows._reservoir_download, "_download_reservoirs_swot") as mock_swot, + patch.object(flows._reservoir_download, "_download_reservoirs_icesat2") as mock_ice, ): flows.download_reservoirs(mock_project_reservoirs) @@ -444,7 +444,7 @@ def test_download_rivers_calls_hydrocron(mock_project_rivers): """download_rivers calls _download_swot_hydrocron_timeseries.""" mock_project_rivers.to_download = ["swot"] - with patch.object(flows, "_download_swot_hydrocron_timeseries") as mock_hydrocron: + with patch.object(flows._river_download, "_download_swot_hydrocron_timeseries") as mock_hydrocron: flows.download_rivers(mock_project_rivers) mock_hydrocron.assert_called_once() @@ -456,12 +456,18 @@ def test_download_rivers_calls_hydrocron(mock_project_rivers): @pytest.mark.unit def test_download_rivers_skips_when_swot_not_in_to_download(mock_project_rivers): - """download_rivers returns early if swot not in to_download.""" + """download_rivers does not call _download_swot_hydrocron_timeseries + when 'swot' is not in to_download, even though other configured + missions (here, icesat2) still proceed normally.""" mock_project_rivers.to_download = ["icesat2"] - with patch.object(flows, "_download_swot_hydrocron_timeseries") as mock_hydrocron: + with ( + patch.object(flows._river_download, "_download_swot_hydrocron_timeseries") as mock_hydrocron, + patch.object(flows._river_download, "_download_rivers_icesat2") as mock_icesat2, + ): flows.download_rivers(mock_project_rivers) mock_hydrocron.assert_not_called() + mock_icesat2.assert_called_once() # ============================================================================ @@ -477,7 +483,7 @@ def test_create_reservoirs_timeseries_orchestrates_steps(mock_project_reservoirs call_order = [] - def track_extract(prj): + def track_extract(prj, **kwargs): call_order.append("extract") def track_clean(prj): @@ -488,10 +494,10 @@ def track_merge(prj): with ( patch.object( - flows, "_extract_reservoirs_timeseries", side_effect=track_extract + flows._reservoir_pipeline, "_extract_reservoirs_timeseries", side_effect=track_extract ), - patch.object(flows, "_clean_reservoirs_timeseries", side_effect=track_clean), - patch.object(flows, "_merge_reservoirs_timeseries", side_effect=track_merge), + patch.object(flows._reservoir_pipeline, "_clean_reservoirs_timeseries", side_effect=track_clean), + patch.object(flows._reservoir_pipeline, "_merge_reservoirs_timeseries", side_effect=track_merge), ): flows.create_reservoirs_timeseries(mock_project_reservoirs) @@ -510,7 +516,7 @@ def test_create_reservoirs_timeseries_calls_export_when_toggled( call_order = [] - def track_extract(prj): + def track_extract(prj, **kwargs): call_order.append("extract") def track_clean(prj): @@ -524,11 +530,11 @@ def track_merge(prj): with ( patch.object( - flows, "_extract_reservoirs_timeseries", side_effect=track_extract + flows._reservoir_pipeline, "_extract_reservoirs_timeseries", side_effect=track_extract ), - patch.object(flows, "_clean_reservoirs_timeseries", side_effect=track_clean), - patch.object(flows, "_export_cleaned_to_dfs0", side_effect=track_export), - patch.object(flows, "_merge_reservoirs_timeseries", side_effect=track_merge), + patch.object(flows._reservoir_pipeline, "_clean_reservoirs_timeseries", side_effect=track_clean), + patch.object(flows._reservoir_pipeline, "_export_cleaned_to_dfs0", side_effect=track_export), + patch.object(flows._reservoir_pipeline, "_merge_reservoirs_timeseries", side_effect=track_merge), ): flows.create_reservoirs_timeseries(mock_project_reservoirs) @@ -546,10 +552,10 @@ def test_create_reservoirs_timeseries_skips_export_when_disabled( mock_project_reservoirs.reservoirs.export_to_dfs0 = False with ( - patch.object(flows, "_extract_reservoirs_timeseries"), - patch.object(flows, "_clean_reservoirs_timeseries"), - patch.object(flows, "_export_cleaned_to_dfs0") as mock_export, - patch.object(flows, "_merge_reservoirs_timeseries"), + patch.object(flows._reservoir_pipeline, "_extract_reservoirs_timeseries"), + patch.object(flows._reservoir_pipeline, "_clean_reservoirs_timeseries"), + patch.object(flows._reservoir_pipeline, "_export_cleaned_to_dfs0") as mock_export, + patch.object(flows._reservoir_pipeline, "_merge_reservoirs_timeseries"), ): flows.create_reservoirs_timeseries(mock_project_reservoirs) @@ -639,7 +645,7 @@ def test_generate_reservoirs_summaries_iterates_ids(mock_project_reservoirs): ) with ( - patch.object(flows, "_load_product_timeseries"), + patch.object(flows._summaries, "_load_product_timeseries"), patch("HydroEO.flows.plotting.plot_crossings") as mock_plot, patch("HydroEO.flows.plotting.plot_cleaning"), patch("HydroEO.flows.plotting.plot_merging"), @@ -652,6 +658,122 @@ def test_generate_reservoirs_summaries_iterates_ids(mock_project_reservoirs): assert mock_plot.call_count >= 2 +# ============================================================================ +# Rivers Timeseries Processing Tests +# ============================================================================ +# +# Mirrors the "Timeseries Processing Tests" section above, which covers the +# reservoirs equivalents (create_reservoirs_timeseries, +# generate_reservoirs_summaries). create_rivers_timeseries and +# generate_rivers_summaries had no coverage at all prior to this section -- +# see tests/unit/test_timeseries.py for the corresponding +# _extract_rivers_*/_clean_rivers_timeseries/_merge_rivers_timeseries tests. + + +@pytest.mark.unit +def test_create_rivers_timeseries_orchestrates_steps(mock_project_rivers): + """create_rivers_timeseries calls extract, clean, and merge in order.""" + mock_project_rivers.to_process = ["swot"] + mock_project_rivers.processing_options = {} + + call_order = [] + + def track_extract(prj, **kwargs): + call_order.append("extract") + + def track_clean(prj): + call_order.append("clean") + + def track_merge(prj): + call_order.append("merge") + + with ( + patch.object(flows._river_pipeline, "_extract_rivers_timeseries", side_effect=track_extract), + patch.object(flows._river_pipeline, "_clean_rivers_timeseries", side_effect=track_clean), + patch.object(flows._river_pipeline, "_merge_rivers_timeseries", side_effect=track_merge), + ): + flows.create_rivers_timeseries(mock_project_rivers) + + # Unlike reservoirs, rivers has no dfs0 export step (see + # flows.create_rivers_timeseries's docstring). + assert call_order == ["extract", "clean", "merge"] + + +@pytest.mark.unit +def test_create_rivers_timeseries_noop_when_rivers_not_configured( + mock_project_reservoirs, +): + """create_rivers_timeseries is a no-op for a reservoirs-only project.""" + with ( + patch.object(flows._river_pipeline, "_extract_rivers_timeseries") as mock_extract, + patch.object(flows._river_pipeline, "_clean_rivers_timeseries") as mock_clean, + patch.object(flows._river_pipeline, "_merge_rivers_timeseries") as mock_merge, + ): + flows.create_rivers_timeseries(mock_project_reservoirs) + + mock_extract.assert_not_called() + mock_clean.assert_not_called() + mock_merge.assert_not_called() + + +@pytest.mark.unit +def test_generate_rivers_summaries_plots_plottable_targets(mock_project_rivers): + """generate_rivers_summaries plots each waterbody with enough observations, + passing the waterbody id and its plottable target ids through to each + plotting call.""" + mock_project_rivers.rivers.target_features = None + mock_project_rivers.rivers.configured_id = "loire" + mock_project_rivers.rivers.target_ids = [101, 102] + + with ( + patch.object(flows._summaries, "_has_enough_observations_to_plot", return_value=True), + patch.object(flows._summaries, "_project_num_months", return_value=3), + patch.object(flows._summaries, "_river_target_corridor", return_value=None), + patch.object(flows._summaries, "_load_merged_timeseries", return_value=None), + patch("HydroEO.flows.plotting.plot_river_crossings") as mock_crossings, + patch("HydroEO.flows.plotting.plot_river_data") as mock_data, + patch("HydroEO.flows.plotting.plot_merging") as mock_merging, + ): + flows.generate_rivers_summaries(mock_project_rivers, show=False, save=False) + + mock_crossings.assert_called_once() + call_args = mock_crossings.call_args + assert call_args[0][1] == "loire" + assert sorted(call_args[0][2]) == [101, 102] + + mock_data.assert_called_once() + # plot_merging is called once per plottable target, not once per waterbody + assert mock_merging.call_count == 2 + + +@pytest.mark.unit +def test_generate_rivers_summaries_skips_waterbody_without_plottable_targets( + mock_project_rivers, caplog +): + """generate_rivers_summaries skips a waterbody entirely (all three plot + types) when none of its targets have enough observations.""" + import logging + + mock_project_rivers.rivers.target_features = None + mock_project_rivers.rivers.configured_id = "loire" + mock_project_rivers.rivers.target_ids = [101, 102] + + with ( + patch.object(flows._summaries, "_has_enough_observations_to_plot", return_value=False), + patch.object(flows._summaries, "_project_num_months", return_value=3), + patch("HydroEO.flows.plotting.plot_river_crossings") as mock_crossings, + patch("HydroEO.flows.plotting.plot_river_data") as mock_data, + patch("HydroEO.flows.plotting.plot_merging") as mock_merging, + caplog.at_level(logging.INFO), + ): + flows.generate_rivers_summaries(mock_project_rivers, show=False, save=False) + + mock_crossings.assert_not_called() + mock_data.assert_not_called() + mock_merging.assert_not_called() + assert "Skipping plots for waterbody" in caplog.text + + # ============================================================================ # Helper Function Tests # ============================================================================ diff --git a/tests/unit/test_project_config.py b/tests/unit/test_project_config.py index d4d4cb1..a76784a 100644 --- a/tests/unit/test_project_config.py +++ b/tests/unit/test_project_config.py @@ -277,7 +277,7 @@ def _unexpected_prepare(*_args, **_kwargs): ) monkeypatch.setattr( - "HydroEO.flows._prepare_rivers_from_sword", + "HydroEO.flows._river_init._prepare_rivers_from_sword", _unexpected_prepare, ) @@ -462,9 +462,16 @@ def test_project_global_date_fallback(tmp_path, monkeypatch, _mock_reservoir_gdf @pytest.mark.unit -def test_project_warns_incompatible_satellites_for_rivers(tmp_path): - """ICESat-2/Sentinel-3/6 with download/process=True should warn when no reservoirs section.""" +def test_project_no_warning_for_icesat2_with_rivers_configured(tmp_path): + """ICESat-2 (and Sentinel-3/6) support rivers directly (see + flows._download_rivers_icesat2/_download_rivers_sentinel), so no + UserWarning should fire when a rivers section is present -- only when + NEITHER reservoirs nor rivers is configured (see + test_project_warns_incompatible_satellites_when_neither_mode_configured + for that case). This replaces a stale test that expected a warning + here from before ICESat-2/Sentinel gained river support.""" from HydroEO.project import Project + import warnings cfg_path = tmp_path / "config.yaml" _write_config( @@ -482,8 +489,46 @@ def test_project_warns_incompatible_satellites_for_rivers(tmp_path): }, ) + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + Project(name="rivers-no-warn", config=str(cfg_path)) + + +@pytest.mark.unit +def test_project_warns_incompatible_satellites_when_neither_mode_configured(tmp_path): + """ICESat-2/Sentinel-3/6 configured for download/process without a + reservoirs or rivers section genuinely has no effect (neither can + spatially filter observations without one), so this is the one case + that should still warn.""" + from HydroEO.project import Project + + cfg_path = tmp_path / "config.yaml" + _write_config( + cfg_path, + { + "project": { + "main_dir": str(tmp_path / "out"), + "startdate": [2024, 1, 1], + "enddate": [2024, 2, 1], + }, + "gis": {"global_crs": "EPSG:4326"}, + "swot_raster": { + "aoi": {"name": "aoi", "type": "bbox", "bbox": [0, 0, 1, 1]}, + "product": "SWOT_L2_HR_Raster_D", + "startdate": [2024, 1, 1], + "enddate": [2024, 2, 1], + }, + "icesat2": { + "download": True, + "process": False, + "startdate": [2024, 1, 1], + "enddate": [2024, 2, 1], + }, + }, + ) + with pytest.warns(UserWarning, match="icesat2"): - Project(name="rivers-warn", config=str(cfg_path)) + Project(name="neither-mode-warn", config=str(cfg_path)) @pytest.mark.unit diff --git a/tests/unit/test_timeseries.py b/tests/unit/test_timeseries.py index 8018250..e99e273 100644 --- a/tests/unit/test_timeseries.py +++ b/tests/unit/test_timeseries.py @@ -11,7 +11,7 @@ from shapely.geometry import box, Point from HydroEO import flows -from HydroEO.waterbody import Reservoirs +from HydroEO.waterbody import Reservoirs, Rivers # ============================================================================ @@ -62,6 +62,80 @@ def mock_project_reservoirs(tmp_path): return prj +@pytest.fixture +def mock_project_rivers(tmp_path): + """Create a mock Project with Rivers configuration. + + Two targets (node_id 101, 102) belonging to the same waterbody + ("loire"), placed ~11km apart (0.1 degrees latitude) so a small + extraction/assignment buffer can't accidentally blur observations + from one target into the other, which would mask assignment bugs + rather than catch them. + """ + gdf = gpd.GeoDataFrame({"geometry": []}, geometry="geometry", crs="EPSG:4326") + prj = SimpleNamespace() + prj.dirs = { + "main": str(tmp_path), + "output": str(tmp_path / "results"), + "swot": str(tmp_path / "raw" / "swot"), + "icesat2_processed": str(tmp_path / "processed" / "icesat2"), + "sentinel3": str(tmp_path / "raw" / "sentinel3"), + "sentinel6": str(tmp_path / "raw" / "sentinel6"), + "sword": str(tmp_path / "aux" / "SWORD" / "gpkg"), + "sword_subset": str(tmp_path / "aux" / "SWORD" / "SWORD_subset.gpkg"), + } + prj.rivers = Rivers(gdf=gdf, id_key="river_id", dirs=prj.dirs) + prj.rivers.target_ids = [101, 102] + prj.rivers.target_id_col = "node_id" + prj.rivers.target_features = gpd.GeoDataFrame( + { + "node_id": [101, 102], + "river_id": ["loire", "loire"], + "geometry": [Point(0, 0), Point(0, 0.1)], + }, + crs="EPSG:4326", + ) + prj.rivers.configured_id = "loire" + prj.rivers.input_mode = "configured_id" + # Fixed buffer (rather than SWORD width-based sizing) so tests don't + # depend on a "width" column being present -- see + # flows._river_target_corridor's docstring for the width-based path. + prj.rivers.extraction_buffer_meters = 500.0 + prj.rivers.width_buffer_factor = 1.05 + prj.rivers.max_node_assignment_meters = 1000.0 + prj.local_crs = "EPSG:3857" + prj.global_crs = "EPSG:4326" + prj.keep_raw_sword = False + prj.startdates = { + "swot": [2024, 1, 1], + "icesat2": [2024, 1, 1], + "sentinel3": [2024, 1, 1], + "sentinel6": [2024, 1, 1], + } + prj.enddates = { + "swot": [2024, 2, 1], + "icesat2": [2024, 2, 1], + "sentinel3": [2024, 2, 1], + "sentinel6": [2024, 2, 1], + } + prj.mission_options = { + "swot": { + "hydrocron_fields": { + "nodes": ["node_id", "node_q", "time_str", "wse"], + "reaches": ["reach_id", "reach_q", "time_str", "wse"], + }, + "quality_filters": { + "nodes": {"max_q": 2}, + "reaches": {"max_q": 2}, + }, + }, + "icesat2": {"atl13_fields": None, "track_keys": None}, + "sentinel3": {"sigma0_max": 1e5, "sigma0_min": 0.0}, + "sentinel6": {"sigma0_max": 1e5, "sigma0_min": 0.0}, + } + return prj + + # ============================================================================ # Timeseries Extraction Tests # ============================================================================ @@ -192,9 +266,9 @@ def test_extract_reservoirs_timeseries_calls_all_enabled_missions( (Path(mock_project_reservoirs.dirs["swot"]) / "dummy.shp").touch() with ( - patch.object(flows, "_extract_swot_observations") as mock_swot, - patch.object(flows, "_extract_icesat2_observations") as mock_icesat2, - patch.object(flows, "_extract_sentinel_observations") as mock_sentinel, + patch.object(flows._reservoir_pipeline, "_extract_swot_observations") as mock_swot, + patch.object(flows._reservoir_pipeline, "_extract_icesat2_observations") as mock_icesat2, + patch.object(flows._reservoir_pipeline, "_extract_sentinel_observations") as mock_sentinel, ): flows._extract_reservoirs_timeseries(mock_project_reservoirs) @@ -217,9 +291,9 @@ def test_extract_reservoirs_timeseries_skips_disabled_missions( (Path(mock_project_reservoirs.dirs["swot"]) / "dummy.shp").touch() with ( - patch.object(flows, "_extract_swot_observations") as mock_swot, - patch.object(flows, "_extract_icesat2_observations") as mock_icesat2, - patch.object(flows, "_extract_sentinel_observations") as mock_sentinel, + patch.object(flows._reservoir_pipeline, "_extract_swot_observations") as mock_swot, + patch.object(flows._reservoir_pipeline, "_extract_icesat2_observations") as mock_icesat2, + patch.object(flows._reservoir_pipeline, "_extract_sentinel_observations") as mock_sentinel, ): flows._extract_reservoirs_timeseries(mock_project_reservoirs) @@ -228,6 +302,223 @@ def test_extract_reservoirs_timeseries_skips_disabled_missions( mock_sentinel.assert_not_called() +# ============================================================================ +# River Timeseries Extraction Tests +# ============================================================================ +# +# Mirrors the reservoirs extraction tests above. Rivers' extraction is more +# involved -- raw data covers a whole waterbody (multiple targets) at once +# and has to be split into the same per-target raw_observations structure +# reservoirs use (see flows._extract_rivers_*_observations) -- so these +# tests exercise the real waterbody-grouping/corridor-buffering/point- +# assignment logic rather than mocking it away, only mocking the actual +# satellite module call each extractor makes. + + +@pytest.mark.unit +def test_extract_rivers_swot_observations_splits_hydrocron_csv_by_target( + mock_project_rivers, +): + """_extract_rivers_swot_observations splits one waterbody's Hydrocron + CSV into separate per-target raw_observations/swot.gpkg files.""" + swot_dir = Path(mock_project_rivers.dirs["swot"]) / "loire" + swot_dir.mkdir(parents=True, exist_ok=True) + + df = pd.DataFrame( + { + "node_id": [101, 101, 102], + "node_q": [0, 0, 1], + "time_str": [ + "2024-01-01T00:00:00Z", + "2024-01-05T00:00:00Z", + "2024-01-01T00:00:00Z", + ], + "wse": [10.0, 10.5, 20.0], + } + ) + df.to_csv(swot_dir / "nodes_timeseries.csv", index=False) + + flows._extract_rivers_swot_observations(mock_project_rivers) + + out_101 = ( + Path(mock_project_rivers.dirs["output"]) / "101" / "raw_observations" / "swot.gpkg" + ) + out_102 = ( + Path(mock_project_rivers.dirs["output"]) / "102" / "raw_observations" / "swot.gpkg" + ) + assert out_101.exists() + assert out_102.exists() + + gdf_101 = gpd.read_file(out_101) + assert len(gdf_101) == 2 + assert set(gdf_101["height"]) == {10.0, 10.5} + assert (gdf_101["platform"] == "swot").all() + + gdf_102 = gpd.read_file(out_102) + assert len(gdf_102) == 1 + assert gdf_102["height"].iloc[0] == 20.0 + + +@pytest.mark.unit +def test_extract_rivers_swot_observations_skips_missing_csv(mock_project_rivers): + """_extract_rivers_swot_observations skips a waterbody with no Hydrocron CSV.""" + flows._extract_rivers_swot_observations(mock_project_rivers) + + out_dir = Path(mock_project_rivers.dirs["output"]) + assert not (out_dir / "101").exists() + assert not (out_dir / "102").exists() + + +@pytest.mark.unit +def test_extract_rivers_icesat2_observations_assigns_points_to_targets( + mock_project_rivers, +): + """_extract_rivers_icesat2_observations assigns ICESat-2 points to their + nearest river target and writes separate per-target output files.""" + from HydroEO.satellites import icesat2 + + parquet_dir = Path(mock_project_rivers.dirs["icesat2_processed"]) / "loire" + parquet_dir.mkdir(parents=True, exist_ok=True) + (parquet_dir / "atl13.parquet").touch() + + def fake_extract_observations(src_dir, dst_path, features, **kwargs): + points = gpd.GeoDataFrame( + { + "height": [101.0, 102.0, 201.0], + "date": pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-01"]), + }, + geometry=[Point(0, 0.0001), Point(0, -0.0001), Point(0, 0.1001)], + crs="EPSG:4326", + ) + points.to_file(dst_path, driver="GPKG") + + with patch.object( + icesat2, "extract_observations", side_effect=fake_extract_observations + ): + flows._extract_rivers_icesat2_observations(mock_project_rivers) + + out_101 = ( + Path(mock_project_rivers.dirs["output"]) + / "101" / "raw_observations" / "icesat2.gpkg" + ) + out_102 = ( + Path(mock_project_rivers.dirs["output"]) + / "102" / "raw_observations" / "icesat2.gpkg" + ) + assert out_101.exists() + assert out_102.exists() + + gdf_101 = gpd.read_file(out_101) + assert len(gdf_101) == 2 + assert set(gdf_101["height"]) == {101.0, 102.0} + + gdf_102 = gpd.read_file(out_102) + assert len(gdf_102) == 1 + assert gdf_102["height"].iloc[0] == 201.0 + + +@pytest.mark.unit +def test_extract_rivers_icesat2_observations_skips_missing_parquet( + mock_project_rivers, +): + """_extract_rivers_icesat2_observations skips a waterbody with no + downloaded atl13.parquet -- no output, no exception.""" + from HydroEO.satellites import icesat2 + + with patch.object(icesat2, "extract_observations") as mock_extract: + flows._extract_rivers_icesat2_observations(mock_project_rivers) + mock_extract.assert_not_called() + + out_dir = Path(mock_project_rivers.dirs["output"]) + assert not (out_dir / "101").exists() + assert not (out_dir / "102").exists() + + +@pytest.mark.unit +def test_extract_rivers_sentinel_observations_filters_sigma0_and_assigns( + mock_project_rivers, +): + """_extract_rivers_sentinel_observations applies the sigma0_min + post-filter before assigning surviving points to their nearest target.""" + from HydroEO.satellites import sentinel + + download_dir = Path(mock_project_rivers.dirs["sentinel3"]) / "loire" + download_dir.mkdir(parents=True, exist_ok=True) + mock_project_rivers.mission_options["sentinel3"]["sigma0_min"] = 0.5 + + def fake_extract_observations(src_dir, dst_path, features, **kwargs): + points = gpd.GeoDataFrame( + { + "height": [101.0, 102.0, 201.0], + "date": pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-01"]), + # the second point (near target 101) should be filtered out + "sigma0": [0.9, 0.1, 0.9], + }, + geometry=[Point(0, 0.0001), Point(0, -0.0001), Point(0, 0.1001)], + crs="EPSG:4326", + ) + points.to_file(dst_path, driver="GPKG") + + with patch.object( + sentinel, "extract_observations", side_effect=fake_extract_observations + ): + flows._extract_rivers_sentinel_observations(mock_project_rivers, "sentinel3", "S3") + + out_101 = ( + Path(mock_project_rivers.dirs["output"]) + / "101" / "raw_observations" / "sentinel3.gpkg" + ) + out_102 = ( + Path(mock_project_rivers.dirs["output"]) + / "102" / "raw_observations" / "sentinel3.gpkg" + ) + assert out_101.exists() + assert out_102.exists() + + gdf_101 = gpd.read_file(out_101) + assert len(gdf_101) == 1 + assert gdf_101["height"].iloc[0] == 101.0 + + gdf_102 = gpd.read_file(out_102) + assert len(gdf_102) == 1 + assert gdf_102["height"].iloc[0] == 201.0 + + +@pytest.mark.unit +def test_extract_rivers_timeseries_calls_all_enabled_missions(mock_project_rivers): + """_extract_rivers_timeseries dispatcher calls extractors for all + enabled missions.""" + mock_project_rivers.to_process = ["swot", "icesat2", "sentinel3", "sentinel6"] + + with ( + patch.object(flows._river_pipeline, "_extract_rivers_icesat2_observations") as mock_icesat2, + patch.object(flows._river_pipeline, "_extract_rivers_sentinel_observations") as mock_sentinel, + patch.object(flows._river_pipeline, "_extract_rivers_swot_observations") as mock_swot, + ): + flows._extract_rivers_timeseries(mock_project_rivers) + + mock_icesat2.assert_called_once() + assert mock_sentinel.call_count == 2 # sentinel3 and sentinel6 + mock_swot.assert_called_once() + + +@pytest.mark.unit +def test_extract_rivers_timeseries_skips_disabled_missions(mock_project_rivers): + """_extract_rivers_timeseries skips missions not in to_process.""" + mock_project_rivers.to_process = ["swot"] + + with ( + patch.object(flows._river_pipeline, "_extract_rivers_icesat2_observations") as mock_icesat2, + patch.object(flows._river_pipeline, "_extract_rivers_sentinel_observations") as mock_sentinel, + patch.object(flows._river_pipeline, "_extract_rivers_swot_observations") as mock_swot, + ): + flows._extract_rivers_timeseries(mock_project_rivers) + + mock_icesat2.assert_not_called() + mock_sentinel.assert_not_called() + mock_swot.assert_called_once() + + # ============================================================================ # Timeseries Cleaning and Filter Tests # ============================================================================ @@ -520,6 +811,79 @@ def test_clean_reservoirs_processes_multiple_products( assert mock_clean.call_count == 2 +# ============================================================================ +# River Timeseries Cleaning Tests +# ============================================================================ +# +# _clean_rivers_timeseries is a thin wrapper around the same +# flows._clean_timeseries engine reservoirs use (target_type="rivers" +# instead of "reservoirs") -- the filter math itself is already covered by +# the reservoirs cleaning tests above, so these focus on the rivers-specific +# dispatch: target ids come from prj.rivers.target_ids rather than +# prj.reservoirs.download_gdf. + + +@pytest.mark.unit +def test_clean_rivers_applies_elevation_filter(mock_project_rivers, tmp_path): + """_clean_rivers_timeseries applies elevation_min/max filter correctly + for a river target.""" + from HydroEO.utils import timeseries + + mock_project_rivers.to_process = ["swot"] + mock_project_rivers.processing_options = { + "swot": { + "processing_filters": ["elevation"], + "elevation_min_m": 5.0, + "elevation_max_m": 8000.0, + } + } + + raw_obs_dir = Path(mock_project_rivers.dirs["output"]) / "101" / "raw_observations" + raw_obs_dir.mkdir(parents=True, exist_ok=True) + raw_gdf = gpd.GeoDataFrame( + { + "height": [3, 10, 20], + "date": pd.date_range("2024-01-01", periods=3), + "geometry": [Point(0, 0)] * 3, + }, + crs="EPSG:4326", + ) + raw_gdf.to_file(raw_obs_dir / "swot.gpkg", driver="GPKG") + + with patch.object(timeseries.Timeseries, "clean") as mock_clean: + flows._clean_rivers_timeseries(mock_project_rivers) + + mock_clean.assert_called_once() + call_args = mock_clean.call_args + assert call_args[0][0] == ["elevation"] + assert call_args[1]["filter_params"]["elevation_min_m"] == 5.0 + assert call_args[1]["filter_params"]["elevation_max_m"] == 8000.0 + + cleaned_path = ( + Path(mock_project_rivers.dirs["output"]) + / "101" / "cleaned_observations" / "swot.csv" + ) + assert cleaned_path.exists() + + # target 102 has no raw_observations at all -- must not be touched + assert not ( + Path(mock_project_rivers.dirs["output"]) / "102" / "cleaned_observations" + ).exists() + + +@pytest.mark.unit +def test_clean_rivers_handles_empty_raw_observations(mock_project_rivers, caplog): + """_clean_rivers_timeseries warns and returns early when no target has + any raw observations yet.""" + import logging + + mock_project_rivers.to_process = ["swot"] + + with caplog.at_level(logging.WARNING): + flows._clean_rivers_timeseries(mock_project_rivers) + assert "No raw observations found for any rivers" in caplog.text + + # ============================================================================ # Timeseries Merging Tests # ============================================================================ @@ -730,6 +1094,66 @@ def test_merge_handles_no_cleaned_observations(mock_project_reservoirs, caplog): assert "No cleaned observations found" in caplog.text +# ============================================================================ +# River Timeseries Merging Tests +# ============================================================================ +# +# _merge_rivers_timeseries is a thin wrapper around the same +# flows._merge_timeseries engine reservoirs use (target_type="rivers" +# instead of "reservoirs") -- the merge algorithm itself is already covered +# by the reservoirs merging tests (and the Nuozhadu baseline regression +# tests) above, so these focus on the rivers-specific dispatch: target ids +# come from prj.rivers.target_ids, and centroid lookup +# (flows._target_centroid) comes from prj.rivers.target_features rather +# than a reservoir polygon. + + +@pytest.mark.unit +def test_merge_rivers_concatenates_cleaned_observations(mock_project_rivers, tmp_path): + """_merge_rivers_timeseries merges a river target's cleaned observations.""" + from HydroEO.utils import timeseries + + mock_project_rivers.to_process = ["swot"] + + cleaned_dir = ( + Path(mock_project_rivers.dirs["output"]) / "101" / "cleaned_observations" + ) + cleaned_dir.mkdir(parents=True, exist_ok=True) + df = pd.DataFrame( + { + "date": pd.date_range("2024-01-01", periods=3), + "height": [10.0, 10.1, 10.2], + } + ) + df.to_csv(cleaned_dir / "swot.csv", index=False) + + with ( + patch.object(timeseries.Timeseries, "merge") as mock_merge, + patch.object(timeseries.Timeseries, "export_csv"), + ): + mock_merge.return_value = mock.MagicMock() + flows._merge_rivers_timeseries(mock_project_rivers) + + assert mock_merge.called + # _target_centroid should have resolved target 101's centroid from + # prj.rivers.target_features rather than failing/using (None, None) + call_kwargs = mock_merge.call_args[1] + assert call_kwargs["ref_lat"] is not None + assert call_kwargs["ref_lon"] is not None + + +@pytest.mark.unit +def test_merge_rivers_handles_no_cleaned_observations(mock_project_rivers, caplog): + """_merge_rivers_timeseries handles the case with no cleaned observations.""" + import logging + + mock_project_rivers.to_process = ["swot"] + + with caplog.at_level(logging.WARNING): + flows._merge_rivers_timeseries(mock_project_rivers) + assert "No cleaned observations found for any rivers" in caplog.text + + # ============================================================================ # Regression): Baseline Tests Using Real Data # ============================================================================ From 08319d27af184c94c017eb0bc5d0d40f533520e5 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Tue, 14 Jul 2026 12:56:46 +0200 Subject: [PATCH 06/19] Test fixing and cleanup --- HydroEO/flows/_reservoir_init.py | 1 - HydroEO/flows/_river_common.py | 5 +++++ HydroEO/flows/_river_download.py | 4 +++- HydroEO/flows/_river_pipeline.py | 2 +- tests/test_integration.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index db7fd99..304c603 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -12,7 +12,6 @@ import geopandas as gpd from HydroEO.downloaders import hydroweb -from HydroEO.utils import general from typing import TYPE_CHECKING diff --git a/HydroEO/flows/_river_common.py b/HydroEO/flows/_river_common.py index 294e511..b196ec2 100644 --- a/HydroEO/flows/_river_common.py +++ b/HydroEO/flows/_river_common.py @@ -9,6 +9,11 @@ import geopandas as gpd +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from HydroEO.project import Project + logger = logging.getLogger(__name__) diff --git a/HydroEO/flows/_river_download.py b/HydroEO/flows/_river_download.py index 2da49a4..2f60762 100644 --- a/HydroEO/flows/_river_download.py +++ b/HydroEO/flows/_river_download.py @@ -11,10 +11,12 @@ import logging import os import datetime +from io import StringIO import pandas as pd +from tqdm import tqdm -from HydroEO.satellites import icesat2, sentinel +from HydroEO.satellites import icesat2 from HydroEO.utils import general from ._river_common import _group_river_targets_by_waterbody, _iter_geometry_pieces from ._sentinel_shared import _sentinel6_use_earthdata, _download_sentinel_for_target diff --git a/HydroEO/flows/_river_pipeline.py b/HydroEO/flows/_river_pipeline.py index 4fab9d8..dda1cf7 100644 --- a/HydroEO/flows/_river_pipeline.py +++ b/HydroEO/flows/_river_pipeline.py @@ -14,7 +14,7 @@ import geopandas as gpd import pandas as pd -from HydroEO.satellites import swot, icesat2, sentinel +from HydroEO.satellites import icesat2, sentinel from HydroEO.utils import general from ._river_common import ( _group_river_targets_by_waterbody, diff --git a/tests/test_integration.py b/tests/test_integration.py index 0191ac2..146efd2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -359,7 +359,7 @@ def test_hydrocron_river_download_writes_filtered_csv(tmp_path): enddate=datetime.date(2024, 6, 1), ) - output_path = swot_dir / "rivers" / "loire" / "nodes_timeseries.csv" + output_path = swot_dir / "loire" / "nodes_timeseries.csv" assert output_path.exists(), f"Expected output CSV at {output_path}" df = pd.read_csv(output_path) From 50f0dd305075e852aca1dac33158c1a83acea254 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Tue, 14 Jul 2026 15:06:47 +0200 Subject: [PATCH 07/19] Fix for failed test Import "name as name" in flows.__init__.py --- HydroEO/flows/__init__.py | 140 ++++++++++++------------- HydroEO/utils/filters/basic_filters.py | 1 - 2 files changed, 70 insertions(+), 71 deletions(-) diff --git a/HydroEO/flows/__init__.py b/HydroEO/flows/__init__.py index f01079d..2c45b79 100644 --- a/HydroEO/flows/__init__.py +++ b/HydroEO/flows/__init__.py @@ -18,100 +18,100 @@ from HydroEO import plotting # noqa: F401 -- re-exported so `flows.plotting`/`HydroEO.flows.plotting` resolves from ._reservoir_init import ( - _assign_pld_id, - _download_pld, - _flag_missing_priors, - initialize_reservoirs, + _assign_pld_id as _assign_pld_id, + _download_pld as _download_pld, + _flag_missing_priors as _flag_missing_priors, + initialize_reservoirs as initialize_reservoirs, ) from ._river_init import ( - _ensure_sword_database, - _prepare_rivers_from_sword, - initialize_rivers, + _ensure_sword_database as _ensure_sword_database, + _prepare_rivers_from_sword as _prepare_rivers_from_sword, + initialize_rivers as initialize_rivers, ) from ._sentinel_shared import ( - _download_sentinel_for_target, - _sentinel6_use_earthdata, + _download_sentinel_for_target as _download_sentinel_for_target, + _sentinel6_use_earthdata as _sentinel6_use_earthdata, ) from ._reservoir_download import ( - _download_reservoirs_icesat2, - _download_reservoirs_sentinel, - _download_reservoirs_swot, - download_reservoirs, + _download_reservoirs_icesat2 as _download_reservoirs_icesat2, + _download_reservoirs_sentinel as _download_reservoirs_sentinel, + _download_reservoirs_swot as _download_reservoirs_swot, + download_reservoirs as download_reservoirs, ) from ._river_download import ( - _download_rivers_icesat2, - _download_rivers_sentinel, - _download_swot_hydrocron_timeseries, - _get_latest_hydrocron_obs_date, - download_rivers, + _download_rivers_icesat2 as _download_rivers_icesat2, + _download_rivers_sentinel as _download_rivers_sentinel, + _download_swot_hydrocron_timeseries as _download_swot_hydrocron_timeseries, + _get_latest_hydrocron_obs_date as _get_latest_hydrocron_obs_date, + download_rivers as download_rivers, ) from ._river_common import ( - _assign_points_to_river_targets, - _group_river_targets_by_waterbody, - _iter_geometry_pieces, - _river_extraction_buffer_meters, - _simplify_to_one_polygon, + _assign_points_to_river_targets as _assign_points_to_river_targets, + _group_river_targets_by_waterbody as _group_river_targets_by_waterbody, + _iter_geometry_pieces as _iter_geometry_pieces, + _river_extraction_buffer_meters as _river_extraction_buffer_meters, + _simplify_to_one_polygon as _simplify_to_one_polygon, ) from ._river_pipeline import ( - _clean_rivers_timeseries, - _extract_rivers_icesat2_observations, - _extract_rivers_sentinel_observations, - _extract_rivers_swot_observations, - _extract_rivers_timeseries, - _merge_rivers_timeseries, - create_rivers_timeseries, + _clean_rivers_timeseries as _clean_rivers_timeseries, + _extract_rivers_icesat2_observations as _extract_rivers_icesat2_observations, + _extract_rivers_sentinel_observations as _extract_rivers_sentinel_observations, + _extract_rivers_swot_observations as _extract_rivers_swot_observations, + _extract_rivers_timeseries as _extract_rivers_timeseries, + _merge_rivers_timeseries as _merge_rivers_timeseries, + create_rivers_timeseries as create_rivers_timeseries, ) from ._reservoir_pipeline import ( - _clean_reservoirs_timeseries, - _export_cleaned_to_dfs0, - _extract_icesat2_observations, - _extract_reservoirs_timeseries, - _extract_sentinel_observations, - _extract_swot_observations, - _merge_reservoirs_timeseries, - create_reservoirs_timeseries, + _clean_reservoirs_timeseries as _clean_reservoirs_timeseries, + _export_cleaned_to_dfs0 as _export_cleaned_to_dfs0, + _extract_icesat2_observations as _extract_icesat2_observations, + _extract_reservoirs_timeseries as _extract_reservoirs_timeseries, + _extract_sentinel_observations as _extract_sentinel_observations, + _extract_swot_observations as _extract_swot_observations, + _merge_reservoirs_timeseries as _merge_reservoirs_timeseries, + create_reservoirs_timeseries as create_reservoirs_timeseries, ) from ._constants import ( - DEFAULT_RESERVOIR_MERGING_OPTIONS, - DEFAULT_RIVER_MERGING_OPTIONS, - PRODUCT_TIMESERIES_KEYS, + DEFAULT_RESERVOIR_MERGING_OPTIONS as DEFAULT_RESERVOIR_MERGING_OPTIONS, + DEFAULT_RIVER_MERGING_OPTIONS as DEFAULT_RIVER_MERGING_OPTIONS, + PRODUCT_TIMESERIES_KEYS as PRODUCT_TIMESERIES_KEYS, ) from ._clean_engine import ( - _clean_timeseries, + _clean_timeseries as _clean_timeseries, ) from ._run_config import ( - _apply_exclusions, - _apply_reach_slope_correction, - _default_run_config, - _exclusion_value_matches, - _fit_reach_slope_correction, - _get_or_fit_spatial_correction_model, - _get_target_ids, - _invalidate_reach_slope_correction_cache, - _invalidate_spatial_correction_cache, - _load_run_config, - _reservoir_centroid, - _run_config_path, - _save_run_config, - _target_centroid, - exclude_from_target, - list_exclusions, - list_target_observations, - remove_exclusion, - set_merging_option, + _apply_exclusions as _apply_exclusions, + _apply_reach_slope_correction as _apply_reach_slope_correction, + _default_run_config as _default_run_config, + _exclusion_value_matches as _exclusion_value_matches, + _fit_reach_slope_correction as _fit_reach_slope_correction, + _get_or_fit_spatial_correction_model as _get_or_fit_spatial_correction_model, + _get_target_ids as _get_target_ids, + _invalidate_reach_slope_correction_cache as _invalidate_reach_slope_correction_cache, + _invalidate_spatial_correction_cache as _invalidate_spatial_correction_cache, + _load_run_config as _load_run_config, + _reservoir_centroid as _reservoir_centroid, + _run_config_path as _run_config_path, + _save_run_config as _save_run_config, + _target_centroid as _target_centroid, + exclude_from_target as exclude_from_target, + list_exclusions as list_exclusions, + list_target_observations as list_target_observations, + remove_exclusion as remove_exclusion, + set_merging_option as set_merging_option, ) from ._merge_engine import ( - _merge_timeseries, + _merge_timeseries as _merge_timeseries, ) from ._summaries import ( - _has_enough_observations_to_plot, - _load_and_parse_cleaned_timeseries, - _load_merged_timeseries, - _load_product_timeseries, - _project_num_months, - _river_target_corridor, - generate_reservoirs_summaries, - generate_rivers_summaries, + _has_enough_observations_to_plot as _has_enough_observations_to_plot, + _load_and_parse_cleaned_timeseries as _load_and_parse_cleaned_timeseries, + _load_merged_timeseries as _load_merged_timeseries, + _load_product_timeseries as _load_product_timeseries, + _project_num_months as _project_num_months, + _river_target_corridor as _river_target_corridor, + generate_reservoirs_summaries as generate_reservoirs_summaries, + generate_rivers_summaries as generate_rivers_summaries, ) __all__ = [ diff --git a/HydroEO/utils/filters/basic_filters.py b/HydroEO/utils/filters/basic_filters.py index f249a75..8086d49 100644 --- a/HydroEO/utils/filters/basic_filters.py +++ b/HydroEO/utils/filters/basic_filters.py @@ -69,7 +69,6 @@ def _resolve_pass_groups(df, date_key, pass_key=None, platform_key=None, orbit_k Returns a pandas Series of group labels (strings), same index as df. """ - n = len(df) group = pd.Series(pd.NA, index=df.index, dtype=object) if pass_key and pass_key in df.columns: From 25ae0bc751a8710c3c351b3505c217d5f5e069ae Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Wed, 15 Jul 2026 11:55:17 +0200 Subject: [PATCH 08/19] Update documentation and .md files to match updates and correct citation - Updated readme, agents, citation to accurately cite DAHITI --- AGENTS.md | 2 + CITATION.cff | 26 ++++++++---- README.md | 10 ++++- configs/reservoirs.md | 15 ++++++- configs/rivers.md | 96 ++++++++++++++++++++++++++++++++++++++++--- configs/rivers.yaml | 12 +++--- 6 files changed, 141 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8ff767f..9f631fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,9 @@ This document provides codebase knowledge for AI coding agents (Claude, Copilot, 2. **Rivers** (`rivers` branch) - SWOT Hydrocron API download (public, no credentials needed) + - Multi-satellite data download (SWOT Lake SP, ICESat-2 ATL13, Sentinel-3, Sentinel-6) - SWORD v17b database integration for river node/reach matching + - Full timeseries extraction, cleaning, filtering, and multi-mission merge aligned to SWORD reaches or nodes - Output: per-river Hydrocron timeseries (CSV) 3. **SWOT Raster Tiles** (`swot_raster` branch) diff --git a/CITATION.cff b/CITATION.cff index e083f0a..b7260d3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,22 +4,32 @@ message: >- HydroEO's core timeseries pipeline additionally implements a methodology described in Schwatke et al. (2015) -- see the "references" section below and HydroEO/utils/filters/basic_filters.py - -- please cite that publication as well when using the Kalman filter / - outlier rejection pipeline it implements. + -- please cite that publication as well when using the Kalman filter, + SVR, or windowed ADM-based outlier/uncertainty weighting it + implements. title: "HydroEO" version: "0.1.0" license: MIT repository-code: "https://github.com/DHI/HydroEO" authors: - - family-names: "Chewning" - given-names: "Connor" - affiliation: "DHI" - family-names: "Kittel" given-names: "Cecile" affiliation: "DHI" + - family-names: "Chewning" + given-names: "Connor" + affiliation: "DHI" - family-names: "Zalite" given-names: "Karlis" affiliation: "DHI" + - family-names: "Monica" + given-names: "Coppo Frias" + affiliation: "DHI" + - family-names: "Sarah" + given-names: "Franze" + affiliation: "DHI" + - family-names: "Paul" + given-names: "Senty" + affiliation: "DHI" references: - type: article authors: @@ -41,5 +51,7 @@ references: notes: >- HydroEO's core timeseries pipeline (see HydroEO/utils/filters/basic_filters.py) follows the windowed - outlier rejection, multi-mission bias correction, and Kalman - filter approach described in this publication. + ADM-based outlier/uncertainty weighting, SVR, and Kalman filter + approach described in this publication. The multi-mission bias + correction and mission-combination logic are this codebase's own + contributions, not sourced from this paper. diff --git a/README.md b/README.md index e9a7564..2ec77e1 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Or via CLI: `hydroeo run configs/reservoirs.yaml` | Branch | Config | Status | Satellites | Description | | --- | --- | --- | --- | --- | | `reservoirs` | [configs/reservoirs.yaml](configs/reservoirs.yaml) | ✅ | SWOT Lake SP, ICESat-2 ATL13, Sentinel-3/6 | Lakes and reservoirs from polygon input. Multi-mission timeseries with cleaning filters. → [Full docs](configs/reservoirs.md) | -| `rivers` | [configs/rivers.yaml](configs/rivers.yaml) | 🧪 partial | SWOT Hydrocron (public API) | River nodes/reaches from SWORD v17b. Download and diagnostic plots. → [Full docs](configs/rivers.md) | +| `rivers` | [configs/rivers.yaml](configs/rivers.yaml) | 🧪 test in progress | SWOT Hydrocron (public API), ICESat-2 ATL13, Sentinel-3/6 | River nodes/reaches from SWORD v17b. Download and diagnostic plots. Virtual station timeseries with cleaning filters → [Full docs](configs/rivers.md) | | `swot_raster` | [configs/swot_raster.yaml](configs/swot_raster.yaml) | ✅ | SWOT L2 HR/LR Raster | Arbitrary AOI. Downloads, clips, and merges raster tiles by date. → [Full docs](configs/swot_raster.md) | | `swot_pixc` | [configs/swot_pixc.yaml](configs/swot_pixc.yaml) | ✅ | SWOT L2 PIXC | Arbitrary AOI. Point cloud gridded to rasters via binned statistics. → [Full docs](configs/swot_pixc.md) | @@ -141,3 +141,11 @@ make test ``` See [tests/README.md](tests/README.md) for the full test suite, pytest markers, and CI/CD setup. + +Citation + +If you use HydroEO, please cite it — see CITATION.cff or use GitHub's "Cite this repository" button (in the sidebar of this page). + +If you use the core timeseries pipeline (HydroEO/utils/filters/basic_filters.py — specifically the windowed ADM-based outlier/uncertainty weighting, SVR, and Kalman filter merge), please also cite the methodology it implements. (The multi-mission bias correction and mission-combination logic are HydroEO's own contributions, not sourced from this paper.) + +Schwatke, C., Dettmering, D., Bosch, W., and Seitz, F.: DAHITI – an innovative approach for estimating water level time series over inland waters using multi-mission satellite altimetry, Hydrology and Earth System Sciences, 19, 4345–4364, 2015. https://doi.org/10.5194/hess-19-4345-2015 diff --git a/configs/reservoirs.md b/configs/reservoirs.md index 211b90b..12e0c36 100644 --- a/configs/reservoirs.md +++ b/configs/reservoirs.md @@ -99,6 +99,19 @@ Applied during `create_timeseries()`. Configured per mission under `processing_f | `hampel` | Spike detection using a sliding median window | | `rolling_median` | Smoothing pass using a rolling median | +## Merging + +Applied during `create_timeseries()`, after cleaning — combines all enabled missions' cleaned observations into one `merged_timeseries.csv` per reservoir using bias correction, SVR, and a Kalman filter. Override any key in `flows.DEFAULT_RESERVOIR_MERGING_OPTIONS` via a `merging_options` dict under `reservoirs:` in your config: + +```yaml +reservoirs: + merging_options: + svr_radial_err: 1.0 + svr_radial_gamma: 0.00438 +``` + +**If you use this pipeline, please also cite Schwatke et al. (2015)** — see the root [README](../README.md)'s Citation section. (DAHITI covers the windowed ADM-based outlier/uncertainty weighting, SVR, and Kalman filter specifically; the bias correction and multi-mission-combination logic are HydroEO's own contributions, not sourced from that paper.) + ## Output structure ``` @@ -159,4 +172,4 @@ hydroeo fetch cop-dem \ Credentials can also be set as environment variables (`EARTHDATA_USERNAME`, `EARTHDATA_PASSWORD`, `CREODIAS_USERNAME`, `CREODIAS_PASSWORD`, `CDSE_USERNAME`, `CDSE_PASSWORD`). -> **Note:** ICESat-2, Sentinel-3, and Sentinel-6 require reservoir polygons for spatial filtering. Using them with `download: true` in a non-reservoirs project emits a `UserWarning`. +> **Note:** ICESat-2, Sentinel-3, and Sentinel-6 require reservoir polygons for spatial filtering. Using them with `download: true` in a non-reservoirs project emits a `UserWarning`. \ No newline at end of file diff --git a/configs/rivers.md b/configs/rivers.md index dee21c8..d323934 100644 --- a/configs/rivers.md +++ b/configs/rivers.md @@ -1,8 +1,22 @@ # Rivers — Configuration Reference -SWOT Hydrocron download for river nodes and reaches, matched to the SWORD v17b river network database. **No credentials required** — Hydrocron is a public API. +SWOT Hydrocron download for river nodes and reaches, matched to the SWORD v17b river network database, with optional ICESat-2 and Sentinel-3/6 as additional missions on the same targets. **SWOT itself needs no credentials** — Hydrocron is a public API — but ICESat-2/Sentinel-3/6 need the same credentials as the reservoirs workflow if enabled (see the root [README](../README.md)'s Credentials table). -> **Status:** Partial. Initialization, SWOT Hydrocron download, and diagnostic plotting are supported. River timeseries filtering/preprocessing is not yet implemented. +> **Status:** Full pipeline supported — initialization, multi-mission download, extraction, cleaning, merge, and diagnostic plotting all work for rivers, sharing the same underlying engine reservoirs use. One gap: no `dfs0` export for rivers yet (reservoirs-only). See [Known limitations](#known-limitations) below. + +## Quick start + +At least one mission (`swot`, `icesat2`, `sentinel3`, or `sentinel6`) must be enabled (`download: true`) or nothing downloads — `download_rivers()` warns and returns early if none are, even with a valid `aoi_path`/SWORD setup (SWORD itself will still have been prepared by `initialize()`, so you may see SWORD files but no timeseries data if you hit this). SWOT via Hydrocron is the natural default/primary source for node/reach WSE, but it is not the *only* mission that can stand alone — ICESat-2/Sentinel-3/6 alone (without SWOT) will also download and process normally. + +```python +from HydroEO.project import Project + +project = Project(name="my_river_project", config="configs/rivers.yaml") +project.initialize() # SWORD download + AOI subsetting +project.download() # whichever of swot/icesat2/sentinel3/sentinel6 are enabled +project.create_timeseries() # extraction + cleaning + merge +project.generate_summaries() # plots +``` ## Config reference @@ -18,11 +32,17 @@ Start from [`configs/rivers.yaml`](rivers.yaml). | `continent_key` | string | ✅ (A) | SWORD continent code (see table below) | | `feature_type` | string | ✅ | `nodes` or `reaches` | | `id_key` | string | ✅ (A) | AOI column used to name per-river output folders | -| `buffer_meters` | float | — | Optional AOI buffer before SWORD spatial subsetting | +| `buffer_meters` | float | — | Optional AOI buffer before SWORD spatial subsetting. Also used as the extraction-corridor fallback if `extraction_buffer_meters` isn't set (see below). | | **Option B — explicit IDs** | | | | | `feature_numbers` | list | ✅ (B) | List of node or reach IDs | | `feature_type` | string | ✅ | Must match IDs: `nodes` or `reaches` | | `id` | string | ✅ (B) | Used as the output folder name | +| **ICESat-2/Sentinel-3/6 extraction (only relevant if those missions are enabled)** | | | | +| `extraction_buffer_meters` | float | — | Corridor half-width (m) around each SWORD target used to decide which raw ICESat-2/Sentinel points are plausibly real river water. Separate from `buffer_meters` above, which only decides which *targets* are in scope. If omitted: falls back to `buffer_meters`, then a flat `500.0` m default. | +| `width_buffer_factor` | float | — | Multiplier applied to SWORD's own `width` column when sizing the corridor per-target instead of using one flat distance (`distance = width/2 × width_buffer_factor`). Default `1.05`. Only used when SWORD `width` data is available; falls back to `extraction_buffer_meters` otherwise. | +| `max_node_assignment_meters` | float | — | Max distance (m) for assigning a raw ICESat-2/Sentinel point to its nearest node/reach. If omitted: falls back to the same resolved `extraction_buffer_meters` value above. | +| `overwrite_extraction` | bool | — | Default `false` (skip re-extracting a target whose output already exists, or — for ICESat-2 — whose downloaded data hasn't changed since the last extraction). `true` forces a full re-extraction from scratch. | +| `merging_options` | dict | — | Per-project override of any key in `flows.DEFAULT_RIVER_MERGING_OPTIONS` (Kalman/SVR/bias-correction/reach-slope-correction parameters). **Currently a direct copy of the reservoir-tuned defaults, not independently validated against real river data** — check kept/rejected observation counts on your own rivers before trusting them as-is. **If you use this pipeline, please also cite Schwatke et al. (2015)** — see the root [README](../README.md)'s Citation section. | **SWORD continent codes:** @@ -51,6 +71,30 @@ When `sword_subset_path` is provided: The SWORD subset is always written to `{main_dir}/aux/SWORD/SWORD_subset.gpkg` for future reuse. +### Additional missions: ICESat-2, Sentinel-3, Sentinel-6 + +Enabled the same way as for reservoirs (`download`/`process` flags per mission), clustered onto the same SWORD targets as SWOT via the corridor parameters above: + +```yaml +icesat2: + download: true + process: true + +sentinel3: + download: true + process: true + sigma0_min: 0.0 # see note below + +sentinel6: + download: true + process: true + source: "earthdata" # CREODIAS only distributes the Low Rate product; + # High Rate (20Hz Ku-band) needs "earthdata" and + # EARTHDATA_USERNAME/PASSWORD credentials +``` + +`sentinel3`/`sentinel6`'s `sigma0_min` is a water-only filter, off (`0.0`, a no-op) by default. Unlike a reservoir polygon, a buffered river corridor genuinely includes riverbank/vegetation, and unlike ICESat-2, Sentinel has no built-in water classification — tune this against real data for your rivers before relying on a nonzero value. + ## Advanced: Hydrocron fields and quality filters By default, a sensible set of fields is requested for each feature type. Add a `swot:` section to override: @@ -69,6 +113,8 @@ swot: reaches: {max_q: 2} # keep records where reach_q <= 2 ``` +`reaches`' `slope`/`slope_u` fields feed the optional reach-level slope correction (`merging_options.use_reach_slope_correction`, off by default, reach-mode only — see `flows.DEFAULT_RIVER_MERGING_OPTIONS`); they're silently unused if you switch to node mode or don't enable that option. + ## Output structure ``` @@ -79,9 +125,47 @@ swot: raw/ swot/ / - nodes_timeseries.csv # or reaches_timeseries.csv + nodes_timeseries.csv # or reaches_timeseries.csv + icesat2// # if icesat2 enabled + sentinel3// # if sentinel3 enabled + sentinel6// # if sentinel6 enabled results/ - / # diagnostic plots (map, timeseries) + / # one map + one combined timeseries plot per waterbody + _map.png + _timeseries.png + / # per node/reach (not nested under ) + raw_observations/ + swot.gpkg + icesat2.gpkg # if enabled + sentinel3.gpkg # if enabled + sentinel6.gpkg # if enabled + cleaned_observations/ + swot.csv, icesat2.csv, ... + merged_timeseries.csv + merged_progress/ # per-processing-step diagnostic CSVs + run_config.yaml # per-target exclusions + merging-option overrides + merging_summary.png # (if save=True) ``` -Each node/reach gets its own subfolder within the AOI-feature folder so multiple nodes from the same AOI feature coexist safely. +`` is the AOI feature's `id_key` value (Option A) or the configured `id` (Option B) — it groups targets for download batching and the two waterbody-level plots, but every target's actual data (`raw_observations/`, `cleaned_observations/`, `merged_timeseries.csv`, `run_config.yaml`) lives directly under its own `` (node or reach ID), not nested inside ``. + +## Per-target exclusion / merging-option overrides + +Same mechanism as reservoirs — reachable via `Project` methods, not by importing `flows` directly: + +```python +project.list_target_observations(23221000160051, target_type="rivers") +project.exclude_observations(23221000160051, platform="S3B", reason="...") +project.list_exclusions(23221000160051, target_type="rivers") +project.remove_exclusion(23221000160051, platform="S3B", orbit=1517) +project.set_merging_option(23221000160051, svr_radial_err=1.0) +``` + +`target_type` can be omitted — it's inferred automatically from whichever of reservoirs/rivers your project actually configures. + +## Known limitations + +- **River `merging_options` defaults are a direct copy of the reservoir-tuned defaults, not independently validated against real river data.** Check kept/rejected observation counts on your own rivers before trusting them as-is. +- **No `dfs0` export for rivers yet** (reservoirs-only). +- **Reach-level slope correction** (`use_reach_slope_correction`) has two assumptions not yet validated against real data: that SWOT's own reach-level WSE is approximately midpoint-referenced, and that the correction's sign convention actually reduces cross-mission scatter rather than increasing it. Confirm both against your own reach before relying on this in production. +- **Sentinel-6's `orbit_key` stability** hasn't been re-verified against real Sentinel-6 data the way Sentinel-3's was (see `flows.PRODUCT_TIMESERIES_KEYS`). \ No newline at end of file diff --git a/configs/rivers.yaml b/configs/rivers.yaml index 11bad2e..a7145f2 100644 --- a/configs/rivers.yaml +++ b/configs/rivers.yaml @@ -100,10 +100,12 @@ rivers: # svr_radial_gamma: 0.00438 -# ─── SWOT (REQUIRED — see quick-start note above) ──────────────────────────── -# Rivers require SWOT Hydrocron for at least node/reach WSE. This section -# must be enabled (download: true, process: true) or nothing will download, -# even with a valid aoi_path/SWORD setup above. +# ─── SWOT (default primary source — see quick-start note above) ───────────── +# SWOT Hydrocron is the natural default source for node/reach WSE, but it is +# not strictly mandatory on its own: at least one mission below (this section, +# or icesat2/sentinel3/sentinel6 further down) must have download: true, or +# nothing will download -- download_rivers() warns and returns early if none +# are enabled, even with a valid aoi_path/SWORD setup above. swot: download: true process: true @@ -153,4 +155,4 @@ sentinel6: # # classification. 0.0 is a safe no-op default -- tune # # against real data before trusting a nonzero value. # sentinel6: -# sigma0_min: 0.0 +# sigma0_min: 0.0 \ No newline at end of file From d50d8bafbe6bb4498f6a92964df89cd1b6f6b7af Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Thu, 16 Jul 2026 13:19:48 +0200 Subject: [PATCH 09/19] Updated documentation, review of flow.py decomposition Updated documentation, review of flow.py decomposition Fixed unary_union depreciation warning Replace unary_union with union_all(), fixed depreciation warning. --- HydroEO/flows/__init__.py | 10 +- HydroEO/flows/_clean_engine.py | 11 +- HydroEO/flows/_constants.py | 147 ++++----------------------- HydroEO/flows/_merge_engine.py | 18 +--- HydroEO/flows/_reservoir_download.py | 25 +---- HydroEO/flows/_reservoir_init.py | 17 +--- HydroEO/flows/_reservoir_pipeline.py | 117 +++++++-------------- HydroEO/flows/_river_common.py | 35 ++----- HydroEO/flows/_river_download.py | 55 ++-------- HydroEO/flows/_river_pipeline.py | 102 ++++++++----------- HydroEO/flows/_run_config.py | 130 ++++++----------------- HydroEO/flows/_sentinel_shared.py | 25 ++--- HydroEO/flows/_summaries.py | 73 +++---------- 13 files changed, 172 insertions(+), 593 deletions(-) diff --git a/HydroEO/flows/__init__.py b/HydroEO/flows/__init__.py index 2c45b79..0ca1b6f 100644 --- a/HydroEO/flows/__init__.py +++ b/HydroEO/flows/__init__.py @@ -4,19 +4,13 @@ logic previously embedded in Reservoirs and Rivers classes. They operate on Project state and external data, with no direct method dependencies. -This package replaces what used to be a single ~3,100-line flows.py file, -split by concern (init/download/extract-clean-merge/run-config/summaries) --- see each submodule's docstring for why its particular grouping was -chosen. Every name below (public and private) is re-exported here so that +Splits single flows.py file by concern (init/download/extract-clean-merge/run-config/summaries) +Every name below (public and private) is re-exported here so that `from HydroEO import flows; flows.` keeps working exactly as it did against the old single-file module, including for tests that patch private helpers via `patch.object(flows, "_name")`. """ -import mikeio # noqa: F401 -- re-exported so `flows.mikeio` resolves (see _reservoir_pipeline.py) - -from HydroEO import plotting # noqa: F401 -- re-exported so `flows.plotting`/`HydroEO.flows.plotting` resolves - from ._reservoir_init import ( _assign_pld_id as _assign_pld_id, _download_pld as _download_pld, diff --git a/HydroEO/flows/_clean_engine.py b/HydroEO/flows/_clean_engine.py index df8470c..817614d 100644 --- a/HydroEO/flows/_clean_engine.py +++ b/HydroEO/flows/_clean_engine.py @@ -1,10 +1,5 @@ -"""Shared cleaning engine used by both reservoirs and rivers. - -_clean_timeseries applies per-mission processing filters generically for -either target_type -- called by _clean_reservoirs_timeseries (in -_reservoir_pipeline.py) and _clean_rivers_timeseries (in -_river_pipeline.py). Not itself patched as a sibling of either wrapper in -the test suite, so it's free to live in its own module. +""" +Shared timeseries cleaning engine for both reservoirs and rivers. """ import logging @@ -82,5 +77,3 @@ def _clean_timeseries(prj: "Project", target_type: str) -> None: ) general.ifnotmakedirs(export_dir) ts.export_csv(os.path.join(export_dir, f"{product}.csv")) - - diff --git a/HydroEO/flows/_constants.py b/HydroEO/flows/_constants.py index 276827c..8e936af 100644 --- a/HydroEO/flows/_constants.py +++ b/HydroEO/flows/_constants.py @@ -1,8 +1,5 @@ -"""Shared constants for the merge/clean pipeline (both reservoirs and rivers). - -Split out of the original flows.py so both _clean_engine.py and -_merge_engine.py (and their reservoir/river wrapper modules) can import -these without needing to import each other. +""" +Shared constants for the merge/clean pipeline (both reservoirs and rivers). """ PRODUCT_TIMESERIES_KEYS = { @@ -10,17 +7,7 @@ lat_key="lat", lon_key="lon", pass_key="file_name", platform_key="platform", orbit_key="relative_orbit", ), - # NOTE: sentinel6 still uses "pass" as orbit_key -- NOT verified to be - # unstable the way it was for sentinel3 (confirmed empirically: on - # real data, "pass" was unique-per-crossing for every S3A/S3B visit, - # i.e. not stable at all, while "relative_orbit" genuinely repeated - # across multiple visits -- e.g. S3B crossed via 2 distinct stable - # configurations, with real biases of -0.14m and +0.22m that a - # platform-only grouping was averaging into one misleading +0.04m). - # Sentinel-6 may have the same "pass" instability and may also have - # its own "relative_orbit"-equivalent column, but this hasn't been - # checked against real S6 data -- don't assume the same fix applies - # without verifying first. + # NOTE: sentinel6 still uses "pass" as orbit_key -- NOT verified "sentinel6": dict( lat_key="lat", lon_key="lon", pass_key="file_name", platform_key="platform", orbit_key="pass", @@ -44,138 +31,38 @@ "window_km": 1.5, "svr_linear_err": 0.1, "svr_linear_epsilon": 0.1, - # Both updated from DAHITI's lake-tuned defaults (err=0.1, gamma= - # 0.0000438) based on real reservoir data validated this session -- - # the lake-tuned gamma implied a ~151-day smoothing lengthscale, far - # too coarse for a reservoir with real multi-week transitions (see - # the svr_radial oversmoothing discussion). err=1.0 (river-like, - # rather than the stricter lake value) and gamma x50 (~21-day - # lengthscale instead of ~151 days) let the trend actually track - # real fast changes instead of rejecting them as if they were noise. - "svr_radial_err": 1.0, + "svr_radial_err": 1.0, # DAHITI default err=0.1 too strict for reservoirs with real multi-week transitions "svr_radial_rbf_c": 10000, - "svr_radial_gamma": 0.0000438 * 50, + "svr_radial_gamma": 0.0000438 * 50, # DAHITI default gamma=0.0000438 (~151 days) too smooth for reservoirs with real multi-week transitions "svr_radial_epsilon": 0.1, - # Confirmed on real data across two reservoirs: revisit sparsity varies a - # lot (e.g. one reservoir had icesat2/S3A/S3B visiting only 7/14/13 - # distinct days all year). At "10D"/3, sparse sources can fail to ever - # find 3 overlapping bins and get dropped as unanchored ENTIRELY (not - # just trimmed) -- confirmed: this silently dropped 2 of 3 missions - # (icesat2, S3B) for one real reservoir. "20D"/1 recovered all of it. - # Widening is monotonically safe against data loss (a wider window can - # only find equal-or-more overlapping bins, never fewer) -- the - # tradeoff is a very wide bin could blur real water-level change within - # the window into the bias estimate; 20D is a modest widening, not an - # extreme one. - "bias_time_bin": "20D", + "bias_time_bin": "20D", #Based on examples - sparse sources can fail and get dropped silently but in order of magnitude of S3 return period "bias_min_overlap": 1, - # Confirmed empirically on real data: "platform_orbit" (using - # orbit_key -- now sentinel3's verified-stable "relative_orbit" - # column, see PRODUCT_TIMESERIES_KEYS) reveals genuine within-platform - # bias heterogeneity that "platform" alone was masking. One real - # reservoir's S3B crosses via two distinct, independently stable - # configurations (5 days on one, 8 on the other) with biases of - # -0.14m and +0.22m respectively -- "platform" grouping averaged - # these into one misleading +0.04m. Same pattern for ICESat-2's - # beams (orbit_key="beam"): per-beam biases ranged 0.06-0.18m under - # "platform_orbit", collapsed to one number under "platform". Total - # kept-row count was IDENTICAL either way on the reservoir tested - # (3182/4901) -- this is a precision gain, not a data-loss risk, at - # least for sentinel3/icesat2. NOTE: sentinel6 still uses "pass" as - # orbit_key (unverified whether it's stable or has a - # relative_orbit-equivalent -- see PRODUCT_TIMESERIES_KEYS) -- if - # it's actually unstable like sentinel3's old "pass" mapping was, - # "platform_orbit" could fragment sentinel6 into single-crossing - # sources. Recheck against real sentinel6 data before trusting this - # default for a project relying heavily on sentinel6. - "bias_group_by": "platform_orbit", - # Not a spatial correction -- just flags (and records in - # ts.bias_correct_diagnostics) when a source's observations are - # centered far from the anchor's, since for a large/elongated - # reservoir some of the estimated bias could be real spatial signal. - # Worth a closer look per-reservoir if this fires, not an error. - "bias_centroid_warn_km": 5.0, - # Off by default -- inflates Kalman input error by distance from the - # reservoir polygon's own centroid, addressing crossings that may be - # hydraulically unrepresentative (e.g. far upstream, subject to real - # slope bias) even when ADM alone reports them as highly precise. Set - # to a real value (m of extra error per km of distance) to enable -- - # the right scale depends on the true magnitude of upstream slope bias - # for your reservoirs, which needs empirical tuning, not a guessed - # default. - "distance_penalty_scale_per_km": None, - # Off by default -- a genuine height correction (not just error - # inflation) using a spatial deviation model fit once from a dense - # source (default ICESat-2) and persisted to disk per reservoir (see - # _get_or_fit_spatial_correction_model) so past corrections don't - # shift retroactively as new data arrives. Turn on once you've - # confirmed (as we did empirically) that the target reservoir shows a - # real, day-to-day-consistent spatial deviation pattern -- fitting - # requires several qualifying dense-source days (see - # fit_spatial_correction_model's min_days), and silently does nothing - # if there isn't enough dense-source data yet. - "use_spatial_correction": False, + "bias_group_by": "platform_orbit", # Should represent individual observation groups (platform and orbit/beam), see note for S6 + "bias_centroid_warn_km": 5.0, #Warning threshold for large/elongated reservoirs + "distance_penalty_scale_per_km": None, # Off by default: inflate Kalman input error by distance from the reservoir centroid + "use_spatial_correction": False, # Off by default: apply a spatial correction model fit from a dense spatial source (ICESat-2) to other missions' observations, if enough qualifying dense-source days exist "spatial_correction_dense_source": "icesat2", } DEFAULT_RIVER_MERGING_OPTIONS = { - # Mostly a starting point copied from the reservoir defaults and NOT - # independently validated against real river data the way the - # reservoir defaults were validated this session -- river dynamics - # differ genuinely (e.g. a real, expected along-reach gradient), so - # do not assume the rest of these are correct without checking. - # svr_radial_err/gamma below ARE an explicit exception (set directly, - # not copied): gamma x100 (~15-day lengthscale, vs DAHITI's ~151-day - # lake value) and err=1.0, matching the same oversmoothing reasoning - # as the reservoir defaults, just with a shorter lengthscale given - # rivers can change faster still. + # Mostly a starting point copied from the reservoir defaults + # NOT independently validated against real river data. "window_km": 1.5, "svr_linear_err": 0.1, "svr_linear_epsilon": 0.1, - "svr_radial_err": 1.0, + "svr_radial_err": 1.0, #Matches reservoir default, DAHITI default err=0.1 too strict for rivers with real multi-week transitions "svr_radial_rbf_c": 10000, - "svr_radial_gamma": 0.0000438 * 100, + "svr_radial_gamma": 0.0000438 * 100, #gamma x100 (~15-day lengthscale, vs DAHITI's ~151-day lake value) "svr_radial_epsilon": 0.1, "bias_time_bin": "20D", "bias_min_overlap": 1, - # Same reasoning/evidence as the reservoir default (see - # DEFAULT_RESERVOIR_MERGING_OPTIONS) for switching from "platform" to - # "platform_orbit" -- but this is carried over, not independently - # verified against real river data. A single river target (node/reach) - # is a much smaller footprint than a reservoir, so it's genuinely - # unclear whether the same within-platform configuration split - # (e.g. S3B's two distinct crossing geometries) would even occur at - # this scale -- check real per-target bias diagnostics once river - # data exists before trusting this. - "bias_group_by": "platform_orbit", - "bias_centroid_warn_km": 5.0, - # Off by default, same reasoning as reservoirs. NOTE: an earlier - # version of this comment claimed a river target's footprint is - # "much smaller than a reservoir" -- that's wrong for reaches - # specifically (confirmed ~10km typical length, comparable to or - # larger than many reservoirs), so distance_penalty/spatial - # correction may matter just as much for reaches as for reservoirs. - # It remains true that these tools address spread WITHIN one - # target's own crossing footprint, never the natural gradient - # BETWEEN different targets, which should never be "corrected away". + "bias_group_by": "platform_orbit", # Less likely for river targets but useful flag + "bias_centroid_warn_km": 5.0, # Relevant for reaches spanning multiple km "distance_penalty_scale_per_km": None, "use_spatial_correction": False, "spatial_correction_dense_source": "icesat2", - # Off by default. ONLY meaningful when - # prj.rivers.target_id_col == "reach_id" -- reference-corrects - # non-SWOT crossings (ICESat-2/Sentinel-3/6) to what they'd read at - # the reach's geometric midpoint, using SWOT's own directly-measured - # "slope" field (see _fit_reach_slope_correction/ - # _apply_reach_slope_correction). Requires "slope" to be present in - # mission_options["swot"]["hydrocron_fields"]["reaches"]. The - # midpoint-referenced assumption for SWOT's own reach WSE is an - # evidence-based inference from the RiverSP processing chain, not a - # fact directly confirmed in SWOT's documentation -- and the sign of - # the correction has not been empirically verified against real - # data in this session. Validate both before trusting this in - # production. - "use_reach_slope_correction": False, + "use_reach_slope_correction": False, #Off by default, meaningful to correct non-SWOT, uses SWOT slope. Not validated. } diff --git a/HydroEO/flows/_merge_engine.py b/HydroEO/flows/_merge_engine.py index a8331a2..b08de40 100644 --- a/HydroEO/flows/_merge_engine.py +++ b/HydroEO/flows/_merge_engine.py @@ -1,11 +1,5 @@ -"""Shared merge engine used by both reservoirs and rivers. - -_merge_timeseries applies the merge()/Kalman/svr_radial pipeline -generically for either target_type -- called by -_merge_reservoirs_timeseries (in _reservoir_pipeline.py) and -_merge_rivers_timeseries (in _river_pipeline.py). Not itself patched as a -sibling of either wrapper in the test suite, so it's free to live in its -own module. +""" +Shared merge engine used by both reservoirs and rivers. """ import logging @@ -127,13 +121,7 @@ def _merge_timeseries(prj: "Project", target_type: str) -> None: ) # Apply exclusions BEFORE exporting all_cleaned_timeseries.csv - # (not just before merge processing) -- this file is meant to - # reflect what's actually being worked with, and writing it - # before exclusions were applied meant it always showed - # excluded data regardless of how many times you re-ran, - # which looked exactly like a stale file from an old run but - # was actually happening on every single run. The full, - # pre-exclusion record is still available per-mission in + # The full, pre-exclusion record is still available per-mission in # cleaned_observations/{product}.csv (written earlier, in # _clean_timeseries, before any exclusion is applied) -- so # nothing is lost by making this file reflect exclusions. diff --git a/HydroEO/flows/_reservoir_download.py b/HydroEO/flows/_reservoir_download.py index fd77df8..fb5497d 100644 --- a/HydroEO/flows/_reservoir_download.py +++ b/HydroEO/flows/_reservoir_download.py @@ -1,10 +1,5 @@ -"""Reservoirs: multi-mission download orchestration. - -download_reservoirs and its three per-mission workers -(_download_reservoirs_swot, _download_reservoirs_icesat2, -_download_reservoirs_sentinel) are tested together via -patch.object(flows, "_name") in tests/unit/test_flows.py -- keep them in -this one module. +""" +Reservoirs: multi-mission download orchestration. """ import logging @@ -58,7 +53,7 @@ def _download_reservoirs_swot(prj: "Project") -> None: coords = [ (x, y) - for x, y in prj.reservoirs.download_gdf.unary_union.envelope.exterior.coords + for x, y in prj.reservoirs.download_gdf.union_all().envelope.exterior.coords ] logger.info( @@ -135,12 +130,7 @@ def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: session_token = None session_start_time = None - # EarthData (Sentinel-6 HR) needs no CREODIAS credentials at all -- - # only require them if we're actually going to use CREODIAS. But it - # does need its OWN upfront check -- without it, earthaccess.login() - # silently falls through to interactive prompting when nothing else - # is configured, which hangs in a non-interactive run instead of - # failing clearly (see Project._require_earthdata_credentials). + # Sentinel-6 can be retrieved via either Copernicus Open Access Hub (only LR) or Earthdata (HR) sentinel_creds = None use_earthdata_s6 = mission == "sentinel6" and _sentinel6_use_earthdata(prj) if use_earthdata_s6: @@ -174,10 +164,3 @@ def _download_reservoirs_sentinel(prj: "Project", mission: str) -> None: prj, mission, product, coords, download_dir, startdate, enddate, sentinel_creds, session_token, session_start_time, ) - - -# ============================================================================ -# RIVERS: Download -# ============================================================================ - - diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 304c603..94faa4d 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -1,9 +1,5 @@ -"""Reservoirs: PLD (Prior Lake Database) initialization. - -initialize_reservoirs and the three helpers it calls (_download_pld, -_assign_pld_id, _flag_missing_priors) are tested together via -patch.object(flows, "_name") in tests/unit/test_flows.py -- keep them in -this one module so those patches keep intercepting the right calls. +""" +Reservoirs: PLD (Prior Lake Database) initialization. """ import logging @@ -61,7 +57,7 @@ def _download_pld(prj: "Project") -> None: logger.info("Downloading PLD") download_dir = os.path.dirname(pld_path) - bounds = list(prj.reservoirs.gdf.unary_union.bounds) + bounds = list(prj.reservoirs.gdf.unary_all.bounds) raw_pld_path = prj.dirs.get("pld_raw") # Determine if raw_pld_path is inside project main_dir @@ -130,10 +126,3 @@ def _flag_missing_priors(prj: "Project") -> None: len(present), len(missing), ) - - -# ============================================================================ -# RIVERS: Initialization -# ============================================================================ - - diff --git a/HydroEO/flows/_reservoir_pipeline.py b/HydroEO/flows/_reservoir_pipeline.py index 81818dd..4fbfdc1 100644 --- a/HydroEO/flows/_reservoir_pipeline.py +++ b/HydroEO/flows/_reservoir_pipeline.py @@ -1,14 +1,5 @@ -"""Reservoirs: extraction + clean + merge + dfs0-export orchestration. - -create_reservoirs_timeseries and everything it (transitively) calls -- -_extract_reservoirs_timeseries, its three per-mission workers, -_clean_reservoirs_timeseries, _merge_reservoirs_timeseries, and -_export_cleaned_to_dfs0 -- are tested together via -patch.object(flows, "_name") in tests/unit/test_flows.py and -tests/unit/test_timeseries.py, so they all live in this one module. -`mikeio` is imported here (rather than at the package level only) because -_export_cleaned_to_dfs0 is the only place it's actually called, and tests -patch it as `flows.mikeio` -- see flows/__init__.py's re-export. +""" +Reservoirs: extraction + clean + merge + dfs0-export orchestration. """ import logging @@ -42,10 +33,6 @@ def create_reservoirs_timeseries(prj: "Project") -> None: if not hasattr(prj, "reservoirs"): return - # Extract raw observations from downloaded files (skips reservoirs whose - # gpkg already exists, unless prj.reservoirs.overwrite_extraction=True -- - # confirmed this re-read/re-extraction was the dominant real-world cost, - # far more than anything in clean()/merge()) _extract_reservoirs_timeseries( prj, overwrite=getattr(prj.reservoirs, "overwrite_extraction", False) ) @@ -69,10 +56,7 @@ def _extract_reservoirs_timeseries(prj: "Project", overwrite: bool = False) -> N overwrite : bool, optional If False (default), any reservoir/mission whose output .gpkg already exists is skipped entirely rather than re-read and - re-extracted. Confirmed on real data that this re-extraction -- - not clean()/merge() -- was the dominant cost in real end-to-end - runs (orders of magnitude larger than the merge pipeline itself). - Set True to force re-extraction (e.g. new raw downloads arrived). + re-extracted. """ if "icesat2" in prj.to_process: _extract_icesat2_observations(prj, overwrite=overwrite) @@ -90,15 +74,16 @@ def _extract_reservoirs_timeseries(prj: "Project", overwrite: bool = False) -> N def _extract_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: """Extract ICESat-2 ATL13 observations for each reservoir. - ICESat-2's raw data is a single atl13.parquet per reservoir that is - rewritten in full (the entire configured date range) on every - download -- unlike SWOT/Sentinel, there's no per-granule file list - to diff against a "processed" log. Re-extraction is instead gated - on whether the parquet is newer than the last extraction (e.g. - after project.update() downloaded through a later date), rather - than simply "does icesat2.gpkg already exist" -- otherwise data - added by update() would never reach the merged timeseries without - overwrite=True forcing a full reprocess of every reservoir. + Update by checking whether the atl13.parquet is newer than the existing geopackage + + Parameters + ---------- + overwrite : bool, optional + If False (default), any reservoir whose output icesat2.gpkg already exists + and is newer than its source atl13.parquet is skipped entirely rather than + re-read and re-extracted. + If True, all reservoirs with an atl13.parquet are re-extracted regardless + of the timestamps. """ available_ids = [ id @@ -175,14 +160,21 @@ def _extract_sentinel_observations( ) -> None: """Extract Sentinel-3 or Sentinel-6 observations for each reservoir. - Uses a per-reservoir "already extracted" file log (see - HydroEO.satellites.sentinel.preprocess.extract_observations's - processed_log_path) rather than a per-reservoir skip based on - whether {mission_key}.gpkg already exists -- so new subset files - downloaded after the first extraction (e.g. via project.update()) - are picked up and appended on the next run instead of being - silently ignored until overwrite=True forces a full reprocess of - every reservoir. + Uses a per-reservoir "already extracted" file log, to ensure new + "sub_" files are read and appended when updating. + + Parameters + ---------- + mission_key : str + Either "sentinel3" or "sentinel6", used to select the correct + download directory and mission-specific options. + product : str + Either "S3" or "S6", used for logging and file naming. + overwrite : bool, optional + If False (default), any reservoir whose output .gpkg already exists + is skipped entirely rather than re-read and re-extracted. + If True, all reservoirs with a download directory are re-extracted regardless + of the timestamps. """ available_ids = [ id @@ -234,17 +226,16 @@ def _extract_sentinel_observations( def _extract_swot_observations(prj: "Project", overwrite: bool = False) -> None: """Extract SWOT Lake SP observations for all reservoirs. - Uses a project-wide "already extracted" file log (see - HydroEO.satellites.swot.preprocess.extract_observations's - processed_log_path) rather than a per-reservoir skip based on - whether swot.gpkg already exists. SWOT granule shapefiles - accumulate forever in one shared download directory across ALL - reservoirs (see satellites.swot.preprocess.subset_by_id), so - whether a granule is "new" can only be decided at the file level, - not per reservoir -- and doing so lets new downloads (e.g. via - project.update()) get picked up and appended on the next run - instead of requiring overwrite=True to reprocess every reservoir - from the full granule history. + Uses a project-wide "already extracted" file log since project swot.gpkg + gets updated at each download. + + Parameters + ---------- + overwrite : bool, optional + If False (default), any reservoir whose output .gpkg already exists + is skipped entirely rather than re-read and re-extracted. + If True, all reservoirs with a download directory are re-extracted regardless + of the timestamps. """ download_dir = prj.dirs["swot"] if not os.path.exists(download_dir): @@ -274,38 +265,6 @@ def _extract_swot_observations(prj: "Project", overwrite: bool = False) -> None: ) -# Per-product mapping from generic Timeseries key attributes to the actual -# column names each mission's extractor writes. Sentinel-3 shares -# Sentinel-6's extractor/schema (same sentinel.extract_observations -# function, see _extract_sentinel_observations). -# -# **************************************************************************** -# TERMINOLOGY TRAP -- read before touching orbit_key/pass_key for Sentinel: -# The raw Sentinel-3/6 data has TWO similarly-named but opposite-meaning -# columns: -# - "orbit": the absolute revolution counter. Unique on every single -# crossing, never repeats. USELESS as orbit_key (bias_correct needs a -# persistent identifier to accumulate overlap against -- grouping by -# something that's different every time means every "source" has -# exactly 1 observation and nothing can ever be calibrated: this is -# exactly the bug that caused every S3A/S3B track to be dropped as -# unanchored in practice). -# - "pass": the satellite-engineering term for the STABLE, REPEATING -# ground track number (same value every ~27-day repeat cycle for -# S3A/S3B). This is what orbit_key actually needs. -# Confusingly, our own framework's `pass_key` means the OPPOSITE thing (one -# specific, one-time crossing -- e.g. file_name) from what "pass" means in -# the satellite data itself (the repeating track). Do not be tempted to -# point pass_key at the raw "pass" column -- file_name is correct there. -# **************************************************************************** -# -# ICESat-2's orbit_key is "beam" (the persistent ground track/virtual -# station) -- cycle_number only matters as an ingredient of the compound -# "pass" column built at extraction time (see -# HydroEO.satellites.icesat2.preprocess.extract_observations). SWOT's -# LakeSP product is already one integrated WSE per crossing with its own -# formal uncertainty (wse_u), so it needs neither lat/lon nor pass_key -- -# see preset_error_key, and daily_mad_error's handling of it. def _clean_reservoirs_timeseries(prj: "Project") -> None: """Apply quality filters to extracted reservoir timeseries.""" _clean_timeseries(prj, "reservoirs") diff --git a/HydroEO/flows/_river_common.py b/HydroEO/flows/_river_common.py index b196ec2..bd200f8 100644 --- a/HydroEO/flows/_river_common.py +++ b/HydroEO/flows/_river_common.py @@ -1,8 +1,5 @@ -"""River geometry helpers shared by download, extraction, and summaries. - -None of these are themselves the target of a sibling patch in the test -suite (only _river_target_corridor is, which is why that one function -lives in _summaries.py instead -- see generate_rivers_summaries's test). +""" +River geometry helpers shared by download, extraction, and summaries. """ import logging @@ -43,11 +40,7 @@ def _group_river_targets_by_waterbody(prj: "Project") -> dict: def _iter_geometry_pieces(geom): """ Yield each individual polygon from a geometry: every part of a - MultiPolygon, or the geometry itself for a plain Polygon. Used for - river downloads, where disconnected corridor pieces should each get - their own query rather than only querying the first piece (silently - dropping coverage of the rest) or merging them into one shape that - would also cover the (possibly large, irrelevant) gap between them. + MultiPolygon, or the geometry itself for a plain Polygon. """ if hasattr(geom, "geoms"): return list(geom.geoms) @@ -57,13 +50,7 @@ def _iter_geometry_pieces(geom): def _simplify_to_one_polygon(geom): """ Collapse a MultiPolygon into a single encompassing polygon via - convex hull. Used for reservoirs: unlike rivers, a reservoir is - treated as one target regardless of how many disconnected parts its - input polygon has, so a single combined query is preferred over - splitting into several separate ones. Convex hull guarantees full - coverage of every part, at the cost of also covering some in-between - area that may not be real water -- an accepted tradeoff for treating - one reservoir as one query rather than several. + convex hull. """ if hasattr(geom, "geoms"): return geom.convex_hull @@ -72,11 +59,10 @@ def _simplify_to_one_polygon(geom): def _river_extraction_buffer_meters(prj: "Project") -> float: """ - Resolve the extraction-corridor buffer distance: prefer an explicit + Resolve the extraction-corridor buffer distance: use prj.rivers.extraction_buffer_meters if set, else fall back to the SWORD-intersection prj.rivers.buffer_meters, else a conservative - default. Kept as its own small function since this fallback chain - is used by both download and extraction. + default (500 m). """ explicit = getattr(prj.rivers, "extraction_buffer_meters", None) if explicit: @@ -90,17 +76,12 @@ def _assign_points_to_river_targets( points, targets, target_id_col, max_distance_meters, local_crs ): """ - Assign each point in `points` to its nearest feature in `targets` + Assign each altimetry point (`points`) to its nearest feature in `targets` (SWORD node or reach geometries, whichever prj.rivers.target_id_col is configured for), dropping points farther than max_distance_meters from any target. - Uses gpd.sjoin_nearest rather than a custom NearestNeighbors/DBSCAN - approach -- it handles point-to-line matching natively (needed for - reaches, not just nodes), and max_distance is expressed directly in - real distance units once both inputs are reprojected to local_crs. - - Returns points (unprojected, original CRS) with target_id_col and a + Return points (unprojected, original CRS) with target_id_col and a _dist_to_target_m column added; rows with no target within range are dropped entirely. """ diff --git a/HydroEO/flows/_river_download.py b/HydroEO/flows/_river_download.py index 2f60762..3bf93e7 100644 --- a/HydroEO/flows/_river_download.py +++ b/HydroEO/flows/_river_download.py @@ -1,11 +1,5 @@ -"""Rivers: multi-mission download orchestration. - -download_rivers and _download_swot_hydrocron_timeseries are tested -together via patch.object(flows, "_name") in tests/unit/test_flows.py -- -keep them in this one module. _download_rivers_icesat2/_sentinel and -_get_latest_hydrocron_obs_date are not themselves patched as siblings of -anything, so they're free to live here too (this mirrors -_reservoir_download.py's structure). +""" +Rivers: multi-mission download orchestration. """ import logging @@ -33,13 +27,6 @@ def download_rivers(prj: "Project") -> None: """Download altimetry data for all configured missions (rivers mode). - SWOT uses the Hydrocron timeseries API directly (per node/reach, no - clustering needed -- see _download_swot_hydrocron_timeseries). - ICESat-2/Sentinel-3/6 download raw observations over a buffered - corridor around each waterbody's SWORD targets (see - _river_target_corridor); associating individual points with a - specific target happens later, during extraction. - Parameters ---------- prj : Project @@ -255,17 +242,7 @@ def _defer_warning(message, *args): def _download_rivers_icesat2(prj: "Project") -> None: """Download ICESat-2 ATL13 data for river waterbody groups. - - Mirrors _download_reservoirs_icesat2, but queries over a buffered - corridor around each waterbody's SWORD targets (see - _river_target_corridor) rather than a single reservoir polygon. If - a waterbody's corridor comes out as disconnected pieces (a - MultiPolygon), queries each piece separately (see - _iter_geometry_pieces) rather than only the first -- unlike - reservoirs, a river waterbody's targets can legitimately be - disjoint (e.g. separate reaches far apart), so collapsing to one - query would either miss coverage or require an artificially large - combined shape. + """ waterbody_groups = _group_river_targets_by_waterbody(prj) explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) @@ -323,17 +300,9 @@ def _download_rivers_icesat2(prj: "Project") -> None: def _download_rivers_sentinel(prj: "Project", mission: str) -> None: """Download Sentinel-3 or Sentinel-6 data for river waterbody groups. - Mirrors _download_reservoirs_sentinel (both now share - _download_sentinel_for_target, including the CREODIAS/EarthData - branching for Sentinel-6). NOTE: sentinel.query/query_earthdata take - a bounding box (envelope), not the exact corridor polygon -- for a - long or winding river corridor this can query/download a - meaningfully larger area than the actual buffered corridor. This is - an existing limitation inherited from the reservoir path (where it - matters far less, since a reservoir's envelope is close to its - actual extent), not something new introduced here -- worth - revisiting if it turns out to matter in practice for a large or - winding waterbody. + NOTE: sentinel.query/query_earthdata take a bounding box (envelope), + not the exact corridor polygon -- for a long or winding river corridor + this can query/download a meaningfully larger area than the actual buffered corridor. """ product = "S3" if mission == "sentinel3" else "S6" @@ -372,11 +341,7 @@ def _download_rivers_sentinel(prj: "Project", mission: str) -> None: ) continue - # Envelope each disconnected piece separately rather than the - # whole (possibly MultiPolygon) corridor at once -- sentinel's - # API only accepts a bounding box, so one envelope covering - # widely separated pieces could be far larger than any of them - # individually. + # Envelope each disconnected piece separately (see note) pieces = _iter_geometry_pieces(corridor_gdf.geometry.iloc[0]) download_dir = os.path.join(prj.dirs[mission], f"{wb_id}") @@ -420,9 +385,3 @@ def _get_latest_hydrocron_obs_date(output_path) -> datetime.date: latest_obs = timestamps.max().to_pydatetime() return datetime.date(latest_obs.year, latest_obs.month, latest_obs.day) - -# ============================================================================ -# RIVERS: Timeseries Processing (extraction) -# ============================================================================ - - diff --git a/HydroEO/flows/_river_pipeline.py b/HydroEO/flows/_river_pipeline.py index dda1cf7..0eb6eaa 100644 --- a/HydroEO/flows/_river_pipeline.py +++ b/HydroEO/flows/_river_pipeline.py @@ -1,11 +1,5 @@ -"""Rivers: extraction + clean + merge orchestration. - -create_rivers_timeseries and everything it (transitively) calls -- -_extract_rivers_timeseries, its three per-mission workers, and the -_clean_rivers_timeseries/_merge_rivers_timeseries wrappers -- are tested -together via patch.object(flows, "_name") in tests/unit/test_flows.py and -tests/unit/test_timeseries.py, so they all live in this one module (this -mirrors _reservoir_pipeline.py's equivalent constraint). +""" +Rivers: extraction + clean + merge orchestration. """ import logging @@ -36,12 +30,11 @@ def _extract_rivers_timeseries(prj: "Project", overwrite: bool = False) -> None: """Extract timeseries observations from raw downloaded files, for rivers. - SWOT still needs a (lightweight) extraction step here: Hydrocron - already returns a per-node/reach timeseries directly, but grouped - per WATERBODY (one CSV covering every target in that waterbody) -- - see _extract_rivers_swot_observations for splitting that into the - same per-target file structure ICESat-2/Sentinel-3/6 use, so the - shared clean/merge pipeline can treat every mission identically. + overwrite: Skip extraction if the target's output .gpkg already exists + (False, default) or re-extract anyway (True). + This is the same semantics as _extract_reservoirs_timeseries. + Lightweight extraction always required for SWOT to match other missions + in post-processing. Parameters ---------- @@ -68,18 +61,16 @@ def _extract_rivers_timeseries(prj: "Project", overwrite: bool = False) -> None: def _extract_rivers_icesat2_observations(prj: "Project", overwrite: bool = False) -> None: - """Extract ICESat-2 ATL13 observations for each river target. - - Unlike reservoirs (one polygon = one target), a river waterbody's - raw download covers many targets at once. This extracts once per - waterbody -- reusing icesat2.extract_observations exactly as - reservoirs use it, with the buffered corridor (see - _river_target_corridor) as the spatial filter instead of a single - reservoir polygon -- then assigns each surviving point to its - nearest target via sjoin_nearest, and splits the result into the - same per-target {output}/{target_id}/raw_observations/icesat2.gpkg - structure reservoirs already use, so everything downstream - (clean/merge) can treat a river target exactly like a reservoir. + """Extract ICESat-2 ATL13 observations for each river target (virtual station). + Uses buffered river corridor and assigns to nearest target. + Clean and merge treats river target (VS) exactly like a reservoir. + + Parameters + ---------- + overwrite : bool, optional + Skip extraction if the target's output .gpkg already exists + (False, default) or re-extract anyway (True). This is the same + semantics as _extract_reservoirs_timeseries. """ waterbody_groups = _group_river_targets_by_waterbody(prj) explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) @@ -171,20 +162,19 @@ def _extract_rivers_sentinel_observations( ) -> None: """Extract Sentinel-3 or Sentinel-6 observations for each river target. - Same per-waterbody-then-split approach as - _extract_rivers_icesat2_observations, plus a water-only filter: a - buffered river corridor is much looser than a reservoir polygon (it - genuinely includes riverbank, fields, vegetation alongside the - channel), and unlike ICESat-2, Sentinel-3/6 have no built-in water - classification. sigma0_min filters this out as a self-contained - post-processing step here (rather than modifying - sentinel.extract_observations itself, whose internals haven't been - verified) -- water gives a strong, consistent specular radar - return; land gives a weaker, noisier one. Needs empirical tuning - against real river data, same as every other threshold in this - pipeline -- the default here (0.0, i.e. no-op) is a safe starting - point, not a verified value; set mission_options[mission_key] - ['sigma0_min'] once you have real data to check it against. + Option to use sigma0 as a quality filter (sigma0_min) + is available in the mission's YAML config section. + + Parameters + ---------- + mission_key : str + "sentinel3" or "sentinel6" + product : str + "S3" or "S6" (used for output file naming) + overwrite : bool, optional + Skip extraction if the target's output .gpkg already exists + (False, default) or re-extract anyway (True). This is the same + semantics as _extract_reservoirs_timeseries. """ waterbody_groups = _group_river_targets_by_waterbody(prj) explicit_buffer = getattr(prj.rivers, "extraction_buffer_meters", None) @@ -293,17 +283,14 @@ def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) - every other mission uses, so the shared clean/merge pipeline can treat SWOT identically to ICESat-2/Sentinel-3/6 for rivers. - Unlike LakeSP for reservoirs, Hydrocron's own CSV doesn't include - per-observation coordinates in the default field lists (see - rivers.yaml) -- but nothing downstream actually needs per-observation - lat/lon for SWOT (see PRODUCT_TIMESERIES_KEYS: no lat_key/lon_key for - "swot"), so this attaches the target's own SWORD geometry as a - constant placeholder purely so the file can be saved/read as .gpkg - like every other mission's output -- the geometry's actual value is - never used downstream, only the height/date/platform/orbit columns. - - Quality filtering (max_q) is already applied at download time (see - _download_swot_hydrocron_timeseries), so it isn't repeated here. + Uses SWORD geometry as placeholder in lieu of "lat/lon". + + Parameters + ---------- + overwrite : bool, optional + Skip extraction if the target's output .gpkg already exists + (False, default) or re-extract anyway (True). This is the same + semantics as _extract_reservoirs_timeseries. """ waterbody_groups = _group_river_targets_by_waterbody(prj) id_label = "nodes" if prj.rivers.target_id_col == "node_id" else "reaches" @@ -381,11 +368,8 @@ def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) - def create_rivers_timeseries(prj: "Project") -> None: """Extract, clean, and merge timeseries for river targets (nodes/reaches). - Mirrors create_reservoirs_timeseries. Not yet done: an export_to_dfs0 - equivalent for rivers, since _export_cleaned_to_dfs0 currently - iterates prj.reservoirs.download_gdf specifically -- left out here - rather than silently generalizing something not explicitly asked - for yet. + Mirrors create_reservoirs_timeseries. + NOTE: Extraction to dsf0 missing. Parameters ---------- @@ -413,9 +397,3 @@ def _merge_rivers_timeseries(prj: "Project") -> None: """Merge multi-mission timeseries into combined datasets, for river targets.""" _merge_timeseries(prj, "rivers") - -# ============================================================================ -# RESERVOIRS: Summaries & Visualization -# ============================================================================ - - diff --git a/HydroEO/flows/_run_config.py b/HydroEO/flows/_run_config.py index 6eacb73..763f02f 100644 --- a/HydroEO/flows/_run_config.py +++ b/HydroEO/flows/_run_config.py @@ -1,14 +1,12 @@ -"""Per-target run_config persistence, exclusions, and spatial/reach-slope +""" +Per-target run_config persistence, exclusions, and spatial/reach-slope correction caching -- shared by both reservoirs and rivers. One YAML file per target ({output}/{id}/run_config.yaml) that is -simultaneously: (a) a human-readable log of decisions made about this -target, (b) the actual source of truth _merge_engine._merge_timeseries -reads to apply those decisions, and (c) something a user can hand-edit -directly for a fully config-driven workflow. - -None of this is itself patched as a sibling of any single caller in the -test suite, so it's free to live in its own module. +simultaneously: (a) readable log of decisions made about this +target, (b) what _merge_engine._merge_timeseries +reads to apply those decisions, and (c) hand-editable +for a fully config-driven workflow. """ import logging @@ -42,14 +40,14 @@ def _get_target_ids(prj: "Project", target_type: str): def _target_centroid(prj: "Project", target_type: str, id): """ - Return (lat, lon) of a target's own geometry centroid -- the - reservoir polygon for target_type="reservoirs", or the SWORD - node/reach geometry for target_type="rivers" -- computed in a - projected (local) CRS for accuracy, then converted back to lat/lon. + Return (lat, lon) of a target's own geometry centroid: + - the reservoir polygon for target_type="reservoirs" + - the SWORD node/reach geometry for target_type="rivers" + computed in a projected (local) CRS for accuracy, then converted back to lat/lon. Used as the reference location for apply_distance_penalty/ - apply_spatial_correction. Returns (None, None) if the target's - geometry can't be found, so callers can treat that as "skip" rather - than fail. + apply_spatial_correction. + + Returns (None, None) if the target's geometry can't be found. """ try: if target_type == "reservoirs": @@ -75,7 +73,7 @@ def _target_centroid(prj: "Project", target_type: str, id): def _reservoir_centroid(prj: "Project", id): - """Backward-compatible wrapper -- see _target_centroid.""" + """Backward-compatible wrapper, see _target_centroid.""" return _target_centroid(prj, "reservoirs", id) @@ -84,20 +82,15 @@ def _get_or_fit_spatial_correction_model( recalibrate=False, **fit_kwargs, ): """ - Load a persisted spatial correction model for this target if one - exists, or fit a fresh one and persist it. Works identically for - reservoirs and river targets -- see _target_centroid. + Load existing spatial correction model if it exists or generates new + model. Works identically for reservoirs and river targets, see _target_centroid. - This is deliberately NOT re-fit automatically every run: doing so - would make past corrections shift retroactively every time new - dense-source data arrives, since the fitted slope would change. - Pass recalibrate=True to explicitly force a re-fit (e.g. as a - deliberate, occasional recalibration step) -- not something that - should happen as a silent side effect of routine reprocessing. + This is only re-fit is requested (recalibrate = True), to avoid + changing past corrections silently. Returns None if no model exists yet and there isn't enough dense - source data to fit one (see fit_spatial_correction_model) -- callers - should treat this the same as "no correction available." + source data to fit one (see fit_spatial_correction_model) + Equivalent to "no correction available." """ model_path = os.path.join( prj.dirs["output"], f"{id}", "spatial_correction_model.json" @@ -149,21 +142,6 @@ def _get_or_fit_spatial_correction_model( return model -# ============================================================================ -# Per-target run config: exclusions + per-target merging option overrides -# ============================================================================ -# -# One YAML file per target ({output}/{id}/run_config.yaml) that is -# simultaneously: (a) a human-readable log of decisions made about this -# target, (b) the actual source of truth _merge_timeseries reads to apply -# those decisions, and (c) something a user can hand-edit directly for a -# fully config-driven workflow. Interactive functions below -# (exclude_from_target, set_merging_option, ...) read-modify-write this -# same file, so a decision made once in a notebook session is exactly the -# same artifact you'd edit by hand or check into version control -- there -# is no separate "notebook state" to keep in sync with "the config". - - def _run_config_path(prj: "Project", id) -> str: return os.path.join(prj.dirs["output"], f"{id}", "run_config.yaml") @@ -201,13 +179,8 @@ def _save_run_config(prj: "Project", id, config: dict) -> None: def _invalidate_spatial_correction_cache(prj: "Project", id) -> None: """ - Delete any cached spatial correction model for this target, forcing - a fresh fit next time use_spatial_correction is used. Called whenever - exclusions or spatial-correction-relevant options change -- the - model may have been fit using observations that are no longer - included, and this is exactly the kind of deliberate, explicit - trigger (not routine reprocessing) that recalibration is meant for -- - see _get_or_fit_spatial_correction_model. + Delete cached spatial correction model for this target, forcing + a fresh fit next time use_spatial_correction is used. """ model_path = os.path.join(prj.dirs["output"], f"{id}", "spatial_correction_model.json") if os.path.exists(model_path): @@ -220,12 +193,7 @@ def _invalidate_spatial_correction_cache(prj: "Project", id) -> None: def _invalidate_reach_slope_correction_cache(prj: "Project", id) -> None: """ - Delete any cached reach slope correction model for this target, - forcing a fresh fit next time use_reach_slope_correction is used. - Called whenever exclusions change -- an exclusion could target SWOT - observations specifically, which is exactly what this model is fit - from (see _fit_reach_slope_correction), so a cached model could - otherwise silently keep reflecting now-excluded SWOT slope values. + Delete any cached reach slope correction model for this target. """ model_path = os.path.join( prj.dirs["output"], f"{id}", "reach_slope_correction_model.json" @@ -241,42 +209,12 @@ def _invalidate_reach_slope_correction_cache(prj: "Project", id) -> None: def _fit_reach_slope_correction(prj: "Project", target_id, recalibrate: bool = False): """ Fit (or load a persisted) reach-level slope correction from SWOT's - own directly-measured "slope" field (RiverSP reach product), used to + own directly-measured "slope" field (RiverSP reach product), to reference-correct OTHER missions' (ICESat-2/Sentinel-3/6) crossings - to what they'd read at the reach's geometric midpoint. - - ONLY meaningful for reaches (prj.rivers.target_id_col == "reach_id") - -- a node is a single ~200m-spaced point, not a ~10km segment with - its own along-reach slope in the same sense. Callers must not invoke - this for node-mode projects. - - Rationale: SWOT's reach-level WSE is an aggregate over the reach's - ~50 constituent, roughly-evenly-spaced nodes, not a value evaluated - at one specific point -- for an evenly-sampled linear profile, the - mean equals the value at the mean position, so this is treated as - approximately midpoint-referenced. This is an evidence-based - inference from the RiverSP processing chain, NOT a fact directly - confirmed in SWOT's product documentation (which does not explicitly - state a reference point) -- validate against real Hydrocron - node-vs-reach output for a known reach before trusting this deeply. - - Uses the MEDIAN of all available SWOT slope observations for this - reach as a single, persistent correction -- not a per-date-specific - one -- consistent with this pipeline's existing preference (see - fit_spatial_correction_model) for a stable, once-fit value over a - per-observation one, and avoiding the complexity/fragility of - matching a specific SWOT overpass date to each individual non-SWOT - observation's date. - - Persisted to {output}/{target_id}/reach_slope_correction_model.json - -- fit once, loaded thereafter, only refit on explicit - recalibrate=True -- so past corrections don't shift retroactively - as new SWOT data arrives, same reasoning as the spatial correction - model's caching. + to the reach's geometric midpoint. Returns None if no model exists yet and there's no usable SWOT slope - data to fit one from -- callers should treat this as "no correction - available," not an error. + data to fit one from, "no correction available," not an error. """ model_path = os.path.join( prj.dirs["output"], f"{target_id}", "reach_slope_correction_model.json" @@ -331,18 +269,10 @@ def _apply_reach_slope_correction( ) -> pd.DataFrame: """ Apply a fitted reach slope correction (see _fit_reach_slope_correction) - to non-SWOT rows in ts_df -- adjusts "height" to what each row would - read at the reach's geometric midpoint, using its along-reach - projected position and the reach's persistent median slope. SWOT's - own rows are left untouched (already assumed midpoint-referenced -- - see _fit_reach_slope_correction's docstring for the reasoning and - its caveats). - - NOTE: the sign convention for "correction = slope x distance" here - has NOT been empirically verified against real data in this - session -- confirm it actually reduces cross-mission scatter for a - real reach (not increases it) before trusting this in production; - flip the sign if it doesn't. + to non-SWOT rows in ts_df. + + NOTE: the sign convention for "correction = slope x distance" + has NOT been empirically verified against real data. Rows without usable lat/lon (or if the target's geometry can't be found) are left uncorrected rather than dropped. diff --git a/HydroEO/flows/_sentinel_shared.py b/HydroEO/flows/_sentinel_shared.py index b041b65..006cf8e 100644 --- a/HydroEO/flows/_sentinel_shared.py +++ b/HydroEO/flows/_sentinel_shared.py @@ -1,10 +1,5 @@ -"""Sentinel-3/6 download logic shared by both reservoirs and rivers. - -_sentinel6_use_earthdata and _download_sentinel_for_target are used by -both _reservoir_download.py and _river_download.py (the CREODIAS/EarthData -branching only needs to exist in one place); neither is itself the target -of a sibling patch in the test suite, so this module has no test-imposed -co-location constraint of its own. +""" +Sentinel-3/6 download logic shared by both reservoirs and rivers. """ import logging @@ -42,21 +37,13 @@ def _download_sentinel_for_target( """ Download + subset Sentinel-3/6 data for one target's AOI (a reservoir polygon, or a river waterbody corridor's envelope) -- - shared by both _download_reservoirs_sentinel and - _download_rivers_sentinel so the CREODIAS/EarthData branching logic - only needs to exist in one place. For Sentinel-6, if _sentinel6_use_earthdata(prj) is True, uses PO.DAAC/EarthData (see sentinel.query_earthdata/download_earthdata) - to get the HR product instead of CREODIAS's LR-only product. - EarthData files arrive flat (no SAFE-zip directory), so the unzip - step is skipped for that path -- subset() already handles both flat - and zipped-folder inputs (see sentinel/preprocess.py's file - discovery, extended for this). - - Returns (session_token, session_start_time) -- unchanged from what - was passed in when using the EarthData path, since that mechanism - (CREODIAS session reuse) doesn't apply to it. + to get the HR product in .nc format instead of CREODIAS's LR-only product + in SAFE-zip format. + + Returns (session_token, session_start_time). """ dir_key = mission use_earthdata = mission == "sentinel6" and _sentinel6_use_earthdata(prj) diff --git a/HydroEO/flows/_summaries.py b/HydroEO/flows/_summaries.py index 93e5b36..036b2db 100644 --- a/HydroEO/flows/_summaries.py +++ b/HydroEO/flows/_summaries.py @@ -1,13 +1,6 @@ -"""Diagnostic plots for reservoirs and rivers, plus shared observation +""" +Diagnostic plots for reservoirs and rivers, plus shared observation loading helpers. - -generate_reservoirs_summaries is tested with _load_product_timeseries -patched as a sibling; generate_rivers_summaries is tested with -_has_enough_observations_to_plot/_project_num_months/_river_target_corridor/ -_load_merged_timeseries patched as siblings (tests/unit/test_flows.py) -- -so all of these live in one module. _river_target_corridor is also called -directly (unpatched) by the download/extraction modules, which import it -from here. """ import logging @@ -37,45 +30,23 @@ def _river_target_corridor( downloading/extracting ICESat-2 and Sentinel-3/6 observations. This is deliberately a SEPARATE buffer distance from - prj.rivers.buffer_meters (used earlier to decide which SWORD - targets intersect the user's AOI at all) -- that question ("is this - target in scope") and this one ("how far from the centerline could - real river water still be, for a raw altimetry point to plausibly - belong to this target") are different, and conflating them risks - the same "one parameter doing two jobs badly" issue found elsewhere - in this pipeline. + prj.rivers.buffer_meters used to select observations over water Parameters ---------- buffer_meters : float or None, optional Explicit, uniform buffer distance (meters), applied to every target regardless of its actual width. If None (default), uses - each target's own SWORD "width" attribute instead: buffer - distance = (width / 2) * width_buffer_factor. This is HALF the - width, not the full width -- buffering a line expands it - symmetrically by the given distance on EACH side, so a buffer - of width/2 gives a corridor whose TOTAL span is approximately - width * width_buffer_factor, matching the river's actual extent - plus a margin, rather than doubling it. Falls back to - _river_extraction_buffer_meters() (a flat scalar) if no usable - "width" column is found -- e.g. if your SWORD data names it - differently than assumed here, this degrades gracefully with a - log message rather than failing. + each target's own SWORD "width" attribute instead width_buffer_factor : float, optional Margin applied on top of each target's own width when using the - width-based default. Default 1.05 -- 5% wider than the river's + SWPORD-based default. Default 1.05 -- 5% wider than the river's actual channel width. Only used when buffer_meters is None. NOTE: "width" is the expected SWORD column name per the standard - SWORD data dictionary -- this has NOT been verified against a real - downloaded SWORD file in this session (no sample data was - available), unlike most other assumptions in this codebase. Check - your actual target_features columns if width-based buffering - doesn't seem to be kicking in. - - Returns a single-row GeoDataFrame in prj.global_crs (matching what - icesat2.extract_observations/sentinel.extract_observations expect - for their `features` argument, same as reservoirs), or None if no + SWORD data dictionary. + + Returns a single-row GeoDataFrame in prj.global_crs, or None if no matching SWORD geometry is found for target_ids. Note the returned geometry may be a MultiPolygon if targets form disconnected pieces (e.g. separate reaches far enough apart that their buffers never @@ -219,11 +190,6 @@ def _load_merged_timeseries(prj, id): return None -# ============================================================================ -# RIVERS: Summaries & Visualization -# ============================================================================ - - def _project_num_months(prj: "Project") -> int: """ Approximate number of months spanned by the project's configured @@ -250,7 +216,7 @@ def _project_num_months(prj: "Project") -> int: def _has_enough_observations_to_plot(prj: "Project", target_id, min_months: int) -> bool: """ Whether a target has enough merged observations to be worth - plotting -- more than min_months (the project's date range in + plotting -- more than min_months/2 (the project's date range in months) or more than 2, whichever is larger. A reach/reservoir with only 1-2 points produces a plot that adds noise without telling you anything. @@ -258,7 +224,7 @@ def _has_enough_observations_to_plot(prj: "Project", target_id, min_months: int) df = _load_merged_timeseries(prj, target_id) if df is None: return False - threshold = max(min_months, 2) + threshold = max(min_months/2, 2) return len(df) > threshold @@ -283,10 +249,7 @@ def generate_rivers_summaries( min_months = _project_num_months(prj) for wb_id, target_ids in waterbody_groups.items(): - # Only plot targets with enough observations to be worth looking - # at -- applies to all three plot types (map, time series, merge - # progress) so a target excluded from one isn't confusingly still - # shown in another. + # Exclude target if fewer than .5 observation per month plottable_ids = [ t for t in target_ids if _has_enough_observations_to_plot(prj, t, min_months) @@ -299,13 +262,7 @@ def generate_rivers_summaries( continue # Compute the actual extraction corridor (same buffer resolution - # used for real extraction, see _river_target_corridor) so the - # shaded area shown is exactly what extraction uses, not an - # approximation -- lets you visually judge whether width-based - # buffering produced a reasonable corridor for this waterbody - # (e.g. a lake-flagged reach whose SWORD width reflects a much - # wider lake extent) without needing external knowledge of the - # real river geometry. + # used for real extraction, see _river_target_corridor) for visual assessment corridor_gdf = _river_target_corridor( prj, plottable_ids, buffer_meters=getattr(prj.rivers, "extraction_buffer_meters", None), @@ -333,9 +290,3 @@ def generate_rivers_summaries( save=save, ) - -# ============================================================================ -# MIKEIO -# ============================================================================ - - From 467e819c0aae434a15b46c93e32e3d9ee868037a Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Thu, 16 Jul 2026 14:13:28 +0200 Subject: [PATCH 10/19] Small bug fixes for unary_union depreciation --- HydroEO/flows/__init__.py | 3 +++ HydroEO/flows/_reservoir_init.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HydroEO/flows/__init__.py b/HydroEO/flows/__init__.py index 0ca1b6f..4224aab 100644 --- a/HydroEO/flows/__init__.py +++ b/HydroEO/flows/__init__.py @@ -10,6 +10,9 @@ against the old single-file module, including for tests that patch private helpers via `patch.object(flows, "_name")`. """ +import mikeio # noqa: F401 -- re-exported so `flows.mikeio` resolves (see _reservoir_pipeline.py) + +from HydroEO import plotting # noqa: F401 -- re-exported so `flows.plotting`/`HydroEO.flows.plotting` resolves from ._reservoir_init import ( _assign_pld_id as _assign_pld_id, diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 94faa4d..204161c 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -57,7 +57,7 @@ def _download_pld(prj: "Project") -> None: logger.info("Downloading PLD") download_dir = os.path.dirname(pld_path) - bounds = list(prj.reservoirs.gdf.unary_all.bounds) + bounds = list(prj.reservoirs.gdf.union_all().bounds) raw_pld_path = prj.dirs.get("pld_raw") # Determine if raw_pld_path is inside project main_dir From 26cdb1026c1a40e17050cb227909742c850b0660 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Thu, 16 Jul 2026 16:09:04 +0200 Subject: [PATCH 11/19] SWOT fixes for robustness - Find lake using "intersects" instead of "within" when comparing to PLD (more robust if using buffered / imprecise polygon) - Fixed potential error if not initializing first and if reservoir in CRS (reprojection order) - Added hydroweb test - Stop download if no lake in PLD. --- HydroEO/downloaders/hydroweb.py | 4 +- HydroEO/flows/_reservoir_download.py | 11 ++ HydroEO/flows/_reservoir_init.py | 11 ++ HydroEO/flows/_river_init.py | 2 +- HydroEO/flows/_summaries.py | 2 +- HydroEO/project.py | 139 +++++----------------- HydroEO/satellites/icesat2/preprocess.py | 2 +- HydroEO/satellites/sentinel/preprocess.py | 2 +- HydroEO/satellites/swot/raster.py | 2 +- tests/unit/test_flows.py | 81 +++++++++++++ tests/unit/test_hydroweb.py | 61 ++++++++++ 11 files changed, 202 insertions(+), 115 deletions(-) create mode 100644 tests/unit/test_hydroweb.py diff --git a/HydroEO/downloaders/hydroweb.py b/HydroEO/downloaders/hydroweb.py index 8372533..7113d8d 100644 --- a/HydroEO/downloaders/hydroweb.py +++ b/HydroEO/downloaders/hydroweb.py @@ -165,8 +165,8 @@ def download_PLD( gdf = gdf.reset_index()[["lake_id", "res_id", "geometry"]] gdf = gdf[["lake_id", "res_id", "geometry"]] - # Filter to bounds and append - gdf = gdf.loc[gdf.within(shapely.Polygon.from_bounds(*bounds))] + # Filter to bounds and append. + gdf = gdf.loc[gdf.intersects(shapely.Polygon.from_bounds(*bounds))] gdf_list.append(gdf) # once we have processed all files, concatenate them diff --git a/HydroEO/flows/_reservoir_download.py b/HydroEO/flows/_reservoir_download.py index fb5497d..56bc1a7 100644 --- a/HydroEO/flows/_reservoir_download.py +++ b/HydroEO/flows/_reservoir_download.py @@ -40,6 +40,17 @@ def download_reservoirs(prj: "Project") -> None: def _download_reservoirs_swot(prj: "Project") -> None: """Download SWOT Lake SP data for reservoirs.""" + if "prior_lake_id" in prj.reservoirs.download_gdf.columns: + matched = (prj.reservoirs.download_gdf["prior_lake_id"] > 0).sum() + if matched == 0: + logger.warning( + "Skipping SWOT download: none of the %d reservoir(s) in this " + "project matched a Prior Lake Database (PLD) lake (see " + "aux/PLD/missing_in_pld.gpkg).", + len(prj.reservoirs.download_gdf), + ) + return + download_dir = prj.dirs["swot"] general.ifnotmakedirs(download_dir) diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 204161c..4912483 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -100,6 +100,7 @@ def _assign_pld_id(prj: "Project") -> None: distance_col="dist_to_pld", ) joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) + joined_gdf = joined_gdf.drop(columns=["index_right", "index_left"], errors="ignore") joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 @@ -126,3 +127,13 @@ def _flag_missing_priors(prj: "Project") -> None: len(present), len(missing), ) + if len(missing) > 0: + logger.warning( + "%d reservoir(s) not matched to any Prior Lake Database (PLD) lake " + "within pld_match_max_distance_m: %s (see aux/PLD/missing_in_pld.gpkg). " + "SWOT Lake SP cannot report observations for these. " + "Consider increasing pld_match_max_distance_m if it's a " + "near miss.", + len(missing), + ", ".join(str(v) for v in missing[prj.reservoirs.id_key]), + ) \ No newline at end of file diff --git a/HydroEO/flows/_river_init.py b/HydroEO/flows/_river_init.py index bf8dee3..e0d5644 100644 --- a/HydroEO/flows/_river_init.py +++ b/HydroEO/flows/_river_init.py @@ -80,7 +80,7 @@ def _prepare_rivers_from_sword(prj: "Project") -> None: # Intersect with SWORD sword_local = sword_gdf.to_crs(prj.local_crs) - subset = sword_local.loc[sword_local.intersects(aoi_local.unary_union)].copy() + subset = sword_local.loc[sword_local.intersects(aoi_local.union_all())].copy() if prj.rivers.id_key not in prj.rivers.aoi_gdf.columns: raise KeyError( diff --git a/HydroEO/flows/_summaries.py b/HydroEO/flows/_summaries.py index 036b2db..03b7b28 100644 --- a/HydroEO/flows/_summaries.py +++ b/HydroEO/flows/_summaries.py @@ -74,7 +74,7 @@ def _river_target_corridor( distances = _river_extraction_buffer_meters(prj) buffered = local.buffer(distances) - corridor = buffered.unary_union + corridor = buffered.union_all() corridor_gdf = gpd.GeoDataFrame( geometry=[corridor], crs=prj.local_crs ).to_crs(prj.global_crs) diff --git a/HydroEO/project.py b/HydroEO/project.py index a62cc24..0035855 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -175,43 +175,34 @@ def __post_init__(self): if "reservoirs" in self.config.keys() and self.config["reservoirs"].get( "enabled", True ): + reservoirs_gdf = gpd.read_file(self.config["reservoirs"]["path"]) + if reservoirs_gdf.crs is None: + raise ValueError( + f"Reservoirs shapefile '{self.config['reservoirs']['path']}' has " + "no CRS defined (missing .prj?). Cannot safely reproject to " + f"global_crs ({self.global_crs}). Set the file's CRS explicitly " + "before using it with HydroEO." + ) + reservoirs_gdf = reservoirs_gdf.to_crs(self.global_crs) + self.reservoirs = Reservoirs( - gdf=gpd.read_file(self.config["reservoirs"]["path"]), - id_key=self.config["reservoirs"]["id_key"], - dirs=self.dirs, + gdf=reservoirs_gdf, + id_key=self.config["reservoirs"]["id_key"], + dirs=self.dirs, ) - - self.reservoirs.gdf = self.reservoirs.gdf.to_crs(self.global_crs) + self.reservoirs.mission_options = self.mission_options self.reservoirs.processing_options = self.processing_options self.reservoirs.export_to_dfs0 = self.config["reservoirs"].get( - "export_to_dfs0", False + "export_to_dfs0", False ) - # NOTE: this was previously read via getattr(prj.reservoirs, - # "overwrite_extraction", False) in flows.py but never - # actually wired to config -- meaning it silently always - # defaulted to False regardless of what a user might have - # tried to set. Fixed here. + self.reservoirs.overwrite_extraction = self.config["reservoirs"].get( "overwrite_extraction", False ) # User-configurable overrides for the merge()/Kalman/svr_radial - # pipeline (see flows.DEFAULT_RESERVOIR_MERGING_OPTIONS for - # every available key and its default). Only the keys the user - # actually sets here are used to override the defaults -- any - # keys not mentioned keep their default value, so a user only - # needs to specify what they want to change. - # NOTE: DEFAULT_RESERVOIR_MERGING_OPTIONS' svr_radial_err/gamma - # trace back to DAHITI's own published values -- and DAHITI's - # calibration is for LAKES specifically. A reservoir with - # managed/operational water level changes (fill/drawdown - # cycles) can have real dynamics on a much faster timescale - # than a natural lake, making the lake-tuned defaults too - # strict (rejecting genuine fast changes as if they were - # noise). If your reservoirs behave more like this, override - # svr_radial_err and svr_radial_gamma here rather than relying - # on the lake-tuned defaults. + # pipeline self.merging_options = self.config["reservoirs"].get( "merging_options", {} ) @@ -278,28 +269,16 @@ def __post_init__(self): self.rivers.target_id_col = target_id_col self.rivers.target_ids = target_ids - # Corridor buffer for ICESat-2/Sentinel-3/6 extraction (see - # flows._river_target_corridor). Deliberately separate from - # buffer_meters above, which only decides which SWORD - # targets count as "in the AOI" at all. If left unset - # (None, the default), each target's own SWORD "width" - # attribute is used instead of one flat value for every - # target -- see width_buffer_factor below. Only falls back - # to a flat value (prj.rivers.buffer_meters, then 500m) if - # no usable "width" column is found. + # Corridor buffer for ICESat-2/Sentinel-3/6 extraction self.rivers.extraction_buffer_meters = rivers_cfg.get( "extraction_buffer_meters" ) # Margin applied on top of each target's own SWORD width - # when using the width-based default above (ignored if - # extraction_buffer_meters is set explicitly). Default 1.05 - # = 5% wider than the target's actual channel width. self.rivers.width_buffer_factor = rivers_cfg.get( "width_buffer_factor", 1.05 ) # Max distance (m) for assigning a raw altimetry point to its - # nearest SWORD target (see flows._assign_points_to_river_targets). - # Falls back to the extraction buffer if not set. + # nearest SWORD target self.rivers.max_node_assignment_meters = rivers_cfg.get( "max_node_assignment_meters" ) @@ -308,18 +287,7 @@ def __post_init__(self): ) # User-configurable overrides for the merge()/Kalman/svr_radial - # pipeline (see flows.DEFAULT_RIVER_MERGING_OPTIONS for every - # available key and its default). Set directly on - # prj.rivers, mirroring how prj.reservoirs.merging_options - # works, rather than routed through the shared project-level - # self.merging_options -- see flows._merge_timeseries's - # per-target-type override lookup. - # NOTE: DEFAULT_RIVER_MERGING_OPTIONS is currently a direct - # copy of the reservoir defaults and has NOT been - # independently validated against real river data -- unlike - # the reservoir defaults, which were tuned this way against - # real reservoirs. Treat it as a starting point to check - # kept/rejected counts against, not a verified value. + # pipeline self.rivers.merging_options = rivers_cfg.get("merging_options", {}) if "swot_raster" in self.config.keys() and self.config["swot_raster"].get( @@ -369,11 +337,7 @@ def __post_init__(self): ) ### Warn when lake/reservoir-only satellites are configured for neither - # reservoirs nor rivers mode. Previously this only checked for - # 'reservoirs' since icesat2/sentinel3/sentinel6 had no river support - # at all -- now that they work for rivers too (see - # flows._download_rivers_icesat2/_download_rivers_sentinel), the - # warning should only fire if NEITHER mode is configured. + # reservoirs nor rivers mode. if not hasattr(self, "reservoirs") and not hasattr(self, "rivers"): incompatible = [ m @@ -484,15 +448,6 @@ def _require_earthdata_credentials(self): """ Check upfront that EarthData credentials are available in some form earthaccess recognizes, before calling earthaccess.login(). - Without this check, earthaccess.login() -- called with no - explicit strategy, same as this codebase's existing SWOT path -- - falls through to environment variables, then a .netrc file, then - INTERACTIVE PROMPTING if neither is found (confirmed against - earthaccess's own documentation). In a non-interactive run (a - scheduled job, a CLI invocation) that prompt either hangs waiting - for input that will never come, or raises a confusing low-level - error deep inside earthaccess -- rather than a clear, immediate - one here. """ has_env = bool( os.environ.get("EARTHDATA_USERNAME") and os.environ.get("EARTHDATA_PASSWORD") @@ -511,10 +466,7 @@ def _require_earthdata_credentials(self): "Set EARTHDATA_USERNAME and EARTHDATA_PASSWORD in the " "environment, or EARTHDATA_TOKEN, or create a .netrc file with " "your Earthdata Login credentials (register free at " - "https://urs.earthdata.nasa.gov). Without one of these, " - "earthaccess.login() falls through to an interactive prompt, " - "which will hang in a non-interactive run rather than fail " - "clearly." + "https://urs.earthdata.nasa.gov)." ) def validate_config(self): @@ -561,34 +513,17 @@ def update(self): - SWOT (satellites.swot._download.download), Sentinel-3/6 via CREODIAS (satellites.sentinel.download), and Sentinel-6 via - EarthData (satellites.sentinel.download_earthdata) all track + EarthData (satellites.sentinel.download_earthdata) track already-downloaded granules in a `downloaded.log` file per - directory and only fetch what's new -- safe and cheap to - re-run the full configured range. + directory and only fetch what's new. - ICESat-2 (satellites.icesat2.download.query) has no such - de-duplication: it always re-submits the full - [startdate, enddate] request to SlideRule and overwrites - atl13.parquet from scratch each call. Still correct (the - file always reflects the complete range afterward), but not - incremental -- update() costs roughly the same as a fresh + de-duplication: update() costs roughly the same as a fresh download() for ICESat-2 specifically. - NOTE: this intentionally does not use the per-mission - get_latest_obs_date() helpers (satellites/{swot,icesat2, - sentinel}) to also advance startdate and skip already-covered - history. satellites.swot.preprocess.get_latest_obs_date() - currently returns the MAX observation date across all - reservoirs rather than the min, which would silently skip the - gap for any reservoir with sparser data if used as a shared - resume point -- fix that first if startdate-advancing is - wanted here. - - This previously called self.reservoirs.download(...) / - self.rivers.download(...), methods that do not exist on - Reservoirs/Rivers (waterbody.py defines only report()) with a - keyword signature (update_existing, enddate_overrides) that - flows.download_reservoirs/download_rivers never implemented -- - every call raised AttributeError. + NOTE: Uses newest observation across project - could be upgraded + to run per reservoir or check missing observations for sparser observed + reservoirs. + """ if not hasattr(self, "reservoirs") and not hasattr(self, "rivers"): logger.warning( @@ -638,16 +573,7 @@ def generate_summaries(self, show=False, save=True): def _infer_target_type(self, target_type=None): """ Resolve which target type (reservoirs/rivers) a per-target call - applies to. If target_type is given explicitly, use it (mainly - useful for tests or direct flows.* calls on a Project built - without going through normal config validation). Otherwise, - infer it from whichever of prj.reservoirs/prj.rivers is present. - - validate_config() only allows one of 'reservoirs'/'rivers'/ - 'swot_raster'/'swot_pixc' to be active per config, so a validly - constructed Project never has both prj.reservoirs and - prj.rivers set -- there is no "both configured" case to - disambiguate here. + applies to. """ if target_type is not None: return target_type @@ -660,10 +586,7 @@ def _infer_target_type(self, target_type=None): def list_target_observations(self, id, target_type=None): """ Summarize what observations exist for a target (reservoir or - river node/reach) at (platform, orbit) granularity -- the "what - could I exclude" view. See generate_summaries()'s plots - (platform-colored merge progress) for where a problem actually - shows up in the data. + river node/reach) at (platform, orbit) granularity """ target_type = self._infer_target_type(target_type) return flows.list_target_observations(self, target_type, id) diff --git a/HydroEO/satellites/icesat2/preprocess.py b/HydroEO/satellites/icesat2/preprocess.py index e96327b..b255933 100644 --- a/HydroEO/satellites/icesat2/preprocess.py +++ b/HydroEO/satellites/icesat2/preprocess.py @@ -27,7 +27,7 @@ def extract_observations( return # Filter observations to those inside the reservoir geometry. - gdf = gdf.loc[gdf.within(features.unary_union)].reset_index(drop=True) + gdf = gdf.loc[gdf.within(features.union_all())].reset_index(drop=True) if len(gdf) == 0: return diff --git a/HydroEO/satellites/sentinel/preprocess.py b/HydroEO/satellites/sentinel/preprocess.py index 28609ab..fe9bf01 100644 --- a/HydroEO/satellites/sentinel/preprocess.py +++ b/HydroEO/satellites/sentinel/preprocess.py @@ -872,7 +872,7 @@ def extract_observations( # filter observations to ensure they fall within geometry data_gdf = data_gdf.loc[ - data_gdf.within(features.unary_union) + data_gdf.within(features.union_all()) ].reset_index(drop=True) if len(data_gdf) > 0: # filter by sigma0 diff --git a/HydroEO/satellites/swot/raster.py b/HydroEO/satellites/swot/raster.py index 13f378a..a15d50b 100644 --- a/HydroEO/satellites/swot/raster.py +++ b/HydroEO/satellites/swot/raster.py @@ -294,7 +294,7 @@ def _defer_warning(message, *args): if aoi_gdf is not None: try: aoi_reproj = aoi_gdf.to_crs(native_crs) - geom = [mapping(aoi_reproj.unary_union)] + geom = [mapping(aoi_reproj.union_all())] clipped = da.rio.clip( geom, aoi_reproj.crs, drop=True, all_touched=False ) diff --git a/tests/unit/test_flows.py b/tests/unit/test_flows.py index 45a54d9..0b37d4d 100644 --- a/tests/unit/test_flows.py +++ b/tests/unit/test_flows.py @@ -402,6 +402,7 @@ def test_download_reservoirs_dispatches_swot(mock_project_reservoirs): mock_sent.assert_not_called() + @pytest.mark.unit def test_download_reservoirs_dispatches_all_missions(mock_project_reservoirs): """download_reservoirs dispatches all enabled satellite missions.""" @@ -423,6 +424,56 @@ def test_download_reservoirs_dispatches_all_missions(mock_project_reservoirs): mock_ice.assert_called_once() assert mock_sent.call_count == 2 # sentinel3 and sentinel6 +@pytest.mark.unit +def test_download_reservoirs_swot_skips_when_none_matched_pld( + mock_project_reservoirs, caplog +): + """_download_reservoirs_swot skips the download entirely (no network call) + when none of the reservoirs matched a PLD lake -- downloading would be + guaranteed-wasted effort since extraction filters strictly by + prior_lake_id and would produce zero usable data regardless.""" + import logging + from HydroEO.satellites import swot + + mock_project_reservoirs.reservoirs.download_gdf = ( + mock_project_reservoirs.reservoirs.download_gdf.assign( + prior_lake_id=[-9999, -9999] + ) + ) + + with ( + patch.object(swot, "query") as mock_query, + caplog.at_level(logging.WARNING), + ): + flows._download_reservoirs_swot(mock_project_reservoirs) + + mock_query.assert_not_called() + assert "Skipping SWOT download" in caplog.text + + +@pytest.mark.unit +def test_download_reservoirs_swot_proceeds_when_some_matched_pld( + mock_project_reservoirs, +): + """_download_reservoirs_swot still downloads if at least one reservoir + matched the PLD, even if others didn't -- the AOI query is shared across + all reservoirs, so a partial match still needs the download to run.""" + from HydroEO.satellites import swot + + mock_project_reservoirs.reservoirs.download_gdf = ( + mock_project_reservoirs.reservoirs.download_gdf.assign( + prior_lake_id=[1001, -9999] + ) + ) + + with patch.object(swot, "query") as mock_query, patch.object(swot, "download") as mock_download: + mock_query.return_value = [] + flows._download_reservoirs_swot(mock_project_reservoirs) + + mock_query.assert_called_once() + mock_download.assert_called_once() + + @pytest.mark.unit def test_download_reservoirs_skips_disabled_missions(mock_project_reservoirs): @@ -935,3 +986,33 @@ def test_assign_pld_id_updates_gdf(mock_project_reservoirs): flows._assign_pld_id(mock_project_reservoirs) mock_sjoin.assert_called_once() + + +@pytest.mark.unit +def test_assign_pld_id_drops_index_right_column(mock_project_reservoirs, tmp_path): + """_assign_pld_id must not leave sjoin_nearest's leftover index_right/ + index_left columns in prj.reservoirs.gdf -- these persisted previously, + which broke any follow-up sjoin_nearest call on prj.reservoirs.gdf later + (ValueError: 'index_right' cannot be a column name in the frames being + joined) since sjoin_nearest wants to use that name itself.""" + pld_path = Path(mock_project_reservoirs.dirs["pld"]) + pld_path.parent.mkdir(parents=True, exist_ok=True) + pld_gdf = gpd.GeoDataFrame( + {"lake_id": [1001, 1002]}, + geometry=[Point(0.5, 0.5), Point(1.5, 1.5)], + crs="EPSG:4326", + ) + pld_gdf.to_file(pld_path, driver="GPKG") + + flows._assign_pld_id(mock_project_reservoirs) + + assert "index_right" not in mock_project_reservoirs.reservoirs.gdf.columns + assert "index_left" not in mock_project_reservoirs.reservoirs.gdf.columns + + # confirm a follow-up sjoin_nearest (e.g. manual diagnostics) doesn't + # collide with a leftover column from this one + gpd.sjoin_nearest( + mock_project_reservoirs.reservoirs.gdf.to_crs("EPSG:3857"), + pld_gdf.to_crs("EPSG:3857"), + distance_col="dist_to_pld", + ) diff --git a/tests/unit/test_hydroweb.py b/tests/unit/test_hydroweb.py new file mode 100644 index 0000000..9fd555a --- /dev/null +++ b/tests/unit/test_hydroweb.py @@ -0,0 +1,61 @@ +"""Unit tests for HydroEO.downloaders.hydroweb.download_PLD.""" + +from unittest.mock import patch + +import geopandas as gpd +import pandas as pd +import pytest +from shapely.geometry import Polygon + +from HydroEO.downloaders import hydroweb + + +@pytest.mark.unit +def test_download_pld_keeps_lakes_that_extend_past_bbox_edge(tmp_path): + """Regression test: download_PLD's bounds filter previously used + .within(), which requires the ENTIRE lake polygon to be contained in the + bounding box -- silently dropping any lake that extends even slightly + past its edge. This is exactly the failure mode for a single-reservoir + project (bounds == that one reservoir's own tight bbox) where the real + PLD lake polygon doesn't align neatly with a buffered input polygon's + bounding box. Fixed to use .intersects() instead, which correctly keeps + any lake that overlaps the bbox at all. + """ + # A pre-extracted PLD directory (raw_pld_path as a directory containing + # .sqlite files directly) skips the download/zip-extraction path + # entirely and goes straight to the buggy bounds-filtering logic. + extracted_dir = tmp_path / "pld_extracted" + extracted_dir.mkdir() + (extracted_dir / "lakes.sqlite").touch() # content irrelevant; gpd.read_file is mocked + + bounds = [0, 0, 1, 1] + + # lake_ok: fully inside the bbox. lake_edge: pokes 0.05 past the top edge + # -- exactly the scenario a buffered reservoir polygon's bbox can miss. + lake_ok = Polygon([(0.2, 0.2), (0.8, 0.2), (0.8, 0.8), (0.2, 0.8)]) + lake_edge = Polygon([(0.2, 0.2), (0.8, 0.2), (0.8, 1.05), (0.2, 1.05)]) + + fake_sqlite_gdf = gpd.GeoDataFrame( + {"res_id": [101, 102]}, + geometry=[lake_ok, lake_edge], + crs="EPSG:4326", + ) + # download_PLD sets gdf.index.name = "lake_id" after reading with + # fid_as_index=True -- give the mock a named index to match. + fake_sqlite_gdf.index = pd.Index([1, 2], name="lake_id") + + download_dir = tmp_path / "output" + + with patch.object(gpd, "read_file", return_value=fake_sqlite_gdf): + hydroweb.download_PLD( + download_dir=str(download_dir), + bounds=bounds, + raw_pld_path=str(extracted_dir), + keep_raw=True, + ) + + result = gpd.read_file(download_dir / "PLD_subset.gpkg") + assert set(result["res_id"]) == {101, 102}, ( + "lake_edge (which pokes past the bbox edge) was incorrectly dropped -- " + "the within()-vs-intersects() bug regressed" + ) From dafb9a76b4e1bbdf01b7b81a40894741321c4950 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Thu, 16 Jul 2026 17:11:40 +0200 Subject: [PATCH 12/19] Fix to PLD if using existing PLD with missing regional detail USe more robust Light sqlight SWORD file to find lake_id and backfill with regional file. Was using random sqlite from unzipped folder, which could lead to lake silently not being found because looking at wrong file. --- HydroEO/downloaders/hydroweb.py | 181 +++++++++++++++++++++++++------ HydroEO/flows/_reservoir_init.py | 1 + HydroEO/flows/_river_init.py | 6 - configs/reservoirs.md | 33 ++++-- tests/unit/test_hydroweb.py | 102 ++++++++++++++++- 5 files changed, 273 insertions(+), 50 deletions(-) diff --git a/HydroEO/downloaders/hydroweb.py b/HydroEO/downloaders/hydroweb.py index 7113d8d..68d435e 100644 --- a/HydroEO/downloaders/hydroweb.py +++ b/HydroEO/downloaders/hydroweb.py @@ -30,8 +30,12 @@ def download_PLD( - download_dir: str, bounds: list, raw_pld_path: str = None, keep_raw: bool = True -): + download_dir: str, + bounds: list, + raw_pld_path: str = None, + keep_raw: bool = True, + continent_codes: list = None, + ): """Download and subset SWOT Prior Lake Database. Parameters @@ -44,7 +48,13 @@ def download_PLD( Path to existing PLD zip file or extracted folder. If provided, skips download. keep_raw : bool, default True If False, delete raw zip and temp extraction folder after subset is created. - """ + continent_codes : list[str], optional + Continent-tile suffixes (e.g. ["AS", "AU"]) identifying which + full-schema PLD files (containing res_id, not just lake_id + + geometry) to use for backfilling res_id only. + Inspect your downloaded PLD folder to see which ones are actually present. + If omitted, all full-schema tiles found are used. + """ # create download directory if needed general.ifnotmakedirs(download_dir) @@ -59,10 +69,12 @@ def download_PLD( # Track whether the zip was downloaded by HydroEO (vs user-provided) hydroweb_managed_zip = True - # Check if extracted files already exist - extracted_files_exist = os.path.isdir(extracted_dir) and any( - f.endswith(".sqlite") for f in os.listdir(extracted_dir) - ) + def _dir_has_pld_files(d): + return any(f.endswith((".sqlite", ".gpkg")) for f in os.listdir(d)) + + # Check if extracted files already exist + extracted_files_exist = os.path.isdir(extracted_dir) and _dir_has_pld_files( + extracted_dir) # Handle user-provided raw_pld_path if raw_pld_path is not None and os.path.exists(raw_pld_path): @@ -74,8 +86,8 @@ def download_PLD( elif os.path.isdir(raw_pld_path): # User provided a directory logger.info("Using provided PLD directory: %s", raw_pld_path) - # Check if it contains .sqlite files directly - if any(f.endswith(".sqlite") for f in os.listdir(raw_pld_path)): + # Check if it contains PLD files (.sqlite or .gpkg) directly + if _dir_has_pld_files(raw_pld_path): extracted_dir = raw_pld_path extracted_files_exist = True else: @@ -83,15 +95,14 @@ def download_PLD( nested_path = os.path.join( raw_pld_path, "SWOT_PRIOR_LAKE_DATABASE", "SWOT_PRIOR_LAKE_DATABASE" ) - if os.path.isdir(nested_path) and any( - f.endswith(".sqlite") for f in os.listdir(nested_path) - ): + if os.path.isdir(nested_path) and _dir_has_pld_files(nested_path): extracted_dir = nested_path extracted_files_exist = True unzipped_dir = raw_pld_path # track parent for deletion logic else: logger.warning( - "Provided directory does not contain .sqlite files: %s", + "Provided directory does not contain .sqlite or " + ".gpkg PLD files: %s", raw_pld_path, ) return @@ -149,28 +160,130 @@ def download_PLD( # Get list of downloaded files downloaded_files = os.listdir(extracted_dir) - # now clean up and merge lake datafiles + def _read_pld_file(filepath, is_sqlite): + if is_sqlite: + # SWOT PLD .sqlite tiles store lake_id as the feature ID + g = gpd.read_file(filepath, layer="lake", fid_as_index=True) + g.columns = [c.lower() for c in g.columns] + g.index.name = "lake_id" + g = g.reset_index() + else: + # .gpkg PLD files already carry lake_id as a normal column. + g = gpd.read_file(filepath) + g.columns = [c.lower() for c in g.columns] + return g + logger.info("Merging products and removing temporary files") - gdf_list = list() - - for file in downloaded_files: - if file.endswith(".sqlite"): - filepath = os.path.join(extracted_dir, file) - logger.info("found %s", filepath) - - # Read layer directly as GeoDataFrame and normalize column names - gdf = gpd.read_file(filepath, layer="lake", fid_as_index=True) - gdf.columns = [c.lower() for c in gdf.columns] - gdf.index.name = "lake_id" - gdf = gdf.reset_index()[["lake_id", "res_id", "geometry"]] - gdf = gdf[["lake_id", "res_id", "geometry"]] - - # Filter to bounds and append. - gdf = gdf.loc[gdf.intersects(shapely.Polygon.from_bounds(*bounds))] - gdf_list.append(gdf) - - # once we have processed all files, concatenate them - gdf = pd.concat(gdf_list).reset_index(drop=True) + light_files = [ + f + for f in downloaded_files + if "_light" in f.lower() and f.lower().endswith((".gpkg", ".sqlite")) + ] + all_tile_files = [ + f + for f in downloaded_files + if f not in light_files and (f.endswith(".sqlite") or f.endswith(".gpkg")) + ] + bbox_poly = shapely.Polygon.from_bounds(*bounds) + + if light_files: + if len(light_files) > 1: + logger.warning( + "Multiple '_light' PLD files found (%s) -- using all of " + "them, but normally there should be exactly one.", + ", ".join(light_files), + ) + coverage_list = [ + _read_pld_file(os.path.join(extracted_dir, f), f.lower().endswith(".sqlite")) + for f in light_files + ] + coverage_gdf = pd.concat(coverage_list, ignore_index=True) + if "res_id" not in coverage_gdf.columns: + coverage_gdf["res_id"] = None + coverage_gdf = coverage_gdf[["lake_id", "res_id", "geometry"]] + + # Find lake using intersection with PLD + coverage_gdf = coverage_gdf.loc[ + coverage_gdf.intersects(bbox_poly) + ].reset_index(drop=True) + + backfill_tile_files = all_tile_files + if continent_codes: + backfill_tile_files = [ + f + for f in all_tile_files + if any(code.lower() in f.lower() for code in continent_codes) + ] + unmatched_codes = [ + code + for code in continent_codes + if not any(code.lower() in f.lower() for f in all_tile_files) + ] + if unmatched_codes: + logger.warning( + "hydroweb.continent_codes %s requested for PLD res_id " + "backfill, but no matching tile file was found among " + "the downloaded PLD data (%s). Re-download the PLD " + "if you expect a tile for these regions.", + unmatched_codes, + ", ".join(downloaded_files), + ) + + if backfill_tile_files and len(coverage_gdf) > 0: + res_id_map = {} + for file in backfill_tile_files: + filepath = os.path.join(extracted_dir, file) + logger.info("found %s (for res_id backfill)", filepath) + tile_gdf = _read_pld_file(filepath, file.endswith(".sqlite")) + if "res_id" in tile_gdf.columns: + res_id_map.update(dict(zip(tile_gdf["lake_id"], tile_gdf["res_id"]))) + if res_id_map: + coverage_gdf["res_id"] = coverage_gdf["lake_id"].map( + res_id_map + ).combine_first(coverage_gdf["res_id"]) + + still_missing = int(coverage_gdf["res_id"].isna().sum()) + if still_missing > 0: + logger.warning( + "%d of %d PLD lake(s) within this project's bounds could " + "not have res_id backfilled from the available " + "tile file(s) (%s). Set hydroweb.continent_codes in " + "your config to the correct region code(s) " + "and/or re-download the PLD to include the missing tile for backfilling res_id.", + still_missing, + len(coverage_gdf), + ", ".join(all_tile_files) if all_tile_files else "none present", + ) + else: + logger.warning( + "No global '_light' PLD file found among the downloaded PLD " + "data (%s) -- falling back to per-continent/region " + "tile files. If this PLD download " + "Check missing_res_id and redownload PLD if data is unexpectedly missing. " \ + "If you expect a global '_light' file, re-download the PLD to include it.", + ", ".join(downloaded_files), + ) + coverage_list = [ + _read_pld_file(os.path.join(extracted_dir, f), f.endswith(".sqlite")) + for f in all_tile_files + ] + if coverage_list: + coverage_gdf = pd.concat(coverage_list, ignore_index=True) + else: + coverage_gdf = gpd.GeoDataFrame( + {"lake_id": [], "res_id": [], "geometry": []}, crs="EPSG:4326" + ) + if "res_id" not in coverage_gdf.columns: + coverage_gdf["res_id"] = None + coverage_gdf = coverage_gdf[["lake_id", "res_id", "geometry"]] + coverage_gdf = coverage_gdf.loc[ + coverage_gdf.intersects(bbox_poly) + ].reset_index(drop=True) + + gdf = coverage_gdf + + gdf["res_id"] = gdf["res_id"].astype("float64") + # save the concatenated dataframe as GPKG export_path = os.path.join(download_dir, "PLD_subset.gpkg") diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 4912483..0c2d03c 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -80,6 +80,7 @@ def _download_pld(prj: "Project") -> None: bounds=bounds, raw_pld_path=raw_pld_path, keep_raw=effective_keep_raw, + continent_codes=getattr(prj, "pld_continent_codes", None), ) diff --git a/HydroEO/flows/_river_init.py b/HydroEO/flows/_river_init.py index e0d5644..b005c5c 100644 --- a/HydroEO/flows/_river_init.py +++ b/HydroEO/flows/_river_init.py @@ -206,9 +206,3 @@ def _ensure_sword_database(prj: "Project") -> None: else: logger.info("Kept raw SWORD zip file at %s (keep_raw_sword=True)", zip_path) - -# ============================================================================ -# RESERVOIRS: Download -# ============================================================================ - - diff --git a/configs/reservoirs.md b/configs/reservoirs.md index 12e0c36..66f34b7 100644 --- a/configs/reservoirs.md +++ b/configs/reservoirs.md @@ -9,7 +9,7 @@ Multi-satellite water surface elevation timeseries for reservoirs and lakes defi | `SWOT_L2_HR_LakeSP_D` | SWOT | Earthdata account + HydroWeb API key | | `ATL13` | ICESat-2 | None (SlideRule public API) | | — | Sentinel-3A/B | CREODIAS account | -| — | Sentinel-6 | CREODIAS account | +| — | Sentinel-6 | CREODIAS account (default, Low Rate product); or Earthdata account for the High Rate product — see `source` under [`sentinel3` / `sentinel6`](#sentinel3--sentinel6) | Credentials can be set in the config file or as environment variables: @@ -38,26 +38,29 @@ Start from [`configs/reservoirs.yaml`](reservoirs.yaml). | Key | Default | Description | | --- | --- | --- | -| `download` | `true` | Download SWOT Lake SP granules | -| `process` | `true` | Extract observations for each reservoir | +| `download` | `false`* | Download SWOT Lake SP granules | +| `process` | `false`* | Extract observations for each reservoir | | `startdate` / `enddate` | project dates | Per-satellite date override | | `pld_match_max_distance_m` | `100.0` | Max nearest-neighbour distance to match reservoirs to PLD lakes | -| `exclude_obs_id_values` | `[]` | SWOT `obs_id` values to drop during extraction | +| `exclude_obs_id_values` | `["no_data"]` | SWOT `obs_id` values to drop during extraction | | `processing_filters` | `["elevation", "MAD"]` | Ordered list of cleaning filters (see [Cleaning filters](#cleaning-filters)) | | `elevation_min_m` | `0.0` | Lower bound for elevation filter (metres) | | `elevation_max_m` | `8000.0` | Upper bound for elevation filter (metres) | | `mad_threshold` | `5.0` | Outlier multiplier for MAD filter | | `download_dir` | `{main_dir}/raw/swot` | Override download location | +*`false` is the code-level default if omitted entirely — you must explicitly set `download: true`/`process: true` to enable SWOT. The shipped [`configs/reservoirs.yaml`](reservoirs.yaml) template sets both to `true` out of the box. + ### `icesat2` | Key | Default | Description | | --- | --- | --- | -| `download` / `process` | — | Enable/disable download and processing | -| `atl13_fields` | `[]` | Extra SlideRule ancillary fields to request | +| `download` / `process` | `false` | Enable/disable download and processing | +| `atl13_fields` | `[]` | Extra SlideRule ancillary fields to request (empty by design -- SlideRule's `atl13x` endpoint already returns the core fields listed below without any extra request) | | `atl13.pass_invalid` | `false` | Include segments flagged as invalid | | `atl13.beams` | `[]` | Subset by beam name (e.g. `["gt1l", "gt2r"]`); empty = all | | `atl13.spots` | `[]` | Subset by spot number (e.g. `[1, 3]`); empty = all | +| `track_keys` | all 6 tracks | Validated against `["gt1l", "gt1r", "gt2l", "gt2r", "gt3l", "gt3r"]` if set, but currently a **no-op** in the SlideRule extraction path -- kept for API compatibility, doesn't yet filter anything. Use `atl13.beams` above for actual beam filtering. | | `processing_filters` | `["elevation", "MAD"]` | Same options as SWOT | | `download_dir` | `{main_dir}/raw/icesat2` | Override download location | @@ -69,9 +72,11 @@ Common output columns: `height` (EGM2008-corrected), `cycle_number`, `beam`, `rg | Key | Default | Description | | --- | --- | --- | -| `download` / `process` | — | Enable/disable | +| `download` / `process` | `false` | Enable/disable | | `sigma0_max` | `100000.0` | Maximum accepted sigma0 during extraction | | `download_threads` | `1` | Parallel CREODIAS download threads | +| `subset_file_id` | `"enhanced_measurement.nc"` | Filename identifying the file to extract from each downloaded product zip | +| `source` (sentinel6 only) | `"creodias"` | CREODIAS only distributes the Sentinel-6 Low Rate product. Set to `"earthdata"` for the High Rate (20Hz Ku-band) product via PO.DAAC/EarthData instead — needs `EARTHDATA_USERNAME`/`PASSWORD` rather than CREODIAS credentials. | | `processing_filters` | `["elevation", "MAD"]` | Same filter options as SWOT | | `download_dir` | `{main_dir}/raw/sentinel3` or `sentinel6` | Override download location | @@ -84,6 +89,10 @@ The PLD matches input reservoirs to SWOT lake identifiers. Required for SWOT ext | `api_key` | — | HydroWeb API key (or env var) | | `raw_pld_path` | — | Optional path to existing PLD zip or folder; skips download | | `keep_raw_pld` | `false` | Retain raw PLD zip and temp folder after subset creation | +| `continent_codes` | — | Optional list of continent-tile suffixes (e.g. `["AS", "AU"]`) identifying which full-schema PLD files to use for backfilling `res_id`. Inspect your downloaded PLD folder (`aux/PLD/PLD_temp/...`) to see which ones are actually present. If omitted, all full-schema tiles found are used. | + +PLD downloads include a global `_light` file (lake matching only, no `res_id`) alongside full-schema per-continent tiles (which additionally carry `res_id` — a cross-reference to external reservoir databases like GeoDAR, present only for PLD lakes classified as man-made reservoirs). HydroEO uses the `_light` file for coverage (guaranteed complete, avoids silently missing a continent your PLD download didn't include) and backfills `res_id` from whichever tile files are present, matched by `lake_id`. +If no `_light` file is present at all, HydroEO falls back to using the tile files directly for coverage too, with a warning that this risks missing continents the download didn't include. The PLD subset is written to `{main_dir}/aux/PLD/PLD_subset.gpkg` with QA/QC files `present_in_pld.gpkg` (matched) and `missing_in_pld.gpkg` (unmatched). @@ -170,6 +179,12 @@ hydroeo fetch cop-dem \ # Add quality layers: --dataset "DEM30,WBM,EDM" (EDM, FLM, HEM, WBM also available) ``` -Credentials can also be set as environment variables (`EARTHDATA_USERNAME`, `EARTHDATA_PASSWORD`, `CREODIAS_USERNAME`, `CREODIAS_PASSWORD`, `CDSE_USERNAME`, `CDSE_PASSWORD`). +Credentials can also be set as environment variables. Note the standalone `fetch` CLI commands use different variable names than the config-driven `Project` workflow above for Earthdata specifically: + +| Command | Environment variables | +| --- | --- | +| `fetch swot-lake` / `swot-raster` / `swot-pixc` | `EARTHACCESS_USERNAME`, `EARTHACCESS_PASSWORD` | +| `fetch sentinel` | `CREODIAS_USERNAME`, `CREODIAS_PASSWORD` | +| `fetch cop-dem` | `CDSE_USERNAME`, `CDSE_PASSWORD` | -> **Note:** ICESat-2, Sentinel-3, and Sentinel-6 require reservoir polygons for spatial filtering. Using them with `download: true` in a non-reservoirs project emits a `UserWarning`. \ No newline at end of file +> **Note:** ICESat-2, Sentinel-3, and Sentinel-6 require reservoir or river targets for spatial filtering. Configuring them with `download: true` while neither a `reservoirs` nor a `rivers` section is present emits a `UserWarning` — they no longer require reservoirs specifically, since ICESat-2/Sentinel-3/6 also support rivers directly (see [rivers.md](rivers.md)). \ No newline at end of file diff --git a/tests/unit/test_hydroweb.py b/tests/unit/test_hydroweb.py index 9fd555a..8c03bd9 100644 --- a/tests/unit/test_hydroweb.py +++ b/tests/unit/test_hydroweb.py @@ -5,7 +5,7 @@ import geopandas as gpd import pandas as pd import pytest -from shapely.geometry import Polygon +from shapely.geometry import Point, Polygon from HydroEO.downloaders import hydroweb @@ -59,3 +59,103 @@ def test_download_pld_keeps_lakes_that_extend_past_bbox_edge(tmp_path): "lake_edge (which pokes past the bbox edge) was incorrectly dropped -- " "the within()-vs-intersects() bug regressed" ) + + +@pytest.mark.unit +def test_download_pld_backfills_res_id_from_tile_files(tmp_path, caplog): + """When a '_light' file (lake_id + geometry only) is present alongside + full-schema continent-tile files, coverage should come from '_light' + (guaranteed complete) while res_id gets backfilled per-lake_id from + whichever tiles are present -- lakes whose continent tile isn't + available should end up with res_id left NA, plus a warning naming how + many, rather than silently missing that information.""" + import logging + + extracted_dir = tmp_path / "pld_extracted" + extracted_dir.mkdir() + (extracted_dir / "SWOT_LakeDatabase_light.gpkg").touch() + (extracted_dir / "SWOT_LakeDatabase_AS.gpkg").touch() + + bounds = [0, 0, 3, 3] + + # Light file: covers everything (lake_id 1 and 2), no res_id column. + light_gdf = gpd.GeoDataFrame( + {"lake_id": [1, 2]}, + geometry=[Point(1, 1), Point(2, 2)], + crs="EPSG:4326", + ) + # AS tile: only has lake 1 (lake 2's continent tile isn't in this + # download), but does carry res_id. + as_tile_gdf = gpd.GeoDataFrame( + {"lake_id": [1], "res_id": [9001]}, + geometry=[Point(1, 1)], + crs="EPSG:4326", + ) + + def fake_read_file(filepath, *args, **kwargs): + if "light" in str(filepath).lower(): + return light_gdf + return as_tile_gdf + + with ( + patch.object(gpd, "read_file", side_effect=fake_read_file), + caplog.at_level(logging.WARNING), + ): + hydroweb.download_PLD( + download_dir=str(tmp_path / "output"), + bounds=bounds, + raw_pld_path=str(extracted_dir), + keep_raw=True, + ) + + result = gpd.read_file(tmp_path / "output" / "PLD_subset.gpkg") + result = result.set_index("lake_id") + + assert result.loc[1, "res_id"] == 9001.0, "lake 1's res_id should be backfilled from the AS tile" + assert pd.isna(result.loc[2, "res_id"]), "lake 2's res_id should stay NA (no matching tile)" + assert "1 of 2 PLD lake(s)" in caplog.text + + +@pytest.mark.unit +def test_download_pld_continent_codes_filters_backfill_tiles(tmp_path, caplog): + """hydroweb.continent_codes should restrict which tile files are used + for res_id backfill, and warn if a requested code has no matching file + among what was actually downloaded.""" + import logging + + extracted_dir = tmp_path / "pld_extracted" + extracted_dir.mkdir() + (extracted_dir / "SWOT_LakeDatabase_light.gpkg").touch() + (extracted_dir / "SWOT_LakeDatabase_AS.gpkg").touch() + + bounds = [0, 0, 3, 3] + light_gdf = gpd.GeoDataFrame( + {"lake_id": [1]}, geometry=[Point(1, 1)], crs="EPSG:4326" + ) + as_tile_gdf = gpd.GeoDataFrame( + {"lake_id": [1], "res_id": [9001]}, geometry=[Point(1, 1)], crs="EPSG:4326" + ) + + def fake_read_file(filepath, *args, **kwargs): + return light_gdf if "light" in str(filepath).lower() else as_tile_gdf + + with ( + patch.object(gpd, "read_file", side_effect=fake_read_file), + caplog.at_level(logging.WARNING), + ): + hydroweb.download_PLD( + download_dir=str(tmp_path / "output"), + bounds=bounds, + raw_pld_path=str(extracted_dir), + keep_raw=True, + continent_codes=["AU"], # doesn't match the AS tile that's present + ) + + result = gpd.read_file(tmp_path / "output" / "PLD_subset.gpkg").set_index("lake_id") + assert pd.isna(result.loc[1, "res_id"]), ( + "res_id should NOT be backfilled from the AS tile since continent_codes " + "only requested AU" + ) + assert "['AU']" in caplog.text or "AU" in caplog.text + assert "no matching tile file" in caplog.text + \ No newline at end of file From 3da717fb135841141954f047b67af00137275f83 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 09:59:29 +0200 Subject: [PATCH 13/19] SWOT fixes Comparison of lake / res id failed due to str / int mismatch. Converted to int in _reser_voir_init. --- HydroEO/flows/_reservoir_init.py | 6 +++++- HydroEO/project.py | 5 +++++ tests/unit/test_flows.py | 33 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 0c2d03c..04b1804 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -6,6 +6,7 @@ import os import geopandas as gpd +import pandas as pd from HydroEO.downloaders import hydroweb @@ -102,7 +103,10 @@ def _assign_pld_id(prj: "Project") -> None: ) joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) joined_gdf = joined_gdf.drop(columns=["index_right", "index_left"], errors="ignore") - + # Make sure lake_id is numeric for future comparison, set missing values to -9999 + joined_gdf["prior_lake_id"] = pd.to_numeric( + joined_gdf["prior_lake_id"], errors="coerce" + ) joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 prj.reservoirs.gdf = joined_gdf diff --git a/HydroEO/project.py b/HydroEO/project.py index 0035855..2b38b6c 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -84,6 +84,11 @@ def __post_init__(self): if "raw_pld_path" in hydroweb_cfg: self.dirs["pld_raw"] = general.normalize_path(hydroweb_cfg["raw_pld_path"]) self.keep_raw_pld = hydroweb_cfg.get("keep_raw_pld", False) + # Continent-tile suffixes (e.g. ["AS", "AU"]) to look for when + # backfilling res_id from the full-schema PLD tiles, in addition to + # the "_light" file (lake_id + geometry only, no res_id) used for + # guaranteed global coverage. T + self.pld_continent_codes = hydroweb_cfg.get("continent_codes") # Set SWORD database paths and configuration sword_db_cfg = self.config.get("sword_db", {}) diff --git a/tests/unit/test_flows.py b/tests/unit/test_flows.py index 0b37d4d..338dc34 100644 --- a/tests/unit/test_flows.py +++ b/tests/unit/test_flows.py @@ -1016,3 +1016,36 @@ def test_assign_pld_id_drops_index_right_column(mock_project_reservoirs, tmp_pat pld_gdf.to_crs("EPSG:3857"), distance_col="dist_to_pld", ) + + +@pytest.mark.unit +def test_assign_pld_id_coerces_string_lake_id_to_numeric(mock_project_reservoirs, tmp_path): + """Regression test: some real PLD files (e.g. the '_light' product) + store lake_id as text rather than integer, which round-trips through + GPKG as a genuine Python str. Without coercion, prior_lake_id ends up + as an object-dtype column mixing strings (matched reservoirs) with the + integer -9999 sentinel (unmatched reservoirs) -- and _flag_missing_priors's + 'prior_lake_id > 0' / '< 0' comparisons raise TypeError: '>' not + supported between instances of 'str' and 'int'. + """ + pld_path = Path(mock_project_reservoirs.dirs["pld"]) + pld_path.parent.mkdir(parents=True, exist_ok=True) + # lake_id stored as text, matching the confirmed round-trip behavior of + # some real PLD files + pld_gdf = gpd.GeoDataFrame( + {"lake_id": ["1001", "1002"]}, + geometry=[Point(0.5, 0.5), Point(1.5, 1.5)], + crs="EPSG:4326", + ) + pld_gdf.to_file(pld_path, driver="GPKG") + + flows._assign_pld_id(mock_project_reservoirs) + + prior_lake_id = mock_project_reservoirs.reservoirs.gdf["prior_lake_id"] + assert pd.api.types.is_numeric_dtype(prior_lake_id), ( + f"prior_lake_id should be numeric, got dtype {prior_lake_id.dtype}" + ) + # this is exactly the comparison that raised TypeError before the fix + present = mock_project_reservoirs.reservoirs.gdf.loc[prior_lake_id > 0] + missing = mock_project_reservoirs.reservoirs.gdf.loc[prior_lake_id < 0] + assert len(present) + len(missing) == len(mock_project_reservoirs.reservoirs.gdf) From 9090ac45d8ab1a01debd430d839d72748bce0bd1 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 10:16:12 +0200 Subject: [PATCH 14/19] Delete unnecessary comments --- HydroEO/flows/_reservoir_pipeline.py | 2 +- HydroEO/flows/_river_pipeline.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/HydroEO/flows/_reservoir_pipeline.py b/HydroEO/flows/_reservoir_pipeline.py index 4fbfdc1..24b0b7f 100644 --- a/HydroEO/flows/_reservoir_pipeline.py +++ b/HydroEO/flows/_reservoir_pipeline.py @@ -351,4 +351,4 @@ def _export_cleaned_to_dfs0(prj: "Project") -> None: product, id, exc, - ) \ No newline at end of file + ) diff --git a/HydroEO/flows/_river_pipeline.py b/HydroEO/flows/_river_pipeline.py index 0eb6eaa..6a91c84 100644 --- a/HydroEO/flows/_river_pipeline.py +++ b/HydroEO/flows/_river_pipeline.py @@ -360,11 +360,6 @@ def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) - gdf.to_file(os.path.join(dst_dir, "swot.gpkg"), driver="GPKG") -# ============================================================================ -# RESERVOIRS: Timeseries Processing -# ============================================================================ - - def create_rivers_timeseries(prj: "Project") -> None: """Extract, clean, and merge timeseries for river targets (nodes/reaches). From 3a99d6d6fa0058913f99d0290e828de041297d52 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 09:59:29 +0200 Subject: [PATCH 15/19] SWOT fixes SWOT fixes and delete unnecessary comments Comparison of lake / res id failed due to str / int mismatch. Converted to int in _reser_voir_init. --- HydroEO/flows/_reservoir_init.py | 6 ++++- HydroEO/flows/_reservoir_pipeline.py | 2 +- HydroEO/flows/_river_pipeline.py | 5 ----- HydroEO/project.py | 5 +++++ tests/unit/test_flows.py | 33 ++++++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 0c2d03c..04b1804 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -6,6 +6,7 @@ import os import geopandas as gpd +import pandas as pd from HydroEO.downloaders import hydroweb @@ -102,7 +103,10 @@ def _assign_pld_id(prj: "Project") -> None: ) joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) joined_gdf = joined_gdf.drop(columns=["index_right", "index_left"], errors="ignore") - + # Make sure lake_id is numeric for future comparison, set missing values to -9999 + joined_gdf["prior_lake_id"] = pd.to_numeric( + joined_gdf["prior_lake_id"], errors="coerce" + ) joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 prj.reservoirs.gdf = joined_gdf diff --git a/HydroEO/flows/_reservoir_pipeline.py b/HydroEO/flows/_reservoir_pipeline.py index 4fbfdc1..24b0b7f 100644 --- a/HydroEO/flows/_reservoir_pipeline.py +++ b/HydroEO/flows/_reservoir_pipeline.py @@ -351,4 +351,4 @@ def _export_cleaned_to_dfs0(prj: "Project") -> None: product, id, exc, - ) \ No newline at end of file + ) diff --git a/HydroEO/flows/_river_pipeline.py b/HydroEO/flows/_river_pipeline.py index 0eb6eaa..6a91c84 100644 --- a/HydroEO/flows/_river_pipeline.py +++ b/HydroEO/flows/_river_pipeline.py @@ -360,11 +360,6 @@ def _extract_rivers_swot_observations(prj: "Project", overwrite: bool = False) - gdf.to_file(os.path.join(dst_dir, "swot.gpkg"), driver="GPKG") -# ============================================================================ -# RESERVOIRS: Timeseries Processing -# ============================================================================ - - def create_rivers_timeseries(prj: "Project") -> None: """Extract, clean, and merge timeseries for river targets (nodes/reaches). diff --git a/HydroEO/project.py b/HydroEO/project.py index 0035855..2b38b6c 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -84,6 +84,11 @@ def __post_init__(self): if "raw_pld_path" in hydroweb_cfg: self.dirs["pld_raw"] = general.normalize_path(hydroweb_cfg["raw_pld_path"]) self.keep_raw_pld = hydroweb_cfg.get("keep_raw_pld", False) + # Continent-tile suffixes (e.g. ["AS", "AU"]) to look for when + # backfilling res_id from the full-schema PLD tiles, in addition to + # the "_light" file (lake_id + geometry only, no res_id) used for + # guaranteed global coverage. T + self.pld_continent_codes = hydroweb_cfg.get("continent_codes") # Set SWORD database paths and configuration sword_db_cfg = self.config.get("sword_db", {}) diff --git a/tests/unit/test_flows.py b/tests/unit/test_flows.py index 0b37d4d..338dc34 100644 --- a/tests/unit/test_flows.py +++ b/tests/unit/test_flows.py @@ -1016,3 +1016,36 @@ def test_assign_pld_id_drops_index_right_column(mock_project_reservoirs, tmp_pat pld_gdf.to_crs("EPSG:3857"), distance_col="dist_to_pld", ) + + +@pytest.mark.unit +def test_assign_pld_id_coerces_string_lake_id_to_numeric(mock_project_reservoirs, tmp_path): + """Regression test: some real PLD files (e.g. the '_light' product) + store lake_id as text rather than integer, which round-trips through + GPKG as a genuine Python str. Without coercion, prior_lake_id ends up + as an object-dtype column mixing strings (matched reservoirs) with the + integer -9999 sentinel (unmatched reservoirs) -- and _flag_missing_priors's + 'prior_lake_id > 0' / '< 0' comparisons raise TypeError: '>' not + supported between instances of 'str' and 'int'. + """ + pld_path = Path(mock_project_reservoirs.dirs["pld"]) + pld_path.parent.mkdir(parents=True, exist_ok=True) + # lake_id stored as text, matching the confirmed round-trip behavior of + # some real PLD files + pld_gdf = gpd.GeoDataFrame( + {"lake_id": ["1001", "1002"]}, + geometry=[Point(0.5, 0.5), Point(1.5, 1.5)], + crs="EPSG:4326", + ) + pld_gdf.to_file(pld_path, driver="GPKG") + + flows._assign_pld_id(mock_project_reservoirs) + + prior_lake_id = mock_project_reservoirs.reservoirs.gdf["prior_lake_id"] + assert pd.api.types.is_numeric_dtype(prior_lake_id), ( + f"prior_lake_id should be numeric, got dtype {prior_lake_id.dtype}" + ) + # this is exactly the comparison that raised TypeError before the fix + present = mock_project_reservoirs.reservoirs.gdf.loc[prior_lake_id > 0] + missing = mock_project_reservoirs.reservoirs.gdf.loc[prior_lake_id < 0] + assert len(present) + len(missing) == len(mock_project_reservoirs.reservoirs.gdf) From 4e5c0792df44acfaf0d09c9d1592fa5347b6eb50 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 10:30:11 +0200 Subject: [PATCH 16/19] Clean up of constants repeated across codebase CLeaned up where and how default processing parameters are defined (elev_min and max, and max_threshold) - previously defined 3 places and hardcoded in function, now called as constant and defined only in 1 place unless provided by users. --- HydroEO/constants.py | 8 ++++++++ HydroEO/flows/_clean_engine.py | 8 +++++++- HydroEO/flows/_constants.py | 13 +++++++++++++ HydroEO/project.py | 23 ++++++++++++++++++----- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/HydroEO/constants.py b/HydroEO/constants.py index f4efac9..189bdf5 100644 --- a/HydroEO/constants.py +++ b/HydroEO/constants.py @@ -88,6 +88,14 @@ SUPPORTED_CLEAN_FILTERS = ["elevation", "MAD", "daily_mean", "hampel", "rolling_median"] +# Project.__sat_init and HydroEO.flows._clean_engine both +# reference these constants +DEFAULT_PROCESSING_FILTERS = ["elevation", "MAD"] +DEFAULT_ELEVATION_MIN_M = 0.0 +DEFAULT_ELEVATION_MAX_M = 8000.0 +DEFAULT_MAD_THRESHOLD = 5.0 + + # ============================================================================ # Mission Defaults # ============================================================================ diff --git a/HydroEO/flows/_clean_engine.py b/HydroEO/flows/_clean_engine.py index 817614d..dd18b3f 100644 --- a/HydroEO/flows/_clean_engine.py +++ b/HydroEO/flows/_clean_engine.py @@ -9,7 +9,13 @@ from tqdm import tqdm from HydroEO.utils import general, timeseries -from ._constants import PRODUCT_TIMESERIES_KEYS +from ._constants import ( + PRODUCT_TIMESERIES_KEYS, + DEFAULT_PROCESSING_FILTERS, + DEFAULT_ELEVATION_MIN_M, + DEFAULT_ELEVATION_MAX_M, + DEFAULT_MAD_THRESHOLD, +) from ._run_config import _get_target_ids from ._summaries import _load_product_timeseries diff --git a/HydroEO/flows/_constants.py b/HydroEO/flows/_constants.py index 8e936af..52be028 100644 --- a/HydroEO/flows/_constants.py +++ b/HydroEO/flows/_constants.py @@ -2,6 +2,19 @@ Shared constants for the merge/clean pipeline (both reservoirs and rivers). """ +# Default cleaning-filter parameters, used by _clean_engine.py as the +# fallback whenever a product's processing_options doesn't specify these +# explicitly. Note: HydroEO.constants.MISSION_DEFAULTS also defines these +# same values (per-mission, for initial config parsing in project.py) -- +# the two aren't wired together, so a change here won't automatically +# propagate there or vice versa. Worth consolidating fully if that +# divergence risk ever matters in practice. +DEFAULT_PROCESSING_FILTERS = ["elevation", "MAD"] +DEFAULT_ELEVATION_MIN_M = 0.0 +DEFAULT_ELEVATION_MAX_M = 8000.0 +DEFAULT_MAD_THRESHOLD = 5.0 + + PRODUCT_TIMESERIES_KEYS = { "sentinel3": dict( lat_key="lat", lon_key="lon", pass_key="file_name", diff --git a/HydroEO/project.py b/HydroEO/project.py index 2b38b6c..b4a3803 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -16,7 +16,13 @@ from HydroEO.satellites.swot.raster import download_raster from HydroEO.satellites.swot.pixc import download_pixc from HydroEO.utils import general -from HydroEO.constants import MISSION_DEFAULTS +from HydroEO.constants import ( + MISSION_DEFAULTS, + DEFAULT_PROCESSING_FILTERS, + DEFAULT_ELEVATION_MIN_M, + DEFAULT_ELEVATION_MAX_M, + DEFAULT_MAD_THRESHOLD, +) from HydroEO.validation import validate_config logger = logging.getLogger(__name__) @@ -412,11 +418,18 @@ def __sat_init(self, name: str): self.processing_options[name] = { "processing_filters": self.config[name].get( - "processing_filters", ["elevation", "MAD"] + "processing_filters", DEFAULT_PROCESSING_FILTERS ), - "elevation_min_m": self.config[name].get("elevation_min_m", 0.0), - "elevation_max_m": self.config[name].get("elevation_max_m", 8000.0), - "mad_threshold": self.config[name].get("mad_threshold", 5.0), + "elevation_min_m": self.config[name].get( + "elevation_min_m", DEFAULT_ELEVATION_MIN_M + ), + "elevation_max_m": self.config[name].get( + "elevation_max_m", DEFAULT_ELEVATION_MAX_M + ), + "mad_threshold": self.config[name].get( + "mad_threshold", DEFAULT_MAD_THRESHOLD + ), + } def _apply_optional_defaults(self): From 1f922589d97616b9f7519fb94c13ce3fc329402e Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 10:40:16 +0200 Subject: [PATCH 17/19] Fix --- HydroEO/constants.py | 32 ++++++++++++++++---------------- HydroEO/flows/__init__.py | 5 +++++ HydroEO/flows/_clean_engine.py | 18 +++++++++++------- HydroEO/flows/_constants.py | 16 +++++++--------- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/HydroEO/constants.py b/HydroEO/constants.py index 189bdf5..28d18df 100644 --- a/HydroEO/constants.py +++ b/HydroEO/constants.py @@ -108,10 +108,10 @@ "exclude_obs_id_values": ["no_data"], "hydrocron_fields": SWOT_DEFAULT_HYDROCRON_FIELDS, "quality_filters": SWOT_DEFAULT_QUALITY_FILTERS, - "processing_filters": ["elevation", "MAD"], - "elevation_min_m": 0.0, - "elevation_max_m": 8000.0, - "mad_threshold": 5.0, + "processing_filters": DEFAULT_PROCESSING_FILTERS, + "elevation_min_m": DEFAULT_ELEVATION_MIN_M, + "elevation_max_m": DEFAULT_ELEVATION_MAX_M, + "mad_threshold": DEFAULT_MAD_THRESHOLD, }, "icesat2": { "download": False, @@ -119,10 +119,10 @@ "atl13_fields": ICESAT2_DEFAULT_FIELDS, "atl13": {"pass_invalid": False, "beams": [], "spots": []}, "track_keys": ICESAT2_SUPPORTED_TRACK_KEYS, - "processing_filters": ["elevation", "MAD"], - "elevation_min_m": 0.0, - "elevation_max_m": 8000.0, - "mad_threshold": 5.0, + "processing_filters": DEFAULT_PROCESSING_FILTERS, + "elevation_min_m": DEFAULT_ELEVATION_MIN_M, + "elevation_max_m": DEFAULT_ELEVATION_MAX_M, + "mad_threshold": DEFAULT_MAD_THRESHOLD, }, "sentinel3": { "download": False, @@ -130,10 +130,10 @@ "subset_file_id": "enhanced_measurement.nc", "sigma0_max": 1e5, "download_threads": 1, - "processing_filters": ["elevation", "MAD"], - "elevation_min_m": 0.0, - "elevation_max_m": 8000.0, - "mad_threshold": 5.0, + "processing_filters": DEFAULT_PROCESSING_FILTERS, + "elevation_min_m": DEFAULT_ELEVATION_MIN_M, + "elevation_max_m": DEFAULT_ELEVATION_MAX_M, + "mad_threshold": DEFAULT_MAD_THRESHOLD, }, "sentinel6": { "download": False, @@ -141,9 +141,9 @@ "subset_file_id": "enhanced_measurement.nc", "sigma0_max": 1e5, "download_threads": 1, - "processing_filters": ["elevation", "MAD"], - "elevation_min_m": 0.0, - "elevation_max_m": 8000.0, - "mad_threshold": 5.0, + "processing_filters": DEFAULT_PROCESSING_FILTERS, + "elevation_min_m": DEFAULT_ELEVATION_MIN_M, + "elevation_max_m": DEFAULT_ELEVATION_MAX_M, + "mad_threshold": DEFAULT_MAD_THRESHOLD, }, } diff --git a/HydroEO/flows/__init__.py b/HydroEO/flows/__init__.py index 4224aab..5637948 100644 --- a/HydroEO/flows/__init__.py +++ b/HydroEO/flows/__init__.py @@ -72,7 +72,12 @@ DEFAULT_RESERVOIR_MERGING_OPTIONS as DEFAULT_RESERVOIR_MERGING_OPTIONS, DEFAULT_RIVER_MERGING_OPTIONS as DEFAULT_RIVER_MERGING_OPTIONS, PRODUCT_TIMESERIES_KEYS as PRODUCT_TIMESERIES_KEYS, + DEFAULT_PROCESSING_FILTERS as DEFAULT_PROCESSING_FILTERS, + DEFAULT_ELEVATION_MIN_M as DEFAULT_ELEVATION_MIN_M, + DEFAULT_ELEVATION_MAX_M as DEFAULT_ELEVATION_MAX_M, + DEFAULT_MAD_THRESHOLD as DEFAULT_MAD_THRESHOLD, ) + from ._clean_engine import ( _clean_timeseries as _clean_timeseries, ) diff --git a/HydroEO/flows/_clean_engine.py b/HydroEO/flows/_clean_engine.py index dd18b3f..79dd479 100644 --- a/HydroEO/flows/_clean_engine.py +++ b/HydroEO/flows/_clean_engine.py @@ -55,10 +55,10 @@ def _clean_timeseries(prj: "Project", target_type: str) -> None: product_options = prj.processing_options.get( product, { - "processing_filters": ["elevation", "MAD"], - "elevation_min_m": 0.0, - "elevation_max_m": 8000.0, - "mad_threshold": 5.0, + "processing_filters": DEFAULT_PROCESSING_FILTERS, + "elevation_min_m": DEFAULT_ELEVATION_MIN_M, + "elevation_max_m": DEFAULT_ELEVATION_MAX_M, + "mad_threshold": DEFAULT_MAD_THRESHOLD, }, ) @@ -70,11 +70,15 @@ def _clean_timeseries(prj: "Project", target_type: str) -> None: ts.clean( product_options.get("processing_filters", ["elevation", "MAD"]), filter_params={ - "elevation_min_m": product_options.get("elevation_min_m", 0.0), + "elevation_min_m": product_options.get( + "elevation_min_m", DEFAULT_ELEVATION_MIN_M + ), "elevation_max_m": product_options.get( - "elevation_max_m", 8000.0 + "elevation_max_m", DEFAULT_ELEVATION_MAX_M + ), + "mad_threshold": product_options.get( + "mad_threshold", DEFAULT_MAD_THRESHOLD ), - "mad_threshold": product_options.get("mad_threshold", 5.0), }, ) diff --git a/HydroEO/flows/_constants.py b/HydroEO/flows/_constants.py index 52be028..c3f1c6f 100644 --- a/HydroEO/flows/_constants.py +++ b/HydroEO/flows/_constants.py @@ -4,15 +4,13 @@ # Default cleaning-filter parameters, used by _clean_engine.py as the # fallback whenever a product's processing_options doesn't specify these -# explicitly. Note: HydroEO.constants.MISSION_DEFAULTS also defines these -# same values (per-mission, for initial config parsing in project.py) -- -# the two aren't wired together, so a change here won't automatically -# propagate there or vice versa. Worth consolidating fully if that -# divergence risk ever matters in practice. -DEFAULT_PROCESSING_FILTERS = ["elevation", "MAD"] -DEFAULT_ELEVATION_MIN_M = 0.0 -DEFAULT_ELEVATION_MAX_M = 8000.0 -DEFAULT_MAD_THRESHOLD = 5.0 +# explicitly. Re-exported from HydroEO.constant. +from HydroEO.constants import ( + DEFAULT_PROCESSING_FILTERS as DEFAULT_PROCESSING_FILTERS, + DEFAULT_ELEVATION_MIN_M as DEFAULT_ELEVATION_MIN_M, + DEFAULT_ELEVATION_MAX_M as DEFAULT_ELEVATION_MAX_M, + DEFAULT_MAD_THRESHOLD as DEFAULT_MAD_THRESHOLD, +) PRODUCT_TIMESERIES_KEYS = { From 586d9244e63de5b1a91ccd6a0fdc2836d47dbc9b Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 13:08:27 +0200 Subject: [PATCH 18/19] Update in handling SWORD overlap - Check overlap in area - proximity allowance now an area overlap (in % not meters). - currently backward compatibility, should be added. --- HydroEO/constants.py | 8 +- HydroEO/flows/_reservoir_init.py | 100 ++++++++++++++---- HydroEO/project.py | 2 +- HydroEO/satellites/swot/preprocess.py | 18 +++- HydroEO/validation.py | 11 +- configs/reservoirs.md | 2 +- configs/reservoirs.yaml | 2 +- tests/unit/test_flows.py | 142 ++++++++++++++++++-------- tests/unit/test_timeseries.py | 69 ++++++++++++- 9 files changed, 280 insertions(+), 74 deletions(-) diff --git a/HydroEO/constants.py b/HydroEO/constants.py index 28d18df..8fd5ceb 100644 --- a/HydroEO/constants.py +++ b/HydroEO/constants.py @@ -95,7 +95,6 @@ DEFAULT_ELEVATION_MAX_M = 8000.0 DEFAULT_MAD_THRESHOLD = 5.0 - # ============================================================================ # Mission Defaults # ============================================================================ @@ -104,7 +103,10 @@ "swot": { "download": False, "process": False, - "pld_match_max_distance_m": 100.0, + # Minimum PLD-lake overlap as a percentage (0-100) of the + # reservoir's own area. Previously used pld_match_max_distance_m. + # NOTE: Note backward compatible. + "pld_match_min_overlap_pct": 10.0, "exclude_obs_id_values": ["no_data"], "hydrocron_fields": SWOT_DEFAULT_HYDROCRON_FIELDS, "quality_filters": SWOT_DEFAULT_QUALITY_FILTERS, @@ -146,4 +148,4 @@ "elevation_max_m": DEFAULT_ELEVATION_MAX_M, "mad_threshold": DEFAULT_MAD_THRESHOLD, }, -} +} \ No newline at end of file diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 04b1804..99ab78a 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -86,28 +86,83 @@ def _download_pld(prj: "Project") -> None: def _assign_pld_id(prj: "Project") -> None: - """Spatial join reservoirs with PLD to assign prior_lake_id.""" + """Spatial join reservoirs with PLD to assign prior_lake_id. + + Prefers the PLD lake with the LARGEST overlapping area over merely the + nearest one. + """ pld = gpd.read_file(prj.dirs["pld"]) + pld = pld.rename(columns={"lake_id": "prior_lake_id", "res_id": "prior_res_id"}) + if "prior_res_id" not in pld.columns: + pld["prior_res_id"] = None + + id_key = prj.reservoirs.id_key + reservoirs_local = prj.reservoirs.gdf.to_crs(prj.local_crs) + pld_local = pld.to_crs(prj.local_crs) + reservoir_areas = reservoirs_local.set_index(id_key).geometry.area + min_overlap_pct = prj.mission_options.get("swot", {}).get( + "pld_match_min_overlap_pct", 10.0 + ) - pld = pld.rename( - columns={"lake_id": "prior_lake_id", "res_id": "prior_res_id"} - ) - joined_gdf = gpd.sjoin_nearest( - prj.reservoirs.gdf.to_crs(prj.local_crs), - pld.to_crs(prj.local_crs), - how="left", - max_distance=prj.mission_options.get("swot", {}).get( - "pld_match_max_distance_m", 100 - ), - distance_col="dist_to_pld", + # Match by largest overlapping area, computed via a true geometric + # intersection (not just "does it intersect" or "how far apart are the + # boundaries"). + overlap = gpd.overlay( + reservoirs_local[[id_key, "geometry"]], + pld_local[["prior_lake_id", "prior_res_id", "geometry"]], + how="intersection", + keep_geom_type=False, ) - joined_gdf = joined_gdf.to_crs(prj.reservoirs.gdf.crs) - joined_gdf = joined_gdf.drop(columns=["index_right", "index_left"], errors="ignore") - # Make sure lake_id is numeric for future comparison, set missing values to -9999 + if len(overlap) > 0: + overlap["dist_to_pld"] = 0.0 + overlap["pld_match_method"] = "overlap" + overlap["_overlap_area"] = overlap.geometry.area + matches = ( + overlap.sort_values("_overlap_area", ascending=False) + .drop_duplicates(subset=id_key, keep="first") + ) + + # Overlap area as a percentage of the RESERVOIR's own area. + matches["_reservoir_area"] = matches[id_key].map(reservoir_areas) + matches["_overlap_pct"] = ( + matches["_overlap_area"] / matches["_reservoir_area"] * 100 + ) + low_overlap = matches.loc[matches["_overlap_pct"] < min_overlap_pct] + if len(low_overlap) > 0: + logger.warning( + "%d reservoir(s) matched a PLD lake covering less than " + "%.0f%% of the reservoir's own area: %s -- still using " + "this match (it's the best candidate available), but " + "verify it's correct rather than a small, unrelated lake " + "that happens to clip the reservoir's edge.", + len(low_overlap), + min_overlap_pct, + ", ".join( + f"{row[id_key]} ({row['_overlap_pct']:.1f}%)" + for _, row in low_overlap.iterrows() + ), + ) + + matches = matches.drop( + columns=["_overlap_area", "_overlap_pct", "_reservoir_area", "geometry"] + ) + else: + matches = pd.DataFrame( + columns=[id_key, "prior_lake_id", "prior_res_id", "dist_to_pld", "pld_match_method"] + ) + + joined_gdf = prj.reservoirs.gdf.merge(matches, on=id_key, how="left") + + # Real PLD files may store lake_id as text rather than integer (e.g. + # the '_light' product) -- coerce here, once, rather than leaving every + # downstream numeric comparison (_flag_missing_priors's "> 0" / "< 0", + # and the equivalent check during SWOT extraction) vulnerable to a + # TypeError comparing str > int. joined_gdf["prior_lake_id"] = pd.to_numeric( joined_gdf["prior_lake_id"], errors="coerce" ) joined_gdf.loc[joined_gdf.prior_lake_id.isnull(), "prior_lake_id"] = -9999 + joined_gdf.loc[joined_gdf["prior_lake_id"] == -9999, "pld_match_method"] = "unmatched" prj.reservoirs.gdf = joined_gdf @@ -132,13 +187,16 @@ def _flag_missing_priors(prj: "Project") -> None: len(present), len(missing), ) + if len(missing) > 0: logger.warning( - "%d reservoir(s) not matched to any Prior Lake Database (PLD) lake " - "within pld_match_max_distance_m: %s (see aux/PLD/missing_in_pld.gpkg). " - "SWOT Lake SP cannot report observations for these. " - "Consider increasing pld_match_max_distance_m if it's a " - "near miss.", + "%d reservoir(s) have no geometrically overlapping Prior Lake " + "Database (PLD) lake: %s (see aux/PLD/missing_in_pld.gpkg). SWOT " + "Lake SP cannot report observations for these regardless of " + "download success. " + "Check the geometry in missing_in_pld.gpkg against your source " + "reservoir polygon before proceeding, or exclude them from this " + "project if they're SWOT-only.", len(missing), ", ".join(str(v) for v in missing[prj.reservoirs.id_key]), - ) \ No newline at end of file + ) diff --git a/HydroEO/project.py b/HydroEO/project.py index b4a3803..b5715c0 100644 --- a/HydroEO/project.py +++ b/HydroEO/project.py @@ -412,7 +412,7 @@ def __sat_init(self, name: str): "short_name", "download_threads", "exclude_obs_id_values", - "pld_match_max_distance_m", + "pld_match_min_overlap_pct", ] } diff --git a/HydroEO/satellites/swot/preprocess.py b/HydroEO/satellites/swot/preprocess.py index 3a86a61..b330b35 100644 --- a/HydroEO/satellites/swot/preprocess.py +++ b/HydroEO/satellites/swot/preprocess.py @@ -142,12 +142,16 @@ def extract_observations( excluded_obs_ids = set(exclude_obs_id_values or ["no_data"]) empty_ids = [] + missing_from_pld_ids = [] # now loop through the ids in the features gdf to extract the observations from the main one for _, feat in tqdm( features.iterrows(), total=len(features), desc="Extracting SWOT Lake SP product" ): dl_id = str(feat[id_key]) - if not np.isnan(feat["prior_lake_id"]): + # prior_lake_id is -9999 (not NaN) for reservoirs unmatched to the PLD -- + # see _assign_pld_id, and _flag_missing_priors's own "> 0" convention for + # "present in PLD" that this mirrors. + if feat["prior_lake_id"] > 0: lake_id = str(int(feat["prior_lake_id"])) # filter observations to keep only the ones associated with this lake/reservoir @@ -186,6 +190,18 @@ def extract_observations( observations.to_file(dst_path) else: empty_ids.append(dl_id) + else: + missing_from_pld_ids.append(dl_id) + + if missing_from_pld_ids: + logger.warning( + "SWOT timeseries unavailable for: %s -- no geometrically " + "overlapping Prior Lake Database (PLD) lake (see " + "aux/PLD/missing_in_pld.gpkg). SWOT Lake SP cannot report " + "observations for a reservoir the PLD doesn't have a matching entry " + "for, regardless of date range or download success.", + ", ".join(missing_from_pld_ids), + ) if processed_log_path: if overwrite: diff --git a/HydroEO/validation.py b/HydroEO/validation.py index e6366e6..37a5b43 100644 --- a/HydroEO/validation.py +++ b/HydroEO/validation.py @@ -537,10 +537,13 @@ def _is_enabled(section_cfg) -> bool: issues.append(f"'{mission}.mad_threshold' must be a positive number.") if mission == "swot": - max_distance = mission_cfg.get("pld_match_max_distance_m", 100.0) - if not isinstance(max_distance, (int, float)) or max_distance < 0: + min_overlap_pct = mission_cfg.get("pld_match_min_overlap_pct", 10.0) + if not isinstance(min_overlap_pct, (int, float)) or not ( + 0 <= min_overlap_pct <= 100 + ): issues.append( - "'swot.pld_match_max_distance_m' must be a non-negative number." + "'swot.pld_match_min_overlap_pct' must be a number " + "between 0 and 100." ) excluded_obs = mission_cfg.get("exclude_obs_id_values", ["no_data"]) @@ -688,4 +691,4 @@ def _is_enabled(section_cfg) -> bool: if issues: raise ValueError("Invalid configuration:\n - " + "\n - ".join(issues)) - return True + return True \ No newline at end of file diff --git a/configs/reservoirs.md b/configs/reservoirs.md index 66f34b7..6e82354 100644 --- a/configs/reservoirs.md +++ b/configs/reservoirs.md @@ -41,7 +41,7 @@ Start from [`configs/reservoirs.yaml`](reservoirs.yaml). | `download` | `false`* | Download SWOT Lake SP granules | | `process` | `false`* | Extract observations for each reservoir | | `startdate` / `enddate` | project dates | Per-satellite date override | -| `pld_match_max_distance_m` | `100.0` | Max nearest-neighbour distance to match reservoirs to PLD lakes | +| `pld_match_min_overlap_pct` | `10.0` | Minimum PLD-lake overlap as a percentage (0-100) of the reservoir's own area. Matching itself requires genuine geometric overlap with no distance-based fallback; a reservoir with zero overlap to any PLD lake is left unmatched (see `aux/PLD/missing_in_pld.gpkg`). Below this percentage, the best-available match is still used, but a warning names it as low-confidence so you can verify it isn't a small, unrelated lake clipping the reservoir's edge. | | `exclude_obs_id_values` | `["no_data"]` | SWOT `obs_id` values to drop during extraction | | `processing_filters` | `["elevation", "MAD"]` | Ordered list of cleaning filters (see [Cleaning filters](#cleaning-filters)) | | `elevation_min_m` | `0.0` | Lower bound for elevation filter (metres) | diff --git a/configs/reservoirs.yaml b/configs/reservoirs.yaml index 111f9f5..7ef0390 100644 --- a/configs/reservoirs.yaml +++ b/configs/reservoirs.yaml @@ -87,7 +87,7 @@ sentinel6: # ────────────────────────────────────────────────────────────────────────────── # swot: -# pld_match_max_distance_m: 100.0 # max distance to match reservoirs to PLD lakes +# pld_match_min_overlap_pct: 10.0 # minimum PLD-lake overlap as a percentage of the reservoir's own area # exclude_obs_id_values: ["no_data"] # SWOT obs_id values to drop during extraction # processing_filters: ["elevation", "MAD"] # elevation | MAD | daily_mean | hampel | rolling_median # elevation_min_m: 0.0 diff --git a/tests/unit/test_flows.py b/tests/unit/test_flows.py index 338dc34..8b54eff 100644 --- a/tests/unit/test_flows.py +++ b/tests/unit/test_flows.py @@ -402,28 +402,6 @@ def test_download_reservoirs_dispatches_swot(mock_project_reservoirs): mock_sent.assert_not_called() - -@pytest.mark.unit -def test_download_reservoirs_dispatches_all_missions(mock_project_reservoirs): - """download_reservoirs dispatches all enabled satellite missions.""" - mock_project_reservoirs.to_download = [ - "swot", - "icesat2", - "sentinel3", - "sentinel6", - ] - - with ( - patch.object(flows._reservoir_download, "_download_reservoirs_swot") as mock_swot, - patch.object(flows._reservoir_download, "_download_reservoirs_icesat2") as mock_ice, - patch.object(flows._reservoir_download, "_download_reservoirs_sentinel") as mock_sent, - ): - flows.download_reservoirs(mock_project_reservoirs) - - mock_swot.assert_called_once() - mock_ice.assert_called_once() - assert mock_sent.call_count == 2 # sentinel3 and sentinel6 - @pytest.mark.unit def test_download_reservoirs_swot_skips_when_none_matched_pld( mock_project_reservoirs, caplog @@ -474,6 +452,27 @@ def test_download_reservoirs_swot_proceeds_when_some_matched_pld( mock_download.assert_called_once() +@pytest.mark.unit +def test_download_reservoirs_dispatches_all_missions(mock_project_reservoirs): + """download_reservoirs dispatches all enabled satellite missions.""" + mock_project_reservoirs.to_download = [ + "swot", + "icesat2", + "sentinel3", + "sentinel6", + ] + + with ( + patch.object(flows._reservoir_download, "_download_reservoirs_swot") as mock_swot, + patch.object(flows._reservoir_download, "_download_reservoirs_icesat2") as mock_ice, + patch.object(flows._reservoir_download, "_download_reservoirs_sentinel") as mock_sent, + ): + flows.download_reservoirs(mock_project_reservoirs) + + mock_swot.assert_called_once() + mock_ice.assert_called_once() + assert mock_sent.call_count == 2 # sentinel3 and sentinel6 + @pytest.mark.unit def test_download_reservoirs_skips_disabled_missions(mock_project_reservoirs): @@ -961,31 +960,94 @@ def test_download_pld_downloads_when_missing(mock_project_reservoirs, caplog): @pytest.mark.unit -def test_assign_pld_id_updates_gdf(mock_project_reservoirs): - """_assign_pld_id joins PLD data and updates reservoirs gdf.""" - # Mock PLD GeoDataFrame +def test_assign_pld_id_updates_gdf(mock_project_reservoirs, tmp_path): + """_assign_pld_id joins PLD data and updates reservoirs gdf, preferring + the overlap-based match (not sjoin_nearest) when a PLD lake genuinely + overlaps the reservoir -- matching how real PLD lakes and reservoirs + are both areal (polygon) features, not points.""" + pld_path = Path(mock_project_reservoirs.dirs["pld"]) + pld_path.parent.mkdir(parents=True, exist_ok=True) + # Reservoirs fixture is box(0,0,1,1) and box(1,1,2,2) -- give each a PLD + # lake polygon genuinely (mostly) overlapping it. pld_gdf = gpd.GeoDataFrame( - { - "lake_id": [1001, 1002], - "res_id": [501, 502], - "geometry": [Point(0.5, 0.5), Point(1.5, 1.5)], - }, + {"lake_id": [1001, 1002], "res_id": [501, 502]}, + geometry=[box(0.1, 0.1, 0.9, 0.9), box(1.1, 1.1, 1.9, 1.9)], crs="EPSG:4326", ) + pld_gdf.to_file(pld_path, driver="GPKG") - with ( - patch("geopandas.read_file", return_value=pld_gdf), - patch("geopandas.sjoin_nearest") as mock_sjoin, - ): - # Mock the sjoin result - joined_gdf = mock_project_reservoirs.reservoirs.gdf.copy() - joined_gdf["prior_lake_id"] = [1001, 1002] - joined_gdf["prior_res_id"] = [501, 502] - mock_sjoin.return_value = joined_gdf + with patch.object(gpd, "sjoin_nearest") as mock_sjoin: + flows._assign_pld_id(mock_project_reservoirs) + + # both reservoirs have a genuine overlapping PLD lake, so the + # nearest-distance fallback should never be needed + mock_sjoin.assert_not_called() + + result = mock_project_reservoirs.reservoirs.gdf.set_index("id") + assert result.loc[1, "prior_lake_id"] == 1001 + assert result.loc[2, "prior_lake_id"] == 1002 + assert (result["pld_match_method"] == "overlap").all() + + +@pytest.mark.unit +def test_assign_pld_id_prefers_largest_overlap_over_nearest(mock_project_reservoirs, tmp_path): + """Regression test: a small, unrelated PLD lake sitting close enough to + also touch the reservoir must not win over the true match just because + it happens to be nearer by centroid/boundary distance -- the one with + the larger actual overlap area should be chosen.""" + pld_path = Path(mock_project_reservoirs.dirs["pld"]) + pld_path.parent.mkdir(parents=True, exist_ok=True) + # Reservoir "id"=1 is box(0,0,1,1). Give it a true, mostly-overlapping + # match AND a tiny noise lake that only clips its corner. + pld_gdf = gpd.GeoDataFrame( + {"lake_id": [1001, 1002], "res_id": [501, None]}, + geometry=[ + box(0.05, 0.05, 0.95, 0.95), # true match: ~0.81 overlap area + box(0.95, 0.95, 1.05, 1.05), # noise: tiny corner clip with reservoir 1 + ], + crs="EPSG:4326", + ) + pld_gdf.to_file(pld_path, driver="GPKG") + + flows._assign_pld_id(mock_project_reservoirs) + + result = mock_project_reservoirs.reservoirs.gdf.set_index("id") + assert result.loc[1, "prior_lake_id"] == 1001, ( + "the small noise lake should not win over the true, larger-overlap match" + ) + assert len(mock_project_reservoirs.reservoirs.gdf) == 2, ( + "no duplicate rows should be introduced for reservoir 1" + ) + + +@pytest.mark.unit +def test_assign_pld_id_warns_but_uses_low_overlap_match(mock_project_reservoirs, tmp_path, caplog): + """A match with overlap below pld_match_min_overlap_pct should still be used, + but flagged with a warning naming the reservoir and its actual overlap percentage.""" + import logging + pld_path = Path(mock_project_reservoirs.dirs["pld"]) + pld_path.parent.mkdir(parents=True, exist_ok=True) + # Reservoir "id"=1 is box(0,0,1,1), area=1.0. Give it only a tiny + # corner-clip lake (area=0.01, i.e. 1% of the reservoir). + pld_gdf = gpd.GeoDataFrame( + {"lake_id": [1001, 1002]}, + geometry=[box(0.9, 0.9, 1.0, 1.0), box(1.1, 1.1, 1.9, 1.9)], + crs="EPSG:4326", + ) + pld_gdf.to_file(pld_path, driver="GPKG") + mock_project_reservoirs.mission_options["swot"]["pld_match_min_overlap_pct"] = 10.0 + + with caplog.at_level(logging.WARNING): flows._assign_pld_id(mock_project_reservoirs) - mock_sjoin.assert_called_once() + assert "1 reservoir(s) matched a PLD lake covering less than 10%" in caplog.text + assert "1 (1.0%)" in caplog.text + + result = mock_project_reservoirs.reservoirs.gdf.set_index("id") + assert result.loc[1, "prior_lake_id"] == 1001, ( + "the low-overlap match should still be used, not rejected" + ) @pytest.mark.unit diff --git a/tests/unit/test_timeseries.py b/tests/unit/test_timeseries.py index e99e273..b131945 100644 --- a/tests/unit/test_timeseries.py +++ b/tests/unit/test_timeseries.py @@ -177,6 +177,73 @@ def test_extract_swot_observations_skips_when_missing(mock_project_reservoirs, c assert "No SWOT downloads found" in caplog.text +@pytest.mark.unit +def test_swot_extract_observations_distinguishes_missing_from_pld(tmp_path, caplog): + """swot.preprocess.extract_observations must treat prior_lake_id == -9999 + (the sentinel _assign_pld_id uses for reservoirs unmatched to the PLD -- + NOT NaN) as genuinely unavailable, with its own distinct warning -- not + silently fall through to the same generic 'no observations matched' + warning used for a reservoir that IS in the PLD but has zero real SWOT + crossings. Regression test for a real bug: the original code checked + `np.isnan(prior_lake_id)`, which is never true since _assign_pld_id + always overwrites nulls with -9999, so both cases were indistinguishable + from the log alone.""" + import logging + from HydroEO.satellites.swot import preprocess as swot_preprocess + + download_dir = tmp_path / "swot" + download_dir.mkdir() + gdf = gpd.GeoDataFrame( + { + "lake_id": [100], + "obs_id": ["ok"], + "time": ["2024-01-01T00:00:00Z"], + "time_str": ["2024-01-01"], + "wse": [10.0], + }, + geometry=[Point(0, 0)], + crs="EPSG:4326", + ) + gdf.to_file(download_dir / "sub_granule.shp") + + # R1 is matched to PLD lake 100 (should extract fine); R2 is unmatched + # (prior_lake_id == -9999, the real sentinel value, not NaN); R3 is + # matched to a PLD lake but that lake has no real SWOT crossings. + features = gpd.GeoDataFrame( + { + "res_id": ["R1", "R2", "R3"], + "prior_lake_id": [100.0, -9999.0, 200.0], + }, + geometry=[Point(0, 0), Point(1, 1), Point(2, 2)], + crs="EPSG:4326", + ) + + with caplog.at_level(logging.WARNING): + empty_ids = swot_preprocess.extract_observations( + src_dir=str(download_dir), + dst_dir=str(tmp_path / "output"), + dst_file_name="swot.gpkg", + features=features, + id_key="res_id", + ) + + # R1: matched and has real data -> written to disk, not in empty_ids + r1_path = tmp_path / "output" / "R1" / "raw_observations" / "swot.gpkg" + assert r1_path.exists() + + # R3: matched to the PLD but zero real crossings -> generic empty_ids path + assert empty_ids == ["R3"] + r3_path = tmp_path / "output" / "R3" / "raw_observations" / "swot.gpkg" + assert not r3_path.exists() + + # R2: missing from PLD entirely -> distinct warning, NOT lumped into empty_ids + assert "R2" not in empty_ids + r2_path = tmp_path / "output" / "R2" / "raw_observations" / "swot.gpkg" + assert not r2_path.exists() + assert "no geometrically overlapping Prior Lake Database" in caplog.text + assert "R2" in caplog.text + + @pytest.mark.unit def test_extract_icesat2_observations_calls_icesat2_module( mock_project_reservoirs, tmp_path @@ -1360,5 +1427,3 @@ def test_merge_baseline_dates_chronological(nuozhadu_baseline_path): is_sorted = merged["date"].is_monotonic_increasing assert is_sorted, "merged_timeseries.csv should be sorted by date" - -# (Old SWORD tests removed - replaced with new comprehensive test suite above) From 906065013ccd8ae472e2ada3a3aedaae364aab01 Mon Sep 17 00:00:00 2001 From: Cecile Kittel Date: Fri, 17 Jul 2026 13:35:49 +0200 Subject: [PATCH 19/19] Catch error if none-unique reservoirs in AOI. --- HydroEO/flows/_reservoir_init.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/HydroEO/flows/_reservoir_init.py b/HydroEO/flows/_reservoir_init.py index 99ab78a..66e0032 100644 --- a/HydroEO/flows/_reservoir_init.py +++ b/HydroEO/flows/_reservoir_init.py @@ -122,6 +122,20 @@ def _assign_pld_id(prj: "Project") -> None: .drop_duplicates(subset=id_key, keep="first") ) + if not reservoir_areas.index.is_unique: + + duplicate_counts = ( + reservoir_areas.index.to_series() + .value_counts() + .loc[lambda x: x > 1] + ) + + raise ValueError( + "reservoir_areas contains duplicate IDs.\n" + f"Number of duplicated IDs: {len(duplicate_counts)}\n" + f"Top duplicates:\n{duplicate_counts.head(20)}" + ) + # Overlap area as a percentage of the RESERVOIR's own area. matches["_reservoir_area"] = matches[id_key].map(reservoir_areas) matches["_overlap_pct"] = (