Skip to content

Commit bc2a4aa

Browse files
kratzertkratzert
authored
Add UK-NRFA fetcher, #34 (#38)
* Add UK-NRFA fetcher, #34 * Add a few more attribute names (#39) * Add a few attribute names * Some more attribute names * Attribute renaming, adding precip * Add parametrized dependency * Use station_name not location
1 parent 522d920 commit bc2a4aa

10 files changed

Lines changed: 2484 additions & 1 deletion

‎examples/test_uk_nrfa_fetcher.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import logging
2+
3+
import matplotlib.pyplot as plt
4+
5+
from rivretrieve import UKNRFAFetcher, constants
6+
7+
logging.basicConfig(level=logging.INFO)
8+
9+
gauge_ids = [
10+
"1001", # Sample gauge from issue #34
11+
]
12+
variable = constants.DISCHARGE
13+
start_date = "2022-01-01"
14+
end_date = "2022-01-31"
15+
16+
plt.figure(figsize=(12, 6))
17+
18+
fetcher = UKNRFAFetcher()
19+
20+
# Test get_metadata
21+
print("Fetching metadata for one gauge...")
22+
metadata = fetcher.get_metadata()
23+
if not metadata.empty:
24+
print(metadata.loc[gauge_ids[0]])
25+
else:
26+
print("Metadata fetching failed or empty.")
27+
28+
for gauge_id in gauge_ids:
29+
print(f"Fetching {variable} for {gauge_id} from {start_date} to {end_date}...")
30+
data = fetcher.get_data(gauge_id=gauge_id, variable=variable, start_date=start_date, end_date=end_date)
31+
if not data.empty:
32+
print(f"Data for {gauge_id}:")
33+
print(data.head())
34+
print(f"Time series from {data[constants.TIME_INDEX].min()} to {data[constants.TIME_INDEX].max()}")
35+
plt.plot(
36+
data[constants.TIME_INDEX],
37+
data[constants.DISCHARGE],
38+
label=gauge_id,
39+
marker=".",
40+
linestyle="-",
41+
)
42+
else:
43+
print(f"No data found for {gauge_id}")
44+
45+
if "data" in locals() and not data.empty:
46+
plt.xlabel(constants.TIME_INDEX)
47+
plt.ylabel(f"{constants.DISCHARGE} (m3/s)")
48+
plt.title(f"UK NRFA River Discharge ({gauge_ids[0]} - {start_date} to {end_date})")
49+
plt.legend()
50+
plt.grid(True)
51+
plt.tight_layout()
52+
plot_path = "uk_nrfa_discharge_plot.png"
53+
plt.savefig(plot_path)
54+
print(f"Plot saved to {plot_path}")
55+
else:
56+
print("No data to plot.")
57+
58+
print("Test finished.")

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ lxml>=4.8.0
77
dataretrieval>=1.0.0
88
openpyxl>=3.0.0
99
xarray
10+
parameterized
1011
tqdm
1112
zarr>=3.0.7

‎rivretrieve/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from .slovenia import SloveniaFetcher
1212
from .southafrica import SouthAfricaFetcher
1313
from .uk import UKFetcher
14+
from .uk_nrfa import UKNRFAFetcher
1415
from .usa import USAFetcher
1516

1617
__version__ = "0.1.0"

‎rivretrieve/cached_site_data/uk_nrfa_sites.csv‎

Lines changed: 1602 additions & 0 deletions
Large diffs are not rendered by default.

‎rivretrieve/constants.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,14 @@
88
DISCHARGE = "discharge"
99
STAGE = "stage"
1010
WATER_TEMPERATURE = "water_temperature"
11+
CATCHMENT_PRECIPITATION = "catchment_precipitation"
1112

1213
# Attributes
1314
ALTITUDE = "altitude"
1415
AREA = "area"
1516
COUNTRY = "country"
1617
LATITUDE = "latitude"
17-
LOCATION = "location"
1818
LONGITUDE = "longitude"
1919
RIVER = "river"
2020
SOURCE = "source"
21+
STATION_NAME = "station_name"

‎rivretrieve/uk_nrfa.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
"""Fetcher for UK National River Flow Archive (NRFA) data."""
2+
3+
import logging
4+
from typing import Any, Dict, Optional
5+
6+
import pandas as pd
7+
import requests
8+
9+
from . import base, constants, utils
10+
11+
logger = logging.getLogger(__name__)
12+
13+
14+
class UKNRFAFetcher(base.RiverDataFetcher):
15+
"""Fetches river gauge data from the UK National River Flow Archive."""
16+
17+
BASE_URL = "https://nrfaapps.ceh.ac.uk/nrfa/ws"
18+
GAUGE_ID_COL = "id"
19+
20+
METADATA_TRANSLATION_MAPPING = {
21+
"name": constants.STATION_NAME,
22+
"catchment-area": constants.AREA,
23+
"latitude": constants.LATITUDE,
24+
"longitude": constants.LONGITUDE,
25+
"river": constants.RIVER,
26+
# Using the catchment median altitude.
27+
"50-percentile-altitude": constants.ALTITUDE,
28+
}
29+
30+
@staticmethod
31+
def get_gauge_ids() -> pd.DataFrame:
32+
"""Retrieves a DataFrame of available NRFA gauge IDs from the cached CSV."""
33+
return utils.load_sites_csv("uk_nrfa")
34+
35+
def get_metadata(self) -> pd.DataFrame:
36+
"""Fetches site metadata from the NRFA API and renames columns."""
37+
query_params = {"station": "*", "format": "json-object", "fields": "all"}
38+
try:
39+
s = utils.requests_retry_session()
40+
response = s.get(f"{UKNRFAFetcher.BASE_URL}/station-info", params=query_params)
41+
response.raise_for_status() # raises an error for non-200 responses
42+
data = response.json()
43+
df = pd.DataFrame(data["data"])
44+
45+
# Rename id column to the standard GAUGE_ID
46+
df = df.rename(columns={UKNRFAFetcher.GAUGE_ID_COL: constants.GAUGE_ID})
47+
df[constants.GAUGE_ID] = df[constants.GAUGE_ID].astype(str)
48+
49+
# Apply translation mapping for renaming
50+
df = df.rename(columns=self.METADATA_TRANSLATION_MAPPING)
51+
52+
return df.set_index(constants.GAUGE_ID)
53+
except requests.exceptions.RequestException as e:
54+
logger.error(f"Error fetching NRFA catalogue: {e}")
55+
raise
56+
except Exception as e:
57+
logger.error(f"Error processing NRFA catalogue: {e}")
58+
raise
59+
60+
@staticmethod
61+
def get_available_variables() -> tuple[str, ...]:
62+
# Based on common NRFA data types, can be expanded
63+
return (constants.DISCHARGE, constants.CATCHMENT_PRECIPITATION)
64+
65+
def _get_nrfa_data_type(self, variable: str) -> str:
66+
if variable == constants.DISCHARGE:
67+
return "gdf" # Mean daily flow
68+
elif variable == constants.CATCHMENT_PRECIPITATION:
69+
return "cdr" # Catchment daily precipitation.
70+
else:
71+
raise ValueError(f"Unsupported variable: {variable} for NRFA")
72+
73+
def _download_data(self, gauge_id: str, variable: str, start_date: str, end_date: str) -> Optional[Dict[str, Any]]:
74+
"""Downloads the raw time series data from the NRFA API."""
75+
data_type = self._get_nrfa_data_type(variable)
76+
query_params = {
77+
"station": str(gauge_id),
78+
"data-type": data_type,
79+
"format": "json-object",
80+
"start-date": f"{start_date}T00:00:00Z",
81+
"end-date": f"{end_date}T23:59:59Z",
82+
}
83+
s = utils.requests_retry_session()
84+
try:
85+
response = s.get(f"{self.BASE_URL}/time-series", params=query_params)
86+
response.raise_for_status()
87+
return response.json()
88+
except requests.exceptions.RequestException as e:
89+
logger.error(f"Error fetching NRFA time series for {gauge_id} ({data_type}): {e}")
90+
return None
91+
92+
def _parse_data(self, gauge_id: str, raw_data: Optional[Dict[str, Any]], variable: str) -> pd.DataFrame:
93+
"""Parses the raw JSON time series data."""
94+
if not raw_data or "data-stream" not in raw_data or not raw_data["data-stream"]:
95+
logger.warning(f"No data stream found for {gauge_id}, variable {variable}")
96+
return pd.DataFrame(columns=[constants.TIME_INDEX, variable])
97+
98+
try:
99+
dates = raw_data["data-stream"][0::2]
100+
values = raw_data["data-stream"][1::2]
101+
df = pd.DataFrame.from_dict({"time": dates, variable: values})
102+
df[constants.TIME_INDEX] = pd.to_datetime(df["time"], format="ISO8601").dt.date
103+
df[constants.TIME_INDEX] = pd.to_datetime(df[constants.TIME_INDEX])
104+
df[variable] = pd.to_numeric(df[variable], errors="coerce")
105+
return df[[constants.TIME_INDEX, variable]].dropna().reset_index(drop=True)
106+
except Exception as e:
107+
logger.error(f"Error parsing NRFA data for {gauge_id}: {e}")
108+
return pd.DataFrame(columns=[constants.TIME_INDEX, variable])
109+
110+
def get_data(
111+
self,
112+
gauge_id: str,
113+
variable: str,
114+
start_date: Optional[str] = None,
115+
end_date: Optional[str] = None,
116+
) -> pd.DataFrame:
117+
"""Fetches and parses UK NRFA river gauge data."""
118+
if variable not in self.get_available_variables():
119+
raise ValueError(f"Unsupported variable: {variable}")
120+
121+
start_date = utils.format_start_date(start_date)
122+
end_date = utils.format_end_date(end_date)
123+
124+
try:
125+
raw_data = self._download_data(gauge_id, variable, start_date, end_date)
126+
df = self._parse_data(gauge_id, raw_data, variable)
127+
128+
# Filter by date range
129+
start_date_dt = pd.to_datetime(start_date)
130+
end_date_dt = pd.to_datetime(end_date)
131+
df = df[(df[constants.TIME_INDEX] >= start_date_dt) & (df[constants.TIME_INDEX] <= end_date_dt)]
132+
return df
133+
except Exception as e:
134+
logger.error(f"Failed to get data for site {gauge_id}, variable {variable}: {e}")
135+
return pd.DataFrame(columns=[constants.TIME_INDEX, variable])

‎tests/test_data/uk_nrfa_1001_discharge_20220101.json‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{
2+
"timestamp": "2025-10-14T08:22:36",
3+
"interval": "R31/2022-01-01/P1D",
4+
"station": {
5+
"id": 1001,
6+
"name": "Wick at Tarroul",
7+
"easting": 326202.0,
8+
"northing": 954915.0,
9+
"latitude": 58.4761957805334,
10+
"longitude": -3.2670605529772914
11+
},
12+
"data-type": {
13+
"id": "gdf",
14+
"name": "Gauged Daily Flow",
15+
"parameter": "Flow",
16+
"units": "m3/s",
17+
"measurement-type": "Mean",
18+
"period": "P1D"
19+
},
20+
"data-stream": [
21+
"2022-01-01",
22+
1.552,
23+
"2022-01-02",
24+
1.461,
25+
"2022-01-03",
26+
2.035,
27+
"2022-01-04",
28+
7.232,
29+
"2022-01-05",
30+
6.539,
31+
"2022-01-06",
32+
3.901,
33+
"2022-01-07",
34+
2.886,
35+
"2022-01-08",
36+
3.556,
37+
"2022-01-09",
38+
2.618,
39+
"2022-01-10",
40+
2.336,
41+
"2022-01-11",
42+
2.173,
43+
"2022-01-12",
44+
1.919,
45+
"2022-01-13",
46+
1.694,
47+
"2022-01-14",
48+
2.088,
49+
"2022-01-15",
50+
2.138,
51+
"2022-01-16",
52+
2.185,
53+
"2022-01-17",
54+
1.378,
55+
"2022-01-18",
56+
1.355,
57+
"2022-01-19",
58+
2.313,
59+
"2022-01-20",
60+
1.624,
61+
"2022-01-21",
62+
1.316,
63+
"2022-01-22",
64+
1.177,
65+
"2022-01-23",
66+
1.054,
67+
"2022-01-24",
68+
1.06,
69+
"2022-01-25",
70+
1.159,
71+
"2022-01-26",
72+
1.573,
73+
"2022-01-27",
74+
1.211,
75+
"2022-01-28",
76+
1.642,
77+
"2022-01-29",
78+
1.999,
79+
"2022-01-30",
80+
1.982,
81+
"2022-01-31",
82+
3.905
83+
]
84+
}

‎tests/test_data/uk_nrfa_1001_precipitation_20220101.json‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{
2+
"timestamp": "2025-10-14T17:58:16",
3+
"interval": "R31/2022-01-01/P1D",
4+
"station": {
5+
"id": 1001,
6+
"name": "Wick at Tarroul",
7+
"easting": 326202.0,
8+
"northing": 954915.0,
9+
"latitude": 58.4761957805334,
10+
"longitude": -3.2670605529772914
11+
},
12+
"data-type": {
13+
"id": "cdr",
14+
"name": "Catchment Daily Rainfall",
15+
"parameter": "Rainfall",
16+
"units": "mm",
17+
"measurement-type": "Accumulation",
18+
"period": "P1D"
19+
},
20+
"data-stream": [
21+
"2022-01-01",
22+
0.2,
23+
"2022-01-02",
24+
1.7,
25+
"2022-01-03",
26+
4.4,
27+
"2022-01-04",
28+
9.1,
29+
"2022-01-05",
30+
0.2,
31+
"2022-01-06",
32+
1,
33+
"2022-01-07",
34+
3.8,
35+
"2022-01-08",
36+
0.3,
37+
"2022-01-09",
38+
0.6,
39+
"2022-01-10",
40+
1,
41+
"2022-01-11",
42+
0.1,
43+
"2022-01-12",
44+
0,
45+
"2022-01-13",
46+
3.6,
47+
"2022-01-14",
48+
0.8,
49+
"2022-01-15",
50+
0.7,
51+
"2022-01-16",
52+
1,
53+
"2022-01-17",
54+
0,
55+
"2022-01-18",
56+
3.8,
57+
"2022-01-19",
58+
1,
59+
"2022-01-20",
60+
0.6,
61+
"2022-01-21",
62+
0,
63+
"2022-01-22",
64+
0,
65+
"2022-01-23",
66+
0.7,
67+
"2022-01-24",
68+
1.1,
69+
"2022-01-25",
70+
0.6,
71+
"2022-01-26",
72+
2.6,
73+
"2022-01-27",
74+
2.1,
75+
"2022-01-28",
76+
1.6,
77+
"2022-01-29",
78+
2.6,
79+
"2022-01-30",
80+
7.7,
81+
"2022-01-31",
82+
6.3
83+
]
84+
}

0 commit comments

Comments
 (0)