diff --git a/examples/test_portugal_fetcher.py b/examples/test_portugal_fetcher.py new file mode 100644 index 0000000..814c758 --- /dev/null +++ b/examples/test_portugal_fetcher.py @@ -0,0 +1,52 @@ +import logging + +import matplotlib.pyplot as plt + +from rivretrieve import PortugalFetcher, constants + +logging.basicConfig(level=logging.INFO) + +# Example gauge IDs from the SITE_MAP in portugal.py +gauge_ids = [ + "19B/01H", # A known gauge ID from the map +] +variables = [constants.STAGE_DAILY_MEAN] +start_date = "1980-01-01" +end_date = None # Defaults to today + +fetcher = PortugalFetcher() + +for variable in variables: + plt.figure(figsize=(12, 6)) + print(f"\n--- Testing variable: {variable} ---") + for gauge_id in gauge_ids: + print(f"Fetching {variable} for {gauge_id} from {start_date} to {end_date}...") + data = fetcher.get_data(gauge_id=gauge_id, variable=variable, start_date=start_date, end_date=end_date) + if not data.empty: + print(f"Data for {gauge_id}:") + print(data.head()) + print(f"Time series from {data.index.min()} to {data.index.max()}") + plt.plot( + data.index, + data[variable], + label=f"{gauge_id} - {variable}", + marker=".", + linestyle="-", + ) + else: + print(f"No {variable} data found for {gauge_id}") + + if plt.gca().has_data(): + plt.xlabel(constants.TIME_INDEX) + plt.ylabel(variable) + plt.title(f"Portugal River Data ({start_date} to {end_date})") + plt.legend() + plt.grid(True) + plt.tight_layout() + plot_path = f"portugal_{variable}_plot.png" + plt.savefig(plot_path) + print(f"Plot saved to {plot_path}") + else: + print(f"No data to plot for {variable}.") + +print("Test finished.") diff --git a/rivretrieve/__init__.py b/rivretrieve/__init__.py index b54e7cb..3878a25 100644 --- a/rivretrieve/__init__.py +++ b/rivretrieve/__init__.py @@ -8,6 +8,7 @@ from .france import FranceFetcher from .japan import JapanFetcher from .poland import PolandFetcher +from .portugal import PortugalFetcher from .slovenia import SloveniaFetcher from .southafrica import SouthAfricaFetcher from .uk_ea import UKEAFetcher diff --git a/rivretrieve/portugal.py b/rivretrieve/portugal.py new file mode 100644 index 0000000..2416db0 --- /dev/null +++ b/rivretrieve/portugal.py @@ -0,0 +1,858 @@ +"""Fetcher for Portuguese river gauge data from SNIRH.""" + +import html +import logging +from datetime import datetime +from io import StringIO +from typing import Optional + +import pandas as pd + +from . import base, constants, utils + +logger = logging.getLogger(__name__) + +# SITE_MAP from https://github.com/kratzert/RivRetrieve-Python/issues/40 +SITE_MAP = { + "19B/01H": 1627743378, + "03J/02H": 1627743350, + "17H/04H": 1627743352, + "15G/02H": 1627743380, + "11H/02H": 1627743382, + "21O/08H": 2443345812, + "21O/03H": 1627743386, + "26J/01H": 1627743392, + "04K/04A": 11508860178, + "11H/01A": 1627743384, + "17M/01A": 1627743458, + "06S/01A": 1627743548, + "28I/01A": 1627743554, + "14H/01A": 1627743562, + "30E/01A": 1627743568, + "05O/01A": 1627743572, + "04H/01A": 1627743674, + "12M/01A": 1627743678, + "12L/02A": 1627751888, + "05P/01A": 1627753166, + "11L/03A": 1627758736, + "13H/05A": 1627758750, + "11H/05A": 7945659548, + "11K/02A": 1627758884, + "03J/01A": 1627758928, + "16K/01A": 1627759292, + "08N/01AE": 1627759302, + "12H/01A": 1627759328, + "07K/01A": 1627759340, + "02Q/01A": 1627759434, + "27L/01A": 1627759452, + "14O/01A": 1627759474, + "07M/02A": 1627759510, + "07K/02A": 1627759512, + "22L/01A": 1627759534, + "29M/04A": 7914972596, + "07K/05A": 7556914548, + "05R/01A": 1627743534, + "17J/01A": 1627743544, + "08H/01A": 1627743574, + "05P/02A": 1627743666, + "26F/01A": 1627743668, + "30M/06A": 7927143708, + "16H/01A": 1627743896, + "15J/01A": 1627751876, + "28G/02A": 1627751880, + "12L/01AE": 1627751890, + "13L/01AE": 1627751892, + "07G/01A": 1627751912, + "04J/08A": 9978398426, + "09G/04A": 7554829608, + "12N/01AE": 1627752558, + "10K/01A": 1627758608, + "06N/01A": 1627758622, + "26G/01AE": 1627758624, + "12I/01A": 1627758668, + "17L/03A": 1627758676, + "05K/02A": 10908513554, + "04I/01A": 1627758686, + "14N/01A": 1627758730, + "22M/01A": 1627758752, + "20E/01A": 1627758756, + "12O/01A": 1627758774, + "22I/01A": 1627758784, + "05T/01A": 1627758786, + "19H/01A": 1627758816, + "29I/01A": 1627758826, + "27H/01A": 1627758846, + "27H/03AE": 1627758832, + "26E/01A": 1627758860, + "18K/02A": 1627758874, + "30L/01A": 1627758894, + "24I/01A": 1627758906, + "18F/03A": 1627758936, + "13O/01A": 1627758954, + "04F/01A": 1627758956, + "23G/01A": 1627758948, + "17L/01A": 1627759290, + "07M/01A": 1627759330, + "21A/01A": 1627759358, + "03I/01A": 1627759378, + "28G/01A": 1627759384, + "13J/01A": 1627759390, + "08P/01A": 1627759396, + "14M/01A": 1627759382, + "17B/01A": 1627759402, + "08N/02AE": 1627759456, + "06P/02AE": 1627759488, + "24H/01A": 1627759492, + "05O/02A": 7556914544, + "02Q/02AE": 8235958464, + "03J/04A": 1627759522, + "03J/05A": 1627759524, + "03I/04A": 12303382358, + "06P/01A": 1627759552, + "08L/01A": 1627759554, + "03H/01A": 1627759560, + "17C/06A": 7796342256, + "19O/01A": 1627743354, + "06M/01A": 1627743406, + "24M/06A": 1627743418, + "24M/07A": 1627743416, + "24M/08A": 7554862818, + "12J/01A": 1627743426, + "03J/02AE": 1627743422, + "02H/01A": 1627743428, + "03J/03A": 1627743430, + "24J/01A": 1627743440, + "06K/05A": 1627743432, + "30G/02A": 1627743460, + "07P/01A": 1627743468, + "04P/01A": 1627743482, + "06O/09A": 7554777518, + "30L/02A": 1627743538, + "14I/01A": 1627743586, + "20O/01A": 1627743648, + "10M/01A": 1627743662, + "07I/01A": 1627743682, + "16G/03A": 7557137960, + "21J/01A": 1627752100, + "26M/01A": 1627752188, + "06O/10A": 7554829602, + "16K/02A": 1627758658, + "30G/01A": 1627758670, + "11L/01A": 1627758738, + "19J/01A": 1627758764, + "27H/02AE": 1627758830, + "22K/01A": 1627758834, + "16I/02A": 1627758876, + "30G/11AE": 3507536920, + "06N/02AE": 1627758894, + "06O/01A": 1627758952, + "05S/01A": 1627758968, + "13L/02A": 1627758972, + "07O/02A": 1627758976, + "16L/01AE": 1627758980, + "09H/06A": 7554777512, + "26I/01A": 1627759368, + "12O/02A": 1627759374, + "06O/02A": 1627759380, + "06K/01A": 1627759446, + "07H/01A": 1627759468, + "03G/01A": 1627759476, + "11L/02A": 1627759502, + "06M/06A": 9321904688, + "29M/03H": 1627743398, + "23F/01H": 1627743394, + "23I/02H": 1627743396, + "17C/01H": 1627743400, + "17I/01H": 1627743402, + "14H/02H": 1627743404, + "11H/01H": 1627743408, + "18F/02H": 1627743410, + "17G/02H": 1627743414, + "15K/01H": 1627743412, + "03J/01H": 1627743424, + "03J/06H": 1627743420, + "08H/05H": 1701658505, + "08H/01H": 1627743434, + "17I/02H": 1627743436, + "24J/01H": 1627743438, + "31F/04H": 1627743442, + "06I/01H": 1627743444, + "06I/04H": 1627743446, + "02I/01H": 1627743448, + "24L/01H": 1627743450, + "11J/01H": 1627743452, + "09F/03H": 1627743456, + "25M/01H": 1627743462, + "24O/01H": 1627743464, + "07F/04H": 1627743466, + "03D/01H": 1627743470, + "30L/04H": 1627743474, + "21A/06H": 10065708580, + "07F/02H": 1627743472, + "26K/02H": 1627743476, + "26I/01H": 1627743478, + "05Q/01H": 1627743480, + "17F/04H": 1627743484, + "13O/02A": 1627743362, + "27L/02A": 1627743364, + "04H/05A": 1627743358, + "04H/02A": 1627743454, + "04J/09A": 11029759146, + "20I/01A": 1627758678, + "05E/02A": 1627743376, + "09H/05A": 1627743356, + "15E/05A": 1627743360, + "25P/01A": 1627743366, + "20H/01A": 1627743368, + "25L/01A": 1627758944, + "12G/01AE": 1627743374, + "25E/02H": 1627743486, + "07K/02H": 1627743488, + "06G/01H": 1627743490, + "24M/01H": 1627743492, + "07P/01H": 1627743494, + "04E/03H": 1627743500, + "04F/02H": 1627743502, + "03H/01H": 1627743504, + "12L/01H": 1627743506, + "17G/05H": 1627743508, + "14H/03H": 1627743512, + "14H/04H": 1627743514, + "03J/04H": 1627743528, + "14I/02H": 1627743516, + "05S/01H": 1627743530, + "30G/10H": 1627743520, + "30G/09H": 1627743522, + "02H/02H": 1627743524, + "30G/04H": 1627743532, + "17N/01H": 1627743536, + "30L/03H": 1627743540, + "30L/01H": 1627743542, + "06S/02H": 1627743546, + "20E/01H": 1627743550, + "31E/01H": 1627743552, + "31K/03H": 1627743556, + "03L/01H": 1627743558, + "14H/01H": 1627743560, + "30E/02H": 1627743566, + "17F/01H": 1627743576, + "25N/01H": 1627743578, + "20I/03H": 1627743580, + "04J/02H": 1627743582, + "04J/07H": 1627743584, + "09I/05H": 1695334936, + "07I/05H": 1627743592, + "07I/04H": 1627743590, + "07I/01H": 1627743616, + "20O/03H": 1627743650, + "19D/03H": 1627743656, + "07F/05H": 1627743654, + "10J/01H": 1627743658, + "10M/07H": 4928135292, + "10M/06H": 1627743660, + "19F/01H": 1627743664, + "12H/04H": 1627743670, + "07H/03H": 1627743672, + "05G/02H": 1627743676, + "07I/02H": 1627743680, + "28K/03H": 1627743688, + "30G/07H": 1627743698, + "01H/03H": 1627743706, + "12M/01H": 1627743716, + "06M/05H": 1627743784, + "06M/01H": 1627743730, + "10P/02H": 1627744042, + "10P/01H": 1627743970, + "08J/02H": 1627744270, + "08J/01H": 1627744152, + "02E/02H": 1627744404, + "09M/01H": 1627744494, + "04J/03H": 1627744618, + "30E/01H": 1627744712, + "17G/08H": 1627744828, + "03M/01H": 1627744996, + "20B/02H": 1627745096, + "12J/02H": 1627744920, + "08O/03H": 1627745264, + "08O/02H": 1627745200, + "31J/01H": 1627751868, + "21A/05H": 1627751870, + "23E/02H": 1627751872, + "17G/01H": 1627751874, + "28F/01H": 1627751878, + "15E/04H": 1627751882, + "31H/01H": 1627751884, + "18L/01H": 1627751886, + "03H/04H": 1627751894, + "07G/01H": 1627751900, + "04J/04H": 1627751922, + "03L/04H": 8259755350, + "03L/02H": 1627751948, + "30L/02H": 1627751978, + "10H/01H": 1627752042, + "31F/02H": 1627752088, + "20M/01H": 1627752122, + "10F/01H": 1627752130, + "27I/01H": 1627752142, + "07H/01H": 1627752166, + "06K/05H": 1627752302, + "06K/01H": 1627752254, + "07J/01H": 1627752204, + "26G/01H": 1627752328, + "08P/03H": 1627752390, + "08P/01H": 1627752346, + "07H/02H": 1627752420, + "08Q/02H": 1627752674, + "08Q/01H": 1627752468, + "16G/02H": 1627752796, + "19D/02H": 1627752886, + "04E/01H": 1627752978, + "21B/02H": 1627753084, + "11I/01H": 1627758610, + "16D/01H": 1627758614, + "06G/04H": 1627758612, + "19K/01H": 1627758616, + "16K/03H": 1627758618, + "23I/01H": 1627758620, + "02G/02H": 8770661530, + "15E/06H": 9087187588, + "04E/06H": 1627758626, + "03F/01H": 1627758628, + "30G/05H": 1627758634, + "04J/01H": 1627758630, + "15K/02H": 1695334942, + "14D/01H": 1627758636, + "01G/03H": 1627758640, + "01G/02H": 1627758638, + "30H/02H": 1627758642, + "16L/01H": 1627758644, + "13E/03H": 1627758646, + "07F/03H": 1627758648, + "16H/01H": 1627758650, + "06M/02H": 1627758652, + "08H/04H": 1627758656, + "08H/02H": 1627758654, + "07F/01H": 1627758662, + "06I/03H": 1627758664, + "12I/01H": 1627758666, + "16G/01H": 1627758606, + "13H/01H": 1627758604, + "17C/05H": 1627758674, + "20I/02H": 1627743370, + "04H/01H": 1627758680, + "13H/04H": 1627758682, + "03Q/01H": 1627758684, + "21L/01H": 1627758690, + "25K/01H": 1627758688, + "22G/02H": 1627758692, + "24G/01H": 1627758694, + "03L/03AE": 1701658501, + "03O/01AE": 1627758724, + "08J/01AE": 1627758700, + "10K/02AE": 1627758702, + "03P/01AE": 1627758704, + "07J/02AE": 1627758706, + "09H/02AE": 1627758710, + "12H/02AE": 1627758712, + "08J/03AE": 1627758714, + "05G/01AE": 1627758716, + "08J/02AE": 1627758718, + "07H/02AE": 1627758726, + "09I/01AE": 1627758696, + "04K/03AE": 2602492446, + "09H/01AE": 1627758698, + "09G/01AE": 1627758708, + "06K/01AE": 1627758720, + "10H/03AE": 4240009782, + "06K/02AE": 1627758722, + "21I/01H": 1627758728, + "14N/01H": 1627758732, + "21A/02H": 1627759318, + "10M/04H": 1627758734, + "21B/03H": 1627758740, + "15E/01H": 1627758744, + "06F/01H": 1627758742, + "27L/04H": 1627758746, + "13H/03H": 1627758748, + "09J/01H": 1627758754, + "11L/02H": 1627758760, + "11L/01H": 1627758758, + "04E/04H": 1627758766, + "18G/02H": 1627758768, + "01F/03H": 1627758770, + "12O/01H": 1627758772, + "25E/01H": 1627758776, + "21B/01H": 1627758778, + "01G/01H": 1627758780, + "13K/01H": 1627758782, + "05T/01H": 1627743526, + "21O/02H": 1627758794, + "04J/06H": 1627758796, + "25G/03H": 1627758798, + "07L/04H": 1627758808, + "07L/01H": 1627758806, + "20O/01H": 1627758810, + "25G/02H": 1627758790, + "16H/02H": 1627758792, + "27H/01H": 1627758800, + "21J/01H": 1627758802, + "18I/01H": 1627758804, + "10K/04H": 1627758812, + "19M/01H": 1627758814, + "24M/03H": 1627758820, + "20I/01H": 1627758824, + "26K/01H": 1627758818, + "27J/01H": 1627758840, + "27H/02H": 1627758848, + "21O/01H": 1627758852, + "24M/02H": 1627758822, + "29L/01H": 1627758828, + "30G/01H": 1627758836, + "19N/01H": 1627758838, + "14D/03H": 1627758842, + "24L/02H": 1627758844, + "12G/06H": 1627758850, + "12E/03H": 1627758854, + "27G/01H": 1627758856, + "19E/02H": 1627758858, + "17F/06H": 1627758862, + "18E/05H": 1627758864, + "07I/03H": 1627758866, + "05M/01H": 1627758868, + "25P/02H": 1627758870, + "26H/01H": 1627758872, + "10K/08H": 1627758880, + "10K/03H": 1627758878, + "28J/01H": 1627758882, + "16K/05H": 1627758886, + "29E/01H": 1627758892, + "30M/05H": 1627758896, + "30M/04H": 1627758898, + "30M/01H": 1627758900, + "30M/02H": 1627758902, + "30E/05H": 1627758890, + "24I/01H": 1627758910, + "28K/02H": 1627758912, + "17E/01H": 1627758914, + "04G/04H": 1627758918, + "11M/03H": 1627758922, + "11M/01H": 1627758920, + "04K/01H": 1627758926, + "03J/03H": 1627758930, + "30G/02H": 1627758932, + "03E/01H": 1627758934, + "20I/04H": 1627758938, + "09H/01H": 1627758940, + "25L/01H": 1627758942, + "28G/02H": 1627758946, + "19C/01H": 1627758950, + "04F/01H": 1627758958, + "06G/03H": 1627758960, + "30F/01H": 1627758962, + "17F/05H": 1627758964, + "09G/02H": 1627758966, + "06L/01H": 1627758970, + "07O/01H": 1627758974, + "09H/02H": 1627758978, + "28L/03H": 7633043532, + "28L/04H": 11791068834, + "22C/02H": 1627758990, + "19C/03H": 1627758992, + "19N/02H": 1627758994, + "26G/04H": 1627758996, + "26G/05H": 1627758998, + "18F/04H": 1627759000, + "22C/01H": 1627759002, + "10G/01H": 1627759004, + "05E/03H": 1627759006, + "12E/01H": 1627759008, + "05Q/03H": 1627759010, + "09I/06H": 10080899844, + "11H/03H": 1627758986, + "14D/02H": 1627759012, + "18E/06H": 1627759014, + "19C/02H": 1627759018, + "07H/04H": 1627759020, + "04G/01H": 1627759024, + "04G/07H": 1627759022, + "05G/01H": 1627759026, + "20H/02H": 1627759028, + "12G/02H": 1627759030, + "12G/09H": 11204879762, + "14I/01H": 1627759032, + "19D/06H": 1627759034, + "03G/05H": 1627759036, + "12G/03H": 1627759040, + "13H/02H": 1627759038, + "20C/01H": 1627759042, + "06I/02H": 1627759044, + "21F/01H": 1627759046, + "25H/01H": 1627759048, + "20C/03H": 1627759050, + "13E/04H": 1627759052, + "22G/03H": 1627759054, + "02P/01H": 1627759058, + "13F/02H": 1627759056, + "04J/05H": 1627759060, + "17G/07H": 1627759062, + "22C/03H": 1627759064, + "12J/01H": 1627759066, + "12G/08H": 1627759068, + "20F/02H": 1627759070, + "19D/05H": 1627759072, + "02P/02H": 1627758788, + "22D/01H": 1627759074, + "03G/02H": 1627759016, + "13I/02H": 11204877412, + "17H/03H": 1627758984, + "13J/03H": 11204877406, + "03F/02H": 1627759116, + "03F/03H": 1627759118, + "03G/01H": 1627759232, + "11I/07H": 1627759076, + "13J/02H": 11204877400, + "10K/06H": 1627759078, + "05J/01H": 1627759080, + "20G/01H": 1627759082, + "10K/02H": 1627759084, + "10N/01H": 1627759086, + "04E/02H": 1627759088, + "21K/01H": 1627759090, + "09O/03H": 1627759094, + "09O/02H": 1627759092, + "18E/01H": 1627759096, + "07K/03H": 1627759100, + "04O/02H": 1627759102, + "13I/01H": 1627759098, + "11K/01H": 1627759106, + "10L/01H": 1627759108, + "05E/01H": 1627759110, + "21B/05H": 1627759112, + "04N/02H": 1627759114, + "21A/01H": 1627759120, + "02F/01H": 1627759122, + "12N/01H": 1627759124, + "30G/08H": 1627759126, + "15E/03H": 1627759128, + "09F/01H": 1627759130, + "21B/06H": 1627759132, + "10M/05H": 1627759134, + "13F/04H": 1627759136, + "06F/02H": 1627759138, + "23M/01H": 1627759140, + "12H/03H": 1627759142, + "19E/03H": 1627759144, + "14N/02H": 1627759146, + "17F/02H": 1627759148, + "10M/01H": 1627759150, + "19D/04H": 1627759154, + "17L/01H": 1627759156, + "04O/01H": 1627759158, + "12M/02H": 1627759160, + "12H/02H": 1627759162, + "30E/04H": 1627759164, + "10G/04H": 1627759166, + "04R/01H": 1627759168, + "21C/01H": 1627759170, + "21A/03H": 1627759172, + "22G/01H": 1627759174, + "12G/05H": 1627759176, + "15E/02H": 1627759178, + "09I/03H": 1627759180, + "04G/02H": 1627759182, + "19C/04H": 1627759184, + "10K/05H": 1627759186, + "12E/02H": 1627759190, + "12F/01H": 1627759192, + "26L/01H": 1627759194, + "02Q/01H": 1627759196, + "10G/05H": 1627759198, + "05Q/04H": 1627759202, + "05Q/02H": 1627759200, + "16F/01H": 1627759204, + "10F/02H": 1627759206, + "21C/02H": 1627759208, + "26K/03H": 1627759210, + "17F/03H": 1627759212, + "03H/05H": 1627759214, + "31H/02H": 1627759218, + "13F/01H": 1627759216, + "06O/02H": 1627759220, + "12G/04H": 1627759222, + "10K/01H": 1627759224, + "11I/03H": 1627759228, + "11I/02H": 1627759226, + "13E/01H": 1627759230, + "18E/02H": 1627759234, + "20E/02H": 1627759236, + "13G/01H": 11204877430, + "26F/02H": 1627759238, + "10F/03H": 7927143702, + "03H/02H": 1627759242, + "11I/06H": 1627759240, + "30K/01H": 10682227146, + "25G/04H": 1627759244, + "09G/01H": 1627759246, + "04N/01H": 1627759248, + "11M/02H": 1627759250, + "06P/01H": 1627759252, + "13E/02H": 1627759254, + "18K/01H": 1627759256, + "04O/04H": 1627759260, + "04O/03H": 1627759258, + "09I/02H": 1627759262, + "20C/02H": 1627759264, + "10G/02H": 1627758988, + "17C/04H": 1627759152, + "02G/01H": 1627759266, + "09I/04H": 1627758982, + "20D/02H": 1627759268, + "25O/01H": 1627759270, + "12H/01H": 1627759272, + "19E/04H": 1627759278, + "19E/01H": 1627759274, + "18F/01H": 1627759276, + "12F/04H": 1627759280, + "11I/05H": 1627759282, + "12G/01H": 1627759284, + "12F/02H": 1627759286, + "04G/05H": 1627759288, + "16K/02H": 1627759294, + "17G/03H": 1627759298, + "27L/01H": 1627759300, + "31L/01H": 10682227148, + "11H/04H": 1627759308, + "07L/03H": 1627759310, + "17G/06H": 1627759306, + "21A/04H": 1627759304, + "06K/02H": 1627759312, + "06O/08H": 1627759316, + "06O/03H": 1627759314, + "21A/07H": 10065708582, + "08L/04H": 1627759322, + "08L/01H": 1627759320, + "02O/01H": 1627759324, + "03G/06H": 8202036962, + "03G/03H": 1627759326, + "17M/01H": 1627759332, + "19O/01H": 1627759334, + "04G/06H": 1627759336, + "03N/01H": 1627759338, + "09J/02H": 1627759344, + "03M/05H": 1627759346, + "09H/03H": 1627759350, + "10G/03H": 1627759352, + "17D/01H": 1627759354, + "07G/03H": 1627759356, + "27L/05H": 1627759362, + "27L/03H": 1627759364, + "15O/01H": 1627759366, + "19B/02H": 1627759370, + "07K/04H": 2443345822, + "07K/01H": 1627759342, + "11O/01H": 1627759372, + "25N/02H": 10018589836, + "03I/02H": 1627759376, + "28G/01H": 1627759386, + "06O/05H": 1627759388, + "17H/01H": 1627759392, + "25G/01H": 1627759394, + "05K/01H": 1627759398, + "29G/01H": 1627759412, + "07L/02H": 1627759414, + "12I/02H": 1627759416, + "01G/04H": 1627759418, + "15P/02H": 1627759422, + "15P/01H": 1627759420, + "05S/02H": 1627759424, + "21B/04H": 1627759426, + "06O/06H": 1627759428, + "09G/03H": 1627759432, + "11I/09H": 1627759438, + "31K/02H": 1627759440, + "11I/10H": 1693384198, + "11I/08H": 1627759442, + "17B/01H": 1627759444, + "13F/05H": 1627759448, + "24H/01H": 1627759400, + "10H/02H": 1627759404, + "29G/02H": 3158125162, + "04S/02H": 1627759408, + "24G/02H": 1627759410, + "17G/04H": 1627759450, + "07G/02H": 1627759454, + "29M/01H": 1627759458, + "02H/01H": 1627759460, + "03I/01H": 1627759464, + "03I/03H": 1627759462, + "17C/02H": 1627759466, + "12G/07H": 1627759472, + "24H/03H": 1627759470, + "17H/02H": 1627759478, + "10M/02H": 1627759480, + "16C/01H": 1627759482, + "24H/02H": 1627759506, + "30H/03H": 1627759486, + "18G/03H": 1627759490, + "11I/04H": 1627759498, + "19F/02H": 1627759500, + "09O/01H": 1627759484, + "03K/01H": 1627759494, + "06L/02H": 1627759496, + "08O/04H": 1693384202, + "08O/01H": 1627759504, + "06M/03H": 1627759508, + "28L/02H": 1627759514, + "12F/03H": 1627759516, + "20L/01H": 1627759518, + "03J/05H": 1627759520, + "23K/01H": 1627759526, + "10M/03H": 1627759528, + "30F/02H": 1627759530, + "15E/07H": 9087187590, + "04K/02H": 1627759532, + "03G/04H": 1627759536, + "09L/01H": 1627759538, + "10L/02H": 1627759540, + "20D/01H": 1627759542, + "13F/03H": 1627759544, + "11G/01H": 1627759546, + "16K/01H": 1627759548, + "08L/02H": 1627759550, + "03M/04H": 1627759556, + "05N/01H": 1627759558, + "03H/03H": 1627759562, + "04R/02H": 1627759564, + "03P/01H": 1627759566, + "05H/01H": 1627759568, + "05H/02H": 1627759570, + "09K/01H": 1627759572, + "19O/02H": 1627759574, + "22E/01H": 1627759576, + "23M/02H": 1627743390, + "17C/03H": 1627758888, + "18E/04H": 1627758916, +} + +# --- SNIRH PARAMETER MAP (Portuguese + English) ---------------------------- +SNIRH_PARAMS = { + 1850: (constants.DISCHARGE_DAILY_MEAN, "Caudal médio diário"), + 1845: (constants.STAGE_DAILY_MEAN, "Nível médio diário"), +} + + +class PortugalFetcher(base.RiverDataFetcher): + """Fetches river gauge data from Portugal's SNIRH.""" + + BASE_URL = "https://snirh.apambiente.pt/snirh/_dadosbase/site/janela_verdados.php" + + @staticmethod + def get_cached_metadata() -> pd.DataFrame: + """Retrieves a DataFrame of available Portuguese gauge IDs and metadata.""" + return utils.load_cached_metadata_csv("portugal") + + @staticmethod + def get_available_variables() -> tuple[str, ...]: + return tuple(val[0] for val in SNIRH_PARAMS.values()) + + def _get_param_id(self, variable: str) -> Optional[int]: + for param_id, (var_name, _) in SNIRH_PARAMS.items(): + if var_name == variable: + return param_id + return None + + def _download_data(self, gauge_id: str, variable: str, start_date: str, end_date: str) -> Optional[pd.DataFrame]: + """Downloads and parses the data table from SNIRH.""" + if gauge_id not in SITE_MAP: + logger.error(f"Gauge ID {gauge_id} not found in SITE_MAP.") + return None + site_id = SITE_MAP[gauge_id] + + param_id = self._get_param_id(variable) + if param_id is None: + raise ValueError(f"Unsupported variable: {variable}") + + _, param_en = SNIRH_PARAMS[param_id] + + # The API seems to return all data regardless of tmin/tmax, so we fetch all and filter later. + tmin = "01/01/1900" + tmax = datetime.now().strftime("%d/%m/%Y") + + url = f"{self.BASE_URL}?sites={site_id}&pars={param_id}&tmin={tmin}&tmax={tmax}" + + headers = { + "User-Agent": "Mozilla/5.0", + "Referer": "https://snirh.apambiente.pt/", + } + + logger.info(f"Fetching {variable} for site {gauge_id} ({site_id}) from SNIRH") + try: + s = utils.requests_retry_session() + r = s.get(url, headers=headers) + r.raise_for_status() + tables = pd.read_html(StringIO(r.text)) + except Exception as e: + logger.error(f"Failed to retrieve data for {gauge_id}, param {param_id}: {e}") + return None + + logger.info(f"Found {len(tables)} tables for {gauge_id}, param {param_id}.") + if len(tables) <= 4: + logger.warning(f"No data table found for {gauge_id}, param {param_id} (expected more than 4 tables).") + return None + + df = tables[4].copy().dropna(how="all").reset_index(drop=True) + logger.debug(f"Table 4 head:\n{df.head()}") + + # Detect header pattern + if len(df) >= 2 and "Data" in str(df.iloc[0, 0]) and "Data" in str(df.iloc[1, 0]): + df = df.iloc[2:].copy() + df.columns = [constants.TIME_INDEX, variable] + elif "Data" in str(df.iloc[0, 0]): + df.columns = [html.unescape(str(c)).strip() for c in df.iloc[0]] + df = df.iloc[1:].copy() + df.columns = [constants.TIME_INDEX, variable] + else: + df.columns = [constants.TIME_INDEX, variable] + + # Clean and format + df[constants.TIME_INDEX] = pd.to_datetime(df[constants.TIME_INDEX], errors="coerce", format="%d/%m/%Y %H:%M") + df[variable] = pd.to_numeric(df[variable], errors="coerce") + + return df + + def _parse_data(self, gauge_id: str, raw_data: Optional[pd.DataFrame], variable: str) -> pd.DataFrame: + """Parses the raw DataFrame.""" + if raw_data is None or raw_data.empty: + return pd.DataFrame(columns=[constants.TIME_INDEX, variable]) + + return raw_data.set_index(constants.TIME_INDEX) + + def get_data( + self, + gauge_id: str, + variable: str, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + ) -> pd.DataFrame: + """Fetches and parses Portuguese river gauge data.""" + start_date = utils.format_start_date(start_date) + end_date = utils.format_end_date(end_date) + + if variable not in self.get_available_variables(): + raise ValueError(f"Unsupported variable: {variable}") + + try: + df = self._download_data(gauge_id, variable, start_date, end_date) + if df is None or df.empty: + return pd.DataFrame(columns=[constants.TIME_INDEX, variable]) + + df = self._parse_data(gauge_id, df, variable) + + # Filter by date range + start_date_dt = pd.to_datetime(start_date) + end_date_dt = pd.to_datetime(end_date) + df = df[(df.index >= start_date_dt) & (df.index <= end_date_dt)] + except Exception as e: + logger.error(f"Failed to get data for site {gauge_id}, variable {variable}: {e}") + return pd.DataFrame(columns=[constants.TIME_INDEX, variable]) + return df diff --git a/tests/test_data/portugal_19B01H_stage.html b/tests/test_data/portugal_19B01H_stage.html new file mode 100644 index 0000000..956fe15 --- /dev/null +++ b/tests/test_data/portugal_19B01H_stage.html @@ -0,0 +1,147 @@ + + + + + + + + + SNIRH > Dados de Base + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Data
Nível médio diário (m)
16/02/2002 00:00 + 0.36
17/02/2002 00:00 + 0.35
18/02/2002 00:00 + 0.37
19/02/2002 00:00 + 0.38
20/02/2002 00:00 + 0.35
+ + +
+
+
+ +
 
+ + + + \ No newline at end of file diff --git a/tests/test_portugal.py b/tests/test_portugal.py new file mode 100644 index 0000000..e635758 --- /dev/null +++ b/tests/test_portugal.py @@ -0,0 +1,46 @@ +import os +import unittest +from unittest.mock import MagicMock, patch + +import pandas as pd +from pandas.testing import assert_frame_equal + +from rivretrieve import PortugalFetcher, constants + + +class TestPortugalFetcher(unittest.TestCase): + def setUp(self): + self.fetcher = PortugalFetcher() + self.test_data_dir = os.path.join(os.path.dirname(__file__), "test_data") + + def _load_mock_html(self, filename): + with open(os.path.join(self.test_data_dir, filename), "r") as f: + return f.read() + + @patch("rivretrieve.utils.requests_retry_session") + def test_get_data_stage(self, mock_session): + mock_response = MagicMock() + mock_html = self._load_mock_html("portugal_19B01H_stage.html") + mock_response.text = mock_html + mock_response.raise_for_status = MagicMock() + mock_session.return_value.get.return_value = mock_response + + gauge_id = "19B/01H" + variable = constants.STAGE_DAILY_MEAN + start_date = "2002-02-16" + end_date = "2002-02-19" + + result_df = self.fetcher.get_data(gauge_id, variable, start_date, end_date) + + expected_dates = pd.to_datetime(["2002-02-16", "2002-02-17", "2002-02-18", "2002-02-19"]) + expected_values = [0.36, 0.35, 0.37, 0.38] + expected_df = pd.DataFrame( + {constants.TIME_INDEX: expected_dates, constants.STAGE_DAILY_MEAN: expected_values} + ).set_index(constants.TIME_INDEX) + + assert_frame_equal(result_df, expected_df) + mock_session.return_value.get.assert_called_once() + + +if __name__ == "__main__": + unittest.main()