diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc8b7f45..c176da6d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ - xxx +# [0.1.20.3] - 2026-07-15 + +## Added +- ESDL profile reading and writing for different types: influxdb and postgres, and inline (in ESDL file). + +## Changed +- Write_result_db_profiles has been replaced by esdl_profiles_output_type (determines the ESDL output profile type) +- Names used for keys in "database_connections" specified +- Using pyesdl for database credential management + +## Fixed +- xxx + + # [0.1.20.2] - 2026-07-15 ## Added @@ -392,4 +406,3 @@ ## Fixed - Error exit when stage 1, priority 1 was not successful - diff --git a/examples/PoCTutorial/src/run_grow_tutorial.py b/examples/PoCTutorial/src/run_grow_tutorial.py index 6b02e8f42..e50dda68a 100644 --- a/examples/PoCTutorial/src/run_grow_tutorial.py +++ b/examples/PoCTutorial/src/run_grow_tutorial.py @@ -1,6 +1,6 @@ from pathlib import Path -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType from mesido.esdl.esdl_parser import ESDLFileParser from mesido.workflows import EndScenarioSizingStaged, run_end_scenario_sizing @@ -13,16 +13,17 @@ class EndScenarioSizingStagedHighs(EndScenarioSizingStaged): import time kwargs = { - "write_result_db_profiles": True, + # '"esdl_output_profiles_type"=None' (default) means no profiles to DB + "esdl_output_profiles_type": ESDLOutputProfilesType.INFLUXDB, "database_connections": [ { "access_type": DBAccessType.WRITE, # DBAccessType.READ or DBAccessType.READ_WRITE - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/examples/municipality/src/run_municipality.py b/examples/municipality/src/run_municipality.py index dc529d66d..bc1d35784 100644 --- a/examples/municipality/src/run_municipality.py +++ b/examples/municipality/src/run_municipality.py @@ -16,12 +16,12 @@ "database_connections": [ { "access_type": DBAccessType.READ, # DBAccessType.WRITE or DBAccessType.READ_WRITE - "influxdb_host": "required_user_input", - "influxdb_port": 1234, - "influxdb_username": "required_user_input", - "influxdb_password": "required_user_input", - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "required_user_input", + "port": 1234, + "username": "required_user_input", + "password": "required_user_input", + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/manual_run/testing_bugs/src/run_grow.py b/manual_run/testing_bugs/src/run_grow.py index 06668149a..696c66d59 100644 --- a/manual_run/testing_bugs/src/run_grow.py +++ b/manual_run/testing_bugs/src/run_grow.py @@ -2,7 +2,7 @@ from mesido.esdl.esdl_parser import ESDLFileParser from mesido.workflows import EndScenarioSizingStaged, run_end_scenario_sizing -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType if __name__ == "__main__": import time @@ -11,27 +11,28 @@ base_folder = Path(__file__).resolve().parent.parent kwargs = { - "use_esdl_ranged_constraint": True, # default value in the code is set to False + "esdl_output_profiles_type": ESDLOutputProfilesType.INFLUXDB, + # None (default) or ESDLOutputProfilesType.POSTGRESQL or + # ESDLOutputProfilesType.TIME_SERIES_PROFILE or ESDLOutputProfilesType.DATE_TIME_PROFILE "database_connections": [ { "access_type": DBAccessType.READ, # or DBAccessType.WRITE or DBAccessType.READ_WRITE - "influxdb_host": "required_user_input", - "influxdb_port": 1234, - "influxdb_username": "required_user_input", - "influxdb_password": "required_user_input", - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "required_user_input", + "port": 1234, + "username": "required_user_input", + "password": "required_user_input", + "ssl": False, + "verify_ssl": False, }, { - "access_type": DBAccessType.WRITE, # or DBAccessType.WRITE or DBAccessType.READ_WRITE - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "access_type": DBAccessType.READ_WRITE, + "host": "localhost", + "port": 8086, + "username": "root", + "password": "9012", + "ssl": False, + "verify_ssl": False, }, - ], } diff --git a/setup.py b/setup.py index fea235196..724b430d9 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ # < 81.0.0 needed for pandapipes (still to be removed) # < 82.0.0 needed for pkg_resources (used in rtctools) "setuptools <= 80.9.0", - "pyesdl == 26.6.1", + "pyesdl[profiles] == 26.7", "pandas >= 1.3.1, < 2.0", "casadi-gil-comp == 3.6.7", "StrEnum == 0.4.15", diff --git a/src/mesido/asset_sizing_mixin.py b/src/mesido/asset_sizing_mixin.py index 382d5419f..533599246 100644 --- a/src/mesido/asset_sizing_mixin.py +++ b/src/mesido/asset_sizing_mixin.py @@ -12,7 +12,7 @@ from mesido.esdl.asset_to_component_base import AssetStateEnum from mesido.esdl.common import Asset from mesido.esdl.esdl_additional_vars_mixin import get_asset_contraints -from mesido.esdl.profile_parser import InfluxDBProfileReader +from mesido.esdl.profile_parser import ESDLProfileReader from mesido.head_loss_class import HeadLossOption from mesido.network_common import NetworkSettings from mesido.pipe_class import CableClass, GasPipeClass, PipeClass @@ -817,7 +817,7 @@ def _get_ub_profile_constraint(asset_name: str, profile_name: str, ub: float) -> if ( qty_asset_profile_constraints > 0 and hasattr(asset_profile_constraints[0], "maximum") - and InfluxDBProfileReader._get_profile_quantity_and_unit( + and ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.WATT @@ -1785,7 +1785,7 @@ def __producer_constraints( if ( qty_asset_profile_constraints > 0 and hasattr(asset_profile_constraints[0], "maximum") - and InfluxDBProfileReader._get_profile_quantity_and_unit( + and ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.WATT @@ -1828,16 +1828,16 @@ def __producer_constraints( # profile is specified without units (xlm/csv) qty_asset_profile_constraints == 0 or ( - InfluxDBProfileReader._get_profile_quantity_and_unit( + ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).physicalQuantity == esdl.PhysicalQuantityEnum.COEFFICIENT and ( - InfluxDBProfileReader._get_profile_quantity_and_unit( + ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.PERCENT - or InfluxDBProfileReader._get_profile_quantity_and_unit( + or ESDLProfileReader._get_profile_quantity_and_unit( asset_profile_constraints[0].maximum ).unit == esdl.UnitEnum.NONE diff --git a/src/mesido/esdl/esdl_mixin.py b/src/mesido/esdl/esdl_mixin.py index 9fcf3daee..425fc1f8a 100644 --- a/src/mesido/esdl/esdl_mixin.py +++ b/src/mesido/esdl/esdl_mixin.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Tuple import esdl.esdl_handler +from esdl.profiles.credentials import Credentials from mesido.component_type_mixin import ( ModelicaComponentTypeMixin, @@ -21,7 +22,7 @@ from mesido.esdl.esdl_model_base import _ESDLModelBase from mesido.esdl.esdl_parser import ESDLStringParser from mesido.esdl.esdl_qth_model import ESDLQTHModel -from mesido.esdl.profile_parser import BaseProfileReader, InfluxDBProfileReader +from mesido.esdl.profile_parser import BaseProfileReader, ESDLProfileReader from mesido.physics_mixin import PhysicsMixin from mesido.pipe_class import GasPipeClass, PipeClass from mesido.pycml.pycml_mixin import PyCMLMixin @@ -57,6 +58,17 @@ class DBAccessType(StrEnum): READ_WRITE = "read_write" +class ESDLOutputProfilesType(StrEnum): + """ + Enumeration for ESDL output profiles type + """ + + INFLUXDB = "influxdb" + POSTGRESQL = "postgresql" + TIME_SERIES_PROFILE = "time_series_profile" + DATE_TIME_PROFILE = "date_time_profile" + + class _ESDLInputException(Exception): pass @@ -132,7 +144,7 @@ def __init__(self, *args, **kwargs) -> None: DBAccessType.WRITE: [], } - profile_reader_class = kwargs.get("profile_reader", InfluxDBProfileReader) + profile_reader_class = kwargs.get("profile_reader", ESDLProfileReader) input_file_name = kwargs.get("input_timeseries_file", None) input_folder = kwargs.get("input_folder") input_file_path = None @@ -141,24 +153,26 @@ def __init__(self, *args, **kwargs) -> None: database_connection_info = kwargs.get("database_connections", {}) read_only_dbase_credentials: Dict[str, Tuple[str, str]] = {} # for profile reader for dbconnection in database_connection_info: + database_host_port = "{}:{}".format( + dbconnection["host"], + dbconnection["port"], + ) if dbconnection["access_type"] != DBAccessType.WRITE: - database_host_port = "{}:{}".format( - dbconnection["influxdb_host"], - dbconnection["influxdb_port"], - ) read_only_dbase_credentials[database_host_port] = ( - dbconnection["influxdb_username"], - dbconnection["influxdb_password"], + dbconnection["username"], + dbconnection["password"], ) if dbconnection["access_type"] != DBAccessType.READ_WRITE: self._database_credentials[dbconnection["access_type"]].append( { - "influxdb_host": dbconnection["influxdb_host"], - "influxdb_port": dbconnection["influxdb_port"], - "influxdb_username": dbconnection["influxdb_username"], - "influxdb_password": dbconnection["influxdb_password"], - "influxdb_ssl": dbconnection["influxdb_ssl"], - "influxdb_verify_ssl": dbconnection["influxdb_verify_ssl"], + "host": dbconnection["host"], + "port": dbconnection["port"], + "username": dbconnection["username"], + "password": dbconnection["password"], + "ssl": dbconnection["ssl"] if "ssl" in dbconnection else False, + "verify_ssl": ( + dbconnection["verify_ssl"] if "verify_ssl" in dbconnection else False + ), } ) elif dbconnection["access_type"] == DBAccessType.READ_WRITE: @@ -166,12 +180,16 @@ def __init__(self, *args, **kwargs) -> None: for rw in both_read_and_write: self._database_credentials[rw].append( { - "influxdb_host": dbconnection["influxdb_host"], - "influxdb_port": dbconnection["influxdb_port"], - "influxdb_username": dbconnection["influxdb_username"], - "influxdb_password": dbconnection["influxdb_password"], - "influxdb_ssl": dbconnection["influxdb_ssl"], - "influxdb_verify_ssl": dbconnection["influxdb_verify_ssl"], + "host": dbconnection["host"], + "port": dbconnection["port"], + "username": dbconnection["username"], + "password": dbconnection["password"], + "ssl": dbconnection["ssl"] if "ssl" in dbconnection else False, + "verify_ssl": ( + dbconnection["verify_ssl"] + if "verify_ssl" in dbconnection + else False + ), } ) else: @@ -181,6 +199,10 @@ def __init__(self, *args, **kwargs) -> None: ) sys.exit(1) + Credentials.add_credential( + database_host_port, dbconnection["username"], dbconnection["password"] + ) + if input_file_name is not None: input_file_path = Path(input_folder) / input_file_name @@ -188,7 +210,6 @@ def __init__(self, *args, **kwargs) -> None: self.__profile_reader: BaseProfileReader = profile_reader_class( energy_system=self.__energy_system_handler.energy_system, file_path=input_file_path, - database_credentials=read_only_dbase_credentials, use_esdl_ranged_contraint=self._ESDLMixin__use_esdl_ranged_constraint, ) else: # read from a file, no database credentials needed diff --git a/src/mesido/esdl/profile_parser.py b/src/mesido/esdl/profile_parser.py index 2ca4aa5ea..169b9d45c 100644 --- a/src/mesido/esdl/profile_parser.py +++ b/src/mesido/esdl/profile_parser.py @@ -3,11 +3,10 @@ import sys from collections import defaultdict from pathlib import Path -from typing import Dict, Optional, Set, Tuple +from typing import Dict, Optional, Set, Union import esdl -from esdl.profiles.influxdbprofilemanager import ConnectionSettings -from esdl.profiles.influxdbprofilemanager import InfluxDBProfileManager +from esdl.profiles.profile_utils import close_db_connections, load_profile_data_and_header from esdl.units.conversion import ENERGY_IN_J, POWER_IN_W, convert_to_unit from mesido.esdl.common import Asset @@ -241,7 +240,7 @@ def _load_profiles_from_source( raise NotImplementedError -class InfluxDBProfileReader(BaseProfileReader): +class ESDLProfileReader(BaseProfileReader): asset_type_to_variable_name_conversion = { esdl.esdl.HeatingDemand: ".target_heat_demand", esdl.esdl.CoolingDemand: ".target_cold_demand", @@ -256,22 +255,30 @@ class InfluxDBProfileReader(BaseProfileReader): esdl.esdl.PVInstallation: ".maximum_electricity_source", } + supported_profiles = ( + esdl.DataTableProfile, + esdl.InfluxDBProfile, + esdl.TimeSeriesProfile, + esdl.DateTimeProfile, + esdl.ProfileReference, + ) + + # Python 3.10 compatible + SupportedProfilesType = Union.__getitem__(tuple(supported_profiles)) + # Do not delete code below. To be used when min Python ver >3.10 + # SupportedProfilesType = Union[*supported_profiles] + def __init__( self, energy_system: esdl.EnergySystem, file_path: Optional[Path], use_esdl_ranged_contraint: bool = False, - database_credentials: Optional[Dict[str, Tuple[str, str]]] = None, ): super().__init__( energy_system=energy_system, file_path=file_path, use_esdl_ranged_contraint=use_esdl_ranged_contraint, ) - self._df = pd.DataFrame() - self._database_credentials = ( - database_credentials if database_credentials is not None else {"": ("", "")} - ) def _load_profiles_from_source( self, @@ -290,7 +297,7 @@ def _load_profiles_from_source( unique_profiles_attributes = [] # a list containning lists of attributes unique_series = [] for profile in [ - x for x in self._energy_system.eAllContents() if isinstance(x, esdl.InfluxDBProfile) + x for x in self._energy_system.eAllContents() if isinstance(x, self.supported_profiles) ]: if profile.profileType == esdl.ProfileTypeEnum.OUTPUT: continue @@ -335,11 +342,13 @@ def _load_profiles_from_source( f"specified to be loaded for each asset covers exactly the " f"same timeseries. " ) + close_db_connections() + # Loop trough all the requried profiles in the energy system and assign the profile data: # - series: use the unique series data, without reading from the database again # - other profile info: get it from the specific profile for profile in [ - x for x in self._energy_system.eAllContents() if isinstance(x, esdl.InfluxDBProfile) + x for x in self._energy_system.eAllContents() if isinstance(x, self.supported_profiles) ]: if profile.profileType == esdl.ProfileTypeEnum.OUTPUT: continue @@ -413,7 +422,7 @@ def _load_profiles_from_source( self._profiles[idx] = profiles.copy() # @staticmethod - def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) -> pd.Series: + def _load_profile_timeseries_from_database(self, profile: SupportedProfilesType) -> pd.Series: """ Function to load the profiles from an InfluxDB. Returns a timeseries with the data for the asset. @@ -430,54 +439,25 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) # to avoid circular import issue from mesido.workflows.utils.error_types import NetworkErrors, potential_error_to_error - if profile.id in self._df: - return self._df[profile.id] - - profile_host = profile.host - - ssl_setting = False - if "https" in profile_host: - profile_host = profile_host[8:] - ssl_setting = True - elif "http" in profile_host: - profile_host = profile_host[7:] - if profile.port == 443: - ssl_setting = True - influx_host = "{}:{}".format(profile_host, profile.port) - - username, password = self._database_credentials.get(influx_host, (None, None)) - - conn_settings = ConnectionSettings( - host=profile.host, - port=profile.port, - username=username, - password=password, - database=profile.database, - ssl=ssl_setting, - verify_ssl=ssl_setting, - ) - try: - time_series_data = InfluxDBProfileManager(conn_settings) - except Exception: + profile_raw_data, _ = load_profile_data_and_header( + profile, + enable_cache=True, + apply_multiplier=False, + close_connection_after_load=False, + ) + except Exception as e: container = profile.eContainer() asset = container.energyasset get_potential_errors().add_potential_issue( MesidoAssetIssueType.ASSET_PROFILE_AVAILABILITY, asset.id, - f"Asset named {asset.name}: Database {profile.database}" - f" is not available in the host.", + f"Error retrieving profile for asset '{asset.name}' from host '{profile.host}'" + f" and database '{profile.database}': {str(e)}.", ) potential_error_to_error(NetworkErrors.HEAT_NETWORK_ERRORS) - time_series_data.load_influxdb( - profile.measurement, - [profile.field], - profile.startDate, - profile.endDate, - ) - - if not time_series_data.profile_data_list: # if time_series_data.profile_data_list == []: + if not profile_raw_data: container = profile.eContainer() asset = container.energyasset get_potential_errors().add_potential_issue( @@ -489,15 +469,15 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) potential_error_to_error(NetworkErrors.HEAT_NETWORK_ERRORS) - for x in time_series_data.profile_data_list: + for x in profile_raw_data: if len(x) != 2: raise RuntimeError( "InfluxDB profile currently only supports parsing exactly one " "profile for each asset" ) - x_0, x_1 = map(list, zip(*time_series_data.profile_data_list)) - if not time_series_data.profile_data_list[0][0].tzinfo: + x_0, x_1 = map(list, zip(*profile_raw_data)) + if not profile_raw_data[0][0].tzinfo: index = pd.DatetimeIndex( data=x_0, tz=datetime.timezone.utc, @@ -508,7 +488,6 @@ def _load_profile_timeseries_from_database(self, profile: esdl.InfluxDBProfile) data = x_1 series = pd.Series(data=data, index=index) - self._df[profile.id] = series return series diff --git a/src/mesido/util.py b/src/mesido/util.py index 6a7777c05..76d0bafec 100644 --- a/src/mesido/util.py +++ b/src/mesido/util.py @@ -1,5 +1,5 @@ from mesido.esdl.esdl_parser import BaseESDLParser -from mesido.esdl.profile_parser import BaseProfileReader, InfluxDBProfileReader +from mesido.esdl.profile_parser import BaseProfileReader, ESDLProfileReader from rtctools.optimization.collocated_integrated_optimization_problem import ( CollocatedIntegratedOptimizationProblem, @@ -13,7 +13,7 @@ def run_esdl_mesido_optimization( esdl_file_name: str = "", base_folder: str = "", esdl_string: str = "", - profile_reader: BaseProfileReader = InfluxDBProfileReader, + profile_reader: BaseProfileReader = ESDLProfileReader, input_timeseries_file: str = "", *args, **kwargs, diff --git a/src/mesido/workflows/gas_elect_workflow.py b/src/mesido/workflows/gas_elect_workflow.py index 946aa1caf..6870699ce 100644 --- a/src/mesido/workflows/gas_elect_workflow.py +++ b/src/mesido/workflows/gas_elect_workflow.py @@ -196,16 +196,16 @@ def main(runinfo_path, log_level): logger.info("Gas and electricity workflow") kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/src/mesido/workflows/grow_workflow.py b/src/mesido/workflows/grow_workflow.py index 228aca403..9413c75d3 100644 --- a/src/mesido/workflows/grow_workflow.py +++ b/src/mesido/workflows/grow_workflow.py @@ -1043,16 +1043,16 @@ def main(runinfo_path, log_level): logger.info("Run Scenario Sizing") kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/src/mesido/workflows/io/write_output.py b/src/mesido/workflows/io/write_output.py index 6143d1f94..3561cea5a 100644 --- a/src/mesido/workflows/io/write_output.py +++ b/src/mesido/workflows/io/write_output.py @@ -11,14 +11,20 @@ from typing import Dict, Union import esdl -from esdl.profiles.influxdbprofilemanager import ConnectionSettings -from esdl.profiles.influxdbprofilemanager import InfluxDBProfileManager -from esdl.profiles.profilemanager import ProfileManager +from esdl.profiles.datatableprofilemanager import DataTableProfileManager +from esdl.profiles.profile_utils import ( + append_values_to_date_time_profile, + close_db_connections, + create_data_table_profile, + create_date_time_profile, + create_time_series_profile, + save_data_table_profiles_to_database, +) import mesido.esdl.esdl_parser from mesido.constants import GRAVITATIONAL_CONSTANT from mesido.esdl.edr_pipe_class import EDRPipeClass -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType from mesido.financial_mixin import calculate_annuity_factor from mesido.network_common import NetworkSettings from mesido.post_processing.post_processing_utils import pipe_pressure, pipe_velocity @@ -33,6 +39,8 @@ logger = logging.getLogger("mesido") +OUTPUT_TIMESERIESPROFILE_TIME_STEP_SECONDS = 3600 + class ScenarioOutput: __optimized_energy_system_handler = None @@ -42,24 +50,29 @@ def __init__(self, **kwargs): self.model_folder = kwargs.get("model_folder") self.output_folder = kwargs.get("output_folder") self.esdl_file_name = kwargs.get("esdl_file_name", "ESDL_file.esdl") - # Settings for influxdb when writing out result profile data to it + # Settings for DataTableProfile when writing out result profile data to it # Default settings - self.write_result_db_profiles = False - self.influxdb_username = None - self.influxdb_password = None + self.username = None + self.password = None + self.esdl_output_profiles_type = kwargs.get("esdl_output_profiles_type", None) + self.write_esdl_profiles_to_db = self.esdl_output_profiles_type in [ + ESDLOutputProfilesType.POSTGRESQL, + ESDLOutputProfilesType.INFLUXDB, + ] base_error_string = "Missing influxdb setting for writing result profile data:" try: - self.write_result_db_profiles = kwargs["write_result_db_profiles"] - - if self.write_result_db_profiles not in [True, False]: + if self.esdl_output_profiles_type is not None and not isinstance( + self.esdl_output_profiles_type, ESDLOutputProfilesType + ): logger.error( - "Current setting of write_result_db_profiles is: " - f"{self.write_result_db_profiles} and it should be set to True or False" + "Current setting of esdl_output_profiles_type is: " + f"{self.esdl_output_profiles_type} and it should be None " + "or a value of ESDLOutputProfilesType" ) sys.exit(1) - if self.write_result_db_profiles: + if self.esdl_output_profiles_type: database_connection_write = self._database_credentials.get(DBAccessType.WRITE, []) if len(database_connection_write) == 0: @@ -74,10 +87,10 @@ def __init__(self, **kwargs): else: database_connection_input = database_connection_write[0] try: - self.influxdb_host = database_connection_input["influxdb_host"] - if len(self.influxdb_host) == 0: + self.host = database_connection_input["host"] + if len(self.host) == 0: logger.error( - "Current setting of influxdb_host is an empty string and it should" + "Current setting of host is an empty string and it should" " be the name of the host" ) sys.exit(1) @@ -85,43 +98,43 @@ def __init__(self, **kwargs): logger.error(f"{base_error_string} host") sys.exit(1) try: - self.influxdb_port = database_connection_input["influxdb_port"] - if not isinstance(self.influxdb_port, int): + self.port = database_connection_input["port"] + if not isinstance(self.port, int): logger.error( - "Current setting of influxdb_port is: " - f"{self.influxdb_port} and it should be set to int value (port number)" + "Current setting of port is: " + f"{self.port} and it should be set to int value (port number)" ) sys.exit(1) except KeyError: logger.error(f"{base_error_string} port") sys.exit(1) try: - self.influxdb_username = database_connection_input["influxdb_username"] + self.username = database_connection_input["username"] except KeyError: logger.error(f"{base_error_string} username") sys.exit(1) try: - self.influxdb_password = database_connection_input["influxdb_password"] + self.password = database_connection_input["password"] except KeyError: logger.error(f"{base_error_string} password") sys.exit(1) try: - self.influxdb_ssl = database_connection_input["influxdb_ssl"] - if self.influxdb_ssl not in [True, False]: + self.ssl = database_connection_input["ssl"] + if self.ssl not in [True, False]: logger.error( - "Current setting of influxdb_ssl is: " - f"{self.influxdb_ssl} and it should be set to True or False" + "Current setting of ssl is: " + f"{self.ssl} and it should be set to True or False" ) sys.exit(1) except KeyError: logger.error(f"{base_error_string} ssl") sys.exit(1) try: - self.influxdb_verify_ssl = database_connection_input["influxdb_verify_ssl"] - if self.influxdb_verify_ssl not in [True, False]: + self.verify_ssl = database_connection_input["verify_ssl"] + if self.verify_ssl not in [True, False]: logger.error( - "Current setting of influxdb_verify_ssl is: " - f"{self.influxdb_verify_ssl} and it should be set to True or False" + "Current setting of verify_ssl is: " + f"{self.verify_ssl} and it should be set to True or False" ) sys.exit(1) except KeyError: @@ -134,6 +147,49 @@ def __init__(self, **kwargs): def get_optimized_esh(self): return self.__optimized_energy_system_handler + def _resample_profile_data_to_fixed_timestep(self, profile_data): + """ + Resample the profile data to a fixed timestep defined by + OUTPUT_TIMESERIESPROFILE_TIME_STEP_SECONDS using a staircase (zero-order hold) method. + + Parameters + ---------- + profile_data : list of tuples + Each tuple contains a datetime and a corresponding value. + + Returns + ------- + list + Resampled values at the fixed timestep. + """ + if not profile_data: + return [] + + sorted_profile_data = sorted(profile_data, key=lambda row: row[0]) + step = datetime.timedelta(seconds=OUTPUT_TIMESERIESPROFILE_TIME_STEP_SECONDS) + + start_datetime = sorted_profile_data[0][0] + end_datetime = sorted_profile_data[-1][0] + duration_seconds = (end_datetime - start_datetime).total_seconds() + n_steps = int(np.ceil(duration_seconds / OUTPUT_TIMESERIESPROFILE_TIME_STEP_SECONDS)) + 1 + + values = [] + row_index = 0 + latest_value = sorted_profile_data[0][1] + + for step_index in range(n_steps): + target_datetime = start_datetime + step_index * step + while ( + row_index + 1 < len(sorted_profile_data) + and sorted_profile_data[row_index + 1][0] <= target_datetime + ): + row_index += 1 + latest_value = sorted_profile_data[row_index][1] + + values.append(latest_value) + + return values + def _write_html_output(self, template_name="mpc_buffer_sizing_output"): from jinja2 import Environment, FileSystemLoader @@ -505,7 +561,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -532,7 +589,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -558,7 +616,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -571,10 +630,12 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) distribution=esdl.StringLabelDistribution( stringItem=[ esdl.StringItem( - label="Installation", value=tot_timehorizon_install_cost_euro + label="Installation", + value=tot_timehorizon_install_cost_euro, ), esdl.StringItem( - label="Investment", value=tot_timehorizon_invest_cost_euro + label="Investment", + value=tot_timehorizon_invest_cost_euro, ), esdl.StringItem( label="Variable OPEX", @@ -587,7 +648,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -602,7 +664,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -618,7 +681,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -634,7 +698,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -652,7 +717,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) ] ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.ENERGY, unit=esdl.UnitEnum.WATTHOUR + physicalQuantity=esdl.PhysicalQuantityEnum.ENERGY, + unit=esdl.UnitEnum.WATTHOUR, ), ) ) @@ -788,7 +854,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) 100.0, ) estimated_energy_from_regional_source_perc[subarea.name] = min( - (100.0 - estimated_energy_from_local_source_perc[subarea.name]), 100.0 + (100.0 - estimated_energy_from_local_source_perc[subarea.name]), + 100.0, ) else: estimated_energy_from_local_source_perc[subarea.name] = 0.0 @@ -870,7 +937,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) kpis.kpi.append( esdl.DoubleKPI( value=round( - estimated_energy_from_regional_source_perc[subarea.name], 1 + estimated_energy_from_regional_source_perc[subarea.name], + 1, ), name="Estimated energy from regional source(s)", quantityAndUnit=esdl.esdl.QuantityAndUnitType( @@ -929,7 +997,8 @@ def _add_kpis_to_energy_system(self, energy_system, optimizer_sim: bool = False) stringItem=polygon_area_string_item ), quantityAndUnit=esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.COST, unit=esdl.UnitEnum.EURO + physicalQuantity=esdl.PhysicalQuantityEnum.COST, + unit=esdl.UnitEnum.EURO, ), ) ) @@ -1070,7 +1139,12 @@ def _write_updated_esdl( logger.warning(f"ESDL update: asset {asset_id} has not been updated") # Pipes: - edr_pipe_properties_to_copy = ["innerDiameter", "outerDiameter", "diameter", "material"] + edr_pipe_properties_to_copy = [ + "innerDiameter", + "outerDiameter", + "diameter", + "material", + ] esh_edr = EnergySystemHandler() @@ -1135,20 +1209,10 @@ def _write_updated_esdl( # - Fields: profile value for the specific variable # - Tags used as filters: simulationRun, assetClass, assetName, assetId, capability - if self.write_result_db_profiles: - logger.info("Writing asset result profile data to influxDB") + if self.esdl_output_profiles_type: + logger.info(f"Writing asset results to profile type '{self.esdl_output_profiles_type}'") results = self.extract_results() - influxdb_conn_settings = ConnectionSettings( - host=self.influxdb_host, - port=self.influxdb_port, - username=self.influxdb_username, - password=self.influxdb_password, - database=output_energy_system_id, - ssl=self.influxdb_ssl, - verify_ssl=self.influxdb_verify_ssl, - ) - capabilities = [ esdl.Transport, esdl.Conversion, @@ -1318,10 +1382,6 @@ def _write_updated_esdl( for v in var_pops: variables_two_hydraulic_system.remove(v) - profiles = ProfileManager() - profiles.profile_type = "DATETIME_LIST" - profiles.profile_header = ["datetime"] # + general_headers - # Get index of outport which will be used to assign the profile data to index_outport = -1 for ip in range(len(asset.port)): @@ -1342,14 +1402,9 @@ def _write_updated_esdl( ) sys.exit(1) + asset_esdl_output_profiles = {} + asset_esdl_output_profiles_data_dict = {} for ii in range(len(self.times())): - if not self.io.datetimes[ii].tzinfo: - data_row = [ - self.io.datetimes[ii].replace(tzinfo=datetime.timezone.utc) - ] - else: - data_row = [self.io.datetimes[ii]] - try: # For all components dealing with one hydraulic system if isinstance( @@ -1373,10 +1428,16 @@ def _write_updated_esdl( traceback.print_exc() sys.exit(1) - for variable in variables_names: + for variable_name in variables_names: + if not self.io.datetimes[ii].tzinfo: + data_row = [ + self.io.datetimes[ii].replace(tzinfo=datetime.timezone.utc) + ] + else: + data_row = [self.io.datetimes[ii]] + if ii == 0: - # Set header for each column - profiles.profile_header.append(variable) + asset_esdl_output_profiles_data_dict[variable_name] = [] # Set profile database attributes for the esdl asset if not self.io.datetimes[0].tzinfo: start_date_time = self.io.datetimes[0].replace( @@ -1385,7 +1446,7 @@ def _write_updated_esdl( logger.warning( f"No timezone specified for the output profile: " f"default UTC has been used for asset {asset_name} " - f"variable {variable}" + f"variable {variable_name}" ) else: start_date_time = self.io.datetimes[0] @@ -1396,81 +1457,117 @@ def _write_updated_esdl( else: end_date_time = self.io.datetimes[-1] - profile_attributes = esdl.InfluxDBProfile( - database=output_energy_system_id, - measurement=carrier_id, - field=profiles.profile_header[-1], - port=self.influxdb_port, - host=self.influxdb_host, - startDate=start_date_time, - endDate=end_date_time, - id=str(uuid.uuid4()), - filters='"assetId"=' + f"'{str(asset_id)}'", - profileType=esdl.ProfileTypeEnum.OUTPUT, - ) # Assign quantity and units variable - if variable in ["Heat_flow", "Pump_power"]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.POWER, - unit=esdl.UnitEnum.WATT, - multiplier=esdl.MultiplierEnum.NONE, - ) + if variable_name in ["Heat_flow", "Pump_power"]: + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.POWER, + unit=esdl.UnitEnum.WATT, + multiplier=esdl.MultiplierEnum.NONE, ) - elif variable in [ + + elif variable_name in [ f"{commodity}In.H", f"Primary.{commodity}In.H", f"Secondary.{commodity}In.H", ]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.PRESSURE, - unit=esdl.UnitEnum.PASCAL, - multiplier=esdl.MultiplierEnum.NONE, - ) + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.PRESSURE, + unit=esdl.UnitEnum.PASCAL, + multiplier=esdl.MultiplierEnum.NONE, ) - elif variable in [ + + elif variable_name in [ f"{commodity}In.Q", f"Primary.{commodity}In.Q", f"Secondary.{commodity}In.Q", ]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.FLOW, - unit=esdl.UnitEnum.CUBIC_METRE, - perTimeUnit=esdl.TimeUnitEnum.SECOND, - multiplier=esdl.MultiplierEnum.NONE, - ) + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.FLOW, + unit=esdl.UnitEnum.CUBIC_METRE, + perTimeUnit=esdl.TimeUnitEnum.SECOND, + multiplier=esdl.MultiplierEnum.NONE, ) - elif variable in ["PostProc.Velocity"]: - profile_attributes.profileQuantityAndUnit = ( - esdl.esdl.QuantityAndUnitType( - physicalQuantity=esdl.PhysicalQuantityEnum.SPEED, - unit=esdl.UnitEnum.METRE, - perTimeUnit=esdl.TimeUnitEnum.SECOND, - multiplier=esdl.MultiplierEnum.NONE, - ) + + elif variable_name in ["PostProc.Velocity"]: + quantity_and_unit = esdl.esdl.QuantityAndUnitType( + physicalQuantity=esdl.PhysicalQuantityEnum.SPEED, + unit=esdl.UnitEnum.METRE, + perTimeUnit=esdl.TimeUnitEnum.SECOND, + multiplier=esdl.MultiplierEnum.NONE, ) + else: logger.warning( f"No profile units will be written to the ESDL for: " - f"{asset_id}. + {variable}" + f"{asset_id}. + {variable_name}" ) # Write the source of profiles (Optimizer) - profile_attributes.dataSource = esdl.esdl.DataSource( + data_source = esdl.DataSource( id=str(uuid.uuid4()), name="Optimizer", description="This was created in the optimizer", type=esdl.DataSourceTypeEnum.MODEL, ) - - # Write result OUTPUT profiles on the optimized esdl - asset.port[index_outport].profile.append(profile_attributes) + if self.write_esdl_profiles_to_db: + if ( + self.esdl_output_profiles_type + == ESDLOutputProfilesType.POSTGRESQL + ): + db_type = esdl.DatabaseTypeEnum.POSTGRESQL + else: + db_type = esdl.DatabaseTypeEnum.INFLUXDB + esdl_profile = create_data_table_profile( + es=energy_system, + database_name=output_energy_system_id, + table_name=carrier_id, + column_name=variable_name, + start_date=start_date_time, + end_date=end_date_time, + db_host=self.host, + db_port=self.port, + filter='"assetId"=' + f"'{str(asset_id)}'", + db_type=db_type, + profile_type=esdl.ProfileTypeEnum.OUTPUT, + quantity_and_unit_type=quantity_and_unit, + data_source=data_source, + ) + asset_esdl_output_profiles[variable_name] = ( + DataTableProfileManager(esdl_profile) + ) + elif ( + self.esdl_output_profiles_type + == ESDLOutputProfilesType.TIME_SERIES_PROFILE + ): + esdl_profile = create_time_series_profile( + es=energy_system, + name=variable_name, + start_date=start_date_time, + timestep_in_seconds=( + OUTPUT_TIMESERIESPROFILE_TIME_STEP_SECONDS + ), + values=[], # fill later + profile_type=esdl.ProfileTypeEnum.OUTPUT, + quantity_and_unit_type=quantity_and_unit, + data_source=data_source, + ) + asset_esdl_output_profiles[variable_name] = esdl_profile + else: # ESDLOutputProfilesType.DATE_TIME_PROFILE + esdl_profile = create_date_time_profile( + es=energy_system, + name=variable_name, + datetime_and_values=[], # fill later + profile_type=esdl.ProfileTypeEnum.OUTPUT, + quantity_and_unit_type=quantity_and_unit, + data_source=data_source, + ) + asset_esdl_output_profiles[variable_name] = esdl_profile + # Write result OUTPUT profiles on the optimized esdl + asset.port[index_outport].profile.append(esdl_profile) # Add variable values in new column conversion_factor = 0.0 - if variable in [ + if variable_name in [ f"{commodity}In.H", f"Primary.{commodity}In.H", f"Secondary.{commodity}In.H", @@ -1478,24 +1575,29 @@ def _write_updated_esdl( conversion_factor = GRAVITATIONAL_CONSTANT * 988.0 else: conversion_factor = 1.0 - if variable not in ["PostProc.Velocity", "PostProc.Pressure"]: + if variable_name not in [ + "PostProc.Velocity", + "PostProc.Pressure", + ]: data_row.append( - results[f"{asset_id}." + variable][ii] * conversion_factor + results[f"{asset_id}." + variable_name][ii] + * conversion_factor ) + # The variable evaluation below seems unnecessary, but it would be # used we expand the list of post process type variables - elif variable in ["PostProc.Velocity", "PostProc.Pressure"]: - data_row.append(post_processed[variable][ii]) + elif variable_name in [ + "PostProc.Velocity", + "PostProc.Pressure", + ]: + data_row.append(post_processed[variable_name][ii]) - profiles.profile_data_list.append(data_row) - # end time steps - profiles.num_profile_items = len(profiles.profile_data_list) - profiles.start_datetime = profiles.profile_data_list[0][0] - profiles.end_datetime = profiles.profile_data_list[-1][0] + if self.write_esdl_profiles_to_db: + asset_esdl_output_profiles[ + variable_name + ].profile_data_list.append(data_row) - influxdb_profile_manager = InfluxDBProfileManager( - influxdb_conn_settings, profiles - ) + asset_esdl_output_profiles_data_dict[variable_name].append(data_row) optim_simulation_tag = { "simulationRun": simulation_id, @@ -1505,11 +1607,31 @@ def _write_updated_esdl( "assetClass": asset_class, "capability": capability, } - _ = influxdb_profile_manager.save_influxdb( - measurement=carrier_id, - field_names=influxdb_profile_manager.profile_header[1:], - tags=optim_simulation_tag, - ) + for ( + variable_name, + profile_data, + ) in asset_esdl_output_profiles_data_dict.items(): + if self.write_esdl_profiles_to_db: + asset_esdl_output_profiles[variable_name].profile_data_list = ( + profile_data + ) + elif ( + self.esdl_output_profiles_type + == ESDLOutputProfilesType.TIME_SERIES_PROFILE + ): + asset_esdl_output_profiles[variable_name].values.extend( + self._resample_profile_data_to_fixed_timestep(profile_data) + ) + else: # ESDLOutputProfilesType.DATE_TIME_PROFILE + append_values_to_date_time_profile( + asset_esdl_output_profiles[variable_name], + profile_data, + ) + if self.write_esdl_profiles_to_db: + save_data_table_profiles_to_database( + list(asset_esdl_output_profiles.values()), + optim_simulation_tag, + ) # -- Test tags -- # do not delete - to be used in test case # prof_loaded_from_influxdb = InfluxDBProfileManager(influxdb_conn_settings) @@ -1610,6 +1732,10 @@ def _write_updated_esdl( # dicts, # ) # test = 0.0 + + # close ESDL output profile database connections + close_db_connections() + # ------------------------------------------------------------------------------------------ # Save esdl file # Edwin_marker_esdl_string - line 1224 diff --git a/src/mesido/workflows/multicommodity_simulator_workflow.py b/src/mesido/workflows/multicommodity_simulator_workflow.py index 58228bbb1..2b77b6baa 100644 --- a/src/mesido/workflows/multicommodity_simulator_workflow.py +++ b/src/mesido/workflows/multicommodity_simulator_workflow.py @@ -824,16 +824,16 @@ def main(runinfo_path, log_level): logger.info("Run Network Simulator") kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/src/mesido/workflows/simulator_workflow.py b/src/mesido/workflows/simulator_workflow.py index 4da36d1c5..d759212a4 100644 --- a/src/mesido/workflows/simulator_workflow.py +++ b/src/mesido/workflows/simulator_workflow.py @@ -352,16 +352,16 @@ def main(runinfo_path, log_level): logger.info("Run Network Simulator") kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/tests/test_head_loss.py b/tests/test_head_loss.py index 8457dd580..c59a10e8d 100644 --- a/tests/test_head_loss.py +++ b/tests/test_head_loss.py @@ -92,16 +92,16 @@ def update_heat_network_settings(self): # Do not delete kwargs: this is used to manualy check writing out of profile data kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/tests/test_multiple_in_and_out_port_components.py b/tests/test_multiple_in_and_out_port_components.py index 2596b43fe..72f97594a 100644 --- a/tests/test_multiple_in_and_out_port_components.py +++ b/tests/test_multiple_in_and_out_port_components.py @@ -56,16 +56,16 @@ def update_heat_network_settings(self): # Do not delete kwargs: this is used to manualy check writing out of profile data kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } @@ -366,16 +366,16 @@ def solver_options(self): # Do not delete kwargs: this is used to manualy check writing out of profile data kwargs = { - "write_result_db_profiles": False, + "esdl_output_profiles_type": None, "database_connections": [ { "access_type": DBAccessType.WRITE, - "influxdb_host": "localhost", - "influxdb_port": 8086, - "influxdb_username": None, - "influxdb_password": None, - "influxdb_ssl": False, - "influxdb_verify_ssl": False, + "host": "localhost", + "port": 8086, + "username": None, + "password": None, + "ssl": False, + "verify_ssl": False, }, ], } diff --git a/tests/test_potential_errors.py b/tests/test_potential_errors.py index 7dcf3bb70..dc2e93993 100644 --- a/tests/test_potential_errors.py +++ b/tests/test_potential_errors.py @@ -7,7 +7,7 @@ import esdl from mesido.esdl.esdl_parser import ESDLFileParser -from mesido.esdl.profile_parser import InfluxDBProfileReader +from mesido.esdl.profile_parser import ESDLProfileReader from mesido.exceptions import MesidoAssetIssueError from mesido.potential_errors import MesidoAssetIssueType, PotentialErrors from mesido.workflows import EndScenarioSizingStaged @@ -20,7 +20,7 @@ from utils_test_scaling import create_log_list_scaling -class MockInfluxDBProfileReader(InfluxDBProfileReader): +class MockESDLProfileReader(ESDLProfileReader): def __init__( self, energy_system: esdl.EnergySystem, @@ -74,7 +74,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_1.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -113,7 +113,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_2.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -141,7 +141,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_3.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -172,7 +172,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_4.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre() @@ -214,8 +214,10 @@ def test_asset_potential_errors(self): ) np.testing.assert_equal( cm.exception.message_per_asset_id["2ab92324-f86e-4976-9a6e-f7454b77ba3c"], - "Asset named HeatingDemand_2ab9: Input profile " - "demand1_MW_wrong_name in WarmingUp default profiles is not available in the database.", + "Error retrieving profile for asset 'HeatingDemand_2ab9' from host " + "'profiles.warmingup.info' and database 'energy_profiles': Failed to load from " + "InfluxDB: Failed to load profile data from InfluxDB for profile with " + "id:c3c18c06-1500-4e55-ae1d-56a82e806387 from host 'profiles.warmingup.info:443'.", ) # Check that the ResidualHeatSource multiplier's error is picked up @@ -229,7 +231,7 @@ def test_asset_potential_errors(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles_error_check_5.esdl", - profile_reader=InfluxDBProfileReader, + profile_reader=ESDLProfileReader, ) problem.pre() np.testing.assert_equal( diff --git a/tests/test_profile_parsing.py b/tests/test_profile_parsing.py index 0daecf1fd..8562947e8 100644 --- a/tests/test_profile_parsing.py +++ b/tests/test_profile_parsing.py @@ -4,12 +4,12 @@ import unittest import unittest.mock from pathlib import Path -from typing import Dict, Optional, Tuple +from typing import Optional import esdl from mesido.esdl.esdl_parser import ESDLFileParser -from mesido.esdl.profile_parser import InfluxDBProfileReader, ProfileReaderFromFile +from mesido.esdl.profile_parser import ESDLProfileReader, ProfileReaderFromFile from mesido.workflows import EndScenarioSizingStaged from mesido.workflows.utils.adapt_profiles import ( adapt_hourly_profile_averages_timestep_size, @@ -21,19 +21,17 @@ import pandas as pd -class MockInfluxDBProfileReader(InfluxDBProfileReader): +class MockESDLProfileReader(ESDLProfileReader): def __init__( self, energy_system: esdl.EnergySystem, file_path: Optional[Path], use_esdl_ranged_contraint: bool, - database_credentials: Optional[Dict[str, Tuple[str, str]]] = None, ): super().__init__( energy_system, file_path, use_esdl_ranged_contraint=use_esdl_ranged_contraint, - database_credentials=database_credentials, ) self._loaded_profiles = pd.read_csv( file_path, @@ -163,7 +161,7 @@ def test_loading_from_influx(self): model_folder=model_folder, input_folder=input_folder, esdl_file_name="1a_with_influx_profiles.esdl", - profile_reader=MockInfluxDBProfileReader, + profile_reader=MockESDLProfileReader, input_timeseries_file="influx_mock.csv", ) problem.pre()