From 6297b9ddf13349204ef915822ebf8a55df01f895 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Fri, 5 Jun 2026 12:36:34 +0200 Subject: [PATCH 1/4] enable postgres profiles --- .vscode/launch.json | 4 +- .vscode/settings.json | 3 +- ci/linux/_load_dot_env.sh | 0 src/grow_worker/worker.py | 79 +++++++++++++++++++++++++++++---------- 4 files changed, 63 insertions(+), 23 deletions(-) mode change 100644 => 100755 ci/linux/_load_dot_env.sh diff --git a/.vscode/launch.json b/.vscode/launch.json index f58ac30..00aff4e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,10 +7,10 @@ "request": "launch", "program": "${workspaceFolder}/src/grow_worker/worker.py", "console": "integratedTerminal", - "cwd": "${workspaceFolder}/src", + "cwd": "${workspaceFolder}", "justMyCode": true, "env": { - "PYTHONPATH": "${workspaceFolder}" + "PYTHONPATH": "${workspaceFolder}/src" }, "envFile": "${workspaceFolder}/.env" } diff --git a/.vscode/settings.json b/.vscode/settings.json index 6728f09..9d29e7e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,10 @@ { + "flake8.enabled": false, "[python]": { "editor.formatOnSave": true, "editor.defaultFormatter": "ms-python.black-formatter", "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, }, "python.analysis.typeCheckingMode": "basic", diff --git a/ci/linux/_load_dot_env.sh b/ci/linux/_load_dot_env.sh old mode 100644 new mode 100755 diff --git a/src/grow_worker/worker.py b/src/grow_worker/worker.py index 9ad9ad8..4961159 100644 --- a/src/grow_worker/worker.py +++ b/src/grow_worker/worker.py @@ -2,29 +2,33 @@ import logging import os from pathlib import Path -from typing import cast, Dict, List, Tuple, Optional +from typing import Dict, List, Optional, Tuple, cast +import esdl +from dotenv import load_dotenv from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_parser import ESDLStringParser +from mesido.esdl.profile_parser import ESDLProfileReader from mesido.exceptions import MesidoAssetIssueError from omotes_sdk.internal.orchestrator_worker_events.esdl_messages import ( EsdlMessage, MessageSeverity, ) -from omotes_sdk.internal.worker.worker import initialize_worker, UpdateProgressHandler +from omotes_sdk.internal.worker.worker import UpdateProgressHandler, initialize_worker from omotes_sdk.types import ProtobufDict -from mesido.esdl.esdl_parser import ESDLStringParser -from mesido.esdl.profile_parser import InfluxDBProfileReader from grow_worker.worker_types import ( - GrowTaskType, GROWProblem, - get_problem_type, + GrowTaskType, get_problem_function, + get_problem_type, get_solver_class, ) logger = logging.getLogger("grow_worker") +load_dotenv() # Load environment variables from .env file + GROW_TASK_TYPES = [GrowTaskType(task_type) for task_type in os.environ["GROW_TASK_TYPE"].split(",")] @@ -63,9 +67,21 @@ def grow_worker_task( mesido_solver = get_solver_class(workflow_type) base_folder = Path(__file__).resolve().parent.parent - write_result_db_profiles = "INFLUXDB_HOSTNAME" in os.environ - influxdb_host = os.environ.get("INFLUXDB_HOSTNAME", "localhost") + write_result_db_profiles = ( + "INFLUXDB_HOSTNAME" in os.environ or "POSTGRES_HOSTNAME" in os.environ + ) + db_type_output_profiles_name = os.environ.get("DB_TYPE_OUTPUT_PROFILES", "POSTGRESQL").upper() + db_type_output_profiles = getattr(esdl.DatabaseTypeEnum, db_type_output_profiles_name, None) + if db_type_output_profiles is None: + logger.warning( + "Unknown DB_TYPE_OUTPUT_PROFILES '%s', defaulting to POSTGRESQL", + db_type_output_profiles_name, + ) + db_type_output_profiles = esdl.DatabaseTypeEnum.POSTGRESQL + influxdb_host = os.environ.get("INFLUXDB_HOSTNAME") influxdb_port = int(os.environ.get("INFLUXDB_PORT", "8086")) + postgres_host = os.environ.get("POSTGRES_HOSTNAME") + postgres_port = int(os.environ.get("POSTGRES_PORT", "5432")) logger.info( "Will write result profiles to influx: %s. At %s:%s", @@ -74,6 +90,38 @@ def grow_worker_task( influxdb_port, ) + database_connection = [] + if influxdb_host: + database_connection.append( + { + "access_type": ( + DBAccessType.READ_WRITE + if db_type_output_profiles == esdl.DatabaseTypeEnum.INFLUXDB + else DBAccessType.READ + ), + "host": influxdb_host, + "port": influxdb_port, + "username": os.environ.get("INFLUXDB_USERNAME"), + "password": os.environ.get("INFLUXDB_PASSWORD"), + "ssl": False, + "verify_ssl": False, + } + ) + if postgres_host: + database_connection.append( + { + "access_type": ( + DBAccessType.READ_WRITE + if db_type_output_profiles == esdl.DatabaseTypeEnum.POSTGRESQL + else DBAccessType.READ + ), + "host": postgres_host, + "port": postgres_port, + "username": os.environ.get("POSTGRES_USERNAME"), + "password": os.environ.get("POSTGRES_PASSWORD"), + } + ) + esdl_str = None esdl_messages = [] try: @@ -84,19 +132,10 @@ def grow_worker_task( esdl_string=base64.encodebytes(input_esdl.encode("utf-8")), esdl_parser=ESDLStringParser, write_result_db_profiles=write_result_db_profiles, - database_connections=[ - { - "access_type": DBAccessType.READ_WRITE, - "influxdb_host": influxdb_host, - "influxdb_port": influxdb_port, - "influxdb_username": os.environ.get("INFLUXDB_USERNAME"), - "influxdb_password": os.environ.get("INFLUXDB_PASSWORD"), - "influxdb_ssl": False, - "influxdb_verify_ssl": False, - }, - ], + db_type_output_profiles=db_type_output_profiles, + database_connections=database_connection, update_progress_function=update_progress_handler, - profile_reader=InfluxDBProfileReader, + profile_reader=ESDLProfileReader, ) esdl_str = cast(str, solution.optimized_esdl_string) # TODO get esdl_messages from successful run after mesido update. From d7864c0359369a519e2dea49868f1b4b64124e1d Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Thu, 16 Jul 2026 13:11:59 +0200 Subject: [PATCH 2/4] to mesido 0.1.20.3 --- .gitignore | 2 - dev-requirements.txt | 71 ++++++++++++++++++++------------- pyproject.toml | 2 +- requirements.txt | 59 +++++++++++++++++----------- src/grow_worker/worker.py | 82 +++++++++++++++------------------------ 5 files changed, 113 insertions(+), 103 deletions(-) diff --git a/.gitignore b/.gitignore index 5f197c2..48f5126 100644 --- a/.gitignore +++ b/.gitignore @@ -221,8 +221,6 @@ fabric.properties unit_test_coverage/ test-results.xml -.env.* - temp/ gurobi/ .idea/ diff --git a/dev-requirements.txt b/dev-requirements.txt index d1e0451..9c4f0ba 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --cert=None --client-cert=None --constraint=requirements.txt --extra=dev --index-url=None --output-file=dev-requirements.txt --pip-args=None pyproject.toml +# pip-compile --constraint=requirements.txt --extra=dev --output-file=dev-requirements.txt pyproject.toml # aio-pika==9.4.3 # via @@ -39,15 +39,15 @@ celery==5.6.3 # via # -c requirements.txt # omotes-sdk-python -certifi==2026.2.25 +certifi==2026.6.17 # via # -c requirements.txt # requests -charset-normalizer==3.4.7 +charset-normalizer==3.4.9 # via # -c requirements.txt # requests -click==8.3.2 +click==8.4.2 # via # -c requirements.txt # black @@ -71,12 +71,16 @@ coolprop==6.6.0 # via # -c requirements.txt # mesido -coverage[toml]==7.13.5 +coverage[toml]==7.15.2 # via pytest-cov defusedxml==0.7.1 # via # -c requirements.txt # rtc-tools-gil-comp +et-xmlfile==2.0.0 + # via + # -c requirements.txt + # openpyxl exceptiongroup==1.3.1 # via # -c requirements.txt @@ -106,15 +110,16 @@ future-fstrings==1.2.0 # via # -c requirements.txt # pyecore -idna==3.11 +idna==3.18 # via # -c requirements.txt # requests # yarl -influxdb==5.3.2 +influxdb==5.3.1 # via # -c requirements.txt # mesido + # pyesdl iniconfig==2.3.0 # via pytest isort==5.13.2 @@ -123,17 +128,17 @@ kombu==5.6.2 # via # -c requirements.txt # celery -lxml==6.0.4 +lxml==6.1.1 # via # -c requirements.txt # pyecore mccabe==0.7.0 # via flake8 -mesido==0.1.18.1 +mesido==0.1.20.3 # via # -c requirements.txt # omotes-grow-worker (pyproject.toml) -msgpack==1.1.2 +msgpack==1.2.1 # via # -c requirements.txt # influxdb @@ -159,15 +164,19 @@ omotes-sdk-protocol==1.2.0 # via # -c requirements.txt # omotes-sdk-python -omotes-sdk-python==4.3.3 +omotes-sdk-python==4.3.4 # via # -c requirements.txt # omotes-grow-worker (pyproject.toml) +openpyxl==3.1.2 + # via + # -c requirements.txt + # pyesdl ordered-set==4.1.0 # via # -c requirements.txt # pyecore -packaging==26.0 +packaging==26.2 # via # -c requirements.txt # black @@ -184,9 +193,9 @@ pandas==1.5.3 # via # -c requirements.txt # mesido -pathspec==1.0.4 +pathspec==1.1.1 # via black -platformdirs==4.9.6 +platformdirs==4.10.0 # via black pluggy==1.6.0 # via pytest @@ -194,7 +203,7 @@ prompt-toolkit==3.0.52 # via # -c requirements.txt # click-repl -propcache==0.4.1 +propcache==0.5.2 # via # -c requirements.txt # yarl @@ -202,6 +211,15 @@ protobuf==5.29.6 # via # -c requirements.txt # omotes-sdk-protocol +psycopg[binary]==3.3.3 + # via + # -c requirements.txt + # pyesdl +psycopg-binary==3.3.3 + # via + # -c requirements.txt + # psycopg + # pyesdl pycodestyle==2.12.1 # via flake8 pydocstyle==6.3.0 @@ -211,7 +229,7 @@ pyecore==0.13.2 # -c requirements.txt # mesido # pyesdl -pyesdl==26.2 +pyesdl[profiles]==26.7 # via # -c requirements.txt # mesido @@ -242,16 +260,16 @@ python-dotenv==1.0.1 # via # -c requirements.txt # omotes-grow-worker (pyproject.toml) -pytz==2026.1.post1 +pytz==2026.2 # via # -c requirements.txt # influxdb # pandas -requests==2.33.1 +requests==2.34.2 # via # -c requirements.txt # influxdb -restrictedpython==8.1 +restrictedpython==8.4 # via # -c requirements.txt # pyecore @@ -275,7 +293,7 @@ six==1.17.0 # flake8-tuple # influxdb # python-dateutil -snowballstemmer==3.0.1 +snowballstemmer==3.1.1 # via pydocstyle streamcapture==1.2.7 # via @@ -294,7 +312,7 @@ tomli==2.4.1 # mypy # pytest # setuptools-git-versioning -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # -c requirements.txt # black @@ -302,15 +320,16 @@ typing-extensions==4.15.0 # multidict # mypy # omotes-sdk-python -tzdata==2026.1 + # psycopg +tzdata==2026.3 # via # -c requirements.txt # kombu -tzlocal==5.3.1 +tzlocal==5.4.4 # via # -c requirements.txt # celery -urllib3==2.6.3 +urllib3==2.7.0 # via # -c requirements.txt # requests @@ -320,13 +339,13 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.6.0 +wcwidth==0.8.2 # via # -c requirements.txt # prompt-toolkit wheel==0.45.1 # via omotes-grow-worker (pyproject.toml) -yarl==1.23.0 +yarl==1.24.2 # via # -c requirements.txt # aio-pika diff --git a/pyproject.toml b/pyproject.toml index ae0b223..7b5ab10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ requires-python = ">=3.10" dependencies = [ "python-dotenv ~= 1.0.0", - "mesido ~= 0.1.18.1", + "mesido ~= 0.1.20.3", "omotes-sdk-python ~= 4.3.3", ] diff --git a/requirements.txt b/requirements.txt index f65c7b6..0a7a941 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --cert=None --client-cert=None --index-url=None --output-file=requirements.txt --pip-args=None pyproject.toml +# pip-compile --output-file=requirements.txt pyproject.toml # aio-pika==9.4.3 # via omotes-sdk-python @@ -20,11 +20,11 @@ casadi-gil-comp==3.6.7 # rtc-tools-gil-comp celery==5.6.3 # via omotes-sdk-python -certifi==2026.2.25 +certifi==2026.6.17 # via requests -charset-normalizer==3.4.7 +charset-normalizer==3.4.9 # via requests -click==8.3.2 +click==8.4.2 # via # celery # click-didyoumean @@ -40,23 +40,27 @@ coolprop==6.6.0 # via mesido defusedxml==0.7.1 # via rtc-tools-gil-comp +et-xmlfile==2.0.0 + # via openpyxl exceptiongroup==1.3.1 # via celery future-fstrings==1.2.0 # via pyecore -idna==3.11 +idna==3.18 # via # requests # yarl -influxdb==5.3.2 - # via mesido +influxdb==5.3.1 + # via + # mesido + # pyesdl kombu==5.6.2 # via celery -lxml==6.0.4 +lxml==6.1.1 # via pyecore -mesido==0.1.18.1 +mesido==0.1.20.3 # via omotes-grow-worker (pyproject.toml) -msgpack==1.1.2 +msgpack==1.2.1 # via influxdb multidict==6.7.1 # via yarl @@ -69,11 +73,13 @@ numpy==1.25.2 # scipy omotes-sdk-protocol==1.2.0 # via omotes-sdk-python -omotes-sdk-python==4.3.3 +omotes-sdk-python==4.3.4 # via omotes-grow-worker (pyproject.toml) +openpyxl==3.1.2 + # via pyesdl ordered-set==4.1.0 # via pyecore -packaging==26.0 +packaging==26.2 # via kombu pamqp==3.3.0 # via @@ -83,15 +89,21 @@ pandas==1.5.3 # via mesido prompt-toolkit==3.0.52 # via click-repl -propcache==0.4.1 +propcache==0.5.2 # via yarl protobuf==5.29.6 # via omotes-sdk-protocol +psycopg[binary]==3.3.3 + # via pyesdl +psycopg-binary==3.3.3 + # via + # psycopg + # pyesdl pyecore==0.13.2 # via # mesido # pyesdl -pyesdl==26.2 +pyesdl[profiles]==26.7 # via # mesido # omotes-sdk-python @@ -107,13 +119,13 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via omotes-grow-worker (pyproject.toml) -pytz==2026.1.post1 +pytz==2026.2 # via # influxdb # pandas -requests==2.33.1 +requests==2.34.2 # via influxdb -restrictedpython==8.1 +restrictedpython==8.4 # via pyecore rtc-tools-channel-flow==1.3.1 # via rtc-tools-gil-comp @@ -129,25 +141,26 @@ streamcapture==1.2.7 # via omotes-sdk-python strenum==0.4.15 # via mesido -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # exceptiongroup # multidict # omotes-sdk-python -tzdata==2026.1 + # psycopg +tzdata==2026.3 # via kombu -tzlocal==5.3.1 +tzlocal==5.4.4 # via celery -urllib3==2.6.3 +urllib3==2.7.0 # via requests vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.6.0 +wcwidth==0.8.2 # via prompt-toolkit -yarl==1.23.0 +yarl==1.24.2 # via # aio-pika # aiormq diff --git a/src/grow_worker/worker.py b/src/grow_worker/worker.py index 4961159..5a35110 100644 --- a/src/grow_worker/worker.py +++ b/src/grow_worker/worker.py @@ -4,9 +4,8 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, cast -import esdl from dotenv import load_dotenv -from mesido.esdl.esdl_mixin import DBAccessType +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType from mesido.esdl.esdl_parser import ESDLStringParser from mesido.esdl.profile_parser import ESDLProfileReader from mesido.exceptions import MesidoAssetIssueError @@ -67,60 +66,42 @@ def grow_worker_task( mesido_solver = get_solver_class(workflow_type) base_folder = Path(__file__).resolve().parent.parent - write_result_db_profiles = ( - "INFLUXDB_HOSTNAME" in os.environ or "POSTGRES_HOSTNAME" in os.environ - ) - db_type_output_profiles_name = os.environ.get("DB_TYPE_OUTPUT_PROFILES", "POSTGRESQL").upper() - db_type_output_profiles = getattr(esdl.DatabaseTypeEnum, db_type_output_profiles_name, None) - if db_type_output_profiles is None: + esdl_output_profiles_type_str = os.environ.get( + "ESDL_OUTPUT_PROFILES_TYPE", "POSTGRESQL" + ).upper() + esdl_output_profiles_type = getattr(ESDLOutputProfilesType, esdl_output_profiles_type_str, None) + if esdl_output_profiles_type is None: logger.warning( - "Unknown DB_TYPE_OUTPUT_PROFILES '%s', defaulting to POSTGRESQL", - db_type_output_profiles_name, + "Unknown ESDL_OUTPUT_PROFILES_TYPE '%s', defaulting to POSTGRESQL", + esdl_output_profiles_type_str, ) - db_type_output_profiles = esdl.DatabaseTypeEnum.POSTGRESQL - influxdb_host = os.environ.get("INFLUXDB_HOSTNAME") - influxdb_port = int(os.environ.get("INFLUXDB_PORT", "8086")) - postgres_host = os.environ.get("POSTGRES_HOSTNAME") - postgres_port = int(os.environ.get("POSTGRES_PORT", "5432")) + esdl_output_profiles_type = ESDLOutputProfilesType.POSTGRESQL + + db_host = os.environ.get("DB_HOSTNAME") + db_port = int(os.environ.get("DB_PORT", "5432")) + db_username = os.environ.get("DB_USERNAME", "") + db_password = os.environ.get("DB_PASSWORD", "") logger.info( - "Will write result profiles to influx: %s. At %s:%s", - write_result_db_profiles, - influxdb_host, - influxdb_port, + "Will write result profiles to '%s' database at %s:%s", + esdl_output_profiles_type_str, + db_host, + db_port, ) database_connection = [] - if influxdb_host: - database_connection.append( - { - "access_type": ( - DBAccessType.READ_WRITE - if db_type_output_profiles == esdl.DatabaseTypeEnum.INFLUXDB - else DBAccessType.READ - ), - "host": influxdb_host, - "port": influxdb_port, - "username": os.environ.get("INFLUXDB_USERNAME"), - "password": os.environ.get("INFLUXDB_PASSWORD"), - "ssl": False, - "verify_ssl": False, - } - ) - if postgres_host: - database_connection.append( - { - "access_type": ( - DBAccessType.READ_WRITE - if db_type_output_profiles == esdl.DatabaseTypeEnum.POSTGRESQL - else DBAccessType.READ - ), - "host": postgres_host, - "port": postgres_port, - "username": os.environ.get("POSTGRES_USERNAME"), - "password": os.environ.get("POSTGRES_PASSWORD"), - } - ) + + database_connection.append( + { + "access_type": DBAccessType.READ_WRITE, + "host": db_host, + "port": db_port, + "username": db_username, + "password": db_password, + "ssl": False, + "verify_ssl": False, + } + ) esdl_str = None esdl_messages = [] @@ -131,8 +112,7 @@ def grow_worker_task( base_folder=base_folder, esdl_string=base64.encodebytes(input_esdl.encode("utf-8")), esdl_parser=ESDLStringParser, - write_result_db_profiles=write_result_db_profiles, - db_type_output_profiles=db_type_output_profiles, + esdl_output_profiles_type=esdl_output_profiles_type, database_connections=database_connection, update_progress_function=update_progress_handler, profile_reader=ESDLProfileReader, From 6efb2c335fd11ba2b6fd2e94797f390a780a791b Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Thu, 16 Jul 2026 13:13:26 +0200 Subject: [PATCH 3/4] add .env.template --- .env.template | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .env.template diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..6351087 --- /dev/null +++ b/.env.template @@ -0,0 +1,20 @@ +ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL + +DB_HOSTNAME=omotes_postgres +DB_PORT=6432 +DB_USERNAME=omotes_timeseries_rw +DB_PASSWORD=somepass4 + +RABBITMQ_USERNAME=celery +RABBITMQ_PASSWORD=somepass2 +RABBITMQ_HOSTNAME=localhost +RABBITMQ_PORT=5672 +RABBITMQ_VIRTUALHOST=omotes_celery + +# May be grow_optimizer_default, grow_optimizer_no_heat_losses, grow_optimizer_with_pressure or grow_simulator +GROW_TASK_TYPE=grow_optimizer_default +LOG_LEVEL=INFO + +TASK_RESULT_QUEUE_NAME=omotes_task_result_events +TASK_PROGRESS_QUEUE_NAME=omotes_task_progress_events +MAX_TASKS_BEFORE_RESTART=0 # 0 means no scheduled restarts From bfccd6d6b23022c772436f7196b36dbd98279539 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Thu, 16 Jul 2026 16:36:59 +0200 Subject: [PATCH 4/4] fix ci --- pyproject.toml | 2 +- src/grow_worker/worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8602df1..020ca7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ enabled = true starting_version = "0.0.1" [tool.pytest.ini_options] -addopts = "--cov=grow_worker --cov-report html --cov-report term-missing --cov-fail-under 35" +addopts = "--cov=grow_worker --cov-report html --cov-report term-missing --cov-fail-under 30" [tool.coverage.run] source = ["src"] diff --git a/src/grow_worker/worker.py b/src/grow_worker/worker.py index 63fbfed..5a35110 100644 --- a/src/grow_worker/worker.py +++ b/src/grow_worker/worker.py @@ -78,7 +78,7 @@ def grow_worker_task( esdl_output_profiles_type = ESDLOutputProfilesType.POSTGRESQL db_host = os.environ.get("DB_HOSTNAME") - db_port = int(os.environ.get("DB_PORT")) + db_port = int(os.environ.get("DB_PORT", "5432")) db_username = os.environ.get("DB_USERNAME", "") db_password = os.environ.get("DB_PASSWORD", "")