diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..14f4455 --- /dev/null +++ b/.env.template @@ -0,0 +1,22 @@ +ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL + +DB_HOSTNAME=omotes_postgres +DB_PORT=6432 +DB_USERNAME=omotes_timeseries_rw +DB_PASSWORD=somepass4 + +RABBITMQ_USERNAME=celery +RABBITMQ_PASSWORD=somepass2 +RABBITMQ_HOSTNAME=localhost +RABBITMQ_PORT=5672 +RABBITMQ_VIRTUALHOST=omotes_celery + +# May be grow_optimizer_default, grow_optimizer_no_heat_losses, grow_optimizer_with_pressure or grow_simulator +GROW_TASK_TYPE=grow_optimizer_default +LOG_LEVEL=INFO + +TASK_RESULT_QUEUE_NAME=omotes_task_result_events +TASK_PROGRESS_QUEUE_NAME=omotes_task_progress_events + +# MAX_TASKS_BEFORE_RESTART: 0 means no scheduled restarts +MAX_TASKS_BEFORE_RESTART=0 diff --git a/.gitignore b/.gitignore index 5f197c2..48f5126 100644 --- a/.gitignore +++ b/.gitignore @@ -221,8 +221,6 @@ fabric.properties unit_test_coverage/ test-results.xml -.env.* - temp/ gurobi/ .idea/ diff --git a/.vscode/launch.json b/.vscode/launch.json index f58ac30..00aff4e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,10 +7,10 @@ "request": "launch", "program": "${workspaceFolder}/src/grow_worker/worker.py", "console": "integratedTerminal", - "cwd": "${workspaceFolder}/src", + "cwd": "${workspaceFolder}", "justMyCode": true, "env": { - "PYTHONPATH": "${workspaceFolder}" + "PYTHONPATH": "${workspaceFolder}/src" }, "envFile": "${workspaceFolder}/.env" } diff --git a/.vscode/settings.json b/.vscode/settings.json index 6728f09..9d29e7e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,9 +1,10 @@ { + "flake8.enabled": false, "[python]": { "editor.formatOnSave": true, "editor.defaultFormatter": "ms-python.black-formatter", "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, }, "python.analysis.typeCheckingMode": "basic", diff --git a/ci/linux/_load_dot_env.sh b/ci/linux/_load_dot_env.sh old mode 100644 new mode 100755 diff --git a/dev-requirements.txt b/dev-requirements.txt index 00e1a4c..990ec4e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -71,12 +71,16 @@ coolprop==6.6.0 # via # -c requirements.txt # mesido -coverage[toml]==7.15.1 +coverage[toml]==7.15.2 # via pytest-cov defusedxml==0.7.1 # via # -c requirements.txt # rtc-tools-gil-comp +et-xmlfile==2.0.0 + # via + # -c requirements.txt + # openpyxl exceptiongroup==1.3.1 # via # -c requirements.txt @@ -111,10 +115,11 @@ idna==3.18 # -c requirements.txt # requests # yarl -influxdb==5.3.2 +influxdb==5.3.1 # via # -c requirements.txt # mesido + # pyesdl iniconfig==2.3.0 # via pytest isort==5.13.2 @@ -129,7 +134,7 @@ lxml==6.1.1 # pyecore mccabe==0.7.0 # via flake8 -mesido==0.1.20.2 +mesido==0.1.20.3 # via # -c requirements.txt # omotes-grow-worker (pyproject.toml) @@ -159,10 +164,14 @@ omotes-sdk-protocol==1.2.0 # via # -c requirements.txt # omotes-sdk-python -omotes-sdk-python==4.3.5 +omotes-sdk-python==4.3.6 # via # -c requirements.txt # omotes-grow-worker (pyproject.toml) +openpyxl==3.1.2 + # via + # -c requirements.txt + # pyesdl ordered-set==4.1.0 # via # -c requirements.txt @@ -202,6 +211,15 @@ protobuf==5.29.6 # via # -c requirements.txt # omotes-sdk-protocol +psycopg[binary]==3.3.3 + # via + # -c requirements.txt + # pyesdl +psycopg-binary==3.3.3 + # via + # -c requirements.txt + # psycopg + # pyesdl pycodestyle==2.12.1 # via flake8 pydocstyle==6.3.0 @@ -211,7 +229,7 @@ pyecore==0.13.2 # -c requirements.txt # mesido # pyesdl -pyesdl==26.6.1 +pyesdl[profiles]==26.7 # via # -c requirements.txt # mesido @@ -302,6 +320,7 @@ typing-extensions==4.16.0 # multidict # mypy # omotes-sdk-python + # psycopg tzdata==2026.3 # via # -c requirements.txt diff --git a/pyproject.toml b/pyproject.toml index bc93890..020ca7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ requires-python = ">=3.10" dependencies = [ "python-dotenv ~= 1.0.0", - "mesido ~= 0.1.20.2", - "omotes-sdk-python ~= 4.3.5", + "mesido ~= 0.1.20.3", + "omotes-sdk-python ~= 4.3.6", ] [project.optional-dependencies] @@ -69,7 +69,7 @@ enabled = true starting_version = "0.0.1" [tool.pytest.ini_options] -addopts = "--cov=grow_worker --cov-report html --cov-report term-missing --cov-fail-under 35" +addopts = "--cov=grow_worker --cov-report html --cov-report term-missing --cov-fail-under 30" [tool.coverage.run] source = ["src"] diff --git a/requirements.txt b/requirements.txt index ffc369d..ce36bda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,8 @@ coolprop==6.6.0 # via mesido defusedxml==0.7.1 # via rtc-tools-gil-comp +et-xmlfile==2.0.0 + # via openpyxl exceptiongroup==1.3.1 # via celery future-fstrings==1.2.0 @@ -48,13 +50,15 @@ idna==3.18 # via # requests # yarl -influxdb==5.3.2 - # via mesido +influxdb==5.3.1 + # via + # mesido + # pyesdl kombu==5.6.2 # via celery lxml==6.1.1 # via pyecore -mesido==0.1.20.2 +mesido==0.1.20.3 # via omotes-grow-worker (pyproject.toml) msgpack==1.2.1 # via influxdb @@ -69,8 +73,10 @@ numpy==1.25.2 # scipy omotes-sdk-protocol==1.2.0 # via omotes-sdk-python -omotes-sdk-python==4.3.5 +omotes-sdk-python==4.3.6 # via omotes-grow-worker (pyproject.toml) +openpyxl==3.1.2 + # via pyesdl ordered-set==4.1.0 # via pyecore packaging==26.2 @@ -87,11 +93,17 @@ propcache==0.5.2 # via yarl protobuf==5.29.6 # via omotes-sdk-protocol +psycopg[binary]==3.3.3 + # via pyesdl +psycopg-binary==3.3.3 + # via + # psycopg + # pyesdl pyecore==0.13.2 # via # mesido # pyesdl -pyesdl==26.6.1 +pyesdl[profiles]==26.7 # via # mesido # omotes-sdk-python @@ -134,6 +146,7 @@ typing-extensions==4.16.0 # exceptiongroup # multidict # omotes-sdk-python + # psycopg tzdata==2026.3 # via kombu tzlocal==5.4.4 diff --git a/src/grow_worker/worker.py b/src/grow_worker/worker.py index 9ad9ad8..5a35110 100644 --- a/src/grow_worker/worker.py +++ b/src/grow_worker/worker.py @@ -2,29 +2,32 @@ import logging import os from pathlib import Path -from typing import cast, Dict, List, Tuple, Optional +from typing import Dict, List, Optional, Tuple, cast -from mesido.esdl.esdl_mixin import DBAccessType +from dotenv import load_dotenv +from mesido.esdl.esdl_mixin import DBAccessType, ESDLOutputProfilesType +from mesido.esdl.esdl_parser import ESDLStringParser +from mesido.esdl.profile_parser import ESDLProfileReader from mesido.exceptions import MesidoAssetIssueError from omotes_sdk.internal.orchestrator_worker_events.esdl_messages import ( EsdlMessage, MessageSeverity, ) -from omotes_sdk.internal.worker.worker import initialize_worker, UpdateProgressHandler +from omotes_sdk.internal.worker.worker import UpdateProgressHandler, initialize_worker from omotes_sdk.types import ProtobufDict -from mesido.esdl.esdl_parser import ESDLStringParser -from mesido.esdl.profile_parser import InfluxDBProfileReader from grow_worker.worker_types import ( - GrowTaskType, GROWProblem, - get_problem_type, + GrowTaskType, get_problem_function, + get_problem_type, get_solver_class, ) logger = logging.getLogger("grow_worker") +load_dotenv() # Load environment variables from .env file + GROW_TASK_TYPES = [GrowTaskType(task_type) for task_type in os.environ["GROW_TASK_TYPE"].split(",")] @@ -63,15 +66,41 @@ def grow_worker_task( mesido_solver = get_solver_class(workflow_type) base_folder = Path(__file__).resolve().parent.parent - write_result_db_profiles = "INFLUXDB_HOSTNAME" in os.environ - influxdb_host = os.environ.get("INFLUXDB_HOSTNAME", "localhost") - influxdb_port = int(os.environ.get("INFLUXDB_PORT", "8086")) + esdl_output_profiles_type_str = os.environ.get( + "ESDL_OUTPUT_PROFILES_TYPE", "POSTGRESQL" + ).upper() + esdl_output_profiles_type = getattr(ESDLOutputProfilesType, esdl_output_profiles_type_str, None) + if esdl_output_profiles_type is None: + logger.warning( + "Unknown ESDL_OUTPUT_PROFILES_TYPE '%s', defaulting to POSTGRESQL", + esdl_output_profiles_type_str, + ) + esdl_output_profiles_type = ESDLOutputProfilesType.POSTGRESQL + + db_host = os.environ.get("DB_HOSTNAME") + db_port = int(os.environ.get("DB_PORT", "5432")) + db_username = os.environ.get("DB_USERNAME", "") + db_password = os.environ.get("DB_PASSWORD", "") logger.info( - "Will write result profiles to influx: %s. At %s:%s", - write_result_db_profiles, - influxdb_host, - influxdb_port, + "Will write result profiles to '%s' database at %s:%s", + esdl_output_profiles_type_str, + db_host, + db_port, + ) + + database_connection = [] + + database_connection.append( + { + "access_type": DBAccessType.READ_WRITE, + "host": db_host, + "port": db_port, + "username": db_username, + "password": db_password, + "ssl": False, + "verify_ssl": False, + } ) esdl_str = None @@ -83,20 +112,10 @@ def grow_worker_task( base_folder=base_folder, esdl_string=base64.encodebytes(input_esdl.encode("utf-8")), esdl_parser=ESDLStringParser, - write_result_db_profiles=write_result_db_profiles, - database_connections=[ - { - "access_type": DBAccessType.READ_WRITE, - "influxdb_host": influxdb_host, - "influxdb_port": influxdb_port, - "influxdb_username": os.environ.get("INFLUXDB_USERNAME"), - "influxdb_password": os.environ.get("INFLUXDB_PASSWORD"), - "influxdb_ssl": False, - "influxdb_verify_ssl": False, - }, - ], + esdl_output_profiles_type=esdl_output_profiles_type, + database_connections=database_connection, update_progress_function=update_progress_handler, - profile_reader=InfluxDBProfileReader, + profile_reader=ESDLProfileReader, ) esdl_str = cast(str, solution.optimized_esdl_string) # TODO get esdl_messages from successful run after mesido update.