174 enable postgres timeseries storage

.env.template

@@ -1,11 +1,15 @@
 POSTGRES_ROOT_USER=root
 POSTGRES_ROOT_PASSWORD=1234
-POSTGRES_DEV_PORT=6432
+POSTGRES_PORT=6432
 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator
 POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3
+POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw
+POSTGRES_TIMESERIES_RW_USER_PASSWORD=somepass4
+POSTGRES_TIMESERIES_RO_USER_NAME=omotes_timeseries_ro
+POSTGRES_TIMESERIES_RO_USER_PASSWORD=somepass5
 
-RABBITMQ_PORT=5672
-RABBITMQ_MANAGEMENT_PORT=15672
+RABBITMQ_PORT=5673
+RABBITMQ_MANAGEMENT_PORT=15673
 RABBITMQ_ROOT_USER=root
 RABBITMQ_ROOT_PASSWORD=5678
 RABBITMQ_OMOTES_USER_NAME=omotes
@@ -28,7 +32,7 @@ TIMEOUT_JOB_MANAGER_START_BUFFER_SEC=60
 TIMEOUT_JOB_HANDLER_RERUN_SEC=30
 
 WORKFLOW_SETTINGS_FILE=./config/workflow_config_nwn_no_gurobi.json
-
 MAX_TASKS_BEFORE_RESTART=1
+ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL
 
 LOG_LEVEL=INFO

.vscode/launch.json

@@ -30,6 +30,24 @@
             "env": {
                 "PYTHONPATH": "${workspaceFolder}/system_tests"
             }
+        },
+        {
+            "name": "test__all",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "pytest",
+            "args": [
+                "test_workflows_steps.py::TestWorkflows",
+                "-v",
+                "-s"
+            ],
+            "jinja": true,
+            "justMyCode": true,
+            "cwd": "${workspaceFolder}/system_tests/src",
+            "envFile": "${workspaceFolder}/.env.test",
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/system_tests"
+            }
         }
     ]
 }

docker-compose.override.dev.yml

@@ -8,13 +8,13 @@ services:
 
   grow_worker_optimizer: &grow_worker
     image: omotes-worker-mesido-dev
-
-  grow_worker_simulator:
-    <<: *grow_worker
     build:
       context: ..
       dockerfile: optimizer-worker/dev.Dockerfile
 
+  grow_worker_simulator:
+    <<: *grow_worker
+
   grow_worker_optimizer_no_heat_losses:
     <<: *grow_worker
 
@@ -32,7 +32,3 @@ services:
     build:
       context: ..
       dockerfile: simulator-worker/dev.Dockerfile
-
-  orchestrator_postgres_db:
-    ports:
-      - "${POSTGRES_DEV_PORT}:5432"

docker-compose.override.setup.yml

@@ -18,6 +18,6 @@ services:
       INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD}
       INFLUXDB_PORT: ${INFLUXDB_PORT}
 
-  orchestrator_postgres_db:
+  omotes_postgres:
     volumes:
       - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro"

docker-compose.yml

@@ -60,16 +60,21 @@ services:
       - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro
       - influxdb_storage:/var/lib/influxdb
 
-  orchestrator_postgres_db: &orchestrator_postgres_db
+  omotes_postgres: &omotes_postgres
     image: postgres:15.4
     restart: unless-stopped
+    command: ["postgres", "-p", "${POSTGRES_PORT}"]
     volumes:
       - "db-data:/var/lib/postgresql/data/"
+    expose:
+      - "${POSTGRES_PORT}"
+    ports:
+      - "${POSTGRES_PORT}:${POSTGRES_PORT}"
     networks:
       - omotes
+#      - mapeditor-net
     environment:
       PGDATA: /var/lib/postgresql/data
-      POSTGRES_DB: omotes_jobs
       POSTGRES_USER: ${POSTGRES_ROOT_USER}
       POSTGRES_PASSWORD: ${POSTGRES_ROOT_PASSWORD}
 
@@ -81,24 +86,15 @@ services:
           "pg_isready",
           "-U",
           "${POSTGRES_ROOT_USER}",
-          "-d",
-          "omotes_jobs",
+          "-p",
+          "${POSTGRES_PORT}"
         ]
       interval: 5s
       timeout: 5s
       retries: 10
 
-  # Skipped in 'docker-compose up', only used to develop database revisions.
-  orchestrator_postgres_db_dev:
-    <<: *orchestrator_postgres_db
-    hostname: orchestrator_postgres_db
-    ports:
-      - "${POSTGRES_DEV_PORT}:5432"
-    profiles:
-      - manual_dev
-
   orchestrator:
-    image: ghcr.io/project-omotes/omotes_orchestrator:1.2.6
+    image: ghcr.io/project-omotes/omotes_orchestrator:1.2.9 # TODO: update to 1.2.10 (with new omotes-sdk-python) when available
     restart: unless-stopped
     networks:
       - omotes
@@ -123,8 +119,8 @@ services:
       TASK_RABBITMQ_PORT: 5672
       TASK_RABBITMQ_VIRTUALHOST: omotes_celery
 
-      POSTGRESQL_HOST: orchestrator_postgres_db
-      POSTGRESQL_PORT: 5432
+      POSTGRESQL_HOST: omotes_postgres
+      POSTGRESQL_PORT: ${POSTGRES_PORT}
       POSTGRESQL_DATABASE: omotes_jobs
       POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME}
       POSTGRESQL_PASSWORD: ${POSTGRES_ORCHESTRATOR_USER_PASSWORD}
@@ -143,13 +139,13 @@ services:
     depends_on:
       rabbitmq:
         condition: service_healthy
-      orchestrator_postgres_db:
+      omotes_postgres:
         condition: service_healthy
       omotes_influxdb:
         condition: service_healthy
 
   grow_worker_optimizer: &grow_worker
-    image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.10
+    image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.15 # TODO: update to 2.1.16 (with new mesido) when available
     restart: unless-stopped
     deploy: &grow_worker_deploy
       replicas: 2
@@ -162,10 +158,13 @@ services:
       - omotes
 #      - mapeditor-net
     environment: &grow_worker_env
-      INFLUXDB_HOSTNAME: omotes_influxdb
-      INFLUXDB_PORT: ${INFLUXDB_PORT}
-      INFLUXDB_USERNAME: ${INFLUXDB_FRONTEND_ADMIN_USER}
-      INFLUXDB_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD}
+      ESDL_OUTPUT_PROFILES_TYPE: POSTGRESQL
+      DB_HOSTNAME: omotes_postgres
+      DB_PORT: ${POSTGRES_PORT}
+      DB_USERNAME: ${POSTGRES_TIMESERIES_RW_USER_NAME}
+      DB_PASSWORD: ${POSTGRES_TIMESERIES_RW_USER_PASSWORD}
+      # Used when ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL
+      PG_DB_TIMESERIES: omotes_timeseries
 
       RABBITMQ_HOSTNAME: rabbitmq-nwn
       RABBITMQ_PORT: 5672

example_sdk_client/optimizer.py

@@ -11,17 +11,17 @@
     JobStatusUpdate,
 )
 
-rabbitmq_config = RabbitMQConfig(username="omotes", password="somepass1", virtual_host="omotes")
+rabbitmq_config = RabbitMQConfig(port=5673,username="omotes", password="somepass1", virtual_host="omotes")
 STOP_EVENT = threading.Event()
 
 
 def handle_on_finished(job: Job, result: JobResult):
-    print("--------------Logs:")
-    print(result.logs)
-    print()
-    print("--------------ESDL:")
-    print(result.output_esdl)
-    print()
+    # print("--------------Logs:")
+    # print(result.logs)
+    # print()
+    # print("--------------ESDL:")
+    # print(result.output_esdl)
+    # print()
     print("--------------ESDL messages:")
     print(result.esdl_messages)
     print("--------------Result")

postgres/postgres-init.sql

@@ -1,8 +1,54 @@
-CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
-ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
-GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME;
-GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME;
-GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME;
-GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME;
-ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME;
-ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME;
+-- create databases
+CREATE DATABASE :PG_DB_JOBS;
+CREATE DATABASE :PG_DB_TIMESERIES;
+
+-- create roles
+CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD';
+CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD';
+CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD';
+
+-- DB1
+\connect :PG_DB_JOBS
+
+-- Make RW user owner
+ALTER DATABASE :PG_DB_JOBS OWNER TO :JOBS_RW_USER_NAME;
+ALTER SCHEMA public OWNER TO :JOBS_RW_USER_NAME;
+
+-- Ensure RW can create objects
+GRANT ALL ON SCHEMA public TO :JOBS_RW_USER_NAME;
+
+-- Existing objects
+GRANT ALL ON ALL TABLES IN SCHEMA public TO :JOBS_RW_USER_NAME;
+GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :JOBS_RW_USER_NAME;
+GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :JOBS_RW_USER_NAME;
+
+-- Default privileges
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :JOBS_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :JOBS_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :JOBS_RW_USER_NAME;
+
+-- DB2
+\connect :PG_DB_TIMESERIES
+
+-- Make RW owner
+ALTER DATABASE :PG_DB_TIMESERIES OWNER TO :TIMESERIES_RW_USER_NAME;
+ALTER SCHEMA public OWNER TO :TIMESERIES_RW_USER_NAME;
+
+-- Schema privileges
+GRANT CONNECT ON DATABASE :PG_DB_TIMESERIES TO :TIMESERIES_RO_USER_NAME;
+GRANT ALL ON SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME;
+
+-- Existing objects
+GRANT ALL ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RO_USER_NAME;
+
+-- Default privileges for public schema objects created by RW user
+ALTER DEFAULT PRIVILEGES FOR ROLE :TIMESERIES_RW_USER_NAME IN SCHEMA public GRANT ALL ON TABLES TO :TIMESERIES_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES FOR ROLE :TIMESERIES_RW_USER_NAME IN SCHEMA public GRANT ALL ON SEQUENCES TO :TIMESERIES_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES FOR ROLE :TIMESERIES_RW_USER_NAME IN SCHEMA public GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME;
+
+-- Default privileges for new schemas created by RW user (one schema per optimization run)
+ALTER DEFAULT PRIVILEGES FOR ROLE :TIMESERIES_RW_USER_NAME GRANT USAGE ON SCHEMAS TO :TIMESERIES_RO_USER_NAME;
+ALTER DEFAULT PRIVILEGES FOR ROLE :TIMESERIES_RW_USER_NAME GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME;

scripts/setup.ps1

@@ -13,21 +13,21 @@ get-content .env | foreach {
 # docker compose stop
 
 # Deploy postgres omotes schema
-docker compose --profile=manual_dev down orchestrator_postgres_db_dev
-docker compose up -d --wait orchestrator_postgres_db
-docker compose exec orchestrator_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;'
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose down
+docker compose up -d --wait postgres_db
+docker compose exec omotes_postgres psql -d postgres -c 'CREATE DATABASE omotes_jobs;'
+docker compose exec omotes_postgres psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
 
 # Upgrade omotes tables
-docker compose build orchestrator_postgres_db_upgrade
-docker compose run --rm orchestrator_postgres_db_upgrade
+docker compose build omotes_postgres_upgrade
+docker compose run --rm omotes_postgres_upgrade
 
 # Setup rabbitmq
 docker compose up -d --wait rabbitmq

scripts/setup.sh

@@ -10,15 +10,22 @@ DOCKER_COMPOSE_FILE=${2:-"./docker-compose.yml -f ./docker-compose.override.setu
 echo "Using docker compose file at: $DOCKER_COMPOSE_FILE"
 
 # stop system before setup
-$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE down
 
 # add postgres user with privileges
-$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait orchestrator_postgres_db
-$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec orchestrator_postgres_db psql \
-  -d omotes_jobs \
-  -v PG_USERNAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \
-  -v PG_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \
-  -v PG_DB=omotes_jobs \
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres psql \
+  -U "$POSTGRES_ROOT_USER" \
+  -p "$POSTGRES_PORT" \
+  -d postgres \
+  -v PG_DB_JOBS=omotes_jobs \
+  -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \
+  -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \
+  -v PG_DB_TIMESERIES=omotes_timeseries \
+  -v TIMESERIES_RW_USER_NAME="$POSTGRES_TIMESERIES_RW_USER_NAME" \
+  -v TIMESERIES_RW_USER_PASSWORD="$POSTGRES_TIMESERIES_RW_USER_PASSWORD" \
+  -v TIMESERIES_RO_USER_NAME="$POSTGRES_TIMESERIES_RO_USER_NAME" \
+  -v TIMESERIES_RO_USER_PASSWORD="$POSTGRES_TIMESERIES_RO_USER_PASSWORD" \
   -f /setup/init.sql
 
 # add rabbitmq 'omotes' and 'celery' vhosts and users

scripts/start-dev.sh

@@ -2,6 +2,6 @@
 
 . scripts/_select_docker_compose.sh
 
-$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev
+$DOCKER_COMPOSE down
 $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build
 $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait

scripts/start.sh

@@ -2,5 +2,5 @@
 
 . scripts/_select_docker_compose.sh
 
-$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev
+$DOCKER_COMPOSE down
 $DOCKER_COMPOSE up --wait

scripts/stop.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 . scripts/_select_docker_compose.sh
 
-$DOCKER_COMPOSE --profile=manual_dev down
+$DOCKER_COMPOSE down

system_tests/docker-compose.override.yml

@@ -19,8 +19,8 @@ services:
       RABBITMQ_HOST: rabbitmq
       RABBITMQ_PORT: 5672
 
-      POSTGRES_HOST: orchestrator_postgres_db
-      POSTGRES_PORT: 5432
+      POSTGRES_HOST: omotes_postgres
+      POSTGRESQL_PORT: ${POSTGRES_PORT}
       POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator
       POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3
 
@@ -33,7 +33,3 @@ services:
       TIMEOUT_JOB_MANAGER_START_BUFFER_SEC: 2
       TIMEOUT_JOB_HANDLER_RERUN_SEC: 5
       JOB_RETENTION_SEC: 20
-
-  orchestrator_postgres_db:
-    ports:
-      - "${POSTGRES_DEV_PORT}:5432"