From 54581f2de319f48b012f076e5e23f77d81484bd9 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 2 Jun 2025 09:10:06 +0000 Subject: [PATCH 001/145] updated config for compatibility with newest ump release --- .../templates/configmap-job-settings.yaml | 28 +++++++++++++---- .../templates/configmap-settings.yaml | 28 +++++++++++++---- charts/urban-model-platform/values.yaml | 31 ++++++++++++++----- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/charts/urban-model-platform/templates/configmap-job-settings.yaml b/charts/urban-model-platform/templates/configmap-job-settings.yaml index cb34e2b..c2d57b6 100644 --- a/charts/urban-model-platform/templates/configmap-job-settings.yaml +++ b/charts/urban-model-platform/templates/configmap-job-settings.yaml @@ -9,9 +9,25 @@ metadata: labels: {{- include "ump.labels" . | nindent 4 }} data: - PROVIDERS_FILE: {{ .Values.config.providersFilePath | quote }} - CORS_URL_REGEX: {{ .Values.config.corsUrlRegex | quote }} - API_SERVER_URL: {{ .Values.config.serverUrl | quote }} - NUMBER_OF_WORKERS: {{ .Values.config.numberOfWorkers | quote }} - POSTGRES_PORT: {{ .Values.config.postgresPort | quote }} - POSTGRES_HOST: {{ .Values.config.postgresHost | quote }} + LOG_LEVEL: {{ .Values.config.logLevel | quote }} + PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} + API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} + REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} + DATABASE_NAME: {{ .Values.config.databaseName | quote }} + DATABASE_HOST: {{ .Values.config.databaseHost | quote }} + DATABASE_PORT: {{ .Values.config.databasePort | quote }} + DATABASE_USER: {{ .Values.config.databaseUser | quote }} + DATABASE_PASSWORD: {{ .Values.config.databasePassword | quote }} + GEOSERVER_URL: {{ .Values.config.geoserverUrl | quote }} + GEOSERVER_DB_HOST: {{ .Values.config.geoserverDbHost | quote }} + GEOSERVER_DB_PORT: {{ .Values.config.geoserverDbPort | quote }} + GEOSERVER_WORKSPACE_NAME: {{ .Values.config.geoserverWorkspaceName | quote }} + GEOSERVER_USER: {{ .Values.config.geoserverUser | quote }} + GEOSERVER_PASSWORD: {{ .Values.config.geoserverPassword | quote }} + GEOSERVER_CONNECTION_TIMEOUT: {{ .Values.config.geoserverConnectionTimeout | quote }} + JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} + KEYCLOAK_URL: {{ .Values.config.keycloakUrl | quote }} + KEYCLOAK_REALM: {{ .Values.config.keycloakRealm | quote }} + KEYCLOAK_CLIENT_ID: {{ .Values.config.keycloakClientId | quote }} + KEYCLOAK_USER: {{ .Values.config.keycloakUser | quote }} + KEYCLOAK_PASSWORD: {{ .Values.config.keycloakPassword | quote }} diff --git a/charts/urban-model-platform/templates/configmap-settings.yaml b/charts/urban-model-platform/templates/configmap-settings.yaml index 7b7efeb..a24c641 100644 --- a/charts/urban-model-platform/templates/configmap-settings.yaml +++ b/charts/urban-model-platform/templates/configmap-settings.yaml @@ -5,9 +5,25 @@ metadata: labels: {{- include "ump.labels" . | nindent 4 }} data: - PROVIDERS_FILE: {{ .Values.config.providersFilePath | quote }} - CORS_URL_REGEX: {{ .Values.config.corsUrlRegex | quote }} - API_SERVER_URL: {{ .Values.config.serverUrl | quote }} - NUMBER_OF_WORKERS: {{ .Values.config.numberOfWorkers | quote }} - POSTGRES_PORT: {{ .Values.config.postgresPort | quote }} - POSTGRES_HOST: {{ .Values.config.postgresHost | quote }} + LOG_LEVEL: {{ .Values.config.logLevel | quote }} + PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} + API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} + REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} + DATABASE_NAME: {{ .Values.config.databaseName | quote }} + DATABASE_HOST: {{ .Values.config.databaseHost | quote }} + DATABASE_PORT: {{ .Values.config.databasePort | quote }} + DATABASE_USER: {{ .Values.config.databaseUser | quote }} + DATABASE_PASSWORD: {{ .Values.config.databasePassword | quote }} + GEOSERVER_URL: {{ .Values.config.geoserverUrl | quote }} + GEOSERVER_DB_HOST: {{ .Values.config.geoserverDbHost | quote }} + GEOSERVER_DB_PORT: {{ .Values.config.geoserverDbPort | quote }} + GEOSERVER_WORKSPACE_NAME: {{ .Values.config.geoserverWorkspaceName | quote }} + GEOSERVER_USER: {{ .Values.config.geoserverUser | quote }} + GEOSERVER_PASSWORD: {{ .Values.config.geoserverPassword | quote }} + GEOSERVER_CONNECTION_TIMEOUT: {{ .Values.config.geoserverConnectionTimeout | quote }} + JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} + KEYCLOAK_URL: {{ .Values.config.keycloakUrl | quote }} + KEYCLOAK_REALM: {{ .Values.config.keycloakRealm | quote }} + KEYCLOAK_CLIENT_ID: {{ .Values.config.keycloakClientId | quote }} + KEYCLOAK_USER: {{ .Values.config.keycloakUser | quote }} + KEYCLOAK_PASSWORD: {{ .Values.config.keycloakPassword | quote }} diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index 184c096..08dd0fe 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -5,7 +5,7 @@ fullnameOverride: "" image: repository: lgvanalytics.azurecr.io/urban-model-platform pullPolicy: IfNotPresent - tag: "dev_improvements" + tag: "1.2.0" pullSecrets: - name: secret @@ -40,13 +40,28 @@ service: targetPort: 5000 port the container itsel uses config: - corsUrlRegex: "*" - serverUrl: "0.0.0.0:5000" - providersFileMountPath: /app - providersFilePath: /app/providers.yaml - numberOfWorkers: "1" - postgresHost: "" - postgresPort: 5432 + logLevel: "DEBUG" + providersFile: "./providers.yaml" + apiServerUrl: "localhost:5003" + remoteJobStatusRequestInterval: "5" + databaseName: "ump" + databaseHost: "api-db" + databasePort: 5432 + databaseUser: "ump" + databasePassword: "ump" + geoserverUrl: "http://geoserver:8080/geoserver" + geoserverDbHost: "localhost" + geoserverDbPort: 5432 + geoserverWorkspaceName: "UMP" + geoserverUser: "admin" + geoserverPassword: "geoserver" + geoserverConnectionTimeout: "60" + jobDeleteInterval: "240" + keycloakUrl: "http://keycloak:8080/auth" + keycloakRealm: "UrbanModelPlatform" + keycloakClientId: "ump-client" + keycloakUser: "ump" + keycloakPassword: "ump" postgresConnection: existingSecret: From 4b79797fb1c7e5a759992e030e1887a945d07dff Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 2 Jun 2025 09:23:40 +0000 Subject: [PATCH 002/145] bump chart version: 0.6.0 --> 0.7.0 --- charts/urban-model-platform/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 526f5fb..7d7ea2f 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.6.0 +version: 0.7.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.9.2" +appVersion: "1.2.0" kubeVersion: ">= 1.31.0" annotations: category: API From 5c7fb299330e535b048a594ea147bb2cd4eb7144 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 2 Jun 2025 12:05:17 +0000 Subject: [PATCH 003/145] updated keycloak credentials --- .../templates/secret-keycloak-job.yaml | 9 +++++++-- .../urban-model-platform/templates/secret-keycloak.yaml | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/charts/urban-model-platform/templates/secret-keycloak-job.yaml b/charts/urban-model-platform/templates/secret-keycloak-job.yaml index f470c72..1c360ab 100644 --- a/charts/urban-model-platform/templates/secret-keycloak-job.yaml +++ b/charts/urban-model-platform/templates/secret-keycloak-job.yaml @@ -1,3 +1,6 @@ +# TODO: currently the full app is loaded when inly migrations are run +# TODO: create an app factory that only runs migrations and dont needs keycloak and geoserver connections +# TODO: then, the respective job secrets can be omitted {{- if not .Values.keycloakConnection.existingSecret.name -}} apiVersion: v1 kind: Secret @@ -10,8 +13,10 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - KEYCLOAK_USER: "" + KEYCLOAK_CLIENT_ID: "" KEYCLOAK_PASSWORD: "" - KEYCLOAK_HOST: "" KEYCLOAK_PROTOCOL: "" + KEYCLOAK_REALM: "" + KEYCLOAK_URL: "" + KEYCLOAK_USER: "" {{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-keycloak.yaml b/charts/urban-model-platform/templates/secret-keycloak.yaml index bd2a1a8..82895d0 100644 --- a/charts/urban-model-platform/templates/secret-keycloak.yaml +++ b/charts/urban-model-platform/templates/secret-keycloak.yaml @@ -6,8 +6,10 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - KEYCLOAK_USER: "" + KEYCLOAK_CLIENT_ID: "" KEYCLOAK_PASSWORD: "" - KEYCLOAK_HOST: "" KEYCLOAK_PROTOCOL: "" + KEYCLOAK_REALM: "" + KEYCLOAK_URL: "" + KEYCLOAK_USER: "" {{- end -}} \ No newline at end of file From 5b2a6d5d738589d7ff1b4f5e19e92bad851704d4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 2 Jun 2025 12:05:28 +0000 Subject: [PATCH 004/145] added geoserver credentials --- .../templates/secret-geoserver-job.yaml | 19 +++++++++++++++++++ .../templates/secret-geoserver.yaml | 15 +++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 charts/urban-model-platform/templates/secret-geoserver-job.yaml create mode 100644 charts/urban-model-platform/templates/secret-geoserver.yaml diff --git a/charts/urban-model-platform/templates/secret-geoserver-job.yaml b/charts/urban-model-platform/templates/secret-geoserver-job.yaml new file mode 100644 index 0000000..fa225f6 --- /dev/null +++ b/charts/urban-model-platform/templates/secret-geoserver-job.yaml @@ -0,0 +1,19 @@ +{{- if not .Values.keycloakConnection.existingSecret.name -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ump.fullname" . }}-job-geoserver-connection + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-6" # Lower weight than job (-5) ensures it runs first + "helm.sh/hook-delete-policy": hook-succeeded + labels: + {{ include "ump.labels" . | nindent 4 }} +data: + url: "" + dbHost: "" + dbPort: "" + workspaceName: "" + user: "" + password: "" +{{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-geoserver.yaml b/charts/urban-model-platform/templates/secret-geoserver.yaml new file mode 100644 index 0000000..a50bc35 --- /dev/null +++ b/charts/urban-model-platform/templates/secret-geoserver.yaml @@ -0,0 +1,15 @@ +{{- if not .Values.keycloakConnection.existingSecret.name -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ump.fullname" . }}-geoserver-connection + labels: + {{ include "ump.labels" . | nindent 4 }} +data: + url: "" + dbHost: "" + dbPort: "" + workspaceName: "" + user: "" + password: "" +{{- end -}} \ No newline at end of file From ef45aa1292f102696beb792f4fd171950548e8b0 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 2 Jun 2025 12:05:50 +0000 Subject: [PATCH 005/145] removed credentials --- charts/urban-model-platform/values.yaml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index 08dd0fe..e17ff32 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -42,26 +42,10 @@ service: config: logLevel: "DEBUG" providersFile: "./providers.yaml" - apiServerUrl: "localhost:5003" + apiServerUrl: "0.0.0.0:5000" remoteJobStatusRequestInterval: "5" - databaseName: "ump" - databaseHost: "api-db" - databasePort: 5432 - databaseUser: "ump" - databasePassword: "ump" - geoserverUrl: "http://geoserver:8080/geoserver" - geoserverDbHost: "localhost" - geoserverDbPort: 5432 - geoserverWorkspaceName: "UMP" - geoserverUser: "admin" - geoserverPassword: "geoserver" geoserverConnectionTimeout: "60" jobDeleteInterval: "240" - keycloakUrl: "http://keycloak:8080/auth" - keycloakRealm: "UrbanModelPlatform" - keycloakClientId: "ump-client" - keycloakUser: "ump" - keycloakPassword: "ump" postgresConnection: existingSecret: @@ -71,6 +55,10 @@ keycloakConnection: existingSecret: name: "" +geoserverConnection: + existingSecret: + name: "" + # If configMap for providers is already existing and should not be overwritten, set this to true. Default: false providers: existingConfigMap: From 10cd4145103f185ce76a43154f9c9925e18fb954 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 3 Jun 2025 07:30:19 +0000 Subject: [PATCH 006/145] fixed: comment --- charts/urban-model-platform/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index e17ff32..e675ee5 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -37,7 +37,7 @@ resources: service: type: ClusterIP port: 5000 # port under which the svc answers - targetPort: 5000 port the container itsel uses + targetPort: 5000 # port the container itself uses config: logLevel: "DEBUG" From 7ba1149c18427f36be09db41d2e1623b1b822e39 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 2 Jul 2025 10:54:42 +0000 Subject: [PATCH 007/145] removed secret-based settings --- .../templates/configmap-settings.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/charts/urban-model-platform/templates/configmap-settings.yaml b/charts/urban-model-platform/templates/configmap-settings.yaml index a24c641..8fff047 100644 --- a/charts/urban-model-platform/templates/configmap-settings.yaml +++ b/charts/urban-model-platform/templates/configmap-settings.yaml @@ -9,21 +9,4 @@ data: PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} - DATABASE_NAME: {{ .Values.config.databaseName | quote }} - DATABASE_HOST: {{ .Values.config.databaseHost | quote }} - DATABASE_PORT: {{ .Values.config.databasePort | quote }} - DATABASE_USER: {{ .Values.config.databaseUser | quote }} - DATABASE_PASSWORD: {{ .Values.config.databasePassword | quote }} - GEOSERVER_URL: {{ .Values.config.geoserverUrl | quote }} - GEOSERVER_DB_HOST: {{ .Values.config.geoserverDbHost | quote }} - GEOSERVER_DB_PORT: {{ .Values.config.geoserverDbPort | quote }} - GEOSERVER_WORKSPACE_NAME: {{ .Values.config.geoserverWorkspaceName | quote }} - GEOSERVER_USER: {{ .Values.config.geoserverUser | quote }} - GEOSERVER_PASSWORD: {{ .Values.config.geoserverPassword | quote }} - GEOSERVER_CONNECTION_TIMEOUT: {{ .Values.config.geoserverConnectionTimeout | quote }} JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} - KEYCLOAK_URL: {{ .Values.config.keycloakUrl | quote }} - KEYCLOAK_REALM: {{ .Values.config.keycloakRealm | quote }} - KEYCLOAK_CLIENT_ID: {{ .Values.config.keycloakClientId | quote }} - KEYCLOAK_USER: {{ .Values.config.keycloakUser | quote }} - KEYCLOAK_PASSWORD: {{ .Values.config.keycloakPassword | quote }} From 9a5bbd27334be31ad8797d54d6f505288c6d37d8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 2 Jul 2025 10:55:08 +0000 Subject: [PATCH 008/145] added env from secrets --- .../templates/deployment.yaml | 68 +++++++++++++++---- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index 4ffe843..cd66484 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -51,43 +51,46 @@ spec: mountPath: {{ .Values.config.providersFileMountPath | quote }} readOnly: true envFrom: - - configMapRef: + - configMapRef: name: {{ include "ump.fullname" . }}-settings {{- if not .Values.keycloakConnection.existingSecret.name }} - secretRef: name: {{ include "ump.fullname" . }}-keycloak-connection {{- end }} + {{- if not .Values.geoserverConnection.existingSecret.name }} + - secretRef: + name: {{ include "ump.fullname" . }}-geoserver-connection + {{- end }} env: - - name: POSTGRES_DB + # postgres connection + {{- if .Values.postgresConnection.existingSecret.name }} + - name: DATABASE_NAME valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: dbname - {{- if not .Values.config.postgresHost }} - - name: POSTGRES_HOST + - name: DATABASE_HOST valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: host - {{- end }} - {{- if not .Values.config.postgresPort }} - - name: POSTGRES_PORT + - name: DATABASE_PORT valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: port - {{- end }} - - name: POSTGRES_USER + - name: DATABASE_USER valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: user - - name: POSTGRES_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: password # keycloak connection + {{- end }} {{- if .Values.keycloakConnection.existingSecret.name }} - name: KEYCLOAK_USER valueFrom: @@ -99,17 +102,54 @@ spec: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} key: password - - name: KEYCLOAK_HOST + - name: KEYCLOAK_REALM valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} - key: host - - name: KEYCLOAK_PROTOCOL + key: realm + - name: KEYCLOAK_URL valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} - key: protocol + key: url + - name: KEYCLOAK_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ .Values.keycloakConnection.existingSecret.name }} + key: clientId {{- end }} + {{- if .Values.geoserverConnection.existingSecret.name }} + - name: GEOSERVER_URL + valueFrom: + secretKeyRef: + name: {{ .Values.geoserverConnection.existingSecret.name }} + key: url + - name: GEOSERVER_DB_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.geoserverConnection.existingSecret.name }} + key: dbHost + - name: GEOSERVER_DB_PORT + valueFrom: + secretKeyRef: + name: {{ .Values.geoserverConnection.existingSecret.name }} + key: dbPort + - name: GEOSERVER_WORKSPACE_NAME + valueFrom: + secretKeyRef: + name: {{ .Values.geoserverConnection.existingSecret.name }} + key: workspaceName + - name: GEOSERVER_USER + valueFrom: + secretKeyRef: + name: {{ .Values.geoserverConnection.existingSecret.name }} + key: user + - name: GEOSERVER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.geoserverConnection.existingSecret.name }} + key: password + {{- end }} readinessProbe: httpGet: path: /api/health/ready From 12f9f770db360b5496a5f07cdfe758a681f034bb Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 2 Jul 2025 10:55:51 +0000 Subject: [PATCH 009/145] fixed env var naemes --- .../templates/secret-geoserver-job.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/urban-model-platform/templates/secret-geoserver-job.yaml b/charts/urban-model-platform/templates/secret-geoserver-job.yaml index fa225f6..b8e0ed8 100644 --- a/charts/urban-model-platform/templates/secret-geoserver-job.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver-job.yaml @@ -10,10 +10,10 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - url: "" - dbHost: "" - dbPort: "" - workspaceName: "" - user: "" - password: "" + GEOSERVER_URL: "" + GEOSERVER_DB_HOST: "" + GEOSERVER_DB_PORT: "" + GEOSERVER_WORKSPACE_NAME: "" + GEOSERVER_USER: "" + GEOSERVER_PASSWORD: "" {{- end -}} \ No newline at end of file From 59fa98bf80505e1a54a87ee55041dd9a416b4ea0 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 2 Jul 2025 10:56:01 +0000 Subject: [PATCH 010/145] fixed env var names --- .../templates/secret-geoserver.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/urban-model-platform/templates/secret-geoserver.yaml b/charts/urban-model-platform/templates/secret-geoserver.yaml index a50bc35..49fbd76 100644 --- a/charts/urban-model-platform/templates/secret-geoserver.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver.yaml @@ -6,10 +6,10 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - url: "" - dbHost: "" - dbPort: "" - workspaceName: "" - user: "" - password: "" + GEOSERVER_URL: "" + GEOSERVER_DB_HOST: "" + GEOSERVER_DB_PORT: "" + GEOSERVER_WORKSPACE_NAME: "" + GEOSERVER_USER: "" + GEOSERVER_PASSWORD: "" {{- end -}} \ No newline at end of file From 32831191bdc6b82301ee7fa5f606e6b2f1543cdf Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 3 Jul 2025 06:17:13 +0000 Subject: [PATCH 011/145] added scratch folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 134e617..051860b 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,4 @@ cython_debug/ geoserver_data postgresql_data +scratch/* \ No newline at end of file From 816e70745c5b429be13eb467d5be0d80bb289678 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 3 Jul 2025 06:17:36 +0000 Subject: [PATCH 012/145] made url prefix customizable --- src/ump/config.py | 1 + src/ump/main.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ump/config.py b/src/ump/config.py index 65fdbf8..a7bc009 100644 --- a/src/ump/config.py +++ b/src/ump/config.py @@ -34,6 +34,7 @@ class UmpSettings(BaseSettings): UMP_KEYCLOAK_CLIENT_ID: str = "ump-client" UMP_KEYCLOAK_USER: str = "admin" UMP_KEYCLOAK_PASSWORD: SecretStr = "admin" + UMP_URL_PREFIX: str = "/" @computed_field @property diff --git a/src/ump/main.py b/src/ump/main.py index 51c51a4..b8177e6 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -129,7 +129,7 @@ def cleanup(): CORS(app) -api = APIBlueprint("api", __name__, url_prefix="/api") +api = APIBlueprint("api", __name__, url_prefix=config.UMP_URL_PREFIX) api.register_blueprint(processes, url_prefix="/processes") api.register_blueprint(jobs, url_prefix="/jobs") api.register_blueprint(ensembles, url_prefix="/ensembles") From 3e9ad619d8f90c5a08b55eccf5a59d20bf5475f8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 3 Jul 2025 06:17:45 +0000 Subject: [PATCH 013/145] removed --- scratch/README.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 scratch/README.md diff --git a/scratch/README.md b/scratch/README.md deleted file mode 100644 index 47fde96..0000000 --- a/scratch/README.md +++ /dev/null @@ -1 +0,0 @@ -Hier kann man kleine Codesnippets ausprobieren und testen. \ No newline at end of file From 8476a3273ef3601aa70bc083b6e54f44e0b6ec28 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 3 Jul 2025 06:39:20 +0000 Subject: [PATCH 014/145] added timeout --- src/ump/api/processes.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 7554732..06a87d8 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -24,7 +24,16 @@ async def load_processes(): auth.get("resource_access", {}).get("ump-client", {}).get("roles", []) ) - async with aiohttp.ClientSession(raise_for_status=True) as session: + client_timeout = ClientTimeout( + total=5, # Set a reasonable timeout for the requests + connect=2, # Connection timeout + sock_connect=2, # Socket connection timeout + sock_read=5, # Socket read timeout + ) # remote server needs to answer in time, because we make multiple requests! + + async with aiohttp.ClientSession( + raise_for_status=True, timeout=client_timeout + ) as session: # Create a list of tasks for fetching processes concurrently tasks = [ fetch_provider_processes( From a3ea658e452edabebb3554225c386c6585544cf9 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 3 Jul 2025 06:42:04 +0000 Subject: [PATCH 015/145] fixed missing engine, removed unused imports --- src/ump/api/routes/jobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ump/api/routes/jobs.py b/src/ump/api/routes/jobs.py index a2fa726..4e6c935 100644 --- a/src/ump/api/routes/jobs.py +++ b/src/ump/api/routes/jobs.py @@ -7,13 +7,12 @@ from flask import Response, g, request from sqlalchemy import or_, select from sqlalchemy.orm import Session -from ump import config from ump.api.models.ensemble import JobsUsers from ump.api.models.job import Job from ump.api.models.job_comments import JobComment from ump.api.jobs import append_ensemble_list, get_jobs from ump.api.keycloak_utils import find_user_id_by_email -from ump.config import app_settings as config +from ump.api.db_handler import engine jobs = APIBlueprint("jobs", __name__) From 6b20910bad13edcc043b99c5db8354a5a2e58fb9 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:27:10 +0000 Subject: [PATCH 016/145] handle remote server errors gracefully --- src/ump/utils.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 src/ump/utils.py diff --git a/src/ump/utils.py b/src/ump/utils.py new file mode 100644 index 0000000..5aa1581 --- /dev/null +++ b/src/ump/utils.py @@ -0,0 +1,55 @@ +import asyncio +import aiohttp + +from ump.api.models.ogc_exception import OGCExceptionResponse +from ump.errors import OGCProcessException + + +async def fetch_json(session, url, **kwargs): + try: + async with session.get(url, **kwargs) as response: + response.raise_for_status() + try: + return await response.json() + except aiohttp.ContentTypeError: + # Not JSON + text = await response.text() + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Invalid Response Content", + status=502, + detail=f"Response from {url} is not valid JSON. Content: {text[:200]}", + instance=url + ) + ) + except asyncio.TimeoutError: + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Upstream Timeout", + status=504, + detail=f"Request to {url} timed out.", + instance=url + ) + ) + except aiohttp.ClientError as e: + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Upstream Connection Error", + status=502, + detail=f"Error connecting to {url}: {str(e)}", + instance=url + ) + ) + except Exception as e: + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Internal Server Error", + status=500, + detail=f"Unexpected error for {url}: {str(e)}", + instance=url + ) + ) \ No newline at end of file From c535e4681bbc3f98e909fea8b5dcec1b2edeb11f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:27:30 +0000 Subject: [PATCH 017/145] added debug config --- .vscode/launch.json | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a11437c..0db9f82 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,31 @@ "version": "0.2.0", "configurations": [ { - "name": "Launch api", + "name": "ump + db", + "type": "debugpy", + "request": "launch", + "module": "flask", + "envFile": "${workspaceFolder}/.env", + "env": { + "FLASK_APP": "src/ump/main.py", + "FLASK_DEBUG": "1" + }, + "args": [ + "--debug", + "run", + "--no-debugger", + "-p", + // "5005", + "${command:pickArgs}" + ], + "jinja": true, + "autoStartBrowser": false, + "preLaunchTask": "start-postgis-container", + "postDebugTask": "stop-postgis-container", + "justMyCode": false, + }, + { + "name": "ump + db + migrations", "type": "debugpy", "request": "launch", "module": "flask", From ab746385aa93d0f05b64241943ba4db9f73af0a5 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:28:11 +0000 Subject: [PATCH 018/145] added an rfc-like exception --- src/ump/api/models/ogc_exception.py | 10 ++++++++++ src/ump/errors.py | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 src/ump/api/models/ogc_exception.py diff --git a/src/ump/api/models/ogc_exception.py b/src/ump/api/models/ogc_exception.py new file mode 100644 index 0000000..35b43fb --- /dev/null +++ b/src/ump/api/models/ogc_exception.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel +from typing import Optional, Dict, Any + +class OGCExceptionResponse(BaseModel): + type: str + title: str + status: int + detail: str + instance: Optional[str] = None + additional: Optional[Dict[str, Any]] = None diff --git a/src/ump/errors.py b/src/ump/errors.py index e45db62..aaa7a35 100644 --- a/src/ump/errors.py +++ b/src/ump/errors.py @@ -1,6 +1,8 @@ import logging import traceback +from ump.api.models.ogc_exception import OGCExceptionResponse + class CustomException(Exception): status_code = 400 @@ -28,3 +30,7 @@ class InvalidUsage(CustomException): class GeoserverException(CustomException): pass + +class OGCProcessException(Exception): + def __init__(self, response: OGCExceptionResponse): + self.response = response \ No newline at end of file From 25e18216b83dd49f5346c6b4b8efd37e1e00dc35 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:28:26 +0000 Subject: [PATCH 019/145] using the rfc-like exception --- src/ump/main.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ump/main.py b/src/ump/main.py index b8177e6..726d9b4 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -29,7 +29,7 @@ from ump.api.routes.processes import processes from ump.api.routes.users import users from ump.config import app_settings as config -from ump.errors import CustomException +from ump.errors import CustomException, OGCProcessException dictConfig( { @@ -146,6 +146,15 @@ def cleanup(): realm_name=config.UMP_KEYCLOAK_REALM, ) + +@app.errorhandler(OGCProcessException) +def handle_ogc_exception(error): + response = jsonify(error.response.model_dump()) + response.status_code = error.response.status + response.content_type = "application/problem+json" + return response + + @app.before_request def check_jwt(): """Decodes the JWT token and runs pending scheduled jobs""" From ea35856809fa459c9ddc4d0995312ee97d754859 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:28:54 +0000 Subject: [PATCH 020/145] added detailed log messages, reduced user facing details --- src/ump/utils.py | 65 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/ump/utils.py b/src/ump/utils.py index 5aa1581..a639ef2 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -1,9 +1,13 @@ import asyncio +import logging + import aiohttp from ump.api.models.ogc_exception import OGCExceptionResponse from ump.errors import OGCProcessException +logger = logging.getLogger(__name__) + async def fetch_json(session, url, **kwargs): try: @@ -12,44 +16,87 @@ async def fetch_json(session, url, **kwargs): try: return await response.json() except aiohttp.ContentTypeError: - # Not JSON text = await response.text() + logger.error( + "Invalid JSON response from remote service. URL: %s, Content: %s", + url, text[:500] + ) raise OGCProcessException( OGCExceptionResponse( type="about:blank", title="Invalid Response Content", status=502, - detail=f"Response from {url} is not valid JSON. Content: {text[:200]}", - instance=url + detail="The response from the remote service was not valid JSON.", + instance=None ) ) except asyncio.TimeoutError: + logger.error( + "Timeout when requesting remote service. URL: %s", + url + ) raise OGCProcessException( OGCExceptionResponse( type="about:blank", title="Upstream Timeout", status=504, - detail=f"Request to {url} timed out.", - instance=url + detail="The request to the remote service timed out.", + instance=None + ) + ) + except aiohttp.ClientResponseError as e: + if e.status == 401: + logger.warning( + "Authentication failed when requesting remote service. URL: %s, Error: %s", + url, str(e) + ) + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Authentication Failed", + status=401, + detail="Authentication with the remote service failed.", + instance=None + ) + ) + logger.error( + "HTTP error when requesting remote service. URL: %s, Status: %s, Error: %s", + url, e.status, str(e) + ) + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Upstream HTTP Error", + status=e.status, + detail="The remote service returned an HTTP error.", + instance=None ) ) except aiohttp.ClientError as e: + logger.error( + "Connection error when requesting remote service. URL: %s, Error: %s", + url, str(e) + ) raise OGCProcessException( OGCExceptionResponse( type="about:blank", title="Upstream Connection Error", status=502, - detail=f"Error connecting to {url}: {str(e)}", - instance=url + detail="There was a connection error with the remote service.", + instance=None ) ) except Exception as e: + logger.error( + "Unexpected error for remote service. URL: %s, Error: %s", + url, str(e) + ) raise OGCProcessException( OGCExceptionResponse( type="about:blank", title="Internal Server Error", status=500, - detail=f"Unexpected error for {url}: {str(e)}", - instance=url + detail="An unexpected error occurred while processing your request.", + instance=None ) ) \ No newline at end of file From e3d201d43d5db9795e25d21092e61b679c6e68eb Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:36:38 +0000 Subject: [PATCH 021/145] renamed process to process_config --- src/ump/api/providers.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ump/api/providers.py b/src/ump/api/providers.py index c54e0f1..eb20c16 100644 --- a/src/ump/api/providers.py +++ b/src/ump/api/providers.py @@ -75,7 +75,9 @@ def authenticate_provider(provider: ProviderConfig): return auth -def check_process_availability(provider: str, process_id: str): +def check_process_availability( + provider: str, process_id: str +) -> tuple[bool, ProcessConfig | None]: available = False with PROVIDERS_LOCK: # Ensure thread-safe access @@ -83,13 +85,14 @@ def check_process_availability(provider: str, process_id: str): provider in PROVIDERS and process_id in PROVIDERS[provider].processes ): - process: ProcessConfig = PROVIDERS[provider].processes[process_id] - available = not process.exclude + # load process configuration + process_config: ProcessConfig = PROVIDERS[provider].processes[process_id] + available = not process_config.exclude - if process.exclude: - logging.debug("Excluding process %s based on configuration", process_id) + if process_config.exclude: + logger.debug("Excluding process %s based on configuration", process_id) - return available + return available, process_config def check_result_storage(provider, process_id): From 5d724e4c6e10f3e90f81cad09277270f051718dd Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:37:39 +0000 Subject: [PATCH 022/145] reworked adn fixed error handling, added (short) timeout when fetching from remote servers --- src/ump/api/models/process.py | 157 ++++++++++++++++++++++------------ 1 file changed, 104 insertions(+), 53 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 0aba40f..83a5365 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -9,18 +9,23 @@ import aiohttp from flask import g +from ump.api.models.ogc_exception import OGCExceptionResponse import ump.api.providers as providers from ump.api.db_handler import engine from ump.api.models.job import Job, JobStatus from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.config import app_settings as config -from ump.errors import CustomException, InvalidUsage - -#TODO: unify logging setup, because this takes not into -# account that an admin wants to configure log level -logging.basicConfig(level=logging.INFO) +from ump.errors import CustomException, InvalidUsage, OGCProcessException +from ump.utils import fetch_json +logger = logging.getLogger(__name__) +client_timeout = aiohttp.ClientTimeout( + total=5, # Set a reasonable timeout for the requests + connect=2, # Connection timeout + sock_connect=2, # Socket connection timeout + sock_read=5, # Socket read timeout +) class Process: def __init__(self, process_id_with_prefix=None): @@ -33,71 +38,117 @@ def __init__(self, process_id_with_prefix=None): match = re.search(r"([^:]+):(.*)", self.process_id_with_prefix) if not match: - raise InvalidUsage( - "Process ID %s is not known! Please check endpoint api/processes " - + "for a list of available processes.", - self.process_id_with_prefix, + logger.warning( + "Process ID '%s' does not match pattern 'provider:process_id'. " + "See /api/processes for a list of available processes.", + self.process_id_with_prefix + ) + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", + title="Invalid Process ID", + status=400, + detail=( + f"Process ID '{self.process_id_with_prefix}' " + "does not match pattern: 'provider:process_id'}. " + "See /api/processes for a list of available processes." + ), + instance=f"/processes/{self.process_id_with_prefix}" + ) ) - self.provider_prefix = match.group(1) self.process_id = match.group(2) - if not providers.check_process_availability( + # this checks if the process is known from providers and confiured as available + # for what purpose? -> security! -if a user selects a process which is not confiured to be available, + # but exists + available, process_config = providers.check_process_availability( self.provider_prefix, self.process_id - ): - raise InvalidUsage( - "Process ID %s is not known! Please check endpoint api/processes " - + "for a list of available processes.", - self.process_id_with_prefix, + ) + if not available: + logger.warning( + "Process ID '%s' is not available. " + "Either the process is not configured or it is excluded from the API.", + self.process_id_with_prefix + ) + # raise OGCProcessException to inform users, but with less detail + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", + title="Not available.", + status=400, + detail=( + f"Process ID '{self.process_id_with_prefix}' " + "is not available." + ), + instance=f"/processes/{self.process_id_with_prefix}" + ) ) + logger.debug( + "Process %s is available. Loading process details.", + self.process_id_with_prefix + ) - process_config = providers.get_providers()[self.provider_prefix].processes[self.process_id] - # if anonymous access isn’t enabled, require a token - restricted_access = not getattr(process_config, "anonymous_access", False) + if not process_config.anonymous_access: + + logger.debug( + "Process %s requires authentication. Checking user roles.", + self.process_id_with_prefix + ) - if restricted_access: auth = getattr(g, "auth_token", None) role = f"{self.provider_prefix}_{self.process_id}" realm_roles = auth.get("realm_access", {}).get("roles", []) if auth else [] - client_roles = (auth.get("resource_access", {})\ - .get("ump-client", {})\ - .get("roles", [])) if auth else [] - allowed = any(r in realm_roles + client_roles for r in [self.provider_prefix, role]) + + # TODO: uses hard-coded client name 'ump-client' to get client roles + client_roles = ( + auth.get("resource_access", {}) + .get("ump-client", {}) + .get("roles", []) + ) if auth else [] + + allowed = any( + r in realm_roles + client_roles + for r in [self.provider_prefix, role] + ) + if not auth or not allowed: - raise InvalidUsage( - "Process ID %s is not known! Please check endpoint api/processes " - + "for a list of available processes.", - self.process_id_with_prefix, + logger.warning( + "User is not allowed to access process %s. " + "Either the process is not configured for anonymous access or " + "the user does not have the required roles.", + self.process_id_with_prefix + ) + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", + title="Not available.", + status=400, + detail=( + f"Process ID '{self.process_id_with_prefix}' " + "is not available." + ), + instance=f"/processes/{self.process_id_with_prefix}" + ) ) - asyncio.run(self.set_details()) + asyncio.run(self.load_process_details()) - async def set_details(self): - provider = providers.get_providers()[self.provider_prefix] - # Check for Authentification - auth = providers.authenticate_provider(provider) + async def load_process_details(self): + provider_config = providers.get_providers()[self.provider_prefix] - async with aiohttp.ClientSession() as session: - response = await session.get( - f"{provider.server_url}processes/{self.process_id}", - auth=auth, - headers={ - "Content-type": "application/json", - "Accept": "application/json", - }, - ) + # return auth (BasicAuth curently, only) + # TODO: add support for other auth methods: JWT, ... + auth = providers.authenticate_provider(provider_config) - if response.status != 200: - # TODO: need to differentiate errors better and give more information(url!) - # widespread usage of InvalidUsage error is impractical, because it - # catches UMP user errors as well as programming errors - raise InvalidUsage( - f"Model/process not found! {response.status}: {response.reason}. " - + "Check /api/processes endpoint for available models/processes.", - ) + async with aiohttp.ClientSession(timeout=client_timeout) as session: - process_details = await response.json() + process_details = await fetch_json( + session, + f"{provider_config.server_url}processes/{self.process_id}", + auth=auth + ) for key in process_details: setattr(self, key, process_details[key]) @@ -264,7 +315,7 @@ async def start_process_execution(self, request_body, user): try: auth = providers.authenticate_provider(provider) - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=client_timeout) as session: process_response = await session.get( f"{provider.server_url}processes/{self.process_id}", auth=auth, @@ -359,7 +410,7 @@ async def _wait_for_results(self, job: Job): try: while not finished: - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=client_timeout) as session: auth = providers.authenticate_provider(provider) async with session.get( From 23cf2eebbcbe13bfc3a2e2626d756e93955cbdef Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:51:05 +0000 Subject: [PATCH 023/145] fixed some formatting issues and replaced root logger --- src/ump/api/models/process.py | 109 +++++++++++++++++----------------- 1 file changed, 54 insertions(+), 55 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 83a5365..4c83afe 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -9,10 +9,10 @@ import aiohttp from flask import g -from ump.api.models.ogc_exception import OGCExceptionResponse import ump.api.providers as providers from ump.api.db_handler import engine from ump.api.models.job import Job, JobStatus +from ump.api.models.ogc_exception import OGCExceptionResponse from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.config import app_settings as config from ump.errors import CustomException, InvalidUsage, OGCProcessException @@ -26,9 +26,10 @@ sock_connect=2, # Socket connection timeout sock_read=5, # Socket read timeout ) -class Process: - def __init__(self, process_id_with_prefix=None): + +class Process: + def __init__(self, process_id_with_prefix): self.inputs: dict self.outputs: dict @@ -41,7 +42,7 @@ def __init__(self, process_id_with_prefix=None): logger.warning( "Process ID '%s' does not match pattern 'provider:process_id'. " "See /api/processes for a list of available processes.", - self.process_id_with_prefix + self.process_id_with_prefix, ) raise OGCProcessException( OGCExceptionResponse( @@ -52,16 +53,16 @@ def __init__(self, process_id_with_prefix=None): f"Process ID '{self.process_id_with_prefix}' " "does not match pattern: 'provider:process_id'}. " "See /api/processes for a list of available processes." - ), - instance=f"/processes/{self.process_id_with_prefix}" + ), + instance=f"/processes/{self.process_id_with_prefix}", ) ) self.provider_prefix = match.group(1) self.process_id = match.group(2) # this checks if the process is known from providers and confiured as available - # for what purpose? -> security! -if a user selects a process which is not confiured to be available, - # but exists + # for what purpose? -> security! -if a user selects a process which is not + # confiured to be available, but exists available, process_config = providers.check_process_availability( self.provider_prefix, self.process_id ) @@ -69,7 +70,7 @@ def __init__(self, process_id_with_prefix=None): logger.warning( "Process ID '%s' is not available. " "Either the process is not configured or it is excluded from the API.", - self.process_id_with_prefix + self.process_id_with_prefix, ) # raise OGCProcessException to inform users, but with less detail raise OGCProcessException( @@ -78,47 +79,44 @@ def __init__(self, process_id_with_prefix=None): title="Not available.", status=400, detail=( - f"Process ID '{self.process_id_with_prefix}' " - "is not available." - ), - instance=f"/processes/{self.process_id_with_prefix}" + f"Process ID '{self.process_id_with_prefix}' is not available." + ), + instance=f"/processes/{self.process_id_with_prefix}", ) ) logger.debug( "Process %s is available. Loading process details.", - self.process_id_with_prefix + self.process_id_with_prefix, ) # if anonymous access isn’t enabled, require a token - if not process_config.anonymous_access: - + if process_config and not process_config.anonymous_access: logger.debug( "Process %s requires authentication. Checking user roles.", - self.process_id_with_prefix + self.process_id_with_prefix, ) auth = getattr(g, "auth_token", None) role = f"{self.provider_prefix}_{self.process_id}" - realm_roles = auth.get("realm_access", {}).get("roles", []) if auth else [] - + realm_roles = auth.get("realm_access", {}).get("roles", []) if auth else [] + # TODO: uses hard-coded client name 'ump-client' to get client roles client_roles = ( - auth.get("resource_access", {}) - .get("ump-client", {}) - .get("roles", []) - ) if auth else [] + (auth.get("resource_access", {}).get("ump-client", {}).get("roles", [])) + if auth + else [] + ) allowed = any( - r in realm_roles + client_roles - for r in [self.provider_prefix, role] + r in realm_roles + client_roles for r in [self.provider_prefix, role] ) - + if not auth or not allowed: logger.warning( "User is not allowed to access process %s. " "Either the process is not configured for anonymous access or " "the user does not have the required roles.", - self.process_id_with_prefix + self.process_id_with_prefix, ) raise OGCProcessException( OGCExceptionResponse( @@ -128,8 +126,8 @@ def __init__(self, process_id_with_prefix=None): detail=( f"Process ID '{self.process_id_with_prefix}' " "is not available." - ), - instance=f"/processes/{self.process_id_with_prefix}" + ), + instance=f"/processes/{self.process_id_with_prefix}", ) ) @@ -143,11 +141,10 @@ async def load_process_details(self): auth = providers.authenticate_provider(provider_config) async with aiohttp.ClientSession(timeout=client_timeout) as session: - process_details = await fetch_json( session, f"{provider_config.server_url}processes/{self.process_id}", - auth=auth + auth=auth, ) for key in process_details: setattr(self, key, process_details[key]) @@ -171,7 +168,7 @@ def validate_params(self, parameters): payload={"parameter_description": parameter_metadata}, ) else: - logging.warning( + logger.warning( "Model execution %s started without parameter %s.", self.process_id_with_prefix, input, @@ -251,12 +248,18 @@ def check_for_cache(self, parameters, user_id): # p = providers.PROVIDERS[self.provider_prefix]["processes"][self.process_id] provider: ProviderConfig = providers.get_providers()[self.provider_prefix] process_config: ProcessConfig = provider.processes[self.process_id] - + if process_config.deterministic: return None sql = """ - select job_id from jobs where hash = encode(sha512((%(parameters)s :: json :: text || %(process_version)s || %(user_id)s) :: bytea), 'base64') + select job_id from jobs where hash = encode( + sha512( + ( + %(parameters)s :: json :: text || %(process_version)s || %(user_id)s) :: bytea + ), + 'base64' + ) """ with engine.begin() as conn: result = conn.exec_driver_sql( @@ -276,7 +279,7 @@ def execute(self, parameters, user): self.validate_params(parameters) - logging.info( + logger.info( " --> Executing %s on model server %s with params %s as process %s for user %s", self.process_id, str(provider.server_url), @@ -297,21 +300,21 @@ async def start_process_execution(self, request_body, user): # execution mode: # to maintain backwards compatibility to models using # pre-1.0.0 versions of OGC api processes - #TODO: need to add prefer-header + # TODO: need to add prefer-header request_body["mode"] = "async" provider: ProviderConfig = providers.get_providers()[self.provider_prefix] # extract job_name from request_body - name = request_body.pop("job_name",None) + name = request_body.pop("job_name", None) job_id = self.check_for_cache(request_body, user) if job_id: - logging.info("Job found, returning cached job.") + logger.info("Job found, returning cached job.") return Job(job_id, user) # TODO: this try...except block is too big! # also it does not catch the exact error from the backend server in some cases - # only telling the user that he has a Bad request + # only telling the user that he has a Bad request try: auth = providers.authenticate_provider(provider) @@ -381,7 +384,7 @@ async def start_process_execution(self, request_body, user): job.status = status_json.get("status") job.save() - logging.info( + logger.info( " --> Job %s for model %s started running.", job.job_id, self.process_id_with_prefix, @@ -396,22 +399,19 @@ def _wait_for_results_async(self, job: Job): asyncio.run(self._wait_for_results(job)) async def _wait_for_results(self, job: Job): - - logging.info(" --> Waiting for results in Thread") + logger.info(" --> Waiting for results in Thread") finished = False - + provider: ProviderConfig = providers.get_providers()[self.provider_prefix] - + timeout = float(provider.timeout) start = time.time() job_details = {} try: while not finished: - async with aiohttp.ClientSession(timeout=client_timeout) as session: - auth = providers.authenticate_provider(provider) async with session.get( f"{provider.server_url}jobs/{job.remote_job_id}?f=json", @@ -421,13 +421,12 @@ async def _wait_for_results(self, job: Job): "Accept": "application/json", }, ) as response: - response.raise_for_status() job_details: dict = await response.json() finished = self.is_finished(job_details) - logging.info(" --> Current Job status: %s", str(job_details)) + logger.info(" --> Current Job status: %s", str(job_details)) # either remote job has progress info or else we cannot provide it either job.progress = job_details.get("progress") @@ -439,12 +438,12 @@ async def _wait_for_results(self, job: Job): if time.time() - start > timeout: raise TimeoutError( - f"Job did not finish within {timeout/60} minutes. Giving up." + f"Job did not finish within {timeout / 60} minutes. Giving up." ) time.sleep(config.UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL) - logging.info( + logger.info( " --> Remote execution job %s: success = %s. Took approx. %s minutes.", job.remote_job_id, finished, @@ -452,8 +451,8 @@ async def _wait_for_results(self, job: Job): ) except Exception as e: - logging.error( - " --> Could not retrieve results for job %s (=%s)/%s from simulation model server: %s", + logger.error( + "Could not retrieve results for job %s (=%s)/%s from remote server: %s", self.process_id_with_prefix, self.process_id, job.job_id, @@ -479,13 +478,13 @@ async def _wait_for_results(self, job: Job): job.updated = job.finished job.progress = 100 job.message = ( - f'Remote execution was not successful! {job_details["message"]}' + f"Remote execution was not successful! {job_details['message']}" ) job.save() raise CustomException(f"Remote job {job.remote_job_id}: {job.message}") except CustomException as e: - logging.error(" --> An error occurred: %s", e) + logger.error(" --> An error occurred: %s", e) job.status = JobStatus.successful.value job.finished = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -501,7 +500,7 @@ async def _wait_for_results(self, job: Job): ): await job.results_to_geoserver() except Exception as e: - logging.error( + logger.error( " --> Could not store results for job %s (=%s)/%s to geoserver: %s", self.process_id_with_prefix, self.process_id, From 8241dc8ca4484d70eaa2dac327ebf28a6d72e02c Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 05:51:29 +0000 Subject: [PATCH 024/145] using ruff formatter instead of black --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b4cb618..0143002 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "--preview" ], "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter", + "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.organizeImports": "explicit" @@ -20,4 +20,5 @@ ], "ruff.importStrategy": "fromEnvironment", "black-formatter.importStrategy": "fromEnvironment", + "python.analysis.typeCheckingMode": "basic", } \ No newline at end of file From 205f13b72667c3c86cf3e5855654ee1321e9ac30 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 07:24:01 +0000 Subject: [PATCH 025/145] added process details, renamed vars and func to refelct its purpose --- src/ump/api/models/process.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 4c83afe..93fb834 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -34,8 +34,14 @@ def __init__(self, process_id_with_prefix): self.outputs: dict self.process_id_with_prefix = process_id_with_prefix - self.process_title = None + self.id = None + self.title = None self.version = None + self.job_control_options = None + self.description = None + self.keywords = None + self.metadata = None + self.links = None match = re.search(r"([^:]+):(.*)", self.process_id_with_prefix) if not match: @@ -149,7 +155,7 @@ async def load_process_details(self): for key in process_details: setattr(self, key, process_details[key]) - def validate_params(self, parameters): + def validate_exec_body(self, parameters): if not self.inputs: return @@ -274,21 +280,22 @@ def check_for_cache(self, parameters, user_id): return row.job_id return None - def execute(self, parameters, user): + def execute(self, exec_body, user): provider: ProviderConfig = providers.get_providers()[self.provider_prefix] - self.validate_params(parameters) + # TODO: make this optional (remote servers should do this themselves) + self.validate_exec_body(exec_body) logger.info( - " --> Executing %s on model server %s with params %s as process %s for user %s", + "Executing %s on model server %s with params %s as process %s for user %s", self.process_id, str(provider.server_url), - parameters, + exec_body, self.process_id_with_prefix, user, ) - job = asyncio.run(self.start_process_execution(parameters, user)) + job = asyncio.run(self.start_process_execution(exec_body, user)) _process = dummy.Process(target=self._wait_for_results_async, args=[job]) _process.start() @@ -305,6 +312,7 @@ async def start_process_execution(self, request_body, user): provider: ProviderConfig = providers.get_providers()[self.provider_prefix] # extract job_name from request_body + # TODO: this is undocumented, non-OGC standard name = request_body.pop("job_name", None) job_id = self.check_for_cache(request_body, user) @@ -318,6 +326,7 @@ async def start_process_execution(self, request_body, user): try: auth = providers.authenticate_provider(provider) + # short timeout and prefer-async: no sync execution supported async with aiohttp.ClientSession(timeout=client_timeout) as session: process_response = await session.get( f"{provider.server_url}processes/{self.process_id}", @@ -343,7 +352,7 @@ async def start_process_execution(self, request_body, user): if process_response.ok: process_details = await process_response.json() - self.process_title = process_details["title"] + self.title = process_details["title"] response.raise_for_status() @@ -360,7 +369,7 @@ async def start_process_execution(self, request_body, user): job.create( remote_job_id=remote_job_id, process_id_with_prefix=self.process_id_with_prefix, - process_title=self.process_title, + process_title=self.title, name=name, parameters=request_body, user=user, From ab1376237ae5406ab2b8a07e23054007a545ec0e Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 09:33:57 +0000 Subject: [PATCH 026/145] mainly formatting, added notes on top, renamed two methods --- src/ump/api/models/job.py | 74 +++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index d18fafd..231b8fa 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -8,14 +8,16 @@ import geopandas as gpd import ump.api.providers as providers -from ump.config import app_settings as config from ump.api.db_handler import DBHandler from ump.api.models.job_status import JobStatus from ump.api.models.providers_config import ProcessConfig, ProviderConfig +from ump.config import app_settings as config from ump.errors import CustomException, InvalidUsage from ump.geoserver.geoserver import Geoserver - +# TODO class violates Single Responsibility Principle (SRP), it mixes +# business logic with data access logic and metadata handling +# TODO methods like insert use redundant fields instead of job instance fields class Job: DISPLAYED_ATTRIBUTES = [ "processID", @@ -29,7 +31,7 @@ class Job: "updated", "progress", "links", - ] + ] SORTABLE_COLUMNS = [ "created", @@ -65,14 +67,14 @@ def __init__(self, job_id=None, user=None): if job_id and not self._init_from_db(job_id, user): raise CustomException("Job could not be found!") - def create( + def insert( self, job_id=None, remote_job_id=None, process_id_with_prefix=None, process_title=None, name=None, - parameters=None, + exec_body=None, user=None, process_version=None, ): @@ -82,7 +84,7 @@ def create( process_id_with_prefix, process_title, name, - parameters, + exec_body, user_id=user, process_version=process_version, ) @@ -134,13 +136,15 @@ def _set_attributes( match = re.search(r"(.*):(.*)", self.process_id_with_prefix) if not match: raise InvalidUsage( - f"Process ID {self.process_id_with_prefix} is not known! " + - "Please check endpoint api/processes for a list of available processes." + f"Process ID {self.process_id_with_prefix} is not known! " + + "Please check endpoint api/processes for a list of available processes." ) self.provider_prefix = match.group(1) self.process_id = match.group(2) - self.provider_url = providers.get_providers()[self.provider_prefix].server_url + self.provider_url = providers.get_providers()[ + self.provider_prefix + ].server_url if not self.job_id: self.job_id = str(uuid.uuid4()) @@ -205,7 +209,7 @@ def _to_dict(self): "process_version": self.process_version, } - def save(self): + def update(self): self.updated = datetime.now(timezone.utc) query = """ @@ -231,9 +235,11 @@ def set_results_metadata(self, results_as_json): values = [] for column in maximal_values_dict: - data_type = str(types[column]) - if data_type == "float64" and results_df[column].apply(float.is_integer).all(): + if ( + data_type == "float64" + and results_df[column].apply(float.is_integer).all() + ): data_type = "int" values.append( @@ -263,7 +269,7 @@ def set_results_metadata(self, results_as_json): return self.results_metadata - def display(self,additional_metadata=False): + def display(self, additional_metadata=False): job_dict = self._to_dict() job_dict["type"] = "process" job_dict["jobID"] = job_dict.pop("job_id") @@ -272,7 +278,6 @@ def display(self,additional_metadata=False): job_dict["processID"] = self.process_id_with_prefix job_dict["links"] = [] - for attr in job_dict: if isinstance(job_dict[attr], datetime): job_dict[attr] = job_dict[attr].strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -282,8 +287,9 @@ def display(self,additional_metadata=False): JobStatus.running.value, JobStatus.accepted.value, ): - - job_result_url = f"{config.UMP_API_SERVER_URL}/api/jobs/{self.job_id}/results" + job_result_url = ( + f"{config.UMP_API_SERVER_URL}/api/jobs/{self.job_id}/results" + ) job_dict["links"] = [ { @@ -291,12 +297,12 @@ def display(self,additional_metadata=False): "rel": "service", "type": "application/json", "hreflang": "en", - "title": f"Results of job {self.job_id} as geojson" + - " - available when job is finished.", + "title": f"Results of job {self.job_id} as geojson" + + " - available when job is finished.", } ] if isinstance(additional_metadata, str): - additional_metadata = additional_metadata.lower() == "true" + additional_metadata = additional_metadata.lower() == "true" if additional_metadata: metadata = {} @@ -312,18 +318,24 @@ def display(self,additional_metadata=False): metadata["process_version"] = self.process_version if self.process_version is not None: metadata["user_id"] = self.user_id - + job_dict["metadata"] = metadata - for key in ["name", "parameters", "results_metadata", "process_title", "process_version","user_id"]: - job_dict.pop(key, None) + for key in [ + "name", + "parameters", + "results_metadata", + "process_title", + "process_version", + "user_id", + ]: + job_dict.pop(key, None) return job_dict - + else: return {k: job_dict[k] for k in self.DISPLAYED_ATTRIBUTES} - async def results(self): if self.status != JobStatus.successful.value: return { @@ -350,19 +362,19 @@ async def results(self): return await response.json() else: raise CustomException( - "Could not retrieve results from model server " + - f"{self.provider_url} - {response.status}: {response.reason}" + "Could not retrieve results from model server " + + f"{self.provider_url} - {response.status}: {response.reason}" ) async def results_to_geoserver(self): try: provider: ProviderConfig = providers.get_providers()[self.provider_prefix] - process_config: ProcessConfig = provider.processes[self.process_id] + process_config: ProcessConfig = provider.processes[self.process_id] results = await self.results() - if result_path:= process_config.result_path: - parts =result_path.split('.') + if result_path := process_config.result_path: + parts = result_path.split(".") for part in parts: results = results[part] @@ -374,7 +386,9 @@ async def results_to_geoserver(self): logging.info( " --> Successfully stored results for job %s (=%s)/%s to geoserver.", - self.process_id_with_prefix, self.process_id, self.job_id + self.process_id_with_prefix, + self.process_id, + self.job_id, ) except Exception as e: From aacf80f6de81e7313099ccbf7a3fe0b7cf553434 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 09:36:46 +0000 Subject: [PATCH 027/145] refactored start_process_execution and splitted it into smaller pieces --- src/ump/api/models/process.py | 243 +++++++++++++++++++++------------- 1 file changed, 153 insertions(+), 90 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 93fb834..9b5f32e 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -27,7 +27,10 @@ sock_read=5, # Socket read timeout ) - +# TODO: this is not an OGC API Process in a strict sense, +# instead it incorporates remote process execution logic +# these two things should probably be separated with regard to +# **Single Responsibility Principle (SRP)** class Process: def __init__(self, process_id_with_prefix): self.inputs: dict @@ -67,7 +70,7 @@ def __init__(self, process_id_with_prefix): self.process_id = match.group(2) # this checks if the process is known from providers and confiured as available - # for what purpose? -> security! -if a user selects a process which is not + # for what purpose? -> security! -if a user selects a process which is not # confiured to be available, but exists available, process_config = providers.check_process_availability( self.provider_prefix, self.process_id @@ -297,22 +300,14 @@ def execute(self, exec_body, user): job = asyncio.run(self.start_process_execution(exec_body, user)) - _process = dummy.Process(target=self._wait_for_results_async, args=[job]) - _process.start() + results_thread = dummy.Process(target=self._wait_for_results_async, args=[job]) + results_thread.start() - result = {"jobID": job.job_id, "status": job.status} - return result + return {"jobID": job.job_id, "status": job.status} async def start_process_execution(self, request_body, user): - # execution mode: - # to maintain backwards compatibility to models using - # pre-1.0.0 versions of OGC api processes - # TODO: need to add prefer-header request_body["mode"] = "async" provider: ProviderConfig = providers.get_providers()[self.provider_prefix] - - # extract job_name from request_body - # TODO: this is undocumented, non-OGC standard name = request_body.pop("job_name", None) job_id = self.check_for_cache(request_body, user) @@ -320,89 +315,157 @@ async def start_process_execution(self, request_body, user): logger.info("Job found, returning cached job.") return Job(job_id, user) - # TODO: this try...except block is too big! - # also it does not catch the exact error from the backend server in some cases - # only telling the user that he has a Bad request - try: - auth = providers.authenticate_provider(provider) - - # short timeout and prefer-async: no sync execution supported - async with aiohttp.ClientSession(timeout=client_timeout) as session: - process_response = await session.get( - f"{provider.server_url}processes/{self.process_id}", - auth=auth, - headers={ - "Content-type": "application/json", - "Accept": "application/json", - }, + auth = providers.authenticate_provider(provider) + + async with aiohttp.ClientSession(timeout=client_timeout) as session: + try: + response = await self._submit_remote_job( + session, str(provider.server_url), request_body, auth ) - response = await session.post( - f"{provider.server_url}processes/{self.process_id}/execution", - json=request_body, - auth=auth, - headers={ - "Content-type": "application/json", - "Accept": "application/json", - # execution mode shall be async, if model supports it - "Prefer": "respond-async", - }, + + job = await self._create_local_job_instance( + response, name, request_body, user ) - process_response.raise_for_status() - - if process_response.ok: - process_details = await process_response.json() - self.title = process_details["title"] - - response.raise_for_status() - - if response.ok and response.headers: - # Retrieve the job id from the simulation model server from the - # location header: - match = re.search( - "http.*/jobs/(.*)$", response.headers["location"] - ) or (re.search(".*/jobs/(.*)$", response.headers["location"])) - if match: - remote_job_id = match.group(1) - - job = Job() - job.create( - remote_job_id=remote_job_id, - process_id_with_prefix=self.process_id_with_prefix, - process_title=self.title, - name=name, - parameters=request_body, - user=user, - process_version=self.version, - ) - job.started = datetime.now(timezone.utc).strftime( - "%Y-%m-%dT%H:%M:%S.%fZ" - ) + # this can probably be omitted here and deferred for _wait_for_status + remote_job_status_info = await self._fetch_remote_job_status( + session, provider.server_url, response, auth + ) - status_response = await session.get( - f"{provider.server_url}jobs/{remote_job_id}?f=json", - auth=auth, - headers={ - "Content-type": "application/json", - "Accept": "application/json", - }, + job.status = remote_job_status_info.get("status") + job.message = remote_job_status_info.get("message", "") + + job.update() + + logger.info( + "Job %s for model %s started.", + job.job_id, + self.process_id_with_prefix, + ) + + return job + + except aiohttp.ClientResponseError as e: + logger.error("HTTP error during job submission: %s", e) + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Remote job submission failed", + status=502, + detail=f"Job could not be started remotely due to HTTP error: {e}", + instance=f"/processes/{self.process_id_with_prefix}/jobs", ) - status_response.raise_for_status() - status_json = await status_response.json() + ) from e - job.status = status_json.get("status") - job.save() + except Exception as e: + logger.exception("Unexpected error during job submission: \n%s", e) - logger.info( - " --> Job %s for model %s started running.", - job.job_id, - self.process_id_with_prefix, + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/server-error", + title="Unexpected error during job submission", + status=500, + detail=f"Job could not be started remotely due to unexpected error. Please see logs for more details.", + instance=f"/processes/{self.process_id_with_prefix}/execution", ) + ) from e + + async def _submit_remote_job( + self, session: aiohttp.ClientSession, + url: str, request_body: dict, auth: aiohttp.BasicAuth | None + ) -> aiohttp.ClientResponse: + response = await session.post( + f"{url}processes/{self.process_id}/execution", + json=request_body, + auth=auth, + headers={ + "Content-type": "application/json", + "Accept": "application/json", + "Prefer": "respond-async", + }, + ) + response.raise_for_status() + + if response.headers or response.content_type == "application/json": + return response + + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="No valid response from remote server.", + status=502, + detail=( + "No application/json response and no headers from " + "remote server received. Response is not valid." + ), + instance=f"/processes/{self.process_id_with_prefix}/jobs", + ) + ) - return job + async def _create_local_job_instance( + self, response: aiohttp.ClientResponse, name, request_body, user + ): + remote_job_id = await self._extract_remote_job_id(response) + + job = Job() + + job.insert( + remote_job_id=remote_job_id, + process_id_with_prefix=self.process_id_with_prefix, + process_title=self.title, + name=name, + exec_body=request_body, + user=user, + process_version=self.version, + ) + return job + + async def _fetch_remote_job_status( + self, session: aiohttp.ClientSession, url, remote_job_id, auth + ): + status_response = await session.get( + f"{url}jobs/{remote_job_id}?f=json", + auth=auth, + headers={ + "Content-type": "application/json", + "Accept": "application/json", + }, + ) + status_response.raise_for_status() + + return await status_response.json() + + async def _extract_remote_job_id(self, response: aiohttp.ClientResponse) -> str: + """ + Extracts the remote job ID from the aiohttp response object. + Tries the Location header first, then the response body for jobID/jobId. + """ + location_header = response.headers.get("location") + if location_header: + match = re.search(r"/jobs/([^/]+)", location_header) + if match: + return match.group(1) + + # Try to extract jobID from response body + try: + if response.content_type == "application/json": + body_json = await response.json() + job_id = body_json.get("jobID") or body_json.get("jobId") + if job_id: + return job_id except Exception as e: - raise CustomException(f"Job could not be started remotely: {e}") from e + logger.warning("Failed to parse jobID from response body: %s", e) + + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", + title="Invalid Location Header", + status=500, + detail="Could not extract remote job ID from Location header or response body.", + instance="/jobs", + ) + ) def _wait_for_results_async(self, job: Job): asyncio.run(self._wait_for_results(job)) @@ -443,7 +506,7 @@ async def _wait_for_results(self, job: Job): job.updated = datetime.now(timezone.utc).strftime( "%Y-%m-%dT%H:%M:%S.%fZ" ) - job.save() + job.update() if time.time() - start > timeout: raise TimeoutError( @@ -472,7 +535,7 @@ async def _wait_for_results(self, job: Job): job.updated = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") job.finished = job.updated job.progress = 100 - job.save() + job.update() raise CustomException( f"Could not retrieve results from simulation model server. {e}" ) from e @@ -489,7 +552,7 @@ async def _wait_for_results(self, job: Job): job.message = ( f"Remote execution was not successful! {job_details['message']}" ) - job.save() + job.update() raise CustomException(f"Remote job {job.remote_job_id}: {job.message}") except CustomException as e: @@ -499,7 +562,7 @@ async def _wait_for_results(self, job: Job): job.finished = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") job.updated = job.finished job.progress = 100 - job.save() + job.update() # Check if results should be stored in the geoserver try: @@ -517,7 +580,7 @@ async def _wait_for_results(self, job: Job): e, ) job.message = str(e) - job.save() + job.update() def is_finished(self, job_details): finished = False From 3cf1d622214f070d3dedb2f688c6e1d62bb509bb Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Fri, 4 Jul 2025 09:58:51 +0000 Subject: [PATCH 028/145] creating job and extracting job id fixed, ware working now --- src/ump/api/models/process.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 9b5f32e..a885463 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -323,13 +323,15 @@ async def start_process_execution(self, request_body, user): session, str(provider.server_url), request_body, auth ) + remote_job_id = await self._extract_remote_job_id(response) + job = await self._create_local_job_instance( - response, name, request_body, user + remote_job_id, name, request_body, user ) # this can probably be omitted here and deferred for _wait_for_status remote_job_status_info = await self._fetch_remote_job_status( - session, provider.server_url, response, auth + session, provider.server_url, remote_job_id, auth ) job.status = remote_job_status_info.get("status") @@ -403,9 +405,8 @@ async def _submit_remote_job( ) async def _create_local_job_instance( - self, response: aiohttp.ClientResponse, name, request_body, user + self, remote_job_id: str, name, request_body, user ): - remote_job_id = await self._extract_remote_job_id(response) job = Job() @@ -471,7 +472,7 @@ def _wait_for_results_async(self, job: Job): asyncio.run(self._wait_for_results(job)) async def _wait_for_results(self, job: Job): - logger.info(" --> Waiting for results in Thread") + logger.info("Thread started to wait for results.") finished = False From 8a81e9a8e52aa1747ec865bbf5e6155807e2b459 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 06:08:24 +0000 Subject: [PATCH 029/145] fixed some typing issues --- src/ump/api/providers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ump/api/providers.py b/src/ump/api/providers.py index eb20c16..cf688f4 100644 --- a/src/ump/api/providers.py +++ b/src/ump/api/providers.py @@ -19,7 +19,7 @@ logger = getLogger(__name__) -PROVIDERS: ModelServers = ModelServers() +PROVIDERS: ModelServers = {} PROVIDERS_LOCK = Lock() # Thread-safe lock for updating PROVIDERS @@ -55,7 +55,7 @@ def load_providers(self): observer = PollingObserver() observer.schedule( provider_loader, - config.UMP_PROVIDERS_FILE.absolute(), # Simplified path handling + config.UMP_PROVIDERS_FILE.absolute().as_posix(), # Simplified path handling recursive=False ) observer.start() @@ -70,7 +70,8 @@ def authenticate_provider(provider: ProviderConfig): auth = None if provider.authentication: auth = aiohttp.BasicAuth( - provider.authentication.user, provider.authentication.password + provider.authentication.user, + provider.authentication.password.get_secret_value() ) return auth From 9e774f57a33d4e543976af3cb3f0ca657c953c18 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 06:09:06 +0000 Subject: [PATCH 030/145] added type hint --- src/ump/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ump/utils.py b/src/ump/utils.py index a639ef2..0b7194d 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -9,7 +9,9 @@ logger = logging.getLogger(__name__) -async def fetch_json(session, url, **kwargs): +async def fetch_json( + session: aiohttp.ClientSession, url, **kwargs +) -> dict: try: async with session.get(url, **kwargs) as response: response.raise_for_status() From 90e0d63363403191e5bdea79b742d5235137c4c4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 06:11:20 +0000 Subject: [PATCH 031/145] fixed job status update --- src/ump/api/models/process.py | 75 +++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index a885463..74309be 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -424,8 +424,9 @@ async def _create_local_job_instance( async def _fetch_remote_job_status( self, session: aiohttp.ClientSession, url, remote_job_id, auth - ): - status_response = await session.get( + ) -> dict: + job_status = await fetch_json( + session, f"{url}jobs/{remote_job_id}?f=json", auth=auth, headers={ @@ -433,9 +434,8 @@ async def _fetch_remote_job_status( "Accept": "application/json", }, ) - status_response.raise_for_status() - return await status_response.json() + return job_status async def _extract_remote_job_id(self, response: aiohttp.ClientResponse) -> str: """ @@ -472,54 +472,59 @@ def _wait_for_results_async(self, job: Job): asyncio.run(self._wait_for_results(job)) async def _wait_for_results(self, job: Job): + # TODO overhaul the whole thing! exceptions job status gaining, results retrieval, etc. logger.info("Thread started to wait for results.") - finished = False + job_finished = False - provider: ProviderConfig = providers.get_providers()[self.provider_prefix] + provider_config: ProviderConfig = providers.get_providers()[self.provider_prefix] - timeout = float(provider.timeout) + timeout_seconds = provider_config.timeout start = time.time() job_details = {} try: - while not finished: - async with aiohttp.ClientSession(timeout=client_timeout) as session: - auth = providers.authenticate_provider(provider) - async with session.get( - f"{provider.server_url}jobs/{job.remote_job_id}?f=json", - auth=auth, - headers={ - "Content-type": "application/json", - "Accept": "application/json", - }, - ) as response: - response.raise_for_status() - job_details: dict = await response.json() - - finished = self.is_finished(job_details) - - logger.info(" --> Current Job status: %s", str(job_details)) - - # either remote job has progress info or else we cannot provide it either - job.progress = job_details.get("progress") - - job.updated = datetime.now(timezone.utc).strftime( - "%Y-%m-%dT%H:%M:%S.%fZ" + while not job_finished: + remote_job_status_info = await self._fetch_remote_job_status( + aiohttp.ClientSession(timeout=client_timeout), + provider_config.server_url, + job.remote_job_id, + providers.authenticate_provider(provider_config), ) + + job.started = remote_job_status_info.get("started") + job.created = remote_job_status_info.get("created") + job.updated = remote_job_status_info.get("updated") + job.finished = remote_job_status_info.get("finished") + job.message = remote_job_status_info.get("message", "") + job.progress = remote_job_status_info.get("progress") + + job_finished = self.is_finished(remote_job_status_info) + + logger.debug( + "Current Job status: %s", str(job_details) + ) + job.update() - if time.time() - start > timeout: - raise TimeoutError( - f"Job did not finish within {timeout / 60} minutes. Giving up." + if time.time() - start > timeout_seconds: + # TODO: dont raise an error, set results with details when reached! + job.status = JobStatus.failed.value + job.finished = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + job.updated = job.finished + job.progress = 100 + job.message = ( + f"Remote execution was not successful! {job_details['message']}" ) + job.update() + raise CustomException(f"Remote job {job.remote_job_id}: {job.message}") time.sleep(config.UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL) logger.info( - " --> Remote execution job %s: success = %s. Took approx. %s minutes.", + "Remote execution job %s: success = %s. Took approx. %s minutes.", job.remote_job_id, - finished, + job_finished, int((time.time() - start) / 60), ) From 7c9b9bfc5425980e333a80dfd69236a7b9de14c1 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 06:11:32 +0000 Subject: [PATCH 032/145] added type hint --- src/ump/api/models/providers_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/models/providers_config.py b/src/ump/api/models/providers_config.py index a70e49b..e47c874 100644 --- a/src/ump/api/models/providers_config.py +++ b/src/ump/api/models/providers_config.py @@ -133,7 +133,7 @@ class ProviderConfig(BaseModel): # a TypeAdapter allows us to use pydantics model_validate method # on arbitrary python types -model_servers_adapter = TypeAdapter(ModelServers) +model_servers_adapter: TypeAdapter[ModelServers] = TypeAdapter(ModelServers) if __name__ == "__main__": From db1f5062b8ef41f769b5028a376b8a303ba4c615 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 06:54:33 +0000 Subject: [PATCH 033/145] added note --- src/ump/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/utils.py b/src/ump/utils.py index 0b7194d..c329ced 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) - +# TODO: retry on timeouts, connection errors, etc. async def fetch_json( session: aiohttp.ClientSession, url, **kwargs ) -> dict: From 04caff72661ab471cf9debb96d5fac5313349da2 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 06:56:23 +0000 Subject: [PATCH 034/145] splitted _wait_for_results into smaller helpers , improved error handling --- src/ump/api/models/process.py | 157 +++++++++++++--------------------- 1 file changed, 60 insertions(+), 97 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 74309be..755f646 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -15,7 +15,7 @@ from ump.api.models.ogc_exception import OGCExceptionResponse from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.config import app_settings as config -from ump.errors import CustomException, InvalidUsage, OGCProcessException +from ump.errors import InvalidUsage, OGCProcessException from ump.utils import fetch_json logger = logging.getLogger(__name__) @@ -472,121 +472,84 @@ def _wait_for_results_async(self, job: Job): asyncio.run(self._wait_for_results(job)) async def _wait_for_results(self, job: Job): - # TODO overhaul the whole thing! exceptions job status gaining, results retrieval, etc. logger.info("Thread started to wait for results.") - - job_finished = False - provider_config: ProviderConfig = providers.get_providers()[self.provider_prefix] - timeout_seconds = provider_config.timeout - start = time.time() - job_details = {} try: - while not job_finished: - remote_job_status_info = await self._fetch_remote_job_status( - aiohttp.ClientSession(timeout=client_timeout), - provider_config.server_url, - job.remote_job_id, - providers.authenticate_provider(provider_config), + await asyncio.wait_for( + self._poll_job_until_finished(job, provider_config), + timeout=timeout_seconds + ) + except asyncio.TimeoutError: + self._set_job_failed( + job, + ( + "While waiting for remote job to finish the" + f" timeout ({provider_config.timeout} sec.) was reached." ) - - job.started = remote_job_status_info.get("started") - job.created = remote_job_status_info.get("created") - job.updated = remote_job_status_info.get("updated") - job.finished = remote_job_status_info.get("finished") - job.message = remote_job_status_info.get("message", "") - job.progress = remote_job_status_info.get("progress") - - job_finished = self.is_finished(remote_job_status_info) - - logger.debug( - "Current Job status: %s", str(job_details) + ) + except Exception as e: + logger.error("Error while waiting for job results: %s", e) + self._set_job_failed( + job, + ( + "An unexpected error occurred while waiting for job results." + "See the logs for details" ) - - job.update() - - if time.time() - start > timeout_seconds: - # TODO: dont raise an error, set results with details when reached! - job.status = JobStatus.failed.value - job.finished = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - job.updated = job.finished - job.progress = 100 - job.message = ( - f"Remote execution was not successful! {job_details['message']}" - ) - job.update() - raise CustomException(f"Remote job {job.remote_job_id}: {job.message}") - - time.sleep(config.UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL) - - logger.info( - "Remote execution job %s: success = %s. Took approx. %s minutes.", + ) + else: + self._set_job_successful(job) + await self._store_results_if_needed(job) + + async def _poll_job_until_finished(self, job, provider_config): + while True: + status_info = await self._fetch_remote_job_status( + aiohttp.ClientSession(timeout=client_timeout), + provider_config.server_url, job.remote_job_id, - job_finished, - int((time.time() - start) / 60), + providers.authenticate_provider(provider_config), ) + self._update_job_from_status(job, status_info) + if self.is_finished(status_info): + break + await asyncio.sleep(config.UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL) + + def _update_job_from_status(self, job: Job, status_info): + job.started = status_info.get("started") + job.created = status_info.get("created") + job.updated = status_info.get("updated") + job.finished = status_info.get("finished") + job.message = status_info.get("message", "") + job.progress = status_info.get("progress") + + # save to database + job.update() - except Exception as e: - logger.error( - "Could not retrieve results for job %s (=%s)/%s from remote server: %s", - self.process_id_with_prefix, - self.process_id, - job.job_id, - e, - ) - job.status = JobStatus.failed.value - job.message = str(e) - job.updated = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - job.finished = job.updated - job.progress = 100 - job.update() - raise CustomException( - f"Could not retrieve results from simulation model server. {e}" - ) from e - - # Check if job was successful - try: - if job_details["status"] != JobStatus.successful.value: - job.status = JobStatus.failed.value - job.finished = datetime.now(timezone.utc).strftime( - "%Y-%m-%dT%H:%M:%S.%fZ" - ) - job.updated = job.finished - job.progress = 100 - job.message = ( - f"Remote execution was not successful! {job_details['message']}" - ) - job.update() - raise CustomException(f"Remote job {job.remote_job_id}: {job.message}") + def _set_job_failed(self, job: Job, message: str): + job.status = JobStatus.failed.value + job.message = message - except CustomException as e: - logger.error(" --> An error occurred: %s", e) + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + job.finished = now + job.updated = now + job.progress = 0 + job.update() + def _set_job_successful(self, job: Job): job.status = JobStatus.successful.value - job.finished = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - job.updated = job.finished + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + job.finished = now + job.updated = now job.progress = 100 job.update() - # Check if results should be stored in the geoserver + async def _store_results_if_needed(self, job: Job): try: - if ( - providers.check_result_storage(self.provider_prefix, self.process_id) - == "geoserver" - ): + if providers.check_result_storage(self.provider_prefix, self.process_id) == "geoserver": await job.results_to_geoserver() except Exception as e: - logger.error( - " --> Could not store results for job %s (=%s)/%s to geoserver: %s", - self.process_id_with_prefix, - self.process_id, - job.job_id, - e, - ) - job.message = str(e) - job.update() + logger.error("Could not store results for job %s: %s", job.job_id, e) def is_finished(self, job_details): finished = False From 14c3821b7c1c190a66d306996ad5c7a6e517f438 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 07:14:49 +0000 Subject: [PATCH 035/145] fix: wrong config reference for workspace --- src/ump/geoserver/geoserver.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/ump/geoserver/geoserver.py b/src/ump/geoserver/geoserver.py index 6e91b36..64fc675 100644 --- a/src/ump/geoserver/geoserver.py +++ b/src/ump/geoserver/geoserver.py @@ -58,7 +58,7 @@ def save_results(self, job_id: str, data: dict): try: self.create_workspace() - logging.info(" --> Workspace should be created now") + logging.info(f"Workspace {self.workspace} created now") self.geojson_to_postgis(data=data, table_name=job_id) @@ -76,9 +76,15 @@ def save_results(self, job_id: str, data: dict): def publish_layer(self, store_name: str, layer_name: str): try: response = requests.post( - f"{config.UMP_GEOSERVER_PATH_WORKSPACE}/{self.workspace}" - + f"/datastores/{store_name}/featuretypes", - auth=(config.UMP_GEOSERVER_USER, config.UMP_GEOSERVER_PASSWORD.get_secret_value()), + ( + f"{config.UMP_GEOSERVER_URL_WORKSPACE}/{self.workspace}" + f"/datastores/{store_name}/featuretypes"), + + auth=( + config.UMP_GEOSERVER_USER, + config.UMP_GEOSERVER_PASSWORD.get_secret_value() + ), + data=f"{layer_name}", headers={"Content-type": "text/xml"}, timeout=config.UMP_GEOSERVER_CONNECTION_TIMEOUT, From cac9c03e7a69fb27d7c600f7a4bc08a1ac82bbf8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 07:15:36 +0000 Subject: [PATCH 036/145] raise exception with details page, fixed types --- src/ump/api/models/job.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index 231b8fa..66d0908 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -7,12 +7,13 @@ import aiohttp import geopandas as gpd +from ump.api.models.ogc_exception import OGCExceptionResponse import ump.api.providers as providers from ump.api.db_handler import DBHandler from ump.api.models.job_status import JobStatus from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.config import app_settings as config -from ump.errors import CustomException, InvalidUsage +from ump.errors import CustomException, InvalidUsage, OGCProcessException from ump.geoserver.geoserver import Geoserver # TODO class violates Single Responsibility Principle (SRP), it mixes @@ -65,7 +66,17 @@ def __init__(self, job_id=None, user=None): self.provider_url = None if job_id and not self._init_from_db(job_id, user): - raise CustomException("Job could not be found!") + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-job", + title="Job not found", + detail="The job with the given ID does not exist.", + status=404, + instance="/".join( + [config.UMP_API_SERVER_URL, "api", "jobs", job_id] + ) + ) + ) def insert( self, @@ -169,8 +180,8 @@ def _init_from_db(self, job_id, user): def _init_from_dict(self, data): self.job_id = data["job_id"] self.remote_job_id = data["remote_job_id"] - self.process_id = data["process_id"] - self.provider_prefix = data["provider_prefix"] + self.process_id: str = data["process_id"] + self.provider_prefix: str = data["provider_prefix"] self.provider_url = data["provider_url"] self.process_id_with_prefix = f"{data['provider_prefix']}:{data['process_id']}" self.status = data["status"] From a3d1734b77d1e4c954bf6d3950ec16fe9c37d06e Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 07:46:44 +0000 Subject: [PATCH 037/145] handle process not found error --- src/ump/api/models/process.py | 66 ++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 755f646..68a9f51 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -27,6 +27,7 @@ sock_read=5, # Socket read timeout ) + # TODO: this is not an OGC API Process in a strict sense, # instead it incorporates remote process execution logic # these two things should probably be separated with regard to @@ -47,6 +48,7 @@ def __init__(self, process_id_with_prefix): self.links = None match = re.search(r"([^:]+):(.*)", self.process_id_with_prefix) + if not match: logger.warning( "Process ID '%s' does not match pattern 'provider:process_id'. " @@ -66,19 +68,35 @@ def __init__(self, process_id_with_prefix): instance=f"/processes/{self.process_id_with_prefix}", ) ) + self.provider_prefix = match.group(1) self.process_id = match.group(2) - # this checks if the process is known from providers and confiured as available + # this checks if the process is known from providers and configured as available # for what purpose? -> security! -if a user selects a process which is not # confiured to be available, but exists - available, process_config = providers.check_process_availability( - self.provider_prefix, self.process_id - ) - if not available: + try: + process_config = providers.load_process_config( + self.provider_prefix, self.process_id + ) + except ValueError as e: + raise OGCProcessException( + OGCExceptionResponse( + type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", + title="Invalid Process ID", + status=400, + detail=( + f"Process ID '{self.process_id_with_prefix}' " + "does not match pattern: 'provider:process_id'}. " + "See /api/processes for a list of available processes." + ), + instance=f"/processes/{self.process_id_with_prefix}", + ) + ) + if process_config.exclude: logger.warning( "Process ID '%s' is not available. " - "Either the process is not configured or it is excluded from the API.", + "Either the process is not configured or it is excluded from this API.", self.process_id_with_prefix, ) # raise OGCProcessException to inform users, but with less detail @@ -93,6 +111,7 @@ def __init__(self, process_id_with_prefix): instance=f"/processes/{self.process_id_with_prefix}", ) ) + logger.debug( "Process %s is available. Loading process details.", self.process_id_with_prefix, @@ -336,15 +355,15 @@ async def start_process_execution(self, request_body, user): job.status = remote_job_status_info.get("status") job.message = remote_job_status_info.get("message", "") - + job.update() - + logger.info( "Job %s for model %s started.", job.job_id, self.process_id_with_prefix, ) - + return job except aiohttp.ClientResponseError as e: @@ -373,8 +392,11 @@ async def start_process_execution(self, request_body, user): ) from e async def _submit_remote_job( - self, session: aiohttp.ClientSession, - url: str, request_body: dict, auth: aiohttp.BasicAuth | None + self, + session: aiohttp.ClientSession, + url: str, + request_body: dict, + auth: aiohttp.BasicAuth | None, ) -> aiohttp.ClientResponse: response = await session.post( f"{url}processes/{self.process_id}/execution", @@ -407,7 +429,6 @@ async def _submit_remote_job( async def _create_local_job_instance( self, remote_job_id: str, name, request_body, user ): - job = Job() job.insert( @@ -423,7 +444,7 @@ async def _create_local_job_instance( return job async def _fetch_remote_job_status( - self, session: aiohttp.ClientSession, url, remote_job_id, auth + self, session: aiohttp.ClientSession, url, remote_job_id, auth ) -> dict: job_status = await fetch_json( session, @@ -434,7 +455,7 @@ async def _fetch_remote_job_status( "Accept": "application/json", }, ) - + return job_status async def _extract_remote_job_id(self, response: aiohttp.ClientResponse) -> str: @@ -473,21 +494,23 @@ def _wait_for_results_async(self, job: Job): async def _wait_for_results(self, job: Job): logger.info("Thread started to wait for results.") - provider_config: ProviderConfig = providers.get_providers()[self.provider_prefix] + provider_config: ProviderConfig = providers.get_providers()[ + self.provider_prefix + ] timeout_seconds = provider_config.timeout try: await asyncio.wait_for( self._poll_job_until_finished(job, provider_config), - timeout=timeout_seconds + timeout=timeout_seconds, ) except asyncio.TimeoutError: self._set_job_failed( job, ( "While waiting for remote job to finish the" - f" timeout ({provider_config.timeout} sec.) was reached." - ) + f" timeout ({provider_config.timeout} sec.) was reached." + ), ) except Exception as e: logger.error("Error while waiting for job results: %s", e) @@ -496,7 +519,7 @@ async def _wait_for_results(self, job: Job): ( "An unexpected error occurred while waiting for job results." "See the logs for details" - ) + ), ) else: self._set_job_successful(job) @@ -546,7 +569,10 @@ def _set_job_successful(self, job: Job): async def _store_results_if_needed(self, job: Job): try: - if providers.check_result_storage(self.provider_prefix, self.process_id) == "geoserver": + if ( + providers.check_result_storage(self.provider_prefix, self.process_id) + == "geoserver" + ): await job.results_to_geoserver() except Exception as e: logger.error("Could not store results for job %s: %s", job.job_id, e) From 63b12245d21ca7e3b76882fb47fc6232b5de1b67 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 07:47:51 +0000 Subject: [PATCH 038/145] check_process_availability -> load_process_config --- src/ump/api/providers.py | 41 ++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/ump/api/providers.py b/src/ump/api/providers.py index cf688f4..e204792 100644 --- a/src/ump/api/providers.py +++ b/src/ump/api/providers.py @@ -56,51 +56,56 @@ def load_providers(self): observer.schedule( provider_loader, config.UMP_PROVIDERS_FILE.absolute().as_posix(), # Simplified path handling - recursive=False + recursive=False, ) observer.start() # Graceful shutdown for observer atexit.register(observer.stop) + def get_providers() -> ModelServers: return PROVIDERS + def authenticate_provider(provider: ProviderConfig): auth = None if provider.authentication: auth = aiohttp.BasicAuth( provider.authentication.user, - provider.authentication.password.get_secret_value() + provider.authentication.password.get_secret_value(), ) return auth -def check_process_availability( - provider: str, process_id: str -) -> tuple[bool, ProcessConfig | None]: - available = False +def load_process_config( + provider: str, process_id: str +) -> ProcessConfig: with PROVIDERS_LOCK: # Ensure thread-safe access - if ( - provider in PROVIDERS and - process_id in PROVIDERS[provider].processes - ): - # load process configuration - process_config: ProcessConfig = PROVIDERS[provider].processes[process_id] - available = not process_config.exclude + try: + processes = PROVIDERS[provider].processes + # load process configuration + process_config: ProcessConfig = processes[process_id] + if process_config.exclude: logger.debug("Excluding process %s based on configuration", process_id) + except KeyError: + logger.error( + "Provider '%s' or process '%s' not found in PROVIDERS", + provider, + process_id, + ) + raise ValueError( + f"Provider '{provider}' or process '{process_id}' not found" + ) - return available, process_config + return process_config def check_result_storage(provider, process_id): with PROVIDERS_LOCK: # Ensure thread-safe access - if ( - provider in PROVIDERS - and process_id in PROVIDERS[provider].processes - ): + if provider in PROVIDERS and process_id in PROVIDERS[provider].processes: return PROVIDERS[provider].processes[process_id].result_storage return None From e3473e56fe1dab1171cb2e1f254e12231974b167 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 09:38:59 +0000 Subject: [PATCH 039/145] type added, some formatting issues resolved --- src/ump/main.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/ump/main.py b/src/ump/main.py index 726d9b4..f518be2 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -1,9 +1,9 @@ -#TODO: this file has become a hodgepodge of very different things, +# TODO: this file has become a hodgepodge of very different things, # it should be split into dedicated files: # - an app factory for migrations # - logging setup # - a geoserver cleanup runner -# - the flask app +# - the flask app import atexit import json import os @@ -17,7 +17,7 @@ from flask_cors import CORS from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy -from keycloak import KeycloakOpenID, KeycloakGetError, KeycloakConnectionError +from keycloak import KeycloakConnectionError, KeycloakGetError, KeycloakOpenID from werkzeug.exceptions import HTTPException from werkzeug.middleware.proxy_fix import ProxyFix @@ -46,10 +46,7 @@ "formatter": "default", } }, - "root": { - "level": config.UMP_LOG_LEVEL, - "handlers": ["wsgi"] - }, + "root": {"level": config.UMP_LOG_LEVEL, "handlers": ["wsgi"]}, "loggers": { "ump.api.providers": { # Configure the logger for providers.py "level": config.UMP_LOG_LEVEL, @@ -69,10 +66,10 @@ def cleanup(): """Cleans up jobs and Geoserver layers of anonymous users""" sql = "delete from jobs where user_id is null and finished < %(finished)s returning job_id, provider_prefix, process_id" - finished = datetime.now() - timedelta(minutes = config.UMP_JOB_DELETE_INTERVAL) - + finished = datetime.now() - timedelta(minutes=config.UMP_JOB_DELETE_INTERVAL) + with DBHandler() as conn: - result = conn.run_query(sql, query_params={'finished': finished}) + result = conn.run_query(sql, query_params={"finished": finished}) for row in result: # get additional job metadata job_id, provider_prefix, process_id = row @@ -90,20 +87,26 @@ def cleanup(): + f"/layers/{job_id}.xml", auth=( config.UMP_GEOSERVER_USER, - config.UMP_GEOSERVER_PASSWORD.get_secret_value + config.UMP_GEOSERVER_PASSWORD.get_secret_value, ), timeout=config.UMP_GEOSERVER_CONNECTION_TIMEOUT, ) requests.delete( f"{config.UMP_GEOSERVER_URL_WORKSPACE}/{config.UMP_GEOSERVER_WORKSPACE_NAME}" + f"/datastores/{job_id}/featuretypes/{job_id}.xml", - auth=(config.UMP_GEOSERVER_USER, config.UMP_GEOSERVER_PASSWORD.get_secret_value()), + auth=( + config.UMP_GEOSERVER_USER, + config.UMP_GEOSERVER_PASSWORD.get_secret_value(), + ), timeout=config.UMP_GEOSERVER_CONNECTION_TIMEOUT, ) requests.delete( f"{config.UMP_GEOSERVER_URL_WORKSPACE}/{config.UMP_GEOSERVER_WORKSPACE_NAME}" + f"/datastores/{job_id}.xml", - auth=(config.UMP_GEOSERVER_USER, config.UMP_GEOSERVER_PASSWORD.get_secret_value()), + auth=( + config.UMP_GEOSERVER_USER, + config.UMP_GEOSERVER_PASSWORD.get_secret_value(), + ), timeout=config.UMP_GEOSERVER_CONNECTION_TIMEOUT, ) @@ -148,7 +151,7 @@ def cleanup(): @app.errorhandler(OGCProcessException) -def handle_ogc_exception(error): +def handle_ogc_exception(error: OGCProcessException): response = jsonify(error.response.model_dump()) response.status_code = error.response.status response.content_type = "application/problem+json" @@ -191,6 +194,7 @@ def check_jwt(): g.auth_token = None pass + @app.after_request def set_headers(response): response.headers["Referrer-Policy"] = "no-referrer" @@ -226,5 +230,6 @@ def shutdown_pool_on_exit(): """Close the connection pool when the application shuts down.""" close_pool() + if __name__ == "__main__": app.run(host="0.0.0.0") From dba2c69c4adc830a9b9ae9a6dbee75a2f4fcd7b3 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 09:39:13 +0000 Subject: [PATCH 040/145] fetch_response_content added --- src/ump/utils.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/ump/utils.py b/src/ump/utils.py index c329ced..ff92db5 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -8,13 +8,35 @@ logger = logging.getLogger(__name__) +async def fetch_response_content( + response: aiohttp.ClientResponse +) -> tuple[str | dict, str, int]: + """ + Reads the content of an aiohttp response, handling both JSON and text, + regardless of HTTP status code. + Returns a tuple: (content, content_type, status_code) + """ + content_type = response.headers.get("Content-Type", "") + status = response.status + + try: + if "application/json" in content_type: + content = await response.json() + else: + content = await response.text() + except Exception: + # If JSON parsing fails, fallback to text + content = await response.text() + content_type = "text/plain" + + return content, content_type, status + # TODO: retry on timeouts, connection errors, etc. async def fetch_json( session: aiohttp.ClientSession, url, **kwargs ) -> dict: try: async with session.get(url, **kwargs) as response: - response.raise_for_status() try: return await response.json() except aiohttp.ContentTypeError: @@ -28,7 +50,10 @@ async def fetch_json( type="about:blank", title="Invalid Response Content", status=502, - detail="The response from the remote service was not valid JSON.", + detail=( + "The response from the remote service was not " + f"valid JSON: '{text[:100]}'" + ), instance=None ) ) From 66a90d787f464487769f52ebc621dd182f628020 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 09:40:23 +0000 Subject: [PATCH 041/145] handling remote server responses (Bad request and the like)to give users meaningful hints, try to prevent from leaking internal details like e.g. server urls --- src/ump/api/models/process.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 68a9f51..87dedfc 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -16,7 +16,7 @@ from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.config import app_settings as config from ump.errors import InvalidUsage, OGCProcessException -from ump.utils import fetch_json +from ump.utils import fetch_json, fetch_response_content logger = logging.getLogger(__name__) @@ -342,6 +342,10 @@ async def start_process_execution(self, request_body, user): session, str(provider.server_url), request_body, auth ) + response_content = await fetch_response_content(response) + + response.raise_for_status() # Raise an error for bad responses + remote_job_id = await self._extract_remote_job_id(response) job = await self._create_local_job_instance( @@ -368,12 +372,16 @@ async def start_process_execution(self, request_body, user): except aiohttp.ClientResponseError as e: logger.error("HTTP error during job submission: %s", e) + # response_body = await response.text() raise OGCProcessException( OGCExceptionResponse( type="about:blank", title="Remote job submission failed", - status=502, - detail=f"Job could not be started remotely due to HTTP error: {e}", + status=e.status, + detail=( + "Job could not be started remotely " + f"due to {response_content[0]}" + ), instance=f"/processes/{self.process_id_with_prefix}/jobs", ) ) from e @@ -408,7 +416,6 @@ async def _submit_remote_job( "Prefer": "respond-async", }, ) - response.raise_for_status() if response.headers or response.content_type == "application/json": return response From 65c70ae65bef65bc161b90c8f50fa1a17e3fb01b Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 09:50:20 +0000 Subject: [PATCH 042/145] rm unused import --- src/ump/api/models/process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 87dedfc..2060ec9 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -2,7 +2,6 @@ import json import logging import re -import time from datetime import datetime, timezone from multiprocessing import dummy From ed6f2cbae4bf9a45d92157444a1a56d878c4abd4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 12:07:47 +0000 Subject: [PATCH 043/145] added notes --- src/ump/api/models/job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index 66d0908..5bb7a1a 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -19,6 +19,8 @@ # TODO class violates Single Responsibility Principle (SRP), it mixes # business logic with data access logic and metadata handling # TODO methods like insert use redundant fields instead of job instance fields +# TODO queries use raw SQL, which is not recommended for different reasons: no migrations, reduced maintainability +# TODO: table schema is missing normalization class Job: DISPLAYED_ATTRIBUTES = [ "processID", From ec5c1e48cee3d4e5b364a6caf1b9336ef008ea14 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 12:40:12 +0000 Subject: [PATCH 044/145] fix: set status, dont set "successfully" manually, it MUST be fetrched from remote --- src/ump/api/models/process.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 2060ec9..fa7f9fb 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -528,7 +528,6 @@ async def _wait_for_results(self, job: Job): ), ) else: - self._set_job_successful(job) await self._store_results_if_needed(job) async def _poll_job_until_finished(self, job, provider_config): @@ -543,6 +542,8 @@ async def _poll_job_until_finished(self, job, provider_config): if self.is_finished(status_info): break await asyncio.sleep(config.UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL) + + return status_info def _update_job_from_status(self, job: Job, status_info): job.started = status_info.get("started") @@ -551,6 +552,7 @@ def _update_job_from_status(self, job: Job, status_info): job.finished = status_info.get("finished") job.message = status_info.get("message", "") job.progress = status_info.get("progress") + job.status = status_info.get("status", "") # save to database job.update() @@ -565,6 +567,8 @@ def _set_job_failed(self, job: Job, message: str): job.progress = 0 job.update() + # left here if we want to set a job as successful, manually + # this is not used in the current implementation, but could be useful def _set_job_successful(self, job: Job): job.status = JobStatus.successful.value now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") From aa0cc4a9dab8a829a02eeb6f4477d0ee421ac69d Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:28:42 +0000 Subject: [PATCH 045/145] reduce timeout to reasonable values (from default 300 seconds), use unified fetch mechanism --- src/ump/api/models/job.py | 40 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index 5bb7a1a..b24f95d 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -15,6 +15,15 @@ from ump.config import app_settings as config from ump.errors import CustomException, InvalidUsage, OGCProcessException from ump.geoserver.geoserver import Geoserver +from ump.utils import fetch_json + + +results_client_timeout = aiohttp.ClientTimeout( + total=5, # Set a reasonable timeout for the requests + connect=2, # Connection timeout + sock_connect=2, # Socket connection timeout + sock_read=5, # Socket read timeout +) # TODO class violates Single Responsibility Principle (SRP), it mixes # business logic with data access logic and metadata handling @@ -351,33 +360,34 @@ def display(self, additional_metadata=False): async def results(self): if self.status != JobStatus.successful.value: - return { - "error": f"No results available. Job status = {self.status}.", - "message": self.message, - } + raise OGCProcessException( + OGCExceptionResponse( + type="InvalidParameterValue", + title="No results available", + detail=self.message, + status=404, + instance=f"{config.UMP_API_SERVER_URL}/api/jobs/{self.job_id}/results", + ) + ) provider: ProviderConfig = providers.get_providers()[self.provider_prefix] self.provider_url = provider.server_url - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=results_client_timeout) as session: auth = providers.authenticate_provider(provider) - response = await session.get( - f"{self.provider_url}jobs/{self.remote_job_id}/results?f=json", - auth=auth, + results = await fetch_json( + session, + url=f"{self.provider_url}jobs/{self.remote_job_id}/results?f=json", + json=self.parameters, headers={ "Content-type": "application/json", "Accept": "application/json", }, + auth=auth ) - if response.status == 200: - return await response.json() - else: - raise CustomException( - "Could not retrieve results from model server " - + f"{self.provider_url} - {response.status}: {response.reason}" - ) + return results async def results_to_geoserver(self): try: From d30a45291c8e502d4a59d77aad35646cade219f6 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:29:21 +0000 Subject: [PATCH 046/145] using parameter names --- src/ump/api/models/process.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index fa7f9fb..d769b00 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -169,8 +169,8 @@ async def load_process_details(self): async with aiohttp.ClientSession(timeout=client_timeout) as session: process_details = await fetch_json( - session, - f"{provider_config.server_url}processes/{self.process_id}", + session=session, + url=f"{provider_config.server_url}processes/{self.process_id}", auth=auth, ) for key in process_details: @@ -405,6 +405,7 @@ async def _submit_remote_job( request_body: dict, auth: aiohttp.BasicAuth | None, ) -> aiohttp.ClientResponse: + response = await session.post( f"{url}processes/{self.process_id}/execution", json=request_body, @@ -453,8 +454,8 @@ async def _fetch_remote_job_status( self, session: aiohttp.ClientSession, url, remote_job_id, auth ) -> dict: job_status = await fetch_json( - session, - f"{url}jobs/{remote_job_id}?f=json", + session=session, + url=f"{url}jobs/{remote_job_id}?f=json", auth=auth, headers={ "Content-type": "application/json", From 5c9439497e227826c00c3858025a6c00007dc18f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:29:38 +0000 Subject: [PATCH 047/145] added optional raise_for_status --- src/ump/utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ump/utils.py b/src/ump/utils.py index ff92db5..89fa0b8 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -33,12 +33,19 @@ async def fetch_response_content( # TODO: retry on timeouts, connection errors, etc. async def fetch_json( - session: aiohttp.ClientSession, url, **kwargs + session: aiohttp.ClientSession, url, + raise_for_status=False, **kwargs ) -> dict: try: async with session.get(url, **kwargs) as response: try: - return await response.json() + response_data = await response.json() + + if raise_for_status: + response.raise_for_status() + + return response_data + except aiohttp.ContentTypeError: text = await response.text() logger.error( @@ -95,7 +102,7 @@ async def fetch_json( type="about:blank", title="Upstream HTTP Error", status=e.status, - detail="The remote service returned an HTTP error.", + detail=f"The remote service returned an HTTP error: {response_data}", instance=None ) ) From cb2a22b8aefe95b8a8df970807cb74a7038cd199 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:44:35 +0000 Subject: [PATCH 048/145] fixed some typing errors --- src/ump/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/ump/config.py b/src/ump/config.py index a7bc009..a422aec 100644 --- a/src/ump/config.py +++ b/src/ump/config.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from pydantic import FilePath, HttpUrl, SecretStr, computed_field, field_validator from pydantic_settings import BaseSettings @@ -9,7 +10,7 @@ # and do automatic type casting in a central place class UmpSettings(BaseSettings): UMP_LOG_LEVEL: str = "INFO" - UMP_PROVIDERS_FILE: FilePath = "providers.yaml" + UMP_PROVIDERS_FILE: FilePath = Path("providers.yaml") UMP_API_SERVER_URL: str = "localhost:3000" UMP_API_SERVER_WORKERS: int = 4 UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: int = 5 @@ -17,24 +18,24 @@ class UmpSettings(BaseSettings): UMP_DATABASE_HOST: str = "postgres" UMP_DATABASE_PORT: int = 5432 UMP_DATABASE_USER: str = "postgres" - UMP_DATABASE_PASSWORD: SecretStr = "postgres" + UMP_DATABASE_PASSWORD: SecretStr = SecretStr("postgres") UMP_GEOSERVER_URL: HttpUrl | None = HttpUrl("http://geoserver:8080/geoserver") UMP_GEOSERVER_DB_HOST: str = "postgis" UMP_GEOSERVER_DB_PORT: int = 5432 UMP_GEOSERVER_DB_NAME: str = "ump" UMP_GEOSERVER_DB_USER: str = "ump" - UMP_GEOSERVER_DB_PASSWORD: SecretStr = "ump" + UMP_GEOSERVER_DB_PASSWORD: SecretStr = SecretStr("ump") UMP_GEOSERVER_WORKSPACE_NAME: str = "UMP" UMP_GEOSERVER_USER: str = "geoserver" - UMP_GEOSERVER_PASSWORD: SecretStr = "geoserver" + UMP_GEOSERVER_PASSWORD: SecretStr = SecretStr("geoserver") UMP_GEOSERVER_CONNECTION_TIMEOUT: int = 60 # seconds UMP_JOB_DELETE_INTERVAL: int = 240 # minutes - UMP_KEYCLOAK_URL: HttpUrl = "http://keycloak:8080/auth" + UMP_KEYCLOAK_URL: HttpUrl = HttpUrl("http://keycloak:8080/auth") UMP_KEYCLOAK_REALM: str = "UrbanModelPlatform" UMP_KEYCLOAK_CLIENT_ID: str = "ump-client" UMP_KEYCLOAK_USER: str = "admin" - UMP_KEYCLOAK_PASSWORD: SecretStr = "admin" - UMP_URL_PREFIX: str = "/" + UMP_KEYCLOAK_PASSWORD: SecretStr = SecretStr("admin") + UMP_API_SERVER_URL_PREFIX: str = "/" @computed_field @property From f3c6e42475dec3335d30b403bc8e2b3de64cfb92 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:44:51 +0000 Subject: [PATCH 049/145] added new setting --- .env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.example b/.env.example index 5d14467..376d8da 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,7 @@ UMP_LOG_LEVEL=DEBUG UMP_PROVIDERS_FILE=providers.yaml UMP_API_SERVER_URL=localhost:5000 +UMP_API_SERVER_URL_PREFIX=/api UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL=5 UMP_DATABASE_NAME=ump UMP_DATABASE_HOST=localhost From ff0d9c41ed42e05dccc281f78bbcbb50e6e8f918 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:53:30 +0000 Subject: [PATCH 050/145] using correct prefix setting --- src/ump/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/main.py b/src/ump/main.py index f518be2..3ab83d0 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -132,7 +132,7 @@ def cleanup(): CORS(app) -api = APIBlueprint("api", __name__, url_prefix=config.UMP_URL_PREFIX) +api = APIBlueprint("api", __name__, url_prefix=config.UMP_API_SERVER_URL_PREFIX) api.register_blueprint(processes, url_prefix="/processes") api.register_blueprint(jobs, url_prefix="/jobs") api.register_blueprint(ensembles, url_prefix="/ensembles") From af09240e139c032d2447a078632a7905851b7b72 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:56:13 +0000 Subject: [PATCH 051/145] server url has schema now --- src/ump/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/config.py b/src/ump/config.py index a422aec..d1a5f64 100644 --- a/src/ump/config.py +++ b/src/ump/config.py @@ -11,7 +11,7 @@ class UmpSettings(BaseSettings): UMP_LOG_LEVEL: str = "INFO" UMP_PROVIDERS_FILE: FilePath = Path("providers.yaml") - UMP_API_SERVER_URL: str = "localhost:3000" + UMP_API_SERVER_URL: str = "http://localhost:3000" UMP_API_SERVER_WORKERS: int = 4 UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: int = 5 UMP_DATABASE_NAME: str = "ump" From 27232d53f34397a73df5a71066eefb4002306cff Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:56:53 +0000 Subject: [PATCH 052/145] full links --- src/ump/api/jobs.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ump/api/jobs.py b/src/ump/api/jobs.py index 2bd6a7c..65ca8c0 100644 --- a/src/ump/api/jobs.py +++ b/src/ump/api/jobs.py @@ -8,6 +8,7 @@ from ump.api.models.ensemble import Ensemble, JobsEnsembles from ump.api.models.job import Job from ump.api.models.job_status import JobStatus +from ump.config import app_settings def append_ensemble_list(job): with Session(engine) as session: @@ -91,7 +92,10 @@ def next_links(page, limit, count_jobs): links = [] if count_jobs > (page - 1) * limit: links.append({ - "href": f"/api/jobs?page={page+1}&limit={limit}", + "href": ( + f"{app_settings.UMP_API_SERVER_URL}/{app_settings.UMP_API_SERVER_URL_PREFIX}/" + f"jobs?page={page+1}&limit={limit}" + ), "rel": "service", "type": "application/json", "hreflang": "en", @@ -100,7 +104,10 @@ def next_links(page, limit, count_jobs): if page > 1: links.append({ - "href": f"/api/jobs?page={page-1}&limit={limit}", + "href": ( + f"{app_settings.UMP_API_SERVER_URL}/{app_settings.UMP_API_SERVER_URL_PREFIX}" + f"/jobs?page={page-1}&limit={limit}" + ), "rel": "service", "type": "application/json", "hreflang": "en", From d8d27fba1efeae869eb323fe6b09834890c44d4c Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 13:57:12 +0000 Subject: [PATCH 053/145] using url and prefix --- src/ump/api/models/job.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index b24f95d..a009582 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -84,7 +84,12 @@ def __init__(self, job_id=None, user=None): detail="The job with the given ID does not exist.", status=404, instance="/".join( - [config.UMP_API_SERVER_URL, "api", "jobs", job_id] + [ + config.UMP_API_SERVER_URL, + f"{config.UMP_API_SERVER_URL_PREFIX}", + "jobs", + job_id + ] ) ) ) @@ -310,7 +315,7 @@ def display(self, additional_metadata=False): JobStatus.accepted.value, ): job_result_url = ( - f"{config.UMP_API_SERVER_URL}/api/jobs/{self.job_id}/results" + f"{config.UMP_API_SERVER_URL}/{config.UMP_API_SERVER_URL_PREFIX}/jobs/{self.job_id}/results" ) job_dict["links"] = [ @@ -366,7 +371,7 @@ async def results(self): title="No results available", detail=self.message, status=404, - instance=f"{config.UMP_API_SERVER_URL}/api/jobs/{self.job_id}/results", + instance=f"{config.UMP_API_SERVER_URL}/{config.UMP_API_SERVER_URL_PREFIX}/jobs/{self.job_id}/results", ) ) From 78279a14401ae375d71b9684fc1647105add9cdf Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:13:25 +0000 Subject: [PATCH 054/145] added clean url joining function --- src/ump/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ump/utils.py b/src/ump/utils.py index 89fa0b8..8bfc22e 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -6,8 +6,12 @@ from ump.api.models.ogc_exception import OGCExceptionResponse from ump.errors import OGCProcessException + logger = logging.getLogger(__name__) +def join_url_parts(*parts): + return '/'.join(str(part).strip('/') for part in parts if part != '') + async def fetch_response_content( response: aiohttp.ClientResponse ) -> tuple[str | dict, str, int]: From 51970ed734ea6c8035df2041db493b3ed5e10769 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:13:39 +0000 Subject: [PATCH 055/145] creating clean urls --- src/ump/api/jobs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ump/api/jobs.py b/src/ump/api/jobs.py index 65ca8c0..9491248 100644 --- a/src/ump/api/jobs.py +++ b/src/ump/api/jobs.py @@ -3,6 +3,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from ump import utils from ump.api.db_handler import DBHandler from ump.api.db_handler import db_engine as engine from ump.api.models.ensemble import Ensemble, JobsEnsembles @@ -92,8 +93,9 @@ def next_links(page, limit, count_jobs): links = [] if count_jobs > (page - 1) * limit: links.append({ - "href": ( - f"{app_settings.UMP_API_SERVER_URL}/{app_settings.UMP_API_SERVER_URL_PREFIX}/" + "href": utils.join_url_parts( + app_settings.UMP_API_SERVER_URL, + app_settings.UMP_API_SERVER_URL_PREFIX, f"jobs?page={page+1}&limit={limit}" ), "rel": "service", @@ -104,9 +106,10 @@ def next_links(page, limit, count_jobs): if page > 1: links.append({ - "href": ( - f"{app_settings.UMP_API_SERVER_URL}/{app_settings.UMP_API_SERVER_URL_PREFIX}" - f"/jobs?page={page-1}&limit={limit}" + "href": utils.join_url_parts( + app_settings.UMP_API_SERVER_URL, + app_settings.UMP_API_SERVER_URL_PREFIX, + f"jobs?page={page-1}&limit={limit}" ), "rel": "service", "type": "application/json", From a9286cdf6ecbeab04f3218cd7a680e4c228add42 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:14:04 +0000 Subject: [PATCH 056/145] fixed results link --- src/ump/api/models/job.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index a009582..0b7a98d 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -321,11 +321,10 @@ def display(self, additional_metadata=False): job_dict["links"] = [ { "href": job_result_url, - "rel": "service", + "rel": "http://www.opengis.net/def/rel/ogc/1.0/results", "type": "application/json", "hreflang": "en", - "title": f"Results of job {self.job_id} as geojson" - + " - available when job is finished.", + "title": "Job result", } ] if isinstance(additional_metadata, str): From b131e211b988d2dead470c04d6a2caff5e5fde08 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:24:37 +0000 Subject: [PATCH 057/145] fix: links fully working --- src/ump/api/models/job.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index 0b7a98d..76d86b3 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -13,9 +13,9 @@ from ump.api.models.job_status import JobStatus from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.config import app_settings as config -from ump.errors import CustomException, InvalidUsage, OGCProcessException +from ump.errors import InvalidUsage, OGCProcessException from ump.geoserver.geoserver import Geoserver -from ump.utils import fetch_json +from ump.utils import fetch_json, join_url_parts results_client_timeout = aiohttp.ClientTimeout( @@ -314,8 +314,11 @@ def display(self, additional_metadata=False): JobStatus.running.value, JobStatus.accepted.value, ): - job_result_url = ( - f"{config.UMP_API_SERVER_URL}/{config.UMP_API_SERVER_URL_PREFIX}/jobs/{self.job_id}/results" + job_result_url = join_url_parts( + config.UMP_API_SERVER_URL, + config.UMP_API_SERVER_URL_PREFIX, + "jobs", + f"{self.job_id}/results" ) job_dict["links"] = [ From c8f8dafa16f35c70834bd59b54dd01a622cadc70 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:27:03 +0000 Subject: [PATCH 058/145] fix: missing jab.started datetime added --- src/ump/api/models/process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index d769b00..281633f 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -356,8 +356,9 @@ async def start_process_execution(self, request_body, user): session, provider.server_url, remote_job_id, auth ) - job.status = remote_job_status_info.get("status") - job.message = remote_job_status_info.get("message", "") + self._update_job_from_status( + job, remote_job_status_info + ) job.update() From 17331e633bb81288bc6e5823b6f33e8de06aa6a8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:38:32 +0000 Subject: [PATCH 059/145] added tag --- Dockerfile | 6 +++++- Makefile | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7631706..6182f0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,8 +42,12 @@ ARG USER_UID=1000 ARG USERNAME=pythonuser ARG USER_GID=2000 ARG SOURCE_COMMIT +ARG TAG=2.0.0 -LABEL maintainer="Urban Data Analytics" name="analytics/urban-model-platform" source_commit=$SOURCE_COMMIT +LABEL maintainer="Urban Data Analytics" \ + name="analytics/urban-model-platform" \ + source_commit=$SOURCE_COMMIT \ + version=${TAG} # add user and group RUN groupadd --gid $USER_GID $USERNAME && \ diff --git a/Makefile b/Makefile index 1c73dd2..9ea5e35 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,10 @@ initiate-dev: build-image: @echo 'Building release ${CONTAINER_REGISTRY}/${CONTAINER_NAMESPACE}/$(IMAGE_NAME):$(IMAGE_TAG)' # build your image - docker compose -f docker-compose-build.yaml build --build-arg SOURCE_COMMIT=$(GIT_COMMIT) api + docker compose -f docker-compose-build.yaml build \ + --build-arg SOURCE_COMMIT=$(GIT_COMMIT) \ + --build-arg TAG=$(IMAGE_TAG) \ + api upload-image: build-image docker compose -f docker-compose-build.yaml push api From 1c0aacd1450407fae6b308a583b4ed418f98813a Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 14:39:11 +0000 Subject: [PATCH 060/145] updated version: 2.0.0 -> 2.1.0.rc1+fix-error-handling --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 03eeef3..047e670 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "1.2.0" +version = "2.1.0rc1+fix-error-handling" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From 89a962e874d528da63ab0c2c56d45154bc3c12ee Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 15:03:44 +0000 Subject: [PATCH 061/145] updated entrypoint script --- scripts/entrypoint.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index f90290f..e092f30 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,8 +1,10 @@ #!/bin/bash set -e +flask db upgrade + echo "Running API Server in production mode." -NUMBER_OF_WORKERS="${NUMBER_OF_WORKERS:-1}" -echo "Running gunicorn with ${NUMBER_OF_WORKERS} workers." +UMP_API_SERVER_WORKERS="${UMP_API_SERVER_WORKERS:-1}" +echo "Running gunicorn with ${UMP_API_SERVER_WORKERS} workers." # export PATH=$PATH:/home/python/.local/bin -exec gunicorn --workers=$NUMBER_OF_WORKERS --bind=0.0.0.0:5000 ump.main:app +exec gunicorn --workers=$UMP_API_SERVER_WORKERS --bind=0.0.0.0:5000 ump.main:app From 93b2786ebdeb7f8a91c0be6aab578385786c28a4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 7 Jul 2025 15:05:22 +0000 Subject: [PATCH 062/145] bump version: 2.1.0rc2+fix-error-handling" --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 047e670..fa0bb26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc1+fix-error-handling" +version = "2.1.0rc2+fix-error-handling" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From cee5d372ec8d21e87bb2c94a43322ee8ca891427 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 06:22:11 +0000 Subject: [PATCH 063/145] added scratch --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 134e617..051860b 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,4 @@ cython_debug/ geoserver_data postgresql_data +scratch/* \ No newline at end of file From 82b5da81dc729a440b491b805eb08e2bbd37c4a9 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 06:36:10 +0000 Subject: [PATCH 064/145] added bump-my-version for automatic version handling --- .bumpmyversion.chart.toml | 10 +++++ .bumpmyversion.toml | 25 +++++++++++ poetry.lock | 88 ++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 .bumpmyversion.chart.toml create mode 100644 .bumpmyversion.toml diff --git a/.bumpmyversion.chart.toml b/.bumpmyversion.chart.toml new file mode 100644 index 0000000..4610990 --- /dev/null +++ b/.bumpmyversion.chart.toml @@ -0,0 +1,10 @@ +[bumpversion] +current_version = "1.2.0" +commit = true +tag = true +tag_name = "chart-v{new_version}" + +[[bumpversion.files]] +filename = "charts/urban-model-platform/Chart.yaml" +search = 'version: {current_version}' +replace = 'version: {new_version}' diff --git a/.bumpmyversion.toml b/.bumpmyversion.toml new file mode 100644 index 0000000..6f88f06 --- /dev/null +++ b/.bumpmyversion.toml @@ -0,0 +1,25 @@ +[bumpversion] +current_version = "2.1.0rc1+fix-error-handling" +commit = true +tag = true +tag_name = "v{new_version}" + +[[bumpversion.files]] +filename = "pyproject.toml" +search = 'version = "{current_version}"' +replace = 'version = "{new_version}"' + +[[bumpversion.files]] +filename = "charts/urban-model-platform/values.yaml" +search = 'tag: "{current_version}"' +replace = 'tag: "{new_version}"' + +[[bumpversion.files]] +filename = ".env" +search = 'IMAGE_TAG={current_version}' +replace = 'IMAGE_TAG={new_version}' + +[[bumpversion.files]] +filename = "Dockerfile" +search = 'ARG IMAGE_TAG={current_version}' +replace = 'ARG IMAGE_TAG={new_version}' \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index db47d24..b08311d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -424,6 +424,39 @@ files = [ {file = "blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf"}, ] +[[package]] +name = "bracex" +version = "2.6" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.9" +files = [ + {file = "bracex-2.6-py3-none-any.whl", hash = "sha256:0b0049264e7340b3ec782b5cb99beb325f36c3782a32e36e876452fd49a09952"}, + {file = "bracex-2.6.tar.gz", hash = "sha256:98f1347cd77e22ee8d967a30ad4e310b233f7754dbf31ff3fceb76145ba47dc7"}, +] + +[[package]] +name = "bump-my-version" +version = "1.2.0" +description = "Version bump your Python project" +optional = false +python-versions = ">=3.8" +files = [ + {file = "bump_my_version-1.2.0-py3-none-any.whl", hash = "sha256:201e6b103ff0f2b240c9d0a6eb83db382840b1f78eb78f6d77726bed39a326d8"}, + {file = "bump_my_version-1.2.0.tar.gz", hash = "sha256:5120d798aaf26468a37ca0f127992dc036688b8e5e106adc8870b13c2a2df22d"}, +] + +[package.dependencies] +click = "*" +httpx = ">=0.28.1" +pydantic = ">=2.0.0" +pydantic-settings = "*" +questionary = "*" +rich = "*" +rich-click = "*" +tomlkit = "*" +wcmatch = ">=8.5.1" + [[package]] name = "certifi" version = "2025.1.31" @@ -4044,6 +4077,20 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} +[[package]] +name = "questionary" +version = "2.1.0" +description = "Python library to build pretty command line user prompts ⭐️" +optional = false +python-versions = ">=3.8" +files = [ + {file = "questionary-2.1.0-py3-none-any.whl", hash = "sha256:44174d237b68bc828e4878c763a9ad6790ee61990e0ae72927694ead57bab8ec"}, + {file = "questionary-2.1.0.tar.gz", hash = "sha256:6302cdd645b19667d8f6e6634774e9538bfcd1aad9be287e743d96cacaf95587"}, +] + +[package.dependencies] +prompt_toolkit = ">=2.0,<4.0" + [[package]] name = "referencing" version = "0.36.2" @@ -4113,6 +4160,26 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "rich-click" +version = "1.8.9" +description = "Format click help output nicely with rich" +optional = false +python-versions = ">=3.7" +files = [ + {file = "rich_click-1.8.9-py3-none-any.whl", hash = "sha256:c3fa81ed8a671a10de65a9e20abf642cfdac6fdb882db1ef465ee33919fbcfe2"}, + {file = "rich_click-1.8.9.tar.gz", hash = "sha256:fd98c0ab9ddc1cf9c0b7463f68daf28b4d0033a74214ceb02f761b3ff2af3136"}, +] + +[package.dependencies] +click = ">=7" +rich = ">=10.7" +typing_extensions = ">=4" + +[package.extras] +dev = ["mypy", "packaging", "pre-commit", "pytest", "pytest-cov", "rich-codex", "ruff", "types-setuptools"] +docs = ["markdown_include", "mkdocs", "mkdocs-glightbox", "mkdocs-material-extensions", "mkdocs-material[imaging] (>=9.5.18,<9.6.0)", "mkdocs-rss-plugin", "mkdocstrings[python]", "rich-codex"] + [[package]] name = "rpds-py" version = "0.23.1" @@ -4256,6 +4323,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a52d48f4e7bf9005e8f0a89209bf9a73f7190ddf0489eee5eb51377385f59f2a"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"}, @@ -4264,6 +4332,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1492a6051dab8d912fc2adeef0e8c72216b24d57bd896ea607cb90bb0c4981d3"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"}, @@ -4272,6 +4341,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b82a7c94a498853aa0b272fd5bc67f29008da798d4f93a2f9f289feb8426a58d"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"}, @@ -4280,6 +4350,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f6f3eac23941b32afccc23081e1f50612bdbe4e982012ef4f5797986828cd01"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"}, @@ -4288,6 +4359,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2c59aa6170b990d8d2719323e628aaf36f3bfbc1c26279c0eeeb24d05d2d11c7"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"}, {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, @@ -5375,6 +5447,20 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "wcmatch" +version = "10.1" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.9" +files = [ + {file = "wcmatch-10.1-py3-none-any.whl", hash = "sha256:5848ace7dbb0476e5e55ab63c6bbd529745089343427caa5537f230cc01beb8a"}, + {file = "wcmatch-10.1.tar.gz", hash = "sha256:f11f94208c8c8484a16f4f48638a85d771d9513f4ab3f37595978801cb9465af"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + [[package]] name = "wcwidth" version = "0.2.13" @@ -5556,4 +5642,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "5b241bf1cb5e085aa28bbe7139e098d802800b08b6ed006f72b9a3e9f3edbe63" +content-hash = "75593a48ecc678f200b1c3203a5cd99e32787d7c65f30258e6cc01545f466093" diff --git a/pyproject.toml b/pyproject.toml index 03eeef3..ef179fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ pre-commit = ">=3.4.0,<4" debugpy = "^1.8.5" flake8 = "^7.1.1" pylint = "^3.3.1" +bump-my-version = "^1.2.0" [tool.poetry.group.docs] optional = true From 0c3e545971a440923db30f13ad002d61beb500ff Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 06:53:33 +0000 Subject: [PATCH 065/145] removed values.yaml and Dockerfile --- .bumpmyversion.toml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.bumpmyversion.toml b/.bumpmyversion.toml index 6f88f06..24917bb 100644 --- a/.bumpmyversion.toml +++ b/.bumpmyversion.toml @@ -9,17 +9,7 @@ filename = "pyproject.toml" search = 'version = "{current_version}"' replace = 'version = "{new_version}"' -[[bumpversion.files]] -filename = "charts/urban-model-platform/values.yaml" -search = 'tag: "{current_version}"' -replace = 'tag: "{new_version}"' - [[bumpversion.files]] filename = ".env" search = 'IMAGE_TAG={current_version}' replace = 'IMAGE_TAG={new_version}' - -[[bumpversion.files]] -filename = "Dockerfile" -search = 'ARG IMAGE_TAG={current_version}' -replace = 'ARG IMAGE_TAG={new_version}' \ No newline at end of file From 6d60c1898844111f30c20074e43a56ddf0d9c0c2 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 06:53:49 +0000 Subject: [PATCH 066/145] using IMAGE_TAG from .env --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6182f0c..db91fdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,12 +42,12 @@ ARG USER_UID=1000 ARG USERNAME=pythonuser ARG USER_GID=2000 ARG SOURCE_COMMIT -ARG TAG=2.0.0 +ARG IMAGE_TAG=2.0.0 LABEL maintainer="Urban Data Analytics" \ name="analytics/urban-model-platform" \ source_commit=$SOURCE_COMMIT \ - version=${TAG} + version=${IMAGE_TAG} # add user and group RUN groupadd --gid $USER_GID $USERNAME && \ From 7797fd0a521ddd0eeb4160643c7f9df69cc8d410 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 06:53:59 +0000 Subject: [PATCH 067/145] added version bump commands --- Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Makefile b/Makefile index 9ea5e35..703e415 100644 --- a/Makefile +++ b/Makefile @@ -98,3 +98,27 @@ build-docs: clean-docs: jupyter-book clean docs + +# Update app version: bump major, minor, or patch +bump-app-version: + @if [ -z "$(part)" ]; then \ + echo "Usage: make bump-app part={major|minor|patch}"; \ + exit 1; \ + fi; \ + bump-my-version bump $(part) + +# Update app version: set to a specific version +set-app-version: + @if [ -z "$(version)" ]; then \ + echo "Usage: make set-app-version version={version}"; \ + exit 1; \ + fi; \ + bump-my-version set $(version) + +# Update chart version: bump major, minor, or patch +bump-chart-version: + @if [ -z "$(part)" ]; then \ + echo "Usage: make bump-chart part={major|minor|patch}"; \ + exit 1; \ + fi; \ + bump-my-version --config .bumpmyversion.chart.toml bump $(part) \ No newline at end of file From c523087411a673fe847d17834375173988c3d9bf Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:03:16 +0000 Subject: [PATCH 068/145] renamed --- .../templates/configmap-job-settings.yaml | 33 ------------------- .../templates/configmap-settings-job.yaml | 18 ++++++++++ 2 files changed, 18 insertions(+), 33 deletions(-) delete mode 100644 charts/urban-model-platform/templates/configmap-job-settings.yaml create mode 100644 charts/urban-model-platform/templates/configmap-settings-job.yaml diff --git a/charts/urban-model-platform/templates/configmap-job-settings.yaml b/charts/urban-model-platform/templates/configmap-job-settings.yaml deleted file mode 100644 index c2d57b6..0000000 --- a/charts/urban-model-platform/templates/configmap-job-settings.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "ump.fullname" . }}-job-settings - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-6" # Lower weight than job (-5) ensures it runs first - "helm.sh/hook-delete-policy": hook-succeeded - labels: - {{- include "ump.labels" . | nindent 4 }} -data: - LOG_LEVEL: {{ .Values.config.logLevel | quote }} - PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} - API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} - REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} - DATABASE_NAME: {{ .Values.config.databaseName | quote }} - DATABASE_HOST: {{ .Values.config.databaseHost | quote }} - DATABASE_PORT: {{ .Values.config.databasePort | quote }} - DATABASE_USER: {{ .Values.config.databaseUser | quote }} - DATABASE_PASSWORD: {{ .Values.config.databasePassword | quote }} - GEOSERVER_URL: {{ .Values.config.geoserverUrl | quote }} - GEOSERVER_DB_HOST: {{ .Values.config.geoserverDbHost | quote }} - GEOSERVER_DB_PORT: {{ .Values.config.geoserverDbPort | quote }} - GEOSERVER_WORKSPACE_NAME: {{ .Values.config.geoserverWorkspaceName | quote }} - GEOSERVER_USER: {{ .Values.config.geoserverUser | quote }} - GEOSERVER_PASSWORD: {{ .Values.config.geoserverPassword | quote }} - GEOSERVER_CONNECTION_TIMEOUT: {{ .Values.config.geoserverConnectionTimeout | quote }} - JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} - KEYCLOAK_URL: {{ .Values.config.keycloakUrl | quote }} - KEYCLOAK_REALM: {{ .Values.config.keycloakRealm | quote }} - KEYCLOAK_CLIENT_ID: {{ .Values.config.keycloakClientId | quote }} - KEYCLOAK_USER: {{ .Values.config.keycloakUser | quote }} - KEYCLOAK_PASSWORD: {{ .Values.config.keycloakPassword | quote }} diff --git a/charts/urban-model-platform/templates/configmap-settings-job.yaml b/charts/urban-model-platform/templates/configmap-settings-job.yaml new file mode 100644 index 0000000..1f13d41 --- /dev/null +++ b/charts/urban-model-platform/templates/configmap-settings-job.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ump.fullname" . }}-job-settings + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-6" # Lower weight than job (-5) ensures it runs first + "helm.sh/hook-delete-policy": hook-succeeded + labels: + {{- include "ump.labels" . | nindent 4 }} +data: + UMP_LOG_LEVEL: {{ .Values.config.logLevel | quote }} + UMP_PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} + UMP_API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} + UMP_API_SERVER_URL_PREFIX: {{ .Values.config.apiServerUrlPrefix | quote }} + UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} + UMP_JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} + UMP_API_SERVER_WORKERS: {{ .Values.config.apiServerWorkers | quote }} From 1a4fcb5b2bba5261647935f88d3afe91afb4577f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:03:31 +0000 Subject: [PATCH 069/145] moved to charts/ --- .bumpmyversion.chart.toml => charts/.bumpmyversion.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .bumpmyversion.chart.toml => charts/.bumpmyversion.toml (100%) diff --git a/.bumpmyversion.chart.toml b/charts/.bumpmyversion.toml similarity index 100% rename from .bumpmyversion.chart.toml rename to charts/.bumpmyversion.toml From 6bd9db251d5f64f88a50b2eaa76e5e6106912dc8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:03:48 +0000 Subject: [PATCH 070/145] updated charts version bump cmd --- Makefile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 703e415..d7676db 100644 --- a/Makefile +++ b/Makefile @@ -101,24 +101,25 @@ clean-docs: # Update app version: bump major, minor, or patch bump-app-version: - @if [ -z "$(part)" ]; then \ - echo "Usage: make bump-app part={major|minor|patch}"; \ - exit 1; \ - fi; \ - bump-my-version bump $(part) + @if [ -z "$(part)" ]; then \ + echo "Usage: make bump-app part={major|minor|patch}"; \ + exit 1; \ + fi; \ + bump-my-version bump $(part) # Update app version: set to a specific version set-app-version: - @if [ -z "$(version)" ]; then \ - echo "Usage: make set-app-version version={version}"; \ - exit 1; \ - fi; \ - bump-my-version set $(version) + @if [ -z "$(version)" ]; then \ + echo "Usage: make set-app-version version={version}"; \ + exit 1; \ + fi; \ + bump-my-version set $(version) # Update chart version: bump major, minor, or patch bump-chart-version: - @if [ -z "$(part)" ]; then \ - echo "Usage: make bump-chart part={major|minor|patch}"; \ - exit 1; \ - fi; \ - bump-my-version --config .bumpmyversion.chart.toml bump $(part) \ No newline at end of file + cd charts + @if [ -z "$(part)" ]; then \ + echo "Usage: make bump-chart part={major|minor|patch}"; \ + exit 1; \ + fi; \ + bump-my-version bump $(part) \ No newline at end of file From 50d9e9388e002e85a768d38f5433232cb79ff968 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:04:08 +0000 Subject: [PATCH 071/145] updated tag --- charts/urban-model-platform/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index e675ee5..20f1cac 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -5,7 +5,7 @@ fullnameOverride: "" image: repository: lgvanalytics.azurecr.io/urban-model-platform pullPolicy: IfNotPresent - tag: "1.2.0" + tag: "2.0.0" pullSecrets: - name: secret @@ -42,7 +42,9 @@ service: config: logLevel: "DEBUG" providersFile: "./providers.yaml" - apiServerUrl: "0.0.0.0:5000" + apiServerUrl: "http://localhost:5000" + apiServerUrlPrefix: "/api" + apiServerWorkers: "4" remoteJobStatusRequestInterval: "5" geoserverConnectionTimeout: "60" jobDeleteInterval: "240" From 86743bb1c10522f2117eeb085fe44c16ef0f8e02 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:04:26 +0000 Subject: [PATCH 072/145] settings updated --- .../templates/configmap-settings.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/charts/urban-model-platform/templates/configmap-settings.yaml b/charts/urban-model-platform/templates/configmap-settings.yaml index 8fff047..274c864 100644 --- a/charts/urban-model-platform/templates/configmap-settings.yaml +++ b/charts/urban-model-platform/templates/configmap-settings.yaml @@ -5,8 +5,10 @@ metadata: labels: {{- include "ump.labels" . | nindent 4 }} data: - LOG_LEVEL: {{ .Values.config.logLevel | quote }} - PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} - API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} - REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} - JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} + UMP_LOG_LEVEL: {{ .Values.config.logLevel | quote }} + UMP_PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} + UMP_API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} + UMP_API_SERVER_URL_PREFIX: {{ .Values.config.apiServerUrlPrefix | quote }} + UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} + UMP_JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} + UMP_API_SERVER_WORKERS: {{ .Values.config.apiServerWorkers | quote }} \ No newline at end of file From b78850f1a45e28edfd65802cf9a9535eae9382ed Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:12:05 +0000 Subject: [PATCH 073/145] env var names updated --- .../templates/deployment.yaml | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index cd66484..66b8f04 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -53,6 +53,7 @@ spec: envFrom: - configMapRef: name: {{ include "ump.fullname" . }}-settings + # if keycloak/geoserver secrets are not referenced, they will be created {{- if not .Values.keycloakConnection.existingSecret.name }} - secretRef: name: {{ include "ump.fullname" . }}-keycloak-connection @@ -64,27 +65,27 @@ spec: env: # postgres connection {{- if .Values.postgresConnection.existingSecret.name }} - - name: DATABASE_NAME + - name: UMP_DATABASE_NAME valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: dbname - - name: DATABASE_HOST + - name: UMP_DATABASE_HOST valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: host - - name: DATABASE_PORT + - name: UMP_DATABASE_PORT valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: port - - name: DATABASE_USER + - name: UMP_DATABASE_USER valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: user - - name: DATABASE_PASSWORD + - name: UMP_DATABASE_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} @@ -92,59 +93,59 @@ spec: # keycloak connection {{- end }} {{- if .Values.keycloakConnection.existingSecret.name }} - - name: KEYCLOAK_USER + - name: UMP_KEYCLOAK_USER valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} key: user - - name: KEYCLOAK_PASSWORD + - name: UMP_KEYCLOAK_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} key: password - - name: KEYCLOAK_REALM + - name: UMP_KEYCLOAK_REALM valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} key: realm - - name: KEYCLOAK_URL + - name: UMP_KEYCLOAK_URL valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} key: url - - name: KEYCLOAK_CLIENT_ID + - name: UMP_KEYCLOAK_CLIENT_ID valueFrom: secretKeyRef: name: {{ .Values.keycloakConnection.existingSecret.name }} key: clientId {{- end }} {{- if .Values.geoserverConnection.existingSecret.name }} - - name: GEOSERVER_URL + - name: UMP_GEOSERVER_URL valueFrom: secretKeyRef: name: {{ .Values.geoserverConnection.existingSecret.name }} key: url - - name: GEOSERVER_DB_HOST + - name: UMP_GEOSERVER_DB_HOST valueFrom: secretKeyRef: name: {{ .Values.geoserverConnection.existingSecret.name }} key: dbHost - - name: GEOSERVER_DB_PORT + - name: UMP_GEOSERVER_DB_PORT valueFrom: secretKeyRef: name: {{ .Values.geoserverConnection.existingSecret.name }} key: dbPort - - name: GEOSERVER_WORKSPACE_NAME + - name: UMP_GEOSERVER_WORKSPACE_NAME valueFrom: secretKeyRef: name: {{ .Values.geoserverConnection.existingSecret.name }} key: workspaceName - - name: GEOSERVER_USER + - name: UMP_GEOSERVER_USER valueFrom: secretKeyRef: name: {{ .Values.geoserverConnection.existingSecret.name }} key: user - - name: GEOSERVER_PASSWORD + - name: UMP_GEOSERVER_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.geoserverConnection.existingSecret.name }} From da7d71e069f39bd043a13cf5adcb2f107cd6035f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:12:18 +0000 Subject: [PATCH 074/145] env var names updated --- .../templates/job-migrations.yaml | 10 +++++----- .../templates/secret-geoserver-job.yaml | 16 ++++++++++------ .../templates/secret-geoserver.yaml | 16 ++++++++++------ .../templates/secret-keycloak-job.yaml | 11 +++++------ .../templates/secret-keycloak.yaml | 11 +++++------ 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/charts/urban-model-platform/templates/job-migrations.yaml b/charts/urban-model-platform/templates/job-migrations.yaml index e873e3a..6326da8 100644 --- a/charts/urban-model-platform/templates/job-migrations.yaml +++ b/charts/urban-model-platform/templates/job-migrations.yaml @@ -33,31 +33,31 @@ spec: - secretRef: name: {{ include "ump.fullname" . }}-job-keycloak-connection env: - - name: POSTGRES_DB + - name: UMP_POSTGRES_DB valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: dbname {{- if not .Values.config.postgresHost }} - - name: POSTGRES_HOST + - name: UMP_POSTGRES_HOST valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: host {{- end }} {{- if not .Values.config.postgresPort }} - - name: POSTGRES_PORT + - name: UMP_POSTGRES_PORT valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: port {{- end }} - - name: POSTGRES_USER + - name: UMP_POSTGRES_USER valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} key: user - - name: POSTGRES_PASSWORD + - name: UMP_POSTGRES_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.postgresConnection.existingSecret.name }} diff --git a/charts/urban-model-platform/templates/secret-geoserver-job.yaml b/charts/urban-model-platform/templates/secret-geoserver-job.yaml index b8e0ed8..d81662a 100644 --- a/charts/urban-model-platform/templates/secret-geoserver-job.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver-job.yaml @@ -10,10 +10,14 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - GEOSERVER_URL: "" - GEOSERVER_DB_HOST: "" - GEOSERVER_DB_PORT: "" - GEOSERVER_WORKSPACE_NAME: "" - GEOSERVER_USER: "" - GEOSERVER_PASSWORD: "" + UMP_GEOSERVER_URL: "" + UMP_GEOSERVER_DB_HOST: "" + UMP_GEOSERVER_DB_PORT: "5432" + UMP_GEOSERVER_DB_NAME: "ump" + UMP_GEOSERVER_DB_USER: "ump" + UMP_GEOSERVER_DB_PASSWORD: "ump" + UMP_GEOSERVER_WORKSPACE_NAME: "UMP" + UMP_GEOSERVER_USER: "geoserver" + UMP_GEOSERVER_PASSWORD: "geoserver" + UMP_GEOSERVER_CONNECTION_TIMEOUT: "60" {{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-geoserver.yaml b/charts/urban-model-platform/templates/secret-geoserver.yaml index 49fbd76..1b23413 100644 --- a/charts/urban-model-platform/templates/secret-geoserver.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver.yaml @@ -6,10 +6,14 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - GEOSERVER_URL: "" - GEOSERVER_DB_HOST: "" - GEOSERVER_DB_PORT: "" - GEOSERVER_WORKSPACE_NAME: "" - GEOSERVER_USER: "" - GEOSERVER_PASSWORD: "" + UMP_GEOSERVER_URL: "" + UMP_GEOSERVER_DB_HOST: "" + UMP_GEOSERVER_DB_PORT: "5432" + UMP_GEOSERVER_DB_NAME: "ump" + UMP_GEOSERVER_DB_USER: "ump" + UMP_GEOSERVER_DB_PASSWORD: "ump" + UMP_GEOSERVER_WORKSPACE_NAME: "UMP" + UMP_GEOSERVER_USER: "geoserver" + UMP_GEOSERVER_PASSWORD: "geoserver" + UMP_GEOSERVER_CONNECTION_TIMEOUT: "60" {{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-keycloak-job.yaml b/charts/urban-model-platform/templates/secret-keycloak-job.yaml index 1c360ab..f149ded 100644 --- a/charts/urban-model-platform/templates/secret-keycloak-job.yaml +++ b/charts/urban-model-platform/templates/secret-keycloak-job.yaml @@ -13,10 +13,9 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - KEYCLOAK_CLIENT_ID: "" - KEYCLOAK_PASSWORD: "" - KEYCLOAK_PROTOCOL: "" - KEYCLOAK_REALM: "" - KEYCLOAK_URL: "" - KEYCLOAK_USER: "" + UMP_KEYCLOAK_CLIENT_ID: "" + UMP_KEYCLOAK_PASSWORD: "" + UMP_KEYCLOAK_REALM: "" + UMP_KEYCLOAK_URL: "" + UMP_KEYCLOAK_USER: "" {{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-keycloak.yaml b/charts/urban-model-platform/templates/secret-keycloak.yaml index 82895d0..e94b390 100644 --- a/charts/urban-model-platform/templates/secret-keycloak.yaml +++ b/charts/urban-model-platform/templates/secret-keycloak.yaml @@ -6,10 +6,9 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - KEYCLOAK_CLIENT_ID: "" - KEYCLOAK_PASSWORD: "" - KEYCLOAK_PROTOCOL: "" - KEYCLOAK_REALM: "" - KEYCLOAK_URL: "" - KEYCLOAK_USER: "" + UMP_KEYCLOAK_CLIENT_ID: "" + UMP_KEYCLOAK_PASSWORD: "" + UMP_KEYCLOAK_REALM: "" + UMP_KEYCLOAK_URL: "" + UMP_KEYCLOAK_USER: "" {{- end -}} \ No newline at end of file From 86b173b04e31adb7b99acaec90f2394b98402677 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:15:58 +0000 Subject: [PATCH 075/145] added line break --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index d7676db..782c237 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,7 @@ set-app-version: # Update chart version: bump major, minor, or patch bump-chart-version: cd charts + @if [ -z "$(part)" ]; then \ echo "Usage: make bump-chart part={major|minor|patch}"; \ exit 1; \ From 313f03f85caaa7a86185ae86c76c1e8d995c2bfc Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:18:30 +0000 Subject: [PATCH 076/145] fixed command --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 782c237..ef5a253 100644 --- a/Makefile +++ b/Makefile @@ -117,10 +117,8 @@ set-app-version: # Update chart version: bump major, minor, or patch bump-chart-version: - cd charts - @if [ -z "$(part)" ]; then \ echo "Usage: make bump-chart part={major|minor|patch}"; \ exit 1; \ fi; \ - bump-my-version bump $(part) \ No newline at end of file + (cd charts && bump-my-version bump $(part)) \ No newline at end of file From 6c43ebed9eaee16ce6a5dcbde8559c2dc3f3a4ab Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:23:57 +0000 Subject: [PATCH 077/145] fixed current version --- charts/.bumpmyversion.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/.bumpmyversion.toml b/charts/.bumpmyversion.toml index 4610990..4c7c744 100644 --- a/charts/.bumpmyversion.toml +++ b/charts/.bumpmyversion.toml @@ -1,5 +1,5 @@ [bumpversion] -current_version = "1.2.0" +current_version = "0.7.0" commit = true tag = true tag_name = "chart-v{new_version}" From fdaf01cbab81a741f3804cf3ec7880ff216988aa Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:25:15 +0000 Subject: [PATCH 078/145] fixed path --- charts/.bumpmyversion.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/.bumpmyversion.toml b/charts/.bumpmyversion.toml index 4c7c744..71ba623 100644 --- a/charts/.bumpmyversion.toml +++ b/charts/.bumpmyversion.toml @@ -5,6 +5,6 @@ tag = true tag_name = "chart-v{new_version}" [[bumpversion.files]] -filename = "charts/urban-model-platform/Chart.yaml" +filename = "urban-model-platform/Chart.yaml" search = 'version: {current_version}' replace = 'version: {new_version}' From ffbde8c8556af937249b069f3caac58f9726e4ef Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 09:38:59 +0000 Subject: [PATCH 079/145] ranmed files --- .bumpmyversion.toml => .bumpversion.toml | 0 charts/{.bumpmyversion.toml => .bumpversion.toml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .bumpmyversion.toml => .bumpversion.toml (100%) rename charts/{.bumpmyversion.toml => .bumpversion.toml} (100%) diff --git a/.bumpmyversion.toml b/.bumpversion.toml similarity index 100% rename from .bumpmyversion.toml rename to .bumpversion.toml diff --git a/charts/.bumpmyversion.toml b/charts/.bumpversion.toml similarity index 100% rename from charts/.bumpmyversion.toml rename to charts/.bumpversion.toml From 85549f9b3f4198a4b3b7711c0954b3faeb2f7bb9 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 10:52:26 +0000 Subject: [PATCH 080/145] bump chart version: 0.7.0 -> 0.8.0 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 7d7ea2f..0a9f460 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.7.0 +version: 0.8.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 15bfb28d809a23971e6785d29c1059786444e658 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 14:11:02 +0000 Subject: [PATCH 081/145] fix: newline --- charts/urban-model-platform/templates/secret-geoserver-job.yaml | 2 +- charts/urban-model-platform/templates/secret-keycloak-job.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/urban-model-platform/templates/secret-geoserver-job.yaml b/charts/urban-model-platform/templates/secret-geoserver-job.yaml index d81662a..0c12303 100644 --- a/charts/urban-model-platform/templates/secret-geoserver-job.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver-job.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.keycloakConnection.existingSecret.name -}} +{{- if not .Values.keycloakConnection.existingSecret.name }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/urban-model-platform/templates/secret-keycloak-job.yaml b/charts/urban-model-platform/templates/secret-keycloak-job.yaml index f149ded..e3921e4 100644 --- a/charts/urban-model-platform/templates/secret-keycloak-job.yaml +++ b/charts/urban-model-platform/templates/secret-keycloak-job.yaml @@ -1,7 +1,7 @@ # TODO: currently the full app is loaded when inly migrations are run # TODO: create an app factory that only runs migrations and dont needs keycloak and geoserver connections # TODO: then, the respective job secrets can be omitted -{{- if not .Values.keycloakConnection.existingSecret.name -}} +{{- if not .Values.keycloakConnection.existingSecret.name }} apiVersion: v1 kind: Secret metadata: From 4a15ed7d4542ecd8c9f1ea1642e31699b11cdfee Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 14:13:50 +0000 Subject: [PATCH 082/145] bump chart version: 0.8.0 -> 0.8.1 --- charts/urban-model-platform/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 0a9f460..1a71cfa 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.0 +version: 0.8.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.2.0" +appVersion: "2.0.0" kubeVersion: ">= 1.31.0" annotations: category: API From 7226435d56360cdb28b7d6f91eed1c0312d5466b Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 14:29:42 +0000 Subject: [PATCH 083/145] empty dummy secrets --- .../templates/secret-geoserver-job.yaml | 16 ++++++++-------- .../templates/secret-geoserver.yaml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/urban-model-platform/templates/secret-geoserver-job.yaml b/charts/urban-model-platform/templates/secret-geoserver-job.yaml index 0c12303..8629ed9 100644 --- a/charts/urban-model-platform/templates/secret-geoserver-job.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver-job.yaml @@ -12,12 +12,12 @@ metadata: data: UMP_GEOSERVER_URL: "" UMP_GEOSERVER_DB_HOST: "" - UMP_GEOSERVER_DB_PORT: "5432" - UMP_GEOSERVER_DB_NAME: "ump" - UMP_GEOSERVER_DB_USER: "ump" - UMP_GEOSERVER_DB_PASSWORD: "ump" - UMP_GEOSERVER_WORKSPACE_NAME: "UMP" - UMP_GEOSERVER_USER: "geoserver" - UMP_GEOSERVER_PASSWORD: "geoserver" - UMP_GEOSERVER_CONNECTION_TIMEOUT: "60" + UMP_GEOSERVER_DB_PORT: "" + UMP_GEOSERVER_DB_NAME: "" + UMP_GEOSERVER_DB_USER: "" + UMP_GEOSERVER_DB_PASSWORD: "" + UMP_GEOSERVER_WORKSPACE_NAME: "" + UMP_GEOSERVER_USER: "" + UMP_GEOSERVER_PASSWORD: "" + UMP_GEOSERVER_CONNECTION_TIMEOUT: "" {{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-geoserver.yaml b/charts/urban-model-platform/templates/secret-geoserver.yaml index 1b23413..1941681 100644 --- a/charts/urban-model-platform/templates/secret-geoserver.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver.yaml @@ -8,12 +8,12 @@ metadata: data: UMP_GEOSERVER_URL: "" UMP_GEOSERVER_DB_HOST: "" - UMP_GEOSERVER_DB_PORT: "5432" - UMP_GEOSERVER_DB_NAME: "ump" - UMP_GEOSERVER_DB_USER: "ump" - UMP_GEOSERVER_DB_PASSWORD: "ump" - UMP_GEOSERVER_WORKSPACE_NAME: "UMP" - UMP_GEOSERVER_USER: "geoserver" - UMP_GEOSERVER_PASSWORD: "geoserver" - UMP_GEOSERVER_CONNECTION_TIMEOUT: "60" + UMP_GEOSERVER_DB_PORT: "" + UMP_GEOSERVER_DB_NAME: "" + UMP_GEOSERVER_DB_USER: "" + UMP_GEOSERVER_DB_PASSWORD: "" + UMP_GEOSERVER_WORKSPACE_NAME: "" + UMP_GEOSERVER_USER: "" + UMP_GEOSERVER_PASSWORD: "" + UMP_GEOSERVER_CONNECTION_TIMEOUT: "" {{- end -}} \ No newline at end of file From f412617c672e381b707ece3c5b42962bace323f4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 14:30:15 +0000 Subject: [PATCH 084/145] bump chart version: 0.8.1 -> 0.8.2 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 1a71cfa..0027b71 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.1 +version: 0.8.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 94974c55751d59651d0312844a826709ccf0050c Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 14:40:57 +0000 Subject: [PATCH 085/145] fix: providersFilePath unified --- .../urban-model-platform/templates/configmap-settings-job.yaml | 2 +- charts/urban-model-platform/templates/configmap-settings.yaml | 2 +- charts/urban-model-platform/templates/deployment.yaml | 2 +- charts/urban-model-platform/values.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/urban-model-platform/templates/configmap-settings-job.yaml b/charts/urban-model-platform/templates/configmap-settings-job.yaml index 1f13d41..ee80407 100644 --- a/charts/urban-model-platform/templates/configmap-settings-job.yaml +++ b/charts/urban-model-platform/templates/configmap-settings-job.yaml @@ -10,7 +10,7 @@ metadata: {{- include "ump.labels" . | nindent 4 }} data: UMP_LOG_LEVEL: {{ .Values.config.logLevel | quote }} - UMP_PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} + UMP_PROVIDERS_FILE: {{ .Values.config.providersFilePath | quote }} UMP_API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} UMP_API_SERVER_URL_PREFIX: {{ .Values.config.apiServerUrlPrefix | quote }} UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} diff --git a/charts/urban-model-platform/templates/configmap-settings.yaml b/charts/urban-model-platform/templates/configmap-settings.yaml index 274c864..be194ec 100644 --- a/charts/urban-model-platform/templates/configmap-settings.yaml +++ b/charts/urban-model-platform/templates/configmap-settings.yaml @@ -6,7 +6,7 @@ metadata: {{- include "ump.labels" . | nindent 4 }} data: UMP_LOG_LEVEL: {{ .Values.config.logLevel | quote }} - UMP_PROVIDERS_FILE: {{ .Values.config.providersFile | quote }} + UMP_PROVIDERS_FILE: {{ .Values.config.providersFilePath | quote }} UMP_API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} UMP_API_SERVER_URL_PREFIX: {{ .Values.config.apiServerUrlPrefix | quote }} UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index 66b8f04..2ab9faa 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -48,7 +48,7 @@ spec: - name: tmp-volume mountPath: /tmp - name: providers-volume - mountPath: {{ .Values.config.providersFileMountPath | quote }} + mountPath: {{ .Values.config.providersFilePath | quote }} readOnly: true envFrom: - configMapRef: diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index 20f1cac..83584d9 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -41,7 +41,7 @@ service: config: logLevel: "DEBUG" - providersFile: "./providers.yaml" + providersFilePath: "./providers.yaml" apiServerUrl: "http://localhost:5000" apiServerUrlPrefix: "/api" apiServerWorkers: "4" From c206cce612a0a2886ab214c42a3e6ec64d2458fc Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 8 Jul 2025 14:41:38 +0000 Subject: [PATCH 086/145] bump chart version: 0.8.2 -> 0.8.3 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 0027b71..f6526aa 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.2 +version: 0.8.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From afb106bc4a98ae8a92faff03bdac066e4af4ad4c Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 08:46:29 +0000 Subject: [PATCH 087/145] deleted job and job related objects --- .../templates/configmap-settings-job.yaml | 18 ----- .../templates/job-migrations.yaml | 77 ------------------- .../templates/role-job-reader.yaml | 10 --- .../templates/rolebinding-job.yaml | 15 ---- .../templates/secret-geoserver-job.yaml | 23 ------ .../templates/secret-keycloak-job.yaml | 21 ----- 6 files changed, 164 deletions(-) delete mode 100644 charts/urban-model-platform/templates/configmap-settings-job.yaml delete mode 100644 charts/urban-model-platform/templates/job-migrations.yaml delete mode 100644 charts/urban-model-platform/templates/role-job-reader.yaml delete mode 100644 charts/urban-model-platform/templates/rolebinding-job.yaml delete mode 100644 charts/urban-model-platform/templates/secret-geoserver-job.yaml delete mode 100644 charts/urban-model-platform/templates/secret-keycloak-job.yaml diff --git a/charts/urban-model-platform/templates/configmap-settings-job.yaml b/charts/urban-model-platform/templates/configmap-settings-job.yaml deleted file mode 100644 index ee80407..0000000 --- a/charts/urban-model-platform/templates/configmap-settings-job.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "ump.fullname" . }}-job-settings - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-6" # Lower weight than job (-5) ensures it runs first - "helm.sh/hook-delete-policy": hook-succeeded - labels: - {{- include "ump.labels" . | nindent 4 }} -data: - UMP_LOG_LEVEL: {{ .Values.config.logLevel | quote }} - UMP_PROVIDERS_FILE: {{ .Values.config.providersFilePath | quote }} - UMP_API_SERVER_URL: {{ .Values.config.apiServerUrl | quote }} - UMP_API_SERVER_URL_PREFIX: {{ .Values.config.apiServerUrlPrefix | quote }} - UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} - UMP_JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} - UMP_API_SERVER_WORKERS: {{ .Values.config.apiServerWorkers | quote }} diff --git a/charts/urban-model-platform/templates/job-migrations.yaml b/charts/urban-model-platform/templates/job-migrations.yaml deleted file mode 100644 index 6326da8..0000000 --- a/charts/urban-model-platform/templates/job-migrations.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "ump.fullname" . }}-db-migrate - labels: - {{- include "ump.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": hook-succeeded -spec: - backoffLimit: 3 - template: - spec: - {{- with .Values.image.pullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: db-migrate - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - command: ["flask", "db", "upgrade"] - volumeMounts: - - name: tmp-volume - mountPath: /tmp - - name: providers-volume - mountPath: {{ .Values.config.providersFilePath | quote }} - subPath: providers.yaml - readOnly: true - envFrom: - - configMapRef: - name: {{ include "ump.fullname" . }}-job-settings - - secretRef: - name: {{ include "ump.fullname" . }}-job-keycloak-connection - env: - - name: UMP_POSTGRES_DB - valueFrom: - secretKeyRef: - name: {{ .Values.postgresConnection.existingSecret.name }} - key: dbname - {{- if not .Values.config.postgresHost }} - - name: UMP_POSTGRES_HOST - valueFrom: - secretKeyRef: - name: {{ .Values.postgresConnection.existingSecret.name }} - key: host - {{- end }} - {{- if not .Values.config.postgresPort }} - - name: UMP_POSTGRES_PORT - valueFrom: - secretKeyRef: - name: {{ .Values.postgresConnection.existingSecret.name }} - key: port - {{- end }} - - name: UMP_POSTGRES_USER - valueFrom: - secretKeyRef: - name: {{ .Values.postgresConnection.existingSecret.name }} - key: user - - name: UMP_POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.postgresConnection.existingSecret.name }} - key: password - - name: FLASK_APP - value: "ump.main" - restartPolicy: Never - volumes: - - name: tmp-volume - emptyDir: {} - - name: providers-volume - configMap: - {{- if not .Values.providers.existingConfigMap.name }} - name: {{ include "ump.fullname" . }}-providers - {{- else }} - name: {{ .Values.providers.existingConfigMap.name }} - {{- end }} diff --git a/charts/urban-model-platform/templates/role-job-reader.yaml b/charts/urban-model-platform/templates/role-job-reader.yaml deleted file mode 100644 index e6039a1..0000000 --- a/charts/urban-model-platform/templates/role-job-reader.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "ump.fullname" . }}-job-reader - labels: - {{- include "ump.labels" . | nindent 4 }} -rules: -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get", "watch", "list"] diff --git a/charts/urban-model-platform/templates/rolebinding-job.yaml b/charts/urban-model-platform/templates/rolebinding-job.yaml deleted file mode 100644 index dd549bc..0000000 --- a/charts/urban-model-platform/templates/rolebinding-job.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# rolebinding-job.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "ump.fullname" . }}-job-reader - labels: - {{- include "ump.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "ump.fullname" . }}-job-reader -subjects: -- kind: ServiceAccount - name: {{ include "ump.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-geoserver-job.yaml b/charts/urban-model-platform/templates/secret-geoserver-job.yaml deleted file mode 100644 index 8629ed9..0000000 --- a/charts/urban-model-platform/templates/secret-geoserver-job.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if not .Values.keycloakConnection.existingSecret.name }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "ump.fullname" . }}-job-geoserver-connection - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-6" # Lower weight than job (-5) ensures it runs first - "helm.sh/hook-delete-policy": hook-succeeded - labels: - {{ include "ump.labels" . | nindent 4 }} -data: - UMP_GEOSERVER_URL: "" - UMP_GEOSERVER_DB_HOST: "" - UMP_GEOSERVER_DB_PORT: "" - UMP_GEOSERVER_DB_NAME: "" - UMP_GEOSERVER_DB_USER: "" - UMP_GEOSERVER_DB_PASSWORD: "" - UMP_GEOSERVER_WORKSPACE_NAME: "" - UMP_GEOSERVER_USER: "" - UMP_GEOSERVER_PASSWORD: "" - UMP_GEOSERVER_CONNECTION_TIMEOUT: "" -{{- end -}} \ No newline at end of file diff --git a/charts/urban-model-platform/templates/secret-keycloak-job.yaml b/charts/urban-model-platform/templates/secret-keycloak-job.yaml deleted file mode 100644 index e3921e4..0000000 --- a/charts/urban-model-platform/templates/secret-keycloak-job.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# TODO: currently the full app is loaded when inly migrations are run -# TODO: create an app factory that only runs migrations and dont needs keycloak and geoserver connections -# TODO: then, the respective job secrets can be omitted -{{- if not .Values.keycloakConnection.existingSecret.name }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "ump.fullname" . }}-job-keycloak-connection - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-6" # Lower weight than job (-5) ensures it runs first - "helm.sh/hook-delete-policy": hook-succeeded - labels: - {{ include "ump.labels" . | nindent 4 }} -data: - UMP_KEYCLOAK_CLIENT_ID: "" - UMP_KEYCLOAK_PASSWORD: "" - UMP_KEYCLOAK_REALM: "" - UMP_KEYCLOAK_URL: "" - UMP_KEYCLOAK_USER: "" -{{- end -}} \ No newline at end of file From 9b9f0c43705c74170e1ce1ab48932e8e99c613fd Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 08:46:40 +0000 Subject: [PATCH 088/145] added a dummy adress --- charts/urban-model-platform/templates/secret-keycloak.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/templates/secret-keycloak.yaml b/charts/urban-model-platform/templates/secret-keycloak.yaml index e94b390..fc5e264 100644 --- a/charts/urban-model-platform/templates/secret-keycloak.yaml +++ b/charts/urban-model-platform/templates/secret-keycloak.yaml @@ -9,6 +9,6 @@ data: UMP_KEYCLOAK_CLIENT_ID: "" UMP_KEYCLOAK_PASSWORD: "" UMP_KEYCLOAK_REALM: "" - UMP_KEYCLOAK_URL: "" + UMP_KEYCLOAK_URL: "aHR0cDovL2tleWNsb2FrOjgwODAvYXV0aA==" UMP_KEYCLOAK_USER: "" {{- end -}} \ No newline at end of file From 165691b322c2bfa47cdb6bb4f48ad4c1f765b6a2 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 08:47:22 +0000 Subject: [PATCH 089/145] version bump: 0.8.3 -> 0.9.0 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index f6526aa..ea7d84f 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.3 +version: 0.9.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 4aec5c8710e659dc8bba2cc2ede359e19d8a71a4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 09:00:54 +0000 Subject: [PATCH 090/145] set flask app --- charts/urban-model-platform/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index 2ab9faa..85ec488 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -63,6 +63,8 @@ spec: name: {{ include "ump.fullname" . }}-geoserver-connection {{- end }} env: + - name: FLASK_APP + value: ump.main # postgres connection {{- if .Values.postgresConnection.existingSecret.name }} - name: UMP_DATABASE_NAME From 0147dc97bc53ed1d797fdf64c022fd74785c05f0 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 09:01:30 +0000 Subject: [PATCH 091/145] bump chart version: 0.9.0 -> 0.9.1 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index ea7d84f..0bc0b55 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.9.0 +version: 0.9.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 42ce4349279f9573d2af4c411d379fb894916d4a Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 09:28:35 +0000 Subject: [PATCH 092/145] fixed: using mount path instead of file --- charts/urban-model-platform/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index 85ec488..3358a30 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -48,7 +48,7 @@ spec: - name: tmp-volume mountPath: /tmp - name: providers-volume - mountPath: {{ .Values.config.providersFilePath | quote }} + mountPath: {{ .Values.config.providersFileMountPath | quote }} readOnly: true envFrom: - configMapRef: From f56d5919950dfa20f83308b319b11d5a5a04b737 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 09:28:46 +0000 Subject: [PATCH 093/145] added mount path --- charts/urban-model-platform/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index 83584d9..37ba243 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -42,6 +42,7 @@ service: config: logLevel: "DEBUG" providersFilePath: "./providers.yaml" + providersFileMountPath: /app apiServerUrl: "http://localhost:5000" apiServerUrlPrefix: "/api" apiServerWorkers: "4" @@ -49,7 +50,7 @@ config: geoserverConnectionTimeout: "60" jobDeleteInterval: "240" -postgresConnection: +postgresConnection: existingSecret: name: postgres-credentials @@ -91,7 +92,7 @@ autoscaling: targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 -tolerations: +tolerations: - key: ump/reservedFor operator: "Equal" value: app From 183cfe8069513324e9713c94c20abf45fe239561 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 09:28:55 +0000 Subject: [PATCH 094/145] added dummy values --- charts/urban-model-platform/templates/secret-geoserver.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/urban-model-platform/templates/secret-geoserver.yaml b/charts/urban-model-platform/templates/secret-geoserver.yaml index 1941681..3c4e746 100644 --- a/charts/urban-model-platform/templates/secret-geoserver.yaml +++ b/charts/urban-model-platform/templates/secret-geoserver.yaml @@ -6,14 +6,14 @@ metadata: labels: {{ include "ump.labels" . | nindent 4 }} data: - UMP_GEOSERVER_URL: "" + UMP_GEOSERVER_URL: "aHR0cDovL2dlb3NlcnZlcjo4MDgw" UMP_GEOSERVER_DB_HOST: "" - UMP_GEOSERVER_DB_PORT: "" + UMP_GEOSERVER_DB_PORT: "NTQzMg==" UMP_GEOSERVER_DB_NAME: "" UMP_GEOSERVER_DB_USER: "" UMP_GEOSERVER_DB_PASSWORD: "" UMP_GEOSERVER_WORKSPACE_NAME: "" UMP_GEOSERVER_USER: "" UMP_GEOSERVER_PASSWORD: "" - UMP_GEOSERVER_CONNECTION_TIMEOUT: "" + UMP_GEOSERVER_CONNECTION_TIMEOUT: "MTA=" {{- end -}} \ No newline at end of file From a3c36c124b6aa275574549686467436416d6dee9 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 09:29:33 +0000 Subject: [PATCH 095/145] bump chart version: 0.9.1 -> 0.9.2 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 0bc0b55..27abcd9 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.9.1 +version: 0.9.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 5738493d142c2bf5cfa4a760ab7405b4f2407cf2 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 12:42:49 +0000 Subject: [PATCH 096/145] readyness probe must use dynamic prefix --- charts/urban-model-platform/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index 3358a30..d84f501 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -155,7 +155,7 @@ spec: {{- end }} readinessProbe: httpGet: - path: /api/health/ready + path: {{ .Values.config.apiServerUrlPrefix }}/health/ready port: http initialDelaySeconds: 5 periodSeconds: 10 From 7c3a41a730391899a15057e858fec93f17cb6d6f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 12:53:25 +0000 Subject: [PATCH 097/145] added missing backendRefs keys --- charts/urban-model-platform/templates/httproute-tls.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/urban-model-platform/templates/httproute-tls.yaml b/charts/urban-model-platform/templates/httproute-tls.yaml index 27a983e..02a5ee1 100644 --- a/charts/urban-model-platform/templates/httproute-tls.yaml +++ b/charts/urban-model-platform/templates/httproute-tls.yaml @@ -23,4 +23,6 @@ spec: - name: {{ include "ump.fullname" . }} port: {{ .Values.service.port }} kind: Service + group: '' + weight: 1 {{- end -}} \ No newline at end of file From 2f5df09f79d0601d9ac70b007eadb1eef99866e4 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 9 Jul 2025 12:54:21 +0000 Subject: [PATCH 098/145] bump chart version: 0.9.2 -> 0.9.3 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index 27abcd9..b9e7043 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.9.2 +version: 0.9.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 5f07f0a37c2b9991e9fc83f27366ae3240509bca Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 10 Jul 2025 10:47:14 +0000 Subject: [PATCH 099/145] improved error details when job is not successful --- src/ump/api/models/job.py | 57 ++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index 76d86b3..e36b8c5 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -367,15 +367,7 @@ def display(self, additional_metadata=False): async def results(self): if self.status != JobStatus.successful.value: - raise OGCProcessException( - OGCExceptionResponse( - type="InvalidParameterValue", - title="No results available", - detail=self.message, - status=404, - instance=f"{config.UMP_API_SERVER_URL}/{config.UMP_API_SERVER_URL_PREFIX}/jobs/{self.job_id}/results", - ) - ) + self.results_not_available() provider: ProviderConfig = providers.get_providers()[self.provider_prefix] self.provider_url = provider.server_url @@ -430,6 +422,53 @@ async def results_to_geoserver(self): e, ) + def results_not_available(self): + """ + Raises an OGCProcessException with a meaningful type and detail + according to the current job status. + """ + status_map = { + JobStatus.failed.value: { + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/failed", + "title": "Job failed", + "detail": self.message or "The job failed and no results are available.", + }, + JobStatus.dismissed.value: { + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/dismissed", + "title": "Job dismissed", + "detail": self.message or "The job was dismissed and no results are available.", + }, + JobStatus.running.value: { + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/result-not-ready", + "title": "Job still running", + "detail": "The job is still running. Results are not yet available.", + }, + JobStatus.accepted.value: { + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/result-not-ready", + "title": "Job accepted", + "detail": "The job has been accepted but has not started yet. Results are not available.", + }, + } + + info = status_map.get( + self.status if self.status is not None else "", + { + "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-job", + "title": "No results available", + "detail": "No results are available for this job.", + }, + ) + + raise OGCProcessException( + OGCExceptionResponse( + type=info["type"], + title=info["title"], + detail=info["detail"], + status=404, + instance=f"{config.UMP_API_SERVER_URL}/{config.UMP_API_SERVER_URL_PREFIX}/jobs/{self.job_id}/results", + ) + ) + def __str__(self): return f""" ----- src.job.Job ----- From 83a71e915d71cd33c96126ff85ddb793181b502a Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 10 Jul 2025 10:48:22 +0000 Subject: [PATCH 100/145] updated version: 2.1.0rc2 -> 2.1.0rc3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fa0bb26..4c93c91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc2+fix-error-handling" +version = "2.1.0rc3+fix-error-handling" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From 57322c10fc020daa2d61edb9da180287398f0329 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 14 Jul 2025 12:28:39 +0000 Subject: [PATCH 101/145] removed read-only from providers mount --- charts/urban-model-platform/templates/deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/urban-model-platform/templates/deployment.yaml b/charts/urban-model-platform/templates/deployment.yaml index d84f501..a61043f 100644 --- a/charts/urban-model-platform/templates/deployment.yaml +++ b/charts/urban-model-platform/templates/deployment.yaml @@ -49,7 +49,6 @@ spec: mountPath: /tmp - name: providers-volume mountPath: {{ .Values.config.providersFileMountPath | quote }} - readOnly: true envFrom: - configMapRef: name: {{ include "ump.fullname" . }}-settings From 6495a4ff810d35248709778e560e193c6b053293 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Mon, 14 Jul 2025 12:29:21 +0000 Subject: [PATCH 102/145] updated chart version: 0.9.3 -> 0.9.4 --- charts/urban-model-platform/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index b9e7043..d66abf6 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.9.3 +version: 0.9.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 47a6e6463add2d2fc991d3a224c9f8673ff83e30 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 15 Jul 2025 10:24:08 +0000 Subject: [PATCH 103/145] added UMP_API_SERVER_URL_PREFIX --- docs/content/02-user_guide/setup.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/02-user_guide/setup.md b/docs/content/02-user_guide/setup.md index 79c5378..c1b4d73 100644 --- a/docs/content/02-user_guide/setup.md +++ b/docs/content/02-user_guide/setup.md @@ -29,6 +29,7 @@ This document describes the configuration options for the Urban Model Platform ( | `UMP_KEYCLOAK_CLIENT_ID` | Keycloak client ID. | `ump-client` | | `UMP_KEYCLOAK_USER` | Keycloak admin username. | `admin` | | `UMP_KEYCLOAK_PASSWORD` | Keycloak admin password. | `admin` | +| `UMP_API_SERVER_URL_PREFIX` | subpath prefix, e.g.: "/api" | `/` | ### Example Modelserver Settings | Variable | Description | Default Value | From 5674699a833326ab01adae50cb86cb0a04e926c6 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 15 Jul 2025 11:43:50 +0000 Subject: [PATCH 104/145] changed providers loader to listen on any event (k8s configmap issue) --- src/ump/api/providers.py | 203 ++++++++++++++++++++++++++++++--------- 1 file changed, 158 insertions(+), 45 deletions(-) diff --git a/src/ump/api/providers.py b/src/ump/api/providers.py index e204792..5b61ac9 100644 --- a/src/ump/api/providers.py +++ b/src/ump/api/providers.py @@ -1,7 +1,9 @@ import atexit -import logging +import time from logging import getLogger -from threading import Lock +from threading import Lock, Timer +import threading +from typing import Optional import aiohttp import yaml @@ -19,33 +21,97 @@ logger = getLogger(__name__) +# Thread-safe provider storage PROVIDERS: ModelServers = {} -PROVIDERS_LOCK = Lock() # Thread-safe lock for updating PROVIDERS +PROVIDERS_LOCK = Lock() +RELOAD_TIMER: Optional[Timer] = None +DEBOUNCE_DELAY = 0.5 # 500ms debounce class ProviderLoader(FileSystemEventHandler): - def on_modified(self, event): - logger.info("File modified: %s", event.src_path) - if event.src_path == config.UMP_PROVIDERS_FILE.absolute().as_posix(): - self.load_providers() + def __init__(self): + self.last_reload = 0 + self.reload_lock = threading.Lock() + + # need to listen on any event, not just file changes for k8s configmap updates + def on_any_event(self, event): + # Ignore directory events + if event.is_directory: + return + + logger.info("File event: %s on %s", event.event_type, event.src_path) + + event.src_path = str(event.src_path) + + # Check if the event affects our config file + config_path = config.UMP_PROVIDERS_FILE.absolute().as_posix() + + endswith = event.src_path.endswith(config.UMP_PROVIDERS_FILE.name) + contains = '..data' in event.src_path + + if ( + event.src_path == config_path or + endswith or + contains + ): + self._debounced_reload() + + def _debounced_reload(self): + """Debounce rapid file changes to avoid reload storms""" + global RELOAD_TIMER + + # Cancel existing timer + if RELOAD_TIMER: + RELOAD_TIMER.cancel() + + # Schedule debounced reload + RELOAD_TIMER = Timer(DEBOUNCE_DELAY, self.load_providers) + RELOAD_TIMER.start() def load_providers(self): logger.info("(Re)Loading providers from %s", config.UMP_PROVIDERS_FILE) + + # Create new providers dict (don't modify global state yet) + new_providers = {} + try: with open(config.UMP_PROVIDERS_FILE, encoding="UTF-8") as file: if content := yaml.safe_load(file): + # Validate before applying validated_content = model_servers_adapter.validate_python(content) - with PROVIDERS_LOCK: # Ensure thread-safe update - # Update PROVIDERS in place - PROVIDERS.clear() - PROVIDERS.update(validated_content) - logger.info("Providers (re)loaded successfully") + new_providers.update(validated_content) + + # Atomic update with rollback capability + self._atomic_update(new_providers) + logger.info("Providers (re)loaded successfully") + else: + logger.warning("Providers file is empty, keeping current configuration") + except FileNotFoundError: logger.error("Providers file not found: %s", config.UMP_PROVIDERS_FILE) except yaml.YAMLError as e: logger.error("Failed to parse providers file: %s", e) except ValidationError as e: logger.error("Validation error in providers file: %s", e) + except Exception as e: + logger.error("Unexpected error loading providers: %s", e) + + def _atomic_update(self, new_providers: ModelServers): + """Atomically update providers with rollback capability""" + global PROVIDERS + + with PROVIDERS_LOCK: + # Store old providers for potential rollback + old_providers = PROVIDERS + try: + # Create a new dict with copied Pydantic models + PROVIDERS = { + name: provider.model_copy(deep=True) + for name, provider in new_providers.items() + } + except Exception as e: + PROVIDERS = old_providers + raise # Initialize the ProviderLoader and load providers initially @@ -55,57 +121,104 @@ def load_providers(self): observer = PollingObserver() observer.schedule( provider_loader, - config.UMP_PROVIDERS_FILE.absolute().as_posix(), # Simplified path handling - recursive=False, + config.UMP_PROVIDERS_FILE.parent.as_posix(), # Watch directory, not file + recursive=False ) observer.start() +def cleanup(): + """Cleanup function for graceful shutdown""" + global RELOAD_TIMER + + if RELOAD_TIMER: + RELOAD_TIMER.cancel() + + observer.stop() + observer.join(timeout=5) # Give it 5 seconds to stop gracefully + # Graceful shutdown for observer -atexit.register(observer.stop) +atexit.register(cleanup) def get_providers() -> ModelServers: - return PROVIDERS + """Get a copy of current providers (thread-safe and immutable)""" + with PROVIDERS_LOCK: + return PROVIDERS.copy() + + +def get_provider(provider_name: str) -> Optional[ProviderConfig]: + """Get a specific provider by name (thread-safe)""" + with PROVIDERS_LOCK: + return PROVIDERS.get(provider_name) def authenticate_provider(provider: ProviderConfig): + """Create authentication object for a provider""" auth = None if provider.authentication: auth = aiohttp.BasicAuth( - provider.authentication.user, - provider.authentication.password.get_secret_value(), + provider.authentication.user, + provider.authentication.password.get_secret_value() ) return auth -def load_process_config( - provider: str, process_id: str -) -> ProcessConfig: - - with PROVIDERS_LOCK: # Ensure thread-safe access - try: - processes = PROVIDERS[provider].processes +def check_process_availability(provider: str, process_id: str) -> bool: + """Check if a process is available and not excluded""" + with PROVIDERS_LOCK: + if ( + provider in PROVIDERS and + process_id in PROVIDERS[provider].processes + ): + process: ProcessConfig = PROVIDERS[provider].processes[process_id] + available = not process.exclude - # load process configuration - process_config: ProcessConfig = processes[process_id] - - if process_config.exclude: + if process.exclude: logger.debug("Excluding process %s based on configuration", process_id) - except KeyError: - logger.error( - "Provider '%s' or process '%s' not found in PROVIDERS", - provider, - process_id, - ) - raise ValueError( - f"Provider '{provider}' or process '{process_id}' not found" - ) - - return process_config - - -def check_result_storage(provider, process_id): - with PROVIDERS_LOCK: # Ensure thread-safe access - if provider in PROVIDERS and process_id in PROVIDERS[provider].processes: + + return available + + return False + + +def check_result_storage(provider: str, process_id: str) -> Optional[str]: + """Get the result storage type for a process""" + with PROVIDERS_LOCK: + if ( + provider in PROVIDERS + and process_id in PROVIDERS[provider].processes + ): return PROVIDERS[provider].processes[process_id].result_storage return None + + +def get_process_config(provider: str, process_id: str) -> Optional[ProcessConfig]: + """Get complete process configuration""" + with PROVIDERS_LOCK: + if ( + provider in PROVIDERS + and process_id in PROVIDERS[provider].processes + ): + return PROVIDERS[provider].processes[process_id] + return None + + +def list_providers() -> list[str]: + """Get list of all provider names""" + with PROVIDERS_LOCK: + return list(PROVIDERS.keys()) + + +def list_processes(provider: str) -> list[str]: + """Get list of all process IDs for a provider""" + with PROVIDERS_LOCK: + if provider in PROVIDERS: + return list(PROVIDERS[provider].processes.keys()) + return [] + + +# Health check function +def is_healthy() -> bool: + """Check if the provider loader is healthy""" + with PROVIDERS_LOCK: + return len(PROVIDERS) > 0 and observer.is_alive() \ No newline at end of file From 31c7840c8a9cf91e54155f23a32faf2762b46477 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 15 Jul 2025 15:23:21 +0000 Subject: [PATCH 105/145] raise ValueError when provider or process not found --- src/ump/api/providers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ump/api/providers.py b/src/ump/api/providers.py index 5b61ac9..2e03759 100644 --- a/src/ump/api/providers.py +++ b/src/ump/api/providers.py @@ -200,7 +200,10 @@ def get_process_config(provider: str, process_id: str) -> Optional[ProcessConfig and process_id in PROVIDERS[provider].processes ): return PROVIDERS[provider].processes[process_id] - return None + + raise ValueError( + f"Process '{process_id}' not found for provider '{provider}'" + ) def list_providers() -> list[str]: From 927a52956d5cec5c66a05e5416ff1cb14947e52f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 15 Jul 2025 15:23:39 +0000 Subject: [PATCH 106/145] "get_" instead of "load_" --- src/ump/api/models/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 281633f..339fa57 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -75,7 +75,7 @@ def __init__(self, process_id_with_prefix): # for what purpose? -> security! -if a user selects a process which is not # confiured to be available, but exists try: - process_config = providers.load_process_config( + process_config = providers.get_process_config( self.provider_prefix, self.process_id ) except ValueError as e: From ebae632af90ac2c7b86ab0eb9610b1f79d0fd9c1 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 08:33:35 +0000 Subject: [PATCH 107/145] fix: guarantee that process.extend receives only real results, not exceptions --- src/ump/api/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 06a87d8..7b0cd31 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -47,7 +47,7 @@ async def load_processes(): # Process results for result in results: - if isinstance(result, Exception): + if isinstance(result, BaseException): logger.error("Error fetching processes: %s", result) else: processes.extend(result) From a3a3c7b34eb75ffdb8c8006d69d0765150b3e3e8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 13:56:21 +0000 Subject: [PATCH 108/145] added new UMP_API_SERVER_URL_PREFIX to control gunicorn --timeout setting --- .env.example | 2 +- scripts/entrypoint.sh | 3 +++ src/ump/config.py | 10 ++++++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 376d8da..75aa41e 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ #---- App settings ---- # The API_SERVER_URL is only used to return the complete URL in the result of the job details as specified in OGC. # Should be the base url to the api. - +UMP_SERVER_TIMEOUT=30 UMP_LOG_LEVEL=DEBUG UMP_PROVIDERS_FILE=providers.yaml UMP_API_SERVER_URL=localhost:5000 diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index e092f30..26eea13 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,6 +1,9 @@ #!/bin/bash set -e + +export UMP_SERVER_TIMEOUT="${UMP_SERVER_TIMEOUT:-30}" + flask db upgrade echo "Running API Server in production mode." diff --git a/src/ump/config.py b/src/ump/config.py index d1a5f64..fdddc53 100644 --- a/src/ump/config.py +++ b/src/ump/config.py @@ -6,6 +6,8 @@ from rich import print logger = logging.getLogger(__name__) + + # using pydantic_settings to manage environment variables # and do automatic type casting in a central place class UmpSettings(BaseSettings): @@ -28,8 +30,8 @@ class UmpSettings(BaseSettings): UMP_GEOSERVER_WORKSPACE_NAME: str = "UMP" UMP_GEOSERVER_USER: str = "geoserver" UMP_GEOSERVER_PASSWORD: SecretStr = SecretStr("geoserver") - UMP_GEOSERVER_CONNECTION_TIMEOUT: int = 60 # seconds - UMP_JOB_DELETE_INTERVAL: int = 240 # minutes + UMP_GEOSERVER_CONNECTION_TIMEOUT: int = 60 # seconds + UMP_JOB_DELETE_INTERVAL: int = 240 # minutes UMP_KEYCLOAK_URL: HttpUrl = HttpUrl("http://keycloak:8080/auth") UMP_KEYCLOAK_REALM: str = "UrbanModelPlatform" UMP_KEYCLOAK_CLIENT_ID: str = "ump-client" @@ -37,6 +39,9 @@ class UmpSettings(BaseSettings): UMP_KEYCLOAK_PASSWORD: SecretStr = SecretStr("admin") UMP_API_SERVER_URL_PREFIX: str = "/" + # Gunicorn default timeout is 30 seconds + UMP_SERVER_TIMEOUT: int = 30 + @computed_field @property def UMP_GEOSERVER_URL_REST(self) -> HttpUrl: @@ -61,5 +66,6 @@ def ensure_trailing_slash(cls, value: str) -> str: value += "/" return value + app_settings = UmpSettings() app_settings.print_settings() From 37d3c7cd101e8146b200017e0a3c1ccd6ad4c55a Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 13:57:23 +0000 Subject: [PATCH 109/145] fetch_json error handling and flow improved --- src/ump/utils.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ump/utils.py b/src/ump/utils.py index 8bfc22e..4c9e34f 100644 --- a/src/ump/utils.py +++ b/src/ump/utils.py @@ -43,12 +43,8 @@ async def fetch_json( try: async with session.get(url, **kwargs) as response: try: + # is the response JSON? response_data = await response.json() - - if raise_for_status: - response.raise_for_status() - - return response_data except aiohttp.ContentTypeError: text = await response.text() @@ -68,6 +64,14 @@ async def fetch_json( instance=None ) ) + + # is the response ok? + if raise_for_status: + response.raise_for_status() + + # if all went good, go! + return response_data + except asyncio.TimeoutError: logger.error( "Timeout when requesting remote service. URL: %s", From 5c77444847cd5be31ccd5fc9e45042da4e277d18 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:02:50 +0000 Subject: [PATCH 110/145] added a log message in case server response does not contain "processes" --- src/ump/api/processes.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 7b0cd31..f3cc16b 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -86,9 +86,18 @@ async def fetch_provider_processes( ): process["id"] = f"{provider_name}:{process_id}" provider_processes.append(process) + + logger.error( + "The response from the remote service was not valid. " + "URL: %s, Content: %s", + provider_config.server_url, + results + ) - except aiohttp.ClientError as e: + # Note: fetch_json raises OGCProcessException on errors + except OGCProcessException as e: logger.error("HTTP error while accessing provider %s: %s", provider_name, e) + except Exception as e: logger.error("Unexpected error while processing provider %s: %s", provider_name, e) traceback.print_exc() From ae1f21e12d57ad4428529c40efb0bddcd2ff8f71 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:05:01 +0000 Subject: [PATCH 111/145] added type hints --- src/ump/api/processes.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index f3cc16b..77ceebf 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -3,24 +3,29 @@ from logging import getLogger import aiohttp -from aiohttp import ClientTimeout +from aiohttp import ClientSession, ClientTimeout from flask import g -from ump.api.models.providers_config import ProcessConfig +from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.api.providers import ( authenticate_provider, get_providers, ) +from ump.errors import OGCProcessException +from ump.utils import fetch_json logger = getLogger(__name__) #TODO: add validation of loaded processes through pydantic model or existing Process class async def load_processes(): processes = [] + auth = g.get("auth_token", {}) or {} - realm_roles = auth.get("realm_access", {}).get("roles", []) + + realm_roles: list = auth.get("realm_access", {}).get("roles", []) + # TODO: another hard-coded one: "ump-client" - client_roles = ( + client_roles: list = ( auth.get("resource_access", {}).get("ump-client", {}).get("roles", []) ) @@ -56,8 +61,9 @@ async def load_processes(): async def fetch_provider_processes( - session, provider_name, provider_config, - realm_roles, client_roles + session: ClientSession, + provider_name: str, provider_config: ProviderConfig, + realm_roles: list, client_roles: list ): """Fetch processes for a specific provider and filter them.""" provider_processes = [] From bf9a814d1b6f0b1eeca54622993db1972d3c3f1b Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:06:32 +0000 Subject: [PATCH 112/145] raise_for_status must be false on clientsession to get a response in case of statusses >=4xx --- src/ump/api/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 77ceebf..73ca12a 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -37,7 +37,7 @@ async def load_processes(): ) # remote server needs to answer in time, because we make multiple requests! async with aiohttp.ClientSession( - raise_for_status=True, timeout=client_timeout + raise_for_status=False, timeout=client_timeout ) as session: # Create a list of tasks for fetching processes concurrently tasks = [ From f9a3828eba8fb32eedcd11a7b3c3b9139c2c48e1 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:07:45 +0000 Subject: [PATCH 113/145] using fetch_json utility function here, too to make sure all possible exceptions are handled when querying remopte servers for processes --- src/ump/api/processes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 73ca12a..50f2a41 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -70,8 +70,12 @@ async def fetch_provider_processes( try: provider_auth = authenticate_provider(provider_config) - results = await fetch_processes_from_provider( - session, provider_config, provider_auth + results = await fetch_json( + session=session, + url=f"{provider_config.server_url}", + raise_for_status=True, + headers={"Content-type": "application/json", "Accept": "application/json"}, + auth=provider_auth ) if "processes" in results: From 314720b029a12aade376478f8088e27d50eab8eb Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:08:18 +0000 Subject: [PATCH 114/145] formatting, note added --- src/ump/api/processes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 50f2a41..ee23359 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -42,7 +42,8 @@ async def load_processes(): # Create a list of tasks for fetching processes concurrently tasks = [ fetch_provider_processes( - session, provider_name, provider_config, realm_roles, client_roles + session, provider_name, + provider_config, realm_roles, client_roles ) for provider_name, provider_config in get_providers().items() ] @@ -78,6 +79,8 @@ async def fetch_provider_processes( auth=provider_auth ) + # TODO: instead of manually checking for a key, we should validate the response + # using a pydantic model or json schema! if "processes" in results: for process in results["processes"]: process_id = process["id"] From 8eae699b8b476b4a03f8a772f37c733e10489e4f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:08:57 +0000 Subject: [PATCH 115/145] log message for grants improved --- src/ump/api/processes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index ee23359..62f008e 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -172,7 +172,8 @@ def is_process_visible( # Log the specific condition(s) that grant access if process_config.anonymous_access: logger.info( - "Granting access for process %s: Anonymous access is allowed.", + "Granting access for process %s:%s: Anonymous access is allowed.", + provider_name, process_id ) From 53db8fa17d516e2a96747d0bffcfbb6cb8ad8280 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:09:16 +0000 Subject: [PATCH 116/145] renamed is_process_visible to has_user_access_rights --- src/ump/api/processes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 62f008e..6df0562 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -93,7 +93,7 @@ async def fetch_provider_processes( continue process_config = provider_config.processes[process_id] - if is_process_visible( + if has_user_access_rights( process_id, provider_name, process_config, realm_roles, client_roles ): @@ -138,7 +138,7 @@ async def fetch_processes_from_provider(session, provider_config, provider_auth) ) raise -def is_process_visible( +def has_user_access_rights( process_id: str, provider_name: str, process_config: ProcessConfig, From 181245916d68142e629b7971a2ed0aa2be5ae6f6 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:10:01 +0000 Subject: [PATCH 117/145] log the error, too --- src/ump/api/models/process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index 339fa57..ba10347 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -79,6 +79,8 @@ def __init__(self, process_id_with_prefix): self.provider_prefix, self.process_id ) except ValueError as e: + logger.error(e) + raise OGCProcessException( OGCExceptionResponse( type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", From af0332b80a2eb8c70ebcdef1399bb27fe615d09f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:19:59 +0000 Subject: [PATCH 118/145] version update: 2.1.0rc4 -> 2.1.0rc5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4c93c91..890bc72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc3+fix-error-handling" +version = "2.1.0rc5+fix-remote-process-timeout" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From ae1229cb44634cebeb3f78d71bfc9966a1314c6f Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:35:37 +0000 Subject: [PATCH 119/145] added missing "processes" endpoint --- src/ump/api/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 6df0562..5ffab0f 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -73,7 +73,7 @@ async def fetch_provider_processes( results = await fetch_json( session=session, - url=f"{provider_config.server_url}", + url=f"{provider_config.server_url}processes", raise_for_status=True, headers={"Content-type": "application/json", "Accept": "application/json"}, auth=provider_auth From 039ea3fe4c03f7976d1e7334298f4f58e96d27a7 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 23 Jul 2025 14:38:16 +0000 Subject: [PATCH 120/145] added field validator to ensure server_url ends with a "/" --- src/ump/api/models/providers_config.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ump/api/models/providers_config.py b/src/ump/api/models/providers_config.py index e47c874..f4bbb21 100644 --- a/src/ump/api/models/providers_config.py +++ b/src/ump/api/models/providers_config.py @@ -1,6 +1,9 @@ from typing import Annotated, Literal, TypeAlias -from pydantic import BaseModel, Field, HttpUrl, SecretStr, TypeAdapter, model_validator +from pydantic import ( + BaseModel, Field, HttpUrl, SecretStr, TypeAdapter, + field_validator, model_validator +) # a type alias to give context to an otherwise generic str ProviderName: TypeAlias = Annotated[str, Field( @@ -120,6 +123,14 @@ class ProviderConfig(BaseModel): ) ) + @field_validator("server_url", mode="before") + def ensure_trailing_slash(cls, value: str) -> HttpUrl: + """Ensure server_url has a trailing slash.""" + + if not str(value).endswith("/"): + value += "/" + return HttpUrl(value) + # a TypeAlias to give context to an otherwise generic dict ModelServers: TypeAlias = Annotated[ dict[str, ProviderConfig], From 72dbefa2714c78b5ab70e63a25a69da445a25113 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 06:32:47 +0000 Subject: [PATCH 121/145] samll fix of missing parenthesis, errorhandler deteails adjusted --- src/ump/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ump/main.py b/src/ump/main.py index 3ab83d0..22c7bb0 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -4,6 +4,7 @@ # - logging setup # - a geoserver cleanup runner # - the flask app +# - the token verification import atexit import json import os @@ -87,7 +88,7 @@ def cleanup(): + f"/layers/{job_id}.xml", auth=( config.UMP_GEOSERVER_USER, - config.UMP_GEOSERVER_PASSWORD.get_secret_value, + config.UMP_GEOSERVER_PASSWORD.get_secret_value(), ), timeout=config.UMP_GEOSERVER_CONNECTION_TIMEOUT, ) @@ -152,9 +153,12 @@ def cleanup(): @app.errorhandler(OGCProcessException) def handle_ogc_exception(error: OGCProcessException): - response = jsonify(error.response.model_dump()) + response = jsonify(error.response.model_dump(exclude_unset=True)) response.status_code = error.response.status response.content_type = "application/problem+json" + + if response.status_code in (401, 403): + response.headers["WWW-Authenticate"] = 'Bearer' return response From 56ef58688aab4feaa17138f57bbc18f23e30a040 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 06:34:06 +0000 Subject: [PATCH 122/145] added note --- src/ump/api/processes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 5ffab0f..635d7b7 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -40,6 +40,8 @@ async def load_processes(): raise_for_status=False, timeout=client_timeout ) as session: # Create a list of tasks for fetching processes concurrently + #TODO: it would make more sense if, not all processes are fetched, + # but only those that are configured and are accessible by the user tasks = [ fetch_provider_processes( session, provider_name, From b1b63c8140ff914ab6db60a3e7a3e81b3810cdbf Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 06:34:19 +0000 Subject: [PATCH 123/145] fixed return type --- src/ump/api/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/providers.py b/src/ump/api/providers.py index 2e03759..d5441a3 100644 --- a/src/ump/api/providers.py +++ b/src/ump/api/providers.py @@ -192,7 +192,7 @@ def check_result_storage(provider: str, process_id: str) -> Optional[str]: return None -def get_process_config(provider: str, process_id: str) -> Optional[ProcessConfig]: +def get_process_config(provider: str, process_id: str) -> ProcessConfig: """Get complete process configuration""" with PROVIDERS_LOCK: if ( From 4242b29af8afee0ca536efc362bc3e2afc723753 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 06:35:39 +0000 Subject: [PATCH 124/145] improved error response and distinguish between "Unauthorized" and "Forbidden" --- src/ump/api/models/process.py | 51 ++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index ba10347..f7821e0 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -72,8 +72,8 @@ def __init__(self, process_id_with_prefix): self.process_id = match.group(2) # this checks if the process is known from providers and configured as available - # for what purpose? -> security! -if a user selects a process which is not - # confiured to be available, but exists + # for what purpose? -> security! -if a user selects a remote process which is + # not (!) explicitly configured to be available, but exists on the remote server try: process_config = providers.get_process_config( self.provider_prefix, self.process_id @@ -96,8 +96,7 @@ def __init__(self, process_id_with_prefix): ) if process_config.exclude: logger.warning( - "Process ID '%s' is not available. " - "Either the process is not configured or it is excluded from this API.", + "Process ID '%s' is explicitly excluded. ", self.process_id_with_prefix, ) # raise OGCProcessException to inform users, but with less detail @@ -119,14 +118,35 @@ def __init__(self, process_id_with_prefix): ) # if anonymous access isn’t enabled, require a token - if process_config and not process_config.anonymous_access: + if not process_config.anonymous_access: logger.debug( "Process %s requires authentication. Checking user roles.", self.process_id_with_prefix, ) + # check if token was provided by the user auth = getattr(g, "auth_token", None) - role = f"{self.provider_prefix}_{self.process_id}" + + if not auth: + logger.warning( + "User is not allowed to access process %s. " + "The request lacks necessary authentication details!", + self.process_id_with_prefix, + ) + raise OGCProcessException( + OGCExceptionResponse( + type="about:blank", + title="Unauthorized.", + status=401, + detail=( + "The request lacks necessary authentication details." + ), + instance=f"/processes/{self.process_id_with_prefix}", + ) + ) + + role_name = f"{self.provider_prefix}_{self.process_id}" + realm_roles = auth.get("realm_access", {}).get("roles", []) if auth else [] # TODO: uses hard-coded client name 'ump-client' to get client roles @@ -137,24 +157,25 @@ def __init__(self, process_id_with_prefix): ) allowed = any( - r in realm_roles + client_roles for r in [self.provider_prefix, role] + r in realm_roles + client_roles for r in [self.provider_prefix, role_name] ) - if not auth or not allowed: + if not allowed: logger.warning( "User is not allowed to access process %s. " - "Either the process is not configured for anonymous access or " - "the user does not have the required roles.", + "User roles: %s, required roles: %s", self.process_id_with_prefix, + realm_roles + client_roles, + [self.provider_prefix, role_name], ) raise OGCProcessException( OGCExceptionResponse( - type="http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", - title="Not available.", - status=400, + type="about:blank", + title="Forbidden.", + status=403, detail=( - f"Process ID '{self.process_id_with_prefix}' " - "is not available." + f"User is not allowed to access process " + f"'{self.process_id_with_prefix}'" ), instance=f"/processes/{self.process_id_with_prefix}", ) From f7352aa84a1d30d03769b608e4e158add7c508b5 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 06:37:14 +0000 Subject: [PATCH 125/145] updated: 2.1.0rc5 -> 2.1.0rc6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 890bc72..23147cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc5+fix-remote-process-timeout" +version = "2.1.0rc6+fix-unauthorized-response" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From f736da8fc040fc4291c7efc46a54c10afe469027 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 07:24:56 +0000 Subject: [PATCH 126/145] added note --- src/ump/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ump/main.py b/src/ump/main.py index 22c7bb0..f0a3b18 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -169,6 +169,8 @@ def check_jwt(): schedule.run_pending() auth = request.authorization + # TODO: this needs to be improved, it should not fail the app + # and error messages must adopt OGCProcessException if auth is not None: # need exception handling here to avoid app failure! try: From 188654155aee052c5bc56cd74e6e9db54b1dc3ab Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 08:29:13 +0000 Subject: [PATCH 127/145] fix: error message was wrongly placed --- src/ump/api/processes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 635d7b7..2fde3ac 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -101,13 +101,13 @@ async def fetch_provider_processes( ): process["id"] = f"{provider_name}:{process_id}" provider_processes.append(process) - - logger.error( - "The response from the remote service was not valid. " - "URL: %s, Content: %s", - provider_config.server_url, - results - ) + else: + logger.error( + "The response from the remote service was not valid. " + "URL: %s, Content: %s", + provider_config.server_url, + results + ) # Note: fetch_json raises OGCProcessException on errors except OGCProcessException as e: From c3e0fe911ffd77636c4a94648b678ac7af47aad8 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 08:29:36 +0000 Subject: [PATCH 128/145] added bug note --- src/ump/api/models/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index f7821e0..cc5dfa6 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -301,7 +301,7 @@ def check_for_cache(self, parameters, user_id): if process_config.deterministic: return None - + # BUG: this fails because of implicit conversion of json to text sql = """ select job_id from jobs where hash = encode( sha512( From 00d40e324450d1347fd3238a7016a8e76f00d43e Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 09:20:35 +0000 Subject: [PATCH 129/145] version update: 2.1.0rc6 -> 2.1.0rc7 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 23147cb..194d9c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc6+fix-unauthorized-response" +version = "2.1.0rc7+fix-unauthorized-response" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From 13d702c6b6de78037e966727cb83a88a2a96ca68 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 12:39:30 +0000 Subject: [PATCH 130/145] added process.py logger --- src/ump/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ump/main.py b/src/ump/main.py index f0a3b18..03fd85a 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -59,6 +59,11 @@ "handlers": ["wsgi"], "propagate": False, }, + "ump.api.models.process": { # Configure the logger for processes.py + "level": config.UMP_LOG_LEVEL, + "handlers": ["wsgi"], + "propagate": False, + }, }, } ) From 45b807386e9ba008fc98791c4f4800c3d19705a2 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 12:44:58 +0000 Subject: [PATCH 131/145] fixed the hash algorithm (it failed when user_id) was actually set --- src/ump/api/models/job.py | 41 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index e36b8c5..f8e2379 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -116,15 +116,52 @@ def insert( process_version=process_version, ) + # TODO: these metadata should come from remote job + # instead of being set here + # because the remote server ultimately decides if a job was accepted! self.status = JobStatus.accepted.value self.created = datetime.now(timezone.utc) self.updated = datetime.now(timezone.utc) + # TODO: we need proper normalization here + # TODO: we need a SQL model for this, raw queries are error prone query = """ INSERT INTO jobs - (job_id, remote_job_id, process_id, provider_prefix, provider_url, status, progress, parameters, message, created, started, finished, updated, user_id, process_title, name, process_version, hash) + ( + job_id, remote_job_id, process_id, + provider_prefix, provider_url, status, + progress, parameters, message, created, + started, finished, updated, user_id, + process_title, name, process_version, hash + ) VALUES - (%(job_id)s, %(remote_job_id)s, %(process_id)s, %(provider_prefix)s, %(provider_url)s, %(status)s, %(progress)s, %(parameters)s, %(message)s, %(created)s, %(started)s, %(finished)s, %(updated)s, %(user_id)s, %(process_title)s, %(name)s, %(process_version)s, encode(sha512((%(parameters)s :: json :: text || %(process_version)s || %(user_id)s) :: bytea), 'base64')) + ( + %(job_id)s, + %(remote_job_id)s, + %(process_id)s, + %(provider_prefix)s, + %(provider_url)s, + %(status)s, + %(progress)s, + %(parameters)s, + %(message)s, + %(created)s, + %(started)s, + %(finished)s, + %(updated)s, + %(user_id)s, + %(process_title)s, + %(name)s, + %(process_version)s, + encode( + sha512( + convert_to( + %(parameters)s :: json :: text || %(process_version)s || %(user_id)s, + 'UTF8' + ) :: bytea + ), 'base64' + ) + ) """ with DBHandler() as db: db.run_query(query, query_params=self._to_dict()) From e4d392cbbc6290eb6a1bb078edef95e1d52b3982 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 12:45:38 +0000 Subject: [PATCH 132/145] added a note --- src/ump/api/routes/processes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ump/api/routes/processes.py b/src/ump/api/routes/processes.py index 0ecbce9..240c65f 100644 --- a/src/ump/api/routes/processes.py +++ b/src/ump/api/routes/processes.py @@ -27,6 +27,8 @@ def show(process_id_with_prefix=None): def execute(process_id_with_prefix=None): auth = g.get('auth_token') process = Process(process_id_with_prefix) + + # extract unique user ID ('sub') from auth token if available result = process.execute(request.json, None if auth is None else auth['sub']) return Response(json.dumps(result), status=201, mimetype="application/json") From 9ff5e4bc17d19e6960c50fa3ca3e04058b7d2b32 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 12:47:59 +0000 Subject: [PATCH 133/145] fixed a problem when deterministic is false AND user_id is set --- src/ump/api/models/process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index cc5dfa6..ee9fe5a 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -305,8 +305,9 @@ def check_for_cache(self, parameters, user_id): sql = """ select job_id from jobs where hash = encode( sha512( - ( - %(parameters)s :: json :: text || %(process_version)s || %(user_id)s) :: bytea + convert_to( + %(parameters)s :: json :: text || %(process_version)s || %(user_id)s) :: bytea, + 'UTF8' ), 'base64' ) From 5020a7d0e530b088baf06ef2241771be71fbfba1 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 13:01:54 +0000 Subject: [PATCH 134/145] version update: 2.1.0rc8 -> 2.1.0rc9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 194d9c1..96ffd46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc7+fix-unauthorized-response" +version = "2.1.0rc8+fix-insert-when-logged-in" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From 6391deca98e7208d2039ccc63d5cbb90d476c508 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 13:27:26 +0000 Subject: [PATCH 135/145] fixed a an order typo --- src/ump/api/models/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index ee9fe5a..e581c46 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -306,9 +306,10 @@ def check_for_cache(self, parameters, user_id): select job_id from jobs where hash = encode( sha512( convert_to( - %(parameters)s :: json :: text || %(process_version)s || %(user_id)s) :: bytea, + %(parameters)s :: json :: text || %(process_version)s || %(user_id)s, 'UTF8' ), + ), 'base64' ) """ From df4b396e53420abeebc5a59e6b5be14acdfe7a53 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Tue, 29 Jul 2025 13:28:25 +0000 Subject: [PATCH 136/145] removed bug note, because the quick fix handled it --- src/ump/api/models/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index e581c46..038eefa 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -301,7 +301,7 @@ def check_for_cache(self, parameters, user_id): if process_config.deterministic: return None - # BUG: this fails because of implicit conversion of json to text + sql = """ select job_id from jobs where hash = encode( sha512( From 9bb1e7e83a560f836faf2e9fa97e679232036432 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 30 Jul 2025 12:20:13 +0000 Subject: [PATCH 137/145] using setting UMP_KEYCLOAK_CLIENT_ID instead of hard-coded "ump-client" --- src/ump/api/models/process.py | 12 ++++++++++-- src/ump/api/processes.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ump/api/models/process.py b/src/ump/api/models/process.py index ba10347..ef696b6 100644 --- a/src/ump/api/models/process.py +++ b/src/ump/api/models/process.py @@ -9,6 +9,8 @@ from flask import g import ump.api.providers as providers + +from ump.config import app_settings from ump.api.db_handler import engine from ump.api.models.job import Job, JobStatus from ump.api.models.ogc_exception import OGCExceptionResponse @@ -129,9 +131,15 @@ def __init__(self, process_id_with_prefix): role = f"{self.provider_prefix}_{self.process_id}" realm_roles = auth.get("realm_access", {}).get("roles", []) if auth else [] - # TODO: uses hard-coded client name 'ump-client' to get client roles client_roles = ( - (auth.get("resource_access", {}).get("ump-client", {}).get("roles", [])) + ( + auth.get( + "resource_access", {} + ).get( + app_settings.UMP_KEYCLOAK_CLIENT_ID, {} + ).get( + "roles", []) + ) if auth else [] ) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 5ffab0f..41d0324 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -6,6 +6,7 @@ from aiohttp import ClientSession, ClientTimeout from flask import g +from ump.config import app_settings from ump.api.models.providers_config import ProcessConfig, ProviderConfig from ump.api.providers import ( authenticate_provider, @@ -24,9 +25,14 @@ async def load_processes(): realm_roles: list = auth.get("realm_access", {}).get("roles", []) - # TODO: another hard-coded one: "ump-client" client_roles: list = ( - auth.get("resource_access", {}).get("ump-client", {}).get("roles", []) + auth.get( + "resource_access", {} + ).get( + app_settings.UMP_KEYCLOAK_CLIENT_ID, {} + ).get( + "roles", [] + ) ) client_timeout = ClientTimeout( From a8c2e413b8a9f1a9de87801c1cb531ee49937aee Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 30 Jul 2025 13:05:31 +0000 Subject: [PATCH 138/145] added notes --- src/ump/api/processes.py | 1 + src/ump/main.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ump/api/processes.py b/src/ump/api/processes.py index 41d0324..d1aebb8 100644 --- a/src/ump/api/processes.py +++ b/src/ump/api/processes.py @@ -23,6 +23,7 @@ async def load_processes(): auth = g.get("auth_token", {}) or {} + # TODO manually parsing jwt is not recommended, use a library like PyJWT or better Authlib realm_roles: list = auth.get("realm_access", {}).get("roles", []) client_roles: list = ( diff --git a/src/ump/main.py b/src/ump/main.py index 3ab83d0..d36af4f 100644 --- a/src/ump/main.py +++ b/src/ump/main.py @@ -170,6 +170,7 @@ def check_jwt(): try: # TODO: generally token verification is done offline, # but decode_token connects to keycloak on every request (for retrieving keys) + # use a library like PyJWT or better Authlib decoded = keycloak_openid.decode_token(auth.token) except KeycloakGetError as e: app.logger.error(e) From e34625476829670857b1c3c2b593b762657de0db Mon Sep 17 00:00:00 2001 From: herzogrh Date: Wed, 30 Jul 2025 16:45:30 +0200 Subject: [PATCH 139/145] Correct deploy docs update --- .github/workflows/deploy-docs.yml | 60 ------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 .github/workflows/deploy-docs.yml diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml deleted file mode 100644 index bd63966..0000000 --- a/.github/workflows/deploy-docs.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: deploy-book - -# Run this when the master or main branch changes -on: - push: - branches: - - master - - main - # If your git repository has the Jupyter Book within some-subfolder next to - # unrelated files, you can make this run only if a file within that specific - # folder has been modified. - # - # paths: - # - some-subfolder/** - -# This job installs dependencies, builds the book, and pushes it to `gh-pages` -jobs: - deploy-book: - runs-on: ubuntu-latest - permissions: - pages: write - id-token: write - steps: - - uses: actions/checkout@v4 - - # Install dependencies - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: pip # Implicitly uses requirements.txt for cache key - - - name: Install dependencies - run: pip install -r requirements.txt - - # (optional) Cache your executed notebooks between runs - # if you have config: - # execute: - # execute_notebooks: cache - - name: cache executed notebooks - uses: actions/cache@v4 - with: - path: _build/.jupyter_cache - key: jupyter-book-cache-${{ hashFiles('requirements.txt') }} - - # Build the book - - name: Build the book - run: | - jupyter-book build . - - # Upload the book's HTML as an artifact - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: "_build/html" - - # Deploy the book's HTML to GitHub Pages - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 From 22e10b3adc0a59030796e5b6d2b1e0f9633c1f18 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 31 Jul 2025 13:48:42 +0000 Subject: [PATCH 140/145] changelog updated --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c845e..440ee01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.1.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +# [2.x] +## [2.1.0] - 2025-07-31 +### Changed: +- improved error handling when requesting remote servers processes and jobs +- helm chart is up-to-date with current UMP +- provider loader listens on any event to be compatible with configmap-updates in k8s +- provider loader improved: debouncing rapid file changes and atomic updates of providers object +- improved server responses in certain situations, especially when something went wrong showing users json information (as this is a json api) in accordance with OGC api process spec +- improved job starting mechanism + +### Added: +- a new setting to control gunicorn worker timout was introduced: UMP_SERVER_TIMEOUT +- a new setting to control ump server path prefix introduced: UMP_API_SERVER_URL_PREFIX +- timeouts for all requests to remote servers + +### Fixed: +- using setting UMP_KEYCLOAK_CLIENT_ID instead of hard-coded "ump-client" +- job insert queries failed when logged-in user created a job +- missing job metadata +- fetch correct job status from remote server + +## [2.0.0] - 2025-06-25 + +### Added +- comprehensive documentation added +- unified database connection pool handling + +### Changed +- created a providers pydantic class for better type safety and concise handling +- improved provider.yaml loading and provider updateing mechanism +- improved logging +- keycloak coinnection error handling improved + +### Fixed +- ump ran out of database connections due to unclosed connections + +# [1.x] ## [1.2.0] - 2024-05-25 ### Added From e817dd1be7bd9c65e708698c5687c1f49d670470 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 31 Jul 2025 14:01:41 +0000 Subject: [PATCH 141/145] added UMP_SERVER_TIMEOUT setting --- charts/urban-model-platform/templates/configmap-settings.yaml | 3 ++- charts/urban-model-platform/values.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/urban-model-platform/templates/configmap-settings.yaml b/charts/urban-model-platform/templates/configmap-settings.yaml index be194ec..b505a90 100644 --- a/charts/urban-model-platform/templates/configmap-settings.yaml +++ b/charts/urban-model-platform/templates/configmap-settings.yaml @@ -11,4 +11,5 @@ data: UMP_API_SERVER_URL_PREFIX: {{ .Values.config.apiServerUrlPrefix | quote }} UMP_REMOTE_JOB_STATUS_REQUEST_INTERVAL: {{ .Values.config.remoteJobStatusRequestInterval | quote }} UMP_JOB_DELETE_INTERVAL: {{ .Values.config.jobDeleteInterval | quote }} - UMP_API_SERVER_WORKERS: {{ .Values.config.apiServerWorkers | quote }} \ No newline at end of file + UMP_API_SERVER_WORKERS: {{ .Values.config.apiServerWorkers | quote }} + UMP_SERVER_TIMEOUT: {{ .Values.config.serverWorkerTimeout | quote }} diff --git a/charts/urban-model-platform/values.yaml b/charts/urban-model-platform/values.yaml index 37ba243..5f8c2b2 100644 --- a/charts/urban-model-platform/values.yaml +++ b/charts/urban-model-platform/values.yaml @@ -47,6 +47,7 @@ config: apiServerUrlPrefix: "/api" apiServerWorkers: "4" remoteJobStatusRequestInterval: "5" + serverWorkerTimeout: "30" # Timeout for server workers, should be higher than all request to remote servers, to avoid server 500 errors geoserverConnectionTimeout: "60" jobDeleteInterval: "240" From a60401c39e4905f24fd77564a274973d32bc7ff3 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 31 Jul 2025 14:08:18 +0000 Subject: [PATCH 142/145] added new UMP_SERVER_TIMEOUT setting to compose --- docker-compose-dev.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index c9ce2f4..9251c1f 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -21,6 +21,7 @@ services: environment: UMP_API_SERVER_URL: localhost:${WEBAPP_PORT_EXTERNAL} UMP_API_SERVER_WORKERS: 1 + UMP_SERVER_TIMEOUT: 30 # seconds UMP_LOG_LEVEL: DEBUG UMP_PROVIDERS_FILE: /home/pythonuser/providers.yaml UMP_KEYCLOAK_USER: admin From 6ba1275bc54be1798bd1af7ec6250ca6a4385d49 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 31 Jul 2025 14:09:09 +0000 Subject: [PATCH 143/145] version update: 2.1.0rc8 -> 2.1.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f0180e0..361d5e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ exclude = [ [tool.poetry] name = "ump" -version = "2.1.0rc8+fix-insert-when-logged-in" +version = "2.1.0" description = "server federation api, OGC Api Processes-based to connect model servers and centralize access to them" authors = [ "Rico Herzog ", From 8919dea4cdae931a24c016ac931ca73121da6b09 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Thu, 31 Jul 2025 14:10:40 +0000 Subject: [PATCH 144/145] chart version update: 0.9.4 -> 0.9.5 --- charts/urban-model-platform/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/urban-model-platform/Chart.yaml b/charts/urban-model-platform/Chart.yaml index d66abf6..f38c991 100644 --- a/charts/urban-model-platform/Chart.yaml +++ b/charts/urban-model-platform/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.9.4 +version: 0.9.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.0.0" +appVersion: "2.1.0" kubeVersion: ">= 1.31.0" annotations: category: API From 2825cee4fd105a074c3c7e4ec43cf1fa6f009473 Mon Sep 17 00:00:00 2001 From: Stefan Schuhart Date: Wed, 6 Aug 2025 15:03:59 +0000 Subject: [PATCH 145/145] added note --- src/ump/api/models/job.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ump/api/models/job.py b/src/ump/api/models/job.py index f8e2379..20cd756 100644 --- a/src/ump/api/models/job.py +++ b/src/ump/api/models/job.py @@ -76,6 +76,9 @@ def __init__(self, job_id=None, user=None): self.process_id = None self.provider_url = None + # TODO: this produces 404 if a job is beeing queried for which was + # stored with a user id, consider to distinguish between + # 404, 401 and 403 here if job_id and not self._init_from_db(job_id, user): raise OGCProcessException( OGCExceptionResponse(