From 264723f78c5bffc98c8acc688ca229b144abf422 Mon Sep 17 00:00:00 2001 From: jland Date: Thu, 16 Jul 2026 10:45:39 -0400 Subject: [PATCH 1/2] Postgres db creds are refreshed every time not just inital reun since they may change --- .../Database/configure_maas_postgres.sh | 108 +++++++++++------- 1 file changed, 66 insertions(+), 42 deletions(-) diff --git a/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh b/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh index 73b5df77c..3229eff60 100755 --- a/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh +++ b/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh @@ -19,8 +19,8 @@ POSTGRES_IMAGE="registry.redhat.io/rhel9/postgresql-15@sha256:90ec347a35ab8a5d53 # Derive MaaS infrastructure namespace (matches MaaS controller logic from PR #1051). # When INFRA_NAMESPACE=AUTO (default since 3.5), the controller expects maas-db-config -# in a separate infra namespace. Postgres stays in APPS_NS; maas-db-config is copied -# to the infra namespace with a cross-namespace connection URL. +# in a separate infra namespace. Postgres stays in APPS_NS; maas-db-config is always +# refreshed from postgres-creds and applied to INFRA_NS (and APPS_NS when they differ). derive_infra_namespace() { case "$1" in redhat-ods-applications) echo "redhat-ai-gateway-infra" ;; @@ -59,6 +59,60 @@ detect_infra_namespace() { fi } +# Always (re)apply maas-db-config from current credentials so namespaces cannot drift. +apply_maas_db_config() { + local target_ns="$1" + local pg_host="$2" + local db_url="postgresql://${PG_USER}:${PG_PASS}@${pg_host}:5432/${PG_DB}?sslmode=disable" + + oc apply -f - </dev/null; then + PG_USER="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)" + PG_PASS="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)" + PG_DB="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)" + if [[ -z "${PG_USER}" || -z "${PG_PASS}" || -z "${PG_DB}" ]]; then + echo "postgres-creds in ${APPS_NS} is missing required keys/values" >&2 + exit 1 + fi + echo "Reusing existing postgres-creds in ${APPS_NS}" + else + PG_USER="maas-$(cat /dev/urandom | tr -dc a-z0-9 | head -c 8)" + PG_PASS="$(cat /dev/urandom | tr -dc A-Za-z0-9 | head -c 32)" + PG_DB="maas-$(cat /dev/urandom | tr -dc a-z0-9 | head -c 8)" + fi +} + INFRA_NS=$(detect_infra_namespace "${APPS_NS}") # Ensure namespaces exist @@ -67,31 +121,20 @@ if [[ "${INFRA_NS}" != "${APPS_NS}" ]]; then oc create namespace "${INFRA_NS}" --dry-run=client -o yaml | oc apply -f - fi -# Skip if all resources already exist and deployment is ready -if oc get secret maas-db-config -n "${INFRA_NS}" &>/dev/null \ - && oc get secret postgres-creds -n "${APPS_NS}" &>/dev/null \ +# If postgres is already provisioned, still refresh maas-db-config every run so +# INFRA_NS (and APPS_NS when separate) cannot keep a stale connection URL. +if oc get secret postgres-creds -n "${APPS_NS}" &>/dev/null \ && oc get service postgres -n "${APPS_NS}" &>/dev/null \ && oc get deployment postgres -n "${APPS_NS}" &>/dev/null; then oc wait deployment/postgres -n "${APPS_NS}" --for=condition=Available --timeout=5m - echo "MaaS PostgreSQL prerequisites already exist (postgres in ${APPS_NS}, maas-db-config in ${INFRA_NS}), skipping." + load_or_generate_postgres_creds + refresh_maas_db_config_secrets + echo "MaaS PostgreSQL already provisioned in ${APPS_NS}; refreshed maas-db-config in ${INFRA_NS}$([[ "${INFRA_NS}" != "${APPS_NS}" ]] && echo " and ${APPS_NS}")." exit 0 fi # Reuse existing credentials if postgres-creds secret is present, otherwise generate new ones -if oc get secret postgres-creds -n "${APPS_NS}" &>/dev/null; then - PG_USER="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)" - PG_PASS="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)" - PG_DB="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)" - if [[ -z "${PG_USER}" || -z "${PG_PASS}" || -z "${PG_DB}" ]]; then - echo "postgres-creds in ${APPS_NS} is missing required keys/values" >&2 - exit 1 - fi - echo "Reusing existing postgres-creds in ${APPS_NS}" -else - PG_USER="maas-$(cat /dev/urandom | tr -dc a-z0-9 | head -c 8)" - PG_PASS="$(cat /dev/urandom | tr -dc A-Za-z0-9 | head -c 32)" - PG_DB="maas-$(cat /dev/urandom | tr -dc a-z0-9 | head -c 8)" -fi +load_or_generate_postgres_creds # 1. postgres-creds secret oc apply -f - < Date: Fri, 17 Jul 2026 08:38:52 -0400 Subject: [PATCH 2/2] Bouncing maas-api pods post secret creation --- .../Database/configure_maas_postgres.sh | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh b/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh index 3229eff60..5c5488b7b 100755 --- a/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh +++ b/ods_ci/tasks/Resources/Database/configure_maas_postgres.sh @@ -31,7 +31,7 @@ derive_infra_namespace() { detect_infra_namespace() { local apps_ns="$1" - + # MaaS provisioning may run before the RHOAI operator has fully deployed # the maas-controller (CRD registration, reconciliation, and pod scheduling # can take 10+ minutes after the CSV is ready). Without waiting, detect @@ -96,6 +96,24 @@ refresh_maas_db_config_secrets() { fi } +# Restart maas-api so it reloads DB_CONNECTION_URL after secret refresh. +# No-op if the deployment is not present yet (e.g. first-time install before DSC). +restart_maas_api_in_ns() { + local ns="$1" + if oc get deployment maas-api -n "${ns}" &>/dev/null; then + echo "Restarting maas-api in ${ns} to pick up refreshed maas-db-config" + oc rollout restart deployment/maas-api -n "${ns}" + oc rollout status deployment/maas-api -n "${ns}" --timeout=5m + fi +} + +restart_maas_api_if_present() { + restart_maas_api_in_ns "${INFRA_NS}" + if [[ "${INFRA_NS}" != "${APPS_NS}" ]]; then + restart_maas_api_in_ns "${APPS_NS}" + fi +} + load_or_generate_postgres_creds() { if oc get secret postgres-creds -n "${APPS_NS}" &>/dev/null; then PG_USER="$(oc get secret postgres-creds -n "${APPS_NS}" -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)" @@ -129,6 +147,7 @@ if oc get secret postgres-creds -n "${APPS_NS}" &>/dev/null \ oc wait deployment/postgres -n "${APPS_NS}" --for=condition=Available --timeout=5m load_or_generate_postgres_creds refresh_maas_db_config_secrets + restart_maas_api_if_present echo "MaaS PostgreSQL already provisioned in ${APPS_NS}; refreshed maas-db-config in ${INFRA_NS}$([[ "${INFRA_NS}" != "${APPS_NS}" ]] && echo " and ${APPS_NS}")." exit 0 fi @@ -244,6 +263,8 @@ if ! oc wait deployment/postgres -n "${APPS_NS}" --for=condition=Available --tim exit 1 fi +restart_maas_api_if_present + if [[ "${INFRA_NS}" != "${APPS_NS}" ]]; then echo "MaaS PostgreSQL prerequisites provisioned (postgres in ${APPS_NS}, maas-db-config in ${INFRA_NS} and ${APPS_NS})" else