diff --git a/.github/workflows/run-pgaftest.yml b/.github/workflows/run-pgaftest.yml new file mode 100644 index 000000000..b6a9085c8 --- /dev/null +++ b/.github/workflows/run-pgaftest.yml @@ -0,0 +1,358 @@ +name: pgaftest + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +# One run per branch; cancel in-progress runs on new push. +concurrency: + group: pgaftest-${{ github.ref }} + cancel-in-progress: true + +jobs: + style_checker: + name: Style check + runs-on: ubuntu-latest + container: citus/stylechecker:no-py + steps: + - name: Checkout repository + uses: actions/checkout@v7.0.0 + + - name: Set safe directory for git + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Check C formatting + run: citus_indent --check + + - name: Check banned functions + run: ci/banned.h.sh + + # --------------------------------------------------------------------------- + # Build one pgaf:run image per Postgres version and upload each as an + # artifact. All downstream test jobs download only the version they need. + # --------------------------------------------------------------------------- + build-images: + name: Build image (PG${{ matrix.PGVERSION }}) + needs: style_checker + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + PGVERSION: [14, 15, 16, 17] + + steps: + - uses: actions/checkout@v7.0.0 + + - name: Generate git-version.h + run: make version + + - name: Build pgaf:run image (PG${{ matrix.PGVERSION }}) + run: | + # Match the CITUSTAG-per-PGVERSION logic from the top-level Makefile: + # PG14–15 require Citus v12 (last version supporting those releases). + # PG16+ use the current default. + case "${{ matrix.PGVERSION }}" in + 14|15) CITUSTAG=v12.1.5 ;; + *) CITUSTAG=v13.2.0 ;; + esac + docker build \ + --build-arg PGVERSION=${{ matrix.PGVERSION }} \ + --build-arg CITUSTAG=${CITUSTAG} \ + --target run \ + -t pgaf:run-pg${{ matrix.PGVERSION }} \ + . + + - name: Build pgaf:debian image (PG${{ matrix.PGVERSION }}) + run: | + # Layer cache from the run build above covers the run stage; + # this step only adds the pg_createcluster layer on top. + case "${{ matrix.PGVERSION }}" in + 14|15) CITUSTAG=v12.1.5 ;; + *) CITUSTAG=v13.2.0 ;; + esac + docker build \ + --build-arg PGVERSION=${{ matrix.PGVERSION }} \ + --build-arg CITUSTAG=${CITUSTAG} \ + --target debian \ + -t pgaf:debian-pg${{ matrix.PGVERSION }} \ + . + + - name: Save images to tarballs + run: | + docker save pgaf:run-pg${{ matrix.PGVERSION }} \ + | gzip > /tmp/pgaf-run-pg${{ matrix.PGVERSION }}.tar.gz + docker save pgaf:debian-pg${{ matrix.PGVERSION }} \ + | gzip > /tmp/pgaf-debian-pg${{ matrix.PGVERSION }}.tar.gz + + - name: Upload run image artifact + uses: actions/upload-artifact@v7.0.1 + with: + name: pgaf-run-image-pg${{ matrix.PGVERSION }} + path: /tmp/pgaf-run-pg${{ matrix.PGVERSION }}.tar.gz + retention-days: 1 + + - name: Upload debian image artifact + uses: actions/upload-artifact@v7.0.1 + with: + name: pgaf-debian-image-pg${{ matrix.PGVERSION }} + path: /tmp/pgaf-debian-pg${{ matrix.PGVERSION }}.tar.gz + retention-days: 1 + + # --------------------------------------------------------------------------- + # Build the pgaf:pgaftest image once from the PG17 build image. + # The Dockerfile pgaftest target bundles the pgaftest binary together with + # its runtime dependencies (libpq5 from PGDG, docker-ce-cli, and the + # docker-compose-plugin) so that test jobs run pgaftest inside this + # container via Docker-out-of-Docker rather than installing libpq on the + # bare runner. That eliminates the fragile apt-get dependency on the + # pre-installed Microsoft package repositories on GitHub's runner image. + # --------------------------------------------------------------------------- + build-pgaftest: + name: Build pgaftest image + needs: style_checker + runs-on: ubuntu-latest + env: + PGVERSION: 17 + + steps: + - uses: actions/checkout@v7.0.0 + + - name: Generate git-version.h + run: make version + + - name: Build pgaf:pgaftest image (PG${{ env.PGVERSION }}) + run: | + docker build \ + --build-arg PGVERSION=${{ env.PGVERSION }} \ + --build-arg CITUSTAG=v13.2.0 \ + --target pgaftest \ + -t pgaf:pgaftest \ + . + + - name: Save image to tarball + run: docker save pgaf:pgaftest | gzip > /tmp/pgaf-pgaftest.tar.gz + + - name: Upload pgaftest image artifact + uses: actions/upload-artifact@v7.0.1 + with: + name: pgaftest-image + path: /tmp/pgaf-pgaftest.tar.gz + retention-days: 1 + + # --------------------------------------------------------------------------- + # Run test schedules. Fast schedules (quick, node, ssl) run against all four + # Postgres versions. Slow schedules (multi-*, citus-*) run PG17 only. + # Total: 3×4 + 5×1 = 17 test jobs — keeps GitHub shared-runner queue short. + # --------------------------------------------------------------------------- + test: + name: pgaftest / ${{ matrix.schedule }} (PG${{ matrix.PGVERSION }}) + needs: [build-images, build-pgaftest] + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + # Fast/medium schedules run against all four Postgres versions. + # Slow schedules (multi-*, citus-*) run PG17 only: they exercise + # pg_auto_failover FSM logic, not Postgres version-specific code paths, + # and restricting them reduces total job count from 36 to 17 — avoiding + # GitHub Actions shared-runner queue saturation that was adding 10-15 min + # of queue wait to every run. + include: + # quick: basic_operation, basic_operation_listen_flag, config_get_set, skip_pg_hba + - { PGVERSION: 14, schedule: quick } + - { PGVERSION: 15, schedule: quick } + - { PGVERSION: 16, schedule: quick } + - { PGVERSION: 17, schedule: quick } + # node: create_standby_with_pgdata, maintenance_and_drop, auth, + # monitor_disabled, replace_monitor, extension_update, + # debian_clusters, tablespaces + - { PGVERSION: 14, schedule: node } + - { PGVERSION: 15, schedule: node } + - { PGVERSION: 16, schedule: node } + - { PGVERSION: 17, schedule: node } + # ssl: enable_ssl, ssl_self_signed, ssl_cert + - { PGVERSION: 14, schedule: ssl } + - { PGVERSION: 15, schedule: ssl } + - { PGVERSION: 16, schedule: ssl } + - { PGVERSION: 17, schedule: ssl } + # slow schedules: PG17 only + - { PGVERSION: 17, schedule: multi-alternate } # multi_alternate + - { PGVERSION: 17, schedule: multi-misc } # multi_standbys, multi_maintenance, ensure, multi_ifdown + - { PGVERSION: 17, schedule: multi-async } # multi_async + - { PGVERSION: 17, schedule: citus-1 } # citus_cluster_name, citus_force_failover, citus_multi_standbys + - { PGVERSION: 17, schedule: citus-2 } # basic_citus_operation, nonha_citus_operation, citus_skip_pg_hba + + steps: + - uses: actions/checkout@v7.0.0 + + - name: Download pgaf:run image (PG${{ matrix.PGVERSION }}) + uses: actions/download-artifact@v8.0.1 + with: + name: pgaf-run-image-pg${{ matrix.PGVERSION }} + path: /tmp + + - name: Load pgaf:run image into Docker + run: | + docker load < /tmp/pgaf-run-pg${{ matrix.PGVERSION }}.tar.gz + # Alias to the name pgaftest expects via PGAF_IMAGE + docker tag pgaf:run-pg${{ matrix.PGVERSION }} pgaf:run + + - name: Download pgaf:debian image (PG${{ matrix.PGVERSION }}) + if: matrix.schedule == 'node' + uses: actions/download-artifact@v8.0.1 + with: + name: pgaf-debian-image-pg${{ matrix.PGVERSION }} + path: /tmp + + - name: Load pgaf:debian image into Docker + if: matrix.schedule == 'node' + run: | + docker load < /tmp/pgaf-debian-pg${{ matrix.PGVERSION }}.tar.gz + docker tag pgaf:debian-pg${{ matrix.PGVERSION }} pgaf:debian + + - name: Download pgaf:pgaftest image + uses: actions/download-artifact@v8.0.1 + with: + name: pgaftest-image + path: /tmp + + - name: Load pgaf:pgaftest image into Docker + run: docker load < /tmp/pgaf-pgaftest.tar.gz + + - name: Run schedule ${{ matrix.schedule }} + timeout-minutes: 35 + run: | + mkdir -p /tmp/pgaftest && chmod 777 /tmp/pgaftest + DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) + docker run --rm \ + --user root \ + --group-add "${DOCKER_GID}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp/pgaftest:/tmp/pgaftest \ + -v "$(pwd)":/work:ro \ + -w /work \ + -e PGAF_IMAGE=pgaf:run \ + -e PGVERSION=${{ matrix.PGVERSION }} \ + -e PGAF_DEBIAN_IMAGE=${{ matrix.schedule == 'node' && 'pgaf:debian' || '' }} \ + -e PGAFTEST_HOST_WORK_DIR="$(pwd)" \ + pgaf:pgaftest \ + pgaftest run --schedule tests/tap/schedules/${{ matrix.schedule }}.sch + + # --------------------------------------------------------------------------- + # installcheck: SQL regression suite via pg_regress. + # Runs on all supported Postgres versions; builds pgaf:testrun inline since + # it needs postgresql-server-dev which isn't in the run image. + # --------------------------------------------------------------------------- + installcheck: + name: pgaftest / installcheck (PG${{ matrix.PGVERSION }}) + needs: build-pgaftest + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + PGVERSION: [14, 15, 16, 17] + + steps: + - uses: actions/checkout@v7.0.0 + + - name: Download pgaf:pgaftest image + uses: actions/download-artifact@v8.0.1 + with: + name: pgaftest-image + path: /tmp + + - name: Load pgaf:pgaftest image into Docker + run: docker load < /tmp/pgaf-pgaftest.tar.gz + + - name: Generate git-version.h + run: make version + + - name: Build pgaf:testrun image (PG${{ matrix.PGVERSION }}) + run: | + case "${{ matrix.PGVERSION }}" in + 14|15) CITUSTAG=v12.1.5 ;; + *) CITUSTAG=v13.2.0 ;; + esac + docker build \ + --build-arg PGVERSION=${{ matrix.PGVERSION }} \ + --build-arg CITUSTAG=${CITUSTAG} \ + --target testrun \ + -t pgaf:testrun \ + . + + - name: Run installcheck spec + timeout-minutes: 15 + run: | + mkdir -p /tmp/pgaftest && chmod 777 /tmp/pgaftest + DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) + docker run --rm \ + --user root \ + --group-add "${DOCKER_GID}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp/pgaftest:/tmp/pgaftest \ + -v "$(pwd)":/work:ro \ + -w /work \ + -e PGAFTEST_HOST_WORK_DIR="$(pwd)" \ + pgaf:pgaftest \ + pgaftest run tests/tap/specs/installcheck.pgaf + + # --------------------------------------------------------------------------- + # Upgrade test: binary + extension swap without container restart. + # Pinned to PG16 (the upgrade path always runs N-1 → current). + # Builds its own pgaf:next and pgaf:current images from the upgrade Makefile. + # --------------------------------------------------------------------------- + upgrade: + name: pgaftest / upgrade + needs: build-pgaftest + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + PGVERSION: 16 + + steps: + - uses: actions/checkout@v7.0.0 + with: + # Full history needed so PREV_TAG auto-detection finds the right tag. + fetch-depth: 0 + + - name: Download pgaf:pgaftest image + uses: actions/download-artifact@v8.0.1 + with: + name: pgaftest-image + path: /tmp + + - name: Load pgaf:pgaftest image into Docker + run: docker load < /tmp/pgaf-pgaftest.tar.gz + + - name: Generate git-version.h + run: make version + + - name: Build pgaf:next (current branch, PG${{ env.PGVERSION }}) + run: make -C tests/upgrade pgaf-next PGVERSION=${{ env.PGVERSION }} + + - name: Build pgaf:current (previous release, PG${{ env.PGVERSION }}) + run: make -C tests/upgrade pgaf-current PGVERSION=${{ env.PGVERSION }} + + - name: Run upgrade spec + timeout-minutes: 25 + run: | + mkdir -p /tmp/pgaftest && chmod 777 /tmp/pgaftest + DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) + docker run --rm \ + --user root \ + --group-add "${DOCKER_GID}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp/pgaftest:/tmp/pgaftest \ + -v "$(pwd)":/work:ro \ + -w /work \ + -e PGVERSION=${{ env.PGVERSION }} \ + -e PGAFTEST_HOST_WORK_DIR="$(pwd)" \ + pgaf:pgaftest \ + pgaftest run tests/tap/specs/upgrade.pgaf diff --git a/.gitignore b/.gitignore index 3d0a5f4fa..de6657523 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,7 @@ docs/tikz/*.png # Exclude our demo/test tmux directory tmux/ valgrind/ +*.tmp +src/bin/pgaftest/test_spec_parse.tab.* +src/bin/pgaftest/test_spec_parse.output +src/bin/pgaftest/pgaftest diff --git a/Dockerfile b/Dockerfile index f9c56cc2b..de72b5084 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ARG PGVERSION=17 # # This base image contains all our target Postgres versions. # -FROM debian:bullseye-slim AS base +FROM debian:bookworm-slim AS base ARG PGVERSION @@ -44,8 +44,6 @@ RUN apt-get update \ make \ autoconf \ openssl \ - python3-nose \ - python3-pytest \ python3 \ python3-setuptools \ python3-psycopg2 \ @@ -58,13 +56,13 @@ RUN apt-get update \ psmisc \ htop \ less \ - mg \ valgrind \ postgresql-common \ && rm -rf /var/lib/apt/lists/* -RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main ${PGVERSION}" > /etc/apt/sources.list.d/pgdg.list +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive-keyring.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/pgdg-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main ${PGVERSION}" > /etc/apt/sources.list.d/pgdg.list # bypass initdb of a "main" cluster RUN echo 'create_main_cluster = false' | sudo tee -a /etc/postgresql-common/createcluster.conf @@ -74,7 +72,7 @@ RUN apt-get update \ postgresql-${PGVERSION} \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install 'pyroute2>=0.5.17' +RUN pip3 install --break-system-packages nose pytest 'pyroute2>=0.5.17' RUN adduser --disabled-password --gecos '' docker RUN adduser docker sudo @@ -84,7 +82,7 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers FROM base AS citus ARG PGVERSION -ARG CITUSTAG=v13.0.1 +ARG CITUSTAG=v13.2.0 ENV PG_CONFIG=/usr/lib/postgresql/${PGVERSION}/bin/pg_config @@ -108,8 +106,8 @@ ENV PG_CONFIG=/usr/lib/postgresql/${PGVERSION}/bin/pg_config WORKDIR /usr/src/pg_auto_failover COPY Makefile ./ -COPY Makefile.azure ./ COPY Makefile.citus ./ +COPY Makefile.azure* ./ COPY ./src/ ./src COPY ./src/bin/pg_autoctl/git-version.h ./src/bin/pg_autoctl/git-version.h # Touch bison/flex generated files so they appear newer than the grammar @@ -146,7 +144,7 @@ ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:/usr/local/sbin:/usr/local/bin:/us # # And finally our "run" images with the bare minimum for run-time. # -FROM debian:bullseye-slim AS run +FROM debian:bookworm-slim AS run ARG PGVERSION @@ -170,8 +168,9 @@ RUN apt-get update \ libpq-dev \ && rm -rf /var/lib/apt/lists/* -RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main ${PGVERSION}" > /etc/apt/sources.list.d/pgdg.list +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive-keyring.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/pgdg-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main ${PGVERSION}" > /etc/apt/sources.list.d/pgdg.list # bypass initdb of a "main" cluster RUN echo 'create_main_cluster = false' | sudo tee -a /etc/postgresql-common/createcluster.conf @@ -206,7 +205,46 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/p ENV PG_AUTOCTL_DEBUG=1 ENV PGDATA=/var/lib/postgres/pgaf -CMD ["pg_autoctl", "do tmux session --nodes 3 --binpath /usr/local/bin/pg_autoctl"] +# +# debian image — like run, but with a Debian-style "main" cluster pre-created +# via pg_createcluster so that pg_autoctl can test adoption of the split-config +# layout (postgresql.conf lives in /etc/postgresql/${PGVERSION}/main/ outside +# PGDATA). pg_autoctl node run detects the missing postgresql.conf in PGDATA, +# moves the conf files in, and proceeds normally — no entrypoint changes needed. +# +FROM run AS debian + +ARG PGVERSION + +USER root +RUN pg_createcluster \ + --user docker --group postgres \ + ${PGVERSION} main \ + -- --auth-local trust --auth-host trust \ + && chown docker /var/lib/postgresql/${PGVERSION} + +USER docker +ENV PGDATA=/var/lib/postgresql/${PGVERSION}/main + +# +# testrun image — like run, but adds postgresql-server-dev and the full source +# tree so that "make installcheck" works inside the monitor container. +# Used by installcheck.pgaf via "monitor image-target testrun". +# +FROM run AS testrun + +ARG PGVERSION + +USER root +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + make \ + postgresql-server-dev-${PGVERSION} \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=docker ./src/ /usr/src/pg_auto_failover/src/ + +USER docker # # pgaftest image — standalone test-runner image. diff --git a/Makefile b/Makefile index 73f062b48..47f05753e 100644 --- a/Makefile +++ b/Makefile @@ -33,13 +33,10 @@ else GIT_VERSION := $(shell awk -F '[ "]' '{print $$4}' $(VERSION_FILE)) endif -# Azure only targets and variables are in a separate Makefile -include Makefile.azure - # # LIST TESTS # -PYTEST = python3 -m pytest +PYTEST = $(shell which pytest || which pytest3) # Tests for the monitor TESTS_MONITOR = test_extension_update diff --git a/Makefile.azure b/Makefile.azure deleted file mode 100644 index 1b29839b8..000000000 --- a/Makefile.azure +++ /dev/null @@ -1,38 +0,0 @@ -# -# AZURE related -# - -# make azcluster arguments -AZURE_PREFIX ?= ha-demo-$(shell whoami) -AZURE_REGION ?= paris -AZURE_LOCATION ?= francecentral - -# Pick a version of Postgres and pg_auto_failover packages to install -# in our target Azure VMs when provisionning -# -# sudo apt-get install -q -y postgresql-13-auto-failover-1.5=1.5.2 -# postgresql-${AZ_PG_VERSION}-auto-failover-${AZ_PGAF_DEB_VERSION}=${AZ_PGAF_VERSION} -AZ_PG_VERSION ?= 13 -AZ_PGAF_DEB_VERSION ?= 1.6 -AZ_PGAF_DEB_REVISION ?= 1.6.4-1 - -export AZ_PG_VERSION -export AZ_PGAF_DEB_VERSION -export AZ_PGAF_DEB_REVISION - -.PHONY: azcluster -azcluster: all - $(PG_AUTOCTL) do azure create \ - --prefix $(AZURE_PREFIX) \ - --region $(AZURE_REGION) \ - --location $(AZURE_LOCATION) \ - --nodes $(NODES) - -# make azcluster has been done before, just re-attach -.PHONY: az -az: all - $(PG_AUTOCTL) do azure tmux session - -.PHONY: azdrop -azdrop: all - $(PG_AUTOCTL) do azure drop diff --git a/docs/citus-quickstart.rst b/docs/citus-quickstart.rst index f4b20e20a..19aae76f2 100644 --- a/docs/citus-quickstart.rst +++ b/docs/citus-quickstart.rst @@ -474,6 +474,83 @@ node too: ------- 75 +.. _citus_tutorial_pgaftest: + +Alternative: Citus cluster with pgaftest +----------------------------------------- + +The named Citus cluster from the previous section can also be started with a +single ``pgaftest`` command. ``pgaftest`` reads the ``.pgaf`` spec below, +generates the compose file, and opens a tmux session once all nodes are +healthy — no manual ``docker compose up`` or ``pg_autoctl watch`` needed. + +.. code-block:: text + :caption: docs/tutorial/citus_tutorial.pgaf + + # Citus cluster: one coordinator pair + three worker pairs + # + # Usage: + # pgaftest setup docs/tutorial/citus_tutorial.pgaf --tmux + + cluster { + monitor + + formation coord { + coord0a coordinator + coord0b coordinator + } + + formation workers num-sync 1 { + worker1a worker group 1 + worker1b worker group 1 + worker2a worker group 2 + worker2b worker group 2 + worker3a worker group 3 + worker3b worker group 3 + } + } + + setup { + wait until primary, secondary timeout 180s + } + + teardown { + compose down + } + +Start it: + +:: + + $ pgaftest setup docs/tutorial/citus_tutorial.pgaf --tmux + +Three panes open immediately: + +- **top** — ``docker compose logs -f`` (streaming container output) +- **middle** — ``pg_autoctl watch`` showing all eight nodes converging +- **bottom** — interactive ``bash`` in the coordinator primary + +From the bottom pane, verify the cluster:: + + pg_autoctl show state \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover + +To trigger a coordinator failover:: + + pg_autoctl perform failover --formation coord \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover + +To trigger a worker failover in group 1:: + + pg_autoctl perform failover --group 1 \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover + +When done:: + + $ pgaftest down --work-dir /tmp/pgaftest/citus_tutorial + +For the full ``pgaftest`` reference see :ref:`pgaftest`. + Cleanup ------- diff --git a/docs/operations.rst b/docs/operations.rst index 60a6566a0..523e3cefd 100644 --- a/docs/operations.rst +++ b/docs/operations.rst @@ -498,3 +498,55 @@ nodes must wait until the monitor is ready. For the full property reference and mutability table see :ref:`pg_autoctl_node`. + + +.. _testing: + +Testing +------- + +pg_auto_failover ships ``pgaftest``, an integration test runner that drives +clusters through scenarios described in ``.pgaf`` spec files. Specs combine +a topology declaration (the Docker Compose layout) with a sequence of named +steps that exercise failover, network partitions, maintenance windows, and +more. + +**Run a spec in CI mode** — full headless run with TAP output:: + + pgaftest run tests/tap/specs/basic_operation.pgaf + +The runner generates a ``docker-compose.yml`` from the ``cluster {}`` block, +starts the stack, runs the ``setup {}`` block to wait for a healthy cluster, +executes each step in ``sequence``, runs ``teardown {}``, and removes the +stack. Exit code 0 means all steps passed. + +**Run the full test schedule** across all supported Postgres versions:: + + pgaftest run --schedule tests/tap/schedule + +**Bring up an interactive cluster** for manual exploration:: + + pgaftest setup tests/tap/specs/basic_operation.pgaf + +**Add** ``--tmux`` to open a three-pane session the moment the compose +stack is ready — no waiting at a blank terminal. The setup block runs in +the bottom pane so you can watch the logs pane while the cluster initialises: + +- **top** — ``docker compose logs -f`` +- **middle** — ``pg_autoctl watch`` +- **bottom** — setup progress, then an interactive ``bash`` shell + +:: + + pgaftest setup tests/tap/specs/basic_operation.pgaf --tmux + +**Run a single named step** against a live cluster:: + + pgaftest step stop_primary --work-dir /tmp/pgaftest/basic_operation + +**Tear down** when done:: + + pgaftest down --work-dir /tmp/pgaftest/basic_operation + +The complete spec language reference, including all DSL commands, environment +variables, and TAP output format, is at :ref:`pgaftest`. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index f9917c3b5..127b23973 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -366,6 +366,46 @@ To dispose of the entire tutorial environment, just use the following command: $ docker compose down +.. _tutorial_pgaftest: + +Alternative: interactive cluster with pgaftest +---------------------------------------------- + +The same two-node cluster from this tutorial can be started with a single +``pgaftest`` command, without writing any compose files or ini files by hand. +``pgaftest`` generates both from the spec file, starts the cluster, and opens +a tmux session so you can explore it immediately. + +The following spec file describes the tutorial topology: + +.. literalinclude:: tutorial/interactive_tutorial.pgaf + :language: text + :caption: docs/tutorial/interactive_tutorial.pgaf + +Start it with: + +:: + + $ pgaftest setup docs/tutorial/interactive_tutorial.pgaf --tmux + +Three tmux panes open as soon as the cluster is healthy: + +- **top** — ``docker compose logs -f`` (live container output) +- **middle** — ``pg_autoctl watch`` state dashboard +- **bottom** — interactive ``bash`` in ``node1`` + +From the bottom pane you can trigger a failover:: + + pg_autoctl perform failover \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover + +Watch the middle pane as the FSM transitions unfold in real time. When you +are done, tear down the cluster from any shell:: + + $ pgaftest down --work-dir /tmp/pgaftest/interactive_tutorial + +For the full ``pgaftest`` reference see :ref:`pgaftest`. + Next steps ---------- diff --git a/src/bin/common/ipaddr.c b/src/bin/common/ipaddr.c index 7abd47451..b1d78ef28 100644 --- a/src/bin/common/ipaddr.c +++ b/src/bin/common/ipaddr.c @@ -79,10 +79,8 @@ fetchLocalIPAddress(char *localIpAddress, int size, hints.ai_socktype = SOCK_STREAM; /* we only want TCP sockets */ hints.ai_protocol = IPPROTO_TCP; /* we only want TCP sockets */ - IntString servicePortStr = intToString(servicePort); - if (!GetAddrInfo(serviceName, - servicePortStr.strValue, + intToString(servicePort).strValue, &hints, &lookup)) { diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index 794c00ae8..08416e0c6 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -407,28 +407,6 @@ cli_common_keeper_getopts(int argc, char **argv, break; } - case 'e': - { - /* { "replication-password", required_argument, NULL, 'e' } */ - strlcpy(LocalOptionConfig.replication_password, optarg, - MAXCONNINFO); - log_trace("--replication-password ****"); - break; - } - - case 'w': - { - /* - * { "monitor-password", required_argument, NULL, 'w' } - * The pgautofailover_monitor health-check role currently uses a - * hardcoded password (PG_AUTOCTL_HEALTH_PASSWORD). Accept the - * option so pg_autoctl node run can pass it without error; it - * is otherwise unused at this time. - */ - log_trace("--monitor-password ****"); - break; - } - case 'V': { /* keeper_cli_print_version prints version and exits. */ @@ -517,6 +495,24 @@ cli_common_keeper_getopts(int argc, char **argv, break; } + case 'W': + { + /* { "monitor-password", required_argument, NULL, 'W' } */ + strlcpy(LocalOptionConfig.monitor_password, optarg, + sizeof(LocalOptionConfig.monitor_password)); + log_trace("--monitor-password ****"); + break; + } + + case 'w': + { + /* { "replication-password", required_argument, NULL, 'w' } */ + strlcpy(LocalOptionConfig.replication_password, optarg, + sizeof(LocalOptionConfig.replication_password)); + log_trace("--replication-password ****"); + break; + } + /* * { "ssl-ca-file", required_argument, &ssl_flag, SSL_CA_FILE_FLAG } * { "ssl-crl-file", required_argument, &ssl_flag, SSL_CA_FILE_FLAG } @@ -1442,9 +1438,7 @@ exit_unless_role_is_keeper(KeeperConfig *kconfig) void keeper_cli_help(int argc, char **argv) { - CommandLine command = root; - - (void) commandline_print_command_tree(&command, stdout); + (void) commandline_print_command_tree(&root, stdout); } diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 8ef2e8643..c8d9c8a02 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -334,6 +334,8 @@ cli_create_postgres_getopts(int argc, char **argv) { "monitor", required_argument, NULL, 'm' }, { "disable-monitor", no_argument, NULL, 'M' }, { "node-id", required_argument, NULL, 'I' }, + { "monitor-password", required_argument, NULL, 'W' }, + { "replication-password", required_argument, NULL, 'w' }, { "version", no_argument, NULL, 'V' }, { "verbose", no_argument, NULL, 'v' }, { "quiet", no_argument, NULL, 'q' }, @@ -354,7 +356,7 @@ cli_create_postgres_getopts(int argc, char **argv) int optind = cli_create_node_getopts(argc, argv, long_options, - "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:RVvqhP:r:xsN", + "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:W:w:RVvqhP:r:xsN", &options); /* publish our option parsing in the global variable */ @@ -793,6 +795,8 @@ cli_create_monitor_getopts(int argc, char **argv) { "listen", required_argument, NULL, 'l' }, { "auth", required_argument, NULL, 'A' }, { "skip-pg-hba", no_argument, NULL, 'S' }, + { "autoctl-node-password", required_argument, NULL, 'W' }, + { "formation", required_argument, NULL, 'f' }, { "version", no_argument, NULL, 'V' }, { "verbose", no_argument, NULL, 'v' }, { "quiet", no_argument, NULL, 'q' }, @@ -821,7 +825,7 @@ cli_create_monitor_getopts(int argc, char **argv) optind = 0; - while ((c = getopt_long(argc, argv, "C:D:p:n:l:A:SVvqhxNs", + while ((c = getopt_long(argc, argv, "C:D:p:n:l:A:SW:f:VvqhxsN", long_options, &option_index)) != -1) { switch (c) @@ -898,6 +902,30 @@ cli_create_monitor_getopts(int argc, char **argv) break; } + case 'W': + { + strlcpy(options.autoctl_node_password, optarg, + sizeof(options.autoctl_node_password)); + log_trace("--autoctl-node-password ****"); + break; + } + + case 'f': + { + /* --formation (may be repeated) */ + if (options.formationCount >= MONITOR_MAX_FORMATIONS) + { + log_error("Too many --formation options (max %d)", + MONITOR_MAX_FORMATIONS); + errors++; + break; + } + int fi = options.formationCount++; + strlcpy(options.formationNames[fi], optarg, NAMEDATALEN); + log_trace("--formation %s", optarg); + break; + } + case 'V': { /* keeper_cli_print_version prints version and exits. */ diff --git a/src/bin/pg_autoctl/cli_do_misc.c b/src/bin/pg_autoctl/cli_do_misc.c index 377e7f75f..68e144a95 100644 --- a/src/bin/pg_autoctl/cli_do_misc.c +++ b/src/bin/pg_autoctl/cli_do_misc.c @@ -38,6 +38,10 @@ #include "primary_standby.h" #include "string_utils.h" +/* Options specific to "pg_autoctl inspect pgsetup wait" */ +static bool pgsetupWaitReadWrite = false; +static int pgsetupWaitTimeout = 30; + /* * keeper_cli_create_replication_slot implements the CLI to create a replication @@ -322,109 +326,150 @@ keeper_cli_pgsetup_is_ready(int argc, char **argv) } -/* timeout parsed by keeper_cli_pgsetup_wait_getopts, consumed by wait_until_ready */ -static int pgsetup_wait_timeout = 30; - /* - * keeper_cli_pgsetup_wait_getopts parses --pgdata and --timeout for the - * "pgsetup wait" subcommand. + * keeper_cli_pgsetup_wait_getopts parses options specific to + * "pg_autoctl inspect pgsetup wait": --read-write, --timeout, plus the + * standard Postgres connection options inherited from the keeper setup. */ int keeper_cli_pgsetup_wait_getopts(int argc, char **argv) { - int c, option_index = 0; - - static struct option long_options[] = { - { "pgdata", required_argument, NULL, 'D' }, - { "timeout", required_argument, NULL, 't' }, - { "version", no_argument, NULL, 'V' }, - { "verbose", no_argument, NULL, 'v' }, - { "quiet", no_argument, NULL, 'q' }, - { "help", no_argument, NULL, 'h' }, + /* + * cli_common_keeper_getopts (called by keeper_cli_keeper_setup_getopts) + * exits with BAD_ARGS when it encounters unknown options. Since --timeout + * and --read-write are not in its options list, we must remove them from + * argv before delegating. + * + * Strategy: + * 1. Scan argv with opterr=0 to capture --timeout / --read-write. + * 2. Build a filtered argv that omits those two options. + * 3. Pass the filtered argv to keeper_cli_keeper_setup_getopts. + */ + + /* Reset module-level wait options */ + pgsetupWaitReadWrite = false; + pgsetupWaitTimeout = 30; + + static struct option wait_options[] = { + { "read-write", no_argument, NULL, 'W' }, + { "timeout", required_argument, NULL, 'T' }, { NULL, 0, NULL, 0 } }; - optind = 0; + optind = 1; + opterr = 0; + + int c; + int option_index = 0; - while ((c = getopt_long(argc, argv, "D:t:Vvqh", - long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "WT:", wait_options, &option_index)) != -1) { switch (c) { - case 'D': + case 'W': { - strlcpy(keeperOptions.pgSetup.pgdata, optarg, MAXPGPATH); - log_trace("--pgdata %s", optarg); + pgsetupWaitReadWrite = true; break; } - case 't': + case 'T': { - if (!stringToInt(optarg, &pgsetup_wait_timeout) || - pgsetup_wait_timeout <= 0) + int t = strtol(optarg, NULL, 10); + if (t <= 0) { - log_fatal( - "--timeout argument is not a valid positive integer: \"%s\"", - optarg); + log_error("--timeout must be a positive integer"); exit(EXIT_CODE_BAD_ARGS); } - log_trace("--timeout %d", pgsetup_wait_timeout); + pgsetupWaitTimeout = t; break; } - case 'V': + default: { - keeper_cli_print_version(argc, argv); + /* standard keeper options; handled by the delegated call below */ break; } + } + } - case 'v': - { - log_set_level(LOG_DEBUG); - break; - } + opterr = 1; - case 'q': - { - log_set_level(LOG_ERROR); - break; - } + /* + * Build a filtered argv that strips --timeout/--read-write (and their + * arguments) so that keeper_cli_keeper_setup_getopts does not see them. + */ + char **filtered_argv = (char **) palloc((argc + 1) * sizeof(char *)); + int filtered_argc = 0; - case 'h': - { - commandline_help(stderr); - exit(EXIT_CODE_QUIT); - break; - } + for (int i = 0; i < argc; i++) + { + if (strcmp(argv[i], "--read-write") == 0 || strcmp(argv[i], "-W") == 0) + { + continue; + } - default: - { - commandline_help(stderr); - exit(EXIT_CODE_BAD_ARGS); - break; - } + if ((strcmp(argv[i], "--timeout") == 0 || strcmp(argv[i], "-T") == 0) && + i + 1 < argc) + { + /* skip both the flag and its argument */ + i++; + continue; } + + filtered_argv[filtered_argc++] = argv[i]; } + filtered_argv[filtered_argc] = NULL; - /* publish parsed options */ - keeperOptions.pgSetup.pgdata[0] = - keeperOptions.pgSetup.pgdata[0]; /* no-op, already set above */ + int rc = keeper_cli_keeper_setup_getopts(filtered_argc, filtered_argv); - return optind; + pfree(filtered_argv); + + return rc; } /* - * keeper_cli_pgsetup_wait_until_ready waits until the local Postgres server - * is ready to accept connections, up to --timeout seconds (default 30). + * keeper_cli_pgsetup_wait_until_ready waits for the local Postgres server to + * become ready. When --read-write is given, it additionally waits until the + * server is accepting read-write connections (not in recovery and not set to + * default_transaction_read_only). + * + * The --timeout value (default 30s) is a single deadline shared by both + * phases: the pg_is_ready poll and the subsequent read-write connection + * attempt. Time spent waiting for Postgres to start counts against the + * budget for the read-write phase. */ void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) { + int timeout = pgsetupWaitTimeout; + ConfigFilePaths pathnames = { 0 }; LocalPostgresServer postgres = { 0 }; PostgresSetup *pgSetup = &(postgres.postgresSetup); + /* Record wall-clock start so all phases share one deadline. */ + time_t startTime = time(NULL); + + /* Wait up to `timeout` seconds for the config file to be created. + * In no-monitor mode, pg_autoctl create postgres runs first and writes the + * config; pgsetup wait may be called before that completes. */ + { + KeeperConfig kconfig = keeperOptions; + if (keeper_config_set_pathnames_from_pgdata(&(kconfig.pathnames), + kconfig.pgSetup.pgdata)) + { + time_t deadline = startTime + timeout; + while (!file_exists(kconfig.pathnames.config) && + time(NULL) < deadline) + { + log_debug("Waiting for config file \"%s\" to appear", + kconfig.pathnames.config); + pg_usleep(500 * 1000); + } + } + } + if (!cli_common_pgsetup_init(&pathnames, pgSetup)) { /* errors have already been logged */ @@ -433,16 +478,105 @@ keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) log_debug("Initialized pgSetup, now calling pg_setup_wait_until_is_ready()"); + /* + * Phase 1: wait for postmaster to signal "ready" in postmaster.pid. + * Pass the remaining timeout so the two phases together stay within the + * single user-visible deadline. + */ + int remainingAfterConfig = timeout - (int) (time(NULL) - startTime); + if (remainingAfterConfig <= 0) + { + log_error("Timed out waiting for Postgres config file to appear"); + exit(EXIT_CODE_PGSQL); + } + bool pgIsReady = - pg_setup_wait_until_is_ready(pgSetup, pgsetup_wait_timeout, LOG_INFO); + pg_setup_wait_until_is_ready(pgSetup, remainingAfterConfig, LOG_INFO); log_info("Postgres status is: \"%s\"", pmStatusToString(pgSetup->pm_status)); - if (pgIsReady) + if (!pgIsReady) + { + exit(EXIT_CODE_PGSQL); + } + + if (!pgsetupWaitReadWrite) { + /* Plain "ready" check — we're done. */ exit(EXIT_CODE_QUIT); } - exit(EXIT_CODE_PGSQL); + + /* + * Phase 2: wait until the server accepts read-write connections. + * + * Postgres is up (phase 1 passed) but may still be in recovery, finishing + * pg_rewind, or in standby mode. We poll with a libpq connection that + * checks pg_is_in_recovery() until it returns false or the deadline fires. + * + * We use the local connection string from pgSetup (Unix socket when + * available, matching whatever auth the node was created with) so that the + * check works regardless of the cluster's auth method. + */ + char connstr[MAXCONNINFO]; + if (!pg_setup_get_local_connection_string(pgSetup, connstr)) + { + log_error("Failed to build local connection string for read-write check"); + exit(EXIT_CODE_BAD_CONFIG); + } + + log_info("Waiting for Postgres to accept read-write connections " + "(timeout %ds)", timeout); + + bool isReadWrite = false; + int attempts = 0; + + while (!isReadWrite) + { + int elapsed = (int) (time(NULL) - startTime); + int remaining = timeout - elapsed; + + if (remaining <= 0) + { + log_error("Timed out after %ds waiting for Postgres " + "to accept read-write connections", timeout); + exit(EXIT_CODE_PGSQL); + } + + /* Use a short per-attempt connect_timeout so we retry briskly. */ + char attemptConnstr[MAXCONNINFO]; + sformat(attemptConnstr, sizeof(attemptConnstr), + "%s connect_timeout=1", connstr); + + PGSQL pgsql = { 0 }; + pgsql_init(&pgsql, attemptConnstr, PGSQL_CONN_LOCAL); + + bool inRecovery = true; /* assume standby until proven otherwise */ + bool queryOk = pgsql_is_in_recovery(&pgsql, &inRecovery); + pgsql_finish(&pgsql); + + if (queryOk && !inRecovery) + { + isReadWrite = true; + break; + } + + /* let's not be THAT verbose about it */ + if (attempts % 10 == 0) + { + log_debug("pgsetup wait --read-write: attempt %d, " + "in_recovery=%s, after %ds", + attempts + 1, + inRecovery ? "true" : "false", + elapsed); + } + + ++attempts; + pg_usleep(100 * 1000); /* 100 ms between probes */ + } + + log_info("Postgres is now accepting read-write connections on port %d", + pgSetup->pgport); + exit(EXIT_CODE_QUIT); } diff --git a/src/bin/pg_autoctl/cli_do_root.c b/src/bin/pg_autoctl/cli_do_root.c index 9ee0752c7..0b473d808 100644 --- a/src/bin/pg_autoctl/cli_do_root.c +++ b/src/bin/pg_autoctl/cli_do_root.c @@ -185,20 +185,13 @@ CommandLine do_pgsetup_is_ready = CommandLine do_pgsetup_wait_until_ready = make_command("wait", "Wait until the local Postgres server is ready", - "[option ...]", - " --pgdata path to data directory\n" - " --timeout seconds to wait, default 30\n", + "[--read-write] [--timeout N] [option ...]", + " --read-write also wait until the server accepts read-write connections\n" + " --timeout N total timeout in seconds (default: 30)\n" + KEEPER_CLI_WORKER_SETUP_OPTIONS, keeper_cli_pgsetup_wait_getopts, keeper_cli_pgsetup_wait_until_ready); -CommandLine do_pgsetup_hba_lan = - make_command("hba-lan", - "Add LAN CIDR trust rules to pg_hba.conf and reload", - "[option ...]", - KEEPER_CLI_WORKER_SETUP_OPTIONS, - keeper_cli_keeper_setup_getopts, - keeper_cli_pgsetup_hba_lan); - CommandLine do_pgsetup_startup_logs = make_command("logs", "Outputs the Postgres startup logs", @@ -215,6 +208,14 @@ CommandLine do_pgsetup_tune = keeper_cli_keeper_setup_getopts, keeper_cli_pgsetup_tune); +CommandLine do_pgsetup_hba_lan = + make_command("hba-lan", + "Append LAN CIDR trust rules to pg_hba.conf and reload Postgres", + "[option ...]", + KEEPER_CLI_WORKER_SETUP_OPTIONS, + keeper_cli_keeper_setup_getopts, + keeper_cli_pgsetup_hba_lan); + CommandLine *do_pgsetup[] = { &do_pgsetup_pg_ctl, &do_pgsetup_discover, @@ -384,54 +385,63 @@ CommandLine do_tmux_commands = "Set of facilities to handle tmux interactive sessions", NULL, NULL, NULL, do_tmux); + /* - * internal service: hidden entry points spawned by the supervisor via - * fork+exec. The supervisor builds argv as: - * pg_autoctl internal service postgres|listener|node-active --pgdata ... - * Use make_hidden_command_set so these never appear in --help output. + * pg_autoctl internal service postgres|listener|node-active + * + * These are the subprocess entry points used by the supervisor (pg_autoctl run + * and pg_autoctl create … --run). The supervisor fork()s and then execv()s + * the pg_autoctl binary itself with one of these sub-commands so that each + * service runs in its own address space. + * + * Using fork+exec (rather than fork alone) is a deliberate design choice for + * live upgrades: when a child process exits with an incompatible monitor + * extension version, the supervisor restarts it via fork()+execv(), which loads + * the current binary from disk. If the binary has been updated in place (e.g. + * by a package manager), the restarted child automatically picks up the new + * version without touching the supervisor process — making pg_autoctl safe to + * use as PID 1 in Docker/Kubernetes containers where replacing the binary and + * sending SIGTERM would lose the container. + * + * See also: keeper.c keeper_check_monitor_extension_version(), which exits on + * version mismatch precisely to trigger this restart-with-new-binary path. + * + * These commands are hidden from --help output (make_hidden_command_set) so + * operators do not accidentally invoke them directly. + * Use "pg_autoctl manual service" for the user-facing controls (restart, pgctl). + * Use "pg_autoctl inspect getpid" to read sub-process PIDs. */ static CommandLine *internal_service_subcommands[] = { - &service_pgcontroller, - &service_postgres, - &service_monitor_listener, - &service_node_active, + &service_pgcontroller, /* debug: supervisor for just the postgres controller */ + &service_postgres, /* spawned by service_postgres_ctl_start() */ + &service_monitor_listener, /* spawned by service_monitor_start() */ + &service_node_active, /* spawned by service_keeper_start() */ NULL }; -CommandLine internal_service_commands = +static CommandLine internal_service_commands = make_hidden_command_set("service", - "Internal subprocess entry points (supervisor use only)", + "Subprocess entry points for the pg_autoctl supervisor", NULL, NULL, NULL, internal_service_subcommands); -static CommandLine *internal_subcommands[] = { - &internal_service_commands, - NULL -}; - -CommandLine internal_commands = - make_hidden_command_set("internal", - "Internal commands for use by the supervisor (not for operators)", - NULL, NULL, NULL, internal_subcommands); - +/* + * pg_autoctl internal + * + * Hidden from --help; routable so the supervisor's execv() calls work. + * Contains only what the supervisor spawns plus dev/QA tooling not yet + * moved to pgaftest (tmux, demo). + */ CommandLine *do_subcommands[] = { - &do_monitor_commands, - &do_coordinator_commands, - &do_fsm_commands, - &do_primary_, - &do_standby_, - &do_show_commands, - &do_pgsetup_commands, - &do_service_postgres_ctl_commands, - &do_service_commands, + &internal_service_commands, &do_tmux_commands, &do_demo_commands, NULL }; -CommandLine do_commands = - make_command_set("do", - "Internal commands and internal QA tooling", NULL, NULL, - NULL, do_subcommands); +CommandLine internal_commands = + make_hidden_command_set("internal", + "Internal subprocess entry points — not for direct use", + NULL, NULL, NULL, do_subcommands); /* diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index 888f68f9f..3cfe3af30 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -1,7 +1,7 @@ /* * src/bin/pg_autoctl/cli_do_root.h - * Implementation of a CLI which lets you run operations on the local - * postgres server directly + * Implementation of a CLI which lets you run individual keeper routines + * directly * * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the PostgreSQL License. @@ -15,33 +15,40 @@ /* src/bin/pg_autoctl/cli_do_fsm.c */ extern CommandLine do_fsm_commands; -extern CommandLine fsm_nodes; -/* Exported individually so cli_inspect.c and cli_manual.c can compose them */ -extern CommandLine fsm_init; +/* read-only sub-commands exposed via "pg_autoctl inspect fsm" */ extern CommandLine fsm_state; +extern CommandLine fsm_node_state; extern CommandLine fsm_list; extern CommandLine fsm_gv; + +/* mutating sub-commands exposed via "pg_autoctl manual fsm" */ +extern CommandLine fsm_init; extern CommandLine fsm_assign; extern CommandLine fsm_step; +extern CommandLine fsm_nodes; /* nodes get + nodes set — kept together in manual */ /* src/bin/pg_autoctl/cli_do_monitor.c */ extern CommandLine do_monitor_commands; -/* Exported individually so cli_inspect.c and cli_manual.c can compose them */ +/* read-only sub-commands exposed via "pg_autoctl inspect monitor" */ extern CommandLine monitor_get_command; extern CommandLine monitor_parse_notification_command; +extern CommandLine monitor_node_state_command; +extern CommandLine monitor_formation_states_command; + +/* mutating sub-commands exposed via "pg_autoctl manual monitor" */ extern CommandLine monitor_register_command; extern CommandLine monitor_node_active_command; extern CommandLine monitor_version_command; /* src/bin/pg_autoctl/cli_do_service.c */ extern CommandLine do_service_commands; -extern CommandLine do_service_restart_commands; extern CommandLine do_service_getpid_commands; +extern CommandLine do_service_restart_commands; extern CommandLine do_service_postgres_ctl_commands; -/* Internal subprocess entry points used in cli_do_root.c's internal_service_commands */ +/* subprocess entry points (spawned by the supervisor via fork+exec) */ extern CommandLine service_pgcontroller; extern CommandLine service_postgres; extern CommandLine service_monitor_listener; @@ -50,24 +57,32 @@ extern CommandLine service_node_active; /* src/bin/pg_autoctl/cli_do_show.c */ extern CommandLine do_show_commands; extern CommandLine do_pgsetup_commands; +extern CommandLine do_service_postgres_ctl_commands; +extern CommandLine do_service_commands; /* src/bin/pg_autoctl/cli_do_demo.c */ extern CommandLine do_demo_commands; -/* src/bin/pg_autoctl/cli_do_coordinator.c */ -extern CommandLine do_coordinator_commands; - /* src/bin/pg_autoctl/cli_do_root.c */ extern CommandLine do_primary_adduser; extern CommandLine *do_primary_adduser_subcommands[]; extern CommandLine do_primary_adduser_monitor; extern CommandLine do_primary_adduser_replica; +extern CommandLine do_primary_syncrep_; +extern CommandLine *do_primary_syncrep[]; +extern CommandLine do_primary_syncrep_enable; +extern CommandLine do_primary_syncrep_disable; + extern CommandLine do_primary_slot_; extern CommandLine *do_primary_slot[]; extern CommandLine do_primary_slot_create; extern CommandLine do_primary_slot_drop; +extern CommandLine do_primary_hba; +extern CommandLine *do_primary_hba_commands[]; +extern CommandLine do_primary_hba_setup; + extern CommandLine do_primary_defaults; extern CommandLine do_primary_identify_system; @@ -80,12 +95,14 @@ extern CommandLine do_standby_init; extern CommandLine do_standby_rewind; extern CommandLine do_standby_promote; +extern CommandLine do_discover; + extern CommandLine do_tmux_commands; -extern CommandLine internal_service_commands; -extern CommandLine internal_commands; +/* src/bin/pg_autoctl/cli_do_coordinator.c */ +extern CommandLine do_coordinator_commands; -extern CommandLine do_commands; +extern CommandLine internal_commands; extern CommandLine *do_subcommands[]; int keeper_cli_keeper_setup_getopts(int argc, char **argv); @@ -98,13 +115,13 @@ void keeper_cli_enable_synchronous_replication(int argc, char **argv); void keeper_cli_disable_synchronous_replication(int argc, char **argv); void keeper_cli_pgsetup_pg_ctl(int argc, char **argv); -void keeper_cli_pgsetup_hba_lan(int argc, char **argv); void keeper_cli_pgsetup_discover(int argc, char **argv); void keeper_cli_pgsetup_is_ready(int argc, char **argv); int keeper_cli_pgsetup_wait_getopts(int argc, char **argv); void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv); void keeper_cli_pgsetup_startup_logs(int argc, char **argv); void keeper_cli_pgsetup_tune(int argc, char **argv); +void keeper_cli_pgsetup_hba_lan(int argc, char **argv); void keeper_cli_add_default_settings(int argc, char **argv); void keeper_cli_create_monitor_user(int argc, char **argv); diff --git a/src/bin/pg_autoctl/cli_drop_node.c b/src/bin/pg_autoctl/cli_drop_node.c index a16e52a6f..04a1bf0c1 100644 --- a/src/bin/pg_autoctl/cli_drop_node.c +++ b/src/bin/pg_autoctl/cli_drop_node.c @@ -48,6 +48,7 @@ */ bool dropAndDestroy = false; static bool dropForce = false; +static bool dropNoWait = false; static void cli_drop_monitor(int argc, char **argv); @@ -84,8 +85,7 @@ CommandLine drop_node_command = " --pgport drop the node with given hostname and pgport\n" " --destroy also destroy Postgres database\n" " --force force dropping the node from the monitor\n" - " --wait how many seconds to wait, default to 60\n" - " --no-wait drop the node without waiting for confirmation\n", + " --wait how many seconds to wait, default to 60 \n", cli_drop_node_getopts, cli_drop_node); @@ -164,8 +164,7 @@ cli_drop_node_getopts(int argc, char **argv) case 'W': { - /* --no-wait: set timeout to zero so the notification loop is skipped */ - options.listen_notifications_timeout = 0; + dropNoWait = true; log_trace("--no-wait"); break; } @@ -605,6 +604,19 @@ cli_drop_local_node(KeeperConfig *config, bool dropAndDestroy) (void) cli_drop_node_from_monitor(config, &nodeId, &groupId); } + /* + * With --no-wait the caller takes responsibility for waiting until the + * supervisor has stopped (e.g. via `docker compose wait` in a container + * environment). The running keeper will detect it has been dropped on its + * next node_active() heartbeat and exit cleanly on its own. + */ + if (dropNoWait) + { + log_info("Node unregistered from monitor; not waiting for the local " + "pg_autoctl process to stop (--no-wait)."); + exit(EXIT_CODE_QUIT); + } + /* * Now, when the pg_autoctl keeper service is still running, wait until * it has reached the DROPPED/DROPPED state on-disk and then exited. diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 3c9221109..df0736974 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -104,8 +104,6 @@ CommandLine *root_subcommands[] = { &manual_commands, &internal_commands, &node_commands, - - &do_commands, &service_run_command, &watch_command, &service_stop_command, diff --git a/src/bin/pg_autoctl/config.c b/src/bin/pg_autoctl/config.c index 4a9f600f1..898867a53 100644 --- a/src/bin/pg_autoctl/config.c +++ b/src/bin/pg_autoctl/config.c @@ -70,10 +70,11 @@ build_xdg_path(char *dst, } default: - + { /* developper error */ log_error("No support for XDG Resource Type %d", xdgType); return false; + } } if (!get_env_copy_with_fallback(envVarName, xdg_topdir, MAXPGPATH, fallback)) diff --git a/src/bin/pg_autoctl/coordinator.c b/src/bin/pg_autoctl/coordinator.c index 61e55c6ec..4845aa96c 100644 --- a/src/bin/pg_autoctl/coordinator.c +++ b/src/bin/pg_autoctl/coordinator.c @@ -159,12 +159,10 @@ coordinator_add_node(Coordinator *coordinator, Keeper *keeper, : keeper->config.pgSetup.citusClusterName; SingleValueResultContext parseContext = { { 0 }, PGSQL_RESULT_INT, false }; - IntString pgportStr = intToString(keeper->config.pgSetup.pgport); - IntString groupIdStr = intToString(keeper->config.groupId); paramValues[0] = keeper->config.hostname; - paramValues[1] = pgportStr.strValue; - paramValues[2] = groupIdStr.strValue; + paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; + paramValues[2] = intToString(keeper->config.groupId).strValue; paramValues[3] = citusRoleStr; paramValues[4] = clusterName; @@ -232,8 +230,6 @@ coordinator_add_inactive_node(Coordinator *coordinator, Keeper *keeper, : keeper->config.pgSetup.citusClusterName; SingleValueResultContext parseContext = { { 0 }, PGSQL_RESULT_INT, false }; - IntString pgportStr = intToString(keeper->config.pgSetup.pgport); - IntString groupIdStr = intToString(keeper->config.groupId); if (!coordinator_master_activate_node_returns_record(pgsql, &returnsRecord)) { @@ -252,8 +248,8 @@ coordinator_add_inactive_node(Coordinator *coordinator, Keeper *keeper, } paramValues[0] = keeper->config.hostname; - paramValues[1] = pgportStr.strValue; - paramValues[2] = groupIdStr.strValue; + paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; + paramValues[2] = intToString(keeper->config.groupId).strValue; paramValues[3] = citusRoleStr; paramValues[4] = clusterName; @@ -311,8 +307,6 @@ coordinator_activate_node(Coordinator *coordinator, Keeper *keeper, parseContext.resultType = PGSQL_RESULT_INT; parseContext.parsedOk = false; - IntString pgportStr = intToString(keeper->config.pgSetup.pgport); - if (!coordinator_master_activate_node_returns_record(pgsql, &returnsRecord)) { log_error("Failed to activate node %s:%d, see above for details", @@ -330,7 +324,7 @@ coordinator_activate_node(Coordinator *coordinator, Keeper *keeper, } paramValues[0] = keeper->config.hostname; - paramValues[1] = pgportStr.strValue; + paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -377,10 +371,9 @@ coordinator_remove_node(Coordinator *coordinator, Keeper *keeper) int paramCount = 2; Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2]; - IntString pgportStr = intToString(keeper->config.pgSetup.pgport); paramValues[0] = keeper->config.hostname; - paramValues[1] = pgportStr.strValue; + paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -612,16 +605,11 @@ coordinator_update_node_prepare(Coordinator *coordinator, Keeper *keeper) * private data handled by the coordinator, and the coordinator is going to * provide for that information itself with the following SQL query. */ - IntString groupIdStr = intToString(groupId); - IntString pgportStr = intToString(keeper->config.pgSetup.pgport); - if (supportForForce) { const int paramCount = 5; Oid paramTypes[5] = { INT4OID, TEXTOID, INT4OID, TEXTOID, INT4OID }; const char *paramValues[5]; - IntString lockCooldownStr = intToString( - keeper->config.citus_master_update_node_lock_cooldown); sformat(sql, sizeof(sql), @@ -633,11 +621,12 @@ coordinator_update_node_prepare(Coordinator *coordinator, Keeper *keeper) " and not exists" " (select 1 from pg_prepared_xacts where gid = $4)"); - paramValues[0] = groupIdStr.strValue; + paramValues[0] = intToString(groupId).strValue; paramValues[1] = keeper->config.hostname; - paramValues[2] = pgportStr.strValue; + paramValues[2] = intToString(keeper->config.pgSetup.pgport).strValue; paramValues[3] = transactionName; - paramValues[4] = lockCooldownStr.strValue; + paramValues[4] = intToString( + keeper->config.citus_master_update_node_lock_cooldown).strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -663,9 +652,9 @@ coordinator_update_node_prepare(Coordinator *coordinator, Keeper *keeper) " and not exists" " (select 1 from pg_prepared_xacts where gid = $4)"); - paramValues[0] = groupIdStr.strValue; + paramValues[0] = intToString(groupId).strValue; paramValues[1] = keeper->config.hostname; - paramValues[2] = pgportStr.strValue; + paramValues[2] = intToString(keeper->config.pgSetup.pgport).strValue; paramValues[3] = transactionName; if (!pgsql_execute_with_params(pgsql, sql, @@ -816,9 +805,7 @@ coordinator_upsert_poolinfo_port(Coordinator *coordinator, Keeper *keeper) sformat(proxyInfo, sizeof(proxyInfo), "host=%s port=%d", keeper->config.hostname, keeper->config.pgSetup.proxyport); - IntString groupIdStr = intToString(keeper->config.groupId); - - paramValues[0] = groupIdStr.strValue; + paramValues[0] = intToString(keeper->config.groupId).strValue; paramValues[1] = proxyInfo; if (!pgsql_execute_with_params(pgsql, sql, diff --git a/src/bin/pg_autoctl/demoapp.c b/src/bin/pg_autoctl/demoapp.c index ac58b066f..609c75c97 100644 --- a/src/bin/pg_autoctl/demoapp.c +++ b/src/bin/pg_autoctl/demoapp.c @@ -481,15 +481,11 @@ demoapp_register_client(const char *pguri, const Oid paramTypes[4] = { INT4OID, INT4OID, INT4OID, INT4OID }; const char *paramValues[4] = { 0 }; - IntString clientIdStr = intToString(clientId); - IntString pidStr = intToString(getpid()); - IntString retrySleepStr = intToString(retrySleep); - IntString retryCapStr = intToString(retryCap); - paramValues[0] = clientIdStr.strValue; - paramValues[1] = pidStr.strValue; - paramValues[2] = retrySleepStr.strValue; - paramValues[3] = retryCapStr.strValue; + paramValues[0] = intToString(clientId).strValue; + paramValues[1] = intToString(getpid()).strValue; + paramValues[2] = intToString(retrySleep).strValue; + paramValues[3] = intToString(retryCap).strValue; pgsql_init(&pgsql, (char *) pguri, PGSQL_CONN_APP); @@ -522,11 +518,9 @@ demoapp_update_client_failovers(const char *pguri, int clientId, int failovers) const Oid paramTypes[2] = { INT4OID, INT4OID }; const char *paramValues[2] = { 0 }; - IntString clientIdStr = intToString(clientId); - IntString failoversStr = intToString(failovers); - paramValues[0] = clientIdStr.strValue; - paramValues[1] = failoversStr.strValue; + paramValues[0] = intToString(clientId).strValue; + paramValues[1] = intToString(failovers).strValue; pgsql_init(&pgsql, (char *) pguri, PGSQL_CONN_APP); @@ -704,15 +698,11 @@ demoapp_start_client(const char *pguri, int clientId, const Oid paramTypes[5] = { INT4OID, INT4OID, INT8OID, INT8OID, BOOLOID }; const char *paramValues[5] = { 0 }; - IntString clientIdStr = intToString(clientId); - IntString indexStr = intToString(index); - IntString attemptsStr = intToString(pgsql.retryPolicy.attempts); - IntString durationUsStr = intToString(INSTR_TIME_GET_MICROSEC(duration)); - - paramValues[0] = clientIdStr.strValue; - paramValues[1] = indexStr.strValue; - paramValues[2] = attemptsStr.strValue; - paramValues[3] = durationUsStr.strValue; + + paramValues[0] = intToString(clientId).strValue; + paramValues[1] = intToString(index).strValue; + paramValues[2] = intToString(pgsql.retryPolicy.attempts).strValue; + paramValues[3] = intToString(INSTR_TIME_GET_MICROSEC(duration)).strValue; paramValues[4] = is_in_recovery ? "true" : "false"; if (!pgsql_execute_with_params(&pgsql, sql, 5, paramTypes, paramValues, diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index b5dcc523a..9680b6d81 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -201,372 +201,526 @@ KeeperFSMTransition KeeperFSM[] = { /* * Started as a single, no nothing */ - { INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_INIT_TO_SINGLE, - &fsm_citus_coordinator_init_primary }, + { + INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_INIT_TO_SINGLE, + &fsm_citus_coordinator_init_primary + }, - { INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_INIT_TO_SINGLE, - &fsm_citus_worker_init_primary }, + { + INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_INIT_TO_SINGLE, + &fsm_citus_worker_init_primary + }, - { INIT_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_SINGLE, - &fsm_init_primary }, + { + INIT_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_SINGLE, + &fsm_init_primary + }, - { DROPPED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_INIT_TO_SINGLE, - &fsm_citus_worker_init_primary }, + { + DROPPED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_INIT_TO_SINGLE, + &fsm_citus_worker_init_primary + }, - { DROPPED_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_SINGLE, - &fsm_init_primary }, + { + DROPPED_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_SINGLE, + &fsm_init_primary + }, - { DROPPED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_DROPPED_TO_REPORT_LSN, - &fsm_init_from_standby }, + { + DROPPED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_DROPPED_TO_REPORT_LSN, + &fsm_init_from_standby + }, /* * other node(s) was forcibly removed, now single */ - { PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, - { WAIT_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + WAIT_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, - { JOIN_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + JOIN_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, /* * failover occurred, primary -> draining/demoted */ - { PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DRAINING, - &fsm_stop_postgres }, + { + PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DRAINING, + &fsm_stop_postgres + }, - { DRAINING_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_DRAINING_TO_DEMOTED, - &fsm_stop_postgres }, + { + DRAINING_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_DRAINING_TO_DEMOTED, + &fsm_stop_postgres + }, - { PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { JOIN_PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DRAINING, - &fsm_stop_postgres }, + { + JOIN_PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DRAINING, + &fsm_stop_postgres + }, - { JOIN_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + JOIN_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { JOIN_PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + JOIN_PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { APPLY_SETTINGS_STATE, DRAINING_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DRAINING, - &fsm_stop_postgres }, + { + APPLY_SETTINGS_STATE, DRAINING_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DRAINING, + &fsm_stop_postgres + }, - { APPLY_SETTINGS_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + APPLY_SETTINGS_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { APPLY_SETTINGS_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + APPLY_SETTINGS_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, /* * primary is put to maintenance */ - { PRIMARY_STATE, PREPARE_MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_PREPARE_MAINTENANCE, - &fsm_stop_postgres_for_primary_maintenance }, + { + PRIMARY_STATE, PREPARE_MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_PREPARE_MAINTENANCE, + &fsm_stop_postgres_for_primary_maintenance + }, - { PREPARE_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_MAINTENANCE, - &fsm_stop_postgres_and_setup_standby }, + { + PREPARE_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_MAINTENANCE, + &fsm_stop_postgres_and_setup_standby + }, - { PRIMARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_MAINTENANCE, - &fsm_stop_postgres_for_primary_maintenance }, + { + PRIMARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_MAINTENANCE, + &fsm_stop_postgres_for_primary_maintenance + }, /* * was demoted, need to be dead now. */ - { DRAINING_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_DRAINING_TO_DEMOTE_TIMEOUT, - &fsm_stop_postgres }, + { + DRAINING_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_DRAINING_TO_DEMOTE_TIMEOUT, + &fsm_stop_postgres + }, - { DEMOTE_TIMEOUT_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_DEMOTE_TIMEOUT_TO_DEMOTED, - &fsm_stop_postgres }, + { + DEMOTE_TIMEOUT_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_DEMOTE_TIMEOUT_TO_DEMOTED, + &fsm_stop_postgres + }, /* * wait_primary stops reporting, is (supposed) dead now */ - { WAIT_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + WAIT_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, /* * was demoted after a failure, but standby was forcibly removed */ - { DEMOTED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_citus_worker_resume_as_primary }, + { + DEMOTED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_citus_worker_resume_as_primary + }, - { DEMOTED_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_resume_as_primary }, + { + DEMOTED_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_resume_as_primary + }, - { DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_citus_worker_resume_as_primary }, + { + DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_citus_worker_resume_as_primary + }, - { DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_resume_as_primary }, + { + DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_resume_as_primary + }, - { DRAINING_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_citus_worker_resume_as_primary }, + { + DRAINING_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_citus_worker_resume_as_primary + }, - { DRAINING_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_resume_as_primary }, + { + DRAINING_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_resume_as_primary + }, /* * primary was forcibly removed */ - { SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_LOST_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_single }, + { + SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_LOST_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_single + }, - { SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_LOST_PRIMARY, - &fsm_citus_worker_promote_standby_to_single }, + { + SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_LOST_PRIMARY, + &fsm_citus_worker_promote_standby_to_single + }, - { SECONDARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_LOST_PRIMARY, - &fsm_promote_standby }, + { + SECONDARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_LOST_PRIMARY, + &fsm_promote_standby + }, - { CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_LOST_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_single }, + { + CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_LOST_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_single + }, - { CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_LOST_PRIMARY, - &fsm_citus_worker_promote_standby_to_single }, + { + CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_LOST_PRIMARY, + &fsm_citus_worker_promote_standby_to_single + }, - { CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_LOST_PRIMARY, - &fsm_promote_standby }, + { + CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_LOST_PRIMARY, + &fsm_promote_standby + }, - { PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_LOST_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_single }, + { + PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_LOST_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_single + }, - { PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_LOST_PRIMARY, - &fsm_citus_worker_promote_standby_to_single }, + { + PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_LOST_PRIMARY, + &fsm_citus_worker_promote_standby_to_single + }, - { PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_LOST_PRIMARY, - &fsm_promote_standby }, + { + PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_LOST_PRIMARY, + &fsm_promote_standby + }, /* * went down to force the primary to time out, but then it was removed */ - { STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_REPLICATION_TO_SINGLE, - &fsm_citus_worker_promote_standby_to_single }, + { + STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_REPLICATION_TO_SINGLE, + &fsm_citus_worker_promote_standby_to_single + }, - { STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_REPLICATION_TO_SINGLE, - &fsm_promote_standby }, + { + STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_REPLICATION_TO_SINGLE, + &fsm_promote_standby + }, /* * all states should lead to SINGLE, including REPORT_LSN */ - { REPORT_LSN_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_SINGLE, - &fsm_promote_standby }, + { + REPORT_LSN_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_SINGLE, + &fsm_promote_standby + }, /* * On the Primary, wait for a standby to be ready: WAIT_PRIMARY */ - { SINGLE_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_SINGLE_TO_WAIT_PRIMARY, - &fsm_prepare_replication }, + { + SINGLE_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_SINGLE_TO_WAIT_PRIMARY, + &fsm_prepare_replication + }, - { PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_JOIN_PRIMARY, - &fsm_prepare_replication }, + { + PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_JOIN_PRIMARY, + &fsm_prepare_replication + }, - { PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_WAIT_PRIMARY, - &fsm_disable_sync_rep }, + { + PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_WAIT_PRIMARY, + &fsm_disable_sync_rep + }, - { JOIN_PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_WAIT_PRIMARY, - &fsm_disable_sync_rep }, + { + JOIN_PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_WAIT_PRIMARY, + &fsm_disable_sync_rep + }, - { WAIT_PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_JOIN_PRIMARY, - &fsm_prepare_replication }, + { + WAIT_PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_JOIN_PRIMARY, + &fsm_prepare_replication + }, /* * Situation is getting back to normal on the primary */ - { WAIT_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_WAIT_PRIMARY_TO_PRIMARY, - &fsm_enable_sync_rep }, + { + WAIT_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_WAIT_PRIMARY_TO_PRIMARY, + &fsm_enable_sync_rep + }, - { JOIN_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_JOIN_PRIMARY_TO_PRIMARY, - &fsm_enable_sync_rep }, + { + JOIN_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_JOIN_PRIMARY_TO_PRIMARY, + &fsm_enable_sync_rep + }, - { DEMOTE_TIMEOUT_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_DEMOTE_TO_PRIMARY, - &fsm_start_postgres }, + { + DEMOTE_TIMEOUT_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_DEMOTE_TO_PRIMARY, + &fsm_start_postgres + }, /* * The primary is now ready to accept a standby, we're the standby */ - { WAIT_STANDBY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_WAIT_STANDBY_TO_CATCHINGUP, - &fsm_init_standby }, + { + WAIT_STANDBY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_WAIT_STANDBY_TO_CATCHINGUP, + &fsm_init_standby + }, - { DEMOTED_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_CATCHINGUP, - &fsm_rewind_or_init }, + { + DEMOTED_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_CATCHINGUP, + &fsm_rewind_or_init + }, - { SECONDARY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_CATCHINGUP, - &fsm_follow_new_primary }, + { + SECONDARY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_CATCHINGUP, + &fsm_follow_new_primary + }, /* * We're asked to be a standby. */ - { CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_CITUS_ANY, - COMMENT_CATCHINGUP_TO_SECONDARY, - &fsm_citus_maintain_replication_slots }, + { + CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_CITUS_ANY, + COMMENT_CATCHINGUP_TO_SECONDARY, + &fsm_citus_maintain_replication_slots + }, - { CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_CATCHINGUP_TO_SECONDARY, - &fsm_prepare_for_secondary }, + { + CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_CATCHINGUP_TO_SECONDARY, + &fsm_prepare_for_secondary + }, /* * The standby is asked to prepare its own promotion */ - { SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_citus_worker_prepare_standby_for_promotion }, + { + SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_citus_worker_prepare_standby_for_promotion + }, - { SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_prepare_standby_for_promotion }, + { + SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_prepare_standby_for_promotion + }, - { CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_citus_worker_prepare_standby_for_promotion }, + { + CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_citus_worker_prepare_standby_for_promotion + }, - { CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_prepare_standby_for_promotion }, + { + CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_prepare_standby_for_promotion + }, /* * Forcefully stop replication by stopping the server. */ - { PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_PROMOTION_TO_STOP_REPLICATION, - &fsm_citus_worker_stop_replication }, + { + PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_PROMOTION_TO_STOP_REPLICATION, + &fsm_citus_worker_stop_replication + }, - { PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_ANY, - COMMENT_PROMOTION_TO_STOP_REPLICATION, - &fsm_stop_replication }, + { + PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_ANY, + COMMENT_PROMOTION_TO_STOP_REPLICATION, + &fsm_stop_replication + }, /* * finish the promotion */ - { STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_primary }, + { + STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_primary + }, - { STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, - &fsm_citus_worker_promote_standby_to_primary }, + { + STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, + &fsm_citus_worker_promote_standby_to_primary + }, - { STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, - &fsm_promote_standby_to_primary }, + { + STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, + &fsm_promote_standby_to_primary + }, - { PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_BLOCKED_WRITES, - &fsm_citus_worker_promote_standby }, + { + PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_BLOCKED_WRITES, + &fsm_citus_worker_promote_standby + }, - { PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_BLOCKED_WRITES, - &fsm_promote_standby }, + { + PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_BLOCKED_WRITES, + &fsm_promote_standby + }, /* * Just wait until primary is ready */ - { INIT_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_WAIT_STANDBY, - NULL }, + { + INIT_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_WAIT_STANDBY, + NULL + }, - { DROPPED_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_WAIT_STANDBY, - NULL }, + { + DROPPED_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_WAIT_STANDBY, + NULL + }, /* * When losing a monitor and then connecting to a new monitor as a * secondary, we need to be able to follow the init sequence again. */ - { SECONDARY_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, - COMMENT_SECONARY_TO_WAIT_STANDBY, - NULL }, + { + SECONDARY_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, + COMMENT_SECONARY_TO_WAIT_STANDBY, + NULL + }, /* * In case of maintenance of the standby server, we stop PostgreSQL. */ - { SECONDARY_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, - NULL }, + { + SECONDARY_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, + NULL + }, - { CATCHINGUP_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, - NULL }, + { + CATCHINGUP_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, + NULL + }, - { SECONDARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_MAINTENANCE, - &fsm_start_maintenance_on_standby }, + { + SECONDARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_MAINTENANCE, + &fsm_start_maintenance_on_standby + }, - { CATCHINGUP_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_MAINTENANCE, - &fsm_start_maintenance_on_standby }, + { + CATCHINGUP_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_MAINTENANCE, + &fsm_start_maintenance_on_standby + }, - { WAIT_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_MAINTENANCE, - &fsm_start_maintenance_on_standby }, + { + WAIT_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_MAINTENANCE, + &fsm_start_maintenance_on_standby + }, - { MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_MAINTENANCE_TO_CATCHINGUP, - &fsm_restart_standby }, + { + MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_MAINTENANCE_TO_CATCHINGUP, + &fsm_restart_standby + }, - { PREPARE_MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_MAINTENANCE_TO_CATCHINGUP, - &fsm_restart_standby }, + { + PREPARE_MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_MAINTENANCE_TO_CATCHINGUP, + &fsm_restart_standby + }, /* * Applying new replication/cluster settings (per node replication quorum, @@ -574,119 +728,167 @@ KeeperFSMTransition KeeperFSM[] = { * have to fetch the new value for synchronous_standby_names from the * monitor. */ - { PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_APPLY_SETTINGS, - NULL }, + { + PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_APPLY_SETTINGS, + NULL + }, - { WAIT_PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_APPLY_SETTINGS, - NULL }, + { + WAIT_PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_APPLY_SETTINGS, + NULL + }, - { APPLY_SETTINGS_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_APPLY_SETTINGS_TO_PRIMARY, - &fsm_enable_sync_rep }, + { + APPLY_SETTINGS_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_APPLY_SETTINGS_TO_PRIMARY, + &fsm_enable_sync_rep + }, - { APPLY_SETTINGS_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + APPLY_SETTINGS_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, - { APPLY_SETTINGS_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_WAIT_PRIMARY, - &fsm_disable_sync_rep }, + { + APPLY_SETTINGS_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_WAIT_PRIMARY, + &fsm_disable_sync_rep + }, - { APPLY_SETTINGS_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_JOIN_PRIMARY, - &fsm_prepare_replication }, + { + APPLY_SETTINGS_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_JOIN_PRIMARY, + &fsm_prepare_replication + }, /* * In case of multiple standbys, failover begins with reporting current LSN */ - { SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { CATCHINGUP_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + CATCHINGUP_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { PREPARE_MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + PREPARE_MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_REPORT_LSN_TO_PREP_PROMOTION, - &fsm_citus_worker_prepare_standby_for_promotion }, + { + REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_REPORT_LSN_TO_PREP_PROMOTION, + &fsm_citus_worker_prepare_standby_for_promotion + }, - { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_PREP_PROMOTION, - &fsm_prepare_standby_for_promotion }, + { + REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_PREP_PROMOTION, + &fsm_prepare_standby_for_promotion + }, - { REPORT_LSN_STATE, FAST_FORWARD_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_FAST_FORWARD, - &fsm_fast_forward }, + { + REPORT_LSN_STATE, FAST_FORWARD_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_FAST_FORWARD, + &fsm_fast_forward + }, - { FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_ANY, - COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, - &fsm_citus_cleanup_and_resume_as_primary }, + { + FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_ANY, + COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, + &fsm_citus_cleanup_and_resume_as_primary + }, - { FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, - &fsm_cleanup_as_primary }, + { + FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, + &fsm_cleanup_as_primary + }, - { REPORT_LSN_STATE, JOIN_SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, - &fsm_checkpoint_and_stop_postgres }, + { + REPORT_LSN_STATE, JOIN_SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, + &fsm_checkpoint_and_stop_postgres + }, - { REPORT_LSN_STATE, SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, - &fsm_follow_new_primary }, + { + REPORT_LSN_STATE, SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, + &fsm_follow_new_primary + }, - { JOIN_SECONDARY_STATE, SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_JOIN_SECONDARY_TO_SECONDARY, - &fsm_follow_new_primary }, + { + JOIN_SECONDARY_STATE, SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_JOIN_SECONDARY_TO_SECONDARY, + &fsm_follow_new_primary + }, /* * When an old primary gets back online and reaches draining/draining, if a * failover is on-going then have it join the selection process. */ - { DRAINING_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_DRAINING_TO_REPORT_LSN, - &fsm_report_lsn_and_drop_replication_slots }, + { + DRAINING_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_DRAINING_TO_REPORT_LSN, + &fsm_report_lsn_and_drop_replication_slots + }, - { DEMOTED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_REPORT_LSN, - &fsm_report_lsn_and_drop_replication_slots }, + { + DEMOTED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_REPORT_LSN, + &fsm_report_lsn_and_drop_replication_slots + }, /* * When adding a new node and there is no primary, but there are existing * nodes that are not candidates for failover. */ - { INIT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_REPORT_LSN, - &fsm_init_from_standby }, + { + INIT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_REPORT_LSN, + &fsm_init_from_standby + }, /* * Dropping a node is a two-step process */ - { ANY_STATE, DROPPED_STATE, NODE_KIND_CITUS_ANY, - COMMENT_ANY_TO_DROPPED, - &fsm_citus_drop_node }, + { + ANY_STATE, DROPPED_STATE, NODE_KIND_CITUS_ANY, + COMMENT_ANY_TO_DROPPED, + &fsm_citus_drop_node + }, - { ANY_STATE, DROPPED_STATE, NODE_KIND_ANY, - COMMENT_ANY_TO_DROPPED, - &fsm_drop_node }, + { + ANY_STATE, DROPPED_STATE, NODE_KIND_ANY, + COMMENT_ANY_TO_DROPPED, + &fsm_drop_node + }, /* * This is the end, my friend. */ - { NO_STATE, NO_STATE, NODE_KIND_ANY, - NULL, - NULL }, + { + NO_STATE, NO_STATE, NODE_KIND_ANY, + NULL, + NULL + }, }; diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index 6315cd00b..d0cb4e351 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -312,9 +312,10 @@ keeper_ensure_current_state(Keeper *keeper) case MAINTENANCE_STATE: default: - + { /* nothing to be done here */ return true; + } } /* should never happen */ @@ -2077,12 +2078,33 @@ keeper_update_group_hba(Keeper *keeper, NodeAddressArray *diffNodesArray) sformat(hbaFilePath, MAXPGPATH, "%s/pg_hba.conf", postgresSetup->pgdata); + /* + * With cert auth, replication connections from standbys use the same + * client certificate (CN=autoctl_node) as monitor connections. The + * database user for replication is pgautofailover_replicator, so cert + * auth needs an ident map to bridge the two names. + */ + bool isCert = (strcmp(authMethod, "cert") == 0); + const char *replAuth = isCert ? "cert map=pgautofailover" : authMethod; + + if (isCert) + { + if (!pghba_ensure_ident_map_entry(postgresSetup->pgdata, + "pgautofailover", + PG_AUTOCTL_MONITOR_USERNAME, + PG_AUTOCTL_REPLICA_USERNAME)) + { + log_error("Failed to add cert ident map entry to pg_ident.conf"); + return false; + } + } + if (!pghba_ensure_host_rules_exist(hbaFilePath, diffNodesArray, postgresSetup->ssl.active, postgresSetup->dbname, PG_AUTOCTL_REPLICA_USERNAME, - authMethod, + replAuth, keeper->config.pgSetup.hbaLevel)) { log_error("Failed to edit HBA file \"%s\" to update rules to current " @@ -2739,8 +2761,8 @@ keeper_config_accept_new(Keeper *keeper, KeeperConfig *newConfig) newConfig->prepare_promotion_walreceiver; } - if (newConfig->postgresql_restart_failure_timeout != - config->postgresql_restart_failure_timeout) + if (newConfig->postgresql_restart_failure_timeout != config-> + postgresql_restart_failure_timeout) { log_info( "Reloading configuration: timeout.postgresql_restart_failure_timeout " @@ -2752,8 +2774,8 @@ keeper_config_accept_new(Keeper *keeper, KeeperConfig *newConfig) newConfig->postgresql_restart_failure_timeout; } - if (newConfig->postgresql_restart_failure_max_retries != - config->postgresql_restart_failure_max_retries) + if (newConfig->postgresql_restart_failure_max_retries != config-> + postgresql_restart_failure_max_retries) { log_info( "Reloading configuration: retries.postgresql_restart_failure_max_retries " diff --git a/src/bin/pg_autoctl/keeper_config.c b/src/bin/pg_autoctl/keeper_config.c index 1459336df..553819ee3 100644 --- a/src/bin/pg_autoctl/keeper_config.c +++ b/src/bin/pg_autoctl/keeper_config.c @@ -133,6 +133,11 @@ config->replication_password, \ REPLICATION_PASSWORD_DEFAULT) +#define OPTION_AUTOCTL_MONITOR_PASSWORD(config) \ + make_strbuf_option_default("pg_autoctl", "monitor_password", NULL, \ + false, MAXCONNINFO, \ + config->monitor_password, "") + #define OPTION_REPLICATION_MAXIMUM_BACKUP_RATE(config) \ make_strbuf_option_default("replication", "maximum_backup_rate", NULL, \ false, MAXIMUM_BACKUP_RATE_LEN, \ @@ -241,6 +246,7 @@ OPTION_REPLICATION_MAXIMUM_BACKUP_RATE(config), \ OPTION_REPLICATION_BACKUP_DIR(config), \ OPTION_REPLICATION_PASSWORD(config), \ + OPTION_AUTOCTL_MONITOR_PASSWORD(config), \ OPTION_TIMEOUT_NETWORK_PARTITION(config), \ OPTION_TIMEOUT_PREPARE_PROMOTION_CATCHUP(config), \ OPTION_TIMEOUT_PREPARE_PROMOTION_WALRECEIVER(config), \ diff --git a/src/bin/pg_autoctl/keeper_config.h b/src/bin/pg_autoctl/keeper_config.h index 3826923d9..73e6cbd62 100644 --- a/src/bin/pg_autoctl/keeper_config.h +++ b/src/bin/pg_autoctl/keeper_config.h @@ -53,6 +53,7 @@ typedef struct KeeperConfig /* PostgreSQL replication / tooling setup */ char replication_slot_name[MAXCONNINFO]; char replication_password[MAXCONNINFO]; + char monitor_password[MAXCONNINFO]; char maximum_backup_rate[MAXIMUM_BACKUP_RATE_LEN]; char backupDirectory[MAXPGPATH]; diff --git a/src/bin/pg_autoctl/keeper_pg_init.c b/src/bin/pg_autoctl/keeper_pg_init.c index 3ff3e555e..043d589f7 100644 --- a/src/bin/pg_autoctl/keeper_pg_init.c +++ b/src/bin/pg_autoctl/keeper_pg_init.c @@ -566,11 +566,12 @@ reach_initial_state(Keeper *keeper) } default: - + { /* we don't support any other state at initialization time */ log_error("reach_initial_state: don't know how to read state %s", NodeStateToString(keeper->state.assigned_role)); return false; + } } /* diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 496793951..a781800c6 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -373,11 +373,17 @@ monitor_get_nodes(Monitor *monitor, char *formation, int groupId, const char *paramValues[2] = { 0 }; NodeAddressArrayParseContext parseContext = { { 0 }, nodeArray, false }; + /* + * Declared at function scope so it outlives the if-block below; paramValues[1] + * points into its strValue member and must remain valid for pgsql_execute_with_params. + */ + IntString myGroupIdString; + paramValues[0] = formation; if (groupId > -1) { - IntString myGroupIdString = intToString(groupId); + myGroupIdString = intToString(groupId); ++paramCount; paramValues[1] = myGroupIdString.strValue; @@ -429,12 +435,13 @@ monitor_print_nodes_as_json(Monitor *monitor, char *formation, int groupId) int paramCount = 1; Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2] = { 0 }; + IntString myGroupIdString; paramValues[0] = formation; if (groupId > -1) { - IntString myGroupIdString = intToString(groupId); + myGroupIdString = intToString(groupId); ++paramCount; paramValues[1] = myGroupIdString.strValue; @@ -847,7 +854,7 @@ monitor_register_node(Monitor *monitor, char *formation, { { 0 }, assignedState, false }; const char *nodeStateString = NodeStateToString(initialState); IntString portStr = intToString(port); - IntString systemIdentifierStr = intToString(system_identifier); + IntString sysIdStr = intToString(system_identifier); IntString desiredNodeIdStr = intToString(desiredNodeId); IntString desiredGroupIdStr = intToString(desiredGroupId); IntString candidatePriorityStr = intToString(candidatePriority); @@ -857,7 +864,7 @@ monitor_register_node(Monitor *monitor, char *formation, paramValues[2] = portStr.strValue; paramValues[3] = dbname; paramValues[4] = name == NULL ? "" : name; - paramValues[5] = systemIdentifierStr.strValue; + paramValues[5] = sysIdStr.strValue; paramValues[6] = desiredNodeIdStr.strValue; paramValues[7] = desiredGroupIdStr.strValue; paramValues[8] = nodeStateString; @@ -950,16 +957,16 @@ monitor_node_active(Monitor *monitor, MonitorAssignedStateParseContext parseContext = { { 0 }, assignedState, false }; const char *nodeStateString = NodeStateToString(currentState); - IntString nodeIdStr = intToString(nodeId); - IntString groupIdStr = intToString(groupId); - IntString currentTLIStr = intToString(currentTLI); + IntString nodeIdString = intToString(nodeId); + IntString groupIdString = intToString(groupId); + IntString currentTLIString = intToString(currentTLI); paramValues[0] = formation; - paramValues[1] = nodeIdStr.strValue; - paramValues[2] = groupIdStr.strValue; + paramValues[1] = nodeIdString.strValue; + paramValues[2] = groupIdString.strValue; paramValues[3] = nodeStateString; paramValues[4] = pgIsRunning ? "true" : "false"; - paramValues[5] = currentTLIStr.strValue; + paramValues[5] = currentTLIString.strValue; paramValues[6] = currentLSN; paramValues[7] = pgsrSyncState; @@ -1009,8 +1016,8 @@ monitor_set_node_candidate_priority(Monitor *monitor, int paramCount = 3; Oid paramTypes[3] = { TEXTOID, TEXTOID, INT4OID }; const char *paramValues[3]; - IntString candidatePriorityStr = intToString(candidate_priority); - char *candidatePriorityText = candidatePriorityStr.strValue; + IntString candidatePriorityString = intToString(candidate_priority); + char *candidatePriorityText = candidatePriorityString.strValue; bool success = true; paramValues[0] = formation; @@ -1227,10 +1234,9 @@ monitor_set_formation_number_sync_standbys(Monitor *monitor, char *formation, Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2]; SingleValueResultContext parseContext = { { 0 }, PGSQL_RESULT_BOOL, false }; - IntString numberSyncStandbysStr = intToString(numberSyncStandbys); - + IntString numberSyncStandbysString = intToString(numberSyncStandbys); paramValues[0] = formation; - paramValues[1] = numberSyncStandbysStr.strValue; + paramValues[1] = numberSyncStandbysString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -1267,10 +1273,10 @@ monitor_remove_by_hostname(Monitor *monitor, char *host, int port, bool force, int paramCount = 3; Oid paramTypes[3] = { TEXTOID, INT4OID, BOOLOID }; const char *paramValues[3]; - IntString portStr = intToString(port); + IntString portString = intToString(port); paramValues[0] = host; - paramValues[1] = portStr.strValue; + paramValues[1] = portString.strValue; paramValues[2] = force ? "true" : "false"; if (!pgsql_execute_with_params(pgsql, sql, @@ -1507,10 +1513,10 @@ monitor_perform_failover(Monitor *monitor, char *formation, int group) int paramCount = 2; Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2]; - IntString groupStr = intToString(group); + IntString groupString = intToString(group); paramValues[0] = formation; - paramValues[1] = groupStr.strValue; + paramValues[1] = groupString.strValue; /* * pgautofailover.perform_failover() returns VOID. @@ -2751,13 +2757,13 @@ monitor_create_formation(Monitor *monitor, int paramCount = 5; Oid paramTypes[5] = { TEXTOID, TEXTOID, TEXTOID, BOOLOID, INT4OID }; const char *paramValues[5]; - IntString numberSyncStandbysStr = intToString(numberSyncStandbys); + IntString numberSyncStandbysString = intToString(numberSyncStandbys); paramValues[0] = formation; paramValues[1] = kind; paramValues[2] = dbname; paramValues[3] = hasSecondary ? "true" : "false"; - paramValues[4] = numberSyncStandbysStr.strValue; + paramValues[4] = numberSyncStandbysString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3443,13 +3449,13 @@ monitor_update_node_metadata(Monitor *monitor, const char *paramValues[4]; SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BOOL, false }; - IntString nodeIdStr = intToString(nodeId); - IntString portStr = intToString(port); + IntString nodeIdString = intToString(nodeId); + IntString portString = intToString(port); - paramValues[0] = nodeIdStr.strValue; + paramValues[0] = nodeIdString.strValue; paramValues[1] = name; paramValues[2] = hostname; - paramValues[3] = portStr.strValue; + paramValues[3] = portString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3493,11 +3499,11 @@ monitor_set_node_system_identifier(Monitor *monitor, NodeAddress node = { 0 }; NodeAddressParseContext parseContext = { { 0 }, &node, false }; - IntString nodeIdStr = intToString(nodeId); - IntString systemIdentifierStr = intToString(system_identifier); + IntString nodeIdString = intToString(nodeId); + IntString systemIdentifierString = intToString(system_identifier); - paramValues[0] = nodeIdStr.strValue; - paramValues[1] = systemIdentifierStr.strValue; + paramValues[0] = nodeIdString.strValue; + paramValues[1] = systemIdentifierString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3543,11 +3549,11 @@ monitor_set_group_system_identifier(Monitor *monitor, const char *paramValues[2]; SingleValueResultContext context = { 0 }; - IntString groupIdStr = intToString(groupId); - IntString systemIdentifierStr = intToString(system_identifier); + IntString groupIdString = intToString(groupId); + IntString systemIdentifierString = intToString(system_identifier); - paramValues[0] = groupIdStr.strValue; - paramValues[1] = systemIdentifierStr.strValue; + paramValues[0] = groupIdString.strValue; + paramValues[1] = systemIdentifierString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3660,9 +3666,9 @@ monitor_start_maintenance(Monitor *monitor, int64_t nodeId, bool *mayRetry) int paramCount = 1; Oid paramTypes[1] = { INT8OID }; const char *paramValues[1]; - IntString nodeIdStr = intToString(nodeId); + IntString nodeIdString = intToString(nodeId); - paramValues[0] = nodeIdStr.strValue; + paramValues[0] = nodeIdString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3709,9 +3715,9 @@ monitor_stop_maintenance(Monitor *monitor, int64_t nodeId, bool *mayRetry) int paramCount = 1; Oid paramTypes[1] = { INT8OID }; const char *paramValues[1]; - IntString nodeIdStr = intToString(nodeId); + IntString nodeIdString = intToString(nodeId); - paramValues[0] = nodeIdStr.strValue; + paramValues[0] = nodeIdString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -4876,12 +4882,12 @@ monitor_find_node_by_nodeid(Monitor *monitor, const char *paramValues[3]; NodeAddressArrayParseContext parseContext = { { 0 }, nodesArray, false }; - IntString groupIdStr = intToString(groupId); - IntString nodeIdStr = intToString(nodeId); + IntString groupIdString = intToString(groupId); + IntString nodeIdString = intToString(nodeId); paramValues[0] = formation; - paramValues[1] = groupIdStr.strValue; - paramValues[2] = nodeIdStr.strValue; + paramValues[1] = groupIdString.strValue; + paramValues[2] = nodeIdString.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, diff --git a/src/bin/pg_autoctl/monitor_config.c b/src/bin/pg_autoctl/monitor_config.c index 77021471e..ce88349b8 100644 --- a/src/bin/pg_autoctl/monitor_config.c +++ b/src/bin/pg_autoctl/monitor_config.c @@ -36,6 +36,11 @@ make_strbuf_option("pg_autoctl", "hostname", "hostname", \ false, _POSIX_HOST_NAME_MAX, config->hostname) +#define OPTION_AUTOCTL_NODE_PASSWORD(config) \ + make_strbuf_option_default("pg_autoctl", "autoctl_node_password", NULL, \ + false, MAXCONNINFO, \ + config->autoctl_node_password, "") + #define OPTION_AUTOCTL_NODENAME(config) \ make_strbuf_compat_option("pg_autoctl", "nodename", \ _POSIX_HOST_NAME_MAX, config->hostname) @@ -104,6 +109,7 @@ OPTION_AUTOCTL_ROLE(config), \ OPTION_AUTOCTL_HOSTNAME(config), \ OPTION_AUTOCTL_NODENAME(config), \ + OPTION_AUTOCTL_NODE_PASSWORD(config), \ OPTION_POSTGRESQL_PGDATA(config), \ OPTION_POSTGRESQL_PG_CTL(config), \ OPTION_POSTGRESQL_USERNAME(config), \ diff --git a/src/bin/pg_autoctl/monitor_config.h b/src/bin/pg_autoctl/monitor_config.h index c7292d992..97c6eee54 100644 --- a/src/bin/pg_autoctl/monitor_config.h +++ b/src/bin/pg_autoctl/monitor_config.h @@ -25,12 +25,19 @@ typedef struct MonitorConfig /* pg_autoctl setup */ char hostname[_POSIX_HOST_NAME_MAX]; + char autoctl_node_password[MAXCONNINFO]; /* PostgreSQL setup */ char role[NAMEDATALEN]; /* PostgreSQL setup */ PostgresSetup pgSetup; + + /* non-default formations to create during monitor init */ +#define MONITOR_MAX_FORMATIONS 16 + int formationCount; + char formationNames[MONITOR_MAX_FORMATIONS][NAMEDATALEN]; + char formationKinds[MONITOR_MAX_FORMATIONS][NAMEDATALEN]; /* "pgsql" = default */ } MonitorConfig; diff --git a/src/bin/pg_autoctl/monitor_pg_init.c b/src/bin/pg_autoctl/monitor_pg_init.c index 2835c7f3a..839cd5111 100644 --- a/src/bin/pg_autoctl/monitor_pg_init.c +++ b/src/bin/pg_autoctl/monitor_pg_init.c @@ -139,7 +139,8 @@ monitor_pg_init(Monitor *monitor) */ bool monitor_install(const char *hostname, - PostgresSetup pgSetupOption, bool checkSettings) + PostgresSetup pgSetupOption, bool checkSettings, + const char *autoctl_node_password) { PostgresSetup pgSetup = { 0 }; bool missingPgdataIsOk = false; @@ -216,6 +217,23 @@ monitor_install(const char *hostname, return false; } + /* + * If a password for the autoctl_node role has been configured, set it now. + * The autoctl_node role is created by the pgautofailover extension (via + * CREATE EXTENSION above), so we ALTER it here after the extension exists. + */ + if (autoctl_node_password != NULL && autoctl_node_password[0] != '\0') + { + if (!pgsql_alter_role_password(&postgres.sqlClient, + PG_AUTOCTL_MONITOR_USERNAME, + autoctl_node_password)) + { + log_error("Failed to set password for role \"%s\"", + PG_AUTOCTL_MONITOR_USERNAME); + return false; + } + } + /* * When installing the monitor on-top of an already running PostgreSQL, we * want to check that our settings have been applied already, and warn the diff --git a/src/bin/pg_autoctl/monitor_pg_init.h b/src/bin/pg_autoctl/monitor_pg_init.h index 38a552c65..aed2b521e 100644 --- a/src/bin/pg_autoctl/monitor_pg_init.h +++ b/src/bin/pg_autoctl/monitor_pg_init.h @@ -18,7 +18,8 @@ bool monitor_pg_init(Monitor *monitor); bool monitor_install(const char *hostname, PostgresSetup pgSetupOption, - bool checkSettings); + bool checkSettings, + const char *autoctl_node_password); bool monitor_add_postgres_default_settings(Monitor *monitor); #endif /* MONITOR_PG_INIT_H */ diff --git a/src/bin/pg_autoctl/nodespec.c b/src/bin/pg_autoctl/nodespec.c index 5f61e6d91..54d142c82 100644 --- a/src/bin/pg_autoctl/nodespec.c +++ b/src/bin/pg_autoctl/nodespec.c @@ -114,8 +114,7 @@ nodespec_read(const char *path, NodeSpec *spec) sizeof(replicationQuorumStr), replicationQuorumStr, "true"), - /* [options] — ssl is mutable (applied via `pg_autoctl enable ssl`); - * auth and pg_hba_lan are create-time only */ + /* [options] — immutable, used only at create time */ make_strbuf_option_default("options", "ssl", NULL, false, sizeof(spec->ssl), spec->ssl, "self-signed"), @@ -684,19 +683,15 @@ nodespec_write_to_path(const NodeSpec *spec, const char *path) * nodespec_apply compares new_spec against old_spec and applies any changes * to the mutable fields by calling into the keeper / monitor APIs. * - * Mutable fields: - * - candidate_priority → pg_autoctl set node candidate-priority - * - replication_quorum → pg_autoctl set node replication-quorum - * - ssl / ssl_*_file → pg_autoctl enable ssl - * - monitor_pguri → pg_autoctl disable monitor --force - * pg_autoctl enable monitor - * - * Immutable fields (kind, pgdata, hostname, port, auth, - * pg_hba_lan) require a node restart to take effect. + * Currently mutable fields: + * - candidate_priority → monitor_set_node_candidate_priority() + * - replication_quorum → monitor_set_node_replication_quorum() * * The [launch] mode field is handled separately by pg_autoctl node start. * Applying a spec with mode=deferred to an already-started node is a * non-fatal warning (ignored). + * + * Immutable fields (kind, pgdata, ssl, auth, pg_hba_lan) are not checked here. */ bool nodespec_apply(const NodeSpec *new_spec, const NodeSpec *old_spec) @@ -757,122 +752,6 @@ nodespec_apply(const NodeSpec *new_spec, const NodeSpec *old_spec) free_program(&prog); } - /* - * Monitor URI changed: disable the current monitor (removing this node - * from it) then re-register to the new one. The --force flag allows - * disable to proceed even if the old monitor is unreachable. - */ - if (strcmp(new_spec->monitor_pguri, old_spec->monitor_pguri) != 0 && - !IS_EMPTY_STRING_BUFFER(new_spec->monitor_pguri)) - { - Program disable_prog = run_program(pg_autoctl_program, - "disable", "monitor", - "--force", - "--pgdata", new_spec->pgdata, - NULL); - - if (disable_prog.returnCode != 0) - { - log_warn("nodespec_apply: disable monitor failed (rc=%d)", - disable_prog.returnCode); - if (disable_prog.stdOut) - { - log_warn("%s", disable_prog.stdOut); - } - free_program(&disable_prog); - } - else - { - free_program(&disable_prog); - - Program enable_prog = run_program(pg_autoctl_program, - "enable", "monitor", - "--pgdata", new_spec->pgdata, - new_spec->monitor_pguri, - NULL); - - if (enable_prog.returnCode != 0) - { - log_warn("nodespec_apply: enable monitor \"%s\" failed (rc=%d)", - new_spec->monitor_pguri, enable_prog.returnCode); - if (enable_prog.stdOut) - { - log_warn("%s", enable_prog.stdOut); - } - } - else - { - log_info("nodespec: applied monitor_pguri = %s", - new_spec->monitor_pguri); - changed = true; - } - free_program(&enable_prog); - } - } - - /* - * SSL mode or certificate paths changed: call `pg_autoctl enable ssl` - * with the appropriate flags so Postgres is reconfigured live. - * - * ssl=off maps to --no-ssl; ssl=self-signed maps to --ssl-self-signed; - * anything else is a CA-verified mode and requires the cert paths. - */ - { - bool ssl_changed = - (strcmp(new_spec->ssl, old_spec->ssl) != 0) || - (strcmp(new_spec->ssl_ca_file, old_spec->ssl_ca_file) != 0) || - (strcmp(new_spec->ssl_cert_file, old_spec->ssl_cert_file) != 0) || - (strcmp(new_spec->ssl_key_file, old_spec->ssl_key_file) != 0); - - if (ssl_changed) - { - Program prog; - - if (strcmp(new_spec->ssl, "off") == 0) - { - prog = run_program(pg_autoctl_program, - "enable", "ssl", - "--pgdata", new_spec->pgdata, - "--no-ssl", NULL); - } - else if (strcmp(new_spec->ssl, "self-signed") == 0 || - IS_EMPTY_STRING_BUFFER(new_spec->ssl_ca_file)) - { - prog = run_program(pg_autoctl_program, - "enable", "ssl", - "--pgdata", new_spec->pgdata, - "--ssl-self-signed", NULL); - } - else - { - prog = run_program(pg_autoctl_program, - "enable", "ssl", - "--pgdata", new_spec->pgdata, - "--ssl-mode", new_spec->ssl, - "--ssl-ca-file", new_spec->ssl_ca_file, - "--server-cert", new_spec->ssl_cert_file, - "--server-key", new_spec->ssl_key_file, - NULL); - } - - if (prog.returnCode != 0) - { - log_warn("nodespec_apply: enable ssl failed (rc=%d)", - prog.returnCode); - if (prog.stdOut) - { - log_warn("%s", prog.stdOut); - } - } - else - { - log_info("nodespec: applied ssl = %s", new_spec->ssl); - changed = true; - } - free_program(&prog); - } - } - /* immediate → deferred on an already-started node: non-fatal, ignored */ if (!old_spec->launchDeferred && new_spec->launchDeferred) { diff --git a/src/bin/pg_autoctl/nodespec.h b/src/bin/pg_autoctl/nodespec.h index 128ef15fb..3328643f1 100644 --- a/src/bin/pg_autoctl/nodespec.h +++ b/src/bin/pg_autoctl/nodespec.h @@ -59,9 +59,7 @@ typedef struct NodeSpec /* [postgresql] */ char pgdata[MAXPGPATH]; - /* [monitor] — empty for kind == monitor - * monitor_pguri is mutable: changing it triggers disable monitor --force - * followed by enable monitor (re-registers without restarting). */ + /* [monitor] — empty for kind == monitor */ char monitor_pguri[MAXCONNINFO]; bool noMonitor; /* [monitor] no_monitor=true: standalone mode */ int nodeId; /* [monitor] node_id: required with --disable-monitor */ diff --git a/src/bin/pg_autoctl/primary_standby.c b/src/bin/pg_autoctl/primary_standby.c index 96a171978..d433e46e5 100644 --- a/src/bin/pg_autoctl/primary_standby.c +++ b/src/bin/pg_autoctl/primary_standby.c @@ -13,6 +13,7 @@ #include "postgres_fe.h" #include "config.h" +#include "env_utils.h" #include "file_utils.h" #include "keeper.h" #include "log.h" @@ -440,6 +441,8 @@ upstream_has_replication_slot(ReplicationSource *upstream, PostgresSetup upstreamSetup = { 0 }; PGSQL upstreamClient = { 0 }; char connectionString[MAXCONNINFO] = { 0 }; + char savedPgPassword[BUFSIZE] = { 0 }; + bool passwordSet = false; /* prepare a PostgresSetup that allows preparing a connection string */ strlcpy(upstreamSetup.username, PG_AUTOCTL_REPLICA_USERNAME, NAMEDATALEN); @@ -454,23 +457,45 @@ upstream_has_replication_slot(ReplicationSource *upstream, */ pg_setup_get_local_connection_string(&upstreamSetup, connectionString); - if (!pgsql_init(&upstreamClient, connectionString, PGSQL_CONN_UPSTREAM)) + /* + * When a replication password is configured (e.g. --auth md5), supply it + * via PGPASSWORD so that the connection to the upstream node can + * authenticate. Save and restore any existing value. + */ + if (!IS_EMPTY_STRING_BUFFER(upstream->password)) { - /* errors have already been logged */ - return false; + if (env_exists("PGPASSWORD")) + { + if (!get_env_copy("PGPASSWORD", savedPgPassword, sizeof(savedPgPassword))) + { + return false; + } + } + setenv("PGPASSWORD", upstream->password, 1); + passwordSet = true; } - if (!pgsql_replication_slot_exists(&upstreamClient, - upstream->slotName, - hasReplicationSlot)) + if (!pgsql_init(&upstreamClient, connectionString, PGSQL_CONN_UPSTREAM)) { /* errors have already been logged */ - PQfinish(upstreamClient.connection); + if (passwordSet) + { + setenv("PGPASSWORD", savedPgPassword, 1); + } return false; } + bool result = pgsql_replication_slot_exists(&upstreamClient, + upstream->slotName, + hasReplicationSlot); PQfinish(upstreamClient.connection); - return true; + + if (passwordSet) + { + setenv("PGPASSWORD", savedPgPassword, 1); + } + + return result; } diff --git a/src/bin/pg_autoctl/service_monitor_init.c b/src/bin/pg_autoctl/service_monitor_init.c index 06998b43c..02d8c9c9e 100644 --- a/src/bin/pg_autoctl/service_monitor_init.c +++ b/src/bin/pg_autoctl/service_monitor_init.c @@ -134,7 +134,8 @@ service_monitor_init_start(void *context, pid_t *pid) (void) set_ps_title(serviceName); /* finish the install if necessary */ - if (!monitor_install(config->hostname, *pgSetup, false)) + if (!monitor_install(config->hostname, *pgSetup, false, + config->autoctl_node_password)) { /* errors have already been logged */ exit(EXIT_CODE_INTERNAL_ERROR); @@ -142,6 +143,27 @@ service_monitor_init_start(void *context, pid_t *pid) log_info("Monitor has been successfully initialized."); + /* create any non-default formations requested via --formation */ + for (int fi = 0; fi < config->formationCount; fi++) + { + char *fname = config->formationNames[fi]; + char *fkind = config->formationKinds[fi][0] + ? config->formationKinds[fi] : "pgsql"; + + log_info("Creating formation \"%s\" (kind %s)", fname, fkind); + + if (!monitor_create_formation(monitor, fname, fkind, + DEFAULT_DATABASE_NAME, + + /* hasSecondary */ true, + + /* numberSyncStandbys */ 0)) + { + log_error("Failed to create formation \"%s\"", fname); + exit(EXIT_CODE_INTERNAL_ERROR); + } + } + if (createAndRun) { /* here we call execv() so we never get back */ diff --git a/src/bin/pg_autoctl/state.c b/src/bin/pg_autoctl/state.c index 553f3bef3..04cdb1d38 100644 --- a/src/bin/pg_autoctl/state.c +++ b/src/bin/pg_autoctl/state.c @@ -488,7 +488,9 @@ NodeStateToString(NodeState s) } default: + { return "Unknown State"; + } } } @@ -662,7 +664,9 @@ PreInitPostgreInstanceStateToString(PreInitPostgreInstanceState pgInitState) } default: + { return "unknown"; + } } /* keep compiler happy */ diff --git a/src/bin/pg_autoctl/watch.c b/src/bin/pg_autoctl/watch.c index eb29a6c81..c37c885d5 100644 --- a/src/bin/pg_autoctl/watch.c +++ b/src/bin/pg_autoctl/watch.c @@ -342,6 +342,7 @@ cli_watch_process_keys(WatchContext *context) } } } + /* left and right moves are conditionnal / relative */ else if (ch == KEY_LEFT || ch == ctrl('b') || ch == 'h') { @@ -365,6 +366,7 @@ cli_watch_process_keys(WatchContext *context) context->move = WATCH_MOVE_FOCUS_NONE; } } + /* left and right moves are conditionnal / relative */ else if (ch == KEY_RIGHT || ch == ctrl('f') || ch == 'l') { @@ -380,6 +382,7 @@ cli_watch_process_keys(WatchContext *context) context->move = WATCH_MOVE_FOCUS_NONE; } } + /* home and end moves are unconditionnal / absolute */ else if (ch == KEY_HOME || ch == ctrl('a') || ch == '0') { @@ -391,6 +394,7 @@ cli_watch_process_keys(WatchContext *context) { context->move = WATCH_MOVE_FOCUS_END; } + /* up is C-p in Emacs, k in vi(m) */ else if (ch == KEY_UP || ch == ctrl('p') || ch == 'k') { @@ -401,6 +405,7 @@ cli_watch_process_keys(WatchContext *context) --context->selectedRow; } } + /* page up, which is also C-u in the terminal with less/more etc */ else if (ch == KEY_PPAGE || ch == ctrl('u')) { @@ -413,6 +418,7 @@ cli_watch_process_keys(WatchContext *context) context->selectedRow -= 5; } } + /* down is C-n in Emacs, j in vi(m) */ else if (ch == KEY_DOWN || ch == ctrl('n') || ch == 'j') { @@ -423,6 +429,7 @@ cli_watch_process_keys(WatchContext *context) ++context->selectedRow; } } + /* page down, which is also C-d in the terminal with less/more etc */ else if (ch == KEY_NPAGE || ch == ctrl('d')) { @@ -436,6 +443,7 @@ cli_watch_process_keys(WatchContext *context) context->selectedRow += 5; } } + /* cancel current selected row */ else if (ch == KEY_DL || ch == KEY_DC) { @@ -561,8 +569,8 @@ cli_watch_render(WatchContext *context, WatchContext *previous) context->startCol != previous->startCol || context->cookedMode != previous->cookedMode || context->eventsArray.count != previous->eventsArray.count || - (context->eventsArray.events[0].eventId != - previous->eventsArray.events[0].eventId)) + (context->eventsArray.events[0].eventId != previous->eventsArray.events[0].eventId + )) { (void) clear_line_at(++printedRows); diff --git a/src/bin/pgaftest/test_runner.c b/src/bin/pgaftest/test_runner.c index 3e8b257dc..675c012e2 100644 --- a/src/bin/pgaftest/test_runner.c +++ b/src/bin/pgaftest/test_runner.c @@ -63,6 +63,58 @@ run_cmd(const char *fmt, ...) } +/* + * Run a shell command, capture both stdout and stderr into buf. + * Returns the exit code. The capture is implemented by appending "2>&1" to + * the shell command string and reading from popen(cmd, "r") — the shell + * redirects file descriptor 2 onto 1 before exec, so both streams arrive on + * the single pipe end that popen hands back to us. We read until EOF, trim + * trailing whitespace, and drain any overflow so pclose() doesn't see an + * unread pipe (which would SIGPIPE the child and make docker report exit 137). + */ +static int __attribute__((format(printf, 3, 4))) +run_cmd_capture_both(char *buf, int buflen, const char *fmt, ...) +{ + char inner[4096]; + va_list ap; + va_start(ap, fmt); + pg_vsnprintf(inner, sizeof(inner), fmt, ap); + va_end(ap); + + char cmd[4096 + 6]; /* room for " 2>&1" */ + sformat(cmd, sizeof(cmd), "%s 2>&1", inner); + + log_debug("$ %s", cmd); + + FILE *p = popen(cmd, "r"); + if (!p) + { + return -1; + } + + int pos = 0; + int c; + while ((c = fgetc(p)) != EOF && pos < buflen - 1) + { + buf[pos++] = (char) c; + } + buf[pos] = '\0'; + + while (pos > 0 && (buf[pos - 1] == '\n' || buf[pos - 1] == '\r' || + buf[pos - 1] == ' ')) + { + buf[--pos] = '\0'; + } + + while (c != EOF) + { + c = fgetc(p); + } + + return pclose(p); +} + + /* Run a shell command, capture stdout into buf */ static int __attribute__((format(printf, 3, 4))) run_cmd_capture(char *buf, int buflen, const char *fmt, ...) @@ -1038,6 +1090,57 @@ runner_wait_socket(TestRunner *r, int remainMs) } +/* + * Post-convergence notification flush for CMD_WAIT_MULTI. + * + * After the subprocess (or LISTEN) confirms all conditions are met, the + * corresponding NOTIFY messages may still be in transit: the monitor commits + * the state change and sends NOTIFY in one transaction, but TCP delivery on + * Docker Desktop for Mac can arrive tens to hundreds of milliseconds after + * the database commit that the subprocess reads. + * + * We loop, draining with marks on each pass, until either all listenSatisfied + * flags are set (every convergence NOTIFY has arrived) or 1 second elapses + * (generous safety cap — the caller already confirmed convergence, so we will + * return true regardless). + */ +static void +notify_flush_until_satisfied(TestRunner *r, + const char *const *mn, + const char *const *ms, + int count, + bool *satisfied) +{ + if (!r->notifyConnected) + { + return; + } + + time_t deadline = time(NULL) + 1; + + while (time(NULL) < deadline) + { + /* stop as soon as every convergence NOTIFY has arrived */ + bool allNotified = true; + for (int i = 0; i < count; i++) + { + if (!satisfied[i]) + { + allNotified = false; + break; + } + } + if (allNotified) + { + break; + } + + runner_wait_socket(r, 200); + runner_drain_notify(r, NULL, mn, ms, count, satisfied); + } +} + + /* * Check that the formation has converged: for each required state, at least * one cluster node must have both reportedstate = assignedstate = that state. @@ -1149,7 +1252,7 @@ monitor_wait_formation_states(TestRunner *r, runner_notify_connect(r); /* pointer arrays for drain marking — NULL node = wildcard (any node) */ - const char *ms[PGAF_MAX_WAIT_STATES]; + const char *ms[PGAF_MAX_WAIT_STATES] = { 0 }; for (int i = 0; i < stateCount && i < PGAF_MAX_WAIT_STATES; i++) { ms[i] = states[i]; @@ -1464,6 +1567,14 @@ runner_wait_notify_goal(TestRunner *r, runner_drain_notify(r, NULL, &nodeName, &targetState, 1, &satisfiedEarly); if (satisfiedEarly) { + /* + * Post-match flush: drain any notifications that arrived in the + * same TCP packet as the convergence event so they appear under + * this step, not interleaved into the next command's drain. + * No marks — these belong to the step that caused the transition, + * not this wait. + */ + runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); return true; } } @@ -2657,11 +2768,14 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) */ case CMD_WAIT_MULTI: { + runner_notify_connect(r); + int timeoutSecs = cmd->timeoutSeconds; time_t deadline = time(NULL) + timeoutSecs; /* build pointer arrays once — used for marking on every drain */ - const char *mn[PGAF_MAX_WAIT_STATES], *ms[PGAF_MAX_WAIT_STATES]; + const char *mn[PGAF_MAX_WAIT_STATES] = { 0 }; + const char *ms[PGAF_MAX_WAIT_STATES] = { 0 }; for (int i = 0; i < cmd->waitStateCount; i++) { mn[i] = cmd->waitNodes[i]; @@ -2672,10 +2786,22 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) * convergence notification arrives for a specific (node, state) pair */ bool listenSatisfied[PGAF_MAX_WAIT_STATES] = { false }; + /* + * Initial drain: consume any notifications buffered during the + * preceding command (e.g. exec that triggered state transitions). + * With marks so that buffered convergence events get '*' here, + * not silently during the next command's inter-drain. + */ + if (r->notifyConnected) + { + runner_drain_notify(r, NULL, mn, ms, + cmd->waitStateCount, listenSatisfied); + } + while (time(NULL) < deadline) { /* - * Try subprocess-based check first. monitor_get_node_state runs + * Try subprocess-based check. monitor_get_node_state runs * "pg_autoctl inspect monitor node-state" which requires v2.2. * When nodes run v2.1 the call returns false; track how many * succeed so we can fall back to LISTEN-based convergence. @@ -2690,7 +2816,6 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) assigned, sizeof(assigned))) { allMet = false; /* can't confirm via subprocess */ - /* don't break — try to drain anyway */ continue; } subproc_ok++; @@ -2710,11 +2835,18 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) } /* - * Drain: log notifications, mark '*' on convergence events, and - * set listenSatisfied[i] when the (node, state) pair converges. + * Drain with marks now — after the subprocess (which may have + * taken ~200ms) — so any notifications that arrived while we + * were polling get their '*' before we decide to return. This + * is the ordering that makes '*' reliable: drain first, check + * allMet / allListenMet second. */ - runner_drain_notify(r, NULL, mn, ms, - cmd->waitStateCount, listenSatisfied); + if (r->notifyConnected) + { + runner_drain_notify(r, NULL, mn, ms, + cmd->waitStateCount, listenSatisfied); + runner_notify_connect(r); + } /* * When subprocess is unavailable (v2.1 nodes), fall back to @@ -2734,14 +2866,21 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) } if (allListenMet) { + notify_flush_until_satisfied(r, mn, ms, + cmd->waitStateCount, + listenSatisfied); return true; } } else if (allMet) { + notify_flush_until_satisfied(r, mn, ms, + cmd->waitStateCount, + listenSatisfied); return true; } + /* wait for next notification */ runner_wait_socket(r, 1000); } @@ -2844,12 +2983,15 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) * Postgres. Equivalent to calling `pg_autoctl manual service pgctl off`. */ log_info("Stopping Postgres on %s", cmd->service); - int rc = run_cmd( - "%s exec %s pg_autoctl manual service pgctl off" + char pgctlOut[4096] = ""; + int rc = run_cmd_capture_both( + pgctlOut, sizeof(pgctlOut), + "%s exec -T %s pg_autoctl manual service pgctl off" " --pgdata /var/lib/postgres/pgaf", r->composeBase, cmd->service); if (rc != 0) { + log_info("%s", pgctlOut); sformat(errBuf, errLen, "stop postgres %s failed (exit %d)", cmd->service, rc); @@ -2861,12 +3003,15 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) case CMD_START_POSTGRES: { log_info("Starting Postgres on %s", cmd->service); - int rc = run_cmd( - "%s exec %s pg_autoctl manual service pgctl on" + char pgctlOut[4096] = ""; + int rc = run_cmd_capture_both( + pgctlOut, sizeof(pgctlOut), + "%s exec -T %s pg_autoctl manual service pgctl on" " --pgdata /var/lib/postgres/pgaf", r->composeBase, cmd->service); if (rc != 0) { + log_info("%s", pgctlOut); sformat(errBuf, errLen, "start postgres %s failed (exit %d)", cmd->service, rc); @@ -3406,17 +3551,28 @@ runner_exec_step(TestRunner *r, TestStep *step, char *errBuf, int errLen, for (TestCmd *cmd = step->commands; cmd; cmd = cmd->next) { /* - * Flush all notifications that arrived during the previous command. + * Flush notifications that arrived during the previous command — + * UNLESS the current command is a wait. Wait commands (CMD_WAIT_STATE, + * CMD_WAIT_STATES, CMD_WAIT_MULTI) each start with their own drain that + * passes the correct mark arrays, so the '*' convergence prefix is + * applied to the right notifications. Draining here without marks would + * consume those notifications before the wait sees them, silencing the + * '*' markers entirely. + * * Loop until the socket is idle for 50 ms so we catch notifications - * still in-flight in the TCP stream, not just what libpq buffered. + * still in-flight in the TCP stream, not just what libpq has buffered. */ - if (r->notifyConnected) + bool isWaitCmd = (cmd->kind == CMD_WAIT_STATE || + cmd->kind == CMD_WAIT_STATES || + cmd->kind == CMD_WAIT_MULTI); + + if (r->notifyConnected && !isWaitCmd) { while (runner_wait_socket(r, 50)) { runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); } - runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); /* one last sweep of the libpq buffer */ + runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); /* one last sweep */ } char label[256]; @@ -3497,15 +3653,27 @@ runner_wait_for_monitor(TestRunner *r) /* * If the direct libpq connection can't be established (e.g. the - * host IP is not in the monitor's pg_hba.conf — common when the - * monitor was initialised by an older pg_autoctl version that only - * added the Docker network CIDR), fall back to a subprocess check - * via docker compose exec. Once the monitor responds to psql we - * return true; wait loops will use subprocess polling instead of - * LISTEN/NOTIFY. + * host IP is not in the monitor's pg_hba.conf — common on Docker + * Desktop for Mac where published-port connections appear as + * 192.168.65.1, outside the Docker bridge CIDR), fall back to a + * subprocess check. Once the monitor responds to psql, patch pg_hba + * to allow all hosts (safe for local test containers with trust auth) + * and retry the LISTEN connection once before falling back to polling. */ if (run_cmd("%s", monitorReadyCmd) == 0) { + run_cmd("%s exec -T monitor sh -c " + "\"echo 'host all all 0.0.0.0/0 trust'" + " >> \\$PGDATA/pg_hba.conf" + " && pg_ctl -D \\$PGDATA reload -s\"" + " >/dev/null 2>&1", + r->composeBase); + pg_usleep(200 * 1000); + if (runner_notify_connect(r)) + { + log_info("Monitor is ready; LISTEN channel open"); + return true; + } log_info("Monitor is ready (subprocess check; LISTEN not available)"); return true; } diff --git a/src/monitor/formation_metadata.c b/src/monitor/formation_metadata.c index debdef1e7..d4974fb6b 100644 --- a/src/monitor/formation_metadata.c +++ b/src/monitor/formation_metadata.c @@ -474,8 +474,10 @@ FormationKindToString(FormationKind kind) } default: + { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unknown formation kind value %d", kind))); + } } /* keep compiler happy */ diff --git a/src/monitor/node_metadata.c b/src/monitor/node_metadata.c index f80d2572b..34fb6adc6 100644 --- a/src/monitor/node_metadata.c +++ b/src/monitor/node_metadata.c @@ -1642,9 +1642,11 @@ SyncStateToString(SyncState pgsrSyncState) } default: + { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unknown SyncState enum value %d", pgsrSyncState))); + } } /* keep compiler happy */ diff --git a/tests/network.py b/tests/network.py index 338d7297b..02e2d91f8 100644 --- a/tests/network.py +++ b/tests/network.py @@ -162,7 +162,8 @@ def run(self, command, user=os.getenv("USER")): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, + encoding="utf-8", + errors="replace", start_new_session=True, ) @@ -190,7 +191,8 @@ def run_unmanaged(self, command, user=os.getenv("USER")): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, + encoding="utf-8", + errors="replace", start_new_session=True, ) diff --git a/tests/tap/README.md b/tests/tap/README.md new file mode 100644 index 000000000..47c6ab798 --- /dev/null +++ b/tests/tap/README.md @@ -0,0 +1,217 @@ +# pg_auto_failover TAP Test Suite + +Tests are written in the `.pgaf` spec DSL and run by the `pgaftest` binary. +Output is [TAP](https://testanything.org/) (Test Anything Protocol), +compatible with `prove` and any TAP harness. + +## Prerequisites + +- `pgaftest` built: `make -C src/bin` +- Docker with the Compose plugin (`docker compose version`) + +## Running tests + +### All tests (via schedule) + +```sh +pgaftest run --schedule tests/tap/schedule +``` + +Runs every spec listed in `tests/tap/schedule` and emits TAP on stdout. +Exit code is 0 only when all tests pass. + +With `prove` for a formatted summary: + +```sh +pgaftest run --schedule tests/tap/schedule | prove --tap - +``` + +### A single spec + +```sh +pgaftest run tests/tap/specs/basic_operation.pgaf +``` + +### Interactive mode (live cluster, shell access) + +Brings up the cluster, runs `setup {}`, then drops you into a shell: + +```sh +pgaftest setup tests/tap/specs/basic_operation.pgaf +``` + +Run individual steps against the live cluster: + +```sh +pgaftest step stop_primary tests/tap/specs/basic_operation.pgaf +pgaftest step check_failover tests/tap/specs/basic_operation.pgaf +``` + +Inspect the generated `docker-compose.yml` without starting anything: + +```sh +pgaftest show tests/tap/specs/basic_operation.pgaf +``` + +Tear down when done: + +```sh +pgaftest down tests/tap/specs/basic_operation.pgaf +``` + +## Spec file format + +Each `.pgaf` file in `specs/` is a complete, self-contained test scenario. + +``` +cluster { + monitor + formation { + node1 + node2 candidate-priority 0 + } +} + +setup { + wait until node1 state is primary timeout 120s + wait until node2 state is secondary timeout 120s +} + +teardown { + compose down +} + +step stop_primary { + compose kill node1 +} + +step check_failover { + wait until node2 state is primary timeout 90s + assert node2 state is primary +} + +sequence + stop_primary + check_failover +``` + +| Block | Purpose | +|---|---| +| `cluster {}` | Topology — drives `docker-compose.yml` generation | +| `setup {}` | Run after `compose up -d`, before the first step | +| `teardown {}` | Always run at end (CI); on-demand in interactive mode | +| `step name {}` | Named command block | +| `sequence` | Step execution order for `pgaftest run` | + +### Commands inside blocks + +**Process control** + +| Command | Effect | +|---|---| +| `exec ` | `docker compose exec -T ` — fails if exit ≠ 0 | +| `exec-fails ` | Same but asserts non-zero exit | +| `compose down` | `docker compose down --volumes` | +| `compose start ` | `docker compose start ` | +| `compose stop ` | `docker compose stop ` | +| `compose kill ` | `docker compose kill ` (SIGKILL) | +| `stop postgres ` | Stop Postgres inside the container without stopping the keeper | +| `start postgres ` | Restart Postgres inside the container via the keeper | + +**State and timing** + +| Command | Effect | +|---|---| +| `wait until state is [timeout Ns]` | Poll monitor; fail on timeout | +| `wait until assigned-state is [timeout Ns]` | Poll assigned state | +| `wait until stopped [timeout Ns]` | Wait until container exits | +| `wait until , , … [in group N] [timeout Ns]` | Formation-wide state convergence | +| `wait until state is and state is … [timeout Ns]` | Multi-node simultaneous wait | +| `assert state is ` | Instant check; no polling | +| `assert assigned-state is ` | Check assigned state column | +| `assert stays while { … }` | Assert state unchanged throughout body | +| `sleep Ns` | Wait N seconds | + +**SQL and expectations** + +| Command | Effect | +|---|---| +| `sql { SQL }` | Run SQL on service, capture output | +| `expect { text }` | Substring-match last `sql` output | +| `expect { { row } { row } }` | Tuple form: match `psql --tuples-only --no-align` rows | +| `expect error [SQLSTATE]` | Assert previous `sql` raised a SQL error | + +**Network** + +| Command | Effect | +|---|---| +| `network disconnect ` | `docker network disconnect` — simulate partition | +| `network connect ` | `docker network connect` — restore | + +**Log inspection** + +| Command | Effect | +|---|---| +| `logs contains ""` | Assert literal `` appears in container logs | +| `logs not contains ""` | Assert literal `` does NOT appear | +| `logs matches ""` | Assert PCRE pattern matches a log line (`grep -P`) | +| `logs not matches ""` | Assert PCRE pattern does NOT match any log line | + +**Cluster lifecycle** + +| Command | Effect | +|---|---| +| `promote [, , …]` | `pg_autoctl perform promotion` per node | +| `set monitor ` | Switch runner's active monitor (for replace-monitor tests) | + +### `%CIDR%` macro + +The literal `%CIDR%` is expanded to the Docker network CIDR in any `exec` +or `exec-fails` argument. Use it to scope HBA rules to the test network: + +``` +exec monitor bash -c "echo 'host all all %CIDR% trust' >> $PGDATA/pg_hba.conf" +``` + +## Schedule file + +`tests/tap/schedule` lists one spec *name* per line (no path, no `.pgaf` +extension). Lines starting with `#` are comments. + +``` +basic_operation +multi_standbys +# citus_basic # requires Citus image — set CITUS_CLUSTER=1 +``` + +## Relationship to Python tests + +Each `.pgaf` spec was ported from a Python integration test in `tests/`. +The original is recorded in the spec header: + +``` +# Predecessor: tests/test_basic_operation.py +``` + +The pgaf framework replaces the Python `pgautofailover_utils` subprocess +harness with declarative DSL commands backed by `docker compose`. Node +creation, cluster initialization, and teardown are handled by the cluster +declaration and `compose up / down`; test steps focus purely on the +scenario logic. + +## CI integration + +```yaml +- name: Run pg_auto_failover tests + run: | + make -C src/bin pgaftest + pgaftest run --schedule tests/tap/schedule +``` + +Or with `prove`: + +```yaml +- run: pgaftest run --schedule tests/tap/schedule | prove --tap - +``` + +For the full DSL reference see [`docs/pgaftest.rst`](../../docs/pgaftest.rst). diff --git a/tests/tap/schedule b/tests/tap/schedule new file mode 100644 index 000000000..a58a8f966 --- /dev/null +++ b/tests/tap/schedule @@ -0,0 +1,36 @@ +# tests/tap/schedule +# One spec name per line; pgaftest run --schedule tests/tap/schedule +# Lines starting with # are comments; empty lines are ignored. +# Citus specs require CITUS_CLUSTER=1 env var to be set. + +basic_operation +basic_operation_listen_flag +maintenance_and_drop +create_standby_with_pgdata +ensure +monitor_disabled +replace_monitor +config_get_set +skip_pg_hba +#debian_clusters +auth +enable_ssl +ssl_cert +ssl_self_signed +multi_standbys +multi_async +multi_ifdown +multi_maintenance +multi_alternate +extension_update +installcheck +# Upgrade test — requires pgaf:current and pgaf:next images to be pre-built: +# make -C tests/upgrade pgaf-current pgaf-next +upgrade +# Citus tests (uncomment when CITUS_CLUSTER=1): +# basic_citus_operation +# citus_cluster_name +# citus_force_failover +# citus_multi_standbys +# citus_skip_pg_hba +# nonha_citus_operation diff --git a/tests/tap/schedules/citus-1.sch b/tests/tap/schedules/citus-1.sch new file mode 100644 index 000000000..f2f780e8e --- /dev/null +++ b/tests/tap/schedules/citus-1.sch @@ -0,0 +1,4 @@ +# Citus basic tests: cluster name, forced failover, multi-standby (~4 min) +citus_cluster_name +citus_force_failover +citus_multi_standbys diff --git a/tests/tap/schedules/citus-2.sch b/tests/tap/schedules/citus-2.sch new file mode 100644 index 000000000..cf792a98c --- /dev/null +++ b/tests/tap/schedules/citus-2.sch @@ -0,0 +1,4 @@ +# Citus advanced tests: full HA, non-HA, skip-pg-hba (~6 min) +basic_citus_operation +nonha_citus_operation +citus_skip_pg_hba diff --git a/tests/tap/schedules/multi-alternate.sch b/tests/tap/schedules/multi-alternate.sch new file mode 100644 index 000000000..123f0cd95 --- /dev/null +++ b/tests/tap/schedules/multi-alternate.sch @@ -0,0 +1,2 @@ +# Multi-node alternate failover scenarios (~6 min) +multi_alternate diff --git a/tests/tap/schedules/multi-async.sch b/tests/tap/schedules/multi-async.sch new file mode 100644 index 000000000..eb8bb56d8 --- /dev/null +++ b/tests/tap/schedules/multi-async.sch @@ -0,0 +1,2 @@ +# Multi-node async replication scenarios (~6 min) +multi_async diff --git a/tests/tap/schedules/multi-misc.sch b/tests/tap/schedules/multi-misc.sch new file mode 100644 index 000000000..6f4e12177 --- /dev/null +++ b/tests/tap/schedules/multi-misc.sch @@ -0,0 +1,5 @@ +# Multi-node misc: standbys, maintenance, ensure, network partition (~8 min) +multi_standbys +multi_maintenance +ensure +multi_ifdown diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch new file mode 100644 index 000000000..8c9960936 --- /dev/null +++ b/tests/tap/schedules/node.sch @@ -0,0 +1,11 @@ +# Node lifecycle, monitor operations, and Debian/tablespace layouts (~30 min). +# Merged from former node, monitor, and node-extra schedules to reduce CI job +# count and GitHub Actions runner queue pressure. +create_standby_with_pgdata +maintenance_and_drop +auth +monitor_disabled +replace_monitor +extension_update +debian_clusters +tablespaces diff --git a/tests/tap/schedules/quick.sch b/tests/tap/schedules/quick.sch new file mode 100644 index 000000000..c9d6e7529 --- /dev/null +++ b/tests/tap/schedules/quick.sch @@ -0,0 +1,5 @@ +# Fast single-node and config tests (~10 min) +basic_operation +basic_operation_listen_flag +config_get_set +skip_pg_hba diff --git a/tests/tap/schedules/ssl.sch b/tests/tap/schedules/ssl.sch new file mode 100644 index 000000000..8a8ec2f1b --- /dev/null +++ b/tests/tap/schedules/ssl.sch @@ -0,0 +1,4 @@ +# SSL: self-signed, enable, cert auth (~20 min; ssl_cert is slowest) +enable_ssl +ssl_self_signed +ssl_cert diff --git a/tests/tap/specs/auth.pgaf b/tests/tap/specs/auth.pgaf new file mode 100644 index 000000000..3e33923ae --- /dev/null +++ b/tests/tap/specs/auth.pgaf @@ -0,0 +1,77 @@ +# Test md5 authentication between nodes and the monitor, including password +# handling and verifying that passwords are not leaked in logs. +# +# Ported from tests/test_auth.py. +# +# Password handling +# ----------------- +# All passwords are declared in the cluster block and written into the ini +# files by compose_gen before any container starts: +# +# cluster.monitorPassword → monitor.ini [pg_auto_failover] autoctl_node_password +# → every node.ini [monitor] pguri (credential in URI) +# → pgaftest service env PG_AUTOCTL_MONITOR (pguri) +# node.monitorPassword → node.ini [pg_auto_failover] monitor_password +# used by fsm_init_primary() when creating the +# pgautofailover_monitor health-check role +# node.replicationPassword → node.ini [replication] password +# used when creating pgautofailover_replicator +# +# All passwords are therefore in place before pg_autoctl runs its first FSM +# transition, removing any need for sequential startup or manual exec steps. +# Predecessor: tests/test_auth.py + +cluster { + monitor password "pg-auto-failover" + auth md5 + formation auth { + node1 monitor-password "pg-hba-check" replication-password "pg-replication" + node2 monitor-password "pg-hba-check" replication-password "pg-replication" + } +} + +setup { + # 180s: cert-auth cluster formation is slower on shared CI runners + wait until primary, secondary timeout 300s + promote node1 +} + +teardown { + compose down +} + +step test_001_init_primary { + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_ } +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } + sql node2 { SELECT count(*) FROM t1; } + expect { 2 } +} + +step test_003_failover { + exec monitor pg_autoctl perform failover --formation auth + wait until node2 state is primary + and node1 state is secondary + timeout 90s + wait until node2 state is primary timeout 90s + wait until node1 state is secondary timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_ } +} + +step test_004_verify_data { + sql node2 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT count(*) FROM t1; } + expect { 3 } +} + +step test_005_logging_of_passwords { + logs node2 not contains "monitor_password" + logs node2 not contains "streaming_password" + logs node2 contains "monitor-password ****" + logs node2 contains "replication-password ****" +} diff --git a/tests/tap/specs/basic_citus_operation.pgaf b/tests/tap/specs/basic_citus_operation.pgaf new file mode 100644 index 000000000..809c1d6cf --- /dev/null +++ b/tests/tap/specs/basic_citus_operation.pgaf @@ -0,0 +1,132 @@ +# Test basic Citus cluster operations: coordinator HA, worker HA with two +# worker groups, distributed table writes/reads, and failover at each level. +# +# Ported from tests/test_basic_citus_operation.py +# Predecessor: tests/test_basic_citus_operation.py + +cluster { + monitor + formation { + coordinator1a coordinator + coordinator1b coordinator + worker1a worker group 1 + worker1b worker group 1 + worker2a worker group 2 + worker2b worker group 2 + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator pair comes up +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + timeout 90s +} + +# +# test_002: worker groups come up +# + +step test_002_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + and worker2a state is primary + and worker2b state is secondary + timeout 90s +} + +step test_002b_wait_metadata_sync { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { + CREATE OR REPLACE FUNCTION public.wait_until_metadata_sync(timeout + INTEGER DEFAULT 15000) RETURNS void LANGUAGE C STRICT AS 'citus'; + } + sql coordinator1a { SELECT public.wait_until_metadata_sync(); } +} + +# +# test_003: create distributed table +# + +step test_003_create_distributed_table { + sql coordinator1a { CREATE TABLE t1 (a int); } + sql coordinator1a { SELECT create_distributed_table('t1', 'a'); } + sql coordinator1a { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_001_fail_worker2 { + network disconnect worker2a + wait until worker2b state is wait_primary timeout 90s +} + +step test_004_003_insert_while_wait_primary { + wait until worker2b state is wait_primary timeout 90s + sql coordinator1a { INSERT INTO t1 VALUES (3); } +} + +step test_004_004_reconnect_worker2a { + network connect worker2a + wait until worker2b state is primary + and worker2a state is secondary + timeout 180s +} + +step test_005_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } } +} + +# +# test_006: write more rows +# + +step test_006_writes_to_coordinator_succeed { + sql coordinator1a { INSERT INTO t1 VALUES (4); } + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +step test_007_fail_worker2b { + network disconnect worker2b + wait until worker2a state is wait_primary timeout 90s +} + +step test_007b_reconnect_worker2b { + network connect worker2b + wait until worker2a state is primary + and worker2b state is secondary + timeout 180s +} + +step test_008_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +step test_009_perform_failover_worker2 { + exec monitor pg_autoctl perform failover --group 2 + wait until worker2b state is primary + and worker2a state is secondary + timeout 180s +} + +step test_010_perform_failover_coordinator { + exec monitor pg_autoctl perform failover --group 0 + wait until coordinator1a state is secondary + and coordinator1b state is primary + timeout 90s +} diff --git a/tests/tap/specs/basic_operation.pgaf b/tests/tap/specs/basic_operation.pgaf new file mode 100644 index 000000000..43b809901 --- /dev/null +++ b/tests/tap/specs/basic_operation.pgaf @@ -0,0 +1,410 @@ +# Basic two-node HA: create primary + secondary, test maintenance, +# failover, network partition detection, and node drop. +# +# Ported from tests/test_basic_operation.py — same step ordering and +# same commands as the Python implementation (run-on-node vs run-on-monitor +# matches the Python test). +# +# Node IDs are deterministic because node2's container depends_on node1 +# (see compose_gen.c), so node1 always registers first and gets nodeid=1, +# node2 gets nodeid=2, node3 gets nodeid=3. Replication slot names embed +# the standby's nodeid: +# pgautofailover_standby_1 — node1's slot (on the primary) +# pgautofailover_standby_2 — node2's slot (on the primary) +# pgautofailover_standby_3 — node3's slot (on the primary) +# +# node3 is declared launch deferred so the container starts with +# `sleep infinity`; test_017 issues `compose start node3` which +# restarts it with the normal `pg_autoctl node run` command. +# Predecessor: tests/test_basic_operation.py + +cluster { + monitor + formation { + node1 + node2 + node3 launch deferred + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +# +# test_003: create table on primary and verify replication to secondary. +# + +step test_003_create_t1 { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node2 { SELECT count(*) FROM t1; } + expect { 2 } +} + +# +# test_004: verify synchronous_standby_names and replication slots +# after secondary joins. Both nodes must have a physical slot +# for the other's replication identity. +# + +step test_004_init_secondary { + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_1 } +} + +# +# test_005: read replicated data from the secondary. +# + +step test_005_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +# +# test_006: verify the secondary rejects writes (read-only replica). +# INSERT must fail with SQLSTATE 25006 (read_only_sql_transaction). +# + +step test_006_001_writes_to_node2_fail { + sql node2 { INSERT INTO t1 VALUES (3); } + expect error 25006 +} + +step test_006_002_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +# +# test_007: maintenance mode on node1 (primary). +# test_007_002: enabling maintenance without --allow-failover on the primary +# must fail (no standby would remain to take over). +# test_007_004: with --allow-failover node2 transitions to wait_primary. +# test_007_005: disabling maintenance rejoins node1 as secondary. +# + +step test_007_002_maintenance_primary { + exec-fails node1 pg_autoctl enable maintenance +} + +step test_007_004_maintenance_primary_allow_failover { + exec node1 pg_autoctl enable maintenance --allow-failover + wait until node1 state is maintenance + and node2 state is wait_primary + timeout 60s + sql node2 { SHOW synchronous_standby_names; } + expect { } +} + +step test_007_005_disable_maintenance { + exec node1 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is primary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +# +# test_008: maintenance mode on node1 (now secondary after test_007). +# test_008_001: enable maintenance, stop postgres on node1, write to primary. +# test_008_002: disable maintenance — node1 pg_rewinds and rejoins as secondary. +# + +step test_008_001_enable_maintenance_secondary { + exec node1 pg_autoctl enable maintenance + wait until node1 state is maintenance timeout 60s + stop postgres node1 + sql node2 { INSERT INTO t1 VALUES (3); } +} + +step test_008_002_disable_maintenance_secondary { + exec node1 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is primary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +# +# test_009: failback — promote node1 back to primary via perform-promotion. +# After: node1=primary, node2=secondary. +# + +step test_009_failback { + exec monitor pg_autoctl perform promotion --name node1 + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } +} + +# +# test_010: disconnect node1 (primary) from the network — simulates a network +# partition. node2 must detect the loss and enter wait_primary +# (it cannot promote yet without confirmation from the monitor). +# + +step test_010_fail_primary { + network disconnect node1 + wait until node2 state is wait_primary timeout 120s +} + +# +# test_011: node2 is now wait_primary — writes must succeed even before +# the full promotion completes (no sync standby required in +# wait_primary state). +# + +step test_011_writes_to_node2_succeed { + sql node2 { INSERT INTO t1 VALUES (4); } + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_012: reconnect node1 — it pg_rewinds against node2 and rejoins as +# secondary. node2 transitions from wait_primary to primary once +# it has a healthy sync standby again. +# + +step test_012_start_node1_again { + network connect node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +# +# test_013: verify all four rows are visible on node1 (now secondary), +# confirming pg_rewind brought it fully up to date. +# + +step test_013_read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_014: node1 is secondary again — writes must be rejected (25006). +# + +step test_014_writes_to_node1_fail { + sql node1 { INSERT INTO t1 VALUES (5); } + expect error 25006 +} + +# +# test_015: stop node1 (secondary) via compose stop — Postgres shuts down +# cleanly. node2 enters wait_primary because its sync standby +# disappeared (number_sync_standbys=1). +# + +step test_015_fail_secondary { + compose stop node1 + # 60s: compose stop can be slow on shared CI runners + wait until node1 stopped timeout 60s + wait until node2 state is wait_primary timeout 60s +} + +# +# test_016: restart node1, let it rejoin, then drop it with --no-wait. +# node2 becomes single. Verify no stale replication slots remain +# on node2 after the drop (pg_auto_failover must clean them up). +# + +step test_016_drop_secondary { + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + exec node1 pg_autoctl drop node --no-wait + # 60s: compose stop can be slow on shared CI runners + wait until node1 stopped timeout 60s + wait until node2 state is single timeout 90s + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { } +} + +# +# test_017–019: add node3 as a new secondary (was launch deferred). +# test_017: issue `pg_autoctl node start` to bring the deferred node live. +# test_018: perform failover must fail immediately — node3 is still +# catching up and no election can complete yet. +# test_019: wait for node3 to reach secondary; verify SSN and slots. +# + +step test_017_add_new_secondary { + exec node3 pg_autoctl node start +} + +step test_018_cant_failover_yet { + exec-fails monitor pg_autoctl perform failover +} + +step test_019_run_secondary { + wait until node3 state is secondary + and node2 state is primary + timeout 120s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3) } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_3 } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } +} + +# +# test_020: two back-to-back manual failovers. After each failover verify +# synchronous_standby_names and that every node has the correct +# physical replication slot for the other nodes — slots must +# survive a role change intact. +# + +step test_020_multiple_manual_failover_verify_replication_slots { + exec monitor pg_autoctl perform failover + wait until node3 state is primary + and node2 state is secondary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_3 } + exec node2 pg_autoctl perform promotion + wait until node2 state is primary + and node3 state is secondary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3) } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_3 } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } +} + +# +# test_021–023: network partition detection and demote-timeout. +# test_021: disconnect node2 (primary) — simulates a hard network failure. +# test_022: wait for demote_timeout on node2 and wait_primary on node3; +# verify pg_autoctl on node2 considers itself unready (no monitor +# contact) and that node3 has no sync standby yet. +# test_023: reconnect node2 — it detects it lost the election, pg_rewinds +# against node3, and rejoins as secondary. +# + +step test_021_ifdown_primary { + wait until node2 state is primary timeout 30s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3) } + network disconnect node2 +} + +step test_022_detect_network_partition { + # node2 is network-disconnected so it cannot report its state back to the + # monitor. Check only the assigned state (monitor's decision); checking + # the full convergence state would wait forever for a node that is offline. + wait until node2 assigned-state = demote_timeout timeout 300s + wait until node3 state is wait_primary timeout 300s + sleep 3s + exec-fails node2 pg_autoctl inspect pgsetup ready + sql node3 { SHOW synchronous_standby_names; } + expect { } +} + +step test_023_ifup_old_primary { + network connect node2 + wait until node2 state is secondary + and node3 state is primary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } +} + +# +# test_024: stop the monitor's Postgres process directly (bypassing +# pg_autoctl) and verify pg_autoctl restarts it automatically. +# The data node must remain in its current state throughout — +# a brief monitor outage must not trigger a spurious failover. +# + +step test_024_stop_postgres_monitor { + # Stop the monitor's postgres directly. The monitor cannot assign state + # changes while postgres is down, so there is no point wrapping this in + # "assert stays primary while" — that construct queries the monitor to + # verify the state, which is impossible when the monitor is the one being + # stopped. The subsequent wait verifies recovery. + stop postgres monitor + sleep 5s + wait until node3 state is primary timeout 60s +} + +# +# test_025: drop the last remaining data node — cluster goes empty. +# + +step test_025_drop_primary { + exec node3 pg_autoctl drop node --no-wait + wait until node3 stopped timeout 60s + wait until node2 state is single timeout 90s +} diff --git a/tests/tap/specs/basic_operation_listen_flag.pgaf b/tests/tap/specs/basic_operation_listen_flag.pgaf new file mode 100644 index 000000000..830f05c3d --- /dev/null +++ b/tests/tap/specs/basic_operation_listen_flag.pgaf @@ -0,0 +1,117 @@ +# Test basic pg_auto_failover operations with the --listen flag, verifying +# that nodes bind on all interfaces for replication and failover. +# +# Ported from tests/test_basic_operation_listen_flag.py +# Predecessor: tests/test_basic_operation_listen_flag.py + +cluster { + monitor + formation { + node1 listen + node2 launch deferred listen + } +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: init primary with listen flag +# + +step test_001_init_primary { + wait until node1 state is single timeout 60s + exec node1 pg_autoctl inspect pgsetup wait +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_003: add secondary node2 with listen flag +# + +step test_003_init_secondary { + exec node2 pg_autoctl node start + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +# +# test_004: read from secondary +# + +step test_004_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +step test_005_writes_to_node2_fail { + sql node2 { INSERT INTO t1 VALUES (3); } + expect error 25006 +} + +# +# test_006: fail primary node1 +# + +step test_006_fail_primary { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} + +# +# test_007: writes to new primary succeed +# + +step test_007_writes_to_node2_succeed { + sql node2 { INSERT INTO t1 VALUES (3); } + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} + +# +# test_008: restart node1 as secondary +# + +step test_008_start_node1_again { + network connect node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} + +# +# test_009: read from new secondary (node1) +# + +step test_009_read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} + +step test_010_writes_to_node1_fail { + sql node1 { INSERT INTO t1 VALUES (3); } + expect error 25006 +} + +# +# test_011: fail secondary node1 +# + +step test_011_fail_secondary { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} diff --git a/tests/tap/specs/citus_cluster_name.pgaf b/tests/tap/specs/citus_cluster_name.pgaf new file mode 100644 index 000000000..ef84bee4d --- /dev/null +++ b/tests/tap/specs/citus_cluster_name.pgaf @@ -0,0 +1,120 @@ +# Test Citus cluster name and read-only secondary cluster routing. +# +# A "readonly" cluster is a set of Citus nodes registered with +# candidate-priority=0, citus-secondary, and citus-cluster-name=readonly. +# These nodes should have: +# citus.use_secondary_nodes = 'always' +# citus.cluster_name = 'readonly' +# in their postgresql.conf, so that distributed queries are routed to the +# secondary (standby) copies of each shard rather than the primaries. +# +# Ported from tests/test_citus_cluster_name.py. +# Predecessor: tests/test_citus_cluster_name.py + +cluster { + monitor + formation { + coordinator1a coordinator + node coordinator1b { + coordinator + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + worker1a worker group 1 + node worker1b { + worker group 1 + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + worker2a worker group 2 + node worker2b { + worker group 2 + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator pair — coordinator1b is read-only secondary cluster +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + timeout 90s +} + +step test_002_001_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + timeout 90s +} + +step test_002_002_init_workers { + wait until worker2a state is primary + and worker2b state is secondary + timeout 90s +} + +# +# test_003: create distributed table and insert data +# + +step test_003_create_distributed_table { + sql coordinator1a { + SET citus.shard_count TO 4; CREATE TABLE t1 (a int); SELECT + create_distributed_table('t1', 'a'); + } + sql coordinator1a { + INSERT INTO t1 SELECT x FROM generate_series(1, 1000) as gs(x); + } + sql coordinator1a { CHECKPOINT; } +} + +# +# test_004: verify read-only cluster settings +# + +step test_004_read_only_cluster { + sql coordinator1a { + SELECT current_setting('citus.use_secondary_nodes'), + current_setting('citus.cluster_name'); + } + expect { never|default } + sql coordinator1b { + SELECT current_setting('citus.use_secondary_nodes'), + current_setting('citus.cluster_name'); + } + expect { always|readonly } + sql coordinator1b { SELECT count(*) FROM t1; } + expect { 1000 } +} + +# +# test_005: drop distributed table (cleanup) +# + +step test_005_drop_table { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true AND nodecluster = '"'"'default'"'"'"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { + CREATE OR REPLACE FUNCTION public.wait_until_metadata_sync(timeout + INTEGER DEFAULT 15000) RETURNS void LANGUAGE C STRICT AS 'citus'; + } + sql coordinator1a { SELECT public.wait_until_metadata_sync(); } + sql coordinator1a { DROP TABLE t1; } +} diff --git a/tests/tap/specs/citus_force_failover.pgaf b/tests/tap/specs/citus_force_failover.pgaf new file mode 100644 index 000000000..858b88787 --- /dev/null +++ b/tests/tap/specs/citus_force_failover.pgaf @@ -0,0 +1,86 @@ +# Test Citus force failover: a transaction in progress competes with a worker +# failover. The failover mechanism must terminate competing backends to make +# progress within a bounded time window. +# Also tests dropping a failed primary worker and a failed coordinator. +# +# Predecessor: tests/test_citus_force_failover.py + +cluster { + monitor + formation { + coordinator1a coordinator + coordinator1b coordinator + worker1a worker group 1 + worker1b worker group 1 + worker2a worker group 2 + worker2b worker group 2 + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator pair +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + timeout 90s +} + +# +# test_002: worker groups +# + +step test_002_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + and worker2a state is primary + and worker2b state is secondary + timeout 90s +} + +# +# test_003: create distributed table +# + +step test_003_create_distributed_table { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { + SET citus.shard_count TO 4; CREATE TABLE t1 (a int); SELECT + create_distributed_table('t1', 'a'); + } +} + +step test_004_fail_while_transaction_is_in_progress { + network disconnect worker1a + wait until worker1b state is wait_primary timeout 90s +} + +# +# test_005: drop failed primary worker2a and coordinator1a +# + +step test_005_drop_primary_worker { + network disconnect worker2a + wait until worker2b state is wait_primary timeout 90s + compose stop worker2a + exec monitor psql -d pg_auto_failover -tAc "SELECT pgautofailover.remove_node(nodehost, nodeport) FROM pgautofailover.node WHERE nodename = 'worker2a'" + wait until worker2b state is single timeout 60s +} + +step test_005_drop_primary_coordinator { + compose stop coordinator1a + exec monitor psql -d pg_auto_failover -tAc "SELECT pgautofailover.remove_node(nodehost, nodeport) FROM pgautofailover.node WHERE nodename = 'coordinator1a'" + wait until coordinator1b state is single timeout 60s +} diff --git a/tests/tap/specs/citus_multi_standbys.pgaf b/tests/tap/specs/citus_multi_standbys.pgaf new file mode 100644 index 000000000..b81b51c69 --- /dev/null +++ b/tests/tap/specs/citus_multi_standbys.pgaf @@ -0,0 +1,176 @@ +# Test Citus cluster with multiple standbys per group (3 nodes per group: +# a primary, a sync secondary, and an async secondary with candidate-priority 0 +# and replication-quorum false). Verifies failover, data consistency, and +# synchronous_standby_names after each failure/recovery cycle. +# +# Ported from tests/test_citus_multi_standbys.py +# Predecessor: tests/test_citus_multi_standbys.py + +cluster { + monitor + formation { + coordinator1a coordinator + coordinator1b coordinator + node coordinator1c { + coordinator + candidate-priority 0 + replication-quorum false + } + worker1a worker group 1 + worker1b worker group 1 + node worker1c { + worker group 1 + candidate-priority 0 + replication-quorum false + } + worker2a worker group 2 + worker2b worker group 2 + node worker2c { + worker group 2 + candidate-priority 0 + replication-quorum false + } + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + wait until coordinator1c state is secondary timeout 90s + wait until worker1c state is secondary timeout 90s + wait until worker2c state is secondary timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator group with async secondary coordinator1c +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + and coordinator1c state is secondary + timeout 90s + exec coordinator1c pg_autoctl get node candidate-priority + expect { 0 } +} + +# +# test_002: worker groups with async standbys +# + +step test_002_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + and worker1c state is secondary + and worker2a state is primary + and worker2b state is secondary + and worker2c state is secondary + timeout 90s + exec worker1c pg_autoctl get node candidate-priority + expect { 0 } + exec worker2c pg_autoctl get node candidate-priority + expect { 0 } +} + +# +# test_003: create distributed table and seed data +# + +step test_003_001_create_distributed_table { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { ALTER DATABASE demo SET citus.shard_count TO 4; } + sql coordinator1a { CREATE TABLE t1 (a int); } + sql coordinator1a { SELECT create_distributed_table('t1', 'a'); } + sql coordinator1a { INSERT INTO t1 VALUES (1), (2); } + sql coordinator1a { SELECT a FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +# +# test_004: fail worker2a, reads for worker1 still work via coordinator +# + +step test_004_001_fail_worker2 { + network disconnect worker2a +} + +step test_004_002_writes_via_coordinator_to_worker2_fail { + sql coordinator1a { INSERT INTO t1 VALUES (3); } + expect error +} + +step test_004_004_wait_for_failover { + wait until worker2b state is wait_primary + and worker2c state is secondary + timeout 90s + exec worker2c pg_autoctl get node candidate-priority + expect { 0 } +} + +step test_004_005_writes_via_coordinator_succeed_after_failover { + sql coordinator1a { INSERT INTO t1 VALUES (3); } +} + +step test_005_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } } +} + +# +# test_006: write more data +# + +step test_006_writes_to_coordinator_succeed { + sql coordinator1a { INSERT INTO t1 VALUES (4); } + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_007: restart worker2a as secondary under new primary worker2b +# + +step test_007_start_worker2a_again { + network connect worker2a + wait until worker2a state is secondary + and worker2b state is primary + timeout 90s +} + +step test_008_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_009: fail worker2b, failover back to worker2a +# + +step test_009_fail_worker2b { + network disconnect worker2b + wait until worker2a state is wait_primary + and worker2c state is secondary + timeout 90s +} + +step test_010_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_011: restart worker2b as secondary +# + +step test_011_start_worker2b_again { + network connect worker2b + wait until worker2b state is secondary + and worker2a state is primary + timeout 90s +} diff --git a/tests/tap/specs/citus_skip_pg_hba.pgaf b/tests/tap/specs/citus_skip_pg_hba.pgaf new file mode 100644 index 000000000..3499b6b9f --- /dev/null +++ b/tests/tap/specs/citus_skip_pg_hba.pgaf @@ -0,0 +1,135 @@ +# Test Citus cluster with authMethod=skip (pg_autoctl does not edit pg_hba.conf). +# +# With auth=skip, pg_autoctl leaves pg_hba.conf untouched after postgres init. +# All data nodes are launch deferred so startup can be sequenced manually: +# +# setup: monitor starts → add LAN HBA to monitor → start coord0a +# test_000: add LAN HBA to coord0a (so coord0b can connect later) +# test_001a: wait until coord0a reaches single +# test_001b: start coord0b, wait until coord0a=primary/coord0b=secondary +# +# worker1a's first run fails because the coordinator tries master_activate_node +# but worker1a's default pg_hba.conf blocks that connection. After manually +# appending HBA rules and retrying, activation succeeds — and pg_autoctl has +# not modified pg_hba.conf itself (verified by assert hba-edited = false). +# +# Predecessor: tests/test_citus_skip_pg_hba.py + +cluster { + monitor + auth skip + formation { + coord0a coordinator launch deferred + coord0b coordinator launch deferred + worker1a worker group 1 launch deferred + worker1b worker group 1 launch deferred + } +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait + exec monitor pg_autoctl inspect pgsetup hba-lan + exec coord0a pg_autoctl node start /etc/pgaf/node.ini +} + +teardown { + compose down +} + +# +# test_000: monitor with auth=skip, append trust rule to pg_hba.conf manually +# + +step test_000_add_coord0a_hba { + exec coord0a pg_autoctl inspect pgsetup wait + exec coord0a pg_autoctl inspect pgsetup hba-lan +} + +# +# test_001a: init coordinator coord0a (single) +# + +step test_001a_init_coordinator { + wait until coord0a state is single timeout 60s +} + +# +# test_001b: init coordinator coord0b (secondary) +# + +step test_001b_init_coordinator { + exec coord0b pg_autoctl node start /etc/pgaf/node.ini + wait until coord0a state is primary + and coord0b state is secondary + timeout 90s +} + +step test_002b_create_worker { + exec worker1a pg_autoctl node start /etc/pgaf/node.ini +} + +# +# test_002b/002d: worker1a — registration initially fails because coordinator +# cannot connect (pg_hba.conf has no trust rule yet). After manually adding +# HBA rules the already-running keeper retries activation automatically. +# +# Wait for postgres to be ready before calling hba-lan: the deferred poll in +# pg_autoctl node run detects launchDeferred=false and execv()s into +# pg_autoctl create citus-worker --run. Calling hba-lan immediately would +# race with that execv and the subsequent supervisor + initdb startup. +# pgsetup wait blocks until pg_autoctl.cfg exists and postgres accepts +# connections, so by the time hba-lan runs pg_hba.conf already exists and the +# container is fully initialised. +# + +step test_002d_run_worker { + exec worker1a pg_autoctl inspect pgsetup wait + exec worker1a pg_autoctl inspect pgsetup hba-lan + wait until worker1a state is single timeout 120s +} + +step test_002e_check_hba { + exec-fails worker1a grep "pgautofailover_replicator" /var/lib/postgres/pgaf/pg_hba.conf +} + +step test_002f_activated_node { + sql coord0a { + SELECT isactive FROM pg_dist_node WHERE nodename = 'worker1a'; + } + expect { t } +} + +# +# test_003: init worker1b as secondary +# + +step test_003_init_worker { + exec worker1b pg_autoctl node start /etc/pgaf/node.ini + exec worker1b pg_autoctl inspect pgsetup wait + exec worker1b pg_autoctl inspect pgsetup hba-lan + wait until worker1a state is primary + and worker1b state is secondary + timeout 120s + exec-fails worker1b grep "pgautofailover_replicator" /var/lib/postgres/pgaf/pg_hba.conf +} + +# +# test_004: create distributed table +# + +step test_004_create_distributed_table { + sql coord0a { CREATE TABLE t1 (a int); } + sql coord0a { SELECT create_distributed_table('t1', 'a'); } + sql coord0a { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_005: manual failover of worker group 1 +# + +step test_005_failover { + exec monitor pg_autoctl perform failover --group 1 + wait until worker1a state is secondary + and worker1b state is primary + timeout 90s +} diff --git a/tests/tap/specs/config_get_set.pgaf b/tests/tap/specs/config_get_set.pgaf new file mode 100644 index 000000000..f1d83fb48 --- /dev/null +++ b/tests/tap/specs/config_get_set.pgaf @@ -0,0 +1,48 @@ +# Test pg_autoctl config get / config set behaviour on both the monitor and a +# keeper node, including validation (rejecting invalid values) and no +# unintended side-effects on other settings. +# +# Ported from tests/test_config_get_set.py +# Predecessor: tests/test_config_get_set.py + +cluster { + monitor + formation { + node1 + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +step test_001_init_primary { + exec node1 pg_autoctl set node metadata --name "node a" + exec node1 pg_autoctl show state + exec node1 pg_autoctl config set pg_autoctl.name a + sleep 2s + exec node1 pg_autoctl show settings +} + +step test_002_config_set_monitor { + exec monitor pg_autoctl config set ssl.sslmode prefer + exec monitor pg_autoctl config get postgresql.pg_ctl + expect { /bin/pg_ctl } + exec-fails monitor pg_autoctl config set postgresql.pg_ctl invalid + exec monitor pg_autoctl config get postgresql.pg_ctl + expect { /bin/pg_ctl } + exec monitor pg_autoctl config get ssl.sslmode + expect { prefer } +} + +step test_002b_config_set_node { + exec node1 pg_autoctl config get postgresql.pg_ctl + expect { /bin/pg_ctl } + exec-fails node1 pg_autoctl config set postgresql.pg_ctl invalid + exec node1 pg_autoctl config get postgresql.pg_ctl + expect { /bin/pg_ctl } +} diff --git a/tests/tap/specs/create_standby_with_pgdata.pgaf b/tests/tap/specs/create_standby_with_pgdata.pgaf new file mode 100644 index 000000000..61699c624 --- /dev/null +++ b/tests/tap/specs/create_standby_with_pgdata.pgaf @@ -0,0 +1,80 @@ +# Test creating a standby from an existing PGDATA directory. +# +# test_003/004/005: verify that registering a node whose PGDATA was created by +# a separate initdb (different system_identifier) is rejected by the monitor. +# +# test_006: start node3 via pg_autoctl node start; pg_autoctl detects the +# pre-existing pg_basebackup PGDATA and sets up streaming replication without +# running another pg_basebackup (skipBaseBackup path in fsm_init_standby). +# +# Both node2 and node3 are declared launch deferred: their containers sleep +# until a step explicitly calls pg_autoctl node start (which rewrites the ini +# from mode=deferred to mode=immediate and unblocks the waiting node run loop). +# +# Ported from tests/test_create_standby_with_pgdata.py +# Predecessor: tests/test_create_standby_with_pgdata.py + +cluster { + monitor + formation { + node1 + node2 launch deferred + node3 launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s + exec node1 pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: init primary +# + +step test_001_init_primary { + wait until node1 state is single timeout 60s +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_003_init_secondary { + exec node2 pg_ctl initdb -D /var/lib/postgres/pgaf +} + +step test_004_create_raises_error { + exec-fails node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --ssl-self-signed --name node2 --hostname node2 +} + +step test_005_cleanup_after_failure { + exec node2 rm -rf /var/lib/postgres/pgaf +} + +step test_006_init_secondary { + exec node3 pg_autoctl node start /etc/pgaf/node.ini + wait until node3 state is secondary + and node1 state is primary + timeout 90s +} + +# +# test_007: manual failover to node3 +# + +step test_007_failover { + exec monitor pg_autoctl perform failover + wait until node3 state is primary + and node1 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/debian_clusters.pgaf b/tests/tap/specs/debian_clusters.pgaf new file mode 100644 index 000000000..5366d3a52 --- /dev/null +++ b/tests/tap/specs/debian_clusters.pgaf @@ -0,0 +1,51 @@ +# Test pg_auto_failover with Debian-style pg_createcluster layouts where the +# postgresql.conf lives outside PGDATA (in /etc/postgresql///). +# pg_autoctl must detect and support this split-config layout. +# +# Ported from tests/test_debian_clusters.py +# Predecessor: tests/test_debian_clusters.py + +cluster { + monitor + formation { + node1 debian-cluster main + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +# +# test_001: pg_autoctl adopts the Debian "main" cluster pre-created by +# pg_createcluster at image build time. postgresql.conf starts outside +# PGDATA (/etc/postgresql/17/main/); pg_autoctl moves it in on first run. +# + +step test_001_single_with_debian_cluster { + exec monitor pg_autoctl inspect pgsetup wait + exec node1 pg_autoctl inspect pgsetup wait +} + +# +# test_002: verify pg_autoctl moved postgresql.conf into PGDATA. +# + +step test_002_conf_in_pgdata { + exec node1 /bin/sh -c 'test -f ${PGDATA}/postgresql.conf' +} + +# +# test_003: basic write/read to confirm Postgres is functional. +# + +step test_003_write_and_read { + sql node1 { CREATE TABLE t1 (a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3); } + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} diff --git a/tests/tap/specs/enable_ssl.pgaf b/tests/tap/specs/enable_ssl.pgaf new file mode 100644 index 000000000..99e2fbbf4 --- /dev/null +++ b/tests/tap/specs/enable_ssl.pgaf @@ -0,0 +1,87 @@ +# Test enabling SSL on a running cluster: nodes start without SSL and SSL is +# enabled live with --ssl-self-signed. After enabling, replication must still +# work and reads from the secondary must return correct data. +# +# CA-signed certificate tests (verify-ca / verify-full) are covered separately +# in ssl_cert.pgaf, which starts the cluster with SSL already configured. +# +# Ported from tests/test_enable_ssl.py +# Predecessor: tests/test_enable_ssl.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 60s + promote node1 +} + +teardown { + compose down +} + +step test_001_init_primary { + sql monitor { SHOW ssl; } + expect { off } + sql node1 { SHOW ssl; } + expect { off } + sql node2 { SHOW ssl; } + expect { off } +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_maintenance { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s +} + +step test_005_enable_ssl_monitor { + exec monitor pg_autoctl enable ssl --ssl-self-signed --ssl-mode require + exec monitor pg_autoctl inspect pgsetup wait --timeout 60 + sql monitor { SHOW ssl; } + expect { on } +} + +step test_006_enable_ssl_primary { + exec node1 pg_autoctl enable ssl --ssl-self-signed --ssl-mode require + exec node1 pg_autoctl inspect pgsetup wait --timeout 60 + sql node1 { SHOW ssl; } + expect { on } +} + +step test_007_enable_ssl_secondary { + exec node2 pg_autoctl enable ssl --ssl-self-signed --ssl-mode require + compose stop node2 + compose start node2 + # node2 is still in maintenance: the monitor assigned state is 'maintenance' + # so pg_autoctl will not start Postgres yet. Postgres starts in test_008 + # after disable maintenance triggers a state transition. +} + +step test_008_disable_maintenance { + exec node2 pg_autoctl disable maintenance + wait until node2 state is secondary + and node1 state is primary + timeout 90s + # 90s: wait for Postgres to be fully ready with SSL after the maintenance + # state transition on shared CI runners + exec node2 pg_autoctl inspect pgsetup wait --timeout 90 +} + +step test_009_read_from_secondary { + exec node2 pg_autoctl inspect pgsetup wait + sql node2 { SHOW ssl; } + expect { on } + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} diff --git a/tests/tap/specs/ensure.pgaf b/tests/tap/specs/ensure.pgaf new file mode 100644 index 000000000..7069b2786 --- /dev/null +++ b/tests/tap/specs/ensure.pgaf @@ -0,0 +1,98 @@ +# Test pg_autoctl "ensure" behaviour: verify that the keeper restarts +# Postgres after an unexpected stop, survives a demoted transition, and +# correctly orchestrates a failover when Postgres is broken on the primary. +# +# node2 is declared launch deferred so it starts in single mode for test_001, +# matching the Python test order (node2 is not created until test_003). +# +# Ported from tests/test_ensure.py +# Predecessor: tests/test_ensure.py + +cluster { + monitor + formation { + node1 + node2 launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s + exec node1 pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +step test_001_init_primary { + stop postgres node1 + wait until node1 state is single timeout 90s + exec node1 pg_autoctl inspect pgsetup wait +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_003_init_secondary { + exec node2 pg_autoctl node start + wait until node2 state is secondary + and node1 state is primary + timeout 90s + stop postgres node2 + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +step test_004_demoted { + # After step 003 stopped and restarted postgres on node2, the cluster may + # still be mid-oscillation (node2 re-catching up, node1 in wait_primary). + # Wait for a fresh stable primary before triggering the failover, otherwise + # the stop below fires while node1 is in wait_primary and no failover occurs. + wait until node1 state is primary timeout 60s + # Stop postgres first while pg_autoctl is still running: pg_autoctl detects + # the outage and reports it to the monitor, which starts the failover FSM. + # (Python: node1.stop_postgres() then node1.stop_pg_autoctl()) + stop postgres node1 + compose stop node1 + # Wait for node2 to reach wait_primary — the failover's point of no return. + # node2 cannot reach 'primary' while node1 is stopped (it needs a sync + # standby), so we must not wait for 'primary' here. Once node2 is in + # wait_primary the monitor has committed to node2 as the new primary; when + # node1 restarts it will come back as secondary, and node2 will then + # transition from wait_primary → primary. + wait until node2 state is wait_primary timeout 90s + # Bring node1 back: pg_autoctl starts, connects to monitor, gets assigned + # 'demoted', executes the demoted transition, then moves to secondary. + # The Python test explicitly asserts the 'demoted' state, but 'demoted' is + # a sub-second transient: node1's pg_autoctl may report 'catchingup' before + # the convergence NOTIFY (demoted ➜ demoted) is ever emitted on the LISTEN + # channel. Skip it and assert the stable end state instead. + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 300s +} + +step test_005_inject_error_in_node2 { + wait until node2 state is primary timeout 60s + wait until node2 state is primary timeout 90s + exec node2 bash -c "echo \"shared_preload_libraries='wrong_extension'\" >> /var/lib/postgres/pgaf/postgresql.conf" + # Stop postgres cleanly (pgctl off), then re-enable auto-start (pgctl on) + # so pg_autoctl keeps trying to restart postgres — which fails each time + # due to the bad config. This mirrors the Python's node2.restart_postgres() + # intent: postgres repeatedly fails, the monitor detects an unhealthy primary + # and triggers a failover to node1. + # We ignore the exit code of pgctl on because postgres cannot start with + # the broken config, so the command always exits with an error; pg_autoctl + # itself keeps running and will retry. + stop postgres node2 + exec node2 bash -c "pg_autoctl manual service pgctl on || true" + wait until node1 state is wait_primary timeout 120s + wait until node2 state is secondary + and node1 state is primary + timeout 120s +} diff --git a/tests/tap/specs/extension_update.pgaf b/tests/tap/specs/extension_update.pgaf new file mode 100644 index 000000000..921cf3dca --- /dev/null +++ b/tests/tap/specs/extension_update.pgaf @@ -0,0 +1,37 @@ +# Test pg_auto_failover extension version management. +# The monitor starts with a "dummy" extension version baked into the Docker +# image. pg_autoctl detects the mismatch at startup, runs ALTER EXTENSION +# pgautofailover UPDATE, and restarts Postgres. The test verifies that the +# installed_version seen in pg_available_extensions matches the dummy version +# and that the monitor comes back healthy after the restart. +# +# Predecessor: tests/test_extension_update.py + +cluster { + monitor + extension-version "dummy" +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: start monitor with a dummy extension version and verify +# that the installed version matches the dummy version. +# + +step test_001_update_extension { + sleep 3s + exec monitor pg_autoctl inspect pgsetup wait + sql monitor { + SELECT installed_version + FROM pg_available_extensions + WHERE name = 'pgautofailover'; + } + expect { dummy } +} diff --git a/tests/tap/specs/installcheck.pgaf b/tests/tap/specs/installcheck.pgaf new file mode 100644 index 000000000..ce8b9005a --- /dev/null +++ b/tests/tap/specs/installcheck.pgaf @@ -0,0 +1,46 @@ +# Test the pgautofailover SQL extension via pg_regress installcheck. +# +# The monitor uses the "testrun" Dockerfile target which includes the full source +# tree and postgresql-server-dev (providing pg_regress). pg_regress connects +# to the running monitor Postgres from within the same container. +# +# Coverage mirrors tests/test_installcheck.py: +# - monitor created and running +# - pg_hba.conf widened so pg_regress can connect (host all all 0.0.0.0/0) +# - src/monitor writable for pg_regress results output +# - make installcheck runs all SQL regression tests in src/monitor/ +# +# Ported from tests/test_installcheck.py +# Predecessor: tests/test_installcheck.py + +cluster { + monitor + image "pgaf:testrun" +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: append a trust HBA entry so pg_regress can connect from localhost. +# + +step test_001_add_hba_entry { + exec monitor bash -c "echo 'host all all %CIDR% trust' >> /var/lib/postgres/pgaf/pg_hba.conf" + exec monitor pg_autoctl reload +} + +# +# test_002: make src/monitor writable for pg_regress results output, +# then run all regression SQL tests. +# + +step test_002_make_installcheck { + exec monitor sudo chmod -R go+w /usr/src/pg_auto_failover/src/monitor + exec monitor make -C /usr/src/pg_auto_failover/src/monitor installcheck PGHOST=127.0.0.1 +} diff --git a/tests/tap/specs/maintenance_and_drop.pgaf b/tests/tap/specs/maintenance_and_drop.pgaf new file mode 100644 index 000000000..1e30d3474 --- /dev/null +++ b/tests/tap/specs/maintenance_and_drop.pgaf @@ -0,0 +1,76 @@ +# Test maintenance mode and clean node drop: enable maintenance with a +# concurrent node failure, verify the cluster recovers when maintenance is +# disabled, then simulate a primary failure, recover, and finally drop a +# node cleanly verifying the formation returns to single. +# +# Predecessor: tests/test_create_run.py + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + # 180s: cluster formation can be slow on shared CI runners + wait until primary, secondary timeout 180s + promote node1 +} + +teardown { + compose down +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +step test_005_maintenance { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s + sql node1 { INSERT INTO t1 VALUES (3); } + exec node2 pg_autoctl disable maintenance + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +step test_006_fail_primary { + compose stop node1 + wait until node2 state is wait_primary timeout 180s +} + +step test_007_start_node1_again { + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} + +step test_008_read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} + +step test_009_fail_secondary { + compose stop node1 + wait until node2 state is wait_primary timeout 90s +} + +step test_010_drop_secondary { + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + exec node1 pg_autoctl drop node --no-wait + wait until node1 stopped timeout 30s + wait until node2 state is single timeout 60s +} diff --git a/tests/tap/specs/monitor_disabled.pgaf b/tests/tap/specs/monitor_disabled.pgaf new file mode 100644 index 000000000..877451af6 --- /dev/null +++ b/tests/tap/specs/monitor_disabled.pgaf @@ -0,0 +1,100 @@ +# Test operating pg_autoctl without a monitor ("monitor disabled" mode). +# The FSM is driven manually via pg_autoctl manual fsm assign / nodes set. +# +# No-monitor nodes start pg_autoctl node run automatically (like all nodes) +# but with no_monitor = true in the ini, so there is no monitor registration. +# Each test step drives the FSM manually. +# +# Ported from tests/test_monitor_disabled.py +# Predecessor: tests/test_monitor_disabled.py + +cluster { + ssl off + formation { + node1 no-monitor + node2 no-monitor + node3 no-monitor + } +} + +# No monitor, so no automatic state convergence — each step drives the FSM +# manually. + +setup { + # No monitor: nodes start pg_autoctl in INIT state (postgres not yet + # running). The first test step assigns SINGLE state to node1 which + # starts postgres. Just give pg_autoctl a moment to initialize the + # data directory before the test steps begin. + sleep 5s +} + +teardown { + compose down +} + +step test_002_init_to_single { + exec node1 pg_autoctl manual fsm assign single + exec node1 pg_autoctl inspect pgsetup wait +} + +step test_003_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_init_secondary { + sleep 5s +} + +step test_005_fsm_nodes_set { + # /etc/pgaf/specs is bind-mounted from the directory containing this spec; + # the JSON files are shipped in git alongside monitor_disabled.pgaf. + exec node1 pg_autoctl manual fsm nodes set /etc/pgaf/specs/monitor_disabled_nodes12.json + exec node2 pg_autoctl manual fsm nodes set /etc/pgaf/specs/monitor_disabled_nodes12.json +} + +step test_006_init_to_wait_standby { + exec node2 pg_autoctl manual fsm assign wait_standby +} + +step test_007_catchingup { + exec node1 pg_autoctl manual fsm assign wait_primary + exec node2 pg_autoctl manual fsm assign catchingup +} + +step test_008_secondary { + exec node1 pg_autoctl manual fsm assign primary + exec node2 pg_autoctl manual fsm assign secondary + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} + +step test_009_init_secondary { + sleep 5s +} + +step test_010_fsm_nodes_set { + exec node1 pg_autoctl manual fsm nodes set /etc/pgaf/specs/monitor_disabled_nodes123.json + exec node2 pg_autoctl manual fsm nodes set /etc/pgaf/specs/monitor_disabled_nodes123.json + exec node3 pg_autoctl manual fsm nodes set /etc/pgaf/specs/monitor_disabled_nodes123.json +} + +step test_011_init_to_wait_standby { + exec node1 pg_autoctl manual fsm assign primary + exec node3 pg_autoctl manual fsm assign wait_standby + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} + +step test_012_catchingup { + exec node3 pg_autoctl manual fsm assign catchingup + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} + +step test_013_secondary { + exec node3 pg_autoctl manual fsm assign secondary + exec node1 pg_autoctl manual fsm assign primary + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} diff --git a/tests/tap/specs/monitor_disabled_nodes12.json b/tests/tap/specs/monitor_disabled_nodes12.json new file mode 100644 index 000000000..470ba66d8 --- /dev/null +++ b/tests/tap/specs/monitor_disabled_nodes12.json @@ -0,0 +1,4 @@ +[ + {"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true}, + {"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false} +] diff --git a/tests/tap/specs/monitor_disabled_nodes123.json b/tests/tap/specs/monitor_disabled_nodes123.json new file mode 100644 index 000000000..73f41031f --- /dev/null +++ b/tests/tap/specs/monitor_disabled_nodes123.json @@ -0,0 +1,5 @@ +[ + {"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true}, + {"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false}, + {"node_id":3,"node_name":"node3","node_host":"node3","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false} +] diff --git a/tests/tap/specs/multi_alternate.pgaf b/tests/tap/specs/multi_alternate.pgaf new file mode 100644 index 000000000..5857f7484 --- /dev/null +++ b/tests/tap/specs/multi_alternate.pgaf @@ -0,0 +1,269 @@ +# Test alternating failover scenarios with three nodes: each node serves as +# primary at least once across three series. Series 003 kills node1 then +# node2 in sequence; series 005 reverses the restart order; series 006 kills +# node3 (the surviving primary) and concurrently restarts a previously failed +# node. Verifies correct election outcomes and that pg_rewind restores each +# node as a healthy secondary after it rejoins. +# +# All kills use SIGKILL (compose kill) so the stopped process has no chance +# to update the monitor or claim a re-election win during shutdown. +# +# Predecessor: tests/test_multi_alternate_primary_failures.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +# +# test_002: verify replication slots after initial 3-node convergence. +# Node ids are assigned in registration order: node1=1, node2=2, node3=3. +# Each primary holds physical replication slots for each standby. +# After setup node1 is primary; it holds slots for standbys 2 and 3. +# node2 and node3 are standbys; each holds a slot for the other two. +# + +step test_002_check_initial_slots { + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } +} + +# +# Series 003: stop node1 (primary) → node2 becomes primary, +# then stop node2 → node3 reports_lsn and waits, +# restart node2 → node2 becomes primary again, +# restart node1 → node1 becomes secondary. +# +# node1 node2 node3 +# primary secondary secondary +# +# demoted primary secondary +# +# demoted draining report_lsn +# +# demoted primary secondary +# +# secondary primary secondary +# + +step test_003_001_stop_primary { + compose kill node1 + wait until node2 state is primary + and node3 state is secondary + timeout 120s +} + +step test_003_002_stop_primary { + compose kill node2 + wait until node2 assigned-state = draining timeout 120s + wait until node3 state is report_lsn timeout 120s +} + +step test_003_003_bringup_last_failed_primary { + compose start node2 + wait until node2 state is primary + and node3 state is secondary + timeout 180s +} + +step test_003_004_bringup_first_failed_primary { + compose start node1 + wait until node1 assigned-state = secondary timeout 90s + wait until node2 assigned-state = primary timeout 90s + wait until node3 assigned-state = secondary timeout 90s +} + +# After series 003: node2 is primary (id=2), node1 and node3 are standbys. + +step test_003_005_check_slots_after_series_003 { + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } +} + +step test_005_001_fail_primary_again { + compose kill node2 + wait until node1 state is primary + and node3 state is secondary + timeout 180s +} + +step test_005_001b_write_and_checkpoint { + wait until node1 state is primary + and node3 state is secondary + timeout 120s + sql node1 { + CREATE TABLE IF NOT EXISTS _lsn_advance(x int); INSERT INTO + _lsn_advance VALUES(1); CHECKPOINT; + } +} + +step test_005_002_fail_primary_again { + compose kill node1 + wait until node3 assigned-state = report_lsn timeout 120s +} + +step test_005_003_bring_up_first_failed_primary { + compose start node2 + # 'demoted' is a transient intermediate state that lasts under a second; + # waiting for it races reliably on shared CI runners. Wait for the stable + # end state instead. + wait until node2 state is secondary + and node3 state is primary + timeout 300s +} + +step test_005_004_bring_up_last_failed_primary { + compose start node1 + wait until node1 state is secondary + and node3 state is primary + timeout 120s +} + +# +# Series 006: stop node3 (primary) → node1 becomes primary, +# stop node1 and concurrently restart node3, +# node2 becomes primary, node3 becomes secondary, +# restart node1 → node1 becomes secondary. +# +# node1 node2 node3 +# secondary secondary primary +# +# primary secondary demoted +# +# +# demoted primary secondary +# +# secondary primary secondary +# + +# After series 005: node3 is primary (id=3), node1 and node2 are standbys. + +step test_005_005_check_slots_after_series_005 { + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } +} + +step test_006_001_fail_primary { + compose kill node3 + wait until node1 state is primary + and node2 state is secondary + timeout 120s +} + +step test_006_002_fail_new_primary { + compose kill node1 + compose start node3 + wait until node2 state is primary + and node3 state is secondary + timeout 120s +} + +step test_006_003_bringup_last_failed_primary { + compose start node1 + wait until node1 state is secondary + and node2 state is primary + and node3 state is secondary + timeout 120s +} + +# After series 006: node2 is primary (id=2), node1 and node3 are standbys. + +step test_006_004_check_slots_after_series_006 { + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } +} diff --git a/tests/tap/specs/multi_async.pgaf b/tests/tap/specs/multi_async.pgaf new file mode 100644 index 000000000..ee4ea1bd8 --- /dev/null +++ b/tests/tap/specs/multi_async.pgaf @@ -0,0 +1,257 @@ +# Test mixed sync/async replication configurations with four nodes. +# Starts with one async node (node3), then exercises: setting the whole +# formation async, async failover via perform-promotion, returning to mixed +# sync/async, dropping a node, and two double-failure scenarios (tests 014 +# and 015) where both the primary and the new primary fail before recovery. +# Also covers: candidate-priority=0 secondary rejoining while primary is +# wait_primary, and async standby failing while primary is already wait_primary. +# +# Predecessor: tests/test_multi_async.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 replication-quorum false + node4 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary + and node4 state is secondary + timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_004: set the whole formation to async +# + +step test_004_set_async { + exec node1 pg_autoctl set formation number-sync-standbys 0 + wait until node1 state is primary timeout 60s + exec node1 pg_autoctl set node replication-quorum false + exec node2 pg_autoctl set node replication-quorum false + exec node3 pg_autoctl set node replication-quorum false + wait until node1 state is primary timeout 60s +} + +# +# test_005: write into primary +# + +step test_005_write_into_primary { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3), (4); } + sql node1 { CHECKPOINT; } +} + +# +# test_006: async failover — promote node2 +# + +step test_006_async_failover { + exec monitor pg_autoctl perform promotion --name node2 + wait until node1 state is secondary + and node3 state is secondary + and node2 state is primary + timeout 90s +} + +# +# test_007: read from new primary +# + +step test_007_read_from_new_primary { + sql node2 { SELECT * FROM t1; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_008: set mixed sync/async configuration +# + +step test_008_set_sync_async { + exec node1 pg_autoctl set node replication-quorum true + exec node2 pg_autoctl set node replication-quorum true + exec node3 pg_autoctl set node replication-quorum false + exec node3 pg_autoctl set node candidate-priority 0 + wait until node2 state is primary timeout 60s +} + +step test_009_add_sync_standby { + exec node2 pg_autoctl set formation number-sync-standbys 1 + wait until node1 state is secondary + and node2 state is primary + and node3 state is secondary + and node4 state is secondary + timeout 60s +} + +# +# test_010: promote node1 via monitor SQL, with node4 pre-disconnected. +# +# The perform_promotion SQL is non-blocking; the LSN election completes in +# under a second on fast runners, so catching node4 AT report_lsn via polling +# is unreliable. Instead, disconnect node4 before the election starts — the +# monitor runs the election without node4, which exercises the same code path +# (a candidate missing from the election) reliably and without a race window. +# + +step test_010_promote_node1 { + network disconnect node4 + sql monitor { + SELECT pgautofailover.perform_promotion('default', 'node1'); + } +} + +# +# test_011: verify the election completed with node4 offline +# + +step test_011_ifdown_node4_at_reportlsn { + wait until node3 state is secondary timeout 180s +} + +# +# test_012: ifup node4 +# + +step test_012_ifup_node4 { + network connect node4 + wait until node3 state is secondary + and node4 state is secondary + and node1 state is primary + and node2 state is secondary + timeout 90s +} + +# +# test_013: drop node4 +# + +step test_013_drop_node4 { + exec monitor pg_autoctl drop node --name node4 + compose stop node4 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 90s +} + +step test_014_001_fail_node1 { + compose stop node1 + wait until node2 state is stop_replication timeout 120s + wait until node2 state is wait_primary + and node3 state is secondary + timeout 120s + sql node2 { SHOW synchronous_standby_names; } + expect { } +} + +step test_014_001b_write_and_checkpoint { + sql node2 { INSERT INTO t1 VALUES (100); } + sql node2 { CHECKPOINT; } + sleep 3s +} + +step test_014_002_stop_new_primary_node2 { + compose stop node2 + wait until node3 state is report_lsn timeout 90s +} + +step test_014_003_restart_node1 { + compose start node1 + wait until node1 state is wait_primary timeout 180s +} + +step test_014_004_restart_node2 { + compose start node2 + wait until node2 state is secondary + and node3 state is secondary + and node1 state is primary + timeout 300s +} + +step test_015_001_fail_primary_node1 { + compose stop node1 + wait until node2 state is wait_primary timeout 120s +} + +step test_015_001b_write_and_checkpoint { + sql node2 { INSERT INTO t1 VALUES (101); } + sql node2 { CHECKPOINT; } + sleep 3s +} + +step test_015_002_fail_new_primary_node2 { + compose stop node2 + wait until node3 state is report_lsn timeout 90s +} + +step test_015_003_restart_node2 { + compose start node2 + wait until node2 state is wait_primary + and node3 state is secondary + timeout 120s +} + +step test_015_004_restart_node1 { + compose start node1 + wait until node3 state is secondary + and node2 state is primary + and node1 state is secondary + timeout 120s +} + +step test_016_001_fail_node3 { + network disconnect node3 + wait until node3 assigned-state = catchingup timeout 90s +} + +step test_016_002_fail_node1 { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} + +step test_016_003_restart_node3 { + network connect node3 + wait until node3 assigned-state = secondary timeout 90s + wait until node2 state is wait_primary timeout 60s + sleep 5s +} + +step test_016_004_restart_node1 { + network connect node1 + wait until node3 state is secondary + and node2 state is primary + and node1 state is secondary + timeout 90s +} + +step test_017_001_fail_node1 { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} + +step test_017_002_fail_node3 { + network disconnect node3 + wait until node3 assigned-state = catchingup timeout 90s +} + +step test_017_003_restart_nodes { + network connect node3 + network connect node1 + wait until node3 state is secondary + and node2 state is primary + and node1 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/multi_ifdown.pgaf b/tests/tap/specs/multi_ifdown.pgaf new file mode 100644 index 000000000..e6fb820b8 --- /dev/null +++ b/tests/tap/specs/multi_ifdown.pgaf @@ -0,0 +1,178 @@ +# Test network-partition (ifdown/ifup) scenarios with three nodes and +# explicit candidate-priority settings. +# Covers: basic ifdown of a standby while writes continue on the primary, +# failover when the primary is disconnected and a lagging candidate reconnects, +# and an advanced split-brain scenario where both the primary and the +# most-advanced secondary are cleanly stopped before a behind async node is +# promoted — requiring pg_rewind to fetch missing WAL from the survivors. +# +# Predecessor: tests/test_multi_ifdown.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_004: write into primary +# + +step test_004_write_into_primary { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3), (4); } + sql node1 { CHECKPOINT; } + sql node1 { SELECT count(*) FROM t1; } + expect { 4 } +} + +# +# test_005: set candidate priorities +# node1 priority=90 (primary) +# node2 priority=0 (not a candidate) +# node3 priority=90 +# + +step test_005_set_candidate_priorities { + exec node1 pg_autoctl set node candidate-priority 90 + exec node2 pg_autoctl set node candidate-priority 0 + exec node3 pg_autoctl set node candidate-priority 90 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3, pgautofailover_standby_2) } +} + +# +# test_006: take node3 offline (ifdown) +# + +step test_006_ifdown_node3 { + network disconnect node3 +} + +# +# test_007: insert rows while node3 is down; node2 is sync and gets WAL +# + +step test_007_insert_rows { + sql node1 { + INSERT INTO t1 SELECT x+10 FROM generate_series(1, 10000) as gs(x); + } + sql node1 { CHECKPOINT; } +} + +# +# test_008: fail node1 (primary), reconnect node3, verify node3 becomes primary +# + +step test_008_failover { + network disconnect node1 + network connect node3 + # 'wait_primary' is transient: node3 passes through it in under a second + # once node2 joins as standby. Skip the intermediate wait and go straight + # to the stable end state. 300s: shared CI runners can be slow. + wait until node2 state is secondary + and node3 state is primary + timeout 300s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_009: read from new primary +# + +step test_009_read_from_new_primary { + sql node3 { SELECT count(*) FROM t1; } + expect { 10004 } +} + +# +# test_010: start node1 again as secondary +# + +step test_010_start_node1_again { + network connect node1 + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_011–012: prepare for advanced scenario +# Promote node2, make node2 async so it can fall behind +# + +step test_011_prepare_candidate_priorities { + exec node2 pg_autoctl set node candidate-priority 100 +} + +step test_012_prepare_replication_quorums { + exec node3 pg_autoctl set formation number-sync-standbys 0 + exec node2 pg_autoctl set node replication-quorum false +} + +step test_013_secondary_gets_behind_primary { + network disconnect node2 + sql node3 { INSERT INTO t1 VALUES (5), (6); } + sql node3 { CHECKPOINT; } + sql node1 { SELECT count(*) FROM t1; } + expect { 10006 } + sleep 2s +} + +# +# test_014: trigger a manual failover — node2 (highest candidate priority 100) +# wins the report_lsn election and becomes the new primary. +# All three nodes are online so the election converges quickly; +# node2 may reach prepare_promotion before polling catches report_lsn, +# so we wait for the stable end state instead. +# + +step test_014_secondary_reports_lsn { + wait until node1 state is secondary + and node3 state is primary + timeout 60s + # Reconnect the behind async node so it participates in the election. + network connect node2 + # Give node2's keeper time to check in so the monitor treats it as healthy. + sleep 10s + # Blocking perform failover: waits until one node reports the new primary + # state before returning (exits 0). With all three nodes healthy the + # report_lsn election finishes in under 5s, well within the exec timeout. + exec monitor pg_autoctl perform failover +} + +# +# test_015: verify the failover result — node2 is primary, node1 and node3 +# rejoined as secondaries, and the data is intact. +# + +step test_015_finalize_failover { + wait until node1 state is secondary + and node3 state is secondary + and node2 state is primary + timeout 300s + sql node2 { SELECT count(*) FROM t1; } + expect { 10006 } +} diff --git a/tests/tap/specs/multi_maintenance.pgaf b/tests/tap/specs/multi_maintenance.pgaf new file mode 100644 index 000000000..cbf55bb95 --- /dev/null +++ b/tests/tap/specs/multi_maintenance.pgaf @@ -0,0 +1,280 @@ +# Test the maintenance mode lifecycle with four nodes (three sync + one async). +# Covers: putting a standby in maintenance while triggering failover, +# both standbys in maintenance while primary keeps running, attempting to +# stop the primary while no standby is available (must fail), enabling +# maintenance on the primary with --allow-failover, and verifying the monitor +# enforces the invariant that at least one non-maintenance standby always +# remains when number-sync-standbys > 0. +# +# node4 starts async / candidate-priority 0; test_013 promotes it to +# full quorum participant to simulate dynamic node addition. +# +# Predecessor: tests/test_multi_maintenance.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + node4 candidate-priority 0 replication-quorum false + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_004: write into primary +# + +step test_004_write_into_primary { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3), (4); } + sql node1 { CHECKPOINT; } + sql node1 { SELECT count(*) FROM t1; } + expect { 4 } +} + +step test_005_set_candidate_priorities { + exec node1 pg_autoctl set node candidate-priority 80 + exec node2 pg_autoctl set node candidate-priority 70 + exec node3 pg_autoctl set node candidate-priority 90 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3, pgautofailover_standby_2) } +} + +# +# test_006a: put node2 in maintenance, failover to node3, then disable maintenance +# + +step test_006a_maintenance_and_failover { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s + stop postgres node2 + wait until node1 state is primary timeout 60s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3, pgautofailover_standby_2) } + exec monitor pg_autoctl perform failover + wait until node3 state is primary + and node1 state is secondary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } + exec node2 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_006b: read from new primary +# + +step test_006b_read_from_new_primary { + sql node3 { SELECT * FROM t1; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_007a: put node1 in maintenance +# + +step test_007a_node1_to_maintenance { + wait until node3 state is primary timeout 60s + exec node1 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s +} + +# +# test_007b: put node2 in maintenance; both standbys now in maintenance +# + +step test_007b_node2_to_maintenance { + exec node2 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_008a: stop the primary (node3) while standbys are in maintenance +# node3 must NOT be assigned draining (both secondaries are in maintenance) +# + +step test_008a_stop_primary { + assert node3 stays primary while { + network disconnect node3 + sleep 30s + } + wait until node3 assigned-state = primary timeout 90s +} + +# +# test_008b: restart primary node3 +# + +step test_008b_start_primary { + network connect node3 + wait until node3 state is primary timeout 90s +} + +step test_009a_enable_maintenance_on_primary_should_fail { + exec-fails node3 pg_autoctl enable maintenance --allow-failover +} + +# +# test_009b: disable maintenance on node1 and node2 +# + +step test_009b_disable_maintenance { + exec node1 pg_autoctl disable maintenance + exec node2 pg_autoctl disable maintenance + # 180s: two nodes rejoining simultaneously (pg_rewind + replication restart) + # can be slow on shared CI runners + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + timeout 180s +} + +step test_010_set_number_sync_standby_to_zero { + exec node3 pg_autoctl set formation number-sync-standbys 0 +} + +# +# test_011: put both standbys in maintenance; primary becomes wait_primary +# + +step test_011_all_to_maintenance { + wait until node3 state is primary timeout 60s + exec node1 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s + exec node2 pg_autoctl enable maintenance + wait until node3 state is wait_primary timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { } +} + +# +# test_012: can write while in wait_primary (number-sync-standbys=0) +# + +step test_012_can_write_during_maintenance { + sql node3 { INSERT INTO t1 VALUES (5), (6); } + sql node3 { CHECKPOINT; } +} + +step test_013_add_standby { + exec node4 pg_autoctl set node replication-quorum true + exec node4 pg_autoctl set node candidate-priority 50 + wait until node4 state is secondary + and node3 state is primary + and node2 state is maintenance + and node1 state is maintenance + timeout 60s +} + +# +# test_014: disable maintenance on node2 and node1 +# + +step test_014_disable_maintenance { + wait until node2 state is maintenance timeout 60s + exec node2 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s + wait until node1 state is maintenance timeout 60s + exec node1 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s +} + +step test_015_set_number_sync_standby_to_one { + exec node3 pg_autoctl set formation number-sync-standbys 1 + wait until node3 state is primary timeout 60s +} + +# +# test_016: put two standbys in maintenance (number-sync-standbys=1, primary stays primary) +# + +step test_016_two_standbys_in_maintenance { + exec node1 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s + exec node2 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s +} + +step test_017_primary_to_maintenance { + exec-fails node3 pg_autoctl enable maintenance +} + +# +# test_018: disable maintenance on node2 and node1 +# + +step test_018_disable_maintenance { + wait until node2 state is maintenance timeout 60s + exec node2 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s + wait until node1 state is maintenance timeout 60s + exec node1 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s +} + +# +# test_019: set priorities so node1 is the preferred candidate +# + +step test_019_set_priorities { + exec node1 pg_autoctl set node candidate-priority 90 + exec node2 pg_autoctl set node candidate-priority 70 + exec node3 pg_autoctl set node candidate-priority 70 + exec node4 pg_autoctl set node candidate-priority 70 + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + and node4 state is secondary + timeout 60s +} + +# +# test_020: put primary (node3) in maintenance with failover allowed +# + +step test_020_primary_to_maintenance { + wait until node3 state is primary timeout 60s + exec node3 pg_autoctl enable maintenance --allow-failover + wait until node3 state is maintenance + and node2 state is secondary + and node4 state is secondary + and node1 state is primary + timeout 90s +} + +# +# test_021: disable maintenance on node3, it rejoins as secondary +# + +step test_021_stop_maintenance { + exec node3 pg_autoctl disable maintenance + wait until node3 state is secondary + and node1 state is primary + and node2 state is secondary + and node4 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/multi_standbys.pgaf b/tests/tap/specs/multi_standbys.pgaf new file mode 100644 index 000000000..791208238 --- /dev/null +++ b/tests/tap/specs/multi_standbys.pgaf @@ -0,0 +1,306 @@ +# Test multi-standby formation features with four nodes. +# Covers: candidate-priority and replication-quorum APIs (get/set/validate), +# number_sync_standbys get/set and the auto-decrement trigger when a node is +# dropped, failover and read-from-all-nodes verification, network-partition +# failover, unblocking writes by setting number-sync-standbys=0 while two +# standbys are down, and the edge case where all candidate priorities are zero +# (no promotion possible until one priority is raised to ≥1). +# +# Predecessor: tests/test_multi_standbys.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + node4 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node2 state is secondary + and node3 state is secondary + and node4 state is secondary + timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_002: candidate priority API +# + +step test_002_candidate_priority { + exec node1 pg_autoctl get node candidate-priority + expect { 50 } + exec-fails node1 pg_autoctl set node candidate-priority -1 + exec node1 pg_autoctl get node candidate-priority + expect { 50 } + exec node1 pg_autoctl set node candidate-priority 99 + exec node1 pg_autoctl get node candidate-priority + expect { 99 } +} + +# +# test_003: replication quorum API +# + +step test_003_replication_quorum { + exec node1 pg_autoctl get node replication-quorum + expect { true } + exec-fails node1 pg_autoctl set node replication-quorum "wrong quorum" + exec node1 pg_autoctl get node replication-quorum + expect { true } + exec node1 pg_autoctl set node replication-quorum false + exec node1 pg_autoctl get node replication-quorum + expect { false } + exec node1 pg_autoctl set node replication-quorum true + exec node1 pg_autoctl get node replication-quorum + expect { true } +} + +step test_004_001_add_three_standbys { + wait until node2 state is secondary + and node1 state is primary + timeout 60s +} + +step test_004_002_add_three_standbys { + wait until node3 state is secondary + and node1 state is primary + timeout 60s +} + +step test_004_003_add_three_standbys { + wait until node4 state is secondary + and node1 state is primary + timeout 60s +} + +# +# test_005: number_sync_standbys +# + +step test_005_number_sync_standbys { + exec node1 pg_autoctl set formation number-sync-standbys 2 + wait until node1 state is primary timeout 30s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 2 (pgautofailover_standby_2, pgautofailover_standby_3, pgautofailover_standby_4) } + exec node1 pg_autoctl set formation number-sync-standbys 0 + wait until node1 state is primary timeout 30s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3, pgautofailover_standby_4) } + exec node1 pg_autoctl set formation number-sync-standbys 1 + wait until node1 state is primary timeout 30s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3, pgautofailover_standby_4) } +} + +step test_006_number_sync_standbys_trigger { + exec node1 pg_autoctl set formation number-sync-standbys 2 + exec monitor pg_autoctl drop node --name node4 + compose stop node4 + wait until node1 state is primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } + sleep 6s +} + +# +# test_007: create table t1 +# + +step test_007_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } + sql node1 { CHECKPOINT; } +} + +step test_008_set_candidate_priorities { + exec node1 pg_autoctl set node candidate-priority 90 + exec node2 pg_autoctl set node candidate-priority 90 + exec node3 pg_autoctl set node candidate-priority 70 + wait until node1 state is primary timeout 60s +} + +# +# test_009: failover — node2 becomes primary +# + +step test_009_failover { + exec monitor pg_autoctl perform failover + wait until node2 state is primary + and node3 state is secondary + and node1 state is secondary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_3) } +} + +# +# test_010: read from all nodes +# + +step test_010_read_from_nodes { + sql node1 { SELECT * FROM t1; } + expect { { 1 } { 2 } } + sql node2 { SELECT * FROM t1; } + expect { { 1 } { 2 } } + sql node3 { SELECT * FROM t1; } + expect { { 1 } { 2 } } +} + +# +# test_011: write into new primary +# + +step test_011_write_into_new_primary { + sql node2 { INSERT INTO t1 VALUES (3), (4); } + sql node2 { CHECKPOINT; } +} + +# +# test_012: fail primary node2 +# + +step test_012_fail_primary { + network disconnect node2 + wait until node1 state is stop_replication timeout 90s + wait until node1 state is primary + and node3 state is secondary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +# +# test_013: restart node2 as secondary +# + +step test_013_restart_node2 { + network connect node2 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_014_001_fail_set_properties { + exec node1 pg_autoctl set node candidate-priority 50 + exec node2 pg_autoctl set node candidate-priority 50 + exec node3 pg_autoctl set node candidate-priority 50 + wait until node1 state is primary timeout 60s +} + +step test_014_002_fail_two_standby_nodes { + compose stop node2 + compose stop node3 + wait until node2 assigned-state = catchingup timeout 180s + wait until node3 assigned-state = catchingup timeout 180s + # node1 remains primary (blocking writes at postgres level) until + # number-sync-standbys is explicitly set to 0 in the next step. + wait until node1 state is primary timeout 180s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_014_003_unblock_writes { + exec node1 pg_autoctl set formation number-sync-standbys 0 + wait until node1 state is wait_primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { } +} + +step test_014_004_restart_nodes { + compose start node3 + compose start node2 + wait until node3 state is secondary + and node2 state is secondary + and node1 state is primary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_015_002_fail_two_standby_nodes { + network disconnect node2 + network disconnect node3 + wait until node1 state is wait_primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { } +} + +step test_015_003_set_properties { + sql monitor { + SELECT pgautofailover.set_formation_number_sync_standbys('default', 1); + } + wait until node1 assigned-state = primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_015_004_restart_nodes { + network connect node3 + network connect node2 + wait until node3 state is secondary + and node2 state is secondary + and node1 state is primary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_016_001_set_candidate_priorities_to_zero { + exec node1 pg_autoctl set node candidate-priority 0 + exec node2 pg_autoctl set node candidate-priority 0 + exec node3 pg_autoctl set node candidate-priority 0 + wait until node1 state is primary timeout 60s +} + +step test_016_002_trigger_failover { + exec monitor bash -c "pg_autoctl perform failover --wait 1 || true" + wait until node3 state is report_lsn + and node2 state is report_lsn + and node1 state is report_lsn + timeout 90s +} + +step test_016_003_set_candidate_priority_to_one { + exec node2 pg_autoctl set node candidate-priority 1 + wait until node2 state is primary + and node1 state is secondary + and node3 state is secondary + timeout 90s +} + +step test_016_004_reset_candidate_priority { + exec node2 pg_autoctl set node candidate-priority 0 + wait until node2 state is primary + and node1 state is secondary + and node3 state is secondary + timeout 60s +} + +step test_016_005_perform_promotion { + exec monitor pg_autoctl perform promotion --name node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 90s +} + +step test_017_remove_old_primary { + exec monitor pg_autoctl drop node --name node2 + compose stop node2 + wait until node1 state is primary + and node3 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/nonha_citus_operation.pgaf b/tests/tap/specs/nonha_citus_operation.pgaf new file mode 100644 index 000000000..e607f025e --- /dev/null +++ b/tests/tap/specs/nonha_citus_operation.pgaf @@ -0,0 +1,104 @@ +# Test non-HA Citus formation lifecycle: start with secondary=false, add +# workers, enable secondary support, add standbys, attempt to disable while +# standbys exist (expect failure), fail-over primaries, drop them, and finally +# disable secondary support once only single nodes remain. +# +# Ported from tests/test_nonha_citus_operation.py +# Predecessor: tests/test_nonha_citus_operation.py + +cluster { + monitor + formation non-ha { + coordinator1a coordinator + coordinator1b coordinator launch deferred + worker1a worker group 1 + worker1b worker group 1 launch deferred + worker2a worker group 2 + worker2b worker group 2 launch deferred + } +} + +setup { + wait until coordinator1a state is single timeout 120s + wait until worker1a state is single timeout 120s + wait until worker2a state is single timeout 120s + sleep 5s +} + +teardown { + compose down +} + +step test_001_disable_secondary_on_formation { + exec monitor pg_autoctl disable secondary --formation non-ha +} + +step test_002_wait_metadata_sync { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' +} + +step test_003_create_distributed_table { + sql coordinator1a { CREATE TABLE t1 (a int); } + sql coordinator1a { SELECT create_distributed_table('t1', 'a'); } + sql coordinator1a { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_enable_secondary { + exec monitor pg_autoctl enable secondary --formation non-ha + exec coordinator1b pg_autoctl node start /etc/pgaf/node.ini + exec worker1b pg_autoctl node start /etc/pgaf/node.ini + exec worker2b pg_autoctl node start /etc/pgaf/node.ini +} + +step test_005_add_secondaries { + wait until coordinator1b state is secondary timeout 120s + wait until worker1b state is secondary timeout 120s + wait until worker2b state is secondary timeout 120s +} + +step test_006_fail_when_disabling_with_secondaries { + exec-fails monitor pg_autoctl disable secondary --formation non-ha +} + +step test_007_failover_coordinator { + network disconnect coordinator1a + wait until coordinator1b state is wait_primary timeout 90s + network connect coordinator1a + wait until coordinator1b state is primary + and coordinator1a state is secondary + timeout 180s +} + +step test_007b_failover_worker1 { + network disconnect worker1a + wait until worker1b state is wait_primary timeout 90s + network connect worker1a + wait until worker1b state is primary + and worker1a state is secondary + timeout 180s +} + +step test_007c_failover_worker2 { + network disconnect worker2a + wait until worker2b state is wait_primary timeout 90s + network connect worker2a + wait until worker2b state is primary + and worker2a state is secondary + timeout 180s +} + +step test_008_remove_old_primaries { + exec monitor pg_autoctl drop node --name coordinator1a --formation non-ha + exec monitor pg_autoctl drop node --name worker1a --formation non-ha + exec monitor pg_autoctl drop node --name worker2a --formation non-ha + compose stop coordinator1a + compose stop worker1a + compose stop worker2a + wait until coordinator1b state is single timeout 60s + wait until worker1b state is single timeout 60s + wait until worker2b state is single timeout 60s +} + +step test_009_disable_secondaries { + exec monitor pg_autoctl disable secondary --formation non-ha +} diff --git a/tests/tap/specs/replace_monitor.pgaf b/tests/tap/specs/replace_monitor.pgaf new file mode 100644 index 000000000..94cef2dd2 --- /dev/null +++ b/tests/tap/specs/replace_monitor.pgaf @@ -0,0 +1,95 @@ +# Test replacing a failed monitor with a new one. +# Sequence: bring up a cluster, drop the old monitor, disable it on both +# nodes, create a new monitor, re-enable it on both nodes, verify the cluster +# re-converges and a failover through the new monitor works. + +cluster { + monitor + monitor newmonitor initially stopped + ssl off + formation { + node1 + node2 + } +} + +setup { + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s +} + +teardown { + compose down +} + +step create_table { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { 1 } + expect { 2 } +} + +step drop_old_monitor { + exec monitor pg_autoctl stop +} + +step disable_monitor_on_nodes { + exec node2 pg_autoctl disable monitor --force + exec node1 pg_autoctl disable monitor --force +} + +step write_while_monitor_down { + sql node1 { INSERT INTO t1 VALUES (3); } +} + +step read_replicated_write { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { 1 } + expect { 2 } + expect { 3 } +} + +step create_new_monitor { + compose start newmonitor + exec newmonitor pg_autoctl inspect pgsetup wait + set monitor newmonitor +} + +step enable_monitor_node1 { + exec node1 pg_autoctl enable monitor postgresql://autoctl_node@newmonitor/pg_auto_failover + wait until node1 state is single timeout 90s +} + +step enable_monitor_node2 { + exec node2 pg_autoctl enable monitor postgresql://autoctl_node@newmonitor/pg_auto_failover + wait until node2 state is catchingup timeout 90s + wait until node1 state is wait_primary timeout 90s +} + +step wait_for_convergence { + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s +} + +step failover_via_new_monitor { + exec newmonitor pg_autoctl perform failover + wait until node2 state is primary timeout 90s + wait until node1 state is secondary timeout 90s + wait until node2 state is primary timeout 90s + wait until node1 state is secondary timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +step read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { 1 } + expect { 2 } + expect { 3 } +} diff --git a/tests/tap/specs/skip_pg_hba.pgaf b/tests/tap/specs/skip_pg_hba.pgaf new file mode 100644 index 000000000..ef14c7f2b --- /dev/null +++ b/tests/tap/specs/skip_pg_hba.pgaf @@ -0,0 +1,58 @@ +# Test --skip-pg-hba (auth-method=skip): pg_autoctl must NOT edit pg_hba.conf +# on either node. HBA rules are managed manually by the operator. +# +# Ported from tests/test_skip_pg_hba.py +# Predecessor: tests/test_skip_pg_hba.py + +cluster { + monitor + formation { + node1 auth skip + node2 auth skip + } +} + +setup { + # With auth=skip, nodes don't add HBA rules automatically. + # Add them now so the monitor can health-check the data nodes. + exec node1 pg_autoctl inspect pgsetup hba-lan + exec node2 pg_autoctl inspect pgsetup hba-lan + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004a_hba_have_not_been_edited { + exec node1 pg_autoctl config get postgresql.hba_level + expect { skip } + exec node2 pg_autoctl config get postgresql.hba_level + expect { skip } +} + +step test_005_failover { + exec monitor pg_autoctl perform failover + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} + +step test_006_restart_secondary { + compose stop node1 + compose start node1 + wait until node1 state is secondary timeout 60s +} + +step test_006a_hba_have_not_been_edited { + exec node1 pg_autoctl config get postgresql.hba_level + expect { skip } + exec node2 pg_autoctl config get postgresql.hba_level + expect { skip } +} diff --git a/tests/tap/specs/ssl_cert.pgaf b/tests/tap/specs/ssl_cert.pgaf new file mode 100644 index 000000000..d107f09b7 --- /dev/null +++ b/tests/tap/specs/ssl_cert.pgaf @@ -0,0 +1,87 @@ +# Test pg_auto_failover with SSL using CA-signed certificates and cert auth. +# +# Monitor and all data nodes are set up with server certificates signed by a +# shared root CA, and client certificates are placed in ~/.postgresql/. +# Authentication uses the "cert" method: pg_autoctl writes hostssl + cert +# rules to pg_hba.conf and the ident map to pg_ident.conf automatically. +# +# Ported from tests/test_ssl_cert.py +# Predecessor: tests/test_ssl_cert.py + +cluster { + monitor + ssl verify-ca + auth cert + formation { + node1 + node2 + } +} + +setup { + # 300s: verify-ca + cert-auth cluster formation is slower than plain auth + # on shared CI runners, especially as the third spec in the ssl schedule + wait until node1 state is primary + and node2 state is secondary + timeout 300s + promote node1 +} + +teardown { + compose down +} + +# +# test_000: verify monitor has SSL enabled with cert auth +# + +step test_000_create_monitor { + exec monitor pg_autoctl inspect pgsetup wait + sql monitor { SHOW ssl; } + expect { on } + exec monitor pg_autoctl config get ssl.sslmode + expect { verify-ca } +} + +# +# test_001: verify primary has SSL enabled with cert auth +# + +step test_001_init_primary { + wait until node1 state is primary timeout 60s + sql node1 { SHOW ssl; } + expect { on } + exec node1 pg_autoctl config get ssl.sslmode + expect { verify-ca } +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_003: verify secondary has SSL enabled with cert auth +# + +step test_003_init_secondary { + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node2 { SHOW ssl; } + expect { on } + exec node2 pg_autoctl config get ssl.sslmode + expect { verify-ca } +} + +# +# test_004: failover +# + +step test_004_failover { + exec monitor pg_autoctl perform failover +} diff --git a/tests/tap/specs/ssl_self_signed.pgaf b/tests/tap/specs/ssl_self_signed.pgaf new file mode 100644 index 000000000..c0a9f4f66 --- /dev/null +++ b/tests/tap/specs/ssl_self_signed.pgaf @@ -0,0 +1,76 @@ +# Test pg_auto_failover with self-signed SSL certificates (ssl-mode = require). +# Both the monitor and data nodes generate their own self-signed certs. +# +# Ported from tests/test_ssl_self_signed.py +# Predecessor: tests/test_ssl_self_signed.py + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +# +# test_000: create monitor with self-signed SSL +# + +step test_000_create_monitor { + exec monitor pg_autoctl inspect pgsetup wait + sql monitor { SHOW ssl; } + expect { on } + exec monitor pg_autoctl config get ssl.sslmode +} + +# +# test_001: init primary with self-signed SSL +# + +step test_001_init_primary { + wait until node1 state is single timeout 60s + sql node1 { SHOW ssl; } + expect { on } + exec node1 pg_autoctl config get ssl.sslmode +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_003: add secondary node2 with self-signed SSL +# + +step test_003_init_secondary { + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node2 { SHOW ssl; } + expect { on } + exec node2 pg_autoctl config get ssl.sslmode +} + +# +# test_004: failover +# + +step test_004_failover { + exec monitor pg_autoctl perform failover + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/tablespaces.pgaf b/tests/tap/specs/tablespaces.pgaf new file mode 100644 index 000000000..abcc9a85b --- /dev/null +++ b/tests/tap/specs/tablespaces.pgaf @@ -0,0 +1,148 @@ +# Tablespace HA: verify that tablespaces stored on extra volumes outside +# PGDATA survive failover, network partitions, and pg_rewind. +# +# Three tablespace directories are used across the test: +# /extra_volumes/extended_a — created before node2 joins +# /extra_volumes/extended_b — created while streaming replication is active +# /extra_volumes/extended_c — created during a network partition (wait_primary) +# +# Each node gets its own named Docker volume for each tablespace path. +# Replication copies data via WAL; volumes are not shared between nodes. +# +# Predecessor: tests/tablespaces/test_tablespaces_0[1-6].py + +cluster { + monitor + formation { + node node1 { + volume extended_a "/extra_volumes/extended_a" + volume extended_b "/extra_volumes/extended_b" + volume extended_c "/extra_volumes/extended_c" + } + node node2 { + volume extended_a "/extra_volumes/extended_a" + volume extended_b "/extra_volumes/extended_b" + volume extended_c "/extra_volumes/extended_c" + } + } +} + +setup { + wait until node1 state is primary + and node2 state is secondary + timeout 90s + exec node1 sudo chown docker /extra_volumes/extended_a /extra_volumes/extended_b /extra_volumes/extended_c + exec node2 sudo chown docker /extra_volumes/extended_a /extra_volumes/extended_b /extra_volumes/extended_c +} + +teardown { + compose down +} + +# +# test_001: create table t1 in default tablespace, then create tablespace +# extended_a and table t2 in it. node2 has not joined yet. +# + +step test_001_create_tablespace_a { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } + sql node1 { + CREATE TABLESPACE extended_a LOCATION '/extra_volumes/extended_a'; + } + sql node1 { CREATE TABLE t2(i int) TABLESPACE extended_a; } + sql node1 { INSERT INTO t2 VALUES (3), (4); } +} + +# +# test_002: node2 joins as secondary; verify that both tables — including +# t2 in extended_a — are accessible on the secondary. +# + +step test_002_init_secondary { + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } + sql node2 { SELECT * FROM t2 ORDER BY i; } + expect { { 3 } { 4 } } +} + +# +# test_003: create tablespace extended_b while streaming replication is +# active; verify the new tablespace and table t3 replicate correctly. +# + +step test_003_tablespace_b_while_streaming { + sql node1 { + CREATE TABLESPACE extended_b LOCATION '/extra_volumes/extended_b'; + } + sql node1 { CREATE TABLE t3(i int) TABLESPACE extended_b; } + sql node1 { INSERT INTO t3 VALUES (5), (6); } + sql node2 { SELECT * FROM t3 ORDER BY i; } + expect { { 5 } { 6 } } +} + +# +# test_004: failover — node2 becomes primary. Write to tablespace tables +# on the new primary, verify the new secondary (node1) reads them back. +# + +step test_004_failover { + exec monitor pg_autoctl perform failover + sql node2 { INSERT INTO t2 VALUES (7); } + sql node1 { SELECT * FROM t2 ORDER BY i; } + expect { { 3 } { 4 } { 7 } } + sql node2 { INSERT INTO t3 VALUES (8); } + sql node1 { SELECT * FROM t3 ORDER BY i; } + expect { { 5 } { 6 } { 8 } } +} + +# +# test_005: network partition — disconnect node1 (now secondary). +# While node2 is in wait_primary, create tablespace extended_c and write +# to all three tablespace tables. +# + +step test_005_network_partition { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s + sql node2 { + CREATE TABLESPACE extended_c LOCATION '/extra_volumes/extended_c'; + } + sql node2 { CREATE TABLE t4(i int) TABLESPACE extended_c; } + sql node2 { INSERT INTO t4 VALUES (10), (11); } + sql node2 { INSERT INTO t2 VALUES (12); } + sql node2 { INSERT INTO t3 VALUES (13); } +} + +# +# test_006: reconnect node1. pg_rewind copies WAL from node2, which +# includes the extended_c tablespace directory. After convergence node1 +# must see all writes that happened during the partition. +# + +step test_006_node1_rejoins { + network connect node1 + wait until node1 state is secondary + and node2 state is primary + timeout 90s + sql node1 { SELECT * FROM t4 ORDER BY i; } + expect { { 10 } { 11 } } + sql node1 { SELECT * FROM t2 ORDER BY i; } + expect { { 3 } { 4 } { 7 } { 12 } } + sql node1 { SELECT * FROM t3 ORDER BY i; } + expect { { 5 } { 6 } { 8 } { 13 } } +} + +# +# test_007: promote the original primary (node1) back via perform-promotion. +# + +step test_007_promote_original_primary { + exec monitor pg_autoctl perform promotion --name node1 + wait until node1 state is primary + and node2 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/upgrade.pgaf b/tests/tap/specs/upgrade.pgaf new file mode 100644 index 000000000..d60f085b0 --- /dev/null +++ b/tests/tap/specs/upgrade.pgaf @@ -0,0 +1,190 @@ +# Upgrade test: live binary + extension swap without container restarts. +# +# Tests the documented production upgrade procedure: +# +# 1. Install new binary on ALL nodes while the cluster is running. +# The keeper's FSM loop fires exit(EXIT_CODE_MONITOR) only when the +# monitor extension version matches the NEW binary's required version, +# so pre-staging the binary is safe and does not trigger self-restart yet. +# +# 2. Install new binary + new extension files on the monitor, then restart +# just the monitor listener child (not the container). The listener +# detects the installed extension version ("2.1") differs from what the +# new binary requires ("2.2"), runs ALTER EXTENSION pgautofailover +# UPDATE TO '2.2', and restarts Postgres — all without stopping the +# supervisor (PID 1). +# +# 3. Each keeper's node-active child polls the monitor, sees the extension +# is now "2.2", compares against the on-disk binary's required version +# ("2.2"), finds a match, and calls exit(EXIT_CODE_MONITOR). The +# supervisor re-execs the node-active child from the new on-disk binary. +# Postgres is never restarted; only the pg_autoctl child re-execs. +# +# Image build requirements — run before this spec: +# make -C tests/upgrade pgaf-current → pgaf:current (built from v2.1 tag) +# make -C tests/upgrade pgaf-next → pgaf:next (built from current branch) +# +# pgaf:current contains BOTH binaries (2.1 and 2.2) and the v2.2 extension +# files pre-staged at /usr/local/bin/pgaf/2.2/. The "inject" steps below +# flip an in-container symlink and copy those pre-staged files — no runtime +# dependency on pgaf:next once the images are built. +# +# PGVERSION defaults to 16 (the latest PG that v2.1 supports). Rebuild with +# PGVERSION=17 make -C tests/upgrade pgaf-current pgaf-next +# if PREV_TAG supports PG17+ (and update the /16/ paths in test_003 below). + +cluster { + monitor + image "pgaf:current" + formation { + node1 + node2 + node3 + } +} + +setup { + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +# ------------------------------------------------------------------------- +# Baseline: cluster running on pgaf:current (v2.1), extension at "2.1". +# Write a marker row that must survive through the upgrade. +# ------------------------------------------------------------------------- + +step test_001_baseline { + sql node1 { CREATE TABLE upgrade_marker(ts timestamptz default now()); } + sql node1 { INSERT INTO upgrade_marker DEFAULT VALUES; } + sql monitor { + SELECT installed_version + FROM pg_available_extensions + WHERE name = 'pgautofailover'; + } + expect { 2.1 } +} + +# ------------------------------------------------------------------------- +# Phase 1: flip the "current" symlink on data nodes to point at the v2.2 +# binary. Both binaries are already baked into pgaf:current — no docker cp +# is needed. The static shim at /usr/local/bin/pg_autoctl delegates to +# pgaf/current/pg_autoctl, so after the flip: +# - The keeper's on-disk version check (argv0 version --json) returns "2.2" +# - The v2.1 supervisor's next child spawn also uses v2.2 (PG_AUTOCTL_DEBUG_BIN_PATH) +# No self-restart fires yet: the monitor extension is still "2.1" and the +# keeper only exits when installed "2.2" matches on-disk required "2.2". +# ------------------------------------------------------------------------- + +step test_002_inject_node_binaries { + exec node1 ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current + exec node2 ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current + exec node3 ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current +} + +# ------------------------------------------------------------------------- +# Phase 2: flip the symlink on the monitor and install the v2.2 extension +# files (also pre-baked into pgaf:current at /usr/local/bin/pgaf/2.2/). +# Then restart just the listener child; the v2.2 listener runs: +# ALTER EXTENSION pgautofailover UPDATE TO '2.2' +# ------------------------------------------------------------------------- + +step test_003_upgrade_monitor { + # Install v2.2 extension files into system paths. The script is baked into + # the image with PGVERSION resolved at build time (no hardcoded /16/ here). + exec monitor sudo /usr/local/bin/pgaf/2.2/install-extension + # Run ALTER EXTENSION via psql BEFORE flipping the symlink so the v2.2 + # listener that starts next already sees installed_version = "2.2" and + # skips the Postgres-restart code path (which races with the supervisor). + exec monitor psql -U docker -d pg_auto_failover -c "ALTER EXTENSION pgautofailover UPDATE TO '2.2'" + # Flip the monitor symlink; from here pg_autoctl = v2.2. + exec monitor ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current + # Restart postgres on the monitor so it loads the v2.2 pgautofailover.so. + # Without this, the old .so is still resident in memory even though the + # control file and SQL catalog were updated; every keeper call to the + # monitor functions fails with "loaded library requires 2.1 but control + # file specifies 2.2", exhausting the supervisor restart limit. + exec monitor pg_autoctl manual service restart postgres --pgdata /var/lib/postgres/pgaf +} + +# ------------------------------------------------------------------------- +# Phase 3: wait for keepers to self-restart. +# +# After the monitor extension becomes "2.2", the keeper children compare +# installed "2.2" against the on-disk binary's required "2.2" — they match, +# so each child calls exit(EXIT_CODE_MONITOR) and the supervisor re-execs +# from the new binary. No Postgres restart, no container restart. +# ------------------------------------------------------------------------- + +step test_003b_wait_keeper_restart { + # Give keepers time to detect the extension upgrade and self-restart. + # v2.1 keeper polls every ~5 s; the listener restart + ALTER EXTENSION adds + # another 1-2 s. 45 s gives ample time on slow CI runners. + exec monitor sleep 45 +} + +step test_004_wait_convergence { + # 300s: 3-node post-upgrade reconvergence can be slow on shared CI runners + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 300s +} + +# ------------------------------------------------------------------------- +# Verify the upgrade mechanism actually fired on each node. +# ------------------------------------------------------------------------- + +step test_005_verify_node_restart { + logs node1 contains "exiting for a restart of the node-active process" + logs node2 contains "exiting for a restart of the node-active process" + logs node3 contains "exiting for a restart of the node-active process" +} + +step test_006_verify_extension_version { + sql monitor { + SELECT installed_version + FROM pg_available_extensions + WHERE name = 'pgautofailover'; + } + expect { 2.2 } +} + +# ------------------------------------------------------------------------- +# Data integrity: row written before the upgrade must be readable on all +# nodes, confirming Postgres was never stopped on the data nodes. +# ------------------------------------------------------------------------- + +step test_007_verify_data_intact { + sql node1 { SELECT count(*) FROM upgrade_marker; } + expect { 1 } + sql node2 { SELECT count(*) FROM upgrade_marker; } + expect { 1 } + sql node3 { SELECT count(*) FROM upgrade_marker; } + expect { 1 } +} + +# ------------------------------------------------------------------------- +# Smoke: failover through the fully-upgraded cluster. +# ------------------------------------------------------------------------- + +step test_008_failover_post_upgrade { + exec monitor pg_autoctl perform promotion --name node2 + wait until node2 state is primary + and node1 state is secondary + and node3 state is secondary + timeout 300s +} + +step test_009_write_on_new_primary { + sql node2 { INSERT INTO upgrade_marker DEFAULT VALUES; } + sql node1 { SELECT count(*) FROM upgrade_marker; } + expect { 2 } + sql node3 { SELECT count(*) FROM upgrade_marker; } + expect { 2 } +} diff --git a/tests/upgrade/Dockerfile.current b/tests/upgrade/Dockerfile.current new file mode 100644 index 000000000..0600e2f39 --- /dev/null +++ b/tests/upgrade/Dockerfile.current @@ -0,0 +1,81 @@ +# pgaf:current — v2.1 with both binaries baked in and a static shim. +# +# Layout: +# /usr/local/bin/pg_autoctl — static bash shim (never replaced) +# /usr/local/bin/pgaf/2.1/pg_autoctl — v2.1 binary +# /usr/local/bin/pgaf/2.2/pg_autoctl — v2.2 binary (from pgaf:next) +# /usr/local/bin/pgaf/current — symlink → pgaf/2.1 (initially) +# +# "Injecting" an upgrade = ln -sf /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current +# Run that inside each container via `docker compose exec`; no docker cp needed. +# +# Why this works for the self-restart mechanism: +# +# The shim always lives at /usr/local/bin/pg_autoctl. It delegates every +# command to pgaf/current/pg_autoctl via `exec -a /usr/local/bin/pg_autoctl` +# so the real binary sees argv[0] = /usr/local/bin/pg_autoctl. That path is +# what pg_autoctl saves as pg_autoctl_argv0, and later uses for the on-disk +# version check: +# +# run_program(pg_autoctl_argv0, "version", "--json") +# +# After the symlink flip, that invocation runs the shim → delegates to v2.2 → +# returns required_extension_version "2.2". Combined with the monitor +# extension just upgraded to "2.2", the keeper detects a match and calls +# exit(EXIT_CODE_MONITOR) so the supervisor re-execs from the new binary. +# +# PG_AUTOCTL_DEBUG_BIN_PATH=/usr/local/bin/pg_autoctl is set in ENV so the +# v2.1 supervisor's pg_autoctl_program also resolves to the shim path, and +# child processes spawned after the symlink flip pick up v2.2. +# +# Startup command translation: +# +# pgaftest generates: pg_autoctl node run /etc/pgaf/node.ini (v2.2 syntax) +# v2.1 does not know "node run"; the shim translates it per node kind. + +ARG BASE_IMAGE=pgaf:current-base +ARG NEXT_IMAGE=pgaf:next + +FROM ${NEXT_IMAGE} AS next +FROM ${BASE_IMAGE} + +USER root + +# bash is required for "exec -a name" (sets argv[0] for the real binary). +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends bash \ + && rm -rf /var/lib/apt/lists/* + +# Place both binaries side by side; current → 2.1 initially. +RUN mkdir -p /usr/local/bin/pgaf/2.1 /usr/local/bin/pgaf/2.2 +RUN mv /usr/local/bin/pg_autoctl /usr/local/bin/pgaf/2.1/pg_autoctl +COPY --from=next /usr/local/bin/pg_autoctl /usr/local/bin/pgaf/2.2/pg_autoctl +RUN ln -s /usr/local/bin/pgaf/2.1 /usr/local/bin/pgaf/current \ + && chown -R docker /usr/local/bin/pgaf + +# Pre-stage v2.2 extension files so the upgrade step is a local install, not +# a docker cp from an external image. Stored alongside the v2.2 binary. +ARG PGVERSION=16 +COPY --from=next /usr/share/postgresql/${PGVERSION}/extension/pgautofailover.control \ + /usr/local/bin/pgaf/2.2/pgautofailover.control +COPY --from=next /usr/share/postgresql/${PGVERSION}/extension/pgautofailover--2.1--2.2.sql \ + /usr/local/bin/pgaf/2.2/pgautofailover--2.1--2.2.sql +COPY --from=next /usr/share/postgresql/${PGVERSION}/extension/pgautofailover--2.2.sql \ + /usr/local/bin/pgaf/2.2/pgautofailover--2.2.sql +COPY --from=next /usr/lib/postgresql/${PGVERSION}/lib/pgautofailover.so \ + /usr/local/bin/pgaf/2.2/pgautofailover.so + +# Install script — uses pg_config at runtime so PGVERSION need not be baked in. +# Usage (inside the container, as root): /usr/local/bin/pgaf/2.2/install-extension +COPY install-extension.sh /usr/local/bin/pgaf/2.2/install-extension +RUN chmod 755 /usr/local/bin/pgaf/2.2/install-extension + +# Static shim — stays at the canonical path forever. +COPY pg_autoctl_shim.sh /usr/local/bin/pg_autoctl +RUN chmod 755 /usr/local/bin/pg_autoctl + +# Tell the v2.1 supervisor to use the shim path for spawning child processes, +# so that after the symlink flip the next child spawn picks up v2.2. +ENV PG_AUTOCTL_DEBUG_BIN_PATH=/usr/local/bin/pg_autoctl + +USER docker diff --git a/tests/upgrade/Makefile b/tests/upgrade/Makefile index 92f17b6da..e178b2022 100644 --- a/tests/upgrade/Makefile +++ b/tests/upgrade/Makefile @@ -1,56 +1,104 @@ -NODES ?= 3 - -PATCH = tests/upgrade/monitor-upgrade-1.7.patch -Q_VERSION = select default_version, installed_version -Q_VERSION += from pg_available_extensions where name = 'pgautofailover' - -build: - docker compose build - -patch: - cd ../.. && git apply $(PATCH) - +# tests/upgrade/Makefile +# +# Builds the two Docker images needed for the upgrade spec and runs it. +# +# Usage: +# make build both images and run the upgrade spec +# make pgaf-current build pgaf:current from PREV_TAG +# make pgaf-next build pgaf:next from current working tree +# make run run tests/tap/specs/upgrade.pgaf +# make clean remove the two images +# +# Customisation: +# PREV_TAG — git tag to build pgaf:current from (default: auto-detected) +# PGVERSION — Postgres major version baked into the image (default: 16) +# +# PGVERSION defaults to 16 because v2.1 (the current PREV_TAG) only supports +# up to PG16 — its monitor extension explicitly rejects PG >= 17. Override +# with PGVERSION=17 only when testing against a PREV_TAG that supports PG17+. +# +# Auto-detection of PREV_TAG: reads PG_AUTOCTL_EXTENSION_VERSION from +# defaults.h to get the current extension version (e.g. "2.2"), then +# finds the most recent tag whose version is strictly lower. The tag +# matching assumes the form "vMAJOR.MINOR". +# +# Examples: +# make PREV_TAG=v2.0 # test 2.0→2.2 on default PG16 +# make PREV_TAG=v2.1 PGVERSION=16 # explicit PG16 + +PGVERSION ?= 16 +PGAFTEST ?= pgaftest +SPEC := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../tap/specs/upgrade.pgaf) +REPO_ROOT := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../..) +DOCKERFILE := $(REPO_ROOT)/Dockerfile + +# Current extension version from source +CURRENT_EXT := $(shell grep 'PG_AUTOCTL_EXTENSION_VERSION ' \ + $(REPO_ROOT)/src/bin/pg_autoctl/defaults.h \ + | grep -o '"[^"]*"' | tr -d '"') + +# Previous release tag: the highest vX.Y tag whose X.Y < CURRENT_EXT. +# Works for two-component versions (2.1, 2.2, …). +# Override with PREV_TAG=v2.0 if the heuristic gives the wrong answer. +_PREV_TAG_DETECT := $(shell \ + cur=$(CURRENT_EXT); \ + git -C $(REPO_ROOT) tag --sort=-version:refname \ + | grep -E '^v[0-9]+[.][0-9]+$$' \ + | grep -v "^v$$cur$$" \ + | head -1) +PREV_TAG ?= $(_PREV_TAG_DETECT) + +.PHONY: all pgaf-current pgaf-next run clean + +all: pgaf-next pgaf-current run + +## Build pgaf:current — the previous release, used as the starting point. +## Built from the git archive of PREV_TAG so that uncommitted changes in +## the working tree do not contaminate the "old" image. +## +## git archive produces a clean tree without git history, so git-version.h +## (a generated file) is absent. We extract the archive into a tempdir, +## synthesise git-version.h from the tag name, then docker build the dir. +pgaf-current: + @if [ -z "$(PREV_TAG)" ]; then \ + echo "ERROR: could not auto-detect PREV_TAG; set it explicitly:"; \ + echo " make PREV_TAG=v2.1"; \ + exit 1; \ + fi + @echo "Building pgaf:current from $(PREV_TAG) (extension version from that tag)" + $(eval _TMP := $(shell mktemp -d)) + git -C $(REPO_ROOT) archive $(PREV_TAG) | tar -x -C $(_TMP) + printf '#define GIT_VERSION "%s"\n' '$(PREV_TAG)' \ + > $(_TMP)/src/bin/pg_autoctl/git-version.h + cp $(REPO_ROOT)/Dockerfile $(_TMP)/Dockerfile + docker build \ + --build-arg PGVERSION=$(PGVERSION) \ + --target run \ + -t pgaf:current-base \ + $(_TMP) + rm -rf $(_TMP) + docker build \ + --build-arg BASE_IMAGE=pgaf:current-base \ + --build-arg NEXT_IMAGE=pgaf:next \ + -f $(dir $(abspath $(lastword $(MAKEFILE_LIST))))Dockerfile.current \ + -t pgaf:current \ + $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +## Build pgaf:next — the current working tree (the new version being tested). +pgaf-next: + @echo "Building pgaf:next from working tree (extension version $(CURRENT_EXT))" + docker build \ + --build-arg PGVERSION=$(PGVERSION) \ + --target run \ + -t pgaf:next \ + $(REPO_ROOT) + +## Run the upgrade spec. +## Requires pgaf:current and pgaf:next to already be built. +run: + @echo "Running upgrade spec: $(SPEC)" + $(PGAFTEST) run $(SPEC) + +## Remove the two upgrade test images. clean: - cd ../.. && git apply --reverse $(PATCH) - -up: create-volumes compose-up tail ; - -down: compose-down rm-volumes ; - -compose-down: - docker compose down --volumes --remove-orphans - -compose-up: - docker compose up -d - -tail: - docker compose logs -f - -create-volumes: - for v in volm vol1 vol2 vol3; do docker volume create $$v; done - -rm-volumes: - for v in volm vol1 vol2 vol3; do docker volume rm $$v; done - -upgrade-monitor: patch - docker compose up -d --no-deps --build monitor - -upgrade-nodes: - docker compose up -d --no-deps --build node3 node2 - docker compose up -d --no-deps --build node1 - -state: - docker compose exec monitor pg_autoctl show state - -version: - docker compose exec monitor pg_autoctl version - docker compose exec monitor psql -d pg_auto_failover -c "$(Q_VERSION)" - -failover: - docker compose exec monitor pg_autoctl perform failover - -watch: - docker compose exec monitor watch -n 0.2 pg_autoctl show state - -.PHONY: build patch clean up down upgrade-monitor state watch -.PHONY: compose-down compose-up create-volumes rm-volumes + -docker rmi pgaf:current pgaf:next 2>/dev/null || true diff --git a/tests/upgrade/install-extension.sh b/tests/upgrade/install-extension.sh new file mode 100644 index 000000000..c9392d58d --- /dev/null +++ b/tests/upgrade/install-extension.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Install the pre-staged v2.2 pgautofailover extension files into the system +# extension and library paths reported by pg_config. Run as root inside the +# pgaf:current container. +set -e + +STAGING=/usr/local/bin/pgaf/2.2 +EXTDIR=$(pg_config --sharedir)/extension +LIBDIR=$(pg_config --pkglibdir) + +install -m 644 "${STAGING}/pgautofailover.control" "${EXTDIR}/" +install -m 644 "${STAGING}/pgautofailover--2.1--2.2.sql" "${EXTDIR}/" +install -m 644 "${STAGING}/pgautofailover--2.2.sql" "${EXTDIR}/" +install -m 755 "${STAGING}/pgautofailover.so" "${LIBDIR}/" diff --git a/tests/upgrade/pg_autoctl_shim.sh b/tests/upgrade/pg_autoctl_shim.sh new file mode 100644 index 000000000..a659ce2fc --- /dev/null +++ b/tests/upgrade/pg_autoctl_shim.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Static shim at /usr/local/bin/pg_autoctl — never replaced during the upgrade. +# +# Delegates to /usr/local/bin/pgaf/current/pg_autoctl, which is a symlink: +# initially → pgaf/2.1 (v2.1 binary) +# after upgrade → pgaf/2.2 (v2.2 binary) +# +# exec -a sets argv[0] to /usr/local/bin/pg_autoctl so the real binary saves +# that path as pg_autoctl_argv0. The keeper uses that path for the on-disk +# version check (pg_autoctl_argv0 version --json). After the symlink flip +# that invocation hits this shim again → delegates to v2.2 → returns "2.2". +# +# PG_AUTOCTL_DEBUG_BIN_PATH is set in ENV (Dockerfile.current) so the v2.1 +# supervisor's pg_autoctl_program is also /usr/local/bin/pg_autoctl; child +# processes after the symlink flip therefore pick up v2.2. +# +# Startup translation: +# pgaftest generates: pg_autoctl node run /etc/pgaf/node.ini (v2.2 syntax) +# v2.1 does not know "node run"; we translate per the [node] kind in the ini. + +SHIM=/usr/local/bin/pg_autoctl +CURRENT=/usr/local/bin/pgaf/current/pg_autoctl + +ini_get() { + awk -F'[[:space:]]*=[[:space:]]*' \ + -v sec="$1" -v key="$2" \ + '/^\[/{in_sec=($0 == "["sec"]")} in_sec && $1==key{print $2; exit}' "$3" +} + +if [ "$1" = "node" ] && [ "$2" = "run" ] && [ -n "$3" ]; then + ini="$3" + + kind=$(ini_get node kind "$ini") + hostname=$(ini_get node hostname "$ini") + port=$(ini_get node port "$ini") + pgdata=$(ini_get postgresql pgdata "$ini") + monitor=$(ini_get monitor pguri "$ini") + name=$(ini_get node name "$ini") + formation=$(ini_get formation name "$ini") + ssl=$(ini_get options ssl "$ini") + auth=$(ini_get options auth "$ini") + + [ -z "$pgdata" ] && { echo "pg_autoctl_shim: missing [postgresql] pgdata in $ini" >&2; exit 1; } + + case "$ssl" in + off|"") ssl_flag="--no-ssl" ;; + *) ssl_flag="--ssl-self-signed" ;; + esac + + set -- \ + ${hostname:+--hostname "$hostname"} \ + ${port:+--pgport "$port"} \ + ${auth:+--auth "$auth"} \ + "$ssl_flag" + + cfg_dir=$(printf '%s' "$pgdata" | sed 's|^/||') + cfg="/var/lib/postgres/.config/pg_autoctl/${cfg_dir}/pg_autoctl.cfg" + + if [ "$kind" = "monitor" ]; then + if [ ! -f "$cfg" ]; then + # v2.1 create monitor supports --run (stays in the foreground). + exec -a "$SHIM" "$CURRENT" create monitor --pgdata "$pgdata" "$@" --run + else + exec -a "$SHIM" "$CURRENT" run --pgdata "$pgdata" + fi + else + set -- "$@" \ + ${monitor:+--monitor "$monitor"} \ + ${name:+--name "$name"} \ + ${formation:+--formation "$formation"} + + if [ ! -f "$cfg" ]; then + # v2.1 create postgres does NOT have --run; exec the run loop after. + "$CURRENT" create postgres --pgdata "$pgdata" "$@" + rc=$?; [ $rc -ne 0 ] && exit $rc + fi + + exec -a "$SHIM" "$CURRENT" run --pgdata "$pgdata" + fi +fi + +# v2.1 supervisor spawns its children as "do service {node-active,postgres,listener}". +# v2.2 renamed the "do" subcommand tree to "internal". After the symlink flip +# ($CURRENT resolves to pgaf/2.2) the v2.1 supervisor's child-spawn calls still +# use "do service X" — translate to "internal service X" so v2.2 accepts them. +# Before the flip $CURRENT → pgaf/2.1 and the v2.1 binary already understands "do". +if [ "$1" = "do" ] && [ "$2" = "service" ]; then + resolved=$(readlink /usr/local/bin/pgaf/current 2>/dev/null) + if [ "$resolved" = "/usr/local/bin/pgaf/2.2" ]; then + exec -a "$SHIM" "$CURRENT" "internal" "${@:2}" + fi +fi + +# All other commands (version --json, run --pgdata respawn, …) +exec -a "$SHIM" "$CURRENT" "$@"