diff --git a/.gitattributes b/.gitattributes index 546a2a29f..82586159d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -25,3 +25,4 @@ src/bin/lib/libs/** -citus-style src/bin/lib/pg/** -citus-style src/bin/lib/subcommands.c/** -citus-style src/monitor/version_compat.c -citus-style +src/bin/pgaftest/** -citus-style diff --git a/.github/workflows/run-pgaftest.yml b/.github/workflows/run-pgaftest.yml new file mode 100644 index 000000000..0f0be634d --- /dev/null +++ b/.github/workflows/run-pgaftest.yml @@ -0,0 +1,259 @@ +name: pgaftest + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +# One run per branch; cancel in-progress runs on new push. +concurrency: + group: pgaftest-${{ github.ref }} + cancel-in-progress: true + +jobs: + # --------------------------------------------------------------------------- + # Build the runtime Docker image and the pgaftest binary once, then share + # both as artifacts with every parallel test job. + # --------------------------------------------------------------------------- + build: + name: Build image & pgaftest + runs-on: ubuntu-latest + env: + PGVERSION: 17 + + steps: + - uses: actions/checkout@v4.2.2 + + # git-version.h is generated from the working tree and must exist on the + # host before the Docker build COPY step reads it. + - name: Generate git-version.h + run: make version + + - name: Build pg_auto_failover run image + run: | + docker build \ + --build-arg PGVERSION=${{ env.PGVERSION }} \ + --target run \ + -t pgaf:run \ + . + + - name: Save pgaf:run image as a tarball + run: docker save pgaf:run | gzip > /tmp/pgaf-run.tar.gz + + - name: Upload pgaf:run image artifact + uses: actions/upload-artifact@v4.4.3 + with: + name: pgaf-run-image + path: /tmp/pgaf-run.tar.gz + # Only needed for the duration of this workflow run. + retention-days: 1 + + # Build the full build image just to extract the pgaftest binary. + # The binary is linux/amd64, linked against PGDG libpq — matches + # ubuntu-latest runners where the test jobs will execute it. + - name: Build pg_auto_failover build image (for pgaftest extraction) + run: | + docker build \ + --build-arg PGVERSION=${{ env.PGVERSION }} \ + --target build \ + -t pgaf:build \ + . + + - name: Extract pgaftest binary from build image + run: | + docker create --name pgaftest-extract pgaf:build + docker cp pgaftest-extract:/usr/lib/postgresql/${{ env.PGVERSION }}/bin/pgaftest \ + /tmp/pgaftest-bin + docker rm pgaftest-extract + chmod +x /tmp/pgaftest-bin + + - name: Upload pgaftest binary artifact + uses: actions/upload-artifact@v4.4.3 + with: + name: pgaftest-binary + path: /tmp/pgaftest-bin + retention-days: 1 + + # --------------------------------------------------------------------------- + # Run all non-upgrade specs from tests/tap/schedule in a matrix. + # Each job is independent; they all share the pre-built image and binary. + # --------------------------------------------------------------------------- + test: + name: pgaftest / ${{ matrix.spec }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + spec: + - basic_operation + - basic_operation_listen_flag + - maintenance_and_drop + - create_standby_with_pgdata + - ensure + - monitor_disabled + - replace_monitor + - config_get_set + - skip_pg_hba + - auth + - enable_ssl + - ssl_cert + - ssl_self_signed + - multi_standbys + - multi_async + - multi_ifdown + - multi_maintenance + - multi_alternate + - extension_update + + steps: + - uses: actions/checkout@v4.2.2 + + - name: Download pgaf:run image + uses: actions/download-artifact@v4.1.8 + with: + name: pgaf-run-image + path: /tmp + + - name: Load pgaf:run image into Docker + run: docker load < /tmp/pgaf-run.tar.gz + + - name: Download pgaftest binary + uses: actions/download-artifact@v4.1.8 + with: + name: pgaftest-binary + path: /tmp + + # pgaftest links against PGDG libpq — install the matching runtime lib. + - name: Install libpq runtime + run: | + sudo apt-get install -y curl ca-certificates gnupg + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | sudo apt-key add - + echo "deb http://apt.postgresql.org/pub/repos/apt \ + $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + sudo apt-get install -y libpq5 + + - name: Run ${{ matrix.spec }} + timeout-minutes: 18 + env: + PGAF_IMAGE: pgaf:run + run: | + chmod +x /tmp/pgaftest-bin + /tmp/pgaftest-bin run tests/tap/specs/${{ matrix.spec }}.pgaf + + # --------------------------------------------------------------------------- + # installcheck: builds pgaf:testrun (run + source tree + postgresql-server-dev) + # and runs the SQL regression test suite via pg_regress. Kept separate from + # the main matrix because it requires a larger image built inline. + # --------------------------------------------------------------------------- + installcheck: + name: pgaftest / installcheck + needs: build + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + PGVERSION: 17 + + steps: + - uses: actions/checkout@v4.2.2 + + - name: Install libpq runtime + run: | + sudo apt-get install -y curl ca-certificates gnupg + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | sudo apt-key add - + echo "deb http://apt.postgresql.org/pub/repos/apt \ + $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + sudo apt-get install -y libpq5 + + - name: Download pgaftest binary + uses: actions/download-artifact@v4.1.8 + with: + name: pgaftest-binary + path: /tmp + + - name: Generate git-version.h + run: make version + + - name: Build pgaf:testrun image + run: | + docker build \ + --build-arg PGVERSION=${{ env.PGVERSION }} \ + --target testrun \ + -t pgaf:testrun \ + . + + - name: Run installcheck spec + timeout-minutes: 15 + run: | + chmod +x /tmp/pgaftest-bin + /tmp/pgaftest-bin run tests/tap/specs/installcheck.pgaf + + # --------------------------------------------------------------------------- + # Upgrade test: builds both pgaf:next (current branch) and pgaf:current + # (the previous release tag, auto-detected), then runs the upgrade spec. + # Kept separate because it needs a PGVERSION=16 image and additional + # build steps (pgaf:current-base from git archive of PREV_TAG). + # --------------------------------------------------------------------------- + upgrade: + name: pgaftest / upgrade + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + PGVERSION: 16 + + steps: + - uses: actions/checkout@v4.2.2 + with: + # Full history needed so PREV_TAG auto-detection (git tag --sort) + # finds the correct previous release tag. + fetch-depth: 0 + + - name: Install libpq runtime + run: | + sudo apt-get install -y curl ca-certificates gnupg + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | sudo apt-key add - + echo "deb http://apt.postgresql.org/pub/repos/apt \ + $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + sudo apt-get install -y libpq5 + + - name: Download pgaftest binary + uses: actions/download-artifact@v4.1.8 + with: + name: pgaftest-binary + path: /tmp + + - name: Generate git-version.h + run: make version + + # The build job already produced pgaf:run at PG16 for the regular tests, + # but the upgrade job runs independently and needs its own images. + # Build pgaf:next first so pgaf:current's Dockerfile.current can COPY + # the v2.2 binary and extension files from it. + - name: Build pgaf:next (current branch, PGVERSION=16) + run: | + make -C tests/upgrade pgaf-next PGVERSION=${{ env.PGVERSION }} + + - name: Build pgaf:current (previous release + both binaries baked in) + run: | + make -C tests/upgrade pgaf-current PGVERSION=${{ env.PGVERSION }} + + - name: Run upgrade spec + timeout-minutes: 25 + run: | + chmod +x /tmp/pgaftest-bin + /tmp/pgaftest-bin run tests/tap/specs/upgrade.pgaf diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c1b626782..8f477a00f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -3,14 +3,30 @@ name: Run Tests on: push: branches: - - main + - main pull_request: branches: - - main - + - main workflow_dispatch: jobs: + style_checker: + name: Style check + runs-on: ubuntu-latest + container: citus/stylechecker:no-py + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.2 + + - name: Set safe directory for git + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Check C formatting + run: citus_indent --check + + - name: Check banned functions + run: ci/banned.h.sh + run_tests: name: Run test runs-on: ubuntu-latest @@ -33,46 +49,19 @@ jobs: include: - PGVERSION: 14 TEST: tablespaces - - PGVERSION: 14 - TEST: linting steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4.2.2 - name: Set environment variables run: | - echo "PGVERSION=${{ matrix.PGVERSION }}" >> $GITHUB_ENV - echo "TEST=${{ matrix.TEST }}" >> $GITHUB_ENV - echo "LINTING=${{ matrix.LINTING }}" >> $GITHUB_ENV - echo "TRAVIS_BUILD_DIR=$(pwd)" >> $GITHUB_ENV - - - name: Clone and install linting tools - if: ${{ env.TEST == 'linting' }} - run: | - sudo apt-get install python3-pip - pip3 install --user black - black --version - gcc --version - # Install uncrustify the Citus way - make -C ci -f tools.mk tools - - - name: Check code formatting and banned function - if: ${{ env.TEST == 'linting' }} - run: | - make lint - - - name: Build documentation - if: ${{ env.TEST == 'linting' }} - run: | - make build-docs + echo "PGVERSION=${{ matrix.PGVERSION }}" >> $GITHUB_ENV + echo "TEST=${{ matrix.TEST }}" >> $GITHUB_ENV + echo "TRAVIS_BUILD_DIR=$(pwd)" >> $GITHUB_ENV - name: Build Docker Test Image - if: ${{ env.TEST != 'linting' }} - run: | - make build-test-image + run: make build-test-image - name: Run Test - if: ${{ env.TEST != 'linting' }} timeout-minutes: 15 - run: | - make ci-test + run: make ci-test diff --git a/.gitignore b/.gitignore index 3d0a5f4fa..94b523b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ docs/tikz/*.png # Exclude our demo/test tmux directory tmux/ valgrind/ +*.tmp +src/bin/pgaftest/test_spec_parse.tab.* +src/bin/pgaftest/test_spec_parse.output diff --git a/Dockerfile b/Dockerfile index 1df68994d..091f8134f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ RUN apt-get update \ postgresql-${PGVERSION} \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install pyroute2>=0.5.17 +RUN pip3 install 'pyroute2>=0.5.17,<0.7.0' RUN adduser --disabled-password --gecos '' docker RUN adduser docker sudo @@ -112,6 +112,16 @@ COPY Makefile.azure ./ COPY Makefile.citus ./ COPY ./src/ ./src COPY ./src/bin/pg_autoctl/git-version.h ./src/bin/pg_autoctl/git-version.h +# Touch bison/flex generated files so they appear newer than the grammar +# sources, preventing make from re-running bison (system bison version may +# differ from the one used to pre-generate the committed .c/.h files). +# Guard with -d: older releases (e.g. v2.1 built via git-archive for upgrade +# tests) do not have src/bin/pgaftest/ at all. +RUN if [ -d src/bin/pgaftest ]; then \ + touch src/bin/pgaftest/test_spec_parse.c \ + src/bin/pgaftest/test_spec_parse.h \ + src/bin/pgaftest/test_spec_scan.c; \ + fi RUN make -s clean && make -s install -j8 @@ -196,4 +206,108 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/p ENV PG_AUTOCTL_DEBUG=1 ENV PGDATA=/var/lib/postgres/pgaf -CMD ["pg_autoctl", "do tmux session --nodes 3 --binpath /usr/local/bin/pg_autoctl"] +# +# debian image — like run, but with a Debian-style "main" cluster pre-created +# via pg_createcluster so that pg_autoctl can test adoption of the split-config +# layout (postgresql.conf lives in /etc/postgresql/${PGVERSION}/main/ outside +# PGDATA). pg_autoctl node run detects the missing postgresql.conf in PGDATA, +# moves the conf files in, and proceeds normally — no entrypoint changes needed. +# +FROM run AS debian + +ARG PGVERSION + +USER root +RUN pg_createcluster \ + --user docker --group postgres \ + ${PGVERSION} main \ + -- --auth-local trust --auth-host trust \ + && chown docker /var/lib/postgresql/${PGVERSION} + +USER docker +ENV PGDATA=/var/lib/postgresql/${PGVERSION}/main + +# +# testrun image — like run, but adds postgresql-server-dev and the full source +# tree so that "make installcheck" works inside the monitor container. +# Used by installcheck.pgaf via "monitor image-target testrun". +# +FROM run AS testrun + +ARG PGVERSION + +USER root +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + make \ + postgresql-server-dev-${PGVERSION} \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=docker ./src/ /usr/src/pg_auto_failover/src/ + +USER docker + +# +# pgaftest image — standalone test-runner image. +# +# Kept entirely separate from the pg_auto_failover run image so that no +# test tooling leaks into the images used for monitor and data nodes. +# +# Runtime dependencies come from two sources: +# - Docker's own apt repo → docker-ce-cli + docker-compose-plugin (v2) +# - PGDG apt repo → libpq5 matching the build's PGVERSION +# +# The pg_auto_failover binaries are copied directly from the run stage; +# nothing is built here. +# +FROM debian:bullseye-slim AS pgaftest + +ARG PGVERSION + +# Minimal runtime libs (libssl, libcurl, libzstd) plus PGDG for libpq +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + libcurl4-gnutls-dev \ + libncurses6 \ + libzstd-dev \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +# PGDG repo — needed for the libpq version that pg_autoctl and pgaftest link against +RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN echo "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main ${PGVERSION}" \ + > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libpq5 \ + && rm -rf /var/lib/apt/lists/* + +# Docker's apt repo — provides docker-ce-cli and the compose v2 plugin +RUN curl -fsSL https://download.docker.com/linux/debian/gpg \ + | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] \ + https://download.docker.com/linux/debian bullseye stable" \ + > /etc/apt/sources.list.d/docker.list \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + docker-ce-cli \ + docker-compose-plugin \ + && rm -rf /var/lib/apt/lists/* + +RUN adduser --disabled-password --gecos '' --home /var/lib/postgres docker \ + && adduser docker sudo \ + && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ + && mkdir -p /var/lib/postgres \ + && chown docker /var/lib/postgres + +# Binaries from the pg_auto_failover run image +COPY --from=run /usr/local/bin/pg_autoctl /usr/local/bin/ + +# pgaftest binary from the build stage +COPY --from=build /usr/lib/postgresql/${PGVERSION}/bin/pgaftest /usr/local/bin/ + +USER docker +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/Makefile b/Makefile index 711a5ca54..77e9379ae 100644 --- a/Makefile +++ b/Makefile @@ -150,17 +150,11 @@ $(VERSION_FILE): # # make ci-test; is run on the GitHub Action workflow # -# In that environment we have a local git checkout of the code, and docker -# is available too. We run our tests in docker, except for the code linting -# parts which requires full access to the git repository, so linter tooling -# is installed directly on the CI vm. # .PHONY: ci-test ci-test: ifeq ($(TEST),tablespaces) $(MAKE) -C tests/tablespaces run-test -else ifeq ($(TEST),linting) - $(MAKE) spellcheck else $(MAKE) run-test endif @@ -172,8 +166,6 @@ endif test: ifeq ($(TEST),tablespaces) $(MAKE) -C tests/tablespaces run-test -else ifeq ($(TEST),linting) - $(MAKE) spellcheck else sudo -E env "PATH=${PATH}" USER=$(shell whoami) \ $(NOSETESTS) \ @@ -187,12 +179,33 @@ endif # # INDENT/LINT/SPELLCHECK # +# citus_indent is run via its official Docker image (citus/stylechecker:no-py) +# so that the local version exactly matches CI. When the local citus_indent +# binary is already in PATH (e.g. inside CI after tools.mk installs it) the +# plain binary is used instead, which is equally authoritative there. +# +# To use Docker locally: +# make docker-indent # auto-fix formatting via Docker +# make docker-check # check-only via Docker +CITUS_INDENT_DOCKER = docker run --rm \ + -v "$(CURDIR):/workdir" \ + -w /workdir \ + citus/stylechecker:no-py \ + citus_indent + # make indent; edits the code when necessary .PHONY: indent indent: citus_indent black --exclude=ci/tools . +# make docker-indent / make docker-check — use Docker image locally +.PHONY: docker-indent docker-check +docker-indent: + $(CITUS_INDENT_DOCKER) +docker-check: + $(CITUS_INDENT_DOCKER) --check + # make lint; is an alias for make spellcheck # make linting; is an alias for make spellcheck .PHONY: lint linting @@ -202,7 +215,7 @@ lint linting: spellcheck ; # reports compliance with the rules. .PHONY: spellcheck spellcheck: - citus_indent --check + $(CITUS_INDENT_DOCKER) --check black --exclude=ci/tools --check . ci/banned.h.sh diff --git a/docs/index.rst b/docs/index.rst index 4e8d8481e..32b6a643c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,6 +63,12 @@ __ https://github.com/hapostgres/pg_auto_failover citus citus-quickstart +.. toctree:: + :hidden: + :caption: Testing + + pgaftest + .. toctree:: :hidden: :caption: Manual Pages diff --git a/docs/pgaftest.rst b/docs/pgaftest.rst new file mode 100644 index 000000000..9d0b7f519 --- /dev/null +++ b/docs/pgaftest.rst @@ -0,0 +1,1071 @@ +.. _pgaftest: + +pgaftest — Integration Test Framework +====================================== + +``pgaftest`` is the integration-test binary for pg_auto_failover. It reads +``.pgaf`` specification files that describe a cluster topology and a sequence +of named test steps, then drives a Docker Compose stack to verify the cluster +behaves correctly. + +The same specification file works in two modes: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Command + - Effect + * - ``pgaftest run spec.pgaf`` + - Headless CI: start → setup → sequence → teardown → TAP output + exit code + * - ``pgaftest setup spec.pgaf`` + - Interactive: start cluster, run setup, then drop into a shell or tmux session + +Quick start +----------- + +.. code-block:: bash + + # Build the pg_auto_failover Docker image once + docker build -t pg_auto_failover:pg17 . + + # Run a spec file headlessly + PGAF_IMAGE=pg_auto_failover:pg17 pgaftest run tests/tap/specs/basic_operation.pgaf + + # Start a cluster interactively + PGAF_IMAGE=pg_auto_failover:pg17 pgaftest setup tests/tap/specs/basic_operation.pgaf + + # Run one named step against the live cluster + # (work dir is auto-derived: $TMPDIR/pgaftest/basic_operation) + pgaftest step fail_primary tests/tap/specs/basic_operation.pgaf + + # Tear down + pgaftest down tests/tap/specs/basic_operation.pgaf + +.. _pgaftest_spec: + +Spec file format +---------------- + +A ``.pgaf`` file contains four kinds of top-level blocks: + +1. ``cluster { }`` — topology declaration +2. ``setup { }`` — commands to run after ``docker compose up`` +3. ``teardown { }`` — commands to run at the end of a run (or on ``pgaftest down``) +4. ``step { }`` — named test step +5. ``sequence`` — ordered list of step names for CI mode + +cluster { } +~~~~~~~~~~~ + +The ``cluster`` block describes the Docker Compose topology. ``pgaftest`` +generates: + +* A ``docker-compose.yml`` with one service per node +* A ``pg_autoctl_node.ini`` file per node (see :ref:`node_spec_format`) +* Each container mounts its ini file at ``/etc/pgaf/node.ini`` and runs the + same command: ``pg_autoctl node run /etc/pgaf/node.ini`` + +The topology follows a three-level hierarchy: +``cluster`` → ``monitor`` + ``formation(s)`` → ``nodes`` + +**Syntax** + +.. code-block:: text + + cluster { + # Cluster-level options (all optional) + monitor [port N] # expose monitor on host port; 0 = auto + image "pg_auto_failover:pg17" # Docker image; "" = build from source + ssl self-signed # self-signed | cert | off + auth trust # trust | md5 | scram + + # One or more formation blocks + formation [name] [num-sync N] { + nodename [kind] [option...] + ... + } + } + +**Cluster-level options** + +``monitor [port N]`` + Declares the monitor service. The optional ``port N`` exposes the + monitor's Postgres port on host port ``N`` so the test runner can connect + directly with libpq. When omitted, a random free port is chosen. + +``image`` + Docker image to use for every container. Overrides the ``PGAF_IMAGE`` + environment variable. When neither is set, a ``build:`` stanza pointing at + the project root is used instead. + +``ssl`` + SSL mode written into the ``[options]`` section of each node's ini file. + Allowed values: ``self-signed`` (default), ``cert``, ``off``. + +``auth`` + Authentication method. Passed as ``--auth`` to ``pg_autoctl create``. + Default: ``trust``. + +**Formation block** + +.. code-block:: text + + formation [name] [num-sync N] { + ...nodes... + } + +``name`` + Formation name on the monitor. Defaults to ``default`` when omitted. + +``num-sync N`` + Sets ``number-sync-standbys`` on this formation after the cluster comes up. + When absent, the monitor default (0) applies. + +**Node lines** (inside a formation block) + +.. code-block:: text + + nodename [kind] [option...] + +``nodename`` + The service name — used as the Docker Compose service name and as the + ``hostname`` in the container. + +``kind`` (optional, default ``postgres``) + * ``postgres`` — standalone Postgres node + * ``coordinator`` — Citus coordinator + * ``worker`` — Citus worker + +Options: + +* ``async`` — makes this node asynchronous (``replication_quorum = false``) +* ``candidate-priority N`` — integer 0–100; default 50. ``0`` prevents this + node from ever being elected primary. Can also be written ``candidate-priority=N``. +* ``replication-quorum false`` — same as ``async``; can also be + written ``replication-quorum=false``. +* ``group N`` — Citus group id (required for ``worker`` kind). + Can also be written ``group=N``. +* ``no-monitor`` — standalone node: does not register with a monitor. +* ``listen`` — bind Postgres to all interfaces (``0.0.0.0``). +* ``citus-secondary`` — marks this node as a Citus secondary. +* ``citus-cluster-name NAME`` — sets the Citus cluster name. +* ``port N`` — override the default Postgres port (5432). +* ``debian-cluster NAME`` — Debian-style Postgres cluster name. The node + will use the Debian ``/var/lib/postgresql//`` layout that + results from ``pg_createcluster``. Requires the ``debian`` Dockerfile + build target; see ``monitor image-target`` below. +* ``ssl MODE`` — per-node SSL mode override (overrides the cluster-level ``ssl``). +* ``auth-method METHOD`` — per-node auth method override. +* ``launch deferred`` — do not start this node automatically at ``compose up`` + time. Use ``compose start `` inside a step to start it later. This + is how tests that add nodes mid-scenario are written. +* ``volume `` — mount a Docker named volume into the + container at ````. The volume is created automatically. + Multiple ``volume`` lines are allowed. + +**Multi-option nodes — block syntax** + +When a node needs several options the block form keeps the spec readable: + +.. code-block:: text + + node coordinator1b { + coordinator + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + + node worker1b { + worker group 1 + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + + node node1 { + volume extra_a "/extra_volumes/extra_a" + volume extra_b "/extra_volumes/extra_b" + } + +The block form requires the ``node`` keyword before the name. Flat (single- +line) form does not use the keyword. ``pgaftest indent`` automatically +promotes a node to block form whenever the flat line would exceed 72 characters +or the node has any ``volume`` entries. + +**Monitor options** + +.. code-block:: text + + cluster { + monitor image-target testrun + monitor debian-cluster main + ... + } + +``monitor image-target `` + Use the specified Dockerfile build stage for the monitor container instead + of the default ``run`` stage. Useful when the monitor needs extra tools + (e.g. ``testrun`` for ``make installcheck``, ``test`` for the build + environment). + +``monitor debian-cluster `` + Mark the monitor container as Debian-style: use the ``debian`` Dockerfile + target and set ``PGDATA`` to ``/var/lib/postgresql//``. + +``monitor initially stopped`` + Declare a second monitor service named ````. The container is + created and initialized at ``compose up`` time (so its data volume is + ready), but ``pgaftest`` immediately stops it. Use this for + monitor-replacement scenarios where the second monitor must exist on disk + but must not be reachable until the test deliberately starts it with + ``compose start ``. + + .. code-block:: text + + cluster { + monitor + monitor monitor2 initially stopped + formation { + node1 + node2 + } + } + +**Examples** + +Minimal two-node HA cluster: + +.. code-block:: text + + cluster { + monitor + formation { + node1 + node2 + } + } + +Three-node cluster, ``node3`` is a non-candidate async standby: + +.. code-block:: text + + cluster { + monitor + formation { + node1 + node2 + node3 async candidate-priority 0 + } + } + +Named formation with ``num-sync``: + +.. code-block:: text + + cluster { + monitor + formation default num-sync 1 { + node1 + node2 + node3 + } + } + +Citus cluster (one coordinator, two worker HA pairs in separate formations): + +.. code-block:: text + + cluster { + image "pg_auto_failover:pg17-citus" + monitor + formation coordinators { + coord coordinator + } + formation workers num-sync 1 { + worker1 worker group 1 + worker2 worker group 1 + worker3 worker group 2 + worker4 worker group 2 + } + } + +Custom image and non-default auth: + +.. code-block:: text + + cluster { + image "myregistry.example.com/pgaf:latest" + ssl self-signed + auth scram-sha-256 + monitor port 15432 + formation { + node1 + node2 + } + } + +setup { } and teardown { } +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These blocks contain the same DSL commands as named steps (see below). +``setup`` runs after ``docker compose up -d``, before the first step. +``teardown`` runs at the end of ``pgaftest run`` (always, even on failure) +and at the end of ``pgaftest down`` in interactive mode. + +A typical setup block waits for all nodes to reach their initial states: + +.. code-block:: text + + setup { + wait until node1 state = primary timeout 120s + wait until node2 state = secondary timeout 120s + } + +step { } +~~~~~~~~~~~~~~~ + +Named blocks of commands that can be run individually (``pgaftest step``) or +collectively via the ``sequence`` directive. + +.. code-block:: text + + step my_step { + + + ... + } + +sequence +~~~~~~~~ + +Declares the ordered list of steps to run in CI mode (``pgaftest run``). +Each step name appears on its own line or space-separated: + +.. code-block:: text + + sequence + create_table + fail_primary + check_failover + restart_old_primary + +Step commands (DSL reference) +------------------------------ + +All commands are available inside ``setup``, ``teardown``, and ``step`` blocks. + +``exec `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run a command inside a running container via ``docker compose exec``. +Fails the step if the command exits with a non-zero status. + +.. code-block:: text + + exec node1 pg_autoctl perform failover + exec monitor psql -c "SELECT count(*) FROM pgautofailover.node" + +``exec-fails `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like ``exec`` but asserts the command **fails** (exits non-zero). +Fails the step if the command unexpectedly succeeds. Use this to verify +that a command is correctly rejected by pg_autoctl. + +.. code-block:: text + + # enable maintenance on a primary without --allow-failover must fail + exec-fails node1 pg_autoctl enable maintenance + + # failover with a single node (no standby) must fail + exec-fails monitor pg_autoctl perform failover + +``wait until state is [timeout Ns]`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Poll until ```` reaches ````, then continue. The timeout +defaults to 90 seconds. Fails the current step if the timeout expires. +Both ``is`` and ``=`` are accepted as the state operator. + +.. code-block:: text + + wait until node1 state is primary timeout 120s + wait until node2 state is secondary timeout 60s + wait until node2 state is demote_timeout timeout 90s + +**Multi-node form** — wait for several nodes simultaneously using ``and``: + +.. code-block:: text + + wait until node1 state is primary + and node2 state is secondary + timeout 90s + + wait until coordinator1a state is primary + and coordinator1b state is secondary + and worker1a state is primary + timeout 120s + +Each ``and state is `` condition must be satisfied at the same +time for the wait to succeed. + +**Pass-through states** — assert that a node visits intermediate states on +the way to the target: + +.. code-block:: text + + wait until node2 state is primary + passing through stop_replication, draining + timeout 120s + +The runner observes monitor LISTEN/NOTIFY notifications; a pass-through state +is satisfied when a notification for that state is received before the target +state notification arrives. + +**State names** — all pg_auto_failover FSM states are first-class tokens in +the spec DSL. Both ``_`` and ``-`` spellings are accepted: + +.. list-table:: + :header-rows: 1 + :widths: 30 30 40 + + * - Token + - Aliases + - Description + * - ``init`` + - + - Node has just been created + * - ``single`` + - + - Single-node cluster (no standby yet) + * - ``primary`` + - + - Running as primary + * - ``wait_primary`` + - ``wait-primary`` + - Waiting for a standby to connect + * - ``wait_standby`` + - ``wait-standby`` + - Primary waiting for first standby + * - ``secondary`` + - + - Running as synchronous standby + * - ``catchingup`` + - + - Standby catching up after promotion or restart + * - ``draining`` + - + - Primary draining before demotion + * - ``demote_timeout`` + - ``demote-timeout`` + - Former primary waiting for demotion timer + * - ``demoted`` + - + - Former primary; postgres stopped + * - ``maintenance`` + - + - Node paused for maintenance + * - ``prepare_maintenance`` + - ``prepare-maintenance`` + - Transition into maintenance + * - ``wait_maintenance`` + - ``wait-maintenance`` + - Standby waiting while primary enters maintenance + * - ``join_primary`` + - ``join-primary`` + - New node joining as primary + * - ``apply_settings`` + - ``apply-settings`` + - Applying configuration changes + * - ``report_lsn`` + - ``report-lsn`` + - Standby reporting its LSN for promotion selection + * - ``fast_forward`` + - ``fast-forward`` + - Standby fast-forwarding to primary's LSN + * - ``join_secondary`` + - ``join-secondary`` + - New node joining as secondary + * - ``prepare_promotion`` + - ``prepare-promotion`` + - Standby preparing to be promoted + * - ``stop_replication`` + - ``stop-replication`` + - Standby stopping replication before promotion + * - ``dropped`` + - + - Node has been dropped from the formation + +**Dual-source polling** — the runner queries two sources in each polling +round, succeeding as soon as either returns a match: + +1. **Monitor** (``pg_autoctl inspect monitor node-state``): checks the + ``reportedstate`` column in ``pgautofailover.node``. This is the primary + source; it captures fast FSM transitions correctly because the monitor's + view lags by exactly one keeper heartbeat. + +2. **Node-local FSM** (``pg_autoctl inspect fsm node-state``): checks the + keeper's on-disk ``current_role`` in the state file inside the container. + This source is consulted **only** when the monitor reports ``health < 0`` + (node is unhealthy / unreachable) for the target node. A network- + partitioned primary self-assigns ``demote_timeout`` locally after + ``network_partition_timeout`` seconds (default 20 s) even though it cannot + report that state back to the monitor. Gating the node-local check on the + monitor's health value avoids false positives from stale on-disk state + while correctly observing partition-driven transitions. + +.. note:: + + The dual-source strategy lets tests write ``wait until node2 state = + demote_timeout`` after a ``network disconnect node2`` without any special + syntax. The runner detects the partition automatically. + +``wait until , , ... [in group N] [timeout Ns]`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Wait until the formation as a whole has at least one node in each of the +listed states simultaneously. Useful for asserting cluster-wide convergence +after a failover. + +.. code-block:: text + + # Wait for primary + secondary simultaneously + wait until primary, secondary timeout 120s + + # Wait for specific group + wait until primary, secondary in group 1 timeout 90s + +``assert state = `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Immediately check that ```` is in ```` on the monitor. Fails +the step if the state does not match. No polling. + +.. code-block:: text + + assert node2 state = primary + assert node1 assigned-state = secondary + +``assert assigned-state = `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like ``assert state`` but checks the ``assigned_state`` column on the monitor +rather than ``reportedstate``. + +``sql { SQL }`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run SQL inside ````'s container and capture the output. Use a +multi-line ``{ ... }`` block for multi-statement SQL. + +.. code-block:: text + + sql node1 { + CREATE TABLE t1 (a int); + INSERT INTO t1 VALUES (1), (2), (3); + } + sql node2 { SELECT count(*) FROM t1; } + +``expect { text }`` +~~~~~~~~~~~~~~~~~~~~ + +Compare the output of the most recent ``sql`` command against ``text``. +Fails the step when the output does not match. Also fails if the +preceding ``sql`` raised a SQL error (use ``expect error`` for that). + +Single-value form: + +.. code-block:: text + + sql node2 { SELECT count(*) FROM t1; } + expect { 3 } + +Multi-line form (matches ``psql --tuples-only --no-align`` output, one row per line): + +.. code-block:: text + + sql node1 { SELECT a, b FROM t ORDER BY a; } + expect { + 1 hello + 2 world + } + +Inline tuple form — each ``{ value }`` group corresponds to one output row. +This is equivalent to the multi-line form above. + +.. code-block:: text + + sql node1 { SELECT count(*) FROM t GROUP BY status; } + expect { { 2 } { 5 } } + +``expect error [SQLSTATE]`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Assert that the most recent ``sql`` command raised a SQL error. +Fails the step if the query succeeded. The optional ``SQLSTATE`` argument +(a 5-character PostgreSQL error code) constrains the check to a specific +error class. + +.. code-block:: text + + # writes to a hot-standby must be rejected (25006 = read_only_sql_transaction) + sql node2 { INSERT INTO t1 VALUES (42); } + expect error 25006 + + # any SQL error is acceptable (no SQLSTATE constraint) + sql node1 { SELECT 1/0; } + expect error + +PostgreSQL SQLSTATE codes of interest: + +* ``25006`` — ``read_only_sql_transaction`` (writes to a standby) +* ``42P01`` — ``undefined_table`` +* ``23505`` — ``unique_violation`` + +The SQLSTATE is extracted from psql's verbose error output +(``VERBOSITY=verbose``). ``ON_ERROR_STOP=1`` is always set so psql +exits non-zero on SQL errors. + +``network disconnect `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Disconnect ````'s container from the Docker network, simulating a +network partition. + +.. code-block:: text + + network disconnect node1 + +``network connect `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reconnect a previously disconnected container. + +.. code-block:: text + + network connect node1 + +``sleep Ns`` +~~~~~~~~~~~~~ + +Wait ``N`` seconds (for example ``sleep 5s`` or ``sleep 30s``). + +``compose down`` +~~~~~~~~~~~~~~~~ + +Run ``docker compose down --volumes``. Used inside ``teardown { }`` to clean +up after a test run. + +.. code-block:: text + + teardown { + compose down + } + +``compose start `` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Start a stopped container. Equivalent to ``docker compose start ``. +Pairs with ``compose stop`` and ``launch deferred`` node declarations. + +.. code-block:: text + + compose start node2 + +``compose stop `` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Stop a container cleanly (SIGTERM + grace period). Equivalent to +``docker compose stop ``. Use this to simulate a process crash or +clean node shutdown. + +.. code-block:: text + + compose stop node1 + wait until node2 state is wait_primary timeout 90s + +``compose kill `` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Kill a container immediately with SIGKILL. Equivalent to +``docker compose kill ``. Use when a clean shutdown would allow +the node to complete its shutdown sequence — which some tests intentionally +want to avoid. + +.. code-block:: text + + compose kill node1 + +``set monitor `` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Switch the runner's active monitor to ````. After this command: + +* LISTEN/NOTIFY reconnects to ```` — subsequent ``wait until`` and + formation-state checks use the new monitor. +* ``monitor_get_node_state`` queries ```` for implicit post-exec + state verification. + +Use this in monitor-replacement tests immediately after starting the new +monitor container, so that all subsequent runner operations target it instead +of the original (stopped) monitor. + +.. code-block:: text + + step switch_to_new_monitor { + compose start monitor2 + sleep 5s + exec monitor2 pg_autoctl inspect pgsetup wait + set monitor monitor2 + } + + step reconnect_node1 { + exec node1 pg_autoctl enable monitor postgresql://autoctl_node@monitor2/pg_auto_failover + wait until node1 state is single timeout 60s + } + +``logs [not] contains ""`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Assert that the container log output of ```` contains (or does not +contain) the literal string ````. The runner runs +``docker compose logs --no-color `` and pipes through ``grep -qF``. + +.. code-block:: text + + logs node2 contains "password=****" + logs node2 not contains "plaintext_secret" + +``logs [not] matches ""`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like ``logs … contains`` but uses PCRE via ``grep -qP``. Use this for +regular-expression patterns, including negative lookahead assertions. + +.. code-block:: text + + logs node2 not matches "^(?!primary_conninfo|Failed to find).*streaming_password" + +The example asserts that no log line contains ``streaming_password`` unless +the line also starts with ``primary_conninfo`` or ``Failed to find`` — the +two expected contexts where the password legitimately appears. + +``stop postgres `` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Stop the Postgres server inside ````'s container without stopping the +``pg_autoctl`` supervisor. Equivalent to calling +``pg_autoctl manual service pgctl off`` inside the container. The supervisor detects +the outage and reacts according to the FSM. + +.. code-block:: text + + exec node1 pg_autoctl enable maintenance + wait until node1 state is maintenance timeout 60s + stop postgres node1 + +``start postgres `` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Restart the Postgres server inside ````'s container via the +``pg_autoctl`` supervisor (``pg_autoctl manual pgctl on``). + +.. code-block:: text + + start postgres node1 + +``promote [, , ...]`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Assign ``pg_autoctl perform promotion`` to the listed nodes, one per group. +Useful after setup to designate which node becomes primary in each group. + +.. code-block:: text + + promote node1 + promote coordinator1a, worker1a, worker2a + +``assert stays while { }`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Assert that ```` remains in ```` throughout the execution of all +commands in the ``while { }`` body. The state is checked before and after +each inner command; any deviation fails the step. + +.. code-block:: text + + assert node1 stays secondary while { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s + exec node2 pg_autoctl disable maintenance + wait until node2 state is secondary timeout 60s + } + +``%CIDR%`` macro +~~~~~~~~~~~~~~~~~ + +The literal token ``%CIDR%`` is expanded to the Docker network CIDR (e.g. +``172.31.0.0/16``) in any ``exec`` or ``exec-fails`` command argument. The +expanded value is logged on a separate line. Use it to scope HBA rules to +the test network instead of ``0.0.0.0/0``: + +.. code-block:: text + + exec monitor bash -c "echo 'host all all %CIDR% trust' >> /var/lib/postgres/pgaf/pg_hba.conf" + +TAP output +---------- + +In CI mode (``pgaftest run``) the runner emits +`TAP `_ (Test Anything Protocol) output to stdout. +One TAP line is emitted per step: + +.. code-block:: text + + 1..5 + ok 1 - create_table + ok 2 - fail_primary + not ok 3 - check_failover + # FAIL: wait until node2 state = primary: timed out after 120s (current: demoted) + ok 4 - restart_old_primary + ok 5 - verify_replication + +Each failing step logs a ``# FAIL: …`` diagnostic line with the exact command +that failed. Standard TAP consumers (``prove``, GitHub Actions, Jenkins) can +parse this output directly. + +The exit code is ``0`` when all steps passed and non-zero otherwise. + +Complete example — basic failover test +--------------------------------------- + +The following is a complete spec file that creates a two-node cluster, +verifies replication, triggers a failover, and checks recovery. + +.. code-block:: text + + # tests/tap/specs/basic_operation.pgaf + # + # Two-node HA: create primary + secondary, test failover and recovery. + + cluster { + monitor + formation { + node1 + node2 + } + } + + setup { + wait until node1 state = primary timeout 120s + wait until node2 state = secondary timeout 120s + } + + teardown { + compose down + } + + # --------------------------------------------------------------- + # Verify that data written on the primary appears on the standby. + # --------------------------------------------------------------- + + step create_table { + sql node1 { + CREATE TABLE t1(a int); + INSERT INTO t1 VALUES (1), (2), (3); + } + sql node2 { SELECT count(*) FROM t1; } + expect { 3 } + } + + # --------------------------------------------------------------- + # Trigger a failover by stopping the primary's pg_autoctl process. + # --------------------------------------------------------------- + + step fail_primary { + exec node1 pg_autoctl stop + } + + step check_failover { + wait until node2 state = wait_primary timeout 120s + wait until node2 state = primary timeout 120s + assert node2 state = primary + } + + step restart_old_primary { + exec node1 pg_autoctl node run /etc/pgaf/node.ini + wait until node1 state = secondary timeout 120s + } + + step verify_replication { + sql node2 { + INSERT INTO t1 VALUES (4), (5); + } + sql node1 { SELECT count(*) FROM t1; } + expect { 5 } + } + + sequence + create_table + fail_primary + check_failover + restart_old_primary + verify_replication + +Running the test suite +---------------------- + +A ``schedule`` file lists the spec files to run, one per line: + +.. code-block:: text + + # tests/tap/schedule + tests/tap/specs/basic_operation.pgaf + tests/tap/specs/multi_standbys.pgaf + +Run all specs in the schedule: + +.. code-block:: bash + + PGAF_IMAGE=pg_auto_failover:pg17 \ + pgaftest run --schedule tests/tap/schedule + +Run a single spec file: + +.. code-block:: bash + + PGAF_IMAGE=pg_auto_failover:pg17 \ + pgaftest run tests/tap/specs/basic_operation.pgaf + +Interactive session (useful for debugging): + +.. code-block:: bash + + # Start the cluster and drop into a shell + PGAF_IMAGE=pg_auto_failover:pg17 \ + pgaftest setup tests/tap/specs/basic_operation.pgaf + + # In a separate terminal, run individual steps + # (work dir auto-derived to $TMPDIR/pgaftest/basic_operation) + pgaftest step create_table tests/tap/specs/basic_operation.pgaf + pgaftest step fail_primary tests/tap/specs/basic_operation.pgaf + pgaftest step check_failover tests/tap/specs/basic_operation.pgaf + + # Tear down when done + pgaftest down tests/tap/specs/basic_operation.pgaf + +Environment variables +--------------------- + +``PGAF_IMAGE`` + Docker image tag to use for all containers. Overrides the ``image`` + directive in the ``cluster { }`` block. When neither is set, ``pgaftest`` + uses a ``build:`` stanza pointing at the project root, which triggers a + local Docker build. + + .. code-block:: bash + + export PGAF_IMAGE=pg_auto_failover:pg17 + +How pgaftest and pg_autoctl node relate +---------------------------------------- + +``pgaftest`` uses ``pg_autoctl node run /etc/pgaf/node.ini`` as the container +command, which means every container benefits from the :ref:`live +reconfiguration ` feature: editing a node's ini file +on the host and saving it causes the running supervisor inside the container +to converge the changed settings within seconds (inotify) or within +``NODESPEC_WATCH_INTERVAL_SECS`` (10 s, on platforms without inotify). + +This is particularly useful during interactive sessions. For example, to +change ``node2``'s candidate priority without restarting the container: + +.. code-block:: bash + + # Edit the generated ini file in the work directory + # (auto-derived from the spec name, e.g. $TMPDIR/pgaftest/basic_operation) + WORKDIR=${TMPDIR:-/tmp}/pgaftest/basic_operation + sed -i 's/candidate_priority = 50/candidate_priority = 0/' \ + $WORKDIR/node2.ini + + # The supervisor inside node2 picks up the change automatically. + # Verify on the monitor: + docker compose exec monitor \ + pg_autoctl get node candidate-priority --pgdata /var/lib/postgres/pgaf + +pg_autoctl inspect commands used by pgaftest +-------------------------------------------- + +``pgaftest`` talks to the containers exclusively through ``pg_autoctl`` +subcommands via ``docker compose exec``. No psql scripts, no port +forwarding. The relevant commands are: + +``pg_autoctl inspect monitor node-state`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runs on the **monitor** container. Queries ``pgautofailover.node`` and +prints one line:: + + || + +``health`` is the integer from the monitor's health-check column: +``1`` = healthy, ``-1`` = unhealthy (keeper unreachable). + +With ``--name `` and ``--state `` the command exits 0 only +when ``reportedstate`` equals ````. ``--timeout N`` retries for up +to ``N`` seconds using exponential back-off. + +.. code-block:: bash + + # One-shot query (prints reported|goal|health) + docker compose exec -T monitor \ + pg_autoctl inspect monitor node-state --name node2 + + # Wait up to 90 s for node2 to report "secondary" + docker compose exec -T monitor \ + pg_autoctl inspect monitor node-state \ + --name node2 --state secondary --timeout 90 + +``pg_autoctl inspect fsm node-state`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runs on a **data node** container. Reads the keeper's on-disk state file +and prints:: + + | + +``current_role`` is the keeper's authoritative local state. It is updated +by the keeper even when the node cannot reach the monitor — for example, a +partitioned primary self-assigns ``demote_timeout`` after +``network_partition_timeout`` (default 20 s) without any monitor contact. + +With ``--state `` exits 0 when ``current_role`` equals ````. +``--timeout N`` retries for up to ``N`` seconds. + +.. code-block:: bash + + # Check that a partitioned primary has self-assigned demote_timeout + docker compose exec -T node2 \ + pg_autoctl inspect fsm node-state --state demote_timeout --timeout 60 + +``pg_autoctl inspect monitor formation-states`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Runs on the **monitor** container. Waits until the formation has at least +one node in each of the listed states simultaneously, then exits 0. + +.. code-block:: bash + + docker compose exec -T monitor \ + pg_autoctl inspect monitor formation-states \ + --timeout 120 primary secondary + +See also +-------- + +* :ref:`pg_autoctl_node` — the ``pg_autoctl node run`` command and node spec + file format used by every container that ``pgaftest`` starts +* :ref:`pg_autoctl_create_postgres` +* :ref:`pg_autoctl_perform_failover` diff --git a/docs/ref/manual.rst b/docs/ref/manual.rst index 12ed2f966..7facd9dec 100644 --- a/docs/ref/manual.rst +++ b/docs/ref/manual.rst @@ -21,6 +21,7 @@ have their own manual page. pg_autoctl_set pg_autoctl_perform pg_autoctl_do + pg_autoctl_node pg_autoctl_run pg_autoctl_watch pg_autoctl_stop diff --git a/docs/ref/pg_autoctl_do_pgsetup.rst b/docs/ref/pg_autoctl_do_pgsetup.rst index fb085d343..0913bb233 100644 --- a/docs/ref/pg_autoctl_do_pgsetup.rst +++ b/docs/ref/pg_autoctl_do_pgsetup.rst @@ -81,8 +81,22 @@ pg_autoctl do pgsetup wait -------------------------- When ``pg_autoctl do pgsetup ready`` would return false because Postgres is -not ready yet, this command continues probing every second for 30 seconds, -and exists as soon as Postgres is ready. +not ready yet, this command continues probing every second (up to +``--timeout`` seconds, default 30) and exits as soon as Postgres is ready. + +Options: + +``--read-write`` + After Postgres reports itself ready, additionally wait until the server + accepts read-write connections (i.e. ``pg_is_in_recovery()`` returns + ``false``). Useful after a failover when you want to block until the new + primary is writable before sending traffic. + +``--timeout N`` + Total timeout in seconds shared across both phases. Time spent waiting + for Postgres to start counts against the budget for the read-write phase, + so a single ``--timeout 60`` means at most 60 seconds total, not 60 + seconds per phase. Default: 30. :: @@ -90,6 +104,14 @@ and exists as soon as Postgres is ready. 16:50:22 70829 INFO Postgres is now serving PGDATA "/Users/dim/dev/MS/pg_auto_failover/tmux/node1" on port 5501 with pid 21029 16:50:22 70829 INFO Postgres status is: "ready" +:: + + $ pg_autoctl do pgsetup wait --pgdata node1 --read-write --timeout 60 + 16:50:22 70829 INFO Postgres is now serving PGDATA "..." on port 5501 with pid 21029 + 16:50:22 70829 INFO Postgres status is: "ready" + 16:50:22 70829 INFO Waiting for Postgres to accept read-write connections (timeout 60s) + 16:50:23 70829 INFO Postgres is now accepting read-write connections on port 5501 + pg_autoctl do pgsetup logs -------------------------- diff --git a/docs/ref/pg_autoctl_node.rst b/docs/ref/pg_autoctl_node.rst new file mode 100644 index 000000000..a7997c99d --- /dev/null +++ b/docs/ref/pg_autoctl_node.rst @@ -0,0 +1,404 @@ +.. _pg_autoctl_node: + +pg_autoctl node +=============== + +pg_autoctl node - Declarative node lifecycle driven by a configuration file + +Synopsis +-------- + +:: + + pg_autoctl node + run Create (if needed) and run from a node spec file + apply Apply mutable settings to a running node + show [ --pgdata ] Dump current config as a node spec file + check Validate a node spec file without creating anything + +Description +----------- + +The ``pg_autoctl node`` command group replaces the manual composition of +``pg_autoctl create [options] --run`` and provides a single declarative +interface for the full lifecycle of a pg_auto_failover node: + +1. **Create** — if ``PGDATA`` does not yet contain a ``pg_autoctl.cfg``, run + the equivalent of ``pg_autoctl create `` using the options in the + file. + +2. **Run** — hand off to the pg_autoctl supervisor loop, which then manages + the Postgres and service subprocesses. + +3. **Watch** — the supervisor watches the node spec file for changes and + automatically **converges mutable settings** (``candidate_priority``, + ``replication_quorum``) without requiring a node restart. + +The node spec file is a small INI file that collects all node parameters. +The default path inside a Docker/Kubernetes container is +``/etc/pgaf/node.ini``. A Docker image built from the pg_auto_failover +project can use a single ``CMD``: + +.. code-block:: dockerfile + + CMD ["pg_autoctl", "node", "run", "/etc/pgaf/node.ini"] + +pg_autoctl node run +------------------- + +:: + + usage: pg_autoctl node run + + Path to the pg_autoctl.ini file. + Defaults to /etc/pgaf/node.ini when no argument is given. + +**First start (node not yet created):** +Builds and executes the equivalent of:: + + pg_autoctl create \ + --pgdata \ + --hostname \ + --pgport \ + --monitor \ + --formation \ + --ssl-self-signed \ # or --no-ssl, depending on [options] ssl = + --auth trust \ + --pg-hba-lan \ + --run + +**Subsequent starts (node already created):** +Reads the spec file, applies any changed mutable settings (``candidate_priority``, +``replication_quorum``), then executes ``pg_autoctl run --pgdata ``. + +Before executing, ``pg_autoctl node run`` sets the environment variable +``PG_AUTOCTL_NODESPEC`` to the path of the spec file. The supervisor process +(PID 1 in Docker/Kubernetes) reads this variable at start-up and initialises a +file watcher so that later edits to the spec file are picked up without +restarting the supervisor. + +pg_autoctl node apply +--------------------- + +:: + + usage: pg_autoctl node apply + +Reads the node spec file and compares the mutable section (``[settings]``) +against the node's current configuration. For each changed field it runs the +appropriate ``pg_autoctl set node`` sub-command: + +.. list-table:: + :header-rows: 1 + + * - INI field + - pg_autoctl sub-command + * - ``candidate_priority`` + - ``pg_autoctl set node candidate-priority`` + * - ``replication_quorum`` + - ``pg_autoctl set node replication-quorum`` + +Use this command when the supervisor is not running (for example, in a +scripted maintenance window) to reconcile the monitor's view of the node with +a freshly edited spec file. + +pg_autoctl node show +-------------------- + +:: + + usage: pg_autoctl node show [ --pgdata ] + +Reads the local ``pg_autoctl.cfg`` configuration file and prints it in +``pg_autoctl.ini`` format on stdout. Useful for bootstrapping a spec +file from an existing node that was created with ``pg_autoctl create``. + +Example:: + + $ pg_autoctl node show --pgdata /var/lib/postgres/pgaf > node.ini + +pg_autoctl node check +--------------------- + +:: + + usage: pg_autoctl node check + +Parses and validates the spec file without creating or modifying anything. +Prints a summary of all resolved fields. Exit code is non-zero when the file +is invalid or a required field is missing. + +Example output:: + + Node spec "/etc/pgaf/node.ini" is valid. + kind : postgres + pgdata : /var/lib/postgres/pgaf + hostname : node1 + port : 5432 + monitor_pguri : postgresql://autoctl_node@monitor/pg_auto_failover + formation : default + group : 0 + candidate_priority : 50 + replication_quorum : true + ssl : self-signed + auth : trust + pg_hba_lan : true + +.. _node_spec_format: + +Node spec file format (pg_autoctl.ini) +------------------------------------------- + +The node spec file uses a simple INI syntax with five sections. Lines +beginning with ``#`` are comments. + +.. code-block:: ini + + # pg_autoctl.ini — describes a single pg_auto_failover node. + # Feed to: pg_autoctl node run /etc/pgaf/node.ini + # Edit and save to converge mutable settings on the running node. + + [node] + kind = postgres # postgres | monitor | coordinator | worker + hostname = node1 # reported to the monitor; default: system hostname + port = 5432 # Postgres port + + [postgresql] + pgdata = /var/lib/postgres/pgaf + + [monitor] + # Required for kind != monitor. Empty for monitor nodes. + pguri = postgresql://autoctl_node@monitor/pg_auto_failover + + [formation] + name = default # formation name; default "default" + group = 0 # Citus group id; 0 = coordinator group + + [settings] + # These fields are mutable. Edit and save to apply without a restart. + candidate_priority = 50 # 0–100; 0 = never a failover candidate + replication_quorum = true # participates in the synchronous replication quorum + + [options] + # These fields are immutable after the node is created. + # They are only used during the first-ever pg_autoctl create run. + ssl = self-signed # self-signed | cert | off + auth = trust # trust | md5 | scram + pg_hba_lan = true # add --pg-hba-lan at create time (allow LAN connections) + +Sections and fields +~~~~~~~~~~~~~~~~~~~ + +``[node]`` +^^^^^^^^^^ + +``kind`` + Node role. One of: ``monitor``, ``postgres``, ``coordinator``, ``worker``. + Determines which ``pg_autoctl create`` sub-command is used. Immutable + after the node is created. + +``hostname`` + Hostname or IP address reported to the pg_auto_failover monitor. Other + nodes use this to reach Postgres. Mutable in the ``[node]`` section but + requires a monitor metadata update (``pg_autoctl set node metadata``) to + take effect on a running node; the file watcher does **not** apply hostname + changes automatically. Defaults to the system hostname when empty. + +``port`` + Postgres port number. Defaults to ``5432``. Immutable after creation. + +``[postgresql]`` +^^^^^^^^^^^^^^^^ + +``pgdata`` + Absolute path to the Postgres data directory. Immutable after creation. + +``[monitor]`` +^^^^^^^^^^^^^ + +``pguri`` + Connection string the node uses to reach the pg_auto_failover monitor. + Required for all ``kind`` values except ``monitor``. Example:: + + pguri = postgresql://autoctl_node@monitor/pg_auto_failover + +``[formation]`` +^^^^^^^^^^^^^^^ + +``name`` + Formation name on the monitor. Defaults to ``default``. + +``group`` + Citus group id. Meaningful only for ``kind = coordinator`` (always ``0``) + and ``kind = worker`` (``1``, ``2``, …). Ignored for other node kinds. + +``[settings]`` — mutable +^^^^^^^^^^^^^^^^^^^^^^^^ + +``candidate_priority`` + Failover candidate priority, ``0``–``100``. ``0`` means the node is never + promoted; ``100`` means it is the preferred candidate. Default ``50``. + + Editing this field and saving the file causes the supervisor to call + ``pg_autoctl set node candidate-priority`` within 10 seconds (or + immediately on Linux systems that support inotify). + +``replication_quorum`` + ``true`` or ``false``. When ``true`` the node participates in the + synchronous replication quorum controlled by + ``synchronous_standby_names``. Default ``true``. + + Editing this field and saving the file causes the supervisor to call + ``pg_autoctl set node replication-quorum``. + +``[options]`` — immutable after creation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``ssl`` + SSL mode used at create time. One of: + + * ``self-signed`` — ``pg_autoctl`` generates a self-signed certificate + and enables SSL (``--ssl-self-signed``). + * ``cert`` — SSL is enabled; certificate files must already be in place. + * ``off`` — no SSL (``--no-ssl``). + +``auth`` + Authentication method passed as ``--auth`` to ``pg_autoctl create``. + Common values: ``trust``, ``md5``, ``scram-sha-256``. + +``pg_hba_lan`` + When ``true``, pass ``--pg-hba-lan`` at create time. This adds a + ``host all all `` rule to ``pg_hba.conf``, which is + required when Postgres is accessed from other containers on the same + Docker network. Set to ``false`` for the monitor node (which does not + serve data connections from other nodes). + +Live reconfiguration +-------------------- + +The supervisor (the process that runs as PID 1 when ``pg_autoctl node run`` +is used) watches the spec file for changes and converges the ``[settings]`` +section automatically. + +On **Linux** the supervisor uses ``inotify`` (``IN_CLOSE_WRITE`` and +``IN_MOVED_TO`` events) so changes are detected within milliseconds of the +file being closed after writing. + +On **macOS and other platforms** the supervisor falls back to polling the +file's modification time every 10 seconds (``NODESPEC_WATCH_INTERVAL_SECS``). + +Workflow — updating candidate priority in a running cluster +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # 1. Edit the spec file on the Docker host + sed -i 's/candidate_priority = 50/candidate_priority = 0/' node1.ini + + # 2. No further action required: the supervisor picks up the change + # within 10 s (mtime poll) or immediately (inotify on Linux). + # On the monitor, verify: + docker compose exec monitor \ + pg_autoctl get node candidate-priority --pgdata /var/lib/postgres/pgaf + +Immutable-field changes are logged as warnings but not applied — a node +restart is required. + +Use in Docker Compose and Kubernetes +------------------------------------- + +The primary motivation for the node spec file is to allow a **single +Docker image** with a **single command** to serve all node roles: + +.. code-block:: dockerfile + + # One image, one CMD — role determined by the mounted ini file + FROM pg_auto_failover:pg17 + CMD ["pg_autoctl", "node", "run", "/etc/pgaf/node.ini"] + +In Docker Compose, each service mounts its own ini file at +``/etc/pgaf/node.ini`` (generated by ``pgaftest`` or written by hand): + +.. code-block:: yaml + + services: + monitor: + image: pg_auto_failover:pg17 + volumes: + - monitor_data:/var/lib/postgres:rw + - ./monitor.ini:/etc/pgaf/node.ini:ro + command: ["pg_autoctl", "node", "run", "/etc/pgaf/node.ini"] + + node1: + image: pg_auto_failover:pg17 + volumes: + - node1_data:/var/lib/postgres:rw + - ./node1.ini:/etc/pgaf/node.ini:ro + command: ["pg_autoctl", "node", "run", "/etc/pgaf/node.ini"] + +In Kubernetes, use a ``ConfigMap`` to store the ini content and mount it as a +file into the Pod. Editing the ``ConfigMap`` propagates the new file into the +Pod within the kubelet's sync period (typically 1 minute); the supervisor +detects the change and converges without a Pod restart. + +.. code-block:: yaml + + apiVersion: v1 + kind: ConfigMap + metadata: + name: node1-spec + data: + node.ini: | + [node] + kind = postgres + hostname = node1.pg-cluster.svc.cluster.local + port = 5432 + + [postgresql] + pgdata = /var/lib/postgres/pgaf + + [monitor] + pguri = postgresql://autoctl_node@monitor.pg-cluster.svc.cluster.local/pg_auto_failover + + [formation] + name = default + group = 0 + + [settings] + candidate_priority = 50 + replication_quorum = true + + [options] + ssl = self-signed + auth = scram-sha-256 + pg_hba_lan = true + + --- + apiVersion: apps/v1 + kind: StatefulSet + # ... + spec: + template: + spec: + containers: + - name: postgres + image: pg_auto_failover:pg17 + command: ["pg_autoctl", "node", "run", "/etc/pgaf/node.ini"] + volumeMounts: + - name: node-spec + mountPath: /etc/pgaf + volumes: + - name: node-spec + configMap: + name: node1-spec + +See also +-------- + +* :ref:`pg_autoctl_create_postgres` +* :ref:`pg_autoctl_create_monitor` +* :ref:`pg_autoctl_run` +* :ref:`pg_autoctl_set_node_candidate_priority` +* :ref:`pg_autoctl_set_node_replication_quorum` +* :ref:`pgaftest_spec` — ``pgaftest`` test spec files use the same ini format + for the per-node files it generates inside ``cluster { }`` blocks diff --git a/src/bin/Makefile b/src/bin/Makefile index 7c743833d..fcdca4ae7 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -1,15 +1,24 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the PostgreSQL License. -all: pg_autoctl ; +all: common pg_autoctl pgaftest ; -pg_autoctl: +common: + $(MAKE) -C common + +pg_autoctl: common $(MAKE) -C pg_autoctl pg_autoctl +pgaftest: common + $(MAKE) -C pgaftest pgaftest + clean: + $(MAKE) -C common clean $(MAKE) -C pg_autoctl clean + $(MAKE) -C pgaftest clean -install: $(pg_autoctl) +install: pg_autoctl pgaftest $(MAKE) -C pg_autoctl install + $(MAKE) -C pgaftest install -.PHONY: all pg_autoctl install clean +.PHONY: all common pg_autoctl pgaftest install clean diff --git a/src/bin/common/Makefile b/src/bin/common/Makefile new file mode 100644 index 000000000..5b106fcc4 --- /dev/null +++ b/src/bin/common/Makefile @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the PostgreSQL License. + +SRC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +.DEFAULT_GOAL := $(SRC_DIR)libpgaf_common.a + +include $(SRC_DIR)Makefile.common + +override CFLAGS += -I$(SRC_DIR)../pg_autoctl + +clean: + rm -f $(COMMON_OBJ) $(COMMON_LIB) + rm -rf $(DEPDIR) + +.PHONY: clean diff --git a/src/bin/common/Makefile.common b/src/bin/common/Makefile.common new file mode 100644 index 000000000..be7a0fe9e --- /dev/null +++ b/src/bin/common/Makefile.common @@ -0,0 +1,129 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the PostgreSQL License. +# +# Makefile.common — shared build variables and common source library. +# Included by both pg_autoctl and pgaftest Makefiles via: +# include ../common/Makefile.common + +PG_CONFIG ?= pg_config + +COMMON_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +LIB_DIR := $(COMMON_DIR)../lib + +PG_SNPRINTF = $(wildcard $(LIB_DIR)/pg/snprintf.*) +LOG_SRC = $(wildcard $(LIB_DIR)/log/src/log.*) +COMMANDLINE_SRC = $(wildcard $(LIB_DIR)/subcommands.c/commandline.*) +PARSON_SRC = $(wildcard $(LIB_DIR)/parson/parson.*) + +COMMON_INCLUDES = -I$(COMMON_DIR) +COMMON_INCLUDES += -I$(LIB_DIR)/pg +COMMON_INCLUDES += -I$(LIB_DIR)/log/src/ +COMMON_INCLUDES += -I$(LIB_DIR)/subcommands.c/ +COMMON_INCLUDES += -I$(LIB_DIR)/libs/ +COMMON_INCLUDES += -I$(LIB_DIR)/parson/ + +CC = $(shell $(PG_CONFIG) --cc) + +DEFAULT_CFLAGS = -std=c99 -D_GNU_SOURCE -g +DEFAULT_CFLAGS += -I $(shell $(PG_CONFIG) --includedir) +DEFAULT_CFLAGS += -I $(shell $(PG_CONFIG) --includedir-server) +DEFAULT_CFLAGS += -I $(shell $(PG_CONFIG) --pkgincludedir)/internal +DEFAULT_CFLAGS += $(shell $(PG_CONFIG) --cflags) +DEFAULT_CFLAGS += -Wformat +DEFAULT_CFLAGS += -Wall +DEFAULT_CFLAGS += -Werror=implicit-int +DEFAULT_CFLAGS += -Werror=implicit-function-declaration +DEFAULT_CFLAGS += -Werror=return-type +DEFAULT_CFLAGS += -Wno-declaration-after-statement +DEFAULT_CFLAGS += -D_WANT_SEMUN +DEFAULT_CFLAGS += -Wno-missing-braces +DEFAULT_CFLAGS += $(COMMON_INCLUDES) + +# On macOS, gettext is keg-only and not in the default include path. +# postgres_fe.h → c.h → libintl.h requires finding it explicitly. +ifeq ($(shell uname -s),Darwin) + GETTEXT_PREFIX := $(shell brew --prefix gettext 2>/dev/null) + ifneq ($(GETTEXT_PREFIX),) + DEFAULT_CFLAGS += -I$(GETTEXT_PREFIX)/include + endif +endif + +override CFLAGS := $(DEFAULT_CFLAGS) $(CFLAGS) + +BINDIR ?= $(shell $(PG_CONFIG) --bindir) + +# pg_config --ldflags from source builds can emit colon-joined paths such as +# -L/path/a:/path/b as a single token, which the linker rejects. Split each +# -L token on ':' so every directory gets its own -L flag. +PG_LDFLAGS_RAW := $(shell $(PG_CONFIG) --ldflags) +PG_LDFLAGS := $(shell echo "$(PG_LDFLAGS_RAW)" | awk '{ \ + for (i = 1; i <= NF; i++) { \ + if (substr($$i, 1, 2) == "-L") { \ + n = split(substr($$i, 3), p, ":"); \ + for (j = 1; j <= n; j++) if (p[j] != "") printf "-L%s ", p[j]; \ + } else { printf "%s ", $$i; } \ + } \ +}') + +LIBS = -L $(shell $(PG_CONFIG) --pkglibdir) +LIBS += -L $(shell $(PG_CONFIG) --libdir) +LIBS += $(PG_LDFLAGS) +LIBS += $(shell $(PG_CONFIG) --libs) +LIBS += -lpq + +DEPDIR = .deps + +# ----------------------------------------------------------------------- +# Common source library (src/bin/common/*.c → libpgaf_common.a) +# Callers link with: $(COMMON_LIB) and add $(COMMON_LIB) to OBJS. +# ----------------------------------------------------------------------- + +COMMON_SRC = $(wildcard $(COMMON_DIR)*.c) +COMMON_OBJ = $(patsubst $(COMMON_DIR)%.c,$(COMMON_DIR)%.o,$(COMMON_SRC)) +COMMON_LIB = $(COMMON_DIR)libpgaf_common.a + +# Compile rule for common objects (run from each caller's directory) +$(COMMON_DIR)%.o: $(COMMON_DIR)%.c + @if test ! -d $(COMMON_DIR)$(DEPDIR); then mkdir -p $(COMMON_DIR)$(DEPDIR); fi + $(CC) $(CFLAGS) -c -MMD -MP -MF$(COMMON_DIR)$(DEPDIR)/$(*F).Po -o $@ $< + +$(COMMON_LIB): $(COMMON_OBJ) + $(AR) rcs $@ $^ + +Po_common_files := $(wildcard $(COMMON_DIR)$(DEPDIR)/*.Po) +ifneq (,$(Po_common_files)) +include $(Po_common_files) +endif + +# ----------------------------------------------------------------------- +# Compile rule for caller's own .c → .o with auto-dependency tracking +# ----------------------------------------------------------------------- +%.o : %.c + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -o $@ $< + +Po_files := $(wildcard $(DEPDIR)/*.Po) +ifneq (,$(Po_files)) +include $(Po_files) +endif + +# Shared rules for vendored libs +lib-snprintf.o: $(PG_SNPRINTF) + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/lib-snprintf.Po -MT$@ \ + -o $@ $(LIB_DIR)/pg/snprintf.c + +lib-strerror.o: $(PG_SNPRINTF) + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/lib-strerror.Po -MT$@ \ + -o $@ $(LIB_DIR)/pg/strerror.c + +lib-log.o: $(LOG_SRC) + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/lib-log.Po -MT$@ \ + -o $@ $(LIB_DIR)/log/src/log.c + +lib-commandline.o: $(COMMANDLINE_SRC) + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/lib-commandline.Po -MT$@ \ + -o $@ $(LIB_DIR)/subcommands.c/commandline.c + +lib-parson.o: $(PARSON_SRC) + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/lib-parson.Po -MT$@ \ + -o $@ $(LIB_DIR)/parson/parson.c diff --git a/src/bin/pg_autoctl/debian.c b/src/bin/common/debian.c similarity index 100% rename from src/bin/pg_autoctl/debian.c rename to src/bin/common/debian.c diff --git a/src/bin/pg_autoctl/debian.h b/src/bin/common/debian.h similarity index 100% rename from src/bin/pg_autoctl/debian.h rename to src/bin/common/debian.h diff --git a/src/bin/pg_autoctl/env_utils.c b/src/bin/common/env_utils.c similarity index 100% rename from src/bin/pg_autoctl/env_utils.c rename to src/bin/common/env_utils.c diff --git a/src/bin/pg_autoctl/env_utils.h b/src/bin/common/env_utils.h similarity index 100% rename from src/bin/pg_autoctl/env_utils.h rename to src/bin/common/env_utils.h diff --git a/src/bin/pg_autoctl/file_utils.c b/src/bin/common/file_utils.c similarity index 100% rename from src/bin/pg_autoctl/file_utils.c rename to src/bin/common/file_utils.c diff --git a/src/bin/pg_autoctl/file_utils.h b/src/bin/common/file_utils.h similarity index 100% rename from src/bin/pg_autoctl/file_utils.h rename to src/bin/common/file_utils.h diff --git a/src/bin/pg_autoctl/ini_file.c b/src/bin/common/ini_file.c similarity index 99% rename from src/bin/pg_autoctl/ini_file.c rename to src/bin/common/ini_file.c index 40c88016e..309f77d57 100644 --- a/src/bin/pg_autoctl/ini_file.c +++ b/src/bin/common/ini_file.c @@ -116,11 +116,12 @@ parse_ini_buffer(const char *filename, } default: - + { /* should never happen, or it's a development bug */ log_fatal("Unknown option type %d", option->type); ini_destroy(ini); return false; + } } } } @@ -219,10 +220,11 @@ ini_validate_options(IniOption *optionList) } default: - + { /* should never happen, or it's a development bug */ log_fatal("Unknown option type %d", option->type); return false; + } } } return true; @@ -660,10 +662,11 @@ ini_merge(IniOption *dstOptionList, IniOption *overrideOptionList) } default: - + { /* should never happen, or it's a development bug */ log_fatal("Unknown option type %d", option->type); return false; + } } } return true; diff --git a/src/bin/pg_autoctl/ini_file.h b/src/bin/common/ini_file.h similarity index 100% rename from src/bin/pg_autoctl/ini_file.h rename to src/bin/common/ini_file.h diff --git a/src/bin/pg_autoctl/ini_implementation.c b/src/bin/common/ini_implementation.c similarity index 100% rename from src/bin/pg_autoctl/ini_implementation.c rename to src/bin/common/ini_implementation.c diff --git a/src/bin/pg_autoctl/ipaddr.c b/src/bin/common/ipaddr.c similarity index 99% rename from src/bin/pg_autoctl/ipaddr.c rename to src/bin/common/ipaddr.c index 1ef03f9a7..b1d78ef28 100644 --- a/src/bin/pg_autoctl/ipaddr.c +++ b/src/bin/common/ipaddr.c @@ -301,7 +301,9 @@ fetchLocalCIDR(const char *localIpAddress, char *localCIDR, int size) } default: + { continue; + } } if (strcmp(address, localIpAddress) == 0) diff --git a/src/bin/pg_autoctl/ipaddr.h b/src/bin/common/ipaddr.h similarity index 100% rename from src/bin/pg_autoctl/ipaddr.h rename to src/bin/common/ipaddr.h diff --git a/src/bin/pg_autoctl/lock_utils.c b/src/bin/common/lock_utils.c similarity index 100% rename from src/bin/pg_autoctl/lock_utils.c rename to src/bin/common/lock_utils.c diff --git a/src/bin/pg_autoctl/lock_utils.h b/src/bin/common/lock_utils.h similarity index 100% rename from src/bin/pg_autoctl/lock_utils.h rename to src/bin/common/lock_utils.h diff --git a/src/bin/pg_autoctl/parsing.c b/src/bin/common/parsing.c similarity index 100% rename from src/bin/pg_autoctl/parsing.c rename to src/bin/common/parsing.c diff --git a/src/bin/pg_autoctl/parsing.h b/src/bin/common/parsing.h similarity index 100% rename from src/bin/pg_autoctl/parsing.h rename to src/bin/common/parsing.h diff --git a/src/bin/pg_autoctl/pgctl.c b/src/bin/common/pgctl.c similarity index 98% rename from src/bin/pg_autoctl/pgctl.c rename to src/bin/common/pgctl.c index 5b9b523bb..8ad756ae3 100644 --- a/src/bin/pg_autoctl/pgctl.c +++ b/src/bin/common/pgctl.c @@ -1264,7 +1264,7 @@ pg_basebackup(const char *pgdata, NodeAddress *primaryNode = &(replicationSource->primaryNode); char primaryConnInfo[MAXCONNINFO] = { 0 }; - char *args[16]; + char *args[18]; /* enough for all pg_basebackup flags incl. --checkpoint=fast */ int argsIndex = 0; char command[BUFSIZE]; @@ -1330,6 +1330,7 @@ pg_basebackup(const char *pgdata, args[argsIndex++] = "--max-rate"; args[argsIndex++] = replicationSource->maximumBackupRate; args[argsIndex++] = "--wal-method=stream"; + args[argsIndex++] = "--checkpoint=fast"; /* we don't use a replication slot e.g. when upstream is a standby */ if (!IS_EMPTY_STRING_BUFFER(replicationSource->slotName)) @@ -1632,7 +1633,8 @@ pg_ctl_postgres(const char *pg_ctl, const char *pgdata, int pgport, args[argsIndex++] = "-D"; args[argsIndex++] = (char *) pgdata; args[argsIndex++] = "-p"; - args[argsIndex++] = (char *) intToString(pgport).strValue; + IntString pgportStr = intToString(pgport); + args[argsIndex++] = (char *) pgportStr.strValue; if (listen) { @@ -2054,6 +2056,33 @@ pg_ctl_promote(const char *pg_ctl, const char *pgdata) } +/* + * pg_ctl_reload reloads Postgres configuration by running "pg_ctl reload". + * Does not require a libpq connection — useful when HBA hasn't been set up yet. + */ +bool +pg_ctl_reload(const char *pg_ctl, const char *pgdata) +{ + Program program = run_program(pg_ctl, "-D", pgdata, "reload", NULL); + int returnCode = program.returnCode; + + if (program.stdErr != NULL) + { + log_debug("%s", program.stdErr); + } + + free_program(&program); + + if (returnCode != 0) + { + log_error("pg_ctl reload -D %s failed (exit %d)", pgdata, returnCode); + return false; + } + + return true; +} + + /* * pg_setup_standby_mode sets up standby mode by either writing a recovery.conf * file or adding the configuration items to postgresql.conf and then creating diff --git a/src/bin/pg_autoctl/pgctl.h b/src/bin/common/pgctl.h similarity index 97% rename from src/bin/pg_autoctl/pgctl.h rename to src/bin/common/pgctl.h index 328c352dc..e030d8217 100644 --- a/src/bin/pg_autoctl/pgctl.h +++ b/src/bin/common/pgctl.h @@ -59,6 +59,7 @@ bool pg_log_recovery_setup(const char *pgdata, int logLevel); bool pg_ctl_stop(const char *pg_ctl, const char *pgdata); int pg_ctl_status(const char *pg_ctl, const char *pgdata, bool log_output); bool pg_ctl_promote(const char *pg_ctl, const char *pgdata); +bool pg_ctl_reload(const char *pg_ctl, const char *pgdata); bool pg_setup_standby_mode(uint32_t pg_control_version, const char *pgdata, diff --git a/src/bin/pg_autoctl/pgsetup.c b/src/bin/common/pgsetup.c similarity index 99% rename from src/bin/pg_autoctl/pgsetup.c rename to src/bin/common/pgsetup.c index 5554d0344..c9e0b0ecf 100644 --- a/src/bin/pg_autoctl/pgsetup.c +++ b/src/bin/common/pgsetup.c @@ -1491,8 +1491,10 @@ nodeKindToString(PgInstanceKind kind) } default: + { log_fatal("nodeKindToString: unknown node kind %d", kind); return NULL; + } } /* can't happen, keep compiler happy */ @@ -1563,7 +1565,9 @@ pmStatusToString(PostmasterStatus pm_status) } case POSTMASTER_STATUS_STANDBY: + { return "standby"; + } } /* keep compiler happy */ @@ -1842,7 +1846,9 @@ pgsetup_sslmode_to_string(SSLMode sslMode) } case SSL_MODE_VERIFY_FULL: + { return "verify-full"; + } } /* This is a huge bug */ @@ -1986,7 +1992,9 @@ pgsetup_hba_level_to_string(HBAEditLevel hbaLevel) } case HBA_EDIT_UNKNOWN: + { return "unknown"; + } } log_error("BUG: hbaLevel %d is unknown", hbaLevel); @@ -2033,7 +2041,9 @@ dbstateToString(DBState state) } case DB_IN_PRODUCTION: + { return "in production"; + } } return "unrecognized status code"; } diff --git a/src/bin/pg_autoctl/pgsetup.h b/src/bin/common/pgsetup.h similarity index 100% rename from src/bin/pg_autoctl/pgsetup.h rename to src/bin/common/pgsetup.h diff --git a/src/bin/pg_autoctl/pgsql.c b/src/bin/common/pgsql.c similarity index 97% rename from src/bin/pg_autoctl/pgsql.c rename to src/bin/common/pgsql.c index 450c7e1a5..4fe32a084 100644 --- a/src/bin/pg_autoctl/pgsql.c +++ b/src/bin/common/pgsql.c @@ -1580,7 +1580,8 @@ BuildNodesArrayValues(NodeAddressArray *nodeArray, for (nodeIndex = 0; nodeIndex < nodeArray->count; nodeIndex++) { NodeAddress *node = &(nodeArray->nodes[nodeIndex]); - char *nodeIdString = intToString(node->nodeId).strValue; + IntString nodeIdStr = intToString(node->nodeId); + char *nodeIdString = nodeIdStr.strValue; int idParamIndex = paramIndex; int lsnParamIndex = paramIndex + 1; @@ -2334,6 +2335,85 @@ pgsql_create_user(PGSQL *pgsql, const char *userName, const char *password, } +/* + * pgsql_alter_role_password runs ALTER ROLE PASSWORD '...'. + * + * The password is never logged; we log "ALTER ROLE PASSWORD '*****'" + * instead. Uses PQescapeIdentifier / PQescapeLiteral so the values are safe + * to interpolate directly into the query string. + */ +bool +pgsql_alter_role_password(PGSQL *pgsql, const char *roleName, + const char *password) +{ + /* open a connection upfront since it is needed by PQescape functions */ + PGconn *connection = pgsql_open_connection(pgsql); + if (connection == NULL) + { + /* error message was logged in pgsql_open_connection */ + return false; + } + + char *escapedRole = PQescapeIdentifier(connection, roleName, strlen(roleName)); + if (escapedRole == NULL) + { + log_error("Failed to escape role name \"%s\": %s", + roleName, PQerrorMessage(connection)); + pgsql_finish(pgsql); + return false; + } + + char *escapedPassword = PQescapeLiteral(connection, password, strlen(password)); + if (escapedPassword == NULL) + { + log_error("Failed to escape password for role \"%s\": %s", + roleName, PQerrorMessage(connection)); + PQfreemem(escapedRole); + pgsql_finish(pgsql); + return false; + } + + /* log without the real password */ + log_debug("ALTER ROLE %s PASSWORD '*****';", escapedRole); + + PQExpBuffer query = createPQExpBuffer(); + appendPQExpBuffer(query, "ALTER ROLE %s PASSWORD %s", escapedRole, escapedPassword); + PQfreemem(escapedRole); + PQfreemem(escapedPassword); + + if (PQExpBufferBroken(query)) + { + log_error("Failed to allocate memory"); + destroyPQExpBuffer(query); + pgsql_finish(pgsql); + return false; + } + + PGresult *result = PQexec(connection, query->data); + destroyPQExpBuffer(query); + + if (!is_response_ok(result)) + { + log_error("Failed to alter role \"%s\" password: %s", + roleName, PQerrorMessage(connection)); + PQclear(result); + clear_results(pgsql); + pgsql_finish(pgsql); + return false; + } + + PQclear(result); + clear_results(pgsql); + + if (pgsql->connectionStatementType == PGSQL_CONNECTION_SINGLE_STATEMENT) + { + pgsql_finish(pgsql); + } + + return true; +} + + /* * pgsql_has_replica returns whether a replica with the given username is active. */ diff --git a/src/bin/pg_autoctl/pgsql.h b/src/bin/common/pgsql.h similarity index 99% rename from src/bin/pg_autoctl/pgsql.h rename to src/bin/common/pgsql.h index cd8387172..978f6b172 100644 --- a/src/bin/pg_autoctl/pgsql.h +++ b/src/bin/common/pgsql.h @@ -366,6 +366,8 @@ bool pgsql_create_extension(PGSQL *pgsql, const char *name); bool pgsql_create_user(PGSQL *pgsql, const char *userName, const char *password, bool login, bool superuser, bool replication, int connlimit); +bool pgsql_alter_role_password(PGSQL *pgsql, const char *roleName, + const char *password); bool pgsql_has_replica(PGSQL *pgsql, char *userName, bool *hasReplica); bool hostname_from_uri(const char *pguri, char *hostname, int maxHostLength, int *port); diff --git a/src/bin/pg_autoctl/pgtuning.c b/src/bin/common/pgtuning.c similarity index 100% rename from src/bin/pg_autoctl/pgtuning.c rename to src/bin/common/pgtuning.c diff --git a/src/bin/pg_autoctl/pgtuning.h b/src/bin/common/pgtuning.h similarity index 100% rename from src/bin/pg_autoctl/pgtuning.h rename to src/bin/common/pgtuning.h diff --git a/src/bin/pg_autoctl/pidfile.c b/src/bin/common/pidfile.c similarity index 100% rename from src/bin/pg_autoctl/pidfile.c rename to src/bin/common/pidfile.c diff --git a/src/bin/pg_autoctl/pidfile.h b/src/bin/common/pidfile.h similarity index 100% rename from src/bin/pg_autoctl/pidfile.h rename to src/bin/common/pidfile.h diff --git a/src/bin/pg_autoctl/signals.c b/src/bin/common/signals.c similarity index 99% rename from src/bin/pg_autoctl/signals.c rename to src/bin/common/signals.c index 0114affe0..a870d66f5 100644 --- a/src/bin/pg_autoctl/signals.c +++ b/src/bin/common/signals.c @@ -253,6 +253,8 @@ signal_to_string(int signal) } default: + { return "unknown signal"; + } } } diff --git a/src/bin/pg_autoctl/signals.h b/src/bin/common/signals.h similarity index 100% rename from src/bin/pg_autoctl/signals.h rename to src/bin/common/signals.h diff --git a/src/bin/pg_autoctl/string_utils.c b/src/bin/common/string_utils.c similarity index 100% rename from src/bin/pg_autoctl/string_utils.c rename to src/bin/common/string_utils.c diff --git a/src/bin/pg_autoctl/string_utils.h b/src/bin/common/string_utils.h similarity index 100% rename from src/bin/pg_autoctl/string_utils.h rename to src/bin/common/string_utils.h diff --git a/src/bin/pg_autoctl/system_utils.c b/src/bin/common/system_utils.c similarity index 100% rename from src/bin/pg_autoctl/system_utils.c rename to src/bin/common/system_utils.c diff --git a/src/bin/pg_autoctl/system_utils.h b/src/bin/common/system_utils.h similarity index 100% rename from src/bin/pg_autoctl/system_utils.h rename to src/bin/common/system_utils.h diff --git a/src/bin/lib/subcommands.c/commandline.c b/src/bin/lib/subcommands.c/commandline.c index bd9eac138..f83c85ede 100644 --- a/src/bin/lib/subcommands.c/commandline.c +++ b/src/bin/lib/subcommands.c/commandline.c @@ -226,6 +226,9 @@ commandline_pretty_print_subcommands(CommandLine *command, FILE *stream) /* pretty printing: reduce maximum length of subcommand names */ for (subcommand = command->subcommands; *subcommand != NULL; subcommand++) { + if ((*subcommand)->hidden) + continue; + int len = strlen((*subcommand)->name); if (maxLength < len) @@ -238,6 +241,9 @@ commandline_pretty_print_subcommands(CommandLine *command, FILE *stream) { const char *description = ""; + if ((*subcommand)->hidden) + continue; + if ((*subcommand)->shortDescription != NULL) { description = (*subcommand)->shortDescription; diff --git a/src/bin/lib/subcommands.c/commandline.h b/src/bin/lib/subcommands.c/commandline.h index 888e58550..a0cb674b9 100644 --- a/src/bin/lib/subcommands.c/commandline.h +++ b/src/bin/lib/subcommands.c/commandline.h @@ -29,15 +29,25 @@ typedef struct CommandLine struct CommandLine **subcommands; char *breadcrumb; + bool hidden; /* if true, omit from --help output */ } CommandLine; extern CommandLine *current_command; #define make_command_set(name, desc, usage, help, getopt, set) \ - { name, desc, usage, help, getopt, NULL, set, NULL } + { name, desc, usage, help, getopt, NULL, set, NULL, false } #define make_command(name, desc, usage, help, getopt, run) \ - { name, desc, usage, help, getopt, run, NULL, NULL } + { name, desc, usage, help, getopt, run, NULL, NULL, false } + +/* + * Like make_command_set but the command is omitted from --help output. + * The command still routes normally when invoked explicitly — used for + * internal subprocess entry points that must remain in the binary but + * are not intended for direct operator use. + */ +#define make_hidden_command_set(name, desc, usage, help, getopt, set) \ + { name, desc, usage, help, getopt, NULL, set, NULL, true } bool commandline_run(CommandLine *command, int argc, char **argv); void commandline_help(FILE *stream); diff --git a/src/bin/pg_autoctl/Makefile b/src/bin/pg_autoctl/Makefile index f1d884db1..b1bc05322 100644 --- a/src/bin/pg_autoctl/Makefile +++ b/src/bin/pg_autoctl/Makefile @@ -5,104 +5,37 @@ PG_AUTOCTL = ./pg_autoctl SRC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -DEPDIR = $(SRC_DIR)/.deps +# Must be set before include so that targets in Makefile.common don't +# become the default goal when included before the all: rule below. +.DEFAULT_GOAL := all -INCLUDES = $(patsubst ${SRC_DIR}%.h,%.h,$(wildcard ${SRC_DIR}*.h)) +include $(SRC_DIR)../common/Makefile.common -SRC = $(patsubst ${SRC_DIR}%.c,%.c,$(wildcard ${SRC_DIR}*.c)) -OBJS = $(patsubst %.c,%.o,$(SRC)) -OBJS += lib-log.o lib-commandline.o lib-parson.o lib-snprintf.o lib-strerror.o - -PG_CONFIG ?= pg_config -BINDIR ?= $(shell $(PG_CONFIG) --bindir) - -PG_SNPRINTF = $(wildcard ${SRC_DIR}../lib/pg/snprintf.*) -LOG_SRC = $(wildcard ${SRC_DIR}../lib/log/src/log.*) -COMMANDLINE_SRC = $(wildcard ${SRC_DIR}../lib/subcommands.c/commandline.*) -PARSON_SRC = $(wildcard ${SRC_DIR}../lib/parson/parson.*) - -COMMON_LIBS = -I${SRC_DIR}../lib/pg -COMMON_LIBS += -I${SRC_DIR}../lib/log/src/ -COMMON_LIBS += -I${SRC_DIR}../lib/subcommands.c/ -COMMON_LIBS += -I${SRC_DIR}../lib/libs/ -COMMON_LIBS += -I${SRC_DIR}../lib/parson/ - -CC = $(shell $(PG_CONFIG) --cc) +# pg_autoctl includes its own source directory too +override CFLAGS += -I$(SRC_DIR) -DEFAULT_CFLAGS = -std=c99 -D_GNU_SOURCE -g -DEFAULT_CFLAGS += -I $(shell $(PG_CONFIG) --includedir) -DEFAULT_CFLAGS += -I $(shell $(PG_CONFIG) --includedir-server) -DEFAULT_CFLAGS += -I $(shell $(PG_CONFIG) --pkgincludedir)/internal -DEFAULT_CFLAGS += $(shell $(PG_CONFIG) --cflags) -DEFAULT_CFLAGS += -Wformat -DEFAULT_CFLAGS += -Wall -DEFAULT_CFLAGS += -Werror=implicit-int -DEFAULT_CFLAGS += -Werror=implicit-function-declaration -DEFAULT_CFLAGS += -Werror=return-type -DEFAULT_CFLAGS += -Wno-declaration-after-statement +INCLUDES = $(patsubst $(SRC_DIR)%.h,%.h,$(wildcard $(SRC_DIR)*.h)) -# Needed for FreeBSD -DEFAULT_CFLAGS += -D_WANT_SEMUN - -# Needed for OSX -DEFAULT_CFLAGS += -Wno-missing-braces -DEFAULT_CFLAGS += $(COMMON_LIBS) - -ifdef USE_SECURITY_FLAGS -# Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide -SECURITY_CFLAGS=-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2 -z noexecstack -fpie -Wl,-pie -Wl,-z,relro -Wl,-z,now -Wformat -Wformat-security -Werror=format-security -DEFAULT_CFLAGS += $(SECURITY_CFLAGS) -endif - -override CFLAGS := $(DEFAULT_CFLAGS) $(CFLAGS) +SRC = $(patsubst $(SRC_DIR)%.c,%.c,$(wildcard $(SRC_DIR)*.c)) +OBJS = $(patsubst %.c,%.o,$(SRC)) +OBJS += lib-log.o lib-commandline.o lib-parson.o lib-snprintf.o lib-strerror.o +OBJS += $(COMMON_LIB) -LIBS = -L $(shell $(PG_CONFIG) --pkglibdir) -LIBS += -L $(shell $(PG_CONFIG) --libdir) -LIBS += $(shell $(PG_CONFIG) --ldflags) -LIBS += $(shell $(PG_CONFIG) --libs) -LIBS += -lpq +# ncurses needed for pg_autoctl watch LIBS += -lncurses -all: $(PG_AUTOCTL) ; - -# Based on Postgres Makefile for automatic dependency generation -# https://github.com/postgres/postgres/blob/1933ae629e7b706c6c23673a381e778819db307d/src/Makefile.global.in#L890-L924 -%.o : %.c - @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi - $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -o $@ $< - -Po_files := $(wildcard $(DEPDIR)/*.Po) -ifneq (,$(Po_files)) -include $(Po_files) -endif - +all: $(COMMON_LIB) $(PG_AUTOCTL) ; $(PG_AUTOCTL): $(OBJS) $(INCLUDES) $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LIBS) -o $@ -lib-snprintf.o: $(PG_SNPRINTF) - $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -MT$@ -o $@ ${SRC_DIR}../lib/pg/snprintf.c - -lib-strerror.o: $(PG_SNPRINTF) - $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -MT$@ -o $@ ${SRC_DIR}../lib/pg/strerror.c - -lib-log.o: $(LOG_SRC) - $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -MT$@ -o $@ ${SRC_DIR}../lib/log/src/log.c - -lib-commandline.o: $(COMMANDLINE_SRC) - $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -MT$@ -o $@ ${SRC_DIR}../lib/subcommands.c/commandline.c - -lib-parson.o: $(PARSON_SRC) - $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/$(*F).Po -MT$@ -o $@ ${SRC_DIR}../lib/parson/parson.c - clean: rm -f $(OBJS) $(PG_AUTOCTL) + rm -f $(COMMON_LIB) $(COMMON_OBJ) rm -rf $(DEPDIR) install: $(PG_AUTOCTL) install -d $(DESTDIR)$(BINDIR) install -m 0755 $(PG_AUTOCTL) $(DESTDIR)$(BINDIR) - - -.PHONY: all monitor clean +.PHONY: all clean install diff --git a/src/bin/pg_autoctl/azure.c b/src/bin/pg_autoctl/azure.c deleted file mode 100644 index 1cfeaddc1..000000000 --- a/src/bin/pg_autoctl/azure.c +++ /dev/null @@ -1,2516 +0,0 @@ -/* - * src/bin/pg_autoctl/azure.c - * Implementation of a CLI which lets you call `az` cli commands to prepare - * a pg_auto_failover demo or QA environment. - * - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the PostgreSQL License. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "postgres_fe.h" -#include "pqexpbuffer.h" -#include "snprintf.h" - -#include "azure.h" -#include "cli_common.h" -#include "cli_do_root.h" -#include "cli_root.h" -#include "commandline.h" -#include "config.h" -#include "env_utils.h" -#include "log.h" -#include "parsing.h" -#include "pgsql.h" -#include "pidfile.h" -#include "signals.h" -#include "string_utils.h" - -#include "runprogram.h" - -char azureCLI[MAXPGPATH] = { 0 }; - -static int azure_run_command(Program *program); -static pid_t azure_start_command(Program *program); -static bool azure_wait_for_commands(int count, pid_t pidArray[]); - -static bool run_ssh(const char *username, const char *ip); - -static bool run_ssh_command(const char *username, - const char *ip, - bool tty, - const char *command); - -static bool start_ssh_command(const char *username, - const char *ip, - const char *command); - -static bool azure_git_toplevel(char *srcDir, size_t size); - -static bool start_rsync_command(const char *username, - const char *ip, - const char *srcDir); - -static bool azure_rsync_vms(AzureRegionResources *azRegion); - -static bool azure_fetch_resource_list(const char *group, - AzureRegionResources *azRegion); - -static bool azure_fetch_vm_addresses(const char *group, const char *vm, - AzureVMipAddresses *addresses); - - -/* log_program_output logs the output of the given program. */ -static void -log_program_output(Program *prog, int outLogLevel, int errorLogLevel) -{ - if (prog->stdOut != NULL) - { - char *outLines[BUFSIZE]; - int lineCount = splitLines(prog->stdOut, outLines, BUFSIZE); - int lineNumber = 0; - - for (lineNumber = 0; lineNumber < lineCount; lineNumber++) - { - log_level(outLogLevel, "%s", outLines[lineNumber]); - } - } - - if (prog->stdErr != NULL) - { - char *errorLines[BUFSIZE]; - int lineCount = splitLines(prog->stdErr, errorLines, BUFSIZE); - int lineNumber = 0; - - for (lineNumber = 0; lineNumber < lineCount; lineNumber++) - { - log_level(errorLogLevel, "%s", errorLines[lineNumber]); - } - } -} - - -/* - * azure_run_command runs a command line using the azure CLI command, and when - * dryRun is true instead of running the command it only shows the command it - * would run as the output of the pg_autoctl command. - */ -static int -azure_run_command(Program *program) -{ - int returnCode; - char command[BUFSIZE] = { 0 }; - - (void) snprintf_program_command_line(program, command, sizeof(command)); - - if (dryRun) - { - appendPQExpBuffer(azureScript, "\n%s", command); - - /* fake successful execution */ - return 0; - } - - log_debug("%s", command); - - (void) execute_subprogram(program); - - returnCode = program->returnCode; - - if (returnCode != 0) - { - (void) log_program_output(program, LOG_INFO, LOG_ERROR); - } - - free_program(program); - - return returnCode; -} - - -/* - * azure_start_command starts a command in the background, as a subprocess of - * the current process, and returns the sub-process pid as soon as the - * sub-process is started. It's the responsibility of the caller to then - * implement waitpid() on the returned pid. - * - * This allows running several commands in parallel, as in the shell sequence: - * - * $ az vm create & - * $ az vm create & - * $ az vm create & - * $ wait - */ -static pid_t -azure_start_command(Program *program) -{ - pid_t fpid; - char command[BUFSIZE] = { 0 }; - - (void) snprintf_program_command_line(program, command, sizeof(command)); - - if (dryRun) - { - appendPQExpBuffer(azureScript, "\n%s &", command); - - /* fake successful execution */ - return 0; - } - - log_debug("%s", command); - - /* Flush stdio channels just before fork, to avoid double-output problems */ - fflush(stdout); - fflush(stderr); - - /* time to create the node_active sub-process */ - fpid = fork(); - - switch (fpid) - { - case -1: - { - log_error("Failed to fork a process for command: %s", command); - return -1; - } - - case 0: - { - /* child process runs the command */ - int returnCode; - - /* initialize the semaphore used for locking log output */ - if (!semaphore_init(&log_semaphore)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } - - /* set our logging facility to use our semaphore as a lock */ - (void) log_set_udata(&log_semaphore); - (void) log_set_lock(&semaphore_log_lock_function); - - (void) execute_subprogram(program); - returnCode = program->returnCode; - - log_debug("Command %s exited with return code %d", - program->args[0], - returnCode); - - if (returnCode != 0) - { - (void) log_program_output(program, LOG_INFO, LOG_ERROR); - free_program(program); - - /* the parent will have to use exit status */ - (void) semaphore_finish(&log_semaphore); - exit(EXIT_CODE_INTERNAL_ERROR); - } - - free_program(program); - (void) semaphore_finish(&log_semaphore); - exit(EXIT_CODE_QUIT); - } - - default: - { - /* fork succeeded, in parent */ - return fpid; - } - } -} - - -/* - * azure_wait_for_commands waits until all processes with pids from the array - * are done. - */ -static bool -azure_wait_for_commands(int count, pid_t pidArray[]) -{ - int subprocessCount = count; - bool allReturnCodeAreZero = true; - - while (subprocessCount > 0) - { - pid_t pid; - int status; - - /* ignore errors */ - pid = waitpid(-1, &status, WNOHANG); - - switch (pid) - { - case -1: - { - if (errno == ECHILD) - { - /* no more childrens */ - return subprocessCount == 0; - } - - pg_usleep(100 * 1000); /* 100 ms */ - break; - } - - case 0: - { - /* - * We're using WNOHANG, 0 means there are no stopped or - * exited children, it's all good. It's the expected case - * when everything is running smoothly, so enjoy and sleep - * for awhile. - */ - pg_usleep(100 * 1000); /* 100 ms */ - break; - } - - default: - { - /* - * One of the az vm create sub-commands has finished, find - * which and if it went all okay. - */ - int returnCode = WEXITSTATUS(status); - - /* find which VM is done now */ - for (int index = 0; index < count; index++) - { - if (pidArray[index] == pid) - { - if (returnCode == 0) - { - log_debug("Process %d exited successfully", - pid); - } - else - { - log_error("Process %d exited with code %d", - pid, returnCode); - - allReturnCodeAreZero = false; - } - } - } - - --subprocessCount; - break; - } - } - } - - return allReturnCodeAreZero; -} - - -/* - * azure_psleep runs count parallel sleep process at the same time. - */ -bool -azure_psleep(int count, bool force) -{ - char sleep[MAXPGPATH] = { 0 }; - pid_t pidArray[26] = { 0 }; - - bool saveDryRun = dryRun; - - if (!search_path_first("sleep", sleep, LOG_ERROR)) - { - log_fatal("Failed to find program sleep in PATH"); - return false; - } - - if (force) - { - dryRun = false; - } - - for (int i = 0; i < count; i++) - { - char *args[3]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = sleep; - args[argsIndex++] = "5"; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - pidArray[i] = azure_start_command(&program); - } - - if (force) - { - dryRun = saveDryRun; - } - - if (!azure_wait_for_commands(count, pidArray)) - { - log_fatal("Failed to sleep concurrently with %d processes", count); - return false; - } - - return true; -} - - -/* - * azure_get_remote_ip gets the local IP address by using the command `curl - * ifconfig.me` - */ -bool -azure_get_remote_ip(char *ipAddress, size_t ipAddressSize) -{ - Program program; - char curl[MAXPGPATH] = { 0 }; - - if (!search_path_first("curl", curl, LOG_ERROR)) - { - log_fatal("Failed to find program curl in PATH"); - return false; - } - - program = run_program(curl, "ifconfig.me", NULL); - - if (program.returnCode != 0) - { - (void) log_program_output(&program, LOG_INFO, LOG_ERROR); - free_program(&program); - return false; - } - else - { - /* we expect a single line of output, no end-of-line */ - strlcpy(ipAddress, program.stdOut, ipAddressSize); - free_program(&program); - - return true; - } -} - - -/* - * azure_create_group creates a new resource group on Azure. - */ -bool -azure_create_group(const char *name, const char *location) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "group"; - args[argsIndex++] = "create"; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = "--location"; - args[argsIndex++] = (char *) location; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Creating group \"%s\" in location \"%s\"", name, location); - - return azure_run_command(&program) == 0; -} - - -/* - * azure_create_vnet creates a new vnet on Azure. - */ -bool -azure_create_vnet(const char *group, const char *name, const char *prefix) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "network"; - args[argsIndex++] = "vnet"; - args[argsIndex++] = "create"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = "--address-prefix"; - args[argsIndex++] = (char *) prefix; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Creating network vnet \"%s\" using address prefix \"%s\"", - name, prefix); - - return azure_run_command(&program) == 0; -} - - -/* - * azure_create_vnet creates a new vnet on Azure. - */ -bool -azure_create_nsg(const char *group, const char *name) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "network"; - args[argsIndex++] = "nsg"; - args[argsIndex++] = "create"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Creating network nsg \"%s\"", name); - - return azure_run_command(&program) == 0; -} - - -/* - * azure_create_vnet creates a new network security rule. - */ -bool -azure_create_nsg_rule(const char *group, - const char *nsgName, - const char *name, - const char *ipAddress) -{ - char *args[38]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "network"; - args[argsIndex++] = "nsg"; - args[argsIndex++] = "rule"; - args[argsIndex++] = "create"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--nsg-name"; - args[argsIndex++] = (char *) nsgName; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = "--access"; - args[argsIndex++] = "allow"; - args[argsIndex++] = "--protocol"; - args[argsIndex++] = "Tcp"; - args[argsIndex++] = "--direction"; - args[argsIndex++] = "Inbound"; - args[argsIndex++] = "--priority"; - args[argsIndex++] = "100"; - args[argsIndex++] = "--source-address-prefixes"; - args[argsIndex++] = (char *) ipAddress; - args[argsIndex++] = "--source-port-range"; - args[argsIndex++] = dryRun ? "\"*\"" : "*"; - args[argsIndex++] = "--destination-address-prefix"; - args[argsIndex++] = dryRun ? "\"*\"" : "*"; - args[argsIndex++] = "--destination-port-ranges"; - args[argsIndex++] = "22"; - args[argsIndex++] = "5432"; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Creating network nsg rules \"%s\" for our IP address \"%s\" " - "for ports 22 and 5432", name, ipAddress); - - return azure_run_command(&program) == 0; -} - - -/* - * azure_create_subnet creates a new subnet on Azure. - */ -bool -azure_create_subnet(const char *group, - const char *vnet, - const char *name, - const char *prefixes, - const char *nsg) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "network"; - args[argsIndex++] = "vnet"; - args[argsIndex++] = "subnet"; - args[argsIndex++] = "create"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--vnet-name"; - args[argsIndex++] = (char *) vnet; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = "--address-prefixes"; - args[argsIndex++] = (char *) prefixes; - args[argsIndex++] = "--network-security-group"; - args[argsIndex++] = (char *) nsg; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Creating network subnet \"%s\" using address prefix \"%s\"", - name, prefixes); - - return azure_run_command(&program) == 0; -} - - -/* - * az_group_delete runs the command az group delete. - */ -bool -az_group_delete(const char *group) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "group"; - args[argsIndex++] = "delete"; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--yes"; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Deleting azure resource group \"%s\"", group); - - return azure_run_command(&program) == 0; -} - - -/* - * azure_prepare_node_name is a utility function that prepares a node name to - * use for a VM in our pg_auto_failover deployment in a target Azure region. - * - * In the resource group "ha-demo-dim-paris" when creating a monitor (index 0), - * an app VM (index 27), and 2 pg nodes VMs we would have the following names: - * - * - [0] ha-demo-dim-paris-monitor - * - [1] ha-demo-dim-paris-a - * - [2] ha-demo-dim-paris-b - * - [27] ha-demo-dim-paris-app - */ -static void -azure_prepare_node(AzureRegionResources *azRegion, int index) -{ - char vmsuffix[] = "abcdefghijklmnopqrstuvwxyz"; - - if (index == 0) - { - sformat(azRegion->vmArray[index].name, - sizeof(azRegion->vmArray[index].name), - "%s-monitor", - azRegion->group); - } - else if (index == MAX_VMS_PER_REGION - 1) - { - sformat(azRegion->vmArray[index].name, - sizeof(azRegion->vmArray[index].name), - "%s-app", - azRegion->group); - } - else - { - sformat(azRegion->vmArray[index].name, - sizeof(azRegion->vmArray[index].name), - "%s-%c", - azRegion->group, - vmsuffix[index - 1]); - } -} - - -/* - * azure_node_index_from_name is the complement to azure_prepare_node. - * Given a VM name such as ha-demo-dim-paris-monitor or ha-demo-dim-paris-a, - * the function returns respectively 0 and 1, which is the array index where we - * want to find information about the VM (name, IP addresses, etc) in an array - * of VMs. - */ -static int -azure_node_index_from_name(const char *group, const char *name) -{ - int groupNameLen = strlen(group); - char *ptr; - - if (strncmp(name, group, groupNameLen) != 0 || - strlen(name) < (groupNameLen + 1)) - { - log_error("VM name \"%s\" does not start with group name \"%s\"", - name, group); - return -1; - } - - /* skip group name and dash: ha-demo-dim-paris- */ - ptr = (char *) name + groupNameLen + 1; - - /* - * ha-demo-dim-paris-monitor is always index 0 - * ha-demo-dim-paris-app is always index 27 (last in the array) - * ha-demo-dim-paris-a is index 1 - * ha-demo-dim-paris-b is index 2 - * ... - * ha-demo-dim-paris-z is index 26 - */ - if (strcmp(ptr, "monitor") == 0) - { - return 0; - } - else if (strcmp(ptr, "app") == 0) - { - return MAX_VMS_PER_REGION - 1; - } - else - { - if (strlen(ptr) != 1) - { - log_error("Failed to parse VM index from name \"%s\"", name); - return -1; - } - - /* 'a' is 1, 'b' is 2, etc */ - return *ptr - 'a' + 1; - } -} - - -/* - * azure_create_vm creates a Virtual Machine in our azure resource group. - */ -bool -azure_create_vm(AzureRegionResources *azRegion, - const char *name, - const char *image, - const char *username) -{ - char *args[26]; - int argsIndex = 0; - - Program program = { 0 }; - - char publicIpAddressName[BUFSIZE] = { 0 }; - - sformat(publicIpAddressName, BUFSIZE, "%s-ip", name); - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "vm"; - args[argsIndex++] = "create"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) azRegion->group; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = "--vnet-name"; - args[argsIndex++] = (char *) azRegion->vnet; - args[argsIndex++] = "--subnet"; - args[argsIndex++] = (char *) azRegion->subnet; - args[argsIndex++] = "--nsg"; - args[argsIndex++] = (char *) azRegion->nsg; - args[argsIndex++] = "--public-ip-address"; - args[argsIndex++] = (char *) publicIpAddressName; - args[argsIndex++] = "--image"; - args[argsIndex++] = (char *) image; - args[argsIndex++] = "--admin-username"; - args[argsIndex++] = (char *) username; - args[argsIndex++] = "--generate-ssh-keys"; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Creating %s virtual machine \"%s\" with user \"%s\"", - image, name, username); - - return azure_start_command(&program); -} - - -/* - * azure_create_vms creates several azure virtual machine in parallel and waits - * until all the commands have finished. - */ -bool -azure_create_vms(AzureRegionResources *azRegion, - const char *image, - const char *username) -{ - int pending = 0; - pid_t pidArray[MAX_VMS_PER_REGION] = { 0 }; - - /* we read from left to right, have the smaller number on the left */ - if (26 < azRegion->nodes) - { - log_error("pg_autoctl only supports up to 26 VMs per region"); - return false; - } - - log_info("Creating Virtual Machines for %s%d Postgres nodes, in parallel", - azRegion->monitor ? "a monitor and " : " ", - azRegion->nodes); - - /* index == 0 for the monitor, then 1..count for the other nodes */ - for (int index = 0; index <= azRegion->nodes; index++) - { - /* skip index 0 when we're not creating a monitor */ - if (index == 0 && !azRegion->monitor) - { - continue; - } - - /* skip VMs that already exist, unless --script is used */ - if (!dryRun && - !IS_EMPTY_STRING_BUFFER(azRegion->vmArray[index].name) && - !IS_EMPTY_STRING_BUFFER(azRegion->vmArray[index].public) && - !IS_EMPTY_STRING_BUFFER(azRegion->vmArray[index].private)) - { - log_info("Skipping creation of VM \"%s\", " - "which already exists with public IP address %s", - azRegion->vmArray[index].name, - azRegion->vmArray[index].public); - continue; - } - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = azure_create_vm(azRegion, - azRegion->vmArray[index].name, - image, - username); - ++pending; - } - - /* also create the application node VM when asked to */ - if (azRegion->appNodes > 0) - { - int index = MAX_VMS_PER_REGION - 1; - - if (!dryRun && - !IS_EMPTY_STRING_BUFFER(azRegion->vmArray[index].name) && - !IS_EMPTY_STRING_BUFFER(azRegion->vmArray[index].public) && - !IS_EMPTY_STRING_BUFFER(azRegion->vmArray[index].private)) - { - log_info("Skipping creation of VM \"%s\", " - "which already exists with public IP address %s", - azRegion->vmArray[index].name, - azRegion->vmArray[index].public); - } - else - { - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = azure_create_vm(azRegion, - azRegion->vmArray[index].name, - image, - username); - ++pending; - } - } - - /* now wait for the child processes to be done */ - if (dryRun && pending > 0) - { - appendPQExpBuffer(azureScript, "\nwait"); - } - else - { - if (!azure_wait_for_commands(pending, pidArray)) - { - log_fatal("Failed to create all %d azure VMs, " - "see above for details", - pending); - return false; - } - } - - return true; -} - - -/* - * azure_git_toplevel calls `git rev-parse --show-toplevel` and uses the result - * as the directory to rsync to our VMs when provisionning from sources. - */ -static bool -azure_git_toplevel(char *srcDir, size_t size) -{ - Program program; - char git[MAXPGPATH] = { 0 }; - - if (!search_path_first("git", git, LOG_ERROR)) - { - log_fatal("Failed to find program git in PATH"); - return false; - } - - program = run_program(git, "rev-parse", "--show-toplevel", NULL); - - if (program.returnCode != 0) - { - (void) log_program_output(&program, LOG_INFO, LOG_ERROR); - free_program(&program); - return false; - } - else - { - char *outLines[BUFSIZE]; - - /* git rev-parse --show-toplevel outputs a single line */ - splitLines(program.stdOut, outLines, BUFSIZE); - strlcpy(srcDir, outLines[0], size); - - free_program(&program); - - return true; - } -} - - -/* - * start_rsync_command is used to sync our local source directory with a remote - * place on a target VM. - */ -static bool -start_rsync_command(const char *username, - const char *ip, - const char *srcDir) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - char ssh[MAXPGPATH] = { 0 }; - char essh[MAXPGPATH] = { 0 }; - char rsync[MAXPGPATH] = { 0 }; - char sourceDir[MAXPGPATH] = { 0 }; - char rsync_remote[MAXPGPATH] = { 0 }; - - if (!search_path_first("rsync", rsync, LOG_ERROR)) - { - log_fatal("Failed to find program rsync in PATH"); - return false; - } - - if (!search_path_first("ssh", ssh, LOG_ERROR)) - { - log_fatal("Failed to find program ssh in PATH"); - return false; - } - - /* use our usual ssh options even when using it through rsync */ - sformat(essh, sizeof(essh), - "%s -o '%s' -o '%s' -o '%s'", - ssh, - "StrictHostKeyChecking=no", - "UserKnownHostsFile /dev/null", - "LogLevel=quiet"); - - /* we need the rsync remote as one string */ - sformat(rsync_remote, sizeof(rsync_remote), - "%s@%s:/home/%s/pg_auto_failover/", - username, ip, username); - - /* we need to ensure that the source directory terminates with a "/" */ - if (strcmp(strrchr(srcDir, '/'), "/") != 0) - { - sformat(sourceDir, sizeof(sourceDir), "%s/", srcDir); - } - else - { - strlcpy(sourceDir, srcDir, sizeof(sourceDir)); - } - - args[argsIndex++] = rsync; - args[argsIndex++] = "-a"; - args[argsIndex++] = "-e"; - args[argsIndex++] = essh; - args[argsIndex++] = "--exclude='.git'"; - args[argsIndex++] = "--exclude='*.o'"; - args[argsIndex++] = "--exclude='*.deps'"; - args[argsIndex++] = "--exclude='./src/bin/pg_autoctl/pg_autoctl'"; - args[argsIndex++] = sourceDir; - args[argsIndex++] = rsync_remote; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - return azure_start_command(&program); -} - - -/* - * azure_rsync_vms runs the rsync command for target VMs in parallel. - */ -static bool -azure_rsync_vms(AzureRegionResources *azRegion) -{ - int pending = 0; - pid_t pidArray[MAX_VMS_PER_REGION] = { 0 }; - - char srcDir[MAXPGPATH] = { 0 }; - - if (!azure_git_toplevel(srcDir, sizeof(srcDir))) - { - /* errors have already been logged */ - return false; - } - - log_info("Syncing local directory \"%s\" to %d Azure VMs", - srcDir, - azRegion->nodes + - azRegion->monitor + - azRegion->appNodes); - - /* index == 0 for the monitor, then 1..count for the other nodes */ - for (int index = 0; index <= azRegion->nodes; index++) - { - /* skip index 0 when we're not creating a monitor */ - if (index == 0 && !azRegion->monitor) - { - continue; - } - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = - start_rsync_command("ha-admin", - azRegion->vmArray[index].public, - srcDir); - - ++pending; - } - - /* also provision the application node VM when asked to */ - if (azRegion->appNodes > 0) - { - int index = MAX_VMS_PER_REGION - 1; - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = - start_rsync_command("ha-admin", - azRegion->vmArray[index].public, - srcDir); - - ++pending; - } - - /* now wait for the child processes to be done */ - if (dryRun) - { - appendPQExpBuffer(azureScript, "\nwait"); - } - else - { - if (!azure_wait_for_commands(pending, pidArray)) - { - log_fatal("Failed to provision all %d azure VMs, " - "see above for details", - pending); - return false; - } - } - - return true; -} - - -/* - * azure_build_pg_autoctl runs `make all` then `make install` on all the target - * VMs in parallel, using an ssh command line. - */ -static bool -azure_build_pg_autoctl(AzureRegionResources *azRegion) -{ - int pending = 0; - pid_t pidArray[MAX_VMS_PER_REGION] = { 0 }; - - char *buildCommand = - "make PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config " - "-C pg_auto_failover -s clean all " - " && " - "sudo make PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config " - "BINDIR=/usr/local/bin -C pg_auto_failover install"; - - log_info("Building pg_auto_failover from sources on %d Azure VMs", - azRegion->nodes + - azRegion->monitor + - azRegion->appNodes); - - log_info("%s", buildCommand); - - /* index == 0 for the monitor, then 1..count for the other nodes */ - for (int index = 0; index <= azRegion->nodes; index++) - { - /* skip index 0 when we're not creating a monitor */ - if (index == 0 && !azRegion->monitor) - { - continue; - } - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = - start_ssh_command("ha-admin", - azRegion->vmArray[index].public, - buildCommand); - ++pending; - } - - /* also provision the application node VM when asked to */ - if (azRegion->appNodes > 0) - { - int index = MAX_VMS_PER_REGION - 1; - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = - start_ssh_command("ha-admin", - azRegion->vmArray[index].public, - buildCommand); - ++pending; - } - - /* now wait for the child processes to be done */ - if (dryRun) - { - appendPQExpBuffer(azureScript, "\nwait"); - } - else - { - if (!azure_wait_for_commands(pending, pidArray)) - { - log_fatal("Failed to provision all %d azure VMs, " - "see above for details", - pending); - return false; - } - } - - return true; -} - - -/* - * azure_prepare_target_versions prepares the environment variables that we - * need to grasp for provisioning our target Azure VMs. We use the following - * environment variables: - * - * AZ_PG_VERSION ?= 13 - * AZ_PGAF_DEB_VERSION ?= 1.5 - * AZ_PGAF_DEB_REVISION ?= 1.5.2-1 - */ -bool -azure_prepare_target_versions(KeyVal *env) -{ - char *keywords[] = { - "AZ_PG_VERSION", "AZ_PGAF_DEB_VERSION", "AZ_PGAF_DEB_REVISION" - }; - - /* set our static set of 3 variables from the environment */ - env->count = 3; - - /* default values */ - sformat(env->values[0], MAXCONNINFO, "13"); /* AZ_PG_VERSION */ - sformat(env->values[1], MAXCONNINFO, "2.2"); /* AZ_PGAF_DEB_VERSION */ - sformat(env->values[2], MAXCONNINFO, "2.2-1"); /* AZ_PGAF_DEB_REVISION */ - - for (int i = 0; i < 3; i++) - { - /* install the environment variable name as the keyword */ - strlcpy(env->keywords[i], keywords[i], MAXCONNINFO); - - /* pick values from the environment when they exist */ - if (env_exists(env->keywords[i])) - { - if (!get_env_copy(env->keywords[i], env->values[i], MAXCONNINFO)) - { - /* errors have already been logged */ - return false; - } - } - } - - return true; -} - - -/* - * azure_prepare_debian_command prepares the debian command to install our - * target pg_auto_failover package on the Azure VMs. - * - * sudo apt-get install -q -y \ - * postgresql-13-auto-failover-1.5=1.5.2-1 \ - * pg-auto-failover-cli-1.5=1.5.2-1 - * - * We are using environment variables to fill in the actual version numbers, - * and we hard-code some defaults in case the environment has not be provided - * for. - */ -static bool -azure_prepare_debian_install_command(char *command, size_t size) -{ - /* re-use our generic data structure from Postgres URI parsing */ - KeyVal env = { 0 }; - - if (!azure_prepare_target_versions(&env)) - { - /* errors have already been logged */ - return false; - } - - sformat(command, size, - "sudo apt-get install -q -y " - " postgresql-%s-auto-failover-%s=%s" - " pg-auto-failover-cli-%s=%s", - env.values[0], /* AZ_PG_VERSION */ - env.values[1], /* AZ_PGAF_DEB_VERSION */ - env.values[2], /* AZ_PGAF_DEB_REVISION */ - env.values[1], /* AZ_PGAF_DEB_VERSION */ - env.values[2]); /* AZ_PGAF_DEB_REVISION */ - - return true; -} - - -/* - * azure_prepare_debian_install_postgres_command prepares the debian command to - * install our target Postgres version when building from sources. - * - * sudo apt-get build-dep -q -y postgresql-16 - */ -static bool -azure_prepare_debian_install_postgres_command(char *command, size_t size) -{ - /* re-use our generic data structure from Postgres URI parsing */ - KeyVal env = { 0 }; - - if (!azure_prepare_target_versions(&env)) - { - /* errors have already been logged */ - return false; - } - - sformat(command, size, - "sudo apt-get build-dep -q -y postgresql-%s", - - /* AZ_PG_VERSION */ - env.values[0]); - - return true; -} - - -/* - * azure_prepare_debian_build_dep_postgres_command_command prepares the debian - * command to install our target Postgres version when building from sources. - * - * As we don't have deb-src for pg_auto_failover packages, we do the list - * manually, and we add also rsync to be able to push sources from the local - * git repository. - * - * sudo apt-get install -q -y \ - * postgresql-server-dev-all libkrb5-dev postgresql-16 rsync - */ -static bool -azure_prepare_debian_build_dep_postgres_command(char *command, size_t size) -{ - /* re-use our generic data structure from Postgres URI parsing */ - KeyVal env = { 0 }; - - if (!azure_prepare_target_versions(&env)) - { - /* errors have already been logged */ - return false; - } - - sformat(command, size, - "sudo apt-get install -q -y " - "postgresql-server-dev-all " - "postgresql-%s " - "libkrb5-dev " - "rsync ", - - /* AZ_PG_VERSION */ - env.values[0]); - - return true; -} - - -/* - * azure_provision_vm runs the command `az vm run-command invoke` with our - * provisioning script. - */ -bool -azure_provision_vm(const char *group, const char *name, bool fromSource) -{ - char *args[26]; - int argsIndex = 0; - - Program program = { 0 }; - - char aptGetInstall[BUFSIZE] = { 0 }; - char aptGetInstallPostgres[BUFSIZE] = { 0 }; - char aptGetBuildDepPostgres[BUFSIZE] = { 0 }; - - const char *scriptsFromPackage[] = - { - "curl https://install.citusdata.com/community/deb.sh | sudo bash", - "sudo apt-get install -q -y postgresql-common", - "echo 'create_main_cluster = false' " - "| sudo tee -a /etc/postgresql-common/createcluster.conf", - aptGetInstall, - "sudo usermod -a -G postgres ha-admin", - NULL - }; - - const char *scriptsFromSource[] = - { - "curl https://install.citusdata.com/community/deb.sh | sudo bash", - "sudo apt-get install -q -y postgresql-common", - "echo 'create_main_cluster = false' " - "| sudo tee -a /etc/postgresql-common/createcluster.conf", - aptGetInstallPostgres, - aptGetBuildDepPostgres, - "sudo usermod -a -G postgres ha-admin", - NULL - }; - - char **scripts = - fromSource ? (char **) scriptsFromSource : (char **) scriptsFromPackage; - - char *quotedScripts[10][BUFSIZE] = { 0 }; - - if (!azure_prepare_debian_install_command(aptGetInstall, BUFSIZE)) - { - /* errors have already been logged */ - return false; - } - - if (!azure_prepare_debian_install_postgres_command(aptGetInstallPostgres, - BUFSIZE)) - { - /* errors have already been logged */ - return false; - } - - if (!azure_prepare_debian_build_dep_postgres_command(aptGetBuildDepPostgres, - BUFSIZE)) - { - /* errors have already been logged */ - return false; - } - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "vm"; - args[argsIndex++] = "run-command"; - args[argsIndex++] = "invoke"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--name"; - args[argsIndex++] = (char *) name; - args[argsIndex++] = "--command-id"; - args[argsIndex++] = "RunShellScript"; - args[argsIndex++] = "--scripts"; - - if (dryRun) - { - for (int i = 0; scripts[i] != NULL; i++) - { - sformat((char *) quotedScripts[i], BUFSIZE, "\"%s\"", scripts[i]); - args[argsIndex++] = (char *) quotedScripts[i]; - } - } - else - { - for (int i = 0; scripts[i] != NULL; i++) - { - args[argsIndex++] = (char *) scripts[i]; - } - } - - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - log_info("Provisioning Virtual Machine \"%s\"", name); - - return azure_start_command(&program); -} - - -/* - * azure_provision_vms provisions several azure virtual machine in parallel and - * waits until all the commands have finished. - */ -bool -azure_provision_vms(AzureRegionResources *azRegion, bool fromSource) -{ - int pending = 0; - pid_t pidArray[MAX_VMS_PER_REGION] = { 0 }; - - char aptGetInstall[BUFSIZE] = { 0 }; - - /* we read from left to right, have the smaller number on the left */ - if (26 < azRegion->nodes) - { - log_error("pg_autoctl only supports up to 26 VMs per region"); - return false; - } - - log_info("Provisioning %d Virtual Machines in parallel", - azRegion->nodes + - azRegion->monitor + - azRegion->appNodes); - - if (!azure_prepare_debian_install_command(aptGetInstall, BUFSIZE)) - { - /* errors have already been logged */ - return false; - } - - log_info("Using: %s", aptGetInstall); - - /* index == 0 for the monitor, then 1..count for the other nodes */ - for (int index = 0; index <= azRegion->nodes; index++) - { - /* skip index 0 when we're not creating a monitor */ - if (index == 0 && azRegion->monitor == 0) - { - continue; - } - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = azure_provision_vm(azRegion->group, - azRegion->vmArray[index].name, - fromSource); - - ++pending; - } - - /* also provision the application node VM when asked to */ - if (azRegion->appNodes > 0) - { - int index = MAX_VMS_PER_REGION - 1; - - (void) azure_prepare_node(azRegion, index); - - pidArray[index] = azure_provision_vm(azRegion->group, - azRegion->vmArray[index].name, - fromSource); - ++pending; - } - - /* now wait for the child processes to be done */ - if (dryRun) - { - appendPQExpBuffer(azureScript, "\nwait"); - } - else - { - if (!azure_wait_for_commands(pending, pidArray)) - { - log_fatal("Failed to provision all %d azure VMs, " - "see above for details", - pending); - return false; - } - } - - return true; -} - - -/* - * azure_resource_list runs the command azure resource list. - * - * az resource list --output table --query "[?resourceGroup=='ha-demo-dim-paris'].{ name: name, flavor: kind, resourceType: type, region: location }" - */ -bool -azure_resource_list(const char *group) -{ - char *args[16]; - int argsIndex = 0; - bool success = true; - - Program program = { 0 }; - - char query[BUFSIZE] = { 0 }; - - char command[BUFSIZE] = { 0 }; - - sformat(query, BUFSIZE, - "[?resourceGroup=='%s']" - ".{ name: name, flavor: kind, resourceType: type, region: location }", - group); - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "resource"; - args[argsIndex++] = "list"; - args[argsIndex++] = "--output"; - args[argsIndex++] = "table"; - args[argsIndex++] = "--query"; - args[argsIndex++] = (char *) query; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - (void) snprintf_program_command_line(&program, command, sizeof(command)); - - log_info("%s", command); - - (void) execute_subprogram(&program); - success = program.returnCode == 0; - - if (success) - { - fformat(stdout, "%s", program.stdOut); - } - else - { - (void) log_program_output(&program, LOG_INFO, LOG_ERROR); - } - free_program(&program); - - return success; -} - - -/* - * azure_fetch_resource_list fetches existing resource names for a short list - * of known objects in a target azure resource group. - */ -static bool -azure_fetch_resource_list(const char *group, AzureRegionResources *azRegion) -{ - char *args[16]; - int argsIndex = 0; - bool success = true; - - Program program = { 0 }; - - char query[BUFSIZE] = { 0 }; - - char command[BUFSIZE] = { 0 }; - - sformat(query, BUFSIZE, - "[?resourceGroup=='%s'].{ name: name, resourceType: type }", - group); - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "resource"; - args[argsIndex++] = "list"; - args[argsIndex++] = "--output"; - args[argsIndex++] = "json"; - args[argsIndex++] = "--query"; - args[argsIndex++] = (char *) query; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - (void) snprintf_program_command_line(&program, command, sizeof(command)); - - log_info("Fetching resources that might already exist from a previous run"); - log_info("%s", command); - - (void) execute_subprogram(&program); - success = program.returnCode == 0; - - if (success) - { - /* parson insists on having fresh heap allocated memory, apparently */ - char *jsonString = strdup(program.stdOut); - JSON_Value *js = json_parse_string(jsonString); - JSON_Array *jsArray = json_value_get_array(js); - int count = json_array_get_count(jsArray); - - if (js == NULL) - { - log_error("Failed to parse JSON string: %s", program.stdOut); - return false; - } - - log_info("Found %d Azure resources already created in group \"%s\"", - count, group); - - for (int index = 0; index < count; index++) - { - JSON_Object *jsObj = json_array_get_object(jsArray, index); - - char *name = (char *) json_object_get_string(jsObj, "name"); - char *type = (char *) json_object_get_string(jsObj, "resourceType"); - - if (streq(type, "Microsoft.Network/virtualNetworks")) - { - strlcpy(azRegion->vnet, name, sizeof(azRegion->vnet)); - - log_info("Found existing vnet \"%s\"", azRegion->vnet); - } - else if (streq(type, "Microsoft.Network/networkSecurityGroups")) - { - strlcpy(azRegion->nsg, name, sizeof(azRegion->nsg)); - - log_info("Found existing nsg \"%s\"", azRegion->nsg); - } - else if (streq(type, "Microsoft.Compute/virtualMachines")) - { - int idx = azure_node_index_from_name(group, name); - - strlcpy(azRegion->vmArray[idx].name, name, NAMEDATALEN); - - log_info("Found existing VM \"%s\"", name); - } - else - { - /* ignore the resource Type listed */ - log_debug("Unknown resource type: \"%s\" with name \"%s\"", - type, name); - } - } - - free(jsonString); - } - else - { - (void) log_program_output(&program, LOG_INFO, LOG_ERROR); - } - free_program(&program); - - return success; -} - - -/* - * azure_show_ip_addresses shows public and private IP addresses for our list - * of nodes created in a specific resource group. - * - * az vm list-ip-addresses -g ha-demo-dim-paris --query '[] [] . { name: virtualMachine.name, "public address": virtualMachine.network.publicIpAddresses[0].ipAddress, "private address": virtualMachine.network.privateIpAddresses[0] }' -o table - */ -bool -azure_show_ip_addresses(const char *group) -{ - char *args[16]; - int argsIndex = 0; - bool success = true; - - Program program = { 0 }; - - char query[BUFSIZE] = { 0 }; - - char command[BUFSIZE] = { 0 }; - - sformat(query, BUFSIZE, - "[] [] . { name: virtualMachine.name, " - "\"public address\": " - "virtualMachine.network.publicIpAddresses[0].ipAddress, " - "\"private address\": " - "virtualMachine.network.privateIpAddresses[0] }"); - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "vm"; - args[argsIndex++] = "list-ip-addresses"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--query"; - args[argsIndex++] = (char *) query; - args[argsIndex++] = "-o"; - args[argsIndex++] = "table"; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - (void) snprintf_program_command_line(&program, command, sizeof(command)); - - log_info("%s", command); - - (void) execute_subprogram(&program); - success = program.returnCode == 0; - - if (success) - { - fformat(stdout, "%s", program.stdOut); - } - else - { - (void) log_program_output(&program, LOG_INFO, LOG_ERROR); - } - free_program(&program); - - return success; -} - - -/* - * azure_fetch_ip_addresses fetches IP address (both public and private) for - * VMs created in an Azure resource group, and fill-in the given array. - */ -bool -azure_fetch_ip_addresses(const char *group, AzureVMipAddresses *vmArray) -{ - char *args[16]; - int argsIndex = 0; - bool success = true; - - Program program = { 0 }; - - char query[BUFSIZE] = { 0 }; - - char command[BUFSIZE] = { 0 }; - - sformat(query, BUFSIZE, - "[] [] . { name: virtualMachine.name, " - "\"public address\": " - "virtualMachine.network.publicIpAddresses[0].ipAddress, " - "\"private address\": " - "virtualMachine.network.privateIpAddresses[0] }"); - - args[argsIndex++] = azureCLI; - args[argsIndex++] = "vm"; - args[argsIndex++] = "list-ip-addresses"; - args[argsIndex++] = "--resource-group"; - args[argsIndex++] = (char *) group; - args[argsIndex++] = "--query"; - args[argsIndex++] = (char *) query; - args[argsIndex++] = "-o"; - args[argsIndex++] = "json"; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - (void) snprintf_program_command_line(&program, command, sizeof(command)); - - if (dryRun) - { - appendPQExpBuffer(azureScript, "\n%s", command); - - return true; - } - - log_info("%s", command); - - (void) execute_subprogram(&program); - success = program.returnCode == 0; - - if (success) - { - JSON_Value *js = json_parse_string(program.stdOut); - JSON_Array *jsArray = json_value_get_array(js); - int count = json_array_get_count(jsArray); - - for (int index = 0; index < count; index++) - { - JSON_Object *jsObj = json_array_get_object(jsArray, index); - char *str = NULL; - int vmIndex = -1; - - str = (char *) json_object_get_string(jsObj, "name"); - - vmIndex = azure_node_index_from_name(group, str); - - if (vmIndex == -1) - { - /* errors have already been logged */ - return false; - } - - strlcpy(vmArray[vmIndex].name, str, NAMEDATALEN); - - str = (char *) json_object_get_string(jsObj, "private address"); - strlcpy(vmArray[vmIndex].private, str, BUFSIZE); - - str = (char *) json_object_get_string(jsObj, "public address"); - strlcpy(vmArray[vmIndex].public, str, BUFSIZE); - - log_debug( - "Parsed VM %d as \"%s\" with public IP %s and private IP %s", - vmIndex, - vmArray[vmIndex].name, - vmArray[vmIndex].public, - vmArray[vmIndex].private); - } - } - else - { - (void) log_program_output(&program, LOG_INFO, LOG_ERROR); - } - free_program(&program); - - return success; -} - - -/* - * run_ssh runs the ssh command to the specified IP address as the given - * username, sharing the current terminal tty. - */ -static bool -run_ssh(const char *username, const char *ip) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - char ssh[MAXPGPATH] = { 0 }; - char command[BUFSIZE] = { 0 }; - - if (!search_path_first("ssh", ssh, LOG_ERROR)) - { - log_fatal("Failed to find program ssh in PATH"); - return false; - } - - args[argsIndex++] = ssh; - args[argsIndex++] = "-o"; - args[argsIndex++] = "StrictHostKeyChecking=no"; - args[argsIndex++] = "-o"; - args[argsIndex++] = "UserKnownHostsFile /dev/null"; - args[argsIndex++] = "-o"; - args[argsIndex++] = "LogLevel=quiet"; - args[argsIndex++] = "-l"; - args[argsIndex++] = (char *) username; - args[argsIndex++] = (char *) ip; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - program.capture = false; /* don't capture output */ - program.tty = true; /* allow sharing the parent's tty */ - - (void) snprintf_program_command_line(&program, command, sizeof(command)); - - log_info("%s", command); - - (void) execute_subprogram(&program); - - return true; -} - - -/* - * run_ssh_command runs the given command on the remote machine given by ip - * address, as the given username. - */ -static bool -run_ssh_command(const char *username, - const char *ip, - bool tty, - const char *command) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - char ssh[MAXPGPATH] = { 0 }; - char ssh_command[BUFSIZE] = { 0 }; - - if (!search_path_first("ssh", ssh, LOG_ERROR)) - { - log_fatal("Failed to find program ssh in PATH"); - return false; - } - - args[argsIndex++] = ssh; - - if (tty) - { - args[argsIndex++] = "-t"; - } - - args[argsIndex++] = "-o"; - args[argsIndex++] = "StrictHostKeyChecking=no"; - args[argsIndex++] = "-o"; - args[argsIndex++] = "UserKnownHostsFile /dev/null"; - args[argsIndex++] = "-o"; - args[argsIndex++] = "LogLevel=quiet"; - args[argsIndex++] = "-l"; - args[argsIndex++] = (char *) username; - args[argsIndex++] = (char *) ip; - args[argsIndex++] = "--"; - args[argsIndex++] = (char *) command; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - program.capture = false; /* don't capture output */ - program.tty = true; /* allow sharing the parent's tty */ - - (void) snprintf_program_command_line(&program, ssh_command, BUFSIZE); - - if (dryRun) - { - appendPQExpBuffer(azureScript, "\n%s", ssh_command); - - return true; - } - - log_info("%s", ssh_command); - - (void) execute_subprogram(&program); - - return true; -} - - -/* - * start_ssh_command starts the given command on the remote machine given by ip - * address, as the given username. - */ -static bool -start_ssh_command(const char *username, - const char *ip, - const char *command) -{ - char *args[16]; - int argsIndex = 0; - - Program program = { 0 }; - - char ssh[MAXPGPATH] = { 0 }; - char ssh_command[BUFSIZE] = { 0 }; - - if (!search_path_first("ssh", ssh, LOG_ERROR)) - { - log_fatal("Failed to find program ssh in PATH"); - return false; - } - - args[argsIndex++] = ssh; - args[argsIndex++] = "-o"; - args[argsIndex++] = "StrictHostKeyChecking=no"; - args[argsIndex++] = "-o"; - args[argsIndex++] = "UserKnownHostsFile /dev/null"; - args[argsIndex++] = "-o"; - args[argsIndex++] = "LogLevel=quiet"; - args[argsIndex++] = "-l"; - args[argsIndex++] = (char *) username; - args[argsIndex++] = (char *) ip; - args[argsIndex++] = "--"; - args[argsIndex++] = (char *) command; - args[argsIndex++] = NULL; - - (void) initialize_program(&program, args, false); - - (void) snprintf_program_command_line(&program, ssh_command, BUFSIZE); - - if (dryRun) - { - appendPQExpBuffer(azureScript, "\n%s", ssh_command); - - return true; - } - - return azure_start_command(&program); -} - - -/* - * azure_fetch_vm_addresses fetches a given VM addresses. - */ -static bool -azure_fetch_vm_addresses(const char *group, const char *vm, - AzureVMipAddresses *addresses) -{ - char groupName[BUFSIZE] = { 0 }; - char vmName[BUFSIZE] = { 0 }; - int vmIndex = -1; - - AzureVMipAddresses vmAddresses[MAX_VMS_PER_REGION] = { 0 }; - - /* if the vmName is already complete, just use it already */ - if (strstr(vm, group) == NULL) - { - sformat(vmName, sizeof(vmName), "%s-%s", group, vm); - } - else - { - sformat(vmName, sizeof(vmName), "%s", vm); - } - - vmIndex = azure_node_index_from_name(group, vmName); - - if (vmIndex == -1) - { - /* errors have already been logged */ - return false; - } - - /* - * It takes as much time fetching all the IP addresses at once compared to - * fetching a single IP address, so we always fetch them all internally. - */ - if (!azure_fetch_ip_addresses(group, vmAddresses)) - { - /* errors have already been logged */ - return false; - } - - if (IS_EMPTY_STRING_BUFFER(vmAddresses[vmIndex].name)) - { - log_error( - "Failed to find Virtual Machine \"%s\" in resource group \"%s\"", - vmName, groupName); - return false; - } - - /* copy the structure wholesale to the target address */ - *addresses = vmAddresses[vmIndex]; - - return true; -} - - -/* - * azure_vm_ssh runs an ssh command to the given VM public IP address. - */ -bool -azure_vm_ssh(const char *group, const char *vm) -{ - AzureVMipAddresses addresses = { 0 }; - - if (!azure_fetch_vm_addresses(group, vm, &addresses)) - { - /* errors have already been logged */ - return false; - } - - return run_ssh("ha-admin", addresses.public); -} - - -/* - * azure_vm_ssh runs an ssh command to the given VM public IP address. - */ -bool -azure_vm_ssh_command(const char *group, - const char *vm, - bool tty, - const char *command) -{ - AzureVMipAddresses addresses = { 0 }; - - if (!azure_fetch_vm_addresses(group, vm, &addresses)) - { - /* errors have already been logged */ - return false; - } - - return run_ssh_command("ha-admin", addresses.public, tty, command); -} - - -/* - * azure_create_region creates a region on Azure and prepares it for - * pg_auto_failover demo/QA activities. - * - * We need to create a vnet, a subnet, a network security group with a rule - * that opens ports 22 (ssh) and 5432 (Postgres) for direct access from the - * current IP address of the "client" machine where this pg_autoctl command is - * being run. - */ -bool -azure_create_region(AzureRegionResources *azRegion) -{ - AzureRegionResources azRegionFound = { 0 }; - - /* - * Fetch Azure objects that might have already been created in the target - * resource group, we're going to re-use them, allowing the command to be - * run several times in a row and then "fix itself", or at least continue - * from where it failed. - */ - if (!dryRun) - { - if (!azure_fetch_resource_list(azRegion->group, &azRegionFound)) - { - /* errors have already been logged */ - return false; - } - } - - /* - * First create the resource group in the target location. - */ - if (!azure_create_group(azRegion->group, azRegion->location)) - { - /* errors have already been logged */ - return false; - } - - /* never skip a step when --script is used */ - if (dryRun || IS_EMPTY_STRING_BUFFER(azRegionFound.vnet)) - { - if (!azure_create_vnet(azRegion->group, - azRegion->vnet, - azRegion->vnetPrefix)) - { - /* errors have already been logged */ - return false; - } - } - else - { - log_info("Skipping creation of vnet \"%s\" which already exist", - azRegion->vnet); - } - - /* - * Get our IP address as seen by the outside world. - */ - if (!azure_get_remote_ip(azRegion->ipAddress, sizeof(azRegion->ipAddress))) - { - /* errors have already been logged */ - return false; - } - - /* - * Create the network security group. - */ - if (dryRun || IS_EMPTY_STRING_BUFFER(azRegionFound.nsg)) - { - if (!azure_create_nsg(azRegion->group, azRegion->nsg)) - { - /* errors have already been logged */ - return false; - } - } - else - { - log_info("Skipping creation of nsg \"%s\" which already exist", - azRegion->nsg); - } - - /* - * Create the network security rules for SSH and Postgres protocols. - * - * Some objects won't show up in the list from azure_fetch_resource_list - * and it would be quite surprising that we find everything but those, so - * we skip their creation even though we don't see them in azRegionFound. - */ - if (dryRun || IS_EMPTY_STRING_BUFFER(azRegionFound.nsg)) - { - if (!azure_create_nsg_rule(azRegion->group, - azRegion->nsg, - azRegion->rule, - azRegion->ipAddress)) - { - /* errors have already been logged */ - return false; - } - } - else - { - log_info("Skipping creation of nsg rule \"%s\", " - "because nsg \"%s\" already exists", - azRegion->rule, - azRegion->nsg); - } - - /* - * Create the network subnet using previous network security group. - */ - if (dryRun || IS_EMPTY_STRING_BUFFER(azRegionFound.vnet)) - { - if (!azure_create_subnet(azRegion->group, - azRegion->vnet, - azRegion->subnet, - azRegion->subnetPrefix, - azRegion->nsg)) - { - /* errors have already been logged */ - return false; - } - } - else - { - log_info("Skipping creation of subnet \"%s\" for prefix \"%s\", " - "because vnet \"%s\" already exists", - azRegion->subnet, - azRegion->subnetPrefix, - azRegion->vnet); - } - - /* - * Now is time to create the virtual machines. - */ - return azure_provision_nodes(azRegion); -} - - -/* - * azure_drop_region runs the command az group delete --name ... --yes - */ -bool -azure_drop_region(AzureRegionResources *azRegion) -{ - return az_group_delete(azRegion->group); -} - - -/* - * azure_provision_nodes creates the pg_autoctl VM nodes that we need, and - * provision them with our provisioning script. - */ -bool -azure_provision_nodes(AzureRegionResources *azRegion) -{ - if (!azure_fetch_ip_addresses(azRegion->group, azRegion->vmArray)) - { - /* errors have already been logged */ - return false; - } - - if (azRegion->monitor > 0 || azRegion->nodes > 0) - { - /* - * Here we run the following commands: - * - * $ az vm create --name a & - * $ az vm create --name b & - * $ wait - * - * $ az vm run-command invoke --name a --scripts ... & - * $ az vm run-command invoke --name b --scripts ... & - * $ wait - * - * We could optimize our code so that we run the provisioning scripts - * for a VM as soon as it's been created, without having to wait until - * the other VMs are created. Two things to keep in mind, though: - * - * - overall, being cleverer here might not be a win as we're going to - * have to wait until all the VMs are provisioned anyway - * - * - in dry-run mode (--script), we still want to produce the more - * naive script as shown above, for lack of known advanced control - * structures in the target shell (we don't require a specific one). - */ - if (!azure_create_vms(azRegion, "debian", "ha-admin")) - { - /* errors have already been logged */ - return false; - } - - if (!azure_provision_vms(azRegion, azRegion->fromSource)) - { - /* errors have already been logged */ - return false; - } - - /* - * When provisioning from sources, after the OS related steps in - * azure_provision_vms, we still need to upload our local sources (this - * requires rsync to have been installed in the previous step), and to - * build our software from same sources. - */ - if (azRegion->fromSource) - { - if (!azure_rsync_vms(azRegion)) - { - /* errors have already been logged */ - return false; - } - - return azure_build_pg_autoctl(azRegion); - } - } - - return true; -} - - -/* - * azure_deploy_monitor deploys pg_autoctl on a monitor node, running both the - * pg_autoctl create monitor command and then the systemd integration commands. - */ -bool -azure_deploy_monitor(AzureRegionResources *azRegion) -{ - KeyVal env = { 0 }; - char create_monitor[BUFSIZE] = { 0 }; - - char *systemd = - "pg_autoctl -q show systemd --pgdata /home/ha-admin/monitor " - "> pgautofailover.service; " - "sudo mv pgautofailover.service /etc/systemd/system; " - "sudo systemctl daemon-reload; " - "sudo systemctl enable pgautofailover; " - "sudo systemctl start pgautofailover"; - - bool tty = false; - char *host = azRegion->vmArray[0].public; - - if (!azure_prepare_target_versions(&env)) - { - /* errors have already been logged */ - return false; - } - - /* build pg_autoctl create monitor command with target Postgres version */ - sformat(create_monitor, sizeof(create_monitor), - "pg_autoctl create monitor " - "--auth trust " - "--ssl-self-signed " - "--pgdata /home/ha-admin/monitor " - "--pgctl /usr/lib/postgresql/%s/bin/pg_ctl", - - /* AZ_PG_VERSION */ - env.values[0]); - - if (azRegion->monitor == 0) - { - /* no monitor to deploy, we're done already */ - return true; - } - - /* the monitor is always at index 0 in the vmArray */ - if (!run_ssh_command("ha-admin", host, tty, create_monitor)) - { - /* errors have already been logged */ - return false; - } - - if (!run_ssh_command("ha-admin", host, tty, systemd)) - { - /* errors have already been logged */ - return false; - } - - return true; -} - - -/* - * azure_deploy_postgres deploys pg_autoctl on a Postgres node, running both - * the pg_autoctl create postgres command and then the systemd integration - * commands. - */ -bool -azure_deploy_postgres(AzureRegionResources *azRegion, int vmIndex) -{ - KeyVal env = { 0 }; - char create_postgres[BUFSIZE] = { 0 }; - - char *systemd = - "pg_autoctl -q show systemd --pgdata /home/ha-admin/pgdata " - "> pgautofailover.service; " - "sudo mv pgautofailover.service /etc/systemd/system; " - "sudo systemctl daemon-reload; " - "sudo systemctl enable pgautofailover; " - "sudo systemctl start pgautofailover"; - - bool tty = false; - char *host = azRegion->vmArray[vmIndex].public; - - if (!azure_prepare_target_versions(&env)) - { - /* errors have already been logged */ - return false; - } - - /* build pg_autoctl create monitor command with target Postgres version */ - sformat(create_postgres, sizeof(create_postgres), - "pg_autoctl create postgres " - "--pgctl /usr/lib/postgresql/%s/bin/pg_ctl " - "--pgdata /home/ha-admin/pgdata " - "--auth trust " - "--ssl-self-signed " - "--username ha-admin " - "--dbname appdb " - "--hostname %s " - "--name %s-%c " - "--monitor " - "'postgres://autoctl_node@%s/pg_auto_failover?sslmode=require'", - - /* AZ_PG_VERSION */ - env.values[0], - azRegion->vmArray[vmIndex].private, - azRegion->region, - 'a' + vmIndex - 1, - azRegion->vmArray[0].private); - - /* the monitor is always at index 0 in the vmArray */ - if (!run_ssh_command("ha-admin", host, tty, create_postgres)) - { - /* errors have already been logged */ - return false; - } - - if (!run_ssh_command("ha-admin", host, tty, systemd)) - { - /* errors have already been logged */ - return false; - } - - return true; -} - - -/* - * azure_create_nodes run the pg_autoctl commands that create our nodes, and - * then register them with systemd on the remote VMs. - */ -bool -azure_create_nodes(AzureRegionResources *azRegion) -{ - bool success = true; - - if (!azure_fetch_ip_addresses(azRegion->group, azRegion->vmArray)) - { - /* errors have already been logged */ - return false; - } - - if (azRegion->monitor > 0) - { - success = success && azure_deploy_monitor(azRegion); - } - - /* - * Now prepare all the other nodes, one at a time, so that we have a the - * primary, etc. It could also be all at once, but one at a time is good - * for a tutorial. - */ - for (int vmIndex = 1; vmIndex <= azRegion->nodes; vmIndex++) - { - success = success && azure_deploy_postgres(azRegion, vmIndex); - } - - return success; -} - - -/* - * azure_deploy_vm deploys a vm given by name ("monitor", "a", ...). - */ -bool -azure_deploy_vm(AzureRegionResources *azRegion, const char *vmName) -{ - int vmIndex = -1; - - if (!azure_fetch_ip_addresses(azRegion->group, azRegion->vmArray)) - { - /* errors have already been logged */ - return false; - } - - vmIndex = azure_node_index_from_name(azRegion->group, vmName); - - switch (vmIndex) - { - case -1: - { - /* errors have already been logged */ - return false; - } - - case 0: - { - return azure_deploy_monitor(azRegion); - } - - default: - { - return azure_deploy_postgres(azRegion, vmIndex); - } - } -} - - -/* - * azure_ls lists the azure resources we created in a specific resource group. - */ -bool -azure_ls(AzureRegionResources *azRegion) -{ - return azure_resource_list(azRegion->group); -} - - -/* - * azure_show_ips shows the azure ip addresses for the VMs we created in a - * specific resource group. - */ -bool -azure_show_ips(AzureRegionResources *azRegion) -{ - return azure_show_ip_addresses(azRegion->group); -} - - -/* - * azure_ssh runs the ssh -l ha-admin command for given - * node in given azure group, identified as usual with a prefix and a name. - */ -bool -azure_ssh(AzureRegionResources *azRegion, const char *vm) -{ - /* return azure_vm_ssh_command(groupName, vm, true, "watch date -R"); */ - return azure_vm_ssh(azRegion->group, vm); -} - - -/* - * azure_ssh_command runs the ssh -l ha-admin for - * given node in given azure group, identified as usual with a prefix and a - * name. - */ -bool -azure_ssh_command(AzureRegionResources *azRegion, - const char *vm, bool tty, const char *command) -{ - return azure_vm_ssh_command(azRegion->group, vm, tty, command); -} - - -/* - * azure_sync_source_dir runs rsync in parallel to all the created VMs. - */ -bool -azure_sync_source_dir(AzureRegionResources *azRegion) -{ - if (!azure_fetch_ip_addresses(azRegion->group, azRegion->vmArray)) - { - /* errors have already been logged */ - return false; - } - - if (!azure_rsync_vms(azRegion)) - { - /* errors have already been logged */ - return false; - } - - return azure_build_pg_autoctl(azRegion); -} diff --git a/src/bin/pg_autoctl/azure.h b/src/bin/pg_autoctl/azure.h deleted file mode 100644 index 46d47200d..000000000 --- a/src/bin/pg_autoctl/azure.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - * src/bin/pg_autoctl/azure.h - * Implementation of a CLI which lets you call `az` cli commands to prepare - * a pg_auto_failover demo or QA environment. - * - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the PostgreSQL License. - * - */ - -#ifndef AZURE_H -#define AZURE_H - -#include - -#include "postgres_fe.h" -#include "pqexpbuffer.h" - -#include "defaults.h" -#include "parsing.h" - -/* global variables from azure.c */ -extern bool dryRun; -extern PQExpBuffer azureScript; -extern char azureCLI[MAXPGPATH]; - -/* command line parsing */ -typedef struct AzureOptions -{ - char prefix[NAMEDATALEN]; - char region[NAMEDATALEN]; - char location[NAMEDATALEN]; - - int nodes; - int cidr; - bool fromSource; - bool appNode; - bool monitor; - bool all; - bool watch; -} AzureOptions; - -#define MAX_VMS_PER_REGION 28 /* monitor, then pg ndoes [a-z], then app */ - -typedef struct AzureVMipAddresses -{ - char name[NAMEDATALEN]; - char public[BUFSIZE]; - char private[BUFSIZE]; -} AzureVMipAddresses; - -typedef struct AzureRegionResources -{ - char filename[MAXPGPATH]; /* on-disk configuration file path */ - char prefix[NAMEDATALEN]; /* ha-demo-dim- */ - char region[NAMEDATALEN]; /* nickname, such as paris */ - char location[NAMEDATALEN]; /* francecentral, eastus, etc */ - - char group[NAMEDATALEN]; /* ha-demo-dim-paris */ - char vnet[BUFSIZE]; /* ha-demo-dim-paris-vnet */ - char nsg[BUFSIZE]; /* ha-demo-dim-paris-nsg */ - char rule[BUFSIZE]; /* ha-demo-dim-paris-ssh-and-pg */ - char subnet[BUFSIZE]; /* ha-demo-dim-paris-subnet */ - - char vnetPrefix[BUFSIZE]; /* 10.%d.0.0/16 */ - char subnetPrefix[BUFSIZE]; /* 10.%d.%d.0/24 */ - char ipAddress[BUFSIZE]; /* our IP address as seen from the outside */ - - int monitor; /* do we want a monitor in that region? */ - int appNodes; /* application nodes count */ - int nodes; /* node count */ - - bool fromSource; - - AzureVMipAddresses vmArray[MAX_VMS_PER_REGION]; -} AzureRegionResources; - - -bool azure_psleep(int count, bool force); -bool azure_get_remote_ip(char *ipAddress, size_t ipAddressSize); - -bool azure_create_group(const char *name, const char *location); -bool azure_create_vnet(const char *group, const char *name, const char *prefix); -bool azure_create_nsg(const char *group, const char *name); - -bool azure_create_nsg_rule(const char *group, - const char *nsgName, - const char *name, - const char *prefixes); - -bool azure_create_subnet(const char *group, - const char *vnet, - const char *name, - const char *prefixes, - const char *nsg); - -bool az_group_delete(const char *group); - -bool azure_create_vm(AzureRegionResources *azRegion, - const char *name, - const char *image, - const char *username); - -bool azure_create_vms(AzureRegionResources *azRegion, - const char *image, - const char *username); - -bool azure_prepare_target_versions(KeyVal *env); -bool azure_provision_vm(const char *group, const char *name, bool fromSource); -bool azure_provision_vms(AzureRegionResources *azRegion, bool fromSource); - -bool azure_fetch_ip_addresses(const char *group, - AzureVMipAddresses *vmArray); - -bool azure_resource_list(const char *group); -bool azure_show_ip_addresses(const char *group); -bool azure_vm_ssh(const char *group, const char *vm); -bool azure_vm_ssh_command(const char *group, - const char *vm, - bool tty, - const char *command); - -bool azure_create_region(AzureRegionResources *azRegion); -bool azure_drop_region(AzureRegionResources *azRegion); -bool azure_provision_nodes(AzureRegionResources *azRegion); - -bool azure_deploy_monitor(AzureRegionResources *azRegion); -bool azure_deploy_postgres(AzureRegionResources *azRegion, int vmIndex); -bool azure_deploy_vm(AzureRegionResources *azRegion, const char *vmName); - -bool azure_create_nodes(AzureRegionResources *azRegion); - -bool azure_ls(AzureRegionResources *azRegion); -bool azure_show_ips(AzureRegionResources *azRegion); -bool azure_ssh(AzureRegionResources *azRegion, const char *vm); -bool azure_ssh_command(AzureRegionResources *azRegion, - const char *vm, bool tty, const char *command); - -bool azure_sync_source_dir(AzureRegionResources *azRegion); - -/* src/bin/pg_autoctl/cli_do_tmux_azure.c */ -bool tmux_azure_start_or_attach_session(AzureRegionResources *azRegion); -bool tmux_azure_kill_session(AzureRegionResources *azRegion); - -#endif /* AZURE_H */ diff --git a/src/bin/pg_autoctl/azure_config.c b/src/bin/pg_autoctl/azure_config.c deleted file mode 100644 index 479068f49..000000000 --- a/src/bin/pg_autoctl/azure_config.c +++ /dev/null @@ -1,209 +0,0 @@ -/* - * src/bin/pg_autoctl/azure_config.c - * Configuration file for azure QA/test environments - * - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the PostgreSQL License. - * - */ - -#include -#include -#include -#include - -#include "postgres_fe.h" - -#include "azure.h" -#include "azure_config.h" -#include "cli_root.h" -#include "config.h" -#include "defaults.h" -#include "ini_file.h" -#include "file_utils.h" -#include "log.h" -#include "string_utils.h" - -#define OPTION_AZURE_PREFIX(config) \ - make_strbuf_option_default("az", "prefix", "prefix", true, NAMEDATALEN, \ - config->prefix, "ha-demo-") - -#define OPTION_AZURE_REGION(config) \ - make_strbuf_option_default("az", "region", "region", true, NAMEDATALEN, \ - config->region, "paris") - -#define OPTION_AZURE_LOCATION(config) \ - make_strbuf_option_default("az", "location", "location", true, NAMEDATALEN, \ - config->location, "francecentral") - -#define OPTION_AZURE_MONITOR(config) \ - make_int_option_default("group", "monitor", "monitor", true, \ - &(config->monitor), 1) - -#define OPTION_AZURE_NODES(config) \ - make_int_option_default("group", "nodes", "nodes", true, \ - &(config->nodes), 2) - -#define OPTION_AZURE_APP_NODES(config) \ - make_int_option_default("group", "appNodes", NULL, true, \ - &(config->appNodes), 0) - -#define OPTION_AZURE_GROUP(config) \ - make_strbuf_option("resource", "group", NULL, false, NAMEDATALEN, \ - config->group) - -#define OPTION_AZURE_VNET(config) \ - make_strbuf_option("resource", "vnet", NULL, false, NAMEDATALEN, \ - config->vnet) - -#define OPTION_AZURE_NSG(config) \ - make_strbuf_option("resource", "nsg", NULL, false, NAMEDATALEN, \ - config->nsg) - -#define OPTION_AZURE_RULE(config) \ - make_strbuf_option("resource", "rule", NULL, false, NAMEDATALEN, \ - config->rule) - -#define OPTION_AZURE_SUBNET(config) \ - make_strbuf_option("resource", "subnet", NULL, false, NAMEDATALEN, \ - config->subnet) - - -#define SET_INI_OPTIONS_ARRAY(config) \ - { \ - OPTION_AZURE_PREFIX(config), \ - OPTION_AZURE_REGION(config), \ - OPTION_AZURE_LOCATION(config), \ - OPTION_AZURE_MONITOR(config), \ - OPTION_AZURE_NODES(config), \ - OPTION_AZURE_APP_NODES(config), \ - OPTION_AZURE_GROUP(config), \ - OPTION_AZURE_VNET(config), \ - OPTION_AZURE_NSG(config), \ - OPTION_AZURE_RULE(config), \ - OPTION_AZURE_SUBNET(config), \ - INI_OPTION_LAST \ - } - - -/* - * azure_config_read_file reads our azure configuration from an INI - * configuration file that has been previously created by our pg_autoctl do - * azure commands. - */ -bool -azure_config_read_file(AzureRegionResources *azRegion) -{ - IniOption azureOptions[] = SET_INI_OPTIONS_ARRAY(azRegion); - - log_debug("Reading azure configuration from %s", azRegion->filename); - - if (!read_ini_file(azRegion->filename, azureOptions)) - { - log_error("Failed to parse azure configuration file \"%s\"", - azRegion->filename); - return false; - } - - return true; -} - - -/* - * azure_config_write write the current azure config to given STREAM. - */ -bool -azure_config_write(FILE *stream, AzureRegionResources *azRegion) -{ - IniOption azureOptions[] = SET_INI_OPTIONS_ARRAY(azRegion); - - return write_ini_to_stream(stream, azureOptions); -} - - -/* - * azure_config_write_file writes the current values in given azRegion to the - * given filename. - */ -bool -azure_config_write_file(AzureRegionResources *azRegion) -{ - bool success = false; - FILE *fileStream = NULL; - - log_trace("azure_config_write_file \"%s\"", azRegion->filename); - - fileStream = fopen_with_umask(azRegion->filename, "w", FOPEN_FLAGS_W, 0644); - if (fileStream == NULL) - { - /* errors have already been logged */ - return false; - } - - success = azure_config_write(fileStream, azRegion); - - if (fclose(fileStream) == EOF) - { - log_error("Failed to write file \"%s\"", azRegion->filename); - return false; - } - - return success; -} - - -/* - * azure_config_prepare prepares the names we use for the different - * Azure network objects that we need: vnet, nsg, nsgrule, subnet. - */ -void -azure_config_prepare(AzureOptions *options, AzureRegionResources *azRegion) -{ - /* build the path to our configuration file on-disk */ - if (!build_xdg_path(azRegion->filename, XDG_CONFIG, ".", "azure.cfg")) - { - log_fatal("Failed to prepare azure configuration filename"); - exit(EXIT_CODE_INTERNAL_ERROR); - } - - strlcpy(azRegion->prefix, options->prefix, sizeof(azRegion->prefix)); - strlcpy(azRegion->region, options->region, sizeof(azRegion->region)); - strlcpy(azRegion->location, options->location, sizeof(azRegion->location)); - - sformat(azRegion->group, sizeof(azRegion->group), - "%s-%s", - options->prefix, - options->region); - - /* - * Prepare our Azure object names from the group objects: vnet, subnet, - * nsg, nsg rule. - */ - sformat(azRegion->vnet, sizeof(azRegion->vnet), "%s-net", azRegion->group); - sformat(azRegion->nsg, sizeof(azRegion->nsg), "%s-nsg", azRegion->group); - - sformat(azRegion->rule, sizeof(azRegion->rule), - "%s-ssh-and-pg", azRegion->group); - - sformat(azRegion->subnet, sizeof(azRegion->subnet), - "%s-subnet", azRegion->group); - - /* transform --monitor and --no-app booleans into integer counts */ - azRegion->monitor = options->monitor ? 1 : 0; - azRegion->appNodes = options->appNode ? 1 : 0; - azRegion->nodes = options->nodes; - - azRegion->fromSource = options->fromSource; - - /* - * Prepare vnet and subnet IP addresses prefixes. - */ - sformat(azRegion->vnetPrefix, sizeof(azRegion->vnetPrefix), - "10.%d.0.0/16", - options->cidr); - - sformat(azRegion->subnetPrefix, sizeof(azRegion->subnetPrefix), - "10.%d.%d.0/24", - options->cidr, - options->cidr); -} diff --git a/src/bin/pg_autoctl/azure_config.h b/src/bin/pg_autoctl/azure_config.h deleted file mode 100644 index aa2b4b13d..000000000 --- a/src/bin/pg_autoctl/azure_config.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * src/bin/pg_autoctl/azure_config.h - * Implementation of a CLI which lets you call `az` cli commands to prepare - * a pg_auto_failover demo or QA environment. - * - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the PostgreSQL License. - * - */ - -#ifndef AZURE_CONFIG_H -#define AZURE_CONFIG_H - -#include - -#include "postgres_fe.h" -#include "pqexpbuffer.h" - -#include "azure.h" -#include "defaults.h" - -bool azure_config_read_file(AzureRegionResources *azRegion); -bool azure_config_write(FILE *stream, AzureRegionResources *azRegion); -bool azure_config_write_file(AzureRegionResources *azRegion); - -void azure_config_prepare(AzureOptions *options, AzureRegionResources *azRegion); -bool azure_get_remote_ip(char *ipAddress, size_t ipAddressSize); - -#endif /* AZURE_CONFIG_H */ diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index a08b8509b..898b7bf5c 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -495,6 +495,24 @@ cli_common_keeper_getopts(int argc, char **argv, break; } + case 'W': + { + /* { "monitor-password", required_argument, NULL, 'W' } */ + strlcpy(LocalOptionConfig.monitor_password, optarg, + sizeof(LocalOptionConfig.monitor_password)); + log_trace("--monitor-password ****"); + break; + } + + case 'w': + { + /* { "replication-password", required_argument, NULL, 'w' } */ + strlcpy(LocalOptionConfig.replication_password, optarg, + sizeof(LocalOptionConfig.replication_password)); + log_trace("--replication-password ****"); + break; + } + /* * { "ssl-ca-file", required_argument, &ssl_flag, SSL_CA_FILE_FLAG } * { "ssl-crl-file", required_argument, &ssl_flag, SSL_CA_FILE_FLAG } @@ -1744,6 +1762,18 @@ cli_pg_autoctl_reload(const char *pidfile) { pid_t pid; + /* + * Ignore SIGHUP in this process before sending it to the daemon. The + * daemon's supervisor_reload_services() broadcasts SIGHUP to all its + * service children. A freshly exec'd one-shot command can share a PID + * with a recently-exited service (PID reuse), and since one-shot + * commands don't install a SIGHUP handler, the default disposition + * (terminate) would kill us before we even return. SIG_IGN is safe + * here because all callers are one-shot CLI commands that exit shortly + * after this call anyway. + */ + signal(SIGHUP, SIG_IGN); + if (read_pidfile(pidfile, &pid)) { if (pid <= 0) diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 8ef2e8643..c8d9c8a02 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -334,6 +334,8 @@ cli_create_postgres_getopts(int argc, char **argv) { "monitor", required_argument, NULL, 'm' }, { "disable-monitor", no_argument, NULL, 'M' }, { "node-id", required_argument, NULL, 'I' }, + { "monitor-password", required_argument, NULL, 'W' }, + { "replication-password", required_argument, NULL, 'w' }, { "version", no_argument, NULL, 'V' }, { "verbose", no_argument, NULL, 'v' }, { "quiet", no_argument, NULL, 'q' }, @@ -354,7 +356,7 @@ cli_create_postgres_getopts(int argc, char **argv) int optind = cli_create_node_getopts(argc, argv, long_options, - "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:RVvqhP:r:xsN", + "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:W:w:RVvqhP:r:xsN", &options); /* publish our option parsing in the global variable */ @@ -793,6 +795,8 @@ cli_create_monitor_getopts(int argc, char **argv) { "listen", required_argument, NULL, 'l' }, { "auth", required_argument, NULL, 'A' }, { "skip-pg-hba", no_argument, NULL, 'S' }, + { "autoctl-node-password", required_argument, NULL, 'W' }, + { "formation", required_argument, NULL, 'f' }, { "version", no_argument, NULL, 'V' }, { "verbose", no_argument, NULL, 'v' }, { "quiet", no_argument, NULL, 'q' }, @@ -821,7 +825,7 @@ cli_create_monitor_getopts(int argc, char **argv) optind = 0; - while ((c = getopt_long(argc, argv, "C:D:p:n:l:A:SVvqhxNs", + while ((c = getopt_long(argc, argv, "C:D:p:n:l:A:SW:f:VvqhxsN", long_options, &option_index)) != -1) { switch (c) @@ -898,6 +902,30 @@ cli_create_monitor_getopts(int argc, char **argv) break; } + case 'W': + { + strlcpy(options.autoctl_node_password, optarg, + sizeof(options.autoctl_node_password)); + log_trace("--autoctl-node-password ****"); + break; + } + + case 'f': + { + /* --formation (may be repeated) */ + if (options.formationCount >= MONITOR_MAX_FORMATIONS) + { + log_error("Too many --formation options (max %d)", + MONITOR_MAX_FORMATIONS); + errors++; + break; + } + int fi = options.formationCount++; + strlcpy(options.formationNames[fi], optarg, NAMEDATALEN); + log_trace("--formation %s", optarg); + break; + } + case 'V': { /* keeper_cli_print_version prints version and exits. */ diff --git a/src/bin/pg_autoctl/cli_do_azure.c b/src/bin/pg_autoctl/cli_do_azure.c deleted file mode 100644 index 6a218849f..000000000 --- a/src/bin/pg_autoctl/cli_do_azure.c +++ /dev/null @@ -1,627 +0,0 @@ -/* - * src/bin/pg_autoctl/cli_do_azure.c - * Implementation of a CLI which lets you call `az` cli commands to prepare - * a pg_auto_failover demo or QA environment. - * - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the PostgreSQL License. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "postgres_fe.h" -#include "pqexpbuffer.h" -#include "snprintf.h" - -#include "azure.h" -#include "azure_config.h" -#include "cli_common.h" -#include "cli_do_root.h" -#include "cli_root.h" -#include "commandline.h" -#include "config.h" -#include "env_utils.h" -#include "log.h" -#include "pidfile.h" -#include "signals.h" -#include "string_utils.h" - -#include "runprogram.h" - -static AzureOptions azOptions = { 0 }; -static AzureRegionResources azRegion = { 0 }; - -bool dryRun = false; -PQExpBuffer azureScript = NULL; - -static void outputAzureScript(void); - - -/* - * cli_print_version_getopts parses the CLI options for the pg_autoctl version - * command, which are the usual suspects. - */ -int -cli_do_azure_getopts(int argc, char **argv) -{ - int c, option_index = 0, errors = 0; - int verboseCount = 0; - bool printVersion = false; - - AzureOptions options = { 0 }; - - static struct option long_options[] = { - { "prefix", required_argument, NULL, 'p' }, - { "region", required_argument, NULL, 'r' }, - { "location", required_argument, NULL, 'l' }, - { "from-source", no_argument, NULL, 's' }, - { "nodes", required_argument, NULL, 'N' }, - { "no-monitor", no_argument, NULL, 'M' }, - { "no-app", no_argument, NULL, 'n' }, - { "all", no_argument, NULL, 'A' }, - { "script", no_argument, NULL, 'S' }, - { "watch", no_argument, NULL, 'T' }, - { "az", no_argument, NULL, 'Z' }, - { "cidr", no_argument, NULL, 'c' }, - { "version", no_argument, NULL, 'V' }, - { "verbose", no_argument, NULL, 'v' }, - { "quiet", no_argument, NULL, 'q' }, - { "help", no_argument, NULL, 'h' }, - { NULL, 0, NULL, 0 } - }; - - optind = 0; - - /* set our defaults */ - options.cidr = 11; /* 10.11.0.0/16 and 10.11.11.0/24 */ - options.nodes = 2; - options.fromSource = false; - options.appNode = true; - options.monitor = true; - options.all = false; - options.watch = false; - - strlcpy(options.prefix, "ha-demo", sizeof(options.prefix)); - - /* - * The only command lines that are using keeper_cli_getopt_pgdata are - * terminal ones: they don't accept subcommands. In that case our option - * parsing can happen in any order and we don't need getopt_long to behave - * in a POSIXLY_CORRECT way. - * - * The unsetenv() call allows getopt_long() to reorder arguments for us. - */ - unsetenv("POSIXLY_CORRECT"); - - while ((c = getopt_long(argc, argv, "p:n:l:N:MAWSTVvqh", - long_options, &option_index)) != -1) - { - switch (c) - { - /* { "prefix", required_argument, NULL, 'p' }, */ - case 'p': - { - strlcpy(options.prefix, optarg, NAMEDATALEN); - log_trace("--prefix %s", options.prefix); - break; - } - - /* { "region", required_argument, NULL, 'r' }, */ - case 'r': - { - strlcpy(options.region, optarg, NAMEDATALEN); - log_trace("--region %s", options.region); - break; - } - - /* { "location", required_argument, NULL, 'l' }, */ - case 'l': - { - strlcpy(options.location, optarg, NAMEDATALEN); - log_trace("--location %s", options.location); - break; - } - - /* { "az", no_argument, NULL, 'Z' }, */ - case 'Z': - { - strlcpy(azureCLI, optarg, NAMEDATALEN); - log_trace("--az %s", azureCLI); - break; - } - - /* { "cidr", no_argument, NULL, 'c' }, */ - case 'c': - { - if (!stringToInt(optarg, &options.cidr)) - { - log_error("Failed to parse --cidr number \"%s\"", optarg); - errors++; - } - else if (options.cidr < 1 || options.cidr > 254) - { - log_error("Failed to parse --cidr number \"%s\"", optarg); - errors++; - } - else - { - log_trace("--cidr %d", options.cidr); - } - break; - } - - /* { "nodes", required_argument, NULL, 'N' }, */ - case 'N': - { - if (!stringToInt(optarg, &options.nodes)) - { - log_error("Failed to parse --nodes number \"%s\"", optarg); - errors++; - } - log_trace("--nodes %d", options.nodes); - break; - } - - /* { "no-monitor", no_argument, NULL, 'M' }, */ - case 'M': - { - options.monitor = false; - log_trace("--no-monitor"); - break; - } - - /* { "no-app", no_argument, NULL, 'n' }, */ - case 'n': - { - options.appNode = false; - log_trace("--no-app"); - break; - } - - /* { "from-source", required_argument, NULL, 's' }, */ - case 's': - { - options.fromSource = true; - log_trace("--from-source"); - break; - } - - /* { "all", no_argument, NULL, 'A' }, */ - case 'A': - { - options.all = true; - log_trace("--monitor"); - break; - } - - /* { "script", no_argument, NULL, 'S' }, */ - case 'S': - { - dryRun = true; - log_trace("--script"); - break; - } - - /* { "watch", no_argument, NULL, 'T' }, */ - case 'T': - { - options.watch = true; - log_trace("--watch"); - break; - } - - case 'h': - { - commandline_help(stderr); - exit(EXIT_CODE_QUIT); - break; - } - - case 'V': - { - /* keeper_cli_print_version prints version and exits. */ - printVersion = true; - break; - } - - case 'v': - { - ++verboseCount; - switch (verboseCount) - { - case 1: - { - log_set_level(LOG_INFO); - break; - } - - case 2: - { - log_set_level(LOG_DEBUG); - break; - } - - default: - { - log_set_level(LOG_TRACE); - break; - } - } - break; - } - - case 'q': - { - log_set_level(LOG_ERROR); - break; - } - - default: - { - /* getopt_long already wrote an error message */ - errors++; - break; - } - } - } - - if (IS_EMPTY_STRING_BUFFER(options.prefix)) - { - ++errors; - log_fatal("--prefix is a mandatory option"); - } - - if (IS_EMPTY_STRING_BUFFER(azureCLI)) - { - if (!search_path_first("az", azureCLI, LOG_ERROR)) - { - ++errors; - log_fatal("Failed to find program \"%s\" in PATH", "az"); - } - } - else - { - if (!file_exists(azureCLI)) - { - ++errors; - log_fatal("No such file or directory: \"%s\"", azureCLI); - } - } - - if (errors > 0) - { - commandline_help(stderr); - exit(EXIT_CODE_BAD_ARGS); - } - - if (printVersion) - { - keeper_cli_print_version(argc, argv); - } - - /* - * From command line options parsing, prepare a AzureRegionResources in our - * static place. - * - * If a configuration file exists already, it takes precendence, because we - * have probably already created all the resources on Azure and deployed - * things there. - * - * If no configuration file exists already, we create one filled with the - * options given in the command line. - */ - (void) azure_config_prepare(&options, &azRegion); - - if (file_exists(azRegion.filename)) - { - log_info("Reading configuration from \"%s\"", azRegion.filename); - - if (!azure_config_read_file(&azRegion)) - { - /* errors have already been logged */ - exit(EXIT_CODE_BAD_CONFIG); - } - - /* maybe late we will merge new options in the pre-existing file */ - log_warn("Ignoring command line options, " - "configuration file takes precedence"); - - log_info("Using --prefix \"%s\" --region \"%s\" --location \"%s\"", - azRegion.prefix, - azRegion.region, - azRegion.location); - } - else - { - if (!azure_config_write_file(&azRegion)) - { - /* errors have already been logged */ - exit(EXIT_CODE_INTERNAL_ERROR); - } - } - - /* when a configuration file already exists, it provides the location */ - if (IS_EMPTY_STRING_BUFFER(azRegion.location)) - { - log_fatal("--location is a mandatory option"); - exit(EXIT_CODE_BAD_ARGS); - } - - /* - * In --script mode (or dry run) we generate a script with the commands we - * would run instead of actually running them. - */ - if (dryRun) - { - azureScript = createPQExpBuffer(); - - if (azureScript == NULL) - { - log_error("Failed to allocate memory"); - exit(EXIT_CODE_INTERNAL_ERROR); - } - - appendPQExpBuffer(azureScript, - "# azure commands for pg_auto_failover demo"); - } - - /* publish parsed options */ - azOptions = options; - - return optind; -} - - -/* - * outputAzureScript writes the azure script to stdout. - */ -static void -outputAzureScript() -{ - if (dryRun) - { - fformat(stdout, "%s\n", azureScript->data); - destroyPQExpBuffer(azureScript); - } -} - - -/* - * cli_do_azure_create_environment creates an Azure region with some nodes and - * network rules for a demo or QA context of pg_auto_failover, then provision - * those VMs with the needed software, and then create pg_auto_failover nodes - * from that, in a tmux session for interactive QA. - */ -void -cli_do_azure_create_environment(int argc, char **argv) -{ - /* - * azure_create_region creates the resources we need (VMs, network, access - * rules, etc) and then provision the VMs with the needed software. - */ - if (!azure_create_region(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } - - (void) outputAzureScript(); - - /* - * tmux_azure_start_or_attach_session then creates a tmux session with a - * shell window for each VM in the Azure resource group, and in each - * session in parallel runs the pg_autoctl create commands, and then add - * the setup to systemd. - * - * Another tmux window is created to run pg_autoctl show state in a watch - * loop. - * - * An extra window is created for interactive tinkering with the QA - * environment thus provided. - */ - if (!tmux_azure_start_or_attach_session(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_create_region creates an Azure region with some nodes and - * network rules for a demo or QA context of pg_auto_failover. - */ -void -cli_do_azure_create_region(int argc, char **argv) -{ - if (!azure_create_region(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } - - (void) outputAzureScript(); -} - - -/* - * cli_do_azure_drop_region drops the azure resource group that has been - * created to host the azure resources in use for the environment. - */ -void -cli_do_azure_drop_region(int argc, char **argv) -{ - if (!azure_drop_region(&azRegion)) - { - log_warn("Configuration file \"%s\" has not been deleted", - azRegion.filename); - exit(EXIT_CODE_INTERNAL_ERROR); - } - - log_info("Killing tmux sessions \"%s\"", azRegion.group); - tmux_azure_kill_session(&azRegion); - - log_info("Removing azure configuration file \"%s\"", azRegion.filename); - - if (!unlink_file(azRegion.filename)) - { - log_fatal("Failed to remove azure configuration file"); - exit(EXIT_CODE_INTERNAL_ERROR); - } - - (void) outputAzureScript(); -} - - -/* - * cli_do_azure_deploy deploys the pg_autoctl services in the target VM, given - * by name (such as "monitor" or "a" or "b", etc). - */ -void -cli_do_azure_deploy(int argc, char **argv) -{ - if (argc != 1) - { - (void) commandline_print_usage(&do_azure_ssh, stderr); - exit(EXIT_CODE_BAD_ARGS); - } - - if (!azure_deploy_vm(&azRegion, argv[0])) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } - - (void) outputAzureScript(); -} - - -/* - * cli_do_azure_create_nodes creates the pg_autoctl services in an Azure - * region that's been created and provisionned before. - */ -void -cli_do_azure_create_nodes(int argc, char **argv) -{ - if (!azure_create_nodes(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } - - (void) outputAzureScript(); -} - - -/* - * cli_do_azure_ls lists Azure resources created in the target region. - */ -void -cli_do_azure_ls(int argc, char **argv) -{ - if (!azure_ls(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_show_ips lists Azure ip addresses assigned to created VMs in a - * specific region. - */ -void -cli_do_azure_show_ips(int argc, char **argv) -{ - if (!azure_show_ips(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_ssh starts an ssh command to the given Azure VM in a specific - * prefix and region name. - */ -void -cli_do_azure_ssh(int argc, char **argv) -{ - if (argc != 1) - { - (void) commandline_print_usage(&do_azure_ssh, stderr); - exit(EXIT_CODE_BAD_ARGS); - } - - if (!azure_ssh(&azRegion, argv[0])) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_rsync uses rsync to upload the current sources to all the - * created VMs in the target region. - */ -void -cli_do_azure_rsync(int argc, char **argv) -{ - if (!azure_sync_source_dir(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_ssh starts an ssh command to the given Azure VM in a specific - * prefix and region name. - */ -void -cli_do_azure_show_state(int argc, char **argv) -{ - char *pg_autoctl_command = - azOptions.watch - ? "watch -n 0.2 pg_autoctl show state --pgdata ./monitor" - : "pg_autoctl show state --pgdata ./monitor"; - - if (!azure_ssh_command(&azRegion, - "monitor", - azOptions.watch, /* tty is needed for watch */ - pg_autoctl_command)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_tmux_session starts or re-attach to a tmux session from where - * to control the VMs in the QA environment on Azure. - */ -void -cli_do_azure_tmux_session(int argc, char **argv) -{ - if (!tmux_azure_start_or_attach_session(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} - - -/* - * cli_do_azure_tmux_session starts or re-attach to a tmux session from where - * to control the VMs in the QA environment on Azure. - */ -void -cli_do_azure_tmux_kill(int argc, char **argv) -{ - if (!tmux_azure_kill_session(&azRegion)) - { - exit(EXIT_CODE_INTERNAL_ERROR); - } -} diff --git a/src/bin/pg_autoctl/cli_do_coordinator.c b/src/bin/pg_autoctl/cli_do_coordinator.c index 72f5dfb55..5506fa29e 100644 --- a/src/bin/pg_autoctl/cli_do_coordinator.c +++ b/src/bin/pg_autoctl/cli_do_coordinator.c @@ -108,7 +108,7 @@ static CommandLine *coordinator_commands[] = { CommandLine do_coordinator_commands = make_command_set("coordinator", - "Query a Citus coordinator", NULL, NULL, + "Manage Citus coordinator node metadata", NULL, NULL, NULL, coordinator_commands); diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index 8c456ff73..4294aceb9 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -29,6 +29,8 @@ static void cli_do_fsm_init(int argc, char **argv); static void cli_do_fsm_state(int argc, char **argv); +static int cli_do_fsm_node_state_getopts(int argc, char **argv); +static void cli_do_fsm_node_state(int argc, char **argv); static void cli_do_fsm_list(int argc, char **argv); static void cli_do_fsm_gv(int argc, char **argv); static void cli_do_fsm_assign(int argc, char **argv); @@ -37,15 +39,8 @@ static void cli_do_fsm_step(int argc, char **argv); static void cli_do_fsm_get_nodes(int argc, char **argv); static void cli_do_fsm_set_nodes(int argc, char **argv); -static CommandLine fsm_init = - make_command("init", - "Initialize the keeper's state on-disk", - CLI_PGDATA_USAGE, - CLI_PGDATA_OPTION, - cli_getopt_pgdata, - cli_do_fsm_init); - -static CommandLine fsm_state = +/* read-only — exported for pg_autoctl inspect fsm */ +CommandLine fsm_state = make_command("state", "Read the keeper's state from disk and display it", CLI_PGDATA_USAGE, @@ -53,7 +48,17 @@ static CommandLine fsm_state = cli_getopt_pgdata, cli_do_fsm_state); -static CommandLine fsm_list = +CommandLine fsm_node_state = + make_command("node-state", + "Poll the keeper's on-disk current_role; exit 0 when it matches --state", + CLI_PGDATA_USAGE, + " --pgdata path to the keeper's data directory\n" + " --state target state to wait for (e.g. demote_timeout)\n" + " --timeout seconds to retry (default: no retry)\n", + cli_do_fsm_node_state_getopts, + cli_do_fsm_node_state); + +CommandLine fsm_list = make_command("list", "List reachable FSM states from current state", CLI_PGDATA_USAGE, @@ -61,12 +66,21 @@ static CommandLine fsm_list = cli_getopt_pgdata, cli_do_fsm_list); -static CommandLine fsm_gv = +CommandLine fsm_gv = make_command("gv", "Output the FSM as a .gv program suitable for graphviz/dot", "", NULL, NULL, cli_do_fsm_gv); -static CommandLine fsm_assign = +/* mutating — exported for pg_autoctl manual fsm */ +CommandLine fsm_init = + make_command("init", + "Initialize the keeper's state on-disk", + CLI_PGDATA_USAGE, + CLI_PGDATA_OPTION, + cli_getopt_pgdata, + cli_do_fsm_init); + +CommandLine fsm_assign = make_command("assign", "Assign a new goal state to the keeper", CLI_PGDATA_USAGE "", @@ -74,7 +88,7 @@ static CommandLine fsm_assign = cli_getopt_pgdata, cli_do_fsm_assign); -static CommandLine fsm_step = +CommandLine fsm_step = make_command("step", "Make a state transition if instructed by the monitor", CLI_PGDATA_USAGE, @@ -228,6 +242,141 @@ cli_do_fsm_state(int argc, char **argv) } +/* option state for fsm node-state */ +static struct +{ + char targetState[64]; + int timeout; +} +fsmNodeStateOpts; + +static int +cli_do_fsm_node_state_getopts(int argc, char **argv) +{ + fsmNodeStateOpts.targetState[0] = '\0'; + fsmNodeStateOpts.timeout = 0; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "state", required_argument, NULL, 's' }, + { "timeout", required_argument, NULL, 't' }, + { NULL, 0, NULL, 0 } + }; + + int c; + optind = 0; + while ((c = getopt_long(argc, argv, "D:s:t:", long_options, NULL)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(keeperOptions.pgSetup.pgdata, optarg, + sizeof(keeperOptions.pgSetup.pgdata)); + break; + } + + case 's': + { + strlcpy(fsmNodeStateOpts.targetState, optarg, + sizeof(fsmNodeStateOpts.targetState)); + break; + } + + case 't': + { + fsmNodeStateOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + default: + { + commandline_print_usage(&fsm_node_state, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + /* fall back to $PGDATA when --pgdata was not given */ + cli_common_get_set_pgdata_or_exit(&keeperOptions.pgSetup); + + if (!keeper_config_set_pathnames_from_pgdata(&keeperOptions.pathnames, + keeperOptions.pgSetup.pgdata)) + { + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +/* + * cli_do_fsm_node_state reads the keeper's on-disk state and prints: + * | + * + * When --state is given the command retries until current_role matches. + * When --timeout is given the retry loop stops after N seconds. + */ +static void +cli_do_fsm_node_state(int argc, char **argv) +{ + KeeperConfig config = keeperOptions; + + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = true; + + if (!keeper_config_read_file(&config, + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + exit(EXIT_CODE_BAD_CONFIG); + } + + time_t deadline = (fsmNodeStateOpts.timeout > 0) + ? time(NULL) + fsmNodeStateOpts.timeout + : 0; + + for (;;) + { + Keeper keeper = { 0 }; + + if (!keeper_init(&keeper, &config)) + { + if (deadline && time(NULL) < deadline) + { + pg_usleep(500 * 1000); + continue; + } + exit(EXIT_CODE_BAD_CONFIG); + } + + KeeperStateData *s = &keeper.state; + const char *current = NodeStateToString(s->current_role); + const char *assigned = NodeStateToString(s->assigned_role); + + if (fsmNodeStateOpts.targetState[0] != '\0' && + strcmp(current, fsmNodeStateOpts.targetState) != 0) + { + log_debug("fsm node-state: current_role=%s, waiting for %s", + current, fsmNodeStateOpts.targetState); + if (deadline && time(NULL) < deadline) + { + pg_usleep(500 * 1000); + continue; + } + + /* timed out or --timeout 0 with no match */ + exit(EXIT_CODE_INTERNAL_ERROR); + } + + fformat(stdout, "%s|%s\n", current, assigned); + exit(0); + } +} + + /* * cli_do_fsm_list lists reachable states from the current one. */ diff --git a/src/bin/pg_autoctl/cli_do_misc.c b/src/bin/pg_autoctl/cli_do_misc.c index 799906cb9..68e144a95 100644 --- a/src/bin/pg_autoctl/cli_do_misc.c +++ b/src/bin/pg_autoctl/cli_do_misc.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "postgres_fe.h" #include "pqexpbuffer.h" @@ -30,10 +31,17 @@ #include "monitor.h" #include "monitor_config.h" #include "pgctl.h" +#include "pghba.h" +#include "pgsetup.h" +#include "pgsql.h" #include "pgtuning.h" #include "primary_standby.h" #include "string_utils.h" +/* Options specific to "pg_autoctl inspect pgsetup wait" */ +static bool pgsetupWaitReadWrite = false; +static int pgsetupWaitTimeout = 30; + /* * keeper_cli_create_replication_slot implements the CLI to create a replication @@ -319,18 +327,149 @@ keeper_cli_pgsetup_is_ready(int argc, char **argv) /* - * keeper_cli_discover_pg_setup implements the CLI to discover a PostgreSQL - * setup thanks to PGDATA and other environment variables. + * keeper_cli_pgsetup_wait_getopts parses options specific to + * "pg_autoctl inspect pgsetup wait": --read-write, --timeout, plus the + * standard Postgres connection options inherited from the keeper setup. + */ +int +keeper_cli_pgsetup_wait_getopts(int argc, char **argv) +{ + /* + * cli_common_keeper_getopts (called by keeper_cli_keeper_setup_getopts) + * exits with BAD_ARGS when it encounters unknown options. Since --timeout + * and --read-write are not in its options list, we must remove them from + * argv before delegating. + * + * Strategy: + * 1. Scan argv with opterr=0 to capture --timeout / --read-write. + * 2. Build a filtered argv that omits those two options. + * 3. Pass the filtered argv to keeper_cli_keeper_setup_getopts. + */ + + /* Reset module-level wait options */ + pgsetupWaitReadWrite = false; + pgsetupWaitTimeout = 30; + + static struct option wait_options[] = { + { "read-write", no_argument, NULL, 'W' }, + { "timeout", required_argument, NULL, 'T' }, + { NULL, 0, NULL, 0 } + }; + + optind = 1; + opterr = 0; + + int c; + int option_index = 0; + + while ((c = getopt_long(argc, argv, "WT:", wait_options, &option_index)) != -1) + { + switch (c) + { + case 'W': + { + pgsetupWaitReadWrite = true; + break; + } + + case 'T': + { + int t = strtol(optarg, NULL, 10); + if (t <= 0) + { + log_error("--timeout must be a positive integer"); + exit(EXIT_CODE_BAD_ARGS); + } + pgsetupWaitTimeout = t; + break; + } + + default: + { + /* standard keeper options; handled by the delegated call below */ + break; + } + } + } + + opterr = 1; + + /* + * Build a filtered argv that strips --timeout/--read-write (and their + * arguments) so that keeper_cli_keeper_setup_getopts does not see them. + */ + char **filtered_argv = (char **) palloc((argc + 1) * sizeof(char *)); + int filtered_argc = 0; + + for (int i = 0; i < argc; i++) + { + if (strcmp(argv[i], "--read-write") == 0 || strcmp(argv[i], "-W") == 0) + { + continue; + } + + if ((strcmp(argv[i], "--timeout") == 0 || strcmp(argv[i], "-T") == 0) && + i + 1 < argc) + { + /* skip both the flag and its argument */ + i++; + continue; + } + + filtered_argv[filtered_argc++] = argv[i]; + } + filtered_argv[filtered_argc] = NULL; + + int rc = keeper_cli_keeper_setup_getopts(filtered_argc, filtered_argv); + + pfree(filtered_argv); + + return rc; +} + + +/* + * keeper_cli_pgsetup_wait_until_ready waits for the local Postgres server to + * become ready. When --read-write is given, it additionally waits until the + * server is accepting read-write connections (not in recovery and not set to + * default_transaction_read_only). + * + * The --timeout value (default 30s) is a single deadline shared by both + * phases: the pg_is_ready poll and the subsequent read-write connection + * attempt. Time spent waiting for Postgres to start counts against the + * budget for the read-write phase. */ void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) { - int timeout = 30; + int timeout = pgsetupWaitTimeout; ConfigFilePaths pathnames = { 0 }; LocalPostgresServer postgres = { 0 }; PostgresSetup *pgSetup = &(postgres.postgresSetup); + /* Record wall-clock start so all phases share one deadline. */ + time_t startTime = time(NULL); + + /* Wait up to `timeout` seconds for the config file to be created. + * In no-monitor mode, pg_autoctl create postgres runs first and writes the + * config; pgsetup wait may be called before that completes. */ + { + KeeperConfig kconfig = keeperOptions; + if (keeper_config_set_pathnames_from_pgdata(&(kconfig.pathnames), + kconfig.pgSetup.pgdata)) + { + time_t deadline = startTime + timeout; + while (!file_exists(kconfig.pathnames.config) && + time(NULL) < deadline) + { + log_debug("Waiting for config file \"%s\" to appear", + kconfig.pathnames.config); + pg_usleep(500 * 1000); + } + } + } + if (!cli_common_pgsetup_init(&pathnames, pgSetup)) { /* errors have already been logged */ @@ -339,15 +478,105 @@ keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) log_debug("Initialized pgSetup, now calling pg_setup_wait_until_is_ready()"); - bool pgIsReady = pg_setup_wait_until_is_ready(pgSetup, timeout, LOG_INFO); + /* + * Phase 1: wait for postmaster to signal "ready" in postmaster.pid. + * Pass the remaining timeout so the two phases together stay within the + * single user-visible deadline. + */ + int remainingAfterConfig = timeout - (int) (time(NULL) - startTime); + if (remainingAfterConfig <= 0) + { + log_error("Timed out waiting for Postgres config file to appear"); + exit(EXIT_CODE_PGSQL); + } + + bool pgIsReady = + pg_setup_wait_until_is_ready(pgSetup, remainingAfterConfig, LOG_INFO); log_info("Postgres status is: \"%s\"", pmStatusToString(pgSetup->pm_status)); - if (pgIsReady) + if (!pgIsReady) { + exit(EXIT_CODE_PGSQL); + } + + if (!pgsetupWaitReadWrite) + { + /* Plain "ready" check — we're done. */ exit(EXIT_CODE_QUIT); } - exit(EXIT_CODE_PGSQL); + + /* + * Phase 2: wait until the server accepts read-write connections. + * + * Postgres is up (phase 1 passed) but may still be in recovery, finishing + * pg_rewind, or in standby mode. We poll with a libpq connection that + * checks pg_is_in_recovery() until it returns false or the deadline fires. + * + * We use the local connection string from pgSetup (Unix socket when + * available, matching whatever auth the node was created with) so that the + * check works regardless of the cluster's auth method. + */ + char connstr[MAXCONNINFO]; + if (!pg_setup_get_local_connection_string(pgSetup, connstr)) + { + log_error("Failed to build local connection string for read-write check"); + exit(EXIT_CODE_BAD_CONFIG); + } + + log_info("Waiting for Postgres to accept read-write connections " + "(timeout %ds)", timeout); + + bool isReadWrite = false; + int attempts = 0; + + while (!isReadWrite) + { + int elapsed = (int) (time(NULL) - startTime); + int remaining = timeout - elapsed; + + if (remaining <= 0) + { + log_error("Timed out after %ds waiting for Postgres " + "to accept read-write connections", timeout); + exit(EXIT_CODE_PGSQL); + } + + /* Use a short per-attempt connect_timeout so we retry briskly. */ + char attemptConnstr[MAXCONNINFO]; + sformat(attemptConnstr, sizeof(attemptConnstr), + "%s connect_timeout=1", connstr); + + PGSQL pgsql = { 0 }; + pgsql_init(&pgsql, attemptConnstr, PGSQL_CONN_LOCAL); + + bool inRecovery = true; /* assume standby until proven otherwise */ + bool queryOk = pgsql_is_in_recovery(&pgsql, &inRecovery); + pgsql_finish(&pgsql); + + if (queryOk && !inRecovery) + { + isReadWrite = true; + break; + } + + /* let's not be THAT verbose about it */ + if (attempts % 10 == 0) + { + log_debug("pgsetup wait --read-write: attempt %d, " + "in_recovery=%s, after %ds", + attempts + 1, + inRecovery ? "true" : "false", + elapsed); + } + + ++attempts; + pg_usleep(100 * 1000); /* 100 ms between probes */ + } + + log_info("Postgres is now accepting read-write connections on port %d", + pgSetup->pgport); + exit(EXIT_CODE_QUIT); } @@ -393,6 +622,169 @@ keeper_cli_pgsetup_tune(int argc, char **argv) } +/* + * keeper_cli_pgsetup_hba_lan appends LAN CIDR trust rules to pg_hba.conf + * without connecting to Postgres (safe when TCP HBA hasn't been set up yet), + * then reloads the configuration via pg_ctl. + * + * Intended for --skip-pg-hba setups where the operator needs to seed the + * initial trust rules before pg_autoctl or replication can connect via TCP. + * + * Sequence: + * 1. Wait up to 60 s for $PGDATA/pg_hba.conf to appear. + * 2. Append "host all all trust" and + * "host replication all trust" directly to the file. + * 3. Wait for Postgres to be running (reads postmaster.pid, no TCP). + * 4. pg_ctl reload -D $PGDATA. + */ +void +keeper_cli_pgsetup_hba_lan(int argc, char **argv) +{ + /* + * Derive pgdata from command-line options directly, without calling + * cli_common_pgsetup_init(). cli_common_pgsetup_init() calls + * ProbeConfigurationFileRole() which fatals when the keeper config file + * does not exist yet (e.g. node still initializing). We do not need the + * full pgSetup here — only pgdata, hostname, auth method, and ssl. + */ + const char *pgdata = keeperOptions.pgSetup.pgdata; + + if (IS_EMPTY_STRING_BUFFER(pgdata)) + { + log_fatal("Please provide --pgdata or set PGDATA"); + exit(EXIT_CODE_BAD_ARGS); + } + + char hbaFile[MAXPGPATH]; + sformat(hbaFile, MAXPGPATH, "%s/pg_hba.conf", pgdata); + + /* Step 1: wait for pg_hba.conf to appear (up to 60 s) */ + { + int timeout = 60; + time_t deadline = time(NULL) + timeout; + while (!file_exists(hbaFile) && time(NULL) < deadline) + { + log_debug("Waiting for \"%s\" to appear", hbaFile); + pg_usleep(500 * 1000); + } + if (!file_exists(hbaFile)) + { + log_error("Timed out waiting for \"%s\" to appear", hbaFile); + exit(EXIT_CODE_PGCTL); + } + } + + /* + * Step 2: determine the hostname for LAN CIDR lookups. + * + * Preference order: + * a) keeper config file hostname (written early during pg_autoctl init) + * b) OS hostname via gethostname() — inside a Docker container this is + * the service name (e.g. "node2") which resolves correctly on the LAN + * c) keeperOptions.pgSetup.pghost (last resort; may be a Unix socket + * path like /var/run/postgresql that does not resolve as a hostname) + * + * The keeper config file is created before postgres initialises, so it is + * normally available by the time pg_hba.conf appears. All three options + * are tried so the command works even when called very early. + */ + char hostname[_POSIX_HOST_NAME_MAX] = ""; + + /* option a: keeper config */ + { + KeeperConfig hbaConfig = keeperOptions; + if (keeper_config_set_pathnames_from_pgdata(&hbaConfig.pathnames, + pgdata) && + keeper_config_read_file(&hbaConfig, + false /* missingPgdataIsOk */, + true /* pgIsNotRunningIsOk */, + true /* monitorDisabledIsOk */) && + !IS_EMPTY_STRING_BUFFER(hbaConfig.hostname)) + { + strlcpy(hostname, hbaConfig.hostname, sizeof(hostname)); + } + } + + /* option b: OS hostname */ + if (IS_EMPTY_STRING_BUFFER(hostname)) + { + if (gethostname(hostname, sizeof(hostname)) == 0) + { + log_debug("hba-lan: using OS hostname \"%s\"", hostname); + } + else + { + hostname[0] = '\0'; + } + } + + /* option c: pghost (may be a socket path — last resort) */ + if (IS_EMPTY_STRING_BUFFER(hostname)) + { + strlcpy(hostname, keeperOptions.pgSetup.pghost, sizeof(hostname)); + } + + /* --auth defaults to "trust"; --ssl enables hostssl rules */ + const char *authMethod = keeperOptions.pgSetup.authMethod; + if (IS_EMPTY_STRING_BUFFER(authMethod)) + { + authMethod = "trust"; + } + bool useSSL = keeperOptions.pgSetup.ssl.active; + + /* cert auth for replication needs an ident map */ + bool isCert = (strcmp(authMethod, "cert") == 0); + const char *replAuth = isCert ? "cert map=pgautofailover" : authMethod; + + if (!pghba_enable_lan_cidr(NULL, useSSL, + HBA_DATABASE_ALL, NULL, + hostname, NULL, authMethod, + HBA_EDIT_MINIMAL, + pgdata)) + { + log_error("Failed to add LAN CIDR HBA rule for \"all\" databases"); + exit(EXIT_CODE_PGCTL); + } + + if (!pghba_enable_lan_cidr(NULL, useSSL, + HBA_DATABASE_REPLICATION, NULL, + hostname, NULL, replAuth, + HBA_EDIT_MINIMAL, + pgdata)) + { + log_error("Failed to add LAN CIDR HBA rule for replication"); + exit(EXIT_CODE_PGCTL); + } + + /* cert auth requires an ident map entry in pg_ident.conf */ + if (isCert) + { + if (!pghba_ensure_ident_map_entry(pgdata, + "pgautofailover", + PG_AUTOCTL_MONITOR_USERNAME, + PG_AUTOCTL_REPLICA_USERNAME)) + { + log_error("Failed to add cert ident map entry to pg_ident.conf"); + exit(EXIT_CODE_PGCTL); + } + } + + /* Step 3: wait for Postgres to be running (PID file, no TCP needed) */ + if (!pg_setup_wait_until_is_ready(&keeperOptions.pgSetup, 60, LOG_INFO)) + { + log_error("Postgres did not become ready within 60 s"); + exit(EXIT_CODE_PGCTL); + } + + /* Step 4: reload so the new HBA rules take effect */ + log_info("Reloading Postgres configuration in \"%s\"", pgdata); + if (!pg_ctl_reload(keeperOptions.pgSetup.pg_ctl, pgdata)) + { + exit(EXIT_CODE_PGCTL); + } +} + + /* * keeper_cli_init_standby initializes a standby */ diff --git a/src/bin/pg_autoctl/cli_do_monitor.c b/src/bin/pg_autoctl/cli_do_monitor.c index f1dfb526c..6426946ee 100644 --- a/src/bin/pg_autoctl/cli_do_monitor.c +++ b/src/bin/pg_autoctl/cli_do_monitor.c @@ -36,7 +36,9 @@ #include "pgctl.h" #include "pgsetup.h" #include "pgsql.h" +#include "pqexpbuffer.h" #include "state.h" +#include "string_utils.h" static void cli_do_monitor_get_primary_node(int argc, char **argv); static void cli_do_monitor_get_other_nodes(int argc, char **argv); @@ -46,6 +48,10 @@ static void cli_do_monitor_register_node(int argc, char **argv); static void cli_do_monitor_node_active(int argc, char **argv); static void cli_do_monitor_version(int argc, char **argv); static void cli_do_monitor_parse_notification(int argc, char **argv); +static int cli_do_monitor_node_state_getopts(int argc, char **argv); +static void cli_do_monitor_node_state(int argc, char **argv); +static int cli_do_monitor_formation_states_getopts(int argc, char **argv); +static void cli_do_monitor_formation_states(int argc, char **argv); static CommandLine monitor_get_primary_command = @@ -88,12 +94,12 @@ static CommandLine *monitor_get_commands[] = { NULL }; -static CommandLine monitor_get_command = +CommandLine monitor_get_command = make_command_set("get", "Get information from the monitor", NULL, NULL, NULL, monitor_get_commands); -static CommandLine monitor_register_command = +CommandLine monitor_register_command = make_command("register", "Register the current node with the monitor", CLI_PGDATA_USAGE "", @@ -101,7 +107,7 @@ static CommandLine monitor_register_command = cli_getopt_pgdata, cli_do_monitor_register_node); -static CommandLine monitor_node_active_command = +CommandLine monitor_node_active_command = make_command("active", "Call in the pg_auto_failover Node Active protocol", CLI_PGDATA_USAGE, @@ -109,7 +115,7 @@ static CommandLine monitor_node_active_command = cli_getopt_pgdata, cli_do_monitor_node_active); -static CommandLine monitor_version_command = +CommandLine monitor_version_command = make_command("version", "Check that monitor version is " PG_AUTOCTL_EXTENSION_VERSION @@ -119,7 +125,7 @@ static CommandLine monitor_version_command = cli_getopt_pgdata, cli_do_monitor_version); -static CommandLine monitor_parse_notification_command = +CommandLine monitor_parse_notification_command = make_command("parse-notification", "parse a raw notification message", " ", @@ -127,6 +133,27 @@ static CommandLine monitor_parse_notification_command = NULL, cli_do_monitor_parse_notification); +CommandLine monitor_node_state_command = + make_command("node-state", + "Print reported|assigned state for a named node (for test runners)", + "--name [--state ] [--monitor ] [--timeout N]", + " --name node name to query\n" + " --state wait until reportedstate matches (requires --timeout)\n" + " --monitor monitor URI (default: local socket)\n" + " --timeout N retry for up to N seconds (default: no retry)\n", + cli_do_monitor_node_state_getopts, + cli_do_monitor_node_state); + +CommandLine monitor_formation_states_command = + make_command("formation-states", + "Exit 0 when every listed state has at least one node (for test runners)", + "[--group N] [--timeout N] [...]", + " --group N restrict to this group id\n" + " --monitor monitor URI (default: local socket)\n" + " --timeout N retry for up to N seconds (default: no retry)\n", + cli_do_monitor_formation_states_getopts, + cli_do_monitor_formation_states); + static CommandLine *monitor_subcommands[] = { &monitor_get_command, &monitor_register_command, @@ -703,3 +730,458 @@ cli_do_monitor_parse_notification(int argc, char **argv) (void) cli_pprint_json(js); } + + +/* ----------------------------------------------------------------------- + * pg_autoctl inspect monitor node-state --name + * pg_autoctl inspect monitor formation-states [--group N] [...] + * + * These two commands run ON the monitor container. They connect to the + * local PostgreSQL socket and return structured output consumed by the + * pgaftest test runner via `docker compose exec -T monitor pg_autoctl …`. + * Using dedicated subcommands (rather than shelling to psql) avoids all + * shell quoting issues. + * ----------------------------------------------------------------------- */ + +#define MONITOR_LOCAL_URI \ + "postgresql://autoctl_node@localhost/pg_auto_failover?sslmode=prefer" + +/* Shared option state for both new commands */ +static struct +{ + char monitorUri[MAXCONNINFO]; + char nodeName[_POSIX_HOST_NAME_MAX]; + char targetState[64]; /* --state: wait until reportedstate matches */ + char formation[NAMEDATALEN]; /* --formation: filter by formation */ + int groupId; /* -1 = all groups */ + int timeout; /* 0 = no retry loop, N = retry for N seconds */ +} +inspectOpts; + +static int +cli_do_monitor_node_state_getopts(int argc, char **argv) +{ + /* defaults */ + strlcpy(inspectOpts.monitorUri, MONITOR_LOCAL_URI, sizeof(inspectOpts.monitorUri)); + inspectOpts.nodeName[0] = '\0'; + inspectOpts.targetState[0] = '\0'; + inspectOpts.groupId = -1; + inspectOpts.timeout = 0; + + static struct option long_options[] = { + { "monitor", required_argument, NULL, 'm' }, + { "name", required_argument, NULL, 'n' }, + { "state", required_argument, NULL, 's' }, + { "timeout", required_argument, NULL, 't' }, + { NULL, 0, NULL, 0 } + }; + + int c; + optind = 0; + while ((c = getopt_long(argc, argv, "m:n:s:t:", long_options, NULL)) != -1) + { + switch (c) + { + case 'm': + { + strlcpy(inspectOpts.monitorUri, optarg, + sizeof(inspectOpts.monitorUri)); + break; + } + + case 'n': + { + strlcpy(inspectOpts.nodeName, optarg, + sizeof(inspectOpts.nodeName)); + break; + } + + case 's': + { + strlcpy(inspectOpts.targetState, optarg, + sizeof(inspectOpts.targetState)); + break; + } + + case 't': + { + inspectOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + default: + { + commandline_print_usage(&monitor_node_state_command, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + if (inspectOpts.nodeName[0] == '\0') + { + log_error("--name is required"); + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +/* + * cli_do_monitor_node_state connects to the local monitor and prints: + * | + * for the named node, then exits 0. Exits non-zero if the node is not + * found or the monitor is unreachable. + * + * When --timeout N is given the command retries for up to N seconds using the + * same exponential-backoff-with-jitter policy used elsewhere in pg_autoctl. + */ +static void +cli_do_monitor_node_state(int argc, char **argv) +{ + const char *sql = + "SELECT reportedstate::text || '|' || goalstate::text || '|' || health::text " + "FROM pgautofailover.node WHERE nodename = $1"; + + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1] = { inspectOpts.nodeName }; + + ConnectionRetryPolicy retryPolicy = { 0 }; + + if (inspectOpts.timeout > 0) + { + pgsql_set_retry_policy(&retryPolicy, + inspectOpts.timeout, + -1, /* unbounded attempts within timeout */ + 2000, /* cap at 2 s between attempts */ + 500); /* start at 500 ms */ + } + else + { + pgsql_set_retry_policy(&retryPolicy, 0, 0, 0, 0); /* no retry */ + } + do { + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, inspectOpts.monitorUri)) + { + /* connection failure: retry if policy allows */ + goto next_attempt; + } + + SingleValueResultContext ctx = { { 0 }, PGSQL_RESULT_STRING, false }; + + if (!pgsql_execute_with_params(&monitor.pgsql, sql, + paramCount, paramTypes, paramValues, + &ctx, &parseSingleValueResult)) + { + pgsql_finish(&monitor.pgsql); + goto next_attempt; + } + + pgsql_finish(&monitor.pgsql); + + if (!ctx.parsedOk || ctx.strVal == NULL) + { + /* node not found yet */ + if (ctx.strVal) + { + free(ctx.strVal); + } + goto next_attempt; + } + + /* + * When --state was given, check that reportedstate (the part + * before the first '|') matches the target. + * Output format: reportedstate|goalstate|health + */ + if (inspectOpts.targetState[0] != '\0') + { + char reported[64] = ""; + char *pipe = strchr(ctx.strVal, '|'); + if (pipe) + { + int len = pipe - ctx.strVal; + if (len >= (int) sizeof(reported)) + { + len = sizeof(reported) - 1; + } + memcpy(reported, ctx.strVal, len); /* IGNORE-BANNED */ + reported[len] = '\0'; + } + if (strcmp(reported, inspectOpts.targetState) != 0) + { + free(ctx.strVal); + goto next_attempt; + } + } + + fformat(stdout, "%s\n", ctx.strVal); + free(ctx.strVal); + exit(0); + +next_attempt: + if (pgsql_retry_policy_expired(&retryPolicy)) + { + break; + } + + int sleepMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); + log_debug("node-state: node \"%s\" not found yet, retrying in %d ms", + inspectOpts.nodeName, sleepMs); + pg_usleep((long) sleepMs * 1000); + } while (!pgsql_retry_policy_expired(&retryPolicy)); + + log_error("Node \"%s\" not found on the monitor", inspectOpts.nodeName); + exit(EXIT_CODE_MONITOR); +} + + +static int +cli_do_monitor_formation_states_getopts(int argc, char **argv) +{ + strlcpy(inspectOpts.monitorUri, MONITOR_LOCAL_URI, sizeof(inspectOpts.monitorUri)); + inspectOpts.formation[0] = '\0'; + inspectOpts.groupId = -1; + inspectOpts.timeout = 0; + + static struct option long_options[] = { + { "monitor", required_argument, NULL, 'm' }, + { "formation", required_argument, NULL, 'F' }, + { "group", required_argument, NULL, 'g' }, + { "timeout", required_argument, NULL, 't' }, + { NULL, 0, NULL, 0 } + }; + + int c; + optind = 0; + while ((c = getopt_long(argc, argv, "m:F:g:t:", long_options, NULL)) != -1) + { + switch (c) + { + case 'm': + { + strlcpy(inspectOpts.monitorUri, optarg, + sizeof(inspectOpts.monitorUri)); + break; + } + + case 'F': + { + strlcpy(inspectOpts.formation, optarg, + sizeof(inspectOpts.formation)); + break; + } + + case 'g': + { + inspectOpts.groupId = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + case 't': + { + inspectOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + default: + { + commandline_print_usage(&monitor_formation_states_command, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + return optind; +} + + +/* + * cli_do_monitor_formation_states checks that every state listed on the + * command line has at least one node currently in that reportedstate. + * + * Exits 0 when all states are satisfied. When --timeout N is given the + * command retries for up to N seconds using the same exponential-backoff- + * with-jitter policy used elsewhere in pg_autoctl; without it the command + * exits 1 immediately when the condition is not met. + */ +static void +cli_do_monitor_formation_states(int argc, char **argv) +{ + if (argc < 1) + { + log_error("Expected at least one state name as argument"); + commandline_print_usage(&monitor_formation_states_command, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + +#define MAX_STATES 16 + if (argc > MAX_STATES) + { + log_error("Too many states (max %d)", MAX_STATES); + exit(EXIT_CODE_BAD_ARGS); + } + + /* + * Build the SQL once; the params are the same across every retry attempt. + * + * SELECT (count(*) FILTER (WHERE reportedstate::text = $1))::text + * || ',' || + * (count(*) FILTER (WHERE reportedstate::text = $2))::text + * ... + * FROM pgautofailover.node + * [WHERE groupid = $N] + */ + PQExpBufferData sqlBuf; + initPQExpBuffer(&sqlBuf); + + appendPQExpBufferStr(&sqlBuf, "SELECT "); + for (int i = 0; i < argc; i++) + { + if (i > 0) + { + appendPQExpBufferStr(&sqlBuf, " || ',' || "); + } + appendPQExpBuffer(&sqlBuf, + "(count(*) FILTER (WHERE reportedstate::text = $%d))::text", + i + 1); + } + appendPQExpBufferStr(&sqlBuf, " FROM pgautofailover.node"); + + bool hasFormation = inspectOpts.formation[0] != '\0'; + bool hasGroup = inspectOpts.groupId >= 0; + bool needWhere = hasFormation || hasGroup; + int extraParams = (hasFormation ? 1 : 0) + (hasGroup ? 1 : 0); + + if (needWhere) + { + appendPQExpBufferStr(&sqlBuf, " WHERE"); + if (hasFormation) + { + appendPQExpBuffer(&sqlBuf, " formationid = $%d", argc + 1); + } + if (hasGroup) + { + appendPQExpBuffer(&sqlBuf, "%s groupid = $%d", + hasFormation ? " AND" : "", + argc + (hasFormation ? 2 : 1)); + } + } + + int paramCount = argc + extraParams; + Oid paramTypes[MAX_STATES + 2]; + const char *paramValues[MAX_STATES + 2]; + IntString groupIdStr = { 0 }; + + for (int i = 0; i < argc; i++) + { + paramTypes[i] = TEXTOID; + paramValues[i] = argv[i]; + } + if (extraParams > 0) + { + int pi = argc; + if (hasFormation) + { + paramTypes[pi] = TEXTOID; + paramValues[pi] = inspectOpts.formation; + pi++; + } + if (hasGroup) + { + groupIdStr = intToString(inspectOpts.groupId); + paramTypes[pi] = INT4OID; + paramValues[pi] = groupIdStr.strValue; + } + } + + ConnectionRetryPolicy retryPolicy = { 0 }; + + if (inspectOpts.timeout > 0) + { + pgsql_set_retry_policy(&retryPolicy, + inspectOpts.timeout, + -1, /* unbounded attempts within timeout */ + 2000, /* cap at 2 s between attempts */ + 500); /* start at 500 ms */ + } + else + { + pgsql_set_retry_policy(&retryPolicy, 0, 0, 0, 0); /* no retry */ + } + do { + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, inspectOpts.monitorUri)) + { + goto next_attempt; + } + + SingleValueResultContext ctx = { { 0 }, PGSQL_RESULT_STRING, false }; + + if (!pgsql_execute_with_params(&monitor.pgsql, sqlBuf.data, + paramCount, paramTypes, paramValues, + &ctx, &parseSingleValueResult)) + { + pgsql_finish(&monitor.pgsql); + goto next_attempt; + } + + pgsql_finish(&monitor.pgsql); + + if (!ctx.parsedOk || ctx.strVal == NULL) + { + if (ctx.strVal) + { + free(ctx.strVal); + } + goto next_attempt; + } + + /* parse "N,N,..." — all must be >= 1 */ + bool allMet = true; + char *p = ctx.strVal; + for (int i = 0; i < argc; i++) + { + int cnt = atoi(p) /* IGNORE-BANNED */; + if (cnt < 1) + { + allMet = false; + break; + } + p = strchr(p, ','); + if (p) + { + p++; + } + else + { + break; + } + } + free(ctx.strVal); + + if (allMet) + { + termPQExpBuffer(&sqlBuf); + exit(0); + } + +next_attempt: + if (pgsql_retry_policy_expired(&retryPolicy)) + { + break; + } + + int sleepMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); + log_debug("formation-states: condition not met yet, retrying in %d ms", + sleepMs); + pg_usleep((long) sleepMs * 1000); + } while (!pgsql_retry_policy_expired(&retryPolicy)); + + termPQExpBuffer(&sqlBuf); + exit(1); +} diff --git a/src/bin/pg_autoctl/cli_do_root.c b/src/bin/pg_autoctl/cli_do_root.c index dacf9792e..59e3dd288 100644 --- a/src/bin/pg_autoctl/cli_do_root.c +++ b/src/bin/pg_autoctl/cli_do_root.c @@ -185,9 +185,11 @@ CommandLine do_pgsetup_is_ready = CommandLine do_pgsetup_wait_until_ready = make_command("wait", "Wait until the local Postgres server is ready", - "[option ...]", + "[--read-write] [--timeout N] [option ...]", + " --read-write also wait until the server accepts read-write connections\n" + " --timeout N total timeout in seconds (default: 30)\n" KEEPER_CLI_WORKER_SETUP_OPTIONS, - keeper_cli_keeper_setup_getopts, + keeper_cli_pgsetup_wait_getopts, keeper_cli_pgsetup_wait_until_ready); CommandLine do_pgsetup_startup_logs = @@ -206,6 +208,14 @@ CommandLine do_pgsetup_tune = keeper_cli_keeper_setup_getopts, keeper_cli_pgsetup_tune); +CommandLine do_pgsetup_hba_lan = + make_command("hba-lan", + "Append LAN CIDR trust rules to pg_hba.conf and reload Postgres", + "[option ...]", + KEEPER_CLI_WORKER_SETUP_OPTIONS, + keeper_cli_keeper_setup_getopts, + keeper_cli_pgsetup_hba_lan); + CommandLine *do_pgsetup[] = { &do_pgsetup_pg_ctl, &do_pgsetup_discover, @@ -213,6 +223,7 @@ CommandLine *do_pgsetup[] = { &do_pgsetup_wait_until_ready, &do_pgsetup_startup_logs, &do_pgsetup_tune, + &do_pgsetup_hba_lan, NULL }; @@ -375,11 +386,12 @@ CommandLine do_tmux_commands = NULL, NULL, NULL, do_tmux); /* - * pg_autoctl do azure ... - * - * Set of commands to prepare and control a full QA environment running in - * Azure VMs, provisionned either from our packages or from local source code. + * Azure integration has been removed. The commands that were here are no + * longer maintained and have been deleted from this file. See pgaftest for + * the replacement QA tooling. */ + +#if 0 /* REMOVED: azure commands — see pgaftest instead */ CommandLine do_azure_provision_region = make_command("region", "Provision an azure region: resource group, network, VMs", @@ -564,27 +576,64 @@ CommandLine do_azure_commands = "Manage a set of Azure resources for a pg_auto_failover demo", NULL, NULL, NULL, do_azure); +#endif /* REMOVED: azure commands */ +/* + * pg_autoctl internal service postgres|listener|node-active + * + * These are the subprocess entry points used by the supervisor (pg_autoctl run + * and pg_autoctl create … --run). The supervisor fork()s and then execv()s + * the pg_autoctl binary itself with one of these sub-commands so that each + * service runs in its own address space. + * + * Using fork+exec (rather than fork alone) is a deliberate design choice for + * live upgrades: when a child process exits with an incompatible monitor + * extension version, the supervisor restarts it via fork()+execv(), which loads + * the current binary from disk. If the binary has been updated in place (e.g. + * by a package manager), the restarted child automatically picks up the new + * version without touching the supervisor process — making pg_autoctl safe to + * use as PID 1 in Docker/Kubernetes containers where replacing the binary and + * sending SIGTERM would lose the container. + * + * See also: keeper.c keeper_check_monitor_extension_version(), which exits on + * version mismatch precisely to trigger this restart-with-new-binary path. + * + * These commands are hidden from --help output (make_hidden_command_set) so + * operators do not accidentally invoke them directly. + * Use "pg_autoctl manual service" for the user-facing controls (restart, pgctl). + * Use "pg_autoctl inspect getpid" to read sub-process PIDs. + */ +static CommandLine *internal_service_subcommands[] = { + &service_pgcontroller, /* debug: supervisor for just the postgres controller */ + &service_postgres, /* spawned by service_postgres_ctl_start() */ + &service_monitor_listener, /* spawned by service_monitor_start() */ + &service_node_active, /* spawned by service_keeper_start() */ + NULL +}; + +static CommandLine internal_service_commands = + make_hidden_command_set("service", + "Subprocess entry points for the pg_autoctl supervisor", + NULL, NULL, NULL, internal_service_subcommands); + +/* + * pg_autoctl internal + * + * Hidden from --help; routable so the supervisor's execv() calls work. + * Contains only what the supervisor spawns plus dev/QA tooling not yet + * moved to pgaftest (tmux, demo). + */ CommandLine *do_subcommands[] = { - &do_monitor_commands, - &do_coordinator_commands, - &do_fsm_commands, - &do_primary_, - &do_standby_, - &do_show_commands, - &do_pgsetup_commands, - &do_service_postgres_ctl_commands, - &do_service_commands, + &internal_service_commands, &do_tmux_commands, - &do_azure_commands, &do_demo_commands, NULL }; CommandLine do_commands = - make_command_set("do", - "Internal commands and internal QA tooling", NULL, NULL, - NULL, do_subcommands); + make_hidden_command_set("internal", + "Internal subprocess entry points — not for direct use", + NULL, NULL, NULL, do_subcommands); /* diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index efb482eaa..1c3112dd6 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -16,13 +16,44 @@ /* src/bin/pg_autoctl/cli_do_fsm.c */ extern CommandLine do_fsm_commands; +/* read-only sub-commands exposed via "pg_autoctl inspect fsm" */ +extern CommandLine fsm_state; +extern CommandLine fsm_node_state; +extern CommandLine fsm_list; +extern CommandLine fsm_gv; + +/* mutating sub-commands exposed via "pg_autoctl manual fsm" */ +extern CommandLine fsm_init; +extern CommandLine fsm_assign; +extern CommandLine fsm_step; +extern CommandLine fsm_nodes; /* nodes get + nodes set — kept together in manual */ + /* src/bin/pg_autoctl/cli_do_monitor.c */ extern CommandLine do_monitor_commands; +/* read-only sub-commands exposed via "pg_autoctl inspect monitor" */ +extern CommandLine monitor_get_command; +extern CommandLine monitor_parse_notification_command; +extern CommandLine monitor_node_state_command; +extern CommandLine monitor_formation_states_command; + +/* mutating sub-commands exposed via "pg_autoctl manual monitor" */ +extern CommandLine monitor_register_command; +extern CommandLine monitor_node_active_command; +extern CommandLine monitor_version_command; + /* src/bin/pg_autoctl/cli_do_service.c */ extern CommandLine do_service_commands; +extern CommandLine do_service_getpid_commands; +extern CommandLine do_service_restart_commands; extern CommandLine do_service_postgres_ctl_commands; +/* subprocess entry points (spawned by the supervisor via fork+exec) */ +extern CommandLine service_pgcontroller; +extern CommandLine service_postgres; +extern CommandLine service_monitor_listener; +extern CommandLine service_node_active; + /* src/bin/pg_autoctl/cli_do_show.c */ extern CommandLine do_show_commands; extern CommandLine do_pgsetup_commands; @@ -71,9 +102,6 @@ extern CommandLine do_tmux_commands; /* src/bin/pg_autoctl/cli_do_coordinator.c */ extern CommandLine do_coordinator_commands; -/* src/bin/pg_autoctl/cli_do_azure.c */ -extern CommandLine do_azure_ssh; - extern CommandLine do_commands; extern CommandLine *do_subcommands[]; @@ -89,9 +117,11 @@ void keeper_cli_disable_synchronous_replication(int argc, char **argv); void keeper_cli_pgsetup_pg_ctl(int argc, char **argv); void keeper_cli_pgsetup_discover(int argc, char **argv); void keeper_cli_pgsetup_is_ready(int argc, char **argv); +int keeper_cli_pgsetup_wait_getopts(int argc, char **argv); void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv); void keeper_cli_pgsetup_startup_logs(int argc, char **argv); void keeper_cli_pgsetup_tune(int argc, char **argv); +void keeper_cli_pgsetup_hba_lan(int argc, char **argv); void keeper_cli_add_default_settings(int argc, char **argv); void keeper_cli_create_monitor_user(int argc, char **argv); @@ -117,19 +147,6 @@ void cli_do_tmux_compose_config(int argc, char **argv); void cli_do_tmux_compose_script(int argc, char **argv); void cli_do_tmux_compose_session(int argc, char **argv); -/* src/bin/pg_autoctl/cli_do_azure.c */ -int cli_do_azure_getopts(int argc, char **argv); -void cli_do_azure_create_environment(int argc, char **argv); -void cli_do_azure_create_region(int argc, char **argv); -void cli_do_azure_drop_region(int argc, char **argv); -void cli_do_azure_deploy(int argc, char **argv); -void cli_do_azure_create_nodes(int argc, char **argv); -void cli_do_azure_ls(int argc, char **argv); -void cli_do_azure_show_ips(int argc, char **argv); -void cli_do_azure_ssh(int argc, char **argv); -void cli_do_azure_rsync(int argc, char **argv); -void cli_do_azure_show_state(int argc, char **argv); -void cli_do_azure_tmux_session(int argc, char **argv); -void cli_do_azure_tmux_kill(int argc, char **argv); +/* Azure integration removed — use pgaftest instead */ #endif /* CLI_DO_ROOT_H */ diff --git a/src/bin/pg_autoctl/cli_do_service.c b/src/bin/pg_autoctl/cli_do_service.c index 94420b7c5..e450417ca 100644 --- a/src/bin/pg_autoctl/cli_do_service.c +++ b/src/bin/pg_autoctl/cli_do_service.c @@ -357,8 +357,8 @@ cli_do_service_restart_node_active(int argc, char **argv) * tree. It is used for debug purposes only. When using this entry point we * have a supervisor process that is responsible for only one service: * - * pg_autoctl do service pgcontroller - * - pg_autoctl do service postgres + * pg_autoctl internal service pgcontroller + * - pg_autoctl internal service postgres * - postgres */ static void @@ -401,9 +401,9 @@ cli_do_service_pgcontroller(int argc, char **argv) * that supervises two sub-processes, one of them is cli_do_service_postgres: * * pg_autoctl - * - pg_autoctl do service postgres + * - pg_autoctl internal service postgres * - postgres - * - pg_autoctl do service keeper|monitor + * - pg_autoctl internal service keeper|monitor */ static void cli_do_service_postgres(int argc, char **argv) diff --git a/src/bin/pg_autoctl/cli_do_tmux_azure.c b/src/bin/pg_autoctl/cli_do_tmux_azure.c deleted file mode 100644 index f4a5373ed..000000000 --- a/src/bin/pg_autoctl/cli_do_tmux_azure.c +++ /dev/null @@ -1,276 +0,0 @@ -/* - * src/bin/pg_autoctl/cli_do_tmux_azure.c - * - * Implementation of commands that create a tmux session to connect to a - * set of Azure VMs where we run pg_autoctl nodes for QA and testing. - * - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the PostgreSQL License. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "postgres_fe.h" -#include "pqexpbuffer.h" -#include "snprintf.h" - -#include "azure.h" -#include "azure_config.h" -#include "cli_common.h" -#include "cli_do_root.h" -#include "cli_do_tmux.h" -#include "cli_root.h" -#include "commandline.h" -#include "config.h" -#include "env_utils.h" -#include "log.h" -#include "parsing.h" -#include "pidfile.h" -#include "signals.h" -#include "string_utils.h" - -#include "runprogram.h" - -static void tmux_azure_new_session(PQExpBuffer script, - AzureRegionResources *azRegion); - -static void tmux_azure_deploy(PQExpBuffer script, - AzureRegionResources *azRegion, - const char *vmName); - -static void tmux_azure_ssh(PQExpBuffer script, AzureRegionResources *azRegion, - const char *vmName); - -static void tmux_azure_systemctl_status(PQExpBuffer script, - AzureRegionResources *azRegion); - -static void prepare_tmux_azure_script(AzureRegionResources *azRegion, - PQExpBuffer script); - - -/* - * tmux_azure_new_session appends a new-session command to the given tmux - * script buffer, using the azure group name for the tmux session name. - */ -static void -tmux_azure_new_session(PQExpBuffer script, AzureRegionResources *azRegion) -{ - tmux_add_command(script, "new-session -s %s", azRegion->group); -} - - -/* - * tmux_azure_deploy_postgres appends a pg_autoctl do azure deploy command for - * the given vmName to the given script buffer. - */ -static void -tmux_azure_deploy(PQExpBuffer script, AzureRegionResources *azRegion, - const char *vmName) -{ - tmux_add_send_keys_command(script, - "%s do azure deploy %s", - pg_autoctl_argv0, - vmName); -} - - -/* - * tmux_azure_ssh appends a pg_autoctl do azure ssh command for the given - * vmName to the given script buffer. - */ -static void -tmux_azure_ssh(PQExpBuffer script, AzureRegionResources *azRegion, - const char *vmName) -{ - tmux_add_send_keys_command(script, - "%s do azure ssh %s", - pg_autoctl_argv0, - vmName); -} - - -/* - * tmux_azure_ssh appends a pg_autoctl do azure ssh command for the given - * vmName to the given script buffer. - */ -static void -tmux_azure_systemctl_status(PQExpBuffer script, - AzureRegionResources *azRegion) -{ - tmux_add_send_keys_command(script, "systemctl status pgautofailover"); -} - - -/* - * tmux_add_environment appends the export VAR=value commands that we need to - * set the environment for pg_autoctl do azure deploy in the shell windows. - */ -static void -tmux_azure_add_environment(PQExpBuffer script, KeyVal *env) -{ - for (int i = 0; i < env->count; i++) - { - tmux_add_send_keys_command(script, - "export %s=%s", - env->keywords[i], - env->values[i]); - } -} - - -/* - * prepare_tmux_script prepares a script for a tmux session with the given - * azure region resources. - */ -static void -prepare_tmux_azure_script(AzureRegionResources *azRegion, PQExpBuffer script) -{ - KeyVal env = { 0 }; - - /* fetch environment and defaults for versions */ - if (!azure_prepare_target_versions(&env)) - { - /* errors have already been logged */ - exit(EXIT_CODE_INTERNAL_ERROR); - } - - tmux_add_command(script, "set-option -g default-shell /bin/bash"); - - tmux_azure_new_session(script, azRegion); - - /* deploy VMs each in a new tmux window */ - for (int vmIndex = 0; vmIndex <= azRegion->nodes; vmIndex++) - { - const char *vmName = azRegion->vmArray[vmIndex].name; - - /* after the first VM, create new tmux windows for each VM */ - if (vmIndex > 0) - { - tmux_add_command(script, "split-window -v"); - tmux_add_command(script, "select-layout even-vertical"); - } - - tmux_azure_add_environment(script, &env); - tmux_azure_deploy(script, azRegion, vmName); - tmux_azure_ssh(script, azRegion, vmName); - tmux_azure_systemctl_status(script, azRegion); - } - - /* add a window for pg_autoctl show state */ - tmux_add_command(script, "split-window -v"); - tmux_add_command(script, "select-layout even-vertical"); - - tmux_add_send_keys_command(script, - "%s do azure show state --watch", - pg_autoctl_argv0); - - /* add a window for interactive pg_autoctl commands */ - tmux_add_command(script, "split-window -v"); - tmux_add_command(script, "select-layout even-vertical"); - tmux_add_send_keys_command(script, - "%s do azure show ips", - pg_autoctl_argv0); -} - - -/* - * cli_do_azure_tmux_session starts a new tmux session for the given azure - * region and resources, or attach an existing session that might be running in - * the background already. - */ -bool -tmux_azure_start_or_attach_session(AzureRegionResources *azRegion) -{ - char tmux[MAXPGPATH] = { 0 }; - - PQExpBuffer script; - char scriptName[MAXPGPATH] = { 0 }; - - if (setenv("PG_AUTOCTL_DEBUG", "1", 1) != 0) - { - log_error("Failed to set environment PG_AUTOCTL_DEBUG: %m"); - return false; - } - - if (!search_path_first("tmux", tmux, LOG_ERROR)) - { - log_fatal("Failed to find program tmux in PATH"); - return false; - } - - /* we might just re-use a pre-existing tmux session */ - if (!dryRun && tmux_has_session(tmux, azRegion->group)) - { - return tmux_attach_session(tmux, azRegion->group); - } - - /* - * Okay, so we have to create the session now. And for that we need the IP - * addresses of the target VMs. - */ - if (!azure_fetch_ip_addresses(azRegion->group, azRegion->vmArray)) - { - /* errors have already been logged */ - return false; - } - - script = createPQExpBuffer(); - if (script == NULL) - { - log_error("Failed to allocate memory"); - return false; - } - - /* prepare the tmux script */ - (void) prepare_tmux_azure_script(azRegion, script); - - /* - * Start a tmux session from the script. - */ - if (dryRun) - { - fformat(stdout, "%s", script->data); - destroyPQExpBuffer(script); - } - else - { - /* write the tmux script to file */ - sformat(scriptName, sizeof(scriptName), "%s.tmux", azRegion->group); - log_info("Writing tmux session script \"%s\"", scriptName); - - if (!write_file(script->data, script->len, scriptName)) - { - log_fatal("Failed to write tmux script at \"%s\"", scriptName); - destroyPQExpBuffer(script); - return false; - } - - if (!tmux_start_server(scriptName, NULL)) - { - log_fatal("Failed to start the tmux session, see above for details"); - destroyPQExpBuffer(script); - return false; - } - } - - return true; -} - - -/* - * tmux_azure_kill_session kills a tmux session for the given QA setup, when - * the tmux session already exists. - */ -bool -tmux_azure_kill_session(AzureRegionResources *azRegion) -{ - return tmux_kill_session_by_name(azRegion->group); -} diff --git a/src/bin/pg_autoctl/cli_drop_node.c b/src/bin/pg_autoctl/cli_drop_node.c index 3ecefffc6..04a1bf0c1 100644 --- a/src/bin/pg_autoctl/cli_drop_node.c +++ b/src/bin/pg_autoctl/cli_drop_node.c @@ -39,6 +39,7 @@ #include "service_monitor.h" #include "service_monitor_init.h" #include "signals.h" +#include "file_utils.h" #include "string_utils.h" /* @@ -47,6 +48,7 @@ */ bool dropAndDestroy = false; static bool dropForce = false; +static bool dropNoWait = false; static void cli_drop_monitor(int argc, char **argv); @@ -103,6 +105,7 @@ cli_drop_node_getopts(int argc, char **argv) { "monitor", required_argument, NULL, 'm' }, { "destroy", no_argument, NULL, 'd' }, { "force", no_argument, NULL, 'F' }, + { "no-wait", no_argument, NULL, 'W' }, { "hostname", required_argument, NULL, 'n' }, { "pgport", required_argument, NULL, 'p' }, { "formation", required_argument, NULL, 'f' }, @@ -159,6 +162,13 @@ cli_drop_node_getopts(int argc, char **argv) break; } + case 'W': + { + dropNoWait = true; + log_trace("--no-wait"); + break; + } + case 'n': { strlcpy(options.hostname, optarg, _POSIX_HOST_NAME_MAX); @@ -594,6 +604,19 @@ cli_drop_local_node(KeeperConfig *config, bool dropAndDestroy) (void) cli_drop_node_from_monitor(config, &nodeId, &groupId); } + /* + * With --no-wait the caller takes responsibility for waiting until the + * supervisor has stopped (e.g. via `docker compose wait` in a container + * environment). The running keeper will detect it has been dropped on its + * next node_active() heartbeat and exit cleanly on its own. + */ + if (dropNoWait) + { + log_info("Node unregistered from monitor; not waiting for the local " + "pg_autoctl process to stop (--no-wait)."); + exit(EXIT_CODE_QUIT); + } + /* * Now, when the pg_autoctl keeper service is still running, wait until * it has reached the DROPPED/DROPPED state on-disk and then exited. @@ -680,6 +703,22 @@ cli_drop_local_node(KeeperConfig *config, bool dropAndDestroy) config->formation, config->groupId); } + else if (pid != 0 && !file_exists(config->pathnames.state)) + { + /* + * The running service stopped and deleted its own state file. The + * service only removes the state file after completing the DROPPED + * protocol (two confirmed monitor contacts in DROPPED state), so the + * absence of the state file is authoritative: the drop succeeded. + */ + log_info("This node with id %lld in formation \"%s\" and group %d " + "has been dropped from the monitor " + "(confirmed by service clean-up)", + (long long) keeperState->current_node_id, + config->formation, + config->groupId); + dropped = true; + } else { log_fatal("Failed to ensure that the local node with id %lld " diff --git a/src/bin/pg_autoctl/cli_formation.c b/src/bin/pg_autoctl/cli_formation.c index 09d546dd1..b06af3bff 100644 --- a/src/bin/pg_autoctl/cli_formation.c +++ b/src/bin/pg_autoctl/cli_formation.c @@ -413,6 +413,12 @@ keeper_cli_formation_create_getopts(int argc, char **argv) } } + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + /* when we have a monitor URI we don't need PGDATA */ if (cli_formation_use_monitor_option(&options)) { diff --git a/src/bin/pg_autoctl/cli_get_set_properties.c b/src/bin/pg_autoctl/cli_get_set_properties.c index f61b1d168..35356ce48 100644 --- a/src/bin/pg_autoctl/cli_get_set_properties.c +++ b/src/bin/pg_autoctl/cli_get_set_properties.c @@ -7,10 +7,16 @@ * Licensed under the PostgreSQL License. * */ +#include +#include + #include "parson.h" #include "cli_common.h" +#include "monitor_pg_init.h" #include "parsing.h" +#include "pgsql.h" +#include "pgsetup.h" #include "string_utils.h" static bool get_node_replication_settings(NodeReplicationSettings *settings); @@ -22,6 +28,8 @@ static void cli_set_node_replication_quorum(int argc, char **argv); static void cli_set_node_candidate_priority(int argc, char **argv); static void cli_set_node_metadata(int argc, char **argv); static void cli_set_formation_number_sync_standbys(int arc, char **argv); +static int cli_set_password_getopts(int argc, char **argv); +static void cli_set_password(int argc, char **argv); static bool set_node_candidate_priority(Keeper *keeper, int candidatePriority); static bool set_node_replication_quorum(Keeper *keeper, bool replicationQuorum); @@ -181,9 +189,21 @@ static CommandLine set_formation_command = set_formation_subcommands); +static CommandLine set_password_command = + make_command("password", + "set the password for a pg_auto_failover role", + " [ --pgdata ] --role [ --password ]", + " --pgdata path to data directory\n" + " --role one of: autoctl_node, pgautofailover_monitor," + " pgautofailover_replicator\n" + " --password new password (prompted if omitted)\n", + cli_set_password_getopts, + cli_set_password); + static CommandLine *set_subcommands[] = { &set_node_command, &set_formation_command, + &set_password_command, NULL }; @@ -852,3 +872,323 @@ set_formation_number_sync_standbys(Monitor *monitor, return true; } + + +/* + * Options for `pg_autoctl set password`. + */ +typedef struct SetPasswordOptions +{ + char pgdata[MAXPGPATH]; + char role[NAMEDATALEN]; + char password[MAXCONNINFO]; +} SetPasswordOptions; + +static SetPasswordOptions setPasswordOptions = { + { + 0 + }, { + 0 + }, { + 0 + } +}; + +static int +cli_set_password_getopts(int argc, char **argv) +{ + static struct option longOptions[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "role", required_argument, NULL, 'r' }, + { "password", required_argument, NULL, 'p' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + int c, option_index = 0; + + optind = 0; + + while ((c = getopt_long(argc, argv, "D:r:p:h", + longOptions, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(setPasswordOptions.pgdata, optarg, MAXPGPATH); + strlcpy(keeperOptions.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", setPasswordOptions.pgdata); + break; + } + + case 'r': + { + strlcpy(setPasswordOptions.role, optarg, NAMEDATALEN); + log_trace("--role %s", setPasswordOptions.role); + break; + } + + case 'p': + { + strlcpy(setPasswordOptions.password, optarg, MAXCONNINFO); + log_trace("--password ****"); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + } + + default: + { + log_error("Unrecognized option: %c", c); + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + if (IS_EMPTY_STRING_BUFFER(setPasswordOptions.role)) + { + log_error("Please provide --role { autoctl_node | " + "pgautofailover_monitor | pgautofailover_replicator }"); + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +/* + * prompt_password reads a password from the terminal without echoing it. + * Returns true on success, false on error. + */ +static bool +prompt_password(const char *prompt, char *buf, size_t buflen) +{ + struct termios old, noecho; + bool ok = false; + + if (tcgetattr(fileno(stdin), &old) != 0) + { + /* stdin is not a tty; just read normally */ + fformat(stderr, "%s", prompt); + fflush(stderr); + + if (fgets(buf, buflen, stdin) == NULL) + { + return false; + } + + /* strip trailing newline */ + buf[strcspn(buf, "\n")] = '\0'; + return true; + } + + noecho = old; + noecho.c_lflag &= ~(ECHO | ECHOE | ECHOK | ECHONL); + + if (tcsetattr(fileno(stdin), TCSAFLUSH, &noecho) != 0) + { + return false; + } + + fformat(stderr, "%s", prompt); + fflush(stderr); + + if (fgets(buf, buflen, stdin) != NULL) + { + buf[strcspn(buf, "\n")] = '\0'; + ok = true; + } + + (void) tcsetattr(fileno(stdin), TCSAFLUSH, &old); + fformat(stderr, "\n"); + + return ok; +} + + +/* + * cli_set_password implements `pg_autoctl set password`. + * + * Supported roles: + * autoctl_node — lives on the monitor; ALTER ROLE there and + * update monitor config autoctl_node_password + * pgautofailover_monitor — lives on each data node; ALTER ROLE there and + * update keeper config monitor_password + * pgautofailover_replicator — lives on each data node; ALTER ROLE there + * and update keeper config replication.password + */ +static void +cli_set_password(int argc, char **argv) +{ + const char *role = setPasswordOptions.role; + char password[MAXCONNINFO] = { 0 }; + + /* validate role name */ + bool isAutoctlNode = (strcmp(role, PG_AUTOCTL_MONITOR_USERNAME) == 0); + bool isHealthUser = (strcmp(role, PG_AUTOCTL_HEALTH_USERNAME) == 0); + bool isReplicaUser = (strcmp(role, PG_AUTOCTL_REPLICA_USERNAME) == 0); + + if (!isAutoctlNode && !isHealthUser && !isReplicaUser) + { + log_error("Unknown role \"%s\". Valid roles: %s, %s, %s", + role, + PG_AUTOCTL_MONITOR_USERNAME, + PG_AUTOCTL_HEALTH_USERNAME, + PG_AUTOCTL_REPLICA_USERNAME); + exit(EXIT_CODE_BAD_ARGS); + } + + /* obtain the password */ + if (!IS_EMPTY_STRING_BUFFER(setPasswordOptions.password)) + { + strlcpy(password, setPasswordOptions.password, sizeof(password)); + } + else + { + char prompt[256]; + sformat(prompt, sizeof(prompt), "Password for role %s: ", role); + + if (!prompt_password(prompt, password, sizeof(password))) + { + log_error("Failed to read password from terminal"); + exit(EXIT_CODE_BAD_ARGS); + } + } + + if (IS_EMPTY_STRING_BUFFER(password)) + { + log_error("Password may not be empty"); + exit(EXIT_CODE_BAD_ARGS); + } + + /* + * autoctl_node lives on the MONITOR. We connect to the monitor's local + * Postgres (via MonitorConfig) and ALTER the role there. + */ + if (isAutoctlNode) + { + MonitorConfig mconfig = { 0 }; + + /* resolve pgdata */ + if (!IS_EMPTY_STRING_BUFFER(setPasswordOptions.pgdata)) + { + strlcpy(mconfig.pgSetup.pgdata, + setPasswordOptions.pgdata, + MAXPGPATH); + } + + if (!monitor_config_set_pathnames_from_pgdata(&mconfig)) + { + log_fatal("Failed to set monitor config pathnames"); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!monitor_config_read_file(&mconfig, false, true)) + { + log_fatal("Failed to read monitor configuration file"); + exit(EXIT_CODE_BAD_CONFIG); + } + + /* connect to the monitor's local postgres */ + PGSQL pgsql = { 0 }; + char connInfo[MAXCONNINFO]; + pg_setup_get_local_connection_string(&mconfig.pgSetup, connInfo); + pgsql_init(&pgsql, connInfo, PGSQL_CONN_LOCAL); + + if (!pgsql_alter_role_password(&pgsql, role, password)) + { + log_error("Failed to ALTER ROLE \"%s\" PASSWORD on the monitor", role); + exit(EXIT_CODE_PGSQL); + } + + /* persist password into monitor ini */ + strlcpy(mconfig.autoctl_node_password, password, + sizeof(mconfig.autoctl_node_password)); + + if (!monitor_config_write_file(&mconfig)) + { + log_error("Failed to update monitor configuration file"); + exit(EXIT_CODE_BAD_CONFIG); + } + + log_info("Password for role \"%s\" updated on the monitor and " + "saved to the monitor configuration file", role); + return; + } + + /* + * pgautofailover_monitor and pgautofailover_replicator both live on the + * local data node. We need a KeeperConfig for the local Postgres + * connection details. + */ + { + KeeperConfig config = keeperOptions; + bool missingPgdataIsOk = false; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = true; + + if (!IS_EMPTY_STRING_BUFFER(setPasswordOptions.pgdata)) + { + strlcpy(config.pgSetup.pgdata, + setPasswordOptions.pgdata, + MAXPGPATH); + } + + if (!keeper_config_set_pathnames_from_pgdata(&config.pathnames, + config.pgSetup.pgdata)) + { + log_fatal("Failed to set keeper config pathnames"); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!keeper_config_read_file(&config, + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + log_fatal("Failed to read keeper configuration file"); + exit(EXIT_CODE_BAD_CONFIG); + } + + /* connect to the local data node's Postgres */ + PGSQL pgsql = { 0 }; + char connInfo[MAXCONNINFO]; + pg_setup_get_local_connection_string(&config.pgSetup, connInfo); + pgsql_init(&pgsql, connInfo, PGSQL_CONN_LOCAL); + + if (!pgsql_alter_role_password(&pgsql, role, password)) + { + log_error("Failed to ALTER ROLE \"%s\" PASSWORD on the local node", + role); + exit(EXIT_CODE_PGSQL); + } + + /* persist password into keeper ini */ + if (isHealthUser) + { + strlcpy(config.monitor_password, password, + sizeof(config.monitor_password)); + } + else + { + /* isReplicaUser */ + strlcpy(config.replication_password, password, + sizeof(config.replication_password)); + } + + if (!keeper_config_write_file(&config)) + { + log_error("Failed to update keeper configuration file"); + exit(EXIT_CODE_BAD_CONFIG); + } + + log_info("Password for role \"%s\" updated on the local node and " + "saved to the keeper configuration file", role); + } +} diff --git a/src/bin/pg_autoctl/cli_inspect.c b/src/bin/pg_autoctl/cli_inspect.c new file mode 100644 index 000000000..4b156c660 --- /dev/null +++ b/src/bin/pg_autoctl/cli_inspect.c @@ -0,0 +1,79 @@ +/* + * src/bin/pg_autoctl/cli_inspect.c + * pg_autoctl inspect — read-only diagnostics, always visible. + * + * All commands here read local or cluster state without mutating anything. + * Safe to run at any time, even while `pg_autoctl run` is active. + * + * For mutating recovery commands (fsm assign, standby promote, etc.) see + * cli_manual.c ("pg_autoctl manual …"). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include "commandline.h" +#include "cli_inspect.h" +#include "cli_do_root.h" + +/* + * Read-only FSM sub-commands: display the current state, list reachable + * transitions, or dump the full FSM as a graphviz .gv file. + * Mutating operations (init, assign, step, nodes set) live under "manual fsm". + */ +static CommandLine *inspect_fsm_subcommands[] = { + &fsm_state, + &fsm_node_state, + &fsm_list, + &fsm_gv, + NULL +}; + +static CommandLine inspect_fsm_commands = + make_command_set("fsm", + "Display keeper FSM state and transitions (read-only)", + NULL, NULL, NULL, inspect_fsm_subcommands); + +/* + * Read-only monitor sub-commands: get primary/others/candidate-count/coordinator + * and parse-notification. We intentionally exclude "register", "active", and + * "version" (ALTER EXTENSION) which are mutating — those live under "manual". + */ +static CommandLine *inspect_monitor_subcommands[] = { + &monitor_get_command, + &monitor_parse_notification_command, + &monitor_node_state_command, + &monitor_formation_states_command, + NULL +}; + +static CommandLine inspect_monitor_commands = + make_command_set("monitor", + "Query the monitor's current state (read-only)", + NULL, NULL, NULL, inspect_monitor_subcommands); + +/* + * Aggregate the read-only do_* command sets under "inspect". + * + * service here is only "getpid" — inspecting PIDs of running sub-processes. + * "restart" and "pgctl on/off" mutate state and belong under "manual service". + */ +static CommandLine *inspect_subcommands[] = { + &do_show_commands, /* ipaddr / cidr / lookup / hostname / reverse */ + &do_pgsetup_commands, /* discover / ready / wait / logs / tune / pg_ctl */ + &inspect_fsm_commands, /* state / list / gv */ + &inspect_monitor_commands, /* get primary|others|candidate-count + parse-notification */ + &do_service_getpid_commands, /* getpid postgres|listener|node-active */ + NULL +}; + +CommandLine inspect_commands = + make_command_set("inspect", + "Read-only diagnostics (safe on live nodes)", + "[sub-command]", + " show Networking and hostname diagnostics\n" + " pgsetup Local PostgreSQL setup inspection\n" + " fsm Display keeper FSM state and reachable transitions\n" + " monitor Query the monitor's current state\n" + " getpid Get PIDs of pg_autoctl sub-processes\n", + NULL, inspect_subcommands); diff --git a/src/bin/pg_autoctl/cli_inspect.h b/src/bin/pg_autoctl/cli_inspect.h new file mode 100644 index 000000000..b9c87d68e --- /dev/null +++ b/src/bin/pg_autoctl/cli_inspect.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_inspect.h + * pg_autoctl inspect — read-only diagnostics. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_INSPECT_H +#define CLI_INSPECT_H + +#include "commandline.h" + +extern CommandLine inspect_commands; + +#endif /* CLI_INSPECT_H */ diff --git a/src/bin/pg_autoctl/cli_manual.c b/src/bin/pg_autoctl/cli_manual.c new file mode 100644 index 000000000..f309a948e --- /dev/null +++ b/src/bin/pg_autoctl/cli_manual.c @@ -0,0 +1,128 @@ +/* + * src/bin/pg_autoctl/cli_manual.c + * pg_autoctl manual — operator-driven FSM operations and low-level controls. + * + * These commands let an operator manually drive individual FSM transitions + * or low-level operations that pg_autoctl automation would normally own. + * Intended for manual recovery when the automated FSM is stopped or stuck, + * or for driving a live upgrade step-by-step. + * + * "manual" is the antonym of the automated behaviour pg_autoctl normally + * provides: you are doing by hand what the system would otherwise do for you. + * + * Read-only diagnostics belong in "pg_autoctl inspect": + * inspect fsm state / list / gv + * inspect monitor get / parse-notification + * inspect getpid postgres / listener / node-active + * + * Internal subprocess entry points belong in the hidden "pg_autoctl do". + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include "commandline.h" +#include "cli_manual.h" +#include "cli_do_root.h" + +/* + * manual fsm: mutating FSM operations only. + * + * Read-only FSM commands (state, list, gv) live under "pg_autoctl inspect fsm" + * because they are safe to run on a live node. + * + * nodes (get + set) is kept as a unit under manual because both sub-commands + * target the offline nodes file (--disable-monitor), making them equally + * operator-oriented regardless of read/write direction. + */ +static CommandLine *manual_fsm_subcommands[] = { + &fsm_init, + &fsm_assign, + &fsm_step, + &fsm_nodes, /* nodes get + nodes set (--disable-monitor file) */ + NULL +}; + +static CommandLine manual_fsm_commands = + make_command_set("fsm", + "Manually drive the keeper FSM (mutating operations)", + NULL, NULL, NULL, manual_fsm_subcommands); + +/* + * manual service: user-visible service controls only. + * + * getpid is read-only and lives under "pg_autoctl inspect getpid". + * + * Intentionally excludes the internal subprocess entry points + * (pgcontroller / postgres / listener / node-active) which are spawned by the + * supervisor via fork+exec and are not meant for direct operator use. + * Those live under the hidden "pg_autoctl internal service" group. + */ +static CommandLine *manual_service_subcommands[] = { + &do_service_restart_commands, /* restart postgres|listener|node-active */ + &do_service_postgres_ctl_commands, /* pgctl on|off */ + NULL +}; + +static CommandLine manual_service_commands = + make_command_set("service", + "Restart pg_autoctl sub-processes or signal the postgres controller", + NULL, NULL, NULL, manual_service_subcommands); + +/* + * manual monitor: mutating monitor operations. + * + * "get" and "parse-notification" are read-only and live under + * "pg_autoctl inspect monitor". Here we expose only the mutating calls: + * + * register — manually step through the node registration protocol + * active — manually call the node_active RPC on the monitor + * version — check monitor extension version and ALTER EXTENSION UPDATE if needed + */ +static CommandLine *manual_monitor_subcommands[] = { + &monitor_register_command, + &monitor_node_active_command, + &monitor_version_command, + NULL +}; + +static CommandLine manual_monitor_commands = + make_command_set("monitor", + "Manually drive monitor RPCs (register / active / version)", + NULL, NULL, NULL, manual_monitor_subcommands); + +/* + * manual coordinator: Citus coordinator metadata management. + * + * These commands let an operator manually replay the coordinator update steps + * that pg_autoctl normally drives automatically during a Citus worker failover: + * + * coordinator update prepare — call master_update_node() in a prepared + * transaction, which blocks shard writes while open + * coordinator update commit — COMMIT PREPARED: makes the new address permanent + * coordinator update rollback — ROLLBACK PREPARED: abandons a stuck prepare + * + * They are typically needed when the coordinator was unavailable during a + * failover and the prepared transaction needs to be resolved out-of-band. + */ +static CommandLine *manual_subcommands[] = { + &manual_fsm_commands, /* init / assign / step / nodes */ + &manual_service_commands, /* restart / pgctl on|off */ + &manual_monitor_commands, /* register / active / version */ + &do_primary_, /* slot create|drop / adduser monitor|replica / defaults / identify */ + &do_standby_, /* init / rewind / crash-recovery / promote */ + &do_coordinator_commands, /* add / activate / remove / update prepare|commit|rollback */ + NULL +}; + +CommandLine manual_commands = + make_command_set("manual", + "Manual FSM operations — drive by hand what automation normally does", + "[sub-command]", + " fsm Manually drive keeper FSM transitions\n" + " service Restart sub-processes or signal the postgres controller\n" + " monitor Manually drive monitor registration protocol\n" + " primary Manual primary-side PostgreSQL operations\n" + " standby Manual standby-side PostgreSQL operations\n" + " coordinator Citus coordinator metadata management\n", + NULL, manual_subcommands); diff --git a/src/bin/pg_autoctl/cli_manual.h b/src/bin/pg_autoctl/cli_manual.h new file mode 100644 index 000000000..d54ec7abd --- /dev/null +++ b/src/bin/pg_autoctl/cli_manual.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_manual.h + * pg_autoctl manual — operator-driven FSM operations and low-level controls. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_MANUAL_H +#define CLI_MANUAL_H + +#include "commandline.h" + +extern CommandLine manual_commands; + +#endif /* CLI_MANUAL_H */ diff --git a/src/bin/pg_autoctl/cli_node.c b/src/bin/pg_autoctl/cli_node.c new file mode 100644 index 000000000..639480743 --- /dev/null +++ b/src/bin/pg_autoctl/cli_node.c @@ -0,0 +1,534 @@ +/* + * src/bin/pg_autoctl/cli_node.c + * pg_autoctl node — declarative node lifecycle driven by a pg_autoctl_node.ini file. + * + * pg_autoctl node run Create (if needed) then run the supervisor. + * pg_autoctl node apply Converge mutable fields on a running node. + * pg_autoctl node show Dump current config as pg_autoctl_node.ini. + * pg_autoctl node check Validate the file without creating anything. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include +#include +#include + +#include "cli_common.h" +#include "cli_node.h" +#include "cli_root.h" +#include "commandline.h" +#include "defaults.h" +#include "env_utils.h" +#include "file_utils.h" +#include "keeper_config.h" +#include "log.h" +#include "nodespec.h" +#include "pgsetup.h" +#include "runprogram.h" +#include "string_utils.h" + + +static int cli_node_run_getopts(int argc, char **argv); +static void cli_node_run(int argc, char **argv); +static int cli_node_apply_getopts(int argc, char **argv); +static void cli_node_apply(int argc, char **argv); +static int cli_node_start_getopts(int argc, char **argv); +static void cli_node_start(int argc, char **argv); +static int cli_node_show_getopts(int argc, char **argv); +static void cli_node_show(int argc, char **argv); +static int cli_node_check_getopts(int argc, char **argv); +static void cli_node_check(int argc, char **argv); + + +static CommandLine node_run_command = + make_command( + "run", + "Create (if needed) and run a node described by a pg_autoctl_node.ini file", + "", + " path to the pg_autoctl_node.ini file\n" + " (default: " PG_AUTOCTL_NODESPEC_PATH ")\n", + cli_node_run_getopts, + cli_node_run); + +static CommandLine node_apply_command = + make_command( + "apply", + "Apply mutable settings from a pg_autoctl_node.ini to a running node", + "", + " path to the pg_autoctl_node.ini file\n", + cli_node_apply_getopts, + cli_node_apply); + +static CommandLine node_start_command = + make_command( + "start", + "Start a node waiting in launch=deferred mode (idempotent)", + "[]", + " path to the pg_autoctl_node.ini file\n" + " (default: " PG_AUTOCTL_NODESPEC_PATH ")\n", + cli_node_start_getopts, + cli_node_start); + +static CommandLine node_show_command = + make_command( + "show", + "Dump current node configuration as a pg_autoctl_node.ini file", + "[--pgdata ]", + " --pgdata location of the Postgres data directory\n", + cli_node_show_getopts, + cli_node_show); + +static CommandLine node_check_command = + make_command( + "check", + "Validate a pg_autoctl_node.ini file without creating anything", + "", + " path to the pg_autoctl_node.ini file\n", + cli_node_check_getopts, + cli_node_check); + +static CommandLine *node_subcommands[] = { + &node_run_command, + &node_apply_command, + &node_start_command, + &node_show_command, + &node_check_command, + NULL +}; + +CommandLine node_commands = + make_command_set( + "node", + "Declarative node lifecycle — create, run, and reconfigure from a .ini file", + NULL, NULL, NULL, node_subcommands); + + +/* ----------------------------------------------------------------------- + * Shared state + * ----------------------------------------------------------------------- */ + +static char nodeSpecPath[MAXPGPATH] = { 0 }; + + +/* ----------------------------------------------------------------------- + * pg_autoctl node run + * ----------------------------------------------------------------------- */ +static int +cli_node_run_getopts(int argc, char **argv) +{ + /* argv[0] is the subcommand name ("run"/"apply"/"check"); the optional + * file path is the first remaining positional argument at argv[1]. */ + if (argc > 1 && argv[1][0] != '-') + { + strlcpy(nodeSpecPath, argv[1], sizeof(nodeSpecPath)); + } + else + { + strlcpy(nodeSpecPath, PG_AUTOCTL_NODESPEC_PATH, sizeof(nodeSpecPath)); + } + + return 0; +} + + +/* + * cli_node_run reads the pg_autoctl_node.ini file, runs `pg_autoctl create + * --run` if PGDATA does not yet exist, or `pg_autoctl run` if it does. + * + * We exec() rather than call the internal functions directly so that: + * 1. The supervisor's fork+exec live-upgrade pattern is preserved. + * 2. The already-running supervisor (PID 1) can restart this process after + * a binary upgrade without special-casing the "node run" path. + */ +static void +cli_node_run(int argc, char **argv) +{ + NodeSpec spec = { 0 }; + char *args[40]; + int nargs; + + if (!nodespec_read(nodeSpecPath, &spec)) + { + exit(EXIT_CODE_BAD_CONFIG); + } + + /* + * [launch] mode = deferred: spin here re-reading nodeSpecPath until + * pg_autoctl node start rewrites it with mode = immediate (or removes + * the [launch] section). The file is the rendezvous point — no external + * sentinel needed. + */ + if (spec.launchDeferred) + { + log_info("Node configured with launch = deferred in \"%s\"; " + "waiting for pg_autoctl node start", nodeSpecPath); + + for (;;) + { + pg_usleep(500 * 1000); /* 0.5 s poll */ + + NodeSpec polled = { 0 }; + if (nodespec_read(nodeSpecPath, &polled) && !polled.launchDeferred) + { + spec = polled; + break; + } + } + + log_info("launch = immediate; proceeding with node initialization"); + } + + /* + * If PGDATA already has a pg_autoctl.cfg, the node was created before. + * In that case run `pg_autoctl run` (no --run, no create flags). + * Otherwise run `pg_autoctl create ... --run`. + */ + char cfgPath[MAXPGPATH]; + bool cfgExists = false; + + if (!IS_EMPTY_STRING_BUFFER(spec.pgdata)) + { + sformat(cfgPath, sizeof(cfgPath), + "%s/pg_autoctl.cfg", spec.pgdata); + cfgExists = file_exists(cfgPath); + } + + if (cfgExists) + { + /* + * Node was already created. Apply any mutable changes that might + * have been made to the spec file since the last run, then hand off + * to the normal run path. + */ + NodeSpec prev = { 0 }; + + /* best-effort: ignore errors if we can't re-read the spec */ + (void) nodespec_read(nodeSpecPath, &prev); + (void) nodespec_apply(&spec, &prev); + + args[0] = (char *) pg_autoctl_program; + args[1] = "run"; + args[2] = "--pgdata"; + args[3] = spec.pgdata; + args[4] = NULL; + nargs = 4; + } + else + { + nargs = nodespec_create_argv(&spec, pg_autoctl_program, + args, 32); + if (nargs < 0) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + } + + /* Tell the supervisor which spec file to watch for live changes */ + setenv("PG_AUTOCTL_NODESPEC", nodeSpecPath, 1); + + /* + * PG_AUTOCTL_TEST_DELAY: stagger node registration so that node IDs + * are assigned in name order (node1 → id 1, node2 → id 2, …). + * Extract the trailing integer from the node name and sleep 2×N seconds. + */ + if (env_exists("PG_AUTOCTL_TEST_DELAY") && spec.name[0] != '\0') + { + const char *p = spec.name + strlen(spec.name); + while (p > spec.name && isdigit((unsigned char) p[-1])) + { + p--; + } + if (*p != '\0') + { + int n = atoi(p) /* IGNORE-BANNED */; + int secs = 2 * n; + log_info("PG_AUTOCTL_TEST_DELAY: sleeping %ds before " + "registration (node %s, index %d)", + secs, spec.name, n); + sleep(secs); + } + } + + /* Log the command we're about to exec, masking password arguments. */ + { + PQExpBuffer cmd = createPQExpBuffer(); + static const char *pwFlags[] = { + "--monitor-password", + "--replication-password", + "--autoctl-node-password", + NULL + }; + for (int i = 0; i < nargs; i++) + { + if (i > 0) + { + appendPQExpBufferChar(cmd, ' '); + } + + /* Check if the previous arg was a password flag. */ + bool maskThis = false; + if (i > 0) + { + for (int k = 0; pwFlags[k]; k++) + { + if (strcmp(args[i - 1], pwFlags[k]) == 0) + { + maskThis = true; + break; + } + } + } + appendPQExpBufferStr(cmd, maskThis ? "****" : args[i]); + } + log_info("pg_autoctl node run: %s", cmd->data); + destroyPQExpBuffer(cmd); + } + + execv(args[0], args); + + /* If we get here execv failed */ + log_fatal("execv(\"%s\"): %m", args[0]); + exit(EXIT_CODE_INTERNAL_ERROR); +} + + +/* ----------------------------------------------------------------------- + * pg_autoctl node apply + * ----------------------------------------------------------------------- */ +static int +cli_node_apply_getopts(int argc, char **argv) +{ + if (argc > 0 && argv[0][0] != '-') + { + strlcpy(nodeSpecPath, argv[0], sizeof(nodeSpecPath)); + } + else + { + log_error("pg_autoctl node apply requires a file argument"); + exit(EXIT_CODE_BAD_ARGS); + } + return 0; +} + + +static void +cli_node_apply(int argc, char **argv) +{ + NodeSpec new_spec = { 0 }; + NodeSpec cur_spec = { 0 }; + + if (!nodespec_read(nodeSpecPath, &new_spec)) + { + exit(EXIT_CODE_BAD_CONFIG); + } + + /* Read the current spec from the same path as a baseline */ + (void) nodespec_read(nodeSpecPath, &cur_spec); + + if (!nodespec_apply(&new_spec, &cur_spec)) + { + log_error("Failed to apply node spec from \"%s\"", nodeSpecPath); + exit(EXIT_CODE_INTERNAL_ERROR); + } +} + + +/* ----------------------------------------------------------------------- + * pg_autoctl node start [] + * + * Clears launch = deferred in the spec file so a waiting pg_autoctl node run + * proceeds. Idempotent: if the node is already immediate, exits 0 quietly. + * ----------------------------------------------------------------------- */ +static int +cli_node_start_getopts(int argc, char **argv) +{ + if (argc > 1 && argv[1][0] != '-') + { + strlcpy(nodeSpecPath, argv[1], sizeof(nodeSpecPath)); + } + else + { + strlcpy(nodeSpecPath, PG_AUTOCTL_NODESPEC_PATH, sizeof(nodeSpecPath)); + } + + return 0; +} + + +static void +cli_node_start(int argc, char **argv) +{ + NodeSpec spec = { 0 }; + + if (!nodespec_read(nodeSpecPath, &spec)) + { + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!spec.launchDeferred) + { + log_info("Node \"%s\" launch is already immediate; nothing to do", + nodeSpecPath); + exit(0); + } + + spec.launchDeferred = false; + + if (!nodespec_write_to_path(&spec, nodeSpecPath)) + { + log_error("Failed to update \"%s\"", nodeSpecPath); + exit(EXIT_CODE_INTERNAL_ERROR); + } + + log_info("Cleared launch = deferred in \"%s\"; node will now start", + nodeSpecPath); +} + + +/* ----------------------------------------------------------------------- + * pg_autoctl node show [--pgdata ] + * ----------------------------------------------------------------------- */ +static int +cli_node_show_getopts(int argc, char **argv) +{ + /* honour --pgdata from the global keeperOptions */ + return cli_getopt_pgdata(argc, argv); +} + + +static void +cli_node_show(int argc, char **argv) +{ + /* + * Build a NodeSpec from the running keeper/monitor config and emit it. + * We read the existing pg_autoctl.cfg rather than the node spec file so + * that `pg_autoctl node show` reflects the live configuration. + */ + KeeperConfig config = keeperOptions; + + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = true; + + if (!keeper_config_read_file(&config, + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + log_error("Failed to read pg_autoctl configuration"); + exit(EXIT_CODE_BAD_CONFIG); + } + + NodeSpec spec = { 0 }; + + spec.kind = config.pgSetup.pgKind; + strlcpy(spec.pgdata, config.pgSetup.pgdata, sizeof(spec.pgdata)); + strlcpy(spec.hostname, config.hostname, sizeof(spec.hostname)); + spec.port = config.pgSetup.pgport; + strlcpy(spec.monitor_pguri, config.monitor_pguri, + sizeof(spec.monitor_pguri)); + strlcpy(spec.formation, config.formation, sizeof(spec.formation)); + spec.group = config.groupId; + + /* Mutable settings defaults — we'd need a monitor round-trip for live values */ + spec.candidate_priority = 50; + spec.replication_quorum = true; + + /* Create-time defaults */ + strlcpy(spec.ssl, "self-signed", sizeof(spec.ssl)); + strlcpy(spec.auth, "trust", sizeof(spec.auth)); + spec.pg_hba_lan = true; + + (void) nodespec_write(&spec, stdout); +} + + +/* ----------------------------------------------------------------------- + * pg_autoctl node check + * ----------------------------------------------------------------------- */ +static int +cli_node_check_getopts(int argc, char **argv) +{ + if (argc > 0 && argv[0][0] != '-') + { + strlcpy(nodeSpecPath, argv[0], sizeof(nodeSpecPath)); + } + else + { + log_error("pg_autoctl node check requires a file argument"); + exit(EXIT_CODE_BAD_ARGS); + } + return 0; +} + + +static void +cli_node_check(int argc, char **argv) +{ + NodeSpec spec = { 0 }; + + if (!nodespec_read(nodeSpecPath, &spec)) + { + log_error("Invalid node spec file \"%s\"", nodeSpecPath); + exit(EXIT_CODE_BAD_CONFIG); + } + + const char *kindStr; + switch (spec.kind) + { + case NODE_KIND_UNKNOWN: + { + kindStr = "monitor"; + break; + } + + case NODE_KIND_STANDALONE: + { + kindStr = "postgres"; + break; + } + + case NODE_KIND_CITUS_COORDINATOR: + { + kindStr = "coordinator"; + break; + } + + case NODE_KIND_CITUS_WORKER: + { + kindStr = "worker"; + break; + } + + default: + { + kindStr = "unknown"; + break; + } + } + + fformat(stdout, "Node spec \"%s\" is valid.\n", nodeSpecPath); + fformat(stdout, " kind : %s\n", kindStr); + fformat(stdout, " pgdata : %s\n", spec.pgdata); + fformat(stdout, " hostname : %s\n", spec.hostname); + fformat(stdout, " port : %d\n", spec.port); + + if (spec.kind != NODE_KIND_UNKNOWN) + { + fformat(stdout, " monitor_pguri : %s\n", spec.monitor_pguri); + fformat(stdout, " formation : %s\n", spec.formation); + fformat(stdout, " group : %d\n", spec.group); + } + + fformat(stdout, " candidate_priority : %d\n", spec.candidate_priority); + fformat(stdout, " replication_quorum : %s\n", + spec.replication_quorum ? "true" : "false"); + fformat(stdout, " ssl : %s\n", spec.ssl); + fformat(stdout, " auth : %s\n", spec.auth); + fformat(stdout, " pg_hba_lan : %s\n", + spec.pg_hba_lan ? "true" : "false"); +} diff --git a/src/bin/pg_autoctl/cli_node.h b/src/bin/pg_autoctl/cli_node.h new file mode 100644 index 000000000..c236eeada --- /dev/null +++ b/src/bin/pg_autoctl/cli_node.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_node.h + * pg_autoctl node — declarative node lifecycle from a .ini file. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_NODE_H +#define CLI_NODE_H + +#include "commandline.h" + +extern CommandLine node_commands; + +#endif /* CLI_NODE_H */ diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 386716b98..ffe11cbe3 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -9,6 +9,10 @@ */ #include "cli_common.h" +#include "cli_do_root.h" +#include "cli_inspect.h" +#include "cli_manual.h" +#include "cli_node.h" #include "cli_root.h" #include "commandline.h" @@ -81,18 +85,24 @@ CommandLine drop_commands = NULL, drop_subcommands); /* - * Binding them all into the top-level command: + * Single root command table — inspect and override are always visible. + * The PG_AUTOCTL_DEBUG env var now only controls log verbosity, not + * command visibility. */ -CommandLine *root_subcommands_with_debug[] = { +CommandLine *root_subcommands[] = { &create_commands, &drop_commands, &config_commands, - &show_commands_with_debug, + &show_commands, &enable_commands, &disable_commands, &get_commands, &set_commands, &perform_commands, + &activate_node_command, + &inspect_commands, + &manual_commands, + &node_commands, &do_commands, &service_run_command, &watch_command, @@ -104,14 +114,15 @@ CommandLine *root_subcommands_with_debug[] = { NULL }; -CommandLine root_with_debug = +CommandLine root = make_command_set("pg_autoctl", "pg_auto_failover control tools and service", - "[ --debug|verbose|quiet ]", NULL, - root_options, root_subcommands_with_debug); - + "[ --verbose --quiet ]", NULL, + root_options, root_subcommands); -CommandLine *root_subcommands[] = { +/* Backward-compat alias: root_with_debug and root_subcommands_with_debug + * previously held the do_commands. They now point to the same root. */ +CommandLine *root_subcommands_with_debug[] = { &create_commands, &drop_commands, &config_commands, @@ -122,6 +133,10 @@ CommandLine *root_subcommands[] = { &set_commands, &perform_commands, &activate_node_command, + &inspect_commands, + &manual_commands, + &node_commands, + &do_commands, &service_run_command, &watch_command, &service_stop_command, @@ -132,11 +147,11 @@ CommandLine *root_subcommands[] = { NULL }; -CommandLine root = +CommandLine root_with_debug = make_command_set("pg_autoctl", "pg_auto_failover control tools and service", "[ --verbose --quiet ]", NULL, - root_options, root_subcommands); + root_options, root_subcommands_with_debug); /* diff --git a/src/bin/pg_autoctl/config.c b/src/bin/pg_autoctl/config.c index 4a9f600f1..898867a53 100644 --- a/src/bin/pg_autoctl/config.c +++ b/src/bin/pg_autoctl/config.c @@ -70,10 +70,11 @@ build_xdg_path(char *dst, } default: - + { /* developper error */ log_error("No support for XDG Resource Type %d", xdgType); return false; + } } if (!get_env_copy_with_fallback(envVarName, xdg_topdir, MAXPGPATH, fallback)) diff --git a/src/bin/pg_autoctl/coordinator.c b/src/bin/pg_autoctl/coordinator.c index 4845aa96c..84bd34f29 100644 --- a/src/bin/pg_autoctl/coordinator.c +++ b/src/bin/pg_autoctl/coordinator.c @@ -160,9 +160,12 @@ coordinator_add_node(Coordinator *coordinator, Keeper *keeper, SingleValueResultContext parseContext = { { 0 }, PGSQL_RESULT_INT, false }; + IntString pgportStr = intToString(keeper->config.pgSetup.pgport); + IntString groupIdStr = intToString(keeper->config.groupId); + paramValues[0] = keeper->config.hostname; - paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; - paramValues[2] = intToString(keeper->config.groupId).strValue; + paramValues[1] = pgportStr.strValue; + paramValues[2] = groupIdStr.strValue; paramValues[3] = citusRoleStr; paramValues[4] = clusterName; @@ -247,9 +250,12 @@ coordinator_add_inactive_node(Coordinator *coordinator, Keeper *keeper, sql = sqlInteger; } + IntString pgportStr = intToString(keeper->config.pgSetup.pgport); + IntString groupIdStr = intToString(keeper->config.groupId); + paramValues[0] = keeper->config.hostname; - paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; - paramValues[2] = intToString(keeper->config.groupId).strValue; + paramValues[1] = pgportStr.strValue; + paramValues[2] = groupIdStr.strValue; paramValues[3] = citusRoleStr; paramValues[4] = clusterName; @@ -323,8 +329,10 @@ coordinator_activate_node(Coordinator *coordinator, Keeper *keeper, sql = sqlInteger; } + IntString pgportStr = intToString(keeper->config.pgSetup.pgport); + paramValues[0] = keeper->config.hostname; - paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; + paramValues[1] = pgportStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -372,8 +380,10 @@ coordinator_remove_node(Coordinator *coordinator, Keeper *keeper) Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2]; + IntString pgportStr = intToString(keeper->config.pgSetup.pgport); + paramValues[0] = keeper->config.hostname; - paramValues[1] = intToString(keeper->config.pgSetup.pgport).strValue; + paramValues[1] = pgportStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -621,12 +631,16 @@ coordinator_update_node_prepare(Coordinator *coordinator, Keeper *keeper) " and not exists" " (select 1 from pg_prepared_xacts where gid = $4)"); - paramValues[0] = intToString(groupId).strValue; + IntString groupIdStr = intToString(groupId); + IntString pgportStr = intToString(keeper->config.pgSetup.pgport); + IntString cooldownStr = intToString( + keeper->config.citus_master_update_node_lock_cooldown); + + paramValues[0] = groupIdStr.strValue; paramValues[1] = keeper->config.hostname; - paramValues[2] = intToString(keeper->config.pgSetup.pgport).strValue; + paramValues[2] = pgportStr.strValue; paramValues[3] = transactionName; - paramValues[4] = intToString( - keeper->config.citus_master_update_node_lock_cooldown).strValue; + paramValues[4] = cooldownStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -652,9 +666,12 @@ coordinator_update_node_prepare(Coordinator *coordinator, Keeper *keeper) " and not exists" " (select 1 from pg_prepared_xacts where gid = $4)"); - paramValues[0] = intToString(groupId).strValue; + IntString groupIdStr = intToString(groupId); + IntString pgportStr = intToString(keeper->config.pgSetup.pgport); + + paramValues[0] = groupIdStr.strValue; paramValues[1] = keeper->config.hostname; - paramValues[2] = intToString(keeper->config.pgSetup.pgport).strValue; + paramValues[2] = pgportStr.strValue; paramValues[3] = transactionName; if (!pgsql_execute_with_params(pgsql, sql, @@ -805,7 +822,9 @@ coordinator_upsert_poolinfo_port(Coordinator *coordinator, Keeper *keeper) sformat(proxyInfo, sizeof(proxyInfo), "host=%s port=%d", keeper->config.hostname, keeper->config.pgSetup.proxyport); - paramValues[0] = intToString(keeper->config.groupId).strValue; + IntString groupIdStr = intToString(keeper->config.groupId); + + paramValues[0] = groupIdStr.strValue; paramValues[1] = proxyInfo; if (!pgsql_execute_with_params(pgsql, sql, diff --git a/src/bin/pg_autoctl/demoapp.c b/src/bin/pg_autoctl/demoapp.c index 609c75c97..dd9cbbdf5 100644 --- a/src/bin/pg_autoctl/demoapp.c +++ b/src/bin/pg_autoctl/demoapp.c @@ -482,10 +482,14 @@ demoapp_register_client(const char *pguri, const Oid paramTypes[4] = { INT4OID, INT4OID, INT4OID, INT4OID }; const char *paramValues[4] = { 0 }; - paramValues[0] = intToString(clientId).strValue; - paramValues[1] = intToString(getpid()).strValue; - paramValues[2] = intToString(retrySleep).strValue; - paramValues[3] = intToString(retryCap).strValue; + IntString clientIdStr = intToString(clientId); + IntString pidStr = intToString(getpid()); + IntString retrySleepStr = intToString(retrySleep); + IntString retryCapStr = intToString(retryCap); + paramValues[0] = clientIdStr.strValue; + paramValues[1] = pidStr.strValue; + paramValues[2] = retrySleepStr.strValue; + paramValues[3] = retryCapStr.strValue; pgsql_init(&pgsql, (char *) pguri, PGSQL_CONN_APP); @@ -519,8 +523,10 @@ demoapp_update_client_failovers(const char *pguri, int clientId, int failovers) const Oid paramTypes[2] = { INT4OID, INT4OID }; const char *paramValues[2] = { 0 }; - paramValues[0] = intToString(clientId).strValue; - paramValues[1] = intToString(failovers).strValue; + IntString clientIdStr = intToString(clientId); + IntString failoversStr = intToString(failovers); + paramValues[0] = clientIdStr.strValue; + paramValues[1] = failoversStr.strValue; pgsql_init(&pgsql, (char *) pguri, PGSQL_CONN_APP); @@ -699,10 +705,14 @@ demoapp_start_client(const char *pguri, int clientId, const Oid paramTypes[5] = { INT4OID, INT4OID, INT8OID, INT8OID, BOOLOID }; const char *paramValues[5] = { 0 }; - paramValues[0] = intToString(clientId).strValue; - paramValues[1] = intToString(index).strValue; - paramValues[2] = intToString(pgsql.retryPolicy.attempts).strValue; - paramValues[3] = intToString(INSTR_TIME_GET_MICROSEC(duration)).strValue; + IntString clientIdStr = intToString(clientId); + IntString indexStr = intToString(index); + IntString attemptsStr = intToString(pgsql.retryPolicy.attempts); + IntString durationStr = intToString(INSTR_TIME_GET_MICROSEC(duration)); + paramValues[0] = clientIdStr.strValue; + paramValues[1] = indexStr.strValue; + paramValues[2] = attemptsStr.strValue; + paramValues[3] = durationStr.strValue; paramValues[4] = is_in_recovery ? "true" : "false"; if (!pgsql_execute_with_params(&pgsql, sql, 5, paramTypes, paramValues, diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index b5dcc523a..9680b6d81 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -201,372 +201,526 @@ KeeperFSMTransition KeeperFSM[] = { /* * Started as a single, no nothing */ - { INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_INIT_TO_SINGLE, - &fsm_citus_coordinator_init_primary }, + { + INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_INIT_TO_SINGLE, + &fsm_citus_coordinator_init_primary + }, - { INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_INIT_TO_SINGLE, - &fsm_citus_worker_init_primary }, + { + INIT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_INIT_TO_SINGLE, + &fsm_citus_worker_init_primary + }, - { INIT_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_SINGLE, - &fsm_init_primary }, + { + INIT_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_SINGLE, + &fsm_init_primary + }, - { DROPPED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_INIT_TO_SINGLE, - &fsm_citus_worker_init_primary }, + { + DROPPED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_INIT_TO_SINGLE, + &fsm_citus_worker_init_primary + }, - { DROPPED_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_SINGLE, - &fsm_init_primary }, + { + DROPPED_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_SINGLE, + &fsm_init_primary + }, - { DROPPED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_DROPPED_TO_REPORT_LSN, - &fsm_init_from_standby }, + { + DROPPED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_DROPPED_TO_REPORT_LSN, + &fsm_init_from_standby + }, /* * other node(s) was forcibly removed, now single */ - { PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, - { WAIT_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + WAIT_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, - { JOIN_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + JOIN_PRIMARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, /* * failover occurred, primary -> draining/demoted */ - { PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DRAINING, - &fsm_stop_postgres }, + { + PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DRAINING, + &fsm_stop_postgres + }, - { DRAINING_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_DRAINING_TO_DEMOTED, - &fsm_stop_postgres }, + { + DRAINING_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_DRAINING_TO_DEMOTED, + &fsm_stop_postgres + }, - { PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { JOIN_PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DRAINING, - &fsm_stop_postgres }, + { + JOIN_PRIMARY_STATE, DRAINING_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DRAINING, + &fsm_stop_postgres + }, - { JOIN_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + JOIN_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { JOIN_PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + JOIN_PRIMARY_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { APPLY_SETTINGS_STATE, DRAINING_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DRAINING, - &fsm_stop_postgres }, + { + APPLY_SETTINGS_STATE, DRAINING_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DRAINING, + &fsm_stop_postgres + }, - { APPLY_SETTINGS_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + APPLY_SETTINGS_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, - { APPLY_SETTINGS_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + APPLY_SETTINGS_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, /* * primary is put to maintenance */ - { PRIMARY_STATE, PREPARE_MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_PREPARE_MAINTENANCE, - &fsm_stop_postgres_for_primary_maintenance }, + { + PRIMARY_STATE, PREPARE_MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_PREPARE_MAINTENANCE, + &fsm_stop_postgres_for_primary_maintenance + }, - { PREPARE_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_MAINTENANCE, - &fsm_stop_postgres_and_setup_standby }, + { + PREPARE_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_MAINTENANCE, + &fsm_stop_postgres_and_setup_standby + }, - { PRIMARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_MAINTENANCE, - &fsm_stop_postgres_for_primary_maintenance }, + { + PRIMARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_MAINTENANCE, + &fsm_stop_postgres_for_primary_maintenance + }, /* * was demoted, need to be dead now. */ - { DRAINING_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, - COMMENT_DRAINING_TO_DEMOTE_TIMEOUT, - &fsm_stop_postgres }, + { + DRAINING_STATE, DEMOTE_TIMEOUT_STATE, NODE_KIND_ANY, + COMMENT_DRAINING_TO_DEMOTE_TIMEOUT, + &fsm_stop_postgres + }, - { DEMOTE_TIMEOUT_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_DEMOTE_TIMEOUT_TO_DEMOTED, - &fsm_stop_postgres }, + { + DEMOTE_TIMEOUT_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_DEMOTE_TIMEOUT_TO_DEMOTED, + &fsm_stop_postgres + }, /* * wait_primary stops reporting, is (supposed) dead now */ - { WAIT_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_DEMOTED, - &fsm_stop_postgres }, + { + WAIT_PRIMARY_STATE, DEMOTED_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_DEMOTED, + &fsm_stop_postgres + }, /* * was demoted after a failure, but standby was forcibly removed */ - { DEMOTED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_citus_worker_resume_as_primary }, + { + DEMOTED_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_citus_worker_resume_as_primary + }, - { DEMOTED_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_resume_as_primary }, + { + DEMOTED_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_resume_as_primary + }, - { DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_citus_worker_resume_as_primary }, + { + DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_citus_worker_resume_as_primary + }, - { DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_resume_as_primary }, + { + DEMOTE_TIMEOUT_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_resume_as_primary + }, - { DRAINING_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_citus_worker_resume_as_primary }, + { + DRAINING_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_citus_worker_resume_as_primary + }, - { DRAINING_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_SINGLE, - &fsm_resume_as_primary }, + { + DRAINING_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_SINGLE, + &fsm_resume_as_primary + }, /* * primary was forcibly removed */ - { SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_LOST_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_single }, + { + SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_LOST_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_single + }, - { SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_LOST_PRIMARY, - &fsm_citus_worker_promote_standby_to_single }, + { + SECONDARY_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_LOST_PRIMARY, + &fsm_citus_worker_promote_standby_to_single + }, - { SECONDARY_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_LOST_PRIMARY, - &fsm_promote_standby }, + { + SECONDARY_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_LOST_PRIMARY, + &fsm_promote_standby + }, - { CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_LOST_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_single }, + { + CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_LOST_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_single + }, - { CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_LOST_PRIMARY, - &fsm_citus_worker_promote_standby_to_single }, + { + CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_LOST_PRIMARY, + &fsm_citus_worker_promote_standby_to_single + }, - { CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_LOST_PRIMARY, - &fsm_promote_standby }, + { + CATCHINGUP_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_LOST_PRIMARY, + &fsm_promote_standby + }, - { PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_LOST_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_single }, + { + PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_LOST_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_single + }, - { PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_LOST_PRIMARY, - &fsm_citus_worker_promote_standby_to_single }, + { + PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_LOST_PRIMARY, + &fsm_citus_worker_promote_standby_to_single + }, - { PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_LOST_PRIMARY, - &fsm_promote_standby }, + { + PREP_PROMOTION_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_LOST_PRIMARY, + &fsm_promote_standby + }, /* * went down to force the primary to time out, but then it was removed */ - { STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_REPLICATION_TO_SINGLE, - &fsm_citus_worker_promote_standby_to_single }, + { + STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_REPLICATION_TO_SINGLE, + &fsm_citus_worker_promote_standby_to_single + }, - { STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_REPLICATION_TO_SINGLE, - &fsm_promote_standby }, + { + STOP_REPLICATION_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_REPLICATION_TO_SINGLE, + &fsm_promote_standby + }, /* * all states should lead to SINGLE, including REPORT_LSN */ - { REPORT_LSN_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_SINGLE, - &fsm_promote_standby }, + { + REPORT_LSN_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_SINGLE, + &fsm_promote_standby + }, /* * On the Primary, wait for a standby to be ready: WAIT_PRIMARY */ - { SINGLE_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_SINGLE_TO_WAIT_PRIMARY, - &fsm_prepare_replication }, + { + SINGLE_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_SINGLE_TO_WAIT_PRIMARY, + &fsm_prepare_replication + }, - { PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_JOIN_PRIMARY, - &fsm_prepare_replication }, + { + PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_JOIN_PRIMARY, + &fsm_prepare_replication + }, - { PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_WAIT_PRIMARY, - &fsm_disable_sync_rep }, + { + PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_WAIT_PRIMARY, + &fsm_disable_sync_rep + }, - { JOIN_PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_WAIT_PRIMARY, - &fsm_disable_sync_rep }, + { + JOIN_PRIMARY_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_WAIT_PRIMARY, + &fsm_disable_sync_rep + }, - { WAIT_PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_JOIN_PRIMARY, - &fsm_prepare_replication }, + { + WAIT_PRIMARY_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_JOIN_PRIMARY, + &fsm_prepare_replication + }, /* * Situation is getting back to normal on the primary */ - { WAIT_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_WAIT_PRIMARY_TO_PRIMARY, - &fsm_enable_sync_rep }, + { + WAIT_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_WAIT_PRIMARY_TO_PRIMARY, + &fsm_enable_sync_rep + }, - { JOIN_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_JOIN_PRIMARY_TO_PRIMARY, - &fsm_enable_sync_rep }, + { + JOIN_PRIMARY_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_JOIN_PRIMARY_TO_PRIMARY, + &fsm_enable_sync_rep + }, - { DEMOTE_TIMEOUT_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_DEMOTE_TO_PRIMARY, - &fsm_start_postgres }, + { + DEMOTE_TIMEOUT_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_DEMOTE_TO_PRIMARY, + &fsm_start_postgres + }, /* * The primary is now ready to accept a standby, we're the standby */ - { WAIT_STANDBY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_WAIT_STANDBY_TO_CATCHINGUP, - &fsm_init_standby }, + { + WAIT_STANDBY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_WAIT_STANDBY_TO_CATCHINGUP, + &fsm_init_standby + }, - { DEMOTED_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_CATCHINGUP, - &fsm_rewind_or_init }, + { + DEMOTED_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_CATCHINGUP, + &fsm_rewind_or_init + }, - { SECONDARY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_CATCHINGUP, - &fsm_follow_new_primary }, + { + SECONDARY_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_CATCHINGUP, + &fsm_follow_new_primary + }, /* * We're asked to be a standby. */ - { CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_CITUS_ANY, - COMMENT_CATCHINGUP_TO_SECONDARY, - &fsm_citus_maintain_replication_slots }, + { + CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_CITUS_ANY, + COMMENT_CATCHINGUP_TO_SECONDARY, + &fsm_citus_maintain_replication_slots + }, - { CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_CATCHINGUP_TO_SECONDARY, - &fsm_prepare_for_secondary }, + { + CATCHINGUP_STATE, SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_CATCHINGUP_TO_SECONDARY, + &fsm_prepare_for_secondary + }, /* * The standby is asked to prepare its own promotion */ - { SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_citus_worker_prepare_standby_for_promotion }, + { + SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_citus_worker_prepare_standby_for_promotion + }, - { SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_prepare_standby_for_promotion }, + { + SECONDARY_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_prepare_standby_for_promotion + }, - { CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_citus_worker_prepare_standby_for_promotion }, + { + CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_citus_worker_prepare_standby_for_promotion + }, - { CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_PREP_PROMOTION, - &fsm_prepare_standby_for_promotion }, + { + CATCHINGUP_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_PREP_PROMOTION, + &fsm_prepare_standby_for_promotion + }, /* * Forcefully stop replication by stopping the server. */ - { PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_PROMOTION_TO_STOP_REPLICATION, - &fsm_citus_worker_stop_replication }, + { + PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_PROMOTION_TO_STOP_REPLICATION, + &fsm_citus_worker_stop_replication + }, - { PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_ANY, - COMMENT_PROMOTION_TO_STOP_REPLICATION, - &fsm_stop_replication }, + { + PREP_PROMOTION_STATE, STOP_REPLICATION_STATE, NODE_KIND_ANY, + COMMENT_PROMOTION_TO_STOP_REPLICATION, + &fsm_stop_replication + }, /* * finish the promotion */ - { STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_COORDINATOR, - COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, - &fsm_citus_coordinator_promote_standby_to_primary }, + { + STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_COORDINATOR, + COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, + &fsm_citus_coordinator_promote_standby_to_primary + }, - { STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, - &fsm_citus_worker_promote_standby_to_primary }, + { + STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, + &fsm_citus_worker_promote_standby_to_primary + }, - { STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, - &fsm_promote_standby_to_primary }, + { + STOP_REPLICATION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_STOP_REPLICATION_TO_WAIT_PRIMARY, + &fsm_promote_standby_to_primary + }, - { PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_BLOCKED_WRITES, - &fsm_citus_worker_promote_standby }, + { + PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_BLOCKED_WRITES, + &fsm_citus_worker_promote_standby + }, - { PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_BLOCKED_WRITES, - &fsm_promote_standby }, + { + PREP_PROMOTION_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_BLOCKED_WRITES, + &fsm_promote_standby + }, /* * Just wait until primary is ready */ - { INIT_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_WAIT_STANDBY, - NULL }, + { + INIT_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_WAIT_STANDBY, + NULL + }, - { DROPPED_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_WAIT_STANDBY, - NULL }, + { + DROPPED_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_WAIT_STANDBY, + NULL + }, /* * When losing a monitor and then connecting to a new monitor as a * secondary, we need to be able to follow the init sequence again. */ - { SECONDARY_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, - COMMENT_SECONARY_TO_WAIT_STANDBY, - NULL }, + { + SECONDARY_STATE, WAIT_STANDBY_STATE, NODE_KIND_ANY, + COMMENT_SECONARY_TO_WAIT_STANDBY, + NULL + }, /* * In case of maintenance of the standby server, we stop PostgreSQL. */ - { SECONDARY_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, - NULL }, + { + SECONDARY_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, + NULL + }, - { CATCHINGUP_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, - NULL }, + { + CATCHINGUP_STATE, WAIT_MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_WAIT_MAINTENANCE, + NULL + }, - { SECONDARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_MAINTENANCE, - &fsm_start_maintenance_on_standby }, + { + SECONDARY_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_MAINTENANCE, + &fsm_start_maintenance_on_standby + }, - { CATCHINGUP_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_MAINTENANCE, - &fsm_start_maintenance_on_standby }, + { + CATCHINGUP_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_MAINTENANCE, + &fsm_start_maintenance_on_standby + }, - { WAIT_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_MAINTENANCE, - &fsm_start_maintenance_on_standby }, + { + WAIT_MAINTENANCE_STATE, MAINTENANCE_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_MAINTENANCE, + &fsm_start_maintenance_on_standby + }, - { MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_MAINTENANCE_TO_CATCHINGUP, - &fsm_restart_standby }, + { + MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_MAINTENANCE_TO_CATCHINGUP, + &fsm_restart_standby + }, - { PREPARE_MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, - COMMENT_MAINTENANCE_TO_CATCHINGUP, - &fsm_restart_standby }, + { + PREPARE_MAINTENANCE_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, + COMMENT_MAINTENANCE_TO_CATCHINGUP, + &fsm_restart_standby + }, /* * Applying new replication/cluster settings (per node replication quorum, @@ -574,119 +728,167 @@ KeeperFSMTransition KeeperFSM[] = { * have to fetch the new value for synchronous_standby_names from the * monitor. */ - { PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_APPLY_SETTINGS, - NULL }, + { + PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_APPLY_SETTINGS, + NULL + }, - { WAIT_PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_APPLY_SETTINGS, - NULL }, + { + WAIT_PRIMARY_STATE, APPLY_SETTINGS_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_APPLY_SETTINGS, + NULL + }, - { APPLY_SETTINGS_STATE, PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_APPLY_SETTINGS_TO_PRIMARY, - &fsm_enable_sync_rep }, + { + APPLY_SETTINGS_STATE, PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_APPLY_SETTINGS_TO_PRIMARY, + &fsm_enable_sync_rep + }, - { APPLY_SETTINGS_STATE, SINGLE_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_SINGLE, - &fsm_disable_replication }, + { + APPLY_SETTINGS_STATE, SINGLE_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_SINGLE, + &fsm_disable_replication + }, - { APPLY_SETTINGS_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_WAIT_PRIMARY, - &fsm_disable_sync_rep }, + { + APPLY_SETTINGS_STATE, WAIT_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_WAIT_PRIMARY, + &fsm_disable_sync_rep + }, - { APPLY_SETTINGS_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, - COMMENT_PRIMARY_TO_JOIN_PRIMARY, - &fsm_prepare_replication }, + { + APPLY_SETTINGS_STATE, JOIN_PRIMARY_STATE, NODE_KIND_ANY, + COMMENT_PRIMARY_TO_JOIN_PRIMARY, + &fsm_prepare_replication + }, /* * In case of multiple standbys, failover begins with reporting current LSN */ - { SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + SECONDARY_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { CATCHINGUP_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + CATCHINGUP_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { PREPARE_MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_SECONDARY_TO_REPORT_LSN, - &fsm_report_lsn }, + { + PREPARE_MAINTENANCE_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_SECONDARY_TO_REPORT_LSN, + &fsm_report_lsn + }, - { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, - COMMENT_REPORT_LSN_TO_PREP_PROMOTION, - &fsm_citus_worker_prepare_standby_for_promotion }, + { + REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_WORKER, + COMMENT_REPORT_LSN_TO_PREP_PROMOTION, + &fsm_citus_worker_prepare_standby_for_promotion + }, - { REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_PREP_PROMOTION, - &fsm_prepare_standby_for_promotion }, + { + REPORT_LSN_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_PREP_PROMOTION, + &fsm_prepare_standby_for_promotion + }, - { REPORT_LSN_STATE, FAST_FORWARD_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_FAST_FORWARD, - &fsm_fast_forward }, + { + REPORT_LSN_STATE, FAST_FORWARD_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_FAST_FORWARD, + &fsm_fast_forward + }, - { FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_ANY, - COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, - &fsm_citus_cleanup_and_resume_as_primary }, + { + FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_CITUS_ANY, + COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, + &fsm_citus_cleanup_and_resume_as_primary + }, - { FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, - COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, - &fsm_cleanup_as_primary }, + { + FAST_FORWARD_STATE, PREP_PROMOTION_STATE, NODE_KIND_ANY, + COMMENT_FAST_FORWARD_TO_PREP_PROMOTION, + &fsm_cleanup_as_primary + }, - { REPORT_LSN_STATE, JOIN_SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, - &fsm_checkpoint_and_stop_postgres }, + { + REPORT_LSN_STATE, JOIN_SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, + &fsm_checkpoint_and_stop_postgres + }, - { REPORT_LSN_STATE, SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, - &fsm_follow_new_primary }, + { + REPORT_LSN_STATE, SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_REPORT_LSN_TO_JOIN_SECONDARY, + &fsm_follow_new_primary + }, - { JOIN_SECONDARY_STATE, SECONDARY_STATE, NODE_KIND_ANY, - COMMENT_JOIN_SECONDARY_TO_SECONDARY, - &fsm_follow_new_primary }, + { + JOIN_SECONDARY_STATE, SECONDARY_STATE, NODE_KIND_ANY, + COMMENT_JOIN_SECONDARY_TO_SECONDARY, + &fsm_follow_new_primary + }, /* * When an old primary gets back online and reaches draining/draining, if a * failover is on-going then have it join the selection process. */ - { DRAINING_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_DRAINING_TO_REPORT_LSN, - &fsm_report_lsn_and_drop_replication_slots }, + { + DRAINING_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_DRAINING_TO_REPORT_LSN, + &fsm_report_lsn_and_drop_replication_slots + }, - { DEMOTED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_DEMOTED_TO_REPORT_LSN, - &fsm_report_lsn_and_drop_replication_slots }, + { + DEMOTED_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_DEMOTED_TO_REPORT_LSN, + &fsm_report_lsn_and_drop_replication_slots + }, /* * When adding a new node and there is no primary, but there are existing * nodes that are not candidates for failover. */ - { INIT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, - COMMENT_INIT_TO_REPORT_LSN, - &fsm_init_from_standby }, + { + INIT_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + COMMENT_INIT_TO_REPORT_LSN, + &fsm_init_from_standby + }, /* * Dropping a node is a two-step process */ - { ANY_STATE, DROPPED_STATE, NODE_KIND_CITUS_ANY, - COMMENT_ANY_TO_DROPPED, - &fsm_citus_drop_node }, + { + ANY_STATE, DROPPED_STATE, NODE_KIND_CITUS_ANY, + COMMENT_ANY_TO_DROPPED, + &fsm_citus_drop_node + }, - { ANY_STATE, DROPPED_STATE, NODE_KIND_ANY, - COMMENT_ANY_TO_DROPPED, - &fsm_drop_node }, + { + ANY_STATE, DROPPED_STATE, NODE_KIND_ANY, + COMMENT_ANY_TO_DROPPED, + &fsm_drop_node + }, /* * This is the end, my friend. */ - { NO_STATE, NO_STATE, NODE_KIND_ANY, - NULL, - NULL }, + { + NO_STATE, NO_STATE, NODE_KIND_ANY, + NULL, + NULL + }, }; diff --git a/src/bin/pg_autoctl/fsm_transition.c b/src/bin/pg_autoctl/fsm_transition.c index bc768cb04..ff6dac1a4 100644 --- a/src/bin/pg_autoctl/fsm_transition.c +++ b/src/bin/pg_autoctl/fsm_transition.c @@ -282,11 +282,34 @@ fsm_init_primary(Keeper *keeper) * leaking information from the passfile, environment variable, or * other places. */ + + /* + * When monitor_password is configured, use it and honour the cluster + * auth method; otherwise fall back to the hard-coded password with + * "trust" so that existing deployments are unaffected. + */ + char healthPassword[MAXCONNINFO]; + char healthAuthMethod[MAXPGPATH]; + + if (config->monitor_password[0] != '\0') + { + strlcpy(healthPassword, config->monitor_password, + sizeof(healthPassword)); + strlcpy(healthAuthMethod, pg_setup_get_auth_method(pgSetup), + sizeof(healthAuthMethod)); + } + else + { + strlcpy(healthPassword, PG_AUTOCTL_HEALTH_PASSWORD, + sizeof(healthPassword)); + strlcpy(healthAuthMethod, "trust", sizeof(healthAuthMethod)); + } + if (!primary_create_user_with_hba(postgres, PG_AUTOCTL_HEALTH_USERNAME, - PG_AUTOCTL_HEALTH_PASSWORD, + healthPassword, monitorHostname, - "trust", + healthAuthMethod, pgSetup->hbaLevel, connlimit)) { @@ -902,6 +925,21 @@ fsm_init_standby_from_upstream(Keeper *keeper) return false; } + /* + * After a pg_basebackup the standby inherits the primary's postgresql.conf + * and postgresql-auto-failover.conf. Rewrite postgresql-auto-failover.conf + * via postgres_add_default_settings() so that citus.cluster_name and + * citus.use_secondary_nodes reflect this node's own configuration rather + * than the primary's. prepare_guc_settings_from_pgsetup() derives those + * settings dynamically from pgSetup->citusClusterName, so the full default + * table (including max_wal_senders = 12) is preserved. + */ + if (!postgres_add_default_settings(postgres, config->hostname)) + { + log_error("Failed to rewrite postgresql-auto-failover.conf for standby"); + return false; + } + /* now, in case we have an init state file around, remove it */ return unlink_file(config->pathnames.init); } @@ -1331,6 +1369,20 @@ fsm_fast_forward(Keeper *keeper) return false; } + /* + * When the most advanced standby IS this node itself, there is no WAL + * to fetch: we are already at the frontier. Skip straight to + * prepare_promotion without setting up a (self-referential) replication + * source that would inevitably fail to connect. + */ + if (upstreamNode.nodeId == keeper->state.current_node_id) + { + log_info("This node is already the most advanced standby " + "(LSN %s); no WAL to fetch, skipping fast-forward", + upstreamNode.lsn); + return true; + } + /* * Postgres 10 does not have pg_replication_slot_advance(), so we don't * support replication slots on standby nodes there. diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index 6315cd00b..4525acee1 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -311,10 +311,22 @@ keeper_ensure_current_state(Keeper *keeper) } case MAINTENANCE_STATE: - default: + { + /* + * Ensure the postgres controller knows postgres should be running. + * This is needed after a process restart (e.g. container restart) + * when the expected-status file may not exist in XDG_DATA_HOME. + * Without writing the status file here, the controller waits + * forever for a file that no one writes. + */ + return ensure_postgres_service_is_running_as_subprocess(postgres); + } + default: + { /* nothing to be done here */ return true; + } } /* should never happen */ @@ -971,11 +983,29 @@ keeper_ensure_configuration(Keeper *keeper, bool postgresNotRunningIsOk) if (replicationSettingsHaveChanged) { - log_info("Replication settings at \"%s\" have changed, " - "restarting Postgres", upstreamConfPath); - - if (pg_setup_is_running(pgSetup)) + /* + * In maintenance state, the operator controls postgres. Skip the + * automatic restart: the updated settings take effect on the next + * postgres startup (e.g. after the node exits maintenance and + * postgres is restarted by the operator or by compose stop/start). + * + * Auto-restarting postgres in maintenance state hangs when the WAL + * receiver is blocked on a connection attempt (e.g. after enabling + * SSL with a new primary_conninfo), because pg_ctl stop --fast + * waits for the WAL receiver to finish. + */ + if (state->current_role == MAINTENANCE_STATE) { + log_info("Replication settings at \"%s\" have changed, " + "skipping Postgres restart in maintenance state " + "(changes will take effect on next restart)", + upstreamConfPath); + } + else if (pg_setup_is_running(pgSetup)) + { + log_info("Replication settings at \"%s\" have changed, " + "restarting Postgres", upstreamConfPath); + if (!pgsql_checkpoint(&(postgres->sqlClient))) { log_warn("Failed to CHECKPOINT before restart, " @@ -991,6 +1021,9 @@ keeper_ensure_configuration(Keeper *keeper, bool postgresNotRunningIsOk) } else { + log_info("Replication settings at \"%s\" have changed, " + "restarting Postgres", upstreamConfPath); + if (!ensure_postgres_service_is_running(postgres)) { log_error("Failed to start Postgres with new " @@ -1630,6 +1663,7 @@ keeper_register_and_init(Keeper *keeper, NodeState initialState) while (!pgsql_retry_policy_expired(&retryPolicy)) { bool mayRetry = false; + bool formationMissing = false; /* * When registering to the monitor, we get assigned a nodeId, that we @@ -1670,6 +1704,7 @@ keeper_register_and_init(Keeper *keeper, NodeState initialState) config->pgSetup.settings.replicationQuorum, config->pgSetup.citusClusterName, &mayRetry, + &formationMissing, &assignedState)) { /* registration was successful, break out of the retry loop */ @@ -1682,22 +1717,8 @@ keeper_register_and_init(Keeper *keeper, NodeState initialState) goto rollback; } - int sleepTimeMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); - - log_warn("Failed to register node %s:%d in group %d of " - "formation \"%s\" with initial state \"%s\" " - "because the monitor is already registering another " - "standby, retrying in %d ms", - config->hostname, - config->pgSetup.pgport, - config->groupId, - config->formation, - NodeStateToString(initialState), - sleepTimeMs); - /* - * The current transaction is dead: we caugth an ERROR from the - * call to pgautofailover.register_node(). + * The current transaction is dead: roll it back before sleeping. */ if (!pgsql_rollback(&(monitor->pgsql))) { @@ -1707,6 +1728,35 @@ keeper_register_and_init(Keeper *keeper, NodeState initialState) return false; } + int sleepTimeMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); + + if (formationMissing) + { + /* + * The formation doesn't exist on the monitor yet. This is + * normal in test environments where "pg_autoctl node run" on the + * monitor creates it shortly after the data nodes start. Wait + * and retry; don't auto-create — a missing formation in + * production likely indicates a configuration error. + */ + log_info("Formation \"%s\" does not exist on the monitor yet, " + "retrying in %d ms", + config->formation, sleepTimeMs); + } + else + { + log_warn("Failed to register node %s:%d in group %d of " + "formation \"%s\" with initial state \"%s\" " + "because the monitor is already registering another " + "standby, retrying in %d ms", + config->hostname, + config->pgSetup.pgport, + config->groupId, + config->formation, + NodeStateToString(initialState), + sleepTimeMs); + } + /* we have milliseconds, pg_usleep() wants microseconds */ (void) pg_usleep(sleepTimeMs * 1000); } @@ -1859,6 +1909,7 @@ keeper_register_again(Keeper *keeper) while (!pgsql_retry_policy_expired(&retryPolicy)) { bool mayRetry = false; + bool formationMissing = false; /* re-registration: formation always exists */ if (monitor_register_node(monitor, config->formation, @@ -1875,6 +1926,7 @@ keeper_register_again(Keeper *keeper) config->pgSetup.settings.replicationQuorum, DEFAULT_CITUS_CLUSTER_NAME, &mayRetry, + &formationMissing, &assignedState)) { /* registration was successful, break out of the retry loop */ @@ -2077,12 +2129,19 @@ keeper_update_group_hba(Keeper *keeper, NodeAddressArray *diffNodesArray) sformat(hbaFilePath, MAXPGPATH, "%s/pg_hba.conf", postgresSetup->pgdata); + /* + * cert auth for replication uses "cert map=pgautofailover" so that the + * autoctl_node client certificate CN maps to pgautofailover_replicator. + */ + bool isCertAuth = (strcmp(authMethod, "cert") == 0); + const char *replAuth = isCertAuth ? "cert map=pgautofailover" : authMethod; + if (!pghba_ensure_host_rules_exist(hbaFilePath, diffNodesArray, - postgresSetup->ssl.active, + postgresSetup->ssl.active || isCertAuth, postgresSetup->dbname, PG_AUTOCTL_REPLICA_USERNAME, - authMethod, + replAuth, keeper->config.pgSetup.hbaLevel)) { log_error("Failed to edit HBA file \"%s\" to update rules to current " @@ -2091,6 +2150,19 @@ keeper_update_group_hba(Keeper *keeper, NodeAddressArray *diffNodesArray) return false; } + if (isCertAuth && + keeper->config.pgSetup.hbaLevel >= HBA_EDIT_MINIMAL) + { + if (!pghba_ensure_ident_map_entry(postgresSetup->pgdata, + "pgautofailover", + PG_AUTOCTL_MONITOR_USERNAME, + PG_AUTOCTL_REPLICA_USERNAME)) + { + log_error("Failed to add cert ident map entry to pg_ident.conf"); + return false; + } + } + /* * Only reload if Postgres is known to be running. If it's not running, we * edited the HBA and it's going to take effect at next restart of @@ -2739,8 +2811,8 @@ keeper_config_accept_new(Keeper *keeper, KeeperConfig *newConfig) newConfig->prepare_promotion_walreceiver; } - if (newConfig->postgresql_restart_failure_timeout != - config->postgresql_restart_failure_timeout) + if (newConfig->postgresql_restart_failure_timeout != config-> + postgresql_restart_failure_timeout) { log_info( "Reloading configuration: timeout.postgresql_restart_failure_timeout " @@ -2752,8 +2824,8 @@ keeper_config_accept_new(Keeper *keeper, KeeperConfig *newConfig) newConfig->postgresql_restart_failure_timeout; } - if (newConfig->postgresql_restart_failure_max_retries != - config->postgresql_restart_failure_max_retries) + if (newConfig->postgresql_restart_failure_max_retries != config-> + postgresql_restart_failure_max_retries) { log_info( "Reloading configuration: retries.postgresql_restart_failure_max_retries " diff --git a/src/bin/pg_autoctl/keeper_config.c b/src/bin/pg_autoctl/keeper_config.c index 1459336df..553819ee3 100644 --- a/src/bin/pg_autoctl/keeper_config.c +++ b/src/bin/pg_autoctl/keeper_config.c @@ -133,6 +133,11 @@ config->replication_password, \ REPLICATION_PASSWORD_DEFAULT) +#define OPTION_AUTOCTL_MONITOR_PASSWORD(config) \ + make_strbuf_option_default("pg_autoctl", "monitor_password", NULL, \ + false, MAXCONNINFO, \ + config->monitor_password, "") + #define OPTION_REPLICATION_MAXIMUM_BACKUP_RATE(config) \ make_strbuf_option_default("replication", "maximum_backup_rate", NULL, \ false, MAXIMUM_BACKUP_RATE_LEN, \ @@ -241,6 +246,7 @@ OPTION_REPLICATION_MAXIMUM_BACKUP_RATE(config), \ OPTION_REPLICATION_BACKUP_DIR(config), \ OPTION_REPLICATION_PASSWORD(config), \ + OPTION_AUTOCTL_MONITOR_PASSWORD(config), \ OPTION_TIMEOUT_NETWORK_PARTITION(config), \ OPTION_TIMEOUT_PREPARE_PROMOTION_CATCHUP(config), \ OPTION_TIMEOUT_PREPARE_PROMOTION_WALRECEIVER(config), \ diff --git a/src/bin/pg_autoctl/keeper_config.h b/src/bin/pg_autoctl/keeper_config.h index 3826923d9..73e6cbd62 100644 --- a/src/bin/pg_autoctl/keeper_config.h +++ b/src/bin/pg_autoctl/keeper_config.h @@ -53,6 +53,7 @@ typedef struct KeeperConfig /* PostgreSQL replication / tooling setup */ char replication_slot_name[MAXCONNINFO]; char replication_password[MAXCONNINFO]; + char monitor_password[MAXCONNINFO]; char maximum_backup_rate[MAXIMUM_BACKUP_RATE_LEN]; char backupDirectory[MAXPGPATH]; diff --git a/src/bin/pg_autoctl/keeper_pg_init.c b/src/bin/pg_autoctl/keeper_pg_init.c index 3ff3e555e..3f1a654f7 100644 --- a/src/bin/pg_autoctl/keeper_pg_init.c +++ b/src/bin/pg_autoctl/keeper_pg_init.c @@ -566,11 +566,12 @@ reach_initial_state(Keeper *keeper) } default: - + { /* we don't support any other state at initialization time */ log_error("reach_initial_state: don't know how to read state %s", NodeStateToString(keeper->state.assigned_role)); return false; + } } /* @@ -907,6 +908,23 @@ create_database_and_extension(Keeper *keeper) } } + /* + * cert auth for replication requires an ident map so the autoctl_node + * client cert CN maps to the pgautofailover_replicator Postgres role. + */ + if (strcmp(pg_setup_get_auth_method(pgSetup), "cert") == 0 && + pgSetup->hbaLevel >= HBA_EDIT_MINIMAL) + { + if (!pghba_ensure_ident_map_entry(pgSetup->pgdata, + "pgautofailover", + PG_AUTOCTL_MONITOR_USERNAME, + PG_AUTOCTL_REPLICA_USERNAME)) + { + log_error("Failed to add cert ident map entry to pg_ident.conf"); + return false; + } + } + /* * In test environments using PG_REGRESS_SOCK_DIR="" to disable unix socket * directory, we have to connect to the address from pghost. diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 7afd5d2cc..b5991b954 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -26,6 +26,7 @@ #define STR_ERRCODE_OBJECT_IN_USE "55006" #define STR_ERRCODE_EXCLUSION_VIOLATION "23P01" +#define STR_ERRCODE_INVALID_OBJECT_DEFINITION "42P17" #define STR_ERRCODE_SERIALIZATION_FAILURE "40001" #define STR_ERRCODE_STATEMENT_COMPLETION_UNKNOWN "40003" @@ -831,6 +832,7 @@ monitor_register_node(Monitor *monitor, char *formation, PgInstanceKind kind, int candidatePriority, bool quorum, char *citusClusterName, bool *mayRetry, + bool *formationMissing, MonitorAssignedState *assignedState) { PGSQL *pgsql = &monitor->pgsql; @@ -838,6 +840,8 @@ monitor_register_node(Monitor *monitor, char *formation, "SELECT * FROM pgautofailover.register_node($1, $2, $3, $4, $5, $6, $7, " "$8, $9::pgautofailover.replication_state, $10, $11, $12, $13)"; int paramCount = 13; + + *formationMissing = false; Oid paramTypes[13] = { TEXTOID, TEXTOID, INT4OID, NAMEOID, TEXTOID, INT8OID, INT8OID, INT4OID, TEXTOID, TEXTOID, INT4OID, BOOLOID, TEXTOID @@ -847,17 +851,23 @@ monitor_register_node(Monitor *monitor, char *formation, { { 0 }, assignedState, false }; const char *nodeStateString = NodeStateToString(initialState); + IntString portStr = intToString(port); + IntString systemIdentifierStr = intToString(system_identifier); + IntString desiredNodeIdStr = intToString(desiredNodeId); + IntString desiredGroupIdStr = intToString(desiredGroupId); + IntString candidatePriorityStr = intToString(candidatePriority); + paramValues[0] = formation; paramValues[1] = host; - paramValues[2] = intToString(port).strValue; + paramValues[2] = portStr.strValue; paramValues[3] = dbname; paramValues[4] = name == NULL ? "" : name; - paramValues[5] = intToString(system_identifier).strValue; - paramValues[6] = intToString(desiredNodeId).strValue; - paramValues[7] = intToString(desiredGroupId).strValue; + paramValues[5] = systemIdentifierStr.strValue; + paramValues[6] = desiredNodeIdStr.strValue; + paramValues[7] = desiredGroupIdStr.strValue; paramValues[8] = nodeStateString; paramValues[9] = nodeKindToString(kind); - paramValues[10] = intToString(candidatePriority).strValue; + paramValues[10] = candidatePriorityStr.strValue; paramValues[11] = quorum ? "true" : "false"; paramValues[12] = IS_EMPTY_STRING_BUFFER(citusClusterName) @@ -874,6 +884,14 @@ monitor_register_node(Monitor *monitor, char *formation, *mayRetry = true; return false; } + else if (strcmp(parseContext.sqlstate, + STR_ERRCODE_INVALID_OBJECT_DEFINITION) == 0) + { + /* Formation doesn't exist yet — caller will create it and retry. */ + *formationMissing = true; + *mayRetry = true; + return false; + } else if (strcmp(parseContext.sqlstate, STR_ERRCODE_EXCLUSION_VIOLATION) == 0) { @@ -946,12 +964,16 @@ monitor_node_active(Monitor *monitor, { { 0 }, assignedState, false }; const char *nodeStateString = NodeStateToString(currentState); + IntString nodeIdStr = intToString(nodeId); + IntString groupIdStr = intToString(groupId); + IntString currentTLIStr = intToString(currentTLI); + paramValues[0] = formation; - paramValues[1] = intToString(nodeId).strValue; - paramValues[2] = intToString(groupId).strValue; + paramValues[1] = nodeIdStr.strValue; + paramValues[2] = groupIdStr.strValue; paramValues[3] = nodeStateString; paramValues[4] = pgIsRunning ? "true" : "false"; - paramValues[5] = intToString(currentTLI).strValue; + paramValues[5] = currentTLIStr.strValue; paramValues[6] = currentLSN; paramValues[7] = pgsrSyncState; @@ -959,6 +981,24 @@ monitor_node_active(Monitor *monitor, paramCount, paramTypes, paramValues, &parseContext, parseNodeState)) { + /* + * ERRCODE_UNDEFINED_OBJECT (42704): the monitor raised "couldn't find + * node with nodeid N". This happens when remove_node() was called with + * --force, or after the node already reported DROPPED and the monitor + * deleted the row. The running keeper should treat this as a DROPPED + * assignment so the FSM exits cleanly rather than looping on errors. + */ + if (strcmp(parseContext.sqlstate, STR_ERRCODE_UNDEFINED_OBJECT) == 0) + { + log_info("Node %" PRId64 " is no longer registered on the monitor " + "(SQLSTATE %s); treating as dropped.", + nodeId, parseContext.sqlstate); + assignedState->nodeId = nodeId; + assignedState->groupId = groupId; + assignedState->state = DROPPED_STATE; + return true; + } + log_error("Failed to get node state for node %" PRId64 " in group %d of formation \"%s\" with initial state " "\"%s\", replication state \"%s\", " @@ -1001,7 +1041,8 @@ monitor_set_node_candidate_priority(Monitor *monitor, int paramCount = 3; Oid paramTypes[3] = { TEXTOID, TEXTOID, INT4OID }; const char *paramValues[3]; - char *candidatePriorityText = intToString(candidate_priority).strValue; + IntString candidatePriorityStr = intToString(candidate_priority); + char *candidatePriorityText = candidatePriorityStr.strValue; bool success = true; paramValues[0] = formation; @@ -1218,8 +1259,9 @@ monitor_set_formation_number_sync_standbys(Monitor *monitor, char *formation, Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2]; SingleValueResultContext parseContext = { { 0 }, PGSQL_RESULT_BOOL, false }; + IntString numSyncStr = intToString(numberSyncStandbys); paramValues[0] = formation; - paramValues[1] = intToString(numberSyncStandbys).strValue; + paramValues[1] = numSyncStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -1257,8 +1299,9 @@ monitor_remove_by_hostname(Monitor *monitor, char *host, int port, bool force, Oid paramTypes[3] = { TEXTOID, INT4OID, BOOLOID }; const char *paramValues[3]; + IntString portStr = intToString(port); paramValues[0] = host; - paramValues[1] = intToString(port).strValue; + paramValues[1] = portStr.strValue; paramValues[2] = force ? "true" : "false"; if (!pgsql_execute_with_params(pgsql, sql, @@ -1496,8 +1539,9 @@ monitor_perform_failover(Monitor *monitor, char *formation, int group) Oid paramTypes[2] = { TEXTOID, INT4OID }; const char *paramValues[2]; + IntString groupStr = intToString(group); paramValues[0] = formation; - paramValues[1] = intToString(group).strValue; + paramValues[1] = groupStr.strValue; /* * pgautofailover.perform_failover() returns VOID. @@ -2743,7 +2787,8 @@ monitor_create_formation(Monitor *monitor, paramValues[1] = kind; paramValues[2] = dbname; paramValues[3] = hasSecondary ? "true" : "false"; - paramValues[4] = intToString(numberSyncStandbys).strValue; + IntString numSyncStr = intToString(numberSyncStandbys); + paramValues[4] = numSyncStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3430,10 +3475,12 @@ monitor_update_node_metadata(Monitor *monitor, SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BOOL, false }; - paramValues[0] = intToString(nodeId).strValue; + IntString nodeIdStr = intToString(nodeId); + IntString portStr = intToString(port); + paramValues[0] = nodeIdStr.strValue; paramValues[1] = name; paramValues[2] = hostname; - paramValues[3] = intToString(port).strValue; + paramValues[3] = portStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3478,8 +3525,10 @@ monitor_set_node_system_identifier(Monitor *monitor, NodeAddress node = { 0 }; NodeAddressParseContext parseContext = { { 0 }, &node, false }; - paramValues[0] = intToString(nodeId).strValue; - paramValues[1] = intToString(system_identifier).strValue; + IntString nodeIdStr = intToString(nodeId); + IntString sysIdStr = intToString(system_identifier); + paramValues[0] = nodeIdStr.strValue; + paramValues[1] = sysIdStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3526,8 +3575,10 @@ monitor_set_group_system_identifier(Monitor *monitor, SingleValueResultContext context = { 0 }; - paramValues[0] = intToString(groupId).strValue; - paramValues[1] = intToString(system_identifier).strValue; + IntString groupIdStr = intToString(groupId); + IntString sysIdStr = intToString(system_identifier); + paramValues[0] = groupIdStr.strValue; + paramValues[1] = sysIdStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3640,8 +3691,9 @@ monitor_start_maintenance(Monitor *monitor, int64_t nodeId, bool *mayRetry) int paramCount = 1; Oid paramTypes[1] = { INT8OID }; const char *paramValues[1]; + IntString nodeIdStr = intToString(nodeId); - paramValues[0] = intToString(nodeId).strValue; + paramValues[0] = nodeIdStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -3688,8 +3740,9 @@ monitor_stop_maintenance(Monitor *monitor, int64_t nodeId, bool *mayRetry) int paramCount = 1; Oid paramTypes[1] = { INT8OID }; const char *paramValues[1]; + IntString nodeIdStr = intToString(nodeId); - paramValues[0] = intToString(nodeId).strValue; + paramValues[0] = nodeIdStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, @@ -4295,20 +4348,27 @@ monitor_wait_until_some_node_reported_state(Monitor *monitor, while (!context.failoverIsDone) { - /* when timeout <= 0 we just never stop waiting */ + int thisLoopTimeout; + if (timeout > 0) { uint64_t now = time(NULL); + uint64_t elapsed = now - start; - if ((now - start) > timeout) + if (elapsed >= (uint64_t) timeout) { log_error("Failed to receive monitor's notifications"); break; } - } - int thisLoopTimeout = - timeout > 0 ? timeout : PG_AUTOCTL_LISTEN_NOTIFICATIONS_TIMEOUT; + /* cap inner wait to remaining time so we don't overshoot by a full timeout */ + thisLoopTimeout = (int) (timeout - elapsed); + } + else + { + /* timeout <= 0: wait indefinitely, using the default block time */ + thisLoopTimeout = PG_AUTOCTL_LISTEN_NOTIFICATIONS_TIMEOUT; + } if (!monitor_process_notifications( monitor, @@ -4855,9 +4915,11 @@ monitor_find_node_by_nodeid(Monitor *monitor, NodeAddressArrayParseContext parseContext = { { 0 }, nodesArray, false }; + IntString groupIdStr = intToString(groupId); + IntString nodeIdStr = intToString(nodeId); paramValues[0] = formation; - paramValues[1] = intToString(groupId).strValue; - paramValues[2] = intToString(nodeId).strValue; + paramValues[1] = groupIdStr.strValue; + paramValues[2] = nodeIdStr.strValue; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 6c6a95477..9dcd12261 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -140,6 +140,7 @@ bool monitor_register_node(Monitor *monitor, bool quorum, char *citusClusterName, bool *mayRetry, + bool *formationMissing, MonitorAssignedState *assignedState); bool monitor_node_active(Monitor *monitor, char *formation, int64_t nodeId, diff --git a/src/bin/pg_autoctl/monitor_config.c b/src/bin/pg_autoctl/monitor_config.c index 77021471e..ce88349b8 100644 --- a/src/bin/pg_autoctl/monitor_config.c +++ b/src/bin/pg_autoctl/monitor_config.c @@ -36,6 +36,11 @@ make_strbuf_option("pg_autoctl", "hostname", "hostname", \ false, _POSIX_HOST_NAME_MAX, config->hostname) +#define OPTION_AUTOCTL_NODE_PASSWORD(config) \ + make_strbuf_option_default("pg_autoctl", "autoctl_node_password", NULL, \ + false, MAXCONNINFO, \ + config->autoctl_node_password, "") + #define OPTION_AUTOCTL_NODENAME(config) \ make_strbuf_compat_option("pg_autoctl", "nodename", \ _POSIX_HOST_NAME_MAX, config->hostname) @@ -104,6 +109,7 @@ OPTION_AUTOCTL_ROLE(config), \ OPTION_AUTOCTL_HOSTNAME(config), \ OPTION_AUTOCTL_NODENAME(config), \ + OPTION_AUTOCTL_NODE_PASSWORD(config), \ OPTION_POSTGRESQL_PGDATA(config), \ OPTION_POSTGRESQL_PG_CTL(config), \ OPTION_POSTGRESQL_USERNAME(config), \ diff --git a/src/bin/pg_autoctl/monitor_config.h b/src/bin/pg_autoctl/monitor_config.h index c7292d992..97c6eee54 100644 --- a/src/bin/pg_autoctl/monitor_config.h +++ b/src/bin/pg_autoctl/monitor_config.h @@ -25,12 +25,19 @@ typedef struct MonitorConfig /* pg_autoctl setup */ char hostname[_POSIX_HOST_NAME_MAX]; + char autoctl_node_password[MAXCONNINFO]; /* PostgreSQL setup */ char role[NAMEDATALEN]; /* PostgreSQL setup */ PostgresSetup pgSetup; + + /* non-default formations to create during monitor init */ +#define MONITOR_MAX_FORMATIONS 16 + int formationCount; + char formationNames[MONITOR_MAX_FORMATIONS][NAMEDATALEN]; + char formationKinds[MONITOR_MAX_FORMATIONS][NAMEDATALEN]; /* "pgsql" = default */ } MonitorConfig; diff --git a/src/bin/pg_autoctl/monitor_pg_init.c b/src/bin/pg_autoctl/monitor_pg_init.c index 2835c7f3a..839cd5111 100644 --- a/src/bin/pg_autoctl/monitor_pg_init.c +++ b/src/bin/pg_autoctl/monitor_pg_init.c @@ -139,7 +139,8 @@ monitor_pg_init(Monitor *monitor) */ bool monitor_install(const char *hostname, - PostgresSetup pgSetupOption, bool checkSettings) + PostgresSetup pgSetupOption, bool checkSettings, + const char *autoctl_node_password) { PostgresSetup pgSetup = { 0 }; bool missingPgdataIsOk = false; @@ -216,6 +217,23 @@ monitor_install(const char *hostname, return false; } + /* + * If a password for the autoctl_node role has been configured, set it now. + * The autoctl_node role is created by the pgautofailover extension (via + * CREATE EXTENSION above), so we ALTER it here after the extension exists. + */ + if (autoctl_node_password != NULL && autoctl_node_password[0] != '\0') + { + if (!pgsql_alter_role_password(&postgres.sqlClient, + PG_AUTOCTL_MONITOR_USERNAME, + autoctl_node_password)) + { + log_error("Failed to set password for role \"%s\"", + PG_AUTOCTL_MONITOR_USERNAME); + return false; + } + } + /* * When installing the monitor on-top of an already running PostgreSQL, we * want to check that our settings have been applied already, and warn the diff --git a/src/bin/pg_autoctl/monitor_pg_init.h b/src/bin/pg_autoctl/monitor_pg_init.h index 38a552c65..aed2b521e 100644 --- a/src/bin/pg_autoctl/monitor_pg_init.h +++ b/src/bin/pg_autoctl/monitor_pg_init.h @@ -18,7 +18,8 @@ bool monitor_pg_init(Monitor *monitor); bool monitor_install(const char *hostname, PostgresSetup pgSetupOption, - bool checkSettings); + bool checkSettings, + const char *autoctl_node_password); bool monitor_add_postgres_default_settings(Monitor *monitor); #endif /* MONITOR_PG_INIT_H */ diff --git a/src/bin/pg_autoctl/nodespec.c b/src/bin/pg_autoctl/nodespec.c new file mode 100644 index 000000000..54d142c82 --- /dev/null +++ b/src/bin/pg_autoctl/nodespec.c @@ -0,0 +1,942 @@ +/* + * src/bin/pg_autoctl/nodespec.c + * Parse, write, and converge a pg_autoctl_node.ini file. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include +#include + +#ifdef __linux__ +#include +#endif + +#include "nodespec.h" +#include "ini_file.h" +#include "ini.h" +#include "log.h" +#include "pgsetup.h" +#include "string_utils.h" +#include "file_utils.h" /* sformat */ +#include "env_utils.h" +#include "cli_root.h" /* pg_autoctl_program */ +#include "runprogram.h" /* Program, run_program, free_program */ + + +/* + * nodespec_read parses a pg_autoctl_node.ini file. + * + * File format: + * + * [node] + * kind = postgres # postgres | monitor | coordinator | worker + * hostname = node1 + * port = 5432 + * + * [postgresql] + * pgdata = /var/lib/postgres/pgaf + * + * [monitor] + * pguri = postgresql://autoctl_node@monitor/pg_auto_failover + * + * [formation] + * name = default + * group = 0 + * + * [settings] + * candidate_priority = 50 + * replication_quorum = true + * + * [options] + * ssl = self-signed + * auth = trust + * pg_hba_lan = true + */ +bool +nodespec_read(const char *path, NodeSpec *spec) +{ + char kindStr[NAMEDATALEN] = { 0 }; + char replicationQuorumStr[8] = { 0 }; + char pgHbaLanStr[8] = { 0 }; + char launchModeStr[16] = { 0 }; + char noMonitorStr[8] = { 0 }; + char citusRoleStr[NAMEDATALEN] = { 0 }; + int port = 5432; + int group = 0; + int candidatePriority = 50; + + /* + * Provide string buffers for booleans and the kind enum — we parse them + * after read_ini_file() returns so we can give decent error messages. + */ + IniOption opts[] = { + /* [node] */ + make_strbuf_option_default("node", "kind", NULL, true, + sizeof(kindStr), kindStr, + "postgres"), + make_strbuf_option_default("node", "name", NULL, false, + sizeof(spec->name), spec->name, + ""), + make_strbuf_option_default("node", "hostname", NULL, false, + sizeof(spec->hostname), spec->hostname, + ""), + make_int_option_default("node", "port", NULL, false, + &port, 5432), + + /* [postgresql] */ + make_strbuf_option("postgresql", "pgdata", NULL, true, + sizeof(spec->pgdata), spec->pgdata), + + /* [monitor] */ + make_strbuf_option_default("monitor", "pguri", NULL, false, + sizeof(spec->monitor_pguri), + spec->monitor_pguri, ""), + make_strbuf_option_default("monitor", "no_monitor", NULL, false, + sizeof(noMonitorStr), noMonitorStr, "false"), + make_int_option_default("monitor", "node_id", NULL, false, + &spec->nodeId, 0), + + /* [formation] */ + make_strbuf_option_default("formation", "name", NULL, false, + sizeof(spec->formation), spec->formation, + "default"), + make_int_option_default("formation", "group", NULL, false, + &group, 0), + + /* [settings] — mutable */ + make_int_option_default("settings", "candidate_priority", NULL, false, + &candidatePriority, 50), + make_strbuf_option_default("settings", "replication_quorum", NULL, false, + sizeof(replicationQuorumStr), + replicationQuorumStr, "true"), + + /* [options] — immutable, used only at create time */ + make_strbuf_option_default("options", "ssl", NULL, false, + sizeof(spec->ssl), spec->ssl, + "self-signed"), + make_strbuf_option_default("options", "auth", NULL, false, + sizeof(spec->auth), spec->auth, + "trust"), + make_strbuf_option_default("options", "pg_hba_lan", NULL, false, + sizeof(pgHbaLanStr), pgHbaLanStr, + "true"), + + /* [ssl] — certificate paths for verify-ca / verify-full mode */ + make_strbuf_option_default("ssl", "ca_file", NULL, false, + sizeof(spec->ssl_ca_file), + spec->ssl_ca_file, ""), + make_strbuf_option_default("ssl", "cert_file", NULL, false, + sizeof(spec->ssl_cert_file), + spec->ssl_cert_file, ""), + make_strbuf_option_default("ssl", "key_file", NULL, false, + sizeof(spec->ssl_key_file), + spec->ssl_key_file, ""), + + /* [launch] — optional section; mode=deferred delays node init */ + make_strbuf_option_default("launch", "mode", NULL, false, + sizeof(launchModeStr), launchModeStr, + "immediate"), + + /* [pg_auto_failover] — monitor: password for autoctl_node role */ + make_strbuf_option_default("pg_auto_failover", "autoctl_node_password", + NULL, false, + sizeof(spec->autoctl_node_password), + spec->autoctl_node_password, ""), + + /* [replication] — postgres: password for pgautofailover_replicator */ + make_strbuf_option_default("replication", "password", NULL, false, + sizeof(spec->replication_password), + spec->replication_password, ""), + + /* [pg_auto_failover] — postgres: password for pgautofailover_monitor */ + make_strbuf_option_default("pg_auto_failover", "monitor_password", + NULL, false, + sizeof(spec->monitor_password), + spec->monitor_password, ""), + + /* [citus] — optional; present only for Citus secondary/read-replica nodes */ + make_strbuf_option_default("citus", "role", NULL, false, + sizeof(citusRoleStr), citusRoleStr, ""), + make_strbuf_option_default("citus", "cluster_name", NULL, false, + sizeof(spec->citusClusterName), + spec->citusClusterName, ""), + + INI_OPTION_LAST + }; + + if (!read_ini_file(path, opts)) + { + log_error("Failed to parse node spec file \"%s\"", path); + return false; + } + + /* resolve kind string → enum */ + if (strcmp(kindStr, "monitor") == 0) + { + spec->kind = NODE_KIND_UNKNOWN; /* handled specially: no formation */ + } + else if (strcmp(kindStr, "postgres") == 0) + { + spec->kind = NODE_KIND_STANDALONE; + } + else if (strcmp(kindStr, "coordinator") == 0) + { + spec->kind = NODE_KIND_CITUS_COORDINATOR; + } + else if (strcmp(kindStr, "worker") == 0) + { + spec->kind = NODE_KIND_CITUS_WORKER; + } + else + { + log_error("Unknown node kind \"%s\" in \"%s\"; " + "expected: monitor, postgres, coordinator, worker", + kindStr, path); + return false; + } + + spec->port = port; + spec->group = group; + spec->candidate_priority = candidatePriority; + + /* parse boolean strings */ + spec->replication_quorum = + (strcmp(replicationQuorumStr, "true") == 0 || + strcmp(replicationQuorumStr, "yes") == 0 || + strcmp(replicationQuorumStr, "1") == 0); + + spec->pg_hba_lan = + (strcmp(pgHbaLanStr, "true") == 0 || + strcmp(pgHbaLanStr, "yes") == 0 || + strcmp(pgHbaLanStr, "1") == 0); + + spec->launchDeferred = (strcmp(launchModeStr, "deferred") == 0); + spec->noMonitor = + (strcmp(noMonitorStr, "true") == 0 || + strcmp(noMonitorStr, "yes") == 0 || + strcmp(noMonitorStr, "1") == 0); + + spec->citusSecondary = (strcmp(citusRoleStr, "secondary") == 0); + + /* + * Second pass: enumerate [formation ] sections. + * The standard IniOption machinery can't handle variable-count sections, + * so we open the file again with ini_load directly. + */ + { + char *fileContents = NULL; + long fileSize = 0L; + + if (read_file(path, &fileContents, &fileSize)) + { + ini_t *raw = ini_load(fileContents, NULL); + free(fileContents); + + if (raw) + { + int nsec = ini_section_count(raw); + spec->formationCount = 0; + + for (int si = 0; si < nsec; si++) + { + const char *sname = ini_section_name(raw, si); + if (!sname) + { + continue; + } + if (strncmp(sname, "formation ", 10) != 0) + { + continue; + } + + const char *fname = sname + 10; /* skip "formation " */ + if (fname[0] == '\0') + { + continue; + } + + if (spec->formationCount >= NODESPEC_MAX_FORMATIONS) + { + log_warn("nodespec: too many [formation ] sections " + "in \"%s\"; max %d", path, NODESPEC_MAX_FORMATIONS); + break; + } + + int fi = spec->formationCount++; + strlcpy(spec->formationNames[fi], fname, + sizeof(spec->formationNames[fi])); + + /* optional: kind = ha (default) */ + int ki = ini_find_property(raw, si, "kind", 0); + if (ki != INI_NOT_FOUND) + { + const char *kv = ini_property_value(raw, si, ki); + if (kv && kv[0]) + { + strlcpy(spec->formationKinds[fi], kv, + sizeof(spec->formationKinds[fi])); + } + else + { + strlcpy(spec->formationKinds[fi], "pgsql", + sizeof(spec->formationKinds[fi])); + } + } + else + { + strlcpy(spec->formationKinds[fi], "pgsql", + sizeof(spec->formationKinds[fi])); + } + } + ini_destroy(raw); + } + } + } + + /* validate: non-monitor nodes need a monitor URI unless no_monitor=true */ + if (spec->kind != NODE_KIND_UNKNOWN && + IS_EMPTY_STRING_BUFFER(spec->monitor_pguri) && + !spec->noMonitor) + { + log_error("Node kind \"%s\" requires [monitor] pguri in \"%s\"", + kindStr, path); + return false; + } + + return true; +} + + +/* + * nodespec_write serialises a NodeSpec as a pg_autoctl_node.ini file. + */ +bool +nodespec_write(const NodeSpec *spec, FILE *out) +{ + const char *kindStr; + + switch (spec->kind) + { + case NODE_KIND_UNKNOWN: + { + kindStr = "monitor"; + break; + } + + case NODE_KIND_STANDALONE: + { + kindStr = "postgres"; + break; + } + + case NODE_KIND_CITUS_COORDINATOR: + { + kindStr = "coordinator"; + break; + } + + case NODE_KIND_CITUS_WORKER: + { + kindStr = "worker"; + break; + } + + default: + { + kindStr = "postgres"; + break; + } + } + + fformat(out, + "[node]\n" + "kind = %s\n", + kindStr); + + if (!IS_EMPTY_STRING_BUFFER(spec->name)) + { + fformat(out, "name = %s\n", spec->name); + } + + fformat(out, + "hostname = %s\n" + "port = %d\n" + "\n" + "[postgresql]\n" + "pgdata = %s\n" + "\n", + spec->hostname, + spec->port, + spec->pgdata); + + if (spec->kind != NODE_KIND_UNKNOWN) + { + if (spec->noMonitor) + { + fformat(out, + "[monitor]\n" + "no_monitor = true\n" + "\n"); + } + else + { + fformat(out, + "[monitor]\n" + "pguri = %s\n" + "\n", + spec->monitor_pguri); + } + + fformat(out, + "[formation]\n" + "name = %s\n" + "group = %d\n" + "\n", + spec->formation, + spec->group); + } + + fformat(out, + "[settings]\n" + "candidate_priority = %d\n" + "replication_quorum = %s\n" + "\n" + "[options]\n" + "ssl = %s\n" + "auth = %s\n" + "pg_hba_lan = %s\n", + spec->candidate_priority, + spec->replication_quorum ? "true" : "false", + spec->ssl, + spec->auth, + spec->pg_hba_lan ? "true" : "false"); + + /* only emit [launch] when deferred — omitting the section means immediate */ + if (spec->launchDeferred) + { + fformat(out, "\n[launch]\nmode = deferred\n"); + } + + /* [formation ] sections — monitor kind only */ + for (int fi = 0; fi < spec->formationCount; fi++) + { + fformat(out, "\n[formation %s]\n", spec->formationNames[fi]); + if (spec->formationKinds[fi][0] && + strcmp(spec->formationKinds[fi], "pgsql") != 0) + { + fformat(out, "kind = %s\n", spec->formationKinds[fi]); + } + } + + return true; +} + + +/* + * nodespec_create_argv builds an argv[] array for: + * + * pg_autoctl create --pgdata

--hostname --pgport + * --monitor [--formation ] [--group ] + * [--ssl-self-signed | --no-ssl] + * [--auth ] [--pg-hba-lan] --run + * + * Returns the number of entries written (terminating NULL not counted). + * args[] must have room for at least 40 pointers. + * + * String values are pointers into *spec — the caller must keep spec alive. + */ +int +nodespec_create_argv(const NodeSpec *spec, + const char *pg_autoctl_path, + char **args, int args_size) +{ + int i = 0; + +#define PUSH(v) do { \ + if (i >= args_size - 1) { \ + log_error("nodespec_create_argv: args[] overflow"); \ + return -1; \ + } \ + args[i++] = (char *) (v); \ +} while (0) + + PUSH(pg_autoctl_path); + PUSH("create"); + + switch (spec->kind) + { + case NODE_KIND_UNKNOWN: + { + PUSH("monitor"); + break; + } + + case NODE_KIND_STANDALONE: + { + PUSH("postgres"); + break; + } + + case NODE_KIND_CITUS_COORDINATOR: + { + PUSH("coordinator"); + break; + } + + case NODE_KIND_CITUS_WORKER: + { + PUSH("worker"); + break; + } + + default: + { + PUSH("postgres"); + break; + } + } + + PUSH("--pgdata"); + PUSH(spec->pgdata); + + if (!IS_EMPTY_STRING_BUFFER(spec->name)) + { + PUSH("--name"); + PUSH(spec->name); + } + + if (!IS_EMPTY_STRING_BUFFER(spec->hostname)) + { + PUSH("--hostname"); + PUSH(spec->hostname); + } + + if (spec->port != 5432) + { + /* static buffer — safe because spec is long-lived */ + static char portbuf[16]; + sformat(portbuf, sizeof(portbuf), "%d", spec->port); + PUSH("--pgport"); + PUSH(portbuf); + } + + if (spec->kind != NODE_KIND_UNKNOWN) + { + if (spec->noMonitor) + { + PUSH("--disable-monitor"); + if (spec->nodeId > 0) + { + static char nodeIdBuf[16]; + sformat(nodeIdBuf, sizeof(nodeIdBuf), "%d", spec->nodeId); + PUSH("--node-id"); + PUSH(nodeIdBuf); + } + } + else + { + PUSH("--monitor"); + PUSH(spec->monitor_pguri); + } + + if (!IS_EMPTY_STRING_BUFFER(spec->formation) && + strcmp(spec->formation, "default") != 0) + { + PUSH("--formation"); + PUSH(spec->formation); + } + + if (spec->kind == NODE_KIND_CITUS_WORKER && spec->group > 0) + { + static char groupbuf[16]; + sformat(groupbuf, sizeof(groupbuf), "%d", spec->group); + PUSH("--group"); + PUSH(groupbuf); + } + } + + /* SSL */ + if (strcmp(spec->ssl, "self-signed") == 0) + { + PUSH("--ssl-self-signed"); + } + else if (strcmp(spec->ssl, "off") == 0) + { + PUSH("--no-ssl"); + } + else if (!IS_EMPTY_STRING_BUFFER(spec->ssl_ca_file)) + { + /* verify-ca / verify-full: pass the cert paths explicitly */ + PUSH("--ssl-ca-file"); + PUSH(spec->ssl_ca_file); + PUSH("--server-cert"); + PUSH(spec->ssl_cert_file); + PUSH("--server-key"); + PUSH(spec->ssl_key_file); + PUSH("--ssl-mode"); + PUSH(spec->ssl); + } + + /* auth */ + if (!IS_EMPTY_STRING_BUFFER(spec->auth)) + { + PUSH("--auth"); + PUSH(spec->auth); + } + + if (spec->pg_hba_lan && spec->kind != NODE_KIND_UNKNOWN) + { + PUSH("--pg-hba-lan"); + } + + /* passwords */ + if (spec->kind == NODE_KIND_UNKNOWN && + !IS_EMPTY_STRING_BUFFER(spec->autoctl_node_password)) + { + PUSH("--autoctl-node-password"); + PUSH(spec->autoctl_node_password); + } + + /* non-default formations to create during monitor init */ + if (spec->kind == NODE_KIND_UNKNOWN) + { + for (int fi = 0; fi < spec->formationCount; fi++) + { + PUSH("--formation"); + PUSH(spec->formationNames[fi]); + } + } + + if (spec->kind != NODE_KIND_UNKNOWN) + { + if (!IS_EMPTY_STRING_BUFFER(spec->monitor_password)) + { + PUSH("--monitor-password"); + PUSH(spec->monitor_password); + } + if (!IS_EMPTY_STRING_BUFFER(spec->replication_password)) + { + PUSH("--replication-password"); + PUSH(spec->replication_password); + } + + /* candidate priority and replication quorum (non-default values) */ + if (spec->candidate_priority != 50) + { + static char pribuf[16]; + sformat(pribuf, sizeof(pribuf), "%d", spec->candidate_priority); + PUSH("--candidate-priority"); + PUSH(pribuf); + } + if (!spec->replication_quorum) + { + PUSH("--replication-quorum"); + PUSH("false"); + } + + /* Citus secondary/read-replica cluster settings */ + if (spec->citusSecondary) + { + PUSH("--citus-secondary"); + } + if (!IS_EMPTY_STRING_BUFFER(spec->citusClusterName)) + { + PUSH("--citus-cluster"); + PUSH(spec->citusClusterName); + } + } + + PUSH("--run"); + + args[i] = NULL; + +#undef PUSH + return i; +} + + +/* + * nodespec_write_to_path writes the spec to the given filesystem path, + * replacing the file in-place. Used by pg_autoctl node start to clear the + * [launch] deferred flag so the waiting pg_autoctl node run can proceed. + */ +bool +nodespec_write_to_path(const NodeSpec *spec, const char *path) +{ + FILE *f = fopen(path, "w"); /* IGNORE-BANNED */ + if (!f) + { + log_error("Cannot open \"%s\" for writing: %m", path); + return false; + } + bool ok = nodespec_write(spec, f); + fclose(f); + return ok; +} + + +/* + * nodespec_apply compares new_spec against old_spec and applies any changes + * to the mutable fields by calling into the keeper / monitor APIs. + * + * Currently mutable fields: + * - candidate_priority → monitor_set_node_candidate_priority() + * - replication_quorum → monitor_set_node_replication_quorum() + * + * The [launch] mode field is handled separately by pg_autoctl node start. + * Applying a spec with mode=deferred to an already-started node is a + * non-fatal warning (ignored). + * + * Immutable fields (kind, pgdata, ssl, auth, pg_hba_lan) are not checked here. + */ +bool +nodespec_apply(const NodeSpec *new_spec, const NodeSpec *old_spec) +{ + bool changed = false; + + if (new_spec->candidate_priority != old_spec->candidate_priority) + { + char prio[16]; + sformat(prio, sizeof(prio), "%d", new_spec->candidate_priority); + + Program prog = run_program(pg_autoctl_program, + "set", "node", "candidate-priority", + "--pgdata", new_spec->pgdata, + prio, NULL); + + if (prog.returnCode != 0) + { + log_warn("nodespec_apply: set candidate-priority %s failed (rc=%d)", + prio, prog.returnCode); + if (prog.stdOut) + { + log_warn("%s", prog.stdOut); + } + } + else + { + log_info("nodespec: applied candidate_priority = %d", + new_spec->candidate_priority); + changed = true; + } + free_program(&prog); + } + + if (new_spec->replication_quorum != old_spec->replication_quorum) + { + const char *quorum = new_spec->replication_quorum ? "true" : "false"; + + Program prog = run_program(pg_autoctl_program, + "set", "node", "replication-quorum", + "--pgdata", new_spec->pgdata, + quorum, NULL); + + if (prog.returnCode != 0) + { + log_warn("nodespec_apply: set replication-quorum %s failed (rc=%d)", + quorum, prog.returnCode); + if (prog.stdOut) + { + log_warn("%s", prog.stdOut); + } + } + else + { + log_info("nodespec: applied replication_quorum = %s", quorum); + changed = true; + } + free_program(&prog); + } + + /* immediate → deferred on an already-started node: non-fatal, ignored */ + if (!old_spec->launchDeferred && new_spec->launchDeferred) + { + log_warn("nodespec_apply: ignoring attempt to set launch=deferred " + "on a node that is already running"); + } + + if (!changed) + { + log_debug("nodespec_apply: no mutable fields changed"); + } + + return true; +} + + +/* ----------------------------------------------------------------------- + * Watcher + * ----------------------------------------------------------------------- */ + +/* + * nodespec_watcher_init sets up file watching for *path. + * On Linux we try inotify first; everywhere else (and on failure) we fall + * back to mtime polling every NODESPEC_WATCH_INTERVAL_SECS. + */ +bool +nodespec_watcher_init(NodeSpecWatcher *w, const char *path) +{ + struct stat st; + + memset(w, 0, sizeof(*w)); + strlcpy(w->path, path, sizeof(w->path)); + + w->last_checked = time(NULL); + +#ifdef __linux__ + w->inotify_fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); + + if (w->inotify_fd >= 0) + { + w->watch_fd = inotify_add_watch(w->inotify_fd, path, + IN_CLOSE_WRITE | IN_MOVED_TO); + if (w->watch_fd < 0) + { + log_debug("nodespec_watcher_init: inotify_add_watch(\"%s\"): %m; " + "falling back to mtime poll", path); + close(w->inotify_fd); + w->inotify_fd = -1; + } + else + { + log_info("nodespec: watching \"%s\" via inotify", path); + } + } + else + { + log_debug("nodespec_watcher_init: inotify_init1: %m; " + "falling back to mtime poll"); + w->inotify_fd = -1; + w->watch_fd = -1; + } +#endif + + /* record initial mtime so we don't fire on the very first check */ + if (stat(path, &st) == 0) + { + w->last_mtime = st.st_mtime; + } + else + { + w->last_mtime = 0; + } + + w->active = true; + return true; +} + + +/* + * nodespec_watcher_check is called from the supervisor's 100 ms tick. + * + * Returns true if the file changed and nodespec_apply() was attempted. + */ +bool +nodespec_watcher_check(NodeSpecWatcher *w, const NodeSpec *current) +{ + bool file_changed = false; + + if (!w->active) + { + return false; + } + +#ifdef __linux__ + if (w->inotify_fd >= 0) + { + /* + * Drain all pending inotify events. We don't care about the event + * details — any event means the file was written. + */ + char buf[sizeof(struct inotify_event) + NAME_MAX + 1]; + ssize_t n; + + while ((n = read(w->inotify_fd, buf, sizeof(buf))) > 0) + { + file_changed = true; + } + + if (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK) + { + log_warn("nodespec watcher: inotify read error: %m; " + "switching to mtime poll"); + close(w->inotify_fd); + w->inotify_fd = -1; + w->watch_fd = -1; + } + } + else +#endif + { + /* mtime poll — only stat every NODESPEC_WATCH_INTERVAL_SECS */ + time_t now = time(NULL); + + if (now - w->last_checked < NODESPEC_WATCH_INTERVAL_SECS) + { + return false; + } + + w->last_checked = now; + + struct stat st; + if (stat(w->path, &st) == 0 && st.st_mtime != w->last_mtime) + { + w->last_mtime = st.st_mtime; + file_changed = true; + } + } + + if (!file_changed) + { + return false; + } + + /* File changed — re-parse and apply */ + log_info("nodespec: \"%s\" changed, re-reading and converging", w->path); + + NodeSpec new_spec = { 0 }; + if (!nodespec_read(w->path, &new_spec)) + { + log_warn("nodespec: failed to parse updated file \"%s\"; " + "keeping current configuration", w->path); + return false; + } + + /* Warn about immutable field changes rather than silently ignoring them */ + if (new_spec.kind != current->kind) + { + log_warn("nodespec: 'kind' changed in \"%s\" but cannot be applied " + "to a running node — restart required", w->path); + } + + if (strcmp(new_spec.pgdata, current->pgdata) != 0) + { + log_warn("nodespec: 'pgdata' changed in \"%s\" but cannot be applied " + "to a running node — restart required", w->path); + } + + (void) nodespec_apply(&new_spec, current); + return true; +} + + +/* + * nodespec_watcher_close releases inotify resources. + */ +void +nodespec_watcher_close(NodeSpecWatcher *w) +{ +#ifdef __linux__ + if (w->inotify_fd >= 0) + { + close(w->inotify_fd); + w->inotify_fd = -1; + w->watch_fd = -1; + } +#endif + w->active = false; +} diff --git a/src/bin/pg_autoctl/nodespec.h b/src/bin/pg_autoctl/nodespec.h new file mode 100644 index 000000000..3328643f1 --- /dev/null +++ b/src/bin/pg_autoctl/nodespec.h @@ -0,0 +1,160 @@ +/* + * src/bin/pg_autoctl/nodespec.h + * Declarative node description file (.ini) — read, write, and converge. + * + * A NodeSpec is the in-memory representation of a pg_autoctl_node.ini file. + * It covers all parameters needed to create and run one pg_auto_failover node. + * + * Lifecycle: + * 1. nodespec_read() — parse the file into a NodeSpec + * 2. nodespec_create() — run `pg_autoctl create ` if PGDATA absent + * 3. nodespec_run() — hand off to the normal run path + * 4. nodespec_apply() — converge mutable fields on an already-running node + * + * The supervisor calls nodespec_check_and_apply() on a timer so that editing + * the file and saving it is sufficient to reconfigure a running node. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef NODESPEC_H +#define NODESPEC_H + +#include +#include +#include + +#include "postgres_fe.h" /* MAXPGPATH, NAMEDATALEN, etc. */ +#include "pgsql.h" /* MAXCONNINFO */ +#include "pgsetup.h" /* PgInstanceKind, NODE_KIND_* */ + +/* + * Fixed location inside every container. The compose generator writes one + * ini file per node and bind-mounts it here. Having a fixed path means the + * Docker image needs only a single CMD: + * + * CMD ["pg_autoctl", "node", "run", PG_AUTOCTL_NODESPEC_PATH] + */ +#define PG_AUTOCTL_NODESPEC_PATH "/etc/pgaf/node.ini" + +/* + * How often the supervisor polls the spec file for changes (seconds). + * inotify/kqueue are used when available; this is the fallback interval. + */ +#define NODESPEC_WATCH_INTERVAL_SECS 10 + +/* ----------------------------------------------------------------------- + * NodeSpec — one-to-one with the [sections] of pg_autoctl_node.ini + * ----------------------------------------------------------------------- */ + +typedef struct NodeSpec +{ + /* [node] */ + PgInstanceKind kind; /* postgres | coordinator | worker | monitor */ + char name[_POSIX_HOST_NAME_MAX]; /* --name; defaults to hostname when empty */ + char hostname[_POSIX_HOST_NAME_MAX]; + int port; /* Postgres port, default 5432 */ + + /* [postgresql] */ + char pgdata[MAXPGPATH]; + + /* [monitor] — empty for kind == monitor */ + char monitor_pguri[MAXCONNINFO]; + bool noMonitor; /* [monitor] no_monitor=true: standalone mode */ + int nodeId; /* [monitor] node_id: required with --disable-monitor */ + + /* [formation] */ + char formation[NAMEDATALEN]; /* default "default" */ + int group; /* Citus group; 0 = coordinator */ + + /* [settings] — mutable; applied on SIGHUP / file change */ + int candidate_priority; /* 0-100, default 50 */ + bool replication_quorum; /* sync quorum participant, default true */ + + /* [options] — immutable; used only at pg_autoctl create time */ + char ssl[32]; /* self-signed | verify-ca | verify-full | off */ + char auth[32]; /* trust | md5 | scram | cert */ + bool pg_hba_lan; /* add --pg-hba-lan flag */ + + /* [ssl] — certificate paths for verify-ca / verify-full mode */ + char ssl_ca_file[MAXPGPATH]; + char ssl_cert_file[MAXPGPATH]; + char ssl_key_file[MAXPGPATH]; + bool launchDeferred; /* [launch] mode=deferred: wait for node start */ + + /* [formation ] — monitor kind: non-default formations to create */ +#define NODESPEC_MAX_FORMATIONS 16 + int formationCount; + char formationNames[NODESPEC_MAX_FORMATIONS][NAMEDATALEN]; + char formationKinds[NODESPEC_MAX_FORMATIONS][NAMEDATALEN]; /* "pgsql" default */ + + /* [pg_auto_failover] — monitor kind: password for autoctl_node role */ + char autoctl_node_password[MAXCONNINFO]; + + /* [replication] — postgres kind: password for pgautofailover_replicator */ + char replication_password[MAXCONNINFO]; + + /* [pg_auto_failover] — postgres kind: password for pgautofailover_monitor */ + char monitor_password[MAXCONNINFO]; + + /* [citus] — Citus secondary/read-replica cluster settings */ + bool citusSecondary; /* role = secondary */ + char citusClusterName[NAMEDATALEN]; /* cluster_name = */ +} NodeSpec; + +/* ----------------------------------------------------------------------- + * File watcher state — embedded in Supervisor + * ----------------------------------------------------------------------- */ + +typedef struct NodeSpecWatcher +{ + bool active; /* true once nodespec_watcher_init() succeeds */ + char path[MAXPGPATH]; /* path of the watched file */ + time_t last_mtime; /* mtime at last check */ + time_t last_checked; /* wall-clock time of last poll */ + +#ifdef __linux__ + int inotify_fd; /* inotify instance fd, -1 if unavailable */ + int watch_fd; /* inotify watch descriptor */ +#endif +} NodeSpecWatcher; + +/* ----------------------------------------------------------------------- + * API + * ----------------------------------------------------------------------- */ + +/* Parse a pg_autoctl_node.ini file into *spec. Returns false on error. */ +bool nodespec_read(const char *path, NodeSpec *spec); + +/* Write *spec as a pg_autoctl_node.ini file (for pg_autoctl node show). */ +bool nodespec_write(const NodeSpec *spec, FILE *out); + +/* Build the argv[] for `pg_autoctl create [flags]`. + * Caller provides args[] with room for at least 40 char* entries. + * Returns the number of entries filled (not counting the trailing NULL). */ +int nodespec_create_argv(const NodeSpec *spec, + const char *pg_autoctl_path, + char **args, int args_size); + +/* Apply mutable fields (candidate_priority, replication_quorum) to a + * running node by calling into the keeper/monitor APIs directly. + * Logs what changed. Returns false only on hard error. */ +bool nodespec_apply(const NodeSpec *new_spec, const NodeSpec *old_spec); +bool nodespec_write_to_path(const NodeSpec *spec, const char *path); + +/* ----------------------------------------------------------------------- + * Watcher — called from supervisor loop + * ----------------------------------------------------------------------- */ + +/* Initialise watcher state. Sets up inotify on Linux if available. */ +bool nodespec_watcher_init(NodeSpecWatcher *w, const char *path); + +/* Called from the supervisor's 100 ms tick. + * Returns true if the file changed and nodespec_apply() was called. */ +bool nodespec_watcher_check(NodeSpecWatcher *w, const NodeSpec *current); + +/* Release inotify fds (called at supervisor shutdown). */ +void nodespec_watcher_close(NodeSpecWatcher *w); + +#endif /* NODESPEC_H */ diff --git a/src/bin/pg_autoctl/pghba.c b/src/bin/pg_autoctl/pghba.c index affe95cdd..921ccc8ce 100644 --- a/src/bin/pg_autoctl/pghba.c +++ b/src/bin/pg_autoctl/pghba.c @@ -700,6 +700,68 @@ pghba_enable_lan_cidr(PGSQL *pgsql, } +/* + * pghba_ensure_ident_map_entry ensures that pg_ident.conf contains the given + * ident map entry, appending it if missing. Used with cert auth so that the + * autoctl_node client certificate CN maps to the pgautofailover_replicator + * PostgreSQL role for replication connections. + */ +bool +pghba_ensure_ident_map_entry(const char *pgdata, + const char *mapName, + const char *systemUser, + const char *pgUser) +{ + char identFilePath[MAXPGPATH]; + char *contents = NULL; + long size = 0L; + + sformat(identFilePath, MAXPGPATH, "%s/pg_ident.conf", pgdata); + + if (!read_file(identFilePath, &contents, &size)) + { + log_error("Failed to read \"%s\"", identFilePath); + return false; + } + + /* check whether this exact entry already exists */ + PQExpBuffer needle = createPQExpBuffer(); + if (!needle) + { + free(contents); + log_error("Failed to allocate memory"); + return false; + } + appendPQExpBuffer(needle, "%s %s %s", mapName, systemUser, pgUser); + + if (strstr(contents, needle->data) != NULL) + { + /* already present */ + destroyPQExpBuffer(needle); + free(contents); + return true; + } + destroyPQExpBuffer(needle); + + /* append the new entry */ + FILE *f = fopen(identFilePath, "a"); /* IGNORE-BANNED */ + if (!f) + { + log_error("Failed to open \"%s\" for appending: %m", identFilePath); + free(contents); + return false; + } + fformat(f, "%s %s %s%s\n", + mapName, systemUser, pgUser, HBA_LINE_COMMENT); + fclose(f); + free(contents); + + log_info("Added ident map entry \"%s %s %s\" to \"%s\"", + mapName, systemUser, pgUser, identFilePath); + return true; +} + + /* * hba_check_hostname returns true when the DNS setting looks compatible with * Postgres expectations for an HBA hostname entry. diff --git a/src/bin/pg_autoctl/pghba.h b/src/bin/pg_autoctl/pghba.h index 4a8450d64..abbc3226e 100644 --- a/src/bin/pg_autoctl/pghba.h +++ b/src/bin/pg_autoctl/pghba.h @@ -49,6 +49,11 @@ bool pghba_enable_lan_cidr(PGSQL *pgsql, HBAEditLevel hbaLevel, const char *pgdata); +bool pghba_ensure_ident_map_entry(const char *pgdata, + const char *mapName, + const char *systemUser, + const char *pgUser); + bool pghba_check_hostname(const char *hostname, char *ipaddr, size_t size, bool *useHostname); diff --git a/src/bin/pg_autoctl/primary_standby.c b/src/bin/pg_autoctl/primary_standby.c index 96a171978..c1ed37419 100644 --- a/src/bin/pg_autoctl/primary_standby.c +++ b/src/bin/pg_autoctl/primary_standby.c @@ -454,6 +454,14 @@ upstream_has_replication_slot(ReplicationSource *upstream, */ pg_setup_get_local_connection_string(&upstreamSetup, connectionString); + if (!IS_EMPTY_STRING_BUFFER(upstream->password)) + { + char connWithPw[MAXCONNINFO]; + sformat(connWithPw, sizeof(connWithPw), "%s password=%s", + connectionString, upstream->password); + strlcpy(connectionString, connWithPw, sizeof(connectionString)); + } + if (!pgsql_init(&upstreamClient, connectionString, PGSQL_CONN_UPSTREAM)) { /* errors have already been logged */ @@ -840,6 +848,10 @@ standby_init_replication_source(LocalPostgresServer *postgres, { strlcpy(upstream->targetLSN, targetLSN, PG_LSN_MAXLENGTH); } + else + { + upstream->targetLSN[0] = '\0'; + } upstream->sslOptions = sslOptions; @@ -1207,7 +1219,7 @@ postgres_maybe_do_crash_recovery(LocalPostgresServer *postgres) default: { /* wait until postgres crash recovery is done */ - for (int attempts = 0;; attempts++) + for (;;) { int timeout = 30; diff --git a/src/bin/pg_autoctl/service_keeper.c b/src/bin/pg_autoctl/service_keeper.c index 5c3d4b557..6f36e5704 100644 --- a/src/bin/pg_autoctl/service_keeper.c +++ b/src/bin/pg_autoctl/service_keeper.c @@ -17,6 +17,7 @@ #include "cli_common.h" #include "cli_root.h" #include "defaults.h" +#include "file_utils.h" #include "fsm.h" #include "keeper.h" #include "keeper_config.h" @@ -143,11 +144,22 @@ service_keeper_start(void *context, pid_t *pid) /* * service_keeper_runprogram runs the node_active protocol service: * - * $ pg_autoctl do service node-active --pgdata ... + * $ pg_autoctl internal service node-active --pgdata ... * * This function is intended to be called from the child process after a fork() * has been successfully done at the parent process level: it's calling * execve() and will never return. + * + * Using execve() (rather than just calling the service function directly) is a + * deliberate design choice for live upgrades: when keeper_node_active() detects + * a monitor extension version mismatch it exits, and the supervisor restarts + * the child via fork()+execve() — which re-execs the pg_autoctl binary from + * disk. If the binary has been updated in place the restarted child picks up + * the new version automatically, making zero-downtime upgrades possible even + * when pg_autoctl runs as PID 1 in a Docker/Kubernetes container. + * + * See keeper_check_monitor_extension_version() in keeper.c for the exit path + * that triggers this mechanism. */ void service_keeper_runprogram(Keeper *keeper) @@ -170,7 +182,7 @@ service_keeper_runprogram(Keeper *keeper) setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "node-active"; args[argsIndex++] = "--pgdata"; @@ -665,7 +677,18 @@ keeper_node_active_loop(Keeper *keeper, pid_t start_pid) if (nodeHasBeenDroppedFromTheMonitor) { - /* signal that it's time to shutdown everything */ + /* + * The monitor has confirmed the drop (the node row has been removed). + * Remove local pg_autoctl files now that it is safe to do so: if we + * were interrupted before this point, the files would still exist and + * the keeper could restart and re-enter this drop sequence. PGDATA is + * intentionally left intact; use "pg_autoctl drop node --destroy" to + * remove it. + */ + (void) unlink_file(keeper->config.pathnames.init); + (void) unlink_file(keeper->config.pathnames.state); + (void) unlink_file(keeper->config.pathnames.config); + exit(EXIT_CODE_DROPPED); } diff --git a/src/bin/pg_autoctl/service_monitor.c b/src/bin/pg_autoctl/service_monitor.c index e2a7f4e3a..c09e43dde 100644 --- a/src/bin/pg_autoctl/service_monitor.c +++ b/src/bin/pg_autoctl/service_monitor.c @@ -119,13 +119,18 @@ service_monitor_start(void *context, pid_t *pid) /* - * service_monitor_runprogram runs the node_active protocol service: + * service_monitor_runprogram runs the monitor listener service: * - * $ pg_autoctl do service monitor --pgdata ... + * $ pg_autoctl internal service listener --pgdata ... * * This function is intended to be called from the child process after a fork() * has been successfully done at the parent process level: it's calling * execve() and will never return. + * + * Using execve() is a deliberate live-upgrade design choice: the supervisor + * restarts children via fork()+execve(), re-execing the binary from disk so + * an in-place update of pg_autoctl is picked up without restarting the + * supervisor (which may be PID 1 in a container). */ void service_monitor_runprogram(Monitor *monitor) @@ -160,7 +165,7 @@ service_monitor_runprogram(Monitor *monitor) setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "listener"; args[argsIndex++] = "--pgdata"; diff --git a/src/bin/pg_autoctl/service_monitor_init.c b/src/bin/pg_autoctl/service_monitor_init.c index 06998b43c..02d8c9c9e 100644 --- a/src/bin/pg_autoctl/service_monitor_init.c +++ b/src/bin/pg_autoctl/service_monitor_init.c @@ -134,7 +134,8 @@ service_monitor_init_start(void *context, pid_t *pid) (void) set_ps_title(serviceName); /* finish the install if necessary */ - if (!monitor_install(config->hostname, *pgSetup, false)) + if (!monitor_install(config->hostname, *pgSetup, false, + config->autoctl_node_password)) { /* errors have already been logged */ exit(EXIT_CODE_INTERNAL_ERROR); @@ -142,6 +143,27 @@ service_monitor_init_start(void *context, pid_t *pid) log_info("Monitor has been successfully initialized."); + /* create any non-default formations requested via --formation */ + for (int fi = 0; fi < config->formationCount; fi++) + { + char *fname = config->formationNames[fi]; + char *fkind = config->formationKinds[fi][0] + ? config->formationKinds[fi] : "pgsql"; + + log_info("Creating formation \"%s\" (kind %s)", fname, fkind); + + if (!monitor_create_formation(monitor, fname, fkind, + DEFAULT_DATABASE_NAME, + + /* hasSecondary */ true, + + /* numberSyncStandbys */ 0)) + { + log_error("Failed to create formation \"%s\"", fname); + exit(EXIT_CODE_INTERNAL_ERROR); + } + } + if (createAndRun) { /* here we call execv() so we never get back */ diff --git a/src/bin/pg_autoctl/service_postgres_ctl.c b/src/bin/pg_autoctl/service_postgres_ctl.c index 82daa6863..fcf20e8a5 100644 --- a/src/bin/pg_autoctl/service_postgres_ctl.c +++ b/src/bin/pg_autoctl/service_postgres_ctl.c @@ -92,7 +92,11 @@ service_postgres_ctl_start(void *context, pid_t *pid) /* * service_postgres_ctl_runprogram runs the postgres controller service: * - * $ pg_autoctl do service postgres --pgdata ... + * $ pg_autoctl internal service postgres --pgdata ... + * + * Uses fork()+execve() so the supervisor can restart the child with the + * current binary on disk — enabling live (zero-downtime) upgrades when + * pg_autoctl runs as PID 1 in a container. */ void service_postgres_ctl_runprogram() @@ -129,7 +133,7 @@ service_postgres_ctl_runprogram() setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "postgres"; args[argsIndex++] = "--pgdata"; diff --git a/src/bin/pg_autoctl/state.c b/src/bin/pg_autoctl/state.c index 553f3bef3..04cdb1d38 100644 --- a/src/bin/pg_autoctl/state.c +++ b/src/bin/pg_autoctl/state.c @@ -488,7 +488,9 @@ NodeStateToString(NodeState s) } default: + { return "Unknown State"; + } } } @@ -662,7 +664,9 @@ PreInitPostgreInstanceStateToString(PreInitPostgreInstanceState pgInitState) } default: + { return "unknown"; + } } /* keep compiler happy */ diff --git a/src/bin/pg_autoctl/supervisor.c b/src/bin/pg_autoctl/supervisor.c index 9e20853d5..c05c19d04 100644 --- a/src/bin/pg_autoctl/supervisor.c +++ b/src/bin/pg_autoctl/supervisor.c @@ -20,6 +20,7 @@ #include "cli_root.h" #include "defaults.h" +#include "nodespec.h" #include "env_utils.h" #include "fsm.h" #include "keeper.h" @@ -76,6 +77,32 @@ supervisor_start(Service services[], int serviceCount, const char *pidfile) /* copy the pidfile over to our supervisor structure */ strlcpy(supervisor.pidfile, pidfile, MAXPGPATH); + /* + * If we were started by `pg_autoctl node run`, the node spec path is + * passed via the PG_AUTOCTL_NODESPEC env var. Set up the file watcher + * so the supervisor can converge mutable settings when the file changes. + */ + { + char specPath[MAXPGPATH] = { 0 }; + + if (env_exists("PG_AUTOCTL_NODESPEC") && + get_env_copy("PG_AUTOCTL_NODESPEC", specPath, sizeof(specPath)) && + !IS_EMPTY_STRING_BUFFER(specPath)) + { + if (nodespec_read(specPath, &supervisor.watchedSpec) && + nodespec_watcher_init(&supervisor.watcher, specPath)) + { + log_info("Supervisor: watching node spec \"%s\"", specPath); + } + else + { + log_warn("Supervisor: failed to initialise node spec watcher " + "for \"%s\"; changes to the file will be ignored", + specPath); + } + } + } + /* * Create our PID file, or quit now if another pg_autoctl instance is * runnning. @@ -219,6 +246,14 @@ supervisor_loop(Supervisor *supervisor) { /* avoid busy looping on waitpid(WNOHANG) */ pg_usleep(100 * 1000); /* 100 ms */ + + /* + * Check if the node spec file has changed and apply mutable + * settings if so. Uses inotify on Linux, mtime poll elsewhere. + * No-op when watcher.active is false (normal run path). + */ + (void) nodespec_watcher_check(&supervisor->watcher, + &supervisor->watchedSpec); } /* ignore errors */ diff --git a/src/bin/pg_autoctl/supervisor.h b/src/bin/pg_autoctl/supervisor.h index 903b587ae..a043f1a11 100644 --- a/src/bin/pg_autoctl/supervisor.h +++ b/src/bin/pg_autoctl/supervisor.h @@ -12,13 +12,15 @@ #include #include +#include "nodespec.h" + /* * pg_autoctl runs sub-processes as "services", and we need to use the same * service names in several places: * * - the main pidfile, * - the per-service name for the pidfile is derived from this, - * - the pg_autoctl do service getpid|restart commands + * - the pg_autoctl manual service getpid|restart commands */ #define SERVICE_NAME_POSTGRES "postgres" #define SERVICE_NAME_KEEPER "node-active" @@ -122,6 +124,16 @@ typedef struct Supervisor bool shutdownSequenceInProgress; int shutdownSignal; int stoppingLoopCounter; + + /* + * Optional node spec watcher. When pg_autoctl is started via + * `pg_autoctl node run `, the supervisor watches the ini file for + * changes and converges mutable settings automatically. + * + * watcher.active is false when not in use (normal create/run path). + */ + NodeSpecWatcher watcher; + NodeSpec watchedSpec; /* last-applied spec — baseline for diff */ } Supervisor; diff --git a/src/bin/pg_autoctl/watch.c b/src/bin/pg_autoctl/watch.c index eb29a6c81..c37c885d5 100644 --- a/src/bin/pg_autoctl/watch.c +++ b/src/bin/pg_autoctl/watch.c @@ -342,6 +342,7 @@ cli_watch_process_keys(WatchContext *context) } } } + /* left and right moves are conditionnal / relative */ else if (ch == KEY_LEFT || ch == ctrl('b') || ch == 'h') { @@ -365,6 +366,7 @@ cli_watch_process_keys(WatchContext *context) context->move = WATCH_MOVE_FOCUS_NONE; } } + /* left and right moves are conditionnal / relative */ else if (ch == KEY_RIGHT || ch == ctrl('f') || ch == 'l') { @@ -380,6 +382,7 @@ cli_watch_process_keys(WatchContext *context) context->move = WATCH_MOVE_FOCUS_NONE; } } + /* home and end moves are unconditionnal / absolute */ else if (ch == KEY_HOME || ch == ctrl('a') || ch == '0') { @@ -391,6 +394,7 @@ cli_watch_process_keys(WatchContext *context) { context->move = WATCH_MOVE_FOCUS_END; } + /* up is C-p in Emacs, k in vi(m) */ else if (ch == KEY_UP || ch == ctrl('p') || ch == 'k') { @@ -401,6 +405,7 @@ cli_watch_process_keys(WatchContext *context) --context->selectedRow; } } + /* page up, which is also C-u in the terminal with less/more etc */ else if (ch == KEY_PPAGE || ch == ctrl('u')) { @@ -413,6 +418,7 @@ cli_watch_process_keys(WatchContext *context) context->selectedRow -= 5; } } + /* down is C-n in Emacs, j in vi(m) */ else if (ch == KEY_DOWN || ch == ctrl('n') || ch == 'j') { @@ -423,6 +429,7 @@ cli_watch_process_keys(WatchContext *context) ++context->selectedRow; } } + /* page down, which is also C-d in the terminal with less/more etc */ else if (ch == KEY_NPAGE || ch == ctrl('d')) { @@ -436,6 +443,7 @@ cli_watch_process_keys(WatchContext *context) context->selectedRow += 5; } } + /* cancel current selected row */ else if (ch == KEY_DL || ch == KEY_DC) { @@ -561,8 +569,8 @@ cli_watch_render(WatchContext *context, WatchContext *previous) context->startCol != previous->startCol || context->cookedMode != previous->cookedMode || context->eventsArray.count != previous->eventsArray.count || - (context->eventsArray.events[0].eventId != - previous->eventsArray.events[0].eventId)) + (context->eventsArray.events[0].eventId != previous->eventsArray.events[0].eventId + )) { (void) clear_line_at(++printedRows); diff --git a/src/bin/pgaftest/Makefile b/src/bin/pgaftest/Makefile new file mode 100644 index 000000000..259b77a89 --- /dev/null +++ b/src/bin/pgaftest/Makefile @@ -0,0 +1,120 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the PostgreSQL License. + +PGAFTEST = ./pgaftest + +SRC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +PGACTL_DIR := $(SRC_DIR)../pg_autoctl/ + +# Must be set before include so that targets in Makefile.common don't +# become the default goal when included before the all: rule below. +.DEFAULT_GOAL := all + +include $(SRC_DIR)../common/Makefile.common + +# Include both our own dir and pg_autoctl for shared headers +override CFLAGS += -I$(SRC_DIR) -I$(PGACTL_DIR) + +# ----------------------------------------------------------------------- +# Sources that live in this directory +# ----------------------------------------------------------------------- +LOCAL_SRC = main.c cli_root.c cli_demo.c cli_indent.c \ + compose_gen.c test_runner.c \ + test_spec_scan.c test_spec_parse.c + +LOCAL_OBJS = $(patsubst %.c,%.o,$(LOCAL_SRC)) + +# ----------------------------------------------------------------------- +# pg_autoctl-specific shared sources (business logic; not in common/). +# Compiled as shared-%.o to distinguish from local objects. +# ----------------------------------------------------------------------- +SHARED_SRCS = cli_common.c config.c coordinator.c fsm.c fsm_transition.c \ + fsm_transition_citus.c keeper.c keeper_config.c keeper_pg_init.c \ + monitor.c monitor_config.c monitor_pg_init.c \ + nodespec.c nodestate_utils.c pghba.c primary_standby.c \ + service_keeper.c service_keeper_init.c service_monitor.c \ + service_monitor_init.c service_postgres.c service_postgres_ctl.c \ + state.c supervisor.c systemd_config.c + +SHARED_OBJS = $(patsubst %.c,shared-%.o,$(SHARED_SRCS)) + +OBJS = $(LOCAL_OBJS) $(SHARED_OBJS) +OBJS += lib-log.o lib-commandline.o lib-parson.o lib-snprintf.o lib-strerror.o +OBJS += $(COMMON_LIB) + +INCLUDES = $(wildcard $(SRC_DIR)*.h) + +# ----------------------------------------------------------------------- +# Flex / Bison +# ----------------------------------------------------------------------- +FLEX = flex +BISON = bison + +test_spec_scan.c: test_spec_scan.l test_spec_parse.h + $(FLEX) -o $@ $< + +test_spec_parse.c test_spec_parse.h: test_spec_parse.y + $(BISON) -d -o test_spec_parse.c $< + +# Convenience target to regenerate parser/scanner sources explicitly. +# Running "make generate" is the only supported way to regenerate the +# committed .c/.h files from the .y/.l grammars. +.PHONY: generate +generate: test_spec_parse.c test_spec_parse.h test_spec_scan.c + +# cli_indent.c includes test_spec_parse.h (bison-generated); ensure it's +# built first so a parallel clean build doesn't race on the generated header. +cli_indent.o: test_spec_parse.h + +# Bison/flex generated files have unavoidable unused tables and helper +# functions that vary by generator version. Suppress those warnings so +# that pg_config --cflags -Werror does not break the build. +GENERATED_CFLAGS = $(CFLAGS) \ + -Wno-unused-variable \ + -Wno-unused-function \ + -Wno-unused-const-variable \ + -Wno-return-type \ + -Wno-error + +test_spec_parse.o: test_spec_parse.c + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(CC) $(GENERATED_CFLAGS) -c -MMD -MP -MF$(DEPDIR)/test_spec_parse.Po -o $@ $< + +test_spec_scan.o: test_spec_scan.c + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(CC) $(GENERATED_CFLAGS) -c -MMD -MP -MF$(DEPDIR)/test_spec_scan.Po -o $@ $< + +# ----------------------------------------------------------------------- +# Rules for shared objects (compiled from pg_autoctl/ sources) +# ----------------------------------------------------------------------- +shared-%.o: $(PGACTL_DIR)%.c + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/shared-$(*F).Po -o $@ $< + +Po_files := $(wildcard $(DEPDIR)/*.Po) +ifneq (,$(Po_files)) +include $(Po_files) +endif + +# ----------------------------------------------------------------------- +all: $(COMMON_LIB) $(PGAFTEST) ; + +$(PGAFTEST): $(OBJS) $(INCLUDES) + $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LIBS) -o $@ + +clean: + rm -f $(OBJS) $(PGAFTEST) + rm -f $(COMMON_LIB) $(COMMON_OBJ) + rm -rf $(DEPDIR) + +# Remove generated parser/scanner sources only when explicitly requested. +# The committed .c files allow building without bison/flex installed and +# without risking version-skew in Docker. +distclean: clean + rm -f test_spec_scan.c test_spec_parse.c test_spec_parse.h + +install: $(PGAFTEST) + install -d $(DESTDIR)$(BINDIR) + install -m 0755 $(PGAFTEST) $(DESTDIR)$(BINDIR) + +.PHONY: all clean install diff --git a/src/bin/pgaftest/cli_demo.c b/src/bin/pgaftest/cli_demo.c new file mode 100644 index 000000000..0ff66a8b9 --- /dev/null +++ b/src/bin/pgaftest/cli_demo.c @@ -0,0 +1,66 @@ +/* + * src/bin/pgaftest/cli_demo.c + * Demo application sub-command for pgaftest (moved from pg_autoctl do demo). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include "commandline.h" +#include "cli_demo.h" +#include "log.h" + +/* + * The demo sub-commands forward to pg_autoctl's demoapp implementation. + * For now we provide a stub that guides the user; the full implementation + * is linked from the shared pg_autoctl sources. + */ + +static void +cli_demo_run(int argc, char **argv) +{ + log_error("pgaftest demo run: not yet implemented in this build."); + log_info("Use: docker compose exec pg_autoctl do demo run ..."); +} + +static void +cli_demo_uri(int argc, char **argv) +{ + log_error("pgaftest demo uri: not yet implemented in this build."); +} + +static void +cli_demo_ping(int argc, char **argv) +{ + log_error("pgaftest demo ping: not yet implemented in this build."); +} + +static void +cli_demo_summary(int argc, char **argv) +{ + log_error("pgaftest demo summary: not yet implemented in this build."); +} + +static CommandLine demo_run = + make_command("run", "Run the demo app", "", "", NULL, cli_demo_run); +static CommandLine demo_uri = + make_command("uri", "Show app URI", "", "", NULL, cli_demo_uri); +static CommandLine demo_ping = + make_command("ping", "Ping the app URI", "", "", NULL, cli_demo_ping); +static CommandLine demo_summary = + make_command("summary", "Show demo summary", "", "", NULL, cli_demo_summary); + +static CommandLine *demo_subcommands[] = { + &demo_run, + &demo_uri, + &demo_ping, + &demo_summary, + NULL +}; + +CommandLine pgaftest_demo_command = + make_command_set("demo", + "Demo application for pg_auto_failover", + "", "", + NULL, demo_subcommands); diff --git a/src/bin/pgaftest/cli_demo.h b/src/bin/pgaftest/cli_demo.h new file mode 100644 index 000000000..8c11bc470 --- /dev/null +++ b/src/bin/pgaftest/cli_demo.h @@ -0,0 +1,16 @@ +/* + * src/bin/pgaftest/cli_demo.h + * Demo application CLI (moved from pg_autoctl do demo). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_DEMO_H +#define CLI_DEMO_H + +#include "commandline.h" + +extern CommandLine pgaftest_demo_command; + +#endif /* CLI_DEMO_H */ diff --git a/src/bin/pgaftest/cli_indent.c b/src/bin/pgaftest/cli_indent.c new file mode 100644 index 000000000..28ab5a2ed --- /dev/null +++ b/src/bin/pgaftest/cli_indent.c @@ -0,0 +1,899 @@ +/* + * src/bin/pgaftest/cli_indent.c + * pgaftest indent — parse a .pgaf file and rewrite it with canonical + * indentation. + * + * The output format follows these conventions: + * - 4-space indentation for all block contents + * - multi-line node specs in cluster{}: each property on its own continuation + * line, indented to align under the node name + * - multi-condition wait until: each "and state is " on its own + * line, indented under the first condition + * - sequence block: one step name per line, indented + * - leading file comment block is preserved verbatim + * - comment blocks immediately before setup/teardown/step are preserved + * - inline comments inside step bodies are dropped (AST does not store them) + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include + +#include "cli_indent.h" +#include "test_spec.h" +#include "test_spec_parse.h" +#include "defaults.h" +#include "log.h" +#include "string_utils.h" + +/* + * StepComment — comment block that precedes a named block in the raw file. + * name is "setup", "teardown", or the step name. + * text is malloc'd; NULL if no comment precedes that block. + */ +#define MAX_STEP_COMMENTS 256 + +typedef struct +{ + char name[128]; + char *text; +} StepComment; + +typedef struct +{ + StepComment entries[MAX_STEP_COMMENTS]; + int count; +} StepCommentMap; + +/* forward declarations */ +static void print_header(FILE *out, const char *path); +static StepCommentMap *collect_comments(const char *path); +static const char *lookup_comment(const StepCommentMap *m, const char *name); +static void free_comments(StepCommentMap *m); +static void print_cluster(FILE *out, const TestSpec *spec); +static void print_step(FILE *out, const TestStep *step, bool named); +static void print_cmd(FILE *out, const TestCmd *cmd, int indent); +static void print_cmd_wait(FILE *out, const TestCmd *cmd, int indent); +static void normalize_sql(const char *in, char *out, int outlen); +static int match_clause_keyword(const char *p); +static void emit_segment(FILE *out, const char *seg, int bodyIndent); +static void print_sql_body(FILE *out, const char *raw_sql, int bodyIndent); + +/* ----------------------------------------------------------------------- + * Entry point called from cli_root.c + * ----------------------------------------------------------------------- */ + +void +cli_indent(int argc, char **argv) +{ + if (argc < 1 || argv[0] == NULL) + { + fprintf(stderr, "Usage: pgaftest indent \n"); + exit(EXIT_CODE_BAD_ARGS); + } + + const char *path = argv[0]; + + TestSpec *spec = parse_test_spec(path); + if (!spec) + { + log_error("Failed to parse \"%s\"", path); + exit(EXIT_CODE_BAD_ARGS); + } + + StepCommentMap *cmap = collect_comments(path); + + FILE *out = stdout; + + print_header(out, path); + + print_cluster(out, spec); + + if (spec->setup) + { + const char *cmt = lookup_comment(cmap, "setup"); + if (cmt) + fprintf(out, "\n%s\nsetup {\n", cmt); + else + fprintf(out, "\nsetup {\n"); + print_step(out, spec->setup, false); + fprintf(out, "}\n"); + } + + if (spec->teardown) + { + const char *cmt = lookup_comment(cmap, "teardown"); + if (cmt) + fprintf(out, "\n%s\nteardown {\n", cmt); + else + fprintf(out, "\nteardown {\n"); + print_step(out, spec->teardown, false); + fprintf(out, "}\n"); + } + + for (TestStep *s = spec->steps; s; s = s->next) + { + const char *cmt = lookup_comment(cmap, s->name); + if (cmt) + fprintf(out, "\n%s\nstep %s {\n", cmt, s->name); + else + fprintf(out, "\nstep %s {\n", s->name); + print_step(out, s, true); + fprintf(out, "}\n"); + } + + /* + * Emit the sequence block only when it differs from definition order. + * When sequence == steps-in-order the block is redundant noise; drop it. + */ + if (spec->sequenceLength > 0) + { + /* Check whether sequence matches step definition order exactly */ + bool redundant = true; + int si = 0; + for (TestStep *s = spec->steps; s && si < spec->sequenceLength; s = s->next, si++) + { + if (strcmp(spec->sequence[si], s->name) != 0) + { redundant = false; break; } + } + /* Also redundant only if counts match */ + if (si != spec->sequenceLength || spec->sequenceLength != spec->stepCount) + redundant = false; + + if (!redundant) + { + fprintf(out, "\nsequence\n"); + for (int i = 0; i < spec->sequenceLength; i++) + fprintf(out, " %s\n", spec->sequence[i]); + } + } + + free_comments(cmap); + exit(0); +} + + +/* ----------------------------------------------------------------------- + * Header: emit the leading # comment block from the original file. + * The parser drops comments, so we re-read the raw file here. + * Falls back to "# " when the file has no leading comment. + * ----------------------------------------------------------------------- */ + +static void +print_header(FILE *out, const char *path) +{ + FILE *f = fopen(path, "r"); + if (!f) + { + fprintf(out, "# %s\n\n", path); + return; + } + + char line[1024]; + bool any = false; + + char pending_blank[1024] = ""; /* blank lines between comment blocks */ + + while (fgets(line, sizeof(line), f)) + { + /* stop at first non-comment, non-blank line */ + if (line[0] != '#' && line[0] != '\n' && line[0] != '\r') + break; + + if (line[0] == '\n' || line[0] == '\r') + { + if (any) + { + /* accumulate blank lines; flush only if more comments follow */ + strncat(pending_blank, line, + sizeof(pending_blank) - strlen(pending_blank) - 1); + } + continue; + } + + /* comment line: flush any accumulated blanks first */ + if (pending_blank[0]) + { + fputs(pending_blank, out); + pending_blank[0] = '\0'; + } + fputs(line, out); + any = true; + } + /* discard trailing blank lines — add a single blank after */ + fclose(f); + + if (!any) + fprintf(out, "# %s\n", path); + + fprintf(out, "\n"); +} + + +/* ----------------------------------------------------------------------- + * Step comment collection — scan the raw file for # blocks that + * immediately precede "step {", "setup {", or "teardown {". + * ----------------------------------------------------------------------- */ + +static StepCommentMap * +collect_comments(const char *path) +{ + StepCommentMap *m = calloc(1, sizeof(StepCommentMap)); + if (!m) return m; + + FILE *f = fopen(path, "r"); + if (!f) return m; + + char line[1024]; + char pending[8192] = ""; /* accumulated # lines not yet assigned */ + bool skip_file_header = true; + + while (fgets(line, sizeof(line), f)) + { + /* Skip the leading file-header comment block (consumed by print_header) */ + if (skip_file_header) + { + if (line[0] == '#' || line[0] == '\n' || line[0] == '\r') + continue; + skip_file_header = false; + } + + if (line[0] == '#') + { + /* Accumulate comment line */ + strncat(pending, line, sizeof(pending) - strlen(pending) - 1); + continue; + } + + if (line[0] == '\n' || line[0] == '\r') + { + /* Blank line: keep pending — comment may continue after blank */ + if (pending[0]) + strncat(pending, line, sizeof(pending) - strlen(pending) - 1); + continue; + } + + /* Check for a block keyword */ + char kw[128] = ""; + if (sscanf(line, "step %127s", kw) == 1) + { + /* strip trailing " {" from name */ + char *brace = strchr(kw, '{'); + if (brace) *brace = '\0'; + } + else if (strncmp(line, "setup", 5) == 0) + strlcpy(kw, "setup", sizeof(kw)); + else if (strncmp(line, "teardown", 8) == 0) + strlcpy(kw, "teardown", sizeof(kw)); + + if (kw[0] && pending[0] && m->count < MAX_STEP_COMMENTS) + { + /* Strip leading blank lines */ + char *start = pending; + while (*start == '\n' || *start == '\r') start++; + + /* Strip trailing blank lines; add exactly one trailing \n */ + int len = (int) strlen(start); + while (len > 0 && (start[len-1] == '\n' || start[len-1] == '\r' || + start[len-1] == ' ')) + len--; + + if (len > 0) + { + char clean[8192]; + snprintf(clean, sizeof(clean), "%.*s\n", len, start); + strlcpy(m->entries[m->count].name, kw, + sizeof(m->entries[0].name)); + m->entries[m->count].text = strdup(clean); + m->count++; + } + } + + /* Reset pending regardless */ + pending[0] = '\0'; + } + + fclose(f); + return m; +} + +static const char * +lookup_comment(const StepCommentMap *m, const char *name) +{ + if (!m) return NULL; + for (int i = 0; i < m->count; i++) + if (strcmp(m->entries[i].name, name) == 0) + return m->entries[i].text; + return NULL; +} + +static void +free_comments(StepCommentMap *m) +{ + if (!m) return; + for (int i = 0; i < m->count; i++) + free(m->entries[i].text); + free(m); +} + + +/* ----------------------------------------------------------------------- + * Cluster block + * ----------------------------------------------------------------------- */ + +/* + * Node property token: a keyword and an optional value string (NULL = flag only). + */ +typedef struct { const char *kw; char val[128]; } NodeProp; + +static void +print_node(FILE *out, const TestNode *n, int baseIndent) +{ + NodeProp props[32]; + int pc = 0; + char numbuf[32]; + + /* kind prefix — emitted inline after the name; for workers group is also inline */ + char kindbuf[64] = ""; + if (n->kind == NODE_KIND_CITUS_COORDINATOR) + strlcpy(kindbuf, "coordinator", sizeof(kindbuf)); + else if (n->kind == NODE_KIND_CITUS_WORKER) + { + if (n->group != 0) + snprintf(kindbuf, sizeof(kindbuf), "worker group %d", n->group); + else + strlcpy(kindbuf, "worker", sizeof(kindbuf)); + } + const char *kind = kindbuf; + +#define ADD(k, v) do { props[pc].kw = (k); strlcpy(props[pc].val, (v), sizeof(props[0].val)); pc++; } while (0) +#define ADDF(k) do { props[pc].kw = (k); props[pc].val[0] = '\0'; pc++; } while (0) + if (n->launchDeferred) ADDF("launch deferred"); + if (n->async) ADDF("async"); + if (n->noMonitor) ADDF("no-monitor"); + if (n->listen) ADDF("listen"); + if (n->candidatePriority != 50) + { snprintf(numbuf, sizeof(numbuf), "%d", n->candidatePriority); ADD("candidate-priority", numbuf); } + if (!n->replicationQuorum) ADD("replication-quorum", "false"); + if (n->citusSecondary) ADDF("citus-secondary"); + if (n->citusClusterName[0]) ADD("citus-cluster-name", n->citusClusterName); + if (n->pgPort != 0) + { snprintf(numbuf, sizeof(numbuf), "%d", n->pgPort); ADD("port", numbuf); } + if (n->ssl[0]) ADD("ssl", n->ssl); + if (n->auth[0]) ADD("auth", n->auth); + if (n->debianCluster[0]) ADD("debian-cluster", n->debianCluster); + /* volumes are always multi-line; handled separately below */ + +#undef ADD + + /* compute inline representation */ + char inline_props[512] = ""; + for (int i = 0; i < pc; i++) + { + strncat(inline_props, " ", sizeof(inline_props)-strlen(inline_props)-1); + strncat(inline_props, props[i].kw, sizeof(inline_props)-strlen(inline_props)-1); + if (props[i].val[0]) + { + strncat(inline_props, " ", sizeof(inline_props)-strlen(inline_props)-1); + strncat(inline_props, props[i].val, sizeof(inline_props)-strlen(inline_props)-1); + } + } + + int nameLen = strlen(n->name); + int kindLen = kind[0] ? (int)strlen(kind) + 1 : 0; + int lineLen = baseIndent + nameLen + kindLen + strlen(inline_props); + + if ((lineLen <= 72 || pc == 0) && n->volumeCount == 0) + { + /* fits on one line — flat syntax, no "node" keyword needed */ + fprintf(out, "%*s%s", baseIndent, "", n->name); + if (kind[0]) fprintf(out, " %s", kind); + fprintf(out, "%s\n", inline_props); + return; + } + + /* multi-line — use block syntax: node { \n kind\n opt\n opt\n } */ + int innerIndent = baseIndent + 4; + fprintf(out, "%*snode %s {\n", baseIndent, "", n->name); + if (kind[0]) fprintf(out, "%*s%s\n", innerIndent, "", kind); + + for (int i = 0; i < pc; i++) + { + fprintf(out, "%*s%s", innerIndent, "", props[i].kw); + if (props[i].val[0]) + fprintf(out, " %s", props[i].val); + fprintf(out, "\n"); + } + for (int vi = 0; vi < n->volumeCount; vi++) + fprintf(out, "%*svolume %s \"%s\"\n", + innerIndent, "", + n->volumes[vi].name, n->volumes[vi].path); + fprintf(out, "%*s}\n", baseIndent, ""); +} + + +static void +print_cluster(FILE *out, const TestSpec *spec) +{ + const TestCluster *cl = &spec->cluster; + + fprintf(out, "cluster {\n"); + + if (cl->withMonitor) + fprintf(out, " monitor\n"); + + if (cl->secondMonitorName[0]) + fprintf(out, " monitor %s initially stopped\n", cl->secondMonitorName); + + if (cl->image[0]) + fprintf(out, " image \"%s\"\n", cl->image); + if (cl->ssl[0] && strcmp(cl->ssl, "self-signed") != 0) + fprintf(out, " ssl %s\n", cl->ssl); + if (cl->auth[0] && strcmp(cl->auth, "trust") != 0) + fprintf(out, " auth %s\n", cl->auth); + if (cl->extensionVersion[0]) + fprintf(out, " extension-version %s\n", cl->extensionVersion); + + for (int fi = 0; fi < cl->formationCount; fi++) + { + const TestFormation *f = &cl->formations[fi]; + + if (strcmp(f->name, "default") == 0) + fprintf(out, " formation {\n"); + else + fprintf(out, " formation %s {\n", f->name); + + for (int ni = 0; ni < f->nodeCount; ni++) + print_node(out, &f->nodes[ni], 8); + + fprintf(out, " }\n"); + } + + fprintf(out, "}\n"); +} + + +/* ----------------------------------------------------------------------- + * Step / command printing + * ----------------------------------------------------------------------- */ + +/* + * normalize_sql — collapse all whitespace runs (including embedded newlines + * from a multi-line block in the source file) to single spaces and trim + * leading/trailing whitespace. This makes `pgaftest indent` idempotent: a + * file that has already been indented produces the same output when indented + * again. + */ +static void +normalize_sql(const char *in, char *out, int outlen) +{ + const char *p = in; + char *q = out; + char *lim = out + outlen - 1; + bool ws = true; /* suppress leading whitespace */ + + while (*p && q < lim) + { + bool isws = (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r'); + if (isws) + { + if (!ws) { *q++ = ' '; } + ws = true; + } + else + { + *q++ = *p; + ws = false; + } + p++; + } + /* trim trailing space */ + if (q > out && q[-1] == ' ') q--; + *q = '\0'; +} + + +/* + * SQL_CLAUSE_KEYWORDS — clause-level keywords that force a new output line. + * Two-word keywords (ORDER BY, GROUP BY) must appear before any single-word + * prefix (BY, ORDER, GROUP) to prevent partial matches. + */ +static const char * const SQL_CLAUSE_KEYWORDS[] = { + "ORDER BY", + "GROUP BY", + "FROM", + "WHERE", + "HAVING", + "LIMIT", + "OFFSET", + NULL +}; + +/* + * match_clause_keyword — if a SQL clause keyword starts at p (case- + * insensitive, followed by space, semicolon, or end-of-string), return its + * length; otherwise return 0. + */ +static int +match_clause_keyword(const char *p) +{ + for (int i = 0; SQL_CLAUSE_KEYWORDS[i]; i++) + { + const char *kw = SQL_CLAUSE_KEYWORDS[i]; + int len = (int) strlen(kw); + if (strncasecmp(p, kw, len) == 0) + { + char next = p[len]; + if (next == ' ' || next == '\0' || next == ';') + return len; + } + } + return 0; +} + + +/* + * emit_segment — word-wrap a single normalized segment (no embedded newlines, + * no leading/trailing spaces) onto one or more lines, each prefixed with + * bodyIndent spaces and at most 79 characters wide total. + */ +static void +emit_segment(FILE *out, const char *seg, int bodyIndent) +{ + int avail = 79 - bodyIndent; + if (avail < 20) avail = 20; + + const char *p = seg; + + while (*p) + { + if ((int) strlen(p) <= avail) + { + fprintf(out, "%*s%s\n", bodyIndent, "", p); + break; + } + + /* Find the last space within avail columns. */ + const char *last_spc = NULL; + for (int i = 0; i < avail && p[i]; i++) + if (p[i] == ' ') last_spc = p + i; + + if (last_spc) + { + fprintf(out, "%*s%.*s\n", + bodyIndent, "", (int)(last_spc - p), p); + p = last_spc + 1; /* skip the space */ + } + else + { + /* No space — hard break (avoids infinite loop on long tokens). */ + fprintf(out, "%*s%.*s\n", bodyIndent, "", avail, p); + p += avail; + } + } +} + + +/* + * print_sql_body — emit normalized SQL with clause-keyword-aware line + * splitting inside a sql { … } block. + * + * Algorithm: + * 1. Normalize whitespace (fixes idempotency when the input was already + * a multi-line block from a previous indent pass). + * 2. If the whole thing fits in avail columns, emit it on one line. + * 3. Otherwise, scan for clause keywords (FROM, WHERE, ORDER BY, …). + * Each keyword starts a new output line; within each segment, apply + * word-wrap at 79 columns. + */ +static void +print_sql_body(FILE *out, const char *raw_sql, int bodyIndent) +{ + char norm[8192]; + normalize_sql(raw_sql, norm, sizeof(norm)); + + int avail = 79 - bodyIndent; + if (avail < 20) avail = 20; + + /* Short enough to fit on one line? */ + if ((int) strlen(norm) <= avail) + { + fprintf(out, "%*s%s\n", bodyIndent, "", norm); + return; + } + + /* + * Scan for clause-keyword split points. A keyword is a split point when + * it is preceded by a space (word boundary in the normalized string). + * We accumulate characters into a segment and flush when we hit a keyword. + */ + const char *p = norm; + const char *seg_start = norm; + + while (*p) + { + if (p > norm && p[-1] == ' ') + { + int kwlen = match_clause_keyword(p); + if (kwlen) + { + /* Flush the segment up to (but not including) the space + * that preceded this keyword. */ + int seglen = (int)((p - 1) - seg_start); + if (seglen > 0) + { + char seg[8192]; + memcpy(seg, seg_start, (size_t) seglen); + seg[seglen] = '\0'; + emit_segment(out, seg, bodyIndent); + } + seg_start = p; /* keyword begins the next segment */ + } + } + p++; + } + + /* Flush the final segment. */ + if (*seg_start) + emit_segment(out, seg_start, bodyIndent); +} + + +static void +print_step(FILE *out, const TestStep *step, bool named) +{ + (void) named; + for (const TestCmd *cmd = step->commands; cmd; cmd = cmd->next) + print_cmd(out, cmd, 4); +} + + +static void +print_cmd(FILE *out, const TestCmd *cmd, int indent) +{ + switch (cmd->kind) + { + case CMD_EXEC: + fprintf(out, "%*sexec %s %s\n", indent, "", cmd->service, cmd->args); + break; + case CMD_EXEC_FAILS: + fprintf(out, "%*sexec-fails %s %s\n", indent, "", + cmd->service, cmd->args); + break; + case CMD_WAIT_STATE: + case CMD_ASSERT_STATE: + case CMD_ASSERT_ASSIGNED: + case CMD_WAIT_MULTI: + print_cmd_wait(out, cmd, indent); + break; + case CMD_WAIT_STATES: + { + /* wait until s1, s2 [in group N] timeout Ns */ + fprintf(out, "%*swait until", indent, ""); + for (int i = 0; i < cmd->waitStateCount; i++) + { + if (i > 0) fprintf(out, ","); + fprintf(out, " %s", cmd->waitStates[i]); + } + if (cmd->waitGroupCount > 0) + { + for (int g = 0; g < cmd->waitGroupCount; g++) + fprintf(out, " in group %d", cmd->waitGroups[g]); + } + fprintf(out, " timeout %ds\n", cmd->timeoutSeconds); + break; + } + case CMD_WAIT_STOPPED: + fprintf(out, "%*swait until %s stopped timeout %ds\n", + indent, "", cmd->service, cmd->timeoutSeconds); + break; + case CMD_PROMOTE: + { + fprintf(out, "%*spromote", indent, ""); + for (int i = 0; i < cmd->promoteCount; i++) + { + if (i > 0) fprintf(out, ","); + fprintf(out, " %s", cmd->promoteNodes[i]); + } + fprintf(out, "\n"); + break; + } + case CMD_SQL: + { + /* + * Normalise the SQL first (collapses embedded newlines from a + * previous multi-line block), then check if the whole thing fits + * on one line: " sql { }". + * If not, open a block and emit clause-keyword-split SQL. + */ + char norm[8192]; + normalize_sql(cmd->args, norm, sizeof(norm)); + + int oneliner_len = indent + + 4 /* "sql " */ + + (int) strlen(cmd->service) + + 3 /* " { " */ + + (int) strlen(norm) + + 2; /* " }" */ + + if (oneliner_len <= 79) + { + fprintf(out, "%*ssql %s { %s }\n", + indent, "", cmd->service, norm); + } + else + { + fprintf(out, "%*ssql %s {\n", indent, "", cmd->service); + print_sql_body(out, norm, indent + 4); + fprintf(out, "%*s}\n", indent, ""); + } + break; + } + case CMD_EXPECT: + { + /* If the stored value has newlines it was converted from + * { r1 } { r2 } tuple syntax by expand_tuple_expect — reverse it. */ + if (strchr(cmd->expected, '\n')) + { + fprintf(out, "%*sexpect {", indent, ""); + const char *p = cmd->expected; + while (*p) + { + const char *nl = strchr(p, '\n'); + int len = nl ? (int)(nl - p) : (int)strlen(p); + fprintf(out, " { %.*s }", len, p); + p += len; + if (*p == '\n') p++; + } + fprintf(out, " }\n"); + } + else + fprintf(out, "%*sexpect { %s }\n", indent, "", cmd->expected); + break; + } + case CMD_EXPECT_ERROR: + fprintf(out, "%*sexpect error %s\n", indent, "", cmd->state); + break; + case CMD_NETWORK_OFF: + fprintf(out, "%*snetwork disconnect %s\n", indent, "", cmd->service); + break; + case CMD_NETWORK_ON: + fprintf(out, "%*snetwork connect %s\n", indent, "", cmd->service); + break; + case CMD_SLEEP: + fprintf(out, "%*ssleep %ds\n", indent, "", cmd->timeoutSeconds); + break; + case CMD_COMPOSE_DOWN: + fprintf(out, "%*scompose down\n", indent, ""); + break; + case CMD_COMPOSE_START: + fprintf(out, "%*scompose start %s\n", indent, "", cmd->service); + break; + case CMD_COMPOSE_STOP: + fprintf(out, "%*scompose stop %s\n", indent, "", cmd->service); + break; + case CMD_COMPOSE_KILL: + fprintf(out, "%*scompose kill %s\n", indent, "", cmd->service); + break; + case CMD_PG_AUTOCTL: + fprintf(out, "%*spg_autoctl %s\n", indent, "", cmd->args); + break; + case CMD_STOP_POSTGRES: + fprintf(out, "%*sstop postgres %s\n", indent, "", cmd->service); + break; + case CMD_START_POSTGRES: + fprintf(out, "%*sstart postgres %s\n", indent, "", cmd->service); + break; + case CMD_STAYS_WHILE: + fprintf(out, "%*sassert %s stays %s while {\n", + indent, "", cmd->service, cmd->state); + for (const TestCmd *bc = cmd->body; bc; bc = bc->next) + print_cmd(out, bc, indent + 4); + fprintf(out, "%*s}\n", indent, ""); + break; + case CMD_COMPOSE_INJECT: + fprintf(out, "%*scompose inject %s %s %s:%s\n", + indent, "", + cmd->expected, /* image */ + cmd->args, /* src path */ + cmd->service, /* dst service */ + cmd->state); /* dst path */ + break; + case CMD_SET_MONITOR: + fprintf(out, "%*sset monitor %s\n", indent, "", cmd->service); + break; + case CMD_LOGS_CHECK: + { + /* + * logs [not] contains|matches "" + * The logsNegate flag adds "not " and allowError picks the verb. + */ + const char *verb = cmd->allowError ? "matches" : "contains"; + fprintf(out, "%*slogs %s %s%s \"%s\"\n", + indent, "", + cmd->service, + cmd->logsNegate ? "not " : "", + verb, + cmd->args); + break; + } + default: + fprintf(out, "%*s(unknown cmd %d)\n", indent, "", cmd->kind); + break; + } +} + + +/* + * print_cmd_wait handles the various wait/assert forms, emitting multi-line + * output for CMD_WAIT_MULTI and for single-node waits with pass-through. + */ +static void +print_cmd_wait(FILE *out, const TestCmd *cmd, int indent) +{ + if (cmd->kind == CMD_WAIT_MULTI) + { + /* first condition on same line, subsequent "and" lines indented */ + fprintf(out, "%*swait until %s state is %s\n", + indent, "", cmd->waitNodes[0], cmd->waitStates[0]); + for (int i = 1; i < cmd->waitStateCount; i++) + fprintf(out, "%*sand %s state is %s\n", + indent + 4, "", + cmd->waitNodes[i], cmd->waitStates[i]); + fprintf(out, "%*s timeout %ds\n", indent, "", cmd->timeoutSeconds); + return; + } + + /* + * CMD_ASSERT_ASSIGNED: grammar uses `=` not `is`. + * Distinguish wait-form (has timeout) from assert-form (no timeout). + */ + if (cmd->kind == CMD_ASSERT_ASSIGNED) + { + if (cmd->timeoutSeconds > 0) + fprintf(out, "%*swait until %s assigned-state = %s timeout %ds\n", + indent, "", cmd->service, cmd->state, cmd->timeoutSeconds); + else + fprintf(out, "%*sassert %s assigned-state = %s\n", + indent, "", cmd->service, cmd->state); + return; + } + + /* CMD_ASSERT_STATE with no timeout: instant check → canonical assert form */ + if (cmd->kind == CMD_ASSERT_STATE && cmd->timeoutSeconds == 0) + { + fprintf(out, "%*sassert %s state = %s\n", + indent, "", cmd->service, cmd->state); + return; + } + + /* CMD_ASSERT_STATE with timeout, and CMD_WAIT_STATE: polling wait */ + const char *op = "wait until"; + + if (cmd->passThroughCount > 0) + { + /* multi-line: target state, then "passing through" line */ + fprintf(out, "%*s%s %s state is %s\n", + indent, "", op, cmd->service, cmd->state); + fprintf(out, "%*s passing through", indent, ""); + for (int i = 0; i < cmd->passThroughCount; i++) + { + if (i > 0) fprintf(out, ","); + fprintf(out, " %s", cmd->passThroughStates[i]); + } + fprintf(out, "\n"); + fprintf(out, "%*s timeout %ds\n", indent, "", cmd->timeoutSeconds); + return; + } + + /* simple single-line */ + if (cmd->timeoutSeconds > 0) + fprintf(out, "%*s%s %s state is %s timeout %ds\n", + indent, "", op, cmd->service, cmd->state, cmd->timeoutSeconds); + else + fprintf(out, "%*s%s %s state is %s\n", + indent, "", op, cmd->service, cmd->state); +} diff --git a/src/bin/pgaftest/cli_indent.h b/src/bin/pgaftest/cli_indent.h new file mode 100644 index 000000000..a625eead8 --- /dev/null +++ b/src/bin/pgaftest/cli_indent.h @@ -0,0 +1,15 @@ +/* + * src/bin/pgaftest/cli_indent.h + * pgaftest indent — parse a .pgaf file and rewrite it with canonical + * indentation. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_INDENT_H +#define CLI_INDENT_H + +void cli_indent(int argc, char **argv); + +#endif /* CLI_INDENT_H */ diff --git a/src/bin/pgaftest/cli_root.c b/src/bin/pgaftest/cli_root.c new file mode 100644 index 000000000..1b6304335 --- /dev/null +++ b/src/bin/pgaftest/cli_root.c @@ -0,0 +1,486 @@ +/* + * src/bin/pgaftest/cli_root.c + * Top-level sub-command table for pgaftest. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include +#include +#include + +#include "commandline.h" +#include "defaults.h" +#include "log.h" +#include "test_spec.h" +#include "test_runner.h" +#include "cli_demo.h" +#include "cli_indent.h" + +/* ----------------------------------------------------------------------- + * Shared option parsing + * ----------------------------------------------------------------------- */ + +typedef struct PgaftestOpts +{ + char specFile[1024]; + char workDir[1024]; /* empty = auto-derive from spec name */ + char stepName[64]; + char schedule[1024]; + char expected[1024]; + bool withTmux; + bool noCleanup; /* --no-cleanup: skip compose down after run */ + int verbose; +} PgaftestOpts; + +static PgaftestOpts pgaftestOpts; /* zero-initialised: workDir = "" */ + +/* + * derive_work_dir — build $TMPDIR/pgaftest/ + * + * Using TMPDIR (or /tmp when unset) + "pgaftest" + the test name means: + * - Every spec gets its own isolated directory. + * - Multiple instances of different specs can run concurrently. + * - The directory name matches the Docker Compose project name, so + * `docker compose -p basic_operation` addresses the right stack. + * + * Examples: + * tests/tap/specs/basic_operation.pgaf → /tmp/pgaftest/basic_operation + * failover.pgaf → /tmp/pgaftest/failover + */ +static void +derive_work_dir(const char *specPath, char *buf, int buflen) +{ + /* base name without directory */ + const char *base = strrchr(specPath, '/'); + base = base ? base + 1 : specPath; + + char name[256] = { 0 }; + strncpy(name, base, sizeof(name) - 1); + + /* strip .pgaf extension */ + char *dot = strrchr(name, '.'); + if (dot && strcmp(dot, ".pgaf") == 0) + *dot = '\0'; + + const char *tmpdir = getenv("TMPDIR"); + if (!tmpdir || *tmpdir == '\0') + tmpdir = "/tmp"; + + snprintf(buf, buflen, "%s/pgaftest/%s", tmpdir, name); +} + +static struct option long_options[] = { + { "work-dir", required_argument, NULL, 'w' }, + { "schedule", required_argument, NULL, 'S' }, + { "expected", required_argument, NULL, 'E' }, + { "tmux", no_argument, NULL, 't' }, + { "no-cleanup", no_argument, NULL, 'n' }, + { "verbose", no_argument, NULL, 'v' }, + { "debug", no_argument, NULL, 'd' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } +}; + +static int +pgaftest_getopts(int argc, char **argv) +{ + int c, option_index = 0; + + while ((c = getopt_long(argc, argv, "w:S:E:tnvdh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'w': + strncpy(pgaftestOpts.workDir, optarg, + sizeof(pgaftestOpts.workDir)-1); + break; + case 'S': + strncpy(pgaftestOpts.schedule, optarg, + sizeof(pgaftestOpts.schedule)-1); + break; + case 'E': + strncpy(pgaftestOpts.expected, optarg, + sizeof(pgaftestOpts.expected)-1); + break; + case 't': + pgaftestOpts.withTmux = true; + break; + case 'n': + pgaftestOpts.noCleanup = true; + break; + case 'v': + pgaftestOpts.verbose++; + log_set_level(LOG_DEBUG); + break; + case 'd': + pgaftestOpts.verbose += 2; + log_set_level(LOG_TRACE); + break; + case 'h': + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + default: + break; + } + } + return optind; +} + +/* ----------------------------------------------------------------------- + * pgaftest run + * ----------------------------------------------------------------------- */ + +static void +cli_run(int argc, char **argv) +{ + /* + * The subcommand library already called pgaftest_getopts() and stripped + * all options from argv before calling us. argv[0] is the spec file + * (if provided), argc is the number of remaining positional arguments. + */ + + /* Handle --schedule file */ + if (pgaftestOpts.schedule[0] != '\0') + { + FILE *f = fopen(pgaftestOpts.schedule, "r"); + if (!f) + { + log_error("Cannot open schedule \"%s\": %m", + pgaftestOpts.schedule); + exit(1); + } + + char line[256]; + int total = 0, failed = 0; + + while (fgets(line, sizeof(line), f)) + { + /* strip newline and comments */ + char *nl = strchr(line, '\n'); + if (nl) *nl = '\0'; + char *hash = strchr(line, '#'); + if (hash) *hash = '\0'; + + /* skip blank lines */ + char *p = line; + while (*p == ' ' || *p == '\t') p++; + if (*p == '\0') continue; + + /* build spec path from schedule entry */ + char specPath[1024]; + if (strchr(p, '/')) + strncpy(specPath, p, sizeof(specPath)-1); + else + snprintf(specPath, sizeof(specPath), + "tests/tap/specs/%s.pgaf", p); + + char workDir[1024]; + if (pgaftestOpts.workDir[0] != '\0') + { + /* explicit --work-dir: append spec name as subdir */ + const char *base = strrchr(specPath, '/'); + base = base ? base+1 : specPath; + char name[128]; + strncpy(name, base, sizeof(name)-1); + char *dot = strrchr(name, '.'); + if (dot) *dot = '\0'; + snprintf(workDir, sizeof(workDir), + "%s/%s", pgaftestOpts.workDir, name); + } + else + { + derive_work_dir(specPath, workDir, sizeof(workDir)); + } + + TestSpec *spec = parse_test_spec(specPath); + if (!spec) { failed++; total++; continue; } + + total++; + if (!runner_run(spec, workDir, pgaftestOpts.noCleanup)) + failed++; + } + fclose(f); + + fprintf(stderr, "\nSchedule complete: %d/%d passed\n", + total - failed, total); + exit(failed > 0 ? 1 : 0); + } + + /* Single spec file */ + if (argc < 1 || argv[0] == NULL) + { + log_error("Usage: pgaftest run [--schedule ] []"); + exit(1); + } + + strncpy(pgaftestOpts.specFile, argv[0], + sizeof(pgaftestOpts.specFile)-1); + + if (pgaftestOpts.workDir[0] == '\0') + derive_work_dir(pgaftestOpts.specFile, + pgaftestOpts.workDir, sizeof(pgaftestOpts.workDir)); + + TestSpec *spec = parse_test_spec(pgaftestOpts.specFile); + if (!spec) exit(1); + + bool ok = runner_run(spec, pgaftestOpts.workDir, pgaftestOpts.noCleanup); + exit(ok ? 0 : 1); +} + +/* ----------------------------------------------------------------------- + * pgaftest setup + * ----------------------------------------------------------------------- */ + +static void +cli_setup(int argc, char **argv) +{ + if (argc < 1 || argv[0] == NULL) + { + log_error("Usage: pgaftest setup [--tmux] [--work-dir

]"); + exit(1); + } + + strncpy(pgaftestOpts.specFile, argv[0], + sizeof(pgaftestOpts.specFile)-1); + + if (pgaftestOpts.workDir[0] == '\0') + derive_work_dir(pgaftestOpts.specFile, + pgaftestOpts.workDir, sizeof(pgaftestOpts.workDir)); + + TestSpec *spec = parse_test_spec(pgaftestOpts.specFile); + if (!spec) exit(1); + + bool ok = runner_setup(spec, pgaftestOpts.workDir, + pgaftestOpts.withTmux); + exit(ok ? 0 : 1); +} + +/* ----------------------------------------------------------------------- + * pgaftest step + * ----------------------------------------------------------------------- */ + +static void +cli_step(int argc, char **argv) +{ + if (argc < 1 || argv[0] == NULL) + { + log_error("Usage: pgaftest step [--work-dir ]"); + exit(1); + } + + /* step name is positional */ + strncpy(pgaftestOpts.stepName, argv[0], + sizeof(pgaftestOpts.stepName)-1); + + /* We need the spec file too — look for it in workDir */ + char specPath[1024]; + snprintf(specPath, sizeof(specPath), "%s/spec.pgaf", + pgaftestOpts.workDir); + + /* If there's a second positional arg, treat it as the spec path */ + if (argc >= 2 && argv[1] != NULL) + strncpy(specPath, argv[1], sizeof(specPath)-1); + + /* derive work dir from spec path when not given explicitly */ + if (pgaftestOpts.workDir[0] == '\0') + derive_work_dir(specPath, pgaftestOpts.workDir, + sizeof(pgaftestOpts.workDir)); + + TestSpec *spec = parse_test_spec(specPath); + if (!spec) exit(1); + + bool ok = runner_step(spec, pgaftestOpts.workDir, + pgaftestOpts.stepName); + exit(ok ? 0 : 1); +} + +/* ----------------------------------------------------------------------- + * pgaftest prepare [] + * + * Writes docker-compose.yml, *.ini files, and a Makefile to the output + * directory. Prints the docker compose command to stdout. + * ----------------------------------------------------------------------- */ + +static void +cli_prepare(int argc, char **argv) +{ + if (argc < 1 || argv[0] == NULL) + { + log_error("Usage: pgaftest prepare []"); + exit(1); + } + + strncpy(pgaftestOpts.specFile, argv[0], sizeof(pgaftestOpts.specFile)-1); + + const char *outDir = (argc >= 2 && argv[1] && argv[1][0] != '-') + ? argv[1] : NULL; + + TestSpec *spec = parse_test_spec(pgaftestOpts.specFile); + if (!spec) exit(1); + + bool ok = runner_prepare(spec, outDir); + exit(ok ? 0 : 1); +} + +/* ----------------------------------------------------------------------- + * pgaftest show + * ----------------------------------------------------------------------- */ + +static void +cli_show(int argc, char **argv) +{ + if (argc < 1 || argv[0] == NULL) + { + log_error("Usage: pgaftest show "); + exit(1); + } + + strncpy(pgaftestOpts.specFile, argv[0], + sizeof(pgaftestOpts.specFile)-1); + + TestSpec *spec = parse_test_spec(pgaftestOpts.specFile); + if (!spec) exit(1); + + bool ok = runner_show(spec); + exit(ok ? 0 : 1); +} + +/* ----------------------------------------------------------------------- + * pgaftest down + * ----------------------------------------------------------------------- */ + +static void +cli_down(int argc, char **argv) +{ + /* optional positional: spec file to derive work dir from */ + if (argc >= 1 && argv[0] && argv[0][0] != '-') + { + strncpy(pgaftestOpts.specFile, argv[0], + sizeof(pgaftestOpts.specFile)-1); + } + + if (pgaftestOpts.workDir[0] == '\0' && pgaftestOpts.specFile[0] != '\0') + derive_work_dir(pgaftestOpts.specFile, + pgaftestOpts.workDir, sizeof(pgaftestOpts.workDir)); + + /* spec file is optional for `down` — teardown may not need it */ + char specPath[1024]; + if (pgaftestOpts.specFile[0] != '\0') + strncpy(specPath, pgaftestOpts.specFile, sizeof(specPath)-1); + else + snprintf(specPath, sizeof(specPath), "%s/spec.pgaf", + pgaftestOpts.workDir); + + TestSpec *spec = NULL; + if (access(specPath, R_OK) == 0) + spec = parse_test_spec(specPath); + + /* If no spec, just run compose down */ + if (!spec) + { + int rc = system("docker compose down --volumes --remove-orphans 2>&1"); + exit(rc == 0 ? 0 : 1); + } + + bool ok = runner_down(spec, pgaftestOpts.workDir); + exit(ok ? 0 : 1); +} + +/* ----------------------------------------------------------------------- + * Root command table + * ----------------------------------------------------------------------- */ + +static CommandLine run_command = + make_command("run", + "Run a .pgaf spec headlessly (CI mode, TAP output)", + "[options] []", + " --schedule Run all specs listed in schedule file\n" + " --expected Directory of expected output files\n" + " --work-dir Working directory\n" + " (default: $TMPDIR/pgaftest/)\n" + " --no-cleanup Leave the compose stack running after the\n" + " run (pass or fail) for post-mortem inspection.\n" + " Use `pgaftest down ` to clean up.\n" + " --verbose Enable DEBUG log level\n" + " --debug Enable TRACE log level\n", + pgaftest_getopts, cli_run); + +static CommandLine setup_command = + make_command("setup", + "Bring up a cluster from a spec file (interactive mode)", + "[options] ", + " --tmux Launch a tmux session after setup\n" + " --work-dir Working directory\n" + " (default: $TMPDIR/pgaftest/)\n" + " --verbose Enable DEBUG log level\n" + " --debug Enable TRACE log level\n", + pgaftest_getopts, cli_setup); + +static CommandLine step_command = + make_command("step", + "Run a single named step against a live compose stack", + "[options] []", + " --work-dir Working directory (default: /tmp/pgaftest)\n", + pgaftest_getopts, cli_step); + +static CommandLine show_command = + make_command("show", + "Print the docker-compose.yml that would be generated", + "[options] ", + "", + pgaftest_getopts, cli_show); + +static CommandLine prepare_command = + make_command("prepare", + "Write compose files and Makefile to a directory for manual use", + " []", + " Path to the spec file\n" + " Output directory (default: -compose/)\n", + pgaftest_getopts, cli_prepare); + +static CommandLine down_command = + make_command("down", + "Run teardown block and stop the compose stack", + "[options]", + " --work-dir Working directory (default: /tmp/pgaftest)\n", + pgaftest_getopts, cli_down); + +extern void cli_indent(int argc, char **argv); + +static CommandLine indent_command = + make_command("indent", + "Parse a .pgaf spec and rewrite it with canonical indentation", + "", + "", + pgaftest_getopts, cli_indent); + +static CommandLine *root_subcommands[] = { + &run_command, + &setup_command, + &step_command, + &show_command, + &prepare_command, + &down_command, + &indent_command, + &pgaftest_demo_command, + NULL +}; + +CommandLine pgaftest_root = + make_command_set("pgaftest", + "pg_auto_failover test runner", + "[command] [options]", + " run Run a .pgaf spec (CI mode)\n" + " setup Bring up a cluster interactively\n" + " step Run one named step\n" + " show Print generated docker-compose.yml\n" + " prepare Write compose files + Makefile to a directory\n" + " down Tear down the cluster\n" + " indent Rewrite a spec with canonical indentation\n" + " demo Demo application\n", + NULL, root_subcommands); diff --git a/src/bin/pgaftest/compose_gen.c b/src/bin/pgaftest/compose_gen.c new file mode 100644 index 000000000..23e0149e7 --- /dev/null +++ b/src/bin/pgaftest/compose_gen.c @@ -0,0 +1,1079 @@ +/* + * src/bin/pgaftest/compose_gen.c + * Generate a docker-compose.yml and per-node pg_autoctl_node.ini files + * from a TestCluster spec. + * + * Design principle — uniform containers + * ====================================== + * Every container in the generated compose file uses: + * - the same Docker image (PGAF_IMAGE env var or a local build) + * - the same command: pg_autoctl node run /etc/pgaf/node.ini + * - the same config file location inside the container + * + * Node-specific behaviour (monitor vs postgres vs coordinator vs worker, + * candidate-priority, replication-quorum, formation, group) is encoded in + * the per-node pg_autoctl_node.ini file, which is bind-mounted into the + * container at /etc/pgaf/node.ini. + * + * This means: + * - No bespoke container command per node type. + * - The running supervisor watches /etc/pgaf/node.ini via inotify/mtime + * and converges mutable settings when the file is edited externally. + * - Changing candidate-priority or replication-quorum for a node is as + * simple as editing the ini file on the host — the container picks it + * up without a restart. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compose_gen.h" +#include "pgsetup.h" +#include "log.h" +#include "file_utils.h" /* sformat */ +#include "string_utils.h" + +/* + * Monitor URI used by data nodes to register. + * The monitor hostname is always "monitor" inside the compose network. + */ +#define MONITOR_PGURI \ + "postgresql://autoctl_node@monitor/pg_auto_failover" + +/* Fixed path inside every container */ +#define NODE_INI_PATH "/etc/pgaf/node.ini" + +/* Fixed PGDATA location inside every container */ +#define NODE_PGDATA "/var/lib/postgres/pgaf" + +/* Debian-style PGDATA prefix: /var/lib/postgresql// */ +#define DEBIAN_PGDATA_PREFIX "/var/lib/postgresql" + +static const char * +debian_pg_version(void) +{ + const char *v = getenv("PGVERSION"); + return (v && v[0]) ? v : "17"; +} + +/* SSL cert directory inside every container when ssl needs CA-signed certs */ +#define SSL_DIR_IN_CONTAINER "/etc/pgaf/ssl" + +/* + * Shell snippet prepended to the container start command when CA-signed certs + * are in use. + * + * - Server cert/key copied to /var/lib/postgres/ (volume root, writable) so + * PostgreSQL can open the key file as the container user. + * - Client cert/key copied to ~/.postgresql/ so libpq uses them automatically + * for all outbound connections (cert auth to monitor and peers). + * - CA cert copied to ~/.postgresql/root.crt so libpq trusts server certs. + */ +#define SSL_COPY_CERTS_CMD \ + "cp " SSL_DIR_IN_CONTAINER "/server/server.crt /var/lib/postgres/server.crt" \ + " && cp " SSL_DIR_IN_CONTAINER "/server/server.key /var/lib/postgres/server.key" \ + " && chmod 0600 /var/lib/postgres/server.key" \ + " && mkdir -p /var/lib/postgres/.postgresql" \ + " && cp " SSL_DIR_IN_CONTAINER "/client/postgresql.crt /var/lib/postgres/.postgresql/postgresql.crt" \ + " && cp " SSL_DIR_IN_CONTAINER "/client/postgresql.key /var/lib/postgres/.postgresql/postgresql.key" \ + " && chmod 0600 /var/lib/postgres/.postgresql/postgresql.key" \ + " && cp " SSL_DIR_IN_CONTAINER "/ca.crt /var/lib/postgres/.postgresql/root.crt" \ + " &&" + + +/* + * ssl_needs_certs returns true when the ssl mode requires CA-signed server + * certificates to be generated on the host: verify-ca and verify-full. + */ +static bool +ssl_needs_certs(const char *ssl) +{ + return strcmp(ssl, "verify-ca") == 0 || strcmp(ssl, "verify-full") == 0; +} + + +/* + * run_openssl runs a single openssl command (argv-style) and returns true on + * success. The last element of argv[] must be NULL. + */ +static bool +run_openssl(const char *const *argv) +{ + char cmd[4096] = "openssl"; + int pos = strlen(cmd); + + for (int i = 1; argv[i]; i++) + { + pos += snprintf(cmd + pos, sizeof(cmd) - pos, " %s", argv[i]); + if (pos >= (int) sizeof(cmd) - 1) + { + log_error("openssl command too long"); + return false; + } + } + + log_debug("Running: %s", cmd); + int rc = system(cmd); + if (rc != 0) + { + log_error("Command failed (exit %d): %s", rc, cmd); + return false; + } + return true; +} + + +/* + * compose_gen_write_ssl_certs generates the CA and per-service server + * certificates in /ssl/ using openssl. Called from + * runner_compose_generate() before compose_gen_write() so the volume paths + * already exist when the compose file is written. + * + * Layout on the host: + * ssl/ca.crt — shared CA certificate + * ssl/ca.key — CA private key (stays on host) + * ssl//server.crt — server certificate for service + * ssl//server.key — server private key for service + * + * The CA cert and the per-service dir are bind-mounted into each container so + * pg_autoctl can find them at the paths recorded in the ini [ssl] section. + */ +bool +compose_gen_write_ssl_certs(const TestCluster *cluster, const char *workDir) +{ + if (!ssl_needs_certs(cluster->ssl)) + return true; + + char sslDir[MAXPGPATH]; + sformat(sslDir, sizeof(sslDir), "%s/ssl", workDir); + + /* Create ssl/ directory */ + if (mkdir(sslDir, 0755) != 0 && errno != EEXIST) + { + log_error("Failed to create %s: %m", sslDir); + return false; + } + + char caKey[MAXPGPATH], caCrt[MAXPGPATH]; + sformat(caKey, sizeof(caKey), "%s/ca.key", sslDir); + sformat(caCrt, sizeof(caCrt), "%s/ca.crt", sslDir); + + /* Generate CA key + self-signed cert with basicConstraints=CA:TRUE */ + bool caRegenerated = false; + if (access(caCrt, F_OK) != 0) + { + const char *genCA[] = { + "openssl", "req", "-new", "-x509", "-nodes", "-text", + "-days", "3650", + "-keyout", caKey, + "-out", caCrt, + "-subj", "/CN=root.pgautofailover.ca", + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign", + NULL + }; + if (!run_openssl(genCA)) + return false; + + chmod(caKey, 0600); /* CA key stays private; not mounted into containers */ + log_info("Generated CA certificate at %s", caCrt); + caRegenerated = true; + } + + /* + * Generate server cert for each service: monitor + all nodes. + * We build a flat list of service names to iterate. + */ + const char *services[PGAF_MAX_NODES + 1]; /* +1 for monitor */ + int svcCount = 0; + + if (cluster->withMonitor) + services[svcCount++] = "monitor"; + + if (cluster->secondMonitorName[0]) + services[svcCount++] = cluster->secondMonitorName; + + for (int fi = 0; fi < cluster->formationCount; fi++) + { + const TestFormation *form = &cluster->formations[fi]; + for (int ni = 0; ni < form->nodeCount; ni++) + services[svcCount++] = form->nodes[ni].name; + } + + char serialFile[MAXPGPATH]; + sformat(serialFile, sizeof(serialFile), "%s/ca.srl", sslDir); + + for (int i = 0; i < svcCount; i++) + { + const char *svc = services[i]; + char svcDir[MAXPGPATH]; + sformat(svcDir, sizeof(svcDir), "%s/%s", sslDir, svc); + + if (mkdir(svcDir, 0755) != 0 && errno != EEXIST) + { + log_error("Failed to create %s: %m", svcDir); + return false; + } + + char srvKey[MAXPGPATH], srvCrt[MAXPGPATH], srvCsr[MAXPGPATH]; + sformat(srvKey, sizeof(srvKey), "%s/server.key", svcDir); + sformat(srvCrt, sizeof(srvCrt), "%s/server.crt", svcDir); + sformat(srvCsr, sizeof(srvCsr), "%s/server.csr", svcDir); + + if (access(srvCrt, F_OK) == 0 && !caRegenerated) + continue; /* already generated and CA hasn't changed */ + + char cn[128]; + sformat(cn, sizeof(cn), "/CN=%s.pgautofailover.ca", svc); + + const char *genReq[] = { + "openssl", "req", "-new", "-nodes", "-text", + "-out", srvCsr, "-keyout", srvKey, + "-subj", cn, + NULL + }; + if (!run_openssl(genReq)) + return false; + + /* 0644: container user must read this for the cp-and-chmod step */ + chmod(srvKey, 0644); + + const char *signSrv[] = { + "openssl", "x509", "-req", "-in", srvCsr, + "-text", "-days", "365", + "-CA", caCrt, "-CAkey", caKey, "-CAcreateserial", + "-out", srvCrt, + NULL + }; + if (!run_openssl(signSrv)) + return false; + + log_info("Generated server certificate for %s at %s", svc, srvCrt); + } + + /* + * Generate client certificate with CN=autoctl_node, used by pg_autoctl + * processes to authenticate to PostgreSQL when ssl.auth = cert. + * Placed in ssl/client/ and mounted read-only into each container; + * the container start command copies it to ~/.postgresql/ with correct + * ownership so libpq can use it for all outbound connections. + */ + char clientDir[MAXPGPATH], clientCrt[MAXPGPATH], + clientKey[MAXPGPATH], clientCsr[MAXPGPATH]; + sformat(clientDir, sizeof(clientDir), "%s/client", sslDir); + sformat(clientCrt, sizeof(clientCrt), "%s/postgresql.crt", clientDir); + sformat(clientKey, sizeof(clientKey), "%s/postgresql.key", clientDir); + sformat(clientCsr, sizeof(clientCsr), "%s/postgresql.csr", clientDir); + + if (mkdir(clientDir, 0755) != 0 && errno != EEXIST) + { + log_error("Failed to create %s: %m", clientDir); + return false; + } + + if (access(clientCrt, F_OK) != 0 || caRegenerated) + { + const char *genReq[] = { + "openssl", "req", "-new", "-nodes", "-text", + "-out", clientCsr, "-keyout", clientKey, + "-subj", "/CN=autoctl_node", + NULL + }; + if (!run_openssl(genReq)) + return false; + + /* 0644: container user must read this for the cp-and-chmod step */ + chmod(clientKey, 0644); + + const char *signClient[] = { + "openssl", "x509", "-req", "-in", clientCsr, + "-text", "-days", "365", + "-CA", caCrt, "-CAkey", caKey, "-CAcreateserial", + "-out", clientCrt, + NULL + }; + if (!run_openssl(signClient)) + return false; + + log_info("Generated client certificate at %s", clientCrt); + } + + return true; +} + + +/* + * image_stanza writes either `image:` (when PGAF_IMAGE is set or the cluster + * spec provides an image) or a `build:` stanza. + */ +static void +write_image_stanza_target(FILE *f, const TestCluster *cluster, + const char *contextDir, const char *target) +{ + /* + * The cluster image (from `image "..."` in the spec) is for data services + * only. The pgaftest service always builds from --target pgaftest so that + * the test runner binary is present regardless of which cluster image is + * used. + */ + bool isTestRunner = (strcmp(target, "pgaftest") == 0); + + /* cluster-level image overrides PGAF_IMAGE env var */ + const char *img = cluster->image[0] ? cluster->image : getenv("PGAF_IMAGE"); + + if (img && *img && !isTestRunner) + fprintf(f, " image: \"%s\"\n", img); + else + fprintf(f, + " build:\n" + " context: \"%s\"\n" + " target: %s\n", + contextDir, target); +} + +static void +write_image_stanza(FILE *f, const TestCluster *cluster, const char *contextDir) +{ + write_image_stanza_target(f, cluster, contextDir, "run"); +} + + +/* ----------------------------------------------------------------------- + * compose_gen_write — the docker-compose.yml + * ----------------------------------------------------------------------- */ + +/* + * compose_gen_write generates the docker-compose.yml. + * + * Every service uses: + * command: pg_autoctl node run /etc/pgaf/node.ini + * + * The per-node ini file is bind-mounted from the workdir. + */ +/* + * Pick a random free TCP port in the range [32768, 60999]. + * Tries up to 64 candidates; exits with a log_fatal if none found. + * + * Ports already handed out in this process are remembered in a static array + * so that consecutive calls never return the same port even though the OS + * releases the probe socket between calls. + */ +static int +pick_free_port(void) +{ + static int reserved[64]; + static int reserved_count = 0; + + srand((unsigned) time(NULL) ^ (unsigned) getpid()); + for (int attempt = 0; attempt < 64; attempt++) + { + int port = 32768 + rand() % (60999 - 32768 + 1); + + /* skip ports already given out this process */ + bool already_used = false; + for (int i = 0; i < reserved_count; i++) + { + if (reserved[i] == port) + { + already_used = true; + break; + } + } + if (already_used) + continue; + + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) continue; + + int opt = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_port = htons((uint16_t) port), + .sin_addr = { .s_addr = htonl(INADDR_LOOPBACK) }, + }; + int rc = bind(fd, (struct sockaddr *) &addr, sizeof(addr)); + close(fd); + + if (rc == 0) + { + if (reserved_count < 64) + reserved[reserved_count++] = port; + return port; + } + } + log_fatal("Could not find a free TCP port for monitor exposure"); + exit(1); +} + +bool +compose_gen_write(TestCluster *cluster, + const char *path, + const char *projectName, + const char *contextDir, + const char *specFile) +{ + FILE *f = fopen(path, "w"); + if (!f) + { + log_error("Failed to open \"%s\" for writing: %m", path); + return false; + } + + fprintf(f, "# Generated by pgaftest — do not edit manually\n"); + fprintf(f, "# Project: %s\n\n", projectName); + fprintf(f, "services:\n"); + + /* ---- monitor (optional) ---- */ + if (cluster->withMonitor) + { + /* allocate a random free host port for the monitor's postgres */ + if (cluster->monitorHostPort == 0) + cluster->monitorHostPort = pick_free_port(); + + fprintf(f, " monitor:\n"); + if (cluster->monitorDebianCluster[0]) + write_image_stanza_target(f, cluster, contextDir, "debian"); + else if (cluster->monitorImageTarget[0]) + write_image_stanza_target(f, cluster, contextDir, + cluster->monitorImageTarget); + else + write_image_stanza(f, cluster, contextDir); + + char monitor_pgdata[MAXPGPATH]; + if (cluster->monitorDebianCluster[0]) + snprintf(monitor_pgdata, sizeof(monitor_pgdata), + DEBIAN_PGDATA_PREFIX "/%s/%s", + debian_pg_version(), cluster->monitorDebianCluster); + else + strlcpy(monitor_pgdata, NODE_PGDATA, sizeof(monitor_pgdata)); + + fprintf(f, + " hostname: monitor\n" + " volumes:\n" + " - monitor_data:/var/lib/postgres:rw\n" + " - ./monitor.ini:" NODE_INI_PATH ":ro\n"); + if (ssl_needs_certs(cluster->ssl)) + fprintf(f, + " - ./ssl/ca.crt:" SSL_DIR_IN_CONTAINER "/ca.crt:ro\n" + " - ./ssl/monitor:" SSL_DIR_IN_CONTAINER "/server:ro\n" + " - ./ssl/client:" SSL_DIR_IN_CONTAINER "/client:ro\n"); + if (cluster->bindSource) + fprintf(f, + " - %s:/usr/src/pg_auto_failover:rw\n", contextDir); + fprintf(f, + " environment:\n" + " PGDATA: %s\n", + monitor_pgdata); + if (cluster->extensionVersion[0]) + fprintf(f, + " PG_AUTOCTL_EXTENSION_VERSION: \"%s\"\n", + cluster->extensionVersion); + fprintf(f, + " ports:\n" + " - \"%d:5432\"\n", + cluster->monitorHostPort); + + fprintf(f, + " command: [\"/bin/sh\", \"-c\"," + " \"%s rm -f /tmp/pg_autoctl%s/pg_autoctl.pid" + " && exec pg_autoctl node run " NODE_INI_PATH "\"]\n" + " stop_grace_period: 60s\n\n", + ssl_needs_certs(cluster->ssl) ? SSL_COPY_CERTS_CMD : "", + monitor_pgdata); + } + + /* ---- second monitor (initially stopped, for replace-monitor tests) ---- */ + if (cluster->secondMonitorName[0]) + { + if (cluster->secondMonitorHostPort == 0) + cluster->secondMonitorHostPort = pick_free_port(); + + const char *svc = cluster->secondMonitorName; + fprintf(f, " %s:\n", svc); + write_image_stanza(f, cluster, contextDir); + fprintf(f, + " hostname: %s\n" + " volumes:\n" + " - %s_data:/var/lib/postgres:rw\n" + " - ./%s.ini:" NODE_INI_PATH ":ro\n", + svc, svc, svc); + if (ssl_needs_certs(cluster->ssl)) + fprintf(f, + " - ./ssl/ca.crt:" SSL_DIR_IN_CONTAINER "/ca.crt:ro\n" + " - ./ssl/%s:" SSL_DIR_IN_CONTAINER "/server:ro\n" + " - ./ssl/client:" SSL_DIR_IN_CONTAINER "/client:ro\n", + svc); + if (cluster->bindSource) + fprintf(f, + " - %s:/usr/src/pg_auto_failover:rw\n", contextDir); + fprintf(f, + " environment:\n" + " PGDATA: " NODE_PGDATA "\n"); + fprintf(f, + " ports:\n" + " - \"%d:5432\"\n", + cluster->secondMonitorHostPort); + fprintf(f, + " command: [\"/bin/sh\", \"-c\"," + " \"%s rm -f /tmp/pg_autoctl" NODE_PGDATA "/pg_autoctl.pid" + " && exec pg_autoctl node run " NODE_INI_PATH "\"]\n" + " stop_grace_period: 60s\n\n", + ssl_needs_certs(cluster->ssl) ? SSL_COPY_CERTS_CMD : ""); + } + + /* ---- data nodes — iterate all formations ---- */ + /* + * Non-default formations are listed in [formation ] sections of the + * monitor.ini. pg_autoctl node run reads them and creates each one after + * the monitor is initialized, before starting the supervisor. + * Data nodes retry registration until their formation exists. + */ + const TestNode *firstNode = NULL; + + for (int fi = 0; fi < cluster->formationCount; fi++) + { + const TestFormation *form = &cluster->formations[fi]; + + for (int ni = 0; ni < form->nodeCount; ni++) + { + const TestNode *n = &form->nodes[ni]; + fprintf(f, " %s:\n", n->name); + if (n->debianCluster[0]) + write_image_stanza_target(f, cluster, contextDir, "debian"); + else + write_image_stanza(f, cluster, contextDir); + + /* debian PGDATA: /var/lib/postgresql// */ + char node_pgdata[MAXPGPATH]; + if (n->debianCluster[0]) + snprintf(node_pgdata, sizeof(node_pgdata), + DEBIAN_PGDATA_PREFIX "/%s/%s", + debian_pg_version(), n->debianCluster); + else + strlcpy(node_pgdata, NODE_PGDATA, sizeof(node_pgdata)); + + fprintf(f, + " hostname: %s\n" + " volumes:\n" + " - %s_data:/var/lib/postgres:rw\n" + " - ./%s.ini:" NODE_INI_PATH ":%s\n", + n->name, + n->name, + n->name, + n->launchDeferred ? "rw" : "ro"); + if (ssl_needs_certs(cluster->ssl)) + fprintf(f, + " - ./ssl/ca.crt:" SSL_DIR_IN_CONTAINER "/ca.crt:ro\n" + " - ./ssl/%s:" SSL_DIR_IN_CONTAINER "/server:ro\n" + " - ./ssl/client:" SSL_DIR_IN_CONTAINER "/client:ro\n", + n->name); + for (int vi = 0; vi < n->volumeCount; vi++) + fprintf(f, " - %s_%s:%s:rw\n", + n->volumes[vi].name, n->name, n->volumes[vi].path); + fprintf(f, + " environment:\n" + " PGDATA: %s\n" + " PGUSER: demo\n" + " PGDATABASE: demo\n" + " PG_AUTOCTL_TEST_DELAY: \"1\"\n" + " expose:\n" + " - 5432\n", + node_pgdata); + /* + * No depends_on: keeper retry loops handle monitor not yet ready, + * and the formation wait in keeper_register_and_init handles the + * case where the formation hasn't been created yet. + */ + + /* per-node ssl override may differ from cluster default */ + const char *node_ssl = n->ssl[0] ? n->ssl : cluster->ssl; + fprintf(f, + " command: [\"/bin/sh\", \"-c\"," + " \"%s rm -f /tmp/pg_autoctl%s/pg_autoctl.pid" + " && exec pg_autoctl node run " NODE_INI_PATH "\"]\n" + " stop_grace_period: 60s\n", + ssl_needs_certs(node_ssl) ? SSL_COPY_CERTS_CMD : "", + node_pgdata); + + /* + * With a monitor: the first data node gets a healthcheck so that + * subsequent nodes use service_healthy depends_on. This ensures + * node1 has registered with the monitor (and become the initial + * primary) before any other node starts, making the initial + * election deterministic. + * + * Without a monitor (no-monitor nodes): the FSM is driven + * manually via pg_autoctl manual fsm assign, so postgres is not + * running when the container starts. No healthcheck; subsequent + * nodes use service_started so they launch as soon as node1 has + * started (they don't need postgres to be ready yet). + */ + if (!firstNode && cluster->withMonitor) + fprintf(f, + " healthcheck:\n" + " test: [\"CMD\", \"pg_autoctl\", \"status\"," + " \"--pgdata\", \"%s\"]\n" + " interval: 2s\n" + " timeout: 5s\n" + " retries: 30\n" + " start_period: 15s\n", + node_pgdata); + + if (firstNode) + fprintf(f, + " depends_on:\n" + " %s:\n" + " condition: %s\n", + firstNode->name, + cluster->withMonitor ? "service_healthy" : "service_started"); + fprintf(f, "\n"); + + if (!firstNode) firstNode = n; + } + } + + /* + * ---- pgaftest service ---- + * + * When specFile is provided, include a pgaftest service that runs the + * spec from inside the compose network. Benefits: + * - direct TCP to monitor/nodes — LISTEN/NOTIFY works, no pg_hba issue + * - pg_autoctl commands run locally (just need --monitor ) + * - compose stop/start and network disconnect/connect via docker socket + * + * CI usage: docker compose up --exit-code-from pgaftest + */ + if (specFile) + { + /* Determine which service pgaftest must wait for before starting. */ + /* It needs the cluster fully ready: monitor healthy + all nodes started. */ + fprintf(f, " pgaftest:\n"); + write_image_stanza_target(f, cluster, contextDir, "pgaftest"); + /* Build monitor pguri for PG_AUTOCTL_MONITOR */ + char monitorPguri[512]; + if (cluster->monitorPassword[0]) + sformat(monitorPguri, sizeof(monitorPguri), + "postgres://autoctl_node:%s@monitor/pg_auto_failover", + cluster->monitorPassword); + else + sformat(monitorPguri, sizeof(monitorPguri), + "postgres://autoctl_node@monitor/pg_auto_failover"); + + /* + * pgaftest has its own ping loop that waits for the monitor before + * starting, so no depends_on is needed. + */ + /* Derive work dir (directory containing the compose file). */ + char workDir[MAXPGPATH]; + strlcpy(workDir, path, sizeof(workDir)); + char *slash = strrchr(workDir, '/'); + if (slash) *slash = '\0'; else strlcpy(workDir, ".", sizeof(workDir)); + + fprintf(f, + " user: root\n" + " volumes:\n" + " - %s:/spec.pgaf:ro\n" + " - /var/run/docker.sock:/var/run/docker.sock\n" + " - %s:%s:ro\n", + specFile, workDir, workDir); + if (ssl_needs_certs(cluster->ssl)) + fprintf(f, + " - %s/ssl/ca.crt:/root/.postgresql/root.crt:ro\n" + " - %s/ssl/client/postgresql.crt:/root/.postgresql/postgresql.crt:ro\n" + " - %s/ssl/client/postgresql.key:/root/.postgresql/postgresql.key:ro\n", + workDir, workDir, workDir); + fprintf(f, + " environment:\n" + " PGAFTEST_COMPOSE_SERVICE: \"1\"\n" + " PGAFTEST_HOST_WORK_DIR: \"%s\"\n" + " COMPOSE_PROJECT_NAME: \"%s\"\n", + workDir, projectName); + if (cluster->withMonitor) + fprintf(f, " PG_AUTOCTL_MONITOR: \"%s\"\n", monitorPguri); + fprintf(f, " command: [\"pgaftest\", \"run\", \"/spec.pgaf\"]\n\n"); + } + + /* ---- volumes ---- */ + fprintf(f, "volumes:\n"); + if (cluster->withMonitor) + fprintf(f, " monitor_data:\n"); + if (cluster->secondMonitorName[0]) + fprintf(f, " %s_data:\n", cluster->secondMonitorName); + for (int fi = 0; fi < cluster->formationCount; fi++) + { + const TestFormation *form = &cluster->formations[fi]; + for (int ni = 0; ni < form->nodeCount; ni++) + { + const TestNode *n = &form->nodes[ni]; + fprintf(f, " %s_data:\n", n->name); + for (int vi = 0; vi < n->volumeCount; vi++) + fprintf(f, " %s_%s:\n", n->volumes[vi].name, n->name); + } + } + + fclose(f); + log_info("Wrote docker-compose.yml to \"%s\"", path); + return true; +} + + +/* ----------------------------------------------------------------------- + * compose_gen_write_monitor_ini + * ----------------------------------------------------------------------- */ + +bool +compose_gen_write_monitor_ini(const TestCluster *cluster, const char *dir) +{ + if (!cluster->withMonitor) + return true; + + char path[1024]; + sformat(path, sizeof(path), "%s/monitor.ini", dir); + + FILE *f = fopen(path, "w"); + if (!f) + { + log_error("Failed to open \"%s\" for writing: %m", path); + return false; + } + + char mon_pgdata[MAXPGPATH]; + if (cluster->monitorDebianCluster[0]) + snprintf(mon_pgdata, sizeof(mon_pgdata), + DEBIAN_PGDATA_PREFIX "/%s/%s", + debian_pg_version(), cluster->monitorDebianCluster); + else + strlcpy(mon_pgdata, NODE_PGDATA, sizeof(mon_pgdata)); + + fprintf(f, + "# Generated by pgaftest — do not edit manually\n" + "# Monitor node configuration\n" + "\n" + "[node]\n" + "kind = monitor\n" + "hostname = monitor\n" + "port = 5432\n" + "\n" + "[postgresql]\n" + "pgdata = %s\n" + "\n" + "[options]\n" + "ssl = %s\n" + "auth = %s\n" + "pg_hba_lan = false\n", + mon_pgdata, + cluster->ssl, + cluster->auth); + + if (ssl_needs_certs(cluster->ssl)) + fprintf(f, + "\n" + "[ssl]\n" + "ca_file = " SSL_DIR_IN_CONTAINER "/ca.crt\n" + /* + * The server cert/key are copied from the read-only bind-mount to + * /var/lib/postgres/ (the volume root, writable by the container + * user) in the start command so the key is owned by the container + * user and PostgreSQL can open it. + */ + "cert_file = /var/lib/postgres/server.crt\n" + "key_file = /var/lib/postgres/server.key\n"); + + if (cluster->monitorPassword[0]) + fprintf(f, + "\n" + "[pg_auto_failover]\n" + "autoctl_node_password = %s\n", + cluster->monitorPassword); + + /* emit [formation ] for every non-default formation */ + for (int fi = 0; fi < cluster->formationCount; fi++) + { + const TestFormation *form = &cluster->formations[fi]; + if (strcmp(form->name, "default") == 0) + continue; + /* kind defaults to "ha" — the monitor uses that default when absent */ + fprintf(f, "\n[formation %s]\n", form->name); + } + + fclose(f); + /* world-writable so the container's docker user can update it at runtime */ + (void) chmod(path, 0666); + log_info("Wrote monitor.ini to \"%s\"", path); + return true; +} + + +/* ----------------------------------------------------------------------- + * compose_gen_write_second_monitor_ini + * ----------------------------------------------------------------------- */ + +bool +compose_gen_write_second_monitor_ini(const TestCluster *cluster, const char *dir) +{ + if (!cluster->secondMonitorName[0]) + return true; + + const char *name = cluster->secondMonitorName; + char path[1024]; + sformat(path, sizeof(path), "%s/%s.ini", dir, name); + + FILE *f = fopen(path, "w"); + if (!f) + { + log_error("Failed to open \"%s\" for writing: %m", path); + return false; + } + + fprintf(f, + "# Generated by pgaftest — do not edit manually\n" + "# Second monitor (replacement): %s\n" + "\n" + "[node]\n" + "kind = monitor\n" + "hostname = %s\n" + "port = 5432\n" + "\n" + "[postgresql]\n" + "pgdata = " NODE_PGDATA "\n" + "\n" + "[options]\n" + "ssl = %s\n" + "auth = %s\n" + "pg_hba_lan = false\n", + name, name, + cluster->ssl, + cluster->auth); + + if (ssl_needs_certs(cluster->ssl)) + fprintf(f, + "\n" + "[ssl]\n" + "ca_file = " SSL_DIR_IN_CONTAINER "/ca.crt\n" + /* + * The server cert/key are copied from the read-only bind-mount to + * /var/lib/postgres/ (the volume root, writable by the container + * user) in the start command so the key is owned by the container + * user and PostgreSQL can open it. + */ + "cert_file = /var/lib/postgres/server.crt\n" + "key_file = /var/lib/postgres/server.key\n"); + + fclose(f); + (void) chmod(path, 0666); + log_info("Wrote %s.ini to \"%s\"", name, path); + return true; +} + + +/* ----------------------------------------------------------------------- + * compose_gen_write_node_ini + * ----------------------------------------------------------------------- */ + +bool +compose_gen_write_node_ini(const TestCluster *cluster, + const TestFormation *formation, + const TestNode *node, + int nodeId, + const char *dir) +{ + char path[1024]; + sformat(path, sizeof(path), "%s/%s.ini", dir, node->name); + + FILE *f = fopen(path, "w"); + if (!f) + { + log_error("Failed to open \"%s\" for writing: %m", path); + return false; + } + + const char *kindStr; + switch (node->kind) + { + case NODE_KIND_CITUS_COORDINATOR: kindStr = "coordinator"; break; + case NODE_KIND_CITUS_WORKER: kindStr = "worker"; break; + default: kindStr = "postgres"; break; + } + + /* Effective ssl/auth: node-level overrides cluster-level default */ + const char *eff_ssl = node->ssl[0] ? node->ssl : cluster->ssl; + const char *eff_auth = node->auth[0] ? node->auth : cluster->auth; + + int pg_port = node->pgPort > 0 ? node->pgPort : 5432; + + fprintf(f, + "# Generated by pgaftest — do not edit manually\n" + "# Node: %s (formation: %s)\n" + "\n" + "[node]\n" + "kind = %s\n" + "name = %s\n" + "hostname = %s\n" + "port = %d\n", + node->name, formation->name, + kindStr, + node->name, + node->name, + pg_port); + + if (node->debianCluster[0]) + fprintf(f, "debian_cluster = %s\n", node->debianCluster); + + if (node->debianCluster[0]) + fprintf(f, + "\n" + "[postgresql]\n" + "pgdata = " DEBIAN_PGDATA_PREFIX "/%s/%s\n" + "\n", + debian_pg_version(), node->debianCluster); + else + fprintf(f, + "\n" + "[postgresql]\n" + "pgdata = " NODE_PGDATA "\n" + "\n"); + + if (node->noMonitor) + { + fprintf(f, "[monitor]\nno_monitor = true\n"); + if (nodeId > 0) + fprintf(f, "node_id = %d\n", nodeId); + fprintf(f, "\n"); + } + else if (cluster->monitorPassword[0]) + fprintf(f, + "[monitor]\n" + "pguri = postgresql://autoctl_node:%s@monitor/pg_auto_failover\n" + "\n", + cluster->monitorPassword); + else + fprintf(f, + "[monitor]\n" + "pguri = " MONITOR_PGURI "\n" + "\n"); + + fprintf(f, + "[formation]\n" + "name = %s\n" + "group = %d\n" + "\n" + "[settings]\n" + "candidate_priority = %d\n" + "replication_quorum = %s\n", + formation->name, + node->group, + node->candidatePriority, + node->replicationQuorum ? "true" : "false"); + + /* Citus role/cluster settings live in their own [citus] section */ + if (node->citusSecondary || node->citusClusterName[0]) + { + fprintf(f, "\n[citus]\n"); + if (node->citusSecondary) + fprintf(f, "role = secondary\n"); + if (node->citusClusterName[0]) + fprintf(f, "cluster_name = %s\n", node->citusClusterName); + } + + fprintf(f, + "\n" + "[options]\n" + "ssl = %s\n" + "auth = %s\n" + "pg_hba_lan = %s\n", + eff_ssl, + eff_auth, + node->noMonitor ? "false" : "true"); + + if (ssl_needs_certs(eff_ssl)) + fprintf(f, + "\n" + "[ssl]\n" + "ca_file = " SSL_DIR_IN_CONTAINER "/ca.crt\n" + /* + * The server cert/key are copied from the read-only bind-mount to + * /var/lib/postgres/ (the volume root, writable by the container + * user) in the start command so the key is owned by the container + * user and PostgreSQL can open it. + */ + "cert_file = /var/lib/postgres/server.crt\n" + "key_file = /var/lib/postgres/server.key\n"); + + if (node->listen) + fprintf(f, "listen = 0.0.0.0\n"); + if (node->launchDeferred) + fprintf(f, "\n[launch]\nmode = deferred\n"); + + /* + * Write pg_auto_failover role passwords when configured. The + * replication.password key already exists in the [replication] section; + * monitor_password and the cluster-level autoctl_node_password go into a + * dedicated [pg_auto_failover] section so that pg_autoctl picks them up + * via the new OPTION_AUTOCTL_NODE_PASSWORD / OPTION_AUTOCTL_MONITOR_PASSWORD + * ini options. + */ + { + const char *eff_monitor_pw = + node->monitorPassword[0] ? node->monitorPassword : ""; + const char *eff_replication_pw = + node->replicationPassword[0] ? node->replicationPassword : ""; + + /* + * cluster->monitorPassword is the autoctl_node password on the monitor. + * It is already embedded in the monitor URI above; it does not belong + * in the node's own [pg_auto_failover] section. + * + * node->monitorPassword is the pgautofailover_monitor health-check role + * password on this data node — set by the monitor when it connects to + * run health checks. + */ + if (eff_replication_pw[0]) + fprintf(f, + "\n" + "[replication]\n" + "password = %s\n", + eff_replication_pw); + + if (eff_monitor_pw[0]) + fprintf(f, + "\n" + "[pg_auto_failover]\n" + "monitor_password = %s\n", + eff_monitor_pw); + } + + fclose(f); + (void) chmod(path, 0666); + log_info("Wrote %s.ini to \"%s\"", node->name, path); + return true; +} + + +/* ----------------------------------------------------------------------- + * Helpers used by test_runner.c + * ----------------------------------------------------------------------- */ + +void +compose_network_name(const char *projectName, char *buf, int buflen) +{ + snprintf(buf, buflen, "%s_default", projectName); +} + +void +compose_container_name(const char *projectName, const char *service, + char *buf, int buflen) +{ + snprintf(buf, buflen, "%s-%s-1", projectName, service); +} diff --git a/src/bin/pgaftest/compose_gen.h b/src/bin/pgaftest/compose_gen.h new file mode 100644 index 000000000..e73774fc4 --- /dev/null +++ b/src/bin/pgaftest/compose_gen.h @@ -0,0 +1,69 @@ +/* + * src/bin/pgaftest/compose_gen.h + * Generate a docker-compose.yml from a TestCluster spec. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef COMPOSE_GEN_H +#define COMPOSE_GEN_H + +#include +#include "test_spec.h" + +/* + * Generate CA and per-service server certificates in /ssl/ using + * openssl. Only called when cluster->ssl is "verify-ca" or "verify-full". + */ +bool compose_gen_write_ssl_certs(const TestCluster *cluster, + const char *workDir); + +/* + * Write a docker-compose.yml to `path` for the given cluster spec. + * Returns true on success. + */ +/* cluster is non-const: monitorHostPort is filled in if zero. + * specFile is the absolute path to the .pgaf spec file; it is bind-mounted + * into the pgaftest service at /spec.pgaf. Pass NULL to omit the service. */ +bool compose_gen_write(TestCluster *cluster, + const char *path, + const char *projectName, + const char *contextDir, + const char *specFile); + +/* + * Write a pg_autoctl_node.ini for the second (replacement) monitor into `dir`. + * Only called when cluster->secondMonitorName is set. + */ +bool compose_gen_write_second_monitor_ini(const TestCluster *cluster, + const char *dir); + +/* + * Write a pg_autoctl_node.ini file for the monitor node into `dir`. + * The file is written as /monitor.ini and will be bind-mounted + * into the container at PG_AUTOCTL_NODESPEC_PATH. + */ +bool compose_gen_write_monitor_ini(const TestCluster *cluster, + const char *dir); + +/* + * Write a pg_autoctl_node.ini file for a data node into `dir`. + * The file is written as /name>.ini. + * `formation` supplies the formation name and is written into [formation]. + */ +bool compose_gen_write_node_ini(const TestCluster *cluster, + const TestFormation *formation, + const TestNode *node, + int nodeId, + const char *dir); + +/* Return the docker network name for a project */ +void compose_network_name(const char *projectName, char *buf, int buflen); + +/* Return the container name for a service in a project */ +void compose_container_name(const char *projectName, + const char *service, + char *buf, int buflen); + +#endif /* COMPOSE_GEN_H */ diff --git a/src/bin/pgaftest/main.c b/src/bin/pgaftest/main.c new file mode 100644 index 000000000..a1a90a1ea --- /dev/null +++ b/src/bin/pgaftest/main.c @@ -0,0 +1,54 @@ +/* + * src/bin/pgaftest/main.c + * pgaftest — pg_auto_failover test runner. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include + +#include "commandline.h" +#include "defaults.h" +#include "log.h" +#include "lock_utils.h" +#include "monitor_config.h" + +extern CommandLine pgaftest_root; + +/* + * Globals required by shared pg_autoctl source files. + * In pg_autoctl these live in cli_root.c / cli_create_node.c / main.c. + * pgaftest owns stub definitions here so the linker is satisfied. + */ +int pgconnect_timeout = 2; +char *ps_buffer; +size_t ps_buffer_size; +size_t last_status_len; +Semaphore log_semaphore = { 0 }; + +char pg_autoctl_argv0[MAXPGPATH] = "pgaftest"; +char pg_autoctl_program[MAXPGPATH] = "pgaftest"; + +MonitorConfig monitorOptions = { 0 }; + +/* Stub command roots referenced by cli_common.c's keeper_cli_help */ +CommandLine root = make_command("pgaftest", "", "", "", NULL, NULL); +CommandLine root_with_debug = make_command("pgaftest", "", "", "", NULL, NULL); + +int +main(int argc, char **argv) +{ + /* default log level: INFO to stderr */ + log_set_level(LOG_INFO); + + if (argc < 2) + { + commandline_print_usage(&pgaftest_root, stderr); + return 1; + } + + return commandline_run(&pgaftest_root, argc, argv); +} diff --git a/src/bin/pgaftest/pgaftest b/src/bin/pgaftest/pgaftest new file mode 100755 index 000000000..f48b2ca01 Binary files /dev/null and b/src/bin/pgaftest/pgaftest differ diff --git a/src/bin/pgaftest/test_runner.c b/src/bin/pgaftest/test_runner.c new file mode 100644 index 000000000..60aaf3283 --- /dev/null +++ b/src/bin/pgaftest/test_runner.c @@ -0,0 +1,3323 @@ +/* + * src/bin/pgaftest/test_runner.c + * Test execution engine for .pgaf specs. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "log.h" +#include "compose_gen.h" +#include "pgsql.h" +#include "parsing.h" +#include "nodestate_utils.h" +#include "state.h" +#include "test_runner.h" +#include "test_spec.h" + +/* Docker binary name */ +#define DOCKER "docker" + +/* ----------------------------------------------------------------------- + * Forward declarations + * ----------------------------------------------------------------------- */ + +static bool wait_for_state(TestRunner *r, const char *nodeName, + const char *targetState, int timeoutSecs, + bool checkAssigned); +static bool runner_wait_assigned_goal(TestRunner *r, const char *nodeName, + const char *targetState, int timeoutSecs); +static void log_output(const char *prefix, const char *out); + +/* ----------------------------------------------------------------------- + * Internal helpers + * ----------------------------------------------------------------------- */ + +/* Run a shell command, return its exit code */ +static int __attribute__((format(printf, 1, 2))) +run_cmd(const char *fmt, ...) +{ + char cmd[4096]; + va_list ap; + va_start(ap, fmt); + vsnprintf(cmd, sizeof(cmd), fmt, ap); + va_end(ap); + + log_debug("$ %s", cmd); + return system(cmd); +} + +/* Run a shell command, capture stdout into buf */ +static int __attribute__((format(printf, 3, 4))) +run_cmd_capture(char *buf, int buflen, const char *fmt, ...) +{ + char cmd[4096]; + va_list ap; + va_start(ap, fmt); + vsnprintf(cmd, sizeof(cmd), fmt, ap); + va_end(ap); + + log_debug("$ %s", cmd); + + FILE *p = popen(cmd, "r"); + if (!p) return -1; + + int pos = 0; + int c; + while ((c = fgetc(p)) != EOF && pos < buflen - 1) + buf[pos++] = (char)c; + buf[pos] = '\0'; + + /* trim trailing whitespace */ + while (pos > 0 && (buf[pos-1] == '\n' || buf[pos-1] == '\r' || + buf[pos-1] == ' ')) + buf[--pos] = '\0'; + + return pclose(p); +} + +/* ----------------------------------------------------------------------- + * Runner initialisation + * ----------------------------------------------------------------------- */ + +static void +runner_init(TestRunner *r, TestSpec *spec, const char *workDir) +{ + memset(r, 0, sizeof(*r)); + r->spec = spec; + + /* + * Project name: inside the compose network COMPOSE_PROJECT_NAME is + * authoritative (the container is invoked with /spec.pgaf which would + * otherwise yield "spec"). On the host, derive from the spec filename. + */ + const char *envProject = getenv("COMPOSE_PROJECT_NAME"); + if (envProject && *envProject) + { + strlcpy(r->projectName, envProject, sizeof(r->projectName)); + } + else + { + const char *base = strrchr(spec->filename, '/'); + base = base ? base + 1 : spec->filename; + strncpy(r->projectName, base, sizeof(r->projectName)-1); + char *dot = strrchr(r->projectName, '.'); + if (dot) *dot = '\0'; + } + + snprintf(r->workDir, sizeof(r->workDir), "%s", workDir); + snprintf(r->composeFile, sizeof(r->composeFile), + "%s/docker-compose.yml", workDir); + + /* + * Inside the compose network the compose file lives on the HOST filesystem + * and is not accessible from the container. Docker Compose v2 can exec + * into a running project by project-name alone; omit -f in that case. + */ + if (getenv("PGAFTEST_COMPOSE_SERVICE")) + { + /* + * Inside the pgaftest container, the compose file lives on the HOST + * at PGAFTEST_HOST_WORK_DIR (bind-mounted to the same path in the + * container so Docker daemon can find it). Use -f to point Docker + * Compose at it so that `up -d` can create new containers. + */ + const char *hostWorkDir = getenv("PGAFTEST_HOST_WORK_DIR"); + if (hostWorkDir && hostWorkDir[0]) + snprintf(r->composeBase, sizeof(r->composeBase), + "docker compose -p %s -f %s/docker-compose.yml", + r->projectName, hostWorkDir); + else + snprintf(r->composeBase, sizeof(r->composeBase), + "docker compose -p %s", r->projectName); + } + else + snprintf(r->composeBase, sizeof(r->composeBase), + "docker compose -p %s -f %s", r->projectName, r->composeFile); + + /* build context = directory from which pgaftest was invoked */ + if (getcwd(r->contextDir, sizeof(r->contextDir)) == NULL) + { + log_error("getcwd failed: %m"); + r->contextDir[0] = '\0'; + } + + /* absolute path to the spec file */ + if (spec->filename[0] == '/') + strlcpy(r->specFile, spec->filename, sizeof(r->specFile)); + else + { + snprintf(r->specFile, sizeof(r->specFile), "%s/%s", + r->contextDir, spec->filename); + } + + /* Start with the primary monitor as the active target. */ + strlcpy(r->activeMonitorService, "monitor", + sizeof(r->activeMonitorService)); +} + +/* ----------------------------------------------------------------------- + * TAP output + * ----------------------------------------------------------------------- */ + +/* Append a line to the TAP buffer (does not print yet). */ +static void __attribute__((format(printf, 2, 3))) +tap_buf(TestRunner *r, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + int n = vsnprintf(r->tapBuffer + r->tapBufferLen, + sizeof(r->tapBuffer) - r->tapBufferLen, + fmt, ap); + va_end(ap); + if (n > 0) + r->tapBufferLen += n; +} + +/* Print all buffered TAP output (plan + results) to stdout and flush. */ +static void +tap_plan(TestRunner *r) +{ + printf("1..%d\n", r->tapTotal); + if (r->tapBufferLen > 0) + fwrite(r->tapBuffer, 1, r->tapBufferLen, stdout); + fflush(stdout); +} + +static void +tap_ok(TestRunner *r, const char *name) +{ + r->tapPass++; + tap_buf(r, "ok %d - %s\n", r->tapPass + r->tapFail, name); +} + +static void +tap_not_ok(TestRunner *r, const char *name, const char *reason) +{ + r->tapFail++; + tap_buf(r, "not ok %d - %s\n", r->tapPass + r->tapFail, name); + if (reason) + tap_buf(r, "# %s\n", reason); +} + +static void __attribute__((format(printf, 1, 2))) +tap_diag(const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + printf("# %s\n", buf); + fflush(stdout); +} + +/* ----------------------------------------------------------------------- + * Compose lifecycle + * ----------------------------------------------------------------------- */ + +static bool +runner_compose_generate(TestRunner *r) +{ + /* ensure workdir exists (create intermediate directories as needed) */ + { + char tmp[sizeof(r->workDir)]; + strncpy(tmp, r->workDir, sizeof(tmp) - 1); + for (char *p = tmp + 1; *p; p++) + { + if (*p == '/') + { + *p = '\0'; + (void) mkdir(tmp, 0755); + *p = '/'; + } + } + if (mkdir(r->workDir, 0755) != 0 && errno != EEXIST) + { + log_error("Failed to create work directory \"%s\": %m", r->workDir); + return false; + } + } + + /* Generate SSL certs if the cluster ssl mode requires them (verify-ca/full) */ + if (!compose_gen_write_ssl_certs(&r->spec->cluster, r->workDir)) + return false; + + /* + * In host mode (no PGAFTEST_COMPOSE_SERVICE) we ARE the test runner, so + * we don't want the compose file to include a pgaftest service — it only + * makes sense for the in-compose CI mode where `docker compose up + * --exit-code-from pgaftest` drives the whole run. Pass NULL to omit it. + */ + const char *specFileForCompose = getenv("PGAFTEST_COMPOSE_SERVICE") + ? r->specFile : NULL; + + if (!compose_gen_write(&r->spec->cluster, + r->composeFile, + r->projectName, + r->contextDir, + specFileForCompose)) + return false; + + /* + * Write per-node pg_autoctl_node.ini files into the workdir. Each one + * is bind-mounted into its container at /etc/pgaf/node.ini so that + * every container uses the same image and command regardless of role. + */ + if (!compose_gen_write_monitor_ini(&r->spec->cluster, r->workDir)) + return false; + + if (!compose_gen_write_second_monitor_ini(&r->spec->cluster, r->workDir)) + return false; + + int globalNodeId = 0; + for (int fi = 0; fi < r->spec->cluster.formationCount; fi++) + { + const TestFormation *form = &r->spec->cluster.formations[fi]; + + for (int ni = 0; ni < form->nodeCount; ni++) + { + if (!compose_gen_write_node_ini(&r->spec->cluster, form, + &form->nodes[ni], + ++globalNodeId, r->workDir)) + return false; + } + } + + return true; +} + +static bool +runner_compose_up(TestRunner *r) +{ + log_info("Starting compose stack (project: %s)", r->projectName); + + int rc = run_cmd("%s up --build -d 2>&1", + r->composeBase); + if (rc != 0) + { + log_error("docker compose up failed (exit %d)", rc); + return false; + } + r->composeUp = true; + + /* + * Stop services declared with "launch deferred" (e.g. a second monitor). + * They are defined in the compose file so their image is built and volumes + * are created, but they must not run until the test explicitly starts them. + */ + if (r->spec->cluster.secondMonitorName[0] && + r->spec->cluster.secondMonitorStopped) + { + log_info("Stopping initially-stopped service %s", + r->spec->cluster.secondMonitorName); + rc = run_cmd("%s stop %s 2>&1", + r->composeBase, r->spec->cluster.secondMonitorName); + if (rc != 0) + { + log_error("docker compose stop %s failed (exit %d)", + r->spec->cluster.secondMonitorName, rc); + return false; + } + } + + return true; +} + +/* + * runner_apply_formation_settings — apply cluster-level formation settings + * that cannot be expressed in the node ini files. + * + * Called once after `docker compose up` and the monitor healthcheck passes. + * Currently applies: + * - number-sync-standbys (when numSync >= 0 in the cluster spec) + * + * We use `docker compose exec` so the call goes through the monitor's own + * pg_autoctl binary and lands on the already-running monitor instance. + */ +static bool +runner_apply_formation_settings(TestRunner *r) +{ + const TestCluster *c = &r->spec->cluster; + bool ok = true; + + for (int fi = 0; fi < c->formationCount; fi++) + { + const TestFormation *form = &c->formations[fi]; + + if (form->numSync < 0) + continue; + + log_info("Setting formation \"%s\" number-sync-standbys = %d", + form->name, form->numSync); + + int rc = run_cmd( + "%s exec -T monitor " + "pg_autoctl set formation number-sync-standbys %d " + "--pgdata /var/lib/postgres/pgaf --formation %s 2>&1", + r->composeBase, + form->numSync, form->name); + + if (rc != 0) + { + log_error("Failed to set number-sync-standbys for formation " + "\"%s\" (exit %d)", form->name, rc); + ok = false; + } + } + + return ok; +} + +static bool +runner_compose_down(TestRunner *r) +{ + if (!r->composeUp) return true; + + /* + * When pgaftest runs as a service inside the compose stack itself, calling + * "compose down" sends SIGKILL to our own container, so skip it — the host + * runner already calls "compose down" after we exit. + */ + if (getenv("PGAFTEST_COMPOSE_SERVICE")) + { + r->composeUp = false; + return true; + } + + log_info("Tearing down compose stack (project: %s)", r->projectName); + + if (r->notifyConnected) + { + pgsql_finish(&r->notifyConn); + r->notifyConnected = false; + } + + run_cmd("%s down --volumes --remove-orphans 2>&1", + r->composeBase); + r->composeUp = false; + return true; +} + +/* ----------------------------------------------------------------------- + * State polling via pg_autoctl inspect monitor subcommands + * + * Instead of shelling to psql (with all its quoting pitfalls), we call + * purpose-built subcommands of pg_autoctl that are installed in the + * monitor container: + * + * pg_autoctl inspect monitor node-state --name + * → stdout: "reported|assigned\n" exit 0 on success + * + * pg_autoctl inspect monitor formation-states [--group N] [...] + * → exit 0 when all listed states have ≥1 node, exit 1 when not + * ----------------------------------------------------------------------- */ + +/* + * Run `docker compose exec -T monitor pg_autoctl inspect monitor node-state + * --name ` and parse the "reported|assigned\n" output. + * + * When timeoutSecs > 0, passes --timeout to let the subcommand do its own + * retry loop (exponential back-off with jitter, same policy as the rest of + * pg_autoctl). When targetState is non-NULL, also passes --state so the + * subcommand only exits 0 when the node has reached that reported state. + */ +static bool +monitor_get_node_state(TestRunner *r, const char *nodeName, + char *reported, int replen, + char *assigned, int asslen) +{ + char out[256]; + int rc = run_cmd_capture(out, sizeof(out), + "%s exec -T %s " + "pg_autoctl inspect monitor node-state --name %s 2>/dev/null", + r->composeBase, r->activeMonitorService, nodeName); + + if (rc != 0 || out[0] == '\0') + { + /* + * Fallback for v2.1 monitors that don't have "pg_autoctl inspect": + * query the pgautofailover.node table directly via psql. + */ + int rc2 = run_cmd_capture(out, sizeof(out), + "%s exec -T %s " + "psql -U autoctl_node -d pg_auto_failover -At " + "-c \"SELECT reportedstate||'|'||goalstate FROM pgautofailover.node " + " WHERE nodename='%s'\" 2>/dev/null", + r->composeBase, r->activeMonitorService, nodeName); + if (rc2 != 0 || out[0] == '\0') + return false; + } + + /* strip trailing whitespace */ + int n = strlen(out); + while (n > 0 && (out[n-1] == '\n' || out[n-1] == '\r' || out[n-1] == ' ')) + out[--n] = '\0'; + + /* format: reported|goal|health */ + char *sep1 = strchr(out, '|'); + if (!sep1) + return false; + *sep1 = '\0'; + strncpy(reported, out, replen - 1); + char *sep2 = strchr(sep1+1, '|'); + if (sep2) + *sep2 = '\0'; + strncpy(assigned, sep1+1, asslen - 1); + return true; +} + +/* + * Read the pgdata path for a node from its .ini file in the work directory. + * Returns true and fills pgdata on success; false if the file can't be read. + */ +static bool +get_node_pgdata(TestRunner *r, const char *nodeName, char *pgdata, int len) +{ + char iniPath[1280]; + snprintf(iniPath, sizeof(iniPath), "%s/%s.ini", r->workDir, nodeName); + + FILE *f = fopen(iniPath, "r"); + if (!f) + return false; + + char line[256]; + bool found = false; + + while (fgets(line, sizeof(line), f)) + { + if (strncmp(line, "pgdata", 6) == 0) + { + char *eq = strchr(line, '='); + if (eq) + { + eq++; + while (*eq == ' ' || *eq == '\t') eq++; + char *end = eq + strlen(eq) - 1; + while (end > eq && (*end == '\n' || *end == '\r' || *end == ' ')) + *end-- = '\0'; + strlcpy(pgdata, eq, len); + found = true; + break; + } + } + } + fclose(f); + return found; +} + + +/* + * Query the monitor for reported|goal|health of a node in one call. + * Returns true if the node was found; health is set to the integer value + * from pgautofailover.node.health (1 = healthy, -1 = unhealthy/unknown). + */ +static bool +monitor_get_node_health(TestRunner *r, const char *nodeName, + char *reported, int replen, + char *goal, int goallen, + int *health) +{ + char out[256]; + int rc = run_cmd_capture(out, sizeof(out), + "%s exec -T monitor " + "pg_autoctl inspect monitor node-state --name %s 2>/dev/null", + r->composeBase, nodeName); + + if (rc != 0 || out[0] == '\0') + return false; + + int n = strlen(out); + while (n > 0 && (out[n-1] == '\n' || out[n-1] == '\r' || out[n-1] == ' ')) + out[--n] = '\0'; + + /* format: reported|goal|health */ + char *p1 = strchr(out, '|'); + if (!p1) return false; + *p1 = '\0'; + strlcpy(reported, out, replen); + + char *p2 = strchr(p1 + 1, '|'); + if (!p2) return false; + *p2 = '\0'; + strlcpy(goal, p1 + 1, goallen); + + *health = atoi(p2 + 1); + return true; +} + + +/* + * Fetch and log all node states from the monitor via pg_autoctl show state. + * Used for progress snapshots during wait loops. + */ +static void +log_formation_state(TestRunner *r) +{ + char out[4096]; + int rc = run_cmd_capture(out, sizeof(out), + "%s exec -T monitor " + "pg_autoctl show state 2>/dev/null", + r->composeBase); + + if (rc != 0 || out[0] == '\0') + return; + + char *line = out; + while (*line) + { + char *nl = strchr(line, '\n'); + if (nl) *nl = '\0'; + if (*line) + log_info(" %s", line); + if (!nl) break; + line = nl + 1; + } +} + +/* ----------------------------------------------------------------------- + * Direct LISTEN/NOTIFY connection to the monitor postgres + * + * compose_gen exposes the monitor's postgres on a random host port. + * We connect directly from the host to receive state-change notifications + * in real time, replacing the polling-via-docker-exec approach. + * ----------------------------------------------------------------------- */ + +/* + * Return the host port for the named monitor service, or 0 if unknown. + * Used to build the direct libpq connection string for LISTEN/NOTIFY and + * to look up the correct service name for monitor_get_node_state(). + */ +static int +runner_monitor_host_port(TestRunner *r, const char *svc) +{ + const TestCluster *cl = &r->spec->cluster; + + if (strcmp(svc, "monitor") == 0) + return cl->monitorHostPort; + + if (cl->secondMonitorName[0] && strcmp(svc, cl->secondMonitorName) == 0) + return cl->secondMonitorHostPort; + + return 0; +} + +/* + * Open (or reopen) a direct libpq connection to the monitor and LISTEN on + * the "state" channel. Safe to call multiple times — no-op if already + * connected. + */ +static bool +runner_notify_connect(TestRunner *r) +{ + if (r->notifyConnected && + r->notifyConn.connection != NULL && + PQstatus(r->notifyConn.connection) == CONNECTION_OK) + return true; + + /* close any stale connection */ + if (r->notifyConn.connection) + { + pgsql_finish(&r->notifyConn); + r->notifyConnected = false; + } + + TestCluster *cl = &r->spec->cluster; + const char *svc = r->activeMonitorService; + + char connstr[512]; + + if (getenv("PGAFTEST_COMPOSE_SERVICE")) + { + /* + * Inside the compose network: connect directly via the monitor service + * hostname. Include the password when the cluster uses md5/scram auth + * so that the autoctl_node role can authenticate. + */ + if (cl->monitorPassword[0]) + snprintf(connstr, sizeof(connstr), + "host=%s port=5432 dbname=pg_auto_failover " + "user=autoctl_node password=%s connect_timeout=5", + svc, cl->monitorPassword); + else + snprintf(connstr, sizeof(connstr), + "host=%s port=5432 dbname=pg_auto_failover " + "user=autoctl_node connect_timeout=5", + svc); + } + else + { + int port = runner_monitor_host_port(r, svc); + if (port == 0) + { + log_debug("host port for monitor service \"%s\" not set, " + "skipping LISTEN connection", svc); + return false; + } + if (cl->monitorPassword[0]) + snprintf(connstr, sizeof(connstr), + "host=localhost port=%d dbname=pg_auto_failover " + "user=autoctl_node password=%s " + "connect_timeout=5", + port, cl->monitorPassword); + else + snprintf(connstr, sizeof(connstr), + "host=localhost port=%d dbname=pg_auto_failover " + "user=autoctl_node " + "connect_timeout=2", + port); + } + + strlcpy(r->notifyConn.connectionString, connstr, + sizeof(r->notifyConn.connectionString)); + + /* + * Suppress libpq's "could not connect" ERROR logs while we're polling — + * connection failures are expected until the monitor is ready. + * Restore the original level immediately after the attempt. + */ + int savedLevel = log_get_level(); + log_set_level(LOG_FATAL); + + char *channels[] = { "state", NULL }; + bool ok = pgsql_listen(&r->notifyConn, channels); + + log_set_level(savedLevel); + + if (!ok) + { + log_debug("LISTEN connection to monitor service \"%s\" failed " + "(will retry)", svc); + return false; + } + + log_debug("Listening on monitor service \"%s\" for state notifications", + svc); + r->notifyConnected = true; + return true; +} + +/* + * Drain pending notifications from the monitor connection, logging each one. + * + * If markNodes/markStates are provided (count > 0), any notification where + * both goalState AND reportedState equal the target state for the matching + * node is printed with a "* [notify]" prefix. Only the convergence event + * (the notification that actually lifts the wait) is marked; the earlier + * assignment event (goalState matches but reportedState doesn't yet) is + * printed with the ordinary " [notify]" prefix. + * + * Callers that have no current wait condition pass count=0 (or NULL arrays). + * Returns the last CurrentNodeState parsed, or leaves *last unchanged if none. + */ +static void +runner_drain_notify(TestRunner *r, CurrentNodeState *last, + const char *const *markNodes, + const char *const *markStates, + int markCount, + bool *satisfied) /* optional: set [i] on convergence */ +{ + PGconn *conn = r->notifyConn.connection; + if (!conn) return; + + PQconsumeInput(conn); + + PGnotify *notify; + while ((notify = PQnotifies(conn)) != NULL) + { + if (strcmp(notify->relname, "state") == 0) + { + CurrentNodeState ns = { 0 }; + if (parse_state_notification_message(&ns, notify->extra)) + { + const char *ns_goal = NodeStateToString(ns.goalState); + const char *ns_rep = NodeStateToString(ns.reportedState); + + /* + * Mark '*' only when this notification IS the convergence event: + * both goalState and reportedState equal the target for this node. + * markNodes[i] == NULL is a wildcard matching any node. + */ + bool matched = false; + for (int i = 0; i < markCount; i++) + { + if (markStates && markStates[i] && + strcmp(ns_goal, markStates[i]) == 0 && + strcmp(ns_rep, markStates[i]) == 0 && + (!markNodes || !markNodes[i] || + strcmp(ns.node.name, markNodes[i]) == 0)) + { + matched = true; + if (satisfied) + satisfied[i] = true; + } + } + const char *prefix = matched ? "* [notify]" : " [notify]"; + + if (ns.health >= 0) + log_info("%s %s: %s \xe2\x9e\x9c %s", + prefix, ns.node.name, ns_rep, ns_goal); + else + log_info("%s %s: %s \xe2\x9e\x9c %s (unhealthy)", + prefix, ns.node.name, ns_rep, ns_goal); + if (last) *last = ns; + } + else + log_debug("unparseable state notification: %s", notify->extra); + } + PQfreemem(notify); + } +} + +/* + * Wait on the monitor socket with select(), up to `remainMs` milliseconds. + * Returns true if there is data to read, false on timeout. + */ +static bool +runner_wait_socket(TestRunner *r, int remainMs) +{ + PGconn *conn = r->notifyConn.connection; + if (!conn) return false; + + int sock = PQsocket(conn); + if (sock < 0) return false; + + fd_set rfds; + FD_ZERO(&rfds); + FD_SET(sock, &rfds); + + struct timeval tv = { + .tv_sec = remainMs / 1000, + .tv_usec = (remainMs % 1000) * 1000, + }; + bool ready = select(sock + 1, &rfds, NULL, NULL, &tv) > 0; + if (ready) + PQconsumeInput(conn); /* pull data into libpq buffer so PQnotifies sees it */ + return ready; +} + +/* + * Check that the formation has converged: for each required state, at least + * one cluster node must have both reportedstate = assignedstate = that state. + * This is the correct convergence condition — checking only assignedstate + * fires as soon as the monitor makes an assignment, before nodes confirm. + * + * groupIds / groupCount optionally restrict the check to specific Citus groups + * (pass NULL / 0 for a plain HA formation). + */ +static bool +monitor_check_formation_converged(TestRunner *r, + const char (*states)[64], int stateCount, + const int *groupIds, int groupCount) +{ + bool satisfied[PGAF_MAX_WAIT_STATES] = {false}; + int nodesQueried = 0; + + const TestCluster *c = &r->spec->cluster; + for (int fi = 0; fi < c->formationCount; fi++) + { + const TestFormation *form = &c->formations[fi]; + for (int ni = 0; ni < form->nodeCount; ni++) + { + const TestNode *node = &form->nodes[ni]; + + /* skip nodes not in the requested groups */ + if (groupCount > 0) + { + bool inGroup = false; + for (int gi = 0; gi < groupCount; gi++) + if (node->group == groupIds[gi]) { inGroup = true; break; } + if (!inGroup) + continue; + } + + char reported[64] = "", assigned[64] = ""; + if (!monitor_get_node_state(r, node->name, + reported, sizeof(reported), + assigned, sizeof(assigned))) + continue; + + nodesQueried++; + for (int si = 0; si < stateCount && si < PGAF_MAX_WAIT_STATES; si++) + { + if (strcmp(reported, states[si]) == 0 && + strcmp(assigned, states[si]) == 0) + satisfied[si] = true; + } + } + } + + /* + * If no nodes could be reached via subprocess (e.g. monitor running v2.1 + * which lacks "pg_autoctl inspect monitor node-state"), the check is + * inconclusive rather than negative. Return true so the caller can trust + * the LISTEN-based satisfied[] from runner_drain_notify instead. + */ + if (nodesQueried == 0) + return true; + + for (int si = 0; si < stateCount; si++) + if (!satisfied[si]) + return false; + return true; +} + +/* + * Wait until the formation has at least one node in each listed state, where + * both reportedstate and assignedstate have converged on the target. + * + * Strategy: + * 1. Fast-path subprocess check: if the formation is already converged on + * entry (e.g. after a synchronous command), return immediately. + * 2. LISTEN-driven main loop: block on the notify socket, drain each batch + * of notifications, and track which target states have been seen with + * BOTH goalState and reportedState matching (convergence events) via + * the satisfied[] array passed to runner_drain_notify. + * When all target states are satisfied by notifications, do ONE subprocess + * double-check before declaring success — guards against a rare race where + * a node races through the target state and moves on before we verify. + * 3. No-LISTEN fallback: if the notify connection is unavailable, fall back + * to subprocess polling every second. + */ +static bool +monitor_wait_formation_states(TestRunner *r, + const char (*states)[64], int stateCount, + const int *groupIds, int groupCount, + int timeoutSecs) +{ + runner_notify_connect(r); + + /* pointer arrays for drain marking — NULL node = wildcard (any node) */ + const char *ms[PGAF_MAX_WAIT_STATES]; + for (int i = 0; i < stateCount && i < PGAF_MAX_WAIT_STATES; i++) + ms[i] = states[i]; + + /* satisfied[i] is set by runner_drain_notify when a convergence + * notification is seen for states[i] (both goal and reported match) */ + bool satisfied[PGAF_MAX_WAIT_STATES] = {false}; + + /* + * Drain any notifications already buffered in libpq before the fast-path + * check. This is important for synchronous commands like perform failover: + * the entire FSM cycle completes inside the blocking exec, so all + * intermediate NOTIFY messages have accumulated in the socket by the time + * we arrive here. Draining first ensures those events appear in the + * current step's log output (with '*' marks on the convergence ones) rather + * than spilling into the next command's inter-drain or into teardown. + */ + if (r->notifyConnected) + runner_drain_notify(r, NULL, NULL, ms, stateCount, satisfied); + + /* fast-path: already converged (confirmed by subprocess) */ + bool allSatisfiedEarly = true; + for (int i = 0; i < stateCount; i++) + if (!satisfied[i]) { allSatisfiedEarly = false; break; } + + if (allSatisfiedEarly && + monitor_check_formation_converged(r, states, stateCount, + groupIds, groupCount)) + return true; + + time_t deadline = time(NULL) + timeoutSecs; + int pollCounter = 0; /* counts 5s wait-socket cycles */ + + while (time(NULL) < deadline) + { + int remainMs = (int) ((deadline - time(NULL)) * 1000); + if (remainMs <= 0) break; + + if (r->notifyConnected) + { + /* block until a notification arrives or up to 5s */ + int waitMs = remainMs < 5000 ? remainMs : 5000; + runner_wait_socket(r, waitMs); + + /* drain: logs, marks '*', and sets satisfied[i] on convergence */ + runner_drain_notify(r, NULL, NULL, ms, stateCount, satisfied); + + /* when every target state has been seen converged via notification, + * do one subprocess double-check before declaring success */ + bool allSatisfied = true; + for (int i = 0; i < stateCount; i++) + if (!satisfied[i]) { allSatisfied = false; break; } + + /* + * Also poll the monitor directly every ~5 cycles (~25 s) even + * without allSatisfied. This handles the case where the LISTEN + * connection reconnects AFTER the state transitions fired — those + * notifications are gone and allSatisfied will never become true, + * but the monitor's current-state query still reflects reality. + */ + ++pollCounter; + + /* + * When notifications say we converged, trust them — return + * immediately without a subprocess double-check. The double-check + * races with fast post-convergence transitions: by the time the + * subprocess queries the monitor the nodes may have already moved + * on, producing a false negative that causes a 120s timeout. + * + * The periodic (every ~25s) subprocess poll remains as a fallback + * for missed notifications (e.g. the LISTEN reconnect after a + * monitor Postgres restart in the upgrade test). + */ + if (allSatisfied) + return true; + if (pollCounter % 5 == 0) + { + if (monitor_check_formation_converged(r, states, stateCount, + groupIds, groupCount)) + return true; + } + + runner_notify_connect(r); /* reconnect if socket went bad */ + } + else + { + /* no LISTEN — subprocess poll once per second */ + pg_usleep(1000 * 1000); + runner_notify_connect(r); + if (monitor_check_formation_converged(r, states, stateCount, + groupIds, groupCount)) + return true; + } + } + + /* deadline reached: one final drain + subprocess check */ + if (r->notifyConnected) + runner_drain_notify(r, NULL, NULL, ms, stateCount, satisfied); + if (monitor_check_formation_converged(r, states, stateCount, + groupIds, groupCount)) + return true; + + log_formation_state(r); + return false; +} + +/* + * Wait until the formation has at least one node in each of the requested + * states (within the given groups), or timeout expires. + */ +static bool +wait_for_states(TestRunner *r, TestCmd *cmd) +{ + /* build a human-readable label for log messages */ + char label[256] = ""; + for (int i = 0; i < cmd->waitStateCount; i++) + { + if (i > 0) strncat(label, ", ", sizeof(label) - strlen(label) - 1); + strncat(label, cmd->waitStates[i], sizeof(label) - strlen(label) - 1); + } + + if (cmd->waitGroupCount > 0) + { + strncat(label, " in group(s) ", sizeof(label) - strlen(label) - 1); + for (int i = 0; i < cmd->waitGroupCount; i++) + { + char g[16]; + snprintf(g, sizeof(g), "%s%d", i > 0 ? "," : "", cmd->waitGroups[i]); + strncat(label, g, sizeof(label) - strlen(label) - 1); + } + } + + time_t t0 = time(NULL); + if (!monitor_wait_formation_states(r, + (const char (*)[64]) cmd->waitStates, + cmd->waitStateCount, + cmd->waitGroups, + cmd->waitGroupCount, + cmd->timeoutSeconds)) + { + log_error("Timed out after %ds waiting for formation states: %s", + (int) (time(NULL) - t0), label); + return false; + } + return true; +} + +/* + * For a single node: query the monitor for its current group and whether it + * is already the primary of that group. If not, call + * pg_autoctl perform promotion --name + * on the monitor and wait until it becomes primary. + * + * pg_autoctl perform promotion is synchronous: it returns only after the new + * primary has reached the 'primary' state. We still poll to confirm the + * monitor agrees, using the existing wait_for_state() helper. + */ +static bool +runner_promote_one(TestRunner *r, const char *nodeName) +{ + /* Check current state */ + char reported[64] = "", assigned[64] = ""; + if (!monitor_get_node_state(r, nodeName, + reported, sizeof(reported), + assigned, sizeof(assigned))) + { + log_error("promote: cannot query monitor for node %s", nodeName); + return false; + } + + if (strcmp(reported, "primary") == 0) + { + log_info(" promote: %s is already primary — no-op", nodeName); + return true; + } + + log_info(" promote: %s is %s, requesting promotion", nodeName, reported); + + /* find the formation this node belongs to */ + const char *formation = "default"; + const TestCluster *c = &r->spec->cluster; + for (int fi = 0; fi < c->formationCount; fi++) + { + const TestFormation *form = &c->formations[fi]; + for (int ni = 0; ni < form->nodeCount; ni++) + { + if (strcmp(form->nodes[ni].name, nodeName) == 0) + { + formation = form->name; + break; + } + } + } + + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "%s exec -T monitor " + "pg_autoctl perform promotion --name %s --formation %s 2>&1", + r->composeBase, nodeName, formation); + + if (rc != 0) + { + if (out[0]) + log_output(" ", out); + log_error("promote: pg_autoctl perform promotion --name %s " + "--formation %s failed (exit %d)", nodeName, formation, rc); + return false; + } + + /* confirm monitor agrees */ + return wait_for_state(r, nodeName, "primary", PGAF_TIMEOUT_DEFAULT, false); +} + +/* + * runner_wait_notify_goal — LISTEN-driven single-node wait (items 5 & 6). + * + * Blocks until a "state" notification arrives where both states have + * converged on the target: + * ns.node.name == nodeName + * AND NodeStateToString(ns.goalState) == targetState + * AND NodeStateToString(ns.reportedState) == targetState + * + * The '*' marker is applied as soon as goalState matches (monitor has + * assigned the target) even before the node confirms, so the log shows + * the assignment event clearly. The wait only lifts when the node + * confirms (reportedState also equals target), verified by a final SQL + * check that both assigned_state and reportedState agree. + * + * Using goalState (== monitor's assigned_state) as the trigger is faster and + * more reliable than polling reportedState: the notification fires the moment + * the monitor makes the assignment decision, before the node has transitioned. + * + * Along the way, any intermediate goalState values for this node are checked + * against passThroughStates[] and seenThrough[] is updated (item 6). + * + * After the target goalState notification arrives: + * - drain remaining notifications (log them all) + * - final SQL check: assigned_state == targetState via monitor_get_node_state() + * - if SQL disagrees (rare race), keep looping + * + * Fallback when LISTEN is unavailable: poll assigned_state via SQL every 500ms; + * track pass-through states from reportedState. + * + * For nodes the monitor has marked unhealthy, also accepts a match from the + * node's local FSM (pg_autoctl inspect fsm node-state) to handle cases where + * the keeper self-assigns a state without monitor contact. + */ +static bool +runner_wait_notify_goal(TestRunner *r, + const char *nodeName, + const char *targetState, + const char passThroughStates[][64], + bool *seenThrough, + int passThroughCount, + int timeoutSecs) +{ + runner_notify_connect(r); + + /* + * Fast path: if the node has already converged on the target state + * (reportedState == assignedState == targetState), return immediately. + * Without this, a node that settled during a preceding command would + * spin the full timeout waiting for a notification that won't arrive. + */ + { + char rep0[64] = "", asgn0[64] = ""; + if (monitor_get_node_state(r, nodeName, rep0, sizeof(rep0), + asgn0, sizeof(asgn0)) && + strcmp(rep0, targetState) == 0 && + strcmp(asgn0, targetState) == 0) + return true; + } + + time_t deadline = time(NULL) + timeoutSecs; + + while (time(NULL) < deadline) + { + if (r->notifyConnected) + { + PQconsumeInput(r->notifyConn.connection); + + PGnotify *notify; + while ((notify = PQnotifies(r->notifyConn.connection)) != NULL) + { + if (strcmp(notify->relname, "state") == 0) + { + CurrentNodeState ns = { 0 }; + if (parse_state_notification_message(&ns, notify->extra)) + { + const char *ns_goal = NodeStateToString(ns.goalState); + const char *ns_rep = NodeStateToString(ns.reportedState); + + /* + * Mark '*' only when this IS the convergence notification: + * both goalState and reportedState equal the target. + * The earlier assignment notification (goalState matches + * but reportedState doesn't yet) prints without a marker. + */ + bool goal_matched = (strcmp(ns.node.name, nodeName) == 0 && + strcmp(ns_goal, targetState) == 0 && + strcmp(ns_rep, targetState) == 0); + const char *prefix = goal_matched ? "* [notify]" : " [notify]"; + + if (ns.health >= 0) + log_info("%s %s: %s \xe2\x9e\x9c %s", + prefix, ns.node.name, ns_rep, ns_goal); + else + log_info("%s %s: %s \xe2\x9e\x9c %s (unhealthy)", + prefix, ns.node.name, ns_rep, ns_goal); + + if (strcmp(ns.node.name, nodeName) == 0) + { + /* track pass-through states from goalState (item 6) */ + for (int i = 0; i < passThroughCount; i++) + { + if (seenThrough && !seenThrough[i] && + strcmp(ns_goal, passThroughStates[i]) == 0) + { + log_info("wait %s: observed pass-through " + "assigned-state %s", + nodeName, ns_goal); + seenThrough[i] = true; + } + } + + /* + * Lift the wait only when both states have converged: + * the monitor assigned the target (goalState) and the + * node confirmed it (reportedState). The notification + * already carries both values so we can test here + * without an extra round-trip; we still do a final SQL + * confirmation to guard against a torn read. + */ + if (strcmp(ns_goal, targetState) == 0 && + strcmp(ns_rep, targetState) == 0) + { + PQfreemem(notify); + /* post-match drain: no marking — belongs to next wait */ + runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); + + /* final SQL: confirm both assigned and reported */ + char rep[64] = "", asgn[64] = ""; + if (monitor_get_node_state(r, nodeName, + rep, sizeof(rep), + asgn, sizeof(asgn))) + { + if (strcmp(rep, targetState) == 0 && + strcmp(asgn, targetState) == 0) + return true; + /* + * Rare torn read: keep looping. + * The next notification will re-trigger. + */ + log_debug(" %s: notify shows convergence on %s " + "but SQL has rep=%s asgn=%s, retrying", + nodeName, targetState, rep, asgn); + } + else + { + /* can't reach monitor — trust the notification */ + return true; + } + goto next_iter; + } + } + } + else + log_debug("unparseable state notification: %s", + notify->extra); + } + PQfreemem(notify); + } + + /* + * For unhealthy nodes: also accept a match from the node's local + * FSM — the keeper self-assigns certain states without monitor + * contact. + * + * For nodes that have been hard-killed (compose kill), the + * container is gone so exec fails. Accept goalState-alone match + * for unhealthy nodes: a dead node can never report the new state, + * but the monitor has already made the assignment decision. + */ + { + char rep[64] = "", goal[64] = ""; + int health = 1; + if (monitor_get_node_health(r, nodeName, + rep, sizeof(rep), + goal, sizeof(goal), + &health) && + health <= 0) + { + if (strcmp(goal, targetState) == 0) + return true; + + char pgdata[1024] = ""; + get_node_pgdata(r, nodeName, pgdata, sizeof(pgdata)); + char out[256]; + if (run_cmd_capture(out, sizeof(out), + "%s exec -T %s " + "pg_autoctl inspect fsm node-state" + " --state %s --timeout 0%s%s 2>/dev/null", + r->composeBase, nodeName, targetState, + pgdata[0] ? " --pgdata " : "", + pgdata[0] ? pgdata : "") == 0) + return true; + } + } + } + else + { + /* + * No LISTEN connection — fall back to polling via SQL. + * Require both reportedState and assignedState to equal the target + * (same convergence criterion as the notification path), except for + * unhealthy nodes where goalState alone is accepted (dead nodes + * can never report the new state). + */ + char rep[64] = "", asgn[64] = ""; + if (monitor_get_node_state(r, nodeName, rep, sizeof(rep), + asgn, sizeof(asgn))) + { + for (int i = 0; i < passThroughCount; i++) + { + if (seenThrough && !seenThrough[i] && + strcmp(rep, passThroughStates[i]) == 0) + { + log_info("wait %s: observed pass-through state %s", + nodeName, rep); + seenThrough[i] = true; + } + } + if (strcmp(rep, targetState) == 0 && + strcmp(asgn, targetState) == 0) + return true; + + /* unhealthy node: accept goalState match alone */ + char rep2[64] = "", goal2[64] = ""; + int health2 = 1; + if (strcmp(asgn, targetState) == 0 && + monitor_get_node_health(r, nodeName, + rep2, sizeof(rep2), + goal2, sizeof(goal2), + &health2) && + health2 <= 0) + return true; + } + } + +next_iter: + { + int remainMs = (int)((deadline - time(NULL)) * 1000); + if (remainMs <= 0) break; + int waitMs = remainMs < 2000 ? remainMs : 2000; + + if (r->notifyConnected) + { + runner_wait_socket(r, waitMs); + runner_notify_connect(r); + } + else + { + pg_usleep(500 * 1000); + runner_notify_connect(r); + } + } + } + + /* one last chance: drain + SQL check — mark the target in case it just arrived */ + if (r->notifyConnected) + runner_drain_notify(r, NULL, &nodeName, &targetState, 1, NULL); + + char rep2[64] = "", asgn2[64] = ""; + if (monitor_get_node_state(r, nodeName, rep2, sizeof(rep2), + asgn2, sizeof(asgn2)) && + strcmp(rep2, targetState) == 0 && + strcmp(asgn2, targetState) == 0) + return true; + + return false; +} + +/* + * runner_wait_assigned_goal — wait until the monitor's goalState for nodeName + * equals targetState, without requiring reportedState to converge. + * + * This implements the semantics of "wait until assigned-state = X": + * succeed as soon as the monitor assigns X as the goalState, even if the node + * has not yet transitioned its own reportedState. This matches the Python + * test's wait_until_assigned_state() behaviour. + * + * Uses NOTIFY fast-path (goalState match alone) and falls back to polling + * the assigned_state column directly via monitor_get_node_state(). + */ +static bool +runner_wait_assigned_goal(TestRunner *r, const char *nodeName, + const char *targetState, int timeoutSecs) +{ + runner_notify_connect(r); + + /* fast path: already there */ + { + char rep[64] = "", asgn[64] = ""; + if (monitor_get_node_state(r, nodeName, rep, sizeof(rep), + asgn, sizeof(asgn)) && + strcmp(asgn, targetState) == 0) + return true; + } + + time_t deadline = time(NULL) + timeoutSecs; + + while (time(NULL) < deadline) + { + if (r->notifyConnected) + { + PQconsumeInput(r->notifyConn.connection); + + PGnotify *notify; + while ((notify = PQnotifies(r->notifyConn.connection)) != NULL) + { + if (strcmp(notify->relname, "state") == 0) + { + CurrentNodeState ns = { 0 }; + if (parse_state_notification_message(&ns, notify->extra)) + { + const char *ns_goal = NodeStateToString(ns.goalState); + const char *ns_rep = NodeStateToString(ns.reportedState); + + if (ns.health >= 0) + log_info(" [notify] %s: %s \xe2\x9e\x9c %s", + ns.node.name, ns_rep, ns_goal); + else + log_info(" [notify] %s: %s \xe2\x9e\x9c %s (unhealthy)", + ns.node.name, ns_rep, ns_goal); + + if (strcmp(ns.node.name, nodeName) == 0 && + strcmp(ns_goal, targetState) == 0) + { + PQfreemem(notify); + /* confirm via SQL */ + char rep[64] = "", asgn[64] = ""; + if (monitor_get_node_state(r, nodeName, + rep, sizeof(rep), + asgn, sizeof(asgn))) + { + if (strcmp(asgn, targetState) == 0) + return true; + /* rare race: goalState changed already; keep looping */ + } + else + { + return true; /* can't reach monitor, trust notify */ + } + goto next_iter_asgn; + } + } + } + PQfreemem(notify); + } + } + else + { + /* fallback: poll assigned_state via SQL */ + char rep[64] = "", asgn[64] = ""; + if (monitor_get_node_state(r, nodeName, rep, sizeof(rep), + asgn, sizeof(asgn)) && + strcmp(asgn, targetState) == 0) + return true; + } + +next_iter_asgn: + { + int remainMs = (int)((deadline - time(NULL)) * 1000); + if (remainMs <= 0) break; + int waitMs = remainMs < 2000 ? remainMs : 2000; + + if (r->notifyConnected) + runner_wait_socket(r, waitMs); + else + pg_usleep(500 * 1000); + + runner_notify_connect(r); + } + } + + return false; +} + +/* + * Wait until the node reaches the target state, or timeout expires. + * + * For state (reported) checks: delegates to runner_wait_notify_goal() which + * waits for both goalState and reportedState to converge on target. + * For assigned-state checks: delegates to runner_wait_assigned_goal() which + * succeeds as soon as the monitor's goalState equals target. + */ +static bool +wait_for_state(TestRunner *r, const char *nodeName, const char *targetState, + int timeoutSecs, bool checkAssigned) +{ + if (checkAssigned) + { + /* + * assigned-state check: succeed as soon as the monitor's goalState + * equals the target, without requiring the node to have transitioned + * its reportedState. This matches the Python test's behaviour for + * wait_until_assigned_state() and is necessary for cases like + * test_015_003 where the primary is assigned "primary" goalstate but + * cannot complete the transition until standbys reconnect. + */ + if (!runner_wait_assigned_goal(r, nodeName, targetState, timeoutSecs)) + { + log_error("Timeout waiting for %s assigned-state = %s", + nodeName, targetState); + return false; + } + return true; + } + + if (!runner_wait_notify_goal(r, nodeName, targetState, + NULL, NULL, 0, timeoutSecs)) + { + log_error("Timeout waiting for %s state = %s", + nodeName, targetState); + return false; + } + return true; +} + +/* + * runner_check_stays_notify — drain pending notifications and verify none + * changed the watched node's goalState away from expectedState (item 9). + * + * Returns true if the state is still as expected (no disruptive notification). + * Returns false and writes into errBuf if a state-change notification arrived. + */ +static bool +runner_check_stays_notify(TestRunner *r, + const char *nodeName, const char *expectedState, + char *errBuf, int errLen) +{ + if (!r->notifyConnected) + return true; /* no LISTEN — caller will poll instead */ + + PQconsumeInput(r->notifyConn.connection); + + PGnotify *notify; + while ((notify = PQnotifies(r->notifyConn.connection)) != NULL) + { + if (strcmp(notify->relname, "state") == 0) + { + CurrentNodeState ns = { 0 }; + if (parse_state_notification_message(&ns, notify->extra)) + { + const char *ns_goal = NodeStateToString(ns.goalState); + const char *ns_rep = NodeStateToString(ns.reportedState); + + if (ns.health >= 0) + log_info(" [notify] %s: %s \xe2\x9e\x9c %s", + ns.node.name, ns_rep, ns_goal); + else + log_info(" [notify] %s: %s \xe2\x9e\x9c %s (unhealthy)", + ns.node.name, ns_rep, ns_goal); + + if (strcmp(ns.node.name, nodeName) == 0 && + strcmp(ns_goal, expectedState) != 0) + { + PQfreemem(notify); + snprintf(errBuf, errLen, + "stays-while: %s was assigned state %s " + "(expected to stay %s)", + nodeName, ns_goal, expectedState); + return false; + } + } + } + PQfreemem(notify); + } + return true; +} + +/* ----------------------------------------------------------------------- + * SQL execution on a service + * ----------------------------------------------------------------------- */ + +/* + * escape_sql_for_shell — escape single-quotes for embedding SQL in a + * single-quoted shell argument using the '\'' trick. + */ +static void +escape_sql_for_shell(const char *sql, char *out, int outlen) +{ + int oi = 0; + for (int i = 0; sql[i] && oi < outlen - 2; i++) + { + if (sql[i] == '\'') + { + out[oi++] = '\''; + out[oi++] = '\\'; + out[oi++] = '\''; + if (oi < outlen - 1) out[oi++] = '\''; + } + else + out[oi++] = sql[i]; + } + out[oi] = '\0'; +} + +/* + * parse_sqlstate — scan output for a PostgreSQL SQLSTATE code. + * + * With VERBOSITY=verbose, psql formats errors as: + * ERROR: XXXXX: message text + * where XXXXX is a 5-character alphanumeric SQLSTATE code. + * Writes up to 6 bytes (5 + NUL) into state. + */ +static void +parse_sqlstate(const char *output, char *state, int statelen) +{ + state[0] = '\0'; + const char *p = output; + while (p && *p) + { + p = strstr(p, "ERROR: "); + if (!p) break; + p += 8; /* skip "ERROR: " */ + /* SQLSTATE: exactly 5 alphanumeric characters followed by ':' */ + if (strlen(p) >= 6 && p[5] == ':') + { + bool valid = true; + for (int i = 0; i < 5 && valid; i++) + valid = isalnum((unsigned char)p[i]); + if (valid) + { + int n = statelen < 6 ? statelen - 1 : 5; + memcpy(state, p, n); + state[n] = '\0'; + return; + } + } + } +} + +static bool +exec_sql_on_service(TestRunner *r, const char *service, + const char *sql, char *outbuf, int outlen) +{ + char escaped[8192]; + escape_sql_for_shell(sql, escaped, sizeof(escaped)); + + /* + * Always run with ON_ERROR_STOP=1 so psql exits non-zero on SQL errors, + * and VERBOSITY=verbose so SQLSTATE codes appear in error output. + * Redirect stderr to stdout so both are captured. + * + * The monitor container only has the pg_auto_failover database (no + * "docker" default database), so supply explicit -U / -d for it. + */ + const char *connArgs = + (strcmp(service, "monitor") == 0) + ? "-U autoctl_node -d pg_auto_failover" + : ""; + + int rc = run_cmd_capture(outbuf, outlen, + "%s exec -T %s " + "psql --tuples-only --no-align " + "-v ON_ERROR_STOP=1 -v VERBOSITY=verbose " + "%s -c '%s' 2>&1", + r->composeBase, service, connArgs, escaped); + + return rc == 0; +} + +/* ----------------------------------------------------------------------- + * Network failure simulation + * ----------------------------------------------------------------------- */ + +static bool +runner_network_off(TestRunner *r, const char *nodeName) +{ + char netName[128], container[128]; + compose_network_name(r->projectName, netName, sizeof(netName)); + compose_container_name(r->projectName, nodeName, container, sizeof(container)); + + log_info(" Disconnecting %s from network %s", container, netName); + char out[1024] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "docker network disconnect %s %s 2>&1", + netName, container); + if (rc != 0 && out[0]) + log_output(" ", out); + return rc == 0; +} + +static bool +runner_network_on(TestRunner *r, const char *nodeName) +{ + char netName[128], container[128]; + compose_network_name(r->projectName, netName, sizeof(netName)); + compose_container_name(r->projectName, nodeName, container, sizeof(container)); + + log_info(" Reconnecting %s to network %s", container, netName); + char out[1024] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "docker network connect %s %s 2>&1", + netName, container); + if (rc != 0 && out[0]) + log_output(" ", out); + return rc == 0; +} + +/* ----------------------------------------------------------------------- + * Execute a single command + * ----------------------------------------------------------------------- */ + +/* + * runner_expand_macros substitutes %CIDR% in the args string with the + * Docker network CIDR reported by `pg_autoctl inspect show cidr` on the + * monitor container. The result is written to dst (at most dstlen bytes). + * Returns false if the CIDR could not be determined. + */ +static bool +runner_expand_macros(TestRunner *r, const char *args, char *dst, int dstlen, + char *errBuf, int errLen) +{ + const char *macro = "%CIDR%"; + const char *found = strstr(args, macro); + + if (!found) + { + strlcpy(dst, args, dstlen); + return true; + } + + /* fetch CIDR from the monitor */ + char cidr[128] = ""; + int rc = run_cmd_capture(cidr, sizeof(cidr), + "%s exec -T monitor pg_autoctl inspect show cidr 2>/dev/null", + r->composeBase); + if (rc != 0 || cidr[0] == '\0') + { + snprintf(errBuf, errLen, + "%%CIDR%%: could not get Docker network CIDR from monitor"); + return false; + } + /* trim trailing newline */ + int n = strlen(cidr); + while (n > 0 && (cidr[n-1] == '\n' || cidr[n-1] == '\r' || cidr[n-1] == ' ')) + cidr[--n] = '\0'; + + log_info(" %%CIDR%% = %s", cidr); + + /* substitute all occurrences */ + int di = 0; + for (const char *p = args; *p && di < dstlen - 1; ) + { + if (strncmp(p, macro, strlen(macro)) == 0) + { + int cl = strlen(cidr); + if (di + cl < dstlen - 1) + { + memcpy(dst + di, cidr, cl); + di += cl; + } + p += strlen(macro); + } + else + dst[di++] = *p++; + } + dst[di] = '\0'; + return true; +} + + +static bool +runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) +{ + switch (cmd->kind) + { + case CMD_EXEC: + { + char expandedArgs[4096] = ""; + if (!runner_expand_macros(r, cmd->args, expandedArgs, + sizeof(expandedArgs), errBuf, errLen)) + return false; + + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "%s exec -T %s %s 2>&1", + r->composeBase, + cmd->service, expandedArgs); + /* store output so a following expect{} can match it */ + strlcpy(r->lastSqlOutput, out, sizeof(r->lastSqlOutput)); + r->lastSqlFailed = false; + if (rc != 0) + { + if (out[0]) + log_output(" ", out); + snprintf(errBuf, errLen, + "exec %s %s failed (exit %d)", + cmd->service, expandedArgs, rc); + return false; + } + + /* + * Several pg_autoctl commands are synchronous from the monitor's + * perspective but leave NOTIFY messages buffered in the libpq + * socket. Drain them with proper '*' marks and confirm the + * expected post-command state before moving to the next step. + * + * perform failover / perform switchover / perform promotion: + * The entire FSM cycle completes inside the blocking exec. + * Wait for the formation to settle at primary+secondary. + * + * enable maintenance: + * The node transitions to maintenance asynchronously after + * the command returns. Wait for cmd->service to reach + * "maintenance" (single-node LISTEN-driven wait). + */ + /* + * Only trigger implicit waits for direct pg_autoctl invocations, + * not for bash -c "..." wrappers where the match is incidental. + */ + bool is_pg_autoctl = (strncmp(expandedArgs, "pg_autoctl", 10) == 0); + if (is_pg_autoctl && + (strstr(expandedArgs, "perform failover") != NULL || + strstr(expandedArgs, "perform switchover") != NULL || + strstr(expandedArgs, "perform promotion") != NULL || + strstr(expandedArgs, "disable maintenance") != NULL)) + { + static const char fsStates[2][64] = {"primary", "secondary"}; + if (!monitor_wait_formation_states(r, fsStates, 2, + NULL, 0, 120)) + { + snprintf(errBuf, errLen, + "exec %s %s: timed out waiting for " + "primary+secondary", + cmd->service, expandedArgs); + return false; + } + } + else if (is_pg_autoctl && strstr(expandedArgs, "enable maintenance") != NULL) + { + /* + * Drain any notifications already buffered from the enable + * maintenance command before calling wait_for_state, so the + * maintenance convergence notification gets the '*' mark here + * rather than spilling unmarked into the next inter-command drain. + */ + if (r->notifyConnected) + { + const char *mn = cmd->service, *ms = "maintenance"; + bool sat = false; + runner_drain_notify(r, NULL, &mn, &ms, 1, &sat); + } + if (!wait_for_state(r, cmd->service, "maintenance", 60, false)) + { + snprintf(errBuf, errLen, + "exec %s %s: timed out waiting for maintenance", + cmd->service, expandedArgs); + return false; + } + } + return true; + } + + case CMD_EXEC_FAILS: + { + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "%s exec -T %s %s 2>&1", + r->composeBase, + cmd->service, cmd->args); + if (rc == 0) + { + if (out[0]) + log_output(" ", out); + snprintf(errBuf, errLen, + "exec-fails %s %s: command succeeded (exit 0) " + "but expected failure", + cmd->service, cmd->args); + return false; + } + log_debug("exec-fails %s %s: exited with %d (expected)", + cmd->service, cmd->args, rc); + return true; + } + + case CMD_WAIT_STATE: + { + /* + * LISTEN-driven wait (items 5 & 6). + * + * runner_wait_notify_goal() watches the "state" channel and fires + * when goalState == target for this node. Pass-through states are + * also tracked from goalState notifications (not by polling). + * After success, verify all pass-through states were observed. + */ + bool seenThrough[PGAF_MAX_WAIT_STATES] = { false }; + + if (!runner_wait_notify_goal(r, cmd->service, cmd->state, + (const char (*)[64]) cmd->passThroughStates, + seenThrough, cmd->passThroughCount, + cmd->timeoutSeconds)) + { + snprintf(errBuf, errLen, + "timeout: %s assigned-state never reached %s", + cmd->service, cmd->state); + return false; + } + + for (int i = 0; i < cmd->passThroughCount; i++) + { + if (!seenThrough[i]) + { + snprintf(errBuf, errLen, + "%s did not pass through assigned-state %s " + "on way to %s", + cmd->service, + cmd->passThroughStates[i], + cmd->state); + return false; + } + } + return true; + } + + case CMD_WAIT_STATES: + if (!wait_for_states(r, cmd)) + { + /* wait_for_states logs its own error detail */ + char label[128] = ""; + for (int i = 0; i < cmd->waitStateCount; i++) + { + if (i > 0) + strncat(label, ",", sizeof(label)-strlen(label)-1); + strncat(label, cmd->waitStates[i], + sizeof(label)-strlen(label)-1); + } + snprintf(errBuf, errLen, + "timeout: formation never reached states {%s}", label); + return false; + } + return true; + + case CMD_PROMOTE: + for (int i = 0; i < cmd->promoteCount; i++) + { + if (!runner_promote_one(r, cmd->promoteNodes[i])) + { + snprintf(errBuf, errLen, + "promote: could not make %s primary", + cmd->promoteNodes[i]); + return false; + } + } + return true; + + case CMD_ASSERT_STATE: + case CMD_ASSERT_ASSIGNED: + { + /* when a timeout is set this is a "wait until" — poll until match */ + if (cmd->timeoutSeconds > 0) + { + if (!wait_for_state(r, cmd->service, cmd->state, + cmd->timeoutSeconds, + cmd->kind == CMD_ASSERT_ASSIGNED)) + { + snprintf(errBuf, errLen, + "timeout: %s %s never reached %s", + cmd->service, + cmd->kind == CMD_ASSERT_ASSIGNED + ? "assigned-state" : "state", + cmd->state); + return false; + } + return true; + } + + char reported[64] = "", assigned[64] = ""; + if (!monitor_get_node_state(r, cmd->service, + reported, sizeof(reported), + assigned, sizeof(assigned))) + { + snprintf(errBuf, errLen, + "cannot reach monitor to assert %s state", + cmd->service); + return false; + } + const char *actual = (cmd->kind == CMD_ASSERT_ASSIGNED) + ? assigned : reported; + if (strcmp(actual, cmd->state) != 0) + { + snprintf(errBuf, errLen, + "%s %s is \"%s\", expected \"%s\"", + cmd->service, + (cmd->kind == CMD_ASSERT_ASSIGNED) + ? "assigned-state" : "state", + actual, cmd->state); + return false; + } + return true; + } + + case CMD_SQL: + { + strlcpy(r->lastSqlService, cmd->service, sizeof(r->lastSqlService)); + r->lastSqlFailed = false; + r->lastSqlState[0] = '\0'; + r->lastSqlOutput[0] = '\0'; + + if (!exec_sql_on_service(r, cmd->service, cmd->args, + r->lastSqlOutput, + sizeof(r->lastSqlOutput))) + { + if (cmd->allowError) + { + /* + * Soft failure: expected by the following CMD_EXPECT_ERROR. + * Record the SQLSTATE and clear the output (it's an error + * message, not usable result rows). + */ + r->lastSqlFailed = true; + parse_sqlstate(r->lastSqlOutput, r->lastSqlState, + sizeof(r->lastSqlState)); + log_debug("sql on %s failed with SQLSTATE %s (expected)", + cmd->service, + r->lastSqlState[0] ? r->lastSqlState : "(unknown)"); + r->lastSqlOutput[0] = '\0'; + return true; + } + snprintf(errBuf, errLen, + "sql on %s failed:\n%s", cmd->service, r->lastSqlOutput); + return false; + } + log_debug("sql output: %s", r->lastSqlOutput); + return true; + } + + case CMD_EXPECT: + { + /* + * Substring match: the expect value is a pattern that must appear + * somewhere in the SQL output. This lets tests avoid depending on + * non-deterministic details like node IDs while still being specific + * enough to catch regressions. + */ + if (strstr(r->lastSqlOutput, cmd->expected) == NULL) + { + snprintf(errBuf, errLen, + "expected \"%s\", got \"%s\"", + cmd->expected, r->lastSqlOutput); + return false; + } + return true; + } + + case CMD_EXPECT_ERROR: + { + if (!r->lastSqlFailed) + { + snprintf(errBuf, errLen, + "expected SQL error on %s, but the query succeeded", + r->lastSqlService); + return false; + } + if (cmd->state[0] && + strcmp(r->lastSqlState, cmd->state) != 0) + { + snprintf(errBuf, errLen, + "expected SQLSTATE %s on %s, got %s", + cmd->state, r->lastSqlService, + r->lastSqlState[0] ? r->lastSqlState : "(unknown)"); + return false; + } + r->lastSqlFailed = false; /* consumed */ + return true; + } + + case CMD_NETWORK_OFF: + if (!runner_network_off(r, cmd->service)) + { + snprintf(errBuf, errLen, + "network disconnect %s failed", cmd->service); + return false; + } + return true; + + case CMD_NETWORK_ON: + if (!runner_network_on(r, cmd->service)) + { + snprintf(errBuf, errLen, + "network connect %s failed", cmd->service); + return false; + } + return true; + + case CMD_SLEEP: + sleep(cmd->timeoutSeconds); + return true; + + case CMD_COMPOSE_DOWN: + return runner_compose_down(r); + + case CMD_COMPOSE_START: + { + /* + * Use `up -d --no-recreate --no-deps` rather than `start`: + * `start` only works for already-created containers, but deferred + * nodes (launch deferred) are never created during compose up, + * so `start` would fail for them. `up -d --no-recreate --no-deps` + * handles both: creates+starts a new container, or starts an + * existing stopped one without touching other services. + * + * `--no-deps` is critical: without it, `start` (and `up`) would + * also start any stopped services listed in `depends_on`, which + * interferes with tests that restart a single node. + */ + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "%s up -d --no-recreate --no-deps %s 2>&1", + r->composeBase, cmd->service); + if (out[0]) + log_output(" compose: ", out); + if (rc != 0) + { + snprintf(errBuf, errLen, + "docker compose start %s failed (exit %d)", + cmd->service, rc); + return false; + } + return true; + } + + case CMD_COMPOSE_STOP: + { + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "%s stop %s 2>&1", + r->composeBase, cmd->service); + if (out[0]) + log_output(" compose: ", out); + if (rc != 0) + { + snprintf(errBuf, errLen, + "docker compose stop %s failed (exit %d)", + cmd->service, rc); + return false; + } + return true; + } + + case CMD_COMPOSE_KILL: + { + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "%s kill %s 2>&1", + r->composeBase, cmd->service); + if (out[0]) + log_output(" compose: ", out); + if (rc != 0) + { + snprintf(errBuf, errLen, + "docker compose kill %s failed (exit %d)", + cmd->service, rc); + return false; + } + return true; + } + + /* + * CMD_COMPOSE_INJECT: copy a file from a Docker image into a running + * service container without restarting it. + * + * compose inject pgaf:next /usr/local/bin/pg_autoctl monitor:/usr/local/bin/pg_autoctl + * + * Fields: expected=image args=src-path service=dst-svc state=dst-path + * + * Sequence: + * 1. docker create --name _pgaf_inject_tmp + * 2. docker cp _pgaf_inject_tmp: /tmp/_pgaf_inject_binary + * 3. docker rm _pgaf_inject_tmp + * 4. docker cp /tmp/_pgaf_inject_binary : + */ + case CMD_COMPOSE_INJECT: + { + char out[4096] = ""; + int rc; + + log_info("compose inject: creating temporary container from %s", + cmd->expected); + + rc = run_cmd_capture(out, sizeof(out), + "docker create --name _pgaf_inject_tmp %s 2>&1", + cmd->expected); + if (out[0]) + log_output(" inject: ", out); + if (rc != 0) + { + snprintf(errBuf, errLen, + "compose inject: docker create %s failed (exit %d)", + cmd->expected, rc); + return false; + } + + out[0] = '\0'; + rc = run_cmd_capture(out, sizeof(out), + "docker cp _pgaf_inject_tmp:%s /tmp/_pgaf_inject_binary 2>&1", + cmd->args); + if (out[0]) + log_output(" inject: ", out); + /* always remove the temp container, even on failure */ + run_cmd_capture(out, sizeof(out), + "docker rm _pgaf_inject_tmp 2>&1"); + if (rc != 0) + { + snprintf(errBuf, errLen, + "compose inject: docker cp from image failed (exit %d)", rc); + return false; + } + + char container[256]; + compose_container_name(r->projectName, cmd->service, + container, sizeof(container)); + + out[0] = '\0'; + rc = run_cmd_capture(out, sizeof(out), + "docker cp /tmp/_pgaf_inject_binary %s:%s 2>&1", + container, cmd->state); + if (out[0]) + log_output(" inject: ", out); + if (rc != 0) + { + snprintf(errBuf, errLen, + "compose inject: docker cp to %s:%s failed (exit %d)", + cmd->service, cmd->state, rc); + return false; + } + + log_info("compose inject: %s:%s → %s:%s", + cmd->expected, cmd->args, cmd->service, cmd->state); + return true; + } + + /* + * CMD_WAIT_MULTI: wait until all (node, state) pairs are satisfied. + * + * wait until node2 state is primary and node1 state is secondary + * with timeout 90s + * + * Polls each condition in turn; repeats until all are true or timeout. + */ + case CMD_WAIT_MULTI: + { + int timeoutSecs = cmd->timeoutSeconds; + time_t deadline = time(NULL) + timeoutSecs; + + /* build pointer arrays once — used for marking on every drain */ + const char *mn[PGAF_MAX_WAIT_STATES], *ms[PGAF_MAX_WAIT_STATES]; + for (int i = 0; i < cmd->waitStateCount; i++) + { + mn[i] = cmd->waitNodes[i]; + ms[i] = cmd->waitStates[i]; + } + + /* LISTEN-based satisfied[]: set by runner_drain_notify when a + * convergence notification arrives for a specific (node, state) pair */ + bool listenSatisfied[PGAF_MAX_WAIT_STATES] = {false}; + + while (time(NULL) < deadline) + { + /* + * Try subprocess-based check first. monitor_get_node_state runs + * "pg_autoctl inspect monitor node-state" which requires v2.2. + * When nodes run v2.1 the call returns false; track how many + * succeed so we can fall back to LISTEN-based convergence. + */ + bool allMet = true; + int subproc_ok = 0; + for (int i = 0; i < cmd->waitStateCount; i++) + { + char reported[64] = "", assigned[64] = ""; + if (!monitor_get_node_state(r, cmd->waitNodes[i], + reported, sizeof(reported), + assigned, sizeof(assigned))) + { + allMet = false; /* can't confirm via subprocess */ + /* don't break — try to drain anyway */ + continue; + } + subproc_ok++; + /* + * Require convergence: the node must have reported the + * target state (reportedState) and the monitor must agree + * it belongs there (assignedState / goalState). Checking + * only reportedState could catch a transient pass-through; + * checking only assignedState fires before the node confirms. + */ + if (strcmp(reported, cmd->waitStates[i]) != 0 || + strcmp(assigned, cmd->waitStates[i]) != 0) + allMet = false; + } + + /* + * Drain: log notifications, mark '*' on convergence events, and + * set listenSatisfied[i] when the (node, state) pair converges. + */ + runner_drain_notify(r, NULL, mn, ms, + cmd->waitStateCount, listenSatisfied); + + /* + * When subprocess is unavailable (v2.1 nodes), fall back to + * LISTEN-based convergence. If all conditions have been seen + * converged via NOTIFY, treat that as success. + */ + if (subproc_ok == 0) + { + bool allListenMet = true; + for (int i = 0; i < cmd->waitStateCount; i++) + if (!listenSatisfied[i]) { allListenMet = false; break; } + if (allListenMet) + return true; + } + else if (allMet) + return true; + + runner_wait_socket(r, 1000); + } + + /* build readable label for error */ + char label[512] = ""; + for (int i = 0; i < cmd->waitStateCount; i++) + { + if (i > 0) + strncat(label, " and ", sizeof(label)-strlen(label)-1); + char part[128]; + snprintf(part, sizeof(part), "%s=%s", + cmd->waitNodes[i], cmd->waitStates[i]); + strncat(label, part, sizeof(label)-strlen(label)-1); + } + snprintf(errBuf, errLen, + "timeout: conditions not met: %s", label); + return false; + } + + /* + * CMD_PG_AUTOCTL: run pg_autoctl locally in the pgaftest process. + * + * When running as a compose service, PG_AUTOCTL_MONITOR is set in the + * container environment (see compose_gen.c) so pg_autoctl finds the + * monitor URI automatically without any --monitor injection here. + * + * When running on the host (pgaftest run outside Docker), the variable + * must be set by the user or in their shell environment. + */ + case CMD_PG_AUTOCTL: + { + char out[4096] = ""; + int rc = run_cmd_capture(out, sizeof(out), + "pg_autoctl %s 2>&1", cmd->args); + if (rc != 0) + { + if (out[0]) + log_output(" ", out); + snprintf(errBuf, errLen, + "pg_autoctl %s failed (exit %d)", cmd->args, rc); + return false; + } + return true; + } + + case CMD_WAIT_STOPPED: + { + const char *svc = cmd->service; + int timeoutSecs = cmd->timeoutSeconds; + time_t deadline = time(NULL) + timeoutSecs; + + log_info(" Waiting for service %s to stop (timeout %ds)", + svc, timeoutSecs); + + while (time(NULL) < deadline) + { + /* + * `docker compose ps --format {{.State}} ` outputs the + * container state: "running", "exited", "paused", etc. + * An empty or non-running result also counts as stopped. + */ + char state[64] = ""; + int rc = run_cmd_capture(state, sizeof(state), + "%s ps --format '{{.State}}' %s" + " 2>/dev/null", + r->composeBase, svc); + + /* strip whitespace */ + int n = strlen(state); + while (n > 0 && (state[n-1] == '\n' || state[n-1] == ' ')) + state[--n] = '\0'; + + if (rc != 0 || state[0] == '\0' || + strcmp(state, "running") != 0) + { + log_info(" Service %s stopped (state: %s)", svc, + state[0] ? state : "gone"); + return true; + } + + usleep(500000); /* 500ms */ + } + + snprintf(errBuf, errLen, + "timeout: service %s did not stop within %ds", + svc, timeoutSecs); + return false; + } + + case CMD_STOP_POSTGRES: + { + /* + * Signal the pg_autoctl postgres-controller service to stop + * Postgres. Equivalent to calling `pg_autoctl manual service pgctl off`. + */ + log_info("Stopping Postgres on %s", cmd->service); + int rc = run_cmd( + "%s exec %s pg_autoctl manual service pgctl off" + " --pgdata /var/lib/postgres/pgaf", + r->composeBase, cmd->service); + if (rc != 0) + { + snprintf(errBuf, errLen, + "stop postgres %s failed (exit %d)", + cmd->service, rc); + return false; + } + return true; + } + + case CMD_START_POSTGRES: + { + log_info("Starting Postgres on %s", cmd->service); + int rc = run_cmd( + "%s exec %s pg_autoctl manual service pgctl on" + " --pgdata /var/lib/postgres/pgaf", + r->composeBase, cmd->service); + if (rc != 0) + { + snprintf(errBuf, errLen, + "start postgres %s failed (exit %d)", + cmd->service, rc); + return false; + } + return true; + } + + case CMD_STAYS_WHILE: + { + /* + * LISTEN-driven stays-while (item 9). + * + * Strategy: + * 1. Connect to LISTEN "state" channel. + * 2. Verify the node's current assigned-state matches expectation. + * 3. Flush any pending notifications (establish clean baseline). + * 4. Run each body command. + * 5. After each command, drain the notification queue and fail + * immediately if any notification for this node has goalState + * != expectedState. + * 6. Final SQL check after the body. + * + * The LISTEN approach catches state-change assignments that happen + * *during* a body command, not just between commands. + */ + log_info("assert %s stays %s while running body", + cmd->service, cmd->state); + + runner_notify_connect(r); + + /* initial assigned-state check */ + char curReported[64] = "", curAssigned[64] = ""; + if (!monitor_get_node_state(r, cmd->service, + curReported, sizeof(curReported), + curAssigned, sizeof(curAssigned))) + { + snprintf(errBuf, errLen, + "stays-while: could not get state of %s", cmd->service); + return false; + } + if (strcmp(curAssigned, cmd->state) != 0) + { + snprintf(errBuf, errLen, + "stays-while: %s assigned-state is %s, " + "expected %s (before body)", + cmd->service, curAssigned, cmd->state); + return false; + } + + /* flush any pre-existing notifications so we start from a clean slate */ + if (r->notifyConnected) + runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); + + /* run each body command, checking notifications after each */ + for (TestCmd *bc = cmd->body; bc; bc = bc->next) + { + if (!runner_exec_cmd(r, bc, errBuf, errLen)) + return false; + + /* check notifications: any goalState change for this node? */ + if (!runner_check_stays_notify(r, cmd->service, cmd->state, + errBuf, errLen)) + return false; + + /* also SQL-poll when LISTEN is unavailable */ + if (!r->notifyConnected) + { + if (!monitor_get_node_state(r, cmd->service, + curReported, sizeof(curReported), + curAssigned, sizeof(curAssigned))) + { + snprintf(errBuf, errLen, + "stays-while: could not get state of %s after command", + cmd->service); + return false; + } + if (strcmp(curAssigned, cmd->state) != 0) + { + snprintf(errBuf, errLen, + "stays-while: %s assigned-state changed to %s, " + "expected %s during body", + cmd->service, curAssigned, cmd->state); + return false; + } + } + } + + /* final check after all body commands */ + if (!runner_check_stays_notify(r, cmd->service, cmd->state, + errBuf, errLen)) + return false; + if (!r->notifyConnected) + { + if (!monitor_get_node_state(r, cmd->service, + curReported, sizeof(curReported), + curAssigned, sizeof(curAssigned)) || + strcmp(curAssigned, cmd->state) != 0) + { + snprintf(errBuf, errLen, + "stays-while: %s assigned-state is %s " + "after body, expected %s", + cmd->service, curAssigned, cmd->state); + return false; + } + } + return true; + } + + case CMD_SET_MONITOR: + { + /* + * Switch the active monitor service: drop the existing LISTEN + * connection (if any) and reconnect to the new monitor. All + * subsequent LISTEN/NOTIFY waits and monitor_get_node_state() + * calls will target the new service. + */ + log_info(" set monitor: switching to service \"%s\"", + cmd->service); + + if (r->notifyConnected) + { + pgsql_finish(&r->notifyConn); + r->notifyConnected = false; + } + + strlcpy(r->activeMonitorService, cmd->service, + sizeof(r->activeMonitorService)); + + /* reconnect immediately so wait loops can start right away */ + runner_notify_connect(r); + return true; + } + + case CMD_LOGS_CHECK: + { + /* + * Grep the container logs. + * cmd->allowError == false → fixed-string grep (-F) + * cmd->allowError == true → PCRE grep (-P) + * cmd->logsNegate → assert pattern NOT found + */ + bool pcre = cmd->allowError; + bool negate = cmd->logsNegate; + const char *flag = pcre ? "-qP" : "-qF"; + + /* Shell-quote the pattern by wrapping in single quotes. + * Replace any embedded ' with '\'' to safely handle patterns + * such as PCRE lookahead strings. + */ + char quotedPat[sizeof(cmd->args) * 4 + 4]; + { + char *q = quotedPat; + *q++ = '\''; + for (const char *p = cmd->args; *p; p++) + { + if (*p == '\'') + { + *q++ = '\''; *q++ = '\\'; *q++ = '\''; *q++ = '\''; + } + else + *q++ = *p; + } + *q++ = '\''; + *q = '\0'; + } + + char grepCmd[8192]; + snprintf(grepCmd, sizeof(grepCmd), + "docker compose --project-directory %s logs --no-color %s 2>&1 | grep %s %s", + r->workDir, + cmd->service, + flag, + quotedPat); + + log_info(" logs %s: %s%s \"%s\"", + cmd->service, + negate ? "not " : "", + pcre ? "matches" : "contains", + cmd->args); + + int rc = system(grepCmd); + bool found = (rc == 0); + + if (negate && found) + { + log_error("logs check failed: pattern found in %s logs: %s", + cmd->service, cmd->args); + return false; + } + else if (!negate && !found) + { + log_error("logs check failed: pattern not found in %s logs: %s", + cmd->service, cmd->args); + return false; + } + return true; + } + } + return true; +} + +/* ----------------------------------------------------------------------- + * Execute a step (list of commands) + * ----------------------------------------------------------------------- */ + +/* + * log_output prints each non-empty line of a captured command output block + * at INFO level, prefixed with `prefix`. Used to fold multi-line output + * from docker compose / docker network into the structured log stream. + */ +static void +log_output(const char *prefix, const char *out) +{ + char buf[4096]; + strlcpy(buf, out, sizeof(buf)); + for (char *line = buf, *nl; line && *line; line = nl ? nl + 1 : NULL) + { + nl = strchr(line, '\n'); + if (nl) *nl = '\0'; + /* skip blank / pure-whitespace lines */ + const char *p = line; + while (*p == ' ' || *p == '\t' || *p == '\r') p++; + if (*p) + log_info("%s%s", prefix, line); + } +} + +/* + * inline_text copies src into dst, collapsing runs of whitespace (including + * newlines) into a single space and trimming leading/trailing whitespace. + * Result is always NUL-terminated and fits in dstlen bytes. + */ +static void +inline_text(const char *src, char *dst, int dstlen) +{ + int di = 0; + bool space = true; /* suppress leading whitespace */ + for (const char *p = src; *p && di < dstlen - 1; p++) + { + if (*p == '\n' || *p == '\r' || *p == '\t' || *p == ' ') + { + if (!space && di < dstlen - 1) + dst[di++] = ' '; + space = true; + } + else + { + dst[di++] = *p; + space = false; + } + } + /* trim trailing space */ + while (di > 0 && dst[di-1] == ' ') + di--; + dst[di] = '\0'; +} + +/* + * cmd_label returns a human-readable description of a command matching the + * DSL syntax from the spec file. The caller owns the buffer. + */ +static void +cmd_label(const TestCmd *cmd, char *buf, int len) +{ + char tmp[512]; + + switch (cmd->kind) + { + case CMD_EXEC: + snprintf(buf, len, "exec %s %s", cmd->service, cmd->args); + break; + case CMD_EXEC_FAILS: + snprintf(buf, len, "exec-fails %s %s", cmd->service, cmd->args); + break; + case CMD_WAIT_STATE: + snprintf(buf, len, "wait until %s state = %s timeout %ds", + cmd->service, cmd->state, cmd->timeoutSeconds); + break; + case CMD_WAIT_STATES: + { + char states[256] = ""; + for (int i = 0; i < cmd->waitStateCount; i++) + { + if (i > 0) strncat(states, ", ", sizeof(states)-strlen(states)-1); + strncat(states, cmd->waitStates[i], sizeof(states)-strlen(states)-1); + } + snprintf(buf, len, "wait until %s timeout %ds", states, + cmd->timeoutSeconds); + break; + } + case CMD_ASSERT_STATE: + snprintf(buf, len, "assert %s state = %s", cmd->service, cmd->state); + break; + case CMD_ASSERT_ASSIGNED: + snprintf(buf, len, "assert %s assigned-state = %s", + cmd->service, cmd->state); + break; + case CMD_PROMOTE: + { + char nodes[256] = ""; + for (int i = 0; i < cmd->promoteCount; i++) + { + if (i > 0) strncat(nodes, ", ", sizeof(nodes)-strlen(nodes)-1); + strncat(nodes, cmd->promoteNodes[i], + sizeof(nodes)-strlen(nodes)-1); + } + snprintf(buf, len, "promote %s", nodes[0] ? nodes : cmd->service); + break; + } + case CMD_SQL: + inline_text(cmd->args, tmp, sizeof(tmp)); + snprintf(buf, len, "sql %s { %s }", cmd->service, tmp); + break; + case CMD_EXPECT: + if (strchr(cmd->expected, '\n')) + { + /* multi-row: reconstruct { row } { row } display form */ + char rows[256] = { 0 }; + int rpos = 0; + const char *p = cmd->expected; + while (*p) + { + const char *nl = strchr(p, '\n'); + int rowlen = nl ? (int)(nl - p) : (int)strlen(p); + int n = snprintf(rows + rpos, sizeof(rows) - rpos, + "%s{ %.*s }", + rpos ? " " : "", rowlen, p); + if (n > 0) rpos += n; + p = nl ? nl + 1 : p + rowlen; + } + snprintf(buf, len, "expect { %s }", rows); + } + else + { + inline_text(cmd->expected, tmp, sizeof(tmp)); + snprintf(buf, len, "expect { %s }", tmp); + } + break; + case CMD_EXPECT_ERROR: + snprintf(buf, len, "expect error %s", cmd->state); + break; + case CMD_NETWORK_OFF: + snprintf(buf, len, "network disconnect %s", cmd->service); + break; + case CMD_NETWORK_ON: + snprintf(buf, len, "network connect %s", cmd->service); + break; + case CMD_SLEEP: + snprintf(buf, len, "sleep %ds", cmd->timeoutSeconds); + break; + case CMD_COMPOSE_DOWN: + snprintf(buf, len, "compose down"); + break; + case CMD_COMPOSE_START: + snprintf(buf, len, "compose start %s", cmd->service); + break; + case CMD_COMPOSE_STOP: + snprintf(buf, len, "compose stop %s", cmd->service); + break; + case CMD_COMPOSE_KILL: + snprintf(buf, len, "compose kill %s", cmd->service); + break; + case CMD_COMPOSE_INJECT: + snprintf(buf, len, "compose inject %s %s %s:%s", + cmd->expected, cmd->args, cmd->service, cmd->state); + break; + case CMD_WAIT_STOPPED: + snprintf(buf, len, "wait until %s stopped timeout %ds", + cmd->service, cmd->timeoutSeconds); + break; + case CMD_WAIT_MULTI: + { + char parts[512] = ""; + for (int i = 0; i < cmd->waitStateCount; i++) + { + if (i > 0) strncat(parts, " and ", sizeof(parts)-strlen(parts)-1); + char piece[128]; + snprintf(piece, sizeof(piece), "%s state = %s", + cmd->waitNodes[i], cmd->waitStates[i]); + strncat(parts, piece, sizeof(parts)-strlen(parts)-1); + } + snprintf(buf, len, "wait until %s timeout %ds", parts, + cmd->timeoutSeconds); + break; + } + case CMD_PG_AUTOCTL: + snprintf(buf, len, "pg_autoctl %s", cmd->args); + break; + case CMD_STOP_POSTGRES: + snprintf(buf, len, "stop postgres %s", cmd->service); + break; + case CMD_START_POSTGRES: + snprintf(buf, len, "start postgres %s", cmd->service); + break; + case CMD_STAYS_WHILE: + snprintf(buf, len, "assert %s stays %s while { ... }", + cmd->service, cmd->state); + break; + case CMD_SET_MONITOR: + snprintf(buf, len, "set monitor %s", cmd->service); + break; + case CMD_LOGS_CHECK: + snprintf(buf, len, "logs %s %s%s \"%s\"", + cmd->service, + cmd->logsNegate ? "not " : "", + cmd->allowError ? "matches" : "contains", + cmd->args); + break; + default: + snprintf(buf, len, "(unknown command %d)", cmd->kind); + break; + } +} + +static bool +runner_exec_step(TestRunner *r, TestStep *step, char *errBuf, int errLen, + int stepNum) +{ + int cmdNum = 0; + for (TestCmd *cmd = step->commands; cmd; cmd = cmd->next) + { + /* + * Flush all notifications that arrived during the previous command. + * Loop until the socket is idle for 50 ms so we catch notifications + * still in-flight in the TCP stream, not just what libpq buffered. + */ + if (r->notifyConnected) + { + while (runner_wait_socket(r, 50)) + runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); + runner_drain_notify(r, NULL, NULL, NULL, 0, NULL); /* one last sweep of the libpq buffer */ + } + + char label[256]; + cmd_label(cmd, label, sizeof(label)); + if (stepNum > 0) + log_info(" [%02d-%02d] %s", stepNum, ++cmdNum, label); + else + log_info(" [%02d] %s", ++cmdNum, label); + if (!runner_exec_cmd(r, cmd, errBuf, errLen)) + return false; + } + return true; +} + +/* ----------------------------------------------------------------------- + * Load runner state from workDir (for step/down subcommands) + * ----------------------------------------------------------------------- */ + +static bool +runner_load_state(TestRunner *r) +{ + /* Check the compose file exists — if it does, the stack should be up */ + struct stat st; + if (stat(r->composeFile, &st) != 0) + { + log_error("No compose file found at \"%s\"; run `pgaftest setup` first", + r->composeFile); + return false; + } + r->composeUp = true; + return true; +} + +/* ----------------------------------------------------------------------- + * Monitor readiness ping loop + * + * When running inside the compose network (PGAFTEST_COMPOSE_SERVICE=1) + * the pgaftest container starts at the same time as the monitor — no + * depends_on ordering. We spin here until the monitor postgres accepts + * connections, then open the LISTEN channel. + * ----------------------------------------------------------------------- */ + +#define MONITOR_WAIT_TIMEOUT_SECS 120 + +static bool +runner_wait_for_monitor(TestRunner *r) +{ + if (!r->spec->cluster.withMonitor) + return true; + + log_info("Waiting for monitor to become ready (timeout %ds)", + MONITOR_WAIT_TIMEOUT_SECS); + + time_t deadline = time(NULL) + MONITOR_WAIT_TIMEOUT_SECS; + + char monitorReadyCmd[sizeof(r->composeBase) + 128]; + snprintf(monitorReadyCmd, sizeof(monitorReadyCmd), + "%s exec -T monitor psql -U autoctl_node -d pg_auto_failover " + "-c 'SELECT 1' >/dev/null 2>&1", + r->composeBase); + + while (time(NULL) < deadline) + { + if (runner_notify_connect(r)) + { + log_info("Monitor is ready; LISTEN channel open"); + return true; + } + + /* + * If the direct libpq connection can't be established (e.g. the + * host IP is not in the monitor's pg_hba.conf — common when the + * monitor was initialised by an older pg_autoctl version that only + * added the Docker network CIDR), fall back to a subprocess check + * via docker compose exec. Once the monitor responds to psql we + * return true; wait loops will use subprocess polling instead of + * LISTEN/NOTIFY. + */ + if (run_cmd("%s", monitorReadyCmd) == 0) + { + log_info("Monitor is ready (subprocess check; LISTEN not available)"); + return true; + } + + pg_usleep(500 * 1000); /* 0.5 s between attempts */ + } + + log_error("Timed out waiting for monitor to become ready after %ds", + MONITOR_WAIT_TIMEOUT_SECS); + return false; +} + + +/* ----------------------------------------------------------------------- + * Public API + * ----------------------------------------------------------------------- */ + +bool +runner_run(TestSpec *spec, const char *workDir, bool noCleanup) +{ + TestRunner r; + runner_init(&r, spec, workDir); + + if (!runner_compose_generate(&r)) + return false; + + /* + * HOST path: start the compose stack, then run tests directly from the host + * using docker-compose exec for all node/monitor interactions. + * + * The compose file does not include a pgaftest service in host mode — the + * current process IS the test runner and has docker access already. + * + * When PGAFTEST_COMPOSE_SERVICE is set the compose file was generated to + * include a pgaftest service and we're running inside that container; skip + * compose up since the stack is already running. + */ + if (!getenv("PGAFTEST_COMPOSE_SERVICE")) + { + log_info("Starting compose stack (project: %s)", r.projectName); + + /* + * Remove any leftover stack from a previous --no-cleanup run so we + * always start with clean volumes. + */ + run_cmd("%s down --volumes --remove-orphans 2>&1", r.composeBase); + + int up_rc = run_cmd("%s up --build -d 2>&1", r.composeBase); + if (up_rc != 0) + { + log_error("docker compose up failed (exit %d)", up_rc); + run_cmd("%s down --volumes 2>&1", r.composeBase); + return false; + } + } + + r.composeUp = true; + + /* + * If no sequence{} block was written, run steps in definition order. + * This lets simple test files omit the redundant sequence block entirely. + */ + if (spec->sequenceLength == 0) + { + for (TestStep *s = spec->steps; s; s = s->next) + { + if (spec->sequenceLength < PGAF_MAX_SEQ) + spec->sequence[spec->sequenceLength++] = s->name; + } + } + + r.tapTotal = spec->sequenceLength; + + if (!runner_wait_for_monitor(&r)) + return false; + + if (!runner_apply_formation_settings(&r)) + return false; + + /* setup{} */ + if (spec->setup) + { + char err[512] = ""; + log_info("Running setup block"); + if (!runner_exec_step(&r, spec->setup, err, sizeof(err), 0)) + { + tap_plan(&r); + tap_diag("setup failed: %s", err); + return false; + } + } + + /* sequence — fail fast: first failure stops the run */ + bool allPassed = true; + for (int i = 0; i < spec->sequenceLength; i++) + { + const char *name = spec->sequence[i]; + TestStep *step = spec_find_step(spec, name); + if (!step) + { + tap_not_ok(&r, name, "step not found in spec"); + allPassed = false; + break; + } + + log_info("STEP %d: %s", i + 1, name); + char err[512] = ""; + if (runner_exec_step(&r, step, err, sizeof(err), i + 1)) + tap_ok(&r, name); + else + { + tap_not_ok(&r, name, err); + allPassed = false; + break; + } + } + + tap_plan(&r); + + /* teardown{} — always runs */ + if (spec->teardown) + { + char err[512] = ""; + log_info("Running teardown block"); + runner_exec_step(&r, spec->teardown, err, sizeof(err), 0); + } + + /* compose lifecycle is owned by the host — do not call compose_down here */ + return allPassed; +} + +bool +runner_setup(TestSpec *spec, const char *workDir, bool withTmux) +{ + TestRunner r; + runner_init(&r, spec, workDir); + + if (!runner_compose_generate(&r)) + return false; + + if (!runner_compose_up(&r)) + return false; + + if (!runner_wait_for_monitor(&r)) + { + runner_compose_down(&r); + return false; + } + + if (!runner_apply_formation_settings(&r)) + { + runner_compose_down(&r); + return false; + } + + /* run setup{} block */ + if (spec->setup) + { + char err[512] = ""; + log_info("Running setup block"); + if (!runner_exec_step(&r, spec->setup, err, sizeof(err), 0)) + { + log_error("Setup failed: %s", err); + runner_compose_down(&r); + return false; + } + } + + if (withTmux) + { + /* Launch tmux with compose logs + pg_autoctl watch */ + log_info("Starting tmux session for project %s", r.projectName); + run_cmd( + "tmux new-session -d -s %s " + "\"%s logs -f\" \\; " + "split-window -v " + "\"%s exec monitor pg_autoctl watch\" \\; " + "split-window -v \\; " + "select-layout even-vertical", + r.projectName, + r.composeBase, + r.composeBase); + + printf("Cluster ready. Attach with: tmux attach -t %s\n", + r.projectName); + printf("Tear down with: pgaftest down --work-dir %s\n", workDir); + } + else + { + printf("\nCluster ready — compose project: %s\n", r.projectName); + printf("Work dir: %s\n", workDir); + printf("\nAvailable steps:"); + for (TestStep *s = spec->steps; s; s = s->next) + printf(" %s", s->name); + printf("\n\nRun a step: pgaftest step --work-dir %s\n", workDir); + printf("Tear down: pgaftest down --work-dir %s\n\n", workDir); + } + + return true; +} + +bool +runner_step(TestSpec *spec, const char *workDir, const char *stepName) +{ + TestRunner r; + runner_init(&r, spec, workDir); + + if (!runner_load_state(&r)) + return false; + + TestStep *step = spec_find_step(spec, stepName); + if (!step) + { + log_error("Step \"%s\" not found in spec", stepName); + return false; + } + + char err[512] = ""; + if (!runner_exec_step(&r, step, err, sizeof(err), 0)) + { + log_error("Step \"%s\" failed: %s", stepName, err); + return false; + } + + log_info("Step \"%s\" passed", stepName); + return true; +} + +bool +runner_down(TestSpec *spec, const char *workDir) +{ + TestRunner r; + runner_init(&r, spec, workDir); + runner_load_state(&r); /* best-effort */ + + /* teardown{} */ + if (spec->teardown) + { + char err[512] = ""; + runner_exec_step(&r, spec->teardown, err, sizeof(err), 0); + } + + return runner_compose_down(&r); +} + +bool +runner_show(TestSpec *spec) +{ + TestRunner r; + runner_init(&r, spec, "/tmp/pgaftest-show"); + + /* write to stdout instead of a file */ + /* No specFile for show — omit the pgaftest service */ + bool ok = compose_gen_write(&spec->cluster, "/dev/stdout", + r.projectName, r.contextDir, NULL); + return ok; +} + +/* + * runner_prepare writes docker-compose.yml, per-node .ini files, and a + * Makefile into outDir, then prints the docker compose command to stdout. + * + * The Makefile has two phony targets: + * test: docker compose up --exit-code-from pgaftest + * down: docker compose down --volumes --remove-orphans + */ +bool +runner_prepare(TestSpec *spec, const char *outDir) +{ + TestRunner r; + + /* If outDir not given, derive from spec name next to spec file */ + char derivedDir[1024]; + if (!outDir || outDir[0] == '\0') + { + const char *base = strrchr(spec->filename, '/'); + base = base ? base + 1 : spec->filename; + + /* strip .pgaf extension for the directory name */ + char stem[256]; + strlcpy(stem, base, sizeof(stem)); + char *dot = strrchr(stem, '.'); + if (dot && strcmp(dot, ".pgaf") == 0) *dot = '\0'; + + /* place it beside the spec file */ + const char *slash = strrchr(spec->filename, '/'); + if (slash) + snprintf(derivedDir, sizeof(derivedDir), "%.*s/%s-compose", + (int)(slash - spec->filename), spec->filename, stem); + else + snprintf(derivedDir, sizeof(derivedDir), "%s-compose", stem); + outDir = derivedDir; + } + + /* Create the output directory */ + if (mkdir(outDir, 0755) != 0 && errno != EEXIST) + { + log_error("mkdir \"%s\": %m", outDir); + return false; + } + + runner_init(&r, spec, outDir); + + /* Generate SSL certs if the cluster ssl mode requires them (verify-ca/full) */ + if (!compose_gen_write_ssl_certs(&spec->cluster, outDir)) + return false; + + /* Write docker-compose.yml (with pgaftest service) */ + if (!compose_gen_write(&spec->cluster, r.composeFile, + r.projectName, r.contextDir, r.specFile)) + return false; + + /* Write monitor.ini */ + if (spec->cluster.withMonitor) + { + if (!compose_gen_write_monitor_ini(&spec->cluster, outDir)) + return false; + } + + if (!compose_gen_write_second_monitor_ini(&spec->cluster, outDir)) + return false; + + /* Write per-node ini files */ + int globalNodeId = 0; + for (int fi = 0; fi < spec->cluster.formationCount; fi++) + { + const TestFormation *form = &spec->cluster.formations[fi]; + for (int ni = 0; ni < form->nodeCount; ni++) + { + if (!compose_gen_write_node_ini(&spec->cluster, form, + &form->nodes[ni], + ++globalNodeId, outDir)) + return false; + } + } + + /* Write Makefile */ + char makefilePath[1280]; + snprintf(makefilePath, sizeof(makefilePath), "%s/Makefile", outDir); + FILE *mf = fopen(makefilePath, "w"); + if (!mf) + { + log_error("Failed to open \"%s\": %m", makefilePath); + return false; + } + fprintf(mf, + ".PHONY: test down\n" + "\n" + "test:\n" + "\tdocker compose -p %s -f docker-compose.yml" + " up --build --exit-code-from pgaftest --attach pgaftest\n" + "\n" + "down:\n" + "\tdocker compose -p %s -f docker-compose.yml" + " down --volumes --remove-orphans\n", + r.projectName, r.projectName); + fclose(mf); + log_info("Wrote Makefile to \"%s\"", makefilePath); + + /* Print the docker compose command to stdout */ + printf("cd %s && \\\n", outDir); + printf(" docker compose -p %s -f docker-compose.yml" + " up --build --exit-code-from pgaftest --attach pgaftest\n", + r.projectName); + + return true; +} diff --git a/src/bin/pgaftest/test_runner.h b/src/bin/pgaftest/test_runner.h new file mode 100644 index 000000000..b17d858f7 --- /dev/null +++ b/src/bin/pgaftest/test_runner.h @@ -0,0 +1,99 @@ +/* + * src/bin/pgaftest/test_runner.h + * Test execution engine for .pgaf specs. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef TEST_RUNNER_H +#define TEST_RUNNER_H + +#include +#include "test_spec.h" +#include "pgsql.h" + +/* Runtime context for a running test */ +typedef struct TestRunner +{ + TestSpec *spec; + char projectName[64]; /* docker compose project name */ + char composeFile[1024]; /* path to generated compose YAML */ + char composeBase[1152]; /* "docker compose -p name [-f file]" */ + char workDir[1024]; /* temp dir for this run */ + char contextDir[1024]; /* absolute path used as docker build context */ + char specFile[1024]; /* absolute path to the .pgaf spec file */ + + /* TAP counters and buffered output */ + int tapTotal; + int tapPass; + int tapFail; + char tapBuffer[65536]; + int tapBufferLen; + + /* last SQL result (for CMD_EXPECT / CMD_EXPECT_ERROR) */ + char lastSqlOutput[4096]; + bool lastSqlFailed; /* true when last sql raised an error */ + char lastSqlState[8]; /* SQLSTATE from last failed sql */ + char lastSqlService[64];/* service name of last sql command */ + + bool composeUp; /* compose stack is running */ + + /* + * Direct libpq connection to the monitor's exposed postgres port. + * Used for LISTEN "state" notifications so wait loops are event-driven + * instead of polling via docker exec. + */ + PGSQL notifyConn; + bool notifyConnected; + + /* + * Which monitor service the runner currently targets for LISTEN/NOTIFY + * and monitor_get_node_state() queries. Defaults to "monitor"; changed + * by the "set monitor " DSL command in a replace-monitor test. + */ + char activeMonitorService[64]; +} TestRunner; + +/* + * CI mode: run the full spec (setup → sequence → teardown). + * Emits TAP to stdout. Returns true if all steps passed. + */ +/* + * noCleanup: when true, skip `compose down` after the run so the stack stays + * up for post-mortem inspection. Use `pgaftest down ` to clean up. + */ +bool runner_run(TestSpec *spec, const char *workDir, bool noCleanup); + +/* + * Interactive mode: bring up compose, run setup{}, then stop. + * The caller gets the running cluster to explore interactively. + * If withTmux is true, launch a tmux session instead of returning. + */ +bool runner_setup(TestSpec *spec, const char *workDir, bool withTmux); + +/* + * Run a single named step against an already-running compose stack. + * Used by `pgaftest step `. + */ +bool runner_step(TestSpec *spec, const char *workDir, const char *stepName); + +/* + * Tear down the compose stack (run teardown{} then compose down). + * Used by `pgaftest down`. + */ +bool runner_down(TestSpec *spec, const char *workDir); + +/* + * Print the generated docker-compose.yml without starting anything. + */ +bool runner_show(TestSpec *spec); + +/* + * Prepare an output directory with docker-compose.yml, *.ini files, and a + * Makefile. Prints the `docker compose up` command to stdout. + * outDir is created if it does not exist. Pass NULL to derive from spec name. + */ +bool runner_prepare(TestSpec *spec, const char *outDir); + +#endif /* TEST_RUNNER_H */ diff --git a/src/bin/pgaftest/test_spec.h b/src/bin/pgaftest/test_spec.h new file mode 100644 index 000000000..9440e28e5 --- /dev/null +++ b/src/bin/pgaftest/test_spec.h @@ -0,0 +1,248 @@ +/* + * src/bin/pgaftest/test_spec.h + * AST types for the .pgaf test specification format. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef TEST_SPEC_H +#define TEST_SPEC_H + +#include +#include "pgsetup.h" /* PgInstanceKind */ + +#define PGAF_MAX_NODES 32 +#define PGAF_MAX_FORMATIONS 16 +#define PGAF_MAX_NODE_VOLUMES 8 +#define PGAF_MAX_STEPS 256 +#define PGAF_MAX_SEQ 256 +#define PGAF_TIMEOUT_DEFAULT 90 + +/* ----------------------------------------------------------------------- + * Cluster topology (from the cluster { } block) + * + * Hierarchy: cluster → monitor + formations → nodes + * + * Syntax: + * + * cluster { + * monitor [port 15432] + * image "pg_auto_failover:pg17" # optional; default: build from src + * ssl self-signed # optional; default: self-signed + * auth trust # optional; default: trust + * + * formation [name] [num-sync N] { + * node1 + * node2 + * node3 async candidate-priority 0 + * coord coordinator + * w1 worker group 1 + * } + * + * # Multiple formations (Citus use case) + * formation workers num-sync 1 { + * w1 worker group 1 + * w2 worker group 1 + * } + * } + * + * When "formation" has no name it defaults to "default". + * When a formation block is omitted entirely the cluster is invalid. + * ----------------------------------------------------------------------- */ + +typedef struct TestNode +{ + char name[128]; + PgInstanceKind kind; /* standalone / coordinator / worker */ + int group; /* Citus group id; 0 = coordinator */ + int candidatePriority; /* 0-100, default 50 */ + bool replicationQuorum; /* participates in sync quorum */ + bool async; /* async standby (replicationQuorum = false) */ + + /* create-time options — passed to pg_autoctl create via [options] ini */ + bool noMonitor; /* --no-monitor: standalone node */ + bool launchDeferred; /* node waits for pg_autoctl node start */ + bool listen; /* --listen 0.0.0.0: bind all interfaces */ + bool citusSecondary; /* --citus-secondary */ + char citusClusterName[64]; /* --citus-cluster-name NAME */ + int pgPort; /* --pg-port N (0 = default 5432) */ + char debianCluster[64]; /* --debian-cluster NAME */ + char ssl[32]; /* per-node ssl override; "" = use cluster */ + char auth[32]; /* per-node auth override; "" = use cluster */ + char replicationPassword[256]; /* replication.password written to node ini */ + char monitorPassword[256]; /* pg_auto_failover.monitor_password written to node ini */ + + /* Extra Docker named volumes: volume */ + struct { char name[64]; char path[256]; } volumes[PGAF_MAX_NODE_VOLUMES]; + int volumeCount; +} TestNode; + +typedef struct TestFormation +{ + char name[128]; /* formation name; default "default" */ + int numSync; /* number-sync-standbys, -1 = unset */ + TestNode nodes[PGAF_MAX_NODES]; + int nodeCount; +} TestFormation; + +typedef struct TestCluster +{ + TestFormation formations[PGAF_MAX_FORMATIONS]; + int formationCount; + + bool withMonitor; /* true when "monitor" keyword appears in cluster{} */ + bool withCitus; + bool bindSource; /* bind-source: mount repo root → /usr/src/pg_auto_failover */ + + /* cluster-level Docker / network options */ + char image[256]; /* Docker image tag; "" = build from source */ + char ssl[32]; /* self-signed | cert | off; default self-signed */ + char auth[32]; /* trust | md5 | scram; default trust */ + char monitorPassword[256]; /* password for autoctl_node; embedded in monitor URI */ + int monitorHostPort; /* random free host port mapped to monitor:5432 */ + char extensionVersion[64]; /* PG_AUTOCTL_EXTENSION_VERSION env var for monitor */ + + /* + * Optional second monitor, used for replace-monitor tests. + * When set, compose_gen emits a second monitor service with the given name + * and (if initiallyStoped) stops it after compose up so that the test can + * start it at the right moment. + */ + char secondMonitorName[64]; /* "" when no second monitor */ + bool secondMonitorStopped; /* true when declared with "launch deferred" */ + int secondMonitorHostPort; /* random free host port for second monitor */ + + char monitorDebianCluster[64]; /* debian-cluster name for the monitor, "" otherwise */ + char monitorImageTarget[64]; /* Dockerfile build target for monitor; "" = "run" */ +} TestCluster; + +/* ----------------------------------------------------------------------- + * Commands (inside step/setup/teardown blocks) + * ----------------------------------------------------------------------- */ + +/* Maximum number of states in a multi-state wait */ +#define PGAF_MAX_WAIT_STATES 8 +/* Maximum number of nodes in a promote list */ +#define PGAF_MAX_PROMOTE_NODES 16 +/* Maximum number of groups in a formation wait */ +#define PGAF_MAX_WAIT_GROUPS 16 + +typedef enum TestCmdKind +{ + CMD_EXEC, /* exec */ + CMD_EXEC_FAILS, /* exec-fails */ + CMD_WAIT_STATE, /* wait until state = [timeout Ns] */ + CMD_WAIT_STATES, /* wait until s1, s2 [in group N,...] [timeout Ns] */ + CMD_ASSERT_STATE, /* assert state = */ + CMD_ASSERT_ASSIGNED, /* assert assigned-state = */ + CMD_PROMOTE, /* promote node1 [, node2, ...] */ + CMD_SQL, /* sql { SQL } */ + CMD_EXPECT, /* expect { text } */ + CMD_EXPECT_ERROR, /* expect error [SQLSTATE] */ + CMD_NETWORK_OFF, /* network disconnect */ + CMD_NETWORK_ON, /* network connect */ + CMD_SLEEP, /* sleep Ns */ + CMD_COMPOSE_DOWN, /* compose down */ + CMD_COMPOSE_START, /* compose start */ + CMD_COMPOSE_STOP, /* compose stop */ + CMD_COMPOSE_KILL, /* compose kill (SIGKILL, no grace) */ + CMD_WAIT_STOPPED, /* wait until stopped [timeout Ns] */ + CMD_WAIT_MULTI, /* wait until n1 state is s1 and n2 state is s2 */ + CMD_PG_AUTOCTL, /* pg_autoctl — runs in pgaftest container */ + CMD_STOP_POSTGRES, /* stop postgres — pg_autoctl manual pgctl off */ + CMD_START_POSTGRES, /* start postgres — pg_autoctl manual pgctl on */ + CMD_STAYS_WHILE, /* assert stays while { cmds } */ + CMD_SET_MONITOR, /* set monitor — switch active monitor service */ + CMD_LOGS_CHECK, /* logs [not] — grep container logs */ + CMD_COMPOSE_INJECT, /* compose inject : */ +} TestCmdKind; + +typedef struct TestCmd +{ + TestCmdKind kind; + char service[64]; /* target service / node name */ + char args[4096]; /* exec args or SQL text */ + char state[64]; /* expected state / SQLSTATE */ + char expected[4096]; /* for CMD_EXPECT */ + int timeoutSeconds; /* for CMD_WAIT_STATE / CMD_WAIT_STATES */ + bool allowError; /* CMD_SQL: don't fail if SQL errors */ + + /* CMD_WAIT_STATES: list of states that must ALL be present */ + char waitStates[PGAF_MAX_WAIT_STATES][64]; + int waitStateCount; + /* CMD_WAIT_STATES: group filter; -1 = all groups */ + int waitGroups[PGAF_MAX_WAIT_GROUPS]; + int waitGroupCount; /* 0 means no group filter (all groups) */ + + /* CMD_WAIT_MULTI: per-node conditions (parallel arrays) */ + char waitNodes[PGAF_MAX_WAIT_STATES][64]; + /* waitStates[] re-used for the per-node state; waitStateCount is the count */ + + /* CMD_PROMOTE: list of nodes to make primary in their respective groups */ + char promoteNodes[PGAF_MAX_PROMOTE_NODES][64]; + int promoteCount; + + /* + * CMD_WAIT_STATE / CMD_WAIT_MULTI: + * Optional list of intermediate states the node must pass through before + * reaching the target state. Checked via LISTEN/NOTIFY on the monitor. + */ + char passThroughStates[PGAF_MAX_WAIT_STATES][64]; + int passThroughCount; + + /* CMD_STAYS_WHILE: nested command list (the "while { }" body) */ + struct TestCmd *body; + + /* CMD_LOGS_CHECK */ + bool logsNegate; /* true → assert pattern NOT found */ + + struct TestCmd *next; +} TestCmd; + +/* ----------------------------------------------------------------------- + * Named step + * ----------------------------------------------------------------------- */ + +typedef struct TestStep +{ + char name[128]; + TestCmd *commands; /* linked list */ + struct TestStep *next; +} TestStep; + +/* ----------------------------------------------------------------------- + * Full test specification + * ----------------------------------------------------------------------- */ + +typedef struct TestSpec +{ + char filename[1024]; + + TestCluster cluster; + + TestStep *setup; /* single anonymous step */ + TestStep *teardown; /* single anonymous step */ + + TestStep *steps; /* named steps, linked list */ + int stepCount; + + char *sequence[PGAF_MAX_SEQ]; /* ordered step names */ + int sequenceLength; +} TestSpec; + +/* ----------------------------------------------------------------------- + * Parser entry point (implemented by bison grammar) + * ----------------------------------------------------------------------- */ + +extern TestSpec *parse_test_spec(const char *filename); + +/* ----------------------------------------------------------------------- + * Helpers + * ----------------------------------------------------------------------- */ + +TestStep *spec_find_step(TestSpec *spec, const char *name); +TestCmd *make_cmd(TestCmdKind kind); +TestStep *make_step(const char *name); + +#endif /* TEST_SPEC_H */ diff --git a/src/bin/pgaftest/test_spec_parse.c b/src/bin/pgaftest/test_spec_parse.c new file mode 100644 index 000000000..05c63d582 --- /dev/null +++ b/src/bin/pgaftest/test_spec_parse.c @@ -0,0 +1,3634 @@ +/* A Bison parser, made by GNU Bison 2.3. */ + +/* Skeleton implementation for Bison's Yacc-like parsers in C + + Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 + Free Software Foundation, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +/* Identify Bison output. */ +#define YYBISON 1 + +/* Bison version. */ +#define YYBISON_VERSION "2.3" + +/* Skeleton name. */ +#define YYSKELETON_NAME "yacc.c" + +/* Pure parsers. */ +#define YYPURE 0 + +/* Using locations. */ +#define YYLSP_NEEDED 0 + + + +/* Tokens. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + /* Put the tokens into the symbol table, so that GDB and other debuggers + know about them. */ + enum yytokentype { + T_CLUSTER = 258, + T_MONITOR = 259, + T_NODE = 260, + T_CITUS_COORDINATOR = 261, + T_CITUS_WORKER = 262, + T_SETUP = 263, + T_TEARDOWN = 264, + T_STEP = 265, + T_SEQUENCE = 266, + T_EQUALS = 267, + T_IMAGE = 268, + T_IMAGE_TARGET = 269, + T_SSL = 270, + T_AUTH = 271, + T_AUTH_METHOD = 272, + T_FORMATION = 273, + T_NUM_SYNC = 274, + T_COORDINATOR = 275, + T_WORKER = 276, + T_ASYNC = 277, + T_NO_MONITOR = 278, + T_LAUNCH = 279, + T_DEFERRED = 280, + T_IMMEDIATE = 281, + T_INITIALLY = 282, + T_VOLUME = 283, + T_LISTEN = 284, + T_CITUS_SECONDARY = 285, + T_CANDIDATE_PRIORITY = 286, + T_PORT = 287, + T_PASSWORD = 288, + T_MONITOR_PASSWORD = 289, + T_CITUS_CLUSTER_NAME = 290, + T_DEBIAN_CLUSTER = 291, + T_REPLICATION_QUORUM = 292, + T_REPLICATION_PASSWORD = 293, + T_EXTENSION_VERSION = 294, + T_BIND_SOURCE = 295, + T_FS_INIT = 296, + T_FS_SINGLE = 297, + T_FS_PRIMARY = 298, + T_FS_WAIT_PRIMARY = 299, + T_FS_WAIT_STANDBY = 300, + T_FS_DEMOTED = 301, + T_FS_DEMOTE_TIMEOUT = 302, + T_FS_DRAINING = 303, + T_FS_SECONDARY = 304, + T_FS_CATCHINGUP = 305, + T_FS_PREP_PROMOTION = 306, + T_FS_STOP_REPLICATION = 307, + T_FS_MAINTENANCE = 308, + T_FS_JOIN_PRIMARY = 309, + T_FS_APPLY_SETTINGS = 310, + T_FS_PREPARE_MAINTENANCE = 311, + T_FS_WAIT_MAINTENANCE = 312, + T_FS_REPORT_LSN = 313, + T_FS_FAST_FORWARD = 314, + T_FS_JOIN_SECONDARY = 315, + T_FS_DROPPED = 316, + T_EXEC = 317, + T_EXEC_FAILS = 318, + T_PG_AUTOCTL = 319, + T_WAIT = 320, + T_UNTIL = 321, + T_TIMEOUT = 322, + T_AND = 323, + T_IS = 324, + T_WITH = 325, + T_ASSERT = 326, + T_SQL = 327, + T_EXPECT = 328, + T_ERROR = 329, + T_PROMOTE = 330, + T_NETWORK = 331, + T_DISCONNECT = 332, + T_CONNECT = 333, + T_SLEEP = 334, + T_COMPOSE = 335, + T_DOWN = 336, + T_START = 337, + T_STOP = 338, + T_STOPPED = 339, + T_KILL = 340, + T_INJECT = 341, + T_STATE = 342, + T_ASSIGNED_STATE = 343, + T_IN = 344, + T_GROUP = 345, + T_LBRACE = 346, + T_RBRACE = 347, + T_COMMA = 348, + T_POSTGRES = 349, + T_STAYS = 350, + T_WHILE = 351, + T_THROUGH = 352, + T_SET = 353, + T_LOGS = 354, + T_NOT = 355, + T_CONTAINS = 356, + T_MATCHES = 357, + T_INTEGER = 358, + T_IDENT = 359, + T_STRING = 360, + T_BLOCK = 361, + T_SHELL_ARGS = 362 + }; +#endif +/* Tokens. */ +#define T_CLUSTER 258 +#define T_MONITOR 259 +#define T_NODE 260 +#define T_CITUS_COORDINATOR 261 +#define T_CITUS_WORKER 262 +#define T_SETUP 263 +#define T_TEARDOWN 264 +#define T_STEP 265 +#define T_SEQUENCE 266 +#define T_EQUALS 267 +#define T_IMAGE 268 +#define T_IMAGE_TARGET 269 +#define T_SSL 270 +#define T_AUTH 271 +#define T_AUTH_METHOD 272 +#define T_FORMATION 273 +#define T_NUM_SYNC 274 +#define T_COORDINATOR 275 +#define T_WORKER 276 +#define T_ASYNC 277 +#define T_NO_MONITOR 278 +#define T_LAUNCH 279 +#define T_DEFERRED 280 +#define T_IMMEDIATE 281 +#define T_INITIALLY 282 +#define T_VOLUME 283 +#define T_LISTEN 284 +#define T_CITUS_SECONDARY 285 +#define T_CANDIDATE_PRIORITY 286 +#define T_PORT 287 +#define T_PASSWORD 288 +#define T_MONITOR_PASSWORD 289 +#define T_CITUS_CLUSTER_NAME 290 +#define T_DEBIAN_CLUSTER 291 +#define T_REPLICATION_QUORUM 292 +#define T_REPLICATION_PASSWORD 293 +#define T_EXTENSION_VERSION 294 +#define T_BIND_SOURCE 295 +#define T_FS_INIT 296 +#define T_FS_SINGLE 297 +#define T_FS_PRIMARY 298 +#define T_FS_WAIT_PRIMARY 299 +#define T_FS_WAIT_STANDBY 300 +#define T_FS_DEMOTED 301 +#define T_FS_DEMOTE_TIMEOUT 302 +#define T_FS_DRAINING 303 +#define T_FS_SECONDARY 304 +#define T_FS_CATCHINGUP 305 +#define T_FS_PREP_PROMOTION 306 +#define T_FS_STOP_REPLICATION 307 +#define T_FS_MAINTENANCE 308 +#define T_FS_JOIN_PRIMARY 309 +#define T_FS_APPLY_SETTINGS 310 +#define T_FS_PREPARE_MAINTENANCE 311 +#define T_FS_WAIT_MAINTENANCE 312 +#define T_FS_REPORT_LSN 313 +#define T_FS_FAST_FORWARD 314 +#define T_FS_JOIN_SECONDARY 315 +#define T_FS_DROPPED 316 +#define T_EXEC 317 +#define T_EXEC_FAILS 318 +#define T_PG_AUTOCTL 319 +#define T_WAIT 320 +#define T_UNTIL 321 +#define T_TIMEOUT 322 +#define T_AND 323 +#define T_IS 324 +#define T_WITH 325 +#define T_ASSERT 326 +#define T_SQL 327 +#define T_EXPECT 328 +#define T_ERROR 329 +#define T_PROMOTE 330 +#define T_NETWORK 331 +#define T_DISCONNECT 332 +#define T_CONNECT 333 +#define T_SLEEP 334 +#define T_COMPOSE 335 +#define T_DOWN 336 +#define T_START 337 +#define T_STOP 338 +#define T_STOPPED 339 +#define T_KILL 340 +#define T_INJECT 341 +#define T_STATE 342 +#define T_ASSIGNED_STATE 343 +#define T_IN 344 +#define T_GROUP 345 +#define T_LBRACE 346 +#define T_RBRACE 347 +#define T_COMMA 348 +#define T_POSTGRES 349 +#define T_STAYS 350 +#define T_WHILE 351 +#define T_THROUGH 352 +#define T_SET 353 +#define T_LOGS 354 +#define T_NOT 355 +#define T_CONTAINS 356 +#define T_MATCHES 357 +#define T_INTEGER 358 +#define T_IDENT 359 +#define T_STRING 360 +#define T_BLOCK 361 +#define T_SHELL_ARGS 362 + + + + +/* Copy the first part of user declarations. */ +#line 1 "test_spec_parse.y" + +/* + * src/bin/pgaftest/test_spec_parse.y + * Bison grammar for .pgaf test specification files. + * + * The outer structure (cluster, setup, teardown, step, sequence) is + * described here as bison rules. Inside step/setup/teardown bodies + * the flex lexer enters the STEP_BODY exclusive state and returns + * individual tokens for every keyword, identifier, integer, and + * punctuation character — no more hand-written strstr/strtok parsing. + * + * The cluster { } block is now also fully parsed by this grammar. + * The flex lexer enters the CLUSTER_BODY exclusive state when it sees + * the opening '{' after "cluster", returning proper tokens for every + * keyword, value, and punctuation inside. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include + +#include "test_spec.h" +#include "pgsetup.h" +#include "file_utils.h" + +/* provided by test_spec_scan.l */ +extern int yylex(void); +extern int pgaf_line_number; +extern FILE *yyin; +extern int pgaf_next_brace_is_while; /* set before T_LBRACE for while body */ + +/* the spec we're building */ +static TestSpec *current_spec = NULL; + +static void yyerror(const char *msg) +{ + fprintf(stderr, "pgaftest: parse error at line %d: %s\n", + pgaf_line_number, msg); + exit(1); +} + +/* helpers */ +static void append_cmd(TestStep *step, TestCmd *cmd) +{ + if (!step->commands) + { + step->commands = cmd; + } + else + { + TestCmd *c = step->commands; + while (c->next) c = c->next; + c->next = cmd; + } +} + +/* + * expand_tuple_expect — convert `{ r1 } { r2 }` tuple syntax into + * the newline-separated form that psql --tuples-only --no-align produces. + */ +static void +expand_tuple_expect(char *buf, int buflen) +{ + const char *p = buf; + while (*p == ' ' || *p == '\t') p++; + + if (p[0] != '{' || (p[1] != ' ' && p[1] != '\t')) + return; + + char tmp[4096] = { 0 }; + int pos = 0; + bool first = true; + + while (*p) + { + while (*p == ' ' || *p == '\t' || *p == '\n') p++; + if (*p == '\0') break; + if (*p != '{') break; + + p++; + while (*p == ' ' || *p == '\t') p++; + + char row[1024] = { 0 }; + int ri = 0; + int depth = 1; + while (*p && depth > 0) + { + if (*p == '{') depth++; + else if (*p == '}') { if (--depth == 0) break; } + if (depth > 0 && ri < (int)sizeof(row) - 1) + row[ri++] = *p; + p++; + } + if (*p == '}') p++; + + while (ri > 0 && (row[ri-1] == ' ' || row[ri-1] == '\t')) ri--; + row[ri] = '\0'; + + if (!first && pos < (int)sizeof(tmp) - 1) + tmp[pos++] = '\n'; + int l = ri; + if (pos + l >= (int)sizeof(tmp)) l = (int)sizeof(tmp) - pos - 1; + if (l > 0) { memcpy(tmp + pos, row, l); pos += l; } + tmp[pos] = '\0'; + first = false; + } + + if (!first) + strncpy(buf, tmp, buflen - 1); +} + +static void register_step(TestSpec *spec, TestStep *step) +{ + if (!spec->steps) + { + spec->steps = step; + } + else + { + TestStep *s = spec->steps; + while (s->next) s = s->next; + s->next = step; + } + spec->stepCount++; +} + +/* ----------------------------------------------------------------------- + * Static state used by multi-element grammar rules. + * + * The parser is single-threaded; these are only live during the reduction + * of a single rule so there is no re-entrancy concern. + * ----------------------------------------------------------------------- */ + +static TestCmd *current_wait_cmd = NULL; +static TestCmd *current_promote_cmd = NULL; +static TestCmd *current_pass_cmd = NULL; /* for opt_passing_through */ +static TestFormation *current_formation = NULL; +static TestNode *current_node = NULL; + + + +/* Enabling traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif + +/* Enabling verbose error messages. */ +#ifdef YYERROR_VERBOSE +# undef YYERROR_VERBOSE +# define YYERROR_VERBOSE 1 +#else +# define YYERROR_VERBOSE 0 +#endif + +/* Enabling the token table. */ +#ifndef YYTOKEN_TABLE +# define YYTOKEN_TABLE 0 +#endif + +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef union YYSTYPE +#line 145 "test_spec_parse.y" +{ + int ival; + char *str; + TestStep *step; + TestCmd *cmd; +} +/* Line 193 of yacc.c. */ +#line 461 "test_spec_parse.c" + YYSTYPE; +# define yystype YYSTYPE /* obsolescent; will be withdrawn */ +# define YYSTYPE_IS_DECLARED 1 +# define YYSTYPE_IS_TRIVIAL 1 +#endif + + + +/* Copy the second part of user declarations. */ + + +/* Line 216 of yacc.c. */ +#line 474 "test_spec_parse.c" + +#ifdef short +# undef short +#endif + +#ifdef YYTYPE_UINT8 +typedef YYTYPE_UINT8 yytype_uint8; +#else +typedef unsigned char yytype_uint8; +#endif + +#ifdef YYTYPE_INT8 +typedef YYTYPE_INT8 yytype_int8; +#elif (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +typedef signed char yytype_int8; +#else +typedef short int yytype_int8; +#endif + +#ifdef YYTYPE_UINT16 +typedef YYTYPE_UINT16 yytype_uint16; +#else +typedef unsigned short int yytype_uint16; +#endif + +#ifdef YYTYPE_INT16 +typedef YYTYPE_INT16 yytype_int16; +#else +typedef short int yytype_int16; +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +# include /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned int +# endif +#endif + +#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_(msgid) dgettext ("bison-runtime", msgid) +# endif +# endif +# ifndef YY_ +# define YY_(msgid) msgid +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YYUSE(e) ((void) (e)) +#else +# define YYUSE(e) /* empty */ +#endif + +/* Identity function, used to suppress warnings about constant conditions. */ +#ifndef lint +# define YYID(n) (n) +#else +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static int +YYID (int i) +#else +static int +YYID (i) + int i; +#endif +{ + return i; +} +#endif + +#if ! defined yyoverflow || YYERROR_VERBOSE + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +# include /* INFRINGES ON USER NAME SPACE */ +# ifndef _STDLIB_H +# define _STDLIB_H 1 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's `empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0)) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined _STDLIB_H \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include /* INFRINGES ON USER NAME SPACE */ +# ifndef _STDLIB_H +# define _STDLIB_H 1 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ + + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yytype_int16 yyss; + YYSTYPE yyvs; + }; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ + + YYSTACK_GAP_MAXIMUM) + +/* Copy COUNT objects from FROM to TO. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(To, From, Count) \ + __builtin_memcpy (To, From, (Count) * sizeof (*(From))) +# else +# define YYCOPY(To, From, Count) \ + do \ + { \ + YYSIZE_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (To)[yyi] = (From)[yyi]; \ + } \ + while (YYID (0)) +# endif +# endif + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack) \ + do \ + { \ + YYSIZE_T yynewbytes; \ + YYCOPY (&yyptr->Stack, Stack, yysize); \ + Stack = &yyptr->Stack; \ + yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / sizeof (*yyptr); \ + } \ + while (YYID (0)) + +#endif + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL 21 +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST 523 + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS 108 +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS 62 +/* YYNRULES -- Number of rules. */ +#define YYNRULES 193 +/* YYNRULES -- Number of states. */ +#define YYNSTATES 312 + +/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ +#define YYUNDEFTOK 2 +#define YYMAXUTOK 362 + +#define YYTRANSLATE(YYX) \ + ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) + +/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ +static const yytype_uint8 yytranslate[] = +{ + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + 105, 106, 107 +}; + +#if YYDEBUG +/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in + YYRHS. */ +static const yytype_uint16 yyprhs[] = +{ + 0, 0, 3, 5, 8, 10, 12, 14, 16, 18, + 19, 25, 26, 29, 31, 33, 35, 37, 39, 41, + 43, 45, 49, 53, 57, 61, 66, 71, 78, 81, + 84, 87, 90, 93, 96, 99, 100, 107, 108, 111, + 113, 115, 117, 119, 121, 123, 126, 127, 130, 132, + 134, 135, 136, 141, 142, 150, 151, 154, 156, 158, + 160, 162, 164, 167, 170, 172, 174, 176, 179, 182, + 185, 188, 191, 194, 197, 200, 203, 206, 209, 213, + 217, 220, 223, 227, 231, 232, 235, 237, 239, 241, + 243, 245, 247, 249, 251, 253, 255, 257, 259, 261, + 265, 268, 272, 275, 279, 282, 284, 286, 288, 293, + 298, 300, 304, 305, 308, 310, 312, 316, 320, 321, + 331, 332, 342, 350, 358, 364, 370, 377, 379, 381, + 385, 389, 390, 393, 396, 401, 402, 405, 409, 416, + 423, 430, 437, 441, 444, 447, 451, 455, 458, 460, + 464, 468, 472, 475, 478, 482, 486, 490, 495, 499, + 503, 504, 510, 516, 520, 525, 531, 536, 542, 545, + 546, 549, 551, 553, 555, 557, 559, 561, 563, 565, + 567, 569, 571, 573, 575, 577, 579, 581, 583, 585, + 587, 589, 591, 593 +}; + +/* YYRHS -- A `-1'-separated list of the rules' RHS. */ +static const yytype_int16 yyrhs[] = +{ + 109, 0, -1, 110, -1, 109, 110, -1, 111, -1, + 133, -1, 134, -1, 135, -1, 166, -1, -1, 3, + 91, 112, 113, 92, -1, -1, 113, 114, -1, 115, + -1, 116, -1, 118, -1, 119, -1, 117, -1, 120, + -1, 40, -1, 4, -1, 4, 36, 104, -1, 4, + 14, 104, -1, 4, 32, 103, -1, 4, 33, 105, + -1, 4, 104, 24, 25, -1, 4, 104, 27, 84, + -1, 4, 104, 24, 25, 33, 105, -1, 13, 105, + -1, 13, 104, -1, 39, 104, -1, 39, 105, -1, + 15, 104, -1, 16, 104, -1, 17, 104, -1, -1, + 18, 121, 122, 91, 125, 92, -1, -1, 122, 124, + -1, 104, -1, 105, -1, 16, -1, 4, -1, 5, + -1, 123, -1, 19, 103, -1, -1, 125, 128, -1, + 104, -1, 4, -1, -1, -1, 126, 127, 129, 131, + -1, -1, 5, 104, 127, 130, 91, 131, 92, -1, + -1, 131, 132, -1, 20, -1, 21, -1, 22, -1, + 23, -1, 25, -1, 24, 25, -1, 24, 26, -1, + 26, -1, 29, -1, 30, -1, 31, 103, -1, 90, + 103, -1, 32, 103, -1, 35, 104, -1, 36, 104, + -1, 15, 104, -1, 16, 104, -1, 17, 104, -1, + 37, 104, -1, 38, 105, -1, 34, 105, -1, 28, + 104, 104, -1, 28, 104, 105, -1, 8, 136, -1, + 9, 136, -1, 10, 169, 136, -1, 91, 137, 92, + -1, -1, 137, 138, -1, 139, -1, 145, -1, 152, + -1, 153, -1, 154, -1, 155, -1, 157, -1, 158, + -1, 159, -1, 160, -1, 163, -1, 164, -1, 165, + -1, 62, 104, 107, -1, 62, 104, -1, 63, 104, + 107, -1, 63, 104, -1, 64, 104, 107, -1, 64, + 104, -1, 64, -1, 12, -1, 69, -1, 104, 87, + 140, 168, -1, 104, 87, 140, 104, -1, 141, -1, + 142, 68, 141, -1, -1, 97, 144, -1, 168, -1, + 104, -1, 144, 93, 168, -1, 144, 93, 104, -1, + -1, 65, 66, 104, 87, 140, 168, 146, 143, 151, + -1, -1, 65, 66, 104, 87, 140, 104, 147, 143, + 151, -1, 65, 66, 104, 88, 140, 168, 151, -1, + 65, 66, 104, 88, 140, 104, 151, -1, 65, 66, + 104, 84, 151, -1, 65, 66, 148, 149, 151, -1, + 65, 66, 141, 68, 142, 151, -1, 168, -1, 104, + -1, 148, 93, 168, -1, 148, 93, 104, -1, -1, + 89, 150, -1, 90, 103, -1, 150, 93, 90, 103, + -1, -1, 67, 103, -1, 70, 67, 103, -1, 71, + 104, 87, 140, 168, 151, -1, 71, 104, 87, 140, + 104, 151, -1, 71, 104, 88, 140, 168, 151, -1, + 71, 104, 88, 140, 104, 151, -1, 72, 104, 106, + -1, 73, 106, -1, 73, 74, -1, 73, 74, 104, + -1, 73, 74, 103, -1, 75, 156, -1, 104, -1, + 156, 93, 104, -1, 76, 77, 104, -1, 76, 78, + 104, -1, 79, 103, -1, 80, 81, -1, 80, 82, + 104, -1, 80, 83, 104, -1, 80, 85, 104, -1, + 80, 86, 104, 107, -1, 83, 94, 126, -1, 82, + 94, 126, -1, -1, 96, 162, 91, 137, 92, -1, + 71, 126, 95, 168, 161, -1, 98, 104, 104, -1, + 99, 104, 101, 105, -1, 99, 104, 100, 101, 105, + -1, 99, 104, 102, 105, -1, 99, 104, 100, 102, + 105, -1, 11, 167, -1, -1, 167, 169, -1, 41, + -1, 42, -1, 43, -1, 44, -1, 45, -1, 46, + -1, 47, -1, 48, -1, 49, -1, 50, -1, 51, + -1, 52, -1, 53, -1, 54, -1, 55, -1, 56, + -1, 57, -1, 58, -1, 59, -1, 60, -1, 61, + -1, 104, -1, 105, -1 +}; + +/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ +static const yytype_uint16 yyrline[] = +{ + 0, 211, 211, 212, 216, 217, 218, 219, 220, 233, + 232, 242, 244, 248, 249, 250, 251, 252, 253, 254, + 267, 271, 278, 285, 291, 298, 305, 312, 325, 331, + 341, 347, 357, 367, 373, 384, 383, 400, 402, 411, + 412, 413, 414, 415, 419, 424, 430, 432, 451, 452, + 461, 478, 477, 485, 484, 492, 494, 498, 503, 508, + 512, 516, 520, 524, 528, 532, 536, 540, 544, 548, + 552, 558, 564, 569, 574, 579, 587, 593, 599, 613, + 634, 641, 652, 670, 685, 688, 696, 697, 698, 699, + 700, 701, 702, 703, 704, 705, 706, 707, 708, 722, + 729, 735, 742, 748, 756, 762, 789, 789, 800, 815, + 833, 834, 849, 851, 855, 863, 871, 878, 890, 889, + 901, 900, 911, 920, 929, 936, 950, 965, 971, 978, + 984, 997, 999, 1003, 1008, 1016, 1017, 1018, 1029, 1037, + 1045, 1053, 1071, 1086, 1093, 1097, 1103, 1116, 1124, 1132, + 1147, 1153, 1166, 1180, 1184, 1190, 1196, 1222, 1256, 1262, + 1279, 1279, 1284, 1303, 1328, 1337, 1346, 1355, 1371, 1374, + 1376, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, + 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, 1415, 1416, + 1417, 1418, 1426, 1427 +}; +#endif + +#if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ + "$end", "error", "$undefined", "T_CLUSTER", "T_MONITOR", "T_NODE", + "T_CITUS_COORDINATOR", "T_CITUS_WORKER", "T_SETUP", "T_TEARDOWN", + "T_STEP", "T_SEQUENCE", "T_EQUALS", "T_IMAGE", "T_IMAGE_TARGET", "T_SSL", + "T_AUTH", "T_AUTH_METHOD", "T_FORMATION", "T_NUM_SYNC", "T_COORDINATOR", + "T_WORKER", "T_ASYNC", "T_NO_MONITOR", "T_LAUNCH", "T_DEFERRED", + "T_IMMEDIATE", "T_INITIALLY", "T_VOLUME", "T_LISTEN", + "T_CITUS_SECONDARY", "T_CANDIDATE_PRIORITY", "T_PORT", "T_PASSWORD", + "T_MONITOR_PASSWORD", "T_CITUS_CLUSTER_NAME", "T_DEBIAN_CLUSTER", + "T_REPLICATION_QUORUM", "T_REPLICATION_PASSWORD", "T_EXTENSION_VERSION", + "T_BIND_SOURCE", "T_FS_INIT", "T_FS_SINGLE", "T_FS_PRIMARY", + "T_FS_WAIT_PRIMARY", "T_FS_WAIT_STANDBY", "T_FS_DEMOTED", + "T_FS_DEMOTE_TIMEOUT", "T_FS_DRAINING", "T_FS_SECONDARY", + "T_FS_CATCHINGUP", "T_FS_PREP_PROMOTION", "T_FS_STOP_REPLICATION", + "T_FS_MAINTENANCE", "T_FS_JOIN_PRIMARY", "T_FS_APPLY_SETTINGS", + "T_FS_PREPARE_MAINTENANCE", "T_FS_WAIT_MAINTENANCE", "T_FS_REPORT_LSN", + "T_FS_FAST_FORWARD", "T_FS_JOIN_SECONDARY", "T_FS_DROPPED", "T_EXEC", + "T_EXEC_FAILS", "T_PG_AUTOCTL", "T_WAIT", "T_UNTIL", "T_TIMEOUT", + "T_AND", "T_IS", "T_WITH", "T_ASSERT", "T_SQL", "T_EXPECT", "T_ERROR", + "T_PROMOTE", "T_NETWORK", "T_DISCONNECT", "T_CONNECT", "T_SLEEP", + "T_COMPOSE", "T_DOWN", "T_START", "T_STOP", "T_STOPPED", "T_KILL", + "T_INJECT", "T_STATE", "T_ASSIGNED_STATE", "T_IN", "T_GROUP", "T_LBRACE", + "T_RBRACE", "T_COMMA", "T_POSTGRES", "T_STAYS", "T_WHILE", "T_THROUGH", + "T_SET", "T_LOGS", "T_NOT", "T_CONTAINS", "T_MATCHES", "T_INTEGER", + "T_IDENT", "T_STRING", "T_BLOCK", "T_SHELL_ARGS", "$accept", "spec", + "spec_item", "cluster_block", "@1", "cluster_item_list", "cluster_item", + "monitor_line", "image_line", "extension_version_line", "ssl_line", + "auth_line", "formation_block", "@2", "formation_opt_list", "bare_name", + "formation_opt", "node_list", "node_name", "init_node_slot", "node_line", + "@3", "@4", "node_opt_list", "node_opt", "setup_block", "teardown_block", + "named_step", "cmd_block", "cmd_list", "step_cmd", "exec_cmd", + "state_op", "wait_multi_condition", "wait_multi_condition_list", + "opt_passing_through", "pass_state_list", "wait_cmd", "@5", "@6", + "state_name_list", "opt_in_group", "group_items", "opt_timeout", + "assert_cmd", "sql_cmd", "expect_cmd", "promote_cmd", "promote_list", + "network_cmd", "sleep_cmd", "compose_cmd", "postgres_ctl_cmd", + "while_body", "@7", "stays_while_cmd", "set_monitor_cmd", "logs_cmd", + "sequence_block", "sequence_names", "fsm_state", "ident_or_string", 0 +}; +#endif + +# ifdef YYPRINT +/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to + token YYLEX-NUM. */ +static const yytype_uint16 yytoknum[] = +{ + 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, + 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, + 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 360, 361, 362 +}; +# endif + +/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ +static const yytype_uint8 yyr1[] = +{ + 0, 108, 109, 109, 110, 110, 110, 110, 110, 112, + 111, 113, 113, 114, 114, 114, 114, 114, 114, 114, + 115, 115, 115, 115, 115, 115, 115, 115, 116, 116, + 117, 117, 118, 119, 119, 121, 120, 122, 122, 123, + 123, 123, 123, 123, 124, 124, 125, 125, 126, 126, + 127, 129, 128, 130, 128, 131, 131, 132, 132, 132, + 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, + 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, + 133, 134, 135, 136, 137, 137, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 139, + 139, 139, 139, 139, 139, 139, 140, 140, 141, 141, + 142, 142, 143, 143, 144, 144, 144, 144, 146, 145, + 147, 145, 145, 145, 145, 145, 145, 148, 148, 148, + 148, 149, 149, 150, 150, 151, 151, 151, 152, 152, + 152, 152, 153, 154, 154, 154, 154, 155, 156, 156, + 157, 157, 158, 159, 159, 159, 159, 159, 160, 160, + 162, 161, 163, 164, 165, 165, 165, 165, 166, 167, + 167, 168, 168, 168, 168, 168, 168, 168, 168, 168, + 168, 168, 168, 168, 168, 168, 168, 168, 168, 168, + 168, 168, 169, 169 +}; + +/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ +static const yytype_uint8 yyr2[] = +{ + 0, 2, 1, 2, 1, 1, 1, 1, 1, 0, + 5, 0, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 3, 3, 3, 4, 4, 6, 2, 2, + 2, 2, 2, 2, 2, 0, 6, 0, 2, 1, + 1, 1, 1, 1, 1, 2, 0, 2, 1, 1, + 0, 0, 4, 0, 7, 0, 2, 1, 1, 1, + 1, 1, 2, 2, 1, 1, 1, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, + 2, 2, 3, 3, 0, 2, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, + 2, 3, 2, 3, 2, 1, 1, 1, 4, 4, + 1, 3, 0, 2, 1, 1, 3, 3, 0, 9, + 0, 9, 7, 7, 5, 5, 6, 1, 1, 3, + 3, 0, 2, 2, 4, 0, 2, 3, 6, 6, + 6, 6, 3, 2, 2, 3, 3, 2, 1, 3, + 3, 3, 2, 2, 3, 3, 3, 4, 3, 3, + 0, 5, 5, 3, 4, 5, 4, 5, 2, 0, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 +}; + +/* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state + STATE-NUM when YYTABLE doesn't specify something else to do. Zero + means the default is an error. */ +static const yytype_uint8 yydefact[] = +{ + 0, 0, 0, 0, 0, 169, 0, 2, 4, 5, + 6, 7, 8, 9, 84, 80, 81, 192, 193, 0, + 168, 1, 3, 11, 0, 82, 170, 0, 0, 0, + 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 83, 0, 0, 85, 86, 87, 88, 89, 90, + 91, 92, 93, 94, 95, 96, 97, 98, 20, 0, + 0, 0, 0, 35, 0, 19, 10, 12, 13, 14, + 17, 15, 16, 18, 100, 102, 104, 0, 49, 48, + 0, 0, 144, 143, 148, 147, 0, 0, 152, 153, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 29, 28, 32, 33, 34, 37, 30, + 31, 99, 101, 103, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 128, 0, 131, 127, 0, + 0, 0, 142, 146, 145, 0, 150, 151, 154, 155, + 156, 0, 48, 159, 158, 163, 0, 0, 0, 22, + 23, 24, 21, 0, 0, 0, 135, 0, 0, 0, + 0, 0, 135, 106, 107, 0, 0, 0, 149, 157, + 0, 0, 164, 166, 25, 26, 42, 43, 41, 0, + 46, 39, 40, 44, 38, 0, 0, 124, 0, 0, + 0, 110, 135, 0, 132, 130, 129, 125, 135, 135, + 135, 135, 160, 162, 165, 167, 0, 45, 0, 136, + 0, 120, 118, 135, 135, 0, 0, 126, 133, 0, + 139, 138, 141, 140, 0, 27, 0, 36, 50, 47, + 137, 112, 112, 123, 122, 0, 111, 0, 84, 50, + 51, 0, 135, 135, 109, 108, 134, 0, 53, 55, + 115, 113, 114, 121, 119, 161, 0, 52, 0, 55, + 0, 0, 0, 57, 58, 59, 60, 0, 61, 64, + 0, 65, 66, 0, 0, 0, 0, 0, 0, 0, + 0, 56, 117, 116, 0, 72, 73, 74, 62, 63, + 0, 67, 69, 77, 70, 71, 75, 76, 68, 54, + 78, 79 +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const yytype_int16 yydefgoto[] = +{ + -1, 6, 7, 8, 23, 27, 67, 68, 69, 70, + 71, 72, 73, 108, 165, 193, 194, 218, 80, 250, + 239, 259, 266, 267, 291, 9, 10, 11, 15, 24, + 44, 45, 175, 136, 202, 252, 261, 46, 242, 241, + 137, 172, 204, 197, 47, 48, 49, 50, 85, 51, + 52, 53, 54, 213, 234, 55, 56, 57, 12, 20, + 138, 19 +}; + +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +#define YYPACT_NINF -160 +static const yytype_int16 yypact[] = +{ + 45, -76, -65, -65, -45, -160, 109, -160, -160, -160, + -160, -160, -160, -160, -160, -160, -160, -160, -160, -65, + -45, -160, -160, -160, 145, -160, -160, 6, -72, -64, + -57, -24, 3, -25, -62, -19, 24, -14, 47, 19, + 32, -160, -9, 27, -160, -160, -160, -160, -160, -160, + -160, -160, -160, -160, -160, -160, -160, -160, -5, 11, + 33, 53, 111, -160, 30, -160, -160, -160, -160, -160, + -160, -160, -160, -160, 106, 112, 126, 122, -160, 55, + 29, 128, 102, -160, -160, 58, 127, 131, -160, -160, + 132, 134, 135, 136, 4, 4, 137, 21, 138, 129, + 140, 142, 73, -160, -160, -160, -160, -160, -160, -160, + -160, -160, -160, -160, -160, -160, -160, -160, -160, -160, + -160, -160, -160, -160, -160, -160, -160, -160, -160, -160, + -160, -160, -160, -160, -160, -51, 244, -75, -160, 17, + 17, 440, -160, -160, -160, 213, -160, -160, -160, -160, + -160, 211, -160, -160, -160, -160, 110, 214, 215, -160, + -160, -160, -160, 296, 241, 1, 44, 17, 17, 224, + 239, 143, 44, -160, -160, 207, 228, 240, -160, -160, + 230, 232, -160, -160, 305, -160, -160, -160, -160, 236, + -160, -160, -160, -160, -160, 237, 274, -160, 249, 313, + 255, -160, 20, 242, 253, -160, -160, -160, 44, 44, + 44, 44, -160, -160, -160, -160, 243, -160, -1, -160, + 248, 276, 279, 44, 44, 17, 224, -160, -160, 262, + -160, -160, -160, -160, 327, -160, 315, -160, -160, -160, + -160, 323, 323, -160, -160, 334, -160, 318, -160, -160, + -160, 355, 44, 44, -160, -160, -160, 251, -160, -160, + -160, 329, -160, -160, -160, -160, 332, 124, 419, -160, + 320, 321, 322, -160, -160, -160, -160, 197, -160, -160, + 324, -160, -160, 326, 328, 325, 330, 331, 333, 335, + 336, -160, -160, -160, 46, -160, -160, -160, -160, -160, + 125, -160, -160, -160, -160, -160, -160, -160, -160, -160, + -160, -160 +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const yytype_int16 yypgoto[] = +{ + -160, -160, 421, -160, -160, -160, -160, -160, -160, -160, + -160, -160, -160, -160, -160, -160, -160, -160, -93, 183, + -160, -160, -160, 164, -160, -160, -160, -160, 22, 188, + -160, -160, -129, -153, -160, 199, -160, -160, -160, -160, + -160, -160, -160, -159, -160, -160, -160, -160, -160, -160, + -160, -160, -160, -160, -160, -160, -160, -160, -160, -160, + -141, 422 +}; + +/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule which + number is the opposite. If zero, do what YYDEFACT says. + If YYTABLE_NINF, syntax error. */ +#define YYTABLE_NINF -110 +static const yytype_int16 yytable[] = +{ + 177, 153, 154, 78, 236, 186, 187, 78, 78, 98, + 58, 176, 82, 207, 170, 13, 201, 188, 171, 59, + 189, 60, 61, 62, 63, 16, 14, 99, 100, 173, + 206, 101, 74, 166, 209, 211, 167, 168, 198, 199, + 75, 25, 77, 227, 83, 64, 65, 76, 1, 230, + 231, 232, 233, 2, 3, 4, 5, 222, 224, 17, + 18, 270, 271, 272, 243, 244, 273, 274, 275, 276, + 277, 278, 279, 246, 280, 281, 282, 283, 284, 81, + 285, 286, 287, 288, 289, 84, 174, 195, 226, 88, + 196, 237, 190, 263, 264, 96, 245, 163, 66, 102, + 164, 86, 87, 152, 255, 191, 192, 79, 152, 21, + 262, 195, 1, 94, 196, 103, 104, 2, 3, 4, + 5, 156, 157, 158, 141, 238, 95, 293, 89, 90, + 91, 97, 92, 93, 109, 110, 290, 105, 309, 270, + 271, 272, 139, 140, 273, 274, 275, 276, 277, 278, + 279, 145, 280, 281, 282, 283, 284, 106, 285, 286, + 287, 288, 289, 114, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, + 131, 132, 133, 134, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 143, 144, 28, 29, 30, + 31, 180, 181, 111, 290, 107, 32, 33, 34, 112, + 35, 36, 298, 299, 37, 38, 135, 39, 40, 310, + 311, 146, 160, 113, 142, 147, 148, 41, 149, 150, + 151, 155, 159, 42, 43, 161, 162, 205, 114, 115, + 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, + 126, 127, 128, 129, 130, 131, 132, 133, 134, 114, + 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, + 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, + 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 208, 169, 28, 29, 30, 31, 178, 179, 182, + 183, 184, 32, 33, 34, 185, 35, 36, 200, 203, + 37, 38, 210, 39, 40, 214, 212, 215, 216, 217, + 219, 220, 225, 265, -109, 228, 229, -108, 235, 42, + 43, 240, 247, 221, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 223, 248, 249, + 251, 256, 268, 269, 295, 296, 297, 22, 300, 301, + 303, 302, 258, 294, 304, 305, 257, 306, 254, 308, + 307, 253, 26, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, + 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 114, 115, 116, 117, 118, 119, 120, 121, 122, + 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 292 +}; + +static const yytype_int16 yycheck[] = +{ + 141, 94, 95, 4, 5, 4, 5, 4, 4, 14, + 4, 140, 74, 172, 89, 91, 169, 16, 93, 13, + 19, 15, 16, 17, 18, 3, 91, 32, 33, 12, + 171, 36, 104, 84, 175, 176, 87, 88, 167, 168, + 104, 19, 66, 202, 106, 39, 40, 104, 3, 208, + 209, 210, 211, 8, 9, 10, 11, 198, 199, 104, + 105, 15, 16, 17, 223, 224, 20, 21, 22, 23, + 24, 25, 26, 226, 28, 29, 30, 31, 32, 104, + 34, 35, 36, 37, 38, 104, 69, 67, 68, 103, + 70, 92, 91, 252, 253, 104, 225, 24, 92, 104, + 27, 77, 78, 104, 245, 104, 105, 104, 104, 0, + 251, 67, 3, 94, 70, 104, 105, 8, 9, 10, + 11, 100, 101, 102, 95, 218, 94, 268, 81, 82, + 83, 104, 85, 86, 104, 105, 90, 104, 92, 15, + 16, 17, 87, 88, 20, 21, 22, 23, 24, 25, + 26, 93, 28, 29, 30, 31, 32, 104, 34, 35, + 36, 37, 38, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 103, 104, 62, 63, 64, + 65, 101, 102, 107, 90, 104, 71, 72, 73, 107, + 75, 76, 25, 26, 79, 80, 104, 82, 83, 104, + 105, 104, 103, 107, 106, 104, 104, 92, 104, 104, + 104, 104, 104, 98, 99, 105, 104, 104, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, 59, 60, 61, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 104, 68, 62, 63, 64, 65, 104, 107, 105, + 105, 25, 71, 72, 73, 84, 75, 76, 104, 90, + 79, 80, 104, 82, 83, 105, 96, 105, 33, 103, + 103, 67, 87, 92, 68, 103, 93, 68, 105, 98, + 99, 103, 90, 104, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 104, 91, 104, + 97, 103, 93, 91, 104, 104, 104, 6, 104, 103, + 105, 103, 249, 269, 104, 104, 248, 104, 104, 103, + 105, 242, 20, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 104, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 104 +}; + +/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing + symbol of state STATE-NUM. */ +static const yytype_uint8 yystos[] = +{ + 0, 3, 8, 9, 10, 11, 109, 110, 111, 133, + 134, 135, 166, 91, 91, 136, 136, 104, 105, 169, + 167, 0, 110, 112, 137, 136, 169, 113, 62, 63, + 64, 65, 71, 72, 73, 75, 76, 79, 80, 82, + 83, 92, 98, 99, 138, 139, 145, 152, 153, 154, + 155, 157, 158, 159, 160, 163, 164, 165, 4, 13, + 15, 16, 17, 18, 39, 40, 92, 114, 115, 116, + 117, 118, 119, 120, 104, 104, 104, 66, 4, 104, + 126, 104, 74, 106, 104, 156, 77, 78, 103, 81, + 82, 83, 85, 86, 94, 94, 104, 104, 14, 32, + 33, 36, 104, 104, 105, 104, 104, 104, 121, 104, + 105, 107, 107, 107, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 104, 141, 148, 168, 87, + 88, 95, 106, 103, 104, 93, 104, 104, 104, 104, + 104, 104, 104, 126, 126, 104, 100, 101, 102, 104, + 103, 105, 104, 24, 27, 122, 84, 87, 88, 68, + 89, 93, 149, 12, 69, 140, 140, 168, 104, 107, + 101, 102, 105, 105, 25, 84, 4, 5, 16, 19, + 91, 104, 105, 123, 124, 67, 70, 151, 140, 140, + 104, 141, 142, 90, 150, 104, 168, 151, 104, 168, + 104, 168, 96, 161, 105, 105, 33, 103, 125, 103, + 67, 104, 168, 104, 168, 87, 68, 151, 103, 93, + 151, 151, 151, 151, 162, 105, 5, 92, 126, 128, + 103, 147, 146, 151, 151, 140, 141, 90, 91, 104, + 127, 97, 143, 143, 104, 168, 103, 137, 127, 129, + 104, 144, 168, 151, 151, 92, 130, 131, 93, 91, + 15, 16, 17, 20, 21, 22, 23, 24, 25, 26, + 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, + 90, 132, 104, 168, 131, 104, 104, 104, 25, 26, + 104, 103, 103, 105, 104, 104, 104, 105, 103, 92, + 104, 105 +}; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) +#define YYEMPTY (-2) +#define YYEOF 0 + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab + + +/* Like YYERROR except do call yyerror. This remains here temporarily + to ease the transition to the new meaning of YYERROR, for GCC. + Once GCC version 2 has supplanted version 1, this can go. */ + +#define YYFAIL goto yyerrlab + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ +do \ + if (yychar == YYEMPTY && yylen == 1) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + yytoken = YYTRANSLATE (yychar); \ + YYPOPSTACK (1); \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ +while (YYID (0)) + + +#define YYTERROR 1 +#define YYERRCODE 256 + + +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (YYID (N)) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (YYID (0)) +#endif + + +/* YY_LOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +#ifndef YY_LOCATION_PRINT +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL +# define YY_LOCATION_PRINT(File, Loc) \ + fprintf (File, "%d.%d-%d.%d", \ + (Loc).first_line, (Loc).first_column, \ + (Loc).last_line, (Loc).last_column) +# else +# define YY_LOCATION_PRINT(File, Loc) ((void) 0) +# endif +#endif + + +/* YYLEX -- calling `yylex' with the right arguments. */ + +#ifdef YYLEX_PARAM +# define YYLEX yylex (YYLEX_PARAM) +#else +# define YYLEX yylex () +#endif + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (YYID (0)) + +# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Type, Value); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (YYID (0)) + + +/*--------------------------------. +| Print this symbol on YYOUTPUT. | +`--------------------------------*/ + +/*ARGSUSED*/ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +#else +static void +yy_symbol_value_print (yyoutput, yytype, yyvaluep) + FILE *yyoutput; + int yytype; + YYSTYPE const * const yyvaluep; +#endif +{ + if (!yyvaluep) + return; +# ifdef YYPRINT + if (yytype < YYNTOKENS) + YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); +# else + YYUSE (yyoutput); +# endif + switch (yytype) + { + default: + break; + } +} + + +/*--------------------------------. +| Print this symbol on YYOUTPUT. | +`--------------------------------*/ + +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +#else +static void +yy_symbol_print (yyoutput, yytype, yyvaluep) + FILE *yyoutput; + int yytype; + YYSTYPE const * const yyvaluep; +#endif +{ + if (yytype < YYNTOKENS) + YYFPRINTF (yyoutput, "token %s (", yytname[yytype]); + else + YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]); + + yy_symbol_value_print (yyoutput, yytype, yyvaluep); + YYFPRINTF (yyoutput, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_stack_print (yytype_int16 *bottom, yytype_int16 *top) +#else +static void +yy_stack_print (bottom, top) + yytype_int16 *bottom; + yytype_int16 *top; +#endif +{ + YYFPRINTF (stderr, "Stack now"); + for (; bottom <= top; ++bottom) + YYFPRINTF (stderr, " %d", *bottom); + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)); \ +} while (YYID (0)) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_reduce_print (YYSTYPE *yyvsp, int yyrule) +#else +static void +yy_reduce_print (yyvsp, yyrule) + YYSTYPE *yyvsp; + int yyrule; +#endif +{ + int yynrhs = yyr2[yyrule]; + int yyi; + unsigned long int yylno = yyrline[yyrule]; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + fprintf (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi], + &(yyvsp[(yyi + 1) - (yynrhs)]) + ); + fprintf (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyvsp, Rule); \ +} while (YYID (0)) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +int yydebug; +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) +# define YY_SYMBOL_PRINT(Title, Type, Value, Location) +# define YY_STACK_PRINT(Bottom, Top) +# define YY_REDUCE_PRINT(Rule) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + + +#if YYERROR_VERBOSE + +# ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen strlen +# else +/* Return the length of YYSTR. */ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static YYSIZE_T +yystrlen (const char *yystr) +#else +static YYSIZE_T +yystrlen (yystr) + const char *yystr; +#endif +{ + YYSIZE_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +# endif + +# ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static char * +yystpcpy (char *yydest, const char *yysrc) +#else +static char * +yystpcpy (yydest, yysrc) + char *yydest; + const char *yysrc; +#endif +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +# endif + +# ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYSIZE_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYSIZE_T yyn = 0; + char const *yyp = yystr; + + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + /* Fall through. */ + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (! yyres) + return yystrlen (yystr); + + return yystpcpy (yyres, yystr) - yyres; +} +# endif + +/* Copy into YYRESULT an error message about the unexpected token + YYCHAR while in state YYSTATE. Return the number of bytes copied, + including the terminating null byte. If YYRESULT is null, do not + copy anything; just return the number of bytes that would be + copied. As a special case, return 0 if an ordinary "syntax error" + message will do. Return YYSIZE_MAXIMUM if overflow occurs during + size calculation. */ +static YYSIZE_T +yysyntax_error (char *yyresult, int yystate, int yychar) +{ + int yyn = yypact[yystate]; + + if (! (YYPACT_NINF < yyn && yyn <= YYLAST)) + return 0; + else + { + int yytype = YYTRANSLATE (yychar); + YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]); + YYSIZE_T yysize = yysize0; + YYSIZE_T yysize1; + int yysize_overflow = 0; + enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; + char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; + int yyx; + +# if 0 + /* This is so xgettext sees the translatable formats that are + constructed on the fly. */ + YY_("syntax error, unexpected %s"); + YY_("syntax error, unexpected %s, expecting %s"); + YY_("syntax error, unexpected %s, expecting %s or %s"); + YY_("syntax error, unexpected %s, expecting %s or %s or %s"); + YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); +# endif + char *yyfmt; + char const *yyf; + static char const yyunexpected[] = "syntax error, unexpected %s"; + static char const yyexpecting[] = ", expecting %s"; + static char const yyor[] = " or %s"; + char yyformat[sizeof yyunexpected + + sizeof yyexpecting - 1 + + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) + * (sizeof yyor - 1))]; + char const *yyprefix = yyexpecting; + + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yycount = 1; + + yyarg[0] = yytname[yytype]; + yyfmt = yystpcpy (yyformat, yyunexpected); + + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) + { + if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) + { + yycount = 1; + yysize = yysize0; + yyformat[sizeof yyunexpected - 1] = '\0'; + break; + } + yyarg[yycount++] = yytname[yyx]; + yysize1 = yysize + yytnamerr (0, yytname[yyx]); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + yyfmt = yystpcpy (yyfmt, yyprefix); + yyprefix = yyor; + } + + yyf = YY_(yyformat); + yysize1 = yysize + yystrlen (yyf); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + + if (yysize_overflow) + return YYSIZE_MAXIMUM; + + if (yyresult) + { + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + char *yyp = yyresult; + int yyi = 0; + while ((*yyp = *yyf) != '\0') + { + if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yyarg[yyi++]); + yyf += 2; + } + else + { + yyp++; + yyf++; + } + } + } + return yysize; + } +} +#endif /* YYERROR_VERBOSE */ + + +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +/*ARGSUSED*/ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep) +#else +static void +yydestruct (yymsg, yytype, yyvaluep) + const char *yymsg; + int yytype; + YYSTYPE *yyvaluep; +#endif +{ + YYUSE (yyvaluep); + + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + + switch (yytype) + { + + default: + break; + } +} + + +/* Prevent warnings from -Wmissing-prototypes. */ + +#ifdef YYPARSE_PARAM +#if defined __STDC__ || defined __cplusplus +int yyparse (void *YYPARSE_PARAM); +#else +int yyparse (); +#endif +#else /* ! YYPARSE_PARAM */ +#if defined __STDC__ || defined __cplusplus +int yyparse (void); +#else +int yyparse (); +#endif +#endif /* ! YYPARSE_PARAM */ + + + +/* The look-ahead symbol. */ +int yychar; + +/* The semantic value of the look-ahead symbol. */ +YYSTYPE yylval; + +/* Number of syntax errors so far. */ +int yynerrs; + + + +/*----------. +| yyparse. | +`----------*/ + +#ifdef YYPARSE_PARAM +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +int +yyparse (void *YYPARSE_PARAM) +#else +int +yyparse (YYPARSE_PARAM) + void *YYPARSE_PARAM; +#endif +#else /* ! YYPARSE_PARAM */ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +int +yyparse (void) +#else +int +yyparse () + +#endif +#endif +{ + + int yystate; + int yyn; + int yyresult; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus; + /* Look-ahead token as an internal (translated) token number. */ + int yytoken = 0; +#if YYERROR_VERBOSE + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYSIZE_T yymsg_alloc = sizeof yymsgbuf; +#endif + + /* Three stacks and their tools: + `yyss': related to states, + `yyvs': related to semantic values, + `yyls': related to locations. + + Refer to the stacks thru separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* The state stack. */ + yytype_int16 yyssa[YYINITDEPTH]; + yytype_int16 *yyss = yyssa; + yytype_int16 *yyssp; + + /* The semantic value stack. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp; + + + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) + + YYSIZE_T yystacksize = YYINITDEPTH; + + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yystate = 0; + yyerrstatus = 0; + yynerrs = 0; + yychar = YYEMPTY; /* Cause a token to be read. */ + + /* Initialize stack pointers. + Waste one element of value and location stack + so that they stay on the same level as the state stack. + The wasted elements are never initialized. */ + + yyssp = yyss; + yyvsp = yyvs; + + goto yysetstate; + +/*------------------------------------------------------------. +| yynewstate -- Push a new state, which is found in yystate. | +`------------------------------------------------------------*/ + yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + yysetstate: + *yyssp = yystate; + + if (yyss + yystacksize - 1 <= yyssp) + { + /* Get the current used size of the three stacks, in elements. */ + YYSIZE_T yysize = yyssp - yyss + 1; + +#ifdef yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + YYSTYPE *yyvs1 = yyvs; + yytype_int16 *yyss1 = yyss; + + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * sizeof (*yyssp), + &yyvs1, yysize * sizeof (*yyvsp), + + &yystacksize); + + yyss = yyss1; + yyvs = yyvs1; + } +#else /* no yyoverflow */ +# ifndef YYSTACK_RELOCATE + goto yyexhaustedlab; +# else + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + goto yyexhaustedlab; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yytype_int16 *yyss1 = yyss; + union yyalloc *yyptr = + (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); + if (! yyptr) + goto yyexhaustedlab; + YYSTACK_RELOCATE (yyss); + YYSTACK_RELOCATE (yyvs); + +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif +#endif /* no yyoverflow */ + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + + + YYDPRINTF ((stderr, "Stack size increased to %lu\n", + (unsigned long int) yystacksize)); + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } + + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + + goto yybackup; + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + + /* Do appropriate processing given the current state. Read a + look-ahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to look-ahead token. */ + yyn = yypact[yystate]; + if (yyn == YYPACT_NINF) + goto yydefault; + + /* Not known => get a look-ahead token if don't already have one. */ + + /* YYCHAR is either YYEMPTY or YYEOF or a valid look-ahead symbol. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token: ")); + yychar = YYLEX; + } + + if (yychar <= YYEOF) + { + yychar = yytoken = YYEOF; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yyn == 0 || yyn == YYTABLE_NINF) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + if (yyn == YYFINAL) + YYACCEPT; + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the look-ahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); + + /* Discard the shifted token unless it is eof. */ + if (yychar != YYEOF) + yychar = YYEMPTY; + + yystate = yyn; + *++yyvsp = yylval; + + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- Do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + `$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; + + + YY_REDUCE_PRINT (yyn); + switch (yyn) + { + case 9: +#line 233 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.ssl, "self-signed", + sizeof(current_spec->cluster.ssl)); + strlcpy(current_spec->cluster.auth, "trust", + sizeof(current_spec->cluster.auth)); + ;} + break; + + case 19: +#line 254 "test_spec_parse.y" + { current_spec->cluster.bindSource = true; ;} + break; + + case 20: +#line 268 "test_spec_parse.y" + { + current_spec->cluster.withMonitor = true; + ;} + break; + + case 21: +#line 272 "test_spec_parse.y" + { + current_spec->cluster.withMonitor = true; + strlcpy(current_spec->cluster.monitorDebianCluster, (yyvsp[(3) - (3)].str), + sizeof(current_spec->cluster.monitorDebianCluster)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 22: +#line 279 "test_spec_parse.y" + { + current_spec->cluster.withMonitor = true; + strlcpy(current_spec->cluster.monitorImageTarget, (yyvsp[(3) - (3)].str), + sizeof(current_spec->cluster.monitorImageTarget)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 23: +#line 286 "test_spec_parse.y" + { + current_spec->cluster.withMonitor = true; + /* monitor port not stored in TestCluster yet; ignore */ + (void) (yyvsp[(3) - (3)].ival); + ;} + break; + + case 24: +#line 292 "test_spec_parse.y" + { + current_spec->cluster.withMonitor = true; + strlcpy(current_spec->cluster.monitorPassword, (yyvsp[(3) - (3)].str), + sizeof(current_spec->cluster.monitorPassword)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 25: +#line 299 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), + sizeof(current_spec->cluster.secondMonitorName)); + current_spec->cluster.secondMonitorStopped = true; + free((yyvsp[(2) - (4)].str)); + ;} + break; + + case 26: +#line 306 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), + sizeof(current_spec->cluster.secondMonitorName)); + current_spec->cluster.secondMonitorStopped = true; + free((yyvsp[(2) - (4)].str)); + ;} + break; + + case 27: +#line 313 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (6)].str), + sizeof(current_spec->cluster.secondMonitorName)); + current_spec->cluster.secondMonitorStopped = true; + free((yyvsp[(2) - (6)].str)); + /* password for second monitor not yet stored */ + free((yyvsp[(6) - (6)].str)); + ;} + break; + + case 28: +#line 326 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.image)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 29: +#line 332 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.image)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 30: +#line 342 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.extensionVersion)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 31: +#line 348 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.extensionVersion)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 32: +#line 358 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.ssl, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.ssl)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 33: +#line 368 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.auth)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 34: +#line 374 "test_spec_parse.y" + { + strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), + sizeof(current_spec->cluster.auth)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 35: +#line 384 "test_spec_parse.y" + { + TestCluster *cl = ¤t_spec->cluster; + if (cl->formationCount >= PGAF_MAX_FORMATIONS) + { + fprintf(stderr, "pgaftest: too many formations (max %d)\n", + PGAF_MAX_FORMATIONS); + exit(1); + } + current_formation = &cl->formations[cl->formationCount++]; + strlcpy(current_formation->name, "default", + sizeof(current_formation->name)); + current_formation->numSync = -1; + ;} + break; + + case 39: +#line 411 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 40: +#line 412 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 41: +#line 413 "test_spec_parse.y" + { (yyval.str) = strdup("auth"); ;} + break; + + case 42: +#line 414 "test_spec_parse.y" + { (yyval.str) = strdup("monitor"); ;} + break; + + case 43: +#line 415 "test_spec_parse.y" + { (yyval.str) = strdup("node"); ;} + break; + + case 44: +#line 420 "test_spec_parse.y" + { + strlcpy(current_formation->name, (yyvsp[(1) - (1)].str), sizeof(current_formation->name)); + free((yyvsp[(1) - (1)].str)); + ;} + break; + + case 45: +#line 425 "test_spec_parse.y" + { + current_formation->numSync = (yyvsp[(2) - (2)].ival); + ;} + break; + + case 48: +#line 451 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 49: +#line 452 "test_spec_parse.y" + { (yyval.str) = strdup("monitor"); ;} + break; + + case 50: +#line 461 "test_spec_parse.y" + { + if (current_formation->nodeCount >= PGAF_MAX_NODES) + { + fprintf(stderr, "pgaftest: too many nodes in formation (max %d)\n", + PGAF_MAX_NODES); + exit(1); + } + current_node = ¤t_formation->nodes[current_formation->nodeCount++]; + current_node->kind = NODE_KIND_STANDALONE; + current_node->candidatePriority = 50; + current_node->replicationQuorum = true; + ;} + break; + + case 51: +#line 478 "test_spec_parse.y" + { + strlcpy(current_node->name, (yyvsp[(1) - (2)].str), sizeof(current_node->name)); + free((yyvsp[(1) - (2)].str)); + ;} + break; + + case 53: +#line 485 "test_spec_parse.y" + { + strlcpy(current_node->name, (yyvsp[(2) - (3)].str), sizeof(current_node->name)); + free((yyvsp[(2) - (3)].str)); + ;} + break; + + case 57: +#line 499 "test_spec_parse.y" + { + current_node->kind = NODE_KIND_CITUS_COORDINATOR; + current_spec->cluster.withCitus = true; + ;} + break; + + case 58: +#line 504 "test_spec_parse.y" + { + current_node->kind = NODE_KIND_CITUS_WORKER; + current_spec->cluster.withCitus = true; + ;} + break; + + case 59: +#line 509 "test_spec_parse.y" + { + current_node->replicationQuorum = false; + ;} + break; + + case 60: +#line 513 "test_spec_parse.y" + { + current_node->noMonitor = true; + ;} + break; + + case 61: +#line 517 "test_spec_parse.y" + { + current_node->launchDeferred = true; + ;} + break; + + case 62: +#line 521 "test_spec_parse.y" + { + current_node->launchDeferred = true; + ;} + break; + + case 63: +#line 525 "test_spec_parse.y" + { + current_node->launchDeferred = false; + ;} + break; + + case 64: +#line 529 "test_spec_parse.y" + { + current_node->launchDeferred = false; + ;} + break; + + case 65: +#line 533 "test_spec_parse.y" + { + current_node->listen = true; + ;} + break; + + case 66: +#line 537 "test_spec_parse.y" + { + current_node->citusSecondary = true; + ;} + break; + + case 67: +#line 541 "test_spec_parse.y" + { + current_node->candidatePriority = (yyvsp[(2) - (2)].ival); + ;} + break; + + case 68: +#line 545 "test_spec_parse.y" + { + current_node->group = (yyvsp[(2) - (2)].ival); + ;} + break; + + case 69: +#line 549 "test_spec_parse.y" + { + current_node->pgPort = (yyvsp[(2) - (2)].ival); + ;} + break; + + case 70: +#line 553 "test_spec_parse.y" + { + strlcpy(current_node->citusClusterName, (yyvsp[(2) - (2)].str), + sizeof(current_node->citusClusterName)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 71: +#line 559 "test_spec_parse.y" + { + strlcpy(current_node->debianCluster, (yyvsp[(2) - (2)].str), + sizeof(current_node->debianCluster)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 72: +#line 565 "test_spec_parse.y" + { + strlcpy(current_node->ssl, (yyvsp[(2) - (2)].str), sizeof(current_node->ssl)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 73: +#line 570 "test_spec_parse.y" + { + strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 74: +#line 575 "test_spec_parse.y" + { + strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 75: +#line 580 "test_spec_parse.y" + { + if (strcmp((yyvsp[(2) - (2)].str), "false") == 0 || strcmp((yyvsp[(2) - (2)].str), "0") == 0) + current_node->replicationQuorum = false; + else + current_node->replicationQuorum = true; + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 76: +#line 588 "test_spec_parse.y" + { + strlcpy(current_node->replicationPassword, (yyvsp[(2) - (2)].str), + sizeof(current_node->replicationPassword)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 77: +#line 594 "test_spec_parse.y" + { + strlcpy(current_node->monitorPassword, (yyvsp[(2) - (2)].str), + sizeof(current_node->monitorPassword)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 78: +#line 600 "test_spec_parse.y" + { + /* volume — adds a named Docker volume */ + int vi = current_node->volumeCount; + if (vi < PGAF_MAX_NODE_VOLUMES) + { + strlcpy(current_node->volumes[vi].name, (yyvsp[(2) - (3)].str), + sizeof(current_node->volumes[0].name)); + strlcpy(current_node->volumes[vi].path, (yyvsp[(3) - (3)].str), + sizeof(current_node->volumes[0].path)); + current_node->volumeCount++; + } + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 79: +#line 614 "test_spec_parse.y" + { + /* volume "/path/with spaces" */ + int vi = current_node->volumeCount; + if (vi < PGAF_MAX_NODE_VOLUMES) + { + strlcpy(current_node->volumes[vi].name, (yyvsp[(2) - (3)].str), + sizeof(current_node->volumes[0].name)); + strlcpy(current_node->volumes[vi].path, (yyvsp[(3) - (3)].str), + sizeof(current_node->volumes[0].path)); + current_node->volumeCount++; + } + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 80: +#line 635 "test_spec_parse.y" + { + current_spec->setup = (yyvsp[(2) - (2)].step); + ;} + break; + + case 81: +#line 642 "test_spec_parse.y" + { + current_spec->teardown = (yyvsp[(2) - (2)].step); + ;} + break; + + case 82: +#line 653 "test_spec_parse.y" + { + TestStep *s = (yyvsp[(3) - (3)].step); + strncpy(s->name, (yyvsp[(2) - (3)].str), sizeof(s->name) - 1); + free((yyvsp[(2) - (3)].str)); + register_step(current_spec, s); + ;} + break; + + case 83: +#line 671 "test_spec_parse.y" + { + /* post-process: CMD_SQL immediately before CMD_EXPECT_ERROR */ + for (TestCmd *c = (yyvsp[(2) - (3)].step)->commands; c; c = c->next) + { + if (c->kind == CMD_SQL && c->next && + c->next->kind == CMD_EXPECT_ERROR) + c->allowError = true; + } + (yyval.step) = (yyvsp[(2) - (3)].step); + ;} + break; + + case 84: +#line 685 "test_spec_parse.y" + { + (yyval.step) = make_step(""); + ;} + break; + + case 85: +#line 689 "test_spec_parse.y" + { + if ((yyvsp[(2) - (2)].cmd)) append_cmd((yyvsp[(1) - (2)].step), (yyvsp[(2) - (2)].cmd)); + (yyval.step) = (yyvsp[(1) - (2)].step); + ;} + break; + + case 86: +#line 696 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 87: +#line 697 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 88: +#line 698 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 89: +#line 699 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 90: +#line 700 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 91: +#line 701 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 92: +#line 702 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 93: +#line 703 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 94: +#line 704 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 95: +#line 705 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 96: +#line 706 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 97: +#line 707 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 98: +#line 708 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 99: +#line 723 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXEC); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->args)); + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 100: +#line 730 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXEC); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 101: +#line 736 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->args)); + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 102: +#line 743 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 103: +#line 749 "test_spec_parse.y" + { + /* "pg_autoctl perform failover --formation auth" + * EXEC_ARGS returns T_IDENT for first word, T_SHELL_ARGS for rest */ + (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), "%s %s", (yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].str)); + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 104: +#line 757 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); + strlcpy((yyval.cmd)->args, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->args)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 105: +#line 763 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); + ;} + break; + + case 108: +#line 801 "test_spec_parse.y" + { + if (!current_wait_cmd) + current_wait_cmd = make_cmd(CMD_WAIT_MULTI); + int i = current_wait_cmd->waitStateCount; + if (i < PGAF_MAX_WAIT_STATES) + { + strlcpy(current_wait_cmd->waitNodes[i], (yyvsp[(1) - (4)].str), + sizeof(current_wait_cmd->waitNodes[0])); + strlcpy(current_wait_cmd->waitStates[i], (yyvsp[(4) - (4)].str), + sizeof(current_wait_cmd->waitStates[0])); + current_wait_cmd->waitStateCount++; + } + free((yyvsp[(1) - (4)].str)); + ;} + break; + + case 109: +#line 816 "test_spec_parse.y" + { + if (!current_wait_cmd) + current_wait_cmd = make_cmd(CMD_WAIT_MULTI); + int i = current_wait_cmd->waitStateCount; + if (i < PGAF_MAX_WAIT_STATES) + { + strlcpy(current_wait_cmd->waitNodes[i], (yyvsp[(1) - (4)].str), + sizeof(current_wait_cmd->waitNodes[0])); + strlcpy(current_wait_cmd->waitStates[i], (yyvsp[(4) - (4)].str), + sizeof(current_wait_cmd->waitStates[0])); + current_wait_cmd->waitStateCount++; + } + free((yyvsp[(1) - (4)].str)); free((yyvsp[(4) - (4)].str)); + ;} + break; + + case 114: +#line 856 "test_spec_parse.y" + { + /* current_pass_cmd set by the enclosing wait_cmd rule */ + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + (yyvsp[(1) - (1)].str), sizeof(current_pass_cmd->passThroughStates[0])); + ;} + break; + + case 115: +#line 864 "test_spec_parse.y" + { + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + (yyvsp[(1) - (1)].str), sizeof(current_pass_cmd->passThroughStates[0])); + free((yyvsp[(1) - (1)].str)); + ;} + break; + + case 116: +#line 872 "test_spec_parse.y" + { + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + (yyvsp[(3) - (3)].str), sizeof(current_pass_cmd->passThroughStates[0])); + ;} + break; + + case 117: +#line 879 "test_spec_parse.y" + { + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + (yyvsp[(3) - (3)].str), sizeof(current_pass_cmd->passThroughStates[0])); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 118: +#line 890 "test_spec_parse.y" + { current_pass_cmd = make_cmd(CMD_WAIT_STATE); + strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); + strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); + free((yyvsp[(3) - (6)].str)); ;} + break; + + case 119: +#line 895 "test_spec_parse.y" + { + current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); + (yyval.cmd) = current_pass_cmd; + current_pass_cmd = NULL; + ;} + break; + + case 120: +#line 901 "test_spec_parse.y" + { current_pass_cmd = make_cmd(CMD_WAIT_STATE); + strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); + strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); + free((yyvsp[(3) - (6)].str)); free((yyvsp[(6) - (6)].str)); ;} + break; + + case 121: +#line 906 "test_spec_parse.y" + { + current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); + (yyval.cmd) = current_pass_cmd; + current_pass_cmd = NULL; + ;} + break; + + case 122: +#line 912 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_STATE); + (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (7)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(6) - (7)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(7) - (7)].ival); + free((yyvsp[(3) - (7)].str)); + ;} + break; + + case 123: +#line 921 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_STATE); + (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (7)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(6) - (7)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(7) - (7)].ival); + free((yyvsp[(3) - (7)].str)); free((yyvsp[(6) - (7)].str)); + ;} + break; + + case 124: +#line 930 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_STOPPED); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(5) - (5)].ival); + free((yyvsp[(3) - (5)].str)); + ;} + break; + + case 125: +#line 937 "test_spec_parse.y" + { + (yyval.cmd) = current_wait_cmd; + (yyval.cmd)->timeoutSeconds = (yyvsp[(5) - (5)].ival); + current_wait_cmd = NULL; + ;} + break; + + case 126: +#line 951 "test_spec_parse.y" + { + (yyval.cmd) = current_wait_cmd; + (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); + current_wait_cmd = NULL; + ;} + break; + + case 127: +#line 966 "test_spec_parse.y" + { + current_wait_cmd = make_cmd(CMD_WAIT_STATES); + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + (yyvsp[(1) - (1)].str), sizeof(current_wait_cmd->waitStates[0])); + ;} + break; + + case 128: +#line 972 "test_spec_parse.y" + { + current_wait_cmd = make_cmd(CMD_WAIT_STATES); + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + (yyvsp[(1) - (1)].str), sizeof(current_wait_cmd->waitStates[0])); + free((yyvsp[(1) - (1)].str)); + ;} + break; + + case 129: +#line 979 "test_spec_parse.y" + { + if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + (yyvsp[(3) - (3)].str), sizeof(current_wait_cmd->waitStates[0])); + ;} + break; + + case 130: +#line 985 "test_spec_parse.y" + { + if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + (yyvsp[(3) - (3)].str), sizeof(current_wait_cmd->waitStates[0])); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 133: +#line 1004 "test_spec_parse.y" + { + if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) + current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(2) - (2)].ival); + ;} + break; + + case 134: +#line 1009 "test_spec_parse.y" + { + if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) + current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(4) - (4)].ival); + ;} + break; + + case 135: +#line 1016 "test_spec_parse.y" + { (yyval.ival) = PGAF_TIMEOUT_DEFAULT; ;} + break; + + case 136: +#line 1017 "test_spec_parse.y" + { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} + break; + + case 137: +#line 1018 "test_spec_parse.y" + { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} + break; + + case 138: +#line 1030 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(5) - (6)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); + free((yyvsp[(2) - (6)].str)); + ;} + break; + + case 139: +#line 1038 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(5) - (6)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); + free((yyvsp[(2) - (6)].str)); free((yyvsp[(5) - (6)].str)); + ;} + break; + + case 140: +#line 1046 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(5) - (6)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); + free((yyvsp[(2) - (6)].str)); + ;} + break; + + case 141: +#line 1054 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(5) - (6)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); + free((yyvsp[(2) - (6)].str)); free((yyvsp[(5) - (6)].str)); + ;} + break; + + case 142: +#line 1072 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_SQL); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->args)); + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 143: +#line 1087 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXPECT); + strlcpy((yyval.cmd)->expected, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->expected)); + expand_tuple_expect((yyval.cmd)->expected, sizeof((yyval.cmd)->expected)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 144: +#line 1094 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); + ;} + break; + + case 145: +#line 1098 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); + strlcpy((yyval.cmd)->state, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->state)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 146: +#line 1104 "test_spec_parse.y" + { + /* SQLSTATE codes like 25006 are all digits, lexed as T_INTEGER */ + (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); + snprintf((yyval.cmd)->state, sizeof((yyval.cmd)->state), "%d", (yyvsp[(3) - (3)].ival)); + ;} + break; + + case 147: +#line 1117 "test_spec_parse.y" + { + (yyval.cmd) = current_promote_cmd; + current_promote_cmd = NULL; + ;} + break; + + case 148: +#line 1125 "test_spec_parse.y" + { + current_promote_cmd = make_cmd(CMD_PROMOTE); + current_promote_cmd->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; + strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], + (yyvsp[(1) - (1)].str), sizeof(current_promote_cmd->promoteNodes[0])); + free((yyvsp[(1) - (1)].str)); + ;} + break; + + case 149: +#line 1133 "test_spec_parse.y" + { + if (current_promote_cmd->promoteCount < PGAF_MAX_PROMOTE_NODES) + strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], + (yyvsp[(3) - (3)].str), sizeof(current_promote_cmd->promoteNodes[0])); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 150: +#line 1148 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_NETWORK_OFF); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 151: +#line 1154 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_NETWORK_ON); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 152: +#line 1167 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_SLEEP); + (yyval.cmd)->timeoutSeconds = (yyvsp[(2) - (2)].ival); + ;} + break; + + case 153: +#line 1181 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_COMPOSE_DOWN); + ;} + break; + + case 154: +#line 1185 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_COMPOSE_START); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 155: +#line 1191 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_COMPOSE_STOP); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 156: +#line 1197 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_COMPOSE_KILL); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 157: +#line 1223 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_COMPOSE_INJECT); + strlcpy((yyval.cmd)->expected, (yyvsp[(3) - (4)].str), sizeof((yyval.cmd)->expected)); /* image */ + + /* Split T_SHELL_ARGS: " :" */ + char tmp[4096]; + strlcpy(tmp, (yyvsp[(4) - (4)].str), sizeof(tmp)); + char *src = tmp; + char *p = tmp; + while (*p && *p != ' ' && *p != '\t') p++; + if (*p) { *p++ = '\0'; while (*p == ' ' || *p == '\t') p++; } + char *svcdst = p; + char *colon = (*svcdst) ? strchr(svcdst, ':') : NULL; + strlcpy((yyval.cmd)->args, src, sizeof((yyval.cmd)->args)); + if (colon) + { + *colon = '\0'; + strlcpy((yyval.cmd)->service, svcdst, sizeof((yyval.cmd)->service)); /* dst svc */ + strlcpy((yyval.cmd)->state, colon + 1, sizeof((yyval.cmd)->state)); /* dst path */ + } + free((yyvsp[(3) - (4)].str)); free((yyvsp[(4) - (4)].str)); + ;} + break; + + case 158: +#line 1257 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_STOP_POSTGRES); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 159: +#line 1263 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_START_POSTGRES); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 160: +#line 1279 "test_spec_parse.y" + { pgaf_next_brace_is_while = 1; ;} + break; + + case 161: +#line 1280 "test_spec_parse.y" + { (yyval.step) = (yyvsp[(4) - (5)].step); ;} + break; + + case 162: +#line 1285 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_STAYS_WHILE); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->state, (yyvsp[(4) - (5)].str), sizeof((yyval.cmd)->state)); + (yyval.cmd)->body = ((yyvsp[(5) - (5)].step)) ? (yyvsp[(5) - (5)].step)->commands : NULL; + free((yyvsp[(2) - (5)].str)); + ;} + break; + + case 163: +#line 1304 "test_spec_parse.y" + { + /* only "set monitor " is supported; $2 must be "monitor" */ + if (strcmp((yyvsp[(2) - (3)].str), "monitor") != 0) + { + fprintf(stderr, "pgaftest: unknown 'set' target '%s' (expected 'monitor')\n", (yyvsp[(2) - (3)].str)); + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + YYERROR; + } + (yyval.cmd) = make_cmd(CMD_SET_MONITOR); + strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); + free((yyvsp[(2) - (3)].str)); free((yyvsp[(3) - (3)].str)); + ;} + break; + + case 164: +#line 1329 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(4) - (4)].str), sizeof((yyval.cmd)->args)); + (yyval.cmd)->logsNegate = false; + (yyval.cmd)->allowError = false; /* false = fixed string, true = PCRE */ + free((yyvsp[(2) - (4)].str)); free((yyvsp[(4) - (4)].str)); + ;} + break; + + case 165: +#line 1338 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(5) - (5)].str), sizeof((yyval.cmd)->args)); + (yyval.cmd)->logsNegate = true; + (yyval.cmd)->allowError = false; + free((yyvsp[(2) - (5)].str)); free((yyvsp[(5) - (5)].str)); + ;} + break; + + case 166: +#line 1347 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(4) - (4)].str), sizeof((yyval.cmd)->args)); + (yyval.cmd)->logsNegate = false; + (yyval.cmd)->allowError = true; /* true = PCRE (-P) */ + free((yyvsp[(2) - (4)].str)); free((yyvsp[(4) - (4)].str)); + ;} + break; + + case 167: +#line 1356 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); + strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(5) - (5)].str), sizeof((yyval.cmd)->args)); + (yyval.cmd)->logsNegate = true; + (yyval.cmd)->allowError = true; + free((yyvsp[(2) - (5)].str)); free((yyvsp[(5) - (5)].str)); + ;} + break; + + case 170: +#line 1377 "test_spec_parse.y" + { + int i = current_spec->sequenceLength; + if (i < PGAF_MAX_SEQ) + current_spec->sequence[current_spec->sequenceLength++] = (yyvsp[(2) - (2)].str); + else + { + fprintf(stderr, "pgaftest: too many steps in sequence (max %d)\n", + PGAF_MAX_SEQ); + exit(1); + } + ;} + break; + + case 171: +#line 1398 "test_spec_parse.y" + { (yyval.str) = "init"; ;} + break; + + case 172: +#line 1399 "test_spec_parse.y" + { (yyval.str) = "single"; ;} + break; + + case 173: +#line 1400 "test_spec_parse.y" + { (yyval.str) = "primary"; ;} + break; + + case 174: +#line 1401 "test_spec_parse.y" + { (yyval.str) = "wait_primary"; ;} + break; + + case 175: +#line 1402 "test_spec_parse.y" + { (yyval.str) = "wait_standby"; ;} + break; + + case 176: +#line 1403 "test_spec_parse.y" + { (yyval.str) = "demoted"; ;} + break; + + case 177: +#line 1404 "test_spec_parse.y" + { (yyval.str) = "demote_timeout"; ;} + break; + + case 178: +#line 1405 "test_spec_parse.y" + { (yyval.str) = "draining"; ;} + break; + + case 179: +#line 1406 "test_spec_parse.y" + { (yyval.str) = "secondary"; ;} + break; + + case 180: +#line 1407 "test_spec_parse.y" + { (yyval.str) = "catchingup"; ;} + break; + + case 181: +#line 1408 "test_spec_parse.y" + { (yyval.str) = "prepare_promotion"; ;} + break; + + case 182: +#line 1409 "test_spec_parse.y" + { (yyval.str) = "stop_replication"; ;} + break; + + case 183: +#line 1410 "test_spec_parse.y" + { (yyval.str) = "maintenance"; ;} + break; + + case 184: +#line 1411 "test_spec_parse.y" + { (yyval.str) = "join_primary"; ;} + break; + + case 185: +#line 1412 "test_spec_parse.y" + { (yyval.str) = "apply_settings"; ;} + break; + + case 186: +#line 1413 "test_spec_parse.y" + { (yyval.str) = "prepare_maintenance"; ;} + break; + + case 187: +#line 1414 "test_spec_parse.y" + { (yyval.str) = "wait_maintenance"; ;} + break; + + case 188: +#line 1415 "test_spec_parse.y" + { (yyval.str) = "report_lsn"; ;} + break; + + case 189: +#line 1416 "test_spec_parse.y" + { (yyval.str) = "fast_forward"; ;} + break; + + case 190: +#line 1417 "test_spec_parse.y" + { (yyval.str) = "join_secondary"; ;} + break; + + case 191: +#line 1418 "test_spec_parse.y" + { (yyval.str) = "dropped"; ;} + break; + + case 192: +#line 1426 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 193: +#line 1427 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + +/* Line 1267 of yacc.c. */ +#line 3360 "test_spec_parse.c" + default: break; + } + YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); + + YYPOPSTACK (yylen); + yylen = 0; + YY_STACK_PRINT (yyss, yyssp); + + *++yyvsp = yyval; + + + /* Now `shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + + yyn = yyr1[yyn]; + + yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; + if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) + yystate = yytable[yystate]; + else + yystate = yydefgoto[yyn - YYNTOKENS]; + + goto yynewstate; + + +/*------------------------------------. +| yyerrlab -- here on detecting error | +`------------------------------------*/ +yyerrlab: + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; +#if ! YYERROR_VERBOSE + yyerror (YY_("syntax error")); +#else + { + YYSIZE_T yysize = yysyntax_error (0, yystate, yychar); + if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) + { + YYSIZE_T yyalloc = 2 * yysize; + if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) + yyalloc = YYSTACK_ALLOC_MAXIMUM; + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = (char *) YYSTACK_ALLOC (yyalloc); + if (yymsg) + yymsg_alloc = yyalloc; + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + } + } + + if (0 < yysize && yysize <= yymsg_alloc) + { + (void) yysyntax_error (yymsg, yystate, yychar); + yyerror (yymsg); + } + else + { + yyerror (YY_("syntax error")); + if (yysize != 0) + goto yyexhaustedlab; + } + } +#endif + } + + + + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse look-ahead token after an + error, discard it. */ + + if (yychar <= YYEOF) + { + /* Return failure if at end of input. */ + if (yychar == YYEOF) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse look-ahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + + /* Pacify compilers like GCC when the user code never invokes + YYERROR and the label yyerrorlab therefore never appears in user + code. */ + if (/*CONSTCOND*/ 0) + goto yyerrorlab; + + /* Do not reclaim the symbols of the rule which action triggered + this YYERROR. */ + YYPOPSTACK (yylen); + yylen = 0; + YY_STACK_PRINT (yyss, yyssp); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + for (;;) + { + yyn = yypact[yystate]; + if (yyn != YYPACT_NINF) + { + yyn += YYTERROR; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + + yydestruct ("Error: popping", + yystos[yystate], yyvsp); + YYPOPSTACK (1); + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp); + } + + if (yyn == YYFINAL) + YYACCEPT; + + *++yyvsp = yylval; + + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturn; + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturn; + +#ifndef yyoverflow +/*-------------------------------------------------. +| yyexhaustedlab -- memory exhaustion comes here. | +`-------------------------------------------------*/ +yyexhaustedlab: + yyerror (YY_("memory exhausted")); + yyresult = 2; + /* Fall through. */ +#endif + +yyreturn: + if (yychar != YYEOF && yychar != YYEMPTY) + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval); + /* Do not reclaim the symbols of the rule which action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + yystos[*yyssp], yyvsp); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif +#if YYERROR_VERBOSE + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); +#endif + /* Make sure YYID is used. */ + return YYID (yyresult); +} + + +#line 1430 "test_spec_parse.y" + + +/* ----------------------------------------------------------------------- + * Public entry point + * ----------------------------------------------------------------------- */ + +TestSpec * +parse_test_spec(const char *filename) +{ + FILE *f = fopen(filename, "r"); + if (!f) + { + fprintf(stderr, "pgaftest: cannot open spec file \"%s\": %s\n", + filename, strerror(errno)); + return NULL; + } + + TestSpec *spec = (TestSpec *) calloc(1, sizeof(TestSpec)); + if (!spec) { fprintf(stderr, "out of memory\n"); exit(1); } + + strncpy(spec->filename, filename, sizeof(spec->filename)-1); + + current_spec = spec; + pgaf_line_number = 1; + yyin = f; + yyparse(); + fclose(f); + + return spec; +} + +TestCmd * +make_cmd(TestCmdKind kind) +{ + TestCmd *c = (TestCmd *) calloc(1, sizeof(TestCmd)); + if (!c) { fprintf(stderr, "out of memory\n"); exit(1); } + c->kind = kind; + c->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; + return c; +} + +TestStep * +make_step(const char *name) +{ + TestStep *s = (TestStep *) calloc(1, sizeof(TestStep)); + if (!s) { fprintf(stderr, "out of memory\n"); exit(1); } + if (name) strncpy(s->name, name, sizeof(s->name)-1); + return s; +} + +TestStep * +spec_find_step(TestSpec *spec, const char *name) +{ + for (TestStep *s = spec->steps; s; s = s->next) + { + if (strcmp(s->name, name) == 0) + return s; + } + return NULL; +} + diff --git a/src/bin/pgaftest/test_spec_parse.h b/src/bin/pgaftest/test_spec_parse.h new file mode 100644 index 000000000..d551e8ad7 --- /dev/null +++ b/src/bin/pgaftest/test_spec_parse.h @@ -0,0 +1,277 @@ +/* A Bison parser, made by GNU Bison 2.3. */ + +/* Skeleton interface for Bison's Yacc-like parsers in C + + Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 + Free Software Foundation, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* Tokens. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + /* Put the tokens into the symbol table, so that GDB and other debuggers + know about them. */ + enum yytokentype { + T_CLUSTER = 258, + T_MONITOR = 259, + T_NODE = 260, + T_CITUS_COORDINATOR = 261, + T_CITUS_WORKER = 262, + T_SETUP = 263, + T_TEARDOWN = 264, + T_STEP = 265, + T_SEQUENCE = 266, + T_EQUALS = 267, + T_IMAGE = 268, + T_IMAGE_TARGET = 269, + T_SSL = 270, + T_AUTH = 271, + T_AUTH_METHOD = 272, + T_FORMATION = 273, + T_NUM_SYNC = 274, + T_COORDINATOR = 275, + T_WORKER = 276, + T_ASYNC = 277, + T_NO_MONITOR = 278, + T_LAUNCH = 279, + T_DEFERRED = 280, + T_IMMEDIATE = 281, + T_INITIALLY = 282, + T_VOLUME = 283, + T_LISTEN = 284, + T_CITUS_SECONDARY = 285, + T_CANDIDATE_PRIORITY = 286, + T_PORT = 287, + T_PASSWORD = 288, + T_MONITOR_PASSWORD = 289, + T_CITUS_CLUSTER_NAME = 290, + T_DEBIAN_CLUSTER = 291, + T_REPLICATION_QUORUM = 292, + T_REPLICATION_PASSWORD = 293, + T_EXTENSION_VERSION = 294, + T_BIND_SOURCE = 295, + T_FS_INIT = 296, + T_FS_SINGLE = 297, + T_FS_PRIMARY = 298, + T_FS_WAIT_PRIMARY = 299, + T_FS_WAIT_STANDBY = 300, + T_FS_DEMOTED = 301, + T_FS_DEMOTE_TIMEOUT = 302, + T_FS_DRAINING = 303, + T_FS_SECONDARY = 304, + T_FS_CATCHINGUP = 305, + T_FS_PREP_PROMOTION = 306, + T_FS_STOP_REPLICATION = 307, + T_FS_MAINTENANCE = 308, + T_FS_JOIN_PRIMARY = 309, + T_FS_APPLY_SETTINGS = 310, + T_FS_PREPARE_MAINTENANCE = 311, + T_FS_WAIT_MAINTENANCE = 312, + T_FS_REPORT_LSN = 313, + T_FS_FAST_FORWARD = 314, + T_FS_JOIN_SECONDARY = 315, + T_FS_DROPPED = 316, + T_EXEC = 317, + T_EXEC_FAILS = 318, + T_PG_AUTOCTL = 319, + T_WAIT = 320, + T_UNTIL = 321, + T_TIMEOUT = 322, + T_AND = 323, + T_IS = 324, + T_WITH = 325, + T_ASSERT = 326, + T_SQL = 327, + T_EXPECT = 328, + T_ERROR = 329, + T_PROMOTE = 330, + T_NETWORK = 331, + T_DISCONNECT = 332, + T_CONNECT = 333, + T_SLEEP = 334, + T_COMPOSE = 335, + T_DOWN = 336, + T_START = 337, + T_STOP = 338, + T_STOPPED = 339, + T_KILL = 340, + T_INJECT = 341, + T_STATE = 342, + T_ASSIGNED_STATE = 343, + T_IN = 344, + T_GROUP = 345, + T_LBRACE = 346, + T_RBRACE = 347, + T_COMMA = 348, + T_POSTGRES = 349, + T_STAYS = 350, + T_WHILE = 351, + T_THROUGH = 352, + T_SET = 353, + T_LOGS = 354, + T_NOT = 355, + T_CONTAINS = 356, + T_MATCHES = 357, + T_INTEGER = 358, + T_IDENT = 359, + T_STRING = 360, + T_BLOCK = 361, + T_SHELL_ARGS = 362 + }; +#endif +/* Tokens. */ +#define T_CLUSTER 258 +#define T_MONITOR 259 +#define T_NODE 260 +#define T_CITUS_COORDINATOR 261 +#define T_CITUS_WORKER 262 +#define T_SETUP 263 +#define T_TEARDOWN 264 +#define T_STEP 265 +#define T_SEQUENCE 266 +#define T_EQUALS 267 +#define T_IMAGE 268 +#define T_IMAGE_TARGET 269 +#define T_SSL 270 +#define T_AUTH 271 +#define T_AUTH_METHOD 272 +#define T_FORMATION 273 +#define T_NUM_SYNC 274 +#define T_COORDINATOR 275 +#define T_WORKER 276 +#define T_ASYNC 277 +#define T_NO_MONITOR 278 +#define T_LAUNCH 279 +#define T_DEFERRED 280 +#define T_IMMEDIATE 281 +#define T_INITIALLY 282 +#define T_VOLUME 283 +#define T_LISTEN 284 +#define T_CITUS_SECONDARY 285 +#define T_CANDIDATE_PRIORITY 286 +#define T_PORT 287 +#define T_PASSWORD 288 +#define T_MONITOR_PASSWORD 289 +#define T_CITUS_CLUSTER_NAME 290 +#define T_DEBIAN_CLUSTER 291 +#define T_REPLICATION_QUORUM 292 +#define T_REPLICATION_PASSWORD 293 +#define T_EXTENSION_VERSION 294 +#define T_BIND_SOURCE 295 +#define T_FS_INIT 296 +#define T_FS_SINGLE 297 +#define T_FS_PRIMARY 298 +#define T_FS_WAIT_PRIMARY 299 +#define T_FS_WAIT_STANDBY 300 +#define T_FS_DEMOTED 301 +#define T_FS_DEMOTE_TIMEOUT 302 +#define T_FS_DRAINING 303 +#define T_FS_SECONDARY 304 +#define T_FS_CATCHINGUP 305 +#define T_FS_PREP_PROMOTION 306 +#define T_FS_STOP_REPLICATION 307 +#define T_FS_MAINTENANCE 308 +#define T_FS_JOIN_PRIMARY 309 +#define T_FS_APPLY_SETTINGS 310 +#define T_FS_PREPARE_MAINTENANCE 311 +#define T_FS_WAIT_MAINTENANCE 312 +#define T_FS_REPORT_LSN 313 +#define T_FS_FAST_FORWARD 314 +#define T_FS_JOIN_SECONDARY 315 +#define T_FS_DROPPED 316 +#define T_EXEC 317 +#define T_EXEC_FAILS 318 +#define T_PG_AUTOCTL 319 +#define T_WAIT 320 +#define T_UNTIL 321 +#define T_TIMEOUT 322 +#define T_AND 323 +#define T_IS 324 +#define T_WITH 325 +#define T_ASSERT 326 +#define T_SQL 327 +#define T_EXPECT 328 +#define T_ERROR 329 +#define T_PROMOTE 330 +#define T_NETWORK 331 +#define T_DISCONNECT 332 +#define T_CONNECT 333 +#define T_SLEEP 334 +#define T_COMPOSE 335 +#define T_DOWN 336 +#define T_START 337 +#define T_STOP 338 +#define T_STOPPED 339 +#define T_KILL 340 +#define T_INJECT 341 +#define T_STATE 342 +#define T_ASSIGNED_STATE 343 +#define T_IN 344 +#define T_GROUP 345 +#define T_LBRACE 346 +#define T_RBRACE 347 +#define T_COMMA 348 +#define T_POSTGRES 349 +#define T_STAYS 350 +#define T_WHILE 351 +#define T_THROUGH 352 +#define T_SET 353 +#define T_LOGS 354 +#define T_NOT 355 +#define T_CONTAINS 356 +#define T_MATCHES 357 +#define T_INTEGER 358 +#define T_IDENT 359 +#define T_STRING 360 +#define T_BLOCK 361 +#define T_SHELL_ARGS 362 + + + + +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef union YYSTYPE +#line 145 "test_spec_parse.y" +{ + int ival; + char *str; + TestStep *step; + TestCmd *cmd; +} +/* Line 1529 of yacc.c. */ +#line 270 "test_spec_parse.h" + YYSTYPE; +# define yystype YYSTYPE /* obsolescent; will be withdrawn */ +# define YYSTYPE_IS_DECLARED 1 +# define YYSTYPE_IS_TRIVIAL 1 +#endif + +extern YYSTYPE yylval; + diff --git a/src/bin/pgaftest/test_spec_parse.y b/src/bin/pgaftest/test_spec_parse.y new file mode 100644 index 000000000..9af5fc113 --- /dev/null +++ b/src/bin/pgaftest/test_spec_parse.y @@ -0,0 +1,1489 @@ +%{ +/* + * src/bin/pgaftest/test_spec_parse.y + * Bison grammar for .pgaf test specification files. + * + * The outer structure (cluster, setup, teardown, step, sequence) is + * described here as bison rules. Inside step/setup/teardown bodies + * the flex lexer enters the STEP_BODY exclusive state and returns + * individual tokens for every keyword, identifier, integer, and + * punctuation character — no more hand-written strstr/strtok parsing. + * + * The cluster { } block is now also fully parsed by this grammar. + * The flex lexer enters the CLUSTER_BODY exclusive state when it sees + * the opening '{' after "cluster", returning proper tokens for every + * keyword, value, and punctuation inside. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include +#include + +#include "test_spec.h" +#include "pgsetup.h" +#include "file_utils.h" + +/* provided by test_spec_scan.l */ +extern int yylex(void); +extern int pgaf_line_number; +extern FILE *yyin; +extern int pgaf_next_brace_is_while; /* set before T_LBRACE for while body */ + +/* the spec we're building */ +static TestSpec *current_spec = NULL; + +static void yyerror(const char *msg) +{ + fprintf(stderr, "pgaftest: parse error at line %d: %s\n", + pgaf_line_number, msg); + exit(1); +} + +/* helpers */ +static void append_cmd(TestStep *step, TestCmd *cmd) +{ + if (!step->commands) + { + step->commands = cmd; + } + else + { + TestCmd *c = step->commands; + while (c->next) c = c->next; + c->next = cmd; + } +} + +/* + * expand_tuple_expect — convert `{ r1 } { r2 }` tuple syntax into + * the newline-separated form that psql --tuples-only --no-align produces. + */ +static void +expand_tuple_expect(char *buf, int buflen) +{ + const char *p = buf; + while (*p == ' ' || *p == '\t') p++; + + if (p[0] != '{' || (p[1] != ' ' && p[1] != '\t')) + return; + + char tmp[4096] = { 0 }; + int pos = 0; + bool first = true; + + while (*p) + { + while (*p == ' ' || *p == '\t' || *p == '\n') p++; + if (*p == '\0') break; + if (*p != '{') break; + + p++; + while (*p == ' ' || *p == '\t') p++; + + char row[1024] = { 0 }; + int ri = 0; + int depth = 1; + while (*p && depth > 0) + { + if (*p == '{') depth++; + else if (*p == '}') { if (--depth == 0) break; } + if (depth > 0 && ri < (int)sizeof(row) - 1) + row[ri++] = *p; + p++; + } + if (*p == '}') p++; + + while (ri > 0 && (row[ri-1] == ' ' || row[ri-1] == '\t')) ri--; + row[ri] = '\0'; + + if (!first && pos < (int)sizeof(tmp) - 1) + tmp[pos++] = '\n'; + int l = ri; + if (pos + l >= (int)sizeof(tmp)) l = (int)sizeof(tmp) - pos - 1; + if (l > 0) { memcpy(tmp + pos, row, l); pos += l; } + tmp[pos] = '\0'; + first = false; + } + + if (!first) + strncpy(buf, tmp, buflen - 1); +} + +static void register_step(TestSpec *spec, TestStep *step) +{ + if (!spec->steps) + { + spec->steps = step; + } + else + { + TestStep *s = spec->steps; + while (s->next) s = s->next; + s->next = step; + } + spec->stepCount++; +} + +/* ----------------------------------------------------------------------- + * Static state used by multi-element grammar rules. + * + * The parser is single-threaded; these are only live during the reduction + * of a single rule so there is no re-entrancy concern. + * ----------------------------------------------------------------------- */ + +static TestCmd *current_wait_cmd = NULL; +static TestCmd *current_promote_cmd = NULL; +static TestCmd *current_pass_cmd = NULL; /* for opt_passing_through */ +static TestFormation *current_formation = NULL; +static TestNode *current_node = NULL; + +%} + +%union { + int ival; + char *str; + TestStep *step; + TestCmd *cmd; +} + +/* ---- Outer-structure tokens (used in INITIAL lex state) ---- */ +%token T_CLUSTER T_MONITOR T_NODE T_CITUS_COORDINATOR T_CITUS_WORKER +%token T_SETUP T_TEARDOWN T_STEP T_SEQUENCE +%token T_EQUALS + +/* ---- Cluster-body tokens ---- */ +%token T_IMAGE T_IMAGE_TARGET T_SSL T_AUTH T_AUTH_METHOD T_FORMATION T_NUM_SYNC +%token T_COORDINATOR T_WORKER T_ASYNC T_NO_MONITOR +%token T_LAUNCH T_DEFERRED T_IMMEDIATE T_INITIALLY T_VOLUME +%token T_LISTEN T_CITUS_SECONDARY T_CANDIDATE_PRIORITY T_PORT T_PASSWORD T_MONITOR_PASSWORD +%token T_CITUS_CLUSTER_NAME T_DEBIAN_CLUSTER T_REPLICATION_QUORUM T_REPLICATION_PASSWORD +%token T_EXTENSION_VERSION T_BIND_SOURCE + +/* ---- FSM state tokens (used in CLUSTER_BODY and STEP_BODY) ---- */ +%token T_FS_INIT T_FS_SINGLE T_FS_PRIMARY +%token T_FS_WAIT_PRIMARY T_FS_WAIT_STANDBY +%token T_FS_DEMOTED T_FS_DEMOTE_TIMEOUT T_FS_DRAINING +%token T_FS_SECONDARY T_FS_CATCHINGUP +%token T_FS_PREP_PROMOTION T_FS_STOP_REPLICATION +%token T_FS_MAINTENANCE T_FS_JOIN_PRIMARY T_FS_APPLY_SETTINGS +%token T_FS_PREPARE_MAINTENANCE T_FS_WAIT_MAINTENANCE +%token T_FS_REPORT_LSN T_FS_FAST_FORWARD T_FS_JOIN_SECONDARY +%token T_FS_DROPPED + +/* ---- Step-body tokens (used in STEP_BODY lex state) ---- */ +%token T_EXEC T_EXEC_FAILS T_PG_AUTOCTL +%token T_WAIT T_UNTIL T_TIMEOUT T_AND T_IS T_WITH +%token T_ASSERT +%token T_SQL T_EXPECT T_ERROR +%token T_PROMOTE +%token T_NETWORK T_DISCONNECT T_CONNECT +%token T_SLEEP +%token T_COMPOSE T_DOWN T_START T_STOP T_STOPPED T_KILL T_INJECT +%token T_STATE T_ASSIGNED_STATE +%token T_IN T_GROUP +%token T_LBRACE T_RBRACE T_COMMA +%token T_POSTGRES T_STAYS T_WHILE T_THROUGH T_SET +%token T_LOGS T_NOT T_CONTAINS T_MATCHES + +/* ---- Tokens with values ---- */ +%token T_INTEGER +%token T_IDENT T_STRING T_BLOCK T_SHELL_ARGS + +/* ---- Non-terminal types ---- */ +%type ident_or_string +%type bare_name +%type fsm_state +%type node_name +%type cmd_block cmd_list +%type step_cmd +%type exec_cmd wait_cmd assert_cmd sql_cmd expect_cmd +%type promote_cmd network_cmd sleep_cmd compose_cmd +%type postgres_ctl_cmd stays_while_cmd set_monitor_cmd logs_cmd +%type opt_timeout +%type while_body + +%% + +spec: + spec_item + | spec spec_item + ; + +spec_item: + cluster_block + | setup_block + | teardown_block + | named_step + | sequence_block + ; + +/* ----------------------------------------------------------------------- + * cluster { } + * + * The flex lexer enters CLUSTER_BODY on the opening '{' and returns to + * INITIAL on the outermost closing '}'. Every keyword and value inside + * the cluster block is a proper token — no hand-written string parsing. + * ----------------------------------------------------------------------- */ + +cluster_block: + T_CLUSTER T_LBRACE + { + strlcpy(current_spec->cluster.ssl, "self-signed", + sizeof(current_spec->cluster.ssl)); + strlcpy(current_spec->cluster.auth, "trust", + sizeof(current_spec->cluster.auth)); + } + cluster_item_list T_RBRACE + ; + +cluster_item_list: + /* empty */ + | cluster_item_list cluster_item + ; + +cluster_item: + monitor_line + | image_line + | ssl_line + | auth_line + | extension_version_line + | formation_block + | T_BIND_SOURCE { current_spec->cluster.bindSource = true; } + ; + +/* + * monitor [port N] + * + * The monitor keyword may optionally be followed by "port N". The ssl and + * auth settings for the monitor are handled by the top-level ssl_line and + * auth_line rules (they write to the same TestCluster fields), so we do not + * duplicate them here. Keeping only T_PORT avoids shift/reduce conflicts + * with ssl_line and auth_line in cluster_item_list. + */ +monitor_line: + T_MONITOR + { + current_spec->cluster.withMonitor = true; + } + | T_MONITOR T_DEBIAN_CLUSTER T_IDENT + { + current_spec->cluster.withMonitor = true; + strlcpy(current_spec->cluster.monitorDebianCluster, $3, + sizeof(current_spec->cluster.monitorDebianCluster)); + free($3); + } + | T_MONITOR T_IMAGE_TARGET T_IDENT + { + current_spec->cluster.withMonitor = true; + strlcpy(current_spec->cluster.monitorImageTarget, $3, + sizeof(current_spec->cluster.monitorImageTarget)); + free($3); + } + | T_MONITOR T_PORT T_INTEGER + { + current_spec->cluster.withMonitor = true; + /* monitor port not stored in TestCluster yet; ignore */ + (void) $3; + } + | T_MONITOR T_PASSWORD T_STRING + { + current_spec->cluster.withMonitor = true; + strlcpy(current_spec->cluster.monitorPassword, $3, + sizeof(current_spec->cluster.monitorPassword)); + free($3); + } + | T_MONITOR T_IDENT T_LAUNCH T_DEFERRED + { + strlcpy(current_spec->cluster.secondMonitorName, $2, + sizeof(current_spec->cluster.secondMonitorName)); + current_spec->cluster.secondMonitorStopped = true; + free($2); + } + | T_MONITOR T_IDENT T_INITIALLY T_STOPPED + { + strlcpy(current_spec->cluster.secondMonitorName, $2, + sizeof(current_spec->cluster.secondMonitorName)); + current_spec->cluster.secondMonitorStopped = true; + free($2); + } + | T_MONITOR T_IDENT T_LAUNCH T_DEFERRED T_PASSWORD T_STRING + { + strlcpy(current_spec->cluster.secondMonitorName, $2, + sizeof(current_spec->cluster.secondMonitorName)); + current_spec->cluster.secondMonitorStopped = true; + free($2); + /* password for second monitor not yet stored */ + free($6); + } + ; + +/* image "tag" | image tag */ +image_line: + T_IMAGE T_STRING + { + strlcpy(current_spec->cluster.image, $2, + sizeof(current_spec->cluster.image)); + free($2); + } + | T_IMAGE T_IDENT + { + strlcpy(current_spec->cluster.image, $2, + sizeof(current_spec->cluster.image)); + free($2); + } + ; + +/* extension-version VALUE — sets PG_AUTOCTL_EXTENSION_VERSION on the monitor */ +extension_version_line: + T_EXTENSION_VERSION T_IDENT + { + strlcpy(current_spec->cluster.extensionVersion, $2, + sizeof(current_spec->cluster.extensionVersion)); + free($2); + } + | T_EXTENSION_VERSION T_STRING + { + strlcpy(current_spec->cluster.extensionVersion, $2, + sizeof(current_spec->cluster.extensionVersion)); + free($2); + } + ; + +/* ssl VALUE */ +ssl_line: + T_SSL T_IDENT + { + strlcpy(current_spec->cluster.ssl, $2, + sizeof(current_spec->cluster.ssl)); + free($2); + } + ; + +/* auth VALUE | auth-method VALUE */ +auth_line: + T_AUTH T_IDENT + { + strlcpy(current_spec->cluster.auth, $2, + sizeof(current_spec->cluster.auth)); + free($2); + } + | T_AUTH_METHOD T_IDENT + { + strlcpy(current_spec->cluster.auth, $2, + sizeof(current_spec->cluster.auth)); + free($2); + } + ; + +/* formation [name] [num-sync N] { node_list } */ +formation_block: + T_FORMATION + { + TestCluster *cl = ¤t_spec->cluster; + if (cl->formationCount >= PGAF_MAX_FORMATIONS) + { + fprintf(stderr, "pgaftest: too many formations (max %d)\n", + PGAF_MAX_FORMATIONS); + exit(1); + } + current_formation = &cl->formations[cl->formationCount++]; + strlcpy(current_formation->name, "default", + sizeof(current_formation->name)); + current_formation->numSync = -1; + } + formation_opt_list T_LBRACE node_list T_RBRACE + ; + +formation_opt_list: + /* empty */ + | formation_opt_list formation_opt + ; + +/* + * bare_name allows any identifier or quoted string as a name, plus keywords + * that are likely to be used as formation/node names (e.g. "auth", "node", + * "monitor"). Using a keyword as a name is a common source of parse errors. + */ +bare_name: + T_IDENT { $$ = $1; } + | T_STRING { $$ = $1; } + | T_AUTH { $$ = strdup("auth"); } + | T_MONITOR { $$ = strdup("monitor"); } + | T_NODE { $$ = strdup("node"); } + ; + +formation_opt: + bare_name + { + strlcpy(current_formation->name, $1, sizeof(current_formation->name)); + free($1); + } + | T_NUM_SYNC T_INTEGER + { + current_formation->numSync = $2; + } + ; + +node_list: + /* empty */ + | node_list node_line + ; + +/* + * node_name — the first token on a node line. + * + * Node names are usually plain identifiers like "node1", "coord", "w1". + * They can also collide with reserved cluster keywords (e.g. a node named + * "monitor" or "node"). We allow T_MONITOR and T_NODE here so the grammar + * doesn't choke on such names. The FSM-state catch-all handles any state + * name used as a node name (unlikely but defensive). + */ +/* + * node_name covers bare-identifier node names used in the flat syntax. + * T_NODE is intentionally excluded: "node" is reserved for the block syntax + * "node foo { ... }" and must not be reduced here to avoid a shift-reduce + * conflict with T_NODE T_IDENT T_LBRACE. + */ +node_name: + T_IDENT { $$ = $1; } + | T_MONITOR { $$ = strdup("monitor"); } + ; + +/* + * init_node_slot — shared mid-rule action that allocates a node slot and + * sets defaults. Used by both node_line productions. + */ +init_node_slot: + /* empty */ + { + if (current_formation->nodeCount >= PGAF_MAX_NODES) + { + fprintf(stderr, "pgaftest: too many nodes in formation (max %d)\n", + PGAF_MAX_NODES); + exit(1); + } + current_node = ¤t_formation->nodes[current_formation->nodeCount++]; + current_node->kind = NODE_KIND_STANDALONE; + current_node->candidatePriority = 50; + current_node->replicationQuorum = true; + } + ; + +node_line: + /* flat syntax: node1 listen port 5432 ... */ + node_name init_node_slot + { + strlcpy(current_node->name, $1, sizeof(current_node->name)); + free($1); + } + node_opt_list + /* block syntax: node foo { listen \n port 5432 \n ... } */ + | T_NODE T_IDENT init_node_slot + { + strlcpy(current_node->name, $2, sizeof(current_node->name)); + free($2); + } + T_LBRACE node_opt_list T_RBRACE + ; + +node_opt_list: + /* empty */ + | node_opt_list node_opt + ; + +node_opt: + T_COORDINATOR + { + current_node->kind = NODE_KIND_CITUS_COORDINATOR; + current_spec->cluster.withCitus = true; + } + | T_WORKER + { + current_node->kind = NODE_KIND_CITUS_WORKER; + current_spec->cluster.withCitus = true; + } + | T_ASYNC + { + current_node->replicationQuorum = false; + } + | T_NO_MONITOR + { + current_node->noMonitor = true; + } + | T_DEFERRED + { + current_node->launchDeferred = true; + } + | T_LAUNCH T_DEFERRED + { + current_node->launchDeferred = true; + } + | T_LAUNCH T_IMMEDIATE + { + current_node->launchDeferred = false; + } + | T_IMMEDIATE + { + current_node->launchDeferred = false; + } + | T_LISTEN + { + current_node->listen = true; + } + | T_CITUS_SECONDARY + { + current_node->citusSecondary = true; + } + | T_CANDIDATE_PRIORITY T_INTEGER + { + current_node->candidatePriority = $2; + } + | T_GROUP T_INTEGER + { + current_node->group = $2; + } + | T_PORT T_INTEGER + { + current_node->pgPort = $2; + } + | T_CITUS_CLUSTER_NAME T_IDENT + { + strlcpy(current_node->citusClusterName, $2, + sizeof(current_node->citusClusterName)); + free($2); + } + | T_DEBIAN_CLUSTER T_IDENT + { + strlcpy(current_node->debianCluster, $2, + sizeof(current_node->debianCluster)); + free($2); + } + | T_SSL T_IDENT + { + strlcpy(current_node->ssl, $2, sizeof(current_node->ssl)); + free($2); + } + | T_AUTH T_IDENT + { + strlcpy(current_node->auth, $2, sizeof(current_node->auth)); + free($2); + } + | T_AUTH_METHOD T_IDENT + { + strlcpy(current_node->auth, $2, sizeof(current_node->auth)); + free($2); + } + | T_REPLICATION_QUORUM T_IDENT + { + if (strcmp($2, "false") == 0 || strcmp($2, "0") == 0) + current_node->replicationQuorum = false; + else + current_node->replicationQuorum = true; + free($2); + } + | T_REPLICATION_PASSWORD T_STRING + { + strlcpy(current_node->replicationPassword, $2, + sizeof(current_node->replicationPassword)); + free($2); + } + | T_MONITOR_PASSWORD T_STRING + { + strlcpy(current_node->monitorPassword, $2, + sizeof(current_node->monitorPassword)); + free($2); + } + | T_VOLUME T_IDENT T_IDENT + { + /* volume — adds a named Docker volume */ + int vi = current_node->volumeCount; + if (vi < PGAF_MAX_NODE_VOLUMES) + { + strlcpy(current_node->volumes[vi].name, $2, + sizeof(current_node->volumes[0].name)); + strlcpy(current_node->volumes[vi].path, $3, + sizeof(current_node->volumes[0].path)); + current_node->volumeCount++; + } + free($2); free($3); + } + | T_VOLUME T_IDENT T_STRING + { + /* volume "/path/with spaces" */ + int vi = current_node->volumeCount; + if (vi < PGAF_MAX_NODE_VOLUMES) + { + strlcpy(current_node->volumes[vi].name, $2, + sizeof(current_node->volumes[0].name)); + strlcpy(current_node->volumes[vi].path, $3, + sizeof(current_node->volumes[0].path)); + current_node->volumeCount++; + } + free($2); free($3); + } + ; + +/* ----------------------------------------------------------------------- + * setup { } and teardown { } + * ----------------------------------------------------------------------- */ + +setup_block: + T_SETUP cmd_block + { + current_spec->setup = $2; + } + ; + +teardown_block: + T_TEARDOWN cmd_block + { + current_spec->teardown = $2; + } + ; + +/* ----------------------------------------------------------------------- + * step { } + * ----------------------------------------------------------------------- */ + +named_step: + T_STEP ident_or_string cmd_block + { + TestStep *s = $3; + strncpy(s->name, $2, sizeof(s->name) - 1); + free($2); + register_step(current_spec, s); + } + ; + +/* ----------------------------------------------------------------------- + * Step body: T_LBRACE ... T_RBRACE + * + * The flex lexer enters STEP_BODY state when it sees the opening T_LBRACE + * and returns to INITIAL state after the closing T_RBRACE. Inside, every + * keyword and value is a distinct token — no strstr/strtok parsing. + * ----------------------------------------------------------------------- */ + +cmd_block: + T_LBRACE cmd_list T_RBRACE + { + /* post-process: CMD_SQL immediately before CMD_EXPECT_ERROR */ + for (TestCmd *c = $2->commands; c; c = c->next) + { + if (c->kind == CMD_SQL && c->next && + c->next->kind == CMD_EXPECT_ERROR) + c->allowError = true; + } + $$ = $2; + } + ; + +cmd_list: + /* empty */ + { + $$ = make_step(""); + } + | cmd_list step_cmd + { + if ($2) append_cmd($1, $2); + $$ = $1; + } + ; + +step_cmd: + exec_cmd { $$ = $1; } + | wait_cmd { $$ = $1; } + | assert_cmd { $$ = $1; } + | sql_cmd { $$ = $1; } + | expect_cmd { $$ = $1; } + | promote_cmd { $$ = $1; } + | network_cmd { $$ = $1; } + | sleep_cmd { $$ = $1; } + | compose_cmd { $$ = $1; } + | postgres_ctl_cmd { $$ = $1; } + | stays_while_cmd { $$ = $1; } + | set_monitor_cmd { $$ = $1; } + | logs_cmd { $$ = $1; } + ; + +/* ----------------------------------------------------------------------- + * exec + * exec-fails + * + * The flex EXEC_ARGS / EXEC_ARGS_REST states return: + * T_EXEC or T_EXEC_FAILS + * T_IDENT (the service name) + * T_SHELL_ARGS (rest of line, trimmed; omitted when line is empty) + * ----------------------------------------------------------------------- */ + +exec_cmd: + T_EXEC T_IDENT T_SHELL_ARGS + { + $$ = make_cmd(CMD_EXEC); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $3, sizeof($$->args)); + free($2); free($3); + } + | T_EXEC T_IDENT + { + $$ = make_cmd(CMD_EXEC); + strlcpy($$->service, $2, sizeof($$->service)); + free($2); + } + | T_EXEC_FAILS T_IDENT T_SHELL_ARGS + { + $$ = make_cmd(CMD_EXEC_FAILS); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $3, sizeof($$->args)); + free($2); free($3); + } + | T_EXEC_FAILS T_IDENT + { + $$ = make_cmd(CMD_EXEC_FAILS); + strlcpy($$->service, $2, sizeof($$->service)); + free($2); + } + | T_PG_AUTOCTL T_IDENT T_SHELL_ARGS + { + /* "pg_autoctl perform failover --formation auth" + * EXEC_ARGS returns T_IDENT for first word, T_SHELL_ARGS for rest */ + $$ = make_cmd(CMD_PG_AUTOCTL); + sformat($$->args, sizeof($$->args), "%s %s", $2, $3); + free($2); free($3); + } + | T_PG_AUTOCTL T_IDENT + { + $$ = make_cmd(CMD_PG_AUTOCTL); + strlcpy($$->args, $2, sizeof($$->args)); + free($2); + } + | T_PG_AUTOCTL + { + $$ = make_cmd(CMD_PG_AUTOCTL); + } + ; + +/* ----------------------------------------------------------------------- + * wait until ... + * + * Four forms, disambiguated cleanly by the token immediately after the + * first T_IDENT (one token of LALR lookahead is sufficient): + * + * wait until state = [timeout Ns] + * wait until assigned-state = [timeout Ns] + * wait until stopped [timeout Ns] + * wait until [, ...] [in group N[, group M]] [timeout Ns] + * + * FSM state names (T_FS_*) are accepted wherever a state string is expected. + * T_IDENT covers node names (e.g. "node1") and any future unknown states. + * ----------------------------------------------------------------------- */ + +/* + * state_op: accept '=' or 'is' as state comparison operator. + * This lets writers use either form: + * wait until node1 state = primary + * wait until node1 state is primary + */ +state_op: T_EQUALS | T_IS ; + +/* + * wait_multi_condition_list: one or more "node state [is|=] state" conditions + * joined by 'and'. Builds a CMD_WAIT_MULTI command via the current_wait_cmd + * global. + * + * wait until node2 state is primary and node1 state is secondary timeout 90s + * wait until node2 state is primary and node1 state is secondary with timeout 90s + */ +wait_multi_condition: + T_IDENT T_STATE state_op fsm_state + { + if (!current_wait_cmd) + current_wait_cmd = make_cmd(CMD_WAIT_MULTI); + int i = current_wait_cmd->waitStateCount; + if (i < PGAF_MAX_WAIT_STATES) + { + strlcpy(current_wait_cmd->waitNodes[i], $1, + sizeof(current_wait_cmd->waitNodes[0])); + strlcpy(current_wait_cmd->waitStates[i], $4, + sizeof(current_wait_cmd->waitStates[0])); + current_wait_cmd->waitStateCount++; + } + free($1); + } + | T_IDENT T_STATE state_op T_IDENT + { + if (!current_wait_cmd) + current_wait_cmd = make_cmd(CMD_WAIT_MULTI); + int i = current_wait_cmd->waitStateCount; + if (i < PGAF_MAX_WAIT_STATES) + { + strlcpy(current_wait_cmd->waitNodes[i], $1, + sizeof(current_wait_cmd->waitNodes[0])); + strlcpy(current_wait_cmd->waitStates[i], $4, + sizeof(current_wait_cmd->waitStates[0])); + current_wait_cmd->waitStateCount++; + } + free($1); free($4); + } + ; + +wait_multi_condition_list: + wait_multi_condition + | wait_multi_condition_list T_AND wait_multi_condition + ; + +/* + * opt_passing_through — optional "passing through s1, s2, ..." clause. + * + * These are intermediate FSM states that the node is expected to transit + * through on the way to the target state. The runner verifies them via + * LISTEN/NOTIFY on the monitor's state-change notifications. Missing any + * listed intermediate state is a test failure. + * + * Syntax (after the target state, before timeout): + * wait until node1 state is primary passing through wait_primary timeout 90s + */ + +opt_passing_through: + /* empty */ + | T_THROUGH pass_state_list + ; + +pass_state_list: + fsm_state + { + /* current_pass_cmd set by the enclosing wait_cmd rule */ + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + $1, sizeof(current_pass_cmd->passThroughStates[0])); + } + | T_IDENT + { + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + $1, sizeof(current_pass_cmd->passThroughStates[0])); + free($1); + } + | pass_state_list T_COMMA fsm_state + { + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + $3, sizeof(current_pass_cmd->passThroughStates[0])); + } + | pass_state_list T_COMMA T_IDENT + { + if (current_pass_cmd && + current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_pass_cmd->passThroughStates[current_pass_cmd->passThroughCount++], + $3, sizeof(current_pass_cmd->passThroughStates[0])); + free($3); + } + ; + +wait_cmd: + T_WAIT T_UNTIL T_IDENT T_STATE state_op fsm_state + { current_pass_cmd = make_cmd(CMD_WAIT_STATE); + strlcpy(current_pass_cmd->service, $3, sizeof(current_pass_cmd->service)); + strlcpy(current_pass_cmd->state, $6, sizeof(current_pass_cmd->state)); + free($3); } + opt_passing_through opt_timeout + { + current_pass_cmd->timeoutSeconds = $9; + $$ = current_pass_cmd; + current_pass_cmd = NULL; + } + | T_WAIT T_UNTIL T_IDENT T_STATE state_op T_IDENT + { current_pass_cmd = make_cmd(CMD_WAIT_STATE); + strlcpy(current_pass_cmd->service, $3, sizeof(current_pass_cmd->service)); + strlcpy(current_pass_cmd->state, $6, sizeof(current_pass_cmd->state)); + free($3); free($6); } + opt_passing_through opt_timeout + { + current_pass_cmd->timeoutSeconds = $9; + $$ = current_pass_cmd; + current_pass_cmd = NULL; + } + | T_WAIT T_UNTIL T_IDENT T_ASSIGNED_STATE state_op fsm_state opt_timeout + { + $$ = make_cmd(CMD_WAIT_STATE); + $$->kind = CMD_ASSERT_ASSIGNED; + strlcpy($$->service, $3, sizeof($$->service)); + strlcpy($$->state, $6, sizeof($$->state)); + $$->timeoutSeconds = $7; + free($3); + } + | T_WAIT T_UNTIL T_IDENT T_ASSIGNED_STATE state_op T_IDENT opt_timeout + { + $$ = make_cmd(CMD_WAIT_STATE); + $$->kind = CMD_ASSERT_ASSIGNED; + strlcpy($$->service, $3, sizeof($$->service)); + strlcpy($$->state, $6, sizeof($$->state)); + $$->timeoutSeconds = $7; + free($3); free($6); + } + | T_WAIT T_UNTIL T_IDENT T_STOPPED opt_timeout + { + $$ = make_cmd(CMD_WAIT_STOPPED); + strlcpy($$->service, $3, sizeof($$->service)); + $$->timeoutSeconds = $5; + free($3); + } + | T_WAIT T_UNTIL state_name_list opt_in_group opt_timeout + { + $$ = current_wait_cmd; + $$->timeoutSeconds = $5; + current_wait_cmd = NULL; + } + /* + * Multi-condition form: + * wait until node2 state is primary and node1 state is secondary timeout 90s + * wait until node2 state is primary and node1 state is secondary with timeout 90s + * + * Parsed via wait_multi_condition_list which populates current_wait_cmd. + * Requires at least two conditions (single-condition uses the forms above). + */ + | T_WAIT T_UNTIL wait_multi_condition T_AND wait_multi_condition_list opt_timeout + { + $$ = current_wait_cmd; + $$->timeoutSeconds = $6; + current_wait_cmd = NULL; + } + ; + +/* + * state_name_list — one or more FSM state names separated by commas. + * + * Accepts both T_FS_* tokens (proper FSM state names) and T_IDENT + * (for forward-compatibility with unknown states). + */ +state_name_list: + fsm_state + { + current_wait_cmd = make_cmd(CMD_WAIT_STATES); + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + $1, sizeof(current_wait_cmd->waitStates[0])); + } + | T_IDENT + { + current_wait_cmd = make_cmd(CMD_WAIT_STATES); + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + $1, sizeof(current_wait_cmd->waitStates[0])); + free($1); + } + | state_name_list T_COMMA fsm_state + { + if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + $3, sizeof(current_wait_cmd->waitStates[0])); + } + | state_name_list T_COMMA T_IDENT + { + if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) + strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], + $3, sizeof(current_wait_cmd->waitStates[0])); + free($3); + } + ; + +/* + * opt_in_group — optional "in group N [, group M ...]" clause. + * When absent, waitGroupCount stays 0 which means "all groups". + */ +opt_in_group: + /* empty */ + | T_IN group_items + ; + +group_items: + T_GROUP T_INTEGER + { + if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) + current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = $2; + } + | group_items T_COMMA T_GROUP T_INTEGER + { + if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) + current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = $4; + } + ; + +opt_timeout: + /* empty */ { $$ = PGAF_TIMEOUT_DEFAULT; } + | T_TIMEOUT T_INTEGER { $$ = $2; } + | T_WITH T_TIMEOUT T_INTEGER { $$ = $3; } + ; + +/* ----------------------------------------------------------------------- + * assert state = (instant check, no timeout) + * assert state is (same, alternate keyword) + * assert state is timeout Ns (polling wait, alias for wait until) + * assert assigned-state = + * ----------------------------------------------------------------------- */ + +assert_cmd: + T_ASSERT T_IDENT T_STATE state_op fsm_state opt_timeout + { + $$ = make_cmd($6 > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->state, $5, sizeof($$->state)); + $$->timeoutSeconds = $6; + free($2); + } + | T_ASSERT T_IDENT T_STATE state_op T_IDENT opt_timeout + { + $$ = make_cmd($6 > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->state, $5, sizeof($$->state)); + $$->timeoutSeconds = $6; + free($2); free($5); + } + | T_ASSERT T_IDENT T_ASSIGNED_STATE state_op fsm_state opt_timeout + { + $$ = make_cmd(CMD_ASSERT_ASSIGNED); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->state, $5, sizeof($$->state)); + $$->timeoutSeconds = $6; + free($2); + } + | T_ASSERT T_IDENT T_ASSIGNED_STATE state_op T_IDENT opt_timeout + { + $$ = make_cmd(CMD_ASSERT_ASSIGNED); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->state, $5, sizeof($$->state)); + $$->timeoutSeconds = $6; + free($2); free($5); + } + ; + +/* ----------------------------------------------------------------------- + * sql { SQL text } + * + * Inside STEP_BODY, '{' triggers the raw block reader which returns + * T_BLOCK with the SQL content already trimmed (braces not included). + * ----------------------------------------------------------------------- */ + +sql_cmd: + T_SQL T_IDENT T_BLOCK + { + $$ = make_cmd(CMD_SQL); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $3, sizeof($$->args)); + free($2); free($3); + } + ; + +/* ----------------------------------------------------------------------- + * expect { text } + * expect error [SQLSTATE] + * ----------------------------------------------------------------------- */ + +expect_cmd: + T_EXPECT T_BLOCK + { + $$ = make_cmd(CMD_EXPECT); + strlcpy($$->expected, $2, sizeof($$->expected)); + expand_tuple_expect($$->expected, sizeof($$->expected)); + free($2); + } + | T_EXPECT T_ERROR + { + $$ = make_cmd(CMD_EXPECT_ERROR); + } + | T_EXPECT T_ERROR T_IDENT + { + $$ = make_cmd(CMD_EXPECT_ERROR); + strlcpy($$->state, $3, sizeof($$->state)); + free($3); + } + | T_EXPECT T_ERROR T_INTEGER + { + /* SQLSTATE codes like 25006 are all digits, lexed as T_INTEGER */ + $$ = make_cmd(CMD_EXPECT_ERROR); + snprintf($$->state, sizeof($$->state), "%d", $3); + } + ; + +/* ----------------------------------------------------------------------- + * promote node1 [, node2, ...] + * ----------------------------------------------------------------------- */ + +promote_cmd: + T_PROMOTE promote_list + { + $$ = current_promote_cmd; + current_promote_cmd = NULL; + } + ; + +promote_list: + T_IDENT + { + current_promote_cmd = make_cmd(CMD_PROMOTE); + current_promote_cmd->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; + strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], + $1, sizeof(current_promote_cmd->promoteNodes[0])); + free($1); + } + | promote_list T_COMMA T_IDENT + { + if (current_promote_cmd->promoteCount < PGAF_MAX_PROMOTE_NODES) + strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], + $3, sizeof(current_promote_cmd->promoteNodes[0])); + free($3); + } + ; + +/* ----------------------------------------------------------------------- + * network disconnect + * network connect + * ----------------------------------------------------------------------- */ + +network_cmd: + T_NETWORK T_DISCONNECT T_IDENT + { + $$ = make_cmd(CMD_NETWORK_OFF); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + | T_NETWORK T_CONNECT T_IDENT + { + $$ = make_cmd(CMD_NETWORK_ON); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + ; + +/* ----------------------------------------------------------------------- + * sleep Ns + * ----------------------------------------------------------------------- */ + +sleep_cmd: + T_SLEEP T_INTEGER + { + $$ = make_cmd(CMD_SLEEP); + $$->timeoutSeconds = $2; + } + ; + +/* ----------------------------------------------------------------------- + * compose down + * compose start + * compose stop + * ----------------------------------------------------------------------- */ + +compose_cmd: + T_COMPOSE T_DOWN + { + $$ = make_cmd(CMD_COMPOSE_DOWN); + } + | T_COMPOSE T_START T_IDENT + { + $$ = make_cmd(CMD_COMPOSE_START); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + | T_COMPOSE T_STOP T_IDENT + { + $$ = make_cmd(CMD_COMPOSE_STOP); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + | T_COMPOSE T_KILL T_IDENT + { + $$ = make_cmd(CMD_COMPOSE_KILL); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + /* + * compose inject : + * + * Injects a file from a Docker image into a running service container + * without restarting it. The sequence: + * docker create --name _pgaf_inject_tmp + * docker cp _pgaf_inject_tmp: /tmp/_pgaf_inject_binary + * docker rm _pgaf_inject_tmp + * docker cp /tmp/_pgaf_inject_binary --1: + * + * Fields used: + * expected → image name (e.g. "pgaf:next") + * args → source path in the image (e.g. "/usr/local/bin/pg_autoctl") + * service → destination service name (e.g. "monitor") + * state → destination path in the container (same as src usually) + * + * T_INJECT triggers BEGIN(EXEC_ARGS) in the lexer, so the image name is + * returned as T_IDENT (matching [^ \t\n]+, which includes ':') and the + * remaining "src svc:dst" tokens come back as T_SHELL_ARGS. + */ + | T_COMPOSE T_INJECT T_IDENT T_SHELL_ARGS + { + $$ = make_cmd(CMD_COMPOSE_INJECT); + strlcpy($$->expected, $3, sizeof($$->expected)); /* image */ + + /* Split T_SHELL_ARGS: " :" */ + char tmp[4096]; + strlcpy(tmp, $4, sizeof(tmp)); + char *src = tmp; + char *p = tmp; + while (*p && *p != ' ' && *p != '\t') p++; + if (*p) { *p++ = '\0'; while (*p == ' ' || *p == '\t') p++; } + char *svcdst = p; + char *colon = (*svcdst) ? strchr(svcdst, ':') : NULL; + strlcpy($$->args, src, sizeof($$->args)); + if (colon) + { + *colon = '\0'; + strlcpy($$->service, svcdst, sizeof($$->service)); /* dst svc */ + strlcpy($$->state, colon + 1, sizeof($$->state)); /* dst path */ + } + free($3); free($4); + } + ; + +/* ----------------------------------------------------------------------- + * stop postgres / start postgres + * + * Sugar for `pg_autoctl manual pgctl off/on --pgdata /var/lib/postgres/pgaf` + * run inside the named node's container. Much cleaner than the raw pg_ctl + * loop that was used before. + * ----------------------------------------------------------------------- */ + +postgres_ctl_cmd: + T_STOP T_POSTGRES node_name + { + $$ = make_cmd(CMD_STOP_POSTGRES); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + | T_START T_POSTGRES node_name + { + $$ = make_cmd(CMD_START_POSTGRES); + strlcpy($$->service, $3, sizeof($$->service)); + free($3); + } + ; + +/* ----------------------------------------------------------------------- + * assert stays while { commands } + * + * Runs the body commands and checks after each one (and at the end) that + * the named node's reported state equals . Fails immediately if the + * state ever changes. + * ----------------------------------------------------------------------- */ + +while_body: + T_WHILE { pgaf_next_brace_is_while = 1; } T_LBRACE cmd_list T_RBRACE + { $$ = $4; } + ; + +stays_while_cmd: + T_ASSERT node_name T_STAYS fsm_state while_body + { + $$ = make_cmd(CMD_STAYS_WHILE); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->state, $4, sizeof($$->state)); + $$->body = ($5) ? $5->commands : NULL; + free($2); + } + ; + +/* ----------------------------------------------------------------------- + * set monitor + * + * Switches the runner's active monitor: LISTEN/NOTIFY reconnects to the + * new service, and monitor_get_node_state() queries it for subsequent + * wait-until / implicit post-failover checks. + * ----------------------------------------------------------------------- */ + +set_monitor_cmd: + T_SET T_IDENT T_IDENT + { + /* only "set monitor " is supported; $2 must be "monitor" */ + if (strcmp($2, "monitor") != 0) + { + fprintf(stderr, "pgaftest: unknown 'set' target '%s' (expected 'monitor')\n", $2); + free($2); free($3); + YYERROR; + } + $$ = make_cmd(CMD_SET_MONITOR); + strlcpy($$->service, $3, sizeof($$->service)); + free($2); free($3); + } + ; + +/* ----------------------------------------------------------------------- + * logs [not] contains + * logs [not] matches + * + * Grep the container's log output (via docker compose logs) for . + * "contains" uses fixed-string grep; "matches" uses grep -P (PCRE). + * With "not", the check asserts the pattern is NOT found. + * ----------------------------------------------------------------------- */ + +logs_cmd: + T_LOGS T_IDENT T_CONTAINS T_STRING + { + $$ = make_cmd(CMD_LOGS_CHECK); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $4, sizeof($$->args)); + $$->logsNegate = false; + $$->allowError = false; /* false = fixed string, true = PCRE */ + free($2); free($4); + } + | T_LOGS T_IDENT T_NOT T_CONTAINS T_STRING + { + $$ = make_cmd(CMD_LOGS_CHECK); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $5, sizeof($$->args)); + $$->logsNegate = true; + $$->allowError = false; + free($2); free($5); + } + | T_LOGS T_IDENT T_MATCHES T_STRING + { + $$ = make_cmd(CMD_LOGS_CHECK); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $4, sizeof($$->args)); + $$->logsNegate = false; + $$->allowError = true; /* true = PCRE (-P) */ + free($2); free($4); + } + | T_LOGS T_IDENT T_NOT T_MATCHES T_STRING + { + $$ = make_cmd(CMD_LOGS_CHECK); + strlcpy($$->service, $2, sizeof($$->service)); + strlcpy($$->args, $5, sizeof($$->args)); + $$->logsNegate = true; + $$->allowError = true; + free($2); free($5); + } + ; + +/* ----------------------------------------------------------------------- + * sequence + * ----------------------------------------------------------------------- */ + +sequence_block: + T_SEQUENCE sequence_names + ; + +sequence_names: + /* empty */ + | sequence_names ident_or_string + { + int i = current_spec->sequenceLength; + if (i < PGAF_MAX_SEQ) + current_spec->sequence[current_spec->sequenceLength++] = $2; + else + { + fprintf(stderr, "pgaftest: too many steps in sequence (max %d)\n", + PGAF_MAX_SEQ); + exit(1); + } + } + ; + +/* ----------------------------------------------------------------------- + * fsm_state — matches any FSM state token and returns its canonical name. + * + * Use %type so callers can strlcpy the name into their cmd->state + * field. The returned string is a string literal — do NOT free() it. + * ----------------------------------------------------------------------- */ + +fsm_state: + T_FS_INIT { $$ = "init"; } + | T_FS_SINGLE { $$ = "single"; } + | T_FS_PRIMARY { $$ = "primary"; } + | T_FS_WAIT_PRIMARY { $$ = "wait_primary"; } + | T_FS_WAIT_STANDBY { $$ = "wait_standby"; } + | T_FS_DEMOTED { $$ = "demoted"; } + | T_FS_DEMOTE_TIMEOUT { $$ = "demote_timeout"; } + | T_FS_DRAINING { $$ = "draining"; } + | T_FS_SECONDARY { $$ = "secondary"; } + | T_FS_CATCHINGUP { $$ = "catchingup"; } + | T_FS_PREP_PROMOTION { $$ = "prepare_promotion"; } + | T_FS_STOP_REPLICATION { $$ = "stop_replication"; } + | T_FS_MAINTENANCE { $$ = "maintenance"; } + | T_FS_JOIN_PRIMARY { $$ = "join_primary"; } + | T_FS_APPLY_SETTINGS { $$ = "apply_settings"; } + | T_FS_PREPARE_MAINTENANCE { $$ = "prepare_maintenance"; } + | T_FS_WAIT_MAINTENANCE { $$ = "wait_maintenance"; } + | T_FS_REPORT_LSN { $$ = "report_lsn"; } + | T_FS_FAST_FORWARD { $$ = "fast_forward"; } + | T_FS_JOIN_SECONDARY { $$ = "join_secondary"; } + | T_FS_DROPPED { $$ = "dropped"; } + ; + +/* ----------------------------------------------------------------------- + * Helpers + * ----------------------------------------------------------------------- */ + +ident_or_string: + T_IDENT { $$ = $1; } + | T_STRING { $$ = $1; } + ; + +%% + +/* ----------------------------------------------------------------------- + * Public entry point + * ----------------------------------------------------------------------- */ + +TestSpec * +parse_test_spec(const char *filename) +{ + FILE *f = fopen(filename, "r"); + if (!f) + { + fprintf(stderr, "pgaftest: cannot open spec file \"%s\": %s\n", + filename, strerror(errno)); + return NULL; + } + + TestSpec *spec = (TestSpec *) calloc(1, sizeof(TestSpec)); + if (!spec) { fprintf(stderr, "out of memory\n"); exit(1); } + + strncpy(spec->filename, filename, sizeof(spec->filename)-1); + + current_spec = spec; + pgaf_line_number = 1; + yyin = f; + yyparse(); + fclose(f); + + return spec; +} + +TestCmd * +make_cmd(TestCmdKind kind) +{ + TestCmd *c = (TestCmd *) calloc(1, sizeof(TestCmd)); + if (!c) { fprintf(stderr, "out of memory\n"); exit(1); } + c->kind = kind; + c->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; + return c; +} + +TestStep * +make_step(const char *name) +{ + TestStep *s = (TestStep *) calloc(1, sizeof(TestStep)); + if (!s) { fprintf(stderr, "out of memory\n"); exit(1); } + if (name) strncpy(s->name, name, sizeof(s->name)-1); + return s; +} + +TestStep * +spec_find_step(TestSpec *spec, const char *name) +{ + for (TestStep *s = spec->steps; s; s = s->next) + { + if (strcmp(s->name, name) == 0) + return s; + } + return NULL; +} diff --git a/src/bin/pgaftest/test_spec_scan.c b/src/bin/pgaftest/test_spec_scan.c new file mode 100644 index 000000000..dc14ed843 --- /dev/null +++ b/src/bin/pgaftest/test_spec_scan.c @@ -0,0 +1,3307 @@ +#line 1 "test_spec_scan.c" + +#line 3 "test_spec_scan.c" + +#define YY_INT_ALIGNED short int + +/* A lexical scanner generated by flex */ + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 6 +#define YY_FLEX_SUBMINOR_VERSION 4 +#if YY_FLEX_SUBMINOR_VERSION > 0 +#define FLEX_BETA +#endif + +/* First, we deal with platform-specific or compiler-specific issues. */ + +/* begin standard C headers. */ +#include +#include +#include +#include + +/* end standard C headers. */ + +/* flex integer type definitions */ + +#ifndef FLEXINT_H +#define FLEXINT_H + +/* C99 systems have . Non-C99 systems may or may not. */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + +/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, + * if you want the limit (max/min) macros for int types. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include +typedef int8_t flex_int8_t; +typedef uint8_t flex_uint8_t; +typedef int16_t flex_int16_t; +typedef uint16_t flex_uint16_t; +typedef int32_t flex_int32_t; +typedef uint32_t flex_uint32_t; +typedef uint64_t flex_uint64_t; +#else +typedef signed char flex_int8_t; +typedef short int flex_int16_t; +typedef int flex_int32_t; +typedef unsigned char flex_uint8_t; +typedef unsigned short int flex_uint16_t; +typedef unsigned int flex_uint32_t; + +/* Limits of integral types. */ +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#ifndef SIZE_MAX +#define SIZE_MAX (~(size_t)0) +#endif + +#endif /* ! C99 */ + +#endif /* ! FLEXINT_H */ + +/* begin standard C++ headers. */ + +/* TODO: this is always defined, so inline it */ +#define yyconst const + +#if defined(__GNUC__) && __GNUC__ >= 3 +#define yynoreturn __attribute__((__noreturn__)) +#else +#define yynoreturn +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an + * integer in range [0..255] for use as an array index. + */ +#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN (yy_start) = 1 + 2 * +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START (((yy_start) - 1) / 2) +#define YYSTATE YY_START +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#ifndef YY_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k. + * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. + * Ditto for the __ia64__ case accordingly. + */ +#define YY_BUF_SIZE 32768 +#else +#define YY_BUF_SIZE 16384 +#endif /* __ia64__ */ +#endif + +/* The state buf must be large enough to hold one state per character in the main buffer. + */ +#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif + +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + +extern yy_size_t yyleng; + +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + + #define YY_LESS_LINENO(n) + #define YY_LINENO_REWIND_TO(ptr) + +/* Return all but the first "n" matched characters back to the input stream. */ +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + *yy_cp = (yy_hold_char); \ + YY_RESTORE_YY_MORE_OFFSET \ + (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) +#define unput(c) yyunput( c, (yytext_ptr) ) + +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + int yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + yy_size_t yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + int yy_bs_lineno; /**< The line count. */ + int yy_bs_column; /**< The column count. */ + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; + +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + + }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ + ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ + : NULL) +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. + */ +#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; +static yy_size_t yy_n_chars; /* number of characters read into yy_ch_buf */ +yy_size_t yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = NULL; +static int yy_init = 0; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart ( FILE *input_file ); +void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); +void yy_delete_buffer ( YY_BUFFER_STATE b ); +void yy_flush_buffer ( YY_BUFFER_STATE b ); +void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state ( void ); + +static void yyensure_buffer_stack ( void ); +static void yy_load_buffer_state ( void ); +static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file ); +#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER ) + +YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); +YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len ); + +void *yyalloc ( yy_size_t ); +void *yyrealloc ( void *, yy_size_t ); +void yyfree ( void * ); + +#define yy_new_buffer yy_create_buffer +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ + } +#define yy_set_bol(at_bol) \ + { \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ + } +#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) + +/* Begin user sect3 */ + +#define yywrap() (/*CONSTCOND*/1) +#define YY_SKIP_YYWRAP +typedef flex_uint8_t YY_CHAR; + +FILE *yyin = NULL, *yyout = NULL; + +typedef int yy_state_type; + +extern int yylineno; +int yylineno = 1; + +extern char *yytext; +#ifdef yytext_ptr +#undef yytext_ptr +#endif +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state ( void ); +static yy_state_type yy_try_NUL_trans ( yy_state_type current_state ); +static int yy_get_next_buffer ( void ); +static void yynoreturn yy_fatal_error ( const char* msg ); + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + (yytext_ptr) = yy_bp; \ + yyleng = (yy_size_t) (yy_cp - yy_bp); \ + (yy_hold_char) = *yy_cp; \ + *yy_cp = '\0'; \ + (yy_c_buf_p) = yy_cp; +#define YY_NUM_RULES 147 +#define YY_END_OF_BUFFER 148 +/* This struct is not used in this scanner, + but its presence is necessary. */ +struct yy_trans_info + { + flex_int32_t yy_verify; + flex_int32_t yy_nxt; + }; +static const flex_int16_t yy_accept[1162] = + { 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 148, 18, 17, 16, 18, 1, 12, 11, 13, 13, + 13, 13, 13, 13, 15, 147, 21, 20, 147, 19, + 55, 54, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 57, 58, 95, 94, 147, 93, 126, 137, 125, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 139, 140, + 143, 142, 144, 145, 146, 17, 0, 14, 1, 12, + 12, 13, 13, 13, 13, 13, 13, 13, 13, 21, + + 0, 56, 19, 55, 55, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 95, + 0, 138, 93, 137, 137, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 117, + 123, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 143, 142, 145, 13, 13, 13, 13, 13, + 13, 13, 13, 92, 92, 92, 92, 92, 92, 92, + + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 25, 92, + 92, 92, 92, 122, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 134, 141, + 141, 141, 141, 141, 141, 141, 131, 141, 141, 103, + 141, 141, 141, 141, 141, 141, 141, 141, 13, 13, + 13, 4, 13, 13, 9, 13, 92, 92, 27, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + + 92, 92, 92, 92, 92, 59, 92, 92, 92, 92, + 92, 92, 30, 92, 92, 47, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 112, 141, 141, 141, + 97, 141, 141, 141, 59, 141, 141, 116, 133, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 114, 141, 141, 141, 99, 141, + 124, 13, 13, 13, 13, 7, 13, 92, 33, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 46, 24, 92, 92, 92, 92, + + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 105, 141, 141, 141, 141, 121, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 110, 113, 118, 128, 141, 141, 141, 141, + 141, 100, 141, 141, 129, 13, 13, 13, 13, 13, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 37, 43, 92, 92, 92, 92, + + 92, 92, 92, 92, 92, 92, 60, 92, 92, 92, + 42, 92, 92, 92, 92, 92, 92, 32, 141, 141, + 102, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 104, 141, 141, 132, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 60, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 13, 13, 2, 3, 13, 13, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 66, 92, 91, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 22, 92, 92, 92, + + 92, 61, 92, 92, 92, 92, 92, 92, 41, 92, + 92, 92, 92, 92, 92, 141, 141, 141, 141, 141, + 111, 109, 141, 141, 141, 66, 141, 141, 91, 141, + 141, 141, 141, 141, 141, 141, 141, 136, 107, 141, + 141, 141, 61, 106, 141, 141, 141, 141, 141, 115, + 130, 101, 141, 141, 141, 141, 141, 141, 13, 13, + 10, 8, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 38, 92, 92, 69, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 29, 35, 92, 92, 92, 92, 92, 92, 92, 92, + + 92, 92, 92, 92, 92, 92, 141, 141, 141, 141, + 141, 135, 141, 141, 141, 69, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 127, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 13, + 13, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 28, 92, 39, 40, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 70, 92, 92, 92, 92, 92, + 92, 92, 92, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + + 141, 141, 141, 141, 141, 141, 70, 141, 141, 141, + 141, 141, 141, 141, 141, 13, 13, 92, 92, 92, + 92, 92, 71, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 34, + 92, 92, 92, 92, 92, 86, 85, 92, 92, 92, + 92, 92, 92, 92, 92, 141, 141, 141, 141, 71, + 141, 141, 108, 96, 141, 141, 141, 141, 141, 141, + 141, 98, 141, 141, 141, 141, 86, 85, 141, 141, + 141, 141, 141, 141, 141, 141, 13, 13, 92, 92, + 26, 53, 92, 92, 92, 31, 92, 92, 92, 92, + + 92, 92, 92, 92, 92, 92, 92, 76, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 76, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 13, 6, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 88, 87, + 23, 78, 92, 77, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 63, 65, 92, 62, 64, 141, + 141, 141, 141, 141, 141, 88, 87, 78, 141, 77, + 141, 141, 141, 141, 141, 141, 141, 141, 63, 65, + + 141, 62, 64, 13, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 13, 80, 79, 92, 92, 92, 49, 68, + 67, 92, 90, 89, 92, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 80, 79, 119, 141, 68, + 67, 90, 89, 141, 141, 141, 141, 141, 141, 141, + 141, 13, 92, 92, 44, 92, 92, 92, 92, 92, + 92, 92, 92, 92, 92, 92, 92, 141, 141, 141, + + 141, 141, 141, 141, 141, 141, 13, 92, 92, 92, + 36, 92, 92, 92, 92, 92, 92, 75, 74, 84, + 83, 141, 141, 141, 141, 141, 75, 74, 84, 83, + 5, 92, 92, 52, 92, 73, 92, 72, 92, 92, + 141, 141, 73, 141, 72, 45, 48, 92, 92, 92, + 50, 120, 141, 141, 82, 81, 92, 82, 81, 51, + 0 + } ; + +static const YY_CHAR yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, + 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 5, 6, 1, 1, 1, 1, 1, + 1, 1, 1, 7, 8, 1, 1, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 1, 1, 1, + 10, 1, 1, 1, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 12, 11, 11, 11, 11, 11, 11, 11, + 1, 1, 1, 1, 13, 1, 14, 15, 16, 17, + + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + 38, 11, 39, 1, 40, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + } ; + +static const YY_CHAR yy_meta[41] = + { 0, + 1, 2, 3, 1, 1, 1, 1, 4, 4, 1, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 1, 1 + } ; + +static const flex_int16_t yy_base[1175] = + { 0, + 0, 0, 40, 0, 80, 0, 119, 121, 1250, 1249, + 1251, 1254, 123, 1254, 1245, 0, 117, 1254, 0, 106, + 1221, 1220, 112, 1229, 1254, 1254, 130, 1254, 1241, 0, + 124, 1254, 0, 106, 1223, 125, 119, 1207, 127, 1212, + 116, 1214, 130, 132, 120, 137, 1223, 139, 1212, 145, + 1254, 1254, 160, 1254, 1234, 0, 1254, 154, 1254, 0, + 147, 153, 160, 138, 1224, 1206, 153, 1208, 1213, 1206, + 1219, 159, 164, 1214, 171, 176, 1204, 185, 1254, 1254, + 0, 1228, 1254, 0, 1254, 198, 1224, 1254, 0, 196, + 1254, 0, 1195, 1193, 1199, 1208, 179, 1206, 1209, 209, + + 1217, 1254, 0, 205, 1254, 0, 1192, 1182, 1186, 1191, + 183, 1184, 1188, 200, 204, 1182, 1182, 1182, 1184, 144, + 1189, 1188, 1175, 1176, 1185, 1179, 186, 1179, 1172, 1172, + 202, 1173, 1185, 1173, 1174, 1170, 1172, 1174, 1164, 219, + 1189, 1254, 0, 213, 1254, 0, 1176, 1163, 1159, 200, + 203, 1164, 1157, 1152, 220, 1156, 213, 1154, 1157, 213, + 0, 1162, 1158, 1162, 216, 1148, 1147, 1166, 1146, 222, + 1148, 223, 1149, 1157, 1149, 227, 1142, 1146, 1138, 1148, + 1147, 1135, 0, 1165, 0, 1132, 1133, 1142, 1145, 1128, + 1127, 1131, 1128, 1133, 1130, 1135, 1138, 1137, 1137, 1118, + + 1120, 1128, 1131, 1120, 1125, 1117, 1127, 1111, 1117, 1108, + 1121, 1122, 1106, 1111, 1110, 1103, 1108, 1112, 1107, 1114, + 1123, 1098, 1096, 1099, 1101, 218, 1098, 1105, 0, 1095, + 1089, 1089, 1097, 0, 1095, 229, 1102, 1102, 1088, 225, + 1088, 1099, 1087, 1091, 1083, 1083, 1094, 1091, 1075, 1073, + 1073, 1087, 1077, 1078, 1070, 1074, 1084, 1063, 0, 1084, + 1064, 1067, 1069, 1068, 1065, 1064, 0, 1071, 1072, 0, + 226, 1060, 1060, 1069, 1064, 1052, 1059, 1062, 1050, 1048, + 1047, 0, 1061, 1049, 0, 1060, 1038, 1059, 1066, 1065, + 1050, 1050, 1038, 1052, 1054, 1036, 1033, 1038, 1035, 1036, + + 252, 1048, 1032, 1042, 1042, 1036, 253, 1041, 1038, 1022, + 1021, 1025, 0, 1020, 1015, 0, 1036, 1035, 1026, 1016, + 1019, 1020, 254, 1018, 255, 1025, 1004, 1010, 1020, 1017, + 1017, 1009, 1018, 1021, 1001, 1005, 0, 1005, 1002, 999, + 1021, 1012, 261, 998, 0, 1010, 262, 0, 0, 992, + 1003, 995, 988, 1001, 1006, 1005, 990, 986, 989, 990, + 985, 980, 994, 979, 263, 976, 981, 983, 264, 989, + 0, 998, 987, 976, 976, 0, 974, 265, 0, 975, + 968, 982, 976, 989, 974, 968, 963, 975, 970, 973, + 958, 970, 969, 954, 0, 978, 963, 970, 250, 252, + + 962, 955, 963, 952, 952, 940, 949, 945, 944, 958, + 940, 955, 953, 939, 938, 950, 949, 259, 261, 935, + 281, 932, 937, 946, 940, 929, 944, 937, 940, 930, + 934, 937, 0, 935, 920, 933, 932, 0, 917, 266, + 267, 931, 930, 916, 913, 914, 913, 912, 909, 908, + 923, 921, 0, 0, 0, 0, 907, 906, 918, 915, + 900, 0, 271, 275, 0, 270, 902, 901, 915, 894, + 897, 896, 909, 898, 911, 897, 286, 896, 914, 903, + 297, 893, 902, 896, 889, 888, 893, 881, 899, 887, + 880, 892, 878, 890, 0, 0, 880, 875, 883, 877, + + 872, 884, 863, 886, 300, 885, 0, 880, 879, 879, + 0, 881, 863, 860, 878, 860, 857, 0, 857, 856, + 0, 869, 872, 858, 866, 850, 855, 303, 854, 853, + 862, 864, 0, 849, 848, 0, 844, 856, 842, 854, + 844, 838, 845, 840, 849, 848, 827, 846, 304, 849, + 0, 844, 843, 843, 838, 825, 843, 825, 822, 840, + 822, 819, 823, 822, 0, 0, 831, 821, 829, 828, + 812, 810, 810, 822, 816, 822, 825, 822, 820, 803, + 802, 0, 814, 0, 805, 801, 800, 802, 815, 795, + 802, 804, 809, 802, 807, 808, 813, 787, 803, 801, + + 311, 0, 784, 791, 790, 783, 784, 783, 0, 789, + 788, 795, 786, 785, 792, 787, 786, 786, 769, 781, + 0, 0, 768, 766, 765, 0, 779, 776, 0, 773, + 763, 762, 770, 775, 768, 773, 774, 0, 0, 771, + 754, 313, 0, 0, 760, 759, 752, 753, 752, 0, + 0, 0, 758, 757, 764, 755, 754, 761, 746, 742, + 0, 0, 739, 738, 749, 738, 750, 733, 732, 749, + 731, 738, 0, 740, 739, 0, 733, 723, 722, 729, + 721, 733, 145, 164, 225, 228, 252, 282, 286, 294, + 0, 0, 299, 301, 301, 297, 299, 294, 308, 309, + + 308, 310, 310, 311, 313, 313, 308, 309, 335, 326, + 311, 0, 324, 325, 332, 0, 324, 314, 315, 326, + 325, 328, 327, 329, 324, 0, 332, 333, 328, 331, + 326, 340, 341, 340, 342, 342, 343, 345, 345, 342, + 350, 342, 343, 349, 362, 371, 351, 349, 354, 355, + 350, 359, 360, 379, 374, 375, 0, 370, 0, 0, + 377, 365, 379, 367, 379, 382, 366, 384, 368, 386, + 370, 374, 376, 377, 0, 383, 384, 374, 394, 392, + 377, 397, 395, 380, 381, 383, 408, 388, 392, 393, + 387, 389, 408, 409, 410, 398, 412, 400, 412, 404, + + 416, 400, 418, 402, 407, 408, 0, 414, 415, 405, + 425, 423, 408, 428, 426, 427, 427, 424, 425, 431, + 431, 421, 0, 418, 425, 422, 422, 437, 438, 422, + 427, 428, 442, 430, 445, 432, 447, 447, 434, 0, + 445, 440, 447, 442, 444, 0, 0, 456, 457, 456, + 444, 461, 459, 447, 464, 458, 459, 449, 454, 0, + 466, 467, 0, 0, 455, 456, 457, 472, 459, 474, + 474, 0, 471, 466, 473, 468, 0, 0, 481, 482, + 481, 469, 486, 484, 472, 489, 483, 475, 480, 481, + 0, 0, 478, 492, 494, 0, 479, 485, 486, 497, + + 499, 500, 485, 481, 506, 483, 508, 0, 491, 497, + 499, 499, 501, 520, 515, 516, 504, 494, 495, 507, + 497, 498, 510, 511, 525, 509, 513, 514, 526, 527, + 507, 532, 509, 534, 0, 522, 524, 524, 526, 539, + 540, 528, 518, 519, 531, 521, 522, 534, 0, 542, + 543, 542, 534, 552, 549, 534, 535, 539, 0, 0, + 0, 0, 540, 0, 541, 537, 541, 547, 543, 549, + 549, 547, 548, 568, 0, 0, 569, 0, 0, 564, + 565, 553, 565, 554, 555, 0, 0, 0, 559, 0, + 560, 559, 565, 561, 567, 563, 564, 584, 0, 0, + + 585, 0, 0, 586, 569, 570, 575, 596, 574, 575, + 574, 575, 577, 572, 573, 584, 595, 581, 597, 583, + 603, 584, 597, 598, 594, 595, 591, 592, 607, 598, + 594, 595, 591, 592, 613, 599, 615, 601, 613, 614, + 610, 611, 606, 0, 0, 609, 614, 604, 0, 0, + 0, 621, 0, 0, 613, 618, 624, 620, 626, 617, + 622, 623, 624, 637, 638, 0, 0, 0, 624, 0, + 0, 0, 0, 629, 635, 631, 637, 632, 633, 646, + 647, 636, 643, 652, 0, 639, 651, 655, 642, 657, + 644, 641, 643, 648, 649, 659, 660, 657, 666, 653, + + 668, 655, 657, 658, 668, 669, 657, 656, 664, 664, + 0, 665, 666, 667, 668, 660, 663, 0, 0, 0, + 0, 665, 672, 673, 674, 675, 0, 0, 0, 0, + 0, 665, 686, 0, 689, 0, 690, 0, 679, 682, + 671, 694, 0, 695, 0, 0, 0, 694, 695, 683, + 0, 0, 697, 698, 0, 0, 700, 0, 0, 0, + 1254, 717, 721, 725, 729, 728, 733, 737, 736, 741, + 745, 744, 749, 753 + } ; + +static const flex_int16_t yy_def[1175] = + { 0, + 1161, 1, 1161, 3, 1161, 5, 1162, 1162, 1163, 1163, + 1161, 1161, 1161, 1161, 1164, 1165, 1161, 1161, 1166, 1166, + 1166, 1166, 1166, 1166, 1161, 1161, 1161, 1161, 1167, 1168, + 1161, 1161, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1161, 1161, 1161, 1161, 1170, 1171, 1161, 1161, 1161, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1161, 1161, + 1173, 1161, 1161, 1174, 1161, 1161, 1164, 1161, 1165, 1161, + 1161, 1166, 1166, 1166, 1166, 1166, 1166, 1166, 1166, 1161, + + 1167, 1161, 1168, 1161, 1161, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1161, + 1170, 1161, 1171, 1161, 1161, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1173, 1161, 1174, 1166, 1166, 1166, 1166, 1166, + 1166, 1166, 1166, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1166, 1166, + 1166, 1166, 1166, 1166, 1166, 1166, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1166, 1166, 1166, 1166, 1166, 1166, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1166, 1166, 1166, 1166, 1166, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1166, 1166, 1166, 1166, 1166, 1166, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1166, 1166, + 1166, 1166, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1166, + 1166, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1166, 1166, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1166, 1166, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1166, 1166, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + + 1172, 1172, 1172, 1166, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1166, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, 1172, 1172, + 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1172, 1166, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1172, 1172, 1172, + + 1172, 1172, 1172, 1172, 1172, 1172, 1166, 1169, 1169, 1169, + 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1169, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, 1172, + 1166, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, 1169, + 1172, 1172, 1172, 1172, 1172, 1169, 1169, 1169, 1169, 1169, + 1169, 1172, 1172, 1172, 1169, 1169, 1169, 1172, 1172, 1169, + 0, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161 + } ; + +static const flex_int16_t yy_nxt[1295] = + { 0, + 12, 13, 14, 13, 15, 16, 12, 12, 17, 18, + 19, 19, 19, 19, 19, 20, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 21, 22, 19, 19, 19, + 19, 23, 24, 19, 19, 19, 19, 19, 25, 12, + 26, 27, 28, 27, 29, 30, 26, 26, 31, 32, + 33, 33, 33, 34, 35, 36, 37, 38, 39, 40, + 33, 41, 42, 33, 43, 44, 45, 33, 46, 33, + 47, 48, 33, 33, 49, 50, 33, 33, 51, 52, + 26, 53, 54, 53, 55, 56, 57, 26, 58, 59, + 60, 60, 60, 61, 60, 62, 63, 64, 65, 66, + + 60, 67, 68, 69, 70, 71, 72, 60, 73, 60, + 74, 75, 76, 77, 60, 78, 60, 60, 79, 80, + 82, 83, 82, 83, 86, 90, 86, 93, 91, 97, + 94, 100, 104, 100, 107, 105, 114, 108, 111, 109, + 117, 120, 121, 123, 98, 125, 112, 127, 91, 115, + 129, 124, 113, 128, 118, 105, 133, 211, 138, 126, + 134, 140, 144, 140, 130, 145, 150, 131, 156, 212, + 135, 136, 139, 147, 157, 148, 166, 152, 149, 160, + 151, 153, 760, 168, 161, 145, 167, 154, 172, 761, + 155, 169, 173, 219, 170, 174, 177, 178, 180, 86, + + 175, 86, 220, 176, 90, 181, 182, 91, 190, 198, + 100, 191, 100, 104, 202, 199, 105, 205, 203, 224, + 140, 144, 140, 225, 145, 204, 237, 91, 239, 240, + 247, 206, 238, 244, 251, 252, 105, 256, 266, 262, + 271, 248, 319, 263, 145, 320, 328, 245, 257, 264, + 329, 333, 762, 763, 272, 267, 362, 334, 363, 392, + 399, 414, 418, 364, 393, 400, 415, 419, 436, 440, + 457, 463, 471, 437, 441, 458, 464, 472, 491, 764, + 493, 492, 416, 494, 512, 563, 515, 513, 519, 516, + 514, 459, 517, 520, 537, 539, 557, 538, 540, 558, + + 560, 575, 559, 561, 580, 564, 562, 604, 765, 581, + 624, 645, 605, 582, 766, 625, 646, 576, 693, 626, + 727, 767, 772, 694, 768, 728, 770, 769, 773, 771, + 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, + 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, + 794, 795, 796, 797, 798, 799, 800, 801, 803, 805, + 802, 804, 806, 807, 808, 809, 810, 811, 812, 813, + 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, + 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, + 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, + + 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, + 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, + 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, + 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, + 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, + 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, + 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, + 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, + 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, + 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, + + 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, + 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, + 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, + 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, + 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, + 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, + 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, + 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, + 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, + 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, + + 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, + 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, + 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, + 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, + 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, + 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, + 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, + 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, + 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, + 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, + + 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, + 1154, 1155, 1156, 1157, 1158, 1159, 1160, 81, 81, 81, + 81, 84, 84, 84, 84, 87, 87, 87, 87, 89, + 89, 92, 89, 101, 101, 101, 101, 103, 103, 106, + 103, 141, 141, 141, 141, 143, 143, 146, 143, 183, + 759, 758, 183, 185, 185, 757, 185, 756, 755, 754, + 753, 752, 751, 750, 749, 748, 747, 746, 745, 744, + 743, 742, 741, 740, 739, 738, 737, 736, 735, 734, + 733, 732, 731, 730, 729, 726, 725, 724, 723, 722, + 721, 720, 719, 718, 717, 716, 715, 714, 713, 712, + + 711, 710, 709, 708, 707, 706, 705, 704, 703, 702, + 701, 700, 699, 698, 697, 696, 695, 692, 691, 690, + 689, 688, 687, 686, 685, 684, 683, 682, 681, 680, + 679, 678, 677, 676, 675, 674, 673, 672, 671, 670, + 669, 668, 667, 666, 665, 664, 663, 662, 661, 660, + 659, 658, 657, 656, 655, 654, 653, 652, 651, 650, + 649, 648, 647, 644, 643, 642, 641, 640, 639, 638, + 637, 636, 635, 634, 633, 632, 631, 630, 629, 628, + 627, 623, 622, 621, 620, 619, 618, 617, 616, 615, + 614, 613, 612, 611, 610, 609, 608, 607, 606, 603, + + 602, 601, 600, 599, 598, 597, 596, 595, 594, 593, + 592, 591, 590, 589, 588, 587, 586, 585, 584, 583, + 579, 578, 577, 574, 573, 572, 571, 570, 569, 568, + 567, 566, 565, 556, 555, 554, 553, 552, 551, 550, + 549, 548, 547, 546, 545, 544, 543, 542, 541, 536, + 535, 534, 533, 532, 531, 530, 529, 528, 527, 526, + 525, 524, 523, 522, 521, 518, 511, 510, 509, 508, + 507, 506, 505, 504, 503, 502, 501, 500, 499, 498, + 497, 496, 495, 490, 489, 488, 487, 486, 485, 484, + 483, 482, 481, 480, 479, 478, 477, 476, 475, 474, + + 473, 470, 469, 468, 467, 466, 465, 462, 461, 460, + 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, + 446, 445, 444, 443, 442, 439, 438, 435, 434, 433, + 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, + 422, 421, 420, 417, 413, 412, 411, 410, 409, 408, + 407, 406, 405, 404, 403, 402, 401, 398, 397, 396, + 395, 394, 391, 390, 389, 388, 387, 386, 385, 384, + 383, 382, 381, 380, 379, 378, 377, 376, 375, 374, + 373, 372, 371, 370, 369, 368, 367, 366, 365, 361, + 360, 359, 358, 357, 356, 355, 354, 353, 352, 351, + + 350, 349, 348, 347, 346, 345, 344, 343, 342, 341, + 340, 339, 338, 337, 336, 335, 332, 331, 330, 327, + 326, 325, 324, 323, 322, 321, 318, 317, 316, 315, + 314, 313, 312, 311, 310, 309, 308, 307, 306, 305, + 304, 303, 302, 301, 300, 299, 298, 297, 296, 295, + 294, 293, 292, 291, 290, 289, 288, 287, 286, 285, + 284, 283, 282, 281, 280, 279, 184, 278, 277, 276, + 275, 274, 273, 270, 269, 268, 265, 261, 260, 259, + 258, 255, 254, 253, 250, 249, 246, 243, 242, 241, + 236, 235, 234, 142, 233, 232, 231, 230, 229, 228, + + 227, 226, 223, 222, 221, 218, 217, 216, 215, 214, + 213, 210, 209, 208, 207, 201, 200, 197, 196, 195, + 194, 102, 193, 192, 189, 188, 187, 186, 88, 184, + 179, 171, 165, 164, 163, 162, 159, 158, 142, 137, + 132, 122, 119, 116, 110, 102, 99, 96, 95, 88, + 1161, 85, 85, 11, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161 + + } ; + +static const flex_int16_t yy_chk[1295] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 7, 7, 8, 8, 13, 17, 13, 20, 17, 23, + 20, 27, 31, 27, 34, 31, 37, 34, 36, 34, + 39, 41, 41, 43, 23, 44, 36, 45, 17, 37, + 46, 43, 36, 45, 39, 31, 48, 120, 50, 44, + 48, 53, 58, 53, 46, 58, 62, 46, 64, 120, + 48, 48, 50, 61, 64, 61, 72, 63, 61, 67, + 62, 63, 683, 73, 67, 58, 72, 63, 75, 684, + 63, 73, 75, 127, 73, 75, 76, 76, 78, 86, + + 75, 86, 127, 75, 90, 78, 78, 90, 97, 111, + 100, 97, 100, 104, 114, 111, 104, 115, 114, 131, + 140, 144, 140, 131, 144, 114, 150, 90, 151, 151, + 157, 115, 150, 155, 160, 160, 104, 165, 172, 170, + 176, 157, 226, 170, 144, 226, 236, 155, 165, 170, + 236, 240, 685, 686, 176, 172, 271, 240, 271, 301, + 307, 323, 325, 271, 301, 307, 323, 325, 343, 347, + 365, 369, 378, 343, 347, 365, 369, 378, 399, 687, + 400, 399, 323, 400, 418, 466, 419, 418, 421, 419, + 418, 365, 419, 421, 440, 441, 463, 440, 441, 463, + + 464, 477, 463, 464, 481, 466, 464, 505, 688, 481, + 528, 549, 505, 481, 689, 528, 549, 477, 601, 528, + 642, 690, 695, 601, 693, 642, 694, 693, 696, 694, + 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, + 707, 708, 709, 710, 711, 713, 714, 715, 717, 718, + 719, 720, 721, 722, 723, 724, 725, 727, 728, 729, + 727, 728, 730, 731, 732, 733, 734, 735, 736, 737, + 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, + 748, 749, 750, 751, 752, 753, 754, 755, 756, 758, + 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, + + 771, 772, 773, 774, 776, 777, 778, 779, 780, 781, + 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, + 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, + 802, 803, 804, 805, 806, 808, 809, 810, 811, 812, + 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, + 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, + 834, 835, 836, 837, 838, 839, 841, 842, 843, 844, + 845, 848, 849, 850, 851, 852, 853, 854, 855, 856, + 857, 858, 859, 861, 862, 865, 866, 867, 868, 869, + 870, 871, 873, 874, 875, 876, 879, 880, 881, 882, + + 883, 884, 885, 886, 887, 888, 889, 890, 893, 894, + 895, 897, 898, 899, 900, 901, 902, 903, 904, 905, + 906, 907, 909, 910, 911, 912, 913, 914, 915, 916, + 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, + 927, 928, 929, 930, 931, 932, 933, 934, 936, 937, + 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, + 948, 950, 951, 952, 953, 954, 955, 956, 957, 958, + 963, 965, 966, 967, 968, 969, 970, 971, 971, 972, + 973, 974, 977, 980, 981, 982, 983, 984, 985, 989, + 991, 992, 993, 994, 995, 996, 997, 998, 1001, 1004, + + 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, + 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, + 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, + 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1046, + 1047, 1048, 1052, 1055, 1056, 1057, 1058, 1059, 1060, 1061, + 1062, 1063, 1064, 1065, 1069, 1074, 1075, 1076, 1077, 1078, + 1079, 1080, 1081, 1082, 1083, 1084, 1086, 1087, 1088, 1089, + 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, + 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, + 1110, 1112, 1113, 1114, 1115, 1116, 1117, 1122, 1123, 1124, + + 1125, 1126, 1132, 1133, 1135, 1137, 1139, 1140, 1141, 1142, + 1144, 1148, 1149, 1150, 1153, 1154, 1157, 1162, 1162, 1162, + 1162, 1163, 1163, 1163, 1163, 1164, 1164, 1164, 1164, 1165, + 1165, 1166, 1165, 1167, 1167, 1167, 1167, 1168, 1168, 1169, + 1168, 1170, 1170, 1170, 1170, 1171, 1171, 1172, 1171, 1173, + 682, 681, 1173, 1174, 1174, 680, 1174, 679, 678, 677, + 675, 674, 672, 671, 670, 669, 668, 667, 666, 665, + 664, 663, 660, 659, 658, 657, 656, 655, 654, 653, + 649, 648, 647, 646, 645, 641, 640, 637, 636, 635, + 634, 633, 632, 631, 630, 628, 627, 625, 624, 623, + + 620, 619, 618, 617, 616, 615, 614, 613, 612, 611, + 610, 608, 607, 606, 605, 604, 603, 600, 599, 598, + 597, 596, 595, 594, 593, 592, 591, 590, 589, 588, + 587, 586, 585, 583, 581, 580, 579, 578, 577, 576, + 575, 574, 573, 572, 571, 570, 569, 568, 567, 564, + 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, + 553, 552, 550, 548, 547, 546, 545, 544, 543, 542, + 541, 540, 539, 538, 537, 535, 534, 532, 531, 530, + 529, 527, 526, 525, 524, 523, 522, 520, 519, 517, + 516, 515, 514, 513, 512, 510, 509, 508, 506, 504, + + 503, 502, 501, 500, 499, 498, 497, 494, 493, 492, + 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, + 480, 479, 478, 476, 475, 474, 473, 472, 471, 470, + 469, 468, 467, 461, 460, 459, 458, 457, 452, 451, + 450, 449, 448, 447, 446, 445, 444, 443, 442, 439, + 437, 436, 435, 434, 432, 431, 430, 429, 428, 427, + 426, 425, 424, 423, 422, 420, 417, 416, 415, 414, + 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, + 403, 402, 401, 398, 397, 396, 394, 393, 392, 391, + 390, 389, 388, 387, 386, 385, 384, 383, 382, 381, + + 380, 377, 375, 374, 373, 372, 370, 368, 367, 366, + 364, 363, 362, 361, 360, 359, 358, 357, 356, 355, + 354, 353, 352, 351, 350, 346, 344, 342, 341, 340, + 339, 338, 336, 335, 334, 333, 332, 331, 330, 329, + 328, 327, 326, 324, 322, 321, 320, 319, 318, 317, + 315, 314, 312, 311, 310, 309, 308, 306, 305, 304, + 303, 302, 300, 299, 298, 297, 296, 295, 294, 293, + 292, 291, 290, 289, 288, 287, 286, 284, 283, 281, + 280, 279, 278, 277, 276, 275, 274, 273, 272, 269, + 268, 266, 265, 264, 263, 262, 261, 260, 258, 257, + + 256, 255, 254, 253, 252, 251, 250, 249, 248, 247, + 246, 245, 244, 243, 242, 241, 239, 238, 237, 235, + 233, 232, 231, 230, 228, 227, 225, 224, 223, 222, + 221, 220, 219, 218, 217, 216, 215, 214, 213, 212, + 211, 210, 209, 208, 207, 206, 205, 204, 203, 202, + 201, 200, 199, 198, 197, 196, 195, 194, 193, 192, + 191, 190, 189, 188, 187, 186, 184, 182, 181, 180, + 179, 178, 177, 175, 174, 173, 171, 169, 168, 167, + 166, 164, 163, 162, 159, 158, 156, 154, 153, 152, + 149, 148, 147, 141, 139, 138, 137, 136, 135, 134, + + 133, 132, 130, 129, 128, 126, 125, 124, 123, 122, + 121, 119, 118, 117, 116, 113, 112, 110, 109, 108, + 107, 101, 99, 98, 96, 95, 94, 93, 87, 82, + 77, 74, 71, 70, 69, 68, 66, 65, 55, 49, + 47, 42, 40, 38, 35, 29, 24, 22, 21, 15, + 11, 10, 9, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, 1161, + 1161, 1161, 1161, 1161 + + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +extern int yy_flex_debug; +int yy_flex_debug = 0; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "test_spec_scan.l" +#line 2 "test_spec_scan.l" +/* + * src/bin/pgaftest/test_spec_scan.l + * Flex lexer for .pgaf test specification files. + * + * The outer structure (cluster, setup, teardown, step, sequence) is + * tokenised in the INITIAL state. When a step/setup/teardown body + * opens, we switch to the STEP_BODY exclusive state where every + * command keyword and value becomes its own token. + * + * Additional exclusive states: + * CLUSTER_BODY - inside a cluster { } block; tokenises the cluster + * mini-language (formations, nodes, options) + * EXEC_ARGS - skip whitespace, then read the service name token + * EXEC_ARGS_REST - read the rest of the line as a raw T_SHELL_ARGS string + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include "test_spec.h" +#include "test_spec_parse.h" + +int pgaf_line_number = 1; + +/* + * When the lexer sees "setup", "teardown", or "step", it sets this flag + * so the next "{" enters STEP_BODY rather than doing the raw block read. + */ +static int pgaf_next_brace_is_step = 0; + +/* + * When the lexer sees "cluster", it sets this flag so the next "{" + * enters CLUSTER_BODY rather than doing the raw block read. + */ +static int pgaf_next_brace_is_cluster = 0; + +/* + * When 1, the next '{' in STEP_BODY returns T_LBRACE (for a while body) + * instead of calling pgaf_read_raw_block(). Set by the grammar via a + * mid-rule action when T_WHILE is seen. Also used by the setup/teardown + * blocks that need sub-block parsing. + */ +int pgaf_next_brace_is_while = 0; + +/* + * Nesting depth for '{' tokens returned as T_LBRACE inside STEP_BODY + * (while bodies). 0 means no nested while bodies are open. Each T_LBRACE + * increments this; each T_RBRACE decrements it when depth > 0; when depth + * reaches 0 the '}' is the step's own closing brace → BEGIN(INITIAL). + */ +static int pgaf_step_brace_depth = 0; + +/* + * Nesting depth inside CLUSTER_BODY - so we know when the outermost "}" + * closes and we can return to INITIAL. + */ +static int pgaf_cluster_depth = 0; + +static char *pgaf_strdup(const char *s) +{ + char *r = strdup(s); + if (!r) { fprintf(stderr, "out of memory\n"); exit(1); } + return r; +} + +/* + * pgaf_read_raw_block: called when '{' is seen and we want to collect + * everything up to the matching '}' as a single string. Used for + * sql/expect content inside step bodies. + * Stores a freshly allocated trimmed string in yylval.str. + * + * Defined in the user-code section (after the second %%) so that it can + * call flex's static input() function. + */ +static void pgaf_read_raw_block(void); +#line 1192 "test_spec_scan.c" + +#line 1194 "test_spec_scan.c" + +#define INITIAL 0 +#define CLUSTER_BODY 1 +#define STEP_BODY 2 +#define EXEC_ARGS 3 +#define EXEC_ARGS_REST 4 + +#ifndef YY_NO_UNISTD_H +/* Special case for "unistd.h", since it is non-ANSI. We include it way + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +#include +#endif + +#ifndef YY_EXTRA_TYPE +#define YY_EXTRA_TYPE void * +#endif + +static int yy_init_globals ( void ); + +/* Accessor methods to globals. + These are made visible to non-reentrant scanners for convenience. */ + +int yylex_destroy ( void ); + +int yyget_debug ( void ); + +void yyset_debug ( int debug_flag ); + +YY_EXTRA_TYPE yyget_extra ( void ); + +void yyset_extra ( YY_EXTRA_TYPE user_defined ); + +FILE *yyget_in ( void ); + +void yyset_in ( FILE * _in_str ); + +FILE *yyget_out ( void ); + +void yyset_out ( FILE * _out_str ); + + yy_size_t yyget_leng ( void ); + +char *yyget_text ( void ); + +int yyget_lineno ( void ); + +void yyset_lineno ( int _line_number ); + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap ( void ); +#else +extern int yywrap ( void ); +#endif +#endif + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef yytext_ptr +static void yy_flex_strncpy ( char *, const char *, int ); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen ( const char * ); +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput ( void ); +#else +static int input ( void ); +#endif + +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k */ +#define YY_READ_BUF_SIZE 16384 +#else +#define YY_READ_BUF_SIZE 8192 +#endif /* __ia64__ */ +#endif + +/* Copy whatever the last rule matched to the standard output. */ +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ + { \ + int c = '*'; \ + yy_size_t n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else \ + { \ + errno=0; \ + while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + { \ + if( errno != EINTR) \ + { \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + break; \ + } \ + errno=0; \ + clearerr(yyin); \ + } \ + }\ +\ + +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* end tables serialization structures and prototypes */ + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL_IS_OURS 1 + +extern int yylex (void); + +#define YY_DECL int yylex (void) +#endif /* !YY_DECL */ + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK /*LINTED*/break; +#endif + +#define YY_RULE_SETUP \ + YY_USER_ACTION + +/** The main scanner function which does all the work. + */ +YY_DECL +{ + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + + if ( !(yy_init) ) + { + (yy_init) = 1; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! (yy_start) ) + (yy_start) = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_load_buffer_state( ); + } + + { +#line 89 "test_spec_scan.l" + + +#line 1416 "test_spec_scan.c" + + while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ + { + yy_cp = (yy_c_buf_p); + + /* Support of yytext. */ + *yy_cp = (yy_hold_char); + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = (yy_start); +yy_match: + do + { + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1162 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + ++yy_cp; + } + while ( yy_current_state != 1161 ); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + + YY_DO_BEFORE_ACTION; + +do_action: /* This label is used only to access EOF actions. */ + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = (yy_hold_char); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + +case 1: +YY_RULE_SETUP +#line 91 "test_spec_scan.l" +{ /* comment */ } + YY_BREAK +case 2: +YY_RULE_SETUP +#line 93 "test_spec_scan.l" +{ pgaf_next_brace_is_cluster = 1; return T_CLUSTER; } + YY_BREAK +case 3: +YY_RULE_SETUP +#line 94 "test_spec_scan.l" +{ return T_MONITOR; } + YY_BREAK +case 4: +YY_RULE_SETUP +#line 95 "test_spec_scan.l" +{ return T_NODE; } + YY_BREAK +case 5: +YY_RULE_SETUP +#line 96 "test_spec_scan.l" +{ return T_CITUS_COORDINATOR; } + YY_BREAK +case 6: +YY_RULE_SETUP +#line 97 "test_spec_scan.l" +{ return T_CITUS_WORKER; } + YY_BREAK +case 7: +YY_RULE_SETUP +#line 99 "test_spec_scan.l" +{ pgaf_next_brace_is_step = 1; return T_SETUP; } + YY_BREAK +case 8: +YY_RULE_SETUP +#line 100 "test_spec_scan.l" +{ pgaf_next_brace_is_step = 1; return T_TEARDOWN; } + YY_BREAK +case 9: +YY_RULE_SETUP +#line 101 "test_spec_scan.l" +{ pgaf_next_brace_is_step = 1; return T_STEP; } + YY_BREAK +case 10: +YY_RULE_SETUP +#line 102 "test_spec_scan.l" +{ return T_SEQUENCE; } + YY_BREAK +case 11: +YY_RULE_SETUP +#line 104 "test_spec_scan.l" +{ return T_EQUALS; } + YY_BREAK +case 12: +YY_RULE_SETUP +#line 106 "test_spec_scan.l" +{ + yylval.ival = atoi(yytext); + return T_INTEGER; +} + YY_BREAK +case 13: +YY_RULE_SETUP +#line 111 "test_spec_scan.l" +{ + yylval.str = pgaf_strdup(yytext); + return T_IDENT; +} + YY_BREAK +case 14: +/* rule 14 can match eol */ +YY_RULE_SETUP +#line 116 "test_spec_scan.l" +{ + yytext[yyleng - 1] = '\0'; + yylval.str = pgaf_strdup(yytext + 1); + return T_STRING; +} + YY_BREAK +case 15: +YY_RULE_SETUP +#line 122 "test_spec_scan.l" +{ + if (pgaf_next_brace_is_step) + { + pgaf_next_brace_is_step = 0; + BEGIN(STEP_BODY); + return T_LBRACE; + } + if (pgaf_next_brace_is_cluster) + { + pgaf_next_brace_is_cluster = 0; + pgaf_cluster_depth = 1; + BEGIN(CLUSTER_BODY); + return T_LBRACE; + } + /* fallback: should not happen in a well-formed file */ + fprintf(stderr, "pgaftest: unexpected '{' at line %d\n", pgaf_line_number); +} + YY_BREAK +case 16: +/* rule 16 can match eol */ +YY_RULE_SETUP +#line 140 "test_spec_scan.l" +{ pgaf_line_number++; } + YY_BREAK +case 17: +YY_RULE_SETUP +#line 141 "test_spec_scan.l" +{ /* whitespace */ } + YY_BREAK +case 18: +YY_RULE_SETUP +#line 143 "test_spec_scan.l" +{ + fprintf(stderr, "pgaftest: unexpected character '%c' at line %d\n", + yytext[0], pgaf_line_number); +} + YY_BREAK +case 19: +YY_RULE_SETUP +#line 148 "test_spec_scan.l" +{ /* comment */ } + YY_BREAK +case 20: +/* rule 20 can match eol */ +YY_RULE_SETUP +#line 149 "test_spec_scan.l" +{ pgaf_line_number++; } + YY_BREAK +case 21: +YY_RULE_SETUP +#line 150 "test_spec_scan.l" +{ /* whitespace */ } + YY_BREAK +case 22: +YY_RULE_SETUP +#line 152 "test_spec_scan.l" +{ return T_MONITOR; } + YY_BREAK +case 23: +YY_RULE_SETUP +#line 153 "test_spec_scan.l" +{ return T_IMAGE_TARGET; } + YY_BREAK +case 24: +YY_RULE_SETUP +#line 154 "test_spec_scan.l" +{ return T_IMAGE; } + YY_BREAK +case 25: +YY_RULE_SETUP +#line 155 "test_spec_scan.l" +{ return T_SSL; } + YY_BREAK +case 26: +YY_RULE_SETUP +#line 156 "test_spec_scan.l" +{ return T_AUTH_METHOD; } + YY_BREAK +case 27: +YY_RULE_SETUP +#line 157 "test_spec_scan.l" +{ return T_AUTH; } + YY_BREAK +case 28: +YY_RULE_SETUP +#line 158 "test_spec_scan.l" +{ return T_FORMATION; } + YY_BREAK +case 29: +YY_RULE_SETUP +#line 159 "test_spec_scan.l" +{ return T_NUM_SYNC; } + YY_BREAK +case 30: +YY_RULE_SETUP +#line 160 "test_spec_scan.l" +{ return T_NODE; } + YY_BREAK +case 31: +YY_RULE_SETUP +#line 162 "test_spec_scan.l" +{ return T_COORDINATOR; } + YY_BREAK +case 32: +YY_RULE_SETUP +#line 163 "test_spec_scan.l" +{ return T_WORKER; } + YY_BREAK +case 33: +YY_RULE_SETUP +#line 164 "test_spec_scan.l" +{ return T_ASYNC; } + YY_BREAK +case 34: +YY_RULE_SETUP +#line 165 "test_spec_scan.l" +{ return T_NO_MONITOR; } + YY_BREAK +case 35: +YY_RULE_SETUP +#line 166 "test_spec_scan.l" +{ return T_PASSWORD; } + YY_BREAK +case 36: +YY_RULE_SETUP +#line 167 "test_spec_scan.l" +{ return T_MONITOR_PASSWORD; } + YY_BREAK +case 37: +YY_RULE_SETUP +#line 168 "test_spec_scan.l" +{ return T_LAUNCH; } + YY_BREAK +case 38: +YY_RULE_SETUP +#line 169 "test_spec_scan.l" +{ return T_DEFERRED; } + YY_BREAK +case 39: +YY_RULE_SETUP +#line 170 "test_spec_scan.l" +{ return T_IMMEDIATE; } + YY_BREAK +case 40: +YY_RULE_SETUP +#line 171 "test_spec_scan.l" +{ return T_INITIALLY; } + YY_BREAK +case 41: +YY_RULE_SETUP +#line 172 "test_spec_scan.l" +{ return T_STOPPED; } + YY_BREAK +case 42: +YY_RULE_SETUP +#line 173 "test_spec_scan.l" +{ return T_VOLUME; } + YY_BREAK +case 43: +YY_RULE_SETUP +#line 174 "test_spec_scan.l" +{ return T_LISTEN; } + YY_BREAK +case 44: +YY_RULE_SETUP +#line 175 "test_spec_scan.l" +{ return T_CITUS_SECONDARY; } + YY_BREAK +case 45: +YY_RULE_SETUP +#line 176 "test_spec_scan.l" +{ return T_CANDIDATE_PRIORITY; } + YY_BREAK +case 46: +YY_RULE_SETUP +#line 177 "test_spec_scan.l" +{ return T_GROUP; } + YY_BREAK +case 47: +YY_RULE_SETUP +#line 178 "test_spec_scan.l" +{ return T_PORT; } + YY_BREAK +case 48: +YY_RULE_SETUP +#line 179 "test_spec_scan.l" +{ return T_CITUS_CLUSTER_NAME; } + YY_BREAK +case 49: +YY_RULE_SETUP +#line 180 "test_spec_scan.l" +{ return T_DEBIAN_CLUSTER; } + YY_BREAK +case 50: +YY_RULE_SETUP +#line 181 "test_spec_scan.l" +{ return T_REPLICATION_QUORUM; } + YY_BREAK +case 51: +YY_RULE_SETUP +#line 182 "test_spec_scan.l" +{ return T_REPLICATION_PASSWORD; } + YY_BREAK +case 52: +YY_RULE_SETUP +#line 183 "test_spec_scan.l" +{ return T_EXTENSION_VERSION; } + YY_BREAK +case 53: +YY_RULE_SETUP +#line 184 "test_spec_scan.l" +{ return T_BIND_SOURCE; } + YY_BREAK +case 54: +YY_RULE_SETUP +#line 186 "test_spec_scan.l" +{ return T_EQUALS; } + YY_BREAK +case 55: +YY_RULE_SETUP +#line 188 "test_spec_scan.l" +{ + yylval.ival = atoi(yytext); + return T_INTEGER; +} + YY_BREAK +case 56: +/* rule 56 can match eol */ +YY_RULE_SETUP +#line 193 "test_spec_scan.l" +{ + yytext[yyleng - 1] = '\0'; + yylval.str = pgaf_strdup(yytext + 1); + return T_STRING; +} + YY_BREAK +case 57: +YY_RULE_SETUP +#line 199 "test_spec_scan.l" +{ + pgaf_cluster_depth++; + return T_LBRACE; +} + YY_BREAK +case 58: +YY_RULE_SETUP +#line 204 "test_spec_scan.l" +{ + pgaf_cluster_depth--; + if (pgaf_cluster_depth == 0) + BEGIN(INITIAL); + return T_RBRACE; +} + YY_BREAK +case 59: +YY_RULE_SETUP +#line 211 "test_spec_scan.l" +{ return T_FS_INIT; } + YY_BREAK +case 60: +YY_RULE_SETUP +#line 212 "test_spec_scan.l" +{ return T_FS_SINGLE; } + YY_BREAK +case 61: +YY_RULE_SETUP +#line 213 "test_spec_scan.l" +{ return T_FS_PRIMARY; } + YY_BREAK +case 62: +YY_RULE_SETUP +#line 214 "test_spec_scan.l" +{ return T_FS_WAIT_PRIMARY; } + YY_BREAK +case 63: +YY_RULE_SETUP +#line 215 "test_spec_scan.l" +{ return T_FS_WAIT_PRIMARY; } + YY_BREAK +case 64: +YY_RULE_SETUP +#line 216 "test_spec_scan.l" +{ return T_FS_WAIT_STANDBY; } + YY_BREAK +case 65: +YY_RULE_SETUP +#line 217 "test_spec_scan.l" +{ return T_FS_WAIT_STANDBY; } + YY_BREAK +case 66: +YY_RULE_SETUP +#line 218 "test_spec_scan.l" +{ return T_FS_DEMOTED; } + YY_BREAK +case 67: +YY_RULE_SETUP +#line 219 "test_spec_scan.l" +{ return T_FS_DEMOTE_TIMEOUT; } + YY_BREAK +case 68: +YY_RULE_SETUP +#line 220 "test_spec_scan.l" +{ return T_FS_DEMOTE_TIMEOUT; } + YY_BREAK +case 69: +YY_RULE_SETUP +#line 221 "test_spec_scan.l" +{ return T_FS_DRAINING; } + YY_BREAK +case 70: +YY_RULE_SETUP +#line 222 "test_spec_scan.l" +{ return T_FS_SECONDARY; } + YY_BREAK +case 71: +YY_RULE_SETUP +#line 223 "test_spec_scan.l" +{ return T_FS_CATCHINGUP; } + YY_BREAK +case 72: +YY_RULE_SETUP +#line 224 "test_spec_scan.l" +{ return T_FS_PREP_PROMOTION; } + YY_BREAK +case 73: +YY_RULE_SETUP +#line 225 "test_spec_scan.l" +{ return T_FS_PREP_PROMOTION; } + YY_BREAK +case 74: +YY_RULE_SETUP +#line 226 "test_spec_scan.l" +{ return T_FS_STOP_REPLICATION; } + YY_BREAK +case 75: +YY_RULE_SETUP +#line 227 "test_spec_scan.l" +{ return T_FS_STOP_REPLICATION; } + YY_BREAK +case 76: +YY_RULE_SETUP +#line 228 "test_spec_scan.l" +{ return T_FS_MAINTENANCE; } + YY_BREAK +case 77: +YY_RULE_SETUP +#line 229 "test_spec_scan.l" +{ return T_FS_JOIN_PRIMARY; } + YY_BREAK +case 78: +YY_RULE_SETUP +#line 230 "test_spec_scan.l" +{ return T_FS_JOIN_PRIMARY; } + YY_BREAK +case 79: +YY_RULE_SETUP +#line 231 "test_spec_scan.l" +{ return T_FS_APPLY_SETTINGS; } + YY_BREAK +case 80: +YY_RULE_SETUP +#line 232 "test_spec_scan.l" +{ return T_FS_APPLY_SETTINGS; } + YY_BREAK +case 81: +YY_RULE_SETUP +#line 233 "test_spec_scan.l" +{ return T_FS_PREPARE_MAINTENANCE; } + YY_BREAK +case 82: +YY_RULE_SETUP +#line 234 "test_spec_scan.l" +{ return T_FS_PREPARE_MAINTENANCE; } + YY_BREAK +case 83: +YY_RULE_SETUP +#line 235 "test_spec_scan.l" +{ return T_FS_WAIT_MAINTENANCE; } + YY_BREAK +case 84: +YY_RULE_SETUP +#line 236 "test_spec_scan.l" +{ return T_FS_WAIT_MAINTENANCE; } + YY_BREAK +case 85: +YY_RULE_SETUP +#line 237 "test_spec_scan.l" +{ return T_FS_REPORT_LSN; } + YY_BREAK +case 86: +YY_RULE_SETUP +#line 238 "test_spec_scan.l" +{ return T_FS_REPORT_LSN; } + YY_BREAK +case 87: +YY_RULE_SETUP +#line 239 "test_spec_scan.l" +{ return T_FS_FAST_FORWARD; } + YY_BREAK +case 88: +YY_RULE_SETUP +#line 240 "test_spec_scan.l" +{ return T_FS_FAST_FORWARD; } + YY_BREAK +case 89: +YY_RULE_SETUP +#line 241 "test_spec_scan.l" +{ return T_FS_JOIN_SECONDARY; } + YY_BREAK +case 90: +YY_RULE_SETUP +#line 242 "test_spec_scan.l" +{ return T_FS_JOIN_SECONDARY; } + YY_BREAK +case 91: +YY_RULE_SETUP +#line 243 "test_spec_scan.l" +{ return T_FS_DROPPED; } + YY_BREAK +case 92: +YY_RULE_SETUP +#line 245 "test_spec_scan.l" +{ + yylval.str = pgaf_strdup(yytext); + return T_IDENT; +} + YY_BREAK +case 93: +YY_RULE_SETUP +#line 250 "test_spec_scan.l" +{ /* comment */ } + YY_BREAK +case 94: +/* rule 94 can match eol */ +YY_RULE_SETUP +#line 251 "test_spec_scan.l" +{ pgaf_line_number++; } + YY_BREAK +case 95: +YY_RULE_SETUP +#line 252 "test_spec_scan.l" +{ /* whitespace */ } + YY_BREAK +case 96: +YY_RULE_SETUP +#line 254 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_EXEC_FAILS; } + YY_BREAK +case 97: +YY_RULE_SETUP +#line 255 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_EXEC; } + YY_BREAK +case 98: +YY_RULE_SETUP +#line 256 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_PG_AUTOCTL; } + YY_BREAK +case 99: +YY_RULE_SETUP +#line 258 "test_spec_scan.l" +{ return T_WAIT; } + YY_BREAK +case 100: +YY_RULE_SETUP +#line 259 "test_spec_scan.l" +{ return T_UNTIL; } + YY_BREAK +case 101: +YY_RULE_SETUP +#line 260 "test_spec_scan.l" +{ return T_TIMEOUT; } + YY_BREAK +case 102: +YY_RULE_SETUP +#line 261 "test_spec_scan.l" +{ return T_ASSERT; } + YY_BREAK +case 103: +YY_RULE_SETUP +#line 262 "test_spec_scan.l" +{ return T_SQL; } + YY_BREAK +case 104: +YY_RULE_SETUP +#line 263 "test_spec_scan.l" +{ return T_EXPECT; } + YY_BREAK +case 105: +YY_RULE_SETUP +#line 264 "test_spec_scan.l" +{ return T_ERROR; } + YY_BREAK +case 106: +YY_RULE_SETUP +#line 265 "test_spec_scan.l" +{ return T_PROMOTE; } + YY_BREAK +case 107: +YY_RULE_SETUP +#line 266 "test_spec_scan.l" +{ return T_NETWORK; } + YY_BREAK +case 108: +YY_RULE_SETUP +#line 267 "test_spec_scan.l" +{ return T_DISCONNECT; } + YY_BREAK +case 109: +YY_RULE_SETUP +#line 268 "test_spec_scan.l" +{ return T_CONNECT; } + YY_BREAK +case 110: +YY_RULE_SETUP +#line 269 "test_spec_scan.l" +{ return T_SLEEP; } + YY_BREAK +case 111: +YY_RULE_SETUP +#line 270 "test_spec_scan.l" +{ return T_COMPOSE; } + YY_BREAK +case 112: +YY_RULE_SETUP +#line 271 "test_spec_scan.l" +{ return T_DOWN; } + YY_BREAK +case 113: +YY_RULE_SETUP +#line 272 "test_spec_scan.l" +{ return T_START; } + YY_BREAK +case 114: +YY_RULE_SETUP +#line 273 "test_spec_scan.l" +{ return T_STOP; } + YY_BREAK +case 115: +YY_RULE_SETUP +#line 274 "test_spec_scan.l" +{ return T_STOPPED; } + YY_BREAK +case 116: +YY_RULE_SETUP +#line 275 "test_spec_scan.l" +{ return T_KILL; } + YY_BREAK +case 117: +YY_RULE_SETUP +#line 276 "test_spec_scan.l" +{ return T_IN; } + YY_BREAK +case 118: +YY_RULE_SETUP +#line 277 "test_spec_scan.l" +{ return T_STATE; } + YY_BREAK +case 119: +YY_RULE_SETUP +#line 278 "test_spec_scan.l" +{ return T_ASSIGNED_STATE; } + YY_BREAK +case 120: +YY_RULE_SETUP +#line 279 "test_spec_scan.l" +{ return T_CANDIDATE_PRIORITY; } + YY_BREAK +case 121: +YY_RULE_SETUP +#line 280 "test_spec_scan.l" +{ return T_GROUP; } + YY_BREAK +case 122: +YY_RULE_SETUP +#line 281 "test_spec_scan.l" +{ return T_AND; } + YY_BREAK +case 123: +YY_RULE_SETUP +#line 282 "test_spec_scan.l" +{ return T_IS; } + YY_BREAK +case 124: +YY_RULE_SETUP +#line 283 "test_spec_scan.l" +{ return T_WITH; } + YY_BREAK +case 125: +YY_RULE_SETUP +#line 284 "test_spec_scan.l" +{ return T_EQUALS; } + YY_BREAK +case 126: +YY_RULE_SETUP +#line 285 "test_spec_scan.l" +{ return T_COMMA; } + YY_BREAK +case 127: +YY_RULE_SETUP +#line 286 "test_spec_scan.l" +{ return T_POSTGRES; } + YY_BREAK +case 128: +YY_RULE_SETUP +#line 287 "test_spec_scan.l" +{ return T_STAYS; } + YY_BREAK +case 129: +YY_RULE_SETUP +#line 288 "test_spec_scan.l" +{ return T_WHILE; } + YY_BREAK +case 130: +YY_RULE_SETUP +#line 289 "test_spec_scan.l" +{ return T_THROUGH; } + YY_BREAK +case 131: +YY_RULE_SETUP +#line 290 "test_spec_scan.l" +{ return T_SET; } + YY_BREAK +case 132: +YY_RULE_SETUP +#line 291 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_INJECT; } + YY_BREAK +case 133: +YY_RULE_SETUP +#line 292 "test_spec_scan.l" +{ return T_LOGS; } + YY_BREAK +case 134: +YY_RULE_SETUP +#line 293 "test_spec_scan.l" +{ return T_NOT; } + YY_BREAK +case 135: +YY_RULE_SETUP +#line 294 "test_spec_scan.l" +{ return T_CONTAINS; } + YY_BREAK +case 136: +YY_RULE_SETUP +#line 295 "test_spec_scan.l" +{ return T_MATCHES; } + YY_BREAK +case 137: +YY_RULE_SETUP +#line 297 "test_spec_scan.l" +{ + yylval.ival = atoi(yytext); + return T_INTEGER; +} + YY_BREAK +case 138: +/* rule 138 can match eol */ +YY_RULE_SETUP +#line 302 "test_spec_scan.l" +{ + yytext[yyleng - 1] = '\0'; + yylval.str = pgaf_strdup(yytext + 1); + return T_STRING; +} + YY_BREAK +case 139: +YY_RULE_SETUP +#line 308 "test_spec_scan.l" +{ + if (pgaf_next_brace_is_while) { + pgaf_next_brace_is_while = 0; + pgaf_step_brace_depth++; + return T_LBRACE; + } + pgaf_read_raw_block(); + return T_BLOCK; +} + YY_BREAK +case 140: +YY_RULE_SETUP +#line 318 "test_spec_scan.l" +{ + if (pgaf_step_brace_depth > 0) { + pgaf_step_brace_depth--; + return T_RBRACE; + } + pgaf_step_brace_depth = 0; + BEGIN(INITIAL); + return T_RBRACE; +} + YY_BREAK +case 141: +YY_RULE_SETUP +#line 328 "test_spec_scan.l" +{ + yylval.str = pgaf_strdup(yytext); + return T_IDENT; +} + YY_BREAK +case 142: +YY_RULE_SETUP +#line 333 "test_spec_scan.l" +{ /* skip whitespace before service name */ } + YY_BREAK +case 143: +YY_RULE_SETUP +#line 335 "test_spec_scan.l" +{ + yylval.str = pgaf_strdup(yytext); + BEGIN(EXEC_ARGS_REST); + return T_IDENT; +} + YY_BREAK +case 144: +/* rule 144 can match eol */ +YY_RULE_SETUP +#line 341 "test_spec_scan.l" +{ + pgaf_line_number++; + BEGIN(STEP_BODY); +} + YY_BREAK +case 145: +YY_RULE_SETUP +#line 346 "test_spec_scan.l" +{ + char *p = yytext; + while (*p == ' ' || *p == '\t') p++; + yylval.str = pgaf_strdup(p); + BEGIN(STEP_BODY); + return T_SHELL_ARGS; +} + YY_BREAK +case 146: +/* rule 146 can match eol */ +YY_RULE_SETUP +#line 354 "test_spec_scan.l" +{ + pgaf_line_number++; + BEGIN(STEP_BODY); +} + YY_BREAK +case 147: +YY_RULE_SETUP +#line 359 "test_spec_scan.l" +ECHO; + YY_BREAK +#line 2301 "test_spec_scan.c" +case YY_STATE_EOF(INITIAL): +case YY_STATE_EOF(CLUSTER_BODY): +case YY_STATE_EOF(STEP_BODY): +case YY_STATE_EOF(EXEC_ARGS): +case YY_STATE_EOF(EXEC_ARGS_REST): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = (yy_hold_char); + YY_RESTORE_YY_MORE_OFFSET + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between YY_CURRENT_BUFFER and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++(yy_c_buf_p); + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_END_OF_FILE: + { + (yy_did_buffer_switch_on_eof) = 0; + + if ( yywrap( ) ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = + (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + (yy_c_buf_p) = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of user's declarations */ +} /* end of yylex */ + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ +static int yy_get_next_buffer (void) +{ + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = (yytext_ptr); + int number_to_move, i; + int ret_val; + + if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1); + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; + + else + { + yy_size_t num_to_read = + YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; + + int yy_c_buf_p_offset = + (int) ((yy_c_buf_p) - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + yy_size_t new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yyrealloc( (void *) b->yy_ch_buf, + (yy_size_t) (b->yy_buf_size + 2) ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = NULL; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - + number_to_move - 1; + + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), + (yy_n_chars), num_to_read ); + + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + if ( (yy_n_chars) == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + /* Extend the array by 50%, plus the number we really need. */ + yy_size_t new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( + (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size ); + if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); + /* "- 2" to take care of EOB's */ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); + } + + (yy_n_chars) += number_to_move; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + + (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; + + return ret_val; +} + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + + static yy_state_type yy_get_previous_state (void) +{ + yy_state_type yy_current_state; + char *yy_cp; + + yy_current_state = (yy_start); + + for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) + { + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1162 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + } + + return yy_current_state; +} + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) +{ + int yy_is_jam; + char *yy_cp = (yy_c_buf_p); + + YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1162 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_is_jam = (yy_current_state == 1161); + + return yy_is_jam ? 0 : yy_current_state; +} + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus + static int yyinput (void) +#else + static int input (void) +#endif + +{ + int c; + + *(yy_c_buf_p) = (yy_hold_char); + + if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + /* This was really a NUL. */ + *(yy_c_buf_p) = '\0'; + + else + { /* need more input */ + yy_size_t offset = (yy_c_buf_p) - (yytext_ptr); + ++(yy_c_buf_p); + + switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /*FALLTHROUGH*/ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap( ) ) + return 0; + + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = (yytext_ptr) + offset; + break; + } + } + } + + c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ + *(yy_c_buf_p) = '\0'; /* preserve yytext */ + (yy_hold_char) = *++(yy_c_buf_p); + + return c; +} +#endif /* ifndef YY_NO_INPUT */ + +/** Immediately switch to a different input stream. + * @param input_file A readable stream. + * + * @note This function does not reset the start condition to @c INITIAL . + */ + void yyrestart (FILE * input_file ) +{ + + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_init_buffer( YY_CURRENT_BUFFER, input_file ); + yy_load_buffer_state( ); +} + +/** Switch to a different input buffer. + * @param new_buffer The new input buffer. + * + */ + void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) +{ + + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (); + if ( YY_CURRENT_BUFFER == new_buffer ) + return; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + YY_CURRENT_BUFFER_LVALUE = new_buffer; + yy_load_buffer_state( ); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + (yy_did_buffer_switch_on_eof) = 1; +} + +static void yy_load_buffer_state (void) +{ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + (yy_hold_char) = *(yy_c_buf_p); +} + +/** Allocate and initialize an input buffer state. + * @param file A readable stream. + * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. + * + * @return the allocated buffer state. + */ + YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) +{ + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; +} + +/** Destroy the buffer. + * @param b a buffer created with yy_create_buffer() + * + */ + void yy_delete_buffer (YY_BUFFER_STATE b ) +{ + + if ( ! b ) + return; + + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yyfree( (void *) b->yy_ch_buf ); + + yyfree( (void *) b ); +} + +/* Initializes or reinitializes a buffer. + * This function is sometimes called more than once on the same buffer, + * such as during a yyrestart() or at EOF. + */ + static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + +{ + int oerrno = errno; + + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + + /* If b is the current buffer, then yy_init_buffer was _probably_ + * called from yyrestart() or through yy_get_next_buffer. + * In that case, we don't want to reset the lineno or column. + */ + if (b != YY_CURRENT_BUFFER){ + b->yy_bs_lineno = 1; + b->yy_bs_column = 0; + } + + b->yy_is_interactive = 0; + + errno = oerrno; +} + +/** Discard all buffered characters. On the next scan, YY_INPUT will be called. + * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. + * + */ + void yy_flush_buffer (YY_BUFFER_STATE b ) +{ + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == YY_CURRENT_BUFFER ) + yy_load_buffer_state( ); +} + +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + * @param new_buffer The new state. + * + */ +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + (yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_LVALUE = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; +} + +/** Removes and deletes the top of the stack, if present. + * The next element becomes the new top. + * + */ +void yypop_buffer_state (void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + if ((yy_buffer_stack_top) > 0) + --(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; + } +} + +/* Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +static void yyensure_buffer_stack (void) +{ + yy_size_t num_to_alloc; + + if (!(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + (yy_buffer_stack_max) = num_to_alloc; + (yy_buffer_stack_top) = 0; + return; + } + + if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + yy_size_t grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = (yy_buffer_stack_max) + grow_size; + (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + ((yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + /* zero only the new slots.*/ + memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + (yy_buffer_stack_max) = num_to_alloc; + } +} + +/** Setup the input buffer state to scan directly from a user-specified character buffer. + * @param base the character buffer + * @param size the size in bytes of the character buffer + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +{ + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return NULL; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = NULL; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; +} + +/** Setup the input buffer state to scan a string. The next call to yylex() will + * scan from a @e copy of @a str. + * @param yystr a NUL-terminated string to scan + * + * @return the newly allocated buffer state object. + * @note If you want to scan bytes that may contain NUL values, then use + * yy_scan_bytes() instead. + */ +YY_BUFFER_STATE yy_scan_string (const char * yystr ) +{ + + return yy_scan_bytes( yystr, (int) strlen(yystr) ); +} + +/** Setup the input buffer state to scan the given bytes. The next call to yylex() will + * scan from a @e copy of @a bytes. + * @param yybytes the byte buffer to scan + * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, yy_size_t _yybytes_len ) +{ + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + yy_size_t i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = (yy_size_t) (_yybytes_len + 2); + buf = (char *) yyalloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < _yybytes_len; ++i ) + buf[i] = yybytes[i]; + + buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; +} + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +static void yynoreturn yy_fatal_error (const char* msg ) +{ + fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); +} + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + yy_size_t yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + yytext[yyleng] = (yy_hold_char); \ + (yy_c_buf_p) = yytext + yyless_macro_arg; \ + (yy_hold_char) = *(yy_c_buf_p); \ + *(yy_c_buf_p) = '\0'; \ + yyleng = yyless_macro_arg; \ + } \ + while ( 0 ) + +/* Accessor methods (get/set functions) to struct members. */ + +/** Get the current line number. + * + */ +int yyget_lineno (void) +{ + + return yylineno; +} + +/** Get the input stream. + * + */ +FILE *yyget_in (void) +{ + return yyin; +} + +/** Get the output stream. + * + */ +FILE *yyget_out (void) +{ + return yyout; +} + +/** Get the length of the current token. + * + */ +yy_size_t yyget_leng (void) +{ + return yyleng; +} + +/** Get the current token. + * + */ + +char *yyget_text (void) +{ + return yytext; +} + +/** Set the current line number. + * @param _line_number line number + * + */ +void yyset_lineno (int _line_number ) +{ + + yylineno = _line_number; +} + +/** Set the input stream. This does not discard the current + * input buffer. + * @param _in_str A readable stream. + * + * @see yy_switch_to_buffer + */ +void yyset_in (FILE * _in_str ) +{ + yyin = _in_str ; +} + +void yyset_out (FILE * _out_str ) +{ + yyout = _out_str ; +} + +int yyget_debug (void) +{ + return yy_flex_debug; +} + +void yyset_debug (int _bdebug ) +{ + yy_flex_debug = _bdebug ; +} + +static int yy_init_globals (void) +{ + /* Initialization is the same as for the non-reentrant scanner. + * This function is called from yylex_destroy(), so don't allocate here. + */ + + (yy_buffer_stack) = NULL; + (yy_buffer_stack_top) = 0; + (yy_buffer_stack_max) = 0; + (yy_c_buf_p) = NULL; + (yy_init) = 0; + (yy_start) = 0; + +/* Defined in main.c */ +#ifdef YY_STDINIT + yyin = stdin; + yyout = stdout; +#else + yyin = NULL; + yyout = NULL; +#endif + + /* For future reference: Set errno on error, since we are called by + * yylex_init() + */ + return 0; +} + +/* yylex_destroy is for both reentrant and non-reentrant scanners. */ +int yylex_destroy (void) +{ + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer( YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + yypop_buffer_state(); + } + + /* Destroy the stack itself. */ + yyfree((yy_buffer_stack) ); + (yy_buffer_stack) = NULL; + + /* Reset the globals. This is important in a non-reentrant scanner so the next time + * yylex() is called, initialization will occur. */ + yy_init_globals( ); + + return 0; +} + +/* + * Internal utility routines. + */ + +#ifndef yytext_ptr +static void yy_flex_strncpy (char* s1, const char * s2, int n ) +{ + + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; +} +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (const char * s ) +{ + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; +} +#endif + +void *yyalloc (yy_size_t size ) +{ + return malloc(size); +} + +void *yyrealloc (void * ptr, yy_size_t size ) +{ + + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return realloc(ptr, size); +} + +void yyfree (void * ptr ) +{ + free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ +} + +#define YYTABLES_NAME "yytables" + +#line 359 "test_spec_scan.l" + + +static void +pgaf_read_raw_block(void) +{ + int depth = 1; + int c; + char buf[65536]; + int pos = 0; + + while (depth > 0 && (c = input()) != EOF) + { + if (c == '{') depth++; + else if (c == '}') { depth--; if (depth == 0) break; } + else if (c == '\n') pgaf_line_number++; + if (depth > 0 && pos < (int)sizeof(buf) - 1) + buf[pos++] = (char) c; + } + buf[pos] = '\0'; + + char *start = buf; + while (*start == ' ' || *start == '\t' || *start == '\n' || *start == '\r') + start++; + char *end = start + strlen(start); + while (end > start && + (end[-1] == ' ' || end[-1] == '\t' || + end[-1] == '\n' || end[-1] == '\r')) + end--; + *end = '\0'; + + yylval.str = pgaf_strdup(start); +} + diff --git a/src/bin/pgaftest/test_spec_scan.l b/src/bin/pgaftest/test_spec_scan.l new file mode 100644 index 000000000..0ee5846d2 --- /dev/null +++ b/src/bin/pgaftest/test_spec_scan.l @@ -0,0 +1,390 @@ +%{ +/* + * src/bin/pgaftest/test_spec_scan.l + * Flex lexer for .pgaf test specification files. + * + * The outer structure (cluster, setup, teardown, step, sequence) is + * tokenised in the INITIAL state. When a step/setup/teardown body + * opens, we switch to the STEP_BODY exclusive state where every + * command keyword and value becomes its own token. + * + * Additional exclusive states: + * CLUSTER_BODY - inside a cluster { } block; tokenises the cluster + * mini-language (formations, nodes, options) + * EXEC_ARGS - skip whitespace, then read the service name token + * EXEC_ARGS_REST - read the rest of the line as a raw T_SHELL_ARGS string + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include +#include "test_spec.h" +#include "test_spec_parse.h" + +int pgaf_line_number = 1; + +/* + * When the lexer sees "setup", "teardown", or "step", it sets this flag + * so the next "{" enters STEP_BODY rather than doing the raw block read. + */ +static int pgaf_next_brace_is_step = 0; + +/* + * When the lexer sees "cluster", it sets this flag so the next "{" + * enters CLUSTER_BODY rather than doing the raw block read. + */ +static int pgaf_next_brace_is_cluster = 0; + +/* + * When 1, the next '{' in STEP_BODY returns T_LBRACE (for a while body) + * instead of calling pgaf_read_raw_block(). Set by the grammar via a + * mid-rule action when T_WHILE is seen. Also used by the setup/teardown + * blocks that need sub-block parsing. + */ +int pgaf_next_brace_is_while = 0; + +/* + * Nesting depth for '{' tokens returned as T_LBRACE inside STEP_BODY + * (while bodies). 0 means no nested while bodies are open. Each T_LBRACE + * increments this; each T_RBRACE decrements it when depth > 0; when depth + * reaches 0 the '}' is the step's own closing brace → BEGIN(INITIAL). + */ +static int pgaf_step_brace_depth = 0; + +/* + * Nesting depth inside CLUSTER_BODY - so we know when the outermost "}" + * closes and we can return to INITIAL. + */ +static int pgaf_cluster_depth = 0; + +static char *pgaf_strdup(const char *s) +{ + char *r = strdup(s); + if (!r) { fprintf(stderr, "out of memory\n"); exit(1); } + return r; +} + +/* + * pgaf_read_raw_block: called when '{' is seen and we want to collect + * everything up to the matching '}' as a single string. Used for + * sql/expect content inside step bodies. + * Stores a freshly allocated trimmed string in yylval.str. + * + * Defined in the user-code section (after the second %%) so that it can + * call flex's static input() function. + */ +static void pgaf_read_raw_block(void); +%} + +%option noyywrap +%option nounput +%option never-interactive + +%x CLUSTER_BODY +%x STEP_BODY +%x EXEC_ARGS +%x EXEC_ARGS_REST + +%% + +"#"[^\n]* { /* comment */ } + +"cluster" { pgaf_next_brace_is_cluster = 1; return T_CLUSTER; } +"monitor" { return T_MONITOR; } +"node" { return T_NODE; } +"citus-coordinator" { return T_CITUS_COORDINATOR; } +"citus-worker" { return T_CITUS_WORKER; } + +"setup" { pgaf_next_brace_is_step = 1; return T_SETUP; } +"teardown" { pgaf_next_brace_is_step = 1; return T_TEARDOWN; } +"step" { pgaf_next_brace_is_step = 1; return T_STEP; } +"sequence" { return T_SEQUENCE; } + +"=" { return T_EQUALS; } + +[0-9]+[sS]? { + yylval.ival = atoi(yytext); + return T_INTEGER; +} + +[A-Za-z_][A-Za-z0-9_-]* { + yylval.str = pgaf_strdup(yytext); + return T_IDENT; +} + +\"[^\"]*\" { + yytext[yyleng - 1] = '\0'; + yylval.str = pgaf_strdup(yytext + 1); + return T_STRING; +} + +"{" { + if (pgaf_next_brace_is_step) + { + pgaf_next_brace_is_step = 0; + BEGIN(STEP_BODY); + return T_LBRACE; + } + if (pgaf_next_brace_is_cluster) + { + pgaf_next_brace_is_cluster = 0; + pgaf_cluster_depth = 1; + BEGIN(CLUSTER_BODY); + return T_LBRACE; + } + /* fallback: should not happen in a well-formed file */ + fprintf(stderr, "pgaftest: unexpected '{' at line %d\n", pgaf_line_number); +} + +\n { pgaf_line_number++; } +[ \t\r]+ { /* whitespace */ } + +. { + fprintf(stderr, "pgaftest: unexpected character '%c' at line %d\n", + yytext[0], pgaf_line_number); +} + +"#"[^\n]* { /* comment */ } +\n { pgaf_line_number++; } +[ \t\r]+ { /* whitespace */ } + +"monitor" { return T_MONITOR; } +"image-target" { return T_IMAGE_TARGET; } +"image" { return T_IMAGE; } +"ssl" { return T_SSL; } +"auth-method" { return T_AUTH_METHOD; } +"auth" { return T_AUTH; } +"formation" { return T_FORMATION; } +"num-sync" { return T_NUM_SYNC; } +"node" { return T_NODE; } + +"coordinator" { return T_COORDINATOR; } +"worker" { return T_WORKER; } +"async" { return T_ASYNC; } +"no-monitor" { return T_NO_MONITOR; } +"password" { return T_PASSWORD; } +"monitor-password" { return T_MONITOR_PASSWORD; } +"launch" { return T_LAUNCH; } +"deferred" { return T_DEFERRED; } +"immediate" { return T_IMMEDIATE; } +"initially" { return T_INITIALLY; } +"stopped" { return T_STOPPED; } +"volume" { return T_VOLUME; } +"listen" { return T_LISTEN; } +"citus-secondary" { return T_CITUS_SECONDARY; } +"candidate-priority" { return T_CANDIDATE_PRIORITY; } +"group" { return T_GROUP; } +"port" { return T_PORT; } +"citus-cluster-name" { return T_CITUS_CLUSTER_NAME; } +"debian-cluster" { return T_DEBIAN_CLUSTER; } +"replication-quorum" { return T_REPLICATION_QUORUM; } +"replication-password" { return T_REPLICATION_PASSWORD; } +"extension-version" { return T_EXTENSION_VERSION; } +"bind-source" { return T_BIND_SOURCE; } + +"=" { return T_EQUALS; } + +[0-9]+[sS]? { + yylval.ival = atoi(yytext); + return T_INTEGER; +} + +\"[^\"]*\" { + yytext[yyleng - 1] = '\0'; + yylval.str = pgaf_strdup(yytext + 1); + return T_STRING; +} + +"{" { + pgaf_cluster_depth++; + return T_LBRACE; +} + +"}" { + pgaf_cluster_depth--; + if (pgaf_cluster_depth == 0) + BEGIN(INITIAL); + return T_RBRACE; +} + +"init" { return T_FS_INIT; } +"single" { return T_FS_SINGLE; } +"primary" { return T_FS_PRIMARY; } +"wait_primary" { return T_FS_WAIT_PRIMARY; } +"wait-primary" { return T_FS_WAIT_PRIMARY; } +"wait_standby" { return T_FS_WAIT_STANDBY; } +"wait-standby" { return T_FS_WAIT_STANDBY; } +"demoted" { return T_FS_DEMOTED; } +"demote_timeout" { return T_FS_DEMOTE_TIMEOUT; } +"demote-timeout" { return T_FS_DEMOTE_TIMEOUT; } +"draining" { return T_FS_DRAINING; } +"secondary" { return T_FS_SECONDARY; } +"catchingup" { return T_FS_CATCHINGUP; } +"prepare_promotion" { return T_FS_PREP_PROMOTION; } +"prepare-promotion" { return T_FS_PREP_PROMOTION; } +"stop_replication" { return T_FS_STOP_REPLICATION; } +"stop-replication" { return T_FS_STOP_REPLICATION; } +"maintenance" { return T_FS_MAINTENANCE; } +"join_primary" { return T_FS_JOIN_PRIMARY; } +"join-primary" { return T_FS_JOIN_PRIMARY; } +"apply_settings" { return T_FS_APPLY_SETTINGS; } +"apply-settings" { return T_FS_APPLY_SETTINGS; } +"prepare_maintenance" { return T_FS_PREPARE_MAINTENANCE; } +"prepare-maintenance" { return T_FS_PREPARE_MAINTENANCE; } +"wait_maintenance" { return T_FS_WAIT_MAINTENANCE; } +"wait-maintenance" { return T_FS_WAIT_MAINTENANCE; } +"report_lsn" { return T_FS_REPORT_LSN; } +"report-lsn" { return T_FS_REPORT_LSN; } +"fast_forward" { return T_FS_FAST_FORWARD; } +"fast-forward" { return T_FS_FAST_FORWARD; } +"join_secondary" { return T_FS_JOIN_SECONDARY; } +"join-secondary" { return T_FS_JOIN_SECONDARY; } +"dropped" { return T_FS_DROPPED; } + +[A-Za-z_][A-Za-z0-9_-]* { + yylval.str = pgaf_strdup(yytext); + return T_IDENT; +} + +"#"[^\n]* { /* comment */ } +\n { pgaf_line_number++; } +[ \t\r]+ { /* whitespace */ } + +"exec-fails" { BEGIN(EXEC_ARGS); return T_EXEC_FAILS; } +"exec" { BEGIN(EXEC_ARGS); return T_EXEC; } +"pg_autoctl" { BEGIN(EXEC_ARGS); return T_PG_AUTOCTL; } + +"wait" { return T_WAIT; } +"until" { return T_UNTIL; } +"timeout" { return T_TIMEOUT; } +"assert" { return T_ASSERT; } +"sql" { return T_SQL; } +"expect" { return T_EXPECT; } +"error" { return T_ERROR; } +"promote" { return T_PROMOTE; } +"network" { return T_NETWORK; } +"disconnect" { return T_DISCONNECT; } +"connect" { return T_CONNECT; } +"sleep" { return T_SLEEP; } +"compose" { return T_COMPOSE; } +"down" { return T_DOWN; } +"start" { return T_START; } +"stop" { return T_STOP; } +"stopped" { return T_STOPPED; } +"kill" { return T_KILL; } +"in" { return T_IN; } +"state" { return T_STATE; } +"assigned-state" { return T_ASSIGNED_STATE; } +"candidate-priority" { return T_CANDIDATE_PRIORITY; } +"group" { return T_GROUP; } +"and" { return T_AND; } +"is" { return T_IS; } +"with" { return T_WITH; } +"=" { return T_EQUALS; } +"," { return T_COMMA; } +"postgres" { return T_POSTGRES; } +"stays" { return T_STAYS; } +"while" { return T_WHILE; } +"through" { return T_THROUGH; } +"set" { return T_SET; } +"inject" { BEGIN(EXEC_ARGS); return T_INJECT; } +"logs" { return T_LOGS; } +"not" { return T_NOT; } +"contains" { return T_CONTAINS; } +"matches" { return T_MATCHES; } + +[0-9]+[sS]? { + yylval.ival = atoi(yytext); + return T_INTEGER; +} + +\"[^\"]*\" { + yytext[yyleng - 1] = '\0'; + yylval.str = pgaf_strdup(yytext + 1); + return T_STRING; +} + +"{" { + if (pgaf_next_brace_is_while) { + pgaf_next_brace_is_while = 0; + pgaf_step_brace_depth++; + return T_LBRACE; + } + pgaf_read_raw_block(); + return T_BLOCK; +} + +"}" { + if (pgaf_step_brace_depth > 0) { + pgaf_step_brace_depth--; + return T_RBRACE; + } + pgaf_step_brace_depth = 0; + BEGIN(INITIAL); + return T_RBRACE; +} + +[A-Za-z_][A-Za-z0-9_-]* { + yylval.str = pgaf_strdup(yytext); + return T_IDENT; +} + +[ \t]+ { /* skip whitespace before service name */ } + +[^ \t\n]+ { + yylval.str = pgaf_strdup(yytext); + BEGIN(EXEC_ARGS_REST); + return T_IDENT; +} + +\n { + pgaf_line_number++; + BEGIN(STEP_BODY); +} + +[^\n]+ { + char *p = yytext; + while (*p == ' ' || *p == '\t') p++; + yylval.str = pgaf_strdup(p); + BEGIN(STEP_BODY); + return T_SHELL_ARGS; +} + +\n { + pgaf_line_number++; + BEGIN(STEP_BODY); +} + +%% + +static void +pgaf_read_raw_block(void) +{ + int depth = 1; + int c; + char buf[65536]; + int pos = 0; + + while (depth > 0 && (c = input()) != EOF) + { + if (c == '{') depth++; + else if (c == '}') { depth--; if (depth == 0) break; } + else if (c == '\n') pgaf_line_number++; + if (depth > 0 && pos < (int)sizeof(buf) - 1) + buf[pos++] = (char) c; + } + buf[pos] = '\0'; + + char *start = buf; + while (*start == ' ' || *start == '\t' || *start == '\n' || *start == '\r') + start++; + char *end = start + strlen(start); + while (end > start && + (end[-1] == ' ' || end[-1] == '\t' || + end[-1] == '\n' || end[-1] == '\r')) + end--; + *end = '\0'; + + yylval.str = pgaf_strdup(start); +} diff --git a/src/monitor/formation_metadata.c b/src/monitor/formation_metadata.c index debdef1e7..d4974fb6b 100644 --- a/src/monitor/formation_metadata.c +++ b/src/monitor/formation_metadata.c @@ -474,8 +474,10 @@ FormationKindToString(FormationKind kind) } default: + { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unknown formation kind value %d", kind))); + } } /* keep compiler happy */ diff --git a/src/monitor/node_active_protocol.c b/src/monitor/node_active_protocol.c index 03eafa2b4..1a4199adb 100644 --- a/src/monitor/node_active_protocol.c +++ b/src/monitor/node_active_protocol.c @@ -476,7 +476,15 @@ NodeActive(char *formationId, AutoFailoverNodeState *currentNodeState) /* * Report the current state. The state might not have changed, but in * that case we still update the last report time. + * + * LockNodeGroup must be taken before ReportAutoFailoverNodeState so + * that every code path acquires advisory locks before row-level locks. + * Reversing the order causes a deadlock with set_node_candidate_priority + * and similar functions that hold the NodeGroup advisory lock while + * updating node-table rows. */ + LockNodeGroup(formationId, currentNodeState->groupId, ExclusiveLock); + ReportAutoFailoverNodeState(pgAutoFailoverNode->nodeHost, pgAutoFailoverNode->nodePort, currentNodeState->replicationState, @@ -486,8 +494,6 @@ NodeActive(char *formationId, AutoFailoverNodeState *currentNodeState) currentNodeState->reportedLSN); } - LockNodeGroup(formationId, currentNodeState->groupId, ExclusiveLock); - ProceedGroupState(pgAutoFailoverNode); AutoFailoverNodeState *assignedNodeState = diff --git a/src/monitor/node_metadata.c b/src/monitor/node_metadata.c index f80d2572b..34fb6adc6 100644 --- a/src/monitor/node_metadata.c +++ b/src/monitor/node_metadata.c @@ -1642,9 +1642,11 @@ SyncStateToString(SyncState pgsrSyncState) } default: + { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unknown SyncState enum value %d", pgsrSyncState))); + } } /* keep compiler happy */ diff --git a/tests/pgautofailover_utils.py b/tests/pgautofailover_utils.py index 0fe9f6a53..4dad8afc2 100644 --- a/tests/pgautofailover_utils.py +++ b/tests/pgautofailover_utils.py @@ -525,7 +525,7 @@ def restart_postgres(self): command = PGAutoCtl(self) command.execute( - "service restart postgres", "do", "service", "restart", "postgres" + "service restart postgres", "manual", "service", "restart", "postgres" ) def pg_is_running(self, timeout=COMMAND_TIMEOUT): @@ -535,7 +535,7 @@ def pg_is_running(self, timeout=COMMAND_TIMEOUT): command = PGAutoCtl(self) try: - command.execute("pgsetup ready", "do", "pgsetup", "ready", "-vvv") + command.execute("pgsetup ready", "inspect", "pgsetup", "ready", "-vvv") except Exception as e: # pg_autoctl uses EXIT_CODE_PGSQL when Postgres is not ready return False @@ -547,7 +547,7 @@ def wait_until_pg_is_running(self, timeout=STATE_CHANGE_TIMEOUT): """ command = PGAutoCtl(self) out, err, ret = command.execute( - "pgsetup ready", "do", "pgsetup", "wait", "-vvv" + "pgsetup ready", "inspect", "pgsetup", "wait", "-vvv" ) return ret == 0 @@ -1217,7 +1217,7 @@ def get_nodeid(self): Fetch the nodeid from the pg_autoctl state file. """ command = PGAutoCtl(self) - out, err, ret = command.execute("get node id", "do", "fsm", "state") + out, err, ret = command.execute("get node id", "inspect", "fsm", "state") self.state = json.loads(out) return self.state["state"]["nodeId"] @@ -1242,7 +1242,7 @@ def get_local_state(self): """ command = PGAutoCtl(self) out, err, ret = command.execute( - "get node id", "-vv", "do", "fsm", "state" + "get node id", "-vv", "inspect", "fsm", "state" ) self.state = json.loads(out) @@ -1418,7 +1418,7 @@ def do_fsm_assign(self, target_state): """ command = PGAutoCtl(self) command.execute( - "do fsm assign", "-vv", "do", "fsm", "assign", target_state + "do fsm assign", "-vv", "manual", "fsm", "assign", target_state ) return True @@ -1435,7 +1435,7 @@ def do_fsm_nodes_set(self, nodesArray): command = PGAutoCtl(self) out, err, ret = command.execute( - "do fsm nodes set", "do", "fsm", "nodes", "set", filename + "do fsm nodes set", "manual", "fsm", "nodes", "set", filename ) return True @@ -1446,7 +1446,7 @@ def do_fsm_step(self): :return: """ command = PGAutoCtl(self) - command.execute("do fsm step", "do", "fsm", "step") + command.execute("do fsm step", "manual", "fsm", "step") return True def set_metadata(self, name=None, host=None, port=None): diff --git a/tests/tap/README.md b/tests/tap/README.md new file mode 100644 index 000000000..47c6ab798 --- /dev/null +++ b/tests/tap/README.md @@ -0,0 +1,217 @@ +# pg_auto_failover TAP Test Suite + +Tests are written in the `.pgaf` spec DSL and run by the `pgaftest` binary. +Output is [TAP](https://testanything.org/) (Test Anything Protocol), +compatible with `prove` and any TAP harness. + +## Prerequisites + +- `pgaftest` built: `make -C src/bin` +- Docker with the Compose plugin (`docker compose version`) + +## Running tests + +### All tests (via schedule) + +```sh +pgaftest run --schedule tests/tap/schedule +``` + +Runs every spec listed in `tests/tap/schedule` and emits TAP on stdout. +Exit code is 0 only when all tests pass. + +With `prove` for a formatted summary: + +```sh +pgaftest run --schedule tests/tap/schedule | prove --tap - +``` + +### A single spec + +```sh +pgaftest run tests/tap/specs/basic_operation.pgaf +``` + +### Interactive mode (live cluster, shell access) + +Brings up the cluster, runs `setup {}`, then drops you into a shell: + +```sh +pgaftest setup tests/tap/specs/basic_operation.pgaf +``` + +Run individual steps against the live cluster: + +```sh +pgaftest step stop_primary tests/tap/specs/basic_operation.pgaf +pgaftest step check_failover tests/tap/specs/basic_operation.pgaf +``` + +Inspect the generated `docker-compose.yml` without starting anything: + +```sh +pgaftest show tests/tap/specs/basic_operation.pgaf +``` + +Tear down when done: + +```sh +pgaftest down tests/tap/specs/basic_operation.pgaf +``` + +## Spec file format + +Each `.pgaf` file in `specs/` is a complete, self-contained test scenario. + +``` +cluster { + monitor + formation { + node1 + node2 candidate-priority 0 + } +} + +setup { + wait until node1 state is primary timeout 120s + wait until node2 state is secondary timeout 120s +} + +teardown { + compose down +} + +step stop_primary { + compose kill node1 +} + +step check_failover { + wait until node2 state is primary timeout 90s + assert node2 state is primary +} + +sequence + stop_primary + check_failover +``` + +| Block | Purpose | +|---|---| +| `cluster {}` | Topology — drives `docker-compose.yml` generation | +| `setup {}` | Run after `compose up -d`, before the first step | +| `teardown {}` | Always run at end (CI); on-demand in interactive mode | +| `step name {}` | Named command block | +| `sequence` | Step execution order for `pgaftest run` | + +### Commands inside blocks + +**Process control** + +| Command | Effect | +|---|---| +| `exec ` | `docker compose exec -T ` — fails if exit ≠ 0 | +| `exec-fails ` | Same but asserts non-zero exit | +| `compose down` | `docker compose down --volumes` | +| `compose start ` | `docker compose start ` | +| `compose stop ` | `docker compose stop ` | +| `compose kill ` | `docker compose kill ` (SIGKILL) | +| `stop postgres ` | Stop Postgres inside the container without stopping the keeper | +| `start postgres ` | Restart Postgres inside the container via the keeper | + +**State and timing** + +| Command | Effect | +|---|---| +| `wait until state is [timeout Ns]` | Poll monitor; fail on timeout | +| `wait until assigned-state is [timeout Ns]` | Poll assigned state | +| `wait until stopped [timeout Ns]` | Wait until container exits | +| `wait until , , … [in group N] [timeout Ns]` | Formation-wide state convergence | +| `wait until state is and state is … [timeout Ns]` | Multi-node simultaneous wait | +| `assert state is ` | Instant check; no polling | +| `assert assigned-state is ` | Check assigned state column | +| `assert stays while { … }` | Assert state unchanged throughout body | +| `sleep Ns` | Wait N seconds | + +**SQL and expectations** + +| Command | Effect | +|---|---| +| `sql { SQL }` | Run SQL on service, capture output | +| `expect { text }` | Substring-match last `sql` output | +| `expect { { row } { row } }` | Tuple form: match `psql --tuples-only --no-align` rows | +| `expect error [SQLSTATE]` | Assert previous `sql` raised a SQL error | + +**Network** + +| Command | Effect | +|---|---| +| `network disconnect ` | `docker network disconnect` — simulate partition | +| `network connect ` | `docker network connect` — restore | + +**Log inspection** + +| Command | Effect | +|---|---| +| `logs contains ""` | Assert literal `` appears in container logs | +| `logs not contains ""` | Assert literal `` does NOT appear | +| `logs matches ""` | Assert PCRE pattern matches a log line (`grep -P`) | +| `logs not matches ""` | Assert PCRE pattern does NOT match any log line | + +**Cluster lifecycle** + +| Command | Effect | +|---|---| +| `promote [, , …]` | `pg_autoctl perform promotion` per node | +| `set monitor ` | Switch runner's active monitor (for replace-monitor tests) | + +### `%CIDR%` macro + +The literal `%CIDR%` is expanded to the Docker network CIDR in any `exec` +or `exec-fails` argument. Use it to scope HBA rules to the test network: + +``` +exec monitor bash -c "echo 'host all all %CIDR% trust' >> $PGDATA/pg_hba.conf" +``` + +## Schedule file + +`tests/tap/schedule` lists one spec *name* per line (no path, no `.pgaf` +extension). Lines starting with `#` are comments. + +``` +basic_operation +multi_standbys +# citus_basic # requires Citus image — set CITUS_CLUSTER=1 +``` + +## Relationship to Python tests + +Each `.pgaf` spec was ported from a Python integration test in `tests/`. +The original is recorded in the spec header: + +``` +# Predecessor: tests/test_basic_operation.py +``` + +The pgaf framework replaces the Python `pgautofailover_utils` subprocess +harness with declarative DSL commands backed by `docker compose`. Node +creation, cluster initialization, and teardown are handled by the cluster +declaration and `compose up / down`; test steps focus purely on the +scenario logic. + +## CI integration + +```yaml +- name: Run pg_auto_failover tests + run: | + make -C src/bin pgaftest + pgaftest run --schedule tests/tap/schedule +``` + +Or with `prove`: + +```yaml +- run: pgaftest run --schedule tests/tap/schedule | prove --tap - +``` + +For the full DSL reference see [`docs/pgaftest.rst`](../../docs/pgaftest.rst). diff --git a/tests/tap/schedule b/tests/tap/schedule new file mode 100644 index 000000000..a58a8f966 --- /dev/null +++ b/tests/tap/schedule @@ -0,0 +1,36 @@ +# tests/tap/schedule +# One spec name per line; pgaftest run --schedule tests/tap/schedule +# Lines starting with # are comments; empty lines are ignored. +# Citus specs require CITUS_CLUSTER=1 env var to be set. + +basic_operation +basic_operation_listen_flag +maintenance_and_drop +create_standby_with_pgdata +ensure +monitor_disabled +replace_monitor +config_get_set +skip_pg_hba +#debian_clusters +auth +enable_ssl +ssl_cert +ssl_self_signed +multi_standbys +multi_async +multi_ifdown +multi_maintenance +multi_alternate +extension_update +installcheck +# Upgrade test — requires pgaf:current and pgaf:next images to be pre-built: +# make -C tests/upgrade pgaf-current pgaf-next +upgrade +# Citus tests (uncomment when CITUS_CLUSTER=1): +# basic_citus_operation +# citus_cluster_name +# citus_force_failover +# citus_multi_standbys +# citus_skip_pg_hba +# nonha_citus_operation diff --git a/tests/tap/specs/auth.pgaf b/tests/tap/specs/auth.pgaf new file mode 100644 index 000000000..4301c2492 --- /dev/null +++ b/tests/tap/specs/auth.pgaf @@ -0,0 +1,76 @@ +# Test md5 authentication between nodes and the monitor, including password +# handling and verifying that passwords are not leaked in logs. +# +# Ported from tests/test_auth.py. +# +# Password handling +# ----------------- +# All passwords are declared in the cluster block and written into the ini +# files by compose_gen before any container starts: +# +# cluster.monitorPassword → monitor.ini [pg_auto_failover] autoctl_node_password +# → every node.ini [monitor] pguri (credential in URI) +# → pgaftest service env PG_AUTOCTL_MONITOR (pguri) +# node.monitorPassword → node.ini [pg_auto_failover] monitor_password +# used by fsm_init_primary() when creating the +# pgautofailover_monitor health-check role +# node.replicationPassword → node.ini [replication] password +# used when creating pgautofailover_replicator +# +# All passwords are therefore in place before pg_autoctl runs its first FSM +# transition, removing any need for sequential startup or manual exec steps. +# Predecessor: tests/test_auth.py + +cluster { + monitor password "pg-auto-failover" + auth md5 + formation auth { + node1 monitor-password "pg-hba-check" replication-password "pg-replication" + node2 monitor-password "pg-hba-check" replication-password "pg-replication" + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_001_init_primary { + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_ } +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } + sql node2 { SELECT count(*) FROM t1; } + expect { 2 } +} + +step test_003_failover { + exec monitor pg_autoctl perform failover --formation auth + wait until node2 state is primary + and node1 state is secondary + timeout 90s + wait until node2 state is primary timeout 90s + wait until node1 state is secondary timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_ } +} + +step test_004_verify_data { + sql node2 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT count(*) FROM t1; } + expect { 3 } +} + +step test_005_logging_of_passwords { + logs node2 not contains "monitor_password" + logs node2 not contains "streaming_password" + logs node2 contains "monitor-password ****" + logs node2 contains "replication-password ****" +} diff --git a/tests/tap/specs/basic_citus_operation.pgaf b/tests/tap/specs/basic_citus_operation.pgaf new file mode 100644 index 000000000..809c1d6cf --- /dev/null +++ b/tests/tap/specs/basic_citus_operation.pgaf @@ -0,0 +1,132 @@ +# Test basic Citus cluster operations: coordinator HA, worker HA with two +# worker groups, distributed table writes/reads, and failover at each level. +# +# Ported from tests/test_basic_citus_operation.py +# Predecessor: tests/test_basic_citus_operation.py + +cluster { + monitor + formation { + coordinator1a coordinator + coordinator1b coordinator + worker1a worker group 1 + worker1b worker group 1 + worker2a worker group 2 + worker2b worker group 2 + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator pair comes up +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + timeout 90s +} + +# +# test_002: worker groups come up +# + +step test_002_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + and worker2a state is primary + and worker2b state is secondary + timeout 90s +} + +step test_002b_wait_metadata_sync { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { + CREATE OR REPLACE FUNCTION public.wait_until_metadata_sync(timeout + INTEGER DEFAULT 15000) RETURNS void LANGUAGE C STRICT AS 'citus'; + } + sql coordinator1a { SELECT public.wait_until_metadata_sync(); } +} + +# +# test_003: create distributed table +# + +step test_003_create_distributed_table { + sql coordinator1a { CREATE TABLE t1 (a int); } + sql coordinator1a { SELECT create_distributed_table('t1', 'a'); } + sql coordinator1a { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_001_fail_worker2 { + network disconnect worker2a + wait until worker2b state is wait_primary timeout 90s +} + +step test_004_003_insert_while_wait_primary { + wait until worker2b state is wait_primary timeout 90s + sql coordinator1a { INSERT INTO t1 VALUES (3); } +} + +step test_004_004_reconnect_worker2a { + network connect worker2a + wait until worker2b state is primary + and worker2a state is secondary + timeout 180s +} + +step test_005_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } } +} + +# +# test_006: write more rows +# + +step test_006_writes_to_coordinator_succeed { + sql coordinator1a { INSERT INTO t1 VALUES (4); } + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +step test_007_fail_worker2b { + network disconnect worker2b + wait until worker2a state is wait_primary timeout 90s +} + +step test_007b_reconnect_worker2b { + network connect worker2b + wait until worker2a state is primary + and worker2b state is secondary + timeout 180s +} + +step test_008_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +step test_009_perform_failover_worker2 { + exec monitor pg_autoctl perform failover --group 2 + wait until worker2b state is primary + and worker2a state is secondary + timeout 180s +} + +step test_010_perform_failover_coordinator { + exec monitor pg_autoctl perform failover --group 0 + wait until coordinator1a state is secondary + and coordinator1b state is primary + timeout 90s +} diff --git a/tests/tap/specs/basic_operation.pgaf b/tests/tap/specs/basic_operation.pgaf new file mode 100644 index 000000000..7dfae59db --- /dev/null +++ b/tests/tap/specs/basic_operation.pgaf @@ -0,0 +1,402 @@ +# Basic two-node HA: create primary + secondary, test maintenance, +# failover, network partition detection, and node drop. +# +# Ported from tests/test_basic_operation.py — same step ordering and +# same commands as the Python implementation (run-on-node vs run-on-monitor +# matches the Python test). +# +# Node IDs are deterministic because node2's container depends_on node1 +# (see compose_gen.c), so node1 always registers first and gets nodeid=1, +# node2 gets nodeid=2, node3 gets nodeid=3. Replication slot names embed +# the standby's nodeid: +# pgautofailover_standby_1 — node1's slot (on the primary) +# pgautofailover_standby_2 — node2's slot (on the primary) +# pgautofailover_standby_3 — node3's slot (on the primary) +# +# node3 is declared launch deferred so the container starts with +# `sleep infinity`; test_017 issues `compose start node3` which +# restarts it with the normal `pg_autoctl node run` command. +# Predecessor: tests/test_basic_operation.py + +cluster { + monitor + formation { + node1 + node2 + node3 launch deferred + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +# +# test_003: create table on primary and verify replication to secondary. +# + +step test_003_create_t1 { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node2 { SELECT count(*) FROM t1; } + expect { 2 } +} + +# +# test_004: verify synchronous_standby_names and replication slots +# after secondary joins. Both nodes must have a physical slot +# for the other's replication identity. +# + +step test_004_init_secondary { + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_1 } +} + +# +# test_005: read replicated data from the secondary. +# + +step test_005_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +# +# test_006: verify the secondary rejects writes (read-only replica). +# INSERT must fail with SQLSTATE 25006 (read_only_sql_transaction). +# + +step test_006_001_writes_to_node2_fail { + sql node2 { INSERT INTO t1 VALUES (3); } + expect error 25006 +} + +step test_006_002_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +# +# test_007: maintenance mode on node1 (primary). +# test_007_002: enabling maintenance without --allow-failover on the primary +# must fail (no standby would remain to take over). +# test_007_004: with --allow-failover node2 transitions to wait_primary. +# test_007_005: disabling maintenance rejoins node1 as secondary. +# + +step test_007_002_maintenance_primary { + exec-fails node1 pg_autoctl enable maintenance +} + +step test_007_004_maintenance_primary_allow_failover { + exec node1 pg_autoctl enable maintenance --allow-failover + wait until node1 state is maintenance + and node2 state is wait_primary + timeout 60s + sql node2 { SHOW synchronous_standby_names; } + expect { } +} + +step test_007_005_disable_maintenance { + exec node1 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is primary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +# +# test_008: maintenance mode on node1 (now secondary after test_007). +# test_008_001: enable maintenance, stop postgres on node1, write to primary. +# test_008_002: disable maintenance — node1 pg_rewinds and rejoins as secondary. +# + +step test_008_001_enable_maintenance_secondary { + exec node1 pg_autoctl enable maintenance + wait until node1 state is maintenance timeout 60s + stop postgres node1 + sql node2 { INSERT INTO t1 VALUES (3); } +} + +step test_008_002_disable_maintenance_secondary { + exec node1 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is primary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +# +# test_009: failback — promote node1 back to primary via perform-promotion. +# After: node1=primary, node2=secondary. +# + +step test_009_failback { + exec monitor pg_autoctl perform promotion --name node1 + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } +} + +# +# test_010: disconnect node1 (primary) from the network — simulates a network +# partition. node2 must detect the loss and enter wait_primary +# (it cannot promote yet without confirmation from the monitor). +# + +step test_010_fail_primary { + network disconnect node1 + wait until node2 state is wait_primary timeout 120s +} + +# +# test_011: node2 is now wait_primary — writes must succeed even before +# the full promotion completes (no sync standby required in +# wait_primary state). +# + +step test_011_writes_to_node2_succeed { + sql node2 { INSERT INTO t1 VALUES (4); } + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_012: reconnect node1 — it pg_rewinds against node2 and rejoins as +# secondary. node2 transitions from wait_primary to primary once +# it has a healthy sync standby again. +# + +step test_012_start_node1_again { + network connect node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +# +# test_013: verify all four rows are visible on node1 (now secondary), +# confirming pg_rewind brought it fully up to date. +# + +step test_013_read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_014: node1 is secondary again — writes must be rejected (25006). +# + +step test_014_writes_to_node1_fail { + sql node1 { INSERT INTO t1 VALUES (5); } + expect error 25006 +} + +# +# test_015: stop node1 (secondary) via compose stop — Postgres shuts down +# cleanly. node2 enters wait_primary because its sync standby +# disappeared (number_sync_standbys=1). +# + +step test_015_fail_secondary { + compose stop node1 + wait until node1 stopped timeout 30s + wait until node2 state is wait_primary timeout 60s +} + +# +# test_016: restart node1, let it rejoin, then drop it with --no-wait. +# node2 becomes single. Verify no stale replication slots remain +# on node2 after the drop (pg_auto_failover must clean them up). +# + +step test_016_drop_secondary { + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + exec node1 pg_autoctl drop node --no-wait + wait until node1 stopped timeout 30s + wait until node2 state is single timeout 90s + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { } +} + +# +# test_017–019: add node3 as a new secondary (was launch deferred). +# test_017: issue `pg_autoctl node start` to bring the deferred node live. +# test_018: perform failover must fail immediately — node3 is still +# catching up and no election can complete yet. +# test_019: wait for node3 to reach secondary; verify SSN and slots. +# + +step test_017_add_new_secondary { + exec node3 pg_autoctl node start +} + +step test_018_cant_failover_yet { + exec-fails monitor pg_autoctl perform failover +} + +step test_019_run_secondary { + wait until node3 state is secondary + and node2 state is primary + timeout 120s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3) } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_3 } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } +} + +# +# test_020: two back-to-back manual failovers. After each failover verify +# synchronous_standby_names and that every node has the correct +# physical replication slot for the other nodes — slots must +# survive a role change intact. +# + +step test_020_multiple_manual_failover_verify_replication_slots { + exec monitor pg_autoctl perform failover + wait until node3 state is primary + and node2 state is secondary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_3 } + exec node2 pg_autoctl perform promotion + wait until node2 state is primary + and node3 state is secondary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3) } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_3 } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { pgautofailover_standby_2 } +} + +# +# test_021–023: network partition detection and demote-timeout. +# test_021: disconnect node2 (primary) — simulates a hard network failure. +# test_022: wait for demote_timeout on node2 and wait_primary on node3; +# verify pg_autoctl on node2 considers itself unready (no monitor +# contact) and that node3 has no sync standby yet. +# test_023: reconnect node2 — it detects it lost the election, pg_rewinds +# against node3, and rejoins as secondary. +# + +step test_021_ifdown_primary { + wait until node2 state is primary timeout 30s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3) } + network disconnect node2 +} + +step test_022_detect_network_partition { + wait until node2 state is demote_timeout timeout 90s + wait until node3 state is wait_primary timeout 90s + sleep 3s + exec-fails node2 pg_autoctl inspect pgsetup ready + sql node3 { SHOW synchronous_standby_names; } + expect { } +} + +step test_023_ifup_old_primary { + network connect node2 + wait until node2 state is secondary + and node3 state is primary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2) } +} + +# +# test_024: stop the monitor's Postgres process directly (bypassing +# pg_autoctl) and verify pg_autoctl restarts it automatically. +# The data node must remain in its current state throughout — +# a brief monitor outage must not trigger a spurious failover. +# + +step test_024_stop_postgres_monitor { + assert node3 stays primary while { + stop postgres monitor + sleep 5s + } + wait until node3 state is primary timeout 60s +} + +# +# test_025: drop the last remaining data node — cluster goes empty. +# + +step test_025_drop_primary { + exec node3 pg_autoctl drop node --no-wait + wait until node3 stopped timeout 60s + wait until node2 state is single timeout 90s +} diff --git a/tests/tap/specs/basic_operation_listen_flag.pgaf b/tests/tap/specs/basic_operation_listen_flag.pgaf new file mode 100644 index 000000000..830f05c3d --- /dev/null +++ b/tests/tap/specs/basic_operation_listen_flag.pgaf @@ -0,0 +1,117 @@ +# Test basic pg_auto_failover operations with the --listen flag, verifying +# that nodes bind on all interfaces for replication and failover. +# +# Ported from tests/test_basic_operation_listen_flag.py +# Predecessor: tests/test_basic_operation_listen_flag.py + +cluster { + monitor + formation { + node1 listen + node2 launch deferred listen + } +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: init primary with listen flag +# + +step test_001_init_primary { + wait until node1 state is single timeout 60s + exec node1 pg_autoctl inspect pgsetup wait +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_003: add secondary node2 with listen flag +# + +step test_003_init_secondary { + exec node2 pg_autoctl node start + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +# +# test_004: read from secondary +# + +step test_004_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +step test_005_writes_to_node2_fail { + sql node2 { INSERT INTO t1 VALUES (3); } + expect error 25006 +} + +# +# test_006: fail primary node1 +# + +step test_006_fail_primary { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} + +# +# test_007: writes to new primary succeed +# + +step test_007_writes_to_node2_succeed { + sql node2 { INSERT INTO t1 VALUES (3); } + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} + +# +# test_008: restart node1 as secondary +# + +step test_008_start_node1_again { + network connect node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} + +# +# test_009: read from new secondary (node1) +# + +step test_009_read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} + +step test_010_writes_to_node1_fail { + sql node1 { INSERT INTO t1 VALUES (3); } + expect error 25006 +} + +# +# test_011: fail secondary node1 +# + +step test_011_fail_secondary { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} diff --git a/tests/tap/specs/citus_cluster_name.pgaf b/tests/tap/specs/citus_cluster_name.pgaf new file mode 100644 index 000000000..ef84bee4d --- /dev/null +++ b/tests/tap/specs/citus_cluster_name.pgaf @@ -0,0 +1,120 @@ +# Test Citus cluster name and read-only secondary cluster routing. +# +# A "readonly" cluster is a set of Citus nodes registered with +# candidate-priority=0, citus-secondary, and citus-cluster-name=readonly. +# These nodes should have: +# citus.use_secondary_nodes = 'always' +# citus.cluster_name = 'readonly' +# in their postgresql.conf, so that distributed queries are routed to the +# secondary (standby) copies of each shard rather than the primaries. +# +# Ported from tests/test_citus_cluster_name.py. +# Predecessor: tests/test_citus_cluster_name.py + +cluster { + monitor + formation { + coordinator1a coordinator + node coordinator1b { + coordinator + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + worker1a worker group 1 + node worker1b { + worker group 1 + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + worker2a worker group 2 + node worker2b { + worker group 2 + candidate-priority 0 + citus-secondary + citus-cluster-name readonly + } + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator pair — coordinator1b is read-only secondary cluster +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + timeout 90s +} + +step test_002_001_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + timeout 90s +} + +step test_002_002_init_workers { + wait until worker2a state is primary + and worker2b state is secondary + timeout 90s +} + +# +# test_003: create distributed table and insert data +# + +step test_003_create_distributed_table { + sql coordinator1a { + SET citus.shard_count TO 4; CREATE TABLE t1 (a int); SELECT + create_distributed_table('t1', 'a'); + } + sql coordinator1a { + INSERT INTO t1 SELECT x FROM generate_series(1, 1000) as gs(x); + } + sql coordinator1a { CHECKPOINT; } +} + +# +# test_004: verify read-only cluster settings +# + +step test_004_read_only_cluster { + sql coordinator1a { + SELECT current_setting('citus.use_secondary_nodes'), + current_setting('citus.cluster_name'); + } + expect { never|default } + sql coordinator1b { + SELECT current_setting('citus.use_secondary_nodes'), + current_setting('citus.cluster_name'); + } + expect { always|readonly } + sql coordinator1b { SELECT count(*) FROM t1; } + expect { 1000 } +} + +# +# test_005: drop distributed table (cleanup) +# + +step test_005_drop_table { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true AND nodecluster = '"'"'default'"'"'"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { + CREATE OR REPLACE FUNCTION public.wait_until_metadata_sync(timeout + INTEGER DEFAULT 15000) RETURNS void LANGUAGE C STRICT AS 'citus'; + } + sql coordinator1a { SELECT public.wait_until_metadata_sync(); } + sql coordinator1a { DROP TABLE t1; } +} diff --git a/tests/tap/specs/citus_force_failover.pgaf b/tests/tap/specs/citus_force_failover.pgaf new file mode 100644 index 000000000..858b88787 --- /dev/null +++ b/tests/tap/specs/citus_force_failover.pgaf @@ -0,0 +1,86 @@ +# Test Citus force failover: a transaction in progress competes with a worker +# failover. The failover mechanism must terminate competing backends to make +# progress within a bounded time window. +# Also tests dropping a failed primary worker and a failed coordinator. +# +# Predecessor: tests/test_citus_force_failover.py + +cluster { + monitor + formation { + coordinator1a coordinator + coordinator1b coordinator + worker1a worker group 1 + worker1b worker group 1 + worker2a worker group 2 + worker2b worker group 2 + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator pair +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + timeout 90s +} + +# +# test_002: worker groups +# + +step test_002_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + and worker2a state is primary + and worker2b state is secondary + timeout 90s +} + +# +# test_003: create distributed table +# + +step test_003_create_distributed_table { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { + SET citus.shard_count TO 4; CREATE TABLE t1 (a int); SELECT + create_distributed_table('t1', 'a'); + } +} + +step test_004_fail_while_transaction_is_in_progress { + network disconnect worker1a + wait until worker1b state is wait_primary timeout 90s +} + +# +# test_005: drop failed primary worker2a and coordinator1a +# + +step test_005_drop_primary_worker { + network disconnect worker2a + wait until worker2b state is wait_primary timeout 90s + compose stop worker2a + exec monitor psql -d pg_auto_failover -tAc "SELECT pgautofailover.remove_node(nodehost, nodeport) FROM pgautofailover.node WHERE nodename = 'worker2a'" + wait until worker2b state is single timeout 60s +} + +step test_005_drop_primary_coordinator { + compose stop coordinator1a + exec monitor psql -d pg_auto_failover -tAc "SELECT pgautofailover.remove_node(nodehost, nodeport) FROM pgautofailover.node WHERE nodename = 'coordinator1a'" + wait until coordinator1b state is single timeout 60s +} diff --git a/tests/tap/specs/citus_multi_standbys.pgaf b/tests/tap/specs/citus_multi_standbys.pgaf new file mode 100644 index 000000000..b81b51c69 --- /dev/null +++ b/tests/tap/specs/citus_multi_standbys.pgaf @@ -0,0 +1,176 @@ +# Test Citus cluster with multiple standbys per group (3 nodes per group: +# a primary, a sync secondary, and an async secondary with candidate-priority 0 +# and replication-quorum false). Verifies failover, data consistency, and +# synchronous_standby_names after each failure/recovery cycle. +# +# Ported from tests/test_citus_multi_standbys.py +# Predecessor: tests/test_citus_multi_standbys.py + +cluster { + monitor + formation { + coordinator1a coordinator + coordinator1b coordinator + node coordinator1c { + coordinator + candidate-priority 0 + replication-quorum false + } + worker1a worker group 1 + worker1b worker group 1 + node worker1c { + worker group 1 + candidate-priority 0 + replication-quorum false + } + worker2a worker group 2 + worker2b worker group 2 + node worker2c { + worker group 2 + candidate-priority 0 + replication-quorum false + } + } +} + +setup { + wait until primary, secondary in group 0 timeout 90s + wait until primary, secondary in group 1 timeout 90s + wait until primary, secondary in group 2 timeout 90s + wait until coordinator1c state is secondary timeout 90s + wait until worker1c state is secondary timeout 90s + wait until worker2c state is secondary timeout 90s + promote coordinator1a, worker1a, worker2a +} + +teardown { + compose down +} + +# +# test_001: coordinator group with async secondary coordinator1c +# + +step test_001_init_coordinator { + wait until coordinator1a state is primary + and coordinator1b state is secondary + and coordinator1c state is secondary + timeout 90s + exec coordinator1c pg_autoctl get node candidate-priority + expect { 0 } +} + +# +# test_002: worker groups with async standbys +# + +step test_002_init_workers { + wait until worker1a state is primary + and worker1b state is secondary + and worker1c state is secondary + and worker2a state is primary + and worker2b state is secondary + and worker2c state is secondary + timeout 90s + exec worker1c pg_autoctl get node candidate-priority + expect { 0 } + exec worker2c pg_autoctl get node candidate-priority + expect { 0 } +} + +# +# test_003: create distributed table and seed data +# + +step test_003_001_create_distributed_table { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' + sql coordinator1a { ALTER DATABASE demo SET citus.shard_count TO 4; } + sql coordinator1a { CREATE TABLE t1 (a int); } + sql coordinator1a { SELECT create_distributed_table('t1', 'a'); } + sql coordinator1a { INSERT INTO t1 VALUES (1), (2); } + sql coordinator1a { SELECT a FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +# +# test_004: fail worker2a, reads for worker1 still work via coordinator +# + +step test_004_001_fail_worker2 { + network disconnect worker2a +} + +step test_004_002_writes_via_coordinator_to_worker2_fail { + sql coordinator1a { INSERT INTO t1 VALUES (3); } + expect error +} + +step test_004_004_wait_for_failover { + wait until worker2b state is wait_primary + and worker2c state is secondary + timeout 90s + exec worker2c pg_autoctl get node candidate-priority + expect { 0 } +} + +step test_004_005_writes_via_coordinator_succeed_after_failover { + sql coordinator1a { INSERT INTO t1 VALUES (3); } +} + +step test_005_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } } +} + +# +# test_006: write more data +# + +step test_006_writes_to_coordinator_succeed { + sql coordinator1a { INSERT INTO t1 VALUES (4); } + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_007: restart worker2a as secondary under new primary worker2b +# + +step test_007_start_worker2a_again { + network connect worker2a + wait until worker2a state is secondary + and worker2b state is primary + timeout 90s +} + +step test_008_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_009: fail worker2b, failover back to worker2a +# + +step test_009_fail_worker2b { + network disconnect worker2b + wait until worker2a state is wait_primary + and worker2c state is secondary + timeout 90s +} + +step test_010_read_from_workers_via_coordinator { + sql coordinator1a { SELECT a FROM t1 ORDER BY a ASC; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_011: restart worker2b as secondary +# + +step test_011_start_worker2b_again { + network connect worker2b + wait until worker2b state is secondary + and worker2a state is primary + timeout 90s +} diff --git a/tests/tap/specs/citus_skip_pg_hba.pgaf b/tests/tap/specs/citus_skip_pg_hba.pgaf new file mode 100644 index 000000000..055aa64dc --- /dev/null +++ b/tests/tap/specs/citus_skip_pg_hba.pgaf @@ -0,0 +1,124 @@ +# Test Citus cluster with authMethod=skip (pg_autoctl does not edit pg_hba.conf). +# +# With auth=skip, pg_autoctl leaves pg_hba.conf untouched after postgres init. +# The coordinator pair starts normally (coord0b pg_basebackup-s from coord0a +# and inherits its HBA; the monitor and coordinator connections work via the +# Docker network with trust auth set at the monitor level). +# +# worker1a is launch deferred. The first manual run fails because the +# coordinator tries master_activate_node on worker1a but worker1a's default +# pg_hba.conf blocks that connection. After we manually append the required +# HBA rules and retry, activation succeeds — and pg_autoctl has not modified +# pg_hba.conf itself (verified by assert hba-edited = false). +# +# worker1b is also launch deferred to ensure it starts only after worker1a is +# the active primary for group 1, matching the Python test's sequential order. +# +# Ported from tests/test_citus_skip_pg_hba.py +# Predecessor: tests/test_citus_skip_pg_hba.py + +cluster { + monitor + auth skip + formation { + coord0a coordinator + coord0b coordinator launch deferred + worker1a worker group 1 launch deferred + worker1b worker group 1 launch deferred + } +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_000: monitor with auth=skip, append trust rule to pg_hba.conf manually +# + +step test_000_create_monitor { + exec monitor pg_autoctl override pgsetup hba-lan + exec coord0a pg_autoctl override pgsetup hba-lan +} + +# +# test_001a: init coordinator coord0a (single) +# + +step test_001a_init_coordinator { + wait until coord0a state is single timeout 60s +} + +# +# test_001b: init coordinator coord0b (secondary) +# + +step test_001b_init_coordinator { + exec coord0b pg_autoctl node start /etc/pgaf/node.ini + wait until coord0a state is primary + and coord0b state is secondary + timeout 90s +} + +step test_002b_create_worker { + exec-fails worker1a pg_autoctl node run /etc/pgaf/node.ini +} + +# +# test_002b/002d: worker1a — registration initially fails because coordinator +# cannot connect; after manually adding HBA rules worker1a runs and activates. +# + +step test_002d_run_worker { + exec worker1a pg_autoctl override pgsetup hba-lan + exec worker1a pg_autoctl node run --background /etc/pgaf/node.ini +} + +step test_002e_check_hba { + exec-fails worker1a grep "Auto-generated by pg_auto_failover" /var/lib/postgres/pgaf/pg_hba.conf +} + +step test_002f_activated_node { + sql coord0a { + SELECT isactive FROM pg_dist_node WHERE nodename = 'worker1a'; + } + expect { true } +} + +# +# test_003: init worker1b as secondary +# + +step test_003_init_worker { + exec worker1b pg_autoctl node run --background /etc/pgaf/node.ini + wait until worker1a state is primary + and worker1b state is secondary + timeout 90s + exec-fails worker1b grep "Auto-generated by pg_auto_failover" /var/lib/postgres/pgaf/pg_hba.conf +} + +# +# test_004: create distributed table +# + +step test_004_create_distributed_table { + sql coord0a { CREATE TABLE t1 (a int); } + sql coord0a { SELECT create_distributed_table('t1', 'a'); } + sql coord0a { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_005: manual failover of worker group 1 +# + +step test_005_failover { + exec monitor pg_autoctl perform failover --group 1 + wait until worker1b state is wait_primary timeout 90s + wait until worker1a state is secondary + and worker1b state is primary + timeout 90s +} diff --git a/tests/tap/specs/config_get_set.pgaf b/tests/tap/specs/config_get_set.pgaf new file mode 100644 index 000000000..0374e05e9 --- /dev/null +++ b/tests/tap/specs/config_get_set.pgaf @@ -0,0 +1,48 @@ +# Test pg_autoctl config get / config set behaviour on both the monitor and a +# keeper node, including validation (rejecting invalid values) and no +# unintended side-effects on other settings. +# +# Ported from tests/test_config_get_set.py +# Predecessor: tests/test_config_get_set.py + +cluster { + monitor + formation { + node1 + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +step test_001_init_primary { + exec node1 pg_autoctl set node metadata --name "node a" + exec node1 pg_autoctl show state + exec node1 pg_autoctl config set pg_autoctl.name a + sleep 2s + exec node1 pg_autoctl show settings +} + +step test_002_config_set_monitor { + exec monitor pg_autoctl config set ssl.sslmode prefer + exec monitor pg_autoctl config get postgresql.pg_ctl + expect { /usr/lib/postgresql/17/bin/pg_ctl } + exec-fails monitor pg_autoctl config set postgresql.pg_ctl invalid + exec monitor pg_autoctl config get postgresql.pg_ctl + expect { /usr/lib/postgresql/17/bin/pg_ctl } + exec monitor pg_autoctl config get ssl.sslmode + expect { prefer } +} + +step test_002b_config_set_node { + exec node1 pg_autoctl config get postgresql.pg_ctl + expect { /usr/lib/postgresql/17/bin/pg_ctl } + exec-fails node1 pg_autoctl config set postgresql.pg_ctl invalid + exec node1 pg_autoctl config get postgresql.pg_ctl + expect { /usr/lib/postgresql/17/bin/pg_ctl } +} diff --git a/tests/tap/specs/create_standby_with_pgdata.pgaf b/tests/tap/specs/create_standby_with_pgdata.pgaf new file mode 100644 index 000000000..61699c624 --- /dev/null +++ b/tests/tap/specs/create_standby_with_pgdata.pgaf @@ -0,0 +1,80 @@ +# Test creating a standby from an existing PGDATA directory. +# +# test_003/004/005: verify that registering a node whose PGDATA was created by +# a separate initdb (different system_identifier) is rejected by the monitor. +# +# test_006: start node3 via pg_autoctl node start; pg_autoctl detects the +# pre-existing pg_basebackup PGDATA and sets up streaming replication without +# running another pg_basebackup (skipBaseBackup path in fsm_init_standby). +# +# Both node2 and node3 are declared launch deferred: their containers sleep +# until a step explicitly calls pg_autoctl node start (which rewrites the ini +# from mode=deferred to mode=immediate and unblocks the waiting node run loop). +# +# Ported from tests/test_create_standby_with_pgdata.py +# Predecessor: tests/test_create_standby_with_pgdata.py + +cluster { + monitor + formation { + node1 + node2 launch deferred + node3 launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s + exec node1 pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: init primary +# + +step test_001_init_primary { + wait until node1 state is single timeout 60s +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_003_init_secondary { + exec node2 pg_ctl initdb -D /var/lib/postgres/pgaf +} + +step test_004_create_raises_error { + exec-fails node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --ssl-self-signed --name node2 --hostname node2 +} + +step test_005_cleanup_after_failure { + exec node2 rm -rf /var/lib/postgres/pgaf +} + +step test_006_init_secondary { + exec node3 pg_autoctl node start /etc/pgaf/node.ini + wait until node3 state is secondary + and node1 state is primary + timeout 90s +} + +# +# test_007: manual failover to node3 +# + +step test_007_failover { + exec monitor pg_autoctl perform failover + wait until node3 state is primary + and node1 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/debian_clusters.pgaf b/tests/tap/specs/debian_clusters.pgaf new file mode 100644 index 000000000..950a7310c --- /dev/null +++ b/tests/tap/specs/debian_clusters.pgaf @@ -0,0 +1,51 @@ +# Test pg_auto_failover with Debian-style pg_createcluster layouts where the +# postgresql.conf lives outside PGDATA (in /etc/postgresql///). +# pg_autoctl must detect and support this split-config layout. +# +# Ported from tests/test_debian_clusters.py +# Predecessor: tests/test_debian_clusters.py + +cluster { + monitor + formation { + node1 debian-cluster main + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +# +# test_001: pg_autoctl adopts the Debian "main" cluster pre-created by +# pg_createcluster at image build time. postgresql.conf starts outside +# PGDATA (/etc/postgresql/17/main/); pg_autoctl moves it in on first run. +# + +step test_001_single_with_debian_cluster { + exec monitor pg_autoctl inspect pgsetup wait + exec node1 pg_autoctl inspect pgsetup wait +} + +# +# test_002: verify pg_autoctl moved postgresql.conf into PGDATA. +# + +step test_002_conf_in_pgdata { + exec node1 test -f /var/lib/postgresql/17/main/postgresql.conf +} + +# +# test_003: basic write/read to confirm Postgres is functional. +# + +step test_003_write_and_read { + sql node1 { CREATE TABLE t1 (a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3); } + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} diff --git a/tests/tap/specs/enable_ssl.pgaf b/tests/tap/specs/enable_ssl.pgaf new file mode 100644 index 000000000..d03d7440d --- /dev/null +++ b/tests/tap/specs/enable_ssl.pgaf @@ -0,0 +1,84 @@ +# Test enabling SSL on a running cluster: nodes start without SSL and SSL is +# enabled live with --ssl-self-signed. After enabling, replication must still +# work and reads from the secondary must return correct data. +# +# CA-signed certificate tests (verify-ca / verify-full) are covered separately +# in ssl_cert.pgaf, which starts the cluster with SSL already configured. +# +# Ported from tests/test_enable_ssl.py +# Predecessor: tests/test_enable_ssl.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 60s + promote node1 +} + +teardown { + compose down +} + +step test_001_init_primary { + sql monitor { SHOW ssl; } + expect { off } + sql node1 { SHOW ssl; } + expect { off } + sql node2 { SHOW ssl; } + expect { off } +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_maintenance { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s +} + +step test_005_enable_ssl_monitor { + exec monitor pg_autoctl enable ssl --ssl-self-signed --ssl-mode require + exec monitor pg_autoctl inspect pgsetup wait --timeout 60 + sql monitor { SHOW ssl; } + expect { on } +} + +step test_006_enable_ssl_primary { + exec node1 pg_autoctl enable ssl --ssl-self-signed --ssl-mode require + exec node1 pg_autoctl inspect pgsetup wait --timeout 60 + sql node1 { SHOW ssl; } + expect { on } +} + +step test_007_enable_ssl_secondary { + exec node2 pg_autoctl enable ssl --ssl-self-signed --ssl-mode require + compose stop node2 + compose start node2 + exec node2 pg_autoctl inspect pgsetup wait --timeout 90 + sql node2 { SHOW ssl; } + expect { on } +} + +step test_008_disable_maintenance { + exec node2 pg_autoctl disable maintenance + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +step test_009_read_from_secondary { + exec node2 pg_autoctl inspect pgsetup wait + sql node2 { SHOW ssl; } + expect { on } + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} diff --git a/tests/tap/specs/ensure.pgaf b/tests/tap/specs/ensure.pgaf new file mode 100644 index 000000000..9020fa7a9 --- /dev/null +++ b/tests/tap/specs/ensure.pgaf @@ -0,0 +1,69 @@ +# Test pg_autoctl "ensure" behaviour: verify that the keeper restarts +# Postgres after an unexpected stop, survives a demoted transition, and +# correctly orchestrates a failover when Postgres is broken on the primary. +# +# node2 is declared launch deferred so it starts in single mode for test_001, +# matching the Python test order (node2 is not created until test_003). +# +# Ported from tests/test_ensure.py +# Predecessor: tests/test_ensure.py + +cluster { + monitor + formation { + node1 + node2 launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s + exec node1 pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +step test_001_init_primary { + stop postgres node1 + wait until node1 state is single timeout 90s + exec node1 pg_autoctl inspect pgsetup wait +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_003_init_secondary { + exec node2 pg_autoctl node start + wait until node2 state is secondary + and node1 state is primary + timeout 90s + stop postgres node2 + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +step test_004_demoted { + compose stop node1 + sleep 30s + compose start node1 + wait until node1 state is demoted timeout 120s + wait until node2 state is primary + and node1 state is secondary + timeout 120s +} + +step test_005_inject_error_in_node2 { + wait until node2 state is primary timeout 60s + wait until node2 state is primary timeout 90s + exec node2 bash -c "echo \"shared_preload_libraries='wrong_extension'\" >> /var/lib/postgres/pgaf/postgresql.conf" + stop postgres node2 + wait until node1 state is wait_primary timeout 120s + wait until node2 state is secondary + and node1 state is primary + timeout 120s +} diff --git a/tests/tap/specs/extension_update.pgaf b/tests/tap/specs/extension_update.pgaf new file mode 100644 index 000000000..921cf3dca --- /dev/null +++ b/tests/tap/specs/extension_update.pgaf @@ -0,0 +1,37 @@ +# Test pg_auto_failover extension version management. +# The monitor starts with a "dummy" extension version baked into the Docker +# image. pg_autoctl detects the mismatch at startup, runs ALTER EXTENSION +# pgautofailover UPDATE, and restarts Postgres. The test verifies that the +# installed_version seen in pg_available_extensions matches the dummy version +# and that the monitor comes back healthy after the restart. +# +# Predecessor: tests/test_extension_update.py + +cluster { + monitor + extension-version "dummy" +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: start monitor with a dummy extension version and verify +# that the installed version matches the dummy version. +# + +step test_001_update_extension { + sleep 3s + exec monitor pg_autoctl inspect pgsetup wait + sql monitor { + SELECT installed_version + FROM pg_available_extensions + WHERE name = 'pgautofailover'; + } + expect { dummy } +} diff --git a/tests/tap/specs/installcheck.pgaf b/tests/tap/specs/installcheck.pgaf new file mode 100644 index 000000000..ce8b9005a --- /dev/null +++ b/tests/tap/specs/installcheck.pgaf @@ -0,0 +1,46 @@ +# Test the pgautofailover SQL extension via pg_regress installcheck. +# +# The monitor uses the "testrun" Dockerfile target which includes the full source +# tree and postgresql-server-dev (providing pg_regress). pg_regress connects +# to the running monitor Postgres from within the same container. +# +# Coverage mirrors tests/test_installcheck.py: +# - monitor created and running +# - pg_hba.conf widened so pg_regress can connect (host all all 0.0.0.0/0) +# - src/monitor writable for pg_regress results output +# - make installcheck runs all SQL regression tests in src/monitor/ +# +# Ported from tests/test_installcheck.py +# Predecessor: tests/test_installcheck.py + +cluster { + monitor + image "pgaf:testrun" +} + +setup { + exec monitor pg_autoctl inspect pgsetup wait +} + +teardown { + compose down +} + +# +# test_001: append a trust HBA entry so pg_regress can connect from localhost. +# + +step test_001_add_hba_entry { + exec monitor bash -c "echo 'host all all %CIDR% trust' >> /var/lib/postgres/pgaf/pg_hba.conf" + exec monitor pg_autoctl reload +} + +# +# test_002: make src/monitor writable for pg_regress results output, +# then run all regression SQL tests. +# + +step test_002_make_installcheck { + exec monitor sudo chmod -R go+w /usr/src/pg_auto_failover/src/monitor + exec monitor make -C /usr/src/pg_auto_failover/src/monitor installcheck PGHOST=127.0.0.1 +} diff --git a/tests/tap/specs/maintenance_and_drop.pgaf b/tests/tap/specs/maintenance_and_drop.pgaf new file mode 100644 index 000000000..b933b5035 --- /dev/null +++ b/tests/tap/specs/maintenance_and_drop.pgaf @@ -0,0 +1,75 @@ +# Test maintenance mode and clean node drop: enable maintenance with a +# concurrent node failure, verify the cluster recovers when maintenance is +# disabled, then simulate a primary failure, recover, and finally drop a +# node cleanly verifying the formation returns to single. +# +# Predecessor: tests/test_create_run.py + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until primary, secondary timeout 90s + promote node1 +} + +teardown { + compose down +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } +} + +step test_005_maintenance { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s + sql node1 { INSERT INTO t1 VALUES (3); } + exec node2 pg_autoctl disable maintenance + wait until node2 state is secondary + and node1 state is primary + timeout 90s +} + +step test_006_fail_primary { + compose stop node1 + wait until node2 state is wait_primary timeout 180s +} + +step test_007_start_node1_again { + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} + +step test_008_read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } { 3 } } +} + +step test_009_fail_secondary { + compose stop node1 + wait until node2 state is wait_primary timeout 90s +} + +step test_010_drop_secondary { + compose start node1 + wait until node2 state is primary + and node1 state is secondary + timeout 90s + exec node1 pg_autoctl drop node --no-wait + wait until node1 stopped timeout 30s + wait until node2 state is single timeout 60s +} diff --git a/tests/tap/specs/monitor_disabled.pgaf b/tests/tap/specs/monitor_disabled.pgaf new file mode 100644 index 000000000..4ebacca3b --- /dev/null +++ b/tests/tap/specs/monitor_disabled.pgaf @@ -0,0 +1,98 @@ +# Test operating pg_autoctl without a monitor ("monitor disabled" mode). +# The FSM is driven manually via pg_autoctl manual fsm assign / nodes set. +# +# No-monitor nodes start pg_autoctl node run automatically (like all nodes) +# but with no_monitor = true in the ini, so there is no monitor registration. +# Each test step drives the FSM manually. +# +# Ported from tests/test_monitor_disabled.py +# Predecessor: tests/test_monitor_disabled.py + +cluster { + ssl off + formation { + node1 no-monitor + node2 no-monitor + node3 no-monitor + } +} + +# No monitor, so no automatic state convergence — each step drives the FSM +# manually. + +setup { + # No monitor: nodes start pg_autoctl in INIT state (postgres not yet + # running). The first test step assigns SINGLE state to node1 which + # starts postgres. Just give pg_autoctl a moment to initialize the + # data directory before the test steps begin. + sleep 5s +} + +teardown { + compose down +} + +step test_002_init_to_single { + exec node1 pg_autoctl manual fsm assign single + exec node1 pg_autoctl inspect pgsetup wait +} + +step test_003_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_init_secondary { + sleep 5s +} + +step test_005_fsm_nodes_set { + exec node1 sh -c 'printf '"'"'[{"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true},{"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false}]'"'"' > /tmp/nodes12.json && pg_autoctl manual fsm nodes set /tmp/nodes12.json' + exec node2 sh -c 'printf '"'"'[{"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true},{"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false}]'"'"' > /tmp/nodes12.json && pg_autoctl manual fsm nodes set /tmp/nodes12.json' +} + +step test_006_init_to_wait_standby { + exec node2 pg_autoctl manual fsm assign wait_standby +} + +step test_007_catchingup { + exec node1 pg_autoctl manual fsm assign wait_primary + exec node2 pg_autoctl manual fsm assign catchingup +} + +step test_008_secondary { + exec node1 pg_autoctl manual fsm assign primary + exec node2 pg_autoctl manual fsm assign secondary + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} + +step test_009_init_secondary { + sleep 5s +} + +step test_010_fsm_nodes_set { + exec node1 sh -c 'printf '"'"'[{"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true},{"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false},{"node_id":3,"node_name":"node3","node_host":"node3","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false}]'"'"' > /tmp/nodes123.json && pg_autoctl manual fsm nodes set /tmp/nodes123.json' + exec node2 sh -c 'printf '"'"'[{"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true},{"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false},{"node_id":3,"node_name":"node3","node_host":"node3","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false}]'"'"' > /tmp/nodes123.json && pg_autoctl manual fsm nodes set /tmp/nodes123.json' + exec node3 sh -c 'printf '"'"'[{"node_id":1,"node_name":"node1","node_host":"node1","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":true},{"node_id":2,"node_name":"node2","node_host":"node2","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false},{"node_id":3,"node_name":"node3","node_host":"node3","node_port":5432,"node_cluster":"default","node_lsn":"0/1","node_is_primary":false}]'"'"' > /tmp/nodes123.json && pg_autoctl manual fsm nodes set /tmp/nodes123.json' +} + +step test_011_init_to_wait_standby { + exec node1 pg_autoctl manual fsm assign primary + exec node3 pg_autoctl manual fsm assign wait_standby + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} + +step test_012_catchingup { + exec node3 pg_autoctl manual fsm assign catchingup + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} + +step test_013_secondary { + exec node3 pg_autoctl manual fsm assign secondary + exec node1 pg_autoctl manual fsm assign primary + sql node1 { SHOW synchronous_standby_names; } + expect { * } +} diff --git a/tests/tap/specs/multi_alternate.pgaf b/tests/tap/specs/multi_alternate.pgaf new file mode 100644 index 000000000..5d1ba9d9c --- /dev/null +++ b/tests/tap/specs/multi_alternate.pgaf @@ -0,0 +1,267 @@ +# Test alternating failover scenarios with three nodes: each node serves as +# primary at least once across three series. Series 003 kills node1 then +# node2 in sequence; series 005 reverses the restart order; series 006 kills +# node3 (the surviving primary) and concurrently restarts a previously failed +# node. Verifies correct election outcomes and that pg_rewind restores each +# node as a healthy secondary after it rejoins. +# +# All kills use SIGKILL (compose kill) so the stopped process has no chance +# to update the monitor or claim a re-election win during shutdown. +# +# Predecessor: tests/test_multi_alternate_primary_failures.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +# +# test_002: verify replication slots after initial 3-node convergence. +# Node ids are assigned in registration order: node1=1, node2=2, node3=3. +# Each primary holds physical replication slots for each standby. +# After setup node1 is primary; it holds slots for standbys 2 and 3. +# node2 and node3 are standbys; each holds a slot for the other two. +# + +step test_002_check_initial_slots { + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } +} + +# +# Series 003: stop node1 (primary) → node2 becomes primary, +# then stop node2 → node3 reports_lsn and waits, +# restart node2 → node2 becomes primary again, +# restart node1 → node1 becomes secondary. +# +# node1 node2 node3 +# primary secondary secondary +# +# demoted primary secondary +# +# demoted draining report_lsn +# +# demoted primary secondary +# +# secondary primary secondary +# + +step test_003_001_stop_primary { + compose kill node1 + wait until node2 state is primary + and node3 state is secondary + timeout 120s +} + +step test_003_002_stop_primary { + compose kill node2 + wait until node2 assigned-state = draining timeout 120s + wait until node3 state is report_lsn timeout 120s +} + +step test_003_003_bringup_last_failed_primary { + compose start node2 + wait until node2 state is primary + and node3 state is secondary + timeout 180s +} + +step test_003_004_bringup_first_failed_primary { + compose start node1 + wait until node1 assigned-state = secondary timeout 90s + wait until node2 assigned-state = primary timeout 90s + wait until node3 assigned-state = secondary timeout 90s +} + +# After series 003: node2 is primary (id=2), node1 and node3 are standbys. + +step test_003_005_check_slots_after_series_003 { + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } +} + +step test_005_001_fail_primary_again { + compose kill node2 + wait until node1 state is primary + and node3 state is secondary + timeout 180s +} + +step test_005_001b_write_and_checkpoint { + wait until node1 state is primary + and node3 state is secondary + timeout 120s + sql node1 { + CREATE TABLE IF NOT EXISTS _lsn_advance(x int); INSERT INTO + _lsn_advance VALUES(1); CHECKPOINT; + } +} + +step test_005_002_fail_primary_again { + compose kill node1 + wait until node3 assigned-state = report_lsn timeout 120s +} + +step test_005_003_bring_up_first_failed_primary { + compose start node2 + wait until node2 state is demoted timeout 120s + wait until node2 state is secondary + and node3 state is primary + timeout 120s +} + +step test_005_004_bring_up_last_failed_primary { + compose start node1 + wait until node1 state is secondary + and node3 state is primary + timeout 120s +} + +# +# Series 006: stop node3 (primary) → node1 becomes primary, +# stop node1 and concurrently restart node3, +# node2 becomes primary, node3 becomes secondary, +# restart node1 → node1 becomes secondary. +# +# node1 node2 node3 +# secondary secondary primary +# +# primary secondary demoted +# +# +# demoted primary secondary +# +# secondary primary secondary +# + +# After series 005: node3 is primary (id=3), node1 and node2 are standbys. + +step test_005_005_check_slots_after_series_005 { + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } +} + +step test_006_001_fail_primary { + compose kill node3 + wait until node1 state is primary + and node2 state is secondary + timeout 120s +} + +step test_006_002_fail_new_primary { + compose kill node1 + compose start node3 + wait until node2 state is primary + and node3 state is secondary + timeout 120s +} + +step test_006_003_bringup_last_failed_primary { + compose start node1 + wait until node1 state is secondary + and node2 state is primary + and node3 state is secondary + timeout 120s +} + +# After series 006: node2 is primary (id=2), node1 and node3 are standbys. + +step test_006_004_check_slots_after_series_006 { + sql node2 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_3 } } + sql node1 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_2 } { pgautofailover_standby_3 } } + sql node3 { + SELECT slot_name + FROM pg_replication_slots + WHERE slot_name ~ '^pgautofailover_standby_' AND slot_type = 'physical' + ORDER BY slot_name; + } + expect { { pgautofailover_standby_1 } { pgautofailover_standby_2 } } +} diff --git a/tests/tap/specs/multi_async.pgaf b/tests/tap/specs/multi_async.pgaf new file mode 100644 index 000000000..d705d6fcb --- /dev/null +++ b/tests/tap/specs/multi_async.pgaf @@ -0,0 +1,251 @@ +# Test mixed sync/async replication configurations with four nodes. +# Starts with one async node (node3), then exercises: setting the whole +# formation async, async failover via perform-promotion, returning to mixed +# sync/async, dropping a node, and two double-failure scenarios (tests 014 +# and 015) where both the primary and the new primary fail before recovery. +# Also covers: candidate-priority=0 secondary rejoining while primary is +# wait_primary, and async standby failing while primary is already wait_primary. +# +# Predecessor: tests/test_multi_async.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 replication-quorum false + node4 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary + and node4 state is secondary + timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_004: set the whole formation to async +# + +step test_004_set_async { + exec node1 pg_autoctl set formation number-sync-standbys 0 + exec node1 pg_autoctl set node replication-quorum false + exec node2 pg_autoctl set node replication-quorum false + exec node3 pg_autoctl set node replication-quorum false + wait until node1 state is primary timeout 60s +} + +# +# test_005: write into primary +# + +step test_005_write_into_primary { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3), (4); } + sql node1 { CHECKPOINT; } +} + +# +# test_006: async failover — promote node2 +# + +step test_006_async_failover { + exec monitor pg_autoctl perform promotion --name node2 + wait until node1 state is secondary + and node3 state is secondary + and node2 state is primary + timeout 90s +} + +# +# test_007: read from new primary +# + +step test_007_read_from_new_primary { + sql node2 { SELECT * FROM t1; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_008: set mixed sync/async configuration +# + +step test_008_set_sync_async { + exec node1 pg_autoctl set node replication-quorum true + exec node2 pg_autoctl set node replication-quorum true + exec node3 pg_autoctl set node replication-quorum false + exec node3 pg_autoctl set node candidate-priority 0 + wait until node2 state is primary timeout 60s +} + +step test_009_add_sync_standby { + exec node2 pg_autoctl set formation number-sync-standbys 1 + wait until node1 state is secondary + and node2 state is primary + and node3 state is secondary + and node4 state is secondary + timeout 60s +} + +# +# test_010: promote node1 via monitor SQL +# + +step test_010_promote_node1 { + sql monitor { + SELECT pgautofailover.perform_promotion('default', 'node1'); + } +} + +# +# test_011: ifdown node4 while it is at report_lsn +# + +step test_011_ifdown_node4_at_reportlsn { + wait until node4 state is report_lsn timeout 120s + network disconnect node4 + wait until node3 state is secondary timeout 120s +} + +# +# test_012: ifup node4 +# + +step test_012_ifup_node4 { + network connect node4 + wait until node3 state is secondary + and node4 state is secondary + and node1 state is primary + and node2 state is secondary + timeout 90s +} + +# +# test_013: drop node4 +# + +step test_013_drop_node4 { + exec monitor pg_autoctl drop node --name node4 + compose stop node4 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 90s +} + +step test_014_001_fail_node1 { + compose stop node1 + wait until node2 state is stop_replication timeout 120s + wait until node2 state is wait_primary + and node3 state is secondary + timeout 120s + sql node2 { SHOW synchronous_standby_names; } + expect { } +} + +step test_014_001b_write_and_checkpoint { + sql node2 { INSERT INTO t1 VALUES (100); } + sql node2 { CHECKPOINT; } + sleep 3s +} + +step test_014_002_stop_new_primary_node2 { + compose stop node2 + wait until node3 state is report_lsn timeout 90s +} + +step test_014_003_restart_node1 { + compose start node1 + wait until node1 state is wait_primary timeout 180s +} + +step test_014_004_restart_node2 { + compose start node2 + wait until node2 state is secondary + and node3 state is secondary + and node1 state is primary + timeout 300s +} + +step test_015_001_fail_primary_node1 { + compose stop node1 + wait until node2 state is wait_primary timeout 120s +} + +step test_015_001b_write_and_checkpoint { + sql node2 { INSERT INTO t1 VALUES (101); } + sql node2 { CHECKPOINT; } + sleep 3s +} + +step test_015_002_fail_new_primary_node2 { + compose stop node2 + wait until node3 state is report_lsn timeout 90s +} + +step test_015_003_restart_node2 { + compose start node2 + wait until node2 state is wait_primary + and node3 state is secondary + timeout 120s +} + +step test_015_004_restart_node1 { + compose start node1 + wait until node3 state is secondary + and node2 state is primary + and node1 state is secondary + timeout 120s +} + +step test_016_001_fail_node3 { + network disconnect node3 + wait until node3 assigned-state = catchingup timeout 90s +} + +step test_016_002_fail_node1 { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} + +step test_016_003_restart_node3 { + network connect node3 + wait until node3 assigned-state = secondary timeout 90s + wait until node2 state is wait_primary timeout 60s + sleep 5s +} + +step test_016_004_restart_node1 { + network connect node1 + wait until node3 state is secondary + and node2 state is primary + and node1 state is secondary + timeout 90s +} + +step test_017_001_fail_node1 { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s +} + +step test_017_002_fail_node3 { + network disconnect node3 + wait until node3 assigned-state = catchingup timeout 90s +} + +step test_017_003_restart_nodes { + network connect node3 + network connect node1 + wait until node3 state is secondary + and node2 state is primary + and node1 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/multi_ifdown.pgaf b/tests/tap/specs/multi_ifdown.pgaf new file mode 100644 index 000000000..a3f1d0417 --- /dev/null +++ b/tests/tap/specs/multi_ifdown.pgaf @@ -0,0 +1,178 @@ +# Test network-partition (ifdown/ifup) scenarios with three nodes and +# explicit candidate-priority settings. +# Covers: basic ifdown of a standby while writes continue on the primary, +# failover when the primary is disconnected and a lagging candidate reconnects, +# and an advanced split-brain scenario where both the primary and the +# most-advanced secondary are cleanly stopped before a behind async node is +# promoted — requiring pg_rewind to fetch missing WAL from the survivors. +# +# Note: the advanced test uses compose stop (clean shutdown) rather than +# network disconnect for the primary/advanced secondary so Postgres does not +# recycle WAL segments, which would corrupt pg_rewind's prev-links. +# +# Predecessor: tests/test_multi_ifdown.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_004: write into primary +# + +step test_004_write_into_primary { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3), (4); } + sql node1 { CHECKPOINT; } + sql node1 { SELECT count(*) FROM t1; } + expect { 4 } +} + +# +# test_005: set candidate priorities +# node1 priority=90 (primary) +# node2 priority=0 (not a candidate) +# node3 priority=90 +# + +step test_005_set_candidate_priorities { + exec node1 pg_autoctl set node candidate-priority 90 + exec node2 pg_autoctl set node candidate-priority 0 + exec node3 pg_autoctl set node candidate-priority 90 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3, pgautofailover_standby_2) } +} + +# +# test_006: take node3 offline (ifdown) +# + +step test_006_ifdown_node3 { + network disconnect node3 +} + +# +# test_007: insert rows while node3 is down; node2 is sync and gets WAL +# + +step test_007_insert_rows { + sql node1 { + INSERT INTO t1 SELECT x+10 FROM generate_series(1, 10000) as gs(x); + } + sql node1 { CHECKPOINT; } +} + +# +# test_008: fail node1 (primary), reconnect node3, verify node3 becomes primary +# + +step test_008_failover { + network disconnect node1 + network connect node3 + wait until node3 state is wait_primary timeout 120s + wait until node2 state is secondary + and node3 state is primary + timeout 120s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_009: read from new primary +# + +step test_009_read_from_new_primary { + sql node3 { SELECT count(*) FROM t1; } + expect { 10004 } +} + +# +# test_010: start node1 again as secondary +# + +step test_010_start_node1_again { + network connect node1 + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_011–012: prepare for advanced scenario +# Promote node2, make node2 async so it can fall behind +# + +step test_011_prepare_candidate_priorities { + exec node2 pg_autoctl set node candidate-priority 100 +} + +step test_012_prepare_replication_quorums { + exec node3 pg_autoctl set formation number-sync-standbys 0 + exec node2 pg_autoctl set node replication-quorum false +} + +step test_013_secondary_gets_behind_primary { + network disconnect node2 + sql node3 { INSERT INTO t1 VALUES (5), (6); } + sql node3 { CHECKPOINT; } + sql node1 { SELECT count(*) FROM t1; } + expect { 10006 } + sleep 2s +} + +# +# test_014: trigger failover with node3 (primary) and node1 (most advanced) +# both offline; only node2 (behind, async candidate) is online +# + +step test_014_secondary_reports_lsn { + wait until node1 state is secondary + and node3 state is primary + timeout 60s + compose stop node3 + compose stop node1 + network connect node2 + exec monitor bash -c "pg_autoctl perform failover || true" + wait until node2 state is report_lsn timeout 90s +} + +step test_015_start_node3_node1 { + compose start node3 + compose start node1 +} + +# +# test_015: bring back node1 and node3 so node2 can fetch missing WAL +# + +step test_015_finalize_failover { + wait until node1 state is secondary + and node3 state is secondary + and node2 state is primary + timeout 300s + sql node2 { SELECT count(*) FROM t1; } + expect { 10006 } +} diff --git a/tests/tap/specs/multi_maintenance.pgaf b/tests/tap/specs/multi_maintenance.pgaf new file mode 100644 index 000000000..85a19ad69 --- /dev/null +++ b/tests/tap/specs/multi_maintenance.pgaf @@ -0,0 +1,278 @@ +# Test the maintenance mode lifecycle with four nodes (three sync + one async). +# Covers: putting a standby in maintenance while triggering failover, +# both standbys in maintenance while primary keeps running, attempting to +# stop the primary while no standby is available (must fail), enabling +# maintenance on the primary with --allow-failover, and verifying the monitor +# enforces the invariant that at least one non-maintenance standby always +# remains when number-sync-standbys > 0. +# +# node4 starts async / candidate-priority 0; test_013 promotes it to +# full quorum participant to simulate dynamic node addition. +# +# Predecessor: tests/test_multi_maintenance.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + node4 candidate-priority 0 replication-quorum false + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node3 state is secondary timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_004: write into primary +# + +step test_004_write_into_primary { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2), (3), (4); } + sql node1 { CHECKPOINT; } + sql node1 { SELECT count(*) FROM t1; } + expect { 4 } +} + +step test_005_set_candidate_priorities { + exec node1 pg_autoctl set node candidate-priority 80 + exec node2 pg_autoctl set node candidate-priority 70 + exec node3 pg_autoctl set node candidate-priority 90 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 60s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3, pgautofailover_standby_2) } +} + +# +# test_006a: put node2 in maintenance, failover to node3, then disable maintenance +# + +step test_006a_maintenance_and_failover { + exec node2 pg_autoctl enable maintenance + wait until node2 state is maintenance timeout 60s + stop postgres node2 + wait until node1 state is primary timeout 60s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_3, pgautofailover_standby_2) } + exec monitor pg_autoctl perform failover + wait until node3 state is primary + and node1 state is secondary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } + exec node2 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_006b: read from new primary +# + +step test_006b_read_from_new_primary { + sql node3 { SELECT * FROM t1; } + expect { { 1 } { 2 } { 3 } { 4 } } +} + +# +# test_007a: put node1 in maintenance +# + +step test_007a_node1_to_maintenance { + wait until node3 state is primary timeout 60s + exec node1 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s +} + +# +# test_007b: put node2 in maintenance; both standbys now in maintenance +# + +step test_007b_node2_to_maintenance { + exec node2 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s + sql node3 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_2) } +} + +# +# test_008a: stop the primary (node3) while standbys are in maintenance +# node3 must NOT be assigned draining (both secondaries are in maintenance) +# + +step test_008a_stop_primary { + assert node3 stays primary while { + network disconnect node3 + sleep 30s + } + wait until node3 assigned-state = primary timeout 90s +} + +# +# test_008b: restart primary node3 +# + +step test_008b_start_primary { + network connect node3 + wait until node3 state is primary timeout 90s +} + +step test_009a_enable_maintenance_on_primary_should_fail { + exec-fails node3 pg_autoctl enable maintenance --allow-failover +} + +# +# test_009b: disable maintenance on node1 and node2 +# + +step test_009b_disable_maintenance { + exec node1 pg_autoctl disable maintenance + exec node2 pg_autoctl disable maintenance + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + timeout 90s +} + +step test_010_set_number_sync_standby_to_zero { + exec node3 pg_autoctl set formation number-sync-standbys 0 +} + +# +# test_011: put both standbys in maintenance; primary becomes wait_primary +# + +step test_011_all_to_maintenance { + wait until node3 state is primary timeout 60s + exec node1 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s + exec node2 pg_autoctl enable maintenance + wait until node3 state is wait_primary timeout 90s + sql node3 { SHOW synchronous_standby_names; } + expect { } +} + +# +# test_012: can write while in wait_primary (number-sync-standbys=0) +# + +step test_012_can_write_during_maintenance { + sql node3 { INSERT INTO t1 VALUES (5), (6); } + sql node3 { CHECKPOINT; } +} + +step test_013_add_standby { + exec node4 pg_autoctl set node replication-quorum true + exec node4 pg_autoctl set node candidate-priority 50 + wait until node4 state is secondary + and node3 state is primary + and node2 state is maintenance + and node1 state is maintenance + timeout 60s +} + +# +# test_014: disable maintenance on node2 and node1 +# + +step test_014_disable_maintenance { + wait until node2 state is maintenance timeout 60s + exec node2 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s + wait until node1 state is maintenance timeout 60s + exec node1 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s +} + +step test_015_set_number_sync_standby_to_one { + exec node3 pg_autoctl set formation number-sync-standbys 1 + wait until node3 state is primary timeout 60s +} + +# +# test_016: put two standbys in maintenance (number-sync-standbys=1, primary stays primary) +# + +step test_016_two_standbys_in_maintenance { + exec node1 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s + exec node2 pg_autoctl enable maintenance + wait until node3 state is primary timeout 60s +} + +step test_017_primary_to_maintenance { + exec-fails node3 pg_autoctl enable maintenance +} + +# +# test_018: disable maintenance on node2 and node1 +# + +step test_018_disable_maintenance { + wait until node2 state is maintenance timeout 60s + exec node2 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s + wait until node1 state is maintenance timeout 60s + exec node1 pg_autoctl disable maintenance + wait until node3 state is primary timeout 90s +} + +# +# test_019: set priorities so node1 is the preferred candidate +# + +step test_019_set_priorities { + exec node1 pg_autoctl set node candidate-priority 90 + exec node2 pg_autoctl set node candidate-priority 70 + exec node3 pg_autoctl set node candidate-priority 70 + exec node4 pg_autoctl set node candidate-priority 70 + wait until node1 state is secondary + and node2 state is secondary + and node3 state is primary + and node4 state is secondary + timeout 60s +} + +# +# test_020: put primary (node3) in maintenance with failover allowed +# + +step test_020_primary_to_maintenance { + wait until node3 state is primary timeout 60s + exec node3 pg_autoctl enable maintenance --allow-failover + wait until node3 state is maintenance + and node2 state is secondary + and node4 state is secondary + and node1 state is primary + timeout 90s +} + +# +# test_021: disable maintenance on node3, it rejoins as secondary +# + +step test_021_stop_maintenance { + exec node3 pg_autoctl disable maintenance + wait until node3 state is secondary + and node1 state is primary + and node2 state is secondary + and node4 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/multi_standbys.pgaf b/tests/tap/specs/multi_standbys.pgaf new file mode 100644 index 000000000..7f55aa14d --- /dev/null +++ b/tests/tap/specs/multi_standbys.pgaf @@ -0,0 +1,303 @@ +# Test multi-standby formation features with four nodes. +# Covers: candidate-priority and replication-quorum APIs (get/set/validate), +# number_sync_standbys get/set and the auto-decrement trigger when a node is +# dropped, failover and read-from-all-nodes verification, network-partition +# failover, unblocking writes by setting number-sync-standbys=0 while two +# standbys are down, and the edge case where all candidate priorities are zero +# (no promotion possible until one priority is raised to ≥1). +# +# Predecessor: tests/test_multi_standbys.py + +cluster { + monitor + ssl off + formation { + node1 + node2 + node3 + node4 + } +} + +setup { + wait until primary, secondary timeout 120s + wait until node2 state is secondary + and node3 state is secondary + and node4 state is secondary + timeout 60s + promote node1 +} + +teardown { + compose down +} + +# +# test_002: candidate priority API +# + +step test_002_candidate_priority { + exec node1 pg_autoctl get node candidate-priority + expect { 50 } + exec-fails node1 pg_autoctl set node candidate-priority -1 + exec node1 pg_autoctl get node candidate-priority + expect { 50 } + exec node1 pg_autoctl set node candidate-priority 99 + exec node1 pg_autoctl get node candidate-priority + expect { 99 } +} + +# +# test_003: replication quorum API +# + +step test_003_replication_quorum { + exec node1 pg_autoctl get node replication-quorum + expect { true } + exec-fails node1 pg_autoctl set node replication-quorum "wrong quorum" + exec node1 pg_autoctl get node replication-quorum + expect { true } + exec node1 pg_autoctl set node replication-quorum false + exec node1 pg_autoctl get node replication-quorum + expect { false } + exec node1 pg_autoctl set node replication-quorum true + exec node1 pg_autoctl get node replication-quorum + expect { true } +} + +step test_004_001_add_three_standbys { + wait until node2 state is secondary + and node1 state is primary + timeout 60s +} + +step test_004_002_add_three_standbys { + wait until node3 state is secondary + and node1 state is primary + timeout 60s +} + +step test_004_003_add_three_standbys { + wait until node4 state is secondary + and node1 state is primary + timeout 60s +} + +# +# test_005: number_sync_standbys +# + +step test_005_number_sync_standbys { + exec node1 pg_autoctl set formation number-sync-standbys 2 + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 2 (pgautofailover_standby_2, pgautofailover_standby_3, pgautofailover_standby_4) } + exec node1 pg_autoctl set formation number-sync-standbys 0 + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3, pgautofailover_standby_4) } + exec node1 pg_autoctl set formation number-sync-standbys 1 + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3, pgautofailover_standby_4) } +} + +step test_006_number_sync_standbys_trigger { + exec node1 pg_autoctl set formation number-sync-standbys 2 + exec monitor pg_autoctl drop node --name node4 + compose stop node4 + wait until node1 state is primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } + sleep 6s +} + +# +# test_007: create table t1 +# + +step test_007_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } + sql node1 { CHECKPOINT; } +} + +step test_008_set_candidate_priorities { + exec node1 pg_autoctl set node candidate-priority 90 + exec node2 pg_autoctl set node candidate-priority 90 + exec node3 pg_autoctl set node candidate-priority 70 + wait until node1 state is primary timeout 60s +} + +# +# test_009: failover — node2 becomes primary +# + +step test_009_failover { + exec monitor pg_autoctl perform failover + wait until node2 state is primary + and node3 state is secondary + and node1 state is secondary + timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1, pgautofailover_standby_3) } +} + +# +# test_010: read from all nodes +# + +step test_010_read_from_nodes { + sql node1 { SELECT * FROM t1; } + expect { { 1 } { 2 } } + sql node2 { SELECT * FROM t1; } + expect { { 1 } { 2 } } + sql node3 { SELECT * FROM t1; } + expect { { 1 } { 2 } } +} + +# +# test_011: write into new primary +# + +step test_011_write_into_new_primary { + sql node2 { INSERT INTO t1 VALUES (3), (4); } + sql node2 { CHECKPOINT; } +} + +# +# test_012: fail primary node2 +# + +step test_012_fail_primary { + network disconnect node2 + wait until node1 state is stop_replication timeout 90s + wait until node1 state is primary + and node3 state is secondary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +# +# test_013: restart node2 as secondary +# + +step test_013_restart_node2 { + network connect node2 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_014_001_fail_set_properties { + exec node1 pg_autoctl set node candidate-priority 50 + exec node2 pg_autoctl set node candidate-priority 50 + exec node3 pg_autoctl set node candidate-priority 50 + wait until node1 state is primary timeout 60s +} + +step test_014_002_fail_two_standby_nodes { + compose stop node2 + compose stop node3 + wait until node2 assigned-state = catchingup timeout 180s + wait until node3 assigned-state = catchingup timeout 180s + # node1 remains primary (blocking writes at postgres level) until + # number-sync-standbys is explicitly set to 0 in the next step. + wait until node1 state is primary timeout 180s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_014_003_unblock_writes { + exec node1 pg_autoctl set formation number-sync-standbys 0 + wait until node1 state is wait_primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { } +} + +step test_014_004_restart_nodes { + compose start node3 + compose start node2 + wait until node3 state is secondary + and node2 state is secondary + and node1 state is primary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_015_002_fail_two_standby_nodes { + network disconnect node2 + network disconnect node3 + wait until node1 state is wait_primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { } +} + +step test_015_003_set_properties { + sql monitor { + SELECT pgautofailover.set_formation_number_sync_standbys('default', 1); + } + wait until node1 assigned-state = primary timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_015_004_restart_nodes { + network connect node3 + network connect node2 + wait until node3 state is secondary + and node2 state is secondary + and node1 state is primary + timeout 90s + sql node1 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_2, pgautofailover_standby_3) } +} + +step test_016_001_set_candidate_priorities_to_zero { + exec node1 pg_autoctl set node candidate-priority 0 + exec node2 pg_autoctl set node candidate-priority 0 + exec node3 pg_autoctl set node candidate-priority 0 + wait until node1 state is primary timeout 60s +} + +step test_016_002_trigger_failover { + exec monitor bash -c "pg_autoctl perform failover --wait 1 || true" + wait until node3 state is report_lsn + and node2 state is report_lsn + and node1 state is report_lsn + timeout 90s +} + +step test_016_003_set_candidate_priority_to_one { + exec node2 pg_autoctl set node candidate-priority 1 + wait until node2 state is primary + and node1 state is secondary + and node3 state is secondary + timeout 90s +} + +step test_016_004_reset_candidate_priority { + exec node2 pg_autoctl set node candidate-priority 0 + wait until node2 state is primary + and node1 state is secondary + and node3 state is secondary + timeout 60s +} + +step test_016_005_perform_promotion { + exec monitor pg_autoctl perform promotion --name node1 + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 90s +} + +step test_017_remove_old_primary { + exec monitor pg_autoctl drop node --name node2 + compose stop node2 + wait until node1 state is primary + and node3 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/nonha_citus_operation.pgaf b/tests/tap/specs/nonha_citus_operation.pgaf new file mode 100644 index 000000000..e607f025e --- /dev/null +++ b/tests/tap/specs/nonha_citus_operation.pgaf @@ -0,0 +1,104 @@ +# Test non-HA Citus formation lifecycle: start with secondary=false, add +# workers, enable secondary support, add standbys, attempt to disable while +# standbys exist (expect failure), fail-over primaries, drop them, and finally +# disable secondary support once only single nodes remain. +# +# Ported from tests/test_nonha_citus_operation.py +# Predecessor: tests/test_nonha_citus_operation.py + +cluster { + monitor + formation non-ha { + coordinator1a coordinator + coordinator1b coordinator launch deferred + worker1a worker group 1 + worker1b worker group 1 launch deferred + worker2a worker group 2 + worker2b worker group 2 launch deferred + } +} + +setup { + wait until coordinator1a state is single timeout 120s + wait until worker1a state is single timeout 120s + wait until worker2a state is single timeout 120s + sleep 5s +} + +teardown { + compose down +} + +step test_001_disable_secondary_on_formation { + exec monitor pg_autoctl disable secondary --formation non-ha +} + +step test_002_wait_metadata_sync { + exec coordinator1a sh -c 'for i in $(seq 1 30); do n=$(psql -U docker -d demo -tAc "SELECT count(*) FROM pg_dist_node WHERE metadatasynced = false AND isactive = true"); [ "$n" = "0" ] && exit 0; sleep 2; done; exit 1' +} + +step test_003_create_distributed_table { + sql coordinator1a { CREATE TABLE t1 (a int); } + sql coordinator1a { SELECT create_distributed_table('t1', 'a'); } + sql coordinator1a { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004_enable_secondary { + exec monitor pg_autoctl enable secondary --formation non-ha + exec coordinator1b pg_autoctl node start /etc/pgaf/node.ini + exec worker1b pg_autoctl node start /etc/pgaf/node.ini + exec worker2b pg_autoctl node start /etc/pgaf/node.ini +} + +step test_005_add_secondaries { + wait until coordinator1b state is secondary timeout 120s + wait until worker1b state is secondary timeout 120s + wait until worker2b state is secondary timeout 120s +} + +step test_006_fail_when_disabling_with_secondaries { + exec-fails monitor pg_autoctl disable secondary --formation non-ha +} + +step test_007_failover_coordinator { + network disconnect coordinator1a + wait until coordinator1b state is wait_primary timeout 90s + network connect coordinator1a + wait until coordinator1b state is primary + and coordinator1a state is secondary + timeout 180s +} + +step test_007b_failover_worker1 { + network disconnect worker1a + wait until worker1b state is wait_primary timeout 90s + network connect worker1a + wait until worker1b state is primary + and worker1a state is secondary + timeout 180s +} + +step test_007c_failover_worker2 { + network disconnect worker2a + wait until worker2b state is wait_primary timeout 90s + network connect worker2a + wait until worker2b state is primary + and worker2a state is secondary + timeout 180s +} + +step test_008_remove_old_primaries { + exec monitor pg_autoctl drop node --name coordinator1a --formation non-ha + exec monitor pg_autoctl drop node --name worker1a --formation non-ha + exec monitor pg_autoctl drop node --name worker2a --formation non-ha + compose stop coordinator1a + compose stop worker1a + compose stop worker2a + wait until coordinator1b state is single timeout 60s + wait until worker1b state is single timeout 60s + wait until worker2b state is single timeout 60s +} + +step test_009_disable_secondaries { + exec monitor pg_autoctl disable secondary --formation non-ha +} diff --git a/tests/tap/specs/replace_monitor.pgaf b/tests/tap/specs/replace_monitor.pgaf new file mode 100644 index 000000000..94cef2dd2 --- /dev/null +++ b/tests/tap/specs/replace_monitor.pgaf @@ -0,0 +1,95 @@ +# Test replacing a failed monitor with a new one. +# Sequence: bring up a cluster, drop the old monitor, disable it on both +# nodes, create a new monitor, re-enable it on both nodes, verify the cluster +# re-converges and a failover through the new monitor works. + +cluster { + monitor + monitor newmonitor initially stopped + ssl off + formation { + node1 + node2 + } +} + +setup { + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s +} + +teardown { + compose down +} + +step create_table { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step read_from_secondary { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { 1 } + expect { 2 } +} + +step drop_old_monitor { + exec monitor pg_autoctl stop +} + +step disable_monitor_on_nodes { + exec node2 pg_autoctl disable monitor --force + exec node1 pg_autoctl disable monitor --force +} + +step write_while_monitor_down { + sql node1 { INSERT INTO t1 VALUES (3); } +} + +step read_replicated_write { + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { 1 } + expect { 2 } + expect { 3 } +} + +step create_new_monitor { + compose start newmonitor + exec newmonitor pg_autoctl inspect pgsetup wait + set monitor newmonitor +} + +step enable_monitor_node1 { + exec node1 pg_autoctl enable monitor postgresql://autoctl_node@newmonitor/pg_auto_failover + wait until node1 state is single timeout 90s +} + +step enable_monitor_node2 { + exec node2 pg_autoctl enable monitor postgresql://autoctl_node@newmonitor/pg_auto_failover + wait until node2 state is catchingup timeout 90s + wait until node1 state is wait_primary timeout 90s +} + +step wait_for_convergence { + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s +} + +step failover_via_new_monitor { + exec newmonitor pg_autoctl perform failover + wait until node2 state is primary timeout 90s + wait until node1 state is secondary timeout 90s + wait until node2 state is primary timeout 90s + wait until node1 state is secondary timeout 90s + sql node2 { SHOW synchronous_standby_names; } + expect { ANY 1 (pgautofailover_standby_1) } +} + +step read_from_new_secondary { + sql node1 { SELECT * FROM t1 ORDER BY a; } + expect { 1 } + expect { 2 } + expect { 3 } +} diff --git a/tests/tap/specs/skip_pg_hba.pgaf b/tests/tap/specs/skip_pg_hba.pgaf new file mode 100644 index 000000000..ef14c7f2b --- /dev/null +++ b/tests/tap/specs/skip_pg_hba.pgaf @@ -0,0 +1,58 @@ +# Test --skip-pg-hba (auth-method=skip): pg_autoctl must NOT edit pg_hba.conf +# on either node. HBA rules are managed manually by the operator. +# +# Ported from tests/test_skip_pg_hba.py +# Predecessor: tests/test_skip_pg_hba.py + +cluster { + monitor + formation { + node1 auth skip + node2 auth skip + } +} + +setup { + # With auth=skip, nodes don't add HBA rules automatically. + # Add them now so the monitor can health-check the data nodes. + exec node1 pg_autoctl inspect pgsetup hba-lan + exec node2 pg_autoctl inspect pgsetup hba-lan + wait until primary, secondary timeout 120s + promote node1 +} + +teardown { + compose down +} + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +step test_004a_hba_have_not_been_edited { + exec node1 pg_autoctl config get postgresql.hba_level + expect { skip } + exec node2 pg_autoctl config get postgresql.hba_level + expect { skip } +} + +step test_005_failover { + exec monitor pg_autoctl perform failover + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} + +step test_006_restart_secondary { + compose stop node1 + compose start node1 + wait until node1 state is secondary timeout 60s +} + +step test_006a_hba_have_not_been_edited { + exec node1 pg_autoctl config get postgresql.hba_level + expect { skip } + exec node2 pg_autoctl config get postgresql.hba_level + expect { skip } +} diff --git a/tests/tap/specs/ssl_cert.pgaf b/tests/tap/specs/ssl_cert.pgaf new file mode 100644 index 000000000..aab711be7 --- /dev/null +++ b/tests/tap/specs/ssl_cert.pgaf @@ -0,0 +1,85 @@ +# Test pg_auto_failover with SSL using CA-signed certificates and cert auth. +# +# Monitor and all data nodes are set up with server certificates signed by a +# shared root CA, and client certificates are placed in ~/.postgresql/. +# Authentication uses the "cert" method: pg_autoctl writes hostssl + cert +# rules to pg_hba.conf and the ident map to pg_ident.conf automatically. +# +# Ported from tests/test_ssl_cert.py +# Predecessor: tests/test_ssl_cert.py + +cluster { + monitor + ssl verify-ca + auth cert + formation { + node1 + node2 + } +} + +setup { + wait until node1 state is primary + and node2 state is secondary + timeout 120s + promote node1 +} + +teardown { + compose down +} + +# +# test_000: verify monitor has SSL enabled with cert auth +# + +step test_000_create_monitor { + exec monitor pg_autoctl inspect pgsetup wait + sql monitor { SHOW ssl; } + expect { on } + exec monitor pg_autoctl config get ssl.sslmode + expect { verify-ca } +} + +# +# test_001: verify primary has SSL enabled with cert auth +# + +step test_001_init_primary { + wait until node1 state is primary timeout 60s + sql node1 { SHOW ssl; } + expect { on } + exec node1 pg_autoctl config get ssl.sslmode + expect { verify-ca } +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_003: verify secondary has SSL enabled with cert auth +# + +step test_003_init_secondary { + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node2 { SHOW ssl; } + expect { on } + exec node2 pg_autoctl config get ssl.sslmode + expect { verify-ca } +} + +# +# test_004: failover +# + +step test_004_failover { + exec monitor pg_autoctl perform failover +} diff --git a/tests/tap/specs/ssl_self_signed.pgaf b/tests/tap/specs/ssl_self_signed.pgaf new file mode 100644 index 000000000..c0a9f4f66 --- /dev/null +++ b/tests/tap/specs/ssl_self_signed.pgaf @@ -0,0 +1,76 @@ +# Test pg_auto_failover with self-signed SSL certificates (ssl-mode = require). +# Both the monitor and data nodes generate their own self-signed certs. +# +# Ported from tests/test_ssl_self_signed.py +# Predecessor: tests/test_ssl_self_signed.py + +cluster { + monitor + formation { + node1 + node2 + } +} + +setup { + wait until node1 state is single timeout 60s +} + +teardown { + compose down +} + +# +# test_000: create monitor with self-signed SSL +# + +step test_000_create_monitor { + exec monitor pg_autoctl inspect pgsetup wait + sql monitor { SHOW ssl; } + expect { on } + exec monitor pg_autoctl config get ssl.sslmode +} + +# +# test_001: init primary with self-signed SSL +# + +step test_001_init_primary { + wait until node1 state is single timeout 60s + sql node1 { SHOW ssl; } + expect { on } + exec node1 pg_autoctl config get ssl.sslmode +} + +# +# test_002: create table t1 +# + +step test_002_create_t1 { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } +} + +# +# test_003: add secondary node2 with self-signed SSL +# + +step test_003_init_secondary { + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node2 { SHOW ssl; } + expect { on } + exec node2 pg_autoctl config get ssl.sslmode +} + +# +# test_004: failover +# + +step test_004_failover { + exec monitor pg_autoctl perform failover + wait until node2 state is primary + and node1 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/tablespaces.pgaf b/tests/tap/specs/tablespaces.pgaf new file mode 100644 index 000000000..abcc9a85b --- /dev/null +++ b/tests/tap/specs/tablespaces.pgaf @@ -0,0 +1,148 @@ +# Tablespace HA: verify that tablespaces stored on extra volumes outside +# PGDATA survive failover, network partitions, and pg_rewind. +# +# Three tablespace directories are used across the test: +# /extra_volumes/extended_a — created before node2 joins +# /extra_volumes/extended_b — created while streaming replication is active +# /extra_volumes/extended_c — created during a network partition (wait_primary) +# +# Each node gets its own named Docker volume for each tablespace path. +# Replication copies data via WAL; volumes are not shared between nodes. +# +# Predecessor: tests/tablespaces/test_tablespaces_0[1-6].py + +cluster { + monitor + formation { + node node1 { + volume extended_a "/extra_volumes/extended_a" + volume extended_b "/extra_volumes/extended_b" + volume extended_c "/extra_volumes/extended_c" + } + node node2 { + volume extended_a "/extra_volumes/extended_a" + volume extended_b "/extra_volumes/extended_b" + volume extended_c "/extra_volumes/extended_c" + } + } +} + +setup { + wait until node1 state is primary + and node2 state is secondary + timeout 90s + exec node1 sudo chown docker /extra_volumes/extended_a /extra_volumes/extended_b /extra_volumes/extended_c + exec node2 sudo chown docker /extra_volumes/extended_a /extra_volumes/extended_b /extra_volumes/extended_c +} + +teardown { + compose down +} + +# +# test_001: create table t1 in default tablespace, then create tablespace +# extended_a and table t2 in it. node2 has not joined yet. +# + +step test_001_create_tablespace_a { + sql node1 { CREATE TABLE t1(a int); } + sql node1 { INSERT INTO t1 VALUES (1), (2); } + sql node1 { + CREATE TABLESPACE extended_a LOCATION '/extra_volumes/extended_a'; + } + sql node1 { CREATE TABLE t2(i int) TABLESPACE extended_a; } + sql node1 { INSERT INTO t2 VALUES (3), (4); } +} + +# +# test_002: node2 joins as secondary; verify that both tables — including +# t2 in extended_a — are accessible on the secondary. +# + +step test_002_init_secondary { + wait until node2 state is secondary + and node1 state is primary + timeout 90s + sql node2 { SELECT * FROM t1 ORDER BY a; } + expect { { 1 } { 2 } } + sql node2 { SELECT * FROM t2 ORDER BY i; } + expect { { 3 } { 4 } } +} + +# +# test_003: create tablespace extended_b while streaming replication is +# active; verify the new tablespace and table t3 replicate correctly. +# + +step test_003_tablespace_b_while_streaming { + sql node1 { + CREATE TABLESPACE extended_b LOCATION '/extra_volumes/extended_b'; + } + sql node1 { CREATE TABLE t3(i int) TABLESPACE extended_b; } + sql node1 { INSERT INTO t3 VALUES (5), (6); } + sql node2 { SELECT * FROM t3 ORDER BY i; } + expect { { 5 } { 6 } } +} + +# +# test_004: failover — node2 becomes primary. Write to tablespace tables +# on the new primary, verify the new secondary (node1) reads them back. +# + +step test_004_failover { + exec monitor pg_autoctl perform failover + sql node2 { INSERT INTO t2 VALUES (7); } + sql node1 { SELECT * FROM t2 ORDER BY i; } + expect { { 3 } { 4 } { 7 } } + sql node2 { INSERT INTO t3 VALUES (8); } + sql node1 { SELECT * FROM t3 ORDER BY i; } + expect { { 5 } { 6 } { 8 } } +} + +# +# test_005: network partition — disconnect node1 (now secondary). +# While node2 is in wait_primary, create tablespace extended_c and write +# to all three tablespace tables. +# + +step test_005_network_partition { + network disconnect node1 + wait until node2 state is wait_primary timeout 90s + sql node2 { + CREATE TABLESPACE extended_c LOCATION '/extra_volumes/extended_c'; + } + sql node2 { CREATE TABLE t4(i int) TABLESPACE extended_c; } + sql node2 { INSERT INTO t4 VALUES (10), (11); } + sql node2 { INSERT INTO t2 VALUES (12); } + sql node2 { INSERT INTO t3 VALUES (13); } +} + +# +# test_006: reconnect node1. pg_rewind copies WAL from node2, which +# includes the extended_c tablespace directory. After convergence node1 +# must see all writes that happened during the partition. +# + +step test_006_node1_rejoins { + network connect node1 + wait until node1 state is secondary + and node2 state is primary + timeout 90s + sql node1 { SELECT * FROM t4 ORDER BY i; } + expect { { 10 } { 11 } } + sql node1 { SELECT * FROM t2 ORDER BY i; } + expect { { 3 } { 4 } { 7 } { 12 } } + sql node1 { SELECT * FROM t3 ORDER BY i; } + expect { { 5 } { 6 } { 8 } { 13 } } +} + +# +# test_007: promote the original primary (node1) back via perform-promotion. +# + +step test_007_promote_original_primary { + exec monitor pg_autoctl perform promotion --name node1 + wait until node1 state is primary + and node2 state is secondary + timeout 90s +} diff --git a/tests/tap/specs/upgrade.pgaf b/tests/tap/specs/upgrade.pgaf new file mode 100644 index 000000000..358dfdd96 --- /dev/null +++ b/tests/tap/specs/upgrade.pgaf @@ -0,0 +1,189 @@ +# Upgrade test: live binary + extension swap without container restarts. +# +# Tests the documented production upgrade procedure: +# +# 1. Install new binary on ALL nodes while the cluster is running. +# The keeper's FSM loop fires exit(EXIT_CODE_MONITOR) only when the +# monitor extension version matches the NEW binary's required version, +# so pre-staging the binary is safe and does not trigger self-restart yet. +# +# 2. Install new binary + new extension files on the monitor, then restart +# just the monitor listener child (not the container). The listener +# detects the installed extension version ("2.1") differs from what the +# new binary requires ("2.2"), runs ALTER EXTENSION pgautofailover +# UPDATE TO '2.2', and restarts Postgres — all without stopping the +# supervisor (PID 1). +# +# 3. Each keeper's node-active child polls the monitor, sees the extension +# is now "2.2", compares against the on-disk binary's required version +# ("2.2"), finds a match, and calls exit(EXIT_CODE_MONITOR). The +# supervisor re-execs the node-active child from the new on-disk binary. +# Postgres is never restarted; only the pg_autoctl child re-execs. +# +# Image build requirements — run before this spec: +# make -C tests/upgrade pgaf-current → pgaf:current (built from v2.1 tag) +# make -C tests/upgrade pgaf-next → pgaf:next (built from current branch) +# +# pgaf:current contains BOTH binaries (2.1 and 2.2) and the v2.2 extension +# files pre-staged at /usr/local/bin/pgaf/2.2/. The "inject" steps below +# flip an in-container symlink and copy those pre-staged files — no runtime +# dependency on pgaf:next once the images are built. +# +# PGVERSION defaults to 16 (the latest PG that v2.1 supports). Rebuild with +# PGVERSION=17 make -C tests/upgrade pgaf-current pgaf-next +# if PREV_TAG supports PG17+ (and update the /16/ paths in test_003 below). + +cluster { + monitor + image "pgaf:current" + formation { + node1 + node2 + node3 + } +} + +setup { + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 120s +} + +teardown { + compose down +} + +# ------------------------------------------------------------------------- +# Baseline: cluster running on pgaf:current (v2.1), extension at "2.1". +# Write a marker row that must survive through the upgrade. +# ------------------------------------------------------------------------- + +step test_001_baseline { + sql node1 { CREATE TABLE upgrade_marker(ts timestamptz default now()); } + sql node1 { INSERT INTO upgrade_marker DEFAULT VALUES; } + sql monitor { + SELECT installed_version + FROM pg_available_extensions + WHERE name = 'pgautofailover'; + } + expect { 2.1 } +} + +# ------------------------------------------------------------------------- +# Phase 1: flip the "current" symlink on data nodes to point at the v2.2 +# binary. Both binaries are already baked into pgaf:current — no docker cp +# is needed. The static shim at /usr/local/bin/pg_autoctl delegates to +# pgaf/current/pg_autoctl, so after the flip: +# - The keeper's on-disk version check (argv0 version --json) returns "2.2" +# - The v2.1 supervisor's next child spawn also uses v2.2 (PG_AUTOCTL_DEBUG_BIN_PATH) +# No self-restart fires yet: the monitor extension is still "2.1" and the +# keeper only exits when installed "2.2" matches on-disk required "2.2". +# ------------------------------------------------------------------------- + +step test_002_inject_node_binaries { + exec node1 ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current + exec node2 ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current + exec node3 ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current +} + +# ------------------------------------------------------------------------- +# Phase 2: flip the symlink on the monitor and install the v2.2 extension +# files (also pre-baked into pgaf:current at /usr/local/bin/pgaf/2.2/). +# Then restart just the listener child; the v2.2 listener runs: +# ALTER EXTENSION pgautofailover UPDATE TO '2.2' +# ------------------------------------------------------------------------- + +step test_003_upgrade_monitor { + # Install v2.2 extension files into system paths. The script is baked into + # the image with PGVERSION resolved at build time (no hardcoded /16/ here). + exec monitor sudo /usr/local/bin/pgaf/2.2/install-extension + # Run ALTER EXTENSION via psql BEFORE flipping the symlink so the v2.2 + # listener that starts next already sees installed_version = "2.2" and + # skips the Postgres-restart code path (which races with the supervisor). + exec monitor psql -U docker -d pg_auto_failover -c "ALTER EXTENSION pgautofailover UPDATE TO '2.2'" + # Flip the monitor symlink; from here pg_autoctl = v2.2. + exec monitor ln -sfn /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current + # Restart postgres on the monitor so it loads the v2.2 pgautofailover.so. + # Without this, the old .so is still resident in memory even though the + # control file and SQL catalog were updated; every keeper call to the + # monitor functions fails with "loaded library requires 2.1 but control + # file specifies 2.2", exhausting the supervisor restart limit. + exec monitor pg_autoctl manual service restart postgres --pgdata /var/lib/postgres/pgaf +} + +# ------------------------------------------------------------------------- +# Phase 3: wait for keepers to self-restart. +# +# After the monitor extension becomes "2.2", the keeper children compare +# installed "2.2" against the on-disk binary's required "2.2" — they match, +# so each child calls exit(EXIT_CODE_MONITOR) and the supervisor re-execs +# from the new binary. No Postgres restart, no container restart. +# ------------------------------------------------------------------------- + +step test_003b_wait_keeper_restart { + # Give keepers time to detect the extension upgrade and self-restart. + # v2.1 keeper polls every ~5 s; the listener restart + ALTER EXTENSION adds + # another 1-2 s. 45 s gives ample time on slow CI runners. + exec monitor sleep 45 +} + +step test_004_wait_convergence { + wait until node1 state is primary + and node2 state is secondary + and node3 state is secondary + timeout 180s +} + +# ------------------------------------------------------------------------- +# Verify the upgrade mechanism actually fired on each node. +# ------------------------------------------------------------------------- + +step test_005_verify_node_restart { + logs node1 contains "exiting for a restart of the node-active process" + logs node2 contains "exiting for a restart of the node-active process" + logs node3 contains "exiting for a restart of the node-active process" +} + +step test_006_verify_extension_version { + sql monitor { + SELECT installed_version + FROM pg_available_extensions + WHERE name = 'pgautofailover'; + } + expect { 2.2 } +} + +# ------------------------------------------------------------------------- +# Data integrity: row written before the upgrade must be readable on all +# nodes, confirming Postgres was never stopped on the data nodes. +# ------------------------------------------------------------------------- + +step test_007_verify_data_intact { + sql node1 { SELECT count(*) FROM upgrade_marker; } + expect { 1 } + sql node2 { SELECT count(*) FROM upgrade_marker; } + expect { 1 } + sql node3 { SELECT count(*) FROM upgrade_marker; } + expect { 1 } +} + +# ------------------------------------------------------------------------- +# Smoke: failover through the fully-upgraded cluster. +# ------------------------------------------------------------------------- + +step test_008_failover_post_upgrade { + exec monitor pg_autoctl perform promotion --name node2 + wait until node2 state is primary + and node1 state is secondary + and node3 state is secondary + timeout 300s +} + +step test_009_write_on_new_primary { + sql node2 { INSERT INTO upgrade_marker DEFAULT VALUES; } + sql node1 { SELECT count(*) FROM upgrade_marker; } + expect { 2 } + sql node3 { SELECT count(*) FROM upgrade_marker; } + expect { 2 } +} diff --git a/tests/upgrade/Dockerfile.current b/tests/upgrade/Dockerfile.current new file mode 100644 index 000000000..0600e2f39 --- /dev/null +++ b/tests/upgrade/Dockerfile.current @@ -0,0 +1,81 @@ +# pgaf:current — v2.1 with both binaries baked in and a static shim. +# +# Layout: +# /usr/local/bin/pg_autoctl — static bash shim (never replaced) +# /usr/local/bin/pgaf/2.1/pg_autoctl — v2.1 binary +# /usr/local/bin/pgaf/2.2/pg_autoctl — v2.2 binary (from pgaf:next) +# /usr/local/bin/pgaf/current — symlink → pgaf/2.1 (initially) +# +# "Injecting" an upgrade = ln -sf /usr/local/bin/pgaf/2.2 /usr/local/bin/pgaf/current +# Run that inside each container via `docker compose exec`; no docker cp needed. +# +# Why this works for the self-restart mechanism: +# +# The shim always lives at /usr/local/bin/pg_autoctl. It delegates every +# command to pgaf/current/pg_autoctl via `exec -a /usr/local/bin/pg_autoctl` +# so the real binary sees argv[0] = /usr/local/bin/pg_autoctl. That path is +# what pg_autoctl saves as pg_autoctl_argv0, and later uses for the on-disk +# version check: +# +# run_program(pg_autoctl_argv0, "version", "--json") +# +# After the symlink flip, that invocation runs the shim → delegates to v2.2 → +# returns required_extension_version "2.2". Combined with the monitor +# extension just upgraded to "2.2", the keeper detects a match and calls +# exit(EXIT_CODE_MONITOR) so the supervisor re-execs from the new binary. +# +# PG_AUTOCTL_DEBUG_BIN_PATH=/usr/local/bin/pg_autoctl is set in ENV so the +# v2.1 supervisor's pg_autoctl_program also resolves to the shim path, and +# child processes spawned after the symlink flip pick up v2.2. +# +# Startup command translation: +# +# pgaftest generates: pg_autoctl node run /etc/pgaf/node.ini (v2.2 syntax) +# v2.1 does not know "node run"; the shim translates it per node kind. + +ARG BASE_IMAGE=pgaf:current-base +ARG NEXT_IMAGE=pgaf:next + +FROM ${NEXT_IMAGE} AS next +FROM ${BASE_IMAGE} + +USER root + +# bash is required for "exec -a name" (sets argv[0] for the real binary). +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends bash \ + && rm -rf /var/lib/apt/lists/* + +# Place both binaries side by side; current → 2.1 initially. +RUN mkdir -p /usr/local/bin/pgaf/2.1 /usr/local/bin/pgaf/2.2 +RUN mv /usr/local/bin/pg_autoctl /usr/local/bin/pgaf/2.1/pg_autoctl +COPY --from=next /usr/local/bin/pg_autoctl /usr/local/bin/pgaf/2.2/pg_autoctl +RUN ln -s /usr/local/bin/pgaf/2.1 /usr/local/bin/pgaf/current \ + && chown -R docker /usr/local/bin/pgaf + +# Pre-stage v2.2 extension files so the upgrade step is a local install, not +# a docker cp from an external image. Stored alongside the v2.2 binary. +ARG PGVERSION=16 +COPY --from=next /usr/share/postgresql/${PGVERSION}/extension/pgautofailover.control \ + /usr/local/bin/pgaf/2.2/pgautofailover.control +COPY --from=next /usr/share/postgresql/${PGVERSION}/extension/pgautofailover--2.1--2.2.sql \ + /usr/local/bin/pgaf/2.2/pgautofailover--2.1--2.2.sql +COPY --from=next /usr/share/postgresql/${PGVERSION}/extension/pgautofailover--2.2.sql \ + /usr/local/bin/pgaf/2.2/pgautofailover--2.2.sql +COPY --from=next /usr/lib/postgresql/${PGVERSION}/lib/pgautofailover.so \ + /usr/local/bin/pgaf/2.2/pgautofailover.so + +# Install script — uses pg_config at runtime so PGVERSION need not be baked in. +# Usage (inside the container, as root): /usr/local/bin/pgaf/2.2/install-extension +COPY install-extension.sh /usr/local/bin/pgaf/2.2/install-extension +RUN chmod 755 /usr/local/bin/pgaf/2.2/install-extension + +# Static shim — stays at the canonical path forever. +COPY pg_autoctl_shim.sh /usr/local/bin/pg_autoctl +RUN chmod 755 /usr/local/bin/pg_autoctl + +# Tell the v2.1 supervisor to use the shim path for spawning child processes, +# so that after the symlink flip the next child spawn picks up v2.2. +ENV PG_AUTOCTL_DEBUG_BIN_PATH=/usr/local/bin/pg_autoctl + +USER docker diff --git a/tests/upgrade/Makefile b/tests/upgrade/Makefile index 92f17b6da..e178b2022 100644 --- a/tests/upgrade/Makefile +++ b/tests/upgrade/Makefile @@ -1,56 +1,104 @@ -NODES ?= 3 - -PATCH = tests/upgrade/monitor-upgrade-1.7.patch -Q_VERSION = select default_version, installed_version -Q_VERSION += from pg_available_extensions where name = 'pgautofailover' - -build: - docker compose build - -patch: - cd ../.. && git apply $(PATCH) - +# tests/upgrade/Makefile +# +# Builds the two Docker images needed for the upgrade spec and runs it. +# +# Usage: +# make build both images and run the upgrade spec +# make pgaf-current build pgaf:current from PREV_TAG +# make pgaf-next build pgaf:next from current working tree +# make run run tests/tap/specs/upgrade.pgaf +# make clean remove the two images +# +# Customisation: +# PREV_TAG — git tag to build pgaf:current from (default: auto-detected) +# PGVERSION — Postgres major version baked into the image (default: 16) +# +# PGVERSION defaults to 16 because v2.1 (the current PREV_TAG) only supports +# up to PG16 — its monitor extension explicitly rejects PG >= 17. Override +# with PGVERSION=17 only when testing against a PREV_TAG that supports PG17+. +# +# Auto-detection of PREV_TAG: reads PG_AUTOCTL_EXTENSION_VERSION from +# defaults.h to get the current extension version (e.g. "2.2"), then +# finds the most recent tag whose version is strictly lower. The tag +# matching assumes the form "vMAJOR.MINOR". +# +# Examples: +# make PREV_TAG=v2.0 # test 2.0→2.2 on default PG16 +# make PREV_TAG=v2.1 PGVERSION=16 # explicit PG16 + +PGVERSION ?= 16 +PGAFTEST ?= pgaftest +SPEC := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../tap/specs/upgrade.pgaf) +REPO_ROOT := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../..) +DOCKERFILE := $(REPO_ROOT)/Dockerfile + +# Current extension version from source +CURRENT_EXT := $(shell grep 'PG_AUTOCTL_EXTENSION_VERSION ' \ + $(REPO_ROOT)/src/bin/pg_autoctl/defaults.h \ + | grep -o '"[^"]*"' | tr -d '"') + +# Previous release tag: the highest vX.Y tag whose X.Y < CURRENT_EXT. +# Works for two-component versions (2.1, 2.2, …). +# Override with PREV_TAG=v2.0 if the heuristic gives the wrong answer. +_PREV_TAG_DETECT := $(shell \ + cur=$(CURRENT_EXT); \ + git -C $(REPO_ROOT) tag --sort=-version:refname \ + | grep -E '^v[0-9]+[.][0-9]+$$' \ + | grep -v "^v$$cur$$" \ + | head -1) +PREV_TAG ?= $(_PREV_TAG_DETECT) + +.PHONY: all pgaf-current pgaf-next run clean + +all: pgaf-next pgaf-current run + +## Build pgaf:current — the previous release, used as the starting point. +## Built from the git archive of PREV_TAG so that uncommitted changes in +## the working tree do not contaminate the "old" image. +## +## git archive produces a clean tree without git history, so git-version.h +## (a generated file) is absent. We extract the archive into a tempdir, +## synthesise git-version.h from the tag name, then docker build the dir. +pgaf-current: + @if [ -z "$(PREV_TAG)" ]; then \ + echo "ERROR: could not auto-detect PREV_TAG; set it explicitly:"; \ + echo " make PREV_TAG=v2.1"; \ + exit 1; \ + fi + @echo "Building pgaf:current from $(PREV_TAG) (extension version from that tag)" + $(eval _TMP := $(shell mktemp -d)) + git -C $(REPO_ROOT) archive $(PREV_TAG) | tar -x -C $(_TMP) + printf '#define GIT_VERSION "%s"\n' '$(PREV_TAG)' \ + > $(_TMP)/src/bin/pg_autoctl/git-version.h + cp $(REPO_ROOT)/Dockerfile $(_TMP)/Dockerfile + docker build \ + --build-arg PGVERSION=$(PGVERSION) \ + --target run \ + -t pgaf:current-base \ + $(_TMP) + rm -rf $(_TMP) + docker build \ + --build-arg BASE_IMAGE=pgaf:current-base \ + --build-arg NEXT_IMAGE=pgaf:next \ + -f $(dir $(abspath $(lastword $(MAKEFILE_LIST))))Dockerfile.current \ + -t pgaf:current \ + $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +## Build pgaf:next — the current working tree (the new version being tested). +pgaf-next: + @echo "Building pgaf:next from working tree (extension version $(CURRENT_EXT))" + docker build \ + --build-arg PGVERSION=$(PGVERSION) \ + --target run \ + -t pgaf:next \ + $(REPO_ROOT) + +## Run the upgrade spec. +## Requires pgaf:current and pgaf:next to already be built. +run: + @echo "Running upgrade spec: $(SPEC)" + $(PGAFTEST) run $(SPEC) + +## Remove the two upgrade test images. clean: - cd ../.. && git apply --reverse $(PATCH) - -up: create-volumes compose-up tail ; - -down: compose-down rm-volumes ; - -compose-down: - docker compose down --volumes --remove-orphans - -compose-up: - docker compose up -d - -tail: - docker compose logs -f - -create-volumes: - for v in volm vol1 vol2 vol3; do docker volume create $$v; done - -rm-volumes: - for v in volm vol1 vol2 vol3; do docker volume rm $$v; done - -upgrade-monitor: patch - docker compose up -d --no-deps --build monitor - -upgrade-nodes: - docker compose up -d --no-deps --build node3 node2 - docker compose up -d --no-deps --build node1 - -state: - docker compose exec monitor pg_autoctl show state - -version: - docker compose exec monitor pg_autoctl version - docker compose exec monitor psql -d pg_auto_failover -c "$(Q_VERSION)" - -failover: - docker compose exec monitor pg_autoctl perform failover - -watch: - docker compose exec monitor watch -n 0.2 pg_autoctl show state - -.PHONY: build patch clean up down upgrade-monitor state watch -.PHONY: compose-down compose-up create-volumes rm-volumes + -docker rmi pgaf:current pgaf:next 2>/dev/null || true diff --git a/tests/upgrade/install-extension.sh b/tests/upgrade/install-extension.sh new file mode 100644 index 000000000..c9392d58d --- /dev/null +++ b/tests/upgrade/install-extension.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Install the pre-staged v2.2 pgautofailover extension files into the system +# extension and library paths reported by pg_config. Run as root inside the +# pgaf:current container. +set -e + +STAGING=/usr/local/bin/pgaf/2.2 +EXTDIR=$(pg_config --sharedir)/extension +LIBDIR=$(pg_config --pkglibdir) + +install -m 644 "${STAGING}/pgautofailover.control" "${EXTDIR}/" +install -m 644 "${STAGING}/pgautofailover--2.1--2.2.sql" "${EXTDIR}/" +install -m 644 "${STAGING}/pgautofailover--2.2.sql" "${EXTDIR}/" +install -m 755 "${STAGING}/pgautofailover.so" "${LIBDIR}/" diff --git a/tests/upgrade/pg_autoctl_shim.sh b/tests/upgrade/pg_autoctl_shim.sh new file mode 100644 index 000000000..a659ce2fc --- /dev/null +++ b/tests/upgrade/pg_autoctl_shim.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Static shim at /usr/local/bin/pg_autoctl — never replaced during the upgrade. +# +# Delegates to /usr/local/bin/pgaf/current/pg_autoctl, which is a symlink: +# initially → pgaf/2.1 (v2.1 binary) +# after upgrade → pgaf/2.2 (v2.2 binary) +# +# exec -a sets argv[0] to /usr/local/bin/pg_autoctl so the real binary saves +# that path as pg_autoctl_argv0. The keeper uses that path for the on-disk +# version check (pg_autoctl_argv0 version --json). After the symlink flip +# that invocation hits this shim again → delegates to v2.2 → returns "2.2". +# +# PG_AUTOCTL_DEBUG_BIN_PATH is set in ENV (Dockerfile.current) so the v2.1 +# supervisor's pg_autoctl_program is also /usr/local/bin/pg_autoctl; child +# processes after the symlink flip therefore pick up v2.2. +# +# Startup translation: +# pgaftest generates: pg_autoctl node run /etc/pgaf/node.ini (v2.2 syntax) +# v2.1 does not know "node run"; we translate per the [node] kind in the ini. + +SHIM=/usr/local/bin/pg_autoctl +CURRENT=/usr/local/bin/pgaf/current/pg_autoctl + +ini_get() { + awk -F'[[:space:]]*=[[:space:]]*' \ + -v sec="$1" -v key="$2" \ + '/^\[/{in_sec=($0 == "["sec"]")} in_sec && $1==key{print $2; exit}' "$3" +} + +if [ "$1" = "node" ] && [ "$2" = "run" ] && [ -n "$3" ]; then + ini="$3" + + kind=$(ini_get node kind "$ini") + hostname=$(ini_get node hostname "$ini") + port=$(ini_get node port "$ini") + pgdata=$(ini_get postgresql pgdata "$ini") + monitor=$(ini_get monitor pguri "$ini") + name=$(ini_get node name "$ini") + formation=$(ini_get formation name "$ini") + ssl=$(ini_get options ssl "$ini") + auth=$(ini_get options auth "$ini") + + [ -z "$pgdata" ] && { echo "pg_autoctl_shim: missing [postgresql] pgdata in $ini" >&2; exit 1; } + + case "$ssl" in + off|"") ssl_flag="--no-ssl" ;; + *) ssl_flag="--ssl-self-signed" ;; + esac + + set -- \ + ${hostname:+--hostname "$hostname"} \ + ${port:+--pgport "$port"} \ + ${auth:+--auth "$auth"} \ + "$ssl_flag" + + cfg_dir=$(printf '%s' "$pgdata" | sed 's|^/||') + cfg="/var/lib/postgres/.config/pg_autoctl/${cfg_dir}/pg_autoctl.cfg" + + if [ "$kind" = "monitor" ]; then + if [ ! -f "$cfg" ]; then + # v2.1 create monitor supports --run (stays in the foreground). + exec -a "$SHIM" "$CURRENT" create monitor --pgdata "$pgdata" "$@" --run + else + exec -a "$SHIM" "$CURRENT" run --pgdata "$pgdata" + fi + else + set -- "$@" \ + ${monitor:+--monitor "$monitor"} \ + ${name:+--name "$name"} \ + ${formation:+--formation "$formation"} + + if [ ! -f "$cfg" ]; then + # v2.1 create postgres does NOT have --run; exec the run loop after. + "$CURRENT" create postgres --pgdata "$pgdata" "$@" + rc=$?; [ $rc -ne 0 ] && exit $rc + fi + + exec -a "$SHIM" "$CURRENT" run --pgdata "$pgdata" + fi +fi + +# v2.1 supervisor spawns its children as "do service {node-active,postgres,listener}". +# v2.2 renamed the "do" subcommand tree to "internal". After the symlink flip +# ($CURRENT resolves to pgaf/2.2) the v2.1 supervisor's child-spawn calls still +# use "do service X" — translate to "internal service X" so v2.2 accepts them. +# Before the flip $CURRENT → pgaf/2.1 and the v2.1 binary already understands "do". +if [ "$1" = "do" ] && [ "$2" = "service" ]; then + resolved=$(readlink /usr/local/bin/pgaf/current 2>/dev/null) + if [ "$resolved" = "/usr/local/bin/pgaf/2.2" ]; then + exec -a "$SHIM" "$CURRENT" "internal" "${@:2}" + fi +fi + +# All other commands (version --json, run --pgdata respawn, …) +exec -a "$SHIM" "$CURRENT" "$@"