From f876e82dc9ded40871e83a82fe7807c3e60b7e7f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 6 Jul 2026 19:52:18 +0200 Subject: [PATCH 1/3] refactor: add pg_autoctl inspect and manual sub-command groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorganise the operator-facing commands to make the binary more approachable without the PG_AUTOCTL_DEBUG environment variable gate: pg_autoctl inspect — read-only diagnostics, always visible inspect pgsetup local PostgreSQL setup inspection inspect fsm FSM state / list / graphviz (read-only subset) inspect monitor get primary/others/candidate-count, parse-notification inspect show ipaddr, cidr, lookup, hostname, reverse pg_autoctl manual — operator-driven FSM operations for manual recovery manual fsm assign / step / nodes get+set manual service restart postgres|listener|node-active; pgctl on|off manual monitor register / active / version manual coordinator add / activate / remove / update Both groups are always visible regardless of PG_AUTOCTL_DEBUG. PG_AUTOCTL_DEBUG now only controls log verbosity, not command visibility. Implementation details: - Add bool hidden field to CommandLine struct so make_hidden_command_set can register the internal sub-process entry points (pg_autoctl internal service postgres|listener|node-active) without surfacing them in --help - The supervisor now spawns pg_autoctl internal service ... instead of the old pg_autoctl do service ... path; pg_autoctl do is trimmed to tmux and demo tooling only - Python test helpers updated to use inspect/manual in place of do Documentation: - Add docs/ref/pg_autoctl_inspect.rst and pg_autoctl_inspect_pgsetup.rst, pg_autoctl_inspect_show.rst - Add docs/ref/pg_autoctl_manual.rst and pg_autoctl_manual_service_restart.rst - Trim docs/ref/pg_autoctl_do.rst to tmux/demo only; remove superseded pg_autoctl_do_pgsetup.rst, pg_autoctl_do_service_restart.rst, pg_autoctl_do_show.rst - Update failover-state-machine.rst: pg_autoctl inspect fsm gv (no longer requires PG_AUTOCTL_DEBUG) --- docs/failover-state-machine.rst | 6 +- docs/ref/manual.rst | 2 + docs/ref/pg_autoctl_do.rst | 113 +--- docs/ref/pg_autoctl_do_pgsetup.rst | 159 ------ docs/ref/pg_autoctl_do_service_restart.rst | 32 -- docs/ref/pg_autoctl_do_show.rst | 130 ----- docs/ref/pg_autoctl_inspect.rst | 60 +++ docs/ref/pg_autoctl_inspect_pgsetup.rst | 116 +++++ docs/ref/pg_autoctl_inspect_show.rst | 90 ++++ docs/ref/pg_autoctl_manual.rst | 88 ++++ .../ref/pg_autoctl_manual_service_restart.rst | 32 ++ src/bin/lib/subcommands.c/commandline.c | 6 + src/bin/lib/subcommands.c/commandline.h | 14 +- src/bin/pg_autoctl/cli_do_coordinator.c | 2 +- src/bin/pg_autoctl/cli_do_fsm.c | 175 ++++++- src/bin/pg_autoctl/cli_do_misc.c | 237 ++++++++- src/bin/pg_autoctl/cli_do_monitor.c | 492 +++++++++++++++++- src/bin/pg_autoctl/cli_do_root.c | 269 +++++++++- src/bin/pg_autoctl/cli_do_root.h | 33 ++ src/bin/pg_autoctl/cli_do_service.c | 8 +- src/bin/pg_autoctl/cli_inspect.c | 79 +++ src/bin/pg_autoctl/cli_inspect.h | 16 + src/bin/pg_autoctl/cli_manual.c | 128 +++++ src/bin/pg_autoctl/cli_manual.h | 16 + src/bin/pg_autoctl/cli_root.c | 34 +- src/bin/pg_autoctl/service_keeper.c | 2 +- src/bin/pg_autoctl/service_monitor.c | 2 +- src/bin/pg_autoctl/service_postgres_ctl.c | 4 +- tests/pgautofailover_utils.py | 24 +- 29 files changed, 1869 insertions(+), 500 deletions(-) delete mode 100644 docs/ref/pg_autoctl_do_pgsetup.rst delete mode 100644 docs/ref/pg_autoctl_do_service_restart.rst delete mode 100644 docs/ref/pg_autoctl_do_show.rst create mode 100644 docs/ref/pg_autoctl_inspect.rst create mode 100644 docs/ref/pg_autoctl_inspect_pgsetup.rst create mode 100644 docs/ref/pg_autoctl_inspect_show.rst create mode 100644 docs/ref/pg_autoctl_manual.rst create mode 100644 docs/ref/pg_autoctl_manual_service_restart.rst create mode 100644 src/bin/pg_autoctl/cli_inspect.c create mode 100644 src/bin/pg_autoctl/cli_inspect.h create mode 100644 src/bin/pg_autoctl/cli_manual.c create mode 100644 src/bin/pg_autoctl/cli_manual.h diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index a1d1e1e9d..026eacec7 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -277,10 +277,10 @@ command, and then the node entry is removed from the monitor. pg_auto_failover keeper's State Machine --------------------------------------- -When built in TEST mode, it is then possible to use the following command to -get a visual representation of the Keeper's Finite State Machine:: +It is possible to use the following command to get a visual representation +of the Keeper's Finite State Machine:: - $ PG_AUTOCTL_DEBUG=1 pg_autoctl do fsm gv | dot -Tsvg > fsm.svg + $ pg_autoctl inspect fsm gv | dot -Tsvg > fsm.svg The `dot` program is part of the Graphviz suite and produces the following output: diff --git a/docs/ref/manual.rst b/docs/ref/manual.rst index 12ed2f966..93638bea5 100644 --- a/docs/ref/manual.rst +++ b/docs/ref/manual.rst @@ -20,6 +20,8 @@ have their own manual page. pg_autoctl_get pg_autoctl_set pg_autoctl_perform + pg_autoctl_inspect + pg_autoctl_manual pg_autoctl_do pg_autoctl_run pg_autoctl_watch diff --git a/docs/ref/pg_autoctl_do.rst b/docs/ref/pg_autoctl_do.rst index d471e63cd..0b39c13e8 100644 --- a/docs/ref/pg_autoctl_do.rst +++ b/docs/ref/pg_autoctl_do.rst @@ -3,121 +3,24 @@ pg_autoctl do ============= -pg_autoctl do - Internal commands and internal QA tooling +pg_autoctl do - Internal QA tooling (tmux sessions, demo app) -The debug commands for ``pg_autoctl`` are only available when the -environment variable ``PG_AUTOCTL_DEBUG`` is set (to any value). - -When testing pg_auto_failover, it is helpful to be able to play with the -local nodes using the same lower-level API as used by the pg_auto_failover -Finite State Machine transitions. Some commands could be useful in contexts -other than pg_auto_failover development and QA work, so some documentation -has been made available. +The ``pg_autoctl do`` command group contains development and QA tooling that +is not intended for production use. For read-only diagnostics see +:ref:`pg_autoctl_inspect`; for manual cluster recovery operations see +:ref:`pg_autoctl_manual`. .. toctree:: :maxdepth: 1 pg_autoctl_do_tmux pg_autoctl_do_demo - pg_autoctl_do_service_restart - pg_autoctl_do_show - pg_autoctl_do_pgsetup -The low-level API is made available through the following ``pg_autoctl do`` -commands, only available in debug environments:: +``pg_autoctl do`` provides the following commands:: pg_autoctl do - + monitor Query a pg_auto_failover monitor - + fsm Manually manage the keeper's state - + primary Manage a PostgreSQL primary server - + standby Manage a PostgreSQL standby server - + show Show some debug level information - + pgsetup Manage a local Postgres setup - + pgctl Signal the pg_autoctl postgres service - + service Run pg_autoctl sub-processes (services) - + tmux Set of facilities to handle tmux interactive sessions - + demo Use a demo application for pg_auto_failover - - pg_autoctl do monitor - + get Get information from the monitor - register Register the current node with the monitor - active Call in the pg_auto_failover Node Active protocol - version Check that monitor version is 1.5.0.1; alter extension update if not - parse-notification parse a raw notification message - - pg_autoctl do monitor get - primary Get the primary node from pg_auto_failover in given formation/group - others Get the other nodes from the pg_auto_failover group of hostname/port - coordinator Get the coordinator node from the pg_auto_failover formation - - pg_autoctl do fsm - init Initialize the keeper's state on-disk - state Read the keeper's state from disk and display it - list List reachable FSM states from current state - gv Output the FSM as a .gv program suitable for graphviz/dot - assign Assign a new goal state to the keeper - step Make a state transition if instructed by the monitor - + nodes Manually manage the keeper's nodes list - - pg_autoctl do fsm nodes - get Get the list of nodes from file (see --disable-monitor) - set Set the list of nodes to file (see --disable-monitor) - - pg_autoctl do primary - + slot Manage replication slot on the primary server - + adduser Create users on primary - defaults Add default settings to postgresql.conf - identify Run the IDENTIFY_SYSTEM replication command on given host - - pg_autoctl do primary slot - create Create a replication slot on the primary server - drop Drop a replication slot on the primary server - - pg_autoctl do primary adduser - monitor add a local user for queries from the monitor - replica add a local user with replication privileges - - pg_autoctl do standby - init Initialize the standby server using pg_basebackup - rewind Rewind a demoted primary server using pg_rewind - promote Promote a standby server to become writable - - pg_autoctl do show - ipaddr Print this node's IP address information - cidr Print this node's CIDR information - lookup Print this node's DNS lookup information - hostname Print this node's default hostname - reverse Lookup given hostname and check reverse DNS setup - - pg_autoctl do pgsetup - pg_ctl Find a non-ambiguous pg_ctl program and Postgres version - discover Discover local PostgreSQL instance, if any - ready Return true is the local Postgres server is ready - wait Wait until the local Postgres server is ready - logs Outputs the Postgres startup logs - tune Compute and log some Postgres tuning options - - pg_autoctl do pgctl - on Signal pg_autoctl postgres service to ensure Postgres is running - off Signal pg_autoctl postgres service to ensure Postgres is stopped - - pg_autoctl do service - + getpid Get the pid of pg_autoctl sub-processes (services) - + restart Restart pg_autoctl sub-processes (services) - pgcontroller pg_autoctl supervised postgres controller - postgres pg_autoctl service that start/stop postgres when asked - listener pg_autoctl service that listens to the monitor notifications - node-active pg_autoctl service that implements the node active protocol - - pg_autoctl do service getpid - postgres Get the pid of the pg_autoctl postgres controller service - listener Get the pid of the pg_autoctl monitor listener service - node-active Get the pid of the pg_autoctl keeper node-active service - - pg_autoctl do service restart - postgres Restart the pg_autoctl postgres controller service - listener Restart the pg_autoctl monitor listener service - node-active Restart the pg_autoctl keeper node-active service + + tmux Set of facilities to handle tmux interactive sessions + + demo Use a demo application for pg_auto_failover pg_autoctl do tmux script Produce a tmux script for a demo or a test case (debug only) diff --git a/docs/ref/pg_autoctl_do_pgsetup.rst b/docs/ref/pg_autoctl_do_pgsetup.rst deleted file mode 100644 index fb085d343..000000000 --- a/docs/ref/pg_autoctl_do_pgsetup.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. _pg_autoctl_do_pgsetup: - -pg_autoctl do pgsetup -===================== - -pg_autoctl do pgsetup - Manage a local Postgres setup - -Synopsis --------- - -The main ``pg_autoctl`` commands implement low-level management tooling for -a local Postgres instance. Some of the low-level Postgres commands can be -used as their own tool in some cases. - -pg_autoctl do pgsetup provides the following commands:: - - pg_autoctl do pgsetup - pg_ctl Find a non-ambiguous pg_ctl program and Postgres version - discover Discover local PostgreSQL instance, if any - ready Return true is the local Postgres server is ready - wait Wait until the local Postgres server is ready - logs Outputs the Postgres startup logs - tune Compute and log some Postgres tuning options - -pg_autoctl do pgsetup pg_ctl ----------------------------- - -In a similar way to ``which -a``, this commands scans your PATH for -``pg_ctl`` commands. Then it runs the ``pg_ctl --version`` command and -parses the output to determine the version of Postgres that is available in -the path. - -:: - - $ pg_autoctl do pgsetup pg_ctl --pgdata node1 - 16:49:18 69684 INFO Environment variable PG_CONFIG is set to "/Applications/Postgres.app//Contents/Versions/12/bin/pg_config" - 16:49:18 69684 INFO `pg_autoctl create postgres` would use "/Applications/Postgres.app/Contents/Versions/12/bin/pg_ctl" for Postgres 12.3 - 16:49:18 69684 INFO `pg_autoctl create monitor` would use "/Applications/Postgres.app/Contents/Versions/12/bin/pg_ctl" for Postgres 12.3 - - -pg_autoctl do pgsetup discover ------------------------------- - -Given a PGDATA or ``--pgdata`` option, the command discovers if a running -Postgres service matches the pg_autoctl setup, and prints the information -that ``pg_autoctl`` typically needs when managing a Postgres instance. - -:: - - $ pg_autoctl do pgsetup discover --pgdata node1 - pgdata: /Users/dim/dev/MS/pg_auto_failover/tmux/node1 - pg_ctl: /Applications/Postgres.app/Contents/Versions/12/bin/pg_ctl - pg_version: 12.3 - pghost: /tmp - pgport: 5501 - proxyport: 0 - pid: 21029 - is in recovery: no - Control Version: 1201 - Catalog Version: 201909212 - System Identifier: 6942422768095393833 - Latest checkpoint LSN: 0/4059C18 - Postmaster status: ready - - -pg_autoctl do pgsetup ready ---------------------------- - -Similar to the `pg_isready`__ command, though uses the Postgres -specifications found in the pg_autoctl node setup. - -__ https://www.postgresql.org/docs/current/app-pg-isready.html - -:: - - $ pg_autoctl do pgsetup ready --pgdata node1 - 16:50:08 70582 INFO Postgres status is: "ready" - - -pg_autoctl do pgsetup wait --------------------------- - -When ``pg_autoctl do pgsetup ready`` would return false because Postgres is -not ready yet, this command continues probing every second for 30 seconds, -and exists as soon as Postgres is ready. - -:: - - $ pg_autoctl do pgsetup wait --pgdata node1 - 16:50:22 70829 INFO Postgres is now serving PGDATA "/Users/dim/dev/MS/pg_auto_failover/tmux/node1" on port 5501 with pid 21029 - 16:50:22 70829 INFO Postgres status is: "ready" - - -pg_autoctl do pgsetup logs --------------------------- - -Outputs the Postgres logs from the most recent log file in the -``PGDATA/log`` directory. - -:: - - $ pg_autoctl do pgsetup logs --pgdata node1 - 16:50:39 71126 WARN Postgres logs from "/Users/dim/dev/MS/pg_auto_failover/tmux/node1/startup.log": - 16:50:39 71126 INFO 2021-03-22 14:43:48.911 CET [21029] LOG: starting PostgreSQL 12.3 on x86_64-apple-darwin16.7.0, compiled by Apple LLVM version 8.1.0 (clang-802.0.42), 64-bit - 16:50:39 71126 INFO 2021-03-22 14:43:48.913 CET [21029] LOG: listening on IPv6 address "::", port 5501 - 16:50:39 71126 INFO 2021-03-22 14:43:48.913 CET [21029] LOG: listening on IPv4 address "0.0.0.0", port 5501 - 16:50:39 71126 INFO 2021-03-22 14:43:48.913 CET [21029] LOG: listening on Unix socket "/tmp/.s.PGSQL.5501" - 16:50:39 71126 INFO 2021-03-22 14:43:48.931 CET [21029] LOG: redirecting log output to logging collector process - 16:50:39 71126 INFO 2021-03-22 14:43:48.931 CET [21029] HINT: Future log output will appear in directory "log". - 16:50:39 71126 WARN Postgres logs from "/Users/dim/dev/MS/pg_auto_failover/tmux/node1/log/postgresql-2021-03-22_144348.log": - 16:50:39 71126 INFO 2021-03-22 14:43:48.937 CET [21033] LOG: database system was shut down at 2021-03-22 14:43:46 CET - 16:50:39 71126 INFO 2021-03-22 14:43:48.937 CET [21033] LOG: entering standby mode - 16:50:39 71126 INFO 2021-03-22 14:43:48.942 CET [21033] LOG: consistent recovery state reached at 0/4022E88 - 16:50:39 71126 INFO 2021-03-22 14:43:48.942 CET [21033] LOG: invalid record length at 0/4022E88: wanted 24, got 0 - 16:50:39 71126 INFO 2021-03-22 14:43:48.946 CET [21029] LOG: database system is ready to accept read only connections - 16:50:39 71126 INFO 2021-03-22 14:43:49.032 CET [21038] LOG: fetching timeline history file for timeline 4 from primary server - 16:50:39 71126 INFO 2021-03-22 14:43:49.037 CET [21038] LOG: started streaming WAL from primary at 0/4000000 on timeline 3 - 16:50:39 71126 INFO 2021-03-22 14:43:49.046 CET [21038] LOG: replication terminated by primary server - 16:50:39 71126 INFO 2021-03-22 14:43:49.046 CET [21038] DETAIL: End of WAL reached on timeline 3 at 0/4022E88. - 16:50:39 71126 INFO 2021-03-22 14:43:49.047 CET [21033] LOG: new target timeline is 4 - 16:50:39 71126 INFO 2021-03-22 14:43:49.049 CET [21038] LOG: restarted WAL streaming at 0/4000000 on timeline 4 - 16:50:39 71126 INFO 2021-03-22 14:43:49.210 CET [21033] LOG: redo starts at 0/4022E88 - 16:50:39 71126 INFO 2021-03-22 14:52:06.692 CET [21029] LOG: received SIGHUP, reloading configuration files - 16:50:39 71126 INFO 2021-03-22 14:52:06.906 CET [21029] LOG: received SIGHUP, reloading configuration files - 16:50:39 71126 FATAL 2021-03-22 15:34:24.920 CET [21038] FATAL: terminating walreceiver due to timeout - 16:50:39 71126 INFO 2021-03-22 15:34:24.973 CET [21033] LOG: invalid record length at 0/4059CC8: wanted 24, got 0 - 16:50:39 71126 INFO 2021-03-22 15:34:25.105 CET [35801] LOG: started streaming WAL from primary at 0/4000000 on timeline 4 - 16:50:39 71126 FATAL 2021-03-22 16:12:56.918 CET [35801] FATAL: terminating walreceiver due to timeout - 16:50:39 71126 INFO 2021-03-22 16:12:57.086 CET [38741] LOG: started streaming WAL from primary at 0/4000000 on timeline 4 - 16:50:39 71126 FATAL 2021-03-22 16:23:39.349 CET [38741] FATAL: terminating walreceiver due to timeout - 16:50:39 71126 INFO 2021-03-22 16:23:39.497 CET [41635] LOG: started streaming WAL from primary at 0/4000000 on timeline 4 - - -pg_autoctl do pgsetup tune --------------------------- - -Outputs the pg_autoctl automated tuning options. Depending on the number of -CPU and amount of RAM detected in the environment where it is run, -``pg_autoctl`` can adjust some very basic Postgres tuning knobs to get -started. - -:: - - $ pg_autoctl do pgsetup tune --pgdata node1 -vv - 13:25:25 77185 DEBUG pgtuning.c:85: Detected 12 CPUs and 16 GB total RAM on this server - 13:25:25 77185 DEBUG pgtuning.c:225: Setting autovacuum_max_workers to 3 - 13:25:25 77185 DEBUG pgtuning.c:228: Setting shared_buffers to 4096 MB - 13:25:25 77185 DEBUG pgtuning.c:231: Setting work_mem to 24 MB - 13:25:25 77185 DEBUG pgtuning.c:235: Setting maintenance_work_mem to 512 MB - 13:25:25 77185 DEBUG pgtuning.c:239: Setting effective_cache_size to 12 GB - # basic tuning computed by pg_auto_failover - track_functions = pl - shared_buffers = '4096 MB' - work_mem = '24 MB' - maintenance_work_mem = '512 MB' - effective_cache_size = '12 GB' - autovacuum_max_workers = 3 - autovacuum_vacuum_scale_factor = 0.08 - autovacuum_analyze_scale_factor = 0.02 diff --git a/docs/ref/pg_autoctl_do_service_restart.rst b/docs/ref/pg_autoctl_do_service_restart.rst deleted file mode 100644 index 8f0442a46..000000000 --- a/docs/ref/pg_autoctl_do_service_restart.rst +++ /dev/null @@ -1,32 +0,0 @@ -.. _pg_autoctl_do_service_restart: - -pg_autoctl do service restart -============================= - -pg_autoctl do service restart - Run pg_autoctl sub-processes (services) - -Synopsis --------- - -pg_autoctl do service restart provides the following commands:: - - pg_autoctl do service restart - postgres Restart the pg_autoctl postgres controller service - listener Restart the pg_autoctl monitor listener service - node-active Restart the pg_autoctl keeper node-active service - - -Description ------------ - -It is possible to restart the ``pg_autoctl`` or the Postgres service without -affecting the other running service. Typically, to restart the -``pg_autoctl`` parts without impacting Postgres:: - - $ pg_autoctl do service restart node-active --pgdata node1 - 14:52:06 31223 INFO Sending the TERM signal to service "node-active" with pid 26626 - 14:52:06 31223 INFO Service "node-active" has been restarted with pid 31230 - 31230 - -The Postgres service has not been impacted by the restart of the -``pg_autoctl`` process. diff --git a/docs/ref/pg_autoctl_do_show.rst b/docs/ref/pg_autoctl_do_show.rst deleted file mode 100644 index 781140216..000000000 --- a/docs/ref/pg_autoctl_do_show.rst +++ /dev/null @@ -1,130 +0,0 @@ -.. _pg_autoctl_do_show: - -pg_autoctl do show -================== - -pg_autoctl do show - Show some debug level information - -Synopsis --------- - -The commands :ref:`pg_autoctl_create_monitor` and -:ref:`pg_autoctl_create_postgres` both implement some level of automated -detection of the node network settings when the option ``--hostname`` is not -used. - -Adding to those commands, when a new node is registered to the monitor, -other nodes also edit their Postgres HBA rules to allow the new node to -connect, unless the option ``--skip-pg-hba`` has been used. - -The debug sub-commands for ``pg_autoctl do show`` can be used to see in -details the network discovery done by ``pg_autoctl``. - -pg_autoctl do show provides the following commands:: - - pg_autoctl do show - ipaddr Print this node's IP address information - cidr Print this node's CIDR information - lookup Print this node's DNS lookup information - hostname Print this node's default hostname - reverse Lookup given hostname and check reverse DNS setup - -pg_autoctl do show ipaddr -------------------------- - -Connects to an external IP address and uses ``getsockname(2)`` to retrieve -the current address to which the socket is bound. - -The external IP address defaults to ``8.8.8.8``, the IP address of a Google -provided public DNS server, or to the monitor IP address or hostname in the -context of :ref:`pg_autoctl_create_postgres`. - -:: - - $ pg_autoctl do show ipaddr - 16:42:40 62631 INFO ipaddr.c:107: Connecting to 8.8.8.8 (port 53) - 192.168.1.156 - -pg_autoctl do show cidr ------------------------ - -Connects to an external IP address in the same way as the previous command -``pg_autoctl do show ipaddr`` and then matches the local socket name with -the list of local network interfaces. When a match is found, uses the -netmask of the interface to compute the CIDR notation from the IP address. - -The computed CIDR notation is then used in HBA rules. - -:: - - $ pg_autoctl do show cidr - 16:43:19 63319 INFO Connecting to 8.8.8.8 (port 53) - 192.168.1.0/24 - - -pg_autoctl do show hostname ---------------------------- - -Uses either its first (and only) argument or the result of -``gethostname(2)`` as the candidate hostname to use in HBA rules, and then -check that the hostname resolves to an IP address that belongs to one of the -machine network interfaces. - -When the hostname forward-dns lookup resolves to an IP address that is local -to the node where the command is run, then a reverse-lookup from the IP -address is made to see if it matches with the candidate hostname. - -:: - - $ pg_autoctl do show hostname - DESKTOP-IC01GOOS.europe.corp.microsoft.com - - $ pg_autoctl -vv do show hostname 'postgres://autoctl_node@localhost:5500/pg_auto_failover' - 13:45:00 93122 INFO cli_do_show.c:256: Using monitor hostname "localhost" and port 5500 - 13:45:00 93122 INFO ipaddr.c:107: Connecting to ::1 (port 5500) - 13:45:00 93122 DEBUG cli_do_show.c:272: cli_show_hostname: ip ::1 - 13:45:00 93122 DEBUG cli_do_show.c:283: cli_show_hostname: host localhost - 13:45:00 93122 DEBUG cli_do_show.c:294: cli_show_hostname: ip ::1 - localhost - -pg_autoctl do show lookup -------------------------- - -Checks that the given argument is an hostname that resolves to a local IP -address, that is an IP address associated with a local network interface. - -:: - - $ pg_autoctl do show lookup DESKTOP-IC01GOOS.europe.corp.microsoft.com - DESKTOP-IC01GOOS.europe.corp.microsoft.com: 192.168.1.156 - -pg_autoctl do show reverse --------------------------- - -Implements the same DNS checks as Postgres HBA matching code: first does a -forward DNS lookup of the given hostname, and then a reverse-lookup from all -the IP addresses obtained. Success is reached when at least one of the IP -addresses from the forward lookup resolves back to the given hostname (as -the first answer to the reverse DNS lookup). - -:: - - $ pg_autoctl do show reverse DESKTOP-IC01GOOS.europe.corp.microsoft.com - 16:44:49 64910 FATAL Failed to find an IP address for hostname "DESKTOP-IC01GOOS.europe.corp.microsoft.com" that matches hostname again in a reverse-DNS lookup. - 16:44:49 64910 INFO Continuing with IP address "192.168.1.156" - - $ pg_autoctl -vv do show reverse DESKTOP-IC01GOOS.europe.corp.microsoft.com - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 192.168.1.156 - 16:44:45 64832 DEBUG ipaddr.c:733: reverse lookup for "192.168.1.156" gives "desktop-ic01goos.europe.corp.microsoft.com" first - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 192.168.1.156 - 16:44:45 64832 DEBUG ipaddr.c:733: reverse lookup for "192.168.1.156" gives "desktop-ic01goos.europe.corp.microsoft.com" first - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 2a01:110:10:40c::2ad - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 2a01:110:10:40c::2ad - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 100.64.34.213 - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 100.64.34.213 - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 FATAL cli_do_show.c:333: Failed to find an IP address for hostname "DESKTOP-IC01GOOS.europe.corp.microsoft.com" that matches hostname again in a reverse-DNS lookup. - 16:44:45 64832 INFO cli_do_show.c:334: Continuing with IP address "192.168.1.156" diff --git a/docs/ref/pg_autoctl_inspect.rst b/docs/ref/pg_autoctl_inspect.rst new file mode 100644 index 000000000..a9c7c009d --- /dev/null +++ b/docs/ref/pg_autoctl_inspect.rst @@ -0,0 +1,60 @@ +.. _pg_autoctl_inspect: + +pg_autoctl inspect +================== + +pg_autoctl inspect - Read-only diagnostics for a pg_auto_failover node + +The ``pg_autoctl inspect`` commands provide read-only access to local and +cluster state. They are always available — no ``PG_AUTOCTL_DEBUG`` environment +variable is required. All commands in this group are safe to run while +``pg_autoctl run`` is active. + +.. toctree:: + :maxdepth: 1 + + pg_autoctl_inspect_pgsetup + pg_autoctl_inspect_show + +``pg_autoctl inspect`` provides the following sub-command groups:: + + pg_autoctl inspect + + show Network and hostname diagnostics + + pgsetup Local PostgreSQL setup inspection + + fsm Display keeper FSM state and transitions (read-only) + + monitor Query the monitor's current state (read-only) + + getpid Get the pid of pg_autoctl sub-processes (services) + + pg_autoctl inspect pgsetup + pg_ctl Find a non-ambiguous pg_ctl program and Postgres version + discover Discover local PostgreSQL instance, if any + ready Return true if the local Postgres server is ready + wait Wait until the local Postgres server is ready + logs Outputs the Postgres startup logs + tune Compute and log some Postgres tuning options + + pg_autoctl inspect fsm + state Read the keeper's state from disk and display it + list List reachable FSM states from current state + gv Output the FSM as a .gv program suitable for graphviz/dot + + pg_autoctl inspect show + ipaddr Print this node's IP address information + cidr Print this node's CIDR information + lookup Print this node's DNS lookup information + hostname Print this node's default hostname + reverse Lookup given hostname and check reverse DNS setup + + pg_autoctl inspect monitor + get Get information from the monitor + parse-notification Parse a raw notification message + + pg_autoctl inspect monitor get + primary Get the primary node from pg_auto_failover in given formation/group + others Get the other nodes from the pg_auto_failover group of hostname/port + coordinator Get the coordinator node from the pg_auto_failover formation + + pg_autoctl inspect getpid + postgres Get the pid of the pg_autoctl postgres controller service + listener Get the pid of the pg_autoctl monitor listener service + node-active Get the pid of the pg_autoctl keeper node-active service diff --git a/docs/ref/pg_autoctl_inspect_pgsetup.rst b/docs/ref/pg_autoctl_inspect_pgsetup.rst new file mode 100644 index 000000000..691d5812b --- /dev/null +++ b/docs/ref/pg_autoctl_inspect_pgsetup.rst @@ -0,0 +1,116 @@ +.. _pg_autoctl_inspect_pgsetup: + +pg_autoctl inspect pgsetup +========================== + +pg_autoctl inspect pgsetup - Inspect the local Postgres setup + +Synopsis +-------- + +``pg_autoctl inspect pgsetup`` provides low-level management tooling for a +local Postgres instance. These commands are always available without +``PG_AUTOCTL_DEBUG``. + +:: + + pg_autoctl inspect pgsetup + pg_ctl Find a non-ambiguous pg_ctl program and Postgres version + discover Discover local PostgreSQL instance, if any + ready Return true if the local Postgres server is ready + wait Wait until the local Postgres server is ready + logs Outputs the Postgres startup logs + tune Compute and log some Postgres tuning options + +pg_autoctl inspect pgsetup pg_ctl +--------------------------------- + +In a similar way to ``which -a``, this command scans your PATH for +``pg_ctl`` commands. Then it runs ``pg_ctl --version`` and parses the output +to determine the version of Postgres available. + +:: + + $ pg_autoctl inspect pgsetup pg_ctl --pgdata node1 + 16:49:18 69684 INFO `pg_autoctl create postgres` would use "/usr/lib/postgresql/17/bin/pg_ctl" for Postgres 17 + + +pg_autoctl inspect pgsetup discover +------------------------------------ + +Given a PGDATA or ``--pgdata`` option, the command discovers whether a +running Postgres service matches the pg_autoctl setup and prints the +information that ``pg_autoctl`` typically needs. + +:: + + $ pg_autoctl inspect pgsetup discover --pgdata node1 + pgdata: /home/postgres/node1 + pg_ctl: /usr/lib/postgresql/17/bin/pg_ctl + pg_version: 17 + pghost: /tmp + pgport: 5501 + pid: 21029 + is in recovery: no + Postmaster status: ready + + +pg_autoctl inspect pgsetup ready +--------------------------------- + +Similar to `pg_isready`__, but uses the Postgres specifications found in the +pg_autoctl node setup. + +__ https://www.postgresql.org/docs/current/app-pg-isready.html + +:: + + $ pg_autoctl inspect pgsetup ready --pgdata node1 + 16:50:08 70582 INFO Postgres status is: "ready" + + +pg_autoctl inspect pgsetup wait +-------------------------------- + +When ``pg_autoctl inspect pgsetup ready`` would return false because Postgres +is not ready yet, this command probes every second for up to 30 seconds and +exits as soon as Postgres is ready. + +:: + + $ pg_autoctl inspect pgsetup wait --pgdata node1 + 16:50:22 70829 INFO Postgres is now serving PGDATA "/home/postgres/node1" on port 5501 with pid 21029 + 16:50:22 70829 INFO Postgres status is: "ready" + + +pg_autoctl inspect pgsetup logs +-------------------------------- + +Outputs the Postgres logs from the most recent log file in the +``PGDATA/log`` directory. + +:: + + $ pg_autoctl inspect pgsetup logs --pgdata node1 + 16:50:39 71126 WARN Postgres logs from "/home/postgres/node1/startup.log": + ... + + +pg_autoctl inspect pgsetup tune +-------------------------------- + +Outputs the pg_autoctl automated tuning options. Depending on the number of +CPUs and amount of RAM detected, ``pg_autoctl`` adjusts basic Postgres tuning +knobs. + +:: + + $ pg_autoctl inspect pgsetup tune --pgdata node1 -vv + 13:25:25 77185 DEBUG Detected 12 CPUs and 16 GB total RAM on this server + 13:25:25 77185 DEBUG Setting shared_buffers to 4096 MB + # basic tuning computed by pg_auto_failover + shared_buffers = '4096 MB' + work_mem = '24 MB' + maintenance_work_mem = '512 MB' + effective_cache_size = '12 GB' + autovacuum_max_workers = 3 diff --git a/docs/ref/pg_autoctl_inspect_show.rst b/docs/ref/pg_autoctl_inspect_show.rst new file mode 100644 index 000000000..d330b1986 --- /dev/null +++ b/docs/ref/pg_autoctl_inspect_show.rst @@ -0,0 +1,90 @@ +.. _pg_autoctl_inspect_show: + +pg_autoctl inspect show +======================= + +pg_autoctl inspect show - Network and hostname diagnostics + +Synopsis +-------- + +The commands :ref:`pg_autoctl_create_monitor` and +:ref:`pg_autoctl_create_postgres` both implement automated detection of node +network settings when the option ``--hostname`` is not used. When a new node +is registered to the monitor, other nodes also update their HBA rules to +allow the new node to connect. + +``pg_autoctl inspect show`` exposes the network discovery logic so that +operators can verify how ``pg_autoctl`` sees the local host. + +:: + + pg_autoctl inspect show + ipaddr Print this node's IP address information + cidr Print this node's CIDR information + lookup Print this node's DNS lookup information + hostname Print this node's default hostname + reverse Lookup given hostname and check reverse DNS setup + +pg_autoctl inspect show ipaddr +------------------------------ + +Connects to an external IP address and uses ``getsockname(2)`` to retrieve +the current address to which the socket is bound. The external IP defaults +to ``8.8.8.8``, or to the monitor IP/hostname in the context of +:ref:`pg_autoctl_create_postgres`. + +:: + + $ pg_autoctl inspect show ipaddr + 192.168.1.156 + +pg_autoctl inspect show cidr +----------------------------- + +Connects to an external IP address in the same way as ``inspect show ipaddr`` +and then matches the local socket name with the list of local network +interfaces. When a match is found, uses the netmask of the interface to +compute the CIDR notation from the IP address. The computed CIDR is used in +HBA rules. + +:: + + $ pg_autoctl inspect show cidr + 192.168.1.0/24 + +pg_autoctl inspect show hostname +--------------------------------- + +Uses either its first argument or the result of ``gethostname(2)`` as the +candidate hostname for HBA rules, then checks that the hostname resolves to +an IP address that belongs to one of the machine's network interfaces. + +:: + + $ pg_autoctl inspect show hostname + node1.example.com + +pg_autoctl inspect show lookup +------------------------------- + +Checks that the given argument is a hostname that resolves to a local IP +address (an IP address associated with a local network interface). + +:: + + $ pg_autoctl inspect show lookup node1.example.com + node1.example.com: 192.168.1.156 + +pg_autoctl inspect show reverse +-------------------------------- + +Implements the same DNS checks as Postgres HBA matching code: first does a +forward DNS lookup of the given hostname, then a reverse-lookup from all the +IP addresses obtained. Success is reached when at least one IP address from +the forward lookup resolves back to the given hostname. + +:: + + $ pg_autoctl inspect show reverse node1.example.com + node1.example.com: 192.168.1.156 diff --git a/docs/ref/pg_autoctl_manual.rst b/docs/ref/pg_autoctl_manual.rst new file mode 100644 index 000000000..1ce1da6a3 --- /dev/null +++ b/docs/ref/pg_autoctl_manual.rst @@ -0,0 +1,88 @@ +.. _pg_autoctl_manual: + +pg_autoctl manual +================= + +pg_autoctl manual - Operator-driven FSM operations for manual cluster recovery + +The ``pg_autoctl manual`` commands provide low-level control over the +pg_auto_failover finite state machine and supporting services. They are +always available — no ``PG_AUTOCTL_DEBUG`` environment variable is required. + +These commands are intended for manual recovery when the automated FSM is +stopped or stuck, and for low-level diagnostic work. Using them while +``pg_autoctl run`` is active can interfere with automated operations. + +.. toctree:: + :maxdepth: 1 + + pg_autoctl_manual_service_restart + +``pg_autoctl manual`` provides the following sub-command groups:: + + pg_autoctl manual + + fsm Manually drive the keeper FSM (mutating operations) + + service Restart pg_autoctl sub-processes or signal the postgres controller + + monitor Manually drive monitor RPCs (register / active / version) + + primary Manage a PostgreSQL primary server + + standby Manage a PostgreSQL standby server + + coordinator Manage Citus coordinator node metadata + + pg_autoctl manual fsm + init Initialize the keeper's state on-disk + assign Assign a new goal state to the keeper + step Make a state transition if instructed by the monitor + + nodes Manually manage the keeper's nodes list + + pg_autoctl manual fsm nodes + get Get the list of nodes from file (see --disable-monitor) + set Set the list of nodes to file (see --disable-monitor) + + pg_autoctl manual service + + restart Restart pg_autoctl sub-processes (services) + + pgctl Signal the pg_autoctl postgres controller + + pg_autoctl manual service restart + postgres Restart the pg_autoctl postgres controller service + listener Restart the pg_autoctl monitor listener service + node-active Restart the pg_autoctl keeper node-active service + + pg_autoctl manual service pgctl + on Signal pg_autoctl postgres service to ensure Postgres is running + off Signal pg_autoctl postgres service to ensure Postgres is stopped + + pg_autoctl manual monitor + register Register the current node with the monitor + active Call in the pg_auto_failover Node Active protocol + version Check that monitor version is current; alter extension update if not + + pg_autoctl manual primary + + slot Manage replication slot on the primary server + + adduser Create users on primary + defaults Add default settings to postgresql.conf + identify Run the IDENTIFY_SYSTEM replication command on given host + + pg_autoctl manual primary slot + create Create a replication slot on the primary server + drop Drop a replication slot on the primary server + + pg_autoctl manual primary adduser + monitor Add a local user for queries from the monitor + replica Add a local user with replication privileges + + pg_autoctl manual standby + init Initialize the standby server using pg_basebackup + rewind Rewind a demoted primary server using pg_rewind + crash-recovery Setup postgres for crash-recovery and start postgres + promote Promote a standby server to become writable + + pg_autoctl manual coordinator + add Add this node to its formation's coordinator + activate Activate this node on its formation's coordinator + remove Remove this node from its formation's coordinator + + update Update current node's host:port on the coordinator + + pg_autoctl manual coordinator update + prepare Prepare a Citus coordinator metadata update + commit Commit a Citus coordinator metadata update + rollback Rollback a Citus coordinator metadata update diff --git a/docs/ref/pg_autoctl_manual_service_restart.rst b/docs/ref/pg_autoctl_manual_service_restart.rst new file mode 100644 index 000000000..af15b7dc7 --- /dev/null +++ b/docs/ref/pg_autoctl_manual_service_restart.rst @@ -0,0 +1,32 @@ +.. _pg_autoctl_manual_service_restart: + +pg_autoctl manual service restart +================================== + +pg_autoctl manual service restart - Restart pg_autoctl sub-processes + +Synopsis +-------- + +:: + + pg_autoctl manual service restart + postgres Restart the pg_autoctl postgres controller service + listener Restart the pg_autoctl monitor listener service + node-active Restart the pg_autoctl keeper node-active service + + +Description +----------- + +It is possible to restart the ``pg_autoctl`` keeper or listener service +without affecting the other running services, and without stopping Postgres. +Typically, to restart the ``pg_autoctl`` keeper without impacting Postgres:: + + $ pg_autoctl manual service restart node-active --pgdata node1 + 14:52:06 31223 INFO Sending the TERM signal to service "node-active" with pid 26626 + 14:52:06 31223 INFO Service "node-active" has been restarted with pid 31230 + 31230 + +The Postgres service is not impacted by the restart of the ``pg_autoctl`` +keeper process. diff --git a/src/bin/lib/subcommands.c/commandline.c b/src/bin/lib/subcommands.c/commandline.c index bd9eac138..f83c85ede 100644 --- a/src/bin/lib/subcommands.c/commandline.c +++ b/src/bin/lib/subcommands.c/commandline.c @@ -226,6 +226,9 @@ commandline_pretty_print_subcommands(CommandLine *command, FILE *stream) /* pretty printing: reduce maximum length of subcommand names */ for (subcommand = command->subcommands; *subcommand != NULL; subcommand++) { + if ((*subcommand)->hidden) + continue; + int len = strlen((*subcommand)->name); if (maxLength < len) @@ -238,6 +241,9 @@ commandline_pretty_print_subcommands(CommandLine *command, FILE *stream) { const char *description = ""; + if ((*subcommand)->hidden) + continue; + if ((*subcommand)->shortDescription != NULL) { description = (*subcommand)->shortDescription; diff --git a/src/bin/lib/subcommands.c/commandline.h b/src/bin/lib/subcommands.c/commandline.h index 888e58550..a0cb674b9 100644 --- a/src/bin/lib/subcommands.c/commandline.h +++ b/src/bin/lib/subcommands.c/commandline.h @@ -29,15 +29,25 @@ typedef struct CommandLine struct CommandLine **subcommands; char *breadcrumb; + bool hidden; /* if true, omit from --help output */ } CommandLine; extern CommandLine *current_command; #define make_command_set(name, desc, usage, help, getopt, set) \ - { name, desc, usage, help, getopt, NULL, set, NULL } + { name, desc, usage, help, getopt, NULL, set, NULL, false } #define make_command(name, desc, usage, help, getopt, run) \ - { name, desc, usage, help, getopt, run, NULL, NULL } + { name, desc, usage, help, getopt, run, NULL, NULL, false } + +/* + * Like make_command_set but the command is omitted from --help output. + * The command still routes normally when invoked explicitly — used for + * internal subprocess entry points that must remain in the binary but + * are not intended for direct operator use. + */ +#define make_hidden_command_set(name, desc, usage, help, getopt, set) \ + { name, desc, usage, help, getopt, NULL, set, NULL, true } bool commandline_run(CommandLine *command, int argc, char **argv); void commandline_help(FILE *stream); diff --git a/src/bin/pg_autoctl/cli_do_coordinator.c b/src/bin/pg_autoctl/cli_do_coordinator.c index 72f5dfb55..5506fa29e 100644 --- a/src/bin/pg_autoctl/cli_do_coordinator.c +++ b/src/bin/pg_autoctl/cli_do_coordinator.c @@ -108,7 +108,7 @@ static CommandLine *coordinator_commands[] = { CommandLine do_coordinator_commands = make_command_set("coordinator", - "Query a Citus coordinator", NULL, NULL, + "Manage Citus coordinator node metadata", NULL, NULL, NULL, coordinator_commands); diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index 8c456ff73..4294aceb9 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -29,6 +29,8 @@ static void cli_do_fsm_init(int argc, char **argv); static void cli_do_fsm_state(int argc, char **argv); +static int cli_do_fsm_node_state_getopts(int argc, char **argv); +static void cli_do_fsm_node_state(int argc, char **argv); static void cli_do_fsm_list(int argc, char **argv); static void cli_do_fsm_gv(int argc, char **argv); static void cli_do_fsm_assign(int argc, char **argv); @@ -37,15 +39,8 @@ static void cli_do_fsm_step(int argc, char **argv); static void cli_do_fsm_get_nodes(int argc, char **argv); static void cli_do_fsm_set_nodes(int argc, char **argv); -static CommandLine fsm_init = - make_command("init", - "Initialize the keeper's state on-disk", - CLI_PGDATA_USAGE, - CLI_PGDATA_OPTION, - cli_getopt_pgdata, - cli_do_fsm_init); - -static CommandLine fsm_state = +/* read-only — exported for pg_autoctl inspect fsm */ +CommandLine fsm_state = make_command("state", "Read the keeper's state from disk and display it", CLI_PGDATA_USAGE, @@ -53,7 +48,17 @@ static CommandLine fsm_state = cli_getopt_pgdata, cli_do_fsm_state); -static CommandLine fsm_list = +CommandLine fsm_node_state = + make_command("node-state", + "Poll the keeper's on-disk current_role; exit 0 when it matches --state", + CLI_PGDATA_USAGE, + " --pgdata path to the keeper's data directory\n" + " --state target state to wait for (e.g. demote_timeout)\n" + " --timeout seconds to retry (default: no retry)\n", + cli_do_fsm_node_state_getopts, + cli_do_fsm_node_state); + +CommandLine fsm_list = make_command("list", "List reachable FSM states from current state", CLI_PGDATA_USAGE, @@ -61,12 +66,21 @@ static CommandLine fsm_list = cli_getopt_pgdata, cli_do_fsm_list); -static CommandLine fsm_gv = +CommandLine fsm_gv = make_command("gv", "Output the FSM as a .gv program suitable for graphviz/dot", "", NULL, NULL, cli_do_fsm_gv); -static CommandLine fsm_assign = +/* mutating — exported for pg_autoctl manual fsm */ +CommandLine fsm_init = + make_command("init", + "Initialize the keeper's state on-disk", + CLI_PGDATA_USAGE, + CLI_PGDATA_OPTION, + cli_getopt_pgdata, + cli_do_fsm_init); + +CommandLine fsm_assign = make_command("assign", "Assign a new goal state to the keeper", CLI_PGDATA_USAGE "", @@ -74,7 +88,7 @@ static CommandLine fsm_assign = cli_getopt_pgdata, cli_do_fsm_assign); -static CommandLine fsm_step = +CommandLine fsm_step = make_command("step", "Make a state transition if instructed by the monitor", CLI_PGDATA_USAGE, @@ -228,6 +242,141 @@ cli_do_fsm_state(int argc, char **argv) } +/* option state for fsm node-state */ +static struct +{ + char targetState[64]; + int timeout; +} +fsmNodeStateOpts; + +static int +cli_do_fsm_node_state_getopts(int argc, char **argv) +{ + fsmNodeStateOpts.targetState[0] = '\0'; + fsmNodeStateOpts.timeout = 0; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "state", required_argument, NULL, 's' }, + { "timeout", required_argument, NULL, 't' }, + { NULL, 0, NULL, 0 } + }; + + int c; + optind = 0; + while ((c = getopt_long(argc, argv, "D:s:t:", long_options, NULL)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(keeperOptions.pgSetup.pgdata, optarg, + sizeof(keeperOptions.pgSetup.pgdata)); + break; + } + + case 's': + { + strlcpy(fsmNodeStateOpts.targetState, optarg, + sizeof(fsmNodeStateOpts.targetState)); + break; + } + + case 't': + { + fsmNodeStateOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + default: + { + commandline_print_usage(&fsm_node_state, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + /* fall back to $PGDATA when --pgdata was not given */ + cli_common_get_set_pgdata_or_exit(&keeperOptions.pgSetup); + + if (!keeper_config_set_pathnames_from_pgdata(&keeperOptions.pathnames, + keeperOptions.pgSetup.pgdata)) + { + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +/* + * cli_do_fsm_node_state reads the keeper's on-disk state and prints: + * | + * + * When --state is given the command retries until current_role matches. + * When --timeout is given the retry loop stops after N seconds. + */ +static void +cli_do_fsm_node_state(int argc, char **argv) +{ + KeeperConfig config = keeperOptions; + + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = true; + + if (!keeper_config_read_file(&config, + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + exit(EXIT_CODE_BAD_CONFIG); + } + + time_t deadline = (fsmNodeStateOpts.timeout > 0) + ? time(NULL) + fsmNodeStateOpts.timeout + : 0; + + for (;;) + { + Keeper keeper = { 0 }; + + if (!keeper_init(&keeper, &config)) + { + if (deadline && time(NULL) < deadline) + { + pg_usleep(500 * 1000); + continue; + } + exit(EXIT_CODE_BAD_CONFIG); + } + + KeeperStateData *s = &keeper.state; + const char *current = NodeStateToString(s->current_role); + const char *assigned = NodeStateToString(s->assigned_role); + + if (fsmNodeStateOpts.targetState[0] != '\0' && + strcmp(current, fsmNodeStateOpts.targetState) != 0) + { + log_debug("fsm node-state: current_role=%s, waiting for %s", + current, fsmNodeStateOpts.targetState); + if (deadline && time(NULL) < deadline) + { + pg_usleep(500 * 1000); + continue; + } + + /* timed out or --timeout 0 with no match */ + exit(EXIT_CODE_INTERNAL_ERROR); + } + + fformat(stdout, "%s|%s\n", current, assigned); + exit(0); + } +} + + /* * cli_do_fsm_list lists reachable states from the current one. */ diff --git a/src/bin/pg_autoctl/cli_do_misc.c b/src/bin/pg_autoctl/cli_do_misc.c index 188faa88f..68e144a95 100644 --- a/src/bin/pg_autoctl/cli_do_misc.c +++ b/src/bin/pg_autoctl/cli_do_misc.c @@ -38,6 +38,10 @@ #include "primary_standby.h" #include "string_utils.h" +/* Options specific to "pg_autoctl inspect pgsetup wait" */ +static bool pgsetupWaitReadWrite = false; +static int pgsetupWaitTimeout = 30; + /* * keeper_cli_create_replication_slot implements the CLI to create a replication @@ -323,18 +327,149 @@ keeper_cli_pgsetup_is_ready(int argc, char **argv) /* - * keeper_cli_discover_pg_setup implements the CLI to discover a PostgreSQL - * setup thanks to PGDATA and other environment variables. + * keeper_cli_pgsetup_wait_getopts parses options specific to + * "pg_autoctl inspect pgsetup wait": --read-write, --timeout, plus the + * standard Postgres connection options inherited from the keeper setup. + */ +int +keeper_cli_pgsetup_wait_getopts(int argc, char **argv) +{ + /* + * cli_common_keeper_getopts (called by keeper_cli_keeper_setup_getopts) + * exits with BAD_ARGS when it encounters unknown options. Since --timeout + * and --read-write are not in its options list, we must remove them from + * argv before delegating. + * + * Strategy: + * 1. Scan argv with opterr=0 to capture --timeout / --read-write. + * 2. Build a filtered argv that omits those two options. + * 3. Pass the filtered argv to keeper_cli_keeper_setup_getopts. + */ + + /* Reset module-level wait options */ + pgsetupWaitReadWrite = false; + pgsetupWaitTimeout = 30; + + static struct option wait_options[] = { + { "read-write", no_argument, NULL, 'W' }, + { "timeout", required_argument, NULL, 'T' }, + { NULL, 0, NULL, 0 } + }; + + optind = 1; + opterr = 0; + + int c; + int option_index = 0; + + while ((c = getopt_long(argc, argv, "WT:", wait_options, &option_index)) != -1) + { + switch (c) + { + case 'W': + { + pgsetupWaitReadWrite = true; + break; + } + + case 'T': + { + int t = strtol(optarg, NULL, 10); + if (t <= 0) + { + log_error("--timeout must be a positive integer"); + exit(EXIT_CODE_BAD_ARGS); + } + pgsetupWaitTimeout = t; + break; + } + + default: + { + /* standard keeper options; handled by the delegated call below */ + break; + } + } + } + + opterr = 1; + + /* + * Build a filtered argv that strips --timeout/--read-write (and their + * arguments) so that keeper_cli_keeper_setup_getopts does not see them. + */ + char **filtered_argv = (char **) palloc((argc + 1) * sizeof(char *)); + int filtered_argc = 0; + + for (int i = 0; i < argc; i++) + { + if (strcmp(argv[i], "--read-write") == 0 || strcmp(argv[i], "-W") == 0) + { + continue; + } + + if ((strcmp(argv[i], "--timeout") == 0 || strcmp(argv[i], "-T") == 0) && + i + 1 < argc) + { + /* skip both the flag and its argument */ + i++; + continue; + } + + filtered_argv[filtered_argc++] = argv[i]; + } + filtered_argv[filtered_argc] = NULL; + + int rc = keeper_cli_keeper_setup_getopts(filtered_argc, filtered_argv); + + pfree(filtered_argv); + + return rc; +} + + +/* + * keeper_cli_pgsetup_wait_until_ready waits for the local Postgres server to + * become ready. When --read-write is given, it additionally waits until the + * server is accepting read-write connections (not in recovery and not set to + * default_transaction_read_only). + * + * The --timeout value (default 30s) is a single deadline shared by both + * phases: the pg_is_ready poll and the subsequent read-write connection + * attempt. Time spent waiting for Postgres to start counts against the + * budget for the read-write phase. */ void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) { - int timeout = 30; + int timeout = pgsetupWaitTimeout; ConfigFilePaths pathnames = { 0 }; LocalPostgresServer postgres = { 0 }; PostgresSetup *pgSetup = &(postgres.postgresSetup); + /* Record wall-clock start so all phases share one deadline. */ + time_t startTime = time(NULL); + + /* Wait up to `timeout` seconds for the config file to be created. + * In no-monitor mode, pg_autoctl create postgres runs first and writes the + * config; pgsetup wait may be called before that completes. */ + { + KeeperConfig kconfig = keeperOptions; + if (keeper_config_set_pathnames_from_pgdata(&(kconfig.pathnames), + kconfig.pgSetup.pgdata)) + { + time_t deadline = startTime + timeout; + while (!file_exists(kconfig.pathnames.config) && + time(NULL) < deadline) + { + log_debug("Waiting for config file \"%s\" to appear", + kconfig.pathnames.config); + pg_usleep(500 * 1000); + } + } + } + if (!cli_common_pgsetup_init(&pathnames, pgSetup)) { /* errors have already been logged */ @@ -343,15 +478,105 @@ keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) log_debug("Initialized pgSetup, now calling pg_setup_wait_until_is_ready()"); - bool pgIsReady = pg_setup_wait_until_is_ready(pgSetup, timeout, LOG_INFO); + /* + * Phase 1: wait for postmaster to signal "ready" in postmaster.pid. + * Pass the remaining timeout so the two phases together stay within the + * single user-visible deadline. + */ + int remainingAfterConfig = timeout - (int) (time(NULL) - startTime); + if (remainingAfterConfig <= 0) + { + log_error("Timed out waiting for Postgres config file to appear"); + exit(EXIT_CODE_PGSQL); + } + + bool pgIsReady = + pg_setup_wait_until_is_ready(pgSetup, remainingAfterConfig, LOG_INFO); log_info("Postgres status is: \"%s\"", pmStatusToString(pgSetup->pm_status)); - if (pgIsReady) + if (!pgIsReady) { + exit(EXIT_CODE_PGSQL); + } + + if (!pgsetupWaitReadWrite) + { + /* Plain "ready" check — we're done. */ exit(EXIT_CODE_QUIT); } - exit(EXIT_CODE_PGSQL); + + /* + * Phase 2: wait until the server accepts read-write connections. + * + * Postgres is up (phase 1 passed) but may still be in recovery, finishing + * pg_rewind, or in standby mode. We poll with a libpq connection that + * checks pg_is_in_recovery() until it returns false or the deadline fires. + * + * We use the local connection string from pgSetup (Unix socket when + * available, matching whatever auth the node was created with) so that the + * check works regardless of the cluster's auth method. + */ + char connstr[MAXCONNINFO]; + if (!pg_setup_get_local_connection_string(pgSetup, connstr)) + { + log_error("Failed to build local connection string for read-write check"); + exit(EXIT_CODE_BAD_CONFIG); + } + + log_info("Waiting for Postgres to accept read-write connections " + "(timeout %ds)", timeout); + + bool isReadWrite = false; + int attempts = 0; + + while (!isReadWrite) + { + int elapsed = (int) (time(NULL) - startTime); + int remaining = timeout - elapsed; + + if (remaining <= 0) + { + log_error("Timed out after %ds waiting for Postgres " + "to accept read-write connections", timeout); + exit(EXIT_CODE_PGSQL); + } + + /* Use a short per-attempt connect_timeout so we retry briskly. */ + char attemptConnstr[MAXCONNINFO]; + sformat(attemptConnstr, sizeof(attemptConnstr), + "%s connect_timeout=1", connstr); + + PGSQL pgsql = { 0 }; + pgsql_init(&pgsql, attemptConnstr, PGSQL_CONN_LOCAL); + + bool inRecovery = true; /* assume standby until proven otherwise */ + bool queryOk = pgsql_is_in_recovery(&pgsql, &inRecovery); + pgsql_finish(&pgsql); + + if (queryOk && !inRecovery) + { + isReadWrite = true; + break; + } + + /* let's not be THAT verbose about it */ + if (attempts % 10 == 0) + { + log_debug("pgsetup wait --read-write: attempt %d, " + "in_recovery=%s, after %ds", + attempts + 1, + inRecovery ? "true" : "false", + elapsed); + } + + ++attempts; + pg_usleep(100 * 1000); /* 100 ms between probes */ + } + + log_info("Postgres is now accepting read-write connections on port %d", + pgSetup->pgport); + exit(EXIT_CODE_QUIT); } diff --git a/src/bin/pg_autoctl/cli_do_monitor.c b/src/bin/pg_autoctl/cli_do_monitor.c index f1dfb526c..6426946ee 100644 --- a/src/bin/pg_autoctl/cli_do_monitor.c +++ b/src/bin/pg_autoctl/cli_do_monitor.c @@ -36,7 +36,9 @@ #include "pgctl.h" #include "pgsetup.h" #include "pgsql.h" +#include "pqexpbuffer.h" #include "state.h" +#include "string_utils.h" static void cli_do_monitor_get_primary_node(int argc, char **argv); static void cli_do_monitor_get_other_nodes(int argc, char **argv); @@ -46,6 +48,10 @@ static void cli_do_monitor_register_node(int argc, char **argv); static void cli_do_monitor_node_active(int argc, char **argv); static void cli_do_monitor_version(int argc, char **argv); static void cli_do_monitor_parse_notification(int argc, char **argv); +static int cli_do_monitor_node_state_getopts(int argc, char **argv); +static void cli_do_monitor_node_state(int argc, char **argv); +static int cli_do_monitor_formation_states_getopts(int argc, char **argv); +static void cli_do_monitor_formation_states(int argc, char **argv); static CommandLine monitor_get_primary_command = @@ -88,12 +94,12 @@ static CommandLine *monitor_get_commands[] = { NULL }; -static CommandLine monitor_get_command = +CommandLine monitor_get_command = make_command_set("get", "Get information from the monitor", NULL, NULL, NULL, monitor_get_commands); -static CommandLine monitor_register_command = +CommandLine monitor_register_command = make_command("register", "Register the current node with the monitor", CLI_PGDATA_USAGE "", @@ -101,7 +107,7 @@ static CommandLine monitor_register_command = cli_getopt_pgdata, cli_do_monitor_register_node); -static CommandLine monitor_node_active_command = +CommandLine monitor_node_active_command = make_command("active", "Call in the pg_auto_failover Node Active protocol", CLI_PGDATA_USAGE, @@ -109,7 +115,7 @@ static CommandLine monitor_node_active_command = cli_getopt_pgdata, cli_do_monitor_node_active); -static CommandLine monitor_version_command = +CommandLine monitor_version_command = make_command("version", "Check that monitor version is " PG_AUTOCTL_EXTENSION_VERSION @@ -119,7 +125,7 @@ static CommandLine monitor_version_command = cli_getopt_pgdata, cli_do_monitor_version); -static CommandLine monitor_parse_notification_command = +CommandLine monitor_parse_notification_command = make_command("parse-notification", "parse a raw notification message", " ", @@ -127,6 +133,27 @@ static CommandLine monitor_parse_notification_command = NULL, cli_do_monitor_parse_notification); +CommandLine monitor_node_state_command = + make_command("node-state", + "Print reported|assigned state for a named node (for test runners)", + "--name [--state ] [--monitor ] [--timeout N]", + " --name node name to query\n" + " --state wait until reportedstate matches (requires --timeout)\n" + " --monitor monitor URI (default: local socket)\n" + " --timeout N retry for up to N seconds (default: no retry)\n", + cli_do_monitor_node_state_getopts, + cli_do_monitor_node_state); + +CommandLine monitor_formation_states_command = + make_command("formation-states", + "Exit 0 when every listed state has at least one node (for test runners)", + "[--group N] [--timeout N] [...]", + " --group N restrict to this group id\n" + " --monitor monitor URI (default: local socket)\n" + " --timeout N retry for up to N seconds (default: no retry)\n", + cli_do_monitor_formation_states_getopts, + cli_do_monitor_formation_states); + static CommandLine *monitor_subcommands[] = { &monitor_get_command, &monitor_register_command, @@ -703,3 +730,458 @@ cli_do_monitor_parse_notification(int argc, char **argv) (void) cli_pprint_json(js); } + + +/* ----------------------------------------------------------------------- + * pg_autoctl inspect monitor node-state --name + * pg_autoctl inspect monitor formation-states [--group N] [...] + * + * These two commands run ON the monitor container. They connect to the + * local PostgreSQL socket and return structured output consumed by the + * pgaftest test runner via `docker compose exec -T monitor pg_autoctl …`. + * Using dedicated subcommands (rather than shelling to psql) avoids all + * shell quoting issues. + * ----------------------------------------------------------------------- */ + +#define MONITOR_LOCAL_URI \ + "postgresql://autoctl_node@localhost/pg_auto_failover?sslmode=prefer" + +/* Shared option state for both new commands */ +static struct +{ + char monitorUri[MAXCONNINFO]; + char nodeName[_POSIX_HOST_NAME_MAX]; + char targetState[64]; /* --state: wait until reportedstate matches */ + char formation[NAMEDATALEN]; /* --formation: filter by formation */ + int groupId; /* -1 = all groups */ + int timeout; /* 0 = no retry loop, N = retry for N seconds */ +} +inspectOpts; + +static int +cli_do_monitor_node_state_getopts(int argc, char **argv) +{ + /* defaults */ + strlcpy(inspectOpts.monitorUri, MONITOR_LOCAL_URI, sizeof(inspectOpts.monitorUri)); + inspectOpts.nodeName[0] = '\0'; + inspectOpts.targetState[0] = '\0'; + inspectOpts.groupId = -1; + inspectOpts.timeout = 0; + + static struct option long_options[] = { + { "monitor", required_argument, NULL, 'm' }, + { "name", required_argument, NULL, 'n' }, + { "state", required_argument, NULL, 's' }, + { "timeout", required_argument, NULL, 't' }, + { NULL, 0, NULL, 0 } + }; + + int c; + optind = 0; + while ((c = getopt_long(argc, argv, "m:n:s:t:", long_options, NULL)) != -1) + { + switch (c) + { + case 'm': + { + strlcpy(inspectOpts.monitorUri, optarg, + sizeof(inspectOpts.monitorUri)); + break; + } + + case 'n': + { + strlcpy(inspectOpts.nodeName, optarg, + sizeof(inspectOpts.nodeName)); + break; + } + + case 's': + { + strlcpy(inspectOpts.targetState, optarg, + sizeof(inspectOpts.targetState)); + break; + } + + case 't': + { + inspectOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + default: + { + commandline_print_usage(&monitor_node_state_command, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + if (inspectOpts.nodeName[0] == '\0') + { + log_error("--name is required"); + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +/* + * cli_do_monitor_node_state connects to the local monitor and prints: + * | + * for the named node, then exits 0. Exits non-zero if the node is not + * found or the monitor is unreachable. + * + * When --timeout N is given the command retries for up to N seconds using the + * same exponential-backoff-with-jitter policy used elsewhere in pg_autoctl. + */ +static void +cli_do_monitor_node_state(int argc, char **argv) +{ + const char *sql = + "SELECT reportedstate::text || '|' || goalstate::text || '|' || health::text " + "FROM pgautofailover.node WHERE nodename = $1"; + + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1] = { inspectOpts.nodeName }; + + ConnectionRetryPolicy retryPolicy = { 0 }; + + if (inspectOpts.timeout > 0) + { + pgsql_set_retry_policy(&retryPolicy, + inspectOpts.timeout, + -1, /* unbounded attempts within timeout */ + 2000, /* cap at 2 s between attempts */ + 500); /* start at 500 ms */ + } + else + { + pgsql_set_retry_policy(&retryPolicy, 0, 0, 0, 0); /* no retry */ + } + do { + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, inspectOpts.monitorUri)) + { + /* connection failure: retry if policy allows */ + goto next_attempt; + } + + SingleValueResultContext ctx = { { 0 }, PGSQL_RESULT_STRING, false }; + + if (!pgsql_execute_with_params(&monitor.pgsql, sql, + paramCount, paramTypes, paramValues, + &ctx, &parseSingleValueResult)) + { + pgsql_finish(&monitor.pgsql); + goto next_attempt; + } + + pgsql_finish(&monitor.pgsql); + + if (!ctx.parsedOk || ctx.strVal == NULL) + { + /* node not found yet */ + if (ctx.strVal) + { + free(ctx.strVal); + } + goto next_attempt; + } + + /* + * When --state was given, check that reportedstate (the part + * before the first '|') matches the target. + * Output format: reportedstate|goalstate|health + */ + if (inspectOpts.targetState[0] != '\0') + { + char reported[64] = ""; + char *pipe = strchr(ctx.strVal, '|'); + if (pipe) + { + int len = pipe - ctx.strVal; + if (len >= (int) sizeof(reported)) + { + len = sizeof(reported) - 1; + } + memcpy(reported, ctx.strVal, len); /* IGNORE-BANNED */ + reported[len] = '\0'; + } + if (strcmp(reported, inspectOpts.targetState) != 0) + { + free(ctx.strVal); + goto next_attempt; + } + } + + fformat(stdout, "%s\n", ctx.strVal); + free(ctx.strVal); + exit(0); + +next_attempt: + if (pgsql_retry_policy_expired(&retryPolicy)) + { + break; + } + + int sleepMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); + log_debug("node-state: node \"%s\" not found yet, retrying in %d ms", + inspectOpts.nodeName, sleepMs); + pg_usleep((long) sleepMs * 1000); + } while (!pgsql_retry_policy_expired(&retryPolicy)); + + log_error("Node \"%s\" not found on the monitor", inspectOpts.nodeName); + exit(EXIT_CODE_MONITOR); +} + + +static int +cli_do_monitor_formation_states_getopts(int argc, char **argv) +{ + strlcpy(inspectOpts.monitorUri, MONITOR_LOCAL_URI, sizeof(inspectOpts.monitorUri)); + inspectOpts.formation[0] = '\0'; + inspectOpts.groupId = -1; + inspectOpts.timeout = 0; + + static struct option long_options[] = { + { "monitor", required_argument, NULL, 'm' }, + { "formation", required_argument, NULL, 'F' }, + { "group", required_argument, NULL, 'g' }, + { "timeout", required_argument, NULL, 't' }, + { NULL, 0, NULL, 0 } + }; + + int c; + optind = 0; + while ((c = getopt_long(argc, argv, "m:F:g:t:", long_options, NULL)) != -1) + { + switch (c) + { + case 'm': + { + strlcpy(inspectOpts.monitorUri, optarg, + sizeof(inspectOpts.monitorUri)); + break; + } + + case 'F': + { + strlcpy(inspectOpts.formation, optarg, + sizeof(inspectOpts.formation)); + break; + } + + case 'g': + { + inspectOpts.groupId = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + case 't': + { + inspectOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; + break; + } + + default: + { + commandline_print_usage(&monitor_formation_states_command, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + } + } + + return optind; +} + + +/* + * cli_do_monitor_formation_states checks that every state listed on the + * command line has at least one node currently in that reportedstate. + * + * Exits 0 when all states are satisfied. When --timeout N is given the + * command retries for up to N seconds using the same exponential-backoff- + * with-jitter policy used elsewhere in pg_autoctl; without it the command + * exits 1 immediately when the condition is not met. + */ +static void +cli_do_monitor_formation_states(int argc, char **argv) +{ + if (argc < 1) + { + log_error("Expected at least one state name as argument"); + commandline_print_usage(&monitor_formation_states_command, stderr); + exit(EXIT_CODE_BAD_ARGS); + } + +#define MAX_STATES 16 + if (argc > MAX_STATES) + { + log_error("Too many states (max %d)", MAX_STATES); + exit(EXIT_CODE_BAD_ARGS); + } + + /* + * Build the SQL once; the params are the same across every retry attempt. + * + * SELECT (count(*) FILTER (WHERE reportedstate::text = $1))::text + * || ',' || + * (count(*) FILTER (WHERE reportedstate::text = $2))::text + * ... + * FROM pgautofailover.node + * [WHERE groupid = $N] + */ + PQExpBufferData sqlBuf; + initPQExpBuffer(&sqlBuf); + + appendPQExpBufferStr(&sqlBuf, "SELECT "); + for (int i = 0; i < argc; i++) + { + if (i > 0) + { + appendPQExpBufferStr(&sqlBuf, " || ',' || "); + } + appendPQExpBuffer(&sqlBuf, + "(count(*) FILTER (WHERE reportedstate::text = $%d))::text", + i + 1); + } + appendPQExpBufferStr(&sqlBuf, " FROM pgautofailover.node"); + + bool hasFormation = inspectOpts.formation[0] != '\0'; + bool hasGroup = inspectOpts.groupId >= 0; + bool needWhere = hasFormation || hasGroup; + int extraParams = (hasFormation ? 1 : 0) + (hasGroup ? 1 : 0); + + if (needWhere) + { + appendPQExpBufferStr(&sqlBuf, " WHERE"); + if (hasFormation) + { + appendPQExpBuffer(&sqlBuf, " formationid = $%d", argc + 1); + } + if (hasGroup) + { + appendPQExpBuffer(&sqlBuf, "%s groupid = $%d", + hasFormation ? " AND" : "", + argc + (hasFormation ? 2 : 1)); + } + } + + int paramCount = argc + extraParams; + Oid paramTypes[MAX_STATES + 2]; + const char *paramValues[MAX_STATES + 2]; + IntString groupIdStr = { 0 }; + + for (int i = 0; i < argc; i++) + { + paramTypes[i] = TEXTOID; + paramValues[i] = argv[i]; + } + if (extraParams > 0) + { + int pi = argc; + if (hasFormation) + { + paramTypes[pi] = TEXTOID; + paramValues[pi] = inspectOpts.formation; + pi++; + } + if (hasGroup) + { + groupIdStr = intToString(inspectOpts.groupId); + paramTypes[pi] = INT4OID; + paramValues[pi] = groupIdStr.strValue; + } + } + + ConnectionRetryPolicy retryPolicy = { 0 }; + + if (inspectOpts.timeout > 0) + { + pgsql_set_retry_policy(&retryPolicy, + inspectOpts.timeout, + -1, /* unbounded attempts within timeout */ + 2000, /* cap at 2 s between attempts */ + 500); /* start at 500 ms */ + } + else + { + pgsql_set_retry_policy(&retryPolicy, 0, 0, 0, 0); /* no retry */ + } + do { + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, inspectOpts.monitorUri)) + { + goto next_attempt; + } + + SingleValueResultContext ctx = { { 0 }, PGSQL_RESULT_STRING, false }; + + if (!pgsql_execute_with_params(&monitor.pgsql, sqlBuf.data, + paramCount, paramTypes, paramValues, + &ctx, &parseSingleValueResult)) + { + pgsql_finish(&monitor.pgsql); + goto next_attempt; + } + + pgsql_finish(&monitor.pgsql); + + if (!ctx.parsedOk || ctx.strVal == NULL) + { + if (ctx.strVal) + { + free(ctx.strVal); + } + goto next_attempt; + } + + /* parse "N,N,..." — all must be >= 1 */ + bool allMet = true; + char *p = ctx.strVal; + for (int i = 0; i < argc; i++) + { + int cnt = atoi(p) /* IGNORE-BANNED */; + if (cnt < 1) + { + allMet = false; + break; + } + p = strchr(p, ','); + if (p) + { + p++; + } + else + { + break; + } + } + free(ctx.strVal); + + if (allMet) + { + termPQExpBuffer(&sqlBuf); + exit(0); + } + +next_attempt: + if (pgsql_retry_policy_expired(&retryPolicy)) + { + break; + } + + int sleepMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); + log_debug("formation-states: condition not met yet, retrying in %d ms", + sleepMs); + pg_usleep((long) sleepMs * 1000); + } while (!pgsql_retry_policy_expired(&retryPolicy)); + + termPQExpBuffer(&sqlBuf); + exit(1); +} diff --git a/src/bin/pg_autoctl/cli_do_root.c b/src/bin/pg_autoctl/cli_do_root.c index ee711d485..59e3dd288 100644 --- a/src/bin/pg_autoctl/cli_do_root.c +++ b/src/bin/pg_autoctl/cli_do_root.c @@ -185,9 +185,11 @@ CommandLine do_pgsetup_is_ready = CommandLine do_pgsetup_wait_until_ready = make_command("wait", "Wait until the local Postgres server is ready", - "[option ...]", + "[--read-write] [--timeout N] [option ...]", + " --read-write also wait until the server accepts read-write connections\n" + " --timeout N total timeout in seconds (default: 30)\n" KEEPER_CLI_WORKER_SETUP_OPTIONS, - keeper_cli_keeper_setup_getopts, + keeper_cli_pgsetup_wait_getopts, keeper_cli_pgsetup_wait_until_ready); CommandLine do_pgsetup_startup_logs = @@ -206,6 +208,14 @@ CommandLine do_pgsetup_tune = keeper_cli_keeper_setup_getopts, keeper_cli_pgsetup_tune); +CommandLine do_pgsetup_hba_lan = + make_command("hba-lan", + "Append LAN CIDR trust rules to pg_hba.conf and reload Postgres", + "[option ...]", + KEEPER_CLI_WORKER_SETUP_OPTIONS, + keeper_cli_keeper_setup_getopts, + keeper_cli_pgsetup_hba_lan); + CommandLine *do_pgsetup[] = { &do_pgsetup_pg_ctl, &do_pgsetup_discover, @@ -213,6 +223,7 @@ CommandLine *do_pgsetup[] = { &do_pgsetup_wait_until_ready, &do_pgsetup_startup_logs, &do_pgsetup_tune, + &do_pgsetup_hba_lan, NULL }; @@ -374,25 +385,255 @@ CommandLine do_tmux_commands = "Set of facilities to handle tmux interactive sessions", NULL, NULL, NULL, do_tmux); +/* + * Azure integration has been removed. The commands that were here are no + * longer maintained and have been deleted from this file. See pgaftest for + * the replacement QA tooling. + */ + +#if 0 /* REMOVED: azure commands — see pgaftest instead */ +CommandLine do_azure_provision_region = + make_command("region", + "Provision an azure region: resource group, network, VMs", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n" + " --location azure location where to create a resource group\n" + " --monitor should we create a monitor in the region (false)\n" + " --nodes number of Postgres nodes to create (2)\n" + " --script output a shell script instead of creating resources\n", + cli_do_azure_getopts, + cli_do_azure_create_region); + +CommandLine do_azure_provision_nodes = + make_command("nodes", + "Provision our pre-created VM with pg_autoctl Postgres nodes", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n" + " --monitor should we create a monitor in the region (false)\n" + " --nodes number of Postgres nodes to create (2)\n" + " --script output a shell script instead of creating resources\n", + cli_do_azure_getopts, + cli_do_azure_create_nodes); + + +CommandLine *do_azure_provision[] = { + &do_azure_provision_region, + &do_azure_provision_nodes, + NULL +}; + +CommandLine do_azure_provision_commands = + make_command_set("provision", + "provision azure resources for a pg_auto_failover demo", + NULL, NULL, NULL, do_azure_provision); + +CommandLine do_azure_create = + make_command("create", + "Create an azure QA environment", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n" + " --location azure location to use for the resources\n" + " --nodes number of Postgres nodes to create (2)\n" + " --script output a script instead of creating resources\n" + " --no-monitor do not create the pg_autoctl monitor node\n" + " --no-app do not create the application node\n" + " --cidr use the 10.CIDR.CIDR.0/24 subnet (11)\n" + " --from-source provision pg_auto_failover from sources\n", + cli_do_azure_getopts, + cli_do_azure_create_environment); + +CommandLine do_azure_drop = + make_command("drop", + "Drop an azure QA environment: resource group, network, VMs", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n" + " --location azure location where to create a resource group\n" + " --monitor should we create a monitor in the region (false)\n" + " --nodes number of Postgres nodes to create (2)\n" + " --script output a shell script instead of creating resources\n", + cli_do_azure_getopts, + cli_do_azure_drop_region); + +CommandLine do_azure_deploy = + make_command("deploy", + "Deploy a pg_autoctl VMs, given by name", + "[option ...] vmName", + "", + cli_do_azure_getopts, + cli_do_azure_deploy); + +CommandLine do_azure_show_ips = + make_command("ips", + "Show public and private IP addresses for selected VMs", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n", + cli_do_azure_getopts, + cli_do_azure_show_ips); + +CommandLine do_azure_show_state = + make_command("state", + "Connect to the monitor node to show the current state", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n" + " --watch run the command again every 0.2s\n", + cli_do_azure_getopts, + cli_do_azure_show_state); + +CommandLine *do_azure_show[] = { + &do_azure_show_ips, + &do_azure_show_state, + NULL +}; + +CommandLine do_azure_show_commands = + make_command_set("show", + "show azure resources for a pg_auto_failover demo", + NULL, NULL, NULL, do_azure_show); + +CommandLine do_azure_ls = + make_command("ls", + "List resources in a given azure region", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n", + cli_do_azure_getopts, + cli_do_azure_ls); + +CommandLine do_azure_ssh = + make_command("ssh", + "Runs ssh -l ha-admin for a given VM name", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region name to use for referencing the region\n", + cli_do_azure_getopts, + cli_do_azure_ssh); + +CommandLine do_azure_sync = + make_command("sync", + "Rsync pg_auto_failover sources on all the target region VMs", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region region to use for referencing the region\n" + " --monitor should we create a monitor in the region (false)\n" + " --nodes number of Postgres nodes to create (2)\n", + cli_do_azure_getopts, + cli_do_azure_rsync); + +CommandLine do_azure_tmux_session = + make_command("session", + "Create or attach a tmux session for the created Azure VMs", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region region to use for referencing the region\n" + " --monitor should we create a monitor in the region (false)\n" + " --nodes number of Postgres nodes to create (2)\n", + cli_do_azure_getopts, + cli_do_azure_tmux_session); + +CommandLine do_azure_tmux_kill = + make_command("kill", + "Kill an existing tmux session for Azure VMs", + "[option ...]", + " --prefix azure group name prefix (ha-demo)\n" + " --region region to use for referencing the region\n" + " --monitor should we create a monitor in the region (false)\n" + " --nodes number of Postgres nodes to create (2)\n", + cli_do_azure_getopts, + cli_do_azure_tmux_kill); + +CommandLine *do_azure_tmux[] = { + &do_azure_tmux_session, + &do_azure_tmux_kill, + NULL +}; + +CommandLine do_azure_tmux_commands = + make_command_set("tmux", + "Run a tmux session with an Azure setup for QA/testing", + NULL, NULL, NULL, do_azure_tmux); + +CommandLine *do_azure[] = { + &do_azure_provision_commands, + &do_azure_tmux_commands, + &do_azure_show_commands, + &do_azure_deploy, + &do_azure_create, + &do_azure_drop, + &do_azure_ls, + &do_azure_ssh, + &do_azure_sync, + NULL +}; + +CommandLine do_azure_commands = + make_command_set("azure", + "Manage a set of Azure resources for a pg_auto_failover demo", + NULL, NULL, NULL, do_azure); + +#endif /* REMOVED: azure commands */ + +/* + * pg_autoctl internal service postgres|listener|node-active + * + * These are the subprocess entry points used by the supervisor (pg_autoctl run + * and pg_autoctl create … --run). The supervisor fork()s and then execv()s + * the pg_autoctl binary itself with one of these sub-commands so that each + * service runs in its own address space. + * + * Using fork+exec (rather than fork alone) is a deliberate design choice for + * live upgrades: when a child process exits with an incompatible monitor + * extension version, the supervisor restarts it via fork()+execv(), which loads + * the current binary from disk. If the binary has been updated in place (e.g. + * by a package manager), the restarted child automatically picks up the new + * version without touching the supervisor process — making pg_autoctl safe to + * use as PID 1 in Docker/Kubernetes containers where replacing the binary and + * sending SIGTERM would lose the container. + * + * See also: keeper.c keeper_check_monitor_extension_version(), which exits on + * version mismatch precisely to trigger this restart-with-new-binary path. + * + * These commands are hidden from --help output (make_hidden_command_set) so + * operators do not accidentally invoke them directly. + * Use "pg_autoctl manual service" for the user-facing controls (restart, pgctl). + * Use "pg_autoctl inspect getpid" to read sub-process PIDs. + */ +static CommandLine *internal_service_subcommands[] = { + &service_pgcontroller, /* debug: supervisor for just the postgres controller */ + &service_postgres, /* spawned by service_postgres_ctl_start() */ + &service_monitor_listener, /* spawned by service_monitor_start() */ + &service_node_active, /* spawned by service_keeper_start() */ + NULL +}; + +static CommandLine internal_service_commands = + make_hidden_command_set("service", + "Subprocess entry points for the pg_autoctl supervisor", + NULL, NULL, NULL, internal_service_subcommands); + +/* + * pg_autoctl internal + * + * Hidden from --help; routable so the supervisor's execv() calls work. + * Contains only what the supervisor spawns plus dev/QA tooling not yet + * moved to pgaftest (tmux, demo). + */ CommandLine *do_subcommands[] = { - &do_monitor_commands, - &do_coordinator_commands, - &do_fsm_commands, - &do_primary_, - &do_standby_, - &do_show_commands, - &do_pgsetup_commands, - &do_service_postgres_ctl_commands, - &do_service_commands, + &internal_service_commands, &do_tmux_commands, &do_demo_commands, NULL }; CommandLine do_commands = - make_command_set("do", - "Internal commands and internal QA tooling", NULL, NULL, - NULL, do_subcommands); + make_hidden_command_set("internal", + "Internal subprocess entry points — not for direct use", + NULL, NULL, NULL, do_subcommands); /* diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index 4de118c88..9156fce31 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -16,13 +16,44 @@ /* src/bin/pg_autoctl/cli_do_fsm.c */ extern CommandLine do_fsm_commands; +/* read-only sub-commands exposed via "pg_autoctl inspect fsm" */ +extern CommandLine fsm_state; +extern CommandLine fsm_node_state; +extern CommandLine fsm_list; +extern CommandLine fsm_gv; + +/* mutating sub-commands exposed via "pg_autoctl manual fsm" */ +extern CommandLine fsm_init; +extern CommandLine fsm_assign; +extern CommandLine fsm_step; +extern CommandLine fsm_nodes; /* nodes get + nodes set — kept together in manual */ + /* src/bin/pg_autoctl/cli_do_monitor.c */ extern CommandLine do_monitor_commands; +/* read-only sub-commands exposed via "pg_autoctl inspect monitor" */ +extern CommandLine monitor_get_command; +extern CommandLine monitor_parse_notification_command; +extern CommandLine monitor_node_state_command; +extern CommandLine monitor_formation_states_command; + +/* mutating sub-commands exposed via "pg_autoctl manual monitor" */ +extern CommandLine monitor_register_command; +extern CommandLine monitor_node_active_command; +extern CommandLine monitor_version_command; + /* src/bin/pg_autoctl/cli_do_service.c */ extern CommandLine do_service_commands; +extern CommandLine do_service_getpid_commands; +extern CommandLine do_service_restart_commands; extern CommandLine do_service_postgres_ctl_commands; +/* subprocess entry points (spawned by the supervisor via fork+exec) */ +extern CommandLine service_pgcontroller; +extern CommandLine service_postgres; +extern CommandLine service_monitor_listener; +extern CommandLine service_node_active; + /* src/bin/pg_autoctl/cli_do_show.c */ extern CommandLine do_show_commands; extern CommandLine do_pgsetup_commands; @@ -86,9 +117,11 @@ void keeper_cli_disable_synchronous_replication(int argc, char **argv); void keeper_cli_pgsetup_pg_ctl(int argc, char **argv); void keeper_cli_pgsetup_discover(int argc, char **argv); void keeper_cli_pgsetup_is_ready(int argc, char **argv); +int keeper_cli_pgsetup_wait_getopts(int argc, char **argv); void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv); void keeper_cli_pgsetup_startup_logs(int argc, char **argv); void keeper_cli_pgsetup_tune(int argc, char **argv); +void keeper_cli_pgsetup_hba_lan(int argc, char **argv); void keeper_cli_add_default_settings(int argc, char **argv); void keeper_cli_create_monitor_user(int argc, char **argv); diff --git a/src/bin/pg_autoctl/cli_do_service.c b/src/bin/pg_autoctl/cli_do_service.c index 94420b7c5..e450417ca 100644 --- a/src/bin/pg_autoctl/cli_do_service.c +++ b/src/bin/pg_autoctl/cli_do_service.c @@ -357,8 +357,8 @@ cli_do_service_restart_node_active(int argc, char **argv) * tree. It is used for debug purposes only. When using this entry point we * have a supervisor process that is responsible for only one service: * - * pg_autoctl do service pgcontroller - * - pg_autoctl do service postgres + * pg_autoctl internal service pgcontroller + * - pg_autoctl internal service postgres * - postgres */ static void @@ -401,9 +401,9 @@ cli_do_service_pgcontroller(int argc, char **argv) * that supervises two sub-processes, one of them is cli_do_service_postgres: * * pg_autoctl - * - pg_autoctl do service postgres + * - pg_autoctl internal service postgres * - postgres - * - pg_autoctl do service keeper|monitor + * - pg_autoctl internal service keeper|monitor */ static void cli_do_service_postgres(int argc, char **argv) diff --git a/src/bin/pg_autoctl/cli_inspect.c b/src/bin/pg_autoctl/cli_inspect.c new file mode 100644 index 000000000..4b156c660 --- /dev/null +++ b/src/bin/pg_autoctl/cli_inspect.c @@ -0,0 +1,79 @@ +/* + * src/bin/pg_autoctl/cli_inspect.c + * pg_autoctl inspect — read-only diagnostics, always visible. + * + * All commands here read local or cluster state without mutating anything. + * Safe to run at any time, even while `pg_autoctl run` is active. + * + * For mutating recovery commands (fsm assign, standby promote, etc.) see + * cli_manual.c ("pg_autoctl manual …"). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include "commandline.h" +#include "cli_inspect.h" +#include "cli_do_root.h" + +/* + * Read-only FSM sub-commands: display the current state, list reachable + * transitions, or dump the full FSM as a graphviz .gv file. + * Mutating operations (init, assign, step, nodes set) live under "manual fsm". + */ +static CommandLine *inspect_fsm_subcommands[] = { + &fsm_state, + &fsm_node_state, + &fsm_list, + &fsm_gv, + NULL +}; + +static CommandLine inspect_fsm_commands = + make_command_set("fsm", + "Display keeper FSM state and transitions (read-only)", + NULL, NULL, NULL, inspect_fsm_subcommands); + +/* + * Read-only monitor sub-commands: get primary/others/candidate-count/coordinator + * and parse-notification. We intentionally exclude "register", "active", and + * "version" (ALTER EXTENSION) which are mutating — those live under "manual". + */ +static CommandLine *inspect_monitor_subcommands[] = { + &monitor_get_command, + &monitor_parse_notification_command, + &monitor_node_state_command, + &monitor_formation_states_command, + NULL +}; + +static CommandLine inspect_monitor_commands = + make_command_set("monitor", + "Query the monitor's current state (read-only)", + NULL, NULL, NULL, inspect_monitor_subcommands); + +/* + * Aggregate the read-only do_* command sets under "inspect". + * + * service here is only "getpid" — inspecting PIDs of running sub-processes. + * "restart" and "pgctl on/off" mutate state and belong under "manual service". + */ +static CommandLine *inspect_subcommands[] = { + &do_show_commands, /* ipaddr / cidr / lookup / hostname / reverse */ + &do_pgsetup_commands, /* discover / ready / wait / logs / tune / pg_ctl */ + &inspect_fsm_commands, /* state / list / gv */ + &inspect_monitor_commands, /* get primary|others|candidate-count + parse-notification */ + &do_service_getpid_commands, /* getpid postgres|listener|node-active */ + NULL +}; + +CommandLine inspect_commands = + make_command_set("inspect", + "Read-only diagnostics (safe on live nodes)", + "[sub-command]", + " show Networking and hostname diagnostics\n" + " pgsetup Local PostgreSQL setup inspection\n" + " fsm Display keeper FSM state and reachable transitions\n" + " monitor Query the monitor's current state\n" + " getpid Get PIDs of pg_autoctl sub-processes\n", + NULL, inspect_subcommands); diff --git a/src/bin/pg_autoctl/cli_inspect.h b/src/bin/pg_autoctl/cli_inspect.h new file mode 100644 index 000000000..b9c87d68e --- /dev/null +++ b/src/bin/pg_autoctl/cli_inspect.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_inspect.h + * pg_autoctl inspect — read-only diagnostics. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_INSPECT_H +#define CLI_INSPECT_H + +#include "commandline.h" + +extern CommandLine inspect_commands; + +#endif /* CLI_INSPECT_H */ diff --git a/src/bin/pg_autoctl/cli_manual.c b/src/bin/pg_autoctl/cli_manual.c new file mode 100644 index 000000000..f309a948e --- /dev/null +++ b/src/bin/pg_autoctl/cli_manual.c @@ -0,0 +1,128 @@ +/* + * src/bin/pg_autoctl/cli_manual.c + * pg_autoctl manual — operator-driven FSM operations and low-level controls. + * + * These commands let an operator manually drive individual FSM transitions + * or low-level operations that pg_autoctl automation would normally own. + * Intended for manual recovery when the automated FSM is stopped or stuck, + * or for driving a live upgrade step-by-step. + * + * "manual" is the antonym of the automated behaviour pg_autoctl normally + * provides: you are doing by hand what the system would otherwise do for you. + * + * Read-only diagnostics belong in "pg_autoctl inspect": + * inspect fsm state / list / gv + * inspect monitor get / parse-notification + * inspect getpid postgres / listener / node-active + * + * Internal subprocess entry points belong in the hidden "pg_autoctl do". + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include "commandline.h" +#include "cli_manual.h" +#include "cli_do_root.h" + +/* + * manual fsm: mutating FSM operations only. + * + * Read-only FSM commands (state, list, gv) live under "pg_autoctl inspect fsm" + * because they are safe to run on a live node. + * + * nodes (get + set) is kept as a unit under manual because both sub-commands + * target the offline nodes file (--disable-monitor), making them equally + * operator-oriented regardless of read/write direction. + */ +static CommandLine *manual_fsm_subcommands[] = { + &fsm_init, + &fsm_assign, + &fsm_step, + &fsm_nodes, /* nodes get + nodes set (--disable-monitor file) */ + NULL +}; + +static CommandLine manual_fsm_commands = + make_command_set("fsm", + "Manually drive the keeper FSM (mutating operations)", + NULL, NULL, NULL, manual_fsm_subcommands); + +/* + * manual service: user-visible service controls only. + * + * getpid is read-only and lives under "pg_autoctl inspect getpid". + * + * Intentionally excludes the internal subprocess entry points + * (pgcontroller / postgres / listener / node-active) which are spawned by the + * supervisor via fork+exec and are not meant for direct operator use. + * Those live under the hidden "pg_autoctl internal service" group. + */ +static CommandLine *manual_service_subcommands[] = { + &do_service_restart_commands, /* restart postgres|listener|node-active */ + &do_service_postgres_ctl_commands, /* pgctl on|off */ + NULL +}; + +static CommandLine manual_service_commands = + make_command_set("service", + "Restart pg_autoctl sub-processes or signal the postgres controller", + NULL, NULL, NULL, manual_service_subcommands); + +/* + * manual monitor: mutating monitor operations. + * + * "get" and "parse-notification" are read-only and live under + * "pg_autoctl inspect monitor". Here we expose only the mutating calls: + * + * register — manually step through the node registration protocol + * active — manually call the node_active RPC on the monitor + * version — check monitor extension version and ALTER EXTENSION UPDATE if needed + */ +static CommandLine *manual_monitor_subcommands[] = { + &monitor_register_command, + &monitor_node_active_command, + &monitor_version_command, + NULL +}; + +static CommandLine manual_monitor_commands = + make_command_set("monitor", + "Manually drive monitor RPCs (register / active / version)", + NULL, NULL, NULL, manual_monitor_subcommands); + +/* + * manual coordinator: Citus coordinator metadata management. + * + * These commands let an operator manually replay the coordinator update steps + * that pg_autoctl normally drives automatically during a Citus worker failover: + * + * coordinator update prepare — call master_update_node() in a prepared + * transaction, which blocks shard writes while open + * coordinator update commit — COMMIT PREPARED: makes the new address permanent + * coordinator update rollback — ROLLBACK PREPARED: abandons a stuck prepare + * + * They are typically needed when the coordinator was unavailable during a + * failover and the prepared transaction needs to be resolved out-of-band. + */ +static CommandLine *manual_subcommands[] = { + &manual_fsm_commands, /* init / assign / step / nodes */ + &manual_service_commands, /* restart / pgctl on|off */ + &manual_monitor_commands, /* register / active / version */ + &do_primary_, /* slot create|drop / adduser monitor|replica / defaults / identify */ + &do_standby_, /* init / rewind / crash-recovery / promote */ + &do_coordinator_commands, /* add / activate / remove / update prepare|commit|rollback */ + NULL +}; + +CommandLine manual_commands = + make_command_set("manual", + "Manual FSM operations — drive by hand what automation normally does", + "[sub-command]", + " fsm Manually drive keeper FSM transitions\n" + " service Restart sub-processes or signal the postgres controller\n" + " monitor Manually drive monitor registration protocol\n" + " primary Manual primary-side PostgreSQL operations\n" + " standby Manual standby-side PostgreSQL operations\n" + " coordinator Citus coordinator metadata management\n", + NULL, manual_subcommands); diff --git a/src/bin/pg_autoctl/cli_manual.h b/src/bin/pg_autoctl/cli_manual.h new file mode 100644 index 000000000..d54ec7abd --- /dev/null +++ b/src/bin/pg_autoctl/cli_manual.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_manual.h + * pg_autoctl manual — operator-driven FSM operations and low-level controls. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_MANUAL_H +#define CLI_MANUAL_H + +#include "commandline.h" + +extern CommandLine manual_commands; + +#endif /* CLI_MANUAL_H */ diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 386716b98..83c26ea62 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -9,6 +9,9 @@ */ #include "cli_common.h" +#include "cli_do_root.h" +#include "cli_inspect.h" +#include "cli_manual.h" #include "cli_root.h" #include "commandline.h" @@ -81,18 +84,24 @@ CommandLine drop_commands = NULL, drop_subcommands); /* - * Binding them all into the top-level command: + * Single root command table — inspect and override are always visible. + * The PG_AUTOCTL_DEBUG env var now only controls log verbosity, not + * command visibility. */ -CommandLine *root_subcommands_with_debug[] = { +CommandLine *root_subcommands[] = { &create_commands, &drop_commands, &config_commands, - &show_commands_with_debug, + &show_commands, &enable_commands, &disable_commands, &get_commands, &set_commands, &perform_commands, + &activate_node_command, + &inspect_commands, + &manual_commands, + &do_commands, &service_run_command, &watch_command, @@ -104,14 +113,15 @@ CommandLine *root_subcommands_with_debug[] = { NULL }; -CommandLine root_with_debug = +CommandLine root = make_command_set("pg_autoctl", "pg_auto_failover control tools and service", - "[ --debug|verbose|quiet ]", NULL, - root_options, root_subcommands_with_debug); - + "[ --verbose --quiet ]", NULL, + root_options, root_subcommands); -CommandLine *root_subcommands[] = { +/* Backward-compat alias: root_with_debug and root_subcommands_with_debug + * previously held the do_commands. They now point to the same root. */ +CommandLine *root_subcommands_with_debug[] = { &create_commands, &drop_commands, &config_commands, @@ -122,6 +132,10 @@ CommandLine *root_subcommands[] = { &set_commands, &perform_commands, &activate_node_command, + &inspect_commands, + &manual_commands, + + &do_commands, &service_run_command, &watch_command, &service_stop_command, @@ -132,11 +146,11 @@ CommandLine *root_subcommands[] = { NULL }; -CommandLine root = +CommandLine root_with_debug = make_command_set("pg_autoctl", "pg_auto_failover control tools and service", "[ --verbose --quiet ]", NULL, - root_options, root_subcommands); + root_options, root_subcommands_with_debug); /* diff --git a/src/bin/pg_autoctl/service_keeper.c b/src/bin/pg_autoctl/service_keeper.c index 5c3d4b557..3b563e211 100644 --- a/src/bin/pg_autoctl/service_keeper.c +++ b/src/bin/pg_autoctl/service_keeper.c @@ -170,7 +170,7 @@ service_keeper_runprogram(Keeper *keeper) setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "node-active"; args[argsIndex++] = "--pgdata"; diff --git a/src/bin/pg_autoctl/service_monitor.c b/src/bin/pg_autoctl/service_monitor.c index e2a7f4e3a..b7ef22e6c 100644 --- a/src/bin/pg_autoctl/service_monitor.c +++ b/src/bin/pg_autoctl/service_monitor.c @@ -160,7 +160,7 @@ service_monitor_runprogram(Monitor *monitor) setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "listener"; args[argsIndex++] = "--pgdata"; diff --git a/src/bin/pg_autoctl/service_postgres_ctl.c b/src/bin/pg_autoctl/service_postgres_ctl.c index 82daa6863..f85cf04a2 100644 --- a/src/bin/pg_autoctl/service_postgres_ctl.c +++ b/src/bin/pg_autoctl/service_postgres_ctl.c @@ -92,7 +92,7 @@ service_postgres_ctl_start(void *context, pid_t *pid) /* * service_postgres_ctl_runprogram runs the postgres controller service: * - * $ pg_autoctl do service postgres --pgdata ... + * $ pg_autoctl internal service postgres --pgdata ... */ void service_postgres_ctl_runprogram() @@ -129,7 +129,7 @@ service_postgres_ctl_runprogram() setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "postgres"; args[argsIndex++] = "--pgdata"; diff --git a/tests/pgautofailover_utils.py b/tests/pgautofailover_utils.py index 0fe9f6a53..7aef6ecfe 100644 --- a/tests/pgautofailover_utils.py +++ b/tests/pgautofailover_utils.py @@ -520,12 +520,12 @@ def reload_postgres(self): def restart_postgres(self): """ - Restart Postgres with pg_autoctl do service restart postgres + Restart Postgres with pg_autoctl manual service restart postgres """ command = PGAutoCtl(self) command.execute( - "service restart postgres", "do", "service", "restart", "postgres" + "service restart postgres", "manual", "service", "restart", "postgres" ) def pg_is_running(self, timeout=COMMAND_TIMEOUT): @@ -535,7 +535,7 @@ def pg_is_running(self, timeout=COMMAND_TIMEOUT): command = PGAutoCtl(self) try: - command.execute("pgsetup ready", "do", "pgsetup", "ready", "-vvv") + command.execute("pgsetup ready", "inspect", "pgsetup", "ready", "-vvv") except Exception as e: # pg_autoctl uses EXIT_CODE_PGSQL when Postgres is not ready return False @@ -547,7 +547,7 @@ def wait_until_pg_is_running(self, timeout=STATE_CHANGE_TIMEOUT): """ command = PGAutoCtl(self) out, err, ret = command.execute( - "pgsetup ready", "do", "pgsetup", "wait", "-vvv" + "pgsetup ready", "inspect", "pgsetup", "wait", "-vvv" ) return ret == 0 @@ -1217,7 +1217,7 @@ def get_nodeid(self): Fetch the nodeid from the pg_autoctl state file. """ command = PGAutoCtl(self) - out, err, ret = command.execute("get node id", "do", "fsm", "state") + out, err, ret = command.execute("get node id", "inspect", "fsm", "state") self.state = json.loads(out) return self.state["state"]["nodeId"] @@ -1242,7 +1242,7 @@ def get_local_state(self): """ command = PGAutoCtl(self) out, err, ret = command.execute( - "get node id", "-vv", "do", "fsm", "state" + "get node id", "-vv", "inspect", "fsm", "state" ) self.state = json.loads(out) @@ -1412,19 +1412,19 @@ def drop(self): def do_fsm_assign(self, target_state): """ - Runs `pg_autoctl do fsm assign` on a node + Runs `pg_autoctl manual fsm assign` on a node :return: """ command = PGAutoCtl(self) command.execute( - "do fsm assign", "-vv", "do", "fsm", "assign", target_state + "manual fsm assign", "-vv", "manual", "fsm", "assign", target_state ) return True def do_fsm_nodes_set(self, nodesArray): """ - Runs `pg_autoctl do fsm nodes set` on a node + Runs `pg_autoctl manual fsm nodes set` on a node :return: """ @@ -1435,18 +1435,18 @@ def do_fsm_nodes_set(self, nodesArray): command = PGAutoCtl(self) out, err, ret = command.execute( - "do fsm nodes set", "do", "fsm", "nodes", "set", filename + "manual fsm nodes set", "manual", "fsm", "nodes", "set", filename ) return True def do_fsm_step(self): """ - Runs `pg_autoctl do fsm step` on a node + Runs `pg_autoctl manual fsm step` on a node :return: """ command = PGAutoCtl(self) - command.execute("do fsm step", "do", "fsm", "step") + command.execute("manual fsm step", "manual", "fsm", "step") return True def set_metadata(self, name=None, host=None, port=None): From 229337206d4c9e3568ed7999a8e0b51fe6b32381 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 6 Jul 2026 22:37:08 +0200 Subject: [PATCH 2/3] refactor: delete root_with_debug, add internal service commands, un-static fsm/monitor symbols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove the PG_AUTOCTL_DEBUG command-dispatch branch: root_with_debug and root_subcommands_with_debug are identical to root now that inspect/manual are always-visible. Delete them from cli_root.c and cli_root.h; remove the env_exists(PG_AUTOCTL_DEBUG) gate from main.c and cli_common.c. - Add pg_autoctl internal service (hidden) — the supervisor spawns subprocess entry points via fork+exec as: pg_autoctl internal service postgres|listener|node-active --pgdata ... make_hidden_command_set keeps these out of --help output. Replaces the old 'pg_autoctl do service' path that was broken when 'do' was renamed to 'internal'. - Export fsm_init/state/list/gv/assign/step and monitor_get_command/ parse_notification/register/node_active/version from their respective cli_do_*.c files (remove static) so cli_inspect.c and cli_manual.c can compose them into the inspect/manual sub-command trees. - Drop fsm_node_state and monitor_node_state/formation_states from cli_inspect.c — these symbols exist only in pgaftest-infra, not origin/main. --- src/bin/pg_autoctl/cli_common.c | 5 - src/bin/pg_autoctl/cli_do_fsm.c | 165 +--------- src/bin/pg_autoctl/cli_do_misc.c | 237 +------------- src/bin/pg_autoctl/cli_do_monitor.c | 482 ---------------------------- src/bin/pg_autoctl/cli_do_root.c | 278 ++-------------- src/bin/pg_autoctl/cli_do_root.h | 45 +-- src/bin/pg_autoctl/cli_inspect.c | 3 - src/bin/pg_autoctl/cli_root.c | 34 +- src/bin/pg_autoctl/cli_root.h | 3 - src/bin/pg_autoctl/main.c | 9 - 10 files changed, 61 insertions(+), 1200 deletions(-) diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index d576cdef9..4b835ab28 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -1422,11 +1422,6 @@ keeper_cli_help(int argc, char **argv) { CommandLine command = root; - if (env_exists(PG_AUTOCTL_DEBUG)) - { - command = root_with_debug; - } - (void) commandline_print_command_tree(&command, stdout); } diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index 4294aceb9..02fbfa337 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -29,8 +29,6 @@ static void cli_do_fsm_init(int argc, char **argv); static void cli_do_fsm_state(int argc, char **argv); -static int cli_do_fsm_node_state_getopts(int argc, char **argv); -static void cli_do_fsm_node_state(int argc, char **argv); static void cli_do_fsm_list(int argc, char **argv); static void cli_do_fsm_gv(int argc, char **argv); static void cli_do_fsm_assign(int argc, char **argv); @@ -39,7 +37,14 @@ static void cli_do_fsm_step(int argc, char **argv); static void cli_do_fsm_get_nodes(int argc, char **argv); static void cli_do_fsm_set_nodes(int argc, char **argv); -/* read-only — exported for pg_autoctl inspect fsm */ +CommandLine fsm_init = + make_command("init", + "Initialize the keeper's state on-disk", + CLI_PGDATA_USAGE, + CLI_PGDATA_OPTION, + cli_getopt_pgdata, + cli_do_fsm_init); + CommandLine fsm_state = make_command("state", "Read the keeper's state from disk and display it", @@ -48,16 +53,6 @@ CommandLine fsm_state = cli_getopt_pgdata, cli_do_fsm_state); -CommandLine fsm_node_state = - make_command("node-state", - "Poll the keeper's on-disk current_role; exit 0 when it matches --state", - CLI_PGDATA_USAGE, - " --pgdata path to the keeper's data directory\n" - " --state target state to wait for (e.g. demote_timeout)\n" - " --timeout seconds to retry (default: no retry)\n", - cli_do_fsm_node_state_getopts, - cli_do_fsm_node_state); - CommandLine fsm_list = make_command("list", "List reachable FSM states from current state", @@ -71,15 +66,6 @@ CommandLine fsm_gv = "Output the FSM as a .gv program suitable for graphviz/dot", "", NULL, NULL, cli_do_fsm_gv); -/* mutating — exported for pg_autoctl manual fsm */ -CommandLine fsm_init = - make_command("init", - "Initialize the keeper's state on-disk", - CLI_PGDATA_USAGE, - CLI_PGDATA_OPTION, - cli_getopt_pgdata, - cli_do_fsm_init); - CommandLine fsm_assign = make_command("assign", "Assign a new goal state to the keeper", @@ -242,141 +228,6 @@ cli_do_fsm_state(int argc, char **argv) } -/* option state for fsm node-state */ -static struct -{ - char targetState[64]; - int timeout; -} -fsmNodeStateOpts; - -static int -cli_do_fsm_node_state_getopts(int argc, char **argv) -{ - fsmNodeStateOpts.targetState[0] = '\0'; - fsmNodeStateOpts.timeout = 0; - - static struct option long_options[] = { - { "pgdata", required_argument, NULL, 'D' }, - { "state", required_argument, NULL, 's' }, - { "timeout", required_argument, NULL, 't' }, - { NULL, 0, NULL, 0 } - }; - - int c; - optind = 0; - while ((c = getopt_long(argc, argv, "D:s:t:", long_options, NULL)) != -1) - { - switch (c) - { - case 'D': - { - strlcpy(keeperOptions.pgSetup.pgdata, optarg, - sizeof(keeperOptions.pgSetup.pgdata)); - break; - } - - case 's': - { - strlcpy(fsmNodeStateOpts.targetState, optarg, - sizeof(fsmNodeStateOpts.targetState)); - break; - } - - case 't': - { - fsmNodeStateOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; - break; - } - - default: - { - commandline_print_usage(&fsm_node_state, stderr); - exit(EXIT_CODE_BAD_ARGS); - } - } - } - - /* fall back to $PGDATA when --pgdata was not given */ - cli_common_get_set_pgdata_or_exit(&keeperOptions.pgSetup); - - if (!keeper_config_set_pathnames_from_pgdata(&keeperOptions.pathnames, - keeperOptions.pgSetup.pgdata)) - { - exit(EXIT_CODE_BAD_ARGS); - } - - return optind; -} - - -/* - * cli_do_fsm_node_state reads the keeper's on-disk state and prints: - * | - * - * When --state is given the command retries until current_role matches. - * When --timeout is given the retry loop stops after N seconds. - */ -static void -cli_do_fsm_node_state(int argc, char **argv) -{ - KeeperConfig config = keeperOptions; - - bool missingPgdataIsOk = true; - bool pgIsNotRunningIsOk = true; - bool monitorDisabledIsOk = true; - - if (!keeper_config_read_file(&config, - missingPgdataIsOk, - pgIsNotRunningIsOk, - monitorDisabledIsOk)) - { - exit(EXIT_CODE_BAD_CONFIG); - } - - time_t deadline = (fsmNodeStateOpts.timeout > 0) - ? time(NULL) + fsmNodeStateOpts.timeout - : 0; - - for (;;) - { - Keeper keeper = { 0 }; - - if (!keeper_init(&keeper, &config)) - { - if (deadline && time(NULL) < deadline) - { - pg_usleep(500 * 1000); - continue; - } - exit(EXIT_CODE_BAD_CONFIG); - } - - KeeperStateData *s = &keeper.state; - const char *current = NodeStateToString(s->current_role); - const char *assigned = NodeStateToString(s->assigned_role); - - if (fsmNodeStateOpts.targetState[0] != '\0' && - strcmp(current, fsmNodeStateOpts.targetState) != 0) - { - log_debug("fsm node-state: current_role=%s, waiting for %s", - current, fsmNodeStateOpts.targetState); - if (deadline && time(NULL) < deadline) - { - pg_usleep(500 * 1000); - continue; - } - - /* timed out or --timeout 0 with no match */ - exit(EXIT_CODE_INTERNAL_ERROR); - } - - fformat(stdout, "%s|%s\n", current, assigned); - exit(0); - } -} - - /* * cli_do_fsm_list lists reachable states from the current one. */ diff --git a/src/bin/pg_autoctl/cli_do_misc.c b/src/bin/pg_autoctl/cli_do_misc.c index 68e144a95..188faa88f 100644 --- a/src/bin/pg_autoctl/cli_do_misc.c +++ b/src/bin/pg_autoctl/cli_do_misc.c @@ -38,10 +38,6 @@ #include "primary_standby.h" #include "string_utils.h" -/* Options specific to "pg_autoctl inspect pgsetup wait" */ -static bool pgsetupWaitReadWrite = false; -static int pgsetupWaitTimeout = 30; - /* * keeper_cli_create_replication_slot implements the CLI to create a replication @@ -327,149 +323,18 @@ keeper_cli_pgsetup_is_ready(int argc, char **argv) /* - * keeper_cli_pgsetup_wait_getopts parses options specific to - * "pg_autoctl inspect pgsetup wait": --read-write, --timeout, plus the - * standard Postgres connection options inherited from the keeper setup. - */ -int -keeper_cli_pgsetup_wait_getopts(int argc, char **argv) -{ - /* - * cli_common_keeper_getopts (called by keeper_cli_keeper_setup_getopts) - * exits with BAD_ARGS when it encounters unknown options. Since --timeout - * and --read-write are not in its options list, we must remove them from - * argv before delegating. - * - * Strategy: - * 1. Scan argv with opterr=0 to capture --timeout / --read-write. - * 2. Build a filtered argv that omits those two options. - * 3. Pass the filtered argv to keeper_cli_keeper_setup_getopts. - */ - - /* Reset module-level wait options */ - pgsetupWaitReadWrite = false; - pgsetupWaitTimeout = 30; - - static struct option wait_options[] = { - { "read-write", no_argument, NULL, 'W' }, - { "timeout", required_argument, NULL, 'T' }, - { NULL, 0, NULL, 0 } - }; - - optind = 1; - opterr = 0; - - int c; - int option_index = 0; - - while ((c = getopt_long(argc, argv, "WT:", wait_options, &option_index)) != -1) - { - switch (c) - { - case 'W': - { - pgsetupWaitReadWrite = true; - break; - } - - case 'T': - { - int t = strtol(optarg, NULL, 10); - if (t <= 0) - { - log_error("--timeout must be a positive integer"); - exit(EXIT_CODE_BAD_ARGS); - } - pgsetupWaitTimeout = t; - break; - } - - default: - { - /* standard keeper options; handled by the delegated call below */ - break; - } - } - } - - opterr = 1; - - /* - * Build a filtered argv that strips --timeout/--read-write (and their - * arguments) so that keeper_cli_keeper_setup_getopts does not see them. - */ - char **filtered_argv = (char **) palloc((argc + 1) * sizeof(char *)); - int filtered_argc = 0; - - for (int i = 0; i < argc; i++) - { - if (strcmp(argv[i], "--read-write") == 0 || strcmp(argv[i], "-W") == 0) - { - continue; - } - - if ((strcmp(argv[i], "--timeout") == 0 || strcmp(argv[i], "-T") == 0) && - i + 1 < argc) - { - /* skip both the flag and its argument */ - i++; - continue; - } - - filtered_argv[filtered_argc++] = argv[i]; - } - filtered_argv[filtered_argc] = NULL; - - int rc = keeper_cli_keeper_setup_getopts(filtered_argc, filtered_argv); - - pfree(filtered_argv); - - return rc; -} - - -/* - * keeper_cli_pgsetup_wait_until_ready waits for the local Postgres server to - * become ready. When --read-write is given, it additionally waits until the - * server is accepting read-write connections (not in recovery and not set to - * default_transaction_read_only). - * - * The --timeout value (default 30s) is a single deadline shared by both - * phases: the pg_is_ready poll and the subsequent read-write connection - * attempt. Time spent waiting for Postgres to start counts against the - * budget for the read-write phase. + * keeper_cli_discover_pg_setup implements the CLI to discover a PostgreSQL + * setup thanks to PGDATA and other environment variables. */ void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) { - int timeout = pgsetupWaitTimeout; + int timeout = 30; ConfigFilePaths pathnames = { 0 }; LocalPostgresServer postgres = { 0 }; PostgresSetup *pgSetup = &(postgres.postgresSetup); - /* Record wall-clock start so all phases share one deadline. */ - time_t startTime = time(NULL); - - /* Wait up to `timeout` seconds for the config file to be created. - * In no-monitor mode, pg_autoctl create postgres runs first and writes the - * config; pgsetup wait may be called before that completes. */ - { - KeeperConfig kconfig = keeperOptions; - if (keeper_config_set_pathnames_from_pgdata(&(kconfig.pathnames), - kconfig.pgSetup.pgdata)) - { - time_t deadline = startTime + timeout; - while (!file_exists(kconfig.pathnames.config) && - time(NULL) < deadline) - { - log_debug("Waiting for config file \"%s\" to appear", - kconfig.pathnames.config); - pg_usleep(500 * 1000); - } - } - } - if (!cli_common_pgsetup_init(&pathnames, pgSetup)) { /* errors have already been logged */ @@ -478,105 +343,15 @@ keeper_cli_pgsetup_wait_until_ready(int argc, char **argv) log_debug("Initialized pgSetup, now calling pg_setup_wait_until_is_ready()"); - /* - * Phase 1: wait for postmaster to signal "ready" in postmaster.pid. - * Pass the remaining timeout so the two phases together stay within the - * single user-visible deadline. - */ - int remainingAfterConfig = timeout - (int) (time(NULL) - startTime); - if (remainingAfterConfig <= 0) - { - log_error("Timed out waiting for Postgres config file to appear"); - exit(EXIT_CODE_PGSQL); - } - - bool pgIsReady = - pg_setup_wait_until_is_ready(pgSetup, remainingAfterConfig, LOG_INFO); + bool pgIsReady = pg_setup_wait_until_is_ready(pgSetup, timeout, LOG_INFO); log_info("Postgres status is: \"%s\"", pmStatusToString(pgSetup->pm_status)); - if (!pgIsReady) - { - exit(EXIT_CODE_PGSQL); - } - - if (!pgsetupWaitReadWrite) + if (pgIsReady) { - /* Plain "ready" check — we're done. */ exit(EXIT_CODE_QUIT); } - - /* - * Phase 2: wait until the server accepts read-write connections. - * - * Postgres is up (phase 1 passed) but may still be in recovery, finishing - * pg_rewind, or in standby mode. We poll with a libpq connection that - * checks pg_is_in_recovery() until it returns false or the deadline fires. - * - * We use the local connection string from pgSetup (Unix socket when - * available, matching whatever auth the node was created with) so that the - * check works regardless of the cluster's auth method. - */ - char connstr[MAXCONNINFO]; - if (!pg_setup_get_local_connection_string(pgSetup, connstr)) - { - log_error("Failed to build local connection string for read-write check"); - exit(EXIT_CODE_BAD_CONFIG); - } - - log_info("Waiting for Postgres to accept read-write connections " - "(timeout %ds)", timeout); - - bool isReadWrite = false; - int attempts = 0; - - while (!isReadWrite) - { - int elapsed = (int) (time(NULL) - startTime); - int remaining = timeout - elapsed; - - if (remaining <= 0) - { - log_error("Timed out after %ds waiting for Postgres " - "to accept read-write connections", timeout); - exit(EXIT_CODE_PGSQL); - } - - /* Use a short per-attempt connect_timeout so we retry briskly. */ - char attemptConnstr[MAXCONNINFO]; - sformat(attemptConnstr, sizeof(attemptConnstr), - "%s connect_timeout=1", connstr); - - PGSQL pgsql = { 0 }; - pgsql_init(&pgsql, attemptConnstr, PGSQL_CONN_LOCAL); - - bool inRecovery = true; /* assume standby until proven otherwise */ - bool queryOk = pgsql_is_in_recovery(&pgsql, &inRecovery); - pgsql_finish(&pgsql); - - if (queryOk && !inRecovery) - { - isReadWrite = true; - break; - } - - /* let's not be THAT verbose about it */ - if (attempts % 10 == 0) - { - log_debug("pgsetup wait --read-write: attempt %d, " - "in_recovery=%s, after %ds", - attempts + 1, - inRecovery ? "true" : "false", - elapsed); - } - - ++attempts; - pg_usleep(100 * 1000); /* 100 ms between probes */ - } - - log_info("Postgres is now accepting read-write connections on port %d", - pgSetup->pgport); - exit(EXIT_CODE_QUIT); + exit(EXIT_CODE_PGSQL); } diff --git a/src/bin/pg_autoctl/cli_do_monitor.c b/src/bin/pg_autoctl/cli_do_monitor.c index 6426946ee..65d9512c5 100644 --- a/src/bin/pg_autoctl/cli_do_monitor.c +++ b/src/bin/pg_autoctl/cli_do_monitor.c @@ -36,9 +36,7 @@ #include "pgctl.h" #include "pgsetup.h" #include "pgsql.h" -#include "pqexpbuffer.h" #include "state.h" -#include "string_utils.h" static void cli_do_monitor_get_primary_node(int argc, char **argv); static void cli_do_monitor_get_other_nodes(int argc, char **argv); @@ -48,10 +46,6 @@ static void cli_do_monitor_register_node(int argc, char **argv); static void cli_do_monitor_node_active(int argc, char **argv); static void cli_do_monitor_version(int argc, char **argv); static void cli_do_monitor_parse_notification(int argc, char **argv); -static int cli_do_monitor_node_state_getopts(int argc, char **argv); -static void cli_do_monitor_node_state(int argc, char **argv); -static int cli_do_monitor_formation_states_getopts(int argc, char **argv); -static void cli_do_monitor_formation_states(int argc, char **argv); static CommandLine monitor_get_primary_command = @@ -133,27 +127,6 @@ CommandLine monitor_parse_notification_command = NULL, cli_do_monitor_parse_notification); -CommandLine monitor_node_state_command = - make_command("node-state", - "Print reported|assigned state for a named node (for test runners)", - "--name [--state ] [--monitor ] [--timeout N]", - " --name node name to query\n" - " --state wait until reportedstate matches (requires --timeout)\n" - " --monitor monitor URI (default: local socket)\n" - " --timeout N retry for up to N seconds (default: no retry)\n", - cli_do_monitor_node_state_getopts, - cli_do_monitor_node_state); - -CommandLine monitor_formation_states_command = - make_command("formation-states", - "Exit 0 when every listed state has at least one node (for test runners)", - "[--group N] [--timeout N] [...]", - " --group N restrict to this group id\n" - " --monitor monitor URI (default: local socket)\n" - " --timeout N retry for up to N seconds (default: no retry)\n", - cli_do_monitor_formation_states_getopts, - cli_do_monitor_formation_states); - static CommandLine *monitor_subcommands[] = { &monitor_get_command, &monitor_register_command, @@ -730,458 +703,3 @@ cli_do_monitor_parse_notification(int argc, char **argv) (void) cli_pprint_json(js); } - - -/* ----------------------------------------------------------------------- - * pg_autoctl inspect monitor node-state --name - * pg_autoctl inspect monitor formation-states [--group N] [...] - * - * These two commands run ON the monitor container. They connect to the - * local PostgreSQL socket and return structured output consumed by the - * pgaftest test runner via `docker compose exec -T monitor pg_autoctl …`. - * Using dedicated subcommands (rather than shelling to psql) avoids all - * shell quoting issues. - * ----------------------------------------------------------------------- */ - -#define MONITOR_LOCAL_URI \ - "postgresql://autoctl_node@localhost/pg_auto_failover?sslmode=prefer" - -/* Shared option state for both new commands */ -static struct -{ - char monitorUri[MAXCONNINFO]; - char nodeName[_POSIX_HOST_NAME_MAX]; - char targetState[64]; /* --state: wait until reportedstate matches */ - char formation[NAMEDATALEN]; /* --formation: filter by formation */ - int groupId; /* -1 = all groups */ - int timeout; /* 0 = no retry loop, N = retry for N seconds */ -} -inspectOpts; - -static int -cli_do_monitor_node_state_getopts(int argc, char **argv) -{ - /* defaults */ - strlcpy(inspectOpts.monitorUri, MONITOR_LOCAL_URI, sizeof(inspectOpts.monitorUri)); - inspectOpts.nodeName[0] = '\0'; - inspectOpts.targetState[0] = '\0'; - inspectOpts.groupId = -1; - inspectOpts.timeout = 0; - - static struct option long_options[] = { - { "monitor", required_argument, NULL, 'm' }, - { "name", required_argument, NULL, 'n' }, - { "state", required_argument, NULL, 's' }, - { "timeout", required_argument, NULL, 't' }, - { NULL, 0, NULL, 0 } - }; - - int c; - optind = 0; - while ((c = getopt_long(argc, argv, "m:n:s:t:", long_options, NULL)) != -1) - { - switch (c) - { - case 'm': - { - strlcpy(inspectOpts.monitorUri, optarg, - sizeof(inspectOpts.monitorUri)); - break; - } - - case 'n': - { - strlcpy(inspectOpts.nodeName, optarg, - sizeof(inspectOpts.nodeName)); - break; - } - - case 's': - { - strlcpy(inspectOpts.targetState, optarg, - sizeof(inspectOpts.targetState)); - break; - } - - case 't': - { - inspectOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; - break; - } - - default: - { - commandline_print_usage(&monitor_node_state_command, stderr); - exit(EXIT_CODE_BAD_ARGS); - } - } - } - - if (inspectOpts.nodeName[0] == '\0') - { - log_error("--name is required"); - exit(EXIT_CODE_BAD_ARGS); - } - - return optind; -} - - -/* - * cli_do_monitor_node_state connects to the local monitor and prints: - * | - * for the named node, then exits 0. Exits non-zero if the node is not - * found or the monitor is unreachable. - * - * When --timeout N is given the command retries for up to N seconds using the - * same exponential-backoff-with-jitter policy used elsewhere in pg_autoctl. - */ -static void -cli_do_monitor_node_state(int argc, char **argv) -{ - const char *sql = - "SELECT reportedstate::text || '|' || goalstate::text || '|' || health::text " - "FROM pgautofailover.node WHERE nodename = $1"; - - int paramCount = 1; - Oid paramTypes[1] = { TEXTOID }; - const char *paramValues[1] = { inspectOpts.nodeName }; - - ConnectionRetryPolicy retryPolicy = { 0 }; - - if (inspectOpts.timeout > 0) - { - pgsql_set_retry_policy(&retryPolicy, - inspectOpts.timeout, - -1, /* unbounded attempts within timeout */ - 2000, /* cap at 2 s between attempts */ - 500); /* start at 500 ms */ - } - else - { - pgsql_set_retry_policy(&retryPolicy, 0, 0, 0, 0); /* no retry */ - } - do { - Monitor monitor = { 0 }; - - if (!monitor_init(&monitor, inspectOpts.monitorUri)) - { - /* connection failure: retry if policy allows */ - goto next_attempt; - } - - SingleValueResultContext ctx = { { 0 }, PGSQL_RESULT_STRING, false }; - - if (!pgsql_execute_with_params(&monitor.pgsql, sql, - paramCount, paramTypes, paramValues, - &ctx, &parseSingleValueResult)) - { - pgsql_finish(&monitor.pgsql); - goto next_attempt; - } - - pgsql_finish(&monitor.pgsql); - - if (!ctx.parsedOk || ctx.strVal == NULL) - { - /* node not found yet */ - if (ctx.strVal) - { - free(ctx.strVal); - } - goto next_attempt; - } - - /* - * When --state was given, check that reportedstate (the part - * before the first '|') matches the target. - * Output format: reportedstate|goalstate|health - */ - if (inspectOpts.targetState[0] != '\0') - { - char reported[64] = ""; - char *pipe = strchr(ctx.strVal, '|'); - if (pipe) - { - int len = pipe - ctx.strVal; - if (len >= (int) sizeof(reported)) - { - len = sizeof(reported) - 1; - } - memcpy(reported, ctx.strVal, len); /* IGNORE-BANNED */ - reported[len] = '\0'; - } - if (strcmp(reported, inspectOpts.targetState) != 0) - { - free(ctx.strVal); - goto next_attempt; - } - } - - fformat(stdout, "%s\n", ctx.strVal); - free(ctx.strVal); - exit(0); - -next_attempt: - if (pgsql_retry_policy_expired(&retryPolicy)) - { - break; - } - - int sleepMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); - log_debug("node-state: node \"%s\" not found yet, retrying in %d ms", - inspectOpts.nodeName, sleepMs); - pg_usleep((long) sleepMs * 1000); - } while (!pgsql_retry_policy_expired(&retryPolicy)); - - log_error("Node \"%s\" not found on the monitor", inspectOpts.nodeName); - exit(EXIT_CODE_MONITOR); -} - - -static int -cli_do_monitor_formation_states_getopts(int argc, char **argv) -{ - strlcpy(inspectOpts.monitorUri, MONITOR_LOCAL_URI, sizeof(inspectOpts.monitorUri)); - inspectOpts.formation[0] = '\0'; - inspectOpts.groupId = -1; - inspectOpts.timeout = 0; - - static struct option long_options[] = { - { "monitor", required_argument, NULL, 'm' }, - { "formation", required_argument, NULL, 'F' }, - { "group", required_argument, NULL, 'g' }, - { "timeout", required_argument, NULL, 't' }, - { NULL, 0, NULL, 0 } - }; - - int c; - optind = 0; - while ((c = getopt_long(argc, argv, "m:F:g:t:", long_options, NULL)) != -1) - { - switch (c) - { - case 'm': - { - strlcpy(inspectOpts.monitorUri, optarg, - sizeof(inspectOpts.monitorUri)); - break; - } - - case 'F': - { - strlcpy(inspectOpts.formation, optarg, - sizeof(inspectOpts.formation)); - break; - } - - case 'g': - { - inspectOpts.groupId = atoi(optarg) /* IGNORE-BANNED */; - break; - } - - case 't': - { - inspectOpts.timeout = atoi(optarg) /* IGNORE-BANNED */; - break; - } - - default: - { - commandline_print_usage(&monitor_formation_states_command, stderr); - exit(EXIT_CODE_BAD_ARGS); - } - } - } - - return optind; -} - - -/* - * cli_do_monitor_formation_states checks that every state listed on the - * command line has at least one node currently in that reportedstate. - * - * Exits 0 when all states are satisfied. When --timeout N is given the - * command retries for up to N seconds using the same exponential-backoff- - * with-jitter policy used elsewhere in pg_autoctl; without it the command - * exits 1 immediately when the condition is not met. - */ -static void -cli_do_monitor_formation_states(int argc, char **argv) -{ - if (argc < 1) - { - log_error("Expected at least one state name as argument"); - commandline_print_usage(&monitor_formation_states_command, stderr); - exit(EXIT_CODE_BAD_ARGS); - } - -#define MAX_STATES 16 - if (argc > MAX_STATES) - { - log_error("Too many states (max %d)", MAX_STATES); - exit(EXIT_CODE_BAD_ARGS); - } - - /* - * Build the SQL once; the params are the same across every retry attempt. - * - * SELECT (count(*) FILTER (WHERE reportedstate::text = $1))::text - * || ',' || - * (count(*) FILTER (WHERE reportedstate::text = $2))::text - * ... - * FROM pgautofailover.node - * [WHERE groupid = $N] - */ - PQExpBufferData sqlBuf; - initPQExpBuffer(&sqlBuf); - - appendPQExpBufferStr(&sqlBuf, "SELECT "); - for (int i = 0; i < argc; i++) - { - if (i > 0) - { - appendPQExpBufferStr(&sqlBuf, " || ',' || "); - } - appendPQExpBuffer(&sqlBuf, - "(count(*) FILTER (WHERE reportedstate::text = $%d))::text", - i + 1); - } - appendPQExpBufferStr(&sqlBuf, " FROM pgautofailover.node"); - - bool hasFormation = inspectOpts.formation[0] != '\0'; - bool hasGroup = inspectOpts.groupId >= 0; - bool needWhere = hasFormation || hasGroup; - int extraParams = (hasFormation ? 1 : 0) + (hasGroup ? 1 : 0); - - if (needWhere) - { - appendPQExpBufferStr(&sqlBuf, " WHERE"); - if (hasFormation) - { - appendPQExpBuffer(&sqlBuf, " formationid = $%d", argc + 1); - } - if (hasGroup) - { - appendPQExpBuffer(&sqlBuf, "%s groupid = $%d", - hasFormation ? " AND" : "", - argc + (hasFormation ? 2 : 1)); - } - } - - int paramCount = argc + extraParams; - Oid paramTypes[MAX_STATES + 2]; - const char *paramValues[MAX_STATES + 2]; - IntString groupIdStr = { 0 }; - - for (int i = 0; i < argc; i++) - { - paramTypes[i] = TEXTOID; - paramValues[i] = argv[i]; - } - if (extraParams > 0) - { - int pi = argc; - if (hasFormation) - { - paramTypes[pi] = TEXTOID; - paramValues[pi] = inspectOpts.formation; - pi++; - } - if (hasGroup) - { - groupIdStr = intToString(inspectOpts.groupId); - paramTypes[pi] = INT4OID; - paramValues[pi] = groupIdStr.strValue; - } - } - - ConnectionRetryPolicy retryPolicy = { 0 }; - - if (inspectOpts.timeout > 0) - { - pgsql_set_retry_policy(&retryPolicy, - inspectOpts.timeout, - -1, /* unbounded attempts within timeout */ - 2000, /* cap at 2 s between attempts */ - 500); /* start at 500 ms */ - } - else - { - pgsql_set_retry_policy(&retryPolicy, 0, 0, 0, 0); /* no retry */ - } - do { - Monitor monitor = { 0 }; - - if (!monitor_init(&monitor, inspectOpts.monitorUri)) - { - goto next_attempt; - } - - SingleValueResultContext ctx = { { 0 }, PGSQL_RESULT_STRING, false }; - - if (!pgsql_execute_with_params(&monitor.pgsql, sqlBuf.data, - paramCount, paramTypes, paramValues, - &ctx, &parseSingleValueResult)) - { - pgsql_finish(&monitor.pgsql); - goto next_attempt; - } - - pgsql_finish(&monitor.pgsql); - - if (!ctx.parsedOk || ctx.strVal == NULL) - { - if (ctx.strVal) - { - free(ctx.strVal); - } - goto next_attempt; - } - - /* parse "N,N,..." — all must be >= 1 */ - bool allMet = true; - char *p = ctx.strVal; - for (int i = 0; i < argc; i++) - { - int cnt = atoi(p) /* IGNORE-BANNED */; - if (cnt < 1) - { - allMet = false; - break; - } - p = strchr(p, ','); - if (p) - { - p++; - } - else - { - break; - } - } - free(ctx.strVal); - - if (allMet) - { - termPQExpBuffer(&sqlBuf); - exit(0); - } - -next_attempt: - if (pgsql_retry_policy_expired(&retryPolicy)) - { - break; - } - - int sleepMs = pgsql_compute_connection_retry_sleep_time(&retryPolicy); - log_debug("formation-states: condition not met yet, retrying in %d ms", - sleepMs); - pg_usleep((long) sleepMs * 1000); - } while (!pgsql_retry_policy_expired(&retryPolicy)); - - termPQExpBuffer(&sqlBuf); - exit(1); -} diff --git a/src/bin/pg_autoctl/cli_do_root.c b/src/bin/pg_autoctl/cli_do_root.c index 59e3dd288..a7f6a86f8 100644 --- a/src/bin/pg_autoctl/cli_do_root.c +++ b/src/bin/pg_autoctl/cli_do_root.c @@ -185,11 +185,9 @@ CommandLine do_pgsetup_is_ready = CommandLine do_pgsetup_wait_until_ready = make_command("wait", "Wait until the local Postgres server is ready", - "[--read-write] [--timeout N] [option ...]", - " --read-write also wait until the server accepts read-write connections\n" - " --timeout N total timeout in seconds (default: 30)\n" + "[option ...]", KEEPER_CLI_WORKER_SETUP_OPTIONS, - keeper_cli_pgsetup_wait_getopts, + keeper_cli_keeper_setup_getopts, keeper_cli_pgsetup_wait_until_ready); CommandLine do_pgsetup_startup_logs = @@ -208,14 +206,6 @@ CommandLine do_pgsetup_tune = keeper_cli_keeper_setup_getopts, keeper_cli_pgsetup_tune); -CommandLine do_pgsetup_hba_lan = - make_command("hba-lan", - "Append LAN CIDR trust rules to pg_hba.conf and reload Postgres", - "[option ...]", - KEEPER_CLI_WORKER_SETUP_OPTIONS, - keeper_cli_keeper_setup_getopts, - keeper_cli_pgsetup_hba_lan); - CommandLine *do_pgsetup[] = { &do_pgsetup_pg_ctl, &do_pgsetup_discover, @@ -223,7 +213,6 @@ CommandLine *do_pgsetup[] = { &do_pgsetup_wait_until_ready, &do_pgsetup_startup_logs, &do_pgsetup_tune, - &do_pgsetup_hba_lan, NULL }; @@ -386,254 +375,53 @@ CommandLine do_tmux_commands = NULL, NULL, NULL, do_tmux); /* - * Azure integration has been removed. The commands that were here are no - * longer maintained and have been deleted from this file. See pgaftest for - * the replacement QA tooling. + * internal service: hidden entry points spawned by the supervisor via + * fork+exec. The supervisor builds argv as: + * pg_autoctl internal service postgres|listener|node-active --pgdata ... + * Use make_hidden_command_set so these never appear in --help output. */ - -#if 0 /* REMOVED: azure commands — see pgaftest instead */ -CommandLine do_azure_provision_region = - make_command("region", - "Provision an azure region: resource group, network, VMs", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n" - " --location azure location where to create a resource group\n" - " --monitor should we create a monitor in the region (false)\n" - " --nodes number of Postgres nodes to create (2)\n" - " --script output a shell script instead of creating resources\n", - cli_do_azure_getopts, - cli_do_azure_create_region); - -CommandLine do_azure_provision_nodes = - make_command("nodes", - "Provision our pre-created VM with pg_autoctl Postgres nodes", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n" - " --monitor should we create a monitor in the region (false)\n" - " --nodes number of Postgres nodes to create (2)\n" - " --script output a shell script instead of creating resources\n", - cli_do_azure_getopts, - cli_do_azure_create_nodes); - - -CommandLine *do_azure_provision[] = { - &do_azure_provision_region, - &do_azure_provision_nodes, - NULL -}; - -CommandLine do_azure_provision_commands = - make_command_set("provision", - "provision azure resources for a pg_auto_failover demo", - NULL, NULL, NULL, do_azure_provision); - -CommandLine do_azure_create = - make_command("create", - "Create an azure QA environment", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n" - " --location azure location to use for the resources\n" - " --nodes number of Postgres nodes to create (2)\n" - " --script output a script instead of creating resources\n" - " --no-monitor do not create the pg_autoctl monitor node\n" - " --no-app do not create the application node\n" - " --cidr use the 10.CIDR.CIDR.0/24 subnet (11)\n" - " --from-source provision pg_auto_failover from sources\n", - cli_do_azure_getopts, - cli_do_azure_create_environment); - -CommandLine do_azure_drop = - make_command("drop", - "Drop an azure QA environment: resource group, network, VMs", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n" - " --location azure location where to create a resource group\n" - " --monitor should we create a monitor in the region (false)\n" - " --nodes number of Postgres nodes to create (2)\n" - " --script output a shell script instead of creating resources\n", - cli_do_azure_getopts, - cli_do_azure_drop_region); - -CommandLine do_azure_deploy = - make_command("deploy", - "Deploy a pg_autoctl VMs, given by name", - "[option ...] vmName", - "", - cli_do_azure_getopts, - cli_do_azure_deploy); - -CommandLine do_azure_show_ips = - make_command("ips", - "Show public and private IP addresses for selected VMs", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n", - cli_do_azure_getopts, - cli_do_azure_show_ips); - -CommandLine do_azure_show_state = - make_command("state", - "Connect to the monitor node to show the current state", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n" - " --watch run the command again every 0.2s\n", - cli_do_azure_getopts, - cli_do_azure_show_state); - -CommandLine *do_azure_show[] = { - &do_azure_show_ips, - &do_azure_show_state, - NULL -}; - -CommandLine do_azure_show_commands = - make_command_set("show", - "show azure resources for a pg_auto_failover demo", - NULL, NULL, NULL, do_azure_show); - -CommandLine do_azure_ls = - make_command("ls", - "List resources in a given azure region", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n", - cli_do_azure_getopts, - cli_do_azure_ls); - -CommandLine do_azure_ssh = - make_command("ssh", - "Runs ssh -l ha-admin for a given VM name", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region name to use for referencing the region\n", - cli_do_azure_getopts, - cli_do_azure_ssh); - -CommandLine do_azure_sync = - make_command("sync", - "Rsync pg_auto_failover sources on all the target region VMs", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region region to use for referencing the region\n" - " --monitor should we create a monitor in the region (false)\n" - " --nodes number of Postgres nodes to create (2)\n", - cli_do_azure_getopts, - cli_do_azure_rsync); - -CommandLine do_azure_tmux_session = - make_command("session", - "Create or attach a tmux session for the created Azure VMs", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region region to use for referencing the region\n" - " --monitor should we create a monitor in the region (false)\n" - " --nodes number of Postgres nodes to create (2)\n", - cli_do_azure_getopts, - cli_do_azure_tmux_session); - -CommandLine do_azure_tmux_kill = - make_command("kill", - "Kill an existing tmux session for Azure VMs", - "[option ...]", - " --prefix azure group name prefix (ha-demo)\n" - " --region region to use for referencing the region\n" - " --monitor should we create a monitor in the region (false)\n" - " --nodes number of Postgres nodes to create (2)\n", - cli_do_azure_getopts, - cli_do_azure_tmux_kill); - -CommandLine *do_azure_tmux[] = { - &do_azure_tmux_session, - &do_azure_tmux_kill, - NULL -}; - -CommandLine do_azure_tmux_commands = - make_command_set("tmux", - "Run a tmux session with an Azure setup for QA/testing", - NULL, NULL, NULL, do_azure_tmux); - -CommandLine *do_azure[] = { - &do_azure_provision_commands, - &do_azure_tmux_commands, - &do_azure_show_commands, - &do_azure_deploy, - &do_azure_create, - &do_azure_drop, - &do_azure_ls, - &do_azure_ssh, - &do_azure_sync, +static CommandLine *internal_service_subcommands[] = { + &service_pgcontroller, + &service_postgres, + &service_monitor_listener, + &service_node_active, NULL }; -CommandLine do_azure_commands = - make_command_set("azure", - "Manage a set of Azure resources for a pg_auto_failover demo", - NULL, NULL, NULL, do_azure); - -#endif /* REMOVED: azure commands */ +CommandLine internal_service_commands = + make_hidden_command_set("service", + "Internal subprocess entry points (supervisor use only)", + NULL, NULL, NULL, internal_service_subcommands); -/* - * pg_autoctl internal service postgres|listener|node-active - * - * These are the subprocess entry points used by the supervisor (pg_autoctl run - * and pg_autoctl create … --run). The supervisor fork()s and then execv()s - * the pg_autoctl binary itself with one of these sub-commands so that each - * service runs in its own address space. - * - * Using fork+exec (rather than fork alone) is a deliberate design choice for - * live upgrades: when a child process exits with an incompatible monitor - * extension version, the supervisor restarts it via fork()+execv(), which loads - * the current binary from disk. If the binary has been updated in place (e.g. - * by a package manager), the restarted child automatically picks up the new - * version without touching the supervisor process — making pg_autoctl safe to - * use as PID 1 in Docker/Kubernetes containers where replacing the binary and - * sending SIGTERM would lose the container. - * - * See also: keeper.c keeper_check_monitor_extension_version(), which exits on - * version mismatch precisely to trigger this restart-with-new-binary path. - * - * These commands are hidden from --help output (make_hidden_command_set) so - * operators do not accidentally invoke them directly. - * Use "pg_autoctl manual service" for the user-facing controls (restart, pgctl). - * Use "pg_autoctl inspect getpid" to read sub-process PIDs. - */ -static CommandLine *internal_service_subcommands[] = { - &service_pgcontroller, /* debug: supervisor for just the postgres controller */ - &service_postgres, /* spawned by service_postgres_ctl_start() */ - &service_monitor_listener, /* spawned by service_monitor_start() */ - &service_node_active, /* spawned by service_keeper_start() */ +static CommandLine *internal_subcommands[] = { + &internal_service_commands, NULL }; -static CommandLine internal_service_commands = - make_hidden_command_set("service", - "Subprocess entry points for the pg_autoctl supervisor", - NULL, NULL, NULL, internal_service_subcommands); +CommandLine internal_commands = + make_hidden_command_set("internal", + "Internal commands for use by the supervisor (not for operators)", + NULL, NULL, NULL, internal_subcommands); -/* - * pg_autoctl internal - * - * Hidden from --help; routable so the supervisor's execv() calls work. - * Contains only what the supervisor spawns plus dev/QA tooling not yet - * moved to pgaftest (tmux, demo). - */ CommandLine *do_subcommands[] = { - &internal_service_commands, + &do_monitor_commands, + &do_coordinator_commands, + &do_fsm_commands, + &do_primary_, + &do_standby_, + &do_show_commands, + &do_pgsetup_commands, + &do_service_postgres_ctl_commands, + &do_service_commands, &do_tmux_commands, &do_demo_commands, NULL }; CommandLine do_commands = - make_hidden_command_set("internal", - "Internal subprocess entry points — not for direct use", - NULL, NULL, NULL, do_subcommands); + make_command_set("do", + "Internal commands and internal QA tooling", NULL, NULL, + NULL, do_subcommands); /* diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index 9156fce31..bc129ad04 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -1,7 +1,7 @@ /* * src/bin/pg_autoctl/cli_do_root.h - * Implementation of a CLI which lets you run individual keeper routines - * directly + * Implementation of a CLI which lets you run operations on the local + * postgres server directly * * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the PostgreSQL License. @@ -15,40 +15,33 @@ /* src/bin/pg_autoctl/cli_do_fsm.c */ extern CommandLine do_fsm_commands; +extern CommandLine fsm_nodes; -/* read-only sub-commands exposed via "pg_autoctl inspect fsm" */ +/* Exported individually so cli_inspect.c and cli_manual.c can compose them */ +extern CommandLine fsm_init; extern CommandLine fsm_state; -extern CommandLine fsm_node_state; extern CommandLine fsm_list; extern CommandLine fsm_gv; - -/* mutating sub-commands exposed via "pg_autoctl manual fsm" */ -extern CommandLine fsm_init; extern CommandLine fsm_assign; extern CommandLine fsm_step; -extern CommandLine fsm_nodes; /* nodes get + nodes set — kept together in manual */ /* src/bin/pg_autoctl/cli_do_monitor.c */ extern CommandLine do_monitor_commands; -/* read-only sub-commands exposed via "pg_autoctl inspect monitor" */ +/* Exported individually so cli_inspect.c and cli_manual.c can compose them */ extern CommandLine monitor_get_command; extern CommandLine monitor_parse_notification_command; -extern CommandLine monitor_node_state_command; -extern CommandLine monitor_formation_states_command; - -/* mutating sub-commands exposed via "pg_autoctl manual monitor" */ extern CommandLine monitor_register_command; extern CommandLine monitor_node_active_command; extern CommandLine monitor_version_command; /* src/bin/pg_autoctl/cli_do_service.c */ extern CommandLine do_service_commands; -extern CommandLine do_service_getpid_commands; extern CommandLine do_service_restart_commands; +extern CommandLine do_service_getpid_commands; extern CommandLine do_service_postgres_ctl_commands; -/* subprocess entry points (spawned by the supervisor via fork+exec) */ +/* Internal subprocess entry points used in cli_do_root.c's internal_service_commands */ extern CommandLine service_pgcontroller; extern CommandLine service_postgres; extern CommandLine service_monitor_listener; @@ -57,32 +50,24 @@ extern CommandLine service_node_active; /* src/bin/pg_autoctl/cli_do_show.c */ extern CommandLine do_show_commands; extern CommandLine do_pgsetup_commands; -extern CommandLine do_service_postgres_ctl_commands; -extern CommandLine do_service_commands; /* src/bin/pg_autoctl/cli_do_demo.c */ extern CommandLine do_demo_commands; +/* src/bin/pg_autoctl/cli_do_coordinator.c */ +extern CommandLine do_coordinator_commands; + /* src/bin/pg_autoctl/cli_do_root.c */ extern CommandLine do_primary_adduser; extern CommandLine *do_primary_adduser_subcommands[]; extern CommandLine do_primary_adduser_monitor; extern CommandLine do_primary_adduser_replica; -extern CommandLine do_primary_syncrep_; -extern CommandLine *do_primary_syncrep[]; -extern CommandLine do_primary_syncrep_enable; -extern CommandLine do_primary_syncrep_disable; - extern CommandLine do_primary_slot_; extern CommandLine *do_primary_slot[]; extern CommandLine do_primary_slot_create; extern CommandLine do_primary_slot_drop; -extern CommandLine do_primary_hba; -extern CommandLine *do_primary_hba_commands[]; -extern CommandLine do_primary_hba_setup; - extern CommandLine do_primary_defaults; extern CommandLine do_primary_identify_system; @@ -95,12 +80,10 @@ extern CommandLine do_standby_init; extern CommandLine do_standby_rewind; extern CommandLine do_standby_promote; -extern CommandLine do_discover; - extern CommandLine do_tmux_commands; -/* src/bin/pg_autoctl/cli_do_coordinator.c */ -extern CommandLine do_coordinator_commands; +extern CommandLine internal_service_commands; +extern CommandLine internal_commands; extern CommandLine do_commands; extern CommandLine *do_subcommands[]; @@ -117,11 +100,9 @@ void keeper_cli_disable_synchronous_replication(int argc, char **argv); void keeper_cli_pgsetup_pg_ctl(int argc, char **argv); void keeper_cli_pgsetup_discover(int argc, char **argv); void keeper_cli_pgsetup_is_ready(int argc, char **argv); -int keeper_cli_pgsetup_wait_getopts(int argc, char **argv); void keeper_cli_pgsetup_wait_until_ready(int argc, char **argv); void keeper_cli_pgsetup_startup_logs(int argc, char **argv); void keeper_cli_pgsetup_tune(int argc, char **argv); -void keeper_cli_pgsetup_hba_lan(int argc, char **argv); void keeper_cli_add_default_settings(int argc, char **argv); void keeper_cli_create_monitor_user(int argc, char **argv); diff --git a/src/bin/pg_autoctl/cli_inspect.c b/src/bin/pg_autoctl/cli_inspect.c index 4b156c660..968c471f2 100644 --- a/src/bin/pg_autoctl/cli_inspect.c +++ b/src/bin/pg_autoctl/cli_inspect.c @@ -23,7 +23,6 @@ */ static CommandLine *inspect_fsm_subcommands[] = { &fsm_state, - &fsm_node_state, &fsm_list, &fsm_gv, NULL @@ -42,8 +41,6 @@ static CommandLine inspect_fsm_commands = static CommandLine *inspect_monitor_subcommands[] = { &monitor_get_command, &monitor_parse_notification_command, - &monitor_node_state_command, - &monitor_formation_states_command, NULL }; diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 83c26ea62..dd5029687 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -101,6 +101,7 @@ CommandLine *root_subcommands[] = { &activate_node_command, &inspect_commands, &manual_commands, + &internal_commands, &do_commands, &service_run_command, @@ -119,39 +120,6 @@ CommandLine root = "[ --verbose --quiet ]", NULL, root_options, root_subcommands); -/* Backward-compat alias: root_with_debug and root_subcommands_with_debug - * previously held the do_commands. They now point to the same root. */ -CommandLine *root_subcommands_with_debug[] = { - &create_commands, - &drop_commands, - &config_commands, - &show_commands, - &enable_commands, - &disable_commands, - &get_commands, - &set_commands, - &perform_commands, - &activate_node_command, - &inspect_commands, - &manual_commands, - - &do_commands, - &service_run_command, - &watch_command, - &service_stop_command, - &service_reload_command, - &service_status_command, - &help, - &version, - NULL -}; - -CommandLine root_with_debug = - make_command_set("pg_autoctl", - "pg_auto_failover control tools and service", - "[ --verbose --quiet ]", NULL, - root_options, root_subcommands_with_debug); - /* * root_options parses flags from the list of arguments that are common to all diff --git a/src/bin/pg_autoctl/cli_root.h b/src/bin/pg_autoctl/cli_root.h index 7b09cff48..03d97639a 100644 --- a/src/bin/pg_autoctl/cli_root.h +++ b/src/bin/pg_autoctl/cli_root.h @@ -40,9 +40,6 @@ extern CommandLine *show_subcommands_with_debug[]; extern CommandLine drop_commands; extern CommandLine *drop_subcommands[]; -extern CommandLine root_with_debug; -extern CommandLine *root_subcommands_with_debug[]; - extern CommandLine root; extern CommandLine *root_subcommands[]; diff --git a/src/bin/pg_autoctl/main.c b/src/bin/pg_autoctl/main.c index a49dae7ed..577ecd540 100644 --- a/src/bin/pg_autoctl/main.c +++ b/src/bin/pg_autoctl/main.c @@ -64,15 +64,6 @@ main(int argc, char **argv) /* register our logging clean-up atexit */ atexit(log_semaphore_unlink_atexit); - /* - * When PG_AUTOCTL_DEBUG is set in the environment, provide the user - * commands available to debug a pg_autoctl instance. - */ - if (env_exists(PG_AUTOCTL_DEBUG)) - { - command = root_with_debug; - } - /* * When PGCONNECT_TIMEOUT is set in the environment, keep a copy of it in * our own global variable pgconnect_timeout. We implement our own From b7a67f8e93a6e5d7e69a393ffdf33f54e5f661f0 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Mon, 6 Jul 2026 22:41:26 +0200 Subject: [PATCH 3/3] docs: fix two conf.py warnings (invalid escape, language=None) --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3c102766a..6c56594b3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -80,7 +80,7 @@ def __init__(self, **options): # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -148,7 +148,7 @@ def setup(app): ( master_doc, "pg_auto_failover.tex", - "pg\_auto\_failover Documentation", + r"pg\_auto\_failover Documentation", "Microsoft", "manual", ),