diff --git a/docs/conf.py b/docs/conf.py index 3c102766a..6c56594b3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -80,7 +80,7 @@ def __init__(self, **options): # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -148,7 +148,7 @@ def setup(app): ( master_doc, "pg_auto_failover.tex", - "pg\_auto\_failover Documentation", + r"pg\_auto\_failover Documentation", "Microsoft", "manual", ), diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index a1d1e1e9d..026eacec7 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -277,10 +277,10 @@ command, and then the node entry is removed from the monitor. pg_auto_failover keeper's State Machine --------------------------------------- -When built in TEST mode, it is then possible to use the following command to -get a visual representation of the Keeper's Finite State Machine:: +It is possible to use the following command to get a visual representation +of the Keeper's Finite State Machine:: - $ PG_AUTOCTL_DEBUG=1 pg_autoctl do fsm gv | dot -Tsvg > fsm.svg + $ pg_autoctl inspect fsm gv | dot -Tsvg > fsm.svg The `dot` program is part of the Graphviz suite and produces the following output: diff --git a/docs/ref/manual.rst b/docs/ref/manual.rst index 12ed2f966..93638bea5 100644 --- a/docs/ref/manual.rst +++ b/docs/ref/manual.rst @@ -20,6 +20,8 @@ have their own manual page. pg_autoctl_get pg_autoctl_set pg_autoctl_perform + pg_autoctl_inspect + pg_autoctl_manual pg_autoctl_do pg_autoctl_run pg_autoctl_watch diff --git a/docs/ref/pg_autoctl_do.rst b/docs/ref/pg_autoctl_do.rst index d471e63cd..0b39c13e8 100644 --- a/docs/ref/pg_autoctl_do.rst +++ b/docs/ref/pg_autoctl_do.rst @@ -3,121 +3,24 @@ pg_autoctl do ============= -pg_autoctl do - Internal commands and internal QA tooling +pg_autoctl do - Internal QA tooling (tmux sessions, demo app) -The debug commands for ``pg_autoctl`` are only available when the -environment variable ``PG_AUTOCTL_DEBUG`` is set (to any value). - -When testing pg_auto_failover, it is helpful to be able to play with the -local nodes using the same lower-level API as used by the pg_auto_failover -Finite State Machine transitions. Some commands could be useful in contexts -other than pg_auto_failover development and QA work, so some documentation -has been made available. +The ``pg_autoctl do`` command group contains development and QA tooling that +is not intended for production use. For read-only diagnostics see +:ref:`pg_autoctl_inspect`; for manual cluster recovery operations see +:ref:`pg_autoctl_manual`. .. toctree:: :maxdepth: 1 pg_autoctl_do_tmux pg_autoctl_do_demo - pg_autoctl_do_service_restart - pg_autoctl_do_show - pg_autoctl_do_pgsetup -The low-level API is made available through the following ``pg_autoctl do`` -commands, only available in debug environments:: +``pg_autoctl do`` provides the following commands:: pg_autoctl do - + monitor Query a pg_auto_failover monitor - + fsm Manually manage the keeper's state - + primary Manage a PostgreSQL primary server - + standby Manage a PostgreSQL standby server - + show Show some debug level information - + pgsetup Manage a local Postgres setup - + pgctl Signal the pg_autoctl postgres service - + service Run pg_autoctl sub-processes (services) - + tmux Set of facilities to handle tmux interactive sessions - + demo Use a demo application for pg_auto_failover - - pg_autoctl do monitor - + get Get information from the monitor - register Register the current node with the monitor - active Call in the pg_auto_failover Node Active protocol - version Check that monitor version is 1.5.0.1; alter extension update if not - parse-notification parse a raw notification message - - pg_autoctl do monitor get - primary Get the primary node from pg_auto_failover in given formation/group - others Get the other nodes from the pg_auto_failover group of hostname/port - coordinator Get the coordinator node from the pg_auto_failover formation - - pg_autoctl do fsm - init Initialize the keeper's state on-disk - state Read the keeper's state from disk and display it - list List reachable FSM states from current state - gv Output the FSM as a .gv program suitable for graphviz/dot - assign Assign a new goal state to the keeper - step Make a state transition if instructed by the monitor - + nodes Manually manage the keeper's nodes list - - pg_autoctl do fsm nodes - get Get the list of nodes from file (see --disable-monitor) - set Set the list of nodes to file (see --disable-monitor) - - pg_autoctl do primary - + slot Manage replication slot on the primary server - + adduser Create users on primary - defaults Add default settings to postgresql.conf - identify Run the IDENTIFY_SYSTEM replication command on given host - - pg_autoctl do primary slot - create Create a replication slot on the primary server - drop Drop a replication slot on the primary server - - pg_autoctl do primary adduser - monitor add a local user for queries from the monitor - replica add a local user with replication privileges - - pg_autoctl do standby - init Initialize the standby server using pg_basebackup - rewind Rewind a demoted primary server using pg_rewind - promote Promote a standby server to become writable - - pg_autoctl do show - ipaddr Print this node's IP address information - cidr Print this node's CIDR information - lookup Print this node's DNS lookup information - hostname Print this node's default hostname - reverse Lookup given hostname and check reverse DNS setup - - pg_autoctl do pgsetup - pg_ctl Find a non-ambiguous pg_ctl program and Postgres version - discover Discover local PostgreSQL instance, if any - ready Return true is the local Postgres server is ready - wait Wait until the local Postgres server is ready - logs Outputs the Postgres startup logs - tune Compute and log some Postgres tuning options - - pg_autoctl do pgctl - on Signal pg_autoctl postgres service to ensure Postgres is running - off Signal pg_autoctl postgres service to ensure Postgres is stopped - - pg_autoctl do service - + getpid Get the pid of pg_autoctl sub-processes (services) - + restart Restart pg_autoctl sub-processes (services) - pgcontroller pg_autoctl supervised postgres controller - postgres pg_autoctl service that start/stop postgres when asked - listener pg_autoctl service that listens to the monitor notifications - node-active pg_autoctl service that implements the node active protocol - - pg_autoctl do service getpid - postgres Get the pid of the pg_autoctl postgres controller service - listener Get the pid of the pg_autoctl monitor listener service - node-active Get the pid of the pg_autoctl keeper node-active service - - pg_autoctl do service restart - postgres Restart the pg_autoctl postgres controller service - listener Restart the pg_autoctl monitor listener service - node-active Restart the pg_autoctl keeper node-active service + + tmux Set of facilities to handle tmux interactive sessions + + demo Use a demo application for pg_auto_failover pg_autoctl do tmux script Produce a tmux script for a demo or a test case (debug only) diff --git a/docs/ref/pg_autoctl_do_pgsetup.rst b/docs/ref/pg_autoctl_do_pgsetup.rst deleted file mode 100644 index fb085d343..000000000 --- a/docs/ref/pg_autoctl_do_pgsetup.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. _pg_autoctl_do_pgsetup: - -pg_autoctl do pgsetup -===================== - -pg_autoctl do pgsetup - Manage a local Postgres setup - -Synopsis --------- - -The main ``pg_autoctl`` commands implement low-level management tooling for -a local Postgres instance. Some of the low-level Postgres commands can be -used as their own tool in some cases. - -pg_autoctl do pgsetup provides the following commands:: - - pg_autoctl do pgsetup - pg_ctl Find a non-ambiguous pg_ctl program and Postgres version - discover Discover local PostgreSQL instance, if any - ready Return true is the local Postgres server is ready - wait Wait until the local Postgres server is ready - logs Outputs the Postgres startup logs - tune Compute and log some Postgres tuning options - -pg_autoctl do pgsetup pg_ctl ----------------------------- - -In a similar way to ``which -a``, this commands scans your PATH for -``pg_ctl`` commands. Then it runs the ``pg_ctl --version`` command and -parses the output to determine the version of Postgres that is available in -the path. - -:: - - $ pg_autoctl do pgsetup pg_ctl --pgdata node1 - 16:49:18 69684 INFO Environment variable PG_CONFIG is set to "/Applications/Postgres.app//Contents/Versions/12/bin/pg_config" - 16:49:18 69684 INFO `pg_autoctl create postgres` would use "/Applications/Postgres.app/Contents/Versions/12/bin/pg_ctl" for Postgres 12.3 - 16:49:18 69684 INFO `pg_autoctl create monitor` would use "/Applications/Postgres.app/Contents/Versions/12/bin/pg_ctl" for Postgres 12.3 - - -pg_autoctl do pgsetup discover ------------------------------- - -Given a PGDATA or ``--pgdata`` option, the command discovers if a running -Postgres service matches the pg_autoctl setup, and prints the information -that ``pg_autoctl`` typically needs when managing a Postgres instance. - -:: - - $ pg_autoctl do pgsetup discover --pgdata node1 - pgdata: /Users/dim/dev/MS/pg_auto_failover/tmux/node1 - pg_ctl: /Applications/Postgres.app/Contents/Versions/12/bin/pg_ctl - pg_version: 12.3 - pghost: /tmp - pgport: 5501 - proxyport: 0 - pid: 21029 - is in recovery: no - Control Version: 1201 - Catalog Version: 201909212 - System Identifier: 6942422768095393833 - Latest checkpoint LSN: 0/4059C18 - Postmaster status: ready - - -pg_autoctl do pgsetup ready ---------------------------- - -Similar to the `pg_isready`__ command, though uses the Postgres -specifications found in the pg_autoctl node setup. - -__ https://www.postgresql.org/docs/current/app-pg-isready.html - -:: - - $ pg_autoctl do pgsetup ready --pgdata node1 - 16:50:08 70582 INFO Postgres status is: "ready" - - -pg_autoctl do pgsetup wait --------------------------- - -When ``pg_autoctl do pgsetup ready`` would return false because Postgres is -not ready yet, this command continues probing every second for 30 seconds, -and exists as soon as Postgres is ready. - -:: - - $ pg_autoctl do pgsetup wait --pgdata node1 - 16:50:22 70829 INFO Postgres is now serving PGDATA "/Users/dim/dev/MS/pg_auto_failover/tmux/node1" on port 5501 with pid 21029 - 16:50:22 70829 INFO Postgres status is: "ready" - - -pg_autoctl do pgsetup logs --------------------------- - -Outputs the Postgres logs from the most recent log file in the -``PGDATA/log`` directory. - -:: - - $ pg_autoctl do pgsetup logs --pgdata node1 - 16:50:39 71126 WARN Postgres logs from "/Users/dim/dev/MS/pg_auto_failover/tmux/node1/startup.log": - 16:50:39 71126 INFO 2021-03-22 14:43:48.911 CET [21029] LOG: starting PostgreSQL 12.3 on x86_64-apple-darwin16.7.0, compiled by Apple LLVM version 8.1.0 (clang-802.0.42), 64-bit - 16:50:39 71126 INFO 2021-03-22 14:43:48.913 CET [21029] LOG: listening on IPv6 address "::", port 5501 - 16:50:39 71126 INFO 2021-03-22 14:43:48.913 CET [21029] LOG: listening on IPv4 address "0.0.0.0", port 5501 - 16:50:39 71126 INFO 2021-03-22 14:43:48.913 CET [21029] LOG: listening on Unix socket "/tmp/.s.PGSQL.5501" - 16:50:39 71126 INFO 2021-03-22 14:43:48.931 CET [21029] LOG: redirecting log output to logging collector process - 16:50:39 71126 INFO 2021-03-22 14:43:48.931 CET [21029] HINT: Future log output will appear in directory "log". - 16:50:39 71126 WARN Postgres logs from "/Users/dim/dev/MS/pg_auto_failover/tmux/node1/log/postgresql-2021-03-22_144348.log": - 16:50:39 71126 INFO 2021-03-22 14:43:48.937 CET [21033] LOG: database system was shut down at 2021-03-22 14:43:46 CET - 16:50:39 71126 INFO 2021-03-22 14:43:48.937 CET [21033] LOG: entering standby mode - 16:50:39 71126 INFO 2021-03-22 14:43:48.942 CET [21033] LOG: consistent recovery state reached at 0/4022E88 - 16:50:39 71126 INFO 2021-03-22 14:43:48.942 CET [21033] LOG: invalid record length at 0/4022E88: wanted 24, got 0 - 16:50:39 71126 INFO 2021-03-22 14:43:48.946 CET [21029] LOG: database system is ready to accept read only connections - 16:50:39 71126 INFO 2021-03-22 14:43:49.032 CET [21038] LOG: fetching timeline history file for timeline 4 from primary server - 16:50:39 71126 INFO 2021-03-22 14:43:49.037 CET [21038] LOG: started streaming WAL from primary at 0/4000000 on timeline 3 - 16:50:39 71126 INFO 2021-03-22 14:43:49.046 CET [21038] LOG: replication terminated by primary server - 16:50:39 71126 INFO 2021-03-22 14:43:49.046 CET [21038] DETAIL: End of WAL reached on timeline 3 at 0/4022E88. - 16:50:39 71126 INFO 2021-03-22 14:43:49.047 CET [21033] LOG: new target timeline is 4 - 16:50:39 71126 INFO 2021-03-22 14:43:49.049 CET [21038] LOG: restarted WAL streaming at 0/4000000 on timeline 4 - 16:50:39 71126 INFO 2021-03-22 14:43:49.210 CET [21033] LOG: redo starts at 0/4022E88 - 16:50:39 71126 INFO 2021-03-22 14:52:06.692 CET [21029] LOG: received SIGHUP, reloading configuration files - 16:50:39 71126 INFO 2021-03-22 14:52:06.906 CET [21029] LOG: received SIGHUP, reloading configuration files - 16:50:39 71126 FATAL 2021-03-22 15:34:24.920 CET [21038] FATAL: terminating walreceiver due to timeout - 16:50:39 71126 INFO 2021-03-22 15:34:24.973 CET [21033] LOG: invalid record length at 0/4059CC8: wanted 24, got 0 - 16:50:39 71126 INFO 2021-03-22 15:34:25.105 CET [35801] LOG: started streaming WAL from primary at 0/4000000 on timeline 4 - 16:50:39 71126 FATAL 2021-03-22 16:12:56.918 CET [35801] FATAL: terminating walreceiver due to timeout - 16:50:39 71126 INFO 2021-03-22 16:12:57.086 CET [38741] LOG: started streaming WAL from primary at 0/4000000 on timeline 4 - 16:50:39 71126 FATAL 2021-03-22 16:23:39.349 CET [38741] FATAL: terminating walreceiver due to timeout - 16:50:39 71126 INFO 2021-03-22 16:23:39.497 CET [41635] LOG: started streaming WAL from primary at 0/4000000 on timeline 4 - - -pg_autoctl do pgsetup tune --------------------------- - -Outputs the pg_autoctl automated tuning options. Depending on the number of -CPU and amount of RAM detected in the environment where it is run, -``pg_autoctl`` can adjust some very basic Postgres tuning knobs to get -started. - -:: - - $ pg_autoctl do pgsetup tune --pgdata node1 -vv - 13:25:25 77185 DEBUG pgtuning.c:85: Detected 12 CPUs and 16 GB total RAM on this server - 13:25:25 77185 DEBUG pgtuning.c:225: Setting autovacuum_max_workers to 3 - 13:25:25 77185 DEBUG pgtuning.c:228: Setting shared_buffers to 4096 MB - 13:25:25 77185 DEBUG pgtuning.c:231: Setting work_mem to 24 MB - 13:25:25 77185 DEBUG pgtuning.c:235: Setting maintenance_work_mem to 512 MB - 13:25:25 77185 DEBUG pgtuning.c:239: Setting effective_cache_size to 12 GB - # basic tuning computed by pg_auto_failover - track_functions = pl - shared_buffers = '4096 MB' - work_mem = '24 MB' - maintenance_work_mem = '512 MB' - effective_cache_size = '12 GB' - autovacuum_max_workers = 3 - autovacuum_vacuum_scale_factor = 0.08 - autovacuum_analyze_scale_factor = 0.02 diff --git a/docs/ref/pg_autoctl_do_service_restart.rst b/docs/ref/pg_autoctl_do_service_restart.rst deleted file mode 100644 index 8f0442a46..000000000 --- a/docs/ref/pg_autoctl_do_service_restart.rst +++ /dev/null @@ -1,32 +0,0 @@ -.. _pg_autoctl_do_service_restart: - -pg_autoctl do service restart -============================= - -pg_autoctl do service restart - Run pg_autoctl sub-processes (services) - -Synopsis --------- - -pg_autoctl do service restart provides the following commands:: - - pg_autoctl do service restart - postgres Restart the pg_autoctl postgres controller service - listener Restart the pg_autoctl monitor listener service - node-active Restart the pg_autoctl keeper node-active service - - -Description ------------ - -It is possible to restart the ``pg_autoctl`` or the Postgres service without -affecting the other running service. Typically, to restart the -``pg_autoctl`` parts without impacting Postgres:: - - $ pg_autoctl do service restart node-active --pgdata node1 - 14:52:06 31223 INFO Sending the TERM signal to service "node-active" with pid 26626 - 14:52:06 31223 INFO Service "node-active" has been restarted with pid 31230 - 31230 - -The Postgres service has not been impacted by the restart of the -``pg_autoctl`` process. diff --git a/docs/ref/pg_autoctl_do_show.rst b/docs/ref/pg_autoctl_do_show.rst deleted file mode 100644 index 781140216..000000000 --- a/docs/ref/pg_autoctl_do_show.rst +++ /dev/null @@ -1,130 +0,0 @@ -.. _pg_autoctl_do_show: - -pg_autoctl do show -================== - -pg_autoctl do show - Show some debug level information - -Synopsis --------- - -The commands :ref:`pg_autoctl_create_monitor` and -:ref:`pg_autoctl_create_postgres` both implement some level of automated -detection of the node network settings when the option ``--hostname`` is not -used. - -Adding to those commands, when a new node is registered to the monitor, -other nodes also edit their Postgres HBA rules to allow the new node to -connect, unless the option ``--skip-pg-hba`` has been used. - -The debug sub-commands for ``pg_autoctl do show`` can be used to see in -details the network discovery done by ``pg_autoctl``. - -pg_autoctl do show provides the following commands:: - - pg_autoctl do show - ipaddr Print this node's IP address information - cidr Print this node's CIDR information - lookup Print this node's DNS lookup information - hostname Print this node's default hostname - reverse Lookup given hostname and check reverse DNS setup - -pg_autoctl do show ipaddr -------------------------- - -Connects to an external IP address and uses ``getsockname(2)`` to retrieve -the current address to which the socket is bound. - -The external IP address defaults to ``8.8.8.8``, the IP address of a Google -provided public DNS server, or to the monitor IP address or hostname in the -context of :ref:`pg_autoctl_create_postgres`. - -:: - - $ pg_autoctl do show ipaddr - 16:42:40 62631 INFO ipaddr.c:107: Connecting to 8.8.8.8 (port 53) - 192.168.1.156 - -pg_autoctl do show cidr ------------------------ - -Connects to an external IP address in the same way as the previous command -``pg_autoctl do show ipaddr`` and then matches the local socket name with -the list of local network interfaces. When a match is found, uses the -netmask of the interface to compute the CIDR notation from the IP address. - -The computed CIDR notation is then used in HBA rules. - -:: - - $ pg_autoctl do show cidr - 16:43:19 63319 INFO Connecting to 8.8.8.8 (port 53) - 192.168.1.0/24 - - -pg_autoctl do show hostname ---------------------------- - -Uses either its first (and only) argument or the result of -``gethostname(2)`` as the candidate hostname to use in HBA rules, and then -check that the hostname resolves to an IP address that belongs to one of the -machine network interfaces. - -When the hostname forward-dns lookup resolves to an IP address that is local -to the node where the command is run, then a reverse-lookup from the IP -address is made to see if it matches with the candidate hostname. - -:: - - $ pg_autoctl do show hostname - DESKTOP-IC01GOOS.europe.corp.microsoft.com - - $ pg_autoctl -vv do show hostname 'postgres://autoctl_node@localhost:5500/pg_auto_failover' - 13:45:00 93122 INFO cli_do_show.c:256: Using monitor hostname "localhost" and port 5500 - 13:45:00 93122 INFO ipaddr.c:107: Connecting to ::1 (port 5500) - 13:45:00 93122 DEBUG cli_do_show.c:272: cli_show_hostname: ip ::1 - 13:45:00 93122 DEBUG cli_do_show.c:283: cli_show_hostname: host localhost - 13:45:00 93122 DEBUG cli_do_show.c:294: cli_show_hostname: ip ::1 - localhost - -pg_autoctl do show lookup -------------------------- - -Checks that the given argument is an hostname that resolves to a local IP -address, that is an IP address associated with a local network interface. - -:: - - $ pg_autoctl do show lookup DESKTOP-IC01GOOS.europe.corp.microsoft.com - DESKTOP-IC01GOOS.europe.corp.microsoft.com: 192.168.1.156 - -pg_autoctl do show reverse --------------------------- - -Implements the same DNS checks as Postgres HBA matching code: first does a -forward DNS lookup of the given hostname, and then a reverse-lookup from all -the IP addresses obtained. Success is reached when at least one of the IP -addresses from the forward lookup resolves back to the given hostname (as -the first answer to the reverse DNS lookup). - -:: - - $ pg_autoctl do show reverse DESKTOP-IC01GOOS.europe.corp.microsoft.com - 16:44:49 64910 FATAL Failed to find an IP address for hostname "DESKTOP-IC01GOOS.europe.corp.microsoft.com" that matches hostname again in a reverse-DNS lookup. - 16:44:49 64910 INFO Continuing with IP address "192.168.1.156" - - $ pg_autoctl -vv do show reverse DESKTOP-IC01GOOS.europe.corp.microsoft.com - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 192.168.1.156 - 16:44:45 64832 DEBUG ipaddr.c:733: reverse lookup for "192.168.1.156" gives "desktop-ic01goos.europe.corp.microsoft.com" first - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 192.168.1.156 - 16:44:45 64832 DEBUG ipaddr.c:733: reverse lookup for "192.168.1.156" gives "desktop-ic01goos.europe.corp.microsoft.com" first - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 2a01:110:10:40c::2ad - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 2a01:110:10:40c::2ad - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 100.64.34.213 - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 DEBUG ipaddr.c:719: DESKTOP-IC01GOOS.europe.corp.microsoft.com has address 100.64.34.213 - 16:44:45 64832 DEBUG ipaddr.c:728: Failed to resolve hostname from address "192.168.1.156": nodename nor servname provided, or not known - 16:44:45 64832 FATAL cli_do_show.c:333: Failed to find an IP address for hostname "DESKTOP-IC01GOOS.europe.corp.microsoft.com" that matches hostname again in a reverse-DNS lookup. - 16:44:45 64832 INFO cli_do_show.c:334: Continuing with IP address "192.168.1.156" diff --git a/docs/ref/pg_autoctl_inspect.rst b/docs/ref/pg_autoctl_inspect.rst new file mode 100644 index 000000000..a9c7c009d --- /dev/null +++ b/docs/ref/pg_autoctl_inspect.rst @@ -0,0 +1,60 @@ +.. _pg_autoctl_inspect: + +pg_autoctl inspect +================== + +pg_autoctl inspect - Read-only diagnostics for a pg_auto_failover node + +The ``pg_autoctl inspect`` commands provide read-only access to local and +cluster state. They are always available — no ``PG_AUTOCTL_DEBUG`` environment +variable is required. All commands in this group are safe to run while +``pg_autoctl run`` is active. + +.. toctree:: + :maxdepth: 1 + + pg_autoctl_inspect_pgsetup + pg_autoctl_inspect_show + +``pg_autoctl inspect`` provides the following sub-command groups:: + + pg_autoctl inspect + + show Network and hostname diagnostics + + pgsetup Local PostgreSQL setup inspection + + fsm Display keeper FSM state and transitions (read-only) + + monitor Query the monitor's current state (read-only) + + getpid Get the pid of pg_autoctl sub-processes (services) + + pg_autoctl inspect pgsetup + pg_ctl Find a non-ambiguous pg_ctl program and Postgres version + discover Discover local PostgreSQL instance, if any + ready Return true if the local Postgres server is ready + wait Wait until the local Postgres server is ready + logs Outputs the Postgres startup logs + tune Compute and log some Postgres tuning options + + pg_autoctl inspect fsm + state Read the keeper's state from disk and display it + list List reachable FSM states from current state + gv Output the FSM as a .gv program suitable for graphviz/dot + + pg_autoctl inspect show + ipaddr Print this node's IP address information + cidr Print this node's CIDR information + lookup Print this node's DNS lookup information + hostname Print this node's default hostname + reverse Lookup given hostname and check reverse DNS setup + + pg_autoctl inspect monitor + get Get information from the monitor + parse-notification Parse a raw notification message + + pg_autoctl inspect monitor get + primary Get the primary node from pg_auto_failover in given formation/group + others Get the other nodes from the pg_auto_failover group of hostname/port + coordinator Get the coordinator node from the pg_auto_failover formation + + pg_autoctl inspect getpid + postgres Get the pid of the pg_autoctl postgres controller service + listener Get the pid of the pg_autoctl monitor listener service + node-active Get the pid of the pg_autoctl keeper node-active service diff --git a/docs/ref/pg_autoctl_inspect_pgsetup.rst b/docs/ref/pg_autoctl_inspect_pgsetup.rst new file mode 100644 index 000000000..691d5812b --- /dev/null +++ b/docs/ref/pg_autoctl_inspect_pgsetup.rst @@ -0,0 +1,116 @@ +.. _pg_autoctl_inspect_pgsetup: + +pg_autoctl inspect pgsetup +========================== + +pg_autoctl inspect pgsetup - Inspect the local Postgres setup + +Synopsis +-------- + +``pg_autoctl inspect pgsetup`` provides low-level management tooling for a +local Postgres instance. These commands are always available without +``PG_AUTOCTL_DEBUG``. + +:: + + pg_autoctl inspect pgsetup + pg_ctl Find a non-ambiguous pg_ctl program and Postgres version + discover Discover local PostgreSQL instance, if any + ready Return true if the local Postgres server is ready + wait Wait until the local Postgres server is ready + logs Outputs the Postgres startup logs + tune Compute and log some Postgres tuning options + +pg_autoctl inspect pgsetup pg_ctl +--------------------------------- + +In a similar way to ``which -a``, this command scans your PATH for +``pg_ctl`` commands. Then it runs ``pg_ctl --version`` and parses the output +to determine the version of Postgres available. + +:: + + $ pg_autoctl inspect pgsetup pg_ctl --pgdata node1 + 16:49:18 69684 INFO `pg_autoctl create postgres` would use "/usr/lib/postgresql/17/bin/pg_ctl" for Postgres 17 + + +pg_autoctl inspect pgsetup discover +------------------------------------ + +Given a PGDATA or ``--pgdata`` option, the command discovers whether a +running Postgres service matches the pg_autoctl setup and prints the +information that ``pg_autoctl`` typically needs. + +:: + + $ pg_autoctl inspect pgsetup discover --pgdata node1 + pgdata: /home/postgres/node1 + pg_ctl: /usr/lib/postgresql/17/bin/pg_ctl + pg_version: 17 + pghost: /tmp + pgport: 5501 + pid: 21029 + is in recovery: no + Postmaster status: ready + + +pg_autoctl inspect pgsetup ready +--------------------------------- + +Similar to `pg_isready`__, but uses the Postgres specifications found in the +pg_autoctl node setup. + +__ https://www.postgresql.org/docs/current/app-pg-isready.html + +:: + + $ pg_autoctl inspect pgsetup ready --pgdata node1 + 16:50:08 70582 INFO Postgres status is: "ready" + + +pg_autoctl inspect pgsetup wait +-------------------------------- + +When ``pg_autoctl inspect pgsetup ready`` would return false because Postgres +is not ready yet, this command probes every second for up to 30 seconds and +exits as soon as Postgres is ready. + +:: + + $ pg_autoctl inspect pgsetup wait --pgdata node1 + 16:50:22 70829 INFO Postgres is now serving PGDATA "/home/postgres/node1" on port 5501 with pid 21029 + 16:50:22 70829 INFO Postgres status is: "ready" + + +pg_autoctl inspect pgsetup logs +-------------------------------- + +Outputs the Postgres logs from the most recent log file in the +``PGDATA/log`` directory. + +:: + + $ pg_autoctl inspect pgsetup logs --pgdata node1 + 16:50:39 71126 WARN Postgres logs from "/home/postgres/node1/startup.log": + ... + + +pg_autoctl inspect pgsetup tune +-------------------------------- + +Outputs the pg_autoctl automated tuning options. Depending on the number of +CPUs and amount of RAM detected, ``pg_autoctl`` adjusts basic Postgres tuning +knobs. + +:: + + $ pg_autoctl inspect pgsetup tune --pgdata node1 -vv + 13:25:25 77185 DEBUG Detected 12 CPUs and 16 GB total RAM on this server + 13:25:25 77185 DEBUG Setting shared_buffers to 4096 MB + # basic tuning computed by pg_auto_failover + shared_buffers = '4096 MB' + work_mem = '24 MB' + maintenance_work_mem = '512 MB' + effective_cache_size = '12 GB' + autovacuum_max_workers = 3 diff --git a/docs/ref/pg_autoctl_inspect_show.rst b/docs/ref/pg_autoctl_inspect_show.rst new file mode 100644 index 000000000..d330b1986 --- /dev/null +++ b/docs/ref/pg_autoctl_inspect_show.rst @@ -0,0 +1,90 @@ +.. _pg_autoctl_inspect_show: + +pg_autoctl inspect show +======================= + +pg_autoctl inspect show - Network and hostname diagnostics + +Synopsis +-------- + +The commands :ref:`pg_autoctl_create_monitor` and +:ref:`pg_autoctl_create_postgres` both implement automated detection of node +network settings when the option ``--hostname`` is not used. When a new node +is registered to the monitor, other nodes also update their HBA rules to +allow the new node to connect. + +``pg_autoctl inspect show`` exposes the network discovery logic so that +operators can verify how ``pg_autoctl`` sees the local host. + +:: + + pg_autoctl inspect show + ipaddr Print this node's IP address information + cidr Print this node's CIDR information + lookup Print this node's DNS lookup information + hostname Print this node's default hostname + reverse Lookup given hostname and check reverse DNS setup + +pg_autoctl inspect show ipaddr +------------------------------ + +Connects to an external IP address and uses ``getsockname(2)`` to retrieve +the current address to which the socket is bound. The external IP defaults +to ``8.8.8.8``, or to the monitor IP/hostname in the context of +:ref:`pg_autoctl_create_postgres`. + +:: + + $ pg_autoctl inspect show ipaddr + 192.168.1.156 + +pg_autoctl inspect show cidr +----------------------------- + +Connects to an external IP address in the same way as ``inspect show ipaddr`` +and then matches the local socket name with the list of local network +interfaces. When a match is found, uses the netmask of the interface to +compute the CIDR notation from the IP address. The computed CIDR is used in +HBA rules. + +:: + + $ pg_autoctl inspect show cidr + 192.168.1.0/24 + +pg_autoctl inspect show hostname +--------------------------------- + +Uses either its first argument or the result of ``gethostname(2)`` as the +candidate hostname for HBA rules, then checks that the hostname resolves to +an IP address that belongs to one of the machine's network interfaces. + +:: + + $ pg_autoctl inspect show hostname + node1.example.com + +pg_autoctl inspect show lookup +------------------------------- + +Checks that the given argument is a hostname that resolves to a local IP +address (an IP address associated with a local network interface). + +:: + + $ pg_autoctl inspect show lookup node1.example.com + node1.example.com: 192.168.1.156 + +pg_autoctl inspect show reverse +-------------------------------- + +Implements the same DNS checks as Postgres HBA matching code: first does a +forward DNS lookup of the given hostname, then a reverse-lookup from all the +IP addresses obtained. Success is reached when at least one IP address from +the forward lookup resolves back to the given hostname. + +:: + + $ pg_autoctl inspect show reverse node1.example.com + node1.example.com: 192.168.1.156 diff --git a/docs/ref/pg_autoctl_manual.rst b/docs/ref/pg_autoctl_manual.rst new file mode 100644 index 000000000..1ce1da6a3 --- /dev/null +++ b/docs/ref/pg_autoctl_manual.rst @@ -0,0 +1,88 @@ +.. _pg_autoctl_manual: + +pg_autoctl manual +================= + +pg_autoctl manual - Operator-driven FSM operations for manual cluster recovery + +The ``pg_autoctl manual`` commands provide low-level control over the +pg_auto_failover finite state machine and supporting services. They are +always available — no ``PG_AUTOCTL_DEBUG`` environment variable is required. + +These commands are intended for manual recovery when the automated FSM is +stopped or stuck, and for low-level diagnostic work. Using them while +``pg_autoctl run`` is active can interfere with automated operations. + +.. toctree:: + :maxdepth: 1 + + pg_autoctl_manual_service_restart + +``pg_autoctl manual`` provides the following sub-command groups:: + + pg_autoctl manual + + fsm Manually drive the keeper FSM (mutating operations) + + service Restart pg_autoctl sub-processes or signal the postgres controller + + monitor Manually drive monitor RPCs (register / active / version) + + primary Manage a PostgreSQL primary server + + standby Manage a PostgreSQL standby server + + coordinator Manage Citus coordinator node metadata + + pg_autoctl manual fsm + init Initialize the keeper's state on-disk + assign Assign a new goal state to the keeper + step Make a state transition if instructed by the monitor + + nodes Manually manage the keeper's nodes list + + pg_autoctl manual fsm nodes + get Get the list of nodes from file (see --disable-monitor) + set Set the list of nodes to file (see --disable-monitor) + + pg_autoctl manual service + + restart Restart pg_autoctl sub-processes (services) + + pgctl Signal the pg_autoctl postgres controller + + pg_autoctl manual service restart + postgres Restart the pg_autoctl postgres controller service + listener Restart the pg_autoctl monitor listener service + node-active Restart the pg_autoctl keeper node-active service + + pg_autoctl manual service pgctl + on Signal pg_autoctl postgres service to ensure Postgres is running + off Signal pg_autoctl postgres service to ensure Postgres is stopped + + pg_autoctl manual monitor + register Register the current node with the monitor + active Call in the pg_auto_failover Node Active protocol + version Check that monitor version is current; alter extension update if not + + pg_autoctl manual primary + + slot Manage replication slot on the primary server + + adduser Create users on primary + defaults Add default settings to postgresql.conf + identify Run the IDENTIFY_SYSTEM replication command on given host + + pg_autoctl manual primary slot + create Create a replication slot on the primary server + drop Drop a replication slot on the primary server + + pg_autoctl manual primary adduser + monitor Add a local user for queries from the monitor + replica Add a local user with replication privileges + + pg_autoctl manual standby + init Initialize the standby server using pg_basebackup + rewind Rewind a demoted primary server using pg_rewind + crash-recovery Setup postgres for crash-recovery and start postgres + promote Promote a standby server to become writable + + pg_autoctl manual coordinator + add Add this node to its formation's coordinator + activate Activate this node on its formation's coordinator + remove Remove this node from its formation's coordinator + + update Update current node's host:port on the coordinator + + pg_autoctl manual coordinator update + prepare Prepare a Citus coordinator metadata update + commit Commit a Citus coordinator metadata update + rollback Rollback a Citus coordinator metadata update diff --git a/docs/ref/pg_autoctl_manual_service_restart.rst b/docs/ref/pg_autoctl_manual_service_restart.rst new file mode 100644 index 000000000..af15b7dc7 --- /dev/null +++ b/docs/ref/pg_autoctl_manual_service_restart.rst @@ -0,0 +1,32 @@ +.. _pg_autoctl_manual_service_restart: + +pg_autoctl manual service restart +================================== + +pg_autoctl manual service restart - Restart pg_autoctl sub-processes + +Synopsis +-------- + +:: + + pg_autoctl manual service restart + postgres Restart the pg_autoctl postgres controller service + listener Restart the pg_autoctl monitor listener service + node-active Restart the pg_autoctl keeper node-active service + + +Description +----------- + +It is possible to restart the ``pg_autoctl`` keeper or listener service +without affecting the other running services, and without stopping Postgres. +Typically, to restart the ``pg_autoctl`` keeper without impacting Postgres:: + + $ pg_autoctl manual service restart node-active --pgdata node1 + 14:52:06 31223 INFO Sending the TERM signal to service "node-active" with pid 26626 + 14:52:06 31223 INFO Service "node-active" has been restarted with pid 31230 + 31230 + +The Postgres service is not impacted by the restart of the ``pg_autoctl`` +keeper process. diff --git a/src/bin/lib/subcommands.c/commandline.c b/src/bin/lib/subcommands.c/commandline.c index bd9eac138..f83c85ede 100644 --- a/src/bin/lib/subcommands.c/commandline.c +++ b/src/bin/lib/subcommands.c/commandline.c @@ -226,6 +226,9 @@ commandline_pretty_print_subcommands(CommandLine *command, FILE *stream) /* pretty printing: reduce maximum length of subcommand names */ for (subcommand = command->subcommands; *subcommand != NULL; subcommand++) { + if ((*subcommand)->hidden) + continue; + int len = strlen((*subcommand)->name); if (maxLength < len) @@ -238,6 +241,9 @@ commandline_pretty_print_subcommands(CommandLine *command, FILE *stream) { const char *description = ""; + if ((*subcommand)->hidden) + continue; + if ((*subcommand)->shortDescription != NULL) { description = (*subcommand)->shortDescription; diff --git a/src/bin/lib/subcommands.c/commandline.h b/src/bin/lib/subcommands.c/commandline.h index 888e58550..a0cb674b9 100644 --- a/src/bin/lib/subcommands.c/commandline.h +++ b/src/bin/lib/subcommands.c/commandline.h @@ -29,15 +29,25 @@ typedef struct CommandLine struct CommandLine **subcommands; char *breadcrumb; + bool hidden; /* if true, omit from --help output */ } CommandLine; extern CommandLine *current_command; #define make_command_set(name, desc, usage, help, getopt, set) \ - { name, desc, usage, help, getopt, NULL, set, NULL } + { name, desc, usage, help, getopt, NULL, set, NULL, false } #define make_command(name, desc, usage, help, getopt, run) \ - { name, desc, usage, help, getopt, run, NULL, NULL } + { name, desc, usage, help, getopt, run, NULL, NULL, false } + +/* + * Like make_command_set but the command is omitted from --help output. + * The command still routes normally when invoked explicitly — used for + * internal subprocess entry points that must remain in the binary but + * are not intended for direct operator use. + */ +#define make_hidden_command_set(name, desc, usage, help, getopt, set) \ + { name, desc, usage, help, getopt, NULL, set, NULL, true } bool commandline_run(CommandLine *command, int argc, char **argv); void commandline_help(FILE *stream); diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index d576cdef9..4b835ab28 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -1422,11 +1422,6 @@ keeper_cli_help(int argc, char **argv) { CommandLine command = root; - if (env_exists(PG_AUTOCTL_DEBUG)) - { - command = root_with_debug; - } - (void) commandline_print_command_tree(&command, stdout); } diff --git a/src/bin/pg_autoctl/cli_do_coordinator.c b/src/bin/pg_autoctl/cli_do_coordinator.c index 72f5dfb55..5506fa29e 100644 --- a/src/bin/pg_autoctl/cli_do_coordinator.c +++ b/src/bin/pg_autoctl/cli_do_coordinator.c @@ -108,7 +108,7 @@ static CommandLine *coordinator_commands[] = { CommandLine do_coordinator_commands = make_command_set("coordinator", - "Query a Citus coordinator", NULL, NULL, + "Manage Citus coordinator node metadata", NULL, NULL, NULL, coordinator_commands); diff --git a/src/bin/pg_autoctl/cli_do_fsm.c b/src/bin/pg_autoctl/cli_do_fsm.c index 8c456ff73..02fbfa337 100644 --- a/src/bin/pg_autoctl/cli_do_fsm.c +++ b/src/bin/pg_autoctl/cli_do_fsm.c @@ -37,7 +37,7 @@ static void cli_do_fsm_step(int argc, char **argv); static void cli_do_fsm_get_nodes(int argc, char **argv); static void cli_do_fsm_set_nodes(int argc, char **argv); -static CommandLine fsm_init = +CommandLine fsm_init = make_command("init", "Initialize the keeper's state on-disk", CLI_PGDATA_USAGE, @@ -45,7 +45,7 @@ static CommandLine fsm_init = cli_getopt_pgdata, cli_do_fsm_init); -static CommandLine fsm_state = +CommandLine fsm_state = make_command("state", "Read the keeper's state from disk and display it", CLI_PGDATA_USAGE, @@ -53,7 +53,7 @@ static CommandLine fsm_state = cli_getopt_pgdata, cli_do_fsm_state); -static CommandLine fsm_list = +CommandLine fsm_list = make_command("list", "List reachable FSM states from current state", CLI_PGDATA_USAGE, @@ -61,12 +61,12 @@ static CommandLine fsm_list = cli_getopt_pgdata, cli_do_fsm_list); -static CommandLine fsm_gv = +CommandLine fsm_gv = make_command("gv", "Output the FSM as a .gv program suitable for graphviz/dot", "", NULL, NULL, cli_do_fsm_gv); -static CommandLine fsm_assign = +CommandLine fsm_assign = make_command("assign", "Assign a new goal state to the keeper", CLI_PGDATA_USAGE "", @@ -74,7 +74,7 @@ static CommandLine fsm_assign = cli_getopt_pgdata, cli_do_fsm_assign); -static CommandLine fsm_step = +CommandLine fsm_step = make_command("step", "Make a state transition if instructed by the monitor", CLI_PGDATA_USAGE, diff --git a/src/bin/pg_autoctl/cli_do_monitor.c b/src/bin/pg_autoctl/cli_do_monitor.c index f1dfb526c..65d9512c5 100644 --- a/src/bin/pg_autoctl/cli_do_monitor.c +++ b/src/bin/pg_autoctl/cli_do_monitor.c @@ -88,12 +88,12 @@ static CommandLine *monitor_get_commands[] = { NULL }; -static CommandLine monitor_get_command = +CommandLine monitor_get_command = make_command_set("get", "Get information from the monitor", NULL, NULL, NULL, monitor_get_commands); -static CommandLine monitor_register_command = +CommandLine monitor_register_command = make_command("register", "Register the current node with the monitor", CLI_PGDATA_USAGE "", @@ -101,7 +101,7 @@ static CommandLine monitor_register_command = cli_getopt_pgdata, cli_do_monitor_register_node); -static CommandLine monitor_node_active_command = +CommandLine monitor_node_active_command = make_command("active", "Call in the pg_auto_failover Node Active protocol", CLI_PGDATA_USAGE, @@ -109,7 +109,7 @@ static CommandLine monitor_node_active_command = cli_getopt_pgdata, cli_do_monitor_node_active); -static CommandLine monitor_version_command = +CommandLine monitor_version_command = make_command("version", "Check that monitor version is " PG_AUTOCTL_EXTENSION_VERSION @@ -119,7 +119,7 @@ static CommandLine monitor_version_command = cli_getopt_pgdata, cli_do_monitor_version); -static CommandLine monitor_parse_notification_command = +CommandLine monitor_parse_notification_command = make_command("parse-notification", "parse a raw notification message", " ", diff --git a/src/bin/pg_autoctl/cli_do_root.c b/src/bin/pg_autoctl/cli_do_root.c index ee711d485..a7f6a86f8 100644 --- a/src/bin/pg_autoctl/cli_do_root.c +++ b/src/bin/pg_autoctl/cli_do_root.c @@ -374,6 +374,35 @@ CommandLine do_tmux_commands = "Set of facilities to handle tmux interactive sessions", NULL, NULL, NULL, do_tmux); +/* + * internal service: hidden entry points spawned by the supervisor via + * fork+exec. The supervisor builds argv as: + * pg_autoctl internal service postgres|listener|node-active --pgdata ... + * Use make_hidden_command_set so these never appear in --help output. + */ +static CommandLine *internal_service_subcommands[] = { + &service_pgcontroller, + &service_postgres, + &service_monitor_listener, + &service_node_active, + NULL +}; + +CommandLine internal_service_commands = + make_hidden_command_set("service", + "Internal subprocess entry points (supervisor use only)", + NULL, NULL, NULL, internal_service_subcommands); + +static CommandLine *internal_subcommands[] = { + &internal_service_commands, + NULL +}; + +CommandLine internal_commands = + make_hidden_command_set("internal", + "Internal commands for use by the supervisor (not for operators)", + NULL, NULL, NULL, internal_subcommands); + CommandLine *do_subcommands[] = { &do_monitor_commands, &do_coordinator_commands, diff --git a/src/bin/pg_autoctl/cli_do_root.h b/src/bin/pg_autoctl/cli_do_root.h index 4de118c88..bc129ad04 100644 --- a/src/bin/pg_autoctl/cli_do_root.h +++ b/src/bin/pg_autoctl/cli_do_root.h @@ -1,7 +1,7 @@ /* * src/bin/pg_autoctl/cli_do_root.h - * Implementation of a CLI which lets you run individual keeper routines - * directly + * Implementation of a CLI which lets you run operations on the local + * postgres server directly * * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the PostgreSQL License. @@ -15,43 +15,59 @@ /* src/bin/pg_autoctl/cli_do_fsm.c */ extern CommandLine do_fsm_commands; +extern CommandLine fsm_nodes; + +/* Exported individually so cli_inspect.c and cli_manual.c can compose them */ +extern CommandLine fsm_init; +extern CommandLine fsm_state; +extern CommandLine fsm_list; +extern CommandLine fsm_gv; +extern CommandLine fsm_assign; +extern CommandLine fsm_step; /* src/bin/pg_autoctl/cli_do_monitor.c */ extern CommandLine do_monitor_commands; +/* Exported individually so cli_inspect.c and cli_manual.c can compose them */ +extern CommandLine monitor_get_command; +extern CommandLine monitor_parse_notification_command; +extern CommandLine monitor_register_command; +extern CommandLine monitor_node_active_command; +extern CommandLine monitor_version_command; + /* src/bin/pg_autoctl/cli_do_service.c */ extern CommandLine do_service_commands; +extern CommandLine do_service_restart_commands; +extern CommandLine do_service_getpid_commands; extern CommandLine do_service_postgres_ctl_commands; +/* Internal subprocess entry points used in cli_do_root.c's internal_service_commands */ +extern CommandLine service_pgcontroller; +extern CommandLine service_postgres; +extern CommandLine service_monitor_listener; +extern CommandLine service_node_active; + /* src/bin/pg_autoctl/cli_do_show.c */ extern CommandLine do_show_commands; extern CommandLine do_pgsetup_commands; -extern CommandLine do_service_postgres_ctl_commands; -extern CommandLine do_service_commands; /* src/bin/pg_autoctl/cli_do_demo.c */ extern CommandLine do_demo_commands; +/* src/bin/pg_autoctl/cli_do_coordinator.c */ +extern CommandLine do_coordinator_commands; + /* src/bin/pg_autoctl/cli_do_root.c */ extern CommandLine do_primary_adduser; extern CommandLine *do_primary_adduser_subcommands[]; extern CommandLine do_primary_adduser_monitor; extern CommandLine do_primary_adduser_replica; -extern CommandLine do_primary_syncrep_; -extern CommandLine *do_primary_syncrep[]; -extern CommandLine do_primary_syncrep_enable; -extern CommandLine do_primary_syncrep_disable; - extern CommandLine do_primary_slot_; extern CommandLine *do_primary_slot[]; extern CommandLine do_primary_slot_create; extern CommandLine do_primary_slot_drop; -extern CommandLine do_primary_hba; -extern CommandLine *do_primary_hba_commands[]; -extern CommandLine do_primary_hba_setup; - extern CommandLine do_primary_defaults; extern CommandLine do_primary_identify_system; @@ -64,12 +80,10 @@ extern CommandLine do_standby_init; extern CommandLine do_standby_rewind; extern CommandLine do_standby_promote; -extern CommandLine do_discover; - extern CommandLine do_tmux_commands; -/* src/bin/pg_autoctl/cli_do_coordinator.c */ -extern CommandLine do_coordinator_commands; +extern CommandLine internal_service_commands; +extern CommandLine internal_commands; extern CommandLine do_commands; extern CommandLine *do_subcommands[]; diff --git a/src/bin/pg_autoctl/cli_do_service.c b/src/bin/pg_autoctl/cli_do_service.c index 94420b7c5..e450417ca 100644 --- a/src/bin/pg_autoctl/cli_do_service.c +++ b/src/bin/pg_autoctl/cli_do_service.c @@ -357,8 +357,8 @@ cli_do_service_restart_node_active(int argc, char **argv) * tree. It is used for debug purposes only. When using this entry point we * have a supervisor process that is responsible for only one service: * - * pg_autoctl do service pgcontroller - * - pg_autoctl do service postgres + * pg_autoctl internal service pgcontroller + * - pg_autoctl internal service postgres * - postgres */ static void @@ -401,9 +401,9 @@ cli_do_service_pgcontroller(int argc, char **argv) * that supervises two sub-processes, one of them is cli_do_service_postgres: * * pg_autoctl - * - pg_autoctl do service postgres + * - pg_autoctl internal service postgres * - postgres - * - pg_autoctl do service keeper|monitor + * - pg_autoctl internal service keeper|monitor */ static void cli_do_service_postgres(int argc, char **argv) diff --git a/src/bin/pg_autoctl/cli_inspect.c b/src/bin/pg_autoctl/cli_inspect.c new file mode 100644 index 000000000..968c471f2 --- /dev/null +++ b/src/bin/pg_autoctl/cli_inspect.c @@ -0,0 +1,76 @@ +/* + * src/bin/pg_autoctl/cli_inspect.c + * pg_autoctl inspect — read-only diagnostics, always visible. + * + * All commands here read local or cluster state without mutating anything. + * Safe to run at any time, even while `pg_autoctl run` is active. + * + * For mutating recovery commands (fsm assign, standby promote, etc.) see + * cli_manual.c ("pg_autoctl manual …"). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include "commandline.h" +#include "cli_inspect.h" +#include "cli_do_root.h" + +/* + * Read-only FSM sub-commands: display the current state, list reachable + * transitions, or dump the full FSM as a graphviz .gv file. + * Mutating operations (init, assign, step, nodes set) live under "manual fsm". + */ +static CommandLine *inspect_fsm_subcommands[] = { + &fsm_state, + &fsm_list, + &fsm_gv, + NULL +}; + +static CommandLine inspect_fsm_commands = + make_command_set("fsm", + "Display keeper FSM state and transitions (read-only)", + NULL, NULL, NULL, inspect_fsm_subcommands); + +/* + * Read-only monitor sub-commands: get primary/others/candidate-count/coordinator + * and parse-notification. We intentionally exclude "register", "active", and + * "version" (ALTER EXTENSION) which are mutating — those live under "manual". + */ +static CommandLine *inspect_monitor_subcommands[] = { + &monitor_get_command, + &monitor_parse_notification_command, + NULL +}; + +static CommandLine inspect_monitor_commands = + make_command_set("monitor", + "Query the monitor's current state (read-only)", + NULL, NULL, NULL, inspect_monitor_subcommands); + +/* + * Aggregate the read-only do_* command sets under "inspect". + * + * service here is only "getpid" — inspecting PIDs of running sub-processes. + * "restart" and "pgctl on/off" mutate state and belong under "manual service". + */ +static CommandLine *inspect_subcommands[] = { + &do_show_commands, /* ipaddr / cidr / lookup / hostname / reverse */ + &do_pgsetup_commands, /* discover / ready / wait / logs / tune / pg_ctl */ + &inspect_fsm_commands, /* state / list / gv */ + &inspect_monitor_commands, /* get primary|others|candidate-count + parse-notification */ + &do_service_getpid_commands, /* getpid postgres|listener|node-active */ + NULL +}; + +CommandLine inspect_commands = + make_command_set("inspect", + "Read-only diagnostics (safe on live nodes)", + "[sub-command]", + " show Networking and hostname diagnostics\n" + " pgsetup Local PostgreSQL setup inspection\n" + " fsm Display keeper FSM state and reachable transitions\n" + " monitor Query the monitor's current state\n" + " getpid Get PIDs of pg_autoctl sub-processes\n", + NULL, inspect_subcommands); diff --git a/src/bin/pg_autoctl/cli_inspect.h b/src/bin/pg_autoctl/cli_inspect.h new file mode 100644 index 000000000..b9c87d68e --- /dev/null +++ b/src/bin/pg_autoctl/cli_inspect.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_inspect.h + * pg_autoctl inspect — read-only diagnostics. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_INSPECT_H +#define CLI_INSPECT_H + +#include "commandline.h" + +extern CommandLine inspect_commands; + +#endif /* CLI_INSPECT_H */ diff --git a/src/bin/pg_autoctl/cli_manual.c b/src/bin/pg_autoctl/cli_manual.c new file mode 100644 index 000000000..f309a948e --- /dev/null +++ b/src/bin/pg_autoctl/cli_manual.c @@ -0,0 +1,128 @@ +/* + * src/bin/pg_autoctl/cli_manual.c + * pg_autoctl manual — operator-driven FSM operations and low-level controls. + * + * These commands let an operator manually drive individual FSM transitions + * or low-level operations that pg_autoctl automation would normally own. + * Intended for manual recovery when the automated FSM is stopped or stuck, + * or for driving a live upgrade step-by-step. + * + * "manual" is the antonym of the automated behaviour pg_autoctl normally + * provides: you are doing by hand what the system would otherwise do for you. + * + * Read-only diagnostics belong in "pg_autoctl inspect": + * inspect fsm state / list / gv + * inspect monitor get / parse-notification + * inspect getpid postgres / listener / node-active + * + * Internal subprocess entry points belong in the hidden "pg_autoctl do". + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#include "commandline.h" +#include "cli_manual.h" +#include "cli_do_root.h" + +/* + * manual fsm: mutating FSM operations only. + * + * Read-only FSM commands (state, list, gv) live under "pg_autoctl inspect fsm" + * because they are safe to run on a live node. + * + * nodes (get + set) is kept as a unit under manual because both sub-commands + * target the offline nodes file (--disable-monitor), making them equally + * operator-oriented regardless of read/write direction. + */ +static CommandLine *manual_fsm_subcommands[] = { + &fsm_init, + &fsm_assign, + &fsm_step, + &fsm_nodes, /* nodes get + nodes set (--disable-monitor file) */ + NULL +}; + +static CommandLine manual_fsm_commands = + make_command_set("fsm", + "Manually drive the keeper FSM (mutating operations)", + NULL, NULL, NULL, manual_fsm_subcommands); + +/* + * manual service: user-visible service controls only. + * + * getpid is read-only and lives under "pg_autoctl inspect getpid". + * + * Intentionally excludes the internal subprocess entry points + * (pgcontroller / postgres / listener / node-active) which are spawned by the + * supervisor via fork+exec and are not meant for direct operator use. + * Those live under the hidden "pg_autoctl internal service" group. + */ +static CommandLine *manual_service_subcommands[] = { + &do_service_restart_commands, /* restart postgres|listener|node-active */ + &do_service_postgres_ctl_commands, /* pgctl on|off */ + NULL +}; + +static CommandLine manual_service_commands = + make_command_set("service", + "Restart pg_autoctl sub-processes or signal the postgres controller", + NULL, NULL, NULL, manual_service_subcommands); + +/* + * manual monitor: mutating monitor operations. + * + * "get" and "parse-notification" are read-only and live under + * "pg_autoctl inspect monitor". Here we expose only the mutating calls: + * + * register — manually step through the node registration protocol + * active — manually call the node_active RPC on the monitor + * version — check monitor extension version and ALTER EXTENSION UPDATE if needed + */ +static CommandLine *manual_monitor_subcommands[] = { + &monitor_register_command, + &monitor_node_active_command, + &monitor_version_command, + NULL +}; + +static CommandLine manual_monitor_commands = + make_command_set("monitor", + "Manually drive monitor RPCs (register / active / version)", + NULL, NULL, NULL, manual_monitor_subcommands); + +/* + * manual coordinator: Citus coordinator metadata management. + * + * These commands let an operator manually replay the coordinator update steps + * that pg_autoctl normally drives automatically during a Citus worker failover: + * + * coordinator update prepare — call master_update_node() in a prepared + * transaction, which blocks shard writes while open + * coordinator update commit — COMMIT PREPARED: makes the new address permanent + * coordinator update rollback — ROLLBACK PREPARED: abandons a stuck prepare + * + * They are typically needed when the coordinator was unavailable during a + * failover and the prepared transaction needs to be resolved out-of-band. + */ +static CommandLine *manual_subcommands[] = { + &manual_fsm_commands, /* init / assign / step / nodes */ + &manual_service_commands, /* restart / pgctl on|off */ + &manual_monitor_commands, /* register / active / version */ + &do_primary_, /* slot create|drop / adduser monitor|replica / defaults / identify */ + &do_standby_, /* init / rewind / crash-recovery / promote */ + &do_coordinator_commands, /* add / activate / remove / update prepare|commit|rollback */ + NULL +}; + +CommandLine manual_commands = + make_command_set("manual", + "Manual FSM operations — drive by hand what automation normally does", + "[sub-command]", + " fsm Manually drive keeper FSM transitions\n" + " service Restart sub-processes or signal the postgres controller\n" + " monitor Manually drive monitor registration protocol\n" + " primary Manual primary-side PostgreSQL operations\n" + " standby Manual standby-side PostgreSQL operations\n" + " coordinator Citus coordinator metadata management\n", + NULL, manual_subcommands); diff --git a/src/bin/pg_autoctl/cli_manual.h b/src/bin/pg_autoctl/cli_manual.h new file mode 100644 index 000000000..d54ec7abd --- /dev/null +++ b/src/bin/pg_autoctl/cli_manual.h @@ -0,0 +1,16 @@ +/* + * src/bin/pg_autoctl/cli_manual.h + * pg_autoctl manual — operator-driven FSM operations and low-level controls. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + */ + +#ifndef CLI_MANUAL_H +#define CLI_MANUAL_H + +#include "commandline.h" + +extern CommandLine manual_commands; + +#endif /* CLI_MANUAL_H */ diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 386716b98..dd5029687 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -9,6 +9,9 @@ */ #include "cli_common.h" +#include "cli_do_root.h" +#include "cli_inspect.h" +#include "cli_manual.h" #include "cli_root.h" #include "commandline.h" @@ -81,36 +84,10 @@ CommandLine drop_commands = NULL, drop_subcommands); /* - * Binding them all into the top-level command: + * Single root command table — inspect and override are always visible. + * The PG_AUTOCTL_DEBUG env var now only controls log verbosity, not + * command visibility. */ -CommandLine *root_subcommands_with_debug[] = { - &create_commands, - &drop_commands, - &config_commands, - &show_commands_with_debug, - &enable_commands, - &disable_commands, - &get_commands, - &set_commands, - &perform_commands, - &do_commands, - &service_run_command, - &watch_command, - &service_stop_command, - &service_reload_command, - &service_status_command, - &help, - &version, - NULL -}; - -CommandLine root_with_debug = - make_command_set("pg_autoctl", - "pg_auto_failover control tools and service", - "[ --debug|verbose|quiet ]", NULL, - root_options, root_subcommands_with_debug); - - CommandLine *root_subcommands[] = { &create_commands, &drop_commands, @@ -122,6 +99,11 @@ CommandLine *root_subcommands[] = { &set_commands, &perform_commands, &activate_node_command, + &inspect_commands, + &manual_commands, + &internal_commands, + + &do_commands, &service_run_command, &watch_command, &service_stop_command, diff --git a/src/bin/pg_autoctl/cli_root.h b/src/bin/pg_autoctl/cli_root.h index 7b09cff48..03d97639a 100644 --- a/src/bin/pg_autoctl/cli_root.h +++ b/src/bin/pg_autoctl/cli_root.h @@ -40,9 +40,6 @@ extern CommandLine *show_subcommands_with_debug[]; extern CommandLine drop_commands; extern CommandLine *drop_subcommands[]; -extern CommandLine root_with_debug; -extern CommandLine *root_subcommands_with_debug[]; - extern CommandLine root; extern CommandLine *root_subcommands[]; diff --git a/src/bin/pg_autoctl/main.c b/src/bin/pg_autoctl/main.c index a49dae7ed..577ecd540 100644 --- a/src/bin/pg_autoctl/main.c +++ b/src/bin/pg_autoctl/main.c @@ -64,15 +64,6 @@ main(int argc, char **argv) /* register our logging clean-up atexit */ atexit(log_semaphore_unlink_atexit); - /* - * When PG_AUTOCTL_DEBUG is set in the environment, provide the user - * commands available to debug a pg_autoctl instance. - */ - if (env_exists(PG_AUTOCTL_DEBUG)) - { - command = root_with_debug; - } - /* * When PGCONNECT_TIMEOUT is set in the environment, keep a copy of it in * our own global variable pgconnect_timeout. We implement our own diff --git a/src/bin/pg_autoctl/service_keeper.c b/src/bin/pg_autoctl/service_keeper.c index 5c3d4b557..3b563e211 100644 --- a/src/bin/pg_autoctl/service_keeper.c +++ b/src/bin/pg_autoctl/service_keeper.c @@ -170,7 +170,7 @@ service_keeper_runprogram(Keeper *keeper) setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "node-active"; args[argsIndex++] = "--pgdata"; diff --git a/src/bin/pg_autoctl/service_monitor.c b/src/bin/pg_autoctl/service_monitor.c index e2a7f4e3a..b7ef22e6c 100644 --- a/src/bin/pg_autoctl/service_monitor.c +++ b/src/bin/pg_autoctl/service_monitor.c @@ -160,7 +160,7 @@ service_monitor_runprogram(Monitor *monitor) setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "listener"; args[argsIndex++] = "--pgdata"; diff --git a/src/bin/pg_autoctl/service_postgres_ctl.c b/src/bin/pg_autoctl/service_postgres_ctl.c index 82daa6863..f85cf04a2 100644 --- a/src/bin/pg_autoctl/service_postgres_ctl.c +++ b/src/bin/pg_autoctl/service_postgres_ctl.c @@ -92,7 +92,7 @@ service_postgres_ctl_start(void *context, pid_t *pid) /* * service_postgres_ctl_runprogram runs the postgres controller service: * - * $ pg_autoctl do service postgres --pgdata ... + * $ pg_autoctl internal service postgres --pgdata ... */ void service_postgres_ctl_runprogram() @@ -129,7 +129,7 @@ service_postgres_ctl_runprogram() setenv(PG_AUTOCTL_DEBUG, "1", 1); args[argsIndex++] = (char *) pg_autoctl_program; - args[argsIndex++] = "do"; + args[argsIndex++] = "internal"; args[argsIndex++] = "service"; args[argsIndex++] = "postgres"; args[argsIndex++] = "--pgdata"; diff --git a/tests/pgautofailover_utils.py b/tests/pgautofailover_utils.py index 0fe9f6a53..7aef6ecfe 100644 --- a/tests/pgautofailover_utils.py +++ b/tests/pgautofailover_utils.py @@ -520,12 +520,12 @@ def reload_postgres(self): def restart_postgres(self): """ - Restart Postgres with pg_autoctl do service restart postgres + Restart Postgres with pg_autoctl manual service restart postgres """ command = PGAutoCtl(self) command.execute( - "service restart postgres", "do", "service", "restart", "postgres" + "service restart postgres", "manual", "service", "restart", "postgres" ) def pg_is_running(self, timeout=COMMAND_TIMEOUT): @@ -535,7 +535,7 @@ def pg_is_running(self, timeout=COMMAND_TIMEOUT): command = PGAutoCtl(self) try: - command.execute("pgsetup ready", "do", "pgsetup", "ready", "-vvv") + command.execute("pgsetup ready", "inspect", "pgsetup", "ready", "-vvv") except Exception as e: # pg_autoctl uses EXIT_CODE_PGSQL when Postgres is not ready return False @@ -547,7 +547,7 @@ def wait_until_pg_is_running(self, timeout=STATE_CHANGE_TIMEOUT): """ command = PGAutoCtl(self) out, err, ret = command.execute( - "pgsetup ready", "do", "pgsetup", "wait", "-vvv" + "pgsetup ready", "inspect", "pgsetup", "wait", "-vvv" ) return ret == 0 @@ -1217,7 +1217,7 @@ def get_nodeid(self): Fetch the nodeid from the pg_autoctl state file. """ command = PGAutoCtl(self) - out, err, ret = command.execute("get node id", "do", "fsm", "state") + out, err, ret = command.execute("get node id", "inspect", "fsm", "state") self.state = json.loads(out) return self.state["state"]["nodeId"] @@ -1242,7 +1242,7 @@ def get_local_state(self): """ command = PGAutoCtl(self) out, err, ret = command.execute( - "get node id", "-vv", "do", "fsm", "state" + "get node id", "-vv", "inspect", "fsm", "state" ) self.state = json.loads(out) @@ -1412,19 +1412,19 @@ def drop(self): def do_fsm_assign(self, target_state): """ - Runs `pg_autoctl do fsm assign` on a node + Runs `pg_autoctl manual fsm assign` on a node :return: """ command = PGAutoCtl(self) command.execute( - "do fsm assign", "-vv", "do", "fsm", "assign", target_state + "manual fsm assign", "-vv", "manual", "fsm", "assign", target_state ) return True def do_fsm_nodes_set(self, nodesArray): """ - Runs `pg_autoctl do fsm nodes set` on a node + Runs `pg_autoctl manual fsm nodes set` on a node :return: """ @@ -1435,18 +1435,18 @@ def do_fsm_nodes_set(self, nodesArray): command = PGAutoCtl(self) out, err, ret = command.execute( - "do fsm nodes set", "do", "fsm", "nodes", "set", filename + "manual fsm nodes set", "manual", "fsm", "nodes", "set", filename ) return True def do_fsm_step(self): """ - Runs `pg_autoctl do fsm step` on a node + Runs `pg_autoctl manual fsm step` on a node :return: """ command = PGAutoCtl(self) - command.execute("do fsm step", "do", "fsm", "step") + command.execute("manual fsm step", "manual", "fsm", "step") return True def set_metadata(self, name=None, host=None, port=None):