From 99e6e97ac41af2db3be5b395d0375ca4e0ba4e06 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:38:22 -0400 Subject: [PATCH 01/44] fix: reconcile explicit restore upload target --- scripts/reconcile-uploads.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/reconcile-uploads.ts b/scripts/reconcile-uploads.ts index cbba276c..a170f940 100644 --- a/scripts/reconcile-uploads.ts +++ b/scripts/reconcile-uploads.ts @@ -2,10 +2,18 @@ import { mkdir, readdir, rm } from "node:fs/promises"; import path from "node:path"; import { prisma } from "../src/lib/prisma"; -const uploadDir = path.join(process.cwd(), "uploads"); -const trashDir = path.join(uploadDir, ".trash"); +function resolveUploadDir() { + const configured = process.env.SYNAPSE_UPLOAD_DIR; + if (!configured) return path.join(process.cwd(), "uploads"); + if (!path.isAbsolute(configured)) { + throw new Error("SYNAPSE_UPLOAD_DIR must be an absolute path when set"); + } + return configured; +} async function main() { + const uploadDir = resolveUploadDir(); + const trashDir = path.join(uploadDir, ".trash"); await mkdir(uploadDir, { recursive: true }); // Query the authoritative database before removing anything. If the database From 9695afd5754fa1b60c6e7b5f44a05012365ae569 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:38:35 -0400 Subject: [PATCH 02/44] fix: validate restored upload directory --- scripts/restore-drill.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 8a936e26..7e4651d5 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -51,6 +51,8 @@ mkdir -p "$RESTORE_UPLOAD_DIR" tar -xzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" -C "$RESTORE_UPLOAD_DIR" DATABASE_URL="$RESTORE_DATABASE_URL" npx prisma migrate status >/dev/null -DATABASE_URL="$RESTORE_DATABASE_URL" npm run uploads:reconcile >/dev/null +SYNAPSE_UPLOAD_DIR="$RESTORE_UPLOAD_DIR" \ +DATABASE_URL="$RESTORE_DATABASE_URL" \ +npm run uploads:reconcile >/dev/null echo "Restore drill passed: database migrations and upload reconciliation are healthy" From bdaff4923d022b576ce039fc32513add939ea55d Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:38:46 -0400 Subject: [PATCH 03/44] test: bind restore reconciliation to restored uploads --- tests/backup-restore.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index df4e7c7c..3ab0c45e 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -15,13 +15,16 @@ test("backup captures database, uploads, manifest, and checksums without secrets test("restore drill is explicit, integrity-checked, and validates application state", async () => { const script = await readFile("scripts/restore-drill.sh", "utf8"); + const reconcile = await readFile("scripts/reconcile-uploads.ts", "utf8"); assert.match(script, /RESTORE_CONFIRM:-.*EMPTY_DISPOSABLE_DATABASE/); assert.match(script, /shasum -a 256 -c checksums\.sha256/); assert.match(script, /pg_restore --exit-on-error --clean --if-exists/); assert.match(script, /url\.searchParams\.delete\("schema"\)/); assert.match(script, /prisma migrate status/); - assert.match(script, /npm run uploads:reconcile/); + assert.match(script, /SYNAPSE_UPLOAD_DIR="\$RESTORE_UPLOAD_DIR"[\s\S]*npm run uploads:reconcile/); + assert.match(reconcile, /process\.env\.SYNAPSE_UPLOAD_DIR/); + assert.match(reconcile, /path\.isAbsolute\(configured\)/); }); test("runbook keeps production recovery gated on owner and operator evidence", async () => { From 7811d8244d656a514ca950cedc594038cb14cc22 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:42:59 -0400 Subject: [PATCH 04/44] fix: gate restore upload override explicitly --- scripts/reconcile-uploads.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/reconcile-uploads.ts b/scripts/reconcile-uploads.ts index a170f940..d82f9253 100644 --- a/scripts/reconcile-uploads.ts +++ b/scripts/reconcile-uploads.ts @@ -1,14 +1,30 @@ import { mkdir, readdir, rm } from "node:fs/promises"; +import { homedir } from "node:os"; import path from "node:path"; import { prisma } from "../src/lib/prisma"; +const RESTORE_UPLOAD_OVERRIDE_CONFIRM = "restore-drill-disposable-upload-target"; + function resolveUploadDir() { const configured = process.env.SYNAPSE_UPLOAD_DIR; if (!configured) return path.join(process.cwd(), "uploads"); + + if (process.env.SYNAPSE_UPLOAD_DIR_CONFIRM !== RESTORE_UPLOAD_OVERRIDE_CONFIRM) { + throw new Error( + `SYNAPSE_UPLOAD_DIR override requires SYNAPSE_UPLOAD_DIR_CONFIRM=${RESTORE_UPLOAD_OVERRIDE_CONFIRM}` + ); + } if (!path.isAbsolute(configured)) { throw new Error("SYNAPSE_UPLOAD_DIR must be an absolute path when set"); } - return configured; + + const resolved = path.resolve(configured); + const root = path.parse(resolved).root; + if (resolved === root || resolved === homedir() || resolved === process.cwd()) { + throw new Error("Refusing unsafe SYNAPSE_UPLOAD_DIR override target"); + } + + return resolved; } async function main() { From 185664a3073044833edbe39e1d00d3e1718056eb Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:43:10 -0400 Subject: [PATCH 05/44] fix: gate restore target reconciliation safely --- scripts/restore-drill.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 7e4651d5..568b8ecd 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -16,6 +16,10 @@ if [[ "$RESTORE_UPLOAD_DIR" == "/" || "$RESTORE_UPLOAD_DIR" == "$HOME" ]]; then echo "Refusing unsafe restore upload target" >&2 exit 2 fi +if [[ -L "$RESTORE_UPLOAD_DIR" ]]; then + echo "Restore upload target must not be a symbolic link" >&2 + exit 2 +fi for file in database.dump uploads.tar.gz manifest.txt checksums.sha256; do if [[ ! -f "$SYNAPSE_BACKUP_PATH/$file" ]]; then @@ -52,6 +56,7 @@ tar -xzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" -C "$RESTORE_UPLOAD_DIR" DATABASE_URL="$RESTORE_DATABASE_URL" npx prisma migrate status >/dev/null SYNAPSE_UPLOAD_DIR="$RESTORE_UPLOAD_DIR" \ +SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target' \ DATABASE_URL="$RESTORE_DATABASE_URL" \ npm run uploads:reconcile >/dev/null From 4073162c279bfeba8273dfdebbf95f9bd0103e51 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:43:22 -0400 Subject: [PATCH 06/44] test: require restore-only upload override safeguards --- tests/backup-restore.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 3ab0c45e..fe8dda0c 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -22,9 +22,19 @@ test("restore drill is explicit, integrity-checked, and validates application st assert.match(script, /pg_restore --exit-on-error --clean --if-exists/); assert.match(script, /url\.searchParams\.delete\("schema"\)/); assert.match(script, /prisma migrate status/); - assert.match(script, /SYNAPSE_UPLOAD_DIR="\$RESTORE_UPLOAD_DIR"[\s\S]*npm run uploads:reconcile/); + assert.match(script, /Restore upload target must not be a symbolic link/); + assert.match( + script, + /SYNAPSE_UPLOAD_DIR="\$RESTORE_UPLOAD_DIR"[\s\S]*SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target'[\s\S]*npm run uploads:reconcile/ + ); assert.match(reconcile, /process\.env\.SYNAPSE_UPLOAD_DIR/); + assert.match(reconcile, /SYNAPSE_UPLOAD_DIR_CONFIRM/); + assert.match(reconcile, /restore-drill-disposable-upload-target/); assert.match(reconcile, /path\.isAbsolute\(configured\)/); + assert.match(reconcile, /resolved === root/); + assert.match(reconcile, /resolved === homedir\(\)/); + assert.match(reconcile, /resolved === process\.cwd\(\)/); + assert.match(reconcile, /path\.join\(process\.cwd\(\), "uploads"\)/); }); test("runbook keeps production recovery gated on owner and operator evidence", async () => { From 120874a561e784971407e1fe97cce0d896bd96a6 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:47:30 -0400 Subject: [PATCH 07/44] fix: make restore upload target explicit --- scripts/reconcile-uploads.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/reconcile-uploads.ts b/scripts/reconcile-uploads.ts index d82f9253..49bb90dd 100644 --- a/scripts/reconcile-uploads.ts +++ b/scripts/reconcile-uploads.ts @@ -6,22 +6,22 @@ import { prisma } from "../src/lib/prisma"; const RESTORE_UPLOAD_OVERRIDE_CONFIRM = "restore-drill-disposable-upload-target"; function resolveUploadDir() { - const configured = process.env.SYNAPSE_UPLOAD_DIR; + const configured = process.argv[2]; if (!configured) return path.join(process.cwd(), "uploads"); if (process.env.SYNAPSE_UPLOAD_DIR_CONFIRM !== RESTORE_UPLOAD_OVERRIDE_CONFIRM) { throw new Error( - `SYNAPSE_UPLOAD_DIR override requires SYNAPSE_UPLOAD_DIR_CONFIRM=${RESTORE_UPLOAD_OVERRIDE_CONFIRM}` + `explicit upload target requires SYNAPSE_UPLOAD_DIR_CONFIRM=${RESTORE_UPLOAD_OVERRIDE_CONFIRM}` ); } if (!path.isAbsolute(configured)) { - throw new Error("SYNAPSE_UPLOAD_DIR must be an absolute path when set"); + throw new Error("explicit upload target must be an absolute path"); } const resolved = path.resolve(configured); const root = path.parse(resolved).root; if (resolved === root || resolved === homedir() || resolved === process.cwd()) { - throw new Error("Refusing unsafe SYNAPSE_UPLOAD_DIR override target"); + throw new Error("Refusing unsafe explicit upload target"); } return resolved; From 29dbed485ca27b064fe1c999c2b9647f5127d13e Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:47:43 -0400 Subject: [PATCH 08/44] fix: pass restore target as explicit argument --- scripts/restore-drill.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 568b8ecd..8ed0cc70 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -55,9 +55,8 @@ mkdir -p "$RESTORE_UPLOAD_DIR" tar -xzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" -C "$RESTORE_UPLOAD_DIR" DATABASE_URL="$RESTORE_DATABASE_URL" npx prisma migrate status >/dev/null -SYNAPSE_UPLOAD_DIR="$RESTORE_UPLOAD_DIR" \ SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target' \ DATABASE_URL="$RESTORE_DATABASE_URL" \ -npm run uploads:reconcile >/dev/null +npm run uploads:reconcile -- "$RESTORE_UPLOAD_DIR" >/dev/null echo "Restore drill passed: database migrations and upload reconciliation are healthy" From 72baa3cef65346dfc2000931040945ed32e9fc33 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:47:58 -0400 Subject: [PATCH 09/44] test: keep restore target out of ambient env --- tests/backup-restore.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index fe8dda0c..661d2c1a 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -25,9 +25,10 @@ test("restore drill is explicit, integrity-checked, and validates application st assert.match(script, /Restore upload target must not be a symbolic link/); assert.match( script, - /SYNAPSE_UPLOAD_DIR="\$RESTORE_UPLOAD_DIR"[\s\S]*SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target'[\s\S]*npm run uploads:reconcile/ + /SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target'[\s\S]*npm run uploads:reconcile -- "\$RESTORE_UPLOAD_DIR"/ ); - assert.match(reconcile, /process\.env\.SYNAPSE_UPLOAD_DIR/); + assert.doesNotMatch(reconcile, /process\.env\.SYNAPSE_UPLOAD_DIR/); + assert.match(reconcile, /process\.argv\[2\]/); assert.match(reconcile, /SYNAPSE_UPLOAD_DIR_CONFIRM/); assert.match(reconcile, /restore-drill-disposable-upload-target/); assert.match(reconcile, /path\.isAbsolute\(configured\)/); From 6ade8fce787dd4dd1d071af0557ff75cad6294da Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:49:04 -0400 Subject: [PATCH 10/44] fix: verify restore database is empty before import --- scripts/restore-drill.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 8ed0cc70..b8a6df07 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -49,7 +49,20 @@ postgres_restore_url="$(RESTORE_DATABASE_URL="$RESTORE_DATABASE_URL" node -e ' url.searchParams.delete("schema"); process.stdout.write(url.toString()); ')" -pg_restore --exit-on-error --clean --if-exists --no-owner --no-privileges \ + +existing_objects="$( + psql "$postgres_restore_url" --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --command "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname !~ '^pg_toast' AND c.relkind IN ('r','p','v','m','S','f');" \ + | tr -d '[:space:]' +)" +if [[ "$existing_objects" != "0" ]]; then + echo "Restore database must be empty; found $existing_objects existing object(s)" >&2 + exit 2 +fi + +# The target was independently verified empty above, so do not use --clean here. +# Omitting destructive drop behavior makes a mistaken or raced target fail closed. +pg_restore --exit-on-error --no-owner --no-privileges \ --dbname="$postgres_restore_url" "$SYNAPSE_BACKUP_PATH/database.dump" mkdir -p "$RESTORE_UPLOAD_DIR" tar -xzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" -C "$RESTORE_UPLOAD_DIR" From 4cb300cd7d6934bd23b44c7d483998f065427b7d Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:49:21 -0400 Subject: [PATCH 11/44] test: require empty restore database before import --- tests/backup-restore.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 661d2c1a..6e4146da 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -19,7 +19,10 @@ test("restore drill is explicit, integrity-checked, and validates application st assert.match(script, /RESTORE_CONFIRM:-.*EMPTY_DISPOSABLE_DATABASE/); assert.match(script, /shasum -a 256 -c checksums\.sha256/); - assert.match(script, /pg_restore --exit-on-error --clean --if-exists/); + assert.match(script, /psql .*--set=ON_ERROR_STOP=1/); + assert.match(script, /Restore database must be empty/); + assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); + assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); assert.match(script, /url\.searchParams\.delete\("schema"\)/); assert.match(script, /prisma migrate status/); assert.match(script, /Restore upload target must not be a symbolic link/); From 80f2ef18d089bad48fb3c3469fa43a9287dec766 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:50:22 -0400 Subject: [PATCH 12/44] fix: reject unsafe upload trees during backup --- scripts/backup.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/backup.sh b/scripts/backup.sh index 81fc4d70..27504507 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -13,12 +13,32 @@ if [[ "$SYNAPSE_BACKUP_DIR" == "/" || "$SYNAPSE_BACKUP_DIR" == "$HOME" ]]; then echo "Refusing unsafe backup root" >&2 exit 2 fi +if [[ -L "$SYNAPSE_BACKUP_DIR" ]]; then + echo "Backup root must not be a symbolic link" >&2 + exit 2 +fi upload_dir="${SYNAPSE_UPLOAD_DIR:-$PWD/uploads}" +case "$upload_dir" in + /*) ;; + *) echo "SYNAPSE_UPLOAD_DIR must be absolute" >&2; exit 2 ;; +esac +if [[ "$upload_dir" == "/" || "$upload_dir" == "$HOME" || "$upload_dir" == "$PWD" ]]; then + echo "Refusing unsafe upload root" >&2 + exit 2 +fi if [[ ! -d "$upload_dir" ]]; then echo "Upload directory does not exist: $upload_dir" >&2 exit 2 fi +if [[ -L "$upload_dir" ]]; then + echo "Upload directory must not be a symbolic link" >&2 + exit 2 +fi +if [[ -n "$(find "$upload_dir" -type l -print -quit)" ]]; then + echo "Upload directory contains symbolic links; refusing unsafe backup" >&2 + exit 2 +fi umask 077 timestamp="$(date -u +%Y%m%dT%H%M%SZ)" From e43f5a0dbe287f9c13aa831b6acf0ee19b3610f5 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:50:46 -0400 Subject: [PATCH 13/44] fix: reject linked entries in restore archive --- scripts/restore-drill.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index b8a6df07..211892c4 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -39,6 +39,11 @@ while IFS= read -r entry; do esac done < <(tar -tzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz") +if tar -tvzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" | grep -Eq '^[lh]'; then + echo "Upload archive contains symbolic or hard links; refusing unsafe restore" >&2 + exit 2 +fi + if [[ -d "$RESTORE_UPLOAD_DIR" && -n "$(find "$RESTORE_UPLOAD_DIR" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then echo "Restore upload target must be empty" >&2 exit 2 From 44cc27fcb4fd08d456871edd4d07ab9be2a80162 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:51:10 -0400 Subject: [PATCH 14/44] test: pin backup and archive link safety --- tests/backup-restore.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 6e4146da..91d61967 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -7,6 +7,10 @@ test("backup captures database, uploads, manifest, and checksums without secrets assert.match(script, /pg_dump --format=custom --no-owner --no-privileges/); assert.match(script, /url\.searchParams\.delete\("schema"\)/); + assert.match(script, /SYNAPSE_UPLOAD_DIR must be absolute/); + assert.match(script, /Refusing unsafe upload root/); + assert.match(script, /Upload directory must not be a symbolic link/); + assert.match(script, /find "\$upload_dir" -type l -print -quit/); assert.match(script, /tar -czf .*uploads\.tar\.gz/); assert.match(script, /secrets_included=false/); assert.match(script, /shasum -a 256 database\.dump uploads\.tar\.gz manifest\.txt/); @@ -19,6 +23,8 @@ test("restore drill is explicit, integrity-checked, and validates application st assert.match(script, /RESTORE_CONFIRM:-.*EMPTY_DISPOSABLE_DATABASE/); assert.match(script, /shasum -a 256 -c checksums\.sha256/); + assert.match(script, /tar -tvzf .*grep -Eq '\^\[lh\]'/); + assert.match(script, /symbolic or hard links/); assert.match(script, /psql .*--set=ON_ERROR_STOP=1/); assert.match(script, /Restore database must be empty/); assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); From b4edb4ab6acc98e09c07d179965f3845011f20e9 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 20:52:25 -0400 Subject: [PATCH 15/44] docs: align recovery runbook with fail-closed drill --- OPERATIONS.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index fbc08406..7b450f5f 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -14,10 +14,12 @@ The backup command creates a private, timestamped directory containing: - `manifest.txt`: format, timestamp, revision, and an explicit no-secrets claim; - `checksums.sha256`: integrity hashes for every artifact. -The destination must be an absolute path and cannot be `/` or the operator's home -directory. The command removes a partial destination if creation fails. It does not -encrypt artifacts, upload them off-site, or choose retention; use an approved encrypted -backup target with access controls before production. +The backup destination and upload root must be absolute paths. The command refuses +filesystem root, the operator's home directory, the current working directory as an +upload root, linked backup/upload roots, and any symbolic links inside the upload tree. +It removes a partial destination if creation fails. It does not encrypt artifacts, +upload them off-site, or choose retention; use an approved encrypted backup target +with access controls before production. ```bash DATABASE_URL='postgresql://…' \ @@ -103,10 +105,14 @@ Never point the database assertion at a different deployment from the HTTP targe ## Restore drill -The drill is deliberately destructive to its target database. Use a newly created, -empty disposable database and an empty disposable uploads directory. It refuses to -run without the exact confirmation value, verifies checksums and archive paths, -restores both stores, confirms Prisma migration state, and reconciles upload metadata. +Use a newly created, empty disposable database and an empty disposable uploads +directory. The drill refuses to run without the exact confirmation value, verifies +checksums and archive paths, rejects symbolic/hard-link archive entries, independently +proves the target database has no non-system objects, and refuses linked or non-empty +upload targets. Because the database is verified empty, restore does not use +`pg_restore --clean` and therefore does not drop existing database objects. It then +restores both stores, confirms Prisma migration state, and reconciles the exact restored +upload directory through a restore-only explicit target argument. ```bash SYNAPSE_BACKUP_PATH='/absolute/path/to/one/synapse-backup' \ From 5957950921b0e63b024d9c56667d39307690359b Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:15:38 -0400 Subject: [PATCH 16/44] fix(recovery): canonicalize and validate backup upload tree --- scripts/backup.sh | 97 ++++++++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/scripts/backup.sh b/scripts/backup.sh index 27504507..982f2f27 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -4,45 +4,74 @@ set -euo pipefail : "${DATABASE_URL:?DATABASE_URL is required}" : "${SYNAPSE_BACKUP_DIR:?SYNAPSE_BACKUP_DIR must be an absolute backup directory}" -case "$SYNAPSE_BACKUP_DIR" in - /*) ;; - *) echo "SYNAPSE_BACKUP_DIR must be absolute" >&2; exit 2 ;; -esac +canonical_existing_directory() { + TARGET_PATH="$1" node <<'NODE' +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); -if [[ "$SYNAPSE_BACKUP_DIR" == "/" || "$SYNAPSE_BACKUP_DIR" == "$HOME" ]]; then - echo "Refusing unsafe backup root" >&2 - exit 2 -fi -if [[ -L "$SYNAPSE_BACKUP_DIR" ]]; then - echo "Backup root must not be a symbolic link" >&2 - exit 2 -fi +const input = process.env.TARGET_PATH; +const resolved = path.resolve(input); +let stat; +try { + stat = fs.lstatSync(resolved); +} catch { + console.error(`Directory does not exist: ${resolved}`); + process.exit(2); +} +if (!stat.isDirectory()) { + console.error(`Path is not a directory: ${resolved}`); + process.exit(2); +} +const canonical = fs.realpathSync(resolved); +if (canonical !== resolved) { + console.error(`Directory path must not contain symbolic-link components: ${resolved}`); + process.exit(2); +} +if (resolved === path.parse(resolved).root || resolved === os.homedir() || resolved === process.cwd()) { + console.error(`Refusing unsafe directory root: ${resolved}`); + process.exit(2); +} +process.stdout.write(resolved); +NODE +} -upload_dir="${SYNAPSE_UPLOAD_DIR:-$PWD/uploads}" -case "$upload_dir" in - /*) ;; - *) echo "SYNAPSE_UPLOAD_DIR must be absolute" >&2; exit 2 ;; -esac -if [[ "$upload_dir" == "/" || "$upload_dir" == "$HOME" || "$upload_dir" == "$PWD" ]]; then - echo "Refusing unsafe upload root" >&2 - exit 2 -fi -if [[ ! -d "$upload_dir" ]]; then - echo "Upload directory does not exist: $upload_dir" >&2 - exit 2 -fi -if [[ -L "$upload_dir" ]]; then - echo "Upload directory must not be a symbolic link" >&2 - exit 2 -fi -if [[ -n "$(find "$upload_dir" -type l -print -quit)" ]]; then - echo "Upload directory contains symbolic links; refusing unsafe backup" >&2 - exit 2 -fi +backup_root="$(canonical_existing_directory "$SYNAPSE_BACKUP_DIR")" +upload_dir="$(canonical_existing_directory "${SYNAPSE_UPLOAD_DIR:-$PWD/uploads}")" + +UPLOAD_ROOT="$upload_dir" node <<'NODE' +const fs = require('node:fs'); +const path = require('node:path'); + +const root = process.env.UPLOAD_ROOT; +function walk(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + const stat = fs.lstatSync(fullPath); + if (stat.isSymbolicLink()) { + console.error(`Upload tree contains a symbolic link: ${fullPath}`); + process.exit(2); + } + if (stat.isDirectory()) { + walk(fullPath); + continue; + } + if (!stat.isFile()) { + console.error(`Upload tree contains a non-regular file: ${fullPath}`); + process.exit(2); + } + if (stat.nlink !== 1) { + console.error(`Upload tree contains a hard-linked file: ${fullPath}`); + process.exit(2); + } + } +} +walk(root); +NODE umask 077 timestamp="$(date -u +%Y%m%dT%H%M%SZ)" -destination="$SYNAPSE_BACKUP_DIR/synapse-$timestamp" +destination="$backup_root/synapse-$timestamp" if [[ -e "$destination" ]]; then echo "Backup destination already exists: $destination" >&2 exit 2 From 3368f9919d3a88bef7579b219fffa59184021eb6 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:16:11 -0400 Subject: [PATCH 17/44] fix(recovery): close restore path and database safety gaps --- scripts/restore-drill.sh | 101 ++++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 13 deletions(-) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 211892c4..c3d1d61a 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -12,14 +12,45 @@ fi case "$SYNAPSE_BACKUP_PATH" in /*) ;; *) echo "SYNAPSE_BACKUP_PATH must be absolute" >&2; exit 2 ;; esac case "$RESTORE_UPLOAD_DIR" in /*) ;; *) echo "RESTORE_UPLOAD_DIR must be absolute" >&2; exit 2 ;; esac -if [[ "$RESTORE_UPLOAD_DIR" == "/" || "$RESTORE_UPLOAD_DIR" == "$HOME" ]]; then - echo "Refusing unsafe restore upload target" >&2 - exit 2 -fi -if [[ -L "$RESTORE_UPLOAD_DIR" ]]; then - echo "Restore upload target must not be a symbolic link" >&2 - exit 2 -fi + +RESTORE_UPLOAD_DIR="$(TARGET_PATH="$RESTORE_UPLOAD_DIR" node <<'NODE' +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const resolved = path.resolve(process.env.TARGET_PATH); +if (resolved === path.parse(resolved).root || resolved === os.homedir() || resolved === process.cwd()) { + console.error(`Refusing unsafe restore upload target: ${resolved}`); + process.exit(2); +} + +let existing = resolved; +while (!fs.existsSync(existing)) { + const parent = path.dirname(existing); + if (parent === existing) { + console.error(`Could not locate an existing restore-target ancestor: ${resolved}`); + process.exit(2); + } + existing = parent; +} +if (fs.realpathSync(existing) !== existing) { + console.error(`Restore upload target path must not contain symbolic-link components: ${resolved}`); + process.exit(2); +} +if (fs.existsSync(resolved)) { + const stat = fs.lstatSync(resolved); + if (!stat.isDirectory()) { + console.error(`Restore upload target is not a directory: ${resolved}`); + process.exit(2); + } + if (fs.realpathSync(resolved) !== resolved) { + console.error(`Restore upload target must not resolve through a symbolic link: ${resolved}`); + process.exit(2); + } +} +process.stdout.write(resolved); +NODE +)" for file in database.dump uploads.tar.gz manifest.txt checksums.sha256; do if [[ ! -f "$SYNAPSE_BACKUP_PATH/$file" ]]; then @@ -33,13 +64,22 @@ done shasum -a 256 -c checksums.sha256 ) +archive_names="$(mktemp)" +archive_listing="$(mktemp)" +cleanup_archive_inspection() { + rm -f -- "$archive_names" "$archive_listing" +} +trap cleanup_archive_inspection EXIT + +tar -tzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" > "$archive_names" while IFS= read -r entry; do case "$entry" in /*|../*|*/../*|*/..) echo "Unsafe upload archive entry: $entry" >&2; exit 2 ;; esac -done < <(tar -tzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz") +done < "$archive_names" -if tar -tvzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" | grep -Eq '^[lh]'; then +tar -tvzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" > "$archive_listing" +if grep -Eq '^[lh]' "$archive_listing"; then echo "Upload archive contains symbolic or hard links; refusing unsafe restore" >&2 exit 2 fi @@ -57,11 +97,46 @@ postgres_restore_url="$(RESTORE_DATABASE_URL="$RESTORE_DATABASE_URL" node -e ' existing_objects="$( psql "$postgres_restore_url" --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ - --command "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname !~ '^pg_toast' AND c.relkind IN ('r','p','v','m','S','f');" \ - | tr -d '[:space:]' + --command " + SELECT count(*) FROM ( + SELECT 1 FROM pg_namespace + WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'public') + AND nspname !~ '^pg_(toast|temp)' + UNION ALL + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid = c.collnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_conversion c JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_operator o JOIN pg_namespace n ON n.oid = o.oprnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_config c JOIN pg_namespace n ON n.oid = c.cfgnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_extension WHERE extname <> 'plpgsql' + UNION ALL + SELECT 1 FROM pg_event_trigger + UNION ALL + SELECT 1 FROM pg_publication + ) AS user_defined_objects; + " | tr -d '[:space:]' )" if [[ "$existing_objects" != "0" ]]; then - echo "Restore database must be empty; found $existing_objects existing object(s)" >&2 + echo "Restore database must be empty; found $existing_objects user-defined object(s)" >&2 exit 2 fi From 8cb82aba1b4af5c6de61acb251453a769bfd5bff Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:16:36 -0400 Subject: [PATCH 18/44] test(recovery): pin canonical path and complete restore guards --- tests/backup-restore.test.ts | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 91d61967..eee86c63 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -2,36 +2,53 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import test from "node:test"; -test("backup captures database, uploads, manifest, and checksums without secrets", async () => { +test("backup captures only a canonical regular upload tree with integrity metadata", async () => { const script = await readFile("scripts/backup.sh", "utf8"); + assert.match(script, /canonical_existing_directory/); + assert.match(script, /fs\.realpathSync\(resolved\)/); + assert.match(script, /Directory path must not contain symbolic-link components/); + assert.match(script, /stat\.isSymbolicLink\(\)/); + assert.match(script, /stat\.nlink !== 1/); + assert.match(script, /hard-linked file/); assert.match(script, /pg_dump --format=custom --no-owner --no-privileges/); assert.match(script, /url\.searchParams\.delete\("schema"\)/); - assert.match(script, /SYNAPSE_UPLOAD_DIR must be absolute/); - assert.match(script, /Refusing unsafe upload root/); - assert.match(script, /Upload directory must not be a symbolic link/); - assert.match(script, /find "\$upload_dir" -type l -print -quit/); assert.match(script, /tar -czf .*uploads\.tar\.gz/); assert.match(script, /secrets_included=false/); assert.match(script, /shasum -a 256 database\.dump uploads\.tar\.gz manifest\.txt/); assert.match(script, /trap cleanup_failed_backup ERR INT TERM/); }); -test("restore drill is explicit, integrity-checked, and validates application state", async () => { +test("restore drill fails closed on archive, target path, and database state", async () => { const script = await readFile("scripts/restore-drill.sh", "utf8"); const reconcile = await readFile("scripts/reconcile-uploads.ts", "utf8"); assert.match(script, /RESTORE_CONFIRM:-.*EMPTY_DISPOSABLE_DATABASE/); assert.match(script, /shasum -a 256 -c checksums\.sha256/); - assert.match(script, /tar -tvzf .*grep -Eq '\^\[lh\]'/); + assert.match(script, /fs\.realpathSync\(existing\) !== existing/); + assert.match(script, /symbolic-link components/); + assert.match(script, /archive_listing="\$\(mktemp\)"/); + assert.match(script, /tar -tvzf .* > "\$archive_listing"/); + assert.match(script, /grep -Eq '\^\[lh\]' "\$archive_listing"/); + assert.doesNotMatch(script, /tar -tvzf[^\n]*\|[^\n]*grep -q/); assert.match(script, /symbolic or hard links/); assert.match(script, /psql .*--set=ON_ERROR_STOP=1/); + for (const catalog of [ + "pg_namespace", + "pg_class", + "pg_proc", + "pg_type", + "pg_extension", + "pg_event_trigger", + "pg_publication", + ]) { + assert.match(script, new RegExp(catalog)); + } assert.match(script, /Restore database must be empty/); assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); assert.match(script, /url\.searchParams\.delete\("schema"\)/); assert.match(script, /prisma migrate status/); - assert.match(script, /Restore upload target must not be a symbolic link/); assert.match( script, /SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target'[\s\S]*npm run uploads:reconcile -- "\$RESTORE_UPLOAD_DIR"/ From d361e2b1203d7f33489926c52c31dd6d4e9c6b5d Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:17:16 -0400 Subject: [PATCH 19/44] fix(recovery): broaden empty database verification --- scripts/restore-drill.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index c3d1d61a..213c451a 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -132,6 +132,23 @@ existing_objects="$( SELECT 1 FROM pg_event_trigger UNION ALL SELECT 1 FROM pg_publication + UNION ALL + SELECT 1 FROM pg_subscription + UNION ALL + SELECT 1 FROM pg_foreign_data_wrapper + UNION ALL + SELECT 1 FROM pg_foreign_server + UNION ALL + SELECT 1 FROM pg_user_mapping + UNION ALL + SELECT 1 FROM pg_transform + UNION ALL + SELECT 1 FROM pg_largeobject_metadata + UNION ALL + SELECT 1 FROM pg_cast WHERE oid >= 16384 + UNION ALL + SELECT 1 FROM pg_language + WHERE lanname NOT IN ('internal', 'c', 'sql', 'plpgsql') ) AS user_defined_objects; " | tr -d '[:space:]' )" From 66e4bc8a37f38dcbf200cf81041f7ed659033558 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:17:42 -0400 Subject: [PATCH 20/44] docs(recovery): document canonical fail-closed recovery paths --- OPERATIONS.md | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 7b450f5f..7029cf65 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -14,17 +14,22 @@ The backup command creates a private, timestamped directory containing: - `manifest.txt`: format, timestamp, revision, and an explicit no-secrets claim; - `checksums.sha256`: integrity hashes for every artifact. -The backup destination and upload root must be absolute paths. The command refuses -filesystem root, the operator's home directory, the current working directory as an -upload root, linked backup/upload roots, and any symbolic links inside the upload tree. -It removes a partial destination if creation fails. It does not encrypt artifacts, -upload them off-site, or choose retention; use an approved encrypted backup target -with access controls before production. +The approved backup root and upload root must already exist as absolute canonical +directories. The command rejects filesystem root, the operator home directory, the +current working directory, any root whose literal path resolves through a symbolic-link +component, symbolic links anywhere inside the upload tree, hard-linked upload files, +and non-regular upload entries. This prevents a successful backup from referring to +files outside the intended upload tree or producing an archive that its own restore +workflow cannot safely consume. A partial timestamped destination is removed if backup +creation fails. + +The command does not encrypt artifacts, upload them off-site, or choose retention; +use an approved encrypted backup target with least-privilege access before production. ```bash DATABASE_URL='postgresql://…' \ -SYNAPSE_UPLOAD_DIR='/absolute/path/to/uploads' \ -SYNAPSE_BACKUP_DIR='/absolute/path/to/private/backups' \ +SYNAPSE_UPLOAD_DIR='/absolute/canonical/path/to/uploads' \ +SYNAPSE_BACKUP_DIR='/absolute/canonical/path/to/private/backups' \ npm run backup ``` @@ -106,18 +111,27 @@ Never point the database assertion at a different deployment from the HTTP targe ## Restore drill Use a newly created, empty disposable database and an empty disposable uploads -directory. The drill refuses to run without the exact confirmation value, verifies -checksums and archive paths, rejects symbolic/hard-link archive entries, independently -proves the target database has no non-system objects, and refuses linked or non-empty -upload targets. Because the database is verified empty, restore does not use -`pg_restore --clean` and therefore does not drop existing database objects. It then -restores both stores, confirms Prisma migration state, and reconciles the exact restored -upload directory through a restore-only explicit target argument. +directory. The drill refuses to run without the exact confirmation value. It +canonicalizes the restore upload target and rejects linked path components, then +verifies backup checksums and fully materializes archive name/type listings before +checking traversal and symbolic/hard-link entries. Link checks therefore cannot be +short-circuited by a producer receiving SIGPIPE. + +Before import, the drill queries PostgreSQL across user schemas, namespace-owned +relations/functions/types/operators/text-search objects, extensions, event triggers, +publications/subscriptions, foreign-data objects, transforms, large objects, custom +casts, and non-default languages. Any detected user-defined object fails the empty-DB +gate. Because the target is independently verified empty, restore does not use +`pg_restore --clean` and does not intentionally drop existing objects. + +The drill then restores both stores, confirms Prisma migration state, and reconciles +the exact restored upload directory through a restore-only explicit argument. Normal +production startup still reconciles only its standard `./uploads` directory. ```bash SYNAPSE_BACKUP_PATH='/absolute/path/to/one/synapse-backup' \ RESTORE_DATABASE_URL='postgresql://…/synapse_restore_drill' \ -RESTORE_UPLOAD_DIR='/absolute/path/to/empty/restore-uploads' \ +RESTORE_UPLOAD_DIR='/absolute/canonical/path/to/empty/restore-uploads' \ RESTORE_CONFIRM='EMPTY_DISPOSABLE_DATABASE' \ npm run restore:drill ``` From bc3af0382ddfb212753254829a59a7eeead1f056 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:27:36 -0400 Subject: [PATCH 21/44] fix(recovery): scope and complete database object preflight --- scripts/restore-drill.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 213c451a..171960b5 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -121,12 +121,27 @@ existing_objects="$( SELECT 1 FROM pg_operator o JOIN pg_namespace n ON n.oid = o.oprnamespace WHERE n.nspname = 'public' UNION ALL + SELECT 1 FROM pg_opclass o JOIN pg_namespace n ON n.oid = o.opcnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_opfamily o JOIN pg_namespace n ON n.oid = o.opfnamespace + WHERE n.nspname = 'public' + UNION ALL SELECT 1 FROM pg_ts_config c JOIN pg_namespace n ON n.oid = c.cfgnamespace WHERE n.nspname = 'public' UNION ALL SELECT 1 FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace WHERE n.nspname = 'public' UNION ALL + SELECT 1 FROM pg_ts_parser p JOIN pg_namespace n ON n.oid = p.prsnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_template t JOIN pg_namespace n ON n.oid = t.tmplnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace + WHERE n.nspname = 'public' + UNION ALL SELECT 1 FROM pg_extension WHERE extname <> 'plpgsql' UNION ALL SELECT 1 FROM pg_event_trigger @@ -134,6 +149,7 @@ existing_objects="$( SELECT 1 FROM pg_publication UNION ALL SELECT 1 FROM pg_subscription + WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) UNION ALL SELECT 1 FROM pg_foreign_data_wrapper UNION ALL From a0001c0c9807be74c955e39062e4871368516ad5 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:27:58 -0400 Subject: [PATCH 22/44] test(recovery): pin complete namespace and subscription guards --- tests/backup-restore.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index eee86c63..ac8044f1 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -38,12 +38,34 @@ test("restore drill fails closed on archive, target path, and database state", a "pg_class", "pg_proc", "pg_type", + "pg_collation", + "pg_conversion", + "pg_operator", + "pg_opclass", + "pg_opfamily", + "pg_ts_config", + "pg_ts_dict", + "pg_ts_parser", + "pg_ts_template", + "pg_statistic_ext", "pg_extension", "pg_event_trigger", "pg_publication", + "pg_subscription", + "pg_foreign_data_wrapper", + "pg_foreign_server", + "pg_user_mapping", + "pg_transform", + "pg_largeobject_metadata", + "pg_cast", + "pg_language", ]) { assert.match(script, new RegExp(catalog)); } + assert.match( + script, + /pg_subscription[\s\S]*subdbid = \(SELECT oid FROM pg_database WHERE datname = current_database\(\)\)/ + ); assert.match(script, /Restore database must be empty/); assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); From 6e2f4c547cee974625672e3fa0e58e154b169767 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:34:19 -0400 Subject: [PATCH 23/44] fix(recovery): reject relative backup and upload roots --- scripts/backup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/backup.sh b/scripts/backup.sh index 982f2f27..9c1ee749 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -11,6 +11,10 @@ const os = require('node:os'); const path = require('node:path'); const input = process.env.TARGET_PATH; +if (!path.isAbsolute(input)) { + console.error(`Directory path must be absolute: ${input}`); + process.exit(2); +} const resolved = path.resolve(input); let stat; try { From abac61f732b21ee1bf9469c7c311667186072f35 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:34:50 -0400 Subject: [PATCH 24/44] fix(recovery): include custom access methods in empty-db gate --- scripts/restore-drill.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 171960b5..9356fdf7 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -161,6 +161,8 @@ existing_objects="$( UNION ALL SELECT 1 FROM pg_largeobject_metadata UNION ALL + SELECT 1 FROM pg_am WHERE oid >= 16384 + UNION ALL SELECT 1 FROM pg_cast WHERE oid >= 16384 UNION ALL SELECT 1 FROM pg_language From 7c35342606a9bd1732ee8b09686edfaf30f7c09d Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Fri, 4 Sep 2026 21:35:17 -0400 Subject: [PATCH 25/44] test(recovery): pin absolute roots and custom access methods --- tests/backup-restore.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index ac8044f1..1ffb2466 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -6,6 +6,8 @@ test("backup captures only a canonical regular upload tree with integrity metada const script = await readFile("scripts/backup.sh", "utf8"); assert.match(script, /canonical_existing_directory/); + assert.match(script, /path\.isAbsolute\(input\)/); + assert.match(script, /Directory path must be absolute/); assert.match(script, /fs\.realpathSync\(resolved\)/); assert.match(script, /Directory path must not contain symbolic-link components/); assert.match(script, /stat\.isSymbolicLink\(\)/); @@ -57,6 +59,7 @@ test("restore drill fails closed on archive, target path, and database state", a "pg_user_mapping", "pg_transform", "pg_largeobject_metadata", + "pg_am", "pg_cast", "pg_language", ]) { @@ -66,6 +69,7 @@ test("restore drill fails closed on archive, target path, and database state", a script, /pg_subscription[\s\S]*subdbid = \(SELECT oid FROM pg_database WHERE datname = current_database\(\)\)/ ); + assert.match(script, /pg_am WHERE oid >= 16384/); assert.match(script, /Restore database must be empty/); assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); From 58a0fcb5320591092cf7be35cf675dd8af9c826c Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 10:38:26 -0400 Subject: [PATCH 26/44] fix: reject unsafe archive members and default ACLs --- scripts/restore-drill.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 9356fdf7..27daeda4 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -79,8 +79,8 @@ while IFS= read -r entry; do done < "$archive_names" tar -tvzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" > "$archive_listing" -if grep -Eq '^[lh]' "$archive_listing"; then - echo "Upload archive contains symbolic or hard links; refusing unsafe restore" >&2 +if grep -Evq '^[-d]' "$archive_listing"; then + echo "Upload archive contains non-regular or non-directory entries; refusing unsafe restore" >&2 exit 2 fi @@ -165,6 +165,8 @@ existing_objects="$( UNION ALL SELECT 1 FROM pg_cast WHERE oid >= 16384 UNION ALL + SELECT 1 FROM pg_default_acl + UNION ALL SELECT 1 FROM pg_language WHERE lanname NOT IN ('internal', 'c', 'sql', 'plpgsql') ) AS user_defined_objects; From 48ba99f53ec1bcb4048e491b43121ecc97a9af0d Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 10:38:49 -0400 Subject: [PATCH 27/44] test: pin archive member and default ACL guards --- tests/backup-restore.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 1ffb2466..c4ae6900 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -31,9 +31,9 @@ test("restore drill fails closed on archive, target path, and database state", a assert.match(script, /symbolic-link components/); assert.match(script, /archive_listing="\$\(mktemp\)"/); assert.match(script, /tar -tvzf .* > "\$archive_listing"/); - assert.match(script, /grep -Eq '\^\[lh\]' "\$archive_listing"/); + assert.match(script, /grep -Evq '\^\[-d\]' "\$archive_listing"/); assert.doesNotMatch(script, /tar -tvzf[^\n]*\|[^\n]*grep -q/); - assert.match(script, /symbolic or hard links/); + assert.match(script, /non-regular or non-directory entries/); assert.match(script, /psql .*--set=ON_ERROR_STOP=1/); for (const catalog of [ "pg_namespace", @@ -61,6 +61,7 @@ test("restore drill fails closed on archive, target path, and database state", a "pg_largeobject_metadata", "pg_am", "pg_cast", + "pg_default_acl", "pg_language", ]) { assert.match(script, new RegExp(catalog)); @@ -70,6 +71,7 @@ test("restore drill fails closed on archive, target path, and database state", a /pg_subscription[\s\S]*subdbid = \(SELECT oid FROM pg_database WHERE datname = current_database\(\)\)/ ); assert.match(script, /pg_am WHERE oid >= 16384/); + assert.match(script, /SELECT 1 FROM pg_default_acl/); assert.match(script, /Restore database must be empty/); assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); From 729094c06e8085310e3df839a5e44b025b873820 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 10:39:26 -0400 Subject: [PATCH 28/44] docs: align recovery runbook with final fail-closed guards --- OPERATIONS.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 7029cf65..21283a43 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -114,15 +114,18 @@ Use a newly created, empty disposable database and an empty disposable uploads directory. The drill refuses to run without the exact confirmation value. It canonicalizes the restore upload target and rejects linked path components, then verifies backup checksums and fully materializes archive name/type listings before -checking traversal and symbolic/hard-link entries. Link checks therefore cannot be -short-circuited by a producer receiving SIGPIPE. +checking traversal and member types. Only ordinary regular-file and directory archive +members are accepted; symbolic links, hard links, FIFOs, devices, and other special +members fail closed before extraction. The complete listing is consumed from a file, +so validation cannot be short-circuited by a producer receiving SIGPIPE. Before import, the drill queries PostgreSQL across user schemas, namespace-owned relations/functions/types/operators/text-search objects, extensions, event triggers, publications/subscriptions, foreign-data objects, transforms, large objects, custom -casts, and non-default languages. Any detected user-defined object fails the empty-DB -gate. Because the target is independently verified empty, restore does not use -`pg_restore --clean` and does not intentionally drop existing objects. +access methods/casts, non-default languages, and pre-existing default ACL rows. Any +detected user-defined object or default privilege fails the empty-DB gate. Because the +target is independently verified empty, restore does not use `pg_restore --clean` and +does not intentionally drop existing objects. The drill then restores both stores, confirms Prisma migration state, and reconciles the exact restored upload directory through a restore-only explicit argument. Normal From 5543396f06dff86268e2a05a199b3d3e6c7c4a05 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 10:50:18 -0400 Subject: [PATCH 29/44] fix: protect restore database preflight from unsafe schema state and races --- scripts/restore-drill.sh | 288 ++++++++++++++++++++++++++------------- 1 file changed, 196 insertions(+), 92 deletions(-) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 27daeda4..aeaf5d7b 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -66,10 +66,30 @@ done archive_names="$(mktemp)" archive_listing="$(mktemp)" -cleanup_archive_inspection() { - rm -f -- "$archive_names" "$archive_listing" +restore_sql="$(mktemp)" +connections_closed=0 +postgres_admin_url="" +restore_database_name="" + +reenable_target_connections() { + if [[ "$connections_closed" != "1" || -z "$postgres_admin_url" || -z "$restore_database_name" ]]; then + return 0 + fi + + psql "$postgres_admin_url" --no-psqlrc --set=ON_ERROR_STOP=1 --set=target_db="$restore_database_name" >/dev/null <<'SQL' +SELECT format('ALTER DATABASE %I ALLOW_CONNECTIONS true', :'target_db') \gexec +SQL + connections_closed=0 } -trap cleanup_archive_inspection EXIT + +cleanup_restore_drill() { + status=$? + set +e + reenable_target_connections + rm -f -- "$archive_names" "$archive_listing" "$restore_sql" + exit "$status" +} +trap cleanup_restore_drill EXIT tar -tzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" > "$archive_names" while IFS= read -r entry; do @@ -89,98 +109,182 @@ if [[ -d "$RESTORE_UPLOAD_DIR" && -n "$(find "$RESTORE_UPLOAD_DIR" -mindepth 1 - exit 2 fi -postgres_restore_url="$(RESTORE_DATABASE_URL="$RESTORE_DATABASE_URL" node -e ' - const url = new URL(process.env.RESTORE_DATABASE_URL); - url.searchParams.delete("schema"); - process.stdout.write(url.toString()); -')" - -existing_objects="$( - psql "$postgres_restore_url" --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ - --command " - SELECT count(*) FROM ( - SELECT 1 FROM pg_namespace - WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'public') - AND nspname !~ '^pg_(toast|temp)' - UNION ALL - SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid = c.collnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_conversion c JOIN pg_namespace n ON n.oid = c.connamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_operator o JOIN pg_namespace n ON n.oid = o.oprnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_opclass o JOIN pg_namespace n ON n.oid = o.opcnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_opfamily o JOIN pg_namespace n ON n.oid = o.opfnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_ts_config c JOIN pg_namespace n ON n.oid = c.cfgnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_ts_parser p JOIN pg_namespace n ON n.oid = p.prsnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_ts_template t JOIN pg_namespace n ON n.oid = t.tmplnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace - WHERE n.nspname = 'public' - UNION ALL - SELECT 1 FROM pg_extension WHERE extname <> 'plpgsql' - UNION ALL - SELECT 1 FROM pg_event_trigger - UNION ALL - SELECT 1 FROM pg_publication - UNION ALL - SELECT 1 FROM pg_subscription - WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) - UNION ALL - SELECT 1 FROM pg_foreign_data_wrapper - UNION ALL - SELECT 1 FROM pg_foreign_server - UNION ALL - SELECT 1 FROM pg_user_mapping - UNION ALL - SELECT 1 FROM pg_transform - UNION ALL - SELECT 1 FROM pg_largeobject_metadata - UNION ALL - SELECT 1 FROM pg_am WHERE oid >= 16384 - UNION ALL - SELECT 1 FROM pg_cast WHERE oid >= 16384 - UNION ALL - SELECT 1 FROM pg_default_acl - UNION ALL - SELECT 1 FROM pg_language - WHERE lanname NOT IN ('internal', 'c', 'sql', 'plpgsql') - ) AS user_defined_objects; - " | tr -d '[:space:]' -)" -if [[ "$existing_objects" != "0" ]]; then - echo "Restore database must be empty; found $existing_objects user-defined object(s)" >&2 +mapfile -t postgres_connection < <(RESTORE_DATABASE_URL="$RESTORE_DATABASE_URL" node <<'NODE' +const target = new URL(process.env.RESTORE_DATABASE_URL); +target.searchParams.delete('schema'); +const databaseName = decodeURIComponent(target.pathname.replace(/^\//, '')); +if (!databaseName || ['postgres', 'template0', 'template1'].includes(databaseName)) { + console.error('RESTORE_DATABASE_URL must name a dedicated disposable database'); + process.exit(2); +} +const admin = new URL(target); +admin.pathname = '/postgres'; +process.stdout.write(`${target.toString()}\n${admin.toString()}\n${databaseName}\n`); +NODE +) + +postgres_restore_url="${postgres_connection[0]:-}" +postgres_admin_url="${postgres_connection[1]:-}" +restore_database_name="${postgres_connection[2]:-}" +if [[ -z "$postgres_restore_url" || -z "$postgres_admin_url" || -z "$restore_database_name" ]]; then + echo "Could not resolve disposable restore database connection details" >&2 exit 2 fi -# The target was independently verified empty above, so do not use --clean here. -# Omitting destructive drop behavior makes a mistaken or raced target fail closed. -pg_restore --exit-on-error --no-owner --no-privileges \ - --dbname="$postgres_restore_url" "$SYNAPSE_BACKUP_PATH/database.dump" +# Generate SQL before closing target connections. Execution happens later through +# one protected psql session so the emptiness check and restore cannot be raced. +pg_restore --no-owner --no-privileges \ + --file="$restore_sql" "$SYNAPSE_BACKUP_PATH/database.dump" + +connections_closed=1 +{ + cat <<'SQL' +\set ON_ERROR_STOP on + +-- Freeze the disposable database before validating it. The current psql session +-- remains connected; new connections are refused until the restore completes. +SELECT format('ALTER DATABASE %I ALLOW_CONNECTIONS false', current_database()) \gexec +SELECT pg_terminate_backend(pid) +FROM pg_stat_activity +WHERE datname = current_database() + AND pid <> pg_backend_pid(); + +DO $synapse_connections$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid() + ) THEN + RAISE EXCEPTION 'Restore database still has another active connection'; + END IF; +END +$synapse_connections$; + +DO $synapse_preflight$ +DECLARE + existing_objects bigint; + public_schema_owner text; + unsafe_public_acl bigint; +BEGIN + IF (SELECT count(*) FROM pg_namespace WHERE nspname = 'public') <> 1 THEN + RAISE EXCEPTION 'Restore database must contain exactly one standard public schema'; + END IF; + + SELECT pg_get_userbyid(nspowner) + INTO public_schema_owner + FROM pg_namespace + WHERE nspname = 'public'; + IF public_schema_owner <> 'pg_database_owner' THEN + RAISE EXCEPTION 'Restore database public schema has unexpected owner: %', public_schema_owner; + END IF; + + SELECT count(*) + INTO unsafe_public_acl + FROM pg_namespace n + CROSS JOIN LATERAL aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) acl + WHERE n.nspname = 'public' + AND acl.grantee <> n.nspowner + AND NOT ( + acl.grantee = 0 + AND acl.privilege_type = 'USAGE' + AND NOT acl.is_grantable + ); + IF unsafe_public_acl <> 0 THEN + RAISE EXCEPTION 'Restore database public schema has unexpected privileges'; + END IF; + + SELECT count(*) + INTO existing_objects + FROM ( + SELECT 1 FROM pg_namespace + WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'public') + AND nspname !~ '^pg_(toast|temp)' + UNION ALL + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid = c.collnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_conversion c JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_operator o JOIN pg_namespace n ON n.oid = o.oprnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_opclass o JOIN pg_namespace n ON n.oid = o.opcnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_opfamily o JOIN pg_namespace n ON n.oid = o.opfnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_config c JOIN pg_namespace n ON n.oid = c.cfgnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_parser p JOIN pg_namespace n ON n.oid = p.prsnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_ts_template t JOIN pg_namespace n ON n.oid = t.tmplnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace + WHERE n.nspname = 'public' + UNION ALL + SELECT 1 FROM pg_extension WHERE extname <> 'plpgsql' + UNION ALL + SELECT 1 FROM pg_event_trigger + UNION ALL + SELECT 1 FROM pg_publication + UNION ALL + SELECT 1 FROM pg_subscription + WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + UNION ALL + SELECT 1 FROM pg_foreign_data_wrapper + UNION ALL + SELECT 1 FROM pg_foreign_server + UNION ALL + SELECT 1 FROM pg_user_mapping + UNION ALL + SELECT 1 FROM pg_transform + UNION ALL + SELECT 1 FROM pg_largeobject_metadata + UNION ALL + SELECT 1 FROM pg_am WHERE oid >= 16384 + UNION ALL + SELECT 1 FROM pg_cast WHERE oid >= 16384 + UNION ALL + SELECT 1 FROM pg_default_acl + UNION ALL + SELECT 1 FROM pg_language + WHERE lanname NOT IN ('internal', 'c', 'sql', 'plpgsql') + ) AS user_defined_objects; + + IF existing_objects <> 0 THEN + RAISE EXCEPTION 'Restore database must be empty; found % user-defined object(s)', existing_objects; + END IF; +END +$synapse_preflight$; +SQL + cat "$restore_sql" + cat <<'SQL' + +-- Re-open the disposable target only after the protected restore finishes. +SELECT format('ALTER DATABASE %I ALLOW_CONNECTIONS true', current_database()) \gexec +SQL +} | psql "$postgres_restore_url" --no-psqlrc --set=ON_ERROR_STOP=1 >/dev/null +connections_closed=0 + mkdir -p "$RESTORE_UPLOAD_DIR" tar -xzf "$SYNAPSE_BACKUP_PATH/uploads.tar.gz" -C "$RESTORE_UPLOAD_DIR" From 1758a9fb354ddc178c7bf38a8948bd98b4fda6ed Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 10:50:57 -0400 Subject: [PATCH 30/44] test: pin public schema and exclusive restore database guards --- tests/backup-restore.test.ts | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index c4ae6900..60edaf6f 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -21,7 +21,7 @@ test("backup captures only a canonical regular upload tree with integrity metada assert.match(script, /trap cleanup_failed_backup ERR INT TERM/); }); -test("restore drill fails closed on archive, target path, and database state", async () => { +test("restore drill fails closed on archive, target path, database state, and concurrent access", async () => { const script = await readFile("scripts/restore-drill.sh", "utf8"); const reconcile = await readFile("scripts/reconcile-uploads.ts", "utf8"); @@ -34,7 +34,20 @@ test("restore drill fails closed on archive, target path, and database state", a assert.match(script, /grep -Evq '\^\[-d\]' "\$archive_listing"/); assert.doesNotMatch(script, /tar -tvzf[^\n]*\|[^\n]*grep -q/); assert.match(script, /non-regular or non-directory entries/); - assert.match(script, /psql .*--set=ON_ERROR_STOP=1/); + + assert.match(script, /dedicated disposable database/); + assert.match(script, /ALLOW_CONNECTIONS false/); + assert.match(script, /ALLOW_CONNECTIONS true/); + assert.match(script, /pg_terminate_backend\(pid\)/); + assert.match(script, /Restore database still has another active connection/); + assert.match(script, /connections_closed=1/); + assert.match(script, /reenable_target_connections/); + assert.match(script, /postgres_admin_url/); + + assert.match(script, /public_schema_owner <> 'pg_database_owner'/); + assert.match(script, /aclexplode\(COALESCE\(n\.nspacl, acldefault\('n', n\.nspowner\)\)\)/); + assert.match(script, /public schema has unexpected privileges/); + for (const catalog of [ "pg_namespace", "pg_class", @@ -73,14 +86,18 @@ test("restore drill fails closed on archive, target path, and database state", a assert.match(script, /pg_am WHERE oid >= 16384/); assert.match(script, /SELECT 1 FROM pg_default_acl/); assert.match(script, /Restore database must be empty/); - assert.match(script, /pg_restore --exit-on-error --no-owner --no-privileges/); + + assert.match(script, /pg_restore --no-owner --no-privileges/); + assert.match(script, /--file="\$restore_sql"/); assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); - assert.match(script, /url\.searchParams\.delete\("schema"\)/); + assert.match(script, /\\set ON_ERROR_STOP on/); + assert.match(script, /url\.searchParams\.delete\('schema'\)|url\.searchParams\.delete\("schema"\)/); assert.match(script, /prisma migrate status/); assert.match( script, /SYNAPSE_UPLOAD_DIR_CONFIRM='restore-drill-disposable-upload-target'[\s\S]*npm run uploads:reconcile -- "\$RESTORE_UPLOAD_DIR"/ ); + assert.doesNotMatch(reconcile, /process\.env\.SYNAPSE_UPLOAD_DIR/); assert.match(reconcile, /process\.argv\[2\]/); assert.match(reconcile, /SYNAPSE_UPLOAD_DIR_CONFIRM/); From 57ece7fc80bbc3ac125131ffc03fc66598b9d7f7 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 10:51:42 -0400 Subject: [PATCH 31/44] docs: document protected restore database session --- OPERATIONS.md | 54 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 21283a43..cc13a13b 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -110,25 +110,41 @@ Never point the database assertion at a different deployment from the HTTP targe ## Restore drill -Use a newly created, empty disposable database and an empty disposable uploads -directory. The drill refuses to run without the exact confirmation value. It -canonicalizes the restore upload target and rejects linked path components, then -verifies backup checksums and fully materializes archive name/type listings before -checking traversal and member types. Only ordinary regular-file and directory archive -members are accepted; symbolic links, hard links, FIFOs, devices, and other special -members fail closed before extraction. The complete listing is consumed from a file, -so validation cannot be short-circuited by a producer receiving SIGPIPE. - -Before import, the drill queries PostgreSQL across user schemas, namespace-owned -relations/functions/types/operators/text-search objects, extensions, event triggers, -publications/subscriptions, foreign-data objects, transforms, large objects, custom -access methods/casts, non-default languages, and pre-existing default ACL rows. Any -detected user-defined object or default privilege fails the empty-DB gate. Because the -target is independently verified empty, restore does not use `pg_restore --clean` and -does not intentionally drop existing objects. - -The drill then restores both stores, confirms Prisma migration state, and reconciles -the exact restored upload directory through a restore-only explicit argument. Normal +Use a newly created, dedicated disposable database and an empty disposable uploads +directory. Do not target `postgres`, `template0`, or `template1`. The drill refuses to +run without the exact confirmation value. It canonicalizes the restore upload target +and rejects linked path components, then verifies backup checksums and fully +materializes archive name/type listings before checking traversal and member types. +Only ordinary regular-file and directory archive members are accepted; symbolic links, +hard links, FIFOs, devices, and other special members fail closed before extraction. +The complete listing is consumed from a file, so validation cannot be short-circuited +by a producer receiving SIGPIPE. + +Before import, the drill creates one protected PostgreSQL session. That session disables +new connections to the disposable target, terminates other target sessions, and refuses +to continue if any competing connection remains. The same protected session then runs +the database preflight and applies SQL generated from the backup, eliminating the race +between an emptiness check and restore. A cleanup path reconnects through the standard +`postgres` administrative database to re-enable target connections if the protected +session fails after closing them. + +The database preflight requires exactly one standard `public` schema owned by +`pg_database_owner`. It rejects unexpected `public` schema ACLs; outside the owner, +only ordinary non-grantable `USAGE` for `PUBLIC` is accepted. It also queries user +schemas, namespace-owned relations/functions/types/operators/text-search objects, +extensions, event triggers, publications/subscriptions, foreign-data objects, +transforms, large objects, custom access methods/casts, non-default languages, and +pre-existing default ACL rows. Any detected user-defined object, unsafe `public` schema +metadata, default privilege, or competing session fails the empty-target gate. + +Because the target is independently verified and isolated, restore does not use +`pg_restore --clean` and does not intentionally drop existing objects. The archive is +rendered to SQL with owner/privilege restoration disabled, then applied with psql +`ON_ERROR_STOP` inside the protected target session. Connections are re-enabled only +after that protected restore completes. + +The drill then restores uploads, confirms Prisma migration state, and reconciles the +exact restored upload directory through a restore-only explicit argument. Normal production startup still reconciles only its standard `./uploads` directory. ```bash From 4c377b1a5071d23aabf4b74ed076c4cee66d2d9b Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 11:54:21 -0400 Subject: [PATCH 32/44] fix: reject prepared transactions before restore --- scripts/restore-drill.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index aeaf5d7b..b9f12cb8 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -266,6 +266,8 @@ BEGIN UNION ALL SELECT 1 FROM pg_default_acl UNION ALL + SELECT 1 FROM pg_prepared_xacts WHERE database = current_database() + UNION ALL SELECT 1 FROM pg_language WHERE lanname NOT IN ('internal', 'c', 'sql', 'plpgsql') ) AS user_defined_objects; From d6dabd8bedab44b32bad6b62ba26a58127eb3c81 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 11:54:46 -0400 Subject: [PATCH 33/44] test: pin prepared transaction restore gate --- tests/backup-restore.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 60edaf6f..db51b442 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -75,6 +75,7 @@ test("restore drill fails closed on archive, target path, database state, and co "pg_am", "pg_cast", "pg_default_acl", + "pg_prepared_xacts", "pg_language", ]) { assert.match(script, new RegExp(catalog)); @@ -85,6 +86,7 @@ test("restore drill fails closed on archive, target path, database state, and co ); assert.match(script, /pg_am WHERE oid >= 16384/); assert.match(script, /SELECT 1 FROM pg_default_acl/); + assert.match(script, /pg_prepared_xacts WHERE database = current_database\(\)/); assert.match(script, /Restore database must be empty/); assert.match(script, /pg_restore --no-owner --no-privileges/); @@ -114,5 +116,6 @@ test("runbook keeps production recovery gated on owner and operator evidence", a assert.match(runbook, /encrypted off-site destination/); assert.match(runbook, /target recovery point and recovery time objectives/); + assert.match(runbook, /prepared transactions/); assert.match(runbook, /backup and recovery remain a release blocker/); }); From c3bfc1c0d9fff0db9bff36a6331b7fb9abdc95d5 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 11:55:16 -0400 Subject: [PATCH 34/44] docs: record prepared transaction restore guard --- OPERATIONS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index cc13a13b..7445ec1f 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -133,9 +133,10 @@ The database preflight requires exactly one standard `public` schema owned by only ordinary non-grantable `USAGE` for `PUBLIC` is accepted. It also queries user schemas, namespace-owned relations/functions/types/operators/text-search objects, extensions, event triggers, publications/subscriptions, foreign-data objects, -transforms, large objects, custom access methods/casts, non-default languages, and -pre-existing default ACL rows. Any detected user-defined object, unsafe `public` schema -metadata, default privilege, or competing session fails the empty-target gate. +transforms, large objects, custom access methods/casts, non-default languages, +pre-existing default ACL rows, and prepared transactions for the target database. Any +detected user-defined object, unsafe `public` schema metadata, default privilege, +prepared transaction, or competing session fails the empty-target gate. Because the target is independently verified and isolated, restore does not use `pg_restore --clean` and does not intentionally drop existing objects. The archive is From a813f33f0c7bbb6dec7c1a46e69adbf9e70fafa1 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 11:59:44 -0400 Subject: [PATCH 35/44] fix: verify referenced uploads during restore --- scripts/reconcile-uploads.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/reconcile-uploads.ts b/scripts/reconcile-uploads.ts index 49bb90dd..8daf2f87 100644 --- a/scripts/reconcile-uploads.ts +++ b/scripts/reconcile-uploads.ts @@ -28,6 +28,7 @@ function resolveUploadDir() { } async function main() { + const explicitRestoreTarget = Boolean(process.argv[2]); const uploadDir = resolveUploadDir(); const trashDir = path.join(uploadDir, ".trash"); await mkdir(uploadDir, { recursive: true }); @@ -43,8 +44,28 @@ async function main() { .filter((filename): filename is string => Boolean(filename)) ); + const uploadEntries = await readdir(uploadDir, { withFileTypes: true }); + const regularFiles = new Set( + uploadEntries.filter((entry) => entry.isFile()).map((entry) => entry.name) + ); + + // The explicit target is available only to the disposable restore drill. In + // that mode, reconciliation must prove completeness as well as remove stale + // files: every attachment row restored from PostgreSQL must have its matching + // regular file before any orphan cleanup can make the result look healthy. + if (explicitRestoreTarget) { + const missingReferenced = [...referenced].filter( + (filename) => !regularFiles.has(filename) + ); + if (missingReferenced.length > 0) { + throw new Error( + `restore upload target is missing ${missingReferenced.length} referenced attachment file(s)` + ); + } + } + let removedOrphans = 0; - for (const entry of await readdir(uploadDir, { withFileTypes: true })) { + for (const entry of uploadEntries) { if (!entry.isFile() || referenced.has(entry.name)) continue; await rm(path.join(uploadDir, entry.name), { force: true }); removedOrphans++; From 215c8b35e73b8843e7783340f48f0addc2c644df Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 12:00:08 -0400 Subject: [PATCH 36/44] test: require referenced uploads in restore reconciliation --- tests/backup-restore.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index db51b442..635a0ef8 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -21,7 +21,7 @@ test("backup captures only a canonical regular upload tree with integrity metada assert.match(script, /trap cleanup_failed_backup ERR INT TERM/); }); -test("restore drill fails closed on archive, target path, database state, and concurrent access", async () => { +test("restore drill fails closed on archive, target path, database state, concurrent access, and missing uploads", async () => { const script = await readFile("scripts/restore-drill.sh", "utf8"); const reconcile = await readFile("scripts/reconcile-uploads.ts", "utf8"); @@ -109,6 +109,9 @@ test("restore drill fails closed on archive, target path, database state, and co assert.match(reconcile, /resolved === homedir\(\)/); assert.match(reconcile, /resolved === process\.cwd\(\)/); assert.match(reconcile, /path\.join\(process\.cwd\(\), "uploads"\)/); + assert.match(reconcile, /explicitRestoreTarget = Boolean\(process\.argv\[2\]\)/); + assert.match(reconcile, /missingReferenced/); + assert.match(reconcile, /restore upload target is missing .* referenced attachment file/); }); test("runbook keeps production recovery gated on owner and operator evidence", async () => { @@ -117,5 +120,6 @@ test("runbook keeps production recovery gated on owner and operator evidence", a assert.match(runbook, /encrypted off-site destination/); assert.match(runbook, /target recovery point and recovery time objectives/); assert.match(runbook, /prepared transactions/); + assert.match(runbook, /every restored attachment row/); assert.match(runbook, /backup and recovery remain a release blocker/); }); From f8de3ee3180a455634a3261f82020675795e85fe Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 12:00:44 -0400 Subject: [PATCH 37/44] docs: require restored attachment completeness --- OPERATIONS.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 7445ec1f..c7cedff7 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -144,9 +144,16 @@ rendered to SQL with owner/privilege restoration disabled, then applied with psq `ON_ERROR_STOP` inside the protected target session. Connections are re-enabled only after that protected restore completes. -The drill then restores uploads, confirms Prisma migration state, and reconciles the -exact restored upload directory through a restore-only explicit argument. Normal -production startup still reconciles only its standard `./uploads` directory. +After the upload archive is extracted, restore-only upload reconciliation verifies +that every restored attachment row has a matching regular file in the restored upload +directory before removing orphan files or rebuilding quota accounting. A database dump +that references a file missing from the archive therefore fails the drill instead of +being reported healthy. Normal production startup does not enable this completeness +check; it retains the standard `./uploads` reconciliation behaviour. + +The drill then confirms Prisma migration state and reconciles the exact restored upload +directory through the restore-only explicit argument. Normal production startup still +reconciles only its standard `./uploads` directory. ```bash SYNAPSE_BACKUP_PATH='/absolute/path/to/one/synapse-backup' \ From 74932ba69042967706a7e39a54d283b90b848c1d Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 13:10:17 -0400 Subject: [PATCH 38/44] fix(recovery): reject user-created system-schema objects --- scripts/restore-drill.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index b9f12cb8..acdd256b 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -165,6 +165,7 @@ $synapse_connections$; DO $synapse_preflight$ DECLARE existing_objects bigint; + system_schema_objects bigint; public_schema_owner text; unsafe_public_acl bigint; BEGIN @@ -195,6 +196,56 @@ BEGIN RAISE EXCEPTION 'Restore database public schema has unexpected privileges'; END IF; + -- PostgreSQL reserves OIDs below FirstNormalObjectId (16384) for built-in/initdb + -- objects. Any namespace-owned object at or above that boundary inside a system + -- schema was created later and makes this supposed disposable target non-pristine. + SELECT count(*) + INTO system_schema_objects + FROM ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND c.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND p.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND t.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid = c.collnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND c.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_conversion c JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND c.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_operator o JOIN pg_namespace n ON n.oid = o.oprnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND o.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_opclass o JOIN pg_namespace n ON n.oid = o.opcnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND o.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_opfamily o JOIN pg_namespace n ON n.oid = o.opfnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND o.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_ts_config c JOIN pg_namespace n ON n.oid = c.cfgnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND c.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND d.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_ts_parser p JOIN pg_namespace n ON n.oid = p.prsnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND p.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_ts_template t JOIN pg_namespace n ON n.oid = t.tmplnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND t.oid >= 16384 + UNION ALL + SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace + WHERE n.nspname IN ('pg_catalog', 'information_schema') AND s.oid >= 16384 + ) AS user_created_system_schema_objects; + + IF system_schema_objects <> 0 THEN + RAISE EXCEPTION 'Restore database system schemas contain % user-created object(s)', system_schema_objects; + END IF; + SELECT count(*) INTO existing_objects FROM ( From 59a3004b621131df7b617e58ba8d6a058e8cf53f Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 13:10:43 -0400 Subject: [PATCH 39/44] test(recovery): pin system-schema contamination gate --- tests/backup-restore.test.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 635a0ef8..de3a34da 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -48,6 +48,31 @@ test("restore drill fails closed on archive, target path, database state, concur assert.match(script, /aclexplode\(COALESCE\(n\.nspacl, acldefault\('n', n\.nspowner\)\)\)/); assert.match(script, /public schema has unexpected privileges/); + const systemSchemaStart = script.indexOf("INTO system_schema_objects"); + const existingObjectsStart = script.indexOf("INTO existing_objects"); + assert.ok(systemSchemaStart >= 0 && existingObjectsStart > systemSchemaStart); + const systemSchemaBlock = script.slice(systemSchemaStart, existingObjectsStart); + assert.match(systemSchemaBlock, /n\.nspname IN \('pg_catalog', 'information_schema'\)/); + assert.match(systemSchemaBlock, /oid >= 16384/); + for (const catalog of [ + "pg_class", + "pg_proc", + "pg_type", + "pg_collation", + "pg_conversion", + "pg_operator", + "pg_opclass", + "pg_opfamily", + "pg_ts_config", + "pg_ts_dict", + "pg_ts_parser", + "pg_ts_template", + "pg_statistic_ext", + ]) { + assert.match(systemSchemaBlock, new RegExp(catalog)); + } + assert.match(script, /system schemas contain % user-created object/); + for (const catalog of [ "pg_namespace", "pg_class", @@ -120,6 +145,7 @@ test("runbook keeps production recovery gated on owner and operator evidence", a assert.match(runbook, /encrypted off-site destination/); assert.match(runbook, /target recovery point and recovery time objectives/); assert.match(runbook, /prepared transactions/); + assert.match(runbook, /system-named schemas/); assert.match(runbook, /every restored attachment row/); assert.match(runbook, /backup and recovery remain a release blocker/); }); From 431d7da80c9bac56aa16423ab61a235a176b6fd8 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sat, 5 Sep 2026 13:11:14 -0400 Subject: [PATCH 40/44] docs(recovery): document system-schema contamination gate --- OPERATIONS.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index c7cedff7..96b55a7b 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -134,9 +134,13 @@ only ordinary non-grantable `USAGE` for `PUBLIC` is accepted. It also queries us schemas, namespace-owned relations/functions/types/operators/text-search objects, extensions, event triggers, publications/subscriptions, foreign-data objects, transforms, large objects, custom access methods/casts, non-default languages, -pre-existing default ACL rows, and prepared transactions for the target database. Any -detected user-defined object, unsafe `public` schema metadata, default privilege, -prepared transaction, or competing session fails the empty-target gate. +pre-existing default ACL rows, and prepared transactions for the target database. +A separate system-schema check rejects namespace-owned objects with user-assigned OIDs +at or above PostgreSQL's `FirstNormalObjectId` boundary (`16384`) inside the +system-named schemas `pg_catalog` and `information_schema`; ordinary low-OID built-in +and initdb objects remain accepted. Any detected user-defined object, unsafe `public` +schema metadata, default privilege, prepared transaction, system-schema contamination, +or competing session fails the empty-target gate. Because the target is independently verified and isolated, restore does not use `pg_restore --clean` and does not intentionally drop existing objects. The archive is From f448f36d882f5b56bb9377b51b9b1226f506db6c Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sun, 6 Sep 2026 02:30:43 -0400 Subject: [PATCH 41/44] fix: verify restore cleanup path before database freeze --- scripts/restore-drill.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index acdd256b..41bf5085 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -131,6 +131,14 @@ if [[ -z "$postgres_restore_url" || -z "$postgres_admin_url" || -z "$restore_dat exit 2 fi +# Prove the exact maintenance connection and permission path needed by failure +# cleanup before making the disposable target inaccessible. This is idempotent while +# the target still accepts connections and fails closed if `postgres` is unavailable, +# CONNECT is denied, the target is missing, or ALTER DATABASE is not permitted. +psql "$postgres_admin_url" --no-psqlrc --set=ON_ERROR_STOP=1 --set=target_db="$restore_database_name" >/dev/null <<'SQL' +SELECT format('ALTER DATABASE %I ALLOW_CONNECTIONS true', :'target_db') \gexec +SQL + # Generate SQL before closing target connections. Execution happens later through # one protected psql session so the emptiness check and restore cannot be raced. pg_restore --no-owner --no-privileges \ From f4ce253581ad71f27021fc11d4569b1c65ffe167 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sun, 6 Sep 2026 02:31:10 -0400 Subject: [PATCH 42/44] test: pin restore cleanup-path preflight --- tests/backup-restore.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index de3a34da..16cb135a 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -21,7 +21,7 @@ test("backup captures only a canonical regular upload tree with integrity metada assert.match(script, /trap cleanup_failed_backup ERR INT TERM/); }); -test("restore drill fails closed on archive, target path, database state, concurrent access, and missing uploads", async () => { +test("restore drill fails closed on archive, target path, database state, concurrent access, cleanup access, and missing uploads", async () => { const script = await readFile("scripts/restore-drill.sh", "utf8"); const reconcile = await readFile("scripts/reconcile-uploads.ts", "utf8"); @@ -43,6 +43,13 @@ test("restore drill fails closed on archive, target path, database state, concur assert.match(script, /connections_closed=1/); assert.match(script, /reenable_target_connections/); assert.match(script, /postgres_admin_url/); + const cleanupProbe = script.indexOf("Prove the exact maintenance connection and permission path"); + const freezeMarker = script.indexOf("connections_closed=1"); + assert.ok(cleanupProbe >= 0 && freezeMarker > cleanupProbe); + assert.match( + script.slice(cleanupProbe, freezeMarker), + /psql "\$postgres_admin_url"[\s\S]*ALTER DATABASE %I ALLOW_CONNECTIONS true/ + ); assert.match(script, /public_schema_owner <> 'pg_database_owner'/); assert.match(script, /aclexplode\(COALESCE\(n\.nspacl, acldefault\('n', n\.nspowner\)\)\)/); @@ -146,6 +153,7 @@ test("runbook keeps production recovery gated on owner and operator evidence", a assert.match(runbook, /target recovery point and recovery time objectives/); assert.match(runbook, /prepared transactions/); assert.match(runbook, /system-named schemas/); + assert.match(runbook, /maintenance connection/); assert.match(runbook, /every restored attachment row/); assert.match(runbook, /backup and recovery remain a release blocker/); }); From f4dd3c9c1bbc833b314de13a7a76a846fdcd40c3 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sun, 6 Sep 2026 02:31:43 -0400 Subject: [PATCH 43/44] docs: document restore cleanup-path preflight --- OPERATIONS.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 96b55a7b..49aa2c7d 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -120,13 +120,17 @@ hard links, FIFOs, devices, and other special members fail closed before extract The complete listing is consumed from a file, so validation cannot be short-circuited by a producer receiving SIGPIPE. -Before import, the drill creates one protected PostgreSQL session. That session disables -new connections to the disposable target, terminates other target sessions, and refuses -to continue if any competing connection remains. The same protected session then runs -the database preflight and applies SQL generated from the backup, eliminating the race -between an emptiness check and restore. A cleanup path reconnects through the standard -`postgres` administrative database to re-enable target connections if the protected -session fails after closing them. +Before import, the drill first proves that its `postgres` maintenance connection can +execute the exact idempotent `ALTER DATABASE … ALLOW_CONNECTIONS true` operation needed +for failure cleanup. If the maintenance database is unavailable, CONNECT is denied, +the target is missing, or the role lacks that permission, the drill aborts before +freezing the disposable target. It then creates one protected PostgreSQL session. That +session disables new connections to the disposable target, terminates other target +sessions, and refuses to continue if any competing connection remains. The same +protected session then runs the database preflight and applies SQL generated from the +backup, eliminating the race between an emptiness check and restore. A cleanup path +reconnects through the already-validated maintenance connection to re-enable target +connections if the protected session fails after closing them. The database preflight requires exactly one standard `public` schema owned by `pg_database_owner`. It rejects unexpected `public` schema ACLs; outside the owner, From 1f0877757c79e16ebdc0b91b68f1ecf8059449c3 Mon Sep 17 00:00:00 2001 From: Cameron Ashley Date: Sun, 6 Sep 2026 02:40:11 -0400 Subject: [PATCH 44/44] test: decouple restore URL assertion from variable name --- tests/backup-restore.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/backup-restore.test.ts b/tests/backup-restore.test.ts index 16cb135a..5cf828e7 100644 --- a/tests/backup-restore.test.ts +++ b/tests/backup-restore.test.ts @@ -125,7 +125,7 @@ test("restore drill fails closed on archive, target path, database state, concur assert.match(script, /--file="\$restore_sql"/); assert.doesNotMatch(script, /pg_restore[^\n]*--clean/); assert.match(script, /\\set ON_ERROR_STOP on/); - assert.match(script, /url\.searchParams\.delete\('schema'\)|url\.searchParams\.delete\("schema"\)/); + assert.match(script, /searchParams\.delete\(['"]schema['"]\)/); assert.match(script, /prisma migrate status/); assert.match( script,