feat(cert-store): certificate store export/import for disaster recovery

README.md

@@ -1179,6 +1179,61 @@ If you want a name approved at handshake time by your own application rather
 than at deploy time, that is what [on-demand TLS](#on-demand-tls) is for. Your
 endpoint answers `2xx` to approve, anything else to deny.
 
+### Backing up and restoring certificates
+
+All certificate state lives on the proxy node's disk: the ACME account key,
+every issued certificate with its private key, the domain-to-certificate
+mappings, and the dynamic domain list. Losing that disk means re-issuing the
+whole estate under the issuance rate limit (about 250 orders per 3 hours) —
+for a large estate, hours of hard TLS failures. Export makes node loss a
+restore instead of an outage:
+
+```bash
+kamal-proxy export certs /backup/certs-$(date +%F).tar.gz
+```
+
+With the proxy running, the snapshot is taken through the proxy, under the
+same lock the certificate managers use for writes, so a backup taken
+mid-renewal is never torn. With no proxy reachable on its socket, the data
+directory is read directly — only do that when the proxy is actually stopped.
+
+**The archive contains private keys** — every certificate key and the ACME
+account key. It is written with mode `0600`; store and transfer it as the
+secret it is.
+
+Backups are only as good as their last verification. `--verify` parses every
+certificate in an archive and reports domains and expiries without touching
+the store, so a cron job or CI can check each backup as it is taken:
+
+```bash
+kamal-proxy import certs --archive /backup/certs-2026-08-09.tar.gz --verify
+```
+
+**Restore runbook** (new node, rebuilt host, or a volume mistake):
+
+1. Stop the proxy.
+2. Restore the estate: `kamal-proxy import certs --archive /backup/certs-2026-08-09.tar.gz`
+   (add `--data-dir` if the proxy runs with one). The import refuses to
+   overwrite a non-empty certificate store unless you pass `--force`.
+3. If you keep a backup of the routing state (`kamal-proxy.state`), restore
+   it now, while the proxy is still stopped — the proxy saves routing state
+   on changes, so a copy restored after startup would be overwritten.
+4. Start the proxy. If no routing state was restored, redeploy your TLS
+   services — the archive holds certificates, not routes, and the proxy
+   refuses a TLS handshake for a host no service is deployed for.
+5. Verify a restored static host with a TLS handshake; the certificate expiry
+   metrics should show the restored estate, with no new ACME orders.
+   (`kamal-proxy domains list` covers only dynamic `--tls-domains-source`
+   domains.)
+
+Restores run offline against the data directory, sharing their writing path
+with the Traefik `acme.json` importer (`import certs --traefik-acme`), so
+there is one code path that knows how to populate the store correctly.
+
+A multi-node shared certificate store is deliberately not what this is: with
+single-node TLS termination plus backups, losing the node is a restore, not
+an outage.
+
 
 ## Specifying `run` options with environment variables
 

internal/cmd/export.go

@@ -0,0 +1,102 @@
+package cmd
+
+import (
+	"fmt"
+	"net/rpc"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/basecamp/kamal-proxy/internal/server"
+)
+
+type exportCommand struct {
+	cmd *cobra.Command
+}
+
+func newExportCommand() *exportCommand {
+	exportCommand := &exportCommand{}
+	exportCommand.cmd = &cobra.Command{
+		Use:   "export",
+		Short: "Export proxy state for backup",
+	}
+
+	exportCommand.cmd.AddCommand(newExportCertsCommand().cmd)
+
+	return exportCommand
+}
+
+// exportCertsCommand archives the certificate store for disaster recovery.
+// Against a running proxy it exports over the RPC socket, under the same lock
+// the certificate managers use for writes, so a backup taken mid-renewal is
+// never torn. Without a reachable proxy it falls back to reading the data
+// directory offline -- only safe when the proxy is actually stopped.
+type exportCertsCommand struct {
+	cmd *cobra.Command
+}
+
+func newExportCertsCommand() *exportCertsCommand {
+	exportCertsCommand := &exportCertsCommand{}
+	exportCertsCommand.cmd = &cobra.Command{
+		Use:   "certs <output-path>",
+		Short: "Export the certificate store to an archive for disaster recovery",
+		Long: "Export the certificate store -- ACME account key, issued certificates,\n" +
+			"domain mappings, and dynamic domain state -- to a gzipped tar archive.\n\n" +
+			"With the proxy running, the snapshot is taken through the proxy under its\n" +
+			"certificate write lock. With no proxy reachable on the socket, the data\n" +
+			"directory is read directly; only do that with the proxy stopped.\n\n" +
+			"The archive contains PRIVATE KEYS (certificate keys and the ACME account\n" +
+			"key). It is written with mode 0600; store and transfer it accordingly.",
+		RunE: exportCertsCommand.run,
+		Args: cobra.ExactArgs(1),
+	}
+
+	exportCertsCommand.cmd.Flags().StringVar(&globalConfig.AlternateConfigDir, "data-dir", getEnvString("DATA_DIR", ""), "Directory for state and certificate storage (default $HOME/.config/kamal-proxy)")
+
+	return exportCertsCommand
+}
+
+func (c *exportCertsCommand) run(cmd *cobra.Command, args []string) error {
+	outputPath, err := filepath.Abs(args[0])
+	if err != nil {
+		return fmt.Errorf("failed to resolve the output path: %w", err)
+	}
+
+	summary, err := c.export(cmd, outputPath)
+	if err != nil {
+		return err
+	}
+
+	for _, warning := range summary.Warnings {
+		fmt.Fprintf(cmd.ErrOrStderr(), "WARN %s\n", warning)
+	}
+
+	fmt.Fprintf(cmd.OutOrStdout(), "Exported %d certificates (%d domains) to %s\n",
+		summary.Certificates, summary.Domains, outputPath)
+
+	return nil
+}
+
+// export snapshots through the running proxy when the socket answers, and
+// falls back to reading the data directory offline when it does not.
+func (c *exportCertsCommand) export(cmd *cobra.Command, outputPath string) (server.CertsExportSummary, error) {
+	var summary server.CertsExportSummary
+
+	client, dialErr := rpc.Dial("unix", globalConfig.SocketPath())
+	if dialErr == nil {
+		defer client.Close()
+		err := client.Call("kamal-proxy.CertsExport", server.CertsExportArgs{Path: outputPath}, &summary)
+		if err != nil && strings.HasPrefix(err.Error(), "rpc: can't find method kamal-proxy.CertsExport") {
+			// A proxy is answering the socket but predates this command. Do
+			// NOT fall back to reading the data dir -- that proxy is live and
+			// writing, which is exactly the torn-snapshot case the RPC path
+			// exists to prevent.
+			return summary, fmt.Errorf("the running proxy does not support certificate export; upgrade it, or stop it and re-run for an offline export: %w", err)
+		}
+		return summary, err
+	}
+
+	fmt.Fprintln(cmd.ErrOrStderr(), "Proxy is not running; exporting offline from the data directory")
+	return server.ExportCertificateStore(globalConfig.CertStorePaths(), outputPath)
+}

internal/cmd/export_test.go

@@ -0,0 +1,129 @@
+package cmd
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/basecamp/kamal-proxy/internal/server"
+)
+
+// runExportCerts executes `export certs` with the given args and returns the
+// combined output. The socket is pointed at nowhere so the command always
+// falls back to the offline path.
+func runExportCerts(t *testing.T, args ...string) (string, error) {
+	t.Helper()
+
+	t.Setenv("KAMAL_PROXY_SOCKET", filepath.Join(t.TempDir(), "no-proxy.sock"))
+
+	previousConfig := globalConfig
+	t.Cleanup(func() {
+		globalConfig = previousConfig
+	})
+	globalConfig = server.Config{}
+	cmd := newExportCommand().cmd
+
+	out := &bytes.Buffer{}
+	cmd.SetOut(out)
+	cmd.SetErr(out)
+	cmd.SetArgs(append([]string{"certs"}, args...))
+
+	err := cmd.Execute()
+	return out.String(), err
+}
+
+// seedCertStore writes a minimal but valid store into dir.
+func seedCertStore(t *testing.T, dir string) {
+	t.Helper()
+
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "acme.state"),
+		[]byte(`{"certificates":{},"domain_map":{},"saved_at":"2026-08-09T00:00:00Z"}`), 0600))
+}
+
+func TestExportCertsCommand_RequiresAnOutputPath(t *testing.T) {
+	_, err := runExportCerts(t)
+	require.Error(t, err)
+}
+
+func TestExportCertsCommand_ExportsOffline(t *testing.T) {
+	dir := t.TempDir()
+	seedCertStore(t, dir)
+
+	archivePath := filepath.Join(t.TempDir(), "backup.tar.gz")
+	out, err := runExportCerts(t, archivePath, "--data-dir", dir)
+	require.NoError(t, err)
+
+	assert.Contains(t, out, "offline")
+	assert.Contains(t, out, "Exported 0 certificates (0 domains)")
+	assert.FileExists(t, archivePath)
+}
+
+func TestExportCertsCommand_EmptyStoreFails(t *testing.T) {
+	_, err := runExportCerts(t, filepath.Join(t.TempDir(), "backup.tar.gz"), "--data-dir", t.TempDir())
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "empty")
+}
+
+func TestImportCertsCommand_ArchiveRestoreRoundTrip(t *testing.T) {
+	source := t.TempDir()
+	seedCertStore(t, source)
+
+	archivePath := filepath.Join(t.TempDir(), "backup.tar.gz")
+	_, err := runExportCerts(t, archivePath, "--data-dir", source)
+	require.NoError(t, err)
+
+	// Restore into an empty data dir.
+	target := t.TempDir()
+	out, err := runImportCerts(t, "--archive", archivePath, "--data-dir", target)
+	require.NoError(t, err)
+	assert.Contains(t, out, "Restored 0 certificates (0 domains)")
+	assert.FileExists(t, filepath.Join(target, "acme.state"))
+
+	// A second restore refuses the now non-empty store...
+	_, err = runImportCerts(t, "--archive", archivePath, "--data-dir", target)
+	require.ErrorIs(t, err, server.ErrCertStoreNotEmpty)
+
+	// ...unless forced.
+	_, err = runImportCerts(t, "--archive", archivePath, "--data-dir", target, "--force")
+	require.NoError(t, err)
+}
+
+func TestImportCertsCommand_VerifyReportsWithoutWriting(t *testing.T) {
+	source := t.TempDir()
+	seedCertStore(t, source)
+
+	archivePath := filepath.Join(t.TempDir(), "backup.tar.gz")
+	_, err := runExportCerts(t, archivePath, "--data-dir", source)
+	require.NoError(t, err)
+
+	target := t.TempDir()
+	out, err := runImportCerts(t, "--archive", archivePath, "--verify", "--data-dir", target)
+	require.NoError(t, err)
+
+	assert.Contains(t, out, "Certificates: 0")
+	assert.NoFileExists(t, filepath.Join(target, "acme.state"),
+		"--verify must not touch the store")
+}
+
+func TestImportCertsCommand_FlagValidation(t *testing.T) {
+	tests := []struct {
+		name string
+		args []string
+	}{
+		{name: "archive and traefik-acme are exclusive", args: []string{"--archive", "a.tar.gz", "--traefik-acme", "acme.json"}},
+		{name: "resolver applies only to traefik", args: []string{"--archive", "a.tar.gz", "--resolver", "le"}},
+		{name: "verify requires archive", args: []string{"--traefik-acme", "acme.json", "--verify"}},
+		{name: "verify and force are exclusive", args: []string{"--archive", "a.tar.gz", "--verify", "--force"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := runImportCerts(t, tt.args...)
+			require.Error(t, err)
+		})
+	}
+}