release: harden installation and lifecycle journeys

README.md

@@ -63,10 +63,11 @@ Kubernetes objects, persistent volumes, or application data.
 
 ## Quick start
 
-The complete procedure, including a working multi-node `ClusterConfig`, PXE
-arguments, disk safety rules, and troubleshooting, is in the
-[installation guide](docs/installing.md). The outline below shows the normal ISO
-path.
+The complete first-cluster procedure is in
+[Build your first KatlOS cluster](docs/getting-started.md). The
+[installation reference](docs/installing.md) covers advanced configuration,
+PXE arguments, disk safety, and troubleshooting. The outline below shows the
+normal ISO path.
 
 ### 1. Download a release
 
@@ -214,7 +215,8 @@ The node agent fetches the selected Kubernetes OCI bundle, verifies its
 manifest and layer digests, stages the sysext, creates generation 1, and runs
 the bounded kubeadm operation. Katl reports phase changes and writes the
 operator kubeconfig to `./kubeconfig`; rerunning the unchanged command resumes
-an interrupted bootstrap.
+an interrupted bootstrap. Nodes normally remain `NotReady` and CoreDNS pending
+until the user installs a CNI; Katl does not choose or manage one.
 
 ## Configuration and upgrades
 
@@ -312,14 +314,20 @@ matching loose artifacts, one explicitly selected disk per node, the matching
 `katlctl`, and kubeadm bootstrap using a compatible published Kubernetes
 bundle. Hardware claims extend only to retained release evidence.
 
-- [Installing KatlOS](docs/installing.md) — complete ISO and PXE workflows.
+- [KatlOS documentation](docs/README.md) — website-style navigation for the
+  complete user journey.
+- [Build your first cluster](docs/getting-started.md) — focused ISO install,
+  generation 0, kubeadm bootstrap, and CNI handoff.
+- [Install with PXE and Matchbox](docs/install-pxe-matchbox.md) — automated
+  network boot using one published machine-config bundle.
+- [Installing KatlOS](docs/installing.md) — complete configuration and install
+  reference.
 - [Operating KatlOS](docs/operations/README.md) — task-oriented runbooks for
-  access, bootstrap, configuration, upgrades, wipe/reinstall, and diagnosis.
+  access, bootstrap, configuration, membership, upgrades, recovery, and
+  diagnosis.
 - [Support boundary](docs/support.md) — compatibility, trust, recovery, and
   reporting expectations.
 - [Developing Katl](docs/developing.md) — build, test, and contribution loop.
-- [North-star architecture](docs/internal/north-star.md) — durable product
-  direction and system boundaries.
 - [GitHub issues](https://github.com/katl-dev/katl/issues) — bugs and feature
   tracking; use [private vulnerability reporting](https://github.com/katl-dev/katl/security/advisories/new)
   for security-sensitive reports.

cmd/katlctl/documentation_test.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+)
+
+func TestDocumentedKatlctlCommandsAndFlagsExist(t *testing.T) {
+	repo, err := filepath.Abs(filepath.Join("..", ".."))
+	if err != nil {
+		t.Fatal(err)
+	}
+	paths := []string{filepath.Join(repo, "README.md")}
+	if err := filepath.WalkDir(filepath.Join(repo, "docs"), func(path string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() && path == filepath.Join(repo, "docs", "internal") {
+			return filepath.SkipDir
+		}
+		if !entry.IsDir() && strings.EqualFold(filepath.Ext(path), ".md") {
+			paths = append(paths, path)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	root := newKatlctlCommand(context.Background(), io.Discard, io.Discard)
+	for _, path := range paths {
+		content, err := os.Open(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		scanner := bufio.NewScanner(content)
+		lineNumber := 0
+		for scanner.Scan() {
+			lineNumber++
+			line := strings.TrimSpace(scanner.Text())
+			if !strings.HasPrefix(line, "katlctl ") {
+				continue
+			}
+			startLine := lineNumber
+			invocation := strings.TrimSuffix(line, "\\")
+			for strings.HasSuffix(line, "\\") && scanner.Scan() {
+				lineNumber++
+				line = strings.TrimSpace(scanner.Text())
+				invocation += " " + strings.TrimSuffix(line, "\\")
+			}
+			checkDocumentedInvocation(t, root, path, startLine, invocation)
+		}
+		if err := scanner.Err(); err != nil {
+			t.Errorf("scan %s: %v", path, err)
+		}
+		content.Close()
+	}
+}
+
+func checkDocumentedInvocation(t *testing.T, root *cobra.Command, path string, line int, invocation string) {
+	t.Helper()
+	fields := strings.Fields(invocation)
+	if len(fields) < 2 {
+		return
+	}
+	command, remaining, err := root.Find(fields[1:])
+	if err != nil {
+		t.Errorf("%s:%d documents an unknown command in %q: %v", path, line, invocation, err)
+		return
+	}
+	if command.HasSubCommands() && len(remaining) > 0 && !strings.HasPrefix(remaining[0], "-") {
+		t.Errorf("%s:%d documents unknown %s subcommand %q", path, line, command.CommandPath(), remaining[0])
+		return
+	}
+	for _, field := range remaining {
+		if !strings.HasPrefix(field, "--") {
+			continue
+		}
+		name := strings.TrimPrefix(strings.SplitN(field, "=", 2)[0], "--")
+		if command.Flags().Lookup(name) == nil && command.InheritedFlags().Lookup(name) == nil {
+			t.Errorf("%s:%d documents unknown %s flag --%s", path, line, command.CommandPath(), name)
+		}
+	}
+}

cmd/katlctl/etcd.go

@@ -40,6 +40,7 @@ type etcdRemovalPlan struct {
 	Target      inventory.PlannedNode
 	Status      *agentapi.EtcdStatus
 	Member      *agentapi.EtcdMember
+	AlreadyGone bool
 }
 
 func newEtcdCommand(ctx context.Context, stdout, stderr io.Writer) *cobra.Command {
@@ -124,6 +125,14 @@ func runEtcdRemove(ctx context.Context, opts etcdRemoveOptions, stdout, stderr i
 }
 
 func planEtcdRemoval(ctx context.Context, inv inventory.Inventory, targetName, coordinatorName, confirmedMemberID string) (etcdRemovalPlan, error) {
+	return planEtcdRemovalState(ctx, inv, targetName, coordinatorName, confirmedMemberID, false)
+}
+
+func planWipeEtcdRemoval(ctx context.Context, inv inventory.Inventory, targetName, coordinatorName string) (etcdRemovalPlan, error) {
+	return planEtcdRemovalState(ctx, inv, targetName, coordinatorName, "", true)
+}
+
+func planEtcdRemovalState(ctx context.Context, inv inventory.Inventory, targetName, coordinatorName, confirmedMemberID string, allowAbsent bool) (etcdRemovalPlan, error) {
 	plan, err := planWipeInventory(inv)
 	if err != nil {
 		return etcdRemovalPlan{}, err
@@ -156,8 +165,17 @@ func planEtcdRemoval(ctx context.Context, inv inventory.Inventory, targetName, c
 			member = candidate
 			break
 		}
+		if candidate.GetName() == target.Name || containsString(candidate.GetPeerUrls(), expectedPeer) {
+			return etcdRemovalPlan{}, fmt.Errorf("etcd member identity for %s does not match expected name and peer URL %s", target.Name, expectedPeer)
+		}
 	}
 	if member == nil {
+		if allowAbsent {
+			if status.GetHealthyMembers() < status.GetQuorum() {
+				return etcdRemovalPlan{}, fmt.Errorf("etcd has %d healthy members, below quorum %d", status.GetHealthyMembers(), status.GetQuorum())
+			}
+			return etcdRemovalPlan{Coordinator: coordinator, Target: target, Status: status, AlreadyGone: true}, nil
+		}
 		return etcdRemovalPlan{}, fmt.Errorf("etcd member for %s with peer URL %s is not present", target.Name, expectedPeer)
 	}
 	if strings.TrimSpace(confirmedMemberID) != "" && !strings.EqualFold(strings.TrimSpace(confirmedMemberID), member.GetId()) {

cmd/katlctl/etcd_test.go

@@ -6,9 +6,11 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/katl-dev/katl/internal/bootstrap/cluster"
+	"github.com/katl-dev/katl/internal/bootstrap/inventory"
 	agentapi "github.com/katl-dev/katl/internal/katlc/agentapi"
 )
 
@@ -42,6 +44,71 @@ func TestWipeNodePlansSafeControlPlaneMembershipRemoval(t *testing.T) {
 	}
 }
 
+func TestWipeNodeResumesAfterEtcdMemberWasRemoved(t *testing.T) {
+	inventoryPath := writeThreeControlPlaneInventory(t)
+	cp3 := readyWipeClusterClient("machine-cp-3")
+	cp3.nodeStatus.Kubernetes = &agentapi.KubernetesStatus{State: "waiting-for-control-plane", Role: "control-plane"}
+	cp1 := readyWipeClusterClient("machine-cp-1")
+	cp1.etcdStatus = healthyTwoMemberEtcdStatus()
+	connector := newFakeWipeClusterConnector(map[string]*fakeKatlcAgentClient{"cp-1": cp1, "cp-3": cp3})
+	oldConnector := newWipeClusterConnector
+	newWipeClusterConnector = func() cluster.AgentConnector { return connector }
+	oldKubectl := operatorKubectlRunner
+	kubectl := &fakeKubectlRunner{}
+	operatorKubectlRunner = kubectl
+	t.Cleanup(func() {
+		newWipeClusterConnector = oldConnector
+		operatorKubectlRunner = oldKubectl
+	})
+
+	var stdout, stderr bytes.Buffer
+	err := run(context.Background(), []string{
+		"node", "wipe", "cp-3", "--inventory", inventoryPath, "--kubeconfig", "admin.conf", "--output", "json",
+	}, &stdout, &stderr)
+	if err != nil {
+		t.Fatalf("run() error = %v, stderr = %s", err, stderr.String())
+	}
+	var report wipeNodeReport
+	if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
+		t.Fatal(err)
+	}
+	if report.EtcdCleanup != "already-removed" || report.EtcdCoordinator != "cp-1" || report.EtcdMemberID != "" || report.KubernetesCleanup != "succeeded" {
+		t.Fatalf("wipe report = %+v", report)
+	}
+	if cp1.submitRequest != nil {
+		t.Fatalf("coordinator submitted another operation: %+v", cp1.submitRequest)
+	}
+	if cp3.submitRequest == nil || cp3.submitRequest.GetDestructiveReset() == nil {
+		t.Fatalf("target wipe request = %+v", cp3.submitRequest)
+	}
+	wantCalls := [][]string{
+		{"kubectl", "--kubeconfig", "admin.conf", "cordon", "cp-3"},
+		{"kubectl", "--kubeconfig", "admin.conf", "drain", "cp-3", "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--timeout=25m"},
+		{"kubectl", "--kubeconfig", "admin.conf", "--server=https://10.0.0.11:6443", "delete", "node", "cp-3", "--ignore-not-found=true"},
+	}
+	if !reflect.DeepEqual(kubectl.calls, wantCalls) {
+		t.Fatalf("kubectl calls = %#v, want %#v", kubectl.calls, wantCalls)
+	}
+}
+
+func TestWipeNodeTextReportsKubernetesCleanupFailure(t *testing.T) {
+	report := wipeNodeReport{wipeClusterReport: wipeClusterReport{Output: "text", Targets: []wipeClusterTarget{{Name: "cp-3", SystemRole: string(inventory.RoleControlPlane), Address: "10.0.0.13"}}}}
+	report.EtcdCleanup = "succeeded"
+	report.EtcdCoordinator = "cp-1"
+	report.EtcdMemberID = "3"
+	report.KubernetesCleanup = "recovery-required"
+	report.KubernetesDiagnostics = []string{"delete node failed: API unavailable"}
+	var output bytes.Buffer
+	if err := printWipeNodeReport(&output, report); err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"etcd cleanup=succeeded coordinator=cp-1 member=3", "Kubernetes cleanup=recovery-required", "Kubernetes: delete node failed: API unavailable"} {
+		if !bytes.Contains(output.Bytes(), []byte(want)) {
+			t.Fatalf("output = %q, want %q", output.String(), want)
+		}
+	}
+}
+
 func TestEtcdRemoveRequiresObservedMemberID(t *testing.T) {
 	err := runEtcdRemove(context.Background(), etcdRemoveOptions{
 		etcdOptions: etcdOptions{configPath: writeThreeControlPlaneInventory(t), output: "text"},
@@ -63,6 +130,14 @@ func healthyThreeMemberEtcdStatus() *agentapi.EtcdStatus {
 	}
 }
 
+func healthyTwoMemberEtcdStatus() *agentapi.EtcdStatus {
+	status := healthyThreeMemberEtcdStatus()
+	status.Members = status.Members[:2]
+	status.HealthyMembers = 2
+	status.Quorum = 2
+	return status
+}
+
 func writeThreeControlPlaneInventory(t *testing.T) string {
 	t.Helper()
 	path := filepath.Join(t.TempDir(), "cluster.yaml")

cmd/katlctl/host_management.go

@@ -46,6 +46,7 @@ type hostStatusReport struct {
 	KatlOSVersion        string                      `json:"katlosVersion,omitempty"`
 	NextBoot             string                      `json:"nextBoot,omitempty"`
 	Activity             string                      `json:"activity"`
+	BootHealthDiagnostic string                      `json:"bootHealthDiagnostic,omitempty"`
 	Kubernetes           *kubernetesStatusReport     `json:"kubernetes,omitempty"`
 	ControlPlaneEndpoint *controlPlaneEndpointReport `json:"controlPlaneEndpoint,omitempty"`
 	Volumes              []volumeStatusReport        `json:"volumes,omitempty"`
@@ -359,6 +360,9 @@ func readHostState(ctx context.Context, client agentapi.KatlcAgentClient, node s
 		return nil, nil, fmt.Errorf("read status from %s: %w", node, err)
 	}
 	generationID := strings.TrimSpace(status.GetCurrentGenerationId())
+	if status.GetBootHealthState() == "failed" && strings.TrimSpace(status.GetSelectedGenerationId()) != "" {
+		generationID = strings.TrimSpace(status.GetSelectedGenerationId())
+	}
 	if generationID == "" {
 		return nil, nil, fmt.Errorf("%s did not report a current KatlOS generation", node)
 	}
@@ -374,13 +378,18 @@ func newHostStatusReport(node, endpoint string, status *agentapi.NodeStatus, cur
 	if status.GetOperationLockHeld() {
 		activity = "busy"
 	}
+	health := displayHostHealth(current)
+	if bootHealth := strings.TrimSpace(status.GetBootHealthState()); bootHealth != "" && bootHealth != "healthy" {
+		health = bootHealth
+	}
 	report := hostStatusReport{
 		Node:                 node,
 		Endpoint:             endpoint,
-		Health:               displayHostHealth(current),
+		Health:               health,
 		Generation:           current.GetGenerationId(),
 		KatlOSVersion:        strings.TrimSpace(current.GetRuntimeVersion()),
 		Activity:             activity,
+		BootHealthDiagnostic: strings.TrimSpace(status.GetBootHealthDiagnostic()),
 		Kubernetes:           newKubernetesStatusReport(status.GetKubernetes()),
 		ControlPlaneEndpoint: newControlPlaneEndpointReport(status.GetControlPlaneEndpoint()),
 	}
@@ -474,6 +483,11 @@ func writeHostStatus(stdout io.Writer, output string, report hostStatusReport) e
 	if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", report.Node, report.Health, kubernetes, version, report.Generation, nextBoot, report.Activity); err != nil {
 		return err
 	}
+	if report.BootHealthDiagnostic != "" {
+		if _, err := fmt.Fprintf(w, "\t%s\n", report.BootHealthDiagnostic); err != nil {
+			return err
+		}
+	}
 	if report.Kubernetes != nil && report.Kubernetes.FailureReason != "" {
 		if _, err := fmt.Fprintf(w, "\t\t%s\n", report.Kubernetes.FailureReason); err != nil {
 			return err

cmd/katlctl/host_management_test.go

@@ -89,6 +89,30 @@ func TestHostStatusJSON(t *testing.T) {
 	}
 }
 
+func TestHostStatusReportsSelectedFallbackAndBootFailure(t *testing.T) {
+	fake := healthyHostClient("machine-a", "agent-a", "generation-0")
+	fake.nodeStatus.CurrentGenerationId = "generation-1"
+	fake.nodeStatus.SelectedGenerationId = "generation-0"
+	fake.nodeStatus.BootTargetGenerationId = "generation-1"
+	fake.nodeStatus.BootHealthState = "failed"
+	fake.nodeStatus.BootHealthDiagnostic = "running generation generation-0 does not match durable boot evidence generation-1"
+	installKatlcDial(t, nil, fake)
+
+	var stdout, stderr bytes.Buffer
+	if err := run(context.Background(), []string{"node", "status", "node-a", "--endpoint", "node-a.test:9443"}, &stdout, &stderr); err != nil {
+		t.Fatalf("run() error = %v, stderr = %s", err, stderr.String())
+	}
+	output := stdout.String()
+	for _, want := range []string{"failed", "generation-0", "generation-1", "does not match durable boot evidence"} {
+		if !strings.Contains(output, want) {
+			t.Fatalf("output missing %q:\n%s", want, output)
+		}
+	}
+	if strings.Contains(output, "\tOK\t") {
+		t.Fatalf("output masks boot failure as healthy:\n%s", output)
+	}
+}
+
 func TestHostRebootDefaultAllowsBootDeadmanRecovery(t *testing.T) {
 	cmd := newHostRebootCommand(context.Background(), io.Discard, io.Discard)
 	timeout, err := cmd.Flags().GetDuration("timeout")