Skip to content

Commit 4c9937e

Browse files
ZarielZariel
authored
upgrade: tolerate versioned root slot labels (#118)
Resolve the active root from the committed generation PARTUUID and find its peer by GPT partition type, so repeated systemd-sysupdate operations do not depend on mutable labels. Fail closed on ambiguous layouts and exercise upgrade, rollback, and repeated staging in the installed-runtime VM journey.
1 parent e9db75d commit 4c9937e

3 files changed

Lines changed: 143 additions & 36 deletions

File tree

‎internal/katlc/agent/host_upgrade_executor.go‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,11 @@ func (e *Executor) executeHostUpgrade(ctx context.Context, record operation.Oper
5454
if err != nil {
5555
return e.failHostUpgrade(record, "verify-katlos-image", fmt.Errorf("read current generation: %w", err))
5656
}
57-
inactiveSlot, inactiveLabel, activeLabel, err := inactiveRoot(previousSpec.Root.Slot)
57+
inactiveSlot, err := inactiveRoot(previousSpec.Root.Slot)
5858
if err != nil {
5959
return e.failHostUpgrade(record, "verify-katlos-image", err)
6060
}
61-
slots, err := e.inspectRootSlots(ctx, activeLabel, inactiveLabel)
61+
slots, err := e.inspectRootSlots(ctx, previousSpec.Root.PartitionUUID)
6262
if err != nil {
6363
return e.failHostUpgrade(record, "verify-katlos-image", err)
6464
}
@@ -262,19 +262,45 @@ type rootSlots struct {
262262
InactivePartUUID string
263263
}
264264

265-
func (e *Executor) inspectRootSlots(ctx context.Context, activeLabel, inactiveLabel string) (rootSlots, error) {
266-
active, err := e.toolOutput(ctx, "blkid", "-t", "PARTLABEL="+activeLabel, "-o", "device")
265+
func (e *Executor) inspectRootSlots(ctx context.Context, activePartUUID string) (rootSlots, error) {
266+
active, err := e.toolOutput(ctx, "blkid", "-t", "PARTUUID="+activePartUUID, "-o", "device")
267267
if err != nil {
268268
return rootSlots{}, fmt.Errorf("find active root slot: %w", err)
269269
}
270-
inactive, err := e.toolOutput(ctx, "blkid", "-t", "PARTLABEL="+inactiveLabel, "-o", "device")
271-
if err != nil {
272-
return rootSlots{}, fmt.Errorf("find inactive root slot: %w", err)
273-
}
274270
activeDisk, activePart, err := e.partitionIdentity(ctx, active)
275271
if err != nil {
276272
return rootSlots{}, err
277273
}
274+
activeType, err := e.toolOutput(ctx, "lsblk", "-no", "PARTTYPE", active)
275+
if err != nil {
276+
return rootSlots{}, fmt.Errorf("inspect active root partition type: %w", err)
277+
}
278+
partitionTable, err := e.toolOutput(ctx, "lsblk", "-rno", "PATH,PARTTYPE", activeDisk)
279+
if err != nil {
280+
return rootSlots{}, fmt.Errorf("inspect root partitions on %s: %w", activeDisk, err)
281+
}
282+
rootDevices := make([]string, 0, 2)
283+
for _, line := range strings.Split(partitionTable, "\n") {
284+
fields := strings.Fields(line)
285+
if len(fields) == 2 && strings.EqualFold(fields[1], activeType) {
286+
rootDevices = append(rootDevices, fields[0])
287+
}
288+
}
289+
if len(rootDevices) != 2 {
290+
return rootSlots{}, fmt.Errorf("find inactive root slot: found %d root partitions with type %q on %s, want 2", len(rootDevices), activeType, activeDisk)
291+
}
292+
inactive := ""
293+
activeFound := false
294+
for _, device := range rootDevices {
295+
if device == active {
296+
activeFound = true
297+
continue
298+
}
299+
inactive = device
300+
}
301+
if !activeFound || inactive == "" {
302+
return rootSlots{}, fmt.Errorf("find inactive root slot: active device %s is not one of %v", active, rootDevices)
303+
}
278304
inactiveDisk, inactivePart, err := e.partitionIdentity(ctx, inactive)
279305
if err != nil {
280306
return rootSlots{}, err
@@ -339,14 +365,14 @@ func (e *Executor) failHostUpgrade(record operation.OperationRecord, phase strin
339365
return errors.Join(cause, err)
340366
}
341367

342-
func inactiveRoot(current string) (string, string, string, error) {
368+
func inactiveRoot(current string) (string, error) {
343369
switch current {
344370
case string(disk.RootSlotA):
345-
return string(disk.RootSlotB), disk.GPTLabelRootB, disk.GPTLabelRootA, nil
371+
return string(disk.RootSlotB), nil
346372
case string(disk.RootSlotB):
347-
return string(disk.RootSlotA), disk.GPTLabelRootA, disk.GPTLabelRootB, nil
373+
return string(disk.RootSlotA), nil
348374
default:
349-
return "", "", "", fmt.Errorf("current generation root slot %q is unsupported", current)
375+
return "", fmt.Errorf("current generation root slot %q is unsupported", current)
350376
}
351377
}
352378

‎internal/katlc/agent/host_upgrade_executor_test.go‎

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,23 +118,35 @@ func TestExecutorStagesHostUpgradeAndArmsTrial(t *testing.T) {
118118
return nil
119119
}
120120
executor.SetBootOneshot = func(_ context.Context, _ string, entry string) error { oneshoot = entry; return nil }
121+
var toolCalls []string
121122
executor.RunTool = func(_ context.Context, argv []string, _ func(int)) ToolResult {
123+
toolCalls = append(toolCalls, strings.Join(argv, " "))
122124
switch filepath.Base(argv[0]) {
123125
case "blkid":
124126
joined := strings.Join(argv, " ")
125127
switch {
126-
case strings.Contains(joined, "PARTLABEL=KATL_ROOT_A"):
128+
case strings.Contains(joined, "PARTUUID=aaaaaaaa-1111-2222-3333-444444444444"):
127129
return ToolResult{Stdout: []byte(activeDevice + "\n")}
128-
case strings.Contains(joined, "PARTLABEL=KATL_ROOT_B"):
129-
return ToolResult{Stdout: []byte(inactiveDevice + "\n")}
130-
default:
130+
case argv[len(argv)-1] == inactiveDevice:
131131
return ToolResult{Stdout: []byte("bbbbbbbb-1111-2222-3333-444444444444\n")}
132+
default:
133+
return ToolResult{Err: os.ErrNotExist, ExitStatus: 2}
132134
}
133135
case "lsblk":
136+
joined := strings.Join(argv, " ")
137+
switch {
138+
case strings.Contains(joined, "-no PARTTYPE "):
139+
return ToolResult{Stdout: []byte("4f68bce3-e8cd-4db1-96e7-fbcaf984b709\n")}
140+
case strings.Contains(joined, "-rno PATH,PARTTYPE "):
141+
return ToolResult{Stdout: []byte("/dev/vda\n/dev/vda1 c12a7328-f81f-11d2-ba4b-00a0c93ec93b\n" + activeDevice + " 4f68bce3-e8cd-4db1-96e7-fbcaf984b709\n" + inactiveDevice + " 4f68bce3-e8cd-4db1-96e7-fbcaf984b709\n/dev/vda4 0fc63daf-8483-4772-8e79-3d69d8477de4\n")}
142+
}
134143
if argv[len(argv)-1] == activeDevice {
135144
return ToolResult{Stdout: []byte("vda 2\n")}
136145
}
137-
return ToolResult{Stdout: []byte("vda 3\n")}
146+
if argv[len(argv)-1] == inactiveDevice {
147+
return ToolResult{Stdout: []byte("vda 3\n")}
148+
}
149+
return ToolResult{Err: os.ErrNotExist, ExitStatus: 1}
138150
case "sfdisk", "partx":
139151
return ToolResult{}
140152
case "systemd-sysupdate":
@@ -152,6 +164,9 @@ func TestExecutorStagesHostUpgradeAndArmsTrial(t *testing.T) {
152164
if err := executor.Execute(context.Background(), record); err != nil {
153165
t.Fatalf("Execute() error = %v", err)
154166
}
167+
if calls := strings.Join(toolCalls, "\n"); strings.Contains(calls, "PARTLABEL=") {
168+
t.Fatalf("host upgrade discovered mutable partition labels:\n%s", calls)
169+
}
155170
if oneshoot != "loader/entries/katl-gen1.conf" {
156171
t.Fatalf("oneshot entry = %q", oneshoot)
157172
}
@@ -182,6 +197,52 @@ func TestExecutorStagesHostUpgradeAndArmsTrial(t *testing.T) {
182197
}
183198
}
184199

200+
func TestInspectRootSlotsRejectsAmbiguousPartitionLayout(t *testing.T) {
201+
const rootType = "4f68bce3-e8cd-4db1-96e7-fbcaf984b709"
202+
for _, test := range []struct {
203+
name string
204+
partition string
205+
want string
206+
}{
207+
{
208+
name: "missing peer",
209+
partition: "/dev/vda2 " + rootType + "\n",
210+
want: "found 1 root partitions",
211+
},
212+
{
213+
name: "extra peer",
214+
partition: "/dev/vda2 " + rootType + "\n/dev/vda3 " + rootType + "\n/dev/vda4 " + rootType + "\n",
215+
want: "found 3 root partitions",
216+
},
217+
{
218+
name: "active absent",
219+
partition: "/dev/vda3 " + rootType + "\n/dev/vda4 " + rootType + "\n",
220+
want: "active device /dev/vda2 is not one of",
221+
},
222+
} {
223+
t.Run(test.name, func(t *testing.T) {
224+
executor := &Executor{RunTool: func(_ context.Context, argv []string, _ func(int)) ToolResult {
225+
switch strings.Join(argv, " ") {
226+
case "blkid -t PARTUUID=active-partuuid -o device":
227+
return ToolResult{Stdout: []byte("/dev/vda2\n")}
228+
case "lsblk -no PKNAME,PARTN /dev/vda2":
229+
return ToolResult{Stdout: []byte("vda 2\n")}
230+
case "lsblk -no PARTTYPE /dev/vda2":
231+
return ToolResult{Stdout: []byte(rootType + "\n")}
232+
case "lsblk -rno PATH,PARTTYPE /dev/vda":
233+
return ToolResult{Stdout: []byte(test.partition)}
234+
default:
235+
return ToolResult{Err: os.ErrNotExist, ExitStatus: 1}
236+
}
237+
}}
238+
_, err := executor.inspectRootSlots(context.Background(), "active-partuuid")
239+
if err == nil || !strings.Contains(err.Error(), test.want) {
240+
t.Fatalf("inspectRootSlots() error = %v, want %q", err, test.want)
241+
}
242+
})
243+
}
244+
}
245+
185246
func testSHA(data []byte) string {
186247
sum := sha256.Sum256(data)
187248
return hex.EncodeToString(sum[:])

‎internal/vmtest/sysupdate_smoke_test.go‎

Lines changed: 39 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -116,28 +116,12 @@ func TestInstalledRuntimeSysupdateRootUKITransfer(t *testing.T) {
116116
conn.Close()
117117
t.Fatalf("read node status before host upgrade: %v", err)
118118
}
119-
accepted, err := katlc.SubmitOperation(ctx, &agentapi.SubmitOperationRequest{
120-
ApiVersion: operation.APIVersion,
121-
Kind: agent.RequestKind,
122-
ClientRequestId: "vmtest-host-upgrade-" + candidateGeneration,
123-
OperationKind: agent.OperationKindHostUpgrade,
124-
Actor: "installed runtime host upgrade vmtest",
125-
ExpectedMachineId: nodeStatus.GetMachineId(),
126-
ExpectedCurrentGenerationId: previousGeneration,
127-
HostUpgrade: &agentapi.HostUpgradeOperationRequest{
128-
ImageLocalRef: localRef,
129-
CandidateGenerationId: candidateGeneration,
130-
},
131-
})
132119
conn.Close()
133-
if err != nil {
134-
t.Fatalf("submit host upgrade operation: %v", err)
135-
}
136-
status := waitKatlcOperationTerminal(t, ctx, endpoint, accepted.GetOperationId())
120+
operationID, status := submitHostUpgradeAndWait(t, ctx, endpoint, nodeStatus.GetMachineId(), previousGeneration, candidateGeneration, localRef)
137121
if status.GetResult() != operation.ResultSucceeded || !status.GetBootHealthPending() || status.GetCandidateGenerationId() != candidateGeneration {
138122
t.Fatalf("host upgrade operation status = %+v", status)
139123
}
140-
recordData := readGuestFile(t, ctx, guest, "/var/lib/katl/operations/"+accepted.GetOperationId()+"/record.json")
124+
recordData := readGuestFile(t, ctx, guest, "/var/lib/katl/operations/"+operationID+"/record.json")
141125
envelope, err := persistedrecord.DecodeEnvelope([]byte(recordData))
142126
if err != nil {
143127
t.Fatalf("decode host upgrade operation envelope: %v", err)
@@ -191,9 +175,22 @@ func TestInstalledRuntimeSysupdateRootUKITransfer(t *testing.T) {
191175
if previousSpec.RuntimeVersion == candidateSpec.RuntimeVersion || previousSpec.Boot.UKIPath == candidateSpec.Boot.UKIPath || previousSpec.Root.Slot == candidateSpec.Root.Slot {
192176
t.Fatalf("upgrade did not produce distinct version, root slot, and UKI identities: previous=%#v candidate=%#v", previousSpec, candidateSpec)
193177
}
178+
repeatedGeneration := candidateGeneration + "-repeat"
179+
_, repeatedStatus := submitHostUpgradeAndWait(t, ctx, endpoint, nodeStatus.GetMachineId(), previousGeneration, repeatedGeneration, localRef)
180+
if repeatedStatus.GetResult() != operation.ResultSucceeded || !repeatedStatus.GetBootHealthPending() || repeatedStatus.GetCandidateGenerationId() != repeatedGeneration {
181+
t.Fatalf("repeated host upgrade operation status = %+v", repeatedStatus)
182+
}
183+
repeatedSpec := generationFromGuest(t, ctx, guest, repeatedGeneration)
184+
if repeatedSpec.Root.Slot != candidateSpec.Root.Slot || repeatedSpec.Root.PartitionUUID != candidateSpec.Root.PartitionUUID {
185+
t.Fatalf("repeated host upgrade root = %#v, want previously upgraded peer %#v", repeatedSpec.Root, candidateSpec.Root)
186+
}
187+
repeatedTrial := bootSelectionFromGuest(t, ctx, guest)
188+
if repeatedTrial.TrialGenerationID != repeatedGeneration || repeatedTrial.PreviousKnownGoodGenerationID != previousGeneration || !repeatedTrial.PendingHealthValidation {
189+
t.Fatalf("repeated host upgrade trial selection = %#v", repeatedTrial)
190+
}
194191
guestCommand(t, ctx, guest, "boot-health-evidence", "systemctl", "show", "katl-boot-health.service", "--property=Result,ExecMainStatus")
195192
guestCommand(t, ctx, guest, "boot-complete-evidence", "systemctl", "is-active", "katl-boot-complete.target")
196-
t.Log("host upgrade and rollback are serialized per node in v0.1; multi-node rollout orchestration remains operator-controlled")
193+
t.Log("host upgrade, rollback, and repeated upgrade staging are serialized per node in v0.1; multi-node rollout orchestration remains operator-controlled")
197194
powerOffGuestForCleanSuccess(t, ctx, &node, guest, client)
198195
client = nil
199196

@@ -208,6 +205,29 @@ func TestInstalledRuntimeSysupdateRootUKITransfer(t *testing.T) {
208205
}
209206
}
210207

208+
func submitHostUpgradeAndWait(t *testing.T, ctx context.Context, endpoint, machineID, currentGeneration, candidateGeneration, localRef string) (string, *agentapi.OperationStatus) {
209+
t.Helper()
210+
conn, katlc := dialKatlcAgentForVMTest(t, ctx, endpoint)
211+
accepted, err := katlc.SubmitOperation(ctx, &agentapi.SubmitOperationRequest{
212+
ApiVersion: operation.APIVersion,
213+
Kind: agent.RequestKind,
214+
ClientRequestId: "vmtest-host-upgrade-" + candidateGeneration,
215+
OperationKind: agent.OperationKindHostUpgrade,
216+
Actor: "installed runtime host upgrade vmtest",
217+
ExpectedMachineId: machineID,
218+
ExpectedCurrentGenerationId: currentGeneration,
219+
HostUpgrade: &agentapi.HostUpgradeOperationRequest{
220+
ImageLocalRef: localRef,
221+
CandidateGenerationId: candidateGeneration,
222+
},
223+
})
224+
conn.Close()
225+
if err != nil {
226+
t.Fatalf("submit host upgrade operation: %v", err)
227+
}
228+
return accepted.GetOperationId(), waitKatlcOperationTerminal(t, ctx, endpoint, accepted.GetOperationId())
229+
}
230+
211231
type builtUpgradeImage struct {
212232
Path string
213233
Version string

0 commit comments

Comments
 (0)