diff --git a/guarantees/agent/golden/scenes/component-boundaries.scene.yaml b/guarantees/agent/golden/scenes/component-boundaries.scene.yaml index 785640cb..348b2d86 100644 --- a/guarantees/agent/golden/scenes/component-boundaries.scene.yaml +++ b/guarantees/agent/golden/scenes/component-boundaries.scene.yaml @@ -829,3 +829,7 @@ workflow: - id: provider-handoff-early-recovery action: { verifier: deployment.golden.provider-handoff-early-recovery } expect: { status: passed } + + - id: disabled-target-resume-native + action: { verifier: deployment.golden.disabled-target-resume-native } + expect: { status: passed } diff --git a/guarantees/verifiers/golden.verifiers.yaml b/guarantees/verifiers/golden.verifiers.yaml index 4a4e7b74..c7f142c1 100644 --- a/guarantees/verifiers/golden.verifiers.yaml +++ b/guarantees/verifiers/golden.verifiers.yaml @@ -101,6 +101,11 @@ verifiers: ownerPackage: "@treeseed/deployment" testFile: tests/workspace-status.test.ts testName: reads the original catalog without changing its canonical bytes and retains malformed device failures as incomplete observations + deployment.golden.disabled-target-resume-native: + kind: vitestCase + ownerPackage: '@treeseed/deployment' + testFile: tests/development-component-status-native.test.ts + testName: native disabled target recovery retains healthy managed processes and exact persisted selections through repeated authoritative reads deployment.golden.stopped-candidate-resume-unit: kind: vitestCase ownerPackage: "@treeseed/deployment" diff --git a/src/supervisor/development-boot.ts b/src/supervisor/development-boot.ts index 0c3e0ef6..2cb24ca7 100644 --- a/src/supervisor/development-boot.ts +++ b/src/supervisor/development-boot.ts @@ -3,12 +3,14 @@ import { resolve } from 'node:path'; import { DevelopmentSessionStore, type ManagedDevelopmentSession } from '../manager/development-sessions.js'; import { assertDevelopmentNotHeld } from '../core/development-backup-hold.js'; import { runtimeStopped } from '../manager/update-state.js'; +import { loadHostConfiguration } from '../core/configuration.js'; type Command = (command: string, args: string[], input?: string) => unknown; const worker = '/usr/lib/treeseed/cli/dist/cli/development/boot-resume.js'; -export function developmentResumeRequired(record: ManagedDevelopmentSession) { - return record.session.targets.some(target => target.mode !== 'released' && target.health !== 'ready'); +export function developmentResumeRequired(record: ManagedDevelopmentSession, components: Readonly> = {}) { + return record.session.targets.some(target => target.mode !== 'released' && target.health !== 'ready' + && components[target.projectId === 'ai' ? target.targetId : target.projectId]?.enabled !== false); } export function captureBootCommand(command: Command, executable: string, args: string[]) { @@ -52,7 +54,7 @@ export function resumeDevelopmentAtBoot(sessionId: string, command: Command) { // RemainAfterExit can be active from an earlier successful boot even after // host stop marked the live selections stopped. Rerun the fixed worker. if (state === 'active') { - if (!developmentResumeRequired(record)) return { ready: true }; + if (!developmentResumeRequired(record, loadHostConfiguration().components)) return { ready: true }; command('/usr/bin/systemctl', ['restart', '--no-block', unit]); return { ready: false }; } diff --git a/tests/development-boot.test.ts b/tests/development-boot.test.ts index eb6dd5e6..d307b186 100644 --- a/tests/development-boot.test.ts +++ b/tests/development-boot.test.ts @@ -35,6 +35,20 @@ describe('development boot process custody', () => { const before = structuredClone(record); expect(developmentResumeRequired(record)).toBe(true); expect(record).toEqual(before); + const disabled = {session:{targets:[{projectId:'admin',targetId:'web',mode:'live',health:'ready',generation:9}, + ...['ai-inference','ai-training','ai-lab'].map(targetId=>({projectId:'ai',targetId,mode:'candidate',health:'stopped',generation:2}))]}} as ManagedDevelopmentSession; + const components = {'ai-inference':{enabled:false},'ai-training':{enabled:false},'ai-lab':{enabled:false},admin:{enabled:true}}; + const held = structuredClone(disabled), configuration = structuredClone(components); + expect(developmentResumeRequired(disabled, components)).toBe(false); + for(const component of ['ai-inference','ai-training','ai-lab'] as const) + expect(developmentResumeRequired(disabled,{...components,[component]:{enabled:true}})).toBe(true); + expect(developmentResumeRequired(disabled,{})).toBe(true); + for(const health of ['pending','stopped','degraded'] as const){ + const failed = structuredClone(disabled);failed.session.targets[0]!.health=health; + expect(developmentResumeRequired(failed,components)).toBe(true); + } + expect(developmentResumeRequired({session:{targets:[{projectId:'admin',targetId:'web',mode:'candidate',health:'stopped'}]}} as ManagedDevelopmentSession,{admin:{enabled:false}})).toBe(false); + expect(disabled).toEqual(held);expect(components).toEqual(configuration); expect(developmentResumeRequired({ session: { targets: [{ mode: 'candidate', health: 'ready' }, { mode: 'released', health: 'stopped' }] } } as ManagedDevelopmentSession)).toBe(false); }); diff --git a/tests/development-component-status-native.test.ts b/tests/development-component-status-native.test.ts index c9ec55ef..ec9672fe 100644 --- a/tests/development-component-status-native.test.ts +++ b/tests/development-component-status-native.test.ts @@ -12,7 +12,8 @@ import {beginDevelopmentBackup,finishDevelopmentBackup,planDevelopmentBackup,typ import {quiescedBackup} from '../src/manager/quiesced-backup.js'; import {assertNoBackupWriters} from '../src/supervisor/backup-writers.js'; import {encryptBackupStream,decryptBackupStream} from '../src/supervisor/backup-stream.js'; -import {component} from './fixtures.js'; +import {component,host} from './fixtures.js'; +import {loadHostConfiguration} from '../src/core/configuration.js'; import {DevelopmentSessionStore} from '../src/manager/development-sessions.js'; import {suspendStoppedDevelopmentSessions} from '../src/manager/development-handoff.js'; import {developmentResumeRequired} from '../src/supervisor/development-boot.js'; @@ -307,3 +308,42 @@ it('enforces the exact selected RAM ceiling through real Compose cgroups and rej expect(f.status()).toMatchObject({ready:false});expect(readFileSync(f.override)).toEqual(held); }finally{f.close();} },30_000); + +it('native disabled target recovery retains healthy managed processes and exact persisted selections through repeated authoritative reads',()=>{ + const f=fixture();let id='';try{ + id=f.create();expect(f.status()).toMatchObject({ready:true}); + const inspect=()=>JSON.parse(f.docker('/usr/bin/docker',['inspect',id,'--format','{{json .State}}'])); + const original=inspect();expect(original.Running).toBe(true);expect(original.Pid).toBeGreaterThan(0); + const contract=(project:string,targetIds:string[])=>developmentRuntimeSchema.parse({schemaVersion:'treeseed.development-runtime/v2', + project:{id:project,repository:'treeseed-ai/'+project},defaults:{restoreOnFailure:true},targets:targetIds.map(targetId=>({ + id:targetId,kind:'rebuild-restart',executionCustody:'manager',platforms:['linux-amd64'],runtimeRequirements:[],sourceRoots:['src'],ignoredPaths:[], + operations:{start:{command:'manager-runtime',args:[],environment:{},timeoutSeconds:30}},ready:{kind:'process',graceSeconds:0},outputs:[],endpoints:[],dependencies:[], + statePolicy:'stateless',migrationPolicy:'none',secretRefs:{},shutdown:{graceSeconds:1,activeWorkPolicy:'block'},resources:{},logs:[],forbiddenOperations:[], + promotion:{liveAdmissible:false,candidateRequiresVerification:true}}))}); + const ai=['ai-inference','ai-training','ai-lab'],sessions=new DevelopmentSessionStore(resolve(f.root,'sessions')); + const session=developmentSessionSchema.parse({schemaVersion:'treeseed.development-session/v2',sessionId:f.input.sessionId,actor:'native-test',hostId:'native-host', + createdAt:new Date().toISOString(),status:'active',repositories:[],leases:[],restoredReceiptId:null,blockers:[],targets:[ + {projectId:'treedx',targetId:'service',mode:'candidate',generation:9,health:'ready'}, + ...ai.map(targetId=>({projectId:'ai',targetId,mode:'candidate',generation:2,health:'stopped'}))]}); + sessions.start(session,[contract('treedx',['service']),contract('ai',ai)]); + const configuration=host();configuration.components.treedx={...configuration.components.api!,enabled:true}; + for(const component of ai)configuration.components[component]={...configuration.components.api!,enabled:false}; + const configPath=resolve(f.root,'host.json');writeFileSync(configPath,JSON.stringify(configuration)); + const saved=sessions.load(session.sessionId),recordPath=resolve(f.root,'sessions',session.sessionId+'.json'); + const recordBytes=readFileSync(recordPath),configurationBytes=readFileSync(configPath); + for(let iteration=0;iteration<3;iteration++){ + expect(developmentResumeRequired(sessions.load(session.sessionId),loadHostConfiguration(configPath).components)).toBe(false); + expect(f.status()).toMatchObject({ready:true});expect(inspect()).toMatchObject({Running:true,Pid:original.Pid,StartedAt:original.StartedAt}); + expect(readFileSync(recordPath)).toEqual(recordBytes);expect(readFileSync(configPath)).toEqual(configurationBytes); + } + for(const component of ai){const enabled=loadHostConfiguration(configPath);enabled.components[component]!.enabled=true; + writeFileSync(configPath,JSON.stringify({...enabled,generation:enabled.generation+1})); + expect(developmentResumeRequired(sessions.load(session.sessionId),loadHostConfiguration(configPath).components)).toBe(true); + writeFileSync(configPath,configurationBytes); + } + const failed=structuredClone(saved);failed.session.targets[0]!.health='stopped';sessions.save(failed); + expect(developmentResumeRequired(sessions.load(session.sessionId),loadHostConfiguration(configPath).components)).toBe(true); + expect(inspect()).toMatchObject({Running:true,Pid:original.Pid,StartedAt:original.StartedAt}); + }finally{f.close();} + expect(existsSync(f.root)).toBe(false);expect(()=>f.docker('/usr/bin/docker',['inspect',id])).toThrow(); +});