Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -829,3 +829,7 @@ workflow:
- id: provider-handoff-early-recovery
action: { verifier: deployment.golden.provider-handoff-early-recovery }
expect: { status: passed }

- id: disabled-target-resume-native
action: { verifier: deployment.golden.disabled-target-resume-native }
expect: { status: passed }
5 changes: 5 additions & 0 deletions guarantees/verifiers/golden.verifiers.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ verifiers:
ownerPackage: "@treeseed/deployment"
testFile: tests/workspace-status.test.ts
testName: reads the original catalog without changing its canonical bytes and retains malformed device failures as incomplete observations
deployment.golden.disabled-target-resume-native:
kind: vitestCase
ownerPackage: '@treeseed/deployment'
testFile: tests/development-component-status-native.test.ts
testName: native disabled target recovery retains healthy managed processes and exact persisted selections through repeated authoritative reads
deployment.golden.stopped-candidate-resume-unit:
kind: vitestCase
ownerPackage: "@treeseed/deployment"
Expand Down
8 changes: 5 additions & 3 deletions src/supervisor/development-boot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@ import { resolve } from 'node:path';
import { DevelopmentSessionStore, type ManagedDevelopmentSession } from '../manager/development-sessions.js';
import { assertDevelopmentNotHeld } from '../core/development-backup-hold.js';
import { runtimeStopped } from '../manager/update-state.js';
import { loadHostConfiguration } from '../core/configuration.js';

type Command = (command: string, args: string[], input?: string) => unknown;
const worker = '/usr/lib/treeseed/cli/dist/cli/development/boot-resume.js';

export function developmentResumeRequired(record: ManagedDevelopmentSession) {
return record.session.targets.some(target => target.mode !== 'released' && target.health !== 'ready');
export function developmentResumeRequired(record: ManagedDevelopmentSession, components: Readonly<Record<string, { enabled?: boolean }>> = {}) {
return record.session.targets.some(target => target.mode !== 'released' && target.health !== 'ready'
&& components[target.projectId === 'ai' ? target.targetId : target.projectId]?.enabled !== false);
}

export function captureBootCommand(command: Command, executable: string, args: string[]) {
Expand Down Expand Up @@ -52,7 +54,7 @@ export function resumeDevelopmentAtBoot(sessionId: string, command: Command) {
// RemainAfterExit can be active from an earlier successful boot even after
// host stop marked the live selections stopped. Rerun the fixed worker.
if (state === 'active') {
if (!developmentResumeRequired(record)) return { ready: true };
if (!developmentResumeRequired(record, loadHostConfiguration().components)) return { ready: true };
command('/usr/bin/systemctl', ['restart', '--no-block', unit]);
return { ready: false };
}
Expand Down
14 changes: 14 additions & 0 deletions tests/development-boot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,20 @@ describe('development boot process custody', () => {
const before = structuredClone(record);
expect(developmentResumeRequired(record)).toBe(true);
expect(record).toEqual(before);
const disabled = {session:{targets:[{projectId:'admin',targetId:'web',mode:'live',health:'ready',generation:9},
...['ai-inference','ai-training','ai-lab'].map(targetId=>({projectId:'ai',targetId,mode:'candidate',health:'stopped',generation:2}))]}} as ManagedDevelopmentSession;
const components = {'ai-inference':{enabled:false},'ai-training':{enabled:false},'ai-lab':{enabled:false},admin:{enabled:true}};
const held = structuredClone(disabled), configuration = structuredClone(components);
expect(developmentResumeRequired(disabled, components)).toBe(false);
for(const component of ['ai-inference','ai-training','ai-lab'] as const)
expect(developmentResumeRequired(disabled,{...components,[component]:{enabled:true}})).toBe(true);
expect(developmentResumeRequired(disabled,{})).toBe(true);
for(const health of ['pending','stopped','degraded'] as const){
const failed = structuredClone(disabled);failed.session.targets[0]!.health=health;
expect(developmentResumeRequired(failed,components)).toBe(true);
}
expect(developmentResumeRequired({session:{targets:[{projectId:'admin',targetId:'web',mode:'candidate',health:'stopped'}]}} as ManagedDevelopmentSession,{admin:{enabled:false}})).toBe(false);
expect(disabled).toEqual(held);expect(components).toEqual(configuration);
expect(developmentResumeRequired({ session: { targets: [{ mode: 'candidate', health: 'ready' },
{ mode: 'released', health: 'stopped' }] } } as ManagedDevelopmentSession)).toBe(false);
});
Expand Down
42 changes: 41 additions & 1 deletion tests/development-component-status-native.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ import {beginDevelopmentBackup,finishDevelopmentBackup,planDevelopmentBackup,typ
import {quiescedBackup} from '../src/manager/quiesced-backup.js';
import {assertNoBackupWriters} from '../src/supervisor/backup-writers.js';
import {encryptBackupStream,decryptBackupStream} from '../src/supervisor/backup-stream.js';
import {component} from './fixtures.js';
import {component,host} from './fixtures.js';
import {loadHostConfiguration} from '../src/core/configuration.js';
import {DevelopmentSessionStore} from '../src/manager/development-sessions.js';
import {suspendStoppedDevelopmentSessions} from '../src/manager/development-handoff.js';
import {developmentResumeRequired} from '../src/supervisor/development-boot.js';
Expand Down Expand Up @@ -307,3 +308,42 @@ it('enforces the exact selected RAM ceiling through real Compose cgroups and rej
expect(f.status()).toMatchObject({ready:false});expect(readFileSync(f.override)).toEqual(held);
}finally{f.close();}
},30_000);

it('native disabled target recovery retains healthy managed processes and exact persisted selections through repeated authoritative reads',()=>{
const f=fixture();let id='';try{
id=f.create();expect(f.status()).toMatchObject({ready:true});
const inspect=()=>JSON.parse(f.docker('/usr/bin/docker',['inspect',id,'--format','{{json .State}}']));
const original=inspect();expect(original.Running).toBe(true);expect(original.Pid).toBeGreaterThan(0);
const contract=(project:string,targetIds:string[])=>developmentRuntimeSchema.parse({schemaVersion:'treeseed.development-runtime/v2',
project:{id:project,repository:'treeseed-ai/'+project},defaults:{restoreOnFailure:true},targets:targetIds.map(targetId=>({
id:targetId,kind:'rebuild-restart',executionCustody:'manager',platforms:['linux-amd64'],runtimeRequirements:[],sourceRoots:['src'],ignoredPaths:[],
operations:{start:{command:'manager-runtime',args:[],environment:{},timeoutSeconds:30}},ready:{kind:'process',graceSeconds:0},outputs:[],endpoints:[],dependencies:[],
statePolicy:'stateless',migrationPolicy:'none',secretRefs:{},shutdown:{graceSeconds:1,activeWorkPolicy:'block'},resources:{},logs:[],forbiddenOperations:[],
promotion:{liveAdmissible:false,candidateRequiresVerification:true}}))});
const ai=['ai-inference','ai-training','ai-lab'],sessions=new DevelopmentSessionStore(resolve(f.root,'sessions'));
const session=developmentSessionSchema.parse({schemaVersion:'treeseed.development-session/v2',sessionId:f.input.sessionId,actor:'native-test',hostId:'native-host',
createdAt:new Date().toISOString(),status:'active',repositories:[],leases:[],restoredReceiptId:null,blockers:[],targets:[
{projectId:'treedx',targetId:'service',mode:'candidate',generation:9,health:'ready'},
...ai.map(targetId=>({projectId:'ai',targetId,mode:'candidate',generation:2,health:'stopped'}))]});
sessions.start(session,[contract('treedx',['service']),contract('ai',ai)]);
const configuration=host();configuration.components.treedx={...configuration.components.api!,enabled:true};
for(const component of ai)configuration.components[component]={...configuration.components.api!,enabled:false};
const configPath=resolve(f.root,'host.json');writeFileSync(configPath,JSON.stringify(configuration));
const saved=sessions.load(session.sessionId),recordPath=resolve(f.root,'sessions',session.sessionId+'.json');
const recordBytes=readFileSync(recordPath),configurationBytes=readFileSync(configPath);
for(let iteration=0;iteration<3;iteration++){
expect(developmentResumeRequired(sessions.load(session.sessionId),loadHostConfiguration(configPath).components)).toBe(false);
expect(f.status()).toMatchObject({ready:true});expect(inspect()).toMatchObject({Running:true,Pid:original.Pid,StartedAt:original.StartedAt});
expect(readFileSync(recordPath)).toEqual(recordBytes);expect(readFileSync(configPath)).toEqual(configurationBytes);
}
for(const component of ai){const enabled=loadHostConfiguration(configPath);enabled.components[component]!.enabled=true;
writeFileSync(configPath,JSON.stringify({...enabled,generation:enabled.generation+1}));
expect(developmentResumeRequired(sessions.load(session.sessionId),loadHostConfiguration(configPath).components)).toBe(true);
writeFileSync(configPath,configurationBytes);
}
const failed=structuredClone(saved);failed.session.targets[0]!.health='stopped';sessions.save(failed);
expect(developmentResumeRequired(sessions.load(session.sessionId),loadHostConfiguration(configPath).components)).toBe(true);
expect(inspect()).toMatchObject({Running:true,Pid:original.Pid,StartedAt:original.StartedAt});
}finally{f.close();}
expect(existsSync(f.root)).toBe(false);expect(()=>f.docker('/usr/bin/docker',['inspect',id])).toThrow();
});
Loading