[rush-daemon] Fail closed on graph-shape changes

Classify graph-defining invalidations before reconciliation and require whole-session recreation without acknowledging stale state.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: mojazayeri

common/changes/@rushstack/rush-daemon/mojazayeri-warm-graph-components_2026-08-19-22-30.json

@@ -2,7 +2,7 @@
   "changes": [
     {
       "packageName": "@rushstack/rush-daemon",
-      "comment": "Add an opt-in all-project engine component factory with explicit phase/plugin shape, retained invalidation reconciliation, and a deterministic engine shutdown contract.",
+      "comment": "Add an opt-in all-project engine component factory with explicit phase/plugin shape, retained invalidation reconciliation, fail-closed graph recreation boundaries, and a deterministic engine shutdown contract.",
       "type": "minor"
     }
   ],

common/reviews/api/rush-daemon.api.md

@@ -21,6 +21,14 @@ export type CreateWorkspaceEngineComponentsAsync = (options: ICreateWorkspaceEng
 // @beta
 export type CreateWorkspaceSessionComponentsAsync = (options: ICreateWorkspaceSessionComponentsOptions) => Promise<IWorkspaceSessionComponents>;
 
+// @beta
+export interface IClassifyWorkspaceInvalidationsOptions {
+    // (undocumented)
+    readonly changedPaths: ReadonlyArray<string>;
+    // (undocumented)
+    readonly rushConfiguration: RushConfiguration;
+}
+
 // @beta
 export interface ICreateWorkspaceEngineComponentsOptions extends IWorkspaceEngineShape {
     readonly projectSelection: ReadonlySet<RushConfigurationProject>;
@@ -83,11 +91,16 @@ export interface IRushDaemonServeOptions extends IRushDaemonHostOptions {
     readonly shutdownSignal?: AbortSignal;
 }
 
+// @beta
+export type IsWorkspaceEngineRecreationRequiredAsync = (options: IClassifyWorkspaceInvalidationsOptions) => Promise<boolean>;
+
 // @beta
 export interface IWorkspaceEngineComponentFactoryOptions {
     // (undocumented)
     readonly createEngineComponentsAsync: CreateWorkspaceEngineComponentsAsync;
     // (undocumented)
+    readonly isEngineRecreationRequiredAsync?: IsWorkspaceEngineRecreationRequiredAsync;
+    // (undocumented)
     readonly mapInvalidationsToOperationsAsync: MapWorkspaceInvalidationsToOperationsAsync;
     // (undocumented)
     readonly shape: IWorkspaceEngineShape;
@@ -259,11 +272,20 @@ export class WorkspaceEngineComponentFactory {
     readonly shape: IWorkspaceEngineShape;
 }
 
+// @beta
+export class WorkspaceEngineRecreationRequiredError extends Error {
+    constructor();
+}
+
 // @beta
 export class WorkspaceInvalidationTracker {
     acknowledgeThrough(sequence: number): void;
     getSnapshot(): IWorkspaceInvalidationSnapshot;
+    // @internal (undocumented)
+    get hasUnattributedUnknownChanges(): boolean;
     invalidate(changedPath?: string): void;
+    // @internal (undocumented)
+    invalidateForInitialization(): void;
     markWatcherUnhealthy(): void;
 }
 

libraries/rush-daemon/README.md

@@ -20,5 +20,10 @@ declare the complete phase and plugin shape because Rush plugins can currently v
 The factory validates graph ownership, serializes retained invalidation reconciliation, and maps path-specific
 changes through the integration. The engine owner must supply one deterministic async disposer because
 `IOperationGraph` does not yet expose an operation that both stops the lifetime and awaits runner cleanup.
+After the initial conservative startup reconciliation, changes to Rush configuration, project package manifests,
+or integration-classified plugin graph inputs fail closed with `WorkspaceEngineRecreationRequiredError` before
+the input baseline advances or the invalidation is acknowledged. The startup watcher-registration boundary has
+no paths to classify and therefore remains a full invalidation. The routing layer must replace the complete
+workspace session rather than run a stale graph.
 The default daemon executable does not construct or route this graph while the command-independent plugin shape and per-iteration runner
 lifetime tracked by [rushstack#5895](https://github.com/microsoft/rushstack/issues/5895) remain incomplete.

libraries/rush-daemon/src/WorkspaceEngineComponentFactory.ts

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
 
+import * as path from 'node:path';
+
 import type {
   GetInputsSnapshotAsyncFn,
   IInputsSnapshot,
@@ -21,7 +23,7 @@ import type {
   WorkspaceInvalidationTracker
 } from './WorkspaceInvalidationTracker';
 
-const INVALIDATION_REASON: string = 'workspace-inputs-changed';
+const INVALIDATION_REASON: 'workspace-inputs-changed' = 'workspace-inputs-changed';
 
 /**
  * The command-dependent phase and plugin shape used to construct a reusable engine graph.
@@ -89,6 +91,29 @@ export interface IMapWorkspaceInvalidationsOptions {
   readonly operationGraph: IOperationGraph;
 }
 
+/**
+ * Context for identifying changes that require a new workspace engine.
+ *
+ * @beta
+ */
+export interface IClassifyWorkspaceInvalidationsOptions {
+  readonly changedPaths: ReadonlyArray<string>;
+  readonly rushConfiguration: RushConfiguration;
+}
+
+/**
+ * Identifies integration-specific graph inputs that cannot be reconciled against an existing graph.
+ *
+ * @remarks
+ * Rush configuration files and project package manifests are classified automatically. Use this callback
+ * for plugin-specific graph inputs outside those locations.
+ *
+ * @beta
+ */
+export type IsWorkspaceEngineRecreationRequiredAsync = (
+  options: IClassifyWorkspaceInvalidationsOptions
+) => Promise<boolean>;
+
 /**
  * Maps path-specific watcher invalidations onto operations in the reusable graph.
  *
@@ -113,27 +138,55 @@ export interface IWorkspaceInvalidationReconciliation {
   readonly sequence: number;
 }
 
+/**
+ * Indicates that retained changes require the owning workspace session to be recreated.
+ *
+ * @remarks
+ * The invalidations remain unacknowledged. Callers must not execute the existing operation graph after
+ * receiving this error.
+ *
+ * @beta
+ */
+export class WorkspaceEngineRecreationRequiredError extends Error {
+  public constructor() {
+    super('Workspace changes require the reusable engine and session to be recreated.');
+    this.name = 'WorkspaceEngineRecreationRequiredError';
+  }
+}
+
 /**
  * Options for {@link WorkspaceEngineComponentFactory}.
  *
  * @beta
  */
 export interface IWorkspaceEngineComponentFactoryOptions {
   readonly createEngineComponentsAsync: CreateWorkspaceEngineComponentsAsync;
+  readonly isEngineRecreationRequiredAsync?: IsWorkspaceEngineRecreationRequiredAsync;
   readonly mapInvalidationsToOperationsAsync: MapWorkspaceInvalidationsToOperationsAsync;
   readonly shape: IWorkspaceEngineShape;
 }
 
 interface IWorkspaceEngineLifecycleOptions {
   readonly components: IWorkspaceEngineComponents;
+  readonly graphDefiningPaths: IGraphDefiningPaths;
   readonly invalidations: WorkspaceInvalidationTracker;
+  readonly isEngineRecreationRequiredAsync: IsWorkspaceEngineRecreationRequiredAsync | undefined;
   readonly mapInvalidationsToOperationsAsync: MapWorkspaceInvalidationsToOperationsAsync;
+  readonly rushConfiguration: RushConfiguration;
+}
+
+interface IGraphDefiningPaths {
+  readonly filePaths: ReadonlySet<string>;
+  readonly folderPaths: ReadonlyArray<string>;
 }
 
 class WorkspaceEngineLifecycle {
   readonly #components: IWorkspaceEngineComponents;
+  readonly #graphDefiningPaths: IGraphDefiningPaths;
   readonly #invalidations: WorkspaceInvalidationTracker;
+  readonly #isEngineRecreationRequiredAsync: IsWorkspaceEngineRecreationRequiredAsync | undefined;
   readonly #mapInvalidationsToOperationsAsync: MapWorkspaceInvalidationsToOperationsAsync;
+  readonly #rushConfiguration: RushConfiguration;
   #currentInputsSnapshot: IInputsSnapshot;
   #disposePromise: Promise<void> | undefined;
   #isDisposing: boolean = false;
@@ -142,9 +195,12 @@ class WorkspaceEngineLifecycle {
 
   public constructor(options: IWorkspaceEngineLifecycleOptions) {
     this.#components = options.components;
+    this.#graphDefiningPaths = options.graphDefiningPaths;
     this.#currentInputsSnapshot = options.components.inputsSnapshot;
     this.#invalidations = options.invalidations;
+    this.#isEngineRecreationRequiredAsync = options.isEngineRecreationRequiredAsync;
     this.#mapInvalidationsToOperationsAsync = options.mapInvalidationsToOperationsAsync;
+    this.#rushConfiguration = options.rushConfiguration;
   }
 
   public get inputsSnapshot(): IInputsSnapshot {
@@ -173,6 +229,9 @@ class WorkspaceEngineLifecycle {
 
   async #reconcileOnceAsync(): Promise<IWorkspaceInvalidationReconciliation> {
     const invalidationSnapshot: IWorkspaceInvalidationSnapshot = this.#invalidations.getSnapshot();
+    if (await this.#requiresEngineRecreationAsync(invalidationSnapshot)) {
+      throw new WorkspaceEngineRecreationRequiredError();
+    }
     const isFullInvalidation: boolean =
       this.#requiresFullInvalidation ||
       invalidationSnapshot.hasUnknownChanges ||
@@ -229,6 +288,43 @@ class WorkspaceEngineLifecycle {
     await this.#reconciliationTail;
     await this.#components[Symbol.asyncDispose]();
   }
+
+  async #requiresEngineRecreationAsync(
+    invalidationSnapshot: IWorkspaceInvalidationSnapshot
+  ): Promise<boolean> {
+    if (
+      invalidationSnapshot.changedPaths.length === 0 &&
+      !invalidationSnapshot.hasUnknownChanges &&
+      invalidationSnapshot.isWatcherHealthy
+    ) {
+      return false;
+    }
+    if (
+      this.#invalidations.hasUnattributedUnknownChanges ||
+      !invalidationSnapshot.isWatcherHealthy
+    ) {
+      return true;
+    }
+
+    if (
+      invalidationSnapshot.changedPaths.some((changedPath: string) =>
+        isBuiltInGraphDefiningPath(
+          changedPath,
+          this.#rushConfiguration.rushJsonFolder,
+          this.#graphDefiningPaths
+        )
+      )
+    ) {
+      return true;
+    }
+
+    return (
+      (await this.#isEngineRecreationRequiredAsync?.({
+        changedPaths: invalidationSnapshot.changedPaths,
+        rushConfiguration: this.#rushConfiguration
+      })) ?? false
+    );
+  }
 }
 
 /**
@@ -242,13 +338,15 @@ class WorkspaceEngineLifecycle {
  */
 export class WorkspaceEngineComponentFactory {
   readonly #createEngineComponentsAsync: CreateWorkspaceEngineComponentsAsync;
+  readonly #isEngineRecreationRequiredAsync: IsWorkspaceEngineRecreationRequiredAsync | undefined;
   readonly #mapInvalidationsToOperationsAsync: MapWorkspaceInvalidationsToOperationsAsync;
 
   public readonly createAsync: CreateWorkspaceSessionComponentsAsync;
   public readonly shape: IWorkspaceEngineShape;
 
   public constructor(options: IWorkspaceEngineComponentFactoryOptions) {
     this.#createEngineComponentsAsync = options.createEngineComponentsAsync;
+    this.#isEngineRecreationRequiredAsync = options.isEngineRecreationRequiredAsync;
     this.#mapInvalidationsToOperationsAsync = options.mapInvalidationsToOperationsAsync;
     this.shape = normalizeShape(options.shape);
     this.createAsync = (createOptions: ICreateWorkspaceSessionComponentsOptions) =>
@@ -259,6 +357,9 @@ export class WorkspaceEngineComponentFactory {
     options: ICreateWorkspaceSessionComponentsOptions
   ): Promise<IWorkspaceSessionComponents> {
     const projects: ReadonlySet<RushConfigurationProject> = new Set(options.rushConfiguration.projects);
+    const graphDefiningPaths: IGraphDefiningPaths = createGraphDefiningPaths(
+      options.rushConfiguration
+    );
     const components: IWorkspaceEngineComponents = await this.#createEngineComponentsAsync({
       phaseNames: this.shape.phaseNames,
       pluginNames: this.shape.pluginNames,
@@ -274,8 +375,11 @@ export class WorkspaceEngineComponentFactory {
 
     const lifecycle: WorkspaceEngineLifecycle = new WorkspaceEngineLifecycle({
       components,
+      graphDefiningPaths,
       invalidations: options.invalidations,
-      mapInvalidationsToOperationsAsync: this.#mapInvalidationsToOperationsAsync
+      isEngineRecreationRequiredAsync: this.#isEngineRecreationRequiredAsync,
+      mapInvalidationsToOperationsAsync: this.#mapInvalidationsToOperationsAsync,
+      rushConfiguration: options.rushConfiguration
     });
     return {
       [Symbol.asyncDispose]: () => lifecycle[Symbol.asyncDispose](),
@@ -290,6 +394,50 @@ export class WorkspaceEngineComponentFactory {
   }
 }
 
+function createGraphDefiningPaths(rushConfiguration: RushConfiguration): IGraphDefiningPaths {
+  const filePaths: Set<string> = new Set([path.resolve(rushConfiguration.rushJsonFile)]);
+  for (const project of rushConfiguration.projects) {
+    filePaths.add(path.join(project.projectFolder, 'package.json'));
+    filePaths.add(path.join(project.projectFolder, 'config', 'rush-project.json'));
+  }
+  return {
+    filePaths,
+    folderPaths: [
+      path.resolve(rushConfiguration.commonRushConfigFolder),
+      ...Array.from(rushConfiguration.subspaces, (subspace) =>
+        path.resolve(subspace.getSubspaceConfigFolderPath())
+      )
+    ]
+  };
+}
+
+function isBuiltInGraphDefiningPath(
+  changedPath: string,
+  rushJsonFolder: string,
+  graphDefiningPaths: IGraphDefiningPaths
+): boolean {
+  const absoluteChangedPath: string = path.resolve(rushJsonFolder, changedPath);
+  if (graphDefiningPaths.filePaths.has(absoluteChangedPath)) {
+    return true;
+  }
+  for (const folderPath of graphDefiningPaths.folderPaths) {
+    if (isPathInside(absoluteChangedPath, folderPath)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function isPathInside(candidatePath: string, folderPath: string): boolean {
+  const relativePath: string = path.relative(path.resolve(folderPath), candidatePath);
+  return (
+    relativePath === '' ||
+    (!path.isAbsolute(relativePath) &&
+      relativePath !== '..' &&
+      !relativePath.startsWith(`..${path.sep}`))
+  );
+}
+
 function normalizeShape(shape: IWorkspaceEngineShape): IWorkspaceEngineShape {
   return Object.freeze({
     phaseNames: normalizeNames(shape.phaseNames, 'phase', true),

libraries/rush-daemon/src/WorkspaceInvalidationTracker.ts

@@ -9,7 +9,7 @@
 export interface IWorkspaceInvalidationSnapshot {
   /** Paths reported by the watcher, sorted for deterministic consumption. */
   readonly changedPaths: ReadonlyArray<string>;
-  /** True when the watcher reported a change without a path or encountered a watcher error. */
+  /** True at initialization, or when the watcher reported a change without a path or encountered an error. */
   readonly hasUnknownChanges: boolean;
   /** False after a watcher error makes subsequent change detection unreliable. */
   readonly isWatcherHealthy: boolean;
@@ -26,6 +26,7 @@ const MAX_TRACKED_CHANGED_PATHS: number = 10_000;
  */
 export class WorkspaceInvalidationTracker {
   readonly #sequenceByPath: Map<string, number> = new Map();
+  #initializationSequence: number | undefined;
   #latestSequence: number = 0;
   #unknownChangeSequence: number | undefined;
   #watcherHealthy: boolean = true;
@@ -50,6 +51,16 @@ export class WorkspaceInvalidationTracker {
     this.#sequenceByPath.set(changedPath, sequence);
   }
 
+  /** @internal */
+  public invalidateForInitialization(): void {
+    this.#initializationSequence = ++this.#latestSequence;
+  }
+
+  /** @internal */
+  public get hasUnattributedUnknownChanges(): boolean {
+    return this.#unknownChangeSequence !== undefined;
+  }
+
   /**
    * Permanently marks the current watcher as unhealthy.
    *
@@ -66,7 +77,8 @@ export class WorkspaceInvalidationTracker {
   public getSnapshot(): IWorkspaceInvalidationSnapshot {
     return {
       changedPaths: Array.from(this.#sequenceByPath.keys()).sort(),
-      hasUnknownChanges: this.#unknownChangeSequence !== undefined,
+      hasUnknownChanges:
+        this.#initializationSequence !== undefined || this.#unknownChangeSequence !== undefined,
       isWatcherHealthy: this.#watcherHealthy,
       sequence: this.#latestSequence
     };
@@ -94,5 +106,8 @@ export class WorkspaceInvalidationTracker {
     ) {
       this.#unknownChangeSequence = undefined;
     }
+    if (this.#initializationSequence !== undefined && this.#initializationSequence <= sequence) {
+      this.#initializationSequence = undefined;
+    }
   }
 }

libraries/rush-daemon/src/WorkspaceSession.ts

@@ -211,7 +211,7 @@ export class WorkspaceSession implements IWorkspaceSession {
         invalidations.invalidate(changedPath)
       );
       // Changes before the watcher registered its callbacks cannot be observed path-by-path.
-      invalidations.invalidate();
+      invalidations.invalidateForInitialization();
       return session;
     } catch (error) {
       const cleanupErrors: unknown[] = [];