From 9c69b602ded336d789cfb6aed0e133d0f867f08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mustafa=20Arif=20=C5=9Ei=C5=9Fman?= Date: Tue, 4 Aug 2026 03:43:06 +0300 Subject: [PATCH] perf(builder): watch bounded source-tree roots instead of one watcher per file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #96. The widened watch list registered one fs watcher per tracked federation source — ~1,900 watchers on a real Nx workspace. Per-file watchers do not scale: macOS FSEvents replays and throttles under that many streams, and every platform pays a file descriptor per watcher. federationWatchPaths() collapses the tracked files into the few top-level workspace source trees that contain them (libs, apps, ...) and both builders watch those roots recursively instead. Because a directory watch surfaces every event under the root, relevance is now filtered where events arrive: only tracked federation sources, project-root files and linked-dir edits pass, and only when their mtime actually advanced (the #96 stale-event filter). Irrelevant events no longer reach the dirty buffer at all. Files outside the workspace, at the workspace root, or under dot dirs, node_modules and build outputs stay as single-file watches; nested candidates deduplicate to their parent tree. Covered by five new unit tests; validated on the same Nx + Angular 22 workspace as #96 (files=~1900 collapses to roots=2). --- src/builders/build/builder.ts | 27 +++++++--- src/builders/remote/builder.ts | 42 ++++++++++++--- src/utils/federation-source-files.spec.ts | 62 ++++++++++++++++++++++- src/utils/federation-source-files.ts | 59 +++++++++++++++++++++ 4 files changed, 174 insertions(+), 16 deletions(-) diff --git a/src/builders/build/builder.ts b/src/builders/build/builder.ts index 4c5bf28..74fb8bb 100644 --- a/src/builders/build/builder.ts +++ b/src/builders/build/builder.ts @@ -49,6 +49,7 @@ import { checkForInvalidImports } from "./../../utils/check-for-invalid-imports. import { describeFederationCache, federationSourceFiles, + federationWatchPaths, } from "./../../utils/federation-source-files.js"; import { createStaleWatchEventFilter } from "./../../utils/stale-watch-event-filter.js"; import { federationBuildNotifier } from "./federation-build-notifier.js"; @@ -460,6 +461,11 @@ export async function* runBuilder( const files = federationSourceFiles( normalized.options.federationCache.bundlerCache, ); + // Watch a handful of top-level source trees, not one watcher per file — + // per-file watchers do not scale (macOS FSEvents replays/throttles under + // thousands of streams). Relevance stays exact via federationWatchedFiles + // in onChange below. + const watchPaths = federationWatchPaths(files, context.workspaceRoot); for (const file of files) { const normalizedFile = path.normalize(file); if (!federationWatchedFiles.has(normalizedFile)) { @@ -467,9 +473,12 @@ export async function* runBuilder( staleEvents.seed(normalizedFile); } } + logger.verbose( + `Federation watch paths: files=${files.length}, roots=${watchPaths.length}`, + ); syncNfFileWatcher( nfWatcher, - { keys: () => files[Symbol.iterator]() }, + { keys: () => watchPaths[Symbol.iterator]() }, linkedDirs, ); }; @@ -481,20 +490,22 @@ export async function* runBuilder( // Coalesce ng-packagr's atomic multi-write bursts into one rebuild. debounceMs: 100, onChange: (p) => { + // Directory-tree watches surface EVERY event under the roots; only + // tracked federation sources and linked-dir edits are relevant. + const normalizedPath = path.normalize(p); + const isFederationSource = federationWatchedFiles.has(normalizedPath); + const isLinkedSource = isUnderLinkedDir(normalizedPath); + if (!isFederationSource && !isLinkedSource) return; // Same-mtime replays never reach the dirty buffer — buffering them // would rebuild federation outputs for files that did not change. - if (!staleEvents.isRealChange(p)) return; + if (!staleEvents.isRealChange(normalizedPath)) return; // Core stops filling the dirty buffer once onChange is set, so refill it // here (Set.add stays idempotent if core is later fixed). Wake the loop // for edits the Angular-driven rebuild will NOT cover: linked dirs and // the federation's own tracked sources, which are externals to the app // build and so never reach Angular's rebuild iterator. - watcherRef.current?.mutate((s) => s.add(p)); - if ( - isUnderLinkedDir(p) || - federationWatchedFiles.has(path.normalize(p)) - ) - notifyChange(); + watcherRef.current?.mutate((s) => s.add(normalizedPath)); + notifyChange(); }, }) : undefined; diff --git a/src/builders/remote/builder.ts b/src/builders/remote/builder.ts index 67dc6a5..ed1858a 100644 --- a/src/builders/remote/builder.ts +++ b/src/builders/remote/builder.ts @@ -29,7 +29,8 @@ import { createAngularBuildAdapter } from '../../tools/esbuild/angular-esbuild-a import { checkForInvalidImports } from '../../utils/check-for-invalid-imports.js'; import { describeFederationCache, - federationSourceFiles + federationSourceFiles, + federationWatchPaths } from '../../utils/federation-source-files.js'; import { createStaleWatchEventFilter } from '../../utils/stale-watch-event-filter.js'; @@ -122,14 +123,32 @@ export async function* runRemoteBuilder( // recently-edited files without a content change, and unguarded that replay // rebuilds forever (see stale-watch-event-filter.ts). const staleEvents = createStaleWatchEventFilter(); + // Directory-tree watches (federationWatchPaths below) surface every event + // under the roots; only project files, linked-dir edits and tracked + // federation sources are relevant rebuild triggers. + const federationProjectRoot = path.dirname( + path.resolve(context.workspaceRoot, federationTsConfig) + ); + const federationWatchedFiles = new Set(); + const isUnder = (file: string, directory: string): boolean => + file === directory || file.startsWith(directory + path.sep); + const isRelevantChange = (file: string): boolean => { + const normalizedFile = path.normalize(file); + return ( + isUnder(normalizedFile, federationProjectRoot) || + linkedDirs.some((directory) => isUnder(normalizedFile, directory)) || + federationWatchedFiles.has(normalizedFile) + ); + }; const changeWatcher = nfBuilderOptions.watch - ? createDebouncedChangeWatcher(nfBuilderOptions.rebuildDelay, staleEvents.isRealChange) + ? createDebouncedChangeWatcher( + nfBuilderOptions.rebuildDelay, + (file) => isRelevantChange(file) && staleEvents.isRealChange(file) + ) : undefined; if (changeWatcher) { - changeWatcher.watcher.addPaths( - path.dirname(path.resolve(context.workspaceRoot, federationTsConfig)) - ); + changeWatcher.watcher.addPaths(federationProjectRoot); for (const assetDir of getAssetWatchDirs(assetEntries, context.workspaceRoot)) { changeWatcher.watcher.addPaths(assetDir); } @@ -155,10 +174,19 @@ export async function* runRemoteBuilder( if (!changeWatcher) return; logger.verbose(describeFederationCache(normalized.options.federationCache.bundlerCache)); const files = federationSourceFiles(normalized.options.federationCache.bundlerCache); - for (const file of files) staleEvents.seed(file); + // Watch a handful of top-level source trees, not one watcher per file — + // per-file watchers do not scale (macOS FSEvents replays/throttles under + // thousands of streams). Relevance stays exact via isRelevantChange above. + const watchPaths = federationWatchPaths(files, context.workspaceRoot); + for (const file of files) { + const normalizedFile = path.normalize(file); + federationWatchedFiles.add(normalizedFile); + staleEvents.seed(normalizedFile); + } + logger.verbose(`Federation watch paths: files=${files.length}, roots=${watchPaths.length}`); syncNfFileWatcher( changeWatcher.watcher, - { keys: () => files[Symbol.iterator]() }, + { keys: () => watchPaths[Symbol.iterator]() }, linkedDirs ); }; diff --git a/src/utils/federation-source-files.spec.ts b/src/utils/federation-source-files.spec.ts index cbb346f..abf3d68 100644 --- a/src/utils/federation-source-files.spec.ts +++ b/src/utils/federation-source-files.spec.ts @@ -1,7 +1,13 @@ import { SourceFileCache } from '@angular/build/private'; import type ts from 'typescript'; -import { describeFederationCache, federationSourceFiles } from './federation-source-files.js'; +import * as path from 'path'; + +import { + describeFederationCache, + federationSourceFiles, + federationWatchPaths, +} from './federation-source-files.js'; function cacheWith(options: { outer?: readonly string[]; @@ -78,3 +84,57 @@ describe('describeFederationCache', () => { ); }); }); + +describe('federationWatchPaths', () => { + const root = path.resolve('/workspace'); + + it('collapses workspace files into their top-level source trees', () => { + const paths = federationWatchPaths( + [ + path.join(root, 'libs', 'a', 'src', 'index.ts'), + path.join(root, 'libs', 'b', 'src', 'index.ts'), + path.join(root, 'apps', 'shell', 'main.ts'), + ], + root, + ); + + expect(paths.sort()).toEqual([path.join(root, 'apps'), path.join(root, 'libs')].sort()); + }); + + it('keeps files outside the workspace as single-file watches', () => { + const outside = path.resolve('/elsewhere/lib/index.ts'); + + expect(federationWatchPaths([outside], root)).toEqual([outside]); + }); + + it('never watches dot dirs, node_modules or build outputs recursively', () => { + const dotFile = path.join(root, '.cache', 'x.ts'); + const nodeModulesFile = path.join(root, 'node_modules', 'pkg', 'index.ts'); + const distFile = path.join(root, 'dist', 'main.js'); + + const paths = federationWatchPaths([dotFile, nodeModulesFile, distFile], root); + + expect(paths.sort()).toEqual([dotFile, nodeModulesFile, distFile].sort()); + }); + + it('keeps a workspace-root-level file as itself', () => { + const rootFile = path.join(root, 'tsconfig.base.json'); + + expect(federationWatchPaths([rootFile], root)).toEqual([rootFile]); + }); + + it('deduplicates nested candidates: a parent tree covers its children', () => { + const paths = federationWatchPaths( + [ + path.join(root, 'libs', 'a', 'src', 'index.ts'), + path.resolve('/elsewhere/libs-extra/index.ts'), + path.join(root, 'libs', 'deep', 'nested', 'file.ts'), + ], + root, + ); + + expect(paths.sort()).toEqual( + [path.join(root, 'libs'), path.resolve('/elsewhere/libs-extra/index.ts')].sort(), + ); + }); +}); diff --git a/src/utils/federation-source-files.ts b/src/utils/federation-source-files.ts index f09ad6c..938001f 100644 --- a/src/utils/federation-source-files.ts +++ b/src/utils/federation-source-files.ts @@ -1,3 +1,5 @@ +import * as path from 'path'; + import type { SourceFileCache } from '@angular/build/private'; /** @@ -25,6 +27,63 @@ export function federationSourceFiles(cache: SourceFileCache): string[] { ].filter((file) => !file.includes('node_modules')); } +/** + * Collapse tracked source files into bounded top-level workspace source trees + * so Native Federation does not create one fs watcher per source file. + * + * A large workspace tracks thousands of sources; per-file watchers do not + * scale (macOS FSEvents in particular replays and throttles under that many + * streams, and every platform pays an fd per watcher). Watching the few + * top-level directories that CONTAIN those files (`libs`, `apps`, ...) keeps + * the watcher count flat while the callers' relevance filter (is the event + * path a tracked federation source?) keeps rebuild triggers exact. + * + * Files outside the workspace root, at the workspace root itself, or under + * top-level directories that must never be watched recursively (dot dirs, + * `node_modules`, build outputs) stay as single-file watches. Nested + * candidates are deduplicated: a parent directory covers its children. + */ +export function federationWatchPaths( + files: readonly string[], + workspaceRoot: string, +): string[] { + const root = path.resolve(workspaceRoot); + const candidates = new Set(); + for (const file of files) { + const resolvedFile = path.resolve(file); + const relative = path.relative(root, resolvedFile); + const isInWorkspace = + relative !== '' && + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative); + if (!isInWorkspace) { + candidates.add(resolvedFile); + continue; + } + const segments = relative.split(path.sep); + const topLevel = segments[0] ?? relative; + const canWatchSourceTree = + segments.length > 1 && + !topLevel.startsWith('.') && + !['node_modules', 'dist', 'out-tsc'].includes(topLevel); + candidates.add(canWatchSourceTree ? path.join(root, topLevel) : resolvedFile); + } + const ordered = [...candidates].sort((left, right) => left.length - right.length); + const watchPaths: string[] = []; + for (const candidate of ordered) { + if ( + watchPaths.some( + (parent) => + candidate === parent || candidate.startsWith(`${parent}${path.sep}`), + ) + ) + continue; + watchPaths.push(candidate); + } + return watchPaths; +} + /** * One-line fingerprint of where the cache tracked its files, for diagnosing * which compilation path a dev server is on: a populated outer Map means