Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 1 addition & 20 deletions packages/nx/src/hasher/check-task-files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { splitTarget } from '../utils/split-target';
import { workspaceRoot as defaultWorkspaceRoot } from '../utils/workspace-root';
import { HashPlanInspector } from './hash-plan-inspector';
import { type ExpandedDepsOutput, getInputs } from './task-hasher';
import { collectUpstreamTaskIds } from '../tasks-runner/task-graph-utils';

// ── Module-level context (loaded once per process) ───────────────────────────

Expand Down Expand Up @@ -269,26 +270,6 @@ function getDepsOutputs(
return result;
}

function collectUpstreamTaskIds(
taskGraph: TaskGraph,
rootTaskId: string,
transitive: boolean
): string[] {
const direct = taskGraph.dependencies[rootTaskId] ?? [];
if (!transitive) return [...direct];

const collected = new Set<string>();
const walk = (id: string): void => {
for (const dep of taskGraph.dependencies[id] ?? []) {
if (collected.has(dep)) continue;
collected.add(dep);
walk(dep);
}
};
walk(rootTaskId);
return [...collected];
}

/**
* Matches a single path against a task's whole output pattern list using the
* native glob engine (`globset`) that the task runner's expand_outputs also
Expand Down
100 changes: 100 additions & 0 deletions packages/nx/src/hasher/hash-task.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import { ProjectGraph } from '../config/project-graph';
import { Task, TaskGraph } from '../config/task-graph';
import { hashTasksThatDoNotDependOnOutputsOfOtherTasks } from './hash-task';

vi.mock('../tasks-runner/task-env', async () => ({
...(await vi.importActual('../tasks-runner/task-env')),
getTaskSpecificEnv: vi.fn(() => process.env),
}));

vi.mock('../tasks-runner/utils', async () => ({
...(await vi.importActual('../tasks-runner/utils')),
getCustomHasher: vi.fn(() => null),
}));

describe('hashTasksThatDoNotDependOnOutputsOfOtherTasks', () => {
function createTask(id: string, outputs: string[]): Task {
const [project, target] = id.split(':');
return {
id,
target: { project, target },
overrides: {},
outputs,
projectRoot: project,
cache: true,
parallelism: true,
} as Task;
}

const projectGraph = {
nodes: {
app: {
name: 'app',
type: 'app',
data: {
root: 'app',
targets: {
build: {
inputs: [
{ dependentTasksOutputFiles: '**/*.d.ts', transitive: true },
],
},
},
},
},
lib: {
name: 'lib',
type: 'lib',
data: { root: 'lib', targets: { build: {} } },
},
tool: {
name: 'tool',
type: 'lib',
data: { root: 'tool', targets: { install: {} } },
},
},
dependencies: { app: [], lib: [], tool: [] },
externalNodes: {},
} as unknown as ProjectGraph;

function hashedIds(taskGraph: TaskGraph) {
const hasher = {
hashTasks: vi.fn(async (tasks: Task[]) =>
tasks.map((t) => ({ value: `${t.id}|hash`, details: {} }))
),
};
return hashTasksThatDoNotDependOnOutputsOfOtherTasks(
hasher as any,
projectGraph,
taskGraph,
{},
null
).then(() => hasher.hashTasks.mock.calls[0][0].map((t: Task) => t.id));
}

it('defers a task whose dep outputs feed its hash', async () => {
const taskGraph: TaskGraph = {
roots: ['lib:build'],
tasks: {
'app:build': createTask('app:build', ['dist/app']),
'lib:build': createTask('lib:build', ['dist/lib']),
},
dependencies: { 'app:build': ['lib:build'], 'lib:build': [] },
continuousDependencies: { 'app:build': [], 'lib:build': [] },
};
expect(await hashedIds(taskGraph)).toEqual(['lib:build']);
});

it('hashes a task up front when its only deps declare no outputs', async () => {
const taskGraph: TaskGraph = {
roots: ['tool:install'],
tasks: {
'app:build': createTask('app:build', ['dist/app']),
'tool:install': createTask('tool:install', []),
},
dependencies: { 'app:build': ['tool:install'], 'tool:install': [] },
continuousDependencies: { 'app:build': [], 'tool:install': [] },
};
expect(await hashedIds(taskGraph)).toEqual(['app:build', 'tool:install']);
});
});
8 changes: 4 additions & 4 deletions packages/nx/src/hasher/hash-task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { getTaskIOService } from '../tasks-runner/task-io-service';
import { getTaskSpecificEnv } from '../tasks-runner/task-env';
import { getCustomHasher } from '../tasks-runner/utils';
import { getDbConnection } from '../utils/db-connection';
import { getInputs, TaskHasher } from './task-hasher';
import { getDependenciesWithOutputsToHash, TaskHasher } from './task-hasher';

let taskDetails: TaskDetails;

Expand Down Expand Up @@ -48,9 +48,9 @@ export async function hashTasksThatDoNotDependOnOutputsOfOtherTasks(
return false;
}

return !(
taskGraph.dependencies[task.id].length > 0 &&
getInputs(task, projectGraph, nxJson).depsOutputs.length > 0
return (
getDependenciesWithOutputsToHash(task, taskGraph, projectGraph, nxJson)
.length === 0
);
})
.map((t) => t.task);
Expand Down
105 changes: 105 additions & 0 deletions packages/nx/src/hasher/task-hasher.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
expandNamedInput,
expandSingleProjectInputs,
filterUsingGlobPatterns,
getDependenciesWithOutputsToHash,
splitInputsIntoSelfAndDependencies,
} from './task-hasher';

Expand Down Expand Up @@ -189,6 +190,110 @@ describe('TaskHasher', () => {
});
});

describe('getDependenciesWithOutputsToHash', () => {
const task = (id: string, outputs: string[]) => {
const [project, target] = id.split(':');
return { id, target: { project, target }, outputs } as any;
};
const graph = (
tasks: any[],
dependencies: Record<string, string[]>
): any => ({
roots: [],
tasks: Object.fromEntries(tasks.map((t) => [t.id, t])),
dependencies,
continuousDependencies: {},
});
const projectGraph = (inputs: any[] | undefined): any => ({
nodes: {
app: {
name: 'app',
type: 'app',
data: {
root: 'app',
targets: { build: { ...(inputs ? { inputs } : {}) } },
},
},
},
dependencies: {},
});
const nxJson: any = { namedInputs: {} };
const app = task('app:build', ['dist/app']);
const depsOutputs = (transitive?: boolean) =>
projectGraph([{ dependentTasksOutputFiles: '**/*.d.ts', transitive }]);

it('returns nothing when the task has no dependentTasksOutputFiles input', () => {
const tg = graph([app, task('lib:build', ['dist/lib'])], {
'app:build': ['lib:build'],
'lib:build': [],
});
expect(
getDependenciesWithOutputsToHash(
app,
tg,
projectGraph([{ fileset: '{projectRoot}/**/*' }]),
nxJson
)
).toEqual([]);
});

it('skips dependencies that declare no outputs', () => {
const tg = graph(
[app, task('tool:install', []), task('lib:build', ['dist/lib'])],
{
'app:build': ['tool:install', 'lib:build'],
'tool:install': [],
'lib:build': [],
}
);
expect(
getDependenciesWithOutputsToHash(app, tg, depsOutputs(), nxJson)
).toEqual(['lib:build']);
});

it('only walks direct dependencies unless transitive', () => {
const tg = graph(
[app, task('mid:build', []), task('leaf:build', ['dist/leaf'])],
{
'app:build': ['mid:build'],
'mid:build': ['leaf:build'],
'leaf:build': [],
}
);
expect(
getDependenciesWithOutputsToHash(app, tg, depsOutputs(false), nxJson)
).toEqual([]);
expect(
getDependenciesWithOutputsToHash(app, tg, depsOutputs(true), nxJson)
).toEqual(['leaf:build']);
});

it('visits diamond dependencies once', () => {
const tg = graph(
[
app,
task('a:build', ['dist/a']),
task('b:build', ['dist/b']),
task('shared:build', ['dist/shared']),
],
{
'app:build': ['a:build', 'b:build'],
'a:build': ['shared:build'],
'b:build': ['shared:build'],
'shared:build': [],
}
);
expect(
getDependenciesWithOutputsToHash(
app,
tg,
depsOutputs(true),
nxJson
).sort()
).toEqual(['a:build', 'b:build', 'shared:build']);
});
});

describe('expandNamedInput', () => {
it('should expand named inputs', () => {
const expanded = expandNamedInput('c', {
Expand Down
25 changes: 25 additions & 0 deletions packages/nx/src/hasher/task-hasher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { NativeTaskHasherImpl } from './native-task-hasher-impl';
import { workspaceRoot } from '../utils/workspace-root';
import { HashInputs, NxWorkspaceFilesExternals } from '../native';
import { getTaskIOService } from '../tasks-runner/task-io-service';
import { collectUpstreamTaskIds } from '../tasks-runner/task-graph-utils';

// Re-export HashInputs from native module for public API
export { HashInputs };
Expand Down Expand Up @@ -352,6 +353,30 @@ export function getInputs(
return { selfInputs, depsInputs, depsOutputs, projectInputs, depsFilesets };
}

/**
* Ids of dependency tasks whose outputs feed `task`'s hash through a
* `dependentTasksOutputFiles` input. Mirrors `process_tasks_outputs` in
* native/tasks/dep_outputs.rs: a dependency with no `outputs` contributes
* nothing, so it does not need to finish before `task` can be hashed.
*/
export function getDependenciesWithOutputsToHash(
task: Task,
taskGraph: TaskGraph,
projectGraph: ProjectGraph,
nxJson: NxJsonConfiguration
): string[] {
const { depsOutputs } = getInputs(task, projectGraph, nxJson);
if (depsOutputs.length === 0) {
return [];
}
// The transitive set is a superset of the direct one, so any transitive
// entry widens the walk for all of them.
const transitive = depsOutputs.some((d) => d.transitive);
return collectUpstreamTaskIds(taskGraph, task.id, transitive).filter(
(id) => taskGraph.tasks[id]?.outputs.length > 0
);
}

export function splitInputsIntoSelfAndDependencies(
inputs: ReadonlyArray<InputDefinition | string>,
namedInputs: { [inputName: string]: ReadonlyArray<InputDefinition | string> }
Expand Down
21 changes: 21 additions & 0 deletions packages/nx/src/tasks-runner/task-graph-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,27 @@ class DependingOnNonParallelContinuousTaskError extends Error {
}
}

/** Ids of `rootTaskId`'s dependencies — direct only, or the whole subtree. */
export function collectUpstreamTaskIds(
taskGraph: TaskGraph,
rootTaskId: string,
transitive: boolean
): string[] {
const direct = taskGraph.dependencies[rootTaskId] ?? [];
if (!transitive) return [...direct];

const collected = new Set<string>();
const walk = (id: string): void => {
for (const dep of taskGraph.dependencies[id] ?? []) {
if (collected.has(dep)) continue;
collected.add(dep);
walk(dep);
}
};
walk(rootTaskId);
return [...collected];
}

export function getLeafTasks(taskGraph: TaskGraph): Set<string> {
const reversed = reverseTaskGraph(taskGraph);
const leafTasks = new Set<string>();
Expand Down
23 changes: 23 additions & 0 deletions packages/nx/src/tasks-runner/task-orchestrator.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,29 @@ describe('TaskOrchestrator', () => {
expect(hasher.hashTasks).toHaveBeenCalledTimes(2);
expect(consumer.hash).toBe('consumer:build|call-2');
});

it('should not re-hash when the only non-cached dep has no outputs', async () => {
// dep runs (cache miss) but declares no outputs, so it contributes
// nothing to the consumer's dependentTasksOutputFiles hash.
const dep = { ...createTask('dep:build'), outputs: [] };
const consumer = createTask('consumer:build');
const taskGraph: TaskGraph = {
roots: ['dep:build'],
tasks: { 'dep:build': dep, 'consumer:build': consumer },
dependencies: { 'dep:build': [], 'consumer:build': ['dep:build'] },
continuousDependencies: { 'dep:build': [], 'consumer:build': [] },
};
const { orchestrator, hasher } = createOrchestrator(taskGraph);

await orchestrator.applyFromCacheOrRunBatch(
true,
{ id: 'batch-1', executorName: 'my-plugin:batch', taskGraph },
0
);

expect(hasher.hashTasks).toHaveBeenCalledTimes(2);
expect(consumer.hash).toBe('consumer:build|call-2');
});
});

describe('cached failures (NX_CACHE_FAILURES)', () => {
Expand Down
Loading
Loading