fix: untracked files written by baseline tests are test artifacts (UPT-105)

.changeset/baseline-test-artifacts.md

@@ -0,0 +1,5 @@
+---
+"uptide": patch
+---
+
+`fix` no longer stops with "baseline tests modified the working tree" when the test run only writes new untracked files, such as a junit `test-report.xml` that is not gitignored. Those files are recorded as test artifacts (`testArtifacts` in the report), kept out of every commit, the agent's diff and the pending-changes check, and removed before each later test run, `uptide verify` included. A change to a tracked file during baseline tests still refuses the run.

packages/core/src/fix/fix.test.ts

@@ -388,6 +388,48 @@ it('rejects source changes made by baseline tests before creating a migration br
   expect(git(root, 'branch', '--show-current')).not.toBe('uptide/zod-4.6.5');
 });
 
+it('keeps an untracked report the tests write out of every commit, and clears it between runs', async () => {
+  const { root, services } = fixture();
+  // A junit reporter: it writes test-report.xml, and fails here if the last run's file is left.
+  const write =
+    "const f=require('fs');if(f.existsSync('test-report.xml'))process.exit(1);f.writeFileSync('test-report.xml','<testsuites/>')";
+  const manifest = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'));
+  writeFileSync(
+    join(root, 'package.json'),
+    JSON.stringify({ ...manifest, scripts: { test: `node -e "${write}"` } }),
+  );
+  git(root, 'commit', '-qam', 'test script');
+  const result = await fix({ cwd: root, only: 'zod' }, services);
+  expect(result.testArtifacts).toEqual(['test-report.xml']);
+  expect(result.verification.baselineTests[0]?.status).toBe('passed');
+  expect(result.verification.tests[0]?.status).toBe('passed');
+  expect(result.verification.passed).toBe(true);
+  expect(result.verificationPending).toBeUndefined();
+  expect(result.changedFiles).not.toContain('test-report.xml');
+  expect(git(root, 'log', '--all', '--name-only', '--format=')).not.toContain('test-report.xml');
+  expect(git(root, 'status', '--porcelain', '--untracked-files=all')).not.toContain('test-report');
+  // A fresh clone for `uptide verify`: no report on disk, no exclude entry; the stored run has it.
+  rmSync(join(root, 'test-report.xml'));
+  writeFileSync(join(root, '.git/info/exclude'), '');
+  const again = await reverify({ cwd: root, tool: CLEAN_TOOL }, services);
+  expect(again.verification.tests[0]?.status).toBe('passed');
+  expect(again.verification.passed).toBe(true);
+  expect(again.verificationPending).toBeUndefined();
+}, 30000);
+
+it('still refuses a tracked file the baseline tests change, next to an untracked report', async () => {
+  const { root, services } = fixture();
+  services.tests = async () => {
+    writeFileSync(join(root, 'test-report.xml'), '<testsuites/>');
+    writeFileSync(join(root, 'index.ts'), 'changed by tests');
+    return [{ workspace: '.', status: 'passed', output: '' }];
+  };
+  await expect(fix({ cwd: root, only: 'zod' }, services)).rejects.toThrow(
+    'baseline tests modified',
+  );
+  expect(git(root, 'branch', '--show-current')).not.toBe('uptide/zod-4.6.5');
+});
+
 it('checks workspace source under its own resolution mode, not the importer bundler mode', () => {
   const { root } = fixture();
   for (const name of ['app', 'shared']) mkdirSync(join(root, name), { recursive: true });

packages/core/src/fix/pin.ts

@@ -19,7 +19,15 @@ import type { FixOptions, FixServices } from './run.js';
 import { commit, safeFile, settle } from './settle.js';
 import { lintFiles } from './style.js';
 import type { FixReport, FixSite } from './types.js';
-import { gitLineMoves, markPreexisting, newDiagnostics, typeResolutionFailure } from './verify.js';
+import {
+  clearing,
+  gitLineMoves,
+  markPreexisting,
+  newDiagnostics,
+  testArtifacts,
+  typeResolutionFailure,
+  untrackedFiles,
+} from './verify.js';
 
 export const PIN_RULE = 'api-version-pin';
 
@@ -128,13 +136,14 @@ export async function pinCurrentApi(
       .filter((w) => w === '.' || file.startsWith(`${w}/`))
       .sort((a, b) => b.length - a.length)[0];
   const edited = workspaces.filter((w) => affected.some((f) => ownerOf(f) === w));
+  const untracked = untrackedFiles(root);
   const baselineTests = await verify(`tests (${names(edited)})`, () =>
     services.tests(root, edited, options.testTimeoutMs, affected, {
       withServices: options.withServices === true,
     }),
   );
-  if (git(root, 'status', '--porcelain', '--untracked-files=all'))
-    throw new Error('baseline tests modified the working tree; review their changes before fixing');
+  const artifacts = testArtifacts(root, untracked);
+  services = clearing(services, root, artifacts);
   const baselineLint = await (services.lint ?? lintFiles)(root, affected);
   const baseHead = git(root, 'rev-parse', 'HEAD');
   git(root, 'switch', '-c', branch);
@@ -200,6 +209,7 @@ export async function pinCurrentApi(
     target: installed,
     mode: 'pin',
     apiVersion,
+    ...(artifacts.length ? { testArtifacts: artifacts } : {}),
     ...(generated.length ? { generated } : {}),
     ...tool,
     verifiedAt: new Date().toISOString(),

packages/core/src/fix/reverify.ts

@@ -15,7 +15,9 @@ import { confirmServices } from './run.js';
 import { type Followed, settle } from './settle.js';
 import type { FixReport, LintResult, TestResult } from './types.js';
 import {
+  clearing,
   diagnostics,
+  excludeArtifacts,
   gitLineMoves,
   newDiagnostics,
   type TestOptions,
@@ -86,6 +88,10 @@ export async function reverify(
     .filter((line) => line.trim() !== '' && !line.slice(3).startsWith('.uptide/'));
   if (pending.length)
     throw new UptideError('DIRTY_WORKING_TREE', 'uptide verify requires a clean working tree');
+  // The run's test artifacts (a junit report) stay out of git and out of each test run here too.
+  const artifacts = report.testArtifacts ?? [];
+  excludeArtifacts(root, artifacts);
+  services = clearing(services, root, artifacts);
   const pack = options.pack ?? activePack(report.package);
   if (!pack) throw new Error(`no migration pack for ${report.package}`);
   const workspaces = workspacePackagesOf(root).filter((w) => {

packages/core/src/fix/run.ts

@@ -76,15 +76,18 @@ import type {
   TestResult,
 } from './types.js';
 import {
+  clearing,
   describeServices,
   diagnostics,
   gitLineMoves,
   markPreexisting,
   newDiagnostics,
   type TestOptions,
+  testArtifacts,
   testWorkspaces,
   typeChecker,
   typeResolutionFailure,
+  untrackedFiles,
 } from './verify.js';
 import {
   assertInstalledVersions,
@@ -610,14 +613,15 @@ async function fixPackage(options: FixOptions, services: FixServices): Promise<F
   // Evidence at a site the compiler rejected is confirmed evidence: the change is real there.
   if (packContext.evidence)
     packContext = { ...packContext, evidence: confirmedEvidence(packContext.evidence, selected) };
+  const untracked = untrackedFiles(root);
   const baselineTests = await verify(`tests (${names(workspaces)})`, () =>
     services.tests(root, workspaces, options.testTimeoutMs, affected, {
       withServices: options.withServices === true,
       fullSuite: true,
     }),
   );
-  if (git(root, 'status', '--porcelain', '--untracked-files=all'))
-    throw new Error('baseline tests modified the working tree; review their changes before fixing');
+  const artifacts = testArtifacts(root, untracked);
+  services = clearing(services, root, artifacts);
   // Lint of the same files before anything changes: a failure that is already there is not ours.
   const baselineLint = await (services.lint ?? lintFiles)(root, affected);
   lap('baseline');
@@ -1119,6 +1123,7 @@ async function fixPackage(options: FixOptions, services: FixServices): Promise<F
     ...(fallback.length ? { agentFallback: fallback } : {}),
     ...(cleanInstall ? { cleanInstall } : {}),
     ...(pending ? { verificationPending: true } : {}),
+    ...(artifacts.length ? { testArtifacts: artifacts } : {}),
     ...(lockfile ? { lockfile } : {}),
     ...(generated.length ? { generated } : {}),
     repo: root,

packages/core/src/fix/types.ts

@@ -267,6 +267,8 @@ export interface FixReport {
   baseHead?: string;
   changedFiles?: string[];
   verificationPending?: boolean;
+  /** Untracked files the baseline tests wrote (a junit report): kept out of every commit. */
+  testArtifacts?: string[];
   branch: string;
   sites: FixSite[];
   /** Generated code produced in the clone (a Prisma client), per workspace and phase. */

packages/core/src/fix/verify.test.ts

@@ -2,14 +2,17 @@ import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'nod
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { afterAll, expect, it } from 'vitest';
+import { git } from './process.js';
 import { isolatedPnpmWorkspace } from './test-fixture.js';
 import {
   diagnostics,
+  excludeArtifacts,
   lineMoves,
   moved,
   newDiagnostics,
   typeChecker,
   typeResolutionFailure,
+  untrackedFiles,
 } from './verify.js';
 
 const scratch = mkdtempSync(join(tmpdir(), 'uptide-verify-tests-'));
@@ -192,3 +195,17 @@ it('finds an explicit lib newer than the target with a TS 7-style install (UPT-1
   }
   expect(diagnostics(root, ['.'])).toEqual([]);
 });
+
+it('hides test artifacts of a project in a subdirectory, names with pattern characters included', () => {
+  const top = realpathSync(mkdtempSync(join(scratch, 'artifacts-')));
+  git(top, 'init', '-q');
+  const root = join(top, 'app');
+  mkdirSync(join(root, 'reports'), { recursive: true });
+  const files = ['test report[1].xml', 'reports/#junit!.xml'];
+  for (const file of files) writeFileSync(join(root, file), '<testsuites/>');
+  expect(untrackedFiles(root).sort()).toEqual([...files].sort());
+  excludeArtifacts(root, files);
+  for (const file of files) writeFileSync(join(root, file), '<testsuites/>');
+  writeFileSync(join(root, 'test report1.xml'), 'not an artifact');
+  expect(untrackedFiles(root)).toEqual(['test report1.xml']);
+});

packages/core/src/fix/verify.ts

@@ -1,8 +1,16 @@
 import { execFileSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { existsSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { builtinModules } from 'node:module';
-import { basename, dirname, isAbsolute, join, relative } from 'node:path';
+import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
 import { ts } from 'ts-morph';
 import { bundledLib } from '../adapters/typescript/compile.js';
 import { resolveCompiler } from '../adapters/typescript/compiler.js';
@@ -968,6 +976,62 @@ function noTestsRan(result: { code: number; timeout: boolean }, output: string):
     !testSummary(output)
   );
 }
+/** Untracked, not ignored files, repository-relative. */
+export function untrackedFiles(root: string): string[] {
+  return git(root, 'ls-files', '-z', '--others', '--exclude-standard').split('\0').filter(Boolean);
+}
+
+/**
+ * What the baseline tests left: a changed tracked file refuses the run; new untracked files
+ * (a junit test-report.xml) are test artifacts. They go in the clone's info/exclude, so no
+ * status, diff, `git add` or agent listing sees them, and are removed now; `clearing` removes
+ * them again before each later test run.
+ */
+export function testArtifacts(root: string, before: string[]): string[] {
+  const known = new Set(before);
+  const created = untrackedFiles(root).filter((f) => !known.has(f));
+  if (git(root, 'status', '--porcelain', '--untracked-files=no'))
+    throw new Error('baseline tests modified the working tree; review their changes before fixing');
+  excludeArtifacts(root, created);
+  return created;
+}
+
+/** Hides recorded test artifacts from git in this clone and removes them. */
+export function excludeArtifacts(root: string, files: string[]): void {
+  if (!files.length) return;
+  // ponytail: the private clone's info/exclude (fix and verify always run in one); restore it if either ever runs in place.
+  const exclude = resolve(root, git(root, 'rev-parse', '--git-path', 'info/exclude'));
+  // Patterns there are anchored at the repository top, not at a project in a subdirectory.
+  const prefix = git(root, 'rev-parse', '--show-prefix');
+  mkdirSync(dirname(exclude), { recursive: true });
+  appendFileSync(
+    exclude,
+    files.map((f) => `/${`${prefix}${f}`.replace(/[\\*?[\]!# ]/g, '\\$&')}\n`).join(''),
+  );
+  clearArtifacts(root, files);
+}
+
+function clearArtifacts(root: string, files: string[]): void {
+  for (const file of files) rmSync(join(root, file), { force: true });
+}
+
+/** The same services, with the test artifacts removed before each test run. */
+export function clearing<S extends { tests: (...args: never[]) => unknown }>(
+  services: S,
+  root: string,
+  artifacts: string[],
+): S {
+  if (!artifacts.length) return services;
+  const tests = services.tests as (...args: unknown[]) => unknown;
+  return {
+    ...services,
+    tests: (...args: unknown[]) => {
+      clearArtifacts(root, artifacts);
+      return tests(...args);
+    },
+  };
+}
+
 const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*[A-Za-z]`, 'g');
 export async function testWorkspaces(
   root: string,