From f8b0e6b440285f0538f0b6c14bfec8188d90fabd Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 29 Jun 2026 15:31:58 +0200 Subject: [PATCH 1/2] feat(results): simplify project + results config to one flat symmetric schema Replace the dual flat/nested results config with a single flat shape shared by the project source repo and the results block: repo (slug or URL), path (local checkout), branch, and auto_push. Drop the removed wire fields entirely (hard cut, no back-compat): results.remote, repo_url/repo_path, nested repo objects, sync.{auto_push,push_conflict_policy,require_push}, and branch_prefix. The internal NormalizedResultsConfig still derives repo_url/repo_path and the origin remote alias; only the YAML wire surface is simplified. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/cli/src/commands/results/remote.ts | 77 +-- .../results/remote-auto-export.test.ts | 8 +- .../src/lib/project-sync-status.test.ts | 1 - apps/dashboard/src/lib/types.ts | 2 - .../docs/docs/evaluation/running-evals.mdx | 8 +- .../src/content/docs/docs/tools/dashboard.mdx | 90 ++-- .../src/content/docs/docs/tools/results.mdx | 6 +- .../src/evaluation/loaders/config-loader.ts | 232 +-------- packages/core/src/evaluation/results-repo.ts | 58 +-- .../evaluation/validation/config-validator.ts | 455 +----------------- packages/core/src/index.ts | 1 + packages/core/src/projects.ts | 197 ++------ .../evaluation/loaders/config-loader.test.ts | 201 ++------ .../core/test/evaluation/results-repo.test.ts | 65 +-- .../validation/config-validator.test.ts | 350 ++------------ packages/core/test/projects.test.ts | 193 +++----- 16 files changed, 322 insertions(+), 1622 deletions(-) diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index bea2d0db2..79109d49e 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -8,6 +8,7 @@ import { type NormalizedResultsConfig, type ResultsConfig, type ResultsRepoStatus, + type RuntimeResultsConfig, confirmResultsMergeAndPull, directPushResultsWithDetails, directorySizeBytes, @@ -154,21 +155,13 @@ function remoteMetadataManifestPath( export interface ResultsPublishOverrides { readonly repo?: string; - readonly repo_url?: string; readonly repo_path?: string; readonly branch?: string; readonly remote?: string; readonly auto_push?: boolean; readonly require_push?: boolean; - readonly push_conflict_policy?: 'block'; } -type RuntimeResultsConfig = Omit & { - readonly sync?: ResultsConfig['sync'] & { - readonly require_push?: boolean; - }; -}; - const REMOTE_RUN_PREFIX = 'remote::'; const SIZE_WARNING_BYTES = 10 * 1024 * 1024; @@ -227,27 +220,10 @@ export async function loadNormalizedResultsConfig( const projectResults = project?.results ? ({ mode: 'github' as const, - ...(project.results.repoUrl !== undefined && { - repo: project.results.repoUrl, - repo_url: project.results.repoUrl, - }), - ...(project.results.repoPath !== undefined && { repo_path: project.results.repoPath }), - ...(project.results.branch !== undefined && { branch: project.results.branch }), + ...(project.results.repo !== undefined && { repo: project.results.repo }), ...(project.results.path !== undefined && { path: project.results.path }), - ...((project.results.sync?.autoPush !== undefined || - project.results.sync?.pushConflictPolicy !== undefined) && { - sync: { - ...(project.results.sync?.autoPush !== undefined && { - auto_push: project.results.sync.autoPush, - }), - ...(project.results.sync?.pushConflictPolicy !== undefined && { - push_conflict_policy: project.results.sync.pushConflictPolicy, - }), - }, - }), - ...(project.results.branchPrefix !== undefined && { - branch_prefix: project.results.branchPrefix, - }), + ...(project.results.branch !== undefined && { branch: project.results.branch }), + ...(project.results.autoPush !== undefined && { auto_push: project.results.autoPush }), } satisfies ResultsConfig) : undefined; const resultsConfig = projectResults ?? resolveResultsConfigForProject(config, project?.id); @@ -257,7 +233,7 @@ export async function loadNormalizedResultsConfig( const baseConfig = resultsConfig ? normalizeResultsConfig(resultsConfig, { baseDir: project?.path ?? repoRoot }) : undefined; - const repoOverride = overrides?.repo ?? overrides?.repo_url ?? overrides?.repo_path; + const repoOverride = overrides?.repo ?? overrides?.repo_path; if (!baseConfig && !repoOverride) { return undefined; } @@ -269,17 +245,15 @@ export async function loadNormalizedResultsConfig( mode: 'github', ...(overrides.repo !== undefined ? { repo: overrides.repo } - : overrides.repo_url !== undefined - ? { repo_url: overrides.repo_url } - : overrides.repo_path !== undefined - ? { repo_path: overrides.repo_path } - : baseConfig?.repo_path - ? { repo_path: baseConfig.repo_path } - : baseConfig?.repo_url - ? { repo_url: baseConfig.repo_url } - : baseConfig?.repo - ? { repo: baseConfig.repo } - : {}), + : overrides.repo_path !== undefined + ? { repo_path: overrides.repo_path } + : baseConfig?.repo_path + ? { repo_path: baseConfig.repo_path } + : baseConfig?.repo_url + ? { repo_url: baseConfig.repo_url } + : baseConfig?.repo + ? { repo: baseConfig.repo } + : {}), ...(overrides.branch !== undefined ? { branch: overrides.branch } : baseConfig?.branch @@ -293,25 +267,12 @@ export async function loadNormalizedResultsConfig( ...(repoOverride === undefined && baseConfig?.repo_path === undefined && baseConfig?.path ? { path: baseConfig.path } : {}), - ...((overrides.auto_push !== undefined || - overrides.require_push !== undefined || - overrides.push_conflict_policy !== undefined || - baseConfig?.auto_push !== undefined || - baseConfig?.require_push !== undefined || - baseConfig?.push_conflict_policy !== undefined) && { - sync: { - ...((overrides.auto_push ?? baseConfig?.auto_push) !== undefined && { - auto_push: overrides.auto_push ?? baseConfig?.auto_push, - }), - ...((overrides.require_push ?? baseConfig?.require_push) !== undefined && { - require_push: overrides.require_push ?? baseConfig?.require_push, - }), - ...((overrides.push_conflict_policy ?? baseConfig?.push_conflict_policy) !== undefined && { - push_conflict_policy: overrides.push_conflict_policy ?? baseConfig?.push_conflict_policy, - }), - }, + ...((overrides.auto_push ?? baseConfig?.auto_push) !== undefined && { + auto_push: overrides.auto_push ?? baseConfig?.auto_push, + }), + ...((overrides.require_push ?? baseConfig?.require_push) !== undefined && { + require_push: overrides.require_push ?? baseConfig?.require_push, }), - ...(baseConfig?.branch_prefix ? { branch_prefix: baseConfig.branch_prefix } : {}), }; return normalizeResultsConfig(merged, { baseDir: project?.path ?? repoRoot }); diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index 880800334..268b1c71c 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -50,11 +50,9 @@ function writeProjectConfig( writeFileSync( path.join(projectDir, '.agentv', 'config.yaml'), `results: - repo: - remote: ${JSON.stringify(params.repo)} -${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path: ${JSON.stringify(params.path)} - sync: - auto_push: ${params.autoPush} + repo: ${JSON.stringify(params.repo)} +${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path: ${JSON.stringify(params.path)} + auto_push: ${params.autoPush} `, ); } diff --git a/apps/dashboard/src/lib/project-sync-status.test.ts b/apps/dashboard/src/lib/project-sync-status.test.ts index 73c773ad0..d152ccc29 100644 --- a/apps/dashboard/src/lib/project-sync-status.test.ts +++ b/apps/dashboard/src/lib/project-sync-status.test.ts @@ -92,7 +92,6 @@ describe('getProjectSyncView', () => { configured: true, available: true, sync_status: 'push_conflict', - push_conflict_policy: 'block', block_reason: 'Results branch push conflict on agentv/results/v1', }), ).toMatchObject({ diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index 033d68fb9..3172fd1ca 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -553,8 +553,6 @@ export interface RemoteStatusResponse { local_dir?: string; path?: string; auto_push?: boolean; - push_conflict_policy?: 'block'; - branch_prefix?: string; run_count?: number; last_synced_at?: string; last_error?: string; diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 2b865dcc4..2d6e3d76f 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -704,12 +704,10 @@ For local workspaces, put portable registry defaults in `$AGENTV_HOME/config.yam projects: - id: agentv name: AgentV - repo: - path: /home/user/projects/agentv + path: /home/user/projects/agentv results: - repo: - path: /home/user/agentv-results - branch: agentv/results/v1 + path: /home/user/agentv-results + branch: agentv/results/v1 ``` When running AgentV from a worktree that needs environment from a primary checkout, load the primary `.env` through the runtime instead of shell-sourcing it: diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index b72810811..d60cb1499 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -211,16 +211,15 @@ folder path, and select a directory that contains `.agentv/`. Each path must contain a `.agentv/` directory. Registered projects are stored under `projects:` in `$AGENTV_HOME/config.yaml`, or `~/.agentv/config.yaml` when `AGENTV_HOME` is unset. -To register a remote repo and keep it synced automatically, add a nested `repo` block to the entry in `$AGENTV_HOME/config.yaml`. `repo.url` is the Git remote URL AgentV passes to `git clone`, so it can be HTTPS or SSH. `repo.branch` is the branch or ref to check out, and `repo.path` is the local checkout path: +To register a remote repo and keep it synced automatically, add the source repo fields directly to the entry in `$AGENTV_HOME/config.yaml`. `repo` is the Git remote slug or URL AgentV passes to `git clone`, so it can be an `owner/name` slug, HTTPS, or SSH. `branch` is the branch or ref to check out, and `path` is the local checkout path: ```yaml projects: - id: my-evals name: My Evals - repo: - url: https://github.com/example/my-evals.git - branch: main - path: /srv/agentv/my-evals + repo: https://github.com/example/my-evals.git + path: /srv/agentv/my-evals + branch: main ``` On each Dashboard startup, AgentV clones the repo if the path is empty (`git clone --depth 1`) or pulls the latest if a clone already exists (`git pull --ff-only`). You can also trigger a sync manually from the Dashboard UI's **Sync** button. @@ -280,58 +279,50 @@ For a registered project, put results repo settings on that project's entry in ` projects: - id: agentv name: AgentV - repo: - url: https://github.com/EntityProcess/agentv.git - branch: main - path: /home/entity/projects/EntityProcess/agentv + repo: https://github.com/EntityProcess/agentv.git + path: /home/entity/projects/EntityProcess/agentv + branch: main results: - repo: - remote: https://github.com/EntityProcess/agentv.git - path: . - branch: agentv/results/v1 - sync: - auto_push: false + repo: https://github.com/EntityProcess/agentv.git + path: /home/entity/projects/EntityProcess/agentv + branch: agentv/results/v1 + auto_push: false ``` -`results.repo.remote` is the Git remote URL used when AgentV creates a fresh results checkout, and the intended remote URL for portable project config. `results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. AgentV does not add or rewrite remotes inside an existing checkout; the checkout's existing `origin` must already point at the repository you want to fetch and push. When `results.repo.remote` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. For CI workflows where a push failure should fail the command after local artifacts are written, invoke the run with `agentv eval run --results-require-push`. The default conflict behavior is block-and-ask; the removed `backup_and_force_push` value is rejected with migration guidance because AgentV never force-pushes result branches. Non-fast-forward result branch pushes are auto-merged with artifact-aware Git merge drivers and pushed as a fast-forward, so the canonical results branch is never force-pushed or rewritten. Genuine overlay conflicts route to a timestamped temp branch plus a GitHub compare link for a human merge instead. +`results.repo` is the Git remote slug or URL used when AgentV creates a fresh results checkout, and the intended remote URL for portable project config. `results.path` is the local Git checkout AgentV writes result commits into; pointing it at the source repository checkout stores completed run artifacts on a dedicated branch of that repository. AgentV does not add or rewrite remotes inside an existing checkout; the checkout's existing `origin` must already point at the repository you want to fetch and push. When `results.repo` is omitted, `results.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. For CI workflows where a push failure should fail the command after local artifacts are written, invoke the run with `agentv eval run --results-require-push`. The default conflict behavior is block-and-ask, because AgentV never force-pushes result branches. Non-fast-forward result branch pushes are auto-merged with artifact-aware Git merge drivers and pushed as a fast-forward, so the canonical results branch is never force-pushed or rewritten. Genuine overlay conflicts route to a timestamped temp branch plus a GitHub compare link for a human merge instead. -For a separate results repository, use `results.repo.remote` and an optional managed clone `results.repo.path`: +For a separate results repository, set `results.repo` and an optional managed clone `results.path`: ```yaml projects: - id: agentv name: AgentV - repo: - path: /home/entity/projects/EntityProcess/agentv + path: /home/entity/projects/EntityProcess/agentv results: - repo: - remote: git@github.com:EntityProcess/agentv-examples-eval-results.git - branch: agentv/results/v1 - path: /home/entity/projects/EntityProcess/agentv-examples-eval-results - sync: - auto_push: true + repo: git@github.com:EntityProcess/agentv-examples-eval-results.git + path: /home/entity/projects/EntityProcess/agentv-examples-eval-results + branch: agentv/results/v1 + auto_push: true ``` -`results.repo.remote` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. When `results.repo.remote` is set and `results.repo.path` is missing or empty, AgentV creates that filesystem location with `git clone`. If `results.repo.path` already points at a Git checkout, AgentV treats that checkout's remotes as user-owned state: it fetches and pushes using the existing configured remote name (`origin` by default), but it does not run `git remote add` or `git remote set-url`. Omit `results.repo.remote` only when `results.repo.path` points at an already-existing local checkout such as `.`. +`results.repo` is the Git remote slug or URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. When `results.repo` is set and `results.path` is missing or empty, AgentV creates that filesystem location with `git clone`. If `results.path` already points at a Git checkout, AgentV treats that checkout's remotes as user-owned state: it fetches and pushes using the existing configured remote name (`origin` by default), but it does not run `git remote add` or `git remote set-url`. Omit `results.repo` only when `results.path` points at an already-existing local checkout. You can also set a top-level global fallback in the same file. This is used when the current project is not registered or its registry entry has no `results` block: ```yaml results: - repo: - remote: https://github.com/EntityProcess/agentv.git - path: . - branch: agentv/results/v1 - sync: - auto_push: false + repo: https://github.com/EntityProcess/agentv.git + path: /home/entity/projects/EntityProcess/agentv + branch: agentv/results/v1 + auto_push: false ``` Project-local `.agentv/config.yaml` is for portable eval defaults such as `execution`, `eval_patterns`, and `dashboard`. Do not put `projects` in project-local config; AgentV warns and ignores it there. `results_by_project` is deprecated; use `projects[].results` in `$AGENTV_HOME/config.yaml`. -The project `repo` block and the `results` block sync different repositories: +The project `repo` and the `results` block sync different repositories: -- `projects[].repo.url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. -- `projects[].results.repo.remote` is the git-backed results store remote URL. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in the local checkout at `projects[].results.repo.path`. +- `projects[].repo` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. +- `projects[].results.repo` is the git-backed results store remote URL. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in the local checkout at `projects[].results.path`. #### Migration from the legacy project schema @@ -358,32 +349,29 @@ After: projects: - id: agentv name: AgentV - repo: - url: https://github.com/EntityProcess/agentv.git - branch: main - path: /home/entity/projects/EntityProcess/agentv + repo: https://github.com/EntityProcess/agentv.git + path: /home/entity/projects/EntityProcess/agentv + branch: main results: - repo: - remote: https://github.com/EntityProcess/agentv-eval-results.git - branch: agentv/results/v1 - path: /home/entity/projects/EntityProcess/agentv-eval-results - sync: - auto_push: true + repo: https://github.com/EntityProcess/agentv-eval-results.git + path: /home/entity/projects/EntityProcess/agentv-eval-results + branch: agentv/results/v1 + auto_push: true ``` -Current flat fields (`path`, `repo_url`, `ref`, `results.repo_url`, `results.repo_path`, `results.branch`, and `results.path`) still load with migration warnings and are written back in nested form the next time AgentV saves the project registry. Removed fields (`source`, `repository`, `results.mode`, `results.repo` as a string, `results.remote`, `results.repository`, `results.local_path`, and `results.auto_push`) fail validation with migration guidance. +Both the source repo and the `results` block use one flat shape: `repo` (slug or Git URL), `path` (local checkout), `branch`, and, for results, `auto_push`. Removed fields (`source`, `repository`, the nested `repo:`/`results.repo:` objects, `repo_url`, `repo_path`, `ref`, `results.remote`, `results.repository`, `results.local_path`, `results.sync`, `results.branch_prefix`, and `results.push_conflict_policy`) fail validation. Use project-level **Sync Project** as the results exchange workflow. It handles pulled remote runs, locally edited metadata, dirty state, and blocked conflict feedback in one project-scoped action. There is no separate `agentv results remote status` or `agentv results remote sync` command. The `agentv results` CLI stays focused on local run workspaces; manual remote exchange is Dashboard/API-only, with eval auto-export covering the common CI/publisher path. -Each run writes to a unique timestamped directory, so concurrent pushes from multiple machines are safe. AgentV creates a missing storage branch automatically and pushes with a non-fast-forward retry. `branch_prefix` remains only the prefix for temporary result/PR branch names; it is not the storage branch. +Each run writes to a unique timestamped directory, so concurrent pushes from multiple machines are safe. AgentV creates a missing storage branch automatically and pushes with a non-fast-forward retry. Temporary result/PR branch names use a fixed prefix; they are not the storage branch. ### What happens to existing local runs? Existing runs already present under `.agentv/results///` stay exactly where they are and continue to appear in Dashboard as **local** runs. Runs in the removed `.agentv/results/runs/**` layout are not discovered by Dashboard. -Adding a `results` block does **not** backfill those historical runs into the results branch automatically. Result publishing only affects runs created after the results repo is configured. `sync.auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. +Adding a `results` block does **not** backfill those historical runs into the results branch automatically. Result publishing only affects runs created after the results repo is configured. `auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. If you want older local-only runs in the remote repo, rerun them or copy the run directories into the managed clone manually before syncing the project. @@ -417,10 +405,10 @@ After sync, newly fetched remote runs appear in the list with a **remote** sourc **Sync Project** fetches the results repo and only changes the clone when Git says it is safe: - A clean clone that is behind the remote is fast-forwarded. -- Safe uncommitted changes under the configured results repo's owned result and metadata paths, such as remote tag overlays under `metadata/runs/**`, are committed and pushed when `sync.auto_push: true`. -- A local results repo that is ahead is pushed when `sync.auto_push: true` and the committed paths are all under `.agentv/results/**`. +- Safe uncommitted changes under the configured results repo's owned result and metadata paths, such as remote tag overlays under `metadata/runs/**`, are committed and pushed when `auto_push: true`. +- A local results repo that is ahead is pushed when `auto_push: true` and the committed paths are all under `.agentv/results/**`. - Dirty non-results files, dirty metadata plus remote changes, unresolved conflicts, missing upstream branches, non-results commits ahead, and rejected pushes are blocked instead of reset. -- Non-fast-forward result branch pushes never force-push. AgentV runs a bounded fetch → merge → push loop that absorbs concurrent remote writes with a real merge commit using artifact-aware Git merge drivers (union for the append-only `run_manifest.jsonl`, a JSON-union driver for tag and feedback overlays), so the common append-mostly case auto-merges and pushes as a fast-forward. When Dashboard sync absorbs concurrent remote changes this way, the success feedback includes **Merged remote (auto)**. The removed `sync.push_conflict_policy: backup_and_force_push` value is rejected with migration guidance; remove the field or set it to `block`. +- Non-fast-forward result branch pushes never force-push. AgentV runs a bounded fetch → merge → push loop that absorbs concurrent remote writes with a real merge commit using artifact-aware Git merge drivers (union for the append-only `run_manifest.jsonl`, a JSON-union driver for tag and feedback overlays), so the common append-mostly case auto-merges and pushes as a fast-forward. When Dashboard sync absorbs concurrent remote changes this way, the success feedback includes **Merged remote (auto)**. - When a genuine overlay conflict cannot be auto-merged, AgentV does not touch the canonical branch. It pushes the local work to a fresh timestamped `agentv/results-sync/--` branch and reports `needs_human_merge` with a `pending_merge` block (temp branch, target branch, and a GitHub compare URL when the remote is on GitHub). The toolbar shows a **Pending merge** card: open the link to merge the branch into the canonical target on GitHub (GitHub's pull request is the conflict surface — AgentV builds no merge UI), then click **I merged it — resync**. That resumes canonical sync by fast-forward-pulling the merged target. A premature click is a safe no-op — local work stays intact and the next sync re-creates a temp branch. When sync is blocked, Dashboard keeps the local clone intact and shows the `block_reason`, `dirty_paths` or `conflicted_paths`, `git_status`, and a compact `git_diff_summary` so you can resolve the results repo manually before syncing again. diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index b7cff2f61..788bd5360 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -11,7 +11,7 @@ import resultsReportDetails from '../../../../assets/screenshots/results-report- The `results` command family works on existing local AgentV run workspaces and `run_manifest.jsonl` manifests. Use it after an eval run to inspect failures, validate manifests, export artifact layouts, combine/delete local run workspaces, or generate a shareable HTML report. -Remote result repository exchange is intentionally not part of `agentv results`. New eval runs publish completed artifacts to a configured results repo or branch; `sync.auto_push: true` additionally pushes that branch to the remote. Manual remote status and sync are Dashboard/API workflows. See [Dashboard Remote Results](/docs/tools/dashboard/#remote-results) for configuration and sync behavior, and [WIP checkpoints](/docs/tools/wip-checkpoints/) for recovering in-progress runs before final publish. +Remote result repository exchange is intentionally not part of `agentv results`. New eval runs publish completed artifacts to a configured results repo or branch; `auto_push: true` additionally pushes that branch to the remote. Manual remote status and sync are Dashboard/API workflows. See [Dashboard Remote Results](/docs/tools/dashboard/#remote-results) for configuration and sync behavior, and [WIP checkpoints](/docs/tools/wip-checkpoints/) for recovering in-progress runs before final publish. ## Subcommands @@ -243,7 +243,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.remote` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `run_manifest.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`run_manifest.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.repo.path` without `results.repo.remote` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `run_manifest.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. The removed `sync.push_conflict_policy: backup_and_force_push` value is rejected with migration guidance; remove the field and rely on the default block-and-ask behavior. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `results.repo` with `results.path` pointing at the source checkout and `results.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `run_manifest.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`run_manifest.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.path` without `results.repo` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `run_manifest.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. -- **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.repo.path` clone with `git` directly, then sync again. +- **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.path` clone with `git` directly, then sync again. diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index c6d894765..7fae7e121 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -49,24 +49,14 @@ export type ResultPushConflictPolicy = 'block'; export type ResultsConfig = { readonly mode?: 'github'; - /** Legacy shorthand or Git remote URL for a managed results clone. */ + /** Git remote slug/URL for a managed results clone, or omit to default to the source repo. */ readonly repo?: string; - /** Git remote URL for a managed results clone. Preferred in YAML wire config. */ - readonly repo_url?: string; - /** Local Git repository path. `.` means the current project/source repository. */ - readonly repo_path?: string; + /** Local Git checkout path for results. */ + readonly path?: string; /** Optional remote branch used as the canonical git-backed results store. */ readonly branch?: string; - /** Runtime-only local Git remote-name override. Persistent config should not set this. */ - readonly remote?: string; - /** Local filesystem path for the results clone. Optional; defaults to ~/.agentv/results//. */ - readonly path?: string; + /** Push committed results to the remote automatically. */ readonly auto_push?: boolean; - readonly sync?: { - readonly auto_push?: boolean; - readonly push_conflict_policy?: ResultPushConflictPolicy; - }; - readonly branch_prefix?: string; }; export type HooksConfig = { @@ -660,92 +650,6 @@ function warnRemovedExperimentPointer(raw: unknown, configPath: string, key: str ); } -function isFilesystemPath(p: string): boolean { - return ( - p.startsWith('/') || - p.startsWith('~/') || - p.startsWith('~\\') || - p === '~' || - /^[A-Za-z]:[/\\]/.test(p) - ); -} - -function readTrimmedString(value: unknown): string | undefined { - if (typeof value !== 'string') { - return undefined; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; -} - -function isGitRemoteUrl(value: string): boolean { - return /^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(value); -} - -type NestedResultsRepoConfig = { - readonly repo_url?: string; - readonly repo_path?: string; - readonly branch?: string; - readonly path?: string; -}; - -function parseNestedResultsRepoConfig( - raw: unknown, - configPath: string, -): NestedResultsRepoConfig | undefined { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { - logWarning(`Invalid results.repo in ${configPath}, expected object`); - return undefined; - } - - const repo = raw as Record; - const url = readTrimmedString(repo.url); - const repoPath = readTrimmedString(repo.path); - const branch = readTrimmedString(repo.branch); - const remote = readTrimmedString(repo.remote); - const remoteUrl = remote ?? url; - - if (repo.url !== undefined && !url) { - logWarning(`Invalid results.repo.url in ${configPath}, expected non-empty string`); - return undefined; - } - if (url && !isGitRemoteUrl(url)) { - logWarning(`Invalid results.repo.url in ${configPath}, expected Git remote URL`); - return undefined; - } - if (repo.path !== undefined && !repoPath) { - logWarning(`Invalid results.repo.path in ${configPath}, expected non-empty string`); - return undefined; - } - if (repo.branch !== undefined && !branch) { - logWarning(`Invalid results.repo.branch in ${configPath}, expected non-empty string`); - return undefined; - } - if (repo.remote !== undefined && !remote) { - logWarning(`Invalid results.repo.remote in ${configPath}, expected non-empty string`); - return undefined; - } - if (remote && !isGitRemoteUrl(remote)) { - logWarning(`Invalid results.repo.remote in ${configPath}, expected Git remote URL`); - return undefined; - } - if (remote && url) { - logWarning(`Invalid results.repo in ${configPath}, set only one of remote or url`); - return undefined; - } - if (!remoteUrl && !repoPath) { - logWarning(`Invalid results.repo in ${configPath}, expected remote or path`); - return undefined; - } - - return { - ...(remoteUrl && { repo_url: remoteUrl }), - ...(repoPath && !remoteUrl && { repo_path: repoPath }), - ...(remoteUrl && repoPath && { path: repoPath }), - ...(branch && { branch }), - }; -} - export function parseResultsConfig(raw: unknown, configPath: string): ResultsConfig | undefined { if (raw === undefined || raw === null) { return undefined; @@ -762,141 +666,49 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon return undefined; } - const hasNestedRepo = obj.repo !== undefined && typeof obj.repo === 'object' && obj.repo !== null; - const nestedRepo = hasNestedRepo ? parseNestedResultsRepoConfig(obj.repo, configPath) : undefined; - if (hasNestedRepo && !nestedRepo) { - return undefined; - } - if (obj.repo !== undefined && !hasNestedRepo && typeof obj.repo !== 'string') { - logWarning(`Invalid results.repo in ${configPath}, expected string or object`); - return undefined; - } - if ( - nestedRepo && - ['repo_url', 'repo_path', 'branch', 'remote', 'path'].some((field) => obj[field] !== undefined) - ) { - logWarning( - `Invalid results in ${configPath}, do not mix nested results.repo with flat repo_url, repo_path, branch, remote, or path fields`, - ); - return undefined; - } - - const legacyRepo = typeof obj.repo === 'string' ? obj.repo.trim() : ''; - const repoUrl = - nestedRepo?.repo_url ?? (typeof obj.repo_url === 'string' ? obj.repo_url.trim() : ''); - const repoPath = - nestedRepo?.repo_path ?? (typeof obj.repo_path === 'string' ? obj.repo_path.trim() : ''); - const repo = legacyRepo || repoUrl; - if (!repo && !repoPath) { - logWarning( - `Invalid results in ${configPath}, expected nested repo.remote/repo.path or compatible repo_url/repo_path`, - ); - return undefined; - } - if (repo && repoPath) { - logWarning( - `Invalid results in ${configPath}, set only one of nested repo.remote/repo.path or compatible repo_url/repo_path`, - ); - return undefined; - } - - let branch: string | undefined; - if (nestedRepo?.branch !== undefined) { - branch = nestedRepo.branch; - } else if (obj.branch !== undefined) { - if (typeof obj.branch !== 'string' || obj.branch.trim().length === 0) { - logWarning(`Invalid results.branch in ${configPath}, expected non-empty string`); - return undefined; - } - branch = obj.branch.trim(); - } - - if (obj.remote !== undefined) { - if (!hasNestedRepo) { - logWarning( - `results.remote in ${configPath} is no longer supported in persistent config. Use results.repo.remote for a portable Git endpoint URL, or omit it and let AgentV use the local checkout remote alias internally.`, - ); + let repo: string | undefined; + if (obj.repo !== undefined) { + if (typeof obj.repo !== 'string' || obj.repo.trim().length === 0) { + logWarning(`Invalid results.repo in ${configPath}, expected non-empty string`); return undefined; } + repo = obj.repo.trim(); } let resultsPath: string | undefined; - if (nestedRepo?.path !== undefined) { - resultsPath = nestedRepo.path; - } else if (obj.path !== undefined) { + if (obj.path !== undefined) { if (typeof obj.path !== 'string' || obj.path.trim().length === 0) { logWarning(`Invalid results.path in ${configPath}, expected non-empty string`); return undefined; } - const trimmedPath = obj.path.trim(); - if (!isFilesystemPath(trimmedPath)) { - logWarning( - `Invalid results.path in ${configPath}: '${trimmedPath}' looks like a repo subdirectory. results.path now specifies the local filesystem directory for the clone (e.g., ~/data/agentv-results). Remove 'path' to use the default or set an absolute/home-relative path.`, - ); - return undefined; - } - resultsPath = trimmedPath; + resultsPath = obj.path.trim(); } - if (obj.auto_push !== undefined && typeof obj.auto_push !== 'boolean') { - logWarning(`Invalid results.auto_push in ${configPath}, expected boolean`); + if (!repo && !resultsPath) { + logWarning(`Invalid results in ${configPath}, expected repo or path`); return undefined; } - let sync: ResultsConfig['sync']; - if (obj.sync !== undefined) { - if (typeof obj.sync !== 'object' || obj.sync === null || Array.isArray(obj.sync)) { - logWarning(`Invalid results.sync in ${configPath}, expected object`); - return undefined; - } - const syncObj = obj.sync as Record; - if (syncObj.auto_push !== undefined && typeof syncObj.auto_push !== 'boolean') { - logWarning(`Invalid results.sync.auto_push in ${configPath}, expected boolean`); - return undefined; - } - if (syncObj.require_push !== undefined) { - logWarning( - `results.sync.require_push in ${configPath} is no longer supported in persistent config. Use the per-run --results-require-push CLI flag instead.`, - ); - return undefined; - } - if (syncObj.push_conflict_policy === 'backup_and_force_push') { - logWarning( - `results.sync.push_conflict_policy: 'backup_and_force_push' in ${configPath} is no longer supported. Remove the field or set it to 'block'; AgentV never force-pushes result branches.`, - ); - return undefined; - } - if (syncObj.push_conflict_policy !== undefined && syncObj.push_conflict_policy !== 'block') { - logWarning(`Invalid results.sync.push_conflict_policy in ${configPath}, expected 'block'`); + let branch: string | undefined; + if (obj.branch !== undefined) { + if (typeof obj.branch !== 'string' || obj.branch.trim().length === 0) { + logWarning(`Invalid results.branch in ${configPath}, expected non-empty string`); return undefined; } - sync = { - ...(typeof syncObj.auto_push === 'boolean' && { auto_push: syncObj.auto_push }), - ...(syncObj.push_conflict_policy === 'block' && { - push_conflict_policy: syncObj.push_conflict_policy, - }), - }; + branch = obj.branch.trim(); } - let branchPrefix: string | undefined; - if (obj.branch_prefix !== undefined) { - if (typeof obj.branch_prefix !== 'string' || obj.branch_prefix.trim().length === 0) { - logWarning(`Invalid results.branch_prefix in ${configPath}, expected non-empty string`); - return undefined; - } - branchPrefix = obj.branch_prefix.trim(); + if (obj.auto_push !== undefined && typeof obj.auto_push !== 'boolean') { + logWarning(`Invalid results.auto_push in ${configPath}, expected boolean`); + return undefined; } return { mode: 'github', ...(repo && { repo }), - ...(repoUrl && { repo_url: repoUrl }), - ...(repoPath && { repo_path: repoPath }), - ...(branch !== undefined && { branch }), ...(resultsPath !== undefined && { path: resultsPath }), + ...(branch !== undefined && { branch }), ...(typeof obj.auto_push === 'boolean' && { auto_push: obj.auto_push }), - ...(sync && { sync }), - ...(branchPrefix && { branch_prefix: branchPrefix }), }; } diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 1c274f917..adbf5cebe 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -244,7 +244,6 @@ export interface ResultsRepoStatus { readonly auto_push?: boolean; readonly require_push?: boolean; readonly push_conflict_policy?: ResultPushConflictPolicy; - readonly branch_prefix?: string; readonly local_dir?: string; readonly last_synced_at?: string; readonly last_error?: string; @@ -294,17 +293,27 @@ export interface NormalizedResultsConfig { readonly auto_push: boolean; readonly require_push: boolean; readonly push_conflict_policy: ResultPushConflictPolicy; - readonly branch_prefix: string; /** @internal Runtime mode; not part of YAML wire format. */ readonly storageBranchWorktree: boolean; } -export type RuntimeResultsConfig = Omit & { - readonly sync?: ResultsConfig['sync'] & { - /** Runtime-only override, set by --results-require-push. Not YAML config. */ - readonly require_push?: boolean; - }; -}; +/** + * Internal runtime shape accepted by {@link normalizeResultsConfig}. Extends the + * flat wire {@link ResultsConfig} with internal/CLI-only fields that never appear + * in persistent YAML: derived `repo_url`/`repo_path`, the local `remote` alias, + * and the per-run `require_push` override (`--results-require-push`). + */ +export interface RuntimeResultsConfig { + readonly mode?: 'github'; + readonly repo?: string; + readonly repo_url?: string; + readonly repo_path?: string; + readonly branch?: string; + readonly remote?: string; + readonly path?: string; + readonly auto_push?: boolean; + readonly require_push?: boolean; +} type StorageBranchResultsConfig = NormalizedResultsConfig & { readonly branch: string }; @@ -399,27 +408,21 @@ export function normalizeResultsConfig( return config; } const baseDir = options?.baseDir ?? process.cwd(); + // `repo`/`repo_url` is a remote slug or URL. `repo_path` is an existing local + // checkout that pushes to its own origin (storage-branch worktree). `path` is + // an explicit clone destination for a remote-backed results repo. const repoUrl = (config.repo_url ?? config.repo)?.trim(); - const repoPath = config.repo_path?.trim(); - const explicitClonePath = config.path?.trim(); + const explicitPath = config.path?.trim(); + // `path` is a local checkout (storage-branch worktree) when no remote repo is + // configured; with a remote repo it is the explicit clone destination. + const repoPath = config.repo_path?.trim() ?? (repoUrl ? undefined : explicitPath); + const explicitClonePath = repoUrl ? explicitPath : undefined; const repo = repoUrl ?? repoPath ?? ''; const branch = config.branch?.trim() || (repoPath ? DEFAULT_RESULTS_BRANCH : undefined); const useStorageBranchWorktree = Boolean(repoPath || (repoUrl && explicitClonePath && branch)); const remote = config.remote?.trim() || 'origin'; - const autoPush = config.sync?.auto_push ?? config.auto_push === true; - const requirePush = config.sync?.require_push === true; - const configuredPushConflictPolicy = ( - config.sync as { push_conflict_policy?: unknown } | undefined - )?.push_conflict_policy; - if (configuredPushConflictPolicy === 'backup_and_force_push') { - throw new Error( - "results.sync.push_conflict_policy: 'backup_and_force_push' is no longer supported. Remove the field or set it to 'block'; AgentV never force-pushes result branches.", - ); - } - if (configuredPushConflictPolicy !== undefined && configuredPushConflictPolicy !== 'block') { - throw new Error("results.sync.push_conflict_policy must be 'block'"); - } - const pushConflictPolicy = configuredPushConflictPolicy ?? 'block'; + const autoPush = config.auto_push === true; + const requirePush = config.require_push === true; const resolvedRepoPath = repoPath ? resolveLocalPath(repoPath, baseDir) : undefined; const resolvedPath = explicitClonePath ? resolveLocalPath(explicitClonePath, baseDir) @@ -436,8 +439,7 @@ export function normalizeResultsConfig( path: resolvedPath, auto_push: autoPush, require_push: requirePush, - push_conflict_policy: pushConflictPolicy, - branch_prefix: config.branch_prefix?.trim() || 'eval-results', + push_conflict_policy: 'block', storageBranchWorktree: useStorageBranchWorktree, }; } @@ -932,8 +934,7 @@ function updateStatusFile( config: ResultsConfig | NormalizedResultsConfig, patch: PersistedStatus, ): void { - const repo = - typeof config.repo === 'string' ? config.repo : (config.repo_url ?? config.repo_path ?? ''); + const repo = normalizeResultsConfig(config).repo; const cachePaths = getResultsRepoLocalPaths(repo); const current = readPersistedStatus(cachePaths.statusFile); writePersistedStatus(cachePaths.statusFile, { @@ -1011,7 +1012,6 @@ export function getResultsRepoStatus(config?: ResultsConfig): ResultsRepoStatus auto_push: normalized.auto_push, require_push: normalized.require_push, push_conflict_policy: normalized.push_conflict_policy, - branch_prefix: normalized.branch_prefix, local_dir: normalized.path, last_synced_at: persisted.last_synced_at, last_error: persisted.last_error, diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 972ba5507..4f9de65f6 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -176,15 +176,6 @@ function validateProjects(errors: ValidationError[], filePath: string, projects: }); } -function addWarning( - errors: ValidationError[], - filePath: string, - location: string, - message: string, -): void { - errors.push({ severity: 'warning', filePath, location, message }); -} - function addError( errors: ValidationError[], filePath: string, @@ -198,116 +189,6 @@ function isPlainObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function isGitRemoteUrlValue(value: string): boolean { - return /^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(value.trim()); -} - -function validateProjectRepoConfig( - errors: ValidationError[], - filePath: string, - projectRecord: Record, - location: string, -): void { - if (projectRecord.source !== undefined) { - addError( - errors, - filePath, - `${location}.source`, - `Field '${location}.source' was removed. Move 'source.url' to '${location}.repo.url', move 'source.ref' to '${location}.repo.branch', and set '${location}.repo.path' to the local checkout path.`, - ); - } - - if (projectRecord.repository !== undefined) { - addError( - errors, - filePath, - `${location}.repository`, - `Field '${location}.repository' was removed. Use '${location}.repo.url' with a Git remote URL instead.`, - ); - } - - if (projectRecord.repo !== undefined) { - if (!isPlainObject(projectRecord.repo)) { - addError( - errors, - filePath, - `${location}.repo`, - `Field '${location}.repo' must be an object with path, optional url, and optional branch.`, - ); - return; - } - - for (const flatField of ['path', 'repo_url', 'ref']) { - if (projectRecord[flatField] !== undefined) { - addError( - errors, - filePath, - `${location}.${flatField}`, - `Do not mix '${location}.${flatField}' with '${location}.repo'. Move source repo fields under '${location}.repo'.`, - ); - } - } - - validateRequiredString(errors, filePath, projectRecord.repo.path, `${location}.repo.path`); - if (projectRecord.repo.url !== undefined) { - validateGitRemoteUrl(errors, filePath, projectRecord.repo.url, `${location}.repo.url`); - } - if (projectRecord.repo.branch !== undefined) { - validateRequiredString( - errors, - filePath, - projectRecord.repo.branch, - `${location}.repo.branch`, - ); - } - if (projectRecord.repo.ref !== undefined) { - addError( - errors, - filePath, - `${location}.repo.ref`, - `Field '${location}.repo.ref' is not supported. Use '${location}.repo.branch'.`, - ); - } - if (projectRecord.repo.remote !== undefined) { - addError( - errors, - filePath, - `${location}.repo.remote`, - `Use '${location}.repo.url' for the source Git URL. '${location}.repo.remote' is only valid inside results repo config.`, - ); - } - return; - } - - validateRequiredString(errors, filePath, projectRecord.path, `${location}.path`); - addWarning( - errors, - filePath, - `${location}.path`, - `Field '${location}.path' is deprecated. Use '${location}.repo.path'. Existing flat project entries still load and are written back in nested form.`, - ); - - if (projectRecord.repo_url !== undefined) { - validateGitRemoteUrl(errors, filePath, projectRecord.repo_url, `${location}.repo_url`); - addWarning( - errors, - filePath, - `${location}.repo_url`, - `Field '${location}.repo_url' is deprecated. Use '${location}.repo.url'.`, - ); - } - - if (projectRecord.ref !== undefined) { - validateRequiredString(errors, filePath, projectRecord.ref, `${location}.ref`); - addWarning( - errors, - filePath, - `${location}.ref`, - `Field '${location}.ref' is deprecated. Use '${location}.repo.branch'.`, - ); - } -} - function validateRequiredString( errors: ValidationError[], filePath: string, @@ -324,269 +205,28 @@ function validateRequiredString( } } -function validateGitRemoteUrl( +function validateOptionalString( errors: ValidationError[], filePath: string, value: unknown, location: string, ): void { - if (typeof value !== 'string' || value.trim().length === 0) { - errors.push({ - severity: 'error', - filePath, - location, - message: `Field '${location}' must be a non-empty Git remote URL (e.g., https://github.com/EntityProcess/agentv.git or git@github.com:EntityProcess/agentv.git)`, - }); - return; - } - - const repoUrl = value.trim(); - if (!isGitRemoteUrlValue(repoUrl)) { - errors.push({ - severity: 'error', - filePath, - location, - message: `Field '${location}' must be a Git remote URL, not an owner/name shorthand. Use https://github.com/owner/repo.git or git@github.com:owner/repo.git.`, - }); - } -} - -function validateResultsRepoBlock( - errors: ValidationError[], - filePath: string, - rawRepo: unknown, - location: string, -): void { - if (!isPlainObject(rawRepo)) { - addError(errors, filePath, location, `Field '${location}' must be an object`); - return; - } - - const repoRecord = rawRepo; - const hasUrl = repoRecord.url !== undefined; - const hasRemote = repoRecord.remote !== undefined; - const hasPath = repoRecord.path !== undefined; - - if (!hasRemote && !hasUrl && !hasPath) { - addError(errors, filePath, location, `Field '${location}' must set remote or path`); - } - - if (hasRemote && hasUrl) { - addError( - errors, - filePath, - location, - `Field '${location}' must set only one remote endpoint. Use '${location}.remote'.`, - ); - } - - if (hasRemote) { - validateGitRemoteUrl(errors, filePath, repoRecord.remote, `${location}.remote`); - } - - if (hasUrl) { - validateGitRemoteUrl(errors, filePath, repoRecord.url, `${location}.url`); - addWarning( - errors, - filePath, - `${location}.url`, - `Field '${location}.url' is accepted for compatibility. Use '${location}.remote' for the Git remote URL.`, - ); - } - - if (hasPath) { - validateRequiredString(errors, filePath, repoRecord.path, `${location}.path`); - } - - if ( - repoRecord.branch !== undefined && - (typeof repoRecord.branch !== 'string' || repoRecord.branch.trim().length === 0) - ) { - addError( - errors, - filePath, - `${location}.branch`, - `Field '${location}.branch' must be a non-empty string`, - ); - } -} - -function validateResultsSyncAndBranchPrefix( - errors: ValidationError[], - filePath: string, - resultsRecord: Record, - location: string, -): void { - if (resultsRecord.auto_push !== undefined && typeof resultsRecord.auto_push !== 'boolean') { - addError( - errors, - filePath, - `${location}.auto_push`, - `Field '${location}.auto_push' must be a boolean`, - ); - } - - if (resultsRecord.sync !== undefined) { - if ( - typeof resultsRecord.sync !== 'object' || - resultsRecord.sync === null || - Array.isArray(resultsRecord.sync) - ) { - addError(errors, filePath, `${location}.sync`, `Field '${location}.sync' must be an object`); - } else { - const syncRecord = resultsRecord.sync as Record; - if (syncRecord.auto_push !== undefined && typeof syncRecord.auto_push !== 'boolean') { - addError( - errors, - filePath, - `${location}.sync.auto_push`, - `Field '${location}.sync.auto_push' must be a boolean`, - ); - } - if (syncRecord.require_push !== undefined) { - addError( - errors, - filePath, - `${location}.sync.require_push`, - `Field '${location}.sync.require_push' was removed from persistent config. Use the per-run --results-require-push CLI flag instead.`, - ); - } - if (syncRecord.push_conflict_policy === 'backup_and_force_push') { - addError( - errors, - filePath, - `${location}.sync.push_conflict_policy`, - `Field '${location}.sync.push_conflict_policy' uses removed value 'backup_and_force_push'; remove it or set it to 'block'. AgentV never force-pushes result branches.`, - ); - } else if ( - syncRecord.push_conflict_policy !== undefined && - syncRecord.push_conflict_policy !== 'block' - ) { - addError( - errors, - filePath, - `${location}.sync.push_conflict_policy`, - `Field '${location}.sync.push_conflict_policy' must be 'block'`, - ); - } - } - } - - if ( - resultsRecord.branch_prefix !== undefined && - (typeof resultsRecord.branch_prefix !== 'string' || - resultsRecord.branch_prefix.trim().length === 0) - ) { - addError( - errors, - filePath, - `${location}.branch_prefix`, - `Field '${location}.branch_prefix' must be a non-empty string`, - ); - } -} - -function validateFlatResultsRepoConfig( - errors: ValidationError[], - filePath: string, - resultsRecord: Record, - location: string, - options: { allowLegacyRepoString: boolean }, -): void { - const hasLegacyRepo = typeof resultsRecord.repo === 'string'; - const hasRepoUrl = resultsRecord.repo_url !== undefined; - const hasRepoPath = resultsRecord.repo_path !== undefined; - const sourceCount = [ - options.allowLegacyRepoString && hasLegacyRepo, - hasRepoUrl, - hasRepoPath, - ].filter(Boolean).length; - if (sourceCount === 0) { - addError(errors, filePath, location, `Field '${location}' must set repo.remote or repo.path`); - } else if (sourceCount > 1) { - addError( - errors, - filePath, - location, - `Field '${location}' must set only one results repo source. Use '${location}.repo.remote' for a managed clone or '${location}.repo.path' for an existing local checkout.`, - ); - } else if (hasLegacyRepo) { - validateRequiredString(errors, filePath, resultsRecord.repo, `${location}.repo`); - } else if (hasRepoUrl) { - validateGitRemoteUrl(errors, filePath, resultsRecord.repo_url, `${location}.repo_url`); - } else { - validateRequiredString(errors, filePath, resultsRecord.repo_path, `${location}.repo_path`); - } - - if ( - resultsRecord.branch !== undefined && - (typeof resultsRecord.branch !== 'string' || resultsRecord.branch.trim().length === 0) - ) { - addError( - errors, - filePath, - `${location}.branch`, - `Field '${location}.branch' must be a non-empty string`, - ); - } - - if (resultsRecord.remote !== undefined) { - addError( - errors, - filePath, - `${location}.remote`, - `Field '${location}.remote' was removed from persistent config because it was a local Git remote-name alias. Use '${location}.repo.remote' for the portable Git endpoint URL, or omit it and let AgentV use the checkout remote alias internally.`, - ); - } - - if (resultsRecord.path !== undefined) { - if (typeof resultsRecord.path !== 'string' || resultsRecord.path.trim().length === 0) { - addError( - errors, - filePath, - `${location}.path`, - `Field '${location}.path' must be a non-empty string`, - ); - } else { - const p = resultsRecord.path.trim(); - if (!isFilesystemPath(p)) { - addError( - errors, - filePath, - `${location}.path`, - `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results). Found: '${p}'. Remove 'path' to use the default.`, - ); - } - } + if (value !== undefined && (typeof value !== 'string' || value.trim().length === 0)) { + addError(errors, filePath, location, `Field '${location}' must be a non-empty string`); } } -function warnFlatResultsMigration( +function validateProjectRepoConfig( errors: ValidationError[], filePath: string, - resultsRecord: Record, + projectRecord: Record, location: string, ): void { - const migrations: Record = { - repo: `${location}.repo.remote`, - repo_url: `${location}.repo.remote`, - repo_path: `${location}.repo.path`, - branch: `${location}.repo.branch`, - path: `${location}.repo.path`, - auto_push: `${location}.sync.auto_push`, - mode: '(remove this field)', - }; - - for (const [field, replacement] of Object.entries(migrations)) { - if (resultsRecord[field] !== undefined) { - addWarning( - errors, - filePath, - `${location}.${field}`, - `Field '${location}.${field}' is deprecated. Use '${replacement}' in the nested results repo schema.`, - ); - } - } + // Source repo is flat: required local checkout `path`, optional `repo` + // (slug or Git URL), and optional `branch`. + validateRequiredString(errors, filePath, projectRecord.path, `${location}.path`); + validateOptionalString(errors, filePath, projectRecord.repo, `${location}.repo`); + validateOptionalString(errors, filePath, projectRecord.branch, `${location}.branch`); } function validateResultsConfigBody( @@ -594,7 +234,7 @@ function validateResultsConfigBody( filePath: string, rawResults: unknown, location: string, - options: { allowLegacyRepoString: boolean; projectScoped: boolean }, + options: { projectScoped: boolean }, ): void { if (rawResults === undefined) { return; @@ -605,72 +245,37 @@ function validateResultsConfigBody( return; } - const resultsRecord = rawResults; - if (resultsRecord.mode !== undefined) { + const r = rawResults; + + if (r.mode !== undefined) { if (options.projectScoped) { addError( errors, filePath, `${location}.mode`, - `Remove '${location}.mode'; project results use '${location}.repo.remote' or '${location}.repo.path'.`, + `Remove '${location}.mode'; project results use '${location}.repo' and '${location}.path'.`, ); - } else if (resultsRecord.mode !== 'github') { + } else if (r.mode !== 'github') { addError(errors, filePath, `${location}.mode`, `Field '${location}.mode' must be 'github'`); } } - for (const [field, message] of Object.entries({ - repository: `Field '${location}.repository' was removed. Use '${location}.repo.remote' with a Git remote URL instead.`, - local_path: `Field '${location}.local_path' was removed. Use '${location}.repo.path' for the local clone path instead.`, - })) { - if (resultsRecord[field] !== undefined) { - addError(errors, filePath, `${location}.${field}`, message); - } + validateOptionalString(errors, filePath, r.repo, `${location}.repo`); + validateOptionalString(errors, filePath, r.path, `${location}.path`); + validateOptionalString(errors, filePath, r.branch, `${location}.branch`); + + if (r.repo === undefined && r.path === undefined) { + addError(errors, filePath, location, `Field '${location}' must set repo or path`); } - if (options.projectScoped && resultsRecord.auto_push !== undefined) { + if (r.auto_push !== undefined && typeof r.auto_push !== 'boolean') { addError( errors, filePath, `${location}.auto_push`, - `Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`, + `Field '${location}.auto_push' must be a boolean`, ); } - - const hasNestedRepo = isPlainObject(resultsRecord.repo); - if (resultsRecord.repo !== undefined && !hasNestedRepo) { - if (typeof resultsRecord.repo === 'string' && options.allowLegacyRepoString) { - // Handled by the flat compatibility branch below. - } else { - addError( - errors, - filePath, - `${location}.repo`, - `Field '${location}.repo' must be an object. Use '${location}.repo.remote' for a Git remote URL or '${location}.repo.path' for an existing local checkout.`, - ); - } - } - - if (hasNestedRepo) { - for (const flatField of ['repo_url', 'repo_path', 'branch', 'remote', 'path']) { - if (resultsRecord[flatField] !== undefined) { - addError( - errors, - filePath, - `${location}.${flatField}`, - `Do not mix '${location}.${flatField}' with '${location}.repo'. Move results repo fields under '${location}.repo'.`, - ); - } - } - validateResultsRepoBlock(errors, filePath, resultsRecord.repo, `${location}.repo`); - } else { - validateFlatResultsRepoConfig(errors, filePath, resultsRecord, location, { - allowLegacyRepoString: options.allowLegacyRepoString, - }); - warnFlatResultsMigration(errors, filePath, resultsRecord, location); - } - - validateResultsSyncAndBranchPrefix(errors, filePath, resultsRecord, location); } function validateProjectResultsConfig( @@ -680,7 +285,6 @@ function validateProjectResultsConfig( location: string, ): void { validateResultsConfigBody(errors, filePath, rawResults, location, { - allowLegacyRepoString: false, projectScoped: true, }); } @@ -692,17 +296,6 @@ function validateResultsConfig( location: string, ): void { validateResultsConfigBody(errors, filePath, rawResults, location, { - allowLegacyRepoString: true, projectScoped: false, }); } - -function isFilesystemPath(p: string): boolean { - return ( - p.startsWith('/') || - p.startsWith('~/') || - p.startsWith('~\\') || - p === '~' || - /^[A-Za-z]:[/\\]/.test(p) - ); -} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 139c6c5a6..483c111c3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -146,6 +146,7 @@ export { type GitResultArtifactReadParams, type GitListedRun, type NormalizedResultsConfig, + type RuntimeResultsConfig, type PreparedResultsRepoBranch, type ResultPushConflictPolicy, type ResultsRepoLocalPaths, diff --git a/packages/core/src/projects.ts b/packages/core/src/projects.ts index e87ba28b5..8e979dc79 100644 --- a/packages/core/src/projects.ts +++ b/packages/core/src/projects.ts @@ -17,17 +17,14 @@ * projects: * - id: my-app * name: My App - * repo: - * url: https://github.com/example/my-app.git - * branch: main - * path: /home/user/projects/my-app + * repo: https://github.com/example/my-app.git + * path: /home/user/projects/my-app + * branch: main * results: - * repo: - * remote: https://github.com/example/my-app.git - * path: . - * branch: agentv/results/v1 - * sync: - * auto_push: true + * repo: https://github.com/example/my-app.git + * path: ~/results/my-app + * branch: agentv/results/v1 + * auto_push: true * added_at: "2026-03-20T10:00:00Z" * last_opened_at: "2026-03-30T14:00:00Z" * @@ -67,18 +64,11 @@ import { getAgentvConfigDir } from './paths.js'; // ── Types ─────────────────────────────────────────────────────────────── -export interface ProjectResultsSyncConfig { - autoPush?: boolean; - pushConflictPolicy?: 'block'; -} - export interface ProjectResultsConfig { - repoUrl?: string; - repoPath?: string; - branch?: string; + repo?: string; path?: string; - sync?: ProjectResultsSyncConfig; - branchPrefix?: string; + branch?: string; + autoPush?: boolean; } export interface ProjectEntry { @@ -107,42 +97,19 @@ export function getProjectsRegistryPath(): string { // internals stay camelCase. fromYaml / toYaml handle the translation; every // other function in this module works in camelCase only. -interface ProjectResultsSyncYaml { - auto_push?: boolean; - push_conflict_policy?: 'block' | string; -} - interface ProjectResultsYaml { - repo?: ProjectResultsRepoYaml; - repo_url?: string; - repo_path?: string; - branch?: string; - remote?: string; + repo?: string; path?: string; - sync?: ProjectResultsSyncYaml; - branch_prefix?: string; -} - -interface ProjectRepoYaml { - url?: string; branch?: string; - path?: string; -} - -interface ProjectResultsRepoYaml { - url?: string; - path?: string; - branch?: string; - remote?: string; + auto_push?: boolean; } interface ProjectEntryYaml { id: string; name: string; - repo?: ProjectRepoYaml; - repo_url?: string; + repo?: string; path?: string; - ref?: string; + branch?: string; added_at: string; last_opened_at: string; results?: ProjectResultsYaml; @@ -154,45 +121,10 @@ function readTrimmedString(value: unknown): string | undefined { return trimmed.length > 0 ? trimmed : undefined; } -let warnedRemovedBackupAndForcePushPolicy = false; -let warnedRemovedRequirePushConfig = false; -let warnedRejectedFlatResultsRemoteConfig = false; - -function warnRemovedBackupAndForcePushPolicy(): void { - if (warnedRemovedBackupAndForcePushPolicy) { - return; - } - warnedRemovedBackupAndForcePushPolicy = true; - console.warn( - "[agentv] projects[].results.sync.push_conflict_policy: 'backup_and_force_push' is no longer supported and was ignored while loading the project registry. Remove the field or set it to 'block'; AgentV never force-pushes result branches.", - ); -} - -function warnRemovedRequirePushConfig(): void { - if (warnedRemovedRequirePushConfig) { - return; - } - warnedRemovedRequirePushConfig = true; - console.warn( - '[agentv] projects[].results.sync.require_push is no longer supported in persistent config and was ignored while loading the project registry. Use the per-run --results-require-push CLI flag instead.', - ); -} - -function warnRejectedFlatResultsRemoteConfig(): void { - if (warnedRejectedFlatResultsRemoteConfig) { - return; - } - warnedRejectedFlatResultsRemoteConfig = true; - console.warn( - '[agentv] projects[].results.remote is no longer supported in persistent config, so that results block was ignored while loading the project registry. Use projects[].results.repo.remote for a portable Git endpoint URL, or omit it and let AgentV use the local checkout remote alias internally.', - ); -} - function fromYaml(raw: unknown): ProjectEntry | null { if (!raw || typeof raw !== 'object') return null; const e = raw as Partial; - const repo = e.repo && typeof e.repo === 'object' ? e.repo : undefined; - const sourcePath = readTrimmedString(repo?.path) ?? readTrimmedString(e.path); + const sourcePath = readTrimmedString(e.path); if (typeof e.id !== 'string' || typeof e.name !== 'string' || !sourcePath) { return null; } @@ -203,63 +135,26 @@ function fromYaml(raw: unknown): ProjectEntry | null { addedAt: typeof e.added_at === 'string' ? e.added_at : '', lastOpenedAt: typeof e.last_opened_at === 'string' ? e.last_opened_at : '', }; - const repoUrl = readTrimmedString(repo?.url) ?? readTrimmedString(e.repo_url); + const repoUrl = readTrimmedString(e.repo); if (repoUrl) { entry.repoUrl = repoUrl; } - const branch = readTrimmedString(repo?.branch) ?? readTrimmedString(e.ref); + const branch = readTrimmedString(e.branch); if (branch) { entry.ref = branch; } - if (e.results && typeof e.results === 'object') { + if (e.results && typeof e.results === 'object' && !Array.isArray(e.results)) { const r = e.results as Partial; - if (r.remote !== undefined) { - warnRejectedFlatResultsRemoteConfig(); - return entry; - } - const resultsRepo = - r.repo && typeof r.repo === 'object' && !Array.isArray(r.repo) ? r.repo : undefined; - const repoUrl = - readTrimmedString(resultsRepo?.remote) ?? - readTrimmedString(resultsRepo?.url) ?? - readTrimmedString(r.repo_url); - const repoPath = repoUrl - ? undefined - : (readTrimmedString(resultsRepo?.path) ?? readTrimmedString(r.repo_path)); - const clonePath = repoUrl - ? (readTrimmedString(resultsRepo?.path) ?? readTrimmedString(r.path)) - : readTrimmedString(r.path); - const resultsBranch = readTrimmedString(resultsRepo?.branch) ?? readTrimmedString(r.branch); - if (repoUrl || repoPath) { - const sync = r.sync && typeof r.sync === 'object' ? r.sync : undefined; + const resultsRepo = readTrimmedString(r.repo); + const resultsPath = readTrimmedString(r.path); + const resultsBranch = readTrimmedString(r.branch); + if (resultsRepo || resultsPath || resultsBranch || typeof r.auto_push === 'boolean') { entry.results = { - ...(repoUrl ? { repoUrl } : {}), - ...(repoPath ? { repoPath } : {}), + ...(resultsRepo ? { repo: resultsRepo } : {}), + ...(resultsPath ? { path: resultsPath } : {}), ...(resultsBranch ? { branch: resultsBranch } : {}), - ...(clonePath ? { path: clonePath } : {}), - ...(sync && - (typeof sync.auto_push === 'boolean' || - sync.push_conflict_policy === 'block' || - sync.push_conflict_policy === 'backup_and_force_push') - ? { - sync: { - ...(typeof sync.auto_push === 'boolean' ? { autoPush: sync.auto_push } : {}), - ...(sync.push_conflict_policy === 'block' - ? { pushConflictPolicy: sync.push_conflict_policy } - : {}), - }, - } - : {}), - ...(typeof r.branch_prefix === 'string' && r.branch_prefix.trim().length > 0 - ? { branchPrefix: r.branch_prefix.trim() } - : {}), + ...(typeof r.auto_push === 'boolean' ? { autoPush: r.auto_push } : {}), }; - if (sync && 'require_push' in sync) { - warnRemovedRequirePushConfig(); - } - if (sync?.push_conflict_policy === 'backup_and_force_push') { - warnRemovedBackupAndForcePushPolicy(); - } } } return entry; @@ -269,46 +164,18 @@ function toYaml(entry: ProjectEntry): ProjectEntryYaml { const yaml: ProjectEntryYaml = { id: entry.id, name: entry.name, - repo: { - ...(entry.repoUrl !== undefined && { url: entry.repoUrl }), - ...(entry.ref !== undefined && { branch: entry.ref }), - path: entry.path, - }, + ...(entry.repoUrl !== undefined && { repo: entry.repoUrl }), + path: entry.path, + ...(entry.ref !== undefined && { branch: entry.ref }), added_at: entry.addedAt, last_opened_at: entry.lastOpenedAt, }; if (entry.results) { - const resultsSync = - entry.results.sync?.autoPush !== undefined || - entry.results.sync?.pushConflictPolicy !== undefined - ? { - sync: { - ...(entry.results.sync?.autoPush !== undefined && { - auto_push: entry.results.sync.autoPush, - }), - ...(entry.results.sync?.pushConflictPolicy !== undefined && { - push_conflict_policy: entry.results.sync.pushConflictPolicy, - }), - }, - } - : {}; - const branchPrefix = - entry.results.branchPrefix !== undefined ? { branch_prefix: entry.results.branchPrefix } : {}; - - const resultsRepo: ProjectResultsRepoYaml = { - ...(entry.results.repoUrl !== undefined && { remote: entry.results.repoUrl }), - ...(entry.results.branch !== undefined && { branch: entry.results.branch }), - ...(entry.results.repoUrl && - entry.results.path !== undefined && { - path: entry.results.path, - }), - ...(entry.results.repoUrl === undefined && - entry.results.repoPath !== undefined && { path: entry.results.repoPath }), - }; yaml.results = { - repo: resultsRepo, - ...resultsSync, - ...branchPrefix, + ...(entry.results.repo !== undefined && { repo: entry.results.repo }), + ...(entry.results.path !== undefined && { path: entry.results.path }), + ...(entry.results.branch !== undefined && { branch: entry.results.branch }), + ...(entry.results.autoPush !== undefined && { auto_push: entry.results.autoPush }), }; } return yaml; diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 9ff7e3dad..69618f8eb 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -108,9 +108,8 @@ describe('loadConfig', () => { ' verbose: true', ' pool_slots: 2', 'results:', - ' repo:', - ' path: .', - ' branch: base-results', + ' path: .', + ' branch: base-results', '', ].join('\n'), ); @@ -123,8 +122,7 @@ describe('loadConfig', () => { ' keep_workspaces: true', ' workspace_path: /tmp/agentv-local-workspace', 'results:', - ' repo:', - ' branch: local-results', + ' branch: local-results', '', ].join('\n'), ); @@ -140,7 +138,7 @@ describe('loadConfig', () => { }); expect(config?.results).toEqual({ mode: 'github', - repo_path: '.', + path: '.', branch: 'local-results', }); } finally { @@ -324,15 +322,14 @@ describe('loadConfig', () => { }); describe('parseResultsConfig', () => { - it('parses valid results config with explicit path', () => { + it('parses valid flat results config', () => { const result = parseResultsConfig( { mode: 'github', repo: 'EntityProcess/agentv-evals', - branch: 'agentv-results', path: '~/data/agentv-results', + branch: 'agentv-results', auto_push: true, - branch_prefix: 'eval-results', }, '/tmp/.agentv/config.yaml', ); @@ -340,10 +337,9 @@ describe('parseResultsConfig', () => { expect(result).toEqual({ mode: 'github', repo: 'EntityProcess/agentv-evals', - branch: 'agentv-results', path: '~/data/agentv-results', + branch: 'agentv-results', auto_push: true, - branch_prefix: 'eval-results', }); }); @@ -376,185 +372,44 @@ describe('parseResultsConfig', () => { }); }); - it('parses repo_path and nested sync config', () => { + it('parses a path-only existing local results checkout', () => { const result = parseResultsConfig( { - repo_path: '.', + path: '~/data/agentv-results', branch: 'agentv/results/v1', - sync: { - auto_push: false, - push_conflict_policy: 'block', - }, - }, - '/tmp/.agentv/config.yaml', - ); - - expect(result).toEqual({ - mode: 'github', - repo_path: '.', - branch: 'agentv/results/v1', - sync: { auto_push: false, - push_conflict_policy: 'block', - }, - }); - }); - - it('rejects flat results.remote in persistent config', () => { - const warn = spyOn(console, 'warn').mockImplementation(() => undefined); - try { - const result = parseResultsConfig( - { - repo_path: '.', - branch: 'agentv/results/v1', - remote: 'origin', - }, - '/tmp/.agentv/config.yaml', - ); - - expect(result).toBeUndefined(); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('results.remote')); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('results.repo.remote')); - } finally { - warn.mockRestore(); - } - }); - - it('rejects require_push in persistent results sync config', () => { - const warn = spyOn(console, 'warn').mockImplementation(() => undefined); - try { - const result = parseResultsConfig( - { - repo_path: '.', - sync: { - require_push: true, - }, - }, - '/tmp/.agentv/config.yaml', - ); - - expect(result).toBeUndefined(); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('results.sync.require_push')); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('--results-require-push')); - } finally { - warn.mockRestore(); - } - }); - - it('rejects removed backup_and_force_push sync policy with migration guidance', () => { - const warn = spyOn(console, 'warn').mockImplementation(() => undefined); - try { - const result = parseResultsConfig( - { - repo_path: '.', - sync: { - auto_push: true, - push_conflict_policy: 'backup_and_force_push', - }, - }, - '/tmp/.agentv/config.yaml', - ); - - expect(result).toBeUndefined(); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('is no longer supported')); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("set it to 'block'")); - } finally { - warn.mockRestore(); - } - }); - - it('parses nested repo config for a managed results clone', () => { - const result = parseResultsConfig( - { - repo: { - remote: 'https://github.com/example/results.git', - branch: 'agentv/results/v1', - path: '~/data/agentv-results', - }, - sync: { - auto_push: true, - }, }, '/tmp/.agentv/config.yaml', ); expect(result).toEqual({ mode: 'github', - repo: 'https://github.com/example/results.git', - repo_url: 'https://github.com/example/results.git', - branch: 'agentv/results/v1', path: '~/data/agentv-results', - sync: { - auto_push: true, - }, - }); - }); - - it('parses nested repo config for a URL-backed source storage branch', () => { - const result = parseResultsConfig( - { - repo: { - remote: 'https://github.com/example/source.git', - path: '.', - branch: 'agentv/results/v1', - }, - sync: { - auto_push: true, - }, - }, - '/tmp/.agentv/config.yaml', - ); - - expect(result).toEqual({ - mode: 'github', - repo: 'https://github.com/example/source.git', - repo_url: 'https://github.com/example/source.git', - path: '.', branch: 'agentv/results/v1', - sync: { - auto_push: true, - }, + auto_push: false, }); }); - it('parses nested repo config for an existing local results checkout', () => { + it('parses repo and path together (existing checkout pushing to repo remote)', () => { const result = parseResultsConfig( { - repo: { - path: '.', - branch: 'agentv/results/v1', - }, - sync: { - auto_push: true, - }, + repo: 'https://github.com/example/results.git', + path: '~/data/agentv-results', + branch: 'agentv/results/v1', + auto_push: true, }, '/tmp/.agentv/config.yaml', ); expect(result).toEqual({ mode: 'github', - repo_path: '.', + repo: 'https://github.com/example/results.git', + path: '~/data/agentv-results', branch: 'agentv/results/v1', - sync: { - auto_push: true, - }, + auto_push: true, }); }); - it('returns undefined when nested repo config is mixed with flat repo fields', () => { - const result = parseResultsConfig( - { - repo: { - url: 'https://github.com/example/results.git', - }, - branch: 'agentv/results/v1', - }, - '/tmp/.agentv/config.yaml', - ); - - expect(result).toBeUndefined(); - }); - it('returns undefined when mode is not github', () => { const result = parseResultsConfig( { @@ -567,12 +422,11 @@ describe('parseResultsConfig', () => { expect(result).toBeUndefined(); }); - it('returns undefined when path looks like a repo subdirectory', () => { + it('returns undefined when neither repo nor path is set', () => { const result = parseResultsConfig( { mode: 'github', - repo: 'EntityProcess/agentv-evals', - path: 'autopilot-dev/runs', + branch: 'agentv/results/v1', }, '/tmp/.agentv/config.yaml', ); @@ -605,11 +459,11 @@ describe('parseResultsConfig', () => { expect(result).toBeUndefined(); }); - it('returns undefined when repo_url and repo_path are both set', () => { + it('returns undefined when repo is not a string', () => { const result = parseResultsConfig( { - repo_url: 'https://github.com/example/results.git', - repo_path: '.', + mode: 'github', + repo: 123, }, '/tmp/.agentv/config.yaml', ); @@ -617,11 +471,12 @@ describe('parseResultsConfig', () => { expect(result).toBeUndefined(); }); - it('returns undefined when repo is not a string', () => { + it('returns undefined when branch is empty', () => { const result = parseResultsConfig( { mode: 'github', - repo: 123, + repo: 'EntityProcess/agentv-evals', + branch: '', }, '/tmp/.agentv/config.yaml', ); @@ -629,12 +484,12 @@ describe('parseResultsConfig', () => { expect(result).toBeUndefined(); }); - it('returns undefined when branch is empty', () => { + it('returns undefined when auto_push is not a boolean', () => { const result = parseResultsConfig( { mode: 'github', repo: 'EntityProcess/agentv-evals', - branch: '', + auto_push: 'yes', }, '/tmp/.agentv/config.yaml', ); diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index ab9490bc1..388828f48 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -129,7 +129,7 @@ async function createStaleResultBranchPushFixture(params: { repo: `file://${params.remoteDir}`, path: params.cloneDir, branch: params.storageBranch, - sync: { auto_push: false }, + auto_push: false, }; const firstSourceDir = path.join(params.rootDir, `${path.basename(params.cloneDir)}-local-1`); writeRunArtifacts(firstSourceDir, 'local-stale', '2026-06-23T09:00:00.000Z'); @@ -755,7 +755,7 @@ describe('results repo write path', () => { const normalized = normalizeResultsConfig( { repo_path: '.', - sync: { auto_push: false }, + auto_push: false, }, { baseDir: '/tmp/source-project' }, ); @@ -780,7 +780,7 @@ describe('results repo write path', () => { repo_url: 'https://github.com/example/source.git', path: '.', branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: true }, + auto_push: true, }, { baseDir: '/tmp/source-project' }, ); @@ -813,7 +813,7 @@ describe('results repo write path', () => { repo_url: `file://${configuredResultsRemoteDir}`, path: projectDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: true }, + auto_push: true, }); expect(git('git remote get-url origin', projectDir)).toBe(originalOrigin); @@ -845,7 +845,7 @@ describe('results repo write path', () => { config: { repo_path: projectDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: false }, + auto_push: false, }, sourceDir: runDir, destinationPath: path.join('current-repo', runTimestamp), @@ -885,7 +885,7 @@ describe('results repo write path', () => { config: { repo_path: projectDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: false }, + auto_push: false, }, sourceDir: runDir, destinationPath: path.join('human-author', runTimestamp), @@ -938,7 +938,7 @@ describe('results repo write path', () => { config: { repo_path: projectDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: false }, + auto_push: false, }, sourceDir: runDir, destinationPath: path.join('env-author', runTimestamp), @@ -994,7 +994,7 @@ describe('results repo write path', () => { config: { repo_path: projectDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: false }, + auto_push: false, }, sourceDir: runDir, destinationPath: path.join('fallback-author', runTimestamp), @@ -1040,7 +1040,7 @@ describe('results repo write path', () => { repo_url: `file://${remoteDir}`, path: projectDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: true }, + auto_push: true, }, sourceDir: runDir, destinationPath: path.join('url-backed-source', runTimestamp), @@ -1075,7 +1075,7 @@ describe('results repo write path', () => { repo_path: projectDir, branch: storageBranch, remote: 'origin', - sync: { auto_push: false }, + auto_push: false, }); expect(status).toMatchObject({ @@ -1118,7 +1118,7 @@ describe('results repo write path', () => { repo_path: projectDir, branch: storageBranch, remote: 'origin', - sync: { auto_push: true }, + auto_push: true, }; await expect(getResultsRepoSyncStatus(config)).resolves.toMatchObject({ @@ -1174,7 +1174,7 @@ describe('results repo write path', () => { repo_path: projectDir, branch: storageBranch, remote: 'origin', - sync: { auto_push: false }, + auto_push: false, }; const status = await syncResultsRepoForProject(config); @@ -1235,7 +1235,7 @@ describe('results repo write path', () => { repo_path: projectDir, branch: storageBranch, remote: 'origin', - sync: { auto_push: true }, + auto_push: true, }; const status = await syncResultsRepoForProject(config); @@ -1286,7 +1286,7 @@ describe('results repo write path', () => { repo_path: projectDir, branch: storageBranch, remote: 'origin', - sync: { auto_push: true }, + auto_push: true, }); expect(status).toMatchObject({ @@ -1339,7 +1339,7 @@ describe('results repo write path', () => { repo_path: projectDir, branch: storageBranch, remote: 'origin', - sync: { auto_push: true }, + auto_push: true, }); expect(status).toMatchObject({ @@ -1380,7 +1380,7 @@ describe('results repo write path', () => { config: { repo_path: resultsRepoDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: false }, + auto_push: false, }, sourceDir: runDir, destinationPath: path.join('external', runTimestamp), @@ -1457,7 +1457,7 @@ describe('results repo write path', () => { }); const result = await directPushResultsWithDetails({ - config: { ...fixture.config, sync: { auto_push: true } }, + config: { ...fixture.config, auto_push: true }, sourceDir: fixture.localSourceDir, destinationPath: fixture.localDestinationPath, commitMessage: 'feat(results): auto-merge push conflict', @@ -1497,27 +1497,6 @@ describe('results repo write path', () => { ).toBe(''); }, 30000); - it('rejects removed backup_and_force_push policy before syncing', async () => { - const sourceDir = path.join(rootDir, 'removed-policy-source'); - writeRunArtifacts(sourceDir, 'removed-policy', '2026-06-23T10:30:00.000Z'); - - await expect( - directPushResultsWithDetails({ - config: { - repo_path: rootDir, - branch: DEFAULT_RESULTS_BRANCH, - sync: { - auto_push: true, - push_conflict_policy: 'backup_and_force_push', - }, - } as unknown as ResultsConfig, - sourceDir, - destinationPath: path.join('removed-policy', '2026-06-23T10-30-00-000Z'), - commitMessage: 'feat(results): removed policy', - }), - ).rejects.toThrow(/backup_and_force_push.*no longer supported/); - }); - it('retries a benign push race without force', async () => { const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); const storageBranch = initializeRemoteStorageBranch(seedDir, DEFAULT_RESULTS_BRANCH); @@ -1526,7 +1505,7 @@ describe('results repo write path', () => { repo: `file://${remoteDir}`, path: cloneDir, branch: storageBranch, - sync: { auto_push: true }, + auto_push: true, }; const baseTip = git(`git --git-dir "${remoteDir}" rev-parse ${storageBranch}`, rootDir); @@ -2564,7 +2543,7 @@ describe('results branch stable genesis', () => { config: { repo_path: cloneDir, branch: DEFAULT_RESULTS_BRANCH, - sync: { auto_push: true }, + auto_push: true, }, sourceDir, destinationPath: path.join(params.experiment, fsTimestamp), @@ -2619,7 +2598,7 @@ describe('results branch stable genesis', () => { const sourceDir = path.join(rootDir, `${label}-run`); writeRunArtifacts(sourceDir, label, runTimestamp); await directPushResults({ - config: { repo_path: repoDir, branch: DEFAULT_RESULTS_BRANCH, sync: { auto_push: false } }, + config: { repo_path: repoDir, branch: DEFAULT_RESULTS_BRANCH, auto_push: false }, sourceDir, destinationPath: path.join(label, fsTimestamp), commitMessage: `feat(results): ${label}`, @@ -2683,7 +2662,7 @@ describe('results branch stable genesis', () => { // Client A publishes first and wins the race to create the remote branch. await directPushResults({ - config: { repo_path: cloneA, branch: DEFAULT_RESULTS_BRANCH, sync: { auto_push: true } }, + config: { repo_path: cloneA, branch: DEFAULT_RESULTS_BRANCH, auto_push: true }, sourceDir: runA, destinationPath: path.join('expA', '2026-06-19T10-00-00-000Z'), commitMessage: 'feat(results): expA', @@ -2693,7 +2672,7 @@ describe('results branch stable genesis', () => { // non-fast-forward, but because both share the deterministic genesis it // reconciles by re-basing onto the remote tip instead of diverging. await directPushResults({ - config: { repo_path: cloneB, branch: DEFAULT_RESULTS_BRANCH, sync: { auto_push: true } }, + config: { repo_path: cloneB, branch: DEFAULT_RESULTS_BRANCH, auto_push: true }, sourceDir: runB, destinationPath: path.join('expB', '2026-06-19T11-00-00-000Z'), commitMessage: 'feat(results): expB', diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index 2b577e047..70e9970ec 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -51,13 +51,10 @@ describe('validateConfigFile', () => { await writeFile( filePath, `results: - repo: - remote: https://github.com/EntityProcess/agentv-evals.git - branch: agentv-results - path: ~/data/agentv-results - sync: - auto_push: true - branch_prefix: eval-results + repo: https://github.com/EntityProcess/agentv-evals.git + branch: agentv-results + path: ~/data/agentv-results + auto_push: true `, ); @@ -89,18 +86,14 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - repo: - url: https://github.com/EntityProcess/agentv.git - branch: main - path: /srv/agentv + repo: https://github.com/EntityProcess/agentv.git + path: /srv/agentv + branch: main results: - repo: - remote: git@github.com:EntityProcess/agentv-results.git - branch: agentv-results - path: /srv/agentv-results - sync: - auto_push: true - branch_prefix: eval-results + repo: git@github.com:EntityProcess/agentv-results.git + branch: agentv-results + path: /srv/agentv-results + auto_push: true `, ); @@ -120,8 +113,7 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - repo: - path: /srv/agentv + path: /srv/agentv added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, @@ -145,111 +137,20 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - repo: - url: https://github.com/EntityProcess/agentv.git - path: /srv/agentv - results: - repo: - remote: https://github.com/EntityProcess/agentv.git - path: . - branch: agentv/results/v1 - sync: - auto_push: false -`, - ); - - const result = await validateConfigFile(filePath, { scope: 'global' }); - - expect(result.valid).toBe(true); - expect(result.errors).toHaveLength(0); - }); - - it('errors on removed require_push persistent results sync config', async () => { - const filePath = path.join(tempDir, 'removed-require-push.yaml'); - await writeFile( - filePath, - `results: - repo: - path: . - sync: - require_push: true -`, - ); - - const result = await validateConfigFile(filePath); - - expect(result.valid).toBe(false); - expect(result.errors).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - severity: 'error', - location: 'results.sync.require_push', - message: expect.stringContaining('--results-require-push'), - }), - ]), - ); - }); - - it('errors on removed flat results.remote persistent config', async () => { - const filePath = path.join(tempDir, 'removed-flat-results-remote.yaml'); - await writeFile( - filePath, - `results: - repo_path: . - branch: agentv/results/v1 - remote: origin -`, - ); - - const result = await validateConfigFile(filePath); - - expect(result.valid).toBe(false); - expect(result.errors).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - severity: 'error', - location: 'results.remote', - message: expect.stringContaining('results.repo.remote'), - }), - ]), - ); - }); - - it('keeps flat project repo fields compatible with migration warnings', async () => { - const filePath = path.join(tempDir, 'global-config-flat-project.yaml'); - await writeFile( - filePath, - `projects: - - id: agentv - name: AgentV - repo_url: https://github.com/EntityProcess/agentv.git + repo: https://github.com/EntityProcess/agentv.git path: /srv/agentv - ref: main results: - repo_url: git@github.com:EntityProcess/agentv-results.git - branch: agentv-results - path: /srv/agentv-results - sync: - auto_push: true + repo: https://github.com/EntityProcess/agentv.git + path: . + branch: agentv/results/v1 + auto_push: false `, ); const result = await validateConfigFile(filePath, { scope: 'global' }); expect(result.valid).toBe(true); - expect(result.errors).toEqual( - expect.arrayContaining([ - expect.objectContaining({ severity: 'warning', location: 'projects[0].repo_url' }), - expect.objectContaining({ severity: 'warning', location: 'projects[0].path' }), - expect.objectContaining({ severity: 'warning', location: 'projects[0].ref' }), - expect.objectContaining({ - severity: 'warning', - location: 'projects[0].results.repo_url', - }), - expect.objectContaining({ severity: 'warning', location: 'projects[0].results.branch' }), - expect.objectContaining({ severity: 'warning', location: 'projects[0].results.path' }), - ]), - ); + expect(result.errors).toHaveLength(0); }); it('infers AGENTV_HOME config.yaml as global even when the home dir is named .agentv', async () => { @@ -262,8 +163,7 @@ describe('validateConfigFile', () => { `projects: - id: agentv name: AgentV - repo: - path: /srv/agentv + path: /srv/agentv `, ); @@ -302,26 +202,20 @@ describe('validateConfigFile', () => { ); }); - it('errors on invalid global project entries and nested results config', async () => { + it('errors on invalid global project entries and flat results config', async () => { const filePath = path.join(tempDir, 'global-config-invalid-projects.yaml'); await writeFile( filePath, `projects: - id: "" name: 42 - repo: - url: EntityProcess/agentv - path: - branch: "" + repo: 99 + path: "" + branch: "" results: - repo: - remote: EntityProcess/results - branch: "" - path: repo/subdir - sync: - auto_push: yes - push_conflict_policy: overwrite - branch_prefix: "" + repo: "" + branch: "" + auto_push: maybe - not-an-object `, ); @@ -333,157 +227,26 @@ describe('validateConfigFile', () => { expect.arrayContaining([ expect.objectContaining({ severity: 'error', location: 'projects[0].id' }), expect.objectContaining({ severity: 'error', location: 'projects[0].name' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].repo.url' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].repo.path' }), - expect.objectContaining({ severity: 'error', location: 'projects[0].repo.branch' }), - expect.objectContaining({ - severity: 'error', - location: 'projects[0].results.repo.branch', - }), + expect.objectContaining({ severity: 'error', location: 'projects[0].repo' }), + expect.objectContaining({ severity: 'error', location: 'projects[0].path' }), + expect.objectContaining({ severity: 'error', location: 'projects[0].branch' }), expect.objectContaining({ severity: 'error', - location: 'projects[0].results.repo.remote', + location: 'projects[0].results.repo', }), expect.objectContaining({ severity: 'error', - location: 'projects[0].results.sync.auto_push', + location: 'projects[0].results.branch', }), expect.objectContaining({ severity: 'error', - location: 'projects[0].results.sync.push_conflict_policy', - }), - expect.objectContaining({ - severity: 'error', - location: 'projects[0].results.branch_prefix', + location: 'projects[0].results.auto_push', }), expect.objectContaining({ severity: 'error', location: 'projects[1]' }), ]), ); }); - it('reports backup_and_force_push as a removed push conflict policy', async () => { - const filePath = path.join(tempDir, 'removed-push-policy.yaml'); - await writeFile( - filePath, - `projects: - - id: demo - name: Demo - repo: - path: /tmp/demo - results: - repo: - path: . - sync: - push_conflict_policy: backup_and_force_push -`, - ); - - const result = await validateConfigFile(filePath, { scope: 'global' }); - - expect(result.valid).toBe(false); - const error = result.errors.find( - (entry) => entry.location === 'projects[0].results.sync.push_conflict_policy', - ); - expect(error?.message).toContain('uses removed value'); - expect(error?.message).toContain("set it to 'block'"); - }); - - it.each([ - { - field: 'repository', - yaml: 'repository: example/repo', - location: 'projects[0].repository', - migration: 'repo.url', - }, - { - field: 'source', - yaml: `source: - url: https://github.com/example/repo - ref: main`, - location: 'projects[0].source', - migration: 'Move', - }, - { - field: 'results.mode', - yaml: `results: - mode: github - repo_url: https://github.com/example/results.git`, - location: 'projects[0].results.mode', - migration: 'Remove', - }, - { - field: 'results.repo', - yaml: `results: - repo: example/legacy-results`, - location: 'projects[0].results.repo', - migration: 'repo.remote', - }, - - { - field: 'results.remote', - yaml: `results: - repo_path: . - branch: agentv/results/v1 - remote: origin`, - location: 'projects[0].results.remote', - migration: 'repo.remote', - }, - { - field: 'results.repository', - yaml: `results: - repo: - remote: https://github.com/example/results.git - repository: example/results`, - location: 'projects[0].results.repository', - migration: 'repo.remote', - }, - { - field: 'results.local_path', - yaml: `results: - repo: - remote: https://github.com/example/results.git - local_path: /srv/results`, - location: 'projects[0].results.local_path', - migration: 'path', - }, - { - field: 'results.auto_push', - yaml: `results: - repo: - remote: https://github.com/example/results.git - auto_push: true`, - location: 'projects[0].results.auto_push', - migration: 'sync.auto_push', - }, - ])('errors on removed legacy project field $field with migration guidance', async (legacy) => { - const filePath = path.join(tempDir, `global-config-legacy-${legacy.field}.yaml`); - await writeFile( - filePath, - `projects: - - id: legacy - name: Legacy - repo: - url: https://github.com/example/repo.git - path: /srv/legacy - branch: main - ${legacy.yaml} -`, - ); - - const result = await validateConfigFile(filePath, { scope: 'global' }); - - expect(result.valid).toBe(false); - expect(result.errors).toContainEqual( - expect.objectContaining({ - severity: 'error', - location: legacy.location, - }), - ); - expect(result.errors.find((e) => e.location === legacy.location)?.message).toContain( - legacy.migration, - ); - }); - it('warns on deprecated results_by_project', async () => { const filePath = path.join(tempDir, 'deprecated-results-by-project.yaml'); await writeFile( @@ -526,9 +289,8 @@ describe('validateConfigFile', () => { await writeFile( filePath, `results: - repo: - path: . - branch: agentv/results/v1 + path: . + branch: agentv/results/v1 `, ); @@ -538,50 +300,6 @@ describe('validateConfigFile', () => { expect(result.errors).toHaveLength(0); }); - it('keeps flat top-level results compatible with migration warnings', async () => { - const filePath = path.join(tempDir, 'config-results-flat-compatible.yaml'); - await writeFile( - filePath, - `results: - repo_url: https://github.com/EntityProcess/agentv-evals.git - branch: agentv-results - path: ~/data/agentv-results - sync: - auto_push: true -`, - ); - - const result = await validateConfigFile(filePath); - - expect(result.valid).toBe(true); - expect(result.errors).toEqual( - expect.arrayContaining([ - expect.objectContaining({ severity: 'warning', location: 'results.repo_url' }), - expect.objectContaining({ severity: 'warning', location: 'results.branch' }), - expect.objectContaining({ severity: 'warning', location: 'results.path' }), - ]), - ); - }); - - it('errors on old-style subdirectory path', async () => { - const filePath = path.join(tempDir, 'config-results-old-path.yaml'); - await writeFile( - filePath, - `results: - mode: github - repo: EntityProcess/agentv-evals - path: autopilot-dev/runs -`, - ); - - const result = await validateConfigFile(filePath); - - const fieldErrors = result.errors.filter( - (e) => e.severity === 'error' && e.location === 'results.path', - ); - expect(fieldErrors).toHaveLength(1); - }); - it('errors on invalid required_version type', async () => { const filePath = path.join(tempDir, 'config-bad-version.yaml'); await writeFile(filePath, 'required_version: 3\n'); diff --git a/packages/core/test/projects.test.ts b/packages/core/test/projects.test.ts index 2ea0b7400..7090364d8 100644 --- a/packages/core/test/projects.test.ts +++ b/packages/core/test/projects.test.ts @@ -109,7 +109,7 @@ describe('projects registry', () => { expect(yamlOnDisk).toContain('projects:'); }); - it('round-trips repo_url and ref fields through YAML', () => { + it('round-trips flat source repo fields through YAML', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); writeFileSync( @@ -117,9 +117,9 @@ describe('projects registry', () => { `projects: - id: remote-bench name: Remote Bench - repo_url: git@github.com:example/repo.git + repo: git@github.com:example/repo.git path: /srv/agentv/repo - ref: main + branch: main added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, @@ -129,22 +129,24 @@ describe('projects registry', () => { const registry = loadProjectRegistry(); expect(registry.projects).toHaveLength(1); const entry = registry.projects[0]; - expect(entry.repoUrl).toBe('git@github.com:example/repo.git'); - expect(entry.ref).toBe('main'); + expect(entry).toMatchObject({ + repoUrl: 'git@github.com:example/repo.git', + ref: 'main', + path: '/srv/agentv/repo', + }); }); - it('round-trips nested source repo fields through YAML', () => { + it('writes flat source repo fields on save', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); writeFileSync( registryPath, `projects: - - id: nested-source - name: Nested Source - repo: - url: git@github.com:example/repo.git - branch: main - path: /srv/agentv/repo + - id: flat-source + name: Flat Source + repo: git@github.com:example/repo.git + path: /srv/agentv/repo + branch: main added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, @@ -152,16 +154,17 @@ describe('projects registry', () => { ); const registry = loadProjectRegistry(); - expect(registry.projects).toHaveLength(1); - const entry = registry.projects[0]; - expect(entry).toMatchObject({ - repoUrl: 'git@github.com:example/repo.git', - ref: 'main', - path: '/srv/agentv/repo', - }); + saveProjectRegistry(registry); + const yamlOnDisk = readFileSync(registryPath, 'utf-8'); + expect(yamlOnDisk).toContain('repo: git@github.com:example/repo.git'); + expect(yamlOnDisk).toContain('path: /srv/agentv/repo'); + expect(yamlOnDisk).toContain('branch: main'); + expect(yamlOnDisk).not.toContain('url:'); + expect(yamlOnDisk).not.toContain('repo_url:'); + expect(yamlOnDisk).not.toContain('ref:'); }); - it('drops removed project results push policy while preserving other sync config', () => { + it('round-trips flat results config through YAML', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); writeFileSync( @@ -171,54 +174,39 @@ describe('projects registry', () => { name: Results Project path: /srv/agentv/repo results: - repo_url: https://github.com/EntityProcess/results-project-runs.git - branch: agentv-results + repo: https://github.com/EntityProcess/results-project-runs.git path: /srv/agentv/results/results-project - sync: - auto_push: true - push_conflict_policy: backup_and_force_push - branch_prefix: eval-results + branch: agentv-results + auto_push: true added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, 'utf-8', ); - const warn = spyOn(console, 'warn').mockImplementation(() => undefined); const registry = loadProjectRegistry(); - try { - expect(registry.projects[0].results).toEqual({ - repoUrl: 'https://github.com/EntityProcess/results-project-runs.git', - branch: 'agentv-results', - path: '/srv/agentv/results/results-project', - sync: { autoPush: true }, - branchPrefix: 'eval-results', - }); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('backup_and_force_push')); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('was ignored')); - } finally { - warn.mockRestore(); - } + expect(registry.projects[0].results).toEqual({ + repo: 'https://github.com/EntityProcess/results-project-runs.git', + path: '/srv/agentv/results/results-project', + branch: 'agentv-results', + autoPush: true, + }); saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); - expect(yamlOnDisk).toContain('repo:'); - expect(yamlOnDisk).toContain( - 'remote: https://github.com/EntityProcess/results-project-runs.git', - ); + expect(yamlOnDisk).toContain('repo: https://github.com/EntityProcess/results-project-runs.git'); expect(yamlOnDisk).toContain('branch: agentv-results'); expect(yamlOnDisk).toContain('path: /srv/agentv/results/results-project'); expect(yamlOnDisk).toContain('auto_push: true'); expect(yamlOnDisk).not.toContain('push_conflict_policy:'); - expect(yamlOnDisk).toContain('branch_prefix: eval-results'); + expect(yamlOnDisk).not.toContain('branch_prefix:'); expect(yamlOnDisk).not.toContain('repo_url:'); - expect(yamlOnDisk).not.toContain('localPath:'); - expect(yamlOnDisk).not.toContain('local_path:'); + expect(yamlOnDisk).not.toContain('repo_path:'); + expect(yamlOnDisk).not.toContain('sync:'); expect(yamlOnDisk).not.toContain('autoPush:'); - expect(yamlOnDisk).not.toContain('branchPrefix:'); }); - it('round-trips branch-backed current-repo results config through YAML', () => { + it('round-trips repo-and-path results config through YAML', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); writeFileSync( @@ -226,16 +214,13 @@ describe('projects registry', () => { `projects: - id: branch-results name: Branch Results - repo: - url: git@github.com:example/source.git - path: /srv/agentv/repo + repo: git@github.com:example/source.git + path: /srv/agentv/repo results: - repo: - remote: git@github.com:example/source.git - path: . - branch: agentv/results/v1 - sync: - auto_push: false + repo: git@github.com:example/source.git + path: . + branch: agentv/results/v1 + auto_push: false added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, @@ -244,61 +229,22 @@ describe('projects registry', () => { const registry = loadProjectRegistry(); expect(registry.projects[0].results).toEqual({ - repoUrl: 'git@github.com:example/source.git', + repo: 'git@github.com:example/source.git', path: '.', branch: 'agentv/results/v1', - sync: { autoPush: false }, + autoPush: false, }); saveProjectRegistry(registry); const yamlOnDisk = readFileSync(registryPath, 'utf-8'); - expect(yamlOnDisk).toContain('repo:'); - expect(yamlOnDisk).toContain('remote: git@github.com:example/source.git'); + expect(yamlOnDisk).toContain('repo: git@github.com:example/source.git'); expect(yamlOnDisk).toContain('path: .'); expect(yamlOnDisk).toContain('branch: agentv/results/v1'); expect(yamlOnDisk).toContain('auto_push: false'); - expect(yamlOnDisk).not.toContain('push_conflict_policy:'); expect(yamlOnDisk).not.toContain('repo_path:'); - expect(yamlOnDisk).not.toContain('repoPath:'); expect(yamlOnDisk).not.toContain('require_push:'); }); - it('warns and rejects results blocks with legacy flat results remote aliases', () => { - const registryPath = getProjectsRegistryPath(); - mkdirSync(path.dirname(registryPath), { recursive: true }); - const warnSpy = spyOn(console, 'warn').mockImplementation(() => undefined); - writeFileSync( - registryPath, - `projects: - - id: legacy-results-remote - name: Legacy Results Remote - path: /srv/agentv/repo - results: - repo_path: . - branch: agentv/results/v1 - remote: upstream - added_at: "2026-01-01T00:00:00Z" - last_opened_at: "2026-01-01T00:00:00Z" -`, - 'utf-8', - ); - - try { - const registry = loadProjectRegistry(); - expect(registry.projects[0].results).toBeUndefined(); - expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('projects[].results.remote')); - - saveProjectRegistry(registry); - const yamlOnDisk = readFileSync(registryPath, 'utf-8'); - expect(yamlOnDisk).not.toContain('results:'); - expect(yamlOnDisk).toContain('repo:'); - expect(yamlOnDisk).not.toContain('remote: upstream'); - expect(yamlOnDisk).not.toContain('repo_path:'); - } finally { - warnSpy.mockRestore(); - } - }); - it('preserves unrelated global config keys when saving projects', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); @@ -322,47 +268,35 @@ dashboard: expect(yamlOnDisk).toContain('projects:'); }); - it('warns and ignores require_push in project registry entries', () => { + it('loads a path-only results checkout from project registry entries', () => { const registryPath = getProjectsRegistryPath(); const localRegistryPath = path.join(path.dirname(registryPath), 'config.local.yaml'); mkdirSync(path.dirname(registryPath), { recursive: true }); - const warnSpy = spyOn(console, 'warn').mockImplementation(() => undefined); writeFileSync( localRegistryPath, `projects: - id: local-results name: Local Results - repo: - path: /srv/agentv/source + path: /srv/agentv/source results: - repo: - path: /srv/agentv/results/local-results - branch: agentv/results/v1 - sync: - require_push: true + path: /srv/agentv/results/local-results + branch: agentv/results/v1 added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, 'utf-8', ); - try { - const registry = loadProjectRegistry(); - - expect(registry.projects).toHaveLength(1); - expect(registry.projects[0]).toMatchObject({ - id: 'local-results', - path: '/srv/agentv/source', - results: { - repoPath: '/srv/agentv/results/local-results', - branch: 'agentv/results/v1', - }, - }); - expect(registry.projects[0].results?.sync).toBeUndefined(); - expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('results.sync.require_push')); - } finally { - warnSpy.mockRestore(); - } + const registry = loadProjectRegistry(); + expect(registry.projects).toHaveLength(1); + expect(registry.projects[0]).toMatchObject({ + id: 'local-results', + path: '/srv/agentv/source', + results: { + path: '/srv/agentv/results/local-results', + branch: 'agentv/results/v1', + }, + }); }); it('keeps registry mutations in config.local.yaml when the overlay owns projects', () => { @@ -375,8 +309,7 @@ dashboard: `projects: - id: local-only name: Local Only - repo: - path: /srv/agentv/source + path: /srv/agentv/source added_at: "2026-01-01T00:00:00Z" last_opened_at: "2026-01-01T00:00:00Z" `, @@ -394,14 +327,14 @@ dashboard: expect(localYaml).not.toContain('dashboard:'); }); - it('interpolates env vars in repo_url', () => { + it('interpolates env vars in repo', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); // Use concatenation to avoid JS template literal evaluating ${{ ... }} const d = '$'; writeFileSync( registryPath, - `projects:\n - id: env-bench\n name: Env Bench\n repo_url: "${d}{{ BENCH_REPO_URL }}"\n path: /srv/agentv/repo\n ref: main\n added_at: "2026-01-01T00:00:00Z"\n last_opened_at: "2026-01-01T00:00:00Z"\n`, + `projects:\n - id: env-bench\n name: Env Bench\n repo: "${d}{{ BENCH_REPO_URL }}"\n path: /srv/agentv/repo\n branch: main\n added_at: "2026-01-01T00:00:00Z"\n last_opened_at: "2026-01-01T00:00:00Z"\n`, 'utf-8', ); From f09120ce60c67182a6b7e305c7d07112b241a5f9 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 29 Jun 2026 15:45:43 +0200 Subject: [PATCH 2/2] test(cli): update serve test fixtures to flat results schema Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/cli/test/commands/results/serve.test.ts | 33 ++++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 6fb87629a..04d59df07 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -196,9 +196,8 @@ function writeResultsConfig( writeFileSync( path.join(projectDir, '.agentv', 'config.yaml'), `results: - repo: - remote: ${JSON.stringify(params.remote)} -${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''}${params.path ? ` path: ${JSON.stringify(params.path)}\n` : ''}${params.autoPush !== undefined ? ` sync:\n auto_push: ${params.autoPush}\n` : ''}`, + repo: ${JSON.stringify(params.remote)} +${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''}${params.path ? ` path: ${JSON.stringify(params.path)}\n` : ''}${params.autoPush !== undefined ? ` auto_push: ${params.autoPush}\n` : ''}`, ); } @@ -2156,9 +2155,9 @@ describe('serve app', () => { name: 'Project No Publish', path: projectDir, results: { - repoUrl: `file://${remoteDir}`, + repo: `file://${remoteDir}`, path: missingCloneDir, - sync: { autoPush: true }, + autoPush: true, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z', @@ -2280,9 +2279,9 @@ describe('serve app', () => { name: 'AgentV', path: projectDir, results: { - repoUrl: 'EntityProcess/agentv-examples-eval-results', + repo: 'EntityProcess/agentv-examples-eval-results', path: '/home/entity/projects/EntityProcess/agentv-examples-eval-results', - sync: { autoPush: true }, + autoPush: true, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z', @@ -2332,9 +2331,9 @@ describe('serve app', () => { name: 'Project Sync Pull', path: projectDir, results: { - repoUrl: `file://${remoteDir}`, + repo: `file://${remoteDir}`, path: cloneDir, - sync: { autoPush: false }, + autoPush: false, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z', @@ -2409,9 +2408,9 @@ describe('serve app', () => { name: 'Project Sync Push', path: projectDir, results: { - repoUrl: `file://${remoteDir}`, + repo: `file://${remoteDir}`, path: cloneDir, - sync: { autoPush: true }, + autoPush: true, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z', @@ -2477,9 +2476,9 @@ describe('serve app', () => { name: 'Project Sync Offline', path: projectDir, results: { - repoUrl: missingRemoteUrl, + repo: missingRemoteUrl, path: cloneDir, - sync: { autoPush: true }, + autoPush: true, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z', @@ -2535,9 +2534,9 @@ describe('serve app', () => { name: 'Project Sync Conflict', path: projectDir, results: { - repoUrl: `file://${remoteDir}`, + repo: `file://${remoteDir}`, path: cloneDir, - sync: { autoPush: true }, + autoPush: true, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z', @@ -2661,9 +2660,9 @@ describe('serve app', () => { name: 'Project Confirm Merge', path: projectDir, results: { - repoUrl: `file://${remoteDir}`, + repo: `file://${remoteDir}`, path: cloneDir, - sync: { autoPush: false }, + autoPush: false, }, addedAt: '2026-01-01T00:00:00.000Z', lastOpenedAt: '2026-01-01T00:00:00.000Z',