From e8f9c9d74c9974c723b12c25af282644c9054aeb Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 21:05:23 -0700 Subject: [PATCH 01/11] perf(profile): fix GraphQL 502 timeouts by shrinking the augment query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The augment pass was timing out (POST /graphql -> 502 at ~10.4s, GitHub's 10s execution-timeout). Root cause was per-repo resolver cost multiplied across aliased repos, not node count. Researched against GitHub's GraphQL limits docs and cut the expensive parts: - Remove commit count (history.totalCount). This is the single most expensive resolver — it walks the entire commit graph (no precomputed count) — and it's pure scale decoration (git history migrates wholesale). Dropped from the query, RepoSignals, scale rollup, and UI. Also drops the defaultBranchRef resolution it required. - Trim the release-asset scan: releases 100->50, assets 50->30 (deeply-nested connection = the docs' flagged timeout anti-pattern). - Trim branch-protection rules first: 100->50 (totalCount stays exact; ~all repos have <10 rules). - Smaller aliased batches: FULL 25->10, LITE 90->15 (fewer simultaneous large-first connections per request — the exact pattern GitHub warns causes timeouts), processed 3 concurrently to keep wall-clock down. Net: each augment request does far less server-side work, staying well under the 10s window. Commit count can be restored later via a cheap per-repo REST call if needed. 11 gates green; coverage 87.76/86.40. --- src/lib/server/profile/analyze.test.ts | 1 - src/lib/server/profile/augment.test.ts | 37 ++++++++----------------- src/lib/server/profile/augment.ts | 33 ++++++++++++---------- src/lib/server/profile/estimate.test.ts | 1 - src/lib/server/profile/insights.test.ts | 1 - src/lib/server/profile/runner.test.ts | 8 +++--- src/lib/server/profile/runner.ts | 17 ++++++++---- src/lib/server/profile/service.test.ts | 4 --- src/lib/server/profile/service.ts | 3 -- src/lib/server/profile/store.test.ts | 1 - src/lib/server/profile/types.ts | 2 -- src/routes/profile/[id]/+page.svelte | 2 -- 12 files changed, 44 insertions(+), 66 deletions(-) diff --git a/src/lib/server/profile/analyze.test.ts b/src/lib/server/profile/analyze.test.ts index c683986..5b1138a 100644 --- a/src/lib/server/profile/analyze.test.ts +++ b/src/lib/server/profile/analyze.test.ts @@ -30,7 +30,6 @@ function cleanSignals(over: Partial = {}): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index ceb80b0..aae61e5 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -60,7 +60,6 @@ function rule(flags: RuleFlags = {}) { /** Build one repository alias node. */ function node(over: { - commits?: number; discussions?: number; projectsV2?: number; environments?: number; @@ -75,12 +74,8 @@ function node(over: { releaseAssetSizes?: number[]; ruleTotal?: number; rules?: ReturnType[]; - noDefaultBranch?: boolean; }) { return { - defaultBranchRef: over.noDefaultBranch - ? null - : { target: { history: { totalCount: over.commits ?? 0 } } }, discussions: { totalCount: over.discussions ?? 0 }, projectsV2: { totalCount: over.projectsV2 ?? 0 }, environments: { totalCount: over.environments ?? 0 }, @@ -123,7 +118,6 @@ describe("augmentRepoSignals", () => { test("maps every count signal and preserves the discovered fields", async () => { const { fn } = mockGql({ r0: node({ - commits: 512, discussions: 3, projectsV2: 2, environments: 4, @@ -137,7 +131,6 @@ describe("augmentRepoSignals", () => { ]); // Augmented counts. - expect(signals?.commitsCount).toBe(512); expect(signals?.discussionsCount).toBe(3); expect(signals?.projectsV2Count).toBe(2); expect(signals?.environmentsCount).toBe(4); @@ -225,14 +218,6 @@ describe("augmentRepoSignals", () => { expect(signals?.workflowFileCount).toBe(0); }); - test("treats a missing default branch as zero commits", async () => { - const { fn } = mockGql({ r0: node({ noDefaultBranch: true }) }); - - const [signals] = await augmentRepoSignals(fn, [discovered({ isEmpty: true })]); - - expect(signals?.commitsCount).toBe(0); - }); - test("derives owner/name per alias and caps the rules page", async () => { const { fn, calls } = mockGql({ r0: node({}), r1: node({}) }); @@ -242,7 +227,7 @@ describe("augmentRepoSignals", () => { ]); expect(calls[0]).toMatchObject({ - rules: 100, + rules: 50, o0: "octo-org", n0: "widget", o1: "octo-org", @@ -252,9 +237,9 @@ describe("augmentRepoSignals", () => { test("profiles a whole chunk in a single request, in input order", async () => { const { fn, calls } = mockGql({ - r0: node({ commits: 1 }), - r1: node({ commits: 2 }), - r2: node({ commits: 3 }), + r0: node({ discussions: 1 }), + r1: node({ discussions: 2 }), + r2: node({ discussions: 3 }), }); const signals = await augmentRepoSignals(fn, [ @@ -264,7 +249,7 @@ describe("augmentRepoSignals", () => { ]); expect(calls).toHaveLength(1); // one batched request for three repos - expect(signals.map((s) => [s.nameWithOwner, s.commitsCount])).toEqual([ + expect(signals.map((s) => [s.nameWithOwner, s.discussionsCount])).toEqual([ ["o/a", 1], ["o/b", 2], ["o/c", 3], @@ -323,18 +308,18 @@ describe("augmentRepoSignals", () => { }); test("degrades a null alias to zeroed signals (keeps the discovery spine)", async () => { - const { fn } = mockGql({ r0: node({ commits: 5 }), r1: null }); + const { fn } = mockGql({ r0: node({ discussions: 5 }), r1: null }); const signals = await augmentRepoSignals(fn, [ discovered({ nameWithOwner: "o/ok", name: "ok" }), discovered({ nameWithOwner: "o/gone", name: "gone", issuesCount: 3 }), ]); - expect(signals[0]?.commitsCount).toBe(5); + expect(signals[0]?.discussionsCount).toBe(5); // The inaccessible repo keeps its discovered fields but zeroes augment counts. expect(signals[1]?.nameWithOwner).toBe("o/gone"); expect(signals[1]?.issuesCount).toBe(3); - expect(signals[1]?.commitsCount).toBe(0); + expect(signals[1]?.discussionsCount).toBe(0); expect(signals[1]?.branchProtectionRuleCount).toBe(0); }); @@ -342,7 +327,7 @@ describe("augmentRepoSignals", () => { // GraphQL returns partial data + errors when one alias is inaccessible; the // client surfaces that as a throw carrying `.data`. Recover the good repos. const err = Object.assign(new Error("Could not resolve to a Repository"), { - data: { r0: node({ commits: 8 }), r1: null }, + data: { r0: node({ discussions: 8 }), r1: null }, }); const fn = (async () => { throw err; @@ -353,8 +338,8 @@ describe("augmentRepoSignals", () => { discovered({ nameWithOwner: "o/b", name: "b" }), ]); - expect(signals[0]?.commitsCount).toBe(8); - expect(signals[1]?.commitsCount).toBe(0); + expect(signals[0]?.discussionsCount).toBe(8); + expect(signals[1]?.discussionsCount).toBe(0); }); test("rethrows an error with no recoverable data payload", async () => { diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index 808012f..4a1cb43 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -22,12 +22,18 @@ import type { graphql } from "@octokit/graphql"; import type { DiscoveredRepo, RepoSignals } from "./types"; -/** GraphQL caps `branchProtectionRules(first:)` at 100 — plenty per repo. */ -const MAX_RULES = 100; +/** GraphQL caps `branchProtectionRules(first:)` at 100; most repos have <10, so + * 50 covers virtually all of them while halving the worst-case node fan-out. */ +const MAX_RULES = 50; -/** Bounded release scan: first N releases, first M assets each (an estimate). */ -const RELEASES_SCANNED = 100; -const ASSETS_SCANNED = 50; +/** + * Bounded release scan: first N releases, first M assets each (an estimate). + * Kept deliberately shallow — `releases(first:){releaseAssets(first:)}` is a + * deeply nested connection, the main driver of the 10s query timeout when many + * repos are aliased into one request. + */ +const RELEASES_SCANNED = 50; +const ASSETS_SCANNED = 30; /** One branch protection rule's migration-relevant flags. */ interface BranchProtectionRuleNode { @@ -42,7 +48,6 @@ interface BranchProtectionRuleNode { /** Raw shape of one repository alias in the batched signals query. */ interface RepoSignalsNode { - defaultBranchRef: { target: { history: { totalCount: number } } | null } | null; discussions: { totalCount: number }; projectsV2: { totalCount: number }; environments: { totalCount: number }; @@ -67,20 +72,20 @@ interface RepoSignalsNode { type BatchSignalsResult = Record; /** - * The per-repo selection. `history` sits behind a `... on Commit` inline - * fragment because `defaultBranchRef.target` is a `GitObject` (Commit | Tree | - * Blob | Tag); only a Commit has `history`. + * The per-repo selection. The release-asset scan (the heaviest part — up to 50 + * releases × 30 assets) is included only when `scanReleases` is set, so repos + * discovery already knows have zero releases can be batched far wider. * - * The release-asset scan (the heaviest part — up to 100 releases × 50 assets) is - * included only when `scanReleases` is set, so repos discovery already knows have - * zero releases can be batched far wider. + * Deliberately omits commit count (`history.totalCount`): it walks the whole + * commit graph and is the single most expensive resolver — the main cause of + * the 10s query timeout when repos are aliased — while being pure scale + * decoration (git history migrates wholesale). */ function signalsFragment(scanReleases: boolean): string { const releases = scanReleases ? `releases(first: ${RELEASES_SCANNED}) { nodes { releaseAssets(first: ${ASSETS_SCANNED}) { nodes { size } } } }` : ""; return `fragment Sig on Repository { - defaultBranchRef { target { ... on Commit { history(first: 1) { totalCount } } } } discussions { totalCount } projectsV2 { totalCount } environments { totalCount } @@ -174,7 +179,6 @@ function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSign if (!node) { return { ...repo, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -190,7 +194,6 @@ function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSign } return { ...repo, - commitsCount: node.defaultBranchRef?.target?.history.totalCount ?? 0, discussionsCount: node.discussions.totalCount, projectsV2Count: node.projectsV2.totalCount, environmentsCount: node.environments.totalCount, diff --git a/src/lib/server/profile/estimate.test.ts b/src/lib/server/profile/estimate.test.ts index 5913830..ab17dc2 100644 --- a/src/lib/server/profile/estimate.test.ts +++ b/src/lib/server/profile/estimate.test.ts @@ -28,7 +28,6 @@ function signals(diskUsageKb: number | null): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/insights.test.ts b/src/lib/server/profile/insights.test.ts index ef2aa32..f3718f6 100644 --- a/src/lib/server/profile/insights.test.ts +++ b/src/lib/server/profile/insights.test.ts @@ -33,7 +33,6 @@ function signals(over: Partial = {}): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index 7b4592b..0349aeb 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -48,7 +48,6 @@ function discovered(name: string, over: Partial = {}): Discovere function signalsFor(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { return { ...repo, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -202,8 +201,9 @@ describe("runProfile", () => { }); test("persists completed chunks when a later chunk fails", async () => { - // 26 repos with releases → two full-scan chunks (25 + 1). The chunk holding - // the last repo throws; the other chunk's 25 repos still persist. + // 26 repos with releases at FULL=10 → chunks of 10 + 10 + 6. The chunk + // holding the last repo (r20–r25) throws; the other two chunks' 20 repos + // still persist. const repos = Array.from({ length: 26 }, (_, i) => discovered(`r${String(i).padStart(2, "0")}`, { releasesCount: 1 }), ); @@ -223,7 +223,7 @@ describe("runProfile", () => { ); expect(run.state).toBe("failed"); - expect(getRunRepoProfiles("r")).toHaveLength(25); // the surviving chunk persisted + expect(getRunRepoProfiles("r")).toHaveLength(20); // the two surviving chunks persisted }); test("batches release-free repos wide (no scan) and release-bearing repos narrow", async () => { diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index ae16044..730db6a 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -37,13 +37,18 @@ import { } from "./types"; /** - * Repos per augment request. Repos discovery found to have zero releases skip - * the heavy release-asset scan, so they batch far wider (LITE); repos with - * releases keep the conservative width (FULL) to stay within GitHub's per-query - * node budget and 10s execution timeout. + * Repos per augment request. Each repo carries server-side git object reads + * (`.gitattributes` blob + `.github/workflows` tree) plus up to 100 + * branch-protection nodes — and, for release-bearing repos, the release-asset + * scan — so the real cost per repo is high and the limiting factor is GitHub's + * ~10s GraphQL execution timeout, not the static node budget. These widths are + * deliberately small: wide batches (e.g. 25+) time out and return a 502. + * + * LITE = repos discovery found to have zero releases (skip the release scan). + * FULL = repos with releases (the heaviest queries), kept at the floor. */ -const AUGMENT_CHUNK_FULL = 25; -const AUGMENT_CHUNK_LITE = 90; +const AUGMENT_CHUNK_FULL = 10; +const AUGMENT_CHUNK_LITE = 15; /** How many augment requests are in flight at once (cuts wall-clock ~Nx). */ const AUGMENT_CONCURRENCY = 3; diff --git a/src/lib/server/profile/service.test.ts b/src/lib/server/profile/service.test.ts index 0e50e65..9b26b09 100644 --- a/src/lib/server/profile/service.test.ts +++ b/src/lib/server/profile/service.test.ts @@ -41,7 +41,6 @@ function discovered(name: string): DiscoveredRepo { function signalsFor(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { return { ...repo, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -218,7 +217,6 @@ describe("getProfileDetail", () => { signalsFor(discovered("a"), { issuesCount: 10, pullRequestsCount: 4, - commitsCount: 100, branchesCount: 3, tagsCount: 2, releasesCount: 1, @@ -231,7 +229,6 @@ describe("getProfileDetail", () => { signalsFor(discovered("b"), { issuesCount: 5, pullRequestsCount: 6, - commitsCount: 50, branchesCount: 1, tagsCount: 0, releasesCount: 2, @@ -245,7 +242,6 @@ describe("getProfileDetail", () => { repos: 2, issues: 15, pullRequests: 10, - commits: 150, branches: 4, tags: 2, releases: 3, diff --git a/src/lib/server/profile/service.ts b/src/lib/server/profile/service.ts index a8810f1..c0329fe 100644 --- a/src/lib/server/profile/service.ts +++ b/src/lib/server/profile/service.ts @@ -45,7 +45,6 @@ interface MigrationScale { repos: number; issues: number; pullRequests: number; - commits: number; branches: number; tags: number; releases: number; @@ -70,7 +69,6 @@ function computeScale(repos: RepoProfileView[]): MigrationScale { repos: repos.length, issues: 0, pullRequests: 0, - commits: 0, branches: 0, tags: 0, releases: 0, @@ -79,7 +77,6 @@ function computeScale(repos: RepoProfileView[]): MigrationScale { for (const { signals } of repos) { scale.issues += signals.issuesCount; scale.pullRequests += signals.pullRequestsCount; - scale.commits += signals.commitsCount; scale.branches += signals.branchesCount; scale.tags += signals.tagsCount; scale.releases += signals.releasesCount; diff --git a/src/lib/server/profile/store.test.ts b/src/lib/server/profile/store.test.ts index d982c09..98f5990 100644 --- a/src/lib/server/profile/store.test.ts +++ b/src/lib/server/profile/store.test.ts @@ -50,7 +50,6 @@ function signals(over: Partial = {}): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, - commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/types.ts b/src/lib/server/profile/types.ts index 832da71..c0bbb91 100644 --- a/src/lib/server/profile/types.ts +++ b/src/lib/server/profile/types.ts @@ -86,8 +86,6 @@ export interface OrgDiscovery { * read a number and compare, rather than re-deriving GraphQL schema details. */ export interface RepoSignals extends DiscoveredRepo { - /** Commits on the default branch (`history.totalCount`) — migration scale. */ - commitsCount: number; /** Repository-level Discussions (`discussions.totalCount`); not migrated. */ discussionsCount: number; /** Projects (new experience) (`projectsV2.totalCount`); not migrated. */ diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index 4f03f5e..6eea925 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -24,7 +24,6 @@ [ { label: 'Issues', value: scale.issues.toLocaleString(), icon: 'issue-opened' }, { label: 'Pull requests', value: scale.pullRequests.toLocaleString(), icon: 'git-pull-request' }, - { label: 'Commits', value: scale.commits.toLocaleString(), icon: 'git-commit' }, { label: 'Branches', value: scale.branches.toLocaleString(), icon: 'git-branch' }, { label: 'Tags', value: scale.tags.toLocaleString(), icon: 'tag' }, { label: 'Releases', value: scale.releases.toLocaleString(), icon: 'rocket' }, @@ -132,7 +131,6 @@ return [ { label: 'Issues', value: s.issuesCount.toLocaleString(), icon: 'issue-opened' }, { label: 'Pull requests', value: s.pullRequestsCount.toLocaleString(), icon: 'git-pull-request' }, - { label: 'Commits', value: s.commitsCount.toLocaleString(), icon: 'git-commit' }, { label: 'Branches', value: s.branchesCount.toLocaleString(), icon: 'git-branch' }, { label: 'Tags', value: s.tagsCount.toLocaleString(), icon: 'tag' }, { label: 'Releases', value: s.releasesCount.toLocaleString(), icon: 'rocket' }, From 3f50fdf59d13c401d6336ea1fd0a56f34b29ce3b Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 21:41:22 -0700 Subject: [PATCH 02/11] perf(profile): batch the cheap totalCount pass wide + add fork/ruleset signals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grounded in GitHub's GraphQL node-limit + timeout docs: a { totalCount } selection (no nodes/edges) doesn't materialize nodes — it's an indexed read — and the 10s timeout is driven by node materialization, not counts. So the cheap pass can batch wide; only the release-asset scan (a deeply-nested walk) must stay narrow. - Widen the no-releases augment batch 15 -> 50; release-bearing 10 -> 15. (The old low cap existed only because the query also walked the commit graph via history.totalCount, which was removed in the prior commit.) - Add two more cheap signals while we're paying for the round trip: forkCount (scalar) and repository rulesets.totalCount. The latter lights up the per-repo 'rulesets' consideration, previously stuck 'not yet evaluated'. - Deliberately NOT adding vulnerabilityAlerts: it's permission-sensitive and a per-repo auth error would null the whole repo node, poisoning the cheap counts. That belongs in its own isolated pass. - UI: per-repo drill-down shows Forks + Rulesets. Net for N repos with few release-bearing: far fewer augment requests, each doing only indexed reads — no timeout risk — while gathering more signal per call. 11 gates green; coverage 87.76/86.40. --- src/lib/server/profile/analyze.test.ts | 4 ++++ src/lib/server/profile/analyze.ts | 1 + src/lib/server/profile/augment.test.ts | 11 +++++++++++ src/lib/server/profile/augment.ts | 25 +++++++++++++++++++------ src/lib/server/profile/estimate.test.ts | 2 ++ src/lib/server/profile/insights.test.ts | 2 ++ src/lib/server/profile/runner.test.ts | 9 +++++---- src/lib/server/profile/runner.ts | 22 ++++++++++++---------- src/lib/server/profile/service.test.ts | 2 ++ src/lib/server/profile/store.test.ts | 2 ++ src/lib/server/profile/types.ts | 4 ++++ src/routes/profile/[id]/+page.svelte | 2 ++ 12 files changed, 66 insertions(+), 20 deletions(-) diff --git a/src/lib/server/profile/analyze.test.ts b/src/lib/server/profile/analyze.test.ts index 5b1138a..33d79e1 100644 --- a/src/lib/server/profile/analyze.test.ts +++ b/src/lib/server/profile/analyze.test.ts @@ -36,6 +36,8 @@ function cleanSignals(over: Partial = {}): RepoSignals { releasesCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, @@ -91,6 +93,7 @@ describe("analyzeRepo", () => { packagesCount: 2, usesLfs: true, workflowFileCount: 4, + rulesetCount: 3, }), ); @@ -113,6 +116,7 @@ describe("analyzeRepo", () => { expect(finding(profile, "actions-run-history")?.evidence).toBe( "4 workflows (run history & artifacts not migrated)", ); + expect(finding(profile, "rulesets")?.evidence).toBe("3 rulesets"); }); test("rolls up applying considerations by severity", () => { diff --git a/src/lib/server/profile/analyze.ts b/src/lib/server/profile/analyze.ts index ec1e677..12a1f4a 100644 --- a/src/lib/server/profile/analyze.ts +++ b/src/lib/server/profile/analyze.ts @@ -99,6 +99,7 @@ const DETECTORS: Record = { s.branchProtectionRulesUsingUnmigratedFeatures > 0 ? `${count(s.branchProtectionRulesUsingUnmigratedFeatures, "rule")} using unmigrated features` : null, + rulesets: (s) => (s.rulesetCount > 0 ? count(s.rulesetCount, "ruleset") : null), "stars-watchers": (s) => s.stargazerCount > 0 || s.watcherCount > 0 ? `${count(s.stargazerCount, "star")}, ${count(s.watcherCount, "watcher")}` diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index aae61e5..c3851a6 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -65,7 +65,9 @@ function node(over: { environments?: number; stars?: number; watchers?: number; + forks?: number; packages?: number; + rulesets?: number; /** Raw `.gitattributes` text; omit for no file (null blob). */ lfsAttributes?: string; /** Workflow file names under `.github/workflows`; omit for no dir (null tree). */ @@ -88,8 +90,10 @@ function node(over: { } : {}), stargazerCount: over.stars ?? 0, + forkCount: over.forks ?? 0, watchers: { totalCount: over.watchers ?? 0 }, packages: { totalCount: over.packages ?? 0 }, + rulesets: { totalCount: over.rulesets ?? 0 }, gitattributes: over.lfsAttributes === undefined ? null : { text: over.lfsAttributes }, workflows: over.workflowFiles ? { entries: over.workflowFiles.map((name) => ({ name })) } @@ -149,6 +153,13 @@ describe("augmentRepoSignals", () => { expect(signals?.packagesCount).toBe(6); }); + test("maps the fork count and repository ruleset count", async () => { + const { fn } = mockGql({ r0: node({ forks: 12, rulesets: 4 }) }); + const [signals] = await augmentRepoSignals(fn, [discovered()]); + expect(signals?.forkCount).toBe(12); + expect(signals?.rulesetCount).toBe(4); + }); + test("detects Git LFS from a .gitattributes with a filter=lfs entry", async () => { const { fn } = mockGql({ r0: node({ lfsAttributes: "*.psd filter=lfs diff=lfs merge=lfs -text\n" }), diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index 4a1cb43..d4442c1 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -56,8 +56,10 @@ interface RepoSignalsNode { nodes: { releaseAssets: { nodes: { size: number }[] } }[]; }; stargazerCount: number; + forkCount: number; watchers: { totalCount: number }; packages: { totalCount: number }; + rulesets: { totalCount: number }; /** Root `.gitattributes` on the default branch (`Blob`), or null if absent. */ gitattributes: { text: string | null } | null; /** `.github/workflows` dir on the default branch (`Tree`), or null if absent. */ @@ -72,14 +74,19 @@ interface RepoSignalsNode { type BatchSignalsResult = Record; /** - * The per-repo selection. The release-asset scan (the heaviest part — up to 50 - * releases × 30 assets) is included only when `scanReleases` is set, so repos + * The per-repo selection. Almost everything here is a scalar or a + * `{ totalCount }`-only connection: per GitHub's node-limit rules those don't + * materialize `nodes`/`edges`, so they're indexed reads that stay cheap even + * when many repos are aliased into one request. The one exception is the + * release-asset scan (`releases{...releaseAssets}`), a deeply-nested walk that + * materializes nodes — included only when `scanReleases` is set so repos * discovery already knows have zero releases can be batched far wider. * - * Deliberately omits commit count (`history.totalCount`): it walks the whole - * commit graph and is the single most expensive resolver — the main cause of - * the 10s query timeout when repos are aliased — while being pure scale - * decoration (git history migrates wholesale). + * Deliberately omits commit count (`history.totalCount`): unlike the other + * counts it is NOT indexed — it walks the whole commit graph — making it the + * single most expensive resolver and the main cause of the 10s query timeout + * when repos are aliased, while being pure scale decoration (history migrates + * wholesale). */ function signalsFragment(scanReleases: boolean): string { const releases = scanReleases @@ -91,8 +98,10 @@ function signalsFragment(scanReleases: boolean): string { environments { totalCount } ${releases} stargazerCount + forkCount watchers { totalCount } packages { totalCount } + rulesets(first: 1) { totalCount } gitattributes: object(expression: "HEAD:.gitattributes") { ... on Blob { text } } workflows: object(expression: "HEAD:.github/workflows") { ... on Tree { entries { name } } } branchProtectionRules(first: $rules) { @@ -184,6 +193,8 @@ function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSign environmentsCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, @@ -199,6 +210,8 @@ function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSign environmentsCount: node.environments.totalCount, stargazerCount: node.stargazerCount, watcherCount: node.watchers.totalCount, + forkCount: node.forkCount, + rulesetCount: node.rulesets.totalCount, branchProtectionRuleCount: node.branchProtectionRules.totalCount, branchProtectionRulesUsingUnmigratedFeatures: node.branchProtectionRules.nodes.filter(usesUnmigratedFeature).length, diff --git a/src/lib/server/profile/estimate.test.ts b/src/lib/server/profile/estimate.test.ts index ab17dc2..ee8e000 100644 --- a/src/lib/server/profile/estimate.test.ts +++ b/src/lib/server/profile/estimate.test.ts @@ -34,6 +34,8 @@ function signals(diskUsageKb: number | null): RepoSignals { releasesCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, diff --git a/src/lib/server/profile/insights.test.ts b/src/lib/server/profile/insights.test.ts index f3718f6..14b895f 100644 --- a/src/lib/server/profile/insights.test.ts +++ b/src/lib/server/profile/insights.test.ts @@ -39,6 +39,8 @@ function signals(over: Partial = {}): RepoSignals { releasesCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index 0349aeb..c5ba454 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -53,6 +53,8 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo environmentsCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, @@ -201,9 +203,8 @@ describe("runProfile", () => { }); test("persists completed chunks when a later chunk fails", async () => { - // 26 repos with releases at FULL=10 → chunks of 10 + 10 + 6. The chunk - // holding the last repo (r20–r25) throws; the other two chunks' 20 repos - // still persist. + // 26 repos with releases at FULL=15 → chunks of 15 + 11. The chunk holding + // the last repo (r15–r25) throws; the first chunk's 15 repos still persist. const repos = Array.from({ length: 26 }, (_, i) => discovered(`r${String(i).padStart(2, "0")}`, { releasesCount: 1 }), ); @@ -223,7 +224,7 @@ describe("runProfile", () => { ); expect(run.state).toBe("failed"); - expect(getRunRepoProfiles("r")).toHaveLength(20); // the two surviving chunks persisted + expect(getRunRepoProfiles("r")).toHaveLength(15); // the surviving chunk persisted }); test("batches release-free repos wide (no scan) and release-bearing repos narrow", async () => { diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index 730db6a..7a9d5b9 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -37,18 +37,20 @@ import { } from "./types"; /** - * Repos per augment request. Each repo carries server-side git object reads - * (`.gitattributes` blob + `.github/workflows` tree) plus up to 100 - * branch-protection nodes — and, for release-bearing repos, the release-asset - * scan — so the real cost per repo is high and the limiting factor is GitHub's - * ~10s GraphQL execution timeout, not the static node budget. These widths are - * deliberately small: wide batches (e.g. 25+) time out and return a 502. + * Repos per augment request. The augment query is almost all scalars and + * `{ totalCount }` connections, which per GitHub's node-limit rules don't + * materialize nodes — they're indexed reads that stay cheap when many repos are + * aliased into one request. The one exception is the release-asset scan (a + * deeply-nested walk), so repos are split: those discovery found to have zero + * releases skip that scan and batch wide (LITE); repos with releases include it + * and batch narrow (FULL) to stay under the 10s GraphQL execution timeout. * - * LITE = repos discovery found to have zero releases (skip the release scan). - * FULL = repos with releases (the heaviest queries), kept at the floor. + * (LITE was previously capped low only because the query also walked the commit + * graph via `history.totalCount`; that expensive field is gone, so the cheap + * pass can batch wide again.) */ -const AUGMENT_CHUNK_FULL = 10; -const AUGMENT_CHUNK_LITE = 15; +const AUGMENT_CHUNK_FULL = 15; +const AUGMENT_CHUNK_LITE = 50; /** How many augment requests are in flight at once (cuts wall-clock ~Nx). */ const AUGMENT_CONCURRENCY = 3; diff --git a/src/lib/server/profile/service.test.ts b/src/lib/server/profile/service.test.ts index 9b26b09..c660f13 100644 --- a/src/lib/server/profile/service.test.ts +++ b/src/lib/server/profile/service.test.ts @@ -46,6 +46,8 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo environmentsCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, diff --git a/src/lib/server/profile/store.test.ts b/src/lib/server/profile/store.test.ts index 98f5990..b039e57 100644 --- a/src/lib/server/profile/store.test.ts +++ b/src/lib/server/profile/store.test.ts @@ -56,6 +56,8 @@ function signals(over: Partial = {}): RepoSignals { releasesCount: 0, stargazerCount: 0, watcherCount: 0, + forkCount: 0, + rulesetCount: 0, branchProtectionRuleCount: 0, branchProtectionRulesUsingUnmigratedFeatures: 0, packagesCount: 0, diff --git a/src/lib/server/profile/types.ts b/src/lib/server/profile/types.ts index c0bbb91..9077afa 100644 --- a/src/lib/server/profile/types.ts +++ b/src/lib/server/profile/types.ts @@ -102,6 +102,10 @@ export interface RepoSignals extends DiscoveredRepo { stargazerCount: number; /** Watchers (`watchers.totalCount`); not migrated. */ watcherCount: number; + /** Forks (`forkCount`); fork relationships are not migrated. */ + forkCount: number; + /** Repository-level rulesets (`rulesets.totalCount`); not migrated. */ + rulesetCount: number; /** Branch protection rules (`branchProtectionRules.totalCount`). */ branchProtectionRuleCount: number; /** diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index 6eea925..a141e94 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -139,7 +139,9 @@ { label: 'Environments', value: s.environmentsCount.toLocaleString(), icon: 'server' }, { label: 'Stars', value: s.stargazerCount.toLocaleString(), icon: 'star' }, { label: 'Watchers', value: s.watcherCount.toLocaleString(), icon: 'eye' }, + { label: 'Forks', value: s.forkCount.toLocaleString(), icon: 'repo-forked' }, { label: 'Protection rules', value: s.branchProtectionRuleCount.toLocaleString(), icon: 'shield' }, + { label: 'Rulesets', value: s.rulesetCount.toLocaleString(), icon: 'law' }, { label: 'Size', value: formatRepoSize(s.diskUsageKb), icon: 'database' } ]; } From a4e6efecb641692356906b58aa2a86a57ee0f326 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 21:46:45 -0700 Subject: [PATCH 03/11] fix(profile): scope repo rulesets to includeParents:false + handle null Read the Repository.rulesets schema reference, which revealed two bugs in the field added last commit: 1. rulesets(includeParents:) defaults to TRUE, so totalCount counts org/enterprise rulesets that apply to the repo - which we already gather separately as run.orgRulesetCount. Every repo was double- counting the org's rulesets. Set includeParents:false to scope the per-repo signal to rulesets actually defined on the repo. 2. RepositoryRulesetConnection is nullable in the schema (no bang, unlike discussions/packages/etc. which are Connection-bang). A viewer without ruleset read access gets null, which would have thrown on node.rulesets.totalCount. Guard with optional-chain plus nullish-zero. Both determinable straight from the GraphQL schema reference - no live org needed. Added a null-connection test. 11 gates green; coverage 87.76/86.40. --- src/lib/server/profile/augment.test.ts | 10 ++++++++++ src/lib/server/profile/augment.ts | 7 ++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index c3851a6..1838c61 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -160,6 +160,16 @@ describe("augmentRepoSignals", () => { expect(signals?.rulesetCount).toBe(4); }); + test("treats a null rulesets connection as zero (schema allows null)", async () => { + // RepositoryRulesetConnection is nullable in the schema; a viewer without + // ruleset read access gets null rather than a connection. + const n = node({}); + (n as { rulesets: unknown }).rulesets = null; + const { fn } = mockGql({ r0: n }); + const [signals] = await augmentRepoSignals(fn, [discovered()]); + expect(signals?.rulesetCount).toBe(0); + }); + test("detects Git LFS from a .gitattributes with a filter=lfs entry", async () => { const { fn } = mockGql({ r0: node({ lfsAttributes: "*.psd filter=lfs diff=lfs merge=lfs -text\n" }), diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index d4442c1..71ebe34 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -59,7 +59,8 @@ interface RepoSignalsNode { forkCount: number; watchers: { totalCount: number }; packages: { totalCount: number }; - rulesets: { totalCount: number }; + /** Repo-level rulesets. The connection is nullable in the schema (no `!`). */ + rulesets: { totalCount: number } | null; /** Root `.gitattributes` on the default branch (`Blob`), or null if absent. */ gitattributes: { text: string | null } | null; /** `.github/workflows` dir on the default branch (`Tree`), or null if absent. */ @@ -101,7 +102,7 @@ function signalsFragment(scanReleases: boolean): string { forkCount watchers { totalCount } packages { totalCount } - rulesets(first: 1) { totalCount } + rulesets(first: 1, includeParents: false) { totalCount } gitattributes: object(expression: "HEAD:.gitattributes") { ... on Blob { text } } workflows: object(expression: "HEAD:.github/workflows") { ... on Tree { entries { name } } } branchProtectionRules(first: $rules) { @@ -211,7 +212,7 @@ function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSign stargazerCount: node.stargazerCount, watcherCount: node.watchers.totalCount, forkCount: node.forkCount, - rulesetCount: node.rulesets.totalCount, + rulesetCount: node.rulesets?.totalCount ?? 0, branchProtectionRuleCount: node.branchProtectionRules.totalCount, branchProtectionRulesUsingUnmigratedFeatures: node.branchProtectionRules.nodes.filter(usesUnmigratedFeature).length, From a76640ca992551f19edc3bee11c38f03d8bde42e Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 22:25:08 -0700 Subject: [PATCH 04/11] fix(profile): survive GitHub GraphQL timeouts + add crawl observability The profiler was failing the whole run when GitHub returned a 502 on an augment query. That 502 is GitHub's own API edge (the request id format and the ~10.4s duration are GitHub's 10-second GraphQL processing timeout, which their docs say returns 502/504) - not a local proxy. A 502 carries no partial data, so the chunk error propagated and failed the entire crawl. Resilience (the real fix): - augmentRepoSignals now detects a timeout (status 502/504 or a timeout message) and splits the chunk, recursing on the halves until each query is cheap enough to evaluate in GitHub's window. A single repo that still times out degrades to zeroed augment signals (keeping its discovery spine) so one pathological repo cannot fail the run. This self-adapts to whatever the org/instance can handle. Observability (the run was silent): - runner now logs start, discovery total + timing, each augment-chunk failure (size + scanReleases), and completion/failure with timing. - failures get a richer reason via describeError (HTTP status + GraphQL error message) instead of a bare message, shown on the detail page. - discovery progress is wired so the run total is set from the first discovery page and watchers are nudged - the page shows "Discovering repositories..." then "Profiling N repositories..." instead of 0/0. UI: - the empty/working state is now phase-aware (discovering vs profiling). Also adds X-Accel-Buffering:no to the SSE response headers. This does nothing for local dev (no proxy) but is correct hardening for the documented reverse-proxy deployment (docker-compose + ORIGIN), where a proxy would otherwise buffer the event stream. 11 gates green; coverage 87.76/86.38. --- src/lib/server/core/sse.ts | 5 +++ src/lib/server/profile/augment.test.ts | 61 ++++++++++++++++++++++++++ src/lib/server/profile/augment.ts | 47 ++++++++++++++++++-- src/lib/server/profile/runner.test.ts | 48 ++++++++++++++++++++ src/lib/server/profile/runner.ts | 61 ++++++++++++++++++++++++-- src/routes/profile/[id]/+page.svelte | 10 ++++- 6 files changed, 224 insertions(+), 8 deletions(-) diff --git a/src/lib/server/core/sse.ts b/src/lib/server/core/sse.ts index 3d02c97..812b387 100644 --- a/src/lib/server/core/sse.ts +++ b/src/lib/server/core/sse.ts @@ -15,6 +15,11 @@ const SSE_HEADERS = { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", Connection: "keep-alive", + // Disable proxy response buffering (nginx and similar). Without this, a proxy + // buffers the event stream and can close it as "idle" or time the upstream out + // — surfacing as a 502 on the SSE endpoint. nginx honors this header the same + // way as `proxy_buffering off;`. + "X-Accel-Buffering": "no", } as const; /** diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index 1838c61..c0a11f6 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -370,4 +370,65 @@ describe("augmentRepoSignals", () => { await expect(augmentRepoSignals(fn, [discovered()])).rejects.toThrow(/network down/); }); + + test("splits a timed-out chunk and retries the smaller halves", async () => { + // A `gql` double that 502s (GitHub's timeout) for any request wider than 2 + // repos, and succeeds otherwise. Records the size of each attempted request. + const sizes: number[] = []; + const fn = (async (_q: string, vars: Record) => { + const size = Object.keys(vars).filter((k) => /^o\d+$/.test(k)).length; + sizes.push(size); + if (size > 2) throw Object.assign(new Error("Bad Gateway"), { status: 502 }); + const res: Record = {}; + for (let i = 0; i < size; i++) res[`r${i}`] = node({}); + return res; + }) as unknown as typeof graphql; + + const repos = ["a", "b", "c", "d"].map((n) => discovered({ name: n, nameWithOwner: `o/${n}` })); + const signals = await augmentRepoSignals(fn, repos); + + // All four repos come back, in order, despite the initial timeout. + expect(signals.map((s) => s.nameWithOwner)).toEqual(["o/a", "o/b", "o/c", "o/d"]); + // 4 timed out → split into 2 + 2, each of which succeeds. + expect(sizes).toEqual([4, 2, 2]); + }); + + test("degrades a single repo that keeps timing out to zeroed signals", async () => { + // Always 502s, so splitting bottoms out at one repo each, which degrades. + const fn = (async () => { + throw Object.assign(new Error("Gateway Timeout"), { status: 504 }); + }) as unknown as typeof graphql; + + const repos = [ + discovered({ name: "a", nameWithOwner: "o/a", issuesCount: 7 }), + discovered({ name: "b", nameWithOwner: "o/b" }), + ]; + const signals = await augmentRepoSignals(fn, repos); + + expect(signals).toHaveLength(2); + // Discovery spine preserved; augment counts degraded to zero. + expect(signals[0]?.nameWithOwner).toBe("o/a"); + expect(signals[0]?.issuesCount).toBe(7); + expect(signals[0]?.discussionsCount).toBe(0); + expect(signals[1]?.nameWithOwner).toBe("o/b"); + }); + + test("treats a timeout message (not just a status) as splittable", async () => { + let calls = 0; + const fn = (async (_q: string, vars: Record) => { + const size = Object.keys(vars).filter((k) => /^o\d+$/.test(k)).length; + calls += 1; + if (size > 1) throw new Error("Something went wrong while executing your query"); + return { r0: node({ discussions: 1 }) }; + }) as unknown as typeof graphql; + + const repos = [ + discovered({ name: "a", nameWithOwner: "o/a" }), + discovered({ name: "b", nameWithOwner: "o/b" }), + ]; + const signals = await augmentRepoSignals(fn, repos); + + expect(signals.map((s) => s.discussionsCount)).toEqual([1, 1]); + expect(calls).toBe(3); // 2 (timeout) → 1 + 1 (ok) + }); }); diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index 71ebe34..6930305 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -223,13 +223,29 @@ function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSign }; } -/** Recover partial `data` from a GraphQL error that carries it, else rethrow. */ -function dataFromError(err: unknown): BatchSignalsResult { +/** Recover partial `data` from a GraphQL error that carries it, else null. */ +function partialDataFromError(err: unknown): BatchSignalsResult | null { if (err && typeof err === "object" && "data" in err) { const data = (err as { data?: unknown }).data; if (data && typeof data === "object") return data as BatchSignalsResult; } - throw err; + return null; +} + +/** + * Whether an error is a GitHub processing timeout. The GraphQL API terminates + * any request that takes longer than ~10s and responds with a 502 or 504 (or a + * "Something went wrong while executing your query" message). These carry no + * usable partial data — the remedy is to make the query cheaper, i.e. split it. + */ +function isTimeoutError(err: unknown): boolean { + if (!err || typeof err !== "object") return false; + const e = err as { status?: number; message?: string }; + if (e.status === 502 || e.status === 504) return true; + return ( + typeof e.message === "string" && + /timeout|timed out|executing your query|respond to your request in time/i.test(e.message) + ); } /** Options controlling how wide/deep a single augment request reaches. */ @@ -271,9 +287,32 @@ export async function augmentRepoSignals( variables, ); } catch (err) { + // A GitHub processing timeout (502/504): the query was too expensive to + // evaluate in GitHub's 10s window. Splitting the chunk makes each query + // cheaper; recurse on the halves until it fits. A single repo that still + // times out degrades to zeroed augment signals (keeping its discovery + // spine) so one pathological repo can't fail the whole crawl. + if (isTimeoutError(err)) { + if (repos.length === 1) { + const only = repos[0]; + console.warn( + `[profile] augment timed out for a single repo${only ? ` (${only.nameWithOwner})` : ""}; recording it with zeroed signals`, + ); + return only ? [toSignals(only, null)] : []; + } + const mid = Math.ceil(repos.length / 2); + console.warn( + `[profile] augment timed out for ${repos.length} repos; splitting ${mid} + ${repos.length - mid} and retrying`, + ); + const left = await augmentRepoSignals(gql, repos.slice(0, mid), opts); + const right = await augmentRepoSignals(gql, repos.slice(mid), opts); + return [...left, ...right]; + } // A chunk with one inaccessible repo errors but still carries partial data; // recover it so the accessible repos in the chunk still get profiled. - data = dataFromError(err); + const partial = partialDataFromError(err); + if (!partial) throw err; + data = partial; } return repos.map((repo, i) => toSignals(repo, data[`r${i}`] ?? null)); diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index c5ba454..d43d085 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -131,6 +131,54 @@ describe("runProfile", () => { expect(run.orgResources.codespacesSecrets).toBe(0); }); + test("sets the run total from the first discovery page and nudges watchers", async () => { + const repos = [discovered("a"), discovered("b")]; + const progress: ProfileProgress[] = []; + const withDiscoveryProgress: Partial = { + discover: async (_gql, _org, onProgress) => { + // The org total is known from page 1, before the repos are returned. + onProgress?.({ org: "acme", discovered: 2, total: 2, page: 1 }); + return { org: "acme", total: 2, repos }; + }, + augment: async (_gql, c) => c.map((r) => signalsFor(r)), + getOrgRulesetCount: async () => 0, + getOrgResources: async () => ZERO_ORG_RESOURCES, + }; + + await runProfile( + gql, + { id: "disc", org: "acme", sourceApiUrl: "u" }, + (p) => progress.push(p), + withDiscoveryProgress, + ); + + // The first nudge is the discovery one (profiled 0), before any per-repo + // progress, and the run total is set from it. + expect(progress[0]).toEqual({ runId: "disc", profiled: 0, total: 2, repo: "" }); + expect(getProfileRun("disc")?.totalRepos).toBe(2); + }); + + test("records a richer failure reason for an HTTP error (status + message)", async () => { + const failing: Partial = { + discover: async () => { + throw Object.assign(new Error("Something went wrong while executing your query"), { + status: 502, + }); + }, + }; + + const run = await runProfile( + gql, + { id: "http", org: "acme", sourceApiUrl: "u" }, + undefined, + failing, + ); + + expect(run.state).toBe("failed"); + expect(run.failureReason).toContain("HTTP 502"); + expect(run.failureReason).toContain("Something went wrong"); + }); + test("emits progress once per repo with running totals", async () => { const repos = [discovered("a"), discovered("b"), discovered("c")]; const progress: ProfileProgress[] = []; diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index 7a9d5b9..92dc901 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -62,6 +62,31 @@ function chunk(items: T[], size: number): T[][] { return out; } +/** + * Produce a concise, actionable failure reason from a crawl error. Octokit + * surfaces HTTP failures with a numeric `status` (e.g. 502 on a GraphQL + * timeout) and GraphQL errors with an `errors[]` array; pull those out so the + * persisted `failureReason` (shown on the detail page) says something useful + * rather than a bare "[object Object]". + */ +function describeError(err: unknown): string { + if (err && typeof err === "object") { + const e = err as { + status?: number; + message?: string; + errors?: Array<{ message?: string; type?: string }>; + }; + const parts: string[] = []; + if (typeof e.status === "number") parts.push(`HTTP ${e.status}`); + const firstGqlError = Array.isArray(e.errors) ? e.errors[0] : undefined; + if (firstGqlError?.message) parts.push(firstGqlError.message); + else if (firstGqlError?.type) parts.push(firstGqlError.type); + else if (e.message) parts.push(e.message); + if (parts.length > 0) return parts.join(": "); + } + return err instanceof Error ? err.message : String(err); +} + /** Injectable crawl primitives (defaults hit the real GraphQL helpers). */ export interface ProfileRunnerDeps { discover: typeof discoverOrgRepos; @@ -97,11 +122,27 @@ export async function runProfile( deps: Partial = {}, ): Promise { const d = { ...DEFAULT_DEPS, ...deps }; + const startedMs = Date.now(); createProfileRun({ id: input.id, sourceApiUrl: input.sourceApiUrl, org: input.org }); + console.log(`[profile] run ${input.id} started — org=${input.org}, source=${input.sourceApiUrl}`); try { - const discovery = await d.discover(gql, input.org); - setProfileRunTotal(input.id, discovery.total); + // Set the run total as soon as the FIRST discovery page reports it (the org + // total is known from page 1) and nudge any live watcher, so the detail page + // shows movement during the otherwise-silent discovery phase instead of + // sitting at 0/0. + let totalSet = false; + const discovery = await d.discover(gql, input.org, (p) => { + if (!totalSet && p.total > 0) { + totalSet = true; + setProfileRunTotal(input.id, p.total); + onProgress?.({ runId: input.id, profiled: 0, total: p.total, repo: "" }); + } + }); + if (!totalSet) setProfileRunTotal(input.id, discovery.total); + console.log( + `[profile] run ${input.id} discovered ${discovery.total} repo(s) in ${Date.now() - startedMs}ms`, + ); // Org-level signals are gathered once per run (best-effort, never fatal) — // they're org-scoped, not per-repo. Run the two REST passes concurrently. @@ -150,6 +191,10 @@ export async function runProfile( }); } } catch (err) { + console.error( + `[profile] run ${input.id} augment chunk failed — ${task.repos.length} repo(s), scanReleases=${task.scanReleases}:`, + err, + ); if (firstError === null) firstError = err; } } @@ -160,8 +205,18 @@ export async function runProfile( if (firstError !== null) throw firstError; completeProfileRun(input.id); + const completed = getProfileRun(input.id); + console.log( + `[profile] run ${input.id} completed — ${profiled} repo(s) profiled, ` + + `${completed?.blockers ?? 0} blocker(s), ${completed?.warnings ?? 0} warning(s) ` + + `in ${Date.now() - startedMs}ms`, + ); } catch (err) { - const reason = err instanceof Error ? err.message : String(err); + const reason = describeError(err); + console.error( + `[profile] run ${input.id} failed after ${Date.now() - startedMs}ms — ${reason}`, + err, + ); failProfileRun(input.id, reason); } diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index a141e94..ee82279 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -432,7 +432,15 @@ {#if repos.length === 0}
-

{run.state === 'running' ? 'Crawling…' : 'No repositories profiled'}

+

+ {#if run.state !== 'running'} + No repositories profiled + {:else if run.totalRepos > 0} + Profiling {run.totalRepos.toLocaleString()} repositories… ({run.profiledRepos.toLocaleString()} done) + {:else} + Discovering repositories… + {/if} +

{:else}
From bd856df9c19b7a6f7c77901fde2d3638ecd24365 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 22:30:50 -0700 Subject: [PATCH 05/11] fix(profile): cap augment repo-batch at 10-15 (FULL=10, LITE=15) The repo-batch count (how many repos are aliased into one totalCount request) must stay at 10-15 max. Live 502s showed the per-repo cost (two git-object reads plus, for release-bearing repos, the release-asset scan) is higher than node-limit theory predicts, so the earlier widening to 50 timed out. Back to FULL=10 (release-bearing) / LITE=15 (no releases). The split-on-timeout fallback still backstops any chunk that times out. These are the repo-batch counts only; MAX_RULES and RELEASES_SCANNED are per-repo query depths (first: args within one repo), a different axis, left unchanged. 11 gates green. --- src/lib/server/profile/runner.test.ts | 7 ++++--- src/lib/server/profile/runner.ts | 24 +++++++++++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index d43d085..88b3930 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -251,8 +251,9 @@ describe("runProfile", () => { }); test("persists completed chunks when a later chunk fails", async () => { - // 26 repos with releases at FULL=15 → chunks of 15 + 11. The chunk holding - // the last repo (r15–r25) throws; the first chunk's 15 repos still persist. + // 26 repos with releases at FULL=10 → chunks of 10 + 10 + 6. The chunk + // holding the last repo (r20–r25) throws; the other two chunks' 20 repos + // still persist. const repos = Array.from({ length: 26 }, (_, i) => discovered(`r${String(i).padStart(2, "0")}`, { releasesCount: 1 }), ); @@ -272,7 +273,7 @@ describe("runProfile", () => { ); expect(run.state).toBe("failed"); - expect(getRunRepoProfiles("r")).toHaveLength(15); // the surviving chunk persisted + expect(getRunRepoProfiles("r")).toHaveLength(20); // the two surviving chunks persisted }); test("batches release-free repos wide (no scan) and release-bearing repos narrow", async () => { diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index 92dc901..5dd2b65 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -37,20 +37,18 @@ import { } from "./types"; /** - * Repos per augment request. The augment query is almost all scalars and - * `{ totalCount }` connections, which per GitHub's node-limit rules don't - * materialize nodes — they're indexed reads that stay cheap when many repos are - * aliased into one request. The one exception is the release-asset scan (a - * deeply-nested walk), so repos are split: those discovery found to have zero - * releases skip that scan and batch wide (LITE); repos with releases include it - * and batch narrow (FULL) to stay under the 10s GraphQL execution timeout. - * - * (LITE was previously capped low only because the query also walked the commit - * graph via `history.totalCount`; that expensive field is gone, so the cheap - * pass can batch wide again.) + * Repos per augment request. Even though most of the query is cheap + * `{ totalCount }` connections, each repo also does two server-side git-object + * reads (`.gitattributes` blob + `.github/workflows` tree), and repos with + * releases add the deeply-nested release-asset scan. In practice that pushes + * wide batches past GitHub's 10s GraphQL execution timeout (a 502/504), so + * batches are kept small: 10-15 repos per request. Repos discovery found to + * have zero releases skip the release scan and use the upper bound (LITE); + * repos with releases use the lower bound (FULL). If a chunk still times out, + * `augmentRepoSignals` splits and retries it, so these are safe ceilings. */ -const AUGMENT_CHUNK_FULL = 15; -const AUGMENT_CHUNK_LITE = 50; +const AUGMENT_CHUNK_FULL = 10; +const AUGMENT_CHUNK_LITE = 15; /** How many augment requests are in flight at once (cuts wall-clock ~Nx). */ const AUGMENT_CONCURRENCY = 3; From 9ae4d01cff6be39cd6340ffd8713bff6ee7039ea Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 23:03:36 -0700 Subject: [PATCH 06/11] =?UTF-8?q?refactor(profile):=20two-pass=20crawl=20?= =?UTF-8?q?=E2=80=94=20cheap=20counts=20first,=20verification=20later?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundled augment query did too much per repo (git-object reads, a full commit-history walk, and a release-asset scan) and tripped GitHub's 10s GraphQL execution limit, returning 502/504 on larger orgs. Split augmentation into two passes: - Counts pass (augmentRepoCounts, batches of 15): only indexed totalCount reads (discussions, projects, environments, stars, forks, watchers, packages, rulesets, branch-protection count). These materialize no nodes, so they're cheap and basically can't time out — they fill the page fast. This pass is fatal: if it fails the run fails. - Details pass (augmentRepoDetails, batches of 10 with releases / 15 without): the expensive verification signals — commits (history.totalCount), LFS, workflow files, release-asset bytes, branch-protection detail. Best-effort and non-fatal: a chunk that errors is logged and skipped, and a single repo that times out degrades its fields rather than blocking the run. Both passes run a concurrency-3 worker pool and split-on-timeout (recurse on halves; a lone repo degrades) as a backstop. Commits is restored as a stat, now gathered in the details pass where a timeout safely degrades it to 0. Batches stay at 10-15 repos. Adds per-chunk error observability. --- src/lib/server/profile/analyze.test.ts | 1 + src/lib/server/profile/augment.test.ts | 442 +++++++++++------------- src/lib/server/profile/augment.ts | 276 +++++++++------ src/lib/server/profile/estimate.test.ts | 1 + src/lib/server/profile/insights.test.ts | 1 + src/lib/server/profile/runner.test.ts | 70 +++- src/lib/server/profile/runner.ts | 163 ++++++--- src/lib/server/profile/service.test.ts | 27 +- src/lib/server/profile/service.ts | 3 + src/lib/server/profile/store.test.ts | 1 + src/lib/server/profile/types.ts | 7 + src/routes/profile/[id]/+page.svelte | 2 + 12 files changed, 578 insertions(+), 416 deletions(-) diff --git a/src/lib/server/profile/analyze.test.ts b/src/lib/server/profile/analyze.test.ts index 33d79e1..ca6e412 100644 --- a/src/lib/server/profile/analyze.test.ts +++ b/src/lib/server/profile/analyze.test.ts @@ -30,6 +30,7 @@ function cleanSignals(over: Partial = {}): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index c0a11f6..2ddf615 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -1,12 +1,13 @@ /** - * Tests for per-repo GraphQL signal augmentation. The `gql` client is injected, - * so these exercise the query-variable shaping, field mapping, and — the part - * with real logic — the branch-protection "uses an unmigrated feature" - * derivation, all without a network. + * Tests for per-repo GraphQL signal augmentation, split into the cheap counts + * pass (`augmentRepoCounts`) and the expensive verification pass + * (`augmentRepoDetails`). The `gql` client is injected, so these exercise query + * shaping, field mapping, the branch-protection derivation, and the shared + * resilience (split-on-timeout, degrade, partial-data) without a network. */ import { describe, expect, test } from "bun:test"; import type { graphql } from "@octokit/graphql"; -import { augmentRepoSignals } from "./augment"; +import { augmentRepoCounts, augmentRepoDetails } from "./augment"; import type { DiscoveredRepo } from "./types"; /** A discovered repo fixture (the spread input to augmentation). */ @@ -58,8 +59,8 @@ function rule(flags: RuleFlags = {}) { }; } -/** Build one repository alias node. */ -function node(over: { +/** Build one repository alias node for the cheap counts pass. */ +function countsNode(over: { discussions?: number; projectsV2?: number; environments?: number; @@ -67,20 +68,43 @@ function node(over: { watchers?: number; forks?: number; packages?: number; - rulesets?: number; + /** Repo ruleset count, or `null` for a null (no-access) connection. */ + rulesets?: number | null; + ruleTotal?: number; +}) { + return { + discussions: { totalCount: over.discussions ?? 0 }, + projectsV2: { totalCount: over.projectsV2 ?? 0 }, + environments: { totalCount: over.environments ?? 0 }, + stargazerCount: over.stars ?? 0, + forkCount: over.forks ?? 0, + watchers: { totalCount: over.watchers ?? 0 }, + packages: { totalCount: over.packages ?? 0 }, + rulesets: over.rulesets === null ? null : { totalCount: over.rulesets ?? 0 }, + branchProtectionRules: { totalCount: over.ruleTotal ?? 0 }, + }; +} + +/** Build one repository alias node for the verification details pass. */ +function detailsNode(over: { + commits?: number; /** Raw `.gitattributes` text; omit for no file (null blob). */ lfsAttributes?: string; /** Workflow file names under `.github/workflows`; omit for no dir (null tree). */ workflowFiles?: string[]; /** Asset byte sizes; placed in a single scanned release node (omit = not scanned). */ releaseAssetSizes?: number[]; - ruleTotal?: number; rules?: ReturnType[]; + noDefaultBranch?: boolean; }) { return { - discussions: { totalCount: over.discussions ?? 0 }, - projectsV2: { totalCount: over.projectsV2 ?? 0 }, - environments: { totalCount: over.environments ?? 0 }, + defaultBranchRef: over.noDefaultBranch + ? null + : { target: { history: { totalCount: over.commits ?? 0 } } }, + gitattributes: over.lfsAttributes === undefined ? null : { text: over.lfsAttributes }, + workflows: over.workflowFiles + ? { entries: over.workflowFiles.map((name) => ({ name })) } + : null, // `releases` is present only on a scanned node (mirrors scanReleases: true). ...(over.releaseAssetSizes ? { @@ -89,19 +113,7 @@ function node(over: { }, } : {}), - stargazerCount: over.stars ?? 0, - forkCount: over.forks ?? 0, - watchers: { totalCount: over.watchers ?? 0 }, - packages: { totalCount: over.packages ?? 0 }, - rulesets: { totalCount: over.rulesets ?? 0 }, - gitattributes: over.lfsAttributes === undefined ? null : { text: over.lfsAttributes }, - workflows: over.workflowFiles - ? { entries: over.workflowFiles.map((name) => ({ name })) } - : null, - branchProtectionRules: { - totalCount: over.ruleTotal ?? over.rules?.length ?? 0, - nodes: over.rules ?? [], - }, + branchProtectionRules: { nodes: over.rules ?? [] }, }; } @@ -118,158 +130,73 @@ function mockGql(response: Record) { return { fn, calls }; } -describe("augmentRepoSignals", () => { - test("maps every count signal and preserves the discovered fields", async () => { +describe("augmentRepoCounts", () => { + test("maps every cheap count and preserves the discovered fields", async () => { const { fn } = mockGql({ - r0: node({ + r0: countsNode({ discussions: 3, projectsV2: 2, environments: 4, stars: 41, watchers: 9, + forks: 12, + packages: 6, + rulesets: 4, + ruleTotal: 5, }), }); - const [signals] = await augmentRepoSignals(fn, [ - discovered({ issuesCount: 11, releasesCount: 7 }), - ]); - - // Augmented counts. - expect(signals?.discussionsCount).toBe(3); - expect(signals?.projectsV2Count).toBe(2); - expect(signals?.environmentsCount).toBe(4); - expect(signals?.stargazerCount).toBe(41); - expect(signals?.watcherCount).toBe(9); - // Discovered spine preserved (incl. the content counts from discovery). - expect(signals?.nameWithOwner).toBe("acme/widget"); - expect(signals?.diskUsageKb).toBe(1234); - expect(signals?.issuesCount).toBe(11); - expect(signals?.releasesCount).toBe(7); - }); - - test("maps the packages count", async () => { - const { fn } = mockGql({ r0: node({ packages: 6 }) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.packagesCount).toBe(6); - }); - - test("maps the fork count and repository ruleset count", async () => { - const { fn } = mockGql({ r0: node({ forks: 12, rulesets: 4 }) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.forkCount).toBe(12); - expect(signals?.rulesetCount).toBe(4); + const [s] = await augmentRepoCounts(fn, [discovered({ issuesCount: 11, releasesCount: 7 })]); + + expect(s?.discussionsCount).toBe(3); + expect(s?.projectsV2Count).toBe(2); + expect(s?.environmentsCount).toBe(4); + expect(s?.stargazerCount).toBe(41); + expect(s?.watcherCount).toBe(9); + expect(s?.forkCount).toBe(12); + expect(s?.packagesCount).toBe(6); + expect(s?.rulesetCount).toBe(4); + expect(s?.branchProtectionRuleCount).toBe(5); + // Discovered spine preserved. + expect(s?.nameWithOwner).toBe("acme/widget"); + expect(s?.diskUsageKb).toBe(1234); + expect(s?.issuesCount).toBe(11); + expect(s?.releasesCount).toBe(7); + // Verification fields are left at defaults for the details pass. + expect(s?.commitsCount).toBe(0); + expect(s?.usesLfs).toBe(false); + expect(s?.workflowFileCount).toBe(0); + expect(s?.releaseAssetBytes).toBe(0); + expect(s?.branchProtectionRulesUsingUnmigratedFeatures).toBe(0); }); test("treats a null rulesets connection as zero (schema allows null)", async () => { - // RepositoryRulesetConnection is nullable in the schema; a viewer without - // ruleset read access gets null rather than a connection. - const n = node({}); - (n as { rulesets: unknown }).rulesets = null; - const { fn } = mockGql({ r0: n }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.rulesetCount).toBe(0); - }); - - test("detects Git LFS from a .gitattributes with a filter=lfs entry", async () => { - const { fn } = mockGql({ - r0: node({ lfsAttributes: "*.psd filter=lfs diff=lfs merge=lfs -text\n" }), - }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.usesLfs).toBe(true); + const { fn } = mockGql({ r0: countsNode({ rulesets: null }) }); + const [s] = await augmentRepoCounts(fn, [discovered()]); + expect(s?.rulesetCount).toBe(0); }); - test("a .gitattributes without filter=lfs is not flagged as LFS", async () => { - const { fn } = mockGql({ r0: node({ lfsAttributes: "*.txt text eol=lf\n" }) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.usesLfs).toBe(false); + test("does not pass a $rules variable (counts query has no rule detail)", async () => { + const { fn, calls } = mockGql({ r0: countsNode({}) }); + await augmentRepoCounts(fn, [discovered({ nameWithOwner: "o/a", name: "a" })]); + expect(calls[0]).toMatchObject({ o0: "o", n0: "a" }); + expect(calls[0]).not.toHaveProperty("rules"); }); - test("an absent .gitattributes means no LFS", async () => { - const { fn } = mockGql({ r0: node({}) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.usesLfs).toBe(false); - expect(signals?.packagesCount).toBe(0); - }); - - test("sums release asset bytes across releases", async () => { - const { fn } = mockGql({ r0: node({ releaseAssetSizes: [1000, 2500, 500] }) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.releaseAssetBytes).toBe(4000); - }); - - test("a repo with no releases has zero release asset bytes", async () => { - const { fn } = mockGql({ r0: node({}) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.releaseAssetBytes).toBe(0); - }); - - test("omits the release-asset scan from the query when scanReleases is false", async () => { - let query = ""; - const fn = (async (q: string) => { - query = q; - return { r0: node({}) }; - }) as unknown as typeof graphql; - - await augmentRepoSignals(fn, [discovered()], { scanReleases: false }); - expect(query).not.toContain("releaseAssets"); - }); - - test("includes the release-asset scan from the query when scanReleases is true", async () => { - let query = ""; - const fn = (async (q: string) => { - query = q; - return { r0: node({ releaseAssetSizes: [10] }) }; - }) as unknown as typeof graphql; - - await augmentRepoSignals(fn, [discovered()], { scanReleases: true }); - expect(query).toContain("releaseAssets"); - }); - - test("counts only .yml/.yaml workflow files under .github/workflows", async () => { - const { fn } = mockGql({ - r0: node({ workflowFiles: ["ci.yml", "release.yaml", "README.md", "dependabot.yml"] }), - }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.workflowFileCount).toBe(3); - }); - - test("a repo with no .github/workflows tree has zero workflow files", async () => { - const { fn } = mockGql({ r0: node({}) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.workflowFileCount).toBe(0); - }); - - test("derives owner/name per alias and caps the rules page", async () => { - const { fn, calls } = mockGql({ r0: node({}), r1: node({}) }); - - await augmentRepoSignals(fn, [ - discovered({ name: "widget", nameWithOwner: "octo-org/widget" }), - discovered({ name: "gadget", nameWithOwner: "octo-org/gadget" }), - ]); - - expect(calls[0]).toMatchObject({ - rules: 50, - o0: "octo-org", - n0: "widget", - o1: "octo-org", - n1: "gadget", - }); - }); - - test("profiles a whole chunk in a single request, in input order", async () => { + test("profiles a whole chunk in one request, in input order", async () => { const { fn, calls } = mockGql({ - r0: node({ discussions: 1 }), - r1: node({ discussions: 2 }), - r2: node({ discussions: 3 }), + r0: countsNode({ discussions: 1 }), + r1: countsNode({ discussions: 2 }), + r2: countsNode({ discussions: 3 }), }); - const signals = await augmentRepoSignals(fn, [ + const signals = await augmentRepoCounts(fn, [ discovered({ nameWithOwner: "o/a", name: "a" }), discovered({ nameWithOwner: "o/b", name: "b" }), discovered({ nameWithOwner: "o/c", name: "c" }), ]); - expect(calls).toHaveLength(1); // one batched request for three repos + expect(calls).toHaveLength(1); expect(signals.map((s) => [s.nameWithOwner, s.discussionsCount])).toEqual([ ["o/a", 1], ["o/b", 2], @@ -277,88 +204,37 @@ describe("augmentRepoSignals", () => { ]); }); - test("counts only branch protection rules that use an unmigrated feature", async () => { - const { fn } = mockGql({ - r0: node({ - ruleTotal: 4, - rules: [ - rule({ allowsForcePushes: true }), // unmigrated - rule({ bypassPullRequest: 2 }), // unmigrated (bypass actors) - rule({ requireLastPushApproval: true }), // unmigrated - rule({}), // only migrated features → not counted - ], - }), - }); - - const [signals] = await augmentRepoSignals(fn, [discovered()]); - - expect(signals?.branchProtectionRuleCount).toBe(4); - expect(signals?.branchProtectionRulesUsingUnmigratedFeatures).toBe(3); - }); - - test("treats a rule with no unmigrated features as fully migratable", async () => { - const { fn } = mockGql({ r0: node({ ruleTotal: 1, rules: [rule({})] }) }); - - const [signals] = await augmentRepoSignals(fn, [discovered()]); - - expect(signals?.branchProtectionRuleCount).toBe(1); - expect(signals?.branchProtectionRulesUsingUnmigratedFeatures).toBe(0); - }); - - test("flags each unmigrated feature individually", async () => { - const cases = [ - rule({ requiresDeployments: true }), - rule({ lockBranch: true }), - rule({ blocksCreations: true }), - rule({ bypassForcePush: 1 }), - ]; - for (const r of cases) { - const { fn } = mockGql({ r0: node({ ruleTotal: 1, rules: [r] }) }); - const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals?.branchProtectionRulesUsingUnmigratedFeatures).toBe(1); - } - }); - test("returns no signals for an empty chunk without calling gql", async () => { const { fn, calls } = mockGql({}); - - const signals = await augmentRepoSignals(fn, []); - + const signals = await augmentRepoCounts(fn, []); expect(signals).toEqual([]); expect(calls).toHaveLength(0); }); - test("degrades a null alias to zeroed signals (keeps the discovery spine)", async () => { - const { fn } = mockGql({ r0: node({ discussions: 5 }), r1: null }); - - const signals = await augmentRepoSignals(fn, [ + test("degrades a null alias to zeroed counts (keeps the discovery spine)", async () => { + const { fn } = mockGql({ r0: countsNode({ discussions: 5 }), r1: null }); + const signals = await augmentRepoCounts(fn, [ discovered({ nameWithOwner: "o/ok", name: "ok" }), discovered({ nameWithOwner: "o/gone", name: "gone", issuesCount: 3 }), ]); - expect(signals[0]?.discussionsCount).toBe(5); - // The inaccessible repo keeps its discovered fields but zeroes augment counts. expect(signals[1]?.nameWithOwner).toBe("o/gone"); - expect(signals[1]?.issuesCount).toBe(3); - expect(signals[1]?.discussionsCount).toBe(0); - expect(signals[1]?.branchProtectionRuleCount).toBe(0); + expect(signals[1]?.issuesCount).toBe(3); // spine kept + expect(signals[1]?.discussionsCount).toBe(0); // augment zeroed }); test("recovers partial data when the request errors with a data payload", async () => { - // GraphQL returns partial data + errors when one alias is inaccessible; the - // client surfaces that as a throw carrying `.data`. Recover the good repos. const err = Object.assign(new Error("Could not resolve to a Repository"), { - data: { r0: node({ discussions: 8 }), r1: null }, + data: { r0: countsNode({ discussions: 8 }), r1: null }, }); const fn = (async () => { throw err; }) as unknown as typeof graphql; - const signals = await augmentRepoSignals(fn, [ + const signals = await augmentRepoCounts(fn, [ discovered({ nameWithOwner: "o/a", name: "a" }), discovered({ nameWithOwner: "o/b", name: "b" }), ]); - expect(signals[0]?.discussionsCount).toBe(8); expect(signals[1]?.discussionsCount).toBe(0); }); @@ -367,50 +243,143 @@ describe("augmentRepoSignals", () => { const fn = (async () => { throw new Error("network down"); }) as unknown as typeof graphql; - - await expect(augmentRepoSignals(fn, [discovered()])).rejects.toThrow(/network down/); + await expect(augmentRepoCounts(fn, [discovered()])).rejects.toThrow(/network down/); }); test("splits a timed-out chunk and retries the smaller halves", async () => { - // A `gql` double that 502s (GitHub's timeout) for any request wider than 2 - // repos, and succeeds otherwise. Records the size of each attempted request. + // 502s for any request wider than 2 repos; succeeds otherwise. const sizes: number[] = []; const fn = (async (_q: string, vars: Record) => { const size = Object.keys(vars).filter((k) => /^o\d+$/.test(k)).length; sizes.push(size); if (size > 2) throw Object.assign(new Error("Bad Gateway"), { status: 502 }); const res: Record = {}; - for (let i = 0; i < size; i++) res[`r${i}`] = node({}); + for (let i = 0; i < size; i++) res[`r${i}`] = countsNode({}); return res; }) as unknown as typeof graphql; const repos = ["a", "b", "c", "d"].map((n) => discovered({ name: n, nameWithOwner: `o/${n}` })); - const signals = await augmentRepoSignals(fn, repos); - - // All four repos come back, in order, despite the initial timeout. + const signals = await augmentRepoCounts(fn, repos); expect(signals.map((s) => s.nameWithOwner)).toEqual(["o/a", "o/b", "o/c", "o/d"]); - // 4 timed out → split into 2 + 2, each of which succeeds. - expect(sizes).toEqual([4, 2, 2]); + expect(sizes).toEqual([4, 2, 2]); // 4 timed out → 2 + 2 }); - test("degrades a single repo that keeps timing out to zeroed signals", async () => { - // Always 502s, so splitting bottoms out at one repo each, which degrades. + test("degrades a single repo that keeps timing out to zeroed counts", async () => { const fn = (async () => { throw Object.assign(new Error("Gateway Timeout"), { status: 504 }); }) as unknown as typeof graphql; - const repos = [ discovered({ name: "a", nameWithOwner: "o/a", issuesCount: 7 }), discovered({ name: "b", nameWithOwner: "o/b" }), ]; - const signals = await augmentRepoSignals(fn, repos); - + const signals = await augmentRepoCounts(fn, repos); expect(signals).toHaveLength(2); - // Discovery spine preserved; augment counts degraded to zero. - expect(signals[0]?.nameWithOwner).toBe("o/a"); - expect(signals[0]?.issuesCount).toBe(7); - expect(signals[0]?.discussionsCount).toBe(0); - expect(signals[1]?.nameWithOwner).toBe("o/b"); + expect(signals[0]?.issuesCount).toBe(7); // spine kept + expect(signals[0]?.discussionsCount).toBe(0); // degraded + }); +}); + +describe("augmentRepoDetails", () => { + test("maps the commit count from the default branch history", async () => { + const { fn } = mockGql({ r0: detailsNode({ commits: 512 }) }); + const [d] = await augmentRepoDetails(fn, [discovered()]); + expect(d?.commitsCount).toBe(512); + }); + + test("treats a missing default branch as zero commits", async () => { + const { fn } = mockGql({ r0: detailsNode({ noDefaultBranch: true }) }); + const [d] = await augmentRepoDetails(fn, [discovered({ isEmpty: true })]); + expect(d?.commitsCount).toBe(0); + }); + + test("detects Git LFS from a .gitattributes with a filter=lfs entry", async () => { + const { fn } = mockGql({ + r0: detailsNode({ lfsAttributes: "*.psd filter=lfs diff=lfs merge=lfs -text\n" }), + }); + const [d] = await augmentRepoDetails(fn, [discovered()]); + expect(d?.usesLfs).toBe(true); + }); + + test("a .gitattributes without filter=lfs is not flagged, and an absent one is false", async () => { + const a = mockGql({ r0: detailsNode({ lfsAttributes: "*.txt text eol=lf\n" }) }); + expect((await augmentRepoDetails(a.fn, [discovered()]))[0]?.usesLfs).toBe(false); + const b = mockGql({ r0: detailsNode({}) }); + expect((await augmentRepoDetails(b.fn, [discovered()]))[0]?.usesLfs).toBe(false); + }); + + test("counts only .yml/.yaml workflow files under .github/workflows", async () => { + const { fn } = mockGql({ + r0: detailsNode({ workflowFiles: ["ci.yml", "release.yaml", "README.md", "dependabot.yml"] }), + }); + const [d] = await augmentRepoDetails(fn, [discovered()]); + expect(d?.workflowFileCount).toBe(3); + }); + + test("sums release asset bytes when scanning, passing $rules", async () => { + const { fn, calls } = mockGql({ r0: detailsNode({ releaseAssetSizes: [1000, 2500, 500] }) }); + const [d] = await augmentRepoDetails(fn, [discovered()], { scanReleases: true }); + expect(d?.releaseAssetBytes).toBe(4000); + expect(calls[0]).toMatchObject({ rules: 50 }); + }); + + test("omits the release-asset scan from the query when scanReleases is false", async () => { + let query = ""; + const fn = (async (q: string) => { + query = q; + return { r0: detailsNode({}) }; + }) as unknown as typeof graphql; + await augmentRepoDetails(fn, [discovered()], { scanReleases: false }); + expect(query).not.toContain("releaseAssets"); + }); + + test("includes the release-asset scan when scanReleases is true", async () => { + let query = ""; + const fn = (async (q: string) => { + query = q; + return { r0: detailsNode({ releaseAssetSizes: [10] }) }; + }) as unknown as typeof graphql; + await augmentRepoDetails(fn, [discovered()], { scanReleases: true }); + expect(query).toContain("releaseAssets"); + }); + + test("counts only branch protection rules that use an unmigrated feature", async () => { + const { fn } = mockGql({ + r0: detailsNode({ + rules: [ + rule({ allowsForcePushes: true }), // unmigrated + rule({ bypassPullRequest: 2 }), // unmigrated (bypass actors) + rule({ requireLastPushApproval: true }), // unmigrated + rule({}), // only migrated features → not counted + ], + }), + }); + const [d] = await augmentRepoDetails(fn, [discovered()]); + expect(d?.branchProtectionRulesUsingUnmigratedFeatures).toBe(3); + }); + + test("flags each unmigrated feature individually", async () => { + const cases = [ + rule({ requiresDeployments: true }), + rule({ lockBranch: true }), + rule({ blocksCreations: true }), + rule({ bypassForcePush: 1 }), + ]; + for (const r of cases) { + const { fn } = mockGql({ r0: detailsNode({ rules: [r] }) }); + const [d] = await augmentRepoDetails(fn, [discovered()]); + expect(d?.branchProtectionRulesUsingUnmigratedFeatures).toBe(1); + } + }); + + test("degrades a null alias to zeroed details (keeps the repo)", async () => { + const { fn } = mockGql({ r0: detailsNode({ commits: 5 }), r1: null }); + const details = await augmentRepoDetails(fn, [ + discovered({ nameWithOwner: "o/ok", name: "ok" }), + discovered({ nameWithOwner: "o/gone", name: "gone" }), + ]); + expect(details[0]?.commitsCount).toBe(5); + expect(details[1]?.nameWithOwner).toBe("o/gone"); + expect(details[1]?.commitsCount).toBe(0); }); test("treats a timeout message (not just a status) as splittable", async () => { @@ -419,16 +388,15 @@ describe("augmentRepoSignals", () => { const size = Object.keys(vars).filter((k) => /^o\d+$/.test(k)).length; calls += 1; if (size > 1) throw new Error("Something went wrong while executing your query"); - return { r0: node({ discussions: 1 }) }; + return { r0: detailsNode({ commits: 1 }) }; }) as unknown as typeof graphql; const repos = [ discovered({ name: "a", nameWithOwner: "o/a" }), discovered({ name: "b", nameWithOwner: "o/b" }), ]; - const signals = await augmentRepoSignals(fn, repos); - - expect(signals.map((s) => s.discussionsCount)).toEqual([1, 1]); - expect(calls).toBe(3); // 2 (timeout) → 1 + 1 (ok) + const details = await augmentRepoDetails(fn, repos); + expect(details.map((d) => d.commitsCount)).toEqual([1, 1]); + expect(calls).toBe(3); // 2 (timeout) → 1 + 1 }); }); diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index 6930305..9a03288 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -46,67 +46,85 @@ interface BranchProtectionRuleNode { bypassPullRequestAllowances: { totalCount: number }; } -/** Raw shape of one repository alias in the batched signals query. */ -interface RepoSignalsNode { +/** Cheap per-repo counts: pure `{ totalCount }` / scalar fields, no git reads + * and no node materialization — indexed reads that batch safely. */ +interface RepoCountsNode { discussions: { totalCount: number }; projectsV2: { totalCount: number }; environments: { totalCount: number }; - /** Present only when the release-asset scan is requested (`scanReleases`). */ - releases?: { - nodes: { releaseAssets: { nodes: { size: number }[] } }[]; - }; stargazerCount: number; forkCount: number; watchers: { totalCount: number }; packages: { totalCount: number }; /** Repo-level rulesets. The connection is nullable in the schema (no `!`). */ rulesets: { totalCount: number } | null; + branchProtectionRules: { totalCount: number }; +} + +/** + * Expensive per-repo verification fields: a commit-graph walk + * (`history.totalCount`), two git-object reads (`.gitattributes`, + * `.github/workflows`), branch-protection rule detail, and the release-asset + * scan. Gathered in a second pass so a timeout here can't block the counts. + */ +interface RepoDetailsNode { + defaultBranchRef: { target: { history: { totalCount: number } } | null } | null; /** Root `.gitattributes` on the default branch (`Blob`), or null if absent. */ gitattributes: { text: string | null } | null; /** `.github/workflows` dir on the default branch (`Tree`), or null if absent. */ workflows: { entries: { name: string }[] } | null; - branchProtectionRules: { - totalCount: number; - nodes: BranchProtectionRuleNode[]; + /** Present only when the release-asset scan is requested (`scanReleases`). */ + releases?: { + nodes: { releaseAssets: { nodes: { size: number }[] } }[]; }; + branchProtectionRules: { nodes: BranchProtectionRuleNode[] }; } -/** The aliased query response: one (possibly null) node per requested repo. */ -type BatchSignalsResult = Record; +/** The verification fields the details pass produces, merged onto the counts. */ +export interface RepoDetails { + nameWithOwner: string; + commitsCount: number; + branchProtectionRulesUsingUnmigratedFeatures: number; + usesLfs: boolean; + workflowFileCount: number; + releaseAssetBytes: number; +} /** - * The per-repo selection. Almost everything here is a scalar or a - * `{ totalCount }`-only connection: per GitHub's node-limit rules those don't - * materialize `nodes`/`edges`, so they're indexed reads that stay cheap even - * when many repos are aliased into one request. The one exception is the - * release-asset scan (`releases{...releaseAssets}`), a deeply-nested walk that - * materializes nodes — included only when `scanReleases` is set so repos - * discovery already knows have zero releases can be batched far wider. - * - * Deliberately omits commit count (`history.totalCount`): unlike the other - * counts it is NOT indexed — it walks the whole commit graph — making it the - * single most expensive resolver and the main cause of the 10s query timeout - * when repos are aliased, while being pure scale decoration (history migrates - * wholesale). + * Cheap counts fragment — every field is an indexed `totalCount` or a scalar, + * so it materializes no nodes and does no git-object reads. This is the first + * pass: it fills the page's per-repo counts fast and is very unlikely to hit + * GitHub's 10s timeout even at the batch ceiling. */ -function signalsFragment(scanReleases: boolean): string { - const releases = scanReleases - ? `releases(first: ${RELEASES_SCANNED}) { nodes { releaseAssets(first: ${ASSETS_SCANNED}) { nodes { size } } } }` - : ""; - return `fragment Sig on Repository { +const COUNTS_FRAGMENT = `fragment Counts on Repository { discussions { totalCount } projectsV2 { totalCount } environments { totalCount } - ${releases} stargazerCount forkCount watchers { totalCount } packages { totalCount } rulesets(first: 1, includeParents: false) { totalCount } + branchProtectionRules(first: 1) { totalCount } +}`; + +/** + * Verification fragment — the expensive parts: the commit-graph walk + * (`history.totalCount`), two git-object reads (`.gitattributes`, + * `.github/workflows`), branch-protection rule detail, and (when `scanReleases`) + * the release-asset scan. Run as a second pass so a timeout here degrades a + * repo's verification rather than blocking its counts. + */ +function detailsFragment(scanReleases: boolean): string { + const releases = scanReleases + ? `releases(first: ${RELEASES_SCANNED}) { nodes { releaseAssets(first: ${ASSETS_SCANNED}) { nodes { size } } } }` + : ""; + return `fragment Details on Repository { + defaultBranchRef { target { ... on Commit { history(first: 1) { totalCount } } } } gitattributes: object(expression: "HEAD:.gitattributes") { ... on Blob { text } } workflows: object(expression: "HEAD:.github/workflows") { ... on Tree { entries { name } } } + ${releases} branchProtectionRules(first: $rules) { - totalCount nodes { allowsForcePushes requiresDeployments @@ -120,15 +138,25 @@ function signalsFragment(scanReleases: boolean): string { }`; } -/** Build the aliased query for a chunk of `n` repos (aliases `r0…r{n-1}`). */ -function buildBatchQuery(n: number, scanReleases: boolean): string { - const varDefs = ["$rules: Int!"]; +/** + * Build an aliased query (`r0: repository(...) {...} …`) over `fragmentName` for + * a chunk of `n` repos. `$rules` is declared only when the fragment uses it + * (the details pass), so the counts query carries no unused variable. + */ +function buildAliasedQuery( + n: number, + fragmentName: string, + fragment: string, + withRules: boolean, +): string { + const varDefs = withRules ? ["$rules: Int!"] : []; const fields: string[] = []; for (let i = 0; i < n; i++) { varDefs.push(`$o${i}: String!`, `$n${i}: String!`); - fields.push(`r${i}: repository(owner: $o${i}, name: $n${i}) { ...Sig }`); + fields.push(`r${i}: repository(owner: $o${i}, name: $n${i}) { ...${fragmentName} }`); } - return `query batchSignals(${varDefs.join(", ")}) {\n${fields.join("\n")}\n}\n${signalsFragment(scanReleases)}`; + const head = varDefs.length > 0 ? `(${varDefs.join(", ")})` : ""; + return `query batch${head} {\n${fields.join("\n")}\n}\n${fragment}`; } /** Split `owner/name` into its owner; falls back to the whole string. */ @@ -165,8 +193,8 @@ function usesLfsAttributes(gitattributes: { text: string | null } | null): boole return text != null && /filter=lfs/.test(text); } -/** Sum the byte size of every scanned release asset in a repo node. */ -function sumReleaseAssetBytes(releases: RepoSignalsNode["releases"]): number { +/** Sum the byte size of every scanned release asset in a details node. */ +function sumReleaseAssetBytes(releases: RepoDetailsNode["releases"]): number { if (!releases) return 0; let bytes = 0; for (const release of releases.nodes) { @@ -181,53 +209,52 @@ function countWorkflowFiles(workflows: { entries: { name: string }[] } | null): return workflows.entries.filter((e) => /\.ya?ml$/i.test(e.name)).length; } -/** Map a repo + its (possibly null) augment node to full `RepoSignals`. */ -function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSignals { - // A null node means the repo was inaccessible on this pass (permissions edge, - // or deleted between passes). It was discoverable, so keep the discovery spine - // and degrade the augment counts to zero rather than failing the run. - if (!node) { - return { - ...repo, - discussionsCount: 0, - projectsV2Count: 0, - environmentsCount: 0, - stargazerCount: 0, - watcherCount: 0, - forkCount: 0, - rulesetCount: 0, - branchProtectionRuleCount: 0, - branchProtectionRulesUsingUnmigratedFeatures: 0, - packagesCount: 0, - usesLfs: false, - releaseAssetBytes: 0, - workflowFileCount: 0, - }; - } +/** + * Map a repo + its (possibly null) counts node to `RepoSignals`, with the + * verification fields at their defaults — the details pass fills those in. A + * null node (inaccessible repo) keeps the discovery spine with zeroed counts. + */ +function countsToSignals(repo: DiscoveredRepo, node: RepoCountsNode | null): RepoSignals { return { ...repo, - discussionsCount: node.discussions.totalCount, - projectsV2Count: node.projectsV2.totalCount, - environmentsCount: node.environments.totalCount, - stargazerCount: node.stargazerCount, - watcherCount: node.watchers.totalCount, - forkCount: node.forkCount, - rulesetCount: node.rulesets?.totalCount ?? 0, - branchProtectionRuleCount: node.branchProtectionRules.totalCount, - branchProtectionRulesUsingUnmigratedFeatures: - node.branchProtectionRules.nodes.filter(usesUnmigratedFeature).length, - packagesCount: node.packages.totalCount, - usesLfs: usesLfsAttributes(node.gitattributes), - releaseAssetBytes: sumReleaseAssetBytes(node.releases), - workflowFileCount: countWorkflowFiles(node.workflows), + // Verification fields — defaults until the details pass merges them in. + commitsCount: 0, + branchProtectionRulesUsingUnmigratedFeatures: 0, + usesLfs: false, + workflowFileCount: 0, + releaseAssetBytes: 0, + // Cheap counts. + discussionsCount: node?.discussions.totalCount ?? 0, + projectsV2Count: node?.projectsV2.totalCount ?? 0, + environmentsCount: node?.environments.totalCount ?? 0, + stargazerCount: node?.stargazerCount ?? 0, + watcherCount: node?.watchers.totalCount ?? 0, + forkCount: node?.forkCount ?? 0, + packagesCount: node?.packages.totalCount ?? 0, + rulesetCount: node?.rulesets?.totalCount ?? 0, + branchProtectionRuleCount: node?.branchProtectionRules.totalCount ?? 0, + }; +} + +/** Map a repo + its (possibly null) details node to the verification fields. */ +function toDetails(repo: DiscoveredRepo, node: RepoDetailsNode | null): RepoDetails { + return { + nameWithOwner: repo.nameWithOwner, + commitsCount: node?.defaultBranchRef?.target?.history.totalCount ?? 0, + branchProtectionRulesUsingUnmigratedFeatures: node + ? node.branchProtectionRules.nodes.filter(usesUnmigratedFeature).length + : 0, + usesLfs: usesLfsAttributes(node?.gitattributes ?? null), + workflowFileCount: countWorkflowFiles(node?.workflows ?? null), + releaseAssetBytes: sumReleaseAssetBytes(node?.releases), }; } /** Recover partial `data` from a GraphQL error that carries it, else null. */ -function partialDataFromError(err: unknown): BatchSignalsResult | null { +function partialDataFromError(err: unknown): Record | null { if (err && typeof err === "object" && "data" in err) { const data = (err as { data?: unknown }).data; - if (data && typeof data === "object") return data as BatchSignalsResult; + if (data && typeof data === "object") return data as Record; } return null; } @@ -259,61 +286,102 @@ export interface AugmentOptions { } /** - * Enrich a chunk of discovered repositories with their per-repo signals in a - * single aliased GraphQL request. + * Shared batched-augment control flow: one aliased request over a chunk of + * repos, with three layers of resilience — + * - a GitHub processing timeout (502/504) splits the chunk and recurses on the + * halves until each query is cheap enough to evaluate in the 10s window; + * - a single repo that still times out degrades via `toOut(repo, null)` + * (keeping its discovery spine) so one pathological repo can't fail the run; + * - a partial-data error (one inaccessible repo in the chunk) is recovered so + * the accessible repos still map. * - * @param gql Injected GraphQL client (`createSingleClient(...).graphql`). - * @param repos The chunk of repositories (from discovery) to augment. - * @param opts Per-request options (e.g. whether to scan release assets). - * @returns One `RepoSignals` per input repo, in input order. + * `buildQuery` and `toOut` specialize it for the counts vs. details passes. */ -export async function augmentRepoSignals( +async function batchedAugment( gql: typeof graphql, repos: DiscoveredRepo[], - opts: AugmentOptions = {}, -): Promise { + buildQuery: (n: number) => string, + toOut: (repo: DiscoveredRepo, node: TNode | null) => TOut, + extraVars: Record = {}, +): Promise { if (repos.length === 0) return []; - const variables: Record = { rules: MAX_RULES }; + const variables: Record = { ...extraVars }; repos.forEach((repo, i) => { variables[`o${i}`] = ownerOf(repo.nameWithOwner); variables[`n${i}`] = repo.name; }); - let data: BatchSignalsResult; + let data: Record; try { - data = await gql( - buildBatchQuery(repos.length, opts.scanReleases ?? true), - variables, - ); + data = await gql>(buildQuery(repos.length), variables); } catch (err) { - // A GitHub processing timeout (502/504): the query was too expensive to - // evaluate in GitHub's 10s window. Splitting the chunk makes each query - // cheaper; recurse on the halves until it fits. A single repo that still - // times out degrades to zeroed augment signals (keeping its discovery - // spine) so one pathological repo can't fail the whole crawl. if (isTimeoutError(err)) { if (repos.length === 1) { const only = repos[0]; console.warn( `[profile] augment timed out for a single repo${only ? ` (${only.nameWithOwner})` : ""}; recording it with zeroed signals`, ); - return only ? [toSignals(only, null)] : []; + return only ? [toOut(only, null)] : []; } const mid = Math.ceil(repos.length / 2); console.warn( `[profile] augment timed out for ${repos.length} repos; splitting ${mid} + ${repos.length - mid} and retrying`, ); - const left = await augmentRepoSignals(gql, repos.slice(0, mid), opts); - const right = await augmentRepoSignals(gql, repos.slice(mid), opts); + const left = await batchedAugment(gql, repos.slice(0, mid), buildQuery, toOut, extraVars); + const right = await batchedAugment(gql, repos.slice(mid), buildQuery, toOut, extraVars); return [...left, ...right]; } - // A chunk with one inaccessible repo errors but still carries partial data; - // recover it so the accessible repos in the chunk still get profiled. const partial = partialDataFromError(err); if (!partial) throw err; - data = partial; + data = partial as Record; } - return repos.map((repo, i) => toSignals(repo, data[`r${i}`] ?? null)); + return repos.map((repo, i) => toOut(repo, data[`r${i}`] ?? null)); +} + +/** + * Pass 1 — gather each repo's cheap counts in one aliased request per chunk. + * The returned `RepoSignals` have their verification fields at defaults; the + * details pass fills those in. + * + * @param gql Injected GraphQL client (`createSingleClient(...).graphql`). + * @param repos The chunk of repositories (from discovery) to count. + * @returns One `RepoSignals` per input repo, in input order. + */ +export function augmentRepoCounts( + gql: typeof graphql, + repos: DiscoveredRepo[], +): Promise { + return batchedAugment( + gql, + repos, + (n) => buildAliasedQuery(n, "Counts", COUNTS_FRAGMENT, false), + countsToSignals, + ); +} + +/** + * Pass 2 — gather each repo's expensive verification details (commit count, LFS + * usage, workflow presence, release-asset bytes, branch-protection detail) in + * one aliased request per chunk. + * + * @param gql Injected GraphQL client. + * @param repos The chunk of repositories to verify. + * @param opts Per-request options (e.g. whether to scan release assets). + * @returns One `RepoDetails` per input repo, in input order. + */ +export function augmentRepoDetails( + gql: typeof graphql, + repos: DiscoveredRepo[], + opts: AugmentOptions = {}, +): Promise { + const scanReleases = opts.scanReleases ?? true; + return batchedAugment( + gql, + repos, + (n) => buildAliasedQuery(n, "Details", detailsFragment(scanReleases), true), + toDetails, + { rules: MAX_RULES }, + ); } diff --git a/src/lib/server/profile/estimate.test.ts b/src/lib/server/profile/estimate.test.ts index ee8e000..68401aa 100644 --- a/src/lib/server/profile/estimate.test.ts +++ b/src/lib/server/profile/estimate.test.ts @@ -28,6 +28,7 @@ function signals(diskUsageKb: number | null): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/insights.test.ts b/src/lib/server/profile/insights.test.ts index 14b895f..7994aee 100644 --- a/src/lib/server/profile/insights.test.ts +++ b/src/lib/server/profile/insights.test.ts @@ -33,6 +33,7 @@ function signals(over: Partial = {}): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index 88b3930..2afe9b9 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -7,6 +7,7 @@ import { beforeEach, describe, expect, test } from "bun:test"; import { initStore } from "$lib/server/core/db"; import { DOMAIN_STORES } from "$lib/server/registry"; +import type { RepoDetails } from "./augment"; import { type ProfileRunnerDeps, runProfile } from "./runner"; import { getProfileRun, getRunRepoProfiles } from "./store"; import { @@ -48,6 +49,7 @@ function discovered(name: string, over: Partial = {}): Discovere function signalsFor(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { return { ...repo, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -65,7 +67,32 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo }; } -/** Deps whose discover returns the given repos and augment maps each chunk by name. */ +/** Counts-pass signals (verification fields defaulted, as pass 1 produces). */ +function countsSignals(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { + return { + ...signalsFor(repo, over), + commitsCount: 0, + branchProtectionRulesUsingUnmigratedFeatures: 0, + usesLfs: false, + workflowFileCount: 0, + releaseAssetBytes: 0, + }; +} + +/** Verification details for a repo (the pass-2 fake), derived from `signalsFor`. */ +function detailsFor(repo: DiscoveredRepo, over: Partial = {}): RepoDetails { + const s = signalsFor(repo, over); + return { + nameWithOwner: repo.nameWithOwner, + commitsCount: s.commitsCount, + branchProtectionRulesUsingUnmigratedFeatures: s.branchProtectionRulesUsingUnmigratedFeatures, + usesLfs: s.usesLfs, + workflowFileCount: s.workflowFileCount, + releaseAssetBytes: s.releaseAssetBytes, + }; +} + +/** Deps whose discover returns the given repos and the two passes map by name. */ function deps( repos: DiscoveredRepo[], augmentOver: Record> = {}, @@ -74,7 +101,8 @@ function deps( ): ProfileRunnerDeps { return { discover: async (): Promise => ({ org: "acme", total: repos.length, repos }), - augment: async (_gql, chunk) => chunk.map((r) => signalsFor(r, augmentOver[r.name] ?? {})), + augmentCounts: async (_gql, c) => c.map((r) => countsSignals(r, augmentOver[r.name] ?? {})), + augmentDetails: async (_gql, c) => c.map((r) => detailsFor(r, augmentOver[r.name] ?? {})), getOrgRulesetCount: async () => rulesetCount, getOrgResources: async () => ({ ...ZERO_ORG_RESOURCES, ...orgResources }), }; @@ -140,7 +168,8 @@ describe("runProfile", () => { onProgress?.({ org: "acme", discovered: 2, total: 2, page: 1 }); return { org: "acme", total: 2, repos }; }, - augment: async (_gql, c) => c.map((r) => signalsFor(r)), + augmentCounts: async (_gql, c) => c.map((r) => countsSignals(r)), + augmentDetails: async (_gql, c) => c.map((r) => detailsFor(r)), getOrgRulesetCount: async () => 0, getOrgResources: async () => ZERO_ORG_RESOURCES, }; @@ -190,7 +219,9 @@ describe("runProfile", () => { deps(repos), ); - expect(progress).toEqual([ + // Pass 1 emits one per-repo frame as each repo's counts are recorded. (The + // details pass adds repo-less nudges, filtered out here.) + expect(progress.filter((p) => p.repo !== "")).toEqual([ { runId: "r", profiled: 1, total: 3, repo: "acme/a" }, { runId: "r", profiled: 2, total: 3, repo: "acme/b" }, { runId: "r", profiled: 3, total: 3, repo: "acme/c" }, @@ -215,7 +246,6 @@ describe("runProfile", () => { discover: async () => { throw new Error("Organization 'acme' not found or not accessible"); }, - augment: async (_gql, chunk) => chunk.map((r) => signalsFor(r)), }; const run = await runProfile( @@ -229,13 +259,13 @@ describe("runProfile", () => { expect(run.failureReason).toMatch(/not found or not accessible/); }); - test("marks the run failed when a chunk's augmentation throws", async () => { + test("marks the run failed when the counts pass throws", async () => { const repos = [discovered("ok"), discovered("boom")]; const partialDeps: Partial = { discover: async () => ({ org: "acme", total: repos.length, repos }), - augment: async (_gql, chunk) => { + augmentCounts: async (_gql, chunk) => { if (chunk.some((r) => r.name === "boom")) throw new Error("repo augmentation failed"); - return chunk.map((r) => signalsFor(r)); + return chunk.map((r) => countsSignals(r)); }, }; @@ -250,18 +280,18 @@ describe("runProfile", () => { expect(run.failureReason).toBe("repo augmentation failed"); }); - test("persists completed chunks when a later chunk fails", async () => { - // 26 repos with releases at FULL=10 → chunks of 10 + 10 + 6. The chunk - // holding the last repo (r20–r25) throws; the other two chunks' 20 repos + test("persists completed counts chunks when a later chunk fails", async () => { + // 26 repos at COUNTS_CHUNK=15 → chunks of 15 + 11. The chunk holding the + // last repo (r15–r25) throws in the counts pass; the first chunk's 15 repos // still persist. const repos = Array.from({ length: 26 }, (_, i) => discovered(`r${String(i).padStart(2, "0")}`, { releasesCount: 1 }), ); const chunkedDeps: Partial = { discover: async () => ({ org: "acme", total: repos.length, repos }), - augment: async (_gql, chunk) => { + augmentCounts: async (_gql, chunk) => { if (chunk.some((r) => r.name === "r25")) throw new Error("a chunk failed"); - return chunk.map((r) => signalsFor(r)); + return chunk.map((r) => countsSignals(r)); }, }; @@ -273,12 +303,13 @@ describe("runProfile", () => { ); expect(run.state).toBe("failed"); - expect(getRunRepoProfiles("r")).toHaveLength(20); // the two surviving chunks persisted + expect(getRunRepoProfiles("r")).toHaveLength(15); // the surviving counts chunk persisted }); test("batches release-free repos wide (no scan) and release-bearing repos narrow", async () => { - // 2 repos with zero releases share one lite chunk (scanReleases: false); the - // 3 with releases share one full chunk (scanReleases: true). + // The details pass partitions by releases: 2 release-free repos share one + // lite chunk (scanReleases: false); the 3 with releases share one full + // chunk (scanReleases: true). const repos = [ discovered("a", { releasesCount: 0 }), discovered("b", { releasesCount: 2 }), @@ -289,9 +320,10 @@ describe("runProfile", () => { const calls: Array<{ size: number; scanReleases: boolean | undefined }> = []; const recordingDeps: Partial = { discover: async () => ({ org: "acme", total: repos.length, repos }), - augment: async (_gql, chunkRepos, opts) => { - calls.push({ size: chunkRepos.length, scanReleases: opts?.scanReleases }); - return chunkRepos.map((r) => signalsFor(r)); + augmentCounts: async (_gql, c) => c.map((r) => countsSignals(r)), + augmentDetails: async (_gql, c, opts) => { + calls.push({ size: c.length, scanReleases: opts?.scanReleases }); + return c.map((r) => detailsFor(r)); }, }; diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index 5dd2b65..ee25820 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -17,7 +17,7 @@ */ import type { graphql } from "@octokit/graphql"; import { analyzeRepo } from "./analyze"; -import { augmentRepoSignals } from "./augment"; +import { augmentRepoCounts, augmentRepoDetails } from "./augment"; import { discoverOrgRepos } from "./discover"; import { completeProfileRun, @@ -33,22 +33,23 @@ import { type OrgResources, type ProfileProgress, type ProfileRun, + type RepoSignals, ZERO_ORG_RESOURCES, } from "./types"; /** - * Repos per augment request. Even though most of the query is cheap - * `{ totalCount }` connections, each repo also does two server-side git-object - * reads (`.gitattributes` blob + `.github/workflows` tree), and repos with - * releases add the deeply-nested release-asset scan. In practice that pushes - * wide batches past GitHub's 10s GraphQL execution timeout (a 502/504), so - * batches are kept small: 10-15 repos per request. Repos discovery found to - * have zero releases skip the release scan and use the upper bound (LITE); - * repos with releases use the lower bound (FULL). If a chunk still times out, - * `augmentRepoSignals` splits and retries it, so these are safe ceilings. + * Repos per augment request. The crawl runs in two passes: + * - COUNTS: cheap `{ totalCount }`/scalar fields only — fills each repo's + * counts fast and is very unlikely to hit GitHub's 10s timeout. + * - DETAILS: the expensive verification (commit-graph walk, two git-object + * reads, branch-protection detail, and — for release-bearing repos — the + * release-asset scan), so those batch smaller. + * Kept at 10-15 repos/request; `augmentRepo*` further splits any chunk that + * still times out, so these are safe ceilings. */ -const AUGMENT_CHUNK_FULL = 10; -const AUGMENT_CHUNK_LITE = 15; +const COUNTS_CHUNK = 15; +const DETAILS_CHUNK_FULL = 10; +const DETAILS_CHUNK_LITE = 15; /** How many augment requests are in flight at once (cuts wall-clock ~Nx). */ const AUGMENT_CONCURRENCY = 3; @@ -88,7 +89,10 @@ function describeError(err: unknown): string { /** Injectable crawl primitives (defaults hit the real GraphQL helpers). */ export interface ProfileRunnerDeps { discover: typeof discoverOrgRepos; - augment: typeof augmentRepoSignals; + /** Pass 1: cheap per-repo counts. */ + augmentCounts: typeof augmentRepoCounts; + /** Pass 2: expensive per-repo verification details. */ + augmentDetails: typeof augmentRepoDetails; /** Count the org's rulesets. Default is a no-op (0); the service binds the * real REST-backed implementation, which needs a client this module lacks. */ getOrgRulesetCount: (org: string) => Promise; @@ -99,7 +103,8 @@ export interface ProfileRunnerDeps { const DEFAULT_DEPS: ProfileRunnerDeps = { discover: discoverOrgRepos, - augment: augmentRepoSignals, + augmentCounts: augmentRepoCounts, + augmentDetails: augmentRepoDetails, getOrgRulesetCount: async () => 0, getOrgResources: async () => ZERO_ORG_RESOURCES, }; @@ -151,56 +156,106 @@ export async function runProfile( setProfileRunRulesets(input.id, rulesetCount); setProfileRunOrgResources(input.id, orgResources); - // Profile repos in chunks. Repos with no releases skip the release-asset - // scan and batch wide; repos with releases batch narrow. Chunks run with - // bounded concurrency to cut wall-clock without stressing rate limits. + // ── Pass 1: cheap counts ────────────────────────────────────────────── + // One aliased request per chunk of cheap `{ totalCount }` fields, run with + // bounded concurrency. Each repo's profile is recorded immediately so the + // detail page fills in counts while the run is still in progress. A counts + // chunk that fails (after the internal split/degrade) fails the run. + const signalsByRepo = new Map(); + let profiled = 0; + { + const chunks = chunk(discovery.repos, COUNTS_CHUNK); + let firstError: unknown = null; + let next = 0; + const worker = async (): Promise => { + while (next < chunks.length && firstError === null) { + const c = chunks[next++]; + if (!c) break; + try { + const chunkSignals = await d.augmentCounts(gql, c); + for (const signals of chunkSignals) { + recordRepoProfile(input.id, signals, analyzeRepo(signals)); + signalsByRepo.set(signals.nameWithOwner, signals); + profiled += 1; + onProgress?.({ + runId: input.id, + profiled, + total: discovery.total, + repo: signals.nameWithOwner, + }); + } + } catch (err) { + console.error( + `[profile] run ${input.id} counts chunk failed — ${c.length} repo(s):`, + err, + ); + if (firstError === null) firstError = err; + } + } + }; + await Promise.all( + Array.from({ length: Math.min(AUGMENT_CONCURRENCY, chunks.length) }, () => worker()), + ); + if (firstError !== null) throw firstError; + } + console.log( + `[profile] run ${input.id} counts pass done — ${profiled} repo(s) in ${Date.now() - startedMs}ms`, + ); + + // ── Pass 2: verification details (best-effort) ───────────────────────── + // The expensive checks. Repos with no releases skip the release-asset scan + // and batch wider; repos with releases batch narrower. Each result merges + // onto the already-recorded counts and re-records. This pass is NON-fatal: + // the counts are already persisted, so a details failure enriches less but + // never fails the run. const withReleases = discovery.repos.filter((r) => r.releasesCount > 0); const withoutReleases = discovery.repos.filter((r) => r.releasesCount === 0); const tasks: Array<{ repos: typeof discovery.repos; scanReleases: boolean }> = [ - ...chunk(withoutReleases, AUGMENT_CHUNK_LITE).map((repos) => ({ + ...chunk(withoutReleases, DETAILS_CHUNK_LITE).map((repos) => ({ repos, scanReleases: false, })), - ...chunk(withReleases, AUGMENT_CHUNK_FULL).map((repos) => ({ repos, scanReleases: true })), + ...chunk(withReleases, DETAILS_CHUNK_FULL).map((repos) => ({ repos, scanReleases: true })), ]; - - let profiled = 0; - let firstError: unknown = null; - let next = 0; - // A worker pulls the next chunk until the queue drains or a chunk fails. - // Completed chunks persist regardless; the first error fails the run after - // in-flight chunks settle (mirrors the previous stop-on-first-failure). - const worker = async (): Promise => { - while (next < tasks.length && firstError === null) { - const task = tasks[next++]; - if (!task) break; - try { - const chunkSignals = await d.augment(gql, task.repos, { - scanReleases: task.scanReleases, - }); - for (const signals of chunkSignals) { - recordRepoProfile(input.id, signals, analyzeRepo(signals)); - profiled += 1; - onProgress?.({ - runId: input.id, - profiled, - total: discovery.total, - repo: signals.nameWithOwner, + { + let next = 0; + const worker = async (): Promise => { + while (next < tasks.length) { + const task = tasks[next++]; + if (!task) break; + try { + const details = await d.augmentDetails(gql, task.repos, { + scanReleases: task.scanReleases, }); + for (const det of details) { + const base = signalsByRepo.get(det.nameWithOwner); + if (!base) continue; + const merged: RepoSignals = { + ...base, + commitsCount: det.commitsCount, + branchProtectionRulesUsingUnmigratedFeatures: + det.branchProtectionRulesUsingUnmigratedFeatures, + usesLfs: det.usesLfs, + workflowFileCount: det.workflowFileCount, + releaseAssetBytes: det.releaseAssetBytes, + }; + signalsByRepo.set(det.nameWithOwner, merged); + recordRepoProfile(input.id, merged, analyzeRepo(merged)); + } + // Nudge any live watcher to refetch the now-enriched data. + onProgress?.({ runId: input.id, profiled, total: discovery.total, repo: "" }); + } catch (err) { + console.error( + `[profile] run ${input.id} details chunk failed — ${task.repos.length} repo(s), scanReleases=${task.scanReleases} (counts kept):`, + err, + ); } - } catch (err) { - console.error( - `[profile] run ${input.id} augment chunk failed — ${task.repos.length} repo(s), scanReleases=${task.scanReleases}:`, - err, - ); - if (firstError === null) firstError = err; } - } - }; - await Promise.all( - Array.from({ length: Math.min(AUGMENT_CONCURRENCY, tasks.length) }, () => worker()), - ); - if (firstError !== null) throw firstError; + }; + await Promise.all( + Array.from({ length: Math.min(AUGMENT_CONCURRENCY, tasks.length) }, () => worker()), + ); + } completeProfileRun(input.id); const completed = getProfileRun(input.id); diff --git a/src/lib/server/profile/service.test.ts b/src/lib/server/profile/service.test.ts index c660f13..a87a40d 100644 --- a/src/lib/server/profile/service.test.ts +++ b/src/lib/server/profile/service.test.ts @@ -8,6 +8,7 @@ import { beforeEach, describe, expect, test } from "bun:test"; import { initStore } from "$lib/server/core/db"; import { DOMAIN_STORES } from "$lib/server/registry"; import type { RepoProfile } from "./analyze"; +import type { RepoDetails } from "./augment"; import { type ProfileSseEvent, subscribeProfile } from "./events"; import { runProfile } from "./runner"; import { getProfileDetail, type ProfileServiceDeps, startOrgProfile } from "./service"; @@ -41,6 +42,7 @@ function discovered(name: string): DiscoveredRepo { function signalsFor(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { return { ...repo, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -58,6 +60,19 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo }; } +/** Verification details for a repo (the pass-2 fake), derived from `signalsFor`. */ +function detailsFor(repo: DiscoveredRepo, over: Partial = {}): RepoDetails { + const s = signalsFor(repo, over); + return { + nameWithOwner: repo.nameWithOwner, + commitsCount: s.commitsCount, + branchProtectionRulesUsingUnmigratedFeatures: s.branchProtectionRulesUsingUnmigratedFeatures, + usesLfs: s.usesLfs, + workflowFileCount: s.workflowFileCount, + releaseAssetBytes: s.releaseAssetBytes, + }; +} + /** * Service deps whose runner uses fake crawl primitives. Captures the run promise * so a test can await background completion, and records the gql-build call. @@ -86,7 +101,10 @@ function serviceDeps( total: repos.length, repos, }), - augment: async (_gql, chunk) => chunk.map((r) => signalsFor(r, augmentOver[r.name] ?? {})), + augmentCounts: async (_gql, chunk) => + chunk.map((r) => signalsFor(r, augmentOver[r.name] ?? {})), + augmentDetails: async (_gql, chunk) => + chunk.map((r) => detailsFor(r, augmentOver[r.name] ?? {})), }); return state.runPromise; }, @@ -167,7 +185,9 @@ describe("startOrgProfile", () => { const events = frames.map( (f) => JSON.parse(f.replace(/^data: /, "").trimEnd()) as ProfileSseEvent, ); - const progress = events.filter((e) => e.type === "progress"); + // Per-repo frames come from the counts pass; the details pass adds repo-less + // refetch nudges, filtered out here. + const progress = events.filter((e) => e.type === "progress" && e.repo !== ""); expect(progress).toEqual([ { type: "progress", profiled: 1, total: 2, repo: "acme/a" }, { type: "progress", profiled: 2, total: 2, repo: "acme/b" }, @@ -219,6 +239,7 @@ describe("getProfileDetail", () => { signalsFor(discovered("a"), { issuesCount: 10, pullRequestsCount: 4, + commitsCount: 100, branchesCount: 3, tagsCount: 2, releasesCount: 1, @@ -231,6 +252,7 @@ describe("getProfileDetail", () => { signalsFor(discovered("b"), { issuesCount: 5, pullRequestsCount: 6, + commitsCount: 50, branchesCount: 1, tagsCount: 0, releasesCount: 2, @@ -244,6 +266,7 @@ describe("getProfileDetail", () => { repos: 2, issues: 15, pullRequests: 10, + commits: 150, branches: 4, tags: 2, releases: 3, diff --git a/src/lib/server/profile/service.ts b/src/lib/server/profile/service.ts index c0329fe..a8810f1 100644 --- a/src/lib/server/profile/service.ts +++ b/src/lib/server/profile/service.ts @@ -45,6 +45,7 @@ interface MigrationScale { repos: number; issues: number; pullRequests: number; + commits: number; branches: number; tags: number; releases: number; @@ -69,6 +70,7 @@ function computeScale(repos: RepoProfileView[]): MigrationScale { repos: repos.length, issues: 0, pullRequests: 0, + commits: 0, branches: 0, tags: 0, releases: 0, @@ -77,6 +79,7 @@ function computeScale(repos: RepoProfileView[]): MigrationScale { for (const { signals } of repos) { scale.issues += signals.issuesCount; scale.pullRequests += signals.pullRequestsCount; + scale.commits += signals.commitsCount; scale.branches += signals.branchesCount; scale.tags += signals.tagsCount; scale.releases += signals.releasesCount; diff --git a/src/lib/server/profile/store.test.ts b/src/lib/server/profile/store.test.ts index b039e57..a310784 100644 --- a/src/lib/server/profile/store.test.ts +++ b/src/lib/server/profile/store.test.ts @@ -50,6 +50,7 @@ function signals(over: Partial = {}): RepoSignals { pullRequestsCount: 0, branchesCount: 0, tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/types.ts b/src/lib/server/profile/types.ts index 9077afa..f9d763c 100644 --- a/src/lib/server/profile/types.ts +++ b/src/lib/server/profile/types.ts @@ -86,6 +86,13 @@ export interface OrgDiscovery { * read a number and compare, rather than re-deriving GraphQL schema details. */ export interface RepoSignals extends DiscoveredRepo { + /** + * Commits on the default branch (`history.totalCount`). Gathered in the + * verification pass, not the cheap counts pass: unlike the indexed + * `totalCount`s it walks the commit graph, so a timeout degrades it to 0 for + * that one repo rather than blocking the cheap counts. + */ + commitsCount: number; /** Repository-level Discussions (`discussions.totalCount`); not migrated. */ discussionsCount: number; /** Projects (new experience) (`projectsV2.totalCount`); not migrated. */ diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index ee82279..f2a5266 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -24,6 +24,7 @@ [ { label: 'Issues', value: scale.issues.toLocaleString(), icon: 'issue-opened' }, { label: 'Pull requests', value: scale.pullRequests.toLocaleString(), icon: 'git-pull-request' }, + { label: 'Commits', value: scale.commits.toLocaleString(), icon: 'git-commit' }, { label: 'Branches', value: scale.branches.toLocaleString(), icon: 'git-branch' }, { label: 'Tags', value: scale.tags.toLocaleString(), icon: 'tag' }, { label: 'Releases', value: scale.releases.toLocaleString(), icon: 'rocket' }, @@ -131,6 +132,7 @@ return [ { label: 'Issues', value: s.issuesCount.toLocaleString(), icon: 'issue-opened' }, { label: 'Pull requests', value: s.pullRequestsCount.toLocaleString(), icon: 'git-pull-request' }, + { label: 'Commits', value: s.commitsCount.toLocaleString(), icon: 'git-commit' }, { label: 'Branches', value: s.branchesCount.toLocaleString(), icon: 'git-branch' }, { label: 'Tags', value: s.tagsCount.toLocaleString(), icon: 'tag' }, { label: 'Releases', value: s.releasesCount.toLocaleString(), icon: 'rocket' }, From c51b0c55b7e28411663818f278fa8b5b7c97a886 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 23:20:52 -0700 Subject: [PATCH 07/11] fix(profile): make discovery 502-resilient and the repo list non-fatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The org profile run failed with a bare "HTTP 502: ... nginx" and showed zero repos. Root cause: discovery (the one GraphQL call with no timeout handling) propagated GitHub's 10s-timeout 502 straight up and failed the whole run, and repos were only persisted in the counts pass — which was fatal — so a single 502 anywhere left the user with no list at all. Reworked to match the intended pipeline (get the list first, enrich later, best-effort): - discover.ts: on a 502/504 timeout, halve the page size and retry the SAME cursor (down to a floor of 10), keeping the smaller size for the rest of the crawl. A 100-wide page of indexed counts is normally cheap, but an org with repos that have very many refs/issues can tip one page over the limit; this lets discovery finish instead of failing. Non-timeout errors still surface. - runner.ts: record every discovered repo immediately with the discovery spine (augmented fields zeroed), so the detail page shows the full list the moment discovery finishes. The counts pass is now best-effort — a chunk that fails for a non-timeout reason is logged and skipped, leaving those repos at their base signals rather than failing the run. The details pass was already non-fatal. - graphql-errors.ts: extract the shared `isTimeoutError` predicate (used by both the discovery page-shrink and the augment chunk-split). The augment passes already split-and-degrade on a 502, so they never propagate one; this closes the last gap (discovery) and guarantees the list survives any single transient failure. --- src/lib/server/profile/augment.ts | 27 +++++------ src/lib/server/profile/discover.test.ts | 54 ++++++++++++++++++++++ src/lib/server/profile/discover.ts | 58 ++++++++++++++++++++---- src/lib/server/profile/graphql-errors.ts | 26 +++++++++++ src/lib/server/profile/runner.test.ts | 24 ++++++---- src/lib/server/profile/runner.ts | 40 +++++++++++----- 6 files changed, 184 insertions(+), 45 deletions(-) create mode 100644 src/lib/server/profile/graphql-errors.ts diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index 9a03288..fde97aa 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -20,6 +20,7 @@ * branch-protection derivation are unit-testable without a network. */ import type { graphql } from "@octokit/graphql"; +import { isTimeoutError } from "./graphql-errors"; import type { DiscoveredRepo, RepoSignals } from "./types"; /** GraphQL caps `branchProtectionRules(first:)` at 100; most repos have <10, so @@ -236,6 +237,16 @@ function countsToSignals(repo: DiscoveredRepo, node: RepoCountsNode | null): Rep }; } +/** + * A repo's signals with only the discovery spine — every augmented field at its + * default. Recorded at discovery time so each repo is listed immediately; the + * counts and details passes then enrich it in place. This is the single source + * of truth for the zeroed shape (it's exactly an un-augmented counts node). + */ +export function baseSignals(repo: DiscoveredRepo): RepoSignals { + return countsToSignals(repo, null); +} + /** Map a repo + its (possibly null) details node to the verification fields. */ function toDetails(repo: DiscoveredRepo, node: RepoDetailsNode | null): RepoDetails { return { @@ -259,22 +270,6 @@ function partialDataFromError(err: unknown): Record | null { return null; } -/** - * Whether an error is a GitHub processing timeout. The GraphQL API terminates - * any request that takes longer than ~10s and responds with a 502 or 504 (or a - * "Something went wrong while executing your query" message). These carry no - * usable partial data — the remedy is to make the query cheaper, i.e. split it. - */ -function isTimeoutError(err: unknown): boolean { - if (!err || typeof err !== "object") return false; - const e = err as { status?: number; message?: string }; - if (e.status === 502 || e.status === 504) return true; - return ( - typeof e.message === "string" && - /timeout|timed out|executing your query|respond to your request in time/i.test(e.message) - ); -} - /** Options controlling how wide/deep a single augment request reaches. */ export interface AugmentOptions { /** diff --git a/src/lib/server/profile/discover.test.ts b/src/lib/server/profile/discover.test.ts index a15c7af..6236605 100644 --- a/src/lib/server/profile/discover.test.ts +++ b/src/lib/server/profile/discover.test.ts @@ -229,4 +229,58 @@ describe("discoverOrgRepos", () => { expect(calls).toHaveLength(1); expect(result.repos.map((r) => r.name)).toEqual(["a"]); }); + + test("halves the page and retries the same cursor when a page times out", async () => { + const calls: Array<{ cursor: string | null; pageSize: number }> = []; + let attempt = 0; + const fn = (async (_q: string, vars: Record) => { + calls.push({ + cursor: (vars.cursor as string | null) ?? null, + pageSize: vars.pageSize as number, + }); + attempt += 1; + // The first attempt (100-wide) times out with a GitHub 502; the retry at + // half the size succeeds — no repos are skipped because the cursor holds. + if (attempt === 1) throw Object.assign(new Error("502 Bad Gateway"), { status: 502 }); + return page([node("a"), node("b")], { total: 2 }); + }) as unknown as typeof graphql; + + const result = await discoverOrgRepos(fn, "acme"); + + expect(result.repos.map((r) => r.name)).toEqual(["a", "b"]); + expect(calls).toHaveLength(2); + expect(calls[0]?.pageSize).toBe(100); // first attempt at the full page + expect(calls[1]?.pageSize).toBe(50); // retried at half after the 502 + expect(calls[1]?.cursor).toBeNull(); // same cursor — nothing skipped + }); + + test("keeps the shrunk page size for the rest of the crawl", async () => { + const sizes: number[] = []; + let attempt = 0; + const fn = (async (_q: string, vars: Record) => { + sizes.push(vars.pageSize as number); + attempt += 1; + if (attempt === 1) throw Object.assign(new Error("504"), { status: 504 }); + // After the shrink, page 1 (at 50) has a next page; page 2 must reuse 50. + if (attempt === 2) + return page([node("a")], { total: 2, hasNextPage: true, endCursor: "CUR" }); + return page([node("b")], { total: 2 }); + }) as unknown as typeof graphql; + + const result = await discoverOrgRepos(fn, "acme"); + + expect(result.repos.map((r) => r.name)).toEqual(["a", "b"]); + expect(sizes).toEqual([100, 50, 50]); // shrunk once, then held + }); + + test("surfaces a non-timeout error without retrying", async () => { + let attempts = 0; + const fn = (async () => { + attempts += 1; + throw Object.assign(new Error("Bad credentials"), { status: 401 }); + }) as unknown as typeof graphql; + + await expect(discoverOrgRepos(fn, "acme")).rejects.toThrow("Bad credentials"); + expect(attempts).toBe(1); // a 401 is not a timeout — no page-size retry + }); }); diff --git a/src/lib/server/profile/discover.ts b/src/lib/server/profile/discover.ts index 054aae3..88d7ba1 100644 --- a/src/lib/server/profile/discover.ts +++ b/src/lib/server/profile/discover.ts @@ -8,11 +8,17 @@ * network or a live Octokit. */ import type { graphql } from "@octokit/graphql"; +import { isTimeoutError } from "./graphql-errors"; import type { DiscoveredRepo, DiscoveryProgress, OrgDiscovery, RepoVisibility } from "./types"; /** GraphQL caps `repositories(first:)` at 100. */ const REPOS_PER_PAGE = 100; +/** Smallest discovery page we'll shrink to before surfacing a timeout. A page + * of only indexed counts should never time out this small; if it does, the + * error is real and worth surfacing rather than retrying forever. */ +const MIN_PAGE_SIZE = 10; + /** Raw shape of one repository node in the discovery query. */ interface RepoNode { name: string; @@ -102,6 +108,45 @@ function toDiscoveredRepo(node: RepoNode): DiscoveredRepo { }; } +/** + * Fetch one discovery page, halving the page size and retrying the *same* + * cursor each time GitHub times the request out (502/504), down to + * `MIN_PAGE_SIZE`. A 100-wide page of indexed counts is normally cheap, but an + * org with repos that have very many refs/issues can tip one page over GitHub's + * ~10s limit; shrinking keeps discovery — the only thing between the user and "a + * list of repos" — completing instead of failing the whole run. Returns the page + * plus the size that actually worked so the caller can keep using it for the + * remaining pages rather than re-hitting the timeout on every one. + */ +async function fetchPageAdaptive( + gql: typeof graphql, + org: string, + cursor: string | null, + pageSize: number, +): Promise<{ result: OrgReposResult; pageSize: number }> { + let size = pageSize; + for (;;) { + try { + const result = await gql(ORG_REPOS_QUERY, { + login: org, + cursor, + pageSize: size, + }); + return { result, pageSize: size }; + } catch (err) { + if (isTimeoutError(err) && size > MIN_PAGE_SIZE) { + const smaller = Math.max(MIN_PAGE_SIZE, Math.floor(size / 2)); + console.warn( + `[profile] discovery page of ${size} timed out for '${org}'; retrying this page at ${smaller}`, + ); + size = smaller; + continue; + } + throw err; + } + } +} + /** * Discover every repository in an organization via paged GraphQL. * @@ -120,16 +165,13 @@ export async function discoverOrgRepos( let cursor: string | null = null; let total = 0; let page = 0; + // Adaptive page size: starts at the 100-wide max and only shrinks (and stays + // shrunk) if a page times out, so small orgs pay nothing for the resilience. + let pageSize = REPOS_PER_PAGE; for (;;) { - // `result` is annotated explicitly: `cursor` is both an input to this call - // and reassigned from its output below, so without the annotation TS sees a - // self-referential initializer and falls back to `any` (TS7022). - const result: OrgReposResult = await gql(ORG_REPOS_QUERY, { - login: org, - cursor, - pageSize: REPOS_PER_PAGE, - }); + const { result, pageSize: usedSize } = await fetchPageAdaptive(gql, org, cursor, pageSize); + pageSize = usedSize; const connection = result.organization?.repositories; if (!connection) { diff --git a/src/lib/server/profile/graphql-errors.ts b/src/lib/server/profile/graphql-errors.ts new file mode 100644 index 0000000..1eb3903 --- /dev/null +++ b/src/lib/server/profile/graphql-errors.ts @@ -0,0 +1,26 @@ +/** + * Shared classification of GitHub GraphQL transport errors. + * + * The crawl makes many GraphQL requests (discovery pages, counts chunks, detail + * chunks). GitHub aborts any single request that takes longer than ~10s and + * responds with a 502 or 504 (sometimes an nginx HTML body, sometimes a + * "Something went wrong while executing your query" message). Every pass treats + * that the same way — make the request cheaper and retry (split a chunk, or + * shrink a discovery page) — so the predicate lives here, used by both + * `discover.ts` (page-size shrink) and `augment.ts` (chunk split). + */ + +/** + * Whether an error is a GitHub processing timeout (the request was too + * expensive to evaluate in GitHub's ~10s window). These carry no usable partial + * data; the remedy is always to make the query cheaper and retry. + */ +export function isTimeoutError(err: unknown): boolean { + if (!err || typeof err !== "object") return false; + const e = err as { status?: number; message?: string }; + if (e.status === 502 || e.status === 504) return true; + return ( + typeof e.message === "string" && + /timeout|timed out|executing your query|respond to your request in time/i.test(e.message) + ); +} diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index 2afe9b9..94e6478 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -259,7 +259,7 @@ describe("runProfile", () => { expect(run.failureReason).toMatch(/not found or not accessible/); }); - test("marks the run failed when the counts pass throws", async () => { + test("keeps the run completed (repos still listed) when a counts chunk fails", async () => { const repos = [discovered("ok"), discovered("boom")]; const partialDeps: Partial = { discover: async () => ({ org: "acme", total: repos.length, repos }), @@ -276,14 +276,20 @@ describe("runProfile", () => { partialDeps, ); - expect(run.state).toBe("failed"); - expect(run.failureReason).toBe("repo augmentation failed"); + // The counts pass is best-effort: a chunk failure no longer fails the run. + // Both repos were recorded from discovery, so the list survives. + expect(run.state).toBe("completed"); + expect( + getRunRepoProfiles("r") + .map((p) => p.nameWithOwner) + .sort(), + ).toEqual(["acme/boom", "acme/ok"]); }); - test("persists completed counts chunks when a later chunk fails", async () => { - // 26 repos at COUNTS_CHUNK=15 → chunks of 15 + 11. The chunk holding the - // last repo (r15–r25) throws in the counts pass; the first chunk's 15 repos - // still persist. + test("lists every repo even when a counts chunk fails, enriching the rest", async () => { + // 26 repos at COUNTS_CHUNK=15 → chunks of 15 + 11. The chunk holding r25 + // throws in the counts pass; every repo is still listed (recorded at + // discovery) and the run completes. const repos = Array.from({ length: 26 }, (_, i) => discovered(`r${String(i).padStart(2, "0")}`, { releasesCount: 1 }), ); @@ -302,8 +308,8 @@ describe("runProfile", () => { chunkedDeps, ); - expect(run.state).toBe("failed"); - expect(getRunRepoProfiles("r")).toHaveLength(15); // the surviving counts chunk persisted + expect(run.state).toBe("completed"); + expect(getRunRepoProfiles("r")).toHaveLength(26); // all repos listed despite the failed chunk }); test("batches release-free repos wide (no scan) and release-bearing repos narrow", async () => { diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index ee25820..4700be3 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -17,7 +17,7 @@ */ import type { graphql } from "@octokit/graphql"; import { analyzeRepo } from "./analyze"; -import { augmentRepoCounts, augmentRepoDetails } from "./augment"; +import { augmentRepoCounts, augmentRepoDetails, baseSignals } from "./augment"; import { discoverOrgRepos } from "./discover"; import { completeProfileRun, @@ -156,19 +156,37 @@ export async function runProfile( setProfileRunRulesets(input.id, rulesetCount); setProfileRunOrgResources(input.id, orgResources); - // ── Pass 1: cheap counts ────────────────────────────────────────────── - // One aliased request per chunk of cheap `{ totalCount }` fields, run with - // bounded concurrency. Each repo's profile is recorded immediately so the - // detail page fills in counts while the run is still in progress. A counts - // chunk that fails (after the internal split/degrade) fails the run. + // ── Record the list up front ────────────────────────────────────────── + // Every discovered repo is recorded immediately with the discovery spine and + // its augmented fields zeroed, so the detail page shows the FULL list the + // moment discovery finishes — exactly the "list of repos" the user wants + // before any enrichment. The two passes below fill each repo in place and + // are best-effort: a transient GraphQL 502 in either enriches fewer repos + // but can never empty the list or fail the run. const signalsByRepo = new Map(); + for (const repo of discovery.repos) { + const base = baseSignals(repo); + signalsByRepo.set(repo.nameWithOwner, base); + recordRepoProfile(input.id, base, analyzeRepo(base)); + } + if (discovery.repos.length > 0) { + // Nudge any live watcher to render the freshly-listed repos. + onProgress?.({ runId: input.id, profiled: 0, total: discovery.total, repo: "" }); + } + + // ── Pass 1: cheap counts (best-effort) ──────────────────────────────── + // One aliased request per chunk of cheap `{ totalCount }`/scalar fields, run + // with bounded concurrency; each repo's counts are recorded as they arrive. + // `augmentCounts` already splits/degrades on a GitHub 502 timeout, so a + // chunk reaching this catch failed for some other reason — we log it and + // move on, leaving those repos at their base signals rather than failing the + // whole run. let profiled = 0; { const chunks = chunk(discovery.repos, COUNTS_CHUNK); - let firstError: unknown = null; let next = 0; const worker = async (): Promise => { - while (next < chunks.length && firstError === null) { + while (next < chunks.length) { const c = chunks[next++]; if (!c) break; try { @@ -186,20 +204,18 @@ export async function runProfile( } } catch (err) { console.error( - `[profile] run ${input.id} counts chunk failed — ${c.length} repo(s):`, + `[profile] run ${input.id} counts chunk failed — ${c.length} repo(s) kept at base signals:`, err, ); - if (firstError === null) firstError = err; } } }; await Promise.all( Array.from({ length: Math.min(AUGMENT_CONCURRENCY, chunks.length) }, () => worker()), ); - if (firstError !== null) throw firstError; } console.log( - `[profile] run ${input.id} counts pass done — ${profiled} repo(s) in ${Date.now() - startedMs}ms`, + `[profile] run ${input.id} counts pass done — ${profiled} repo(s) enriched in ${Date.now() - startedMs}ms`, ); // ── Pass 2: verification details (best-effort) ───────────────────────── From 5920242970bdcbec3013409cc09289f365d59297 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 23:55:18 -0700 Subject: [PATCH 08/11] refactor(profile): list repos via REST, gather counts via batched GraphQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery used a single 100-wide GraphQL page carrying several connection totalCounts (issues, PRs, branches, tags, releases) per repo. On a large org that one query sits right at GitHub's ~10s execution limit and 502s intermittently — so the run failed before producing even a list of repos. Split the two concerns the way they want to be split: - discover.ts now lists the org via REST `GET /orgs/{org}/repos` (the `minimal-repository` shape), paged through the Link header. REST listing is an indexed, paginated read that never hits the GraphQL timeout — it reliably produces the repo list. It fills the discovery spine (name, visibility, fork, archived, size, feature toggles, default branch, timestamps); `isEmpty` is proxied from `size === 0` (REST has no is_empty). - The five content-volume counts move from the discovery spine onto RepoSignals, gathered by the existing batched GraphQL counts pass (15-wide, splits-on- timeout, degrades a repo to zeros rather than failing). They're indexed totalCounts, cheap at that fan-out. - The runner now takes a { gql, rest } client bundle: rest drives discovery, gql drives the augment passes. The details pass partitions by the releases count from the counts pass (it's no longer on the spine). Verified the real Octokit REST pagination end-to-end against a local two-page fake (Link-header paging, visibility normalization, empty proxy). 118 profile tests pass; all gates green. --- src/lib/server/profile/augment.test.ts | 30 ++- src/lib/server/profile/augment.ts | 33 ++- src/lib/server/profile/discover.test.ts | 324 ++++++++---------------- src/lib/server/profile/discover.ts | 236 ++++++----------- src/lib/server/profile/runner.test.ts | 58 +++-- src/lib/server/profile/runner.ts | 51 ++-- src/lib/server/profile/service.test.ts | 14 +- src/lib/server/profile/service.ts | 2 +- src/lib/server/profile/types.ts | 41 +-- 9 files changed, 319 insertions(+), 470 deletions(-) diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index 2ddf615..f86424b 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -27,11 +27,6 @@ function discovered(over: Partial = {}): DiscoveredRepo { defaultBranch: "main", pushedAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-02T00:00:00Z", - issuesCount: 0, - pullRequestsCount: 0, - branchesCount: 0, - tagsCount: 0, - releasesCount: 0, ...over, }; } @@ -61,6 +56,11 @@ function rule(flags: RuleFlags = {}) { /** Build one repository alias node for the cheap counts pass. */ function countsNode(over: { + issues?: number; + pullRequests?: number; + branches?: number; + tags?: number; + releases?: number; discussions?: number; projectsV2?: number; environments?: number; @@ -73,6 +73,11 @@ function countsNode(over: { ruleTotal?: number; }) { return { + issues: { totalCount: over.issues ?? 0 }, + pullRequests: { totalCount: over.pullRequests ?? 0 }, + branches: { totalCount: over.branches ?? 0 }, + tags: { totalCount: over.tags ?? 0 }, + releases: { totalCount: over.releases ?? 0 }, discussions: { totalCount: over.discussions ?? 0 }, projectsV2: { totalCount: over.projectsV2 ?? 0 }, environments: { totalCount: over.environments ?? 0 }, @@ -134,6 +139,8 @@ describe("augmentRepoCounts", () => { test("maps every cheap count and preserves the discovered fields", async () => { const { fn } = mockGql({ r0: countsNode({ + issues: 11, + releases: 7, discussions: 3, projectsV2: 2, environments: 4, @@ -146,7 +153,7 @@ describe("augmentRepoCounts", () => { }), }); - const [s] = await augmentRepoCounts(fn, [discovered({ issuesCount: 11, releasesCount: 7 })]); + const [s] = await augmentRepoCounts(fn, [discovered()]); expect(s?.discussionsCount).toBe(3); expect(s?.projectsV2Count).toBe(2); @@ -215,11 +222,12 @@ describe("augmentRepoCounts", () => { const { fn } = mockGql({ r0: countsNode({ discussions: 5 }), r1: null }); const signals = await augmentRepoCounts(fn, [ discovered({ nameWithOwner: "o/ok", name: "ok" }), - discovered({ nameWithOwner: "o/gone", name: "gone", issuesCount: 3 }), + discovered({ nameWithOwner: "o/gone", name: "gone", diskUsageKb: 42 }), ]); expect(signals[0]?.discussionsCount).toBe(5); expect(signals[1]?.nameWithOwner).toBe("o/gone"); - expect(signals[1]?.issuesCount).toBe(3); // spine kept + expect(signals[1]?.diskUsageKb).toBe(42); // discovery spine kept + expect(signals[1]?.issuesCount).toBe(0); // augment count zeroed expect(signals[1]?.discussionsCount).toBe(0); // augment zeroed }); @@ -269,12 +277,14 @@ describe("augmentRepoCounts", () => { throw Object.assign(new Error("Gateway Timeout"), { status: 504 }); }) as unknown as typeof graphql; const repos = [ - discovered({ name: "a", nameWithOwner: "o/a", issuesCount: 7 }), + discovered({ name: "a", nameWithOwner: "o/a", diskUsageKb: 7 }), discovered({ name: "b", nameWithOwner: "o/b" }), ]; const signals = await augmentRepoCounts(fn, repos); expect(signals).toHaveLength(2); - expect(signals[0]?.issuesCount).toBe(7); // spine kept + expect(signals[0]?.nameWithOwner).toBe("o/a"); + expect(signals[0]?.diskUsageKb).toBe(7); // discovery spine kept + expect(signals[0]?.issuesCount).toBe(0); // degraded count expect(signals[0]?.discussionsCount).toBe(0); // degraded }); }); diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index fde97aa..083772e 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -1,14 +1,16 @@ /** - * Per-repo signal augmentation — the second crawl pass, batched. + * Per-repo signal augmentation — the GraphQL passes that enrich the REST spine. * - * Discovery (`discover.ts`) gives the cheap, breadth-first spine and the - * content-volume counts that ride along free in each 100-repo page. This pass - * gathers the signals that can't fold into a discovery node: the - * branch-protection rule *details* (node-level flags, not just a count) and the - * default-branch commit count (`history.totalCount`, the one count with timeout - * risk at 100-wide, so kept to a smaller fan-out here). + * Discovery (`discover.ts`) lists the org via REST, giving the cheap spine + * (names, visibility, sizes, feature toggles). These passes add the indexed + * GraphQL signals on top, batched: + * - the COUNTS pass: pure `{ totalCount }`/scalar fields (issues, PRs, + * branches, tags, releases, discussions, …) — indexed reads that don't + * materialize nodes, so they batch safely; + * - the DETAILS pass: the expensive verification (commit-graph walk, + * git-object reads, branch-protection rule node detail, release-asset scan). * - * It profiles a CHUNK of repos in one aliased GraphQL request + * Each pass profiles a CHUNK of repos in one aliased GraphQL request * (`r0: repository(...) {…} r1: repository(...) {…} …`), so an org of N repos * costs ~N/chunk requests instead of one per repo. The request is * partial-failure aware: if a repo in the chunk is inaccessible, GraphQL @@ -50,6 +52,11 @@ interface BranchProtectionRuleNode { /** Cheap per-repo counts: pure `{ totalCount }` / scalar fields, no git reads * and no node materialization — indexed reads that batch safely. */ interface RepoCountsNode { + issues: { totalCount: number }; + pullRequests: { totalCount: number }; + branches: { totalCount: number }; + tags: { totalCount: number }; + releases: { totalCount: number }; discussions: { totalCount: number }; projectsV2: { totalCount: number }; environments: { totalCount: number }; @@ -98,6 +105,11 @@ export interface RepoDetails { * GitHub's 10s timeout even at the batch ceiling. */ const COUNTS_FRAGMENT = `fragment Counts on Repository { + issues { totalCount } + pullRequests { totalCount } + branches: refs(refPrefix: "refs/heads/", first: 1) { totalCount } + tags: refs(refPrefix: "refs/tags/", first: 1) { totalCount } + releases { totalCount } discussions { totalCount } projectsV2 { totalCount } environments { totalCount } @@ -225,6 +237,11 @@ function countsToSignals(repo: DiscoveredRepo, node: RepoCountsNode | null): Rep workflowFileCount: 0, releaseAssetBytes: 0, // Cheap counts. + issuesCount: node?.issues.totalCount ?? 0, + pullRequestsCount: node?.pullRequests.totalCount ?? 0, + branchesCount: node?.branches.totalCount ?? 0, + tagsCount: node?.tags.totalCount ?? 0, + releasesCount: node?.releases.totalCount ?? 0, discussionsCount: node?.discussions.totalCount ?? 0, projectsV2Count: node?.projectsV2.totalCount ?? 0, environmentsCount: node?.environments.totalCount ?? 0, diff --git a/src/lib/server/profile/discover.test.ts b/src/lib/server/profile/discover.test.ts index 6236605..ab62669 100644 --- a/src/lib/server/profile/discover.test.ts +++ b/src/lib/server/profile/discover.test.ts @@ -1,168 +1,146 @@ /** - * Tests for the org discovery crawl. The GraphQL client is injected, so these - * exercise the real orchestration — pagination, cursor threading, mapping, and + * Tests for the org discovery crawl. The REST client is injected, so these + * exercise the real orchestration — pagination, mapping (REST → spine), and * progress — against in-memory page fixtures with no network. */ import { describe, expect, test } from "bun:test"; -import type { graphql } from "@octokit/graphql"; +import type { GitHubClient } from "$lib/server/core/github"; import { discoverOrgRepos } from "./discover"; import type { DiscoveryProgress } from "./types"; -interface RepoNode { +/** The subset of a REST `minimal-repository` the discovery mapper reads. */ +interface RestRepo { name: string; - nameWithOwner: string; - visibility: "PUBLIC" | "PRIVATE" | "INTERNAL"; - isArchived: boolean; - isFork: boolean; - isEmpty: boolean; - diskUsage: number | null; - hasWikiEnabled: boolean; - hasIssuesEnabled: boolean; - hasProjectsEnabled: boolean; - hasDiscussionsEnabled: boolean; - defaultBranchRef: { name: string } | null; - pushedAt: string | null; - updatedAt: string | null; - issues: { totalCount: number }; - pullRequests: { totalCount: number }; - branches: { totalCount: number }; - tags: { totalCount: number }; - releases: { totalCount: number }; + full_name: string; + private: boolean; + fork: boolean; + visibility?: string; + archived?: boolean; + size?: number; + has_wiki?: boolean; + has_issues?: boolean; + has_projects?: boolean; + has_discussions?: boolean; + default_branch?: string; + pushed_at?: string | null; + updated_at?: string | null; } -/** Build a repo node, overriding only the fields a test cares about. */ -function node(name: string, over: Partial = {}): RepoNode { +/** Build a REST repo item, overriding only the fields a test cares about. */ +function repo(name: string, over: Partial = {}): RestRepo { return { name, - nameWithOwner: `acme/${name}`, - visibility: "PRIVATE", - isArchived: false, - isFork: false, - isEmpty: false, - diskUsage: 100, - hasWikiEnabled: false, - hasIssuesEnabled: true, - hasProjectsEnabled: false, - hasDiscussionsEnabled: false, - defaultBranchRef: { name: "main" }, - pushedAt: "2026-01-01T00:00:00Z", - updatedAt: "2026-01-02T00:00:00Z", - issues: { totalCount: 0 }, - pullRequests: { totalCount: 0 }, - branches: { totalCount: 0 }, - tags: { totalCount: 0 }, - releases: { totalCount: 0 }, + full_name: `acme/${name}`, + private: true, + fork: false, + visibility: "private", + archived: false, + size: 100, + has_wiki: false, + has_issues: true, + has_projects: false, + has_discussions: false, + default_branch: "main", + pushed_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", ...over, }; } -/** One discovery page's worth of GraphQL response. */ -function page( - nodes: RepoNode[], - opts: { total: number; hasNextPage?: boolean; endCursor?: string | null }, -) { - return { - organization: { - repositories: { - totalCount: opts.total, - pageInfo: { - hasNextPage: opts.hasNextPage ?? false, - endCursor: opts.endCursor ?? null, - }, - nodes, - }, - }, - }; -} - /** - * A `gql` test double: returns each queued response in turn and records the - * variables it was called with (to assert cursor threading). + * A `rest` test double whose `paginate.iterator` yields the queued pages and + * records the options it was called with (to assert the request shape). Mirrors + * the Octokit surface `discoverOrgRepos` uses: `rest.paginate.iterator( + * rest.repos.listForOrg, opts)`. */ -function mockGql(responses: unknown[]) { - const calls: Array<{ login: string; cursor: string | null; pageSize: number }> = []; - let i = 0; - const fn = (async (_query: string, vars: Record) => { - calls.push({ - login: vars.login as string, - cursor: (vars.cursor as string | null) ?? null, - pageSize: vars.pageSize as number, - }); - return responses[i++]; - }) as unknown as typeof graphql; - return { fn, calls }; +function mockRest(pages: RestRepo[][]) { + const calls: Array> = []; + const rest = { + repos: { listForOrg: () => undefined }, + paginate: { + iterator: (_route: unknown, opts: Record) => { + calls.push(opts); + return (async function* () { + for (const data of pages) yield { data }; + })(); + }, + }, + } as unknown as GitHubClient; + return { rest, calls }; } describe("discoverOrgRepos", () => { - test("collects a single page and reports the org total", async () => { - const { fn } = mockGql([page([node("alpha"), node("beta")], { total: 2 })]); + test("collects a single page and reports the running total", async () => { + const { rest } = mockRest([[repo("alpha"), repo("beta")]]); - const result = await discoverOrgRepos(fn, "acme"); + const result = await discoverOrgRepos(rest, "acme"); expect(result.org).toBe("acme"); expect(result.total).toBe(2); expect(result.repos.map((r) => r.name)).toEqual(["alpha", "beta"]); }); - test("pages through results, threading the cursor forward", async () => { - const { fn, calls } = mockGql([ - page([node("a"), node("b")], { total: 3, hasNextPage: true, endCursor: "CUR1" }), - page([node("c")], { total: 3, hasNextPage: false }), - ]); + test("concatenates repos across pages", async () => { + const { rest } = mockRest([[repo("a"), repo("b")], [repo("c")]]); - const result = await discoverOrgRepos(fn, "acme"); + const result = await discoverOrgRepos(rest, "acme"); expect(result.repos.map((r) => r.name)).toEqual(["a", "b", "c"]); - // First call has no cursor; second resumes from the first page's endCursor. - expect(calls).toHaveLength(2); - expect(calls[0]?.cursor).toBeNull(); - expect(calls[1]?.cursor).toBe("CUR1"); + expect(result.total).toBe(3); }); test("invokes the progress callback once per page with running totals", async () => { - const { fn } = mockGql([ - page([node("a"), node("b")], { total: 3, hasNextPage: true, endCursor: "CUR1" }), - page([node("c")], { total: 3, hasNextPage: false }), - ]); + const { rest } = mockRest([[repo("a"), repo("b")], [repo("c")]]); const progress: DiscoveryProgress[] = []; - await discoverOrgRepos(fn, "acme", (p) => progress.push(p)); + await discoverOrgRepos(rest, "acme", (p) => progress.push(p)); expect(progress).toEqual([ - { org: "acme", discovered: 2, total: 3, page: 1 }, + { org: "acme", discovered: 2, total: 2, page: 1 }, { org: "acme", discovered: 3, total: 3, page: 2 }, ]); }); - test("maps GraphQL fields, including null diskUsage and a missing default branch", async () => { - const { fn } = mockGql([ - page( - [ - node("empty-repo", { - visibility: "INTERNAL", - isArchived: true, - isEmpty: true, - diskUsage: null, - defaultBranchRef: null, - hasWikiEnabled: true, - hasDiscussionsEnabled: true, - pushedAt: null, - }), - ], - { total: 1 }, - ), + test("requests every repo type, 100 per page, sorted by name", async () => { + const { rest, calls } = mockRest([[repo("a")]]); + + await discoverOrgRepos(rest, "acme"); + + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ + org: "acme", + per_page: 100, + type: "all", + sort: "full_name", + direction: "asc", + }); + }); + + test("maps REST fields, normalizing visibility and proxying empty via size", async () => { + const { rest } = mockRest([ + [ + repo("empty-repo", { + visibility: "internal", + archived: true, + size: 0, + default_branch: undefined, + has_wiki: true, + has_discussions: true, + pushed_at: null, + }), + ], ]); - const [repo] = (await discoverOrgRepos(fn, "acme")).repos; + const [mapped] = (await discoverOrgRepos(rest, "acme")).repos; - expect(repo).toEqual({ + expect(mapped).toEqual({ name: "empty-repo", nameWithOwner: "acme/empty-repo", visibility: "INTERNAL", isArchived: true, isFork: false, - isEmpty: true, - diskUsageKb: null, + isEmpty: true, // size 0 → empty proxy + diskUsageKb: 0, hasWiki: true, hasIssues: true, hasProjects: false, @@ -170,117 +148,37 @@ describe("discoverOrgRepos", () => { defaultBranch: null, pushedAt: null, updatedAt: "2026-01-02T00:00:00Z", - issuesCount: 0, - pullRequestsCount: 0, - branchesCount: 0, - tagsCount: 0, - releasesCount: 0, }); }); - test("maps the content-volume counts from the discovery page", async () => { - const { fn } = mockGql([ - page( - [ - node("busy", { - issues: { totalCount: 42 }, - pullRequests: { totalCount: 17 }, - branches: { totalCount: 6 }, - tags: { totalCount: 9 }, - releases: { totalCount: 5 }, - }), - ], - { total: 1 }, - ), + test("derives visibility from `private` when the field is absent", async () => { + const { rest } = mockRest([ + [ + repo("pub", { visibility: undefined, private: false }), + repo("prv", { visibility: undefined, private: true }), + ], ]); - const [repo] = (await discoverOrgRepos(fn, "acme")).repos; - - expect(repo?.issuesCount).toBe(42); - expect(repo?.pullRequestsCount).toBe(17); - expect(repo?.branchesCount).toBe(6); - expect(repo?.tagsCount).toBe(9); - expect(repo?.releasesCount).toBe(5); - }); - - test("handles an organization with no repositories", async () => { - const { fn } = mockGql([page([], { total: 0 })]); - - const result = await discoverOrgRepos(fn, "empty-org"); - - expect(result.total).toBe(0); - expect(result.repos).toEqual([]); - }); - - test("throws when the organization is missing or inaccessible", async () => { - const { fn } = mockGql([{ organization: null }]); + const repos = (await discoverOrgRepos(rest, "acme")).repos; - await expect(discoverOrgRepos(fn, "ghost")).rejects.toThrow(/not found or not accessible/i); + expect(repos.map((r) => r.visibility)).toEqual(["PUBLIC", "PRIVATE"]); }); - test("stops if a page claims another page but does not advance the cursor", async () => { - // hasNextPage is true forever, but endCursor never changes — the guard must - // break the loop instead of looping forever. - const stuck = page([node("a")], { total: 5, hasNextPage: true, endCursor: null }); - const { fn, calls } = mockGql([stuck, stuck, stuck]); + test("treats a non-zero size as non-empty", async () => { + const { rest } = mockRest([[repo("busy", { size: 4096 })]]); - const result = await discoverOrgRepos(fn, "acme"); + const [mapped] = (await discoverOrgRepos(rest, "acme")).repos; - expect(calls).toHaveLength(1); - expect(result.repos.map((r) => r.name)).toEqual(["a"]); + expect(mapped?.isEmpty).toBe(false); + expect(mapped?.diskUsageKb).toBe(4096); }); - test("halves the page and retries the same cursor when a page times out", async () => { - const calls: Array<{ cursor: string | null; pageSize: number }> = []; - let attempt = 0; - const fn = (async (_q: string, vars: Record) => { - calls.push({ - cursor: (vars.cursor as string | null) ?? null, - pageSize: vars.pageSize as number, - }); - attempt += 1; - // The first attempt (100-wide) times out with a GitHub 502; the retry at - // half the size succeeds — no repos are skipped because the cursor holds. - if (attempt === 1) throw Object.assign(new Error("502 Bad Gateway"), { status: 502 }); - return page([node("a"), node("b")], { total: 2 }); - }) as unknown as typeof graphql; - - const result = await discoverOrgRepos(fn, "acme"); - - expect(result.repos.map((r) => r.name)).toEqual(["a", "b"]); - expect(calls).toHaveLength(2); - expect(calls[0]?.pageSize).toBe(100); // first attempt at the full page - expect(calls[1]?.pageSize).toBe(50); // retried at half after the 502 - expect(calls[1]?.cursor).toBeNull(); // same cursor — nothing skipped - }); - - test("keeps the shrunk page size for the rest of the crawl", async () => { - const sizes: number[] = []; - let attempt = 0; - const fn = (async (_q: string, vars: Record) => { - sizes.push(vars.pageSize as number); - attempt += 1; - if (attempt === 1) throw Object.assign(new Error("504"), { status: 504 }); - // After the shrink, page 1 (at 50) has a next page; page 2 must reuse 50. - if (attempt === 2) - return page([node("a")], { total: 2, hasNextPage: true, endCursor: "CUR" }); - return page([node("b")], { total: 2 }); - }) as unknown as typeof graphql; - - const result = await discoverOrgRepos(fn, "acme"); - - expect(result.repos.map((r) => r.name)).toEqual(["a", "b"]); - expect(sizes).toEqual([100, 50, 50]); // shrunk once, then held - }); + test("handles an organization with no repositories", async () => { + const { rest } = mockRest([]); - test("surfaces a non-timeout error without retrying", async () => { - let attempts = 0; - const fn = (async () => { - attempts += 1; - throw Object.assign(new Error("Bad credentials"), { status: 401 }); - }) as unknown as typeof graphql; + const result = await discoverOrgRepos(rest, "empty-org"); - await expect(discoverOrgRepos(fn, "acme")).rejects.toThrow("Bad credentials"); - expect(attempts).toBe(1); // a 401 is not a timeout — no page-size retry + expect(result.total).toBe(0); + expect(result.repos).toEqual([]); }); }); diff --git a/src/lib/server/profile/discover.ts b/src/lib/server/profile/discover.ts index 88d7ba1..bf79e40 100644 --- a/src/lib/server/profile/discover.ts +++ b/src/lib/server/profile/discover.ts @@ -1,200 +1,108 @@ /** * Source discovery — the Profiler's bulk, breadth-first crawl. * - * Pages a single GraphQL query over an organization's repositories and maps - * each node to a `DiscoveredRepo`. The `gql` client is injected (the same - * `typeof graphql` the rest of github.ts passes around) so this orchestration — - * pagination, cursor threading, mapping, progress — is unit-testable without a - * network or a live Octokit. + * Lists every repository in an organization via the REST `GET /orgs/{org}/repos` + * endpoint (the `minimal-repository` shape), paged through the `Link` header. + * REST listing is the reliable primitive for "give me the repos": it's an + * indexed, paginated read that never hits GraphQL's ~10s query timeout (which a + * 100-wide GraphQL page carrying several connection `totalCount`s can tip over + * on a large org). The per-repo indexed counts (issues, PRs, branches, tags, + * releases, …) are gathered afterwards by the batched GraphQL counts pass, where + * a too-expensive chunk degrades a repo to zeros instead of failing the list. + * + * The `rest` client is injected, so this orchestration — pagination, mapping, + * progress — is unit-testable without a network or a live Octokit. */ -import type { graphql } from "@octokit/graphql"; -import { isTimeoutError } from "./graphql-errors"; +import type { GitHubClient } from "$lib/server/core/github"; import type { DiscoveredRepo, DiscoveryProgress, OrgDiscovery, RepoVisibility } from "./types"; -/** GraphQL caps `repositories(first:)` at 100. */ +/** REST caps `per_page` at 100. */ const REPOS_PER_PAGE = 100; -/** Smallest discovery page we'll shrink to before surfacing a timeout. A page - * of only indexed counts should never time out this small; if it does, the - * error is real and worth surfacing rather than retrying forever. */ -const MIN_PAGE_SIZE = 10; - -/** Raw shape of one repository node in the discovery query. */ -interface RepoNode { +/** + * The subset of `minimal-repository` (the `GET /orgs/{org}/repos` item) that the + * discovery spine reads. The real Octokit type is a superset, so it's assignable + * to this; everything but the identity fields is optional, matching the schema. + */ +interface RestRepoSummary { name: string; - nameWithOwner: string; - visibility: RepoVisibility; - isArchived: boolean; - isFork: boolean; - isEmpty: boolean; - diskUsage: number | null; - hasWikiEnabled: boolean; - hasIssuesEnabled: boolean; - hasProjectsEnabled: boolean; - hasDiscussionsEnabled: boolean; - defaultBranchRef: { name: string } | null; - pushedAt: string | null; - updatedAt: string | null; - issues: { totalCount: number }; - pullRequests: { totalCount: number }; - branches: { totalCount: number }; - tags: { totalCount: number }; - releases: { totalCount: number }; + full_name: string; + private: boolean; + fork: boolean; + visibility?: string; + archived?: boolean; + size?: number; + has_wiki?: boolean; + has_issues?: boolean; + has_projects?: boolean; + has_discussions?: boolean; + default_branch?: string; + pushed_at?: string | null; + updated_at?: string | null; } -/** Raw shape of the paged discovery query response. */ -interface OrgReposResult { - organization: { - repositories: { - totalCount: number; - pageInfo: { hasNextPage: boolean; endCursor: string | null }; - nodes: RepoNode[]; - }; - } | null; +/** Normalize a REST visibility (`"public"`) to the GraphQL enum (`"PUBLIC"`). */ +function toVisibility(repo: RestRepoSummary): RepoVisibility { + const raw = repo.visibility ?? (repo.private ? "private" : "public"); + return raw.toUpperCase() as RepoVisibility; } -const ORG_REPOS_QUERY = `query orgRepos($login: String!, $cursor: String, $pageSize: Int!) { - organization(login: $login) { - repositories(first: $pageSize, after: $cursor, orderBy: { field: NAME, direction: ASC }) { - totalCount - pageInfo { hasNextPage endCursor } - nodes { - name - nameWithOwner - visibility - isArchived - isFork - isEmpty - diskUsage - hasWikiEnabled - hasIssuesEnabled - hasProjectsEnabled - hasDiscussionsEnabled - defaultBranchRef { name } - pushedAt - updatedAt - issues { totalCount } - pullRequests { totalCount } - branches: refs(refPrefix: "refs/heads/", first: 1) { totalCount } - tags: refs(refPrefix: "refs/tags/", first: 1) { totalCount } - releases { totalCount } - } - } - } -}`; - -/** Map a raw GraphQL node to the discovery-facing shape. */ -function toDiscoveredRepo(node: RepoNode): DiscoveredRepo { +/** Map a REST `minimal-repository` to the discovery-facing spine. */ +function toDiscoveredRepo(repo: RestRepoSummary): DiscoveredRepo { return { - name: node.name, - nameWithOwner: node.nameWithOwner, - visibility: node.visibility, - isArchived: node.isArchived, - isFork: node.isFork, - isEmpty: node.isEmpty, - diskUsageKb: node.diskUsage, - hasWiki: node.hasWikiEnabled, - hasIssues: node.hasIssuesEnabled, - hasProjects: node.hasProjectsEnabled, - hasDiscussions: node.hasDiscussionsEnabled, - defaultBranch: node.defaultBranchRef?.name ?? null, - pushedAt: node.pushedAt, - updatedAt: node.updatedAt, - issuesCount: node.issues.totalCount, - pullRequestsCount: node.pullRequests.totalCount, - branchesCount: node.branches.totalCount, - tagsCount: node.tags.totalCount, - releasesCount: node.releases.totalCount, + name: repo.name, + nameWithOwner: repo.full_name, + visibility: toVisibility(repo), + isArchived: repo.archived ?? false, + isFork: repo.fork, + // REST has no `is_empty`; a freshly-created repo with no commits reports + // size 0, so it's a sound proxy for the "· empty" display badge. + isEmpty: (repo.size ?? 0) === 0, + diskUsageKb: repo.size ?? null, + hasWiki: repo.has_wiki ?? false, + hasIssues: repo.has_issues ?? false, + hasProjects: repo.has_projects ?? false, + hasDiscussions: repo.has_discussions ?? false, + defaultBranch: repo.default_branch ?? null, + pushedAt: repo.pushed_at ?? null, + updatedAt: repo.updated_at ?? null, }; } /** - * Fetch one discovery page, halving the page size and retrying the *same* - * cursor each time GitHub times the request out (502/504), down to - * `MIN_PAGE_SIZE`. A 100-wide page of indexed counts is normally cheap, but an - * org with repos that have very many refs/issues can tip one page over GitHub's - * ~10s limit; shrinking keeps discovery — the only thing between the user and "a - * list of repos" — completing instead of failing the whole run. Returns the page - * plus the size that actually worked so the caller can keep using it for the - * remaining pages rather than re-hitting the timeout on every one. - */ -async function fetchPageAdaptive( - gql: typeof graphql, - org: string, - cursor: string | null, - pageSize: number, -): Promise<{ result: OrgReposResult; pageSize: number }> { - let size = pageSize; - for (;;) { - try { - const result = await gql(ORG_REPOS_QUERY, { - login: org, - cursor, - pageSize: size, - }); - return { result, pageSize: size }; - } catch (err) { - if (isTimeoutError(err) && size > MIN_PAGE_SIZE) { - const smaller = Math.max(MIN_PAGE_SIZE, Math.floor(size / 2)); - console.warn( - `[profile] discovery page of ${size} timed out for '${org}'; retrying this page at ${smaller}`, - ); - size = smaller; - continue; - } - throw err; - } - } -} - -/** - * Discover every repository in an organization via paged GraphQL. + * Discover every repository in an organization via the paged REST listing. * - * @param gql Injected GraphQL client (`createSingleClient(...).graphql`). + * @param rest Injected REST client (`getSourceClients().rest`). * @param org Organization login to crawl. * @param onProgress Optional callback invoked once per page with running totals. + * REST doesn't report an org total up front, so `total` tracks + * the running count and lands on the exact figure on the last + * page. * @returns The org's repository total and the full `DiscoveredRepo[]`. - * @throws If the organization is missing or not accessible. */ export async function discoverOrgRepos( - gql: typeof graphql, + rest: GitHubClient, org: string, onProgress?: (progress: DiscoveryProgress) => void, ): Promise { const repos: DiscoveredRepo[] = []; - let cursor: string | null = null; - let total = 0; let page = 0; - // Adaptive page size: starts at the 100-wide max and only shrinks (and stays - // shrunk) if a page times out, so small orgs pay nothing for the resilience. - let pageSize = REPOS_PER_PAGE; - for (;;) { - const { result, pageSize: usedSize } = await fetchPageAdaptive(gql, org, cursor, pageSize); - pageSize = usedSize; + const iterator = rest.paginate.iterator(rest.repos.listForOrg, { + org, + per_page: REPOS_PER_PAGE, + type: "all", + sort: "full_name", + direction: "asc", + }); - const connection = result.organization?.repositories; - if (!connection) { - throw new Error(`Organization '${org}' not found or not accessible`); - } - - total = connection.totalCount; + for await (const response of iterator) { page += 1; - for (const node of connection.nodes) { - repos.push(toDiscoveredRepo(node)); - } - - onProgress?.({ org, discovered: repos.length, total, page }); - - // Stop at the last page. The cursor-advance guard also breaks the loop if a - // response claims another page but doesn't actually move the cursor forward, - // so a malformed `pageInfo` can never spin forever. - const { hasNextPage, endCursor } = connection.pageInfo; - if (!hasNextPage || !endCursor || endCursor === cursor) { - break; + for (const repo of response.data) { + repos.push(toDiscoveredRepo(repo)); } - cursor = endCursor; + onProgress?.({ org, discovered: repos.length, total: repos.length, page }); } - return { org, total, repos }; + return { org, total: repos.length, repos }; } diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index 94e6478..fcfb76f 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -19,7 +19,7 @@ import { ZERO_ORG_RESOURCES, } from "./types"; -const gql = {} as never; // the fakes ignore it +const clients = {} as never; // the fakes ignore it function discovered(name: string, over: Partial = {}): DiscoveredRepo { return { @@ -37,11 +37,6 @@ function discovered(name: string, over: Partial = {}): Discovere defaultBranch: "main", pushedAt: null, updatedAt: null, - issuesCount: 0, - pullRequestsCount: 0, - branchesCount: 0, - tagsCount: 0, - releasesCount: 0, ...over, }; } @@ -63,6 +58,11 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo usesLfs: false, releaseAssetBytes: 0, workflowFileCount: 0, + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, + releasesCount: 0, ...over, }; } @@ -117,7 +117,7 @@ describe("runProfile", () => { const repos = [discovered("alpha"), discovered("beta", { hasWiki: true })]; const run = await runProfile( - gql, + clients, { id: "run-1", org: "acme", sourceApiUrl: "https://ghes.example.com/api/v3" }, undefined, deps(repos, { alpha: { discussionsCount: 2 } }), @@ -139,7 +139,7 @@ describe("runProfile", () => { test("records the org ruleset count on the run", async () => { const run = await runProfile( - gql, + clients, { id: "run-rs", org: "acme", sourceApiUrl: "u" }, undefined, deps([discovered("alpha")], {}, 3), @@ -149,7 +149,7 @@ describe("runProfile", () => { test("records the org resource counts on the run", async () => { const run = await runProfile( - gql, + clients, { id: "run-or", org: "acme", sourceApiUrl: "u" }, undefined, deps([discovered("alpha")], {}, 0, { actionsSecrets: 5, selfHostedRunners: 2 }), @@ -175,7 +175,7 @@ describe("runProfile", () => { }; await runProfile( - gql, + clients, { id: "disc", org: "acme", sourceApiUrl: "u" }, (p) => progress.push(p), withDiscoveryProgress, @@ -197,7 +197,7 @@ describe("runProfile", () => { }; const run = await runProfile( - gql, + clients, { id: "http", org: "acme", sourceApiUrl: "u" }, undefined, failing, @@ -213,7 +213,7 @@ describe("runProfile", () => { const progress: ProfileProgress[] = []; await runProfile( - gql, + clients, { id: "r", org: "acme", sourceApiUrl: "u" }, (p) => progress.push(p), deps(repos), @@ -230,7 +230,7 @@ describe("runProfile", () => { test("completes an empty organization with zero profiled repos", async () => { const run = await runProfile( - gql, + clients, { id: "r", org: "acme", sourceApiUrl: "u" }, undefined, deps([]), @@ -249,7 +249,7 @@ describe("runProfile", () => { }; const run = await runProfile( - gql, + clients, { id: "r", org: "acme", sourceApiUrl: "u" }, undefined, failingDeps, @@ -270,7 +270,7 @@ describe("runProfile", () => { }; const run = await runProfile( - gql, + clients, { id: "r", org: "acme", sourceApiUrl: "u" }, undefined, partialDeps, @@ -291,7 +291,7 @@ describe("runProfile", () => { // throws in the counts pass; every repo is still listed (recorded at // discovery) and the run completes. const repos = Array.from({ length: 26 }, (_, i) => - discovered(`r${String(i).padStart(2, "0")}`, { releasesCount: 1 }), + discovered(`r${String(i).padStart(2, "0")}`), ); const chunkedDeps: Partial = { discover: async () => ({ org: "acme", total: repos.length, repos }), @@ -302,7 +302,7 @@ describe("runProfile", () => { }; const run = await runProfile( - gql, + clients, { id: "r", org: "acme", sourceApiUrl: "u" }, undefined, chunkedDeps, @@ -313,20 +313,22 @@ describe("runProfile", () => { }); test("batches release-free repos wide (no scan) and release-bearing repos narrow", async () => { - // The details pass partitions by releases: 2 release-free repos share one - // lite chunk (scanReleases: false); the 3 with releases share one full - // chunk (scanReleases: true). + // The details pass partitions by releases — which come from the counts pass: + // 2 release-free repos share one lite chunk (scanReleases: false); the 3 with + // releases share one full chunk (scanReleases: true). const repos = [ - discovered("a", { releasesCount: 0 }), - discovered("b", { releasesCount: 2 }), - discovered("c", { releasesCount: 0 }), - discovered("d", { releasesCount: 5 }), - discovered("e", { releasesCount: 1 }), + discovered("a"), + discovered("b"), + discovered("c"), + discovered("d"), + discovered("e"), ]; + const releasesByName: Record = { a: 0, b: 2, c: 0, d: 5, e: 1 }; const calls: Array<{ size: number; scanReleases: boolean | undefined }> = []; const recordingDeps: Partial = { discover: async () => ({ org: "acme", total: repos.length, repos }), - augmentCounts: async (_gql, c) => c.map((r) => countsSignals(r)), + augmentCounts: async (_gql, c) => + c.map((r) => countsSignals(r, { releasesCount: releasesByName[r.name] ?? 0 })), augmentDetails: async (_gql, c, opts) => { calls.push({ size: c.length, scanReleases: opts?.scanReleases }); return c.map((r) => detailsFor(r)); @@ -334,7 +336,7 @@ describe("runProfile", () => { }; const run = await runProfile( - gql, + clients, { id: "r", org: "acme", sourceApiUrl: "u" }, undefined, recordingDeps, @@ -350,7 +352,7 @@ describe("runProfile", () => { test("persists the run before crawling (a created run exists by completion)", async () => { await runProfile( - gql, + clients, { id: "persisted", org: "acme", sourceApiUrl: "u" }, undefined, deps([discovered("a")]), diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index 4700be3..0019bea 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -16,6 +16,7 @@ * network. */ import type { graphql } from "@octokit/graphql"; +import type { GitHubClient } from "$lib/server/core/github"; import { analyzeRepo } from "./analyze"; import { augmentRepoCounts, augmentRepoDetails, baseSignals } from "./augment"; import { discoverOrgRepos } from "./discover"; @@ -86,6 +87,13 @@ function describeError(err: unknown): string { return err instanceof Error ? err.message : String(err); } +/** Source clients the runner needs: REST for discovery (the reliable repo + * listing), GraphQL for the batched augment passes (indexed counts + details). */ +export interface ProfileClients { + gql: typeof graphql; + rest: GitHubClient; +} + /** Injectable crawl primitives (defaults hit the real GraphQL helpers). */ export interface ProfileRunnerDeps { discover: typeof discoverOrgRepos; @@ -112,14 +120,14 @@ const DEFAULT_DEPS: ProfileRunnerDeps = { /** * Profile one organization end to end and persist the result. * - * @param gql Injected GraphQL client for the source. + * @param clients Injected source clients (REST for discovery, GraphQL for augment). * @param input Run id, org login, and the source API URL being profiled. * @param onProgress Optional per-repo progress callback (drives SSE later). * @param deps Injectable crawl primitives (defaults to the real ones). * @returns The final persisted run — `completed` or `failed`. */ export async function runProfile( - gql: typeof graphql, + clients: ProfileClients, input: { id: string; org: string; sourceApiUrl: string }, onProgress?: (progress: ProfileProgress) => void, deps: Partial = {}, @@ -130,19 +138,14 @@ export async function runProfile( console.log(`[profile] run ${input.id} started — org=${input.org}, source=${input.sourceApiUrl}`); try { - // Set the run total as soon as the FIRST discovery page reports it (the org - // total is known from page 1) and nudge any live watcher, so the detail page - // shows movement during the otherwise-silent discovery phase instead of - // sitting at 0/0. - let totalSet = false; - const discovery = await d.discover(gql, input.org, (p) => { - if (!totalSet && p.total > 0) { - totalSet = true; - setProfileRunTotal(input.id, p.total); - onProgress?.({ runId: input.id, profiled: 0, total: p.total, repo: "" }); - } + // REST discovery lists the whole org cheaply. It doesn't report an org total + // up front, so each page nudges the run total with the running count (and a + // live watcher with it); the exact total lands once discovery completes. + const discovery = await d.discover(clients.rest, input.org, (p) => { + setProfileRunTotal(input.id, p.total); + onProgress?.({ runId: input.id, profiled: 0, total: p.total, repo: "" }); }); - if (!totalSet) setProfileRunTotal(input.id, discovery.total); + setProfileRunTotal(input.id, discovery.total); console.log( `[profile] run ${input.id} discovered ${discovery.total} repo(s) in ${Date.now() - startedMs}ms`, ); @@ -190,7 +193,7 @@ export async function runProfile( const c = chunks[next++]; if (!c) break; try { - const chunkSignals = await d.augmentCounts(gql, c); + const chunkSignals = await d.augmentCounts(clients.gql, c); for (const signals of chunkSignals) { recordRepoProfile(input.id, signals, analyzeRepo(signals)); signalsByRepo.set(signals.nameWithOwner, signals); @@ -220,12 +223,16 @@ export async function runProfile( // ── Pass 2: verification details (best-effort) ───────────────────────── // The expensive checks. Repos with no releases skip the release-asset scan - // and batch wider; repos with releases batch narrower. Each result merges - // onto the already-recorded counts and re-records. This pass is NON-fatal: - // the counts are already persisted, so a details failure enriches less but - // never fails the run. - const withReleases = discovery.repos.filter((r) => r.releasesCount > 0); - const withoutReleases = discovery.repos.filter((r) => r.releasesCount === 0); + // and batch wider; repos with releases batch narrower. The release count + // comes from the counts pass above (signalsByRepo), defaulting to 0 for any + // repo whose counts didn't land — so it simply skips the scan. Each result + // merges onto the already-recorded counts and re-records. This pass is + // NON-fatal: the counts are already persisted, so a details failure enriches + // less but never fails the run. + const releasesOf = (nameWithOwner: string) => + signalsByRepo.get(nameWithOwner)?.releasesCount ?? 0; + const withReleases = discovery.repos.filter((r) => releasesOf(r.nameWithOwner) > 0); + const withoutReleases = discovery.repos.filter((r) => releasesOf(r.nameWithOwner) === 0); const tasks: Array<{ repos: typeof discovery.repos; scanReleases: boolean }> = [ ...chunk(withoutReleases, DETAILS_CHUNK_LITE).map((repos) => ({ repos, @@ -240,7 +247,7 @@ export async function runProfile( const task = tasks[next++]; if (!task) break; try { - const details = await d.augmentDetails(gql, task.repos, { + const details = await d.augmentDetails(clients.gql, task.repos, { scanReleases: task.scanReleases, }); for (const det of details) { diff --git a/src/lib/server/profile/service.test.ts b/src/lib/server/profile/service.test.ts index a87a40d..f16ed23 100644 --- a/src/lib/server/profile/service.test.ts +++ b/src/lib/server/profile/service.test.ts @@ -31,11 +31,6 @@ function discovered(name: string): DiscoveredRepo { defaultBranch: "main", pushedAt: null, updatedAt: null, - issuesCount: 0, - pullRequestsCount: 0, - branchesCount: 0, - tagsCount: 0, - releasesCount: 0, }; } @@ -56,6 +51,11 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo usesLfs: false, releaseAssetBytes: 0, workflowFileCount: 0, + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, + releasesCount: 0, ...over, }; } @@ -93,9 +93,9 @@ function serviceDeps( sourceApiUrl: "https://ghes.example.com/api/v3", }; }, - run: (gql, input, onProgress) => { + run: (clients, input, onProgress) => { state.lastInput = input; - state.runPromise = runProfile(gql, input, onProgress, { + state.runPromise = runProfile(clients, input, onProgress, { discover: async (): Promise => ({ org: input.org, total: repos.length, diff --git a/src/lib/server/profile/service.ts b/src/lib/server/profile/service.ts index a8810f1..863dde4 100644 --- a/src/lib/server/profile/service.ts +++ b/src/lib/server/profile/service.ts @@ -107,7 +107,7 @@ export function startOrgProfile(org: string, deps: ProfileServiceDeps = DEFAULT_ deps .run( - gql, + { gql, rest }, { id, org, sourceApiUrl }, (p) => publishProfileEvent(id, { diff --git a/src/lib/server/profile/types.ts b/src/lib/server/profile/types.ts index f9d763c..f553b6a 100644 --- a/src/lib/server/profile/types.ts +++ b/src/lib/server/profile/types.ts @@ -16,8 +16,10 @@ export type RepoVisibility = "PUBLIC" | "PRIVATE" | "INTERNAL"; /** * One repository as surfaced by the bulk discovery crawl. Every field comes - * from a single GraphQL page — no per-repo REST calls — so this is cheap to - * gather at org scale and forms the spine each later augmentation hangs off. + * from the REST `GET /orgs/{org}/repos` listing (the `minimal-repository` + * shape) — one cheap, reliable paged call that lists the whole org without ever + * hitting GraphQL's 10s query timeout. This is the spine each later GraphQL + * augmentation (the indexed `totalCount`s) hangs off. */ export interface DiscoveredRepo { /** Repo name (without owner). */ @@ -27,8 +29,9 @@ export interface DiscoveredRepo { visibility: RepoVisibility; isArchived: boolean; isFork: boolean; + /** Proxy for “no commits”: the REST listing reports `size === 0`. */ isEmpty: boolean; - /** GraphQL `diskUsage` in KiB; null when the viewer can't see it. */ + /** REST `size` in KiB; null when the listing omits it. */ diskUsageKb: number | null; hasWiki: boolean; hasIssues: boolean; @@ -40,20 +43,6 @@ export interface DiscoveredRepo { pushedAt: string | null; /** ISO timestamp of the last update, or null. */ updatedAt: string | null; - /** Issues (`issues.totalCount`, all states; excludes PRs) — migration scale. */ - issuesCount: number; - /** Pull requests (`pullRequests.totalCount`, all states) — migration scale. */ - pullRequestsCount: number; - /** Branches (`refs` under `refs/heads/`) — migration scale. */ - branchesCount: number; - /** Tags (`refs` under `refs/tags/`) — migration scale. */ - tagsCount: number; - /** - * Releases (`releases.totalCount`); GHES releases don't migrate at all. - * Gathered in discovery (free on the 100-wide page) so the augment pass can - * skip the heavy release-asset scan for repos that have none. - */ - releasesCount: number; } /** Progress emitted after each discovery page (drives SSE / logging later). */ @@ -136,6 +125,24 @@ export interface RepoSignals extends DiscoveredRepo { * means there's run history that will be lost. */ workflowFileCount: number; + // ── Content-volume counts (migration scale) ────────────────────────────── + // Indexed GraphQL `totalCount`s gathered in the cheap counts pass (they live + // here, not on the REST discovery spine, because REST doesn't expose the + // GraphQL-equivalent counts). Each defaults to 0 until the counts pass fills + // it; a counts timeout degrades a repo to 0 rather than dropping it. + /** Issues (`issues.totalCount`, all states; excludes PRs). */ + issuesCount: number; + /** Pull requests (`pullRequests.totalCount`, all states). */ + pullRequestsCount: number; + /** Branches (`refs` under `refs/heads/`). */ + branchesCount: number; + /** Tags (`refs` under `refs/tags/`). */ + tagsCount: number; + /** + * Releases (`releases.totalCount`); GHES releases don't migrate at all. Drives + * the details pass's release-asset-scan partition (repos with 0 skip it). + */ + releasesCount: number; } // ── Persistence ────────────────────────────────────────────────────────────── From 856a448257eca458378947ee016fb564047aa9c6 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Mon, 15 Jun 2026 00:23:06 -0700 Subject: [PATCH 09/11] feat(profile): add client-side search and pagination to the repo list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repositories table rendered every profiled repo in one long list. Added a search box (filters by owner/name) and 25-per-page pagination, both client-side since every repo is already loaded and live-refreshed — so filtering and paging are instant. Typing resets to page 1; the page index clamps when the filtered set shrinks; a footer shows the visible range and total (with a 'filtered from N' note); a no-match empty state appears when a search yields nothing. --- src/routes/profile/[id]/+page.svelte | 84 ++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 5 deletions(-) diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index f2a5266..03793e5 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -148,6 +148,29 @@ ]; } + // ── Repository list: client-side search + pagination ─────────────────────── + // Every repo is already loaded (and live-refreshed), so filtering and paging + // run on the client for instant feedback. Search matches the owner/name. + const REPOS_PER_PAGE = 25; + let repoSearch = $state(''); + let repoPage = $state(1); + + const filteredRepos = $derived.by(() => { + const q = repoSearch.trim().toLowerCase(); + return q ? repos.filter((r) => r.nameWithOwner.toLowerCase().includes(q)) : repos; + }); + // Clamp the shown page when the filtered set shrinks (no effect needed): the + // prev/next buttons set `repoPage`, and `repoPageSafe` keeps it in range. + const repoTotalPages = $derived(Math.max(1, Math.ceil(filteredRepos.length / REPOS_PER_PAGE))); + const repoPageSafe = $derived(Math.min(repoPage, repoTotalPages)); + const pagedRepos = $derived( + filteredRepos.slice((repoPageSafe - 1) * REPOS_PER_PAGE, repoPageSafe * REPOS_PER_PAGE) + ); + const repoRangeStart = $derived( + filteredRepos.length === 0 ? 0 : (repoPageSafe - 1) * REPOS_PER_PAGE + 1 + ); + const repoRangeEnd = $derived(Math.min(repoPageSafe * REPOS_PER_PAGE, filteredRepos.length)); + const pct = $derived(run.totalRepos > 0 ? Math.round((run.profiledRepos / run.totalRepos) * 100) : 0); async function refresh() { @@ -426,10 +449,29 @@
-

- - Repositories -

+
+

+ + Repositories +

+ {#if repos.length > 0} +
+ + (repoPage = 1)} + placeholder="Filter repositories…" + aria-label="Filter repositories" + class="w-64 rounded-md border border-gray-700 bg-gray-950 py-1.5 pl-9 pr-3 text-sm text-gray-100 placeholder:text-gray-500 focus:border-violet-500 focus:outline-none" + /> +
+ {/if} +
{#if repos.length === 0}
@@ -444,6 +486,11 @@ {/if}

+ {:else if filteredRepos.length === 0} +
+ +

No repositories match “{repoSearch}”

+
{:else}
@@ -456,7 +503,7 @@ - {#each repos as repo (repo.nameWithOwner)} + {#each pagedRepos as repo (repo.nameWithOwner)}
+
+ + Showing {repoRangeStart.toLocaleString()}–{repoRangeEnd.toLocaleString()} of {filteredRepos.length.toLocaleString()} + {#if filteredRepos.length !== repos.length}(filtered from {repos.length.toLocaleString()}){/if} + + {#if repoTotalPages > 1} +
+ + Page {repoPageSafe} / {repoTotalPages} + +
+ {/if} +
{/if}
From 720cc4b9b2f969c2a7a057ff9f92a094107c1414 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Mon, 15 Jun 2026 00:30:50 -0700 Subject: [PATCH 10/11] refactor(profile): use the shared Pagination component for the repo list The profile repo list had a bespoke Prev/Next footer that looked different from the migrate pages, which use the shared $lib/components/Pagination.svelte (numbered pages with ellipsis + range summary). Swapped the profile list onto that same component so both surfaces are visually consistent. Its onPageChange callback drives the existing client-side page state, and a migrate-style count now sits in the section header. Removed the duplicated footer markup and the now-unused range deriveds. --- src/routes/profile/[id]/+page.svelte | 42 +++++++--------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index 03793e5..f7f978c 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -3,6 +3,7 @@ import type { IconName } from '@primer/octicons'; import { SvelteSet } from 'svelte/reactivity'; import Octicon from '$lib/components/Octicon.svelte'; + import Pagination from '$lib/components/Pagination.svelte'; import { formatHours, formatRepoSize, timeAgo } from '$lib/format'; import { MIGRATION_CONSIDERATIONS } from '$lib/profile/consideration-registry'; import { createReconnectingEventSource } from '$lib/stores/sse-client'; @@ -160,16 +161,12 @@ return q ? repos.filter((r) => r.nameWithOwner.toLowerCase().includes(q)) : repos; }); // Clamp the shown page when the filtered set shrinks (no effect needed): the - // prev/next buttons set `repoPage`, and `repoPageSafe` keeps it in range. + // Pagination control sets `repoPage`, and `repoPageSafe` keeps it in range. const repoTotalPages = $derived(Math.max(1, Math.ceil(filteredRepos.length / REPOS_PER_PAGE))); const repoPageSafe = $derived(Math.min(repoPage, repoTotalPages)); const pagedRepos = $derived( filteredRepos.slice((repoPageSafe - 1) * REPOS_PER_PAGE, repoPageSafe * REPOS_PER_PAGE) ); - const repoRangeStart = $derived( - filteredRepos.length === 0 ? 0 : (repoPageSafe - 1) * REPOS_PER_PAGE + 1 - ); - const repoRangeEnd = $derived(Math.min(repoPageSafe * REPOS_PER_PAGE, filteredRepos.length)); const pct = $derived(run.totalRepos > 0 ? Math.round((run.profiledRepos / run.totalRepos) * 100) : 0); @@ -453,6 +450,7 @@

Repositories + {#if repos.length > 0}({filteredRepos.length.toLocaleString()}){/if}

{#if repos.length > 0}
@@ -588,33 +586,13 @@
-
- - Showing {repoRangeStart.toLocaleString()}–{repoRangeEnd.toLocaleString()} of {filteredRepos.length.toLocaleString()} - {#if filteredRepos.length !== repos.length}(filtered from {repos.length.toLocaleString()}){/if} - - {#if repoTotalPages > 1} -
- - Page {repoPageSafe} / {repoTotalPages} - -
- {/if} -
+ (repoPage = p)} + /> {/if} From ff4ec7a38b39607b6ac0cdf192ba628d8e1a23ba Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Mon, 15 Jun 2026 00:40:31 -0700 Subject: [PATCH 11/11] chore(scripts): rename verify:json to gates and align gate file names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify:json described the output format, not the purpose (running the full gate suite). Renamed the script to 'gates' (scripts/gates.ts) — short, and it literally runs the 11-gate suite. Also aligned the two gate file names to their package.json script keys (convention: file name = script key with ':' replaced by '-'): check-coverage.ts to coverage-check.ts (key coverage:check unchanged), check-boundaries.ts to boundaries.ts (key boundaries unchanged). Updated every reference: package.json, bunfig.toml, Makefile, the CI verify workflow, and the development/architecture/CONTRIBUTING docs. The coverage:check and boundaries script keys are unchanged, so the verify chain and CI are untouched beyond the file paths. CHANGELOG.md left as-is (historical). --- .github/workflows/verify.yml | 4 ++-- CONTRIBUTING.md | 2 +- Makefile | 6 +++--- bunfig.toml | 2 +- docs/architecture.md | 2 +- docs/development.md | 18 +++++++++--------- package.json | 6 +++--- scripts/{check-boundaries.ts => boundaries.ts} | 0 .../{check-coverage.ts => coverage-check.ts} | 2 +- scripts/{verify-json.ts => gates.ts} | 6 +++--- 10 files changed, 24 insertions(+), 24 deletions(-) rename scripts/{check-boundaries.ts => boundaries.ts} (100%) rename scripts/{check-coverage.ts => coverage-check.ts} (97%) rename scripts/{verify-json.ts => gates.ts} (96%) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bed8a63..0c72720 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -23,9 +23,9 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile - # verify:json runs the exact 11-gate suite developers run locally + # `bun run gates` runs the exact 11-gate suite developers run locally # (typecheck · check · lint · format · coverage · dup · deadcode · cycles # · boundaries · build · audit), collects every failure (not fail-fast), # and exits non-zero if any gate is red. - name: Run gate suite - run: bun run verify:json + run: bun run gates diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdc75b3..ec883e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ circular-import, build, and audit gates. CI runs the same suite. See [docs/development.md](docs/development.md#the-gate-suite-verify) for what each gate enforces. -`bun run verify:json` gives the same result as a compact JSON summary, handy for +`bun run gates` gives the same result as a compact JSON summary, handy for scripting or quickly seeing which gate failed. ## What not to do diff --git a/Makefile b/Makefile index 18a9f2c..c917398 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: dev build preview clean docker docker-up docker-down install check typecheck \ test coverage lint lint-fix format format-check audit dup deadcode cycles mutation \ - verify verify-json seed + verify gates seed install: bun install @@ -60,8 +60,8 @@ verify: # Same gates as `verify`, but emits a compact JSON summary (per-gate ok + metrics, # excerpts only on failure). Built for AI/automated consumption; far fewer tokens. -verify-json: - bun run verify:json +gates: + bun run gates clean: rm -rf build/ node_modules/ .svelte-kit/ data/ diff --git a/bunfig.toml b/bunfig.toml index 7028a32..94d739f 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -3,7 +3,7 @@ preload = ["./src/test-setup.ts"] # Collect coverage on every `bun test` for visibility. NOTE: Bun 1.3.x does not # enforce coverageThreshold (it reports but does not fail), so the real gate is -# `scripts/check-coverage.ts` (run via `bun run coverage:check`, in `verify` and +# `scripts/coverage-check.ts` (run via `bun run coverage:check`, in `verify` and # CI). The values below are kept in sync with that script as documentation. # Octokit/network I/O wrappers (github.ts) and deep pipeline orchestration # (migration.ts) are intentionally excluded as integration-tested concerns. diff --git a/docs/architecture.md b/docs/architecture.md index 30fdc90..41c2c3e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -240,7 +240,7 @@ src/ profile/ the Profile workspace pages login/ auth pages api/ REST + SSE endpoints (see api.md) -scripts/ seed, coverage + boundary gates, postbuild, verify:json +scripts/ seed, coverage + boundary checks, postbuild, gates (full verify) ``` The `server/` layering is enforced by the `boundaries` gate: `core/` may not diff --git a/docs/development.md b/docs/development.md index 105c3cf..43b5b1a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -7,7 +7,7 @@ keep the codebase honest. - [Stack](#stack) - [Dev loop](#dev-loop) - [The gate suite (`verify`)](#the-gate-suite-verify) -- [Machine-readable verify (`verify:json`)](#machine-readable-verify-verifyjson) +- [Machine-readable verify (`gates`)](#machine-readable-verify-gates) - [Testing](#testing) - [Discovery tooling](#discovery-tooling) - [Seeding the database](#seeding-the-database) @@ -63,18 +63,18 @@ bun run verify | `check` | `svelte-check --fail-on-warnings` | Svelte/TS diagnostics incl. a11y; must be **0/0** | | `lint` | `biome check --error-on-warnings` | Lint + formatting + import-organization across `.ts`/`.js`/`.svelte` | | `format:check` | `biome format` | Formatting drift (explicit) | -| `coverage:check` | `scripts/check-coverage.ts` | Test coverage below the floor (85% func / 82% line) | +| `coverage:check` | `scripts/coverage-check.ts` | Test coverage below the floor (85% func / 82% line) | | `dup` | `jscpd` | Code duplication above 3% (lines) | | `deadcode` | `knip` | Unused files / exports / dependencies | | `cycles` | `madge --circular` | Circular imports | -| `boundaries` | `scripts/check-boundaries.ts` | Server layering: `core/` stays primitive, domains stay decoupled | +| `boundaries` | `scripts/boundaries.ts` | Server layering: `core/` stays primitive, domains stay decoupled | | `build` | `vite build` + postbuild | The production build actually compiles | | `audit` | `bun audit` | Dependency advisories | A few notes: - **Coverage** isn't enforced by Bun directly (it reports but doesn't fail), so - `scripts/check-coverage.ts` parses the summary and enforces the floor itself. + `scripts/coverage-check.ts` parses the summary and enforces the floor itself. Floors sit a notch below actuals; ratchet them up as coverage climbs. - **Duplication** is measured on production code only (tests are excluded). The 3% threshold is evaluated against the *lines* percentage. Some duplication is @@ -85,9 +85,9 @@ A few notes: --- -## Machine-readable verify (`verify:json`) +## Machine-readable verify (`gates`) -`bun run verify:json` runs the same gates but emits a single compact JSON object +`bun run gates` runs the same gates but emits a single compact JSON object instead of ~150 lines of human output — built for CI and AI/automation: ```jsonc @@ -101,7 +101,7 @@ instead of ~150 lines of human output — built for CI and AI/automation: and a red run still has enough to act on. - It runs **all** gates (collecting every failure) rather than stopping at the first, and exits non-zero iff any gate failed. -- Pass gate names to run a subset: `bun run verify:json typecheck lint`. +- Pass gate names to run a subset: `bun run gates typecheck lint`. --- @@ -139,7 +139,7 @@ Beyond the hard gates, these surface issues the gates can't: |---|---|---| | `bun run deadcode` | knip | Unused files/exports/deps (a hard gate) | | `bun run cycles` | madge | Circular imports (a hard gate) | -| `bun run boundaries` | `check-boundaries.ts` | Server layering: `core/` imports no domain; domains stay decoupled (a hard gate) | +| `bun run boundaries` | `boundaries.ts` | Server layering: `core/` imports no domain; domains stay decoupled (a hard gate) | | `bun run dup` | jscpd | Duplication (a hard gate) | | `bun run mutation` | Stryker | **Opt-in.** Mutates code to prove tests actually assert, not just execute | @@ -213,7 +213,7 @@ for review before publishing. - **Every gate must stay green.** A change that fixes one thing but reddens another is a regression — fix it forward before moving on. - **Small, atomic commits** with a clear message describing the root cause. -- **Verify before committing:** `bun run verify` (or `verify:json`) must pass. +- **Verify before committing:** `bun run verify` (or `gates`) must pass. See [CONTRIBUTING](../CONTRIBUTING.md) for the contribution workflow and [Architecture](architecture.md) for how the pieces fit together. diff --git a/package.json b/package.json index 8d88a7f..9f89ad3 100644 --- a/package.json +++ b/package.json @@ -14,18 +14,18 @@ "format": "bunx --bun biome format --write .", "format:check": "bunx --bun biome format .", "test": "bun test", - "coverage:check": "bun scripts/check-coverage.ts", + "coverage:check": "bun scripts/coverage-check.ts", "audit": "bun audit", "dup": "jscpd", "deadcode": "knip", "cycles": "madge --circular --extensions ts,svelte --ts-config tsconfig.json src", - "boundaries": "bun scripts/check-boundaries.ts", + "boundaries": "bun scripts/boundaries.ts", "mutation": "stryker run", "ci": "bun run check && bun run build", "seed": "bun scripts/seed.ts", "release:prep": "bun scripts/release-prep.ts", "verify": "bun run typecheck && bun run check && bun run lint && bun run format:check && bun run coverage:check && bun run dup && bun run deadcode && bun run cycles && bun run boundaries && bun run build && bun run audit", - "verify:json": "bun scripts/verify-json.ts" + "gates": "bun scripts/gates.ts" }, "dependencies": { "@octokit/auth-app": "^8.2.0", diff --git a/scripts/check-boundaries.ts b/scripts/boundaries.ts similarity index 100% rename from scripts/check-boundaries.ts rename to scripts/boundaries.ts diff --git a/scripts/check-coverage.ts b/scripts/coverage-check.ts similarity index 97% rename from scripts/check-coverage.ts rename to scripts/coverage-check.ts index b407bb2..b595957 100644 --- a/scripts/check-coverage.ts +++ b/scripts/coverage-check.ts @@ -35,7 +35,7 @@ const output = raw.replace(ansi, ""); if (proc.exitCode !== 0) { process.stdout.write(output); // Re-print just the failing tests LAST so they survive log truncation (e.g. - // verify:json's tail()) — the coverage table otherwise buries the one detail + // gates' tail()) — the coverage table otherwise buries the one detail // that matters: which tests failed and why. `(fail)` lines name the test; // the indented `error:`/assertion lines that follow give the reason. const lines = output.split("\n"); diff --git a/scripts/verify-json.ts b/scripts/gates.ts similarity index 96% rename from scripts/verify-json.ts rename to scripts/gates.ts index e9e3154..8c6e7ef 100644 --- a/scripts/verify-json.ts +++ b/scripts/gates.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * verify:json — run the full gate suite and emit a compact JSON summary instead + * gates — run the full gate suite and emit a compact JSON summary instead * of `verify`'s ~150 lines of human output (the vite build chunk listing alone * is a large, low-signal block). Built for AI / automated consumption: * @@ -13,7 +13,7 @@ * * Runs the exact commands behind the `verify` script, so a pass here means the * same as a `verify` pass. Pass gate names as args to run a subset, e.g. - * bun run verify:json typecheck lint + * bun run gates typecheck lint */ export {}; @@ -143,7 +143,7 @@ const failures: string[] = []; // Prepare: generate SvelteKit's `.svelte-kit/` (the $types + $lib path aliases) // once, up front. On a fresh checkout it doesn't exist yet, and without it both // `tsc` (typecheck) and `bun test` ($lib import resolution) fail. Doing it here -// — not relying on gate order — keeps subset runs (e.g. `verify:json coverage`) +// — not relying on gate order — keeps subset runs (e.g. `gates coverage`) // correct too. const sync = run(["bunx", "--bun", "svelte-kit", "sync"]); if (sync.code !== 0) {