From 7770e9839e36c627a6211e6f03a8d74ece51e614 Mon Sep 17 00:00:00 2001 From: Casey Vega Date: Sun, 14 Jun 2026 16:36:52 -0700 Subject: [PATCH 1/3] feat(profile): collect content counts, batch augment, size blockers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The profiler only gathered the per-feature signals the consideration checks need, one repo per request, and never the actual content volume. This adds real repo metadata, batches the second pass, and turns the content/size data into a scale rollup and the first working blocker. - Discovery now also pulls each repo's content-volume totalCounts (issues, pull requests, branches, tags) — they ride along free in the 100-repo page already being fetched. - The augment pass is batched: one aliased GraphQL request profiles a chunk of 25 repos (rule details + the default-branch commit count) instead of one request per repo, so an org of N repos costs ~N/25 requests. It's partial-failure aware — an inaccessible repo in a chunk degrades to zeroed signals via the error's partial data instead of failing the whole chunk. - analyze wires the git-archive-size blocker off diskUsage (a proxy, flagged for git-sizer confirmation), so blockers are no longer always 0. - getProfileDetail derives an org-wide migration-scale rollup (issues, PRs, commits, branches, tags, releases, total size) by summing the persisted signals on read — no schema change — surfaced as a "Migration scale" section on the run page. --- src/lib/server/profile/analyze.test.ts | 24 +++ src/lib/server/profile/analyze.ts | 14 ++ src/lib/server/profile/augment.test.ts | 204 ++++++++++++++++------- src/lib/server/profile/augment.ts | 209 +++++++++++++++--------- src/lib/server/profile/discover.test.ts | 35 ++++ src/lib/server/profile/discover.ts | 12 ++ src/lib/server/profile/insights.test.ts | 5 + src/lib/server/profile/runner.test.ts | 48 +++++- src/lib/server/profile/runner.ts | 33 ++-- src/lib/server/profile/service.test.ts | 54 +++++- src/lib/server/profile/service.ts | 46 +++++- src/lib/server/profile/store.test.ts | 5 + src/lib/server/profile/types.ts | 10 ++ src/routes/profile/[id]/+page.svelte | 31 +++- 14 files changed, 576 insertions(+), 154 deletions(-) diff --git a/src/lib/server/profile/analyze.test.ts b/src/lib/server/profile/analyze.test.ts index 00815df..132b113 100644 --- a/src/lib/server/profile/analyze.test.ts +++ b/src/lib/server/profile/analyze.test.ts @@ -26,6 +26,11 @@ function cleanSignals(over: Partial = {}): RepoSignals { defaultBranch: "main", pushedAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-02T00:00:00Z", + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -117,6 +122,25 @@ describe("analyzeRepo", () => { expect(profile.summary.blockers).toBe(0); }); + test("flags an oversized repo as a git-archive-size blocker", () => { + const big = analyzeRepo(cleanSignals({ diskUsageKb: 3 * 1024 * 1024 })); // ~3 GiB + const f = finding(big, "git-archive-size-limit"); + expect(f?.status).toBe("applies"); + expect(f?.evidence).toMatch(/3\.0 GiB.*git-sizer/); + expect(big.summary.blockers).toBe(1); + }); + + test("a repo under the archive proxy threshold is clear of the size blocker", () => { + const small = analyzeRepo(cleanSignals({ diskUsageKb: 500 * 1024 })); // 500 MiB + expect(finding(small, "git-archive-size-limit")?.status).toBe("clear"); + expect(small.summary.blockers).toBe(0); + }); + + test("null diskUsage is clear of the size blocker (not a false positive)", () => { + const unknown = analyzeRepo(cleanSignals({ diskUsageKb: null })); + expect(finding(unknown, "git-archive-size-limit")?.status).toBe("clear"); + }); + test("branch-protection consideration ignores rules that use only migratable features", () => { // Rules exist, but none use an unmigrated feature. const profile = analyzeRepo( diff --git a/src/lib/server/profile/analyze.ts b/src/lib/server/profile/analyze.ts index 1e2fce7..9a2fdfb 100644 --- a/src/lib/server/profile/analyze.ts +++ b/src/lib/server/profile/analyze.ts @@ -63,6 +63,15 @@ function count(n: number, noun: string): string { return `${n} ${noun}${n === 1 ? "" : "s"}`; } +/** + * Smallest GHES Git-archive size limit (2 GiB, in KiB). `diskUsage` is the + * uncompressed working size and only a proxy for the compressed archive — a repo + * already over this uncompressed is a genuine candidate to exceed the limit, so + * it's flagged (as an estimate) for git-sizer confirmation. Repos under it are + * cleared rather than nagged. + */ +const GIT_ARCHIVE_PROXY_KB = 2 * 1024 * 1024; + /** * Detectors for the considerations the crawl can currently evaluate, keyed by * consideration id. Add an entry here as each new signal is gathered. @@ -82,6 +91,11 @@ const DETECTORS: Record = { : null, "fork-relationships": (s) => (s.isFork ? "repository is a fork" : null), "wiki-attachments": (s) => (s.hasWiki ? "wiki enabled (attachments not migrated)" : null), + "git-archive-size-limit": (s) => { + if (s.diskUsageKb == null || s.diskUsageKb < GIT_ARCHIVE_PROXY_KB) return null; + const gib = (s.diskUsageKb / (1024 * 1024)).toFixed(1); + return `repository size ~${gib} GiB (estimate — confirm with git-sizer)`; + }, }; /** Consideration ids the engine can evaluate (the rest report indeterminate). */ diff --git a/src/lib/server/profile/augment.test.ts b/src/lib/server/profile/augment.test.ts index d383477..70e32c9 100644 --- a/src/lib/server/profile/augment.test.ts +++ b/src/lib/server/profile/augment.test.ts @@ -26,6 +26,10 @@ function discovered(over: Partial = {}): DiscoveredRepo { defaultBranch: "main", pushedAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-02T00:00:00Z", + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, ...over, }; } @@ -53,8 +57,9 @@ function rule(flags: RuleFlags = {}) { }; } -/** Build the repository query response. */ -function repoResult(over: { +/** Build one repository alias node. */ +function node(over: { + commits?: number; discussions?: number; projectsV2?: number; environments?: number; @@ -63,32 +68,33 @@ function repoResult(over: { watchers?: number; ruleTotal?: number; rules?: ReturnType[]; + noDefaultBranch?: boolean; }) { return { - repository: { - discussions: { totalCount: over.discussions ?? 0 }, - projectsV2: { totalCount: over.projectsV2 ?? 0 }, - environments: { totalCount: over.environments ?? 0 }, - releases: { totalCount: over.releases ?? 0 }, - stargazerCount: over.stars ?? 0, - watchers: { totalCount: over.watchers ?? 0 }, - branchProtectionRules: { - totalCount: over.ruleTotal ?? over.rules?.length ?? 0, - nodes: over.rules ?? [], - }, + defaultBranchRef: over.noDefaultBranch + ? null + : { target: { history: { totalCount: over.commits ?? 0 } } }, + discussions: { totalCount: over.discussions ?? 0 }, + projectsV2: { totalCount: over.projectsV2 ?? 0 }, + environments: { totalCount: over.environments ?? 0 }, + releases: { totalCount: over.releases ?? 0 }, + stargazerCount: over.stars ?? 0, + watchers: { totalCount: over.watchers ?? 0 }, + branchProtectionRules: { + totalCount: over.ruleTotal ?? over.rules?.length ?? 0, + nodes: over.rules ?? [], }, }; } -/** A `gql` double returning one queued response and recording its variables. */ -function mockGql(response: unknown) { - const calls: Array<{ owner: string; name: string; rules: number }> = []; +/** + * A `gql` double returning a queued aliased response (keyed `r0`, `r1`, …) and + * recording the variables it was called with. + */ +function mockGql(response: Record) { + const calls: Array> = []; const fn = (async (_query: string, vars: Record) => { - calls.push({ - owner: vars.owner as string, - name: vars.name as string, - rules: vars.rules as number, - }); + calls.push(vars); return response; }) as unknown as typeof graphql; return { fn, calls }; @@ -96,8 +102,9 @@ function mockGql(response: unknown) { describe("augmentRepoSignals", () => { test("maps every count signal and preserves the discovered fields", async () => { - const { fn } = mockGql( - repoResult({ + const { fn } = mockGql({ + r0: node({ + commits: 512, discussions: 3, projectsV2: 2, environments: 4, @@ -105,34 +112,73 @@ describe("augmentRepoSignals", () => { stars: 41, watchers: 9, }), - ); + }); - const signals = await augmentRepoSignals(fn, discovered({ name: "widget" })); + const [signals] = await augmentRepoSignals(fn, [discovered({ issuesCount: 11 })]); // Augmented counts. - expect(signals.discussionsCount).toBe(3); - expect(signals.projectsV2Count).toBe(2); - expect(signals.environmentsCount).toBe(4); - expect(signals.releasesCount).toBe(7); - expect(signals.stargazerCount).toBe(41); - expect(signals.watcherCount).toBe(9); - // Discovered spine preserved. - expect(signals.nameWithOwner).toBe("acme/widget"); - expect(signals.diskUsageKb).toBe(1234); - expect(signals.hasDiscussions).toBe(true); + expect(signals?.commitsCount).toBe(512); + expect(signals?.discussionsCount).toBe(3); + expect(signals?.projectsV2Count).toBe(2); + expect(signals?.environmentsCount).toBe(4); + expect(signals?.releasesCount).toBe(7); + expect(signals?.stargazerCount).toBe(41); + expect(signals?.watcherCount).toBe(9); + // Discovered spine preserved (incl. the content counts from discovery). + expect(signals?.nameWithOwner).toBe("acme/widget"); + expect(signals?.diskUsageKb).toBe(1234); + expect(signals?.issuesCount).toBe(11); + }); + + test("treats a missing default branch as zero commits", async () => { + const { fn } = mockGql({ r0: node({ noDefaultBranch: true }) }); + + const [signals] = await augmentRepoSignals(fn, [discovered({ isEmpty: true })]); + + expect(signals?.commitsCount).toBe(0); }); - test("derives the owner from nameWithOwner and caps the rules page", async () => { - const { fn, calls } = mockGql(repoResult({})); + test("derives owner/name per alias and caps the rules page", async () => { + const { fn, calls } = mockGql({ r0: node({}), r1: node({}) }); + + await augmentRepoSignals(fn, [ + discovered({ name: "widget", nameWithOwner: "octo-org/widget" }), + discovered({ name: "gadget", nameWithOwner: "octo-org/gadget" }), + ]); + + expect(calls[0]).toMatchObject({ + rules: 100, + o0: "octo-org", + n0: "widget", + o1: "octo-org", + n1: "gadget", + }); + }); + + test("profiles a whole chunk in a single request, in input order", async () => { + const { fn, calls } = mockGql({ + r0: node({ commits: 1 }), + r1: node({ commits: 2 }), + r2: node({ commits: 3 }), + }); - await augmentRepoSignals(fn, discovered({ name: "widget", nameWithOwner: "octo-org/widget" })); + const signals = await augmentRepoSignals(fn, [ + discovered({ nameWithOwner: "o/a", name: "a" }), + discovered({ nameWithOwner: "o/b", name: "b" }), + discovered({ nameWithOwner: "o/c", name: "c" }), + ]); - expect(calls[0]).toEqual({ owner: "octo-org", name: "widget", rules: 100 }); + expect(calls).toHaveLength(1); // one batched request for three repos + expect(signals.map((s) => [s.nameWithOwner, s.commitsCount])).toEqual([ + ["o/a", 1], + ["o/b", 2], + ["o/c", 3], + ]); }); test("counts only branch protection rules that use an unmigrated feature", async () => { - const { fn } = mockGql( - repoResult({ + const { fn } = mockGql({ + r0: node({ ruleTotal: 4, rules: [ rule({ allowsForcePushes: true }), // unmigrated @@ -141,21 +187,21 @@ describe("augmentRepoSignals", () => { rule({}), // only migrated features → not counted ], }), - ); + }); - const signals = await augmentRepoSignals(fn, discovered()); + const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals.branchProtectionRuleCount).toBe(4); - expect(signals.branchProtectionRulesUsingUnmigratedFeatures).toBe(3); + expect(signals?.branchProtectionRuleCount).toBe(4); + expect(signals?.branchProtectionRulesUsingUnmigratedFeatures).toBe(3); }); test("treats a rule with no unmigrated features as fully migratable", async () => { - const { fn } = mockGql(repoResult({ ruleTotal: 1, rules: [rule({})] })); + const { fn } = mockGql({ r0: node({ ruleTotal: 1, rules: [rule({})] }) }); - const signals = await augmentRepoSignals(fn, discovered()); + const [signals] = await augmentRepoSignals(fn, [discovered()]); - expect(signals.branchProtectionRuleCount).toBe(1); - expect(signals.branchProtectionRulesUsingUnmigratedFeatures).toBe(0); + expect(signals?.branchProtectionRuleCount).toBe(1); + expect(signals?.branchProtectionRulesUsingUnmigratedFeatures).toBe(0); }); test("flags each unmigrated feature individually", async () => { @@ -166,17 +212,61 @@ describe("augmentRepoSignals", () => { rule({ bypassForcePush: 1 }), ]; for (const r of cases) { - const { fn } = mockGql(repoResult({ ruleTotal: 1, rules: [r] })); - const signals = await augmentRepoSignals(fn, discovered()); - expect(signals.branchProtectionRulesUsingUnmigratedFeatures).toBe(1); + const { fn } = mockGql({ r0: node({ ruleTotal: 1, rules: [r] }) }); + const [signals] = await augmentRepoSignals(fn, [discovered()]); + expect(signals?.branchProtectionRulesUsingUnmigratedFeatures).toBe(1); } }); - test("throws when the repository is missing or inaccessible", async () => { - const { fn } = mockGql({ repository: null }); + test("returns no signals for an empty chunk without calling gql", async () => { + const { fn, calls } = mockGql({}); + + const signals = await augmentRepoSignals(fn, []); + + expect(signals).toEqual([]); + expect(calls).toHaveLength(0); + }); + + test("degrades a null alias to zeroed signals (keeps the discovery spine)", async () => { + const { fn } = mockGql({ r0: node({ commits: 5 }), r1: null }); + + const signals = await augmentRepoSignals(fn, [ + discovered({ nameWithOwner: "o/ok", name: "ok" }), + discovered({ nameWithOwner: "o/gone", name: "gone", issuesCount: 3 }), + ]); + + expect(signals[0]?.commitsCount).toBe(5); + // The inaccessible repo keeps its discovered fields but zeroes augment counts. + expect(signals[1]?.nameWithOwner).toBe("o/gone"); + expect(signals[1]?.issuesCount).toBe(3); + expect(signals[1]?.commitsCount).toBe(0); + expect(signals[1]?.branchProtectionRuleCount).toBe(0); + }); + + test("recovers partial data when the request errors with a data payload", async () => { + // GraphQL returns partial data + errors when one alias is inaccessible; the + // client surfaces that as a throw carrying `.data`. Recover the good repos. + const err = Object.assign(new Error("Could not resolve to a Repository"), { + data: { r0: node({ commits: 8 }), r1: null }, + }); + const fn = (async () => { + throw err; + }) as unknown as typeof graphql; + + const signals = await augmentRepoSignals(fn, [ + discovered({ nameWithOwner: "o/a", name: "a" }), + discovered({ nameWithOwner: "o/b", name: "b" }), + ]); + + expect(signals[0]?.commitsCount).toBe(8); + expect(signals[1]?.commitsCount).toBe(0); + }); + + test("rethrows an error with no recoverable data payload", async () => { + const fn = (async () => { + throw new Error("network down"); + }) as unknown as typeof graphql; - await expect(augmentRepoSignals(fn, discovered())).rejects.toThrow( - /not found or not accessible/i, - ); + await expect(augmentRepoSignals(fn, [discovered()])).rejects.toThrow(/network down/); }); }); diff --git a/src/lib/server/profile/augment.ts b/src/lib/server/profile/augment.ts index 734ec80..ae0be31 100644 --- a/src/lib/server/profile/augment.ts +++ b/src/lib/server/profile/augment.ts @@ -1,13 +1,20 @@ /** - * Per-repo signal augmentation — the second crawl pass. + * Per-repo signal augmentation — the second crawl pass, batched. * - * Discovery (`discover.ts`) gives the cheap, breadth-first spine; this pass - * enriches one repository with the GraphQL count signals the consideration - * analysis needs (discussions, projects, environments, releases, stars/watchers, - * branch - * protection rules). It's a single bounded query per repo — no asset/ref paging - * — so deeper size scans (release assets, git-sizer, ref-name length) stay in a - * separate, heavier pass. + * Discovery (`discover.ts`) gives the cheap, breadth-first spine and the + * content-volume counts that ride along free in each 100-repo page. This pass + * gathers the signals that can't fold into a discovery node: the + * branch-protection rule *details* (node-level flags, not just a count) and the + * default-branch commit count (`history.totalCount`, the one count with timeout + * risk at 100-wide, so kept to a smaller fan-out here). + * + * It profiles a CHUNK of repos in one aliased GraphQL request + * (`r0: repository(...) {…} r1: repository(...) {…} …`), so an org of N repos + * costs ~N/chunk requests instead of one per repo. The request is + * partial-failure aware: if a repo in the chunk is inaccessible, GraphQL + * returns a null alias plus an `errors` entry (surfaced by the client as an + * error carrying partial `data`) — that repo degrades to zeroed signals rather + * than failing the whole chunk. * * The `gql` client is injected, so the query shaping, mapping, and the * branch-protection derivation are unit-testable without a network. @@ -29,49 +36,72 @@ interface BranchProtectionRuleNode { bypassPullRequestAllowances: { totalCount: number }; } -/** Raw shape of the per-repo signals query response. */ -interface RepoSignalsResult { - repository: { - discussions: { totalCount: number }; - projectsV2: { totalCount: number }; - environments: { totalCount: number }; - releases: { totalCount: number }; - stargazerCount: number; - watchers: { totalCount: number }; - branchProtectionRules: { - totalCount: number; - nodes: BranchProtectionRuleNode[]; - }; - } | null; +/** Raw shape of one repository alias in the batched signals query. */ +interface RepoSignalsNode { + defaultBranchRef: { target: { history: { totalCount: number } } | null } | null; + discussions: { totalCount: number }; + projectsV2: { totalCount: number }; + environments: { totalCount: number }; + releases: { totalCount: number }; + stargazerCount: number; + watchers: { totalCount: number }; + branchProtectionRules: { + totalCount: number; + nodes: BranchProtectionRuleNode[]; + }; } -const REPO_SIGNALS_QUERY = `query repoSignals($owner: String!, $name: String!, $rules: Int!) { - repository(owner: $owner, name: $name) { - discussions { totalCount } - projectsV2 { totalCount } - environments { totalCount } - releases { totalCount } - stargazerCount - watchers { totalCount } - branchProtectionRules(first: $rules) { - totalCount - nodes { - allowsForcePushes - requiresDeployments - lockBranch - blocksCreations - requireLastPushApproval - bypassForcePushAllowances { totalCount } - bypassPullRequestAllowances { totalCount } - } +/** The aliased query response: one (possibly null) node per requested repo. */ +type BatchSignalsResult = Record; + +/** + * The per-repo selection. `history` sits behind a `... on Commit` inline + * fragment because `defaultBranchRef.target` is a `GitObject` (Commit | Tree | + * Blob | Tag); only a Commit has `history`. + */ +const SIGNALS_FRAGMENT = `fragment Sig on Repository { + defaultBranchRef { target { ... on Commit { history(first: 1) { totalCount } } } } + discussions { totalCount } + projectsV2 { totalCount } + environments { totalCount } + releases { totalCount } + stargazerCount + watchers { totalCount } + branchProtectionRules(first: $rules) { + totalCount + nodes { + allowsForcePushes + requiresDeployments + lockBranch + blocksCreations + requireLastPushApproval + bypassForcePushAllowances { totalCount } + bypassPullRequestAllowances { totalCount } } } }`; +/** Build the aliased query for a chunk of `n` repos (aliases `r0…r{n-1}`). */ +function buildBatchQuery(n: number): string { + const varDefs = ["$rules: Int!"]; + const fields: string[] = []; + for (let i = 0; i < n; i++) { + varDefs.push(`$o${i}: String!`, `$n${i}: String!`); + fields.push(`r${i}: repository(owner: $o${i}, name: $n${i}) { ...Sig }`); + } + return `query batchSignals(${varDefs.join(", ")}) {\n${fields.join("\n")}\n}\n${SIGNALS_FRAGMENT}`; +} + +/** Split `owner/name` into its owner; falls back to the whole string. */ +function ownerOf(nameWithOwner: string): string { + const slash = nameWithOwner.indexOf("/"); + return slash > 0 ? nameWithOwner.slice(0, slash) : nameWithOwner; +} + /** - * Whether a branch protection rule uses any feature GEI does not migrate. - * (Plain required-reviews/status-checks DO migrate, so a rule using only those - * counts as zero here.) + * Whether a branch protection rule uses any feature GitHub's importer does not + * migrate. (Plain required-reviews/status-checks DO migrate, so a rule using + * only those counts as zero here.) */ function usesUnmigratedFeature(rule: BranchProtectionRuleNode): boolean { return ( @@ -85,46 +115,77 @@ function usesUnmigratedFeature(rule: BranchProtectionRuleNode): boolean { ); } -/** Split `owner/name` into its owner; falls back to the whole string. */ -function ownerOf(nameWithOwner: string): string { - const slash = nameWithOwner.indexOf("/"); - return slash > 0 ? nameWithOwner.slice(0, slash) : nameWithOwner; +/** Map a repo + its (possibly null) augment node to full `RepoSignals`. */ +function toSignals(repo: DiscoveredRepo, node: RepoSignalsNode | null): RepoSignals { + // A null node means the repo was inaccessible on this pass (permissions edge, + // or deleted between passes). It was discoverable, so keep the discovery spine + // and degrade the augment counts to zero rather than failing the run. + if (!node) { + return { + ...repo, + commitsCount: 0, + discussionsCount: 0, + projectsV2Count: 0, + environmentsCount: 0, + releasesCount: 0, + stargazerCount: 0, + watcherCount: 0, + branchProtectionRuleCount: 0, + branchProtectionRulesUsingUnmigratedFeatures: 0, + }; + } + return { + ...repo, + commitsCount: node.defaultBranchRef?.target?.history.totalCount ?? 0, + discussionsCount: node.discussions.totalCount, + projectsV2Count: node.projectsV2.totalCount, + environmentsCount: node.environments.totalCount, + releasesCount: node.releases.totalCount, + stargazerCount: node.stargazerCount, + watcherCount: node.watchers.totalCount, + branchProtectionRuleCount: node.branchProtectionRules.totalCount, + branchProtectionRulesUsingUnmigratedFeatures: + node.branchProtectionRules.nodes.filter(usesUnmigratedFeature).length, + }; +} + +/** Recover partial `data` from a GraphQL error that carries it, else rethrow. */ +function dataFromError(err: unknown): BatchSignalsResult { + if (err && typeof err === "object" && "data" in err) { + const data = (err as { data?: unknown }).data; + if (data && typeof data === "object") return data as BatchSignalsResult; + } + throw err; } /** - * Enrich a discovered repository with its per-repo GraphQL signals. + * Enrich a chunk of discovered repositories with their per-repo signals in a + * single aliased GraphQL request. * * @param gql Injected GraphQL client (`createSingleClient(...).graphql`). - * @param repo The repository from the discovery pass to augment. - * @returns `repo` plus the gathered count signals (`RepoSignals`). - * @throws If the repository is missing or not accessible. + * @param repos The chunk of repositories (from discovery) to augment. + * @returns One `RepoSignals` per input repo, in input order. */ export async function augmentRepoSignals( gql: typeof graphql, - repo: DiscoveredRepo, -): Promise { - const result: RepoSignalsResult = await gql(REPO_SIGNALS_QUERY, { - owner: ownerOf(repo.nameWithOwner), - name: repo.name, - rules: MAX_RULES, + repos: DiscoveredRepo[], +): Promise { + if (repos.length === 0) return []; + + const variables: Record = { rules: MAX_RULES }; + repos.forEach((repo, i) => { + variables[`o${i}`] = ownerOf(repo.nameWithOwner); + variables[`n${i}`] = repo.name; }); - const r = result.repository; - if (!r) { - throw new Error(`Repository '${repo.nameWithOwner}' not found or not accessible`); + let data: BatchSignalsResult; + try { + data = await gql(buildBatchQuery(repos.length), variables); + } catch (err) { + // A chunk with one inaccessible repo errors but still carries partial data; + // recover it so the accessible repos in the chunk still get profiled. + data = dataFromError(err); } - const rulesUsingUnmigrated = r.branchProtectionRules.nodes.filter(usesUnmigratedFeature).length; - - return { - ...repo, - discussionsCount: r.discussions.totalCount, - projectsV2Count: r.projectsV2.totalCount, - environmentsCount: r.environments.totalCount, - releasesCount: r.releases.totalCount, - stargazerCount: r.stargazerCount, - watcherCount: r.watchers.totalCount, - branchProtectionRuleCount: r.branchProtectionRules.totalCount, - branchProtectionRulesUsingUnmigratedFeatures: rulesUsingUnmigrated, - }; + return repos.map((repo, i) => toSignals(repo, data[`r${i}`] ?? null)); } diff --git a/src/lib/server/profile/discover.test.ts b/src/lib/server/profile/discover.test.ts index 51fd520..b7042cd 100644 --- a/src/lib/server/profile/discover.test.ts +++ b/src/lib/server/profile/discover.test.ts @@ -23,6 +23,10 @@ interface RepoNode { defaultBranchRef: { name: string } | null; pushedAt: string | null; updatedAt: string | null; + issues: { totalCount: number }; + pullRequests: { totalCount: number }; + branches: { totalCount: number }; + tags: { totalCount: number }; } /** Build a repo node, overriding only the fields a test cares about. */ @@ -42,6 +46,10 @@ function node(name: string, over: Partial = {}): RepoNode { defaultBranchRef: { name: "main" }, pushedAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-02T00:00:00Z", + issues: { totalCount: 0 }, + pullRequests: { totalCount: 0 }, + branches: { totalCount: 0 }, + tags: { totalCount: 0 }, ...over, }; } @@ -160,9 +168,36 @@ describe("discoverOrgRepos", () => { defaultBranch: null, pushedAt: null, updatedAt: "2026-01-02T00:00:00Z", + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, }); }); + test("maps the content-volume counts from the discovery page", async () => { + const { fn } = mockGql([ + page( + [ + node("busy", { + issues: { totalCount: 42 }, + pullRequests: { totalCount: 17 }, + branches: { totalCount: 6 }, + tags: { totalCount: 9 }, + }), + ], + { total: 1 }, + ), + ]); + + const [repo] = (await discoverOrgRepos(fn, "acme")).repos; + + expect(repo?.issuesCount).toBe(42); + expect(repo?.pullRequestsCount).toBe(17); + expect(repo?.branchesCount).toBe(6); + expect(repo?.tagsCount).toBe(9); + }); + test("handles an organization with no repositories", async () => { const { fn } = mockGql([page([], { total: 0 })]); diff --git a/src/lib/server/profile/discover.ts b/src/lib/server/profile/discover.ts index 2e3d9e3..6ecb347 100644 --- a/src/lib/server/profile/discover.ts +++ b/src/lib/server/profile/discover.ts @@ -29,6 +29,10 @@ interface RepoNode { defaultBranchRef: { name: string } | null; pushedAt: string | null; updatedAt: string | null; + issues: { totalCount: number }; + pullRequests: { totalCount: number }; + branches: { totalCount: number }; + tags: { totalCount: number }; } /** Raw shape of the paged discovery query response. */ @@ -62,6 +66,10 @@ const ORG_REPOS_QUERY = `query orgRepos($login: String!, $cursor: String, $pageS defaultBranchRef { name } pushedAt updatedAt + issues { totalCount } + pullRequests { totalCount } + branches: refs(refPrefix: "refs/heads/", first: 1) { totalCount } + tags: refs(refPrefix: "refs/tags/", first: 1) { totalCount } } } } @@ -84,6 +92,10 @@ function toDiscoveredRepo(node: RepoNode): DiscoveredRepo { defaultBranch: node.defaultBranchRef?.name ?? null, pushedAt: node.pushedAt, updatedAt: node.updatedAt, + issuesCount: node.issues.totalCount, + pullRequestsCount: node.pullRequests.totalCount, + branchesCount: node.branches.totalCount, + tagsCount: node.tags.totalCount, }; } diff --git a/src/lib/server/profile/insights.test.ts b/src/lib/server/profile/insights.test.ts index 07be6b6..1504048 100644 --- a/src/lib/server/profile/insights.test.ts +++ b/src/lib/server/profile/insights.test.ts @@ -29,6 +29,11 @@ function signals(over: Partial = {}): RepoSignals { defaultBranch: "main", pushedAt: monthsAgo(1), updatedAt: monthsAgo(1), + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/runner.test.ts b/src/lib/server/profile/runner.test.ts index 2cd7703..1a9ae0e 100644 --- a/src/lib/server/profile/runner.test.ts +++ b/src/lib/server/profile/runner.test.ts @@ -29,6 +29,10 @@ function discovered(name: string, over: Partial = {}): Discovere defaultBranch: "main", pushedAt: null, updatedAt: null, + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, ...over, }; } @@ -36,6 +40,7 @@ function discovered(name: string, over: Partial = {}): Discovere function signalsFor(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { return { ...repo, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -48,14 +53,14 @@ function signalsFor(repo: DiscoveredRepo, over: Partial = {}): Repo }; } -/** Deps whose discover returns the given repos and augment maps each by name. */ +/** Deps whose discover returns the given repos and augment maps each chunk by name. */ function deps( repos: DiscoveredRepo[], augmentOver: Record> = {}, ): ProfileRunnerDeps { return { discover: async (): Promise => ({ org: "acme", total: repos.length, repos }), - augment: async (_gql, repo) => signalsFor(repo, augmentOver[repo.name] ?? {}), + augment: async (_gql, chunk) => chunk.map((r) => signalsFor(r, augmentOver[r.name] ?? {})), }; } @@ -124,7 +129,7 @@ describe("runProfile", () => { discover: async () => { throw new Error("Organization 'acme' not found or not accessible"); }, - augment: async (_gql, repo) => signalsFor(repo), + augment: async (_gql, chunk) => chunk.map((r) => signalsFor(r)), }; const run = await runProfile( @@ -138,13 +143,13 @@ describe("runProfile", () => { expect(run.failureReason).toMatch(/not found or not accessible/); }); - test("marks the run failed if a repo augmentation throws mid-crawl", async () => { + test("marks the run failed when a chunk's augmentation throws", async () => { const repos = [discovered("ok"), discovered("boom")]; const partialDeps: ProfileRunnerDeps = { discover: async () => ({ org: "acme", total: repos.length, repos }), - augment: async (_gql, repo) => { - if (repo.name === "boom") throw new Error("repo augmentation failed"); - return signalsFor(repo); + augment: async (_gql, chunk) => { + if (chunk.some((r) => r.name === "boom")) throw new Error("repo augmentation failed"); + return chunk.map((r) => signalsFor(r)); }, }; @@ -157,8 +162,33 @@ describe("runProfile", () => { expect(run.state).toBe("failed"); expect(run.failureReason).toBe("repo augmentation failed"); - // The repo profiled before the failure was still persisted. - expect(getRunRepoProfiles("r").map((p) => p.nameWithOwner)).toEqual(["acme/ok"]); + }); + + test("persists completed chunks when a later chunk fails", async () => { + // 26 repos → two augment chunks (25 + 1). The first chunk succeeds and is + // persisted; the second throws, failing the run — the first chunk survives. + const repos = Array.from({ length: 26 }, (_, i) => + discovered(`r${String(i).padStart(2, "0")}`), + ); + let call = 0; + const chunkedDeps: ProfileRunnerDeps = { + discover: async () => ({ org: "acme", total: repos.length, repos }), + augment: async (_gql, chunk) => { + call += 1; + if (call >= 2) throw new Error("second chunk failed"); + return chunk.map((r) => signalsFor(r)); + }, + }; + + const run = await runProfile( + gql, + { id: "r", org: "acme", sourceApiUrl: "u" }, + undefined, + chunkedDeps, + ); + + expect(run.state).toBe("failed"); + expect(getRunRepoProfiles("r")).toHaveLength(25); // the first chunk persisted }); test("persists the run before crawling (a created run exists by completion)", async () => { diff --git a/src/lib/server/profile/runner.ts b/src/lib/server/profile/runner.ts index 6029ea4..56c85d6 100644 --- a/src/lib/server/profile/runner.ts +++ b/src/lib/server/profile/runner.ts @@ -29,6 +29,13 @@ import { } from "./store"; import type { ProfileProgress, ProfileRun } from "./types"; +/** + * Repos per augment request. Each chunk is one aliased GraphQL query; 25 keeps + * the per-request node budget (≤ 25 × 100 branch-protection rules) and query + * complexity comfortably within GitHub's limits while cutting requests 25×. + */ +const AUGMENT_CHUNK = 25; + /** Injectable crawl primitives (defaults hit the real GraphQL helpers). */ export interface ProfileRunnerDeps { discover: typeof discoverOrgRepos; @@ -61,17 +68,23 @@ export async function runProfile( const discovery = await deps.discover(gql, input.org); setProfileRunTotal(input.id, discovery.total); + // Profile repos in chunks: each `augment` call is one aliased GraphQL + // request covering up to AUGMENT_CHUNK repos, so an org of N repos costs + // ~N/AUGMENT_CHUNK requests instead of one per repo. let profiled = 0; - for (const repo of discovery.repos) { - const signals = await deps.augment(gql, repo); - recordRepoProfile(input.id, signals, analyzeRepo(signals)); - profiled += 1; - onProgress?.({ - runId: input.id, - profiled, - total: discovery.total, - repo: repo.nameWithOwner, - }); + for (let i = 0; i < discovery.repos.length; i += AUGMENT_CHUNK) { + const chunk = discovery.repos.slice(i, i + AUGMENT_CHUNK); + const chunkSignals = await deps.augment(gql, chunk); + for (const signals of chunkSignals) { + recordRepoProfile(input.id, signals, analyzeRepo(signals)); + profiled += 1; + onProgress?.({ + runId: input.id, + profiled, + total: discovery.total, + repo: signals.nameWithOwner, + }); + } } completeProfileRun(input.id); diff --git a/src/lib/server/profile/service.test.ts b/src/lib/server/profile/service.test.ts index 7a5d962..d245907 100644 --- a/src/lib/server/profile/service.test.ts +++ b/src/lib/server/profile/service.test.ts @@ -30,12 +30,17 @@ function discovered(name: string): DiscoveredRepo { defaultBranch: "main", pushedAt: null, updatedAt: null, + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, }; } function signalsFor(repo: DiscoveredRepo, over: Partial = {}): RepoSignals { return { ...repo, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, @@ -72,7 +77,7 @@ function serviceDeps( total: repos.length, repos, }), - augment: async (_gql, repo) => signalsFor(repo, augmentOver[repo.name] ?? {}), + augment: async (_gql, chunk) => chunk.map((r) => signalsFor(r, augmentOver[r.name] ?? {})), }); return state.runPromise; }, @@ -193,6 +198,53 @@ describe("getProfileDetail", () => { expect(detail?.repos[0]?.insights.map((i) => i.id)).toContain("archived-move-now"); }); + test("derives the org migration-scale rollup by summing repo signals", () => { + createProfileRun({ id: "r", sourceApiUrl: "u", org: "acme" }); + const emptyProfile = (nameWithOwner: string): RepoProfile => ({ + nameWithOwner, + findings: [], + summary: { applies: 0, blockers: 0, warnings: 0, infos: 0, clear: 0, indeterminate: 0 }, + }); + recordRepoProfile( + "r", + signalsFor(discovered("a"), { + issuesCount: 10, + pullRequestsCount: 4, + commitsCount: 100, + branchesCount: 3, + tagsCount: 2, + releasesCount: 1, + diskUsageKb: 500, + }), + emptyProfile("acme/a"), + ); + recordRepoProfile( + "r", + signalsFor(discovered("b"), { + issuesCount: 5, + pullRequestsCount: 6, + commitsCount: 50, + branchesCount: 1, + tagsCount: 0, + releasesCount: 2, + diskUsageKb: 250, + }), + emptyProfile("acme/b"), + ); + + const scale = getProfileDetail("r")?.scale; + expect(scale).toEqual({ + repos: 2, + issues: 15, + pullRequests: 10, + commits: 150, + branches: 4, + tags: 2, + releases: 3, + diskUsageKb: 750, + }); + }); + test("returns null for an unknown run", () => { expect(getProfileDetail("nope")).toBeNull(); }); diff --git a/src/lib/server/profile/service.ts b/src/lib/server/profile/service.ts index d058f3f..f2ade32 100644 --- a/src/lib/server/profile/service.ts +++ b/src/lib/server/profile/service.ts @@ -32,10 +32,52 @@ interface RepoProfileView extends StoredRepoProfile { insights: Insight[]; } -/** A run plus its per-repo results (each enriched with insights). */ +/** + * Org-wide content-volume totals across a run's repos — the migration's scale. + * Derived on read by summing the persisted per-repo signals (no schema change), + * so it stays correct however many times a repo was re-recorded. + */ +interface MigrationScale { + repos: number; + issues: number; + pullRequests: number; + commits: number; + branches: number; + tags: number; + releases: number; + /** Sum of repo disk usage in KiB (null entries count as 0). */ + diskUsageKb: number; +} + +/** A run plus its per-repo results (each enriched with insights) and scale. */ export interface ProfileDetail { run: ProfileRun; repos: RepoProfileView[]; + scale: MigrationScale; +} + +/** Sum the per-repo signals into the org-wide migration-scale rollup. */ +function computeScale(repos: RepoProfileView[]): MigrationScale { + const scale: MigrationScale = { + repos: repos.length, + issues: 0, + pullRequests: 0, + commits: 0, + branches: 0, + tags: 0, + releases: 0, + diskUsageKb: 0, + }; + for (const { signals } of repos) { + scale.issues += signals.issuesCount; + scale.pullRequests += signals.pullRequestsCount; + scale.commits += signals.commitsCount; + scale.branches += signals.branchesCount; + scale.tags += signals.tagsCount; + scale.releases += signals.releasesCount; + scale.diskUsageKb += signals.diskUsageKb ?? 0; + } + return scale; } /** @@ -85,5 +127,5 @@ export function getProfileDetail(id: string): ProfileDetail | null { ...repo, insights: deriveInsights(repo.signals), })); - return { run, repos }; + return { run, repos, scale: computeScale(repos) }; } diff --git a/src/lib/server/profile/store.test.ts b/src/lib/server/profile/store.test.ts index 84db3d6..c074ab5 100644 --- a/src/lib/server/profile/store.test.ts +++ b/src/lib/server/profile/store.test.ts @@ -44,6 +44,11 @@ function signals(over: Partial = {}): RepoSignals { defaultBranch: "main", pushedAt: null, updatedAt: null, + issuesCount: 0, + pullRequestsCount: 0, + branchesCount: 0, + tagsCount: 0, + commitsCount: 0, discussionsCount: 0, projectsV2Count: 0, environmentsCount: 0, diff --git a/src/lib/server/profile/types.ts b/src/lib/server/profile/types.ts index 005b455..c5a2df8 100644 --- a/src/lib/server/profile/types.ts +++ b/src/lib/server/profile/types.ts @@ -40,6 +40,14 @@ export interface DiscoveredRepo { pushedAt: string | null; /** ISO timestamp of the last update, or null. */ updatedAt: string | null; + /** Issues (`issues.totalCount`, all states; excludes PRs) — migration scale. */ + issuesCount: number; + /** Pull requests (`pullRequests.totalCount`, all states) — migration scale. */ + pullRequestsCount: number; + /** Branches (`refs` under `refs/heads/`) — migration scale. */ + branchesCount: number; + /** Tags (`refs` under `refs/tags/`) — migration scale. */ + tagsCount: number; } /** Progress emitted after each discovery page (drives SSE / logging later). */ @@ -72,6 +80,8 @@ export interface OrgDiscovery { * read a number and compare, rather than re-deriving GraphQL schema details. */ export interface RepoSignals extends DiscoveredRepo { + /** Commits on the default branch (`history.totalCount`) — migration scale. */ + commitsCount: number; /** Repository-level Discussions (`discussions.totalCount`); not migrated. */ discussionsCount: number; /** Projects (new experience) (`projectsV2.totalCount`); not migrated. */ diff --git a/src/routes/profile/[id]/+page.svelte b/src/routes/profile/[id]/+page.svelte index 6ec84b9..a18e428 100644 --- a/src/routes/profile/[id]/+page.svelte +++ b/src/routes/profile/[id]/+page.svelte @@ -1,7 +1,7 @@