diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb7da3..3aaafcd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,3 +40,33 @@ jobs: - name: Inspect publish contents run: npm pack --dry-run + + pagination-baseline: + name: Pagination baseline (nonblocking) + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + + - name: Setup Node + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 22 + cache: npm + + - name: Install + run: npm ci + + - name: Measure guarded pagination baseline + run: npm run benchmark:pagination + + - name: Upload machine-readable baseline + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: pagination-benchmark-results + path: artifacts/pagination-benchmark.json + if-no-files-found: error + retention-days: 30 diff --git a/.gitignore b/.gitignore index aa67b6a..e4534a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ coverage/ .vitest/ +artifacts/ *.log .DS_Store Thumbs.db diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ba9e1d..6289d01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ - Additive per-control security capability/enabled results with sanitized recovery, deterministic enabled/disabled/omitted/403/404/partial fixtures, and preserved legacy `state` booleans/`ok` semantics. +- Exported pagination hard-guard maximums, deterministic 100-page/10,000-item + measurement with guard fixtures and machine-readable latency/memory evidence, and a + nonblocking hosted baseline artifact job. ### Changed diff --git a/README.md b/README.md index 512919e..1909282 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,9 @@ This package now covers the smallest high-value GitHub admin workflow end to end retain their requested limit and return `truncated` when more results exist - internal GitHub requests use a 10-second default timeout, retry bounded read-only transient failures, and surface sanitized structured recovery metadata +- pagination hard guard maximums are exported as `MAX_COLLECTION_PAGES` and + `MAX_COLLECTION_ITEMS`; the deterministic offline benchmark records hosted baseline + evidence without enforcing a budget until stable shared-runner samples exist - security verification keeps legacy `state` booleans and `ok` while adding per-control capability, enabled-state, sanitized reason/status, and bounded recovery results; masked 404 responses remain `unknown` rather than being treated as proof of unsupported state diff --git a/benchmarks/pagination.measurement.test.ts b/benchmarks/pagination.measurement.test.ts new file mode 100644 index 0000000..d164d14 --- /dev/null +++ b/benchmarks/pagination.measurement.test.ts @@ -0,0 +1,152 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { describe, expect, it } from "vitest"; +import { + MAX_COLLECTION_ITEMS, + MAX_COLLECTION_PAGES, + paginate, + type GitHubClient, + type PaginatedResult, +} from "../src/http.js"; + +const WARMUP_RUNS = 3; +const SAMPLE_RUNS = 25; +const ITEMS_PER_PAGE = MAX_COLLECTION_ITEMS / MAX_COLLECTION_PAGES; +const START_PATH = "/benchmark/items?page=1"; +const FIXTURE_DIRECTORY = path.resolve(import.meta.dirname, "../tests/fixtures/pagination"); +const OUTPUT_PATH = path.resolve(process.env.PAGINATION_BENCHMARK_OUTPUT ?? "artifacts/pagination-benchmark.json"); + +interface FixturePage { + path: string; + items: number[]; + next?: string; +} + +interface PaginationFixture { + name: string; + startPath: string; + pages: FixturePage[]; + options: { maxItems?: number; maxPages?: number }; + expected: PaginatedResult; +} + +function link(next?: string): Headers { + return new Headers(next ? { link: `; rel="next"` } : {}); +} + +function fixtureClient(fixture: PaginationFixture): GitHubClient { + const pages = new Map(fixture.pages.map((page) => [page.path, page])); + return { + request: async () => undefined, + requestWithMeta: async (requestPath: string) => { + const page = pages.get(requestPath); + if (!page) throw new Error(`fixture '${fixture.name}' has no page for ${requestPath}`); + return { data: page.items, status: 200, headers: link(page.next) }; + }, + } as unknown as GitHubClient; +} + +function maximumCollectionClient(observeMemory: () => void): GitHubClient { + return { + request: async () => undefined, + requestWithMeta: async (requestPath: string) => { + if (!requestPath.startsWith("/benchmark/items?page=")) { + throw new Error(`benchmark attempted unexpected request: ${requestPath}`); + } + const page = Number(new URL(requestPath, "https://api.github.com").searchParams.get("page")); + if (!Number.isInteger(page) || page < 1 || page > MAX_COLLECTION_PAGES) { + throw new Error(`benchmark attempted out-of-range page: ${requestPath}`); + } + const first = (page - 1) * ITEMS_PER_PAGE; + const data = Array.from({ length: ITEMS_PER_PAGE }, (_, index) => first + index); + observeMemory(); + const next = page < MAX_COLLECTION_PAGES ? `/benchmark/items?page=${page + 1}` : undefined; + return { data, status: 200, headers: link(next) }; + }, + } as unknown as GitHubClient; +} + +function percentile(sorted: number[], quantile: number): number { + return sorted[Math.max(0, Math.ceil(sorted.length * quantile) - 1)]!; +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +async function readFixtures(): Promise { + const names = ["max-items.json", "max-pages.json", "repeated-link.json", "exact-boundary.json"]; + return Promise.all(names.map(async (name) => JSON.parse(await readFile(path.join(FIXTURE_DIRECTORY, name), "utf8")) as PaginationFixture)); +} + +describe("guarded pagination measurement", () => { + it("measures a deterministic fake 100-page/10,000-item collection", async () => { + const fixtureResults = []; + for (const fixture of await readFixtures()) { + const actual = await paginate(fixtureClient(fixture), fixture.startPath, fixture.options); + expect(actual).toEqual(fixture.expected); + fixtureResults.push({ name: fixture.name, passed: true, result: actual }); + } + + let peakRssBytes = process.memoryUsage().rss; + let peakHeapUsedBytes = process.memoryUsage().heapUsed; + const observeMemory = () => { + const memory = process.memoryUsage(); + peakRssBytes = Math.max(peakRssBytes, memory.rss); + peakHeapUsedBytes = Math.max(peakHeapUsedBytes, memory.heapUsed); + }; + const run = async () => { + const result = await paginate(maximumCollectionClient(observeMemory), START_PATH, { + maxItems: MAX_COLLECTION_ITEMS, + maxPages: MAX_COLLECTION_PAGES, + }); + observeMemory(); + return result; + }; + const assertMaximumResult = (result: PaginatedResult) => { + expect(result.pages).toBe(MAX_COLLECTION_PAGES); + expect(result.truncated).toBe(false); + expect(result.items).toHaveLength(MAX_COLLECTION_ITEMS); + expect(result.items[0]).toBe(0); + expect(result.items.at(-1)).toBe(MAX_COLLECTION_ITEMS - 1); + }; + + for (let index = 0; index < WARMUP_RUNS; index += 1) assertMaximumResult(await run()); + const samplesMs: number[] = []; + for (let index = 0; index < SAMPLE_RUNS; index += 1) { + const started = performance.now(); + const result = await run(); + samplesMs.push(performance.now() - started); + assertMaximumResult(result); + } + const sorted = [...samplesMs].sort((left, right) => left - right); + const output = { + schemaVersion: 1, + benchmark: "guarded-pagination-maximum-collection", + measurementOnly: true, + budget: null, + configuration: { + warmupRuns: WARMUP_RUNS, + sampleRuns: SAMPLE_RUNS, + pages: MAX_COLLECTION_PAGES, + items: MAX_COLLECTION_ITEMS, + itemsPerPage: ITEMS_PER_PAGE, + transport: "deterministic-in-memory-fake", + liveGitHubApi: false, + }, + environment: { node: process.version, platform: process.platform, arch: process.arch }, + latencyMs: { + p50: round(percentile(sorted, 0.5)), + p95: round(percentile(sorted, 0.95)), + p99: round(percentile(sorted, 0.99)), + samples: samplesMs.map(round), + }, + memoryBytes: { peakRss: peakRssBytes, peakHeapUsed: peakHeapUsedBytes }, + fixtures: fixtureResults, + }; + await mkdir(path.dirname(OUTPUT_PATH), { recursive: true }); + await writeFile(OUTPUT_PATH, `${JSON.stringify(output, null, 2)}\n`, "utf8"); + console.log(`PAGINATION_BENCHMARK_JSON=${JSON.stringify(output)}`); + }); +}); diff --git a/benchmarks/vitest.measurement.config.ts b/benchmarks/vitest.measurement.config.ts new file mode 100644 index 0000000..4abcd40 --- /dev/null +++ b/benchmarks/vitest.measurement.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["benchmarks/pagination.measurement.test.ts"], + }, +}); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 679cd19..d4baee9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -91,7 +91,9 @@ npm pack --dry-run `npm run verify` runs type checking, the Vitest suite with committed coverage thresholds, and release/changelog metadata verification. CI runs the same checks on -pull requests and pushes to `main`. `npm run benchmark` is required for -performance-sensitive helper or planning changes but is not a shared-runner gate. +pull requests and pushes to `main`. `npm run benchmark` runs the helper microbenchmarks +and the deterministic guarded-pagination measurement but does not enforce a shared-runner +budget. CI uploads the nonblocking hosted pagination baseline described in +[`PAGINATION_BENCHMARK.md`](PAGINATION_BENCHMARK.md). There is no separate compile or bundle artifact: Pi loads the TypeScript extension entry point declared in `package.json`. diff --git a/docs/PAGINATION_BENCHMARK.md b/docs/PAGINATION_BENCHMARK.md new file mode 100644 index 0000000..9645024 --- /dev/null +++ b/docs/PAGINATION_BENCHMARK.md @@ -0,0 +1,19 @@ +# Guarded pagination benchmark + +## Purpose + +The pagination benchmark measures the existing bounded collection path with a deterministic, in-memory `GitHubClient` fake. It performs no network requests and never calls the live GitHub API. The maximum workload is 100 pages and 10,000 items, matching the exported hard collection guards. + +Run it locally with: + +```bash +npm run benchmark:pagination +``` + +The command performs three warmup runs and 25 measured runs. It writes machine-readable JSON to `artifacts/pagination-benchmark.json` (override with `PAGINATION_BENCHMARK_OUTPUT`) and reports p50, p95, and p99 latency plus the peak observed RSS and heap usage. Guard fixtures cover maximum-item truncation, maximum-page truncation, repeated links, and an exact-boundary non-truncated result. + +## Measurement versus decision + +This change establishes evidence collection only. The hosted `pagination-baseline` job is deliberately nonblocking and uploads `pagination-benchmark-results` for comparison across GitHub-hosted runs. It does not encode a latency or memory budget. + +After enough comparable hosted samples exist, maintainers should review run-to-run variance and open a separate budget PR if stable bounds can be justified. That decision PR should document the selected statistic, tolerated variance, runner assumptions, and failure/rebaseline process before turning a threshold into a required CI gate. A single local or hosted sample is not sufficient evidence for a hard budget. diff --git a/package.json b/package.json index 8ae6f5e..2abd7a5 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,9 @@ "typecheck": "tsc --noEmit", "test": "vitest run", "test:coverage": "vitest run --coverage", - "benchmark": "vitest bench --run", + "benchmark:core": "vitest bench --run", + "benchmark:pagination": "vitest run --config benchmarks/vitest.measurement.config.ts", + "benchmark": "npm run benchmark:core && npm run benchmark:pagination", "verify:release": "node scripts/verify-release.mjs", "verify:workflows": "node scripts/verify-workflows.mjs", "verify": "npm run verify:workflows && npm run typecheck && npm run test:coverage && npm run verify:release" diff --git a/src/http.ts b/src/http.ts index 56bff1b..c521bcc 100644 --- a/src/http.ts +++ b/src/http.ts @@ -4,6 +4,8 @@ export const DEFAULT_REQUEST_TIMEOUT_MS = 10_000; export const DEFAULT_MAX_RETRIES = 2; export const DEFAULT_MAX_PAGES = 20; export const DEFAULT_MAX_ITEMS = 2_000; +export const MAX_COLLECTION_PAGES = 100; +export const MAX_COLLECTION_ITEMS = 10_000; export const MAX_ERROR_BODY_LENGTH = 512; export interface GitHubRequestOptions { @@ -176,8 +178,8 @@ export async function paginateMapped( mapPage: (page: TPage) => TItem[], options: PaginationOptions = {}, ): Promise> { - const maxPages = boundedInteger(options.maxPages, DEFAULT_MAX_PAGES, 1, 100); - const maxItems = boundedInteger(options.maxItems, DEFAULT_MAX_ITEMS, 1, 10_000); + const maxPages = boundedInteger(options.maxPages, DEFAULT_MAX_PAGES, 1, MAX_COLLECTION_PAGES); + const maxItems = boundedInteger(options.maxItems, DEFAULT_MAX_ITEMS, 1, MAX_COLLECTION_ITEMS); const items: TItem[] = []; const seen = new Set(); let next: string | undefined = path; diff --git a/tests/fixtures/pagination/exact-boundary.json b/tests/fixtures/pagination/exact-boundary.json new file mode 100644 index 0000000..0c1e387 --- /dev/null +++ b/tests/fixtures/pagination/exact-boundary.json @@ -0,0 +1,9 @@ +{ + "name": "exact-boundary-not-truncated", + "startPath": "/items?page=1", + "pages": [ + { "path": "/items?page=1", "items": [1, 2] } + ], + "options": { "maxItems": 2 }, + "expected": { "items": [1, 2], "pages": 1, "truncated": false } +} diff --git a/tests/fixtures/pagination/max-items.json b/tests/fixtures/pagination/max-items.json new file mode 100644 index 0000000..c6e5c63 --- /dev/null +++ b/tests/fixtures/pagination/max-items.json @@ -0,0 +1,9 @@ +{ + "name": "max-items", + "startPath": "/items?page=1", + "pages": [ + { "path": "/items?page=1", "items": [1, 2, 3], "next": "/items?page=2" } + ], + "options": { "maxItems": 2 }, + "expected": { "items": [1, 2], "pages": 1, "truncated": true, "reason": "max_items" } +} diff --git a/tests/fixtures/pagination/max-pages.json b/tests/fixtures/pagination/max-pages.json new file mode 100644 index 0000000..6864d8b --- /dev/null +++ b/tests/fixtures/pagination/max-pages.json @@ -0,0 +1,9 @@ +{ + "name": "max-pages", + "startPath": "/items?page=1", + "pages": [ + { "path": "/items?page=1", "items": [1], "next": "/items?page=2" } + ], + "options": { "maxPages": 1 }, + "expected": { "items": [1], "pages": 1, "truncated": true, "reason": "max_pages" } +} diff --git a/tests/fixtures/pagination/repeated-link.json b/tests/fixtures/pagination/repeated-link.json new file mode 100644 index 0000000..f0f76e6 --- /dev/null +++ b/tests/fixtures/pagination/repeated-link.json @@ -0,0 +1,9 @@ +{ + "name": "repeated-link", + "startPath": "/items?page=1", + "pages": [ + { "path": "/items?page=1", "items": [1], "next": "/items?page=1" } + ], + "options": {}, + "expected": { "items": [1], "pages": 1, "truncated": true, "reason": "repeated_link" } +} diff --git a/tests/http.test.ts b/tests/http.test.ts index 14da29b..bcf7902 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -2,6 +2,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createGitHubClient, GitHubApiError, + MAX_COLLECTION_ITEMS, + MAX_COLLECTION_PAGES, paginate, paginateMapped, parseNextLink, @@ -115,6 +117,21 @@ describe("pagination", () => { await expect(paginate(clientFor([{ data: [1, 2], link }]), "/items", { maxItems: 1 })).resolves.toMatchObject({ items: [1], reason: "max_items", truncated: true }); }); + it("clamps caller overrides to the exported hard collection guards", async () => { + const pages = Array.from({ length: MAX_COLLECTION_PAGES }, (_, index) => ({ + data: Array.from({ length: MAX_COLLECTION_ITEMS / MAX_COLLECTION_PAGES }, () => index), + link: index + 1 < MAX_COLLECTION_PAGES + ? `; rel="next"` + : undefined, + })); + const result = await paginate(clientFor(pages), "/items", { + maxItems: Number.MAX_SAFE_INTEGER, + maxPages: Number.MAX_SAFE_INTEGER, + }); + expect(result.items).toHaveLength(MAX_COLLECTION_ITEMS); + expect(result).toMatchObject({ pages: MAX_COLLECTION_PAGES, truncated: false }); + }); + it("maps wrapped collection pages", async () => { const client = clientFor([{ data: { check_runs: [{ name: "one" }] } }]); await expect(paginateMapped(client, "/checks", (page: any) => page.check_runs)).resolves.toMatchObject({ items: [{ name: "one" }] }); diff --git a/tests/pagination-fixtures.test.ts b/tests/pagination-fixtures.test.ts new file mode 100644 index 0000000..4296aad --- /dev/null +++ b/tests/pagination-fixtures.test.ts @@ -0,0 +1,41 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { paginate, type GitHubClient, type PaginatedResult, type PaginationOptions } from "../src/http.js"; + +interface Fixture { + name: string; + startPath: string; + pages: Array<{ path: string; items: number[]; next?: string }>; + options: PaginationOptions; + expected: PaginatedResult; +} + +const directory = path.join(import.meta.dirname, "fixtures", "pagination"); +const fixtureNames = ["max-items.json", "max-pages.json", "repeated-link.json", "exact-boundary.json"]; + +async function loadFixture(name: string): Promise { + return JSON.parse(await readFile(path.join(directory, name), "utf8")) as Fixture; +} + +function clientFor(fixture: Fixture): GitHubClient { + const pages = new Map(fixture.pages.map((page) => [page.path, page])); + return { + request: async () => undefined, + requestWithMeta: async (requestPath: string) => { + const page = pages.get(requestPath); + if (!page) throw new Error(`fixture '${fixture.name}' has no page for ${requestPath}`); + const headers = new Headers(page.next ? { link: `; rel="next"` } : {}); + return { data: page.items, status: 200, headers }; + }, + } as unknown as GitHubClient; +} + +describe("pagination guard fixtures", () => { + for (const fixtureName of fixtureNames) { + it(fixtureName, async () => { + const fixture = await loadFixture(fixtureName); + await expect(paginate(clientFor(fixture), fixture.startPath, fixture.options)).resolves.toEqual(fixture.expected); + }); + } +}); diff --git a/tsconfig.json b/tsconfig.json index 2c03a46..872885f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,5 +10,5 @@ "resolveJsonModule": true, "types": ["node", "vitest/globals"] }, - "include": ["extensions/**/*.ts", "src/**/*.ts", "tests/**/*.ts"] + "include": ["extensions/**/*.ts", "src/**/*.ts", "tests/**/*.ts", "benchmarks/**/*.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts index 4a94cdb..8e6638a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { + include: ["tests/**/*.test.ts"], coverage: { provider: "v8", reporter: ["text", "json-summary"],