diff --git a/.github/actions/sdk-compliance-check-drift/action.yml b/.github/actions/sdk-compliance-check-drift/action.yml new file mode 100644 index 0000000..8c54347 --- /dev/null +++ b/.github/actions/sdk-compliance-check-drift/action.yml @@ -0,0 +1,68 @@ +name: Check capability matrix drift +description: Non-blocking check — warns when an `implemented` capability's registered symbols can't be found by the parser, or has none registered at all. + +inputs: + compliance-file: + description: Path to sdk-compliance.yaml in the calling repo + default: sdk-compliance.yaml + +runs: + using: composite + steps: + - name: Check capability matrix drift + shell: bash + env: + COMPLIANCE_FILE: ${{ inputs.compliance-file }} + run: | + npm run check-drift -- \ + "$GITHUB_WORKSPACE/_sdk-pr/$COMPLIANCE_FILE" \ + "$GITHUB_WORKSPACE/pr-symbols.json" \ + "$COMPLIANCE_FILE" + working-directory: _sdk-spec/scripts/capability-matrix + + - name: Upsert or clear drift comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('fs'); + const marker = ''; + const summaryPath = '_sdk-spec/scripts/capability-matrix/drift-summary.md'; + + try { + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find((c) => c.body && c.body.includes(marker)); + + if (fs.existsSync(summaryPath)) { + const body = fs.readFileSync(summaryPath, 'utf8'); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + } else if (existing) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + }); + } + } catch (error) { + // Fork PRs get a read-only GITHUB_TOKEN, so writes here 403. This + // comment is a best-effort enhancement — the ::warning:: annotations + // from the CLI step already carry the signal, so never fail the job. + console.error('Could not upsert/clear drift comment:', error); + } diff --git a/.github/workflows/validate-sdk-compliance-dart.yml b/.github/workflows/validate-sdk-compliance-dart.yml index dd803e5..9c494c7 100644 --- a/.github/workflows/validate-sdk-compliance-dart.yml +++ b/.github/workflows/validate-sdk-compliance-dart.yml @@ -26,6 +26,9 @@ jobs: name: Check public API against capability matrix if: github.event_name == 'pull_request' runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: supabase/sdk/.github/actions/sdk-compliance-check-setup@main with: @@ -52,3 +55,7 @@ jobs: - uses: supabase/sdk/.github/actions/sdk-compliance-check-symbols@main with: compliance-file: ${{ inputs.compliance-file }} + + - uses: supabase/sdk/.github/actions/sdk-compliance-check-drift@main + with: + compliance-file: ${{ inputs.compliance-file }} diff --git a/.github/workflows/validate-sdk-compliance-javascript.yml b/.github/workflows/validate-sdk-compliance-javascript.yml index 006a60b..efc28ab 100644 --- a/.github/workflows/validate-sdk-compliance-javascript.yml +++ b/.github/workflows/validate-sdk-compliance-javascript.yml @@ -34,6 +34,9 @@ jobs: name: Check public API against capability matrix if: github.event_name == 'pull_request' runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: supabase/sdk/.github/actions/sdk-compliance-check-setup@main with: @@ -71,3 +74,7 @@ jobs: - uses: supabase/sdk/.github/actions/sdk-compliance-check-symbols@main with: compliance-file: ${{ inputs.compliance-file }} + + - uses: supabase/sdk/.github/actions/sdk-compliance-check-drift@main + with: + compliance-file: ${{ inputs.compliance-file }} diff --git a/.github/workflows/validate-sdk-compliance-python.yml b/.github/workflows/validate-sdk-compliance-python.yml index 871100a..596f2c8 100644 --- a/.github/workflows/validate-sdk-compliance-python.yml +++ b/.github/workflows/validate-sdk-compliance-python.yml @@ -34,6 +34,9 @@ jobs: name: Check public API against capability matrix if: github.event_name == 'pull_request' runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: supabase/sdk/.github/actions/sdk-compliance-check-setup@main with: @@ -79,3 +82,7 @@ jobs: - uses: supabase/sdk/.github/actions/sdk-compliance-check-symbols@main with: compliance-file: ${{ inputs.compliance-file }} + + - uses: supabase/sdk/.github/actions/sdk-compliance-check-drift@main + with: + compliance-file: ${{ inputs.compliance-file }} diff --git a/.github/workflows/validate-sdk-compliance-swift.yml b/.github/workflows/validate-sdk-compliance-swift.yml index 49ad9fb..fbc141c 100644 --- a/.github/workflows/validate-sdk-compliance-swift.yml +++ b/.github/workflows/validate-sdk-compliance-swift.yml @@ -26,6 +26,9 @@ jobs: name: Check public API against capability matrix if: github.event_name == 'pull_request' runs-on: macos-latest + permissions: + contents: read + pull-requests: write steps: - uses: supabase/sdk/.github/actions/sdk-compliance-check-setup@main with: @@ -69,3 +72,7 @@ jobs: - uses: supabase/sdk/.github/actions/sdk-compliance-check-symbols@main with: compliance-file: ${{ inputs.compliance-file }} + + - uses: supabase/sdk/.github/actions/sdk-compliance-check-drift@main + with: + compliance-file: ${{ inputs.compliance-file }} diff --git a/docs/capability-matrix.md b/docs/capability-matrix.md new file mode 100644 index 0000000..f43d3d5 --- /dev/null +++ b/docs/capability-matrix.md @@ -0,0 +1,30 @@ +# Capability Matrix + +The capability matrix is the canonical feature registry for all Supabase client SDKs. It defines what features exist (ID, name, description, grouping) across the supported client SDKs, and each SDK repo declares which features it implements via its own `sdk-compliance.yaml` file. + +## Feature IDs + +Feature IDs use three segments: `{area}.{group}.{method}` (e.g., `auth.sign_in.email`, `storage.buckets.create`). IDs are defined in this repo's `capabilities/*.yaml` files and must be globally unique. + +## SDK Compliance Format + +Each SDK repo hosts a `sdk-compliance.yaml` at a known path: + +```yaml +sdk: javascript +features: + auth.sign_in.email: implemented + auth.mfa.enroll: + status: partially_implemented + note: "TOTP only" + symbols: + - GoTrueClient.mfaEnroll + storage.objects.upload: not_implemented +``` + +Valid status values: `implemented`, `partially_implemented`, `not_implemented`, `not_applicable`. + +The optional `symbols` field lists the public API symbol(s) that implement a feature. CI uses it two ways: + +- **New-symbol check** (blocking): a PR that adds a new public symbol not listed under any feature's `symbols` fails, prompting the author to register it. +- **Drift check** (non-blocking warning): every feature marked `implemented` with a `symbols` list is periodically re-verified against the SDK's actual public API. If a registered symbol can no longer be found (renamed, removed) — or if an `implemented` feature has no `symbols` registered at all to verify against — CI posts a warning so the entry can be corrected. diff --git a/scripts/capability-matrix/package.json b/scripts/capability-matrix/package.json index 2f75e49..8248e5a 100644 --- a/scripts/capability-matrix/package.json +++ b/scripts/capability-matrix/package.json @@ -12,6 +12,7 @@ "normalize-symbolgraph": "tsx src/normalize-symbolgraph-cli.ts", "normalize-griffe": "tsx src/normalize-griffe-cli.ts", "check-api-symbols": "tsx src/check-api-symbols.ts", + "check-drift": "tsx src/check-drift.ts", "aggregate": "tsx src/aggregate.ts", "report": "tsx src/cli.ts report", "test": "vitest run", diff --git a/scripts/capability-matrix/src/check-drift.ts b/scripts/capability-matrix/src/check-drift.ts new file mode 100644 index 0000000..8febf2f --- /dev/null +++ b/scripts/capability-matrix/src/check-drift.ts @@ -0,0 +1,72 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parse } from "yaml"; +import { checkDrift, formatDriftSummary } from "./drift-check.js"; +import { buildSourceMap } from "./compliance-source-map.js"; +import type { RawCompliance } from "./compliance.js"; +import type { ParseResult } from "./normalize-typedoc.js"; + +// ponytail: workflow-command annotations are single-line; strip newlines so a +// crafted featureId/symbol in committed YAML can't inject extra `::` commands. +function sanitize(s: string): string { + return s.replace(/[\r\n]+/g, " "); +} + +async function main(): Promise { + const [compliancePath, prFile, annotationPath] = process.argv.slice(2); + + if (!compliancePath || !prFile) { + console.error("Usage: check-drift [annotation-file-path]"); + return; + } + + let complianceText: string; + let compliance: RawCompliance; + + try { + complianceText = readFileSync(resolve(compliancePath), "utf8"); + compliance = parse(complianceText) as RawCompliance; + } catch (e) { + console.error(`Failed to read compliance file: ${(e as Error).message}`); + return; + } + + let prResult: ParseResult; + try { + prResult = JSON.parse(readFileSync(resolve(prFile), "utf8")) as ParseResult; + } catch (e) { + console.error(`Failed to read PR symbols: ${(e as Error).message}`); + return; + } + + const findings = checkDrift(prResult.symbols, compliance); + + if (findings.length === 0) { + console.log("✅ No capability matrix drift detected."); + return; + } + + const sourceMap = buildSourceMap(complianceText); + const annotationFile = annotationPath ?? compliancePath; + + for (const finding of findings) { + const line = finding.symbol + ? sourceMap.symbolLines.get(finding.symbol) + : sourceMap.featureLines.get(finding.featureId); + const featureId = sanitize(finding.featureId); + const message = finding.symbol + ? `${featureId}: expected symbol ${sanitize(finding.symbol)} not found in ${compliance.sdk}` + : `${featureId}: marked implemented but has no registered symbols to verify`; + console.log( + line !== undefined + ? `::warning file=${annotationFile},line=${line}::${message}` + : `::warning::${message}`, + ); + } + + const summary = formatDriftSummary(findings, compliance.sdk); + writeFileSync(resolve("drift-summary.md"), summary, "utf8"); + console.error(summary); +} + +main().catch((e) => { console.error(e); }); diff --git a/scripts/capability-matrix/src/compliance-source-map.ts b/scripts/capability-matrix/src/compliance-source-map.ts new file mode 100644 index 0000000..73b92d5 --- /dev/null +++ b/scripts/capability-matrix/src/compliance-source-map.ts @@ -0,0 +1,37 @@ +import { parseDocument, LineCounter, isMap, isSeq, isScalar } from "yaml"; + +export interface SourceMap { + symbolLines: Map; + featureLines: Map; +} + +export function buildSourceMap(complianceYamlText: string): SourceMap { + const symbolLines = new Map(); + const featureLines = new Map(); + + const lineCounter = new LineCounter(); + const doc = parseDocument(complianceYamlText, { lineCounter }); + + const features = doc.get("features", true); + if (!isMap(features)) return { symbolLines, featureLines }; + + for (const pair of features.items) { + const key = pair.key; + if (!isScalar(key) || typeof key.value !== "string") continue; + if (key.range) featureLines.set(key.value, lineCounter.linePos(key.range[0]).line); + + const entry = pair.value; + if (!isMap(entry)) continue; + + const symbols = entry.get("symbols", true); + if (!isSeq(symbols)) continue; + + for (const item of symbols.items) { + if (isScalar(item) && typeof item.value === "string" && item.range) { + symbolLines.set(item.value, lineCounter.linePos(item.range[0]).line); + } + } + } + + return { symbolLines, featureLines }; +} diff --git a/scripts/capability-matrix/src/drift-check.ts b/scripts/capability-matrix/src/drift-check.ts new file mode 100644 index 0000000..03ba83f --- /dev/null +++ b/scripts/capability-matrix/src/drift-check.ts @@ -0,0 +1,78 @@ +import type { RawCompliance } from "./compliance.js"; +import type { ParsedSymbol } from "./normalize-typedoc.js"; + +export interface DriftFinding { + featureId: string; + symbol?: string; +} + +export function checkDrift( + prSymbols: ParsedSymbol[], + compliance: RawCompliance, +): DriftFinding[] { + const prNames = new Set(prSymbols.map((s) => s.name)); + const findings: DriftFinding[] = []; + + for (const [featureId, value] of Object.entries(compliance.features ?? {})) { + let status: string | undefined; + let symbols: string[] | undefined; + + if (typeof value === "string") { + status = value; + } else if (typeof value === "object" && value !== null) { + status = value.status; + symbols = value.symbols; + } + + if (status !== "implemented") continue; + + if (!symbols || symbols.length === 0) { + findings.push({ featureId }); + continue; + } + + for (const symbol of symbols) { + if (!prNames.has(symbol)) { + findings.push({ featureId, symbol }); + } + } + } + + return findings; +} + +export function formatDriftSummary(findings: DriftFinding[], sdkName: string): string { + const missingSymbol = findings.filter((f) => f.symbol !== undefined); + const unverifiable = findings.filter((f) => f.symbol === undefined); + + const lines: string[] = [ + "", + "⚠️ Capability matrix drift detected", + ]; + + if (missingSymbol.length > 0) { + lines.push( + "", + `The following capabilities are marked \`implemented\` in the matrix but could not be found in ${sdkName}:`, + ...missingSymbol.map((f) => ` - ${f.featureId} → expected symbol: ${f.symbol}`), + ); + } + + if (unverifiable.length > 0) { + lines.push( + "", + `The following capabilities are marked \`implemented\` in ${sdkName} but have no registered symbols to verify:`, + ...unverifiable.map( + (f) => ` - ${f.featureId} (no \`symbols\` list — cannot confirm implementation exists)`, + ), + ); + } + + lines.push( + "", + "These may have been renamed, removed, or never registered. Please update the capability matrix.", + "See: https://github.com/supabase/sdk/blob/main/docs/capability-matrix.md", + ); + + return lines.join("\n"); +} diff --git a/scripts/capability-matrix/test/compliance-source-map.test.ts b/scripts/capability-matrix/test/compliance-source-map.test.ts new file mode 100644 index 0000000..524acd8 --- /dev/null +++ b/scripts/capability-matrix/test/compliance-source-map.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from "vitest"; +import { buildSourceMap } from "../src/compliance-source-map"; + +describe("buildSourceMap", () => { + it("maps each symbol to its 1-based line number", () => { + const yamlText = [ + "sdk: javascript", + "features:", + " auth.sign_up:", + " status: implemented", + " symbols:", + " - AuthClient.signUp", + " - AuthClient.signUpAnon", + " auth.sign_in:", + " status: implemented", + " symbols:", + " - AuthClient.signIn", + ].join("\n"); + + const { symbolLines } = buildSourceMap(yamlText); + + expect(symbolLines.get("AuthClient.signUp")).toBe(6); + expect(symbolLines.get("AuthClient.signUpAnon")).toBe(7); + expect(symbolLines.get("AuthClient.signIn")).toBe(11); + }); + + it("maps each feature id to its own key line", () => { + const yamlText = [ + "sdk: javascript", + "features:", + " auth.sign_up:", + " status: implemented", + " auth.mfa.enroll:", + " status: implemented", + ].join("\n"); + + const { featureLines } = buildSourceMap(yamlText); + + expect(featureLines.get("auth.sign_up")).toBe(3); + expect(featureLines.get("auth.mfa.enroll")).toBe(5); + }); + + it("returns empty maps for a compliance file with no features", () => { + const { symbolLines, featureLines } = buildSourceMap("sdk: javascript\nfeatures: {}\n"); + expect(symbolLines.size).toBe(0); + expect(featureLines.size).toBe(0); + }); +}); diff --git a/scripts/capability-matrix/test/drift-check.test.ts b/scripts/capability-matrix/test/drift-check.test.ts new file mode 100644 index 0000000..5f48090 --- /dev/null +++ b/scripts/capability-matrix/test/drift-check.test.ts @@ -0,0 +1,120 @@ +import { describe, it, expect } from "vitest"; +import { checkDrift, formatDriftSummary } from "../src/drift-check"; +import type { ParsedSymbol } from "../src/normalize-typedoc"; + +function sym(name: string): ParsedSymbol { + return { name, kind: "method", file: "src/index.ts" }; +} + +describe("checkDrift", () => { + it("returns no finding when a registered symbol is present", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.sign_up": { status: "implemented", symbols: ["AuthClient.signUp"] }, + }, + }; + expect(checkDrift([sym("AuthClient.signUp")], compliance)).toEqual([]); + }); + + it("reports a finding when a registered symbol is missing", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.mfa.enroll": { status: "implemented", symbols: ["MFAApi.enroll"] }, + }, + }; + expect(checkDrift([], compliance)).toEqual([ + { featureId: "auth.mfa.enroll", symbol: "MFAApi.enroll" }, + ]); + }); + + it("reports one finding per missing symbol when multiple are registered", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.sign_up": { + status: "implemented", + symbols: ["AuthClient.signUp", "AuthClient.signUpAnon"], + }, + }, + }; + expect(checkDrift([sym("AuthClient.signUp")], compliance)).toEqual([ + { featureId: "auth.sign_up", symbol: "AuthClient.signUpAnon" }, + ]); + }); + + it("skips entries that are not status implemented", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.sign_up": { status: "not_implemented", symbols: ["AuthClient.signUp"] }, + }, + }; + expect(checkDrift([], compliance)).toEqual([]); + }); + + it("reports an unverifiable finding when an implemented entry has no symbols list", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.mfa.enroll": { status: "implemented" }, + }, + }; + expect(checkDrift([], compliance)).toEqual([{ featureId: "auth.mfa.enroll" }]); + }); + + it("reports an unverifiable finding when an implemented entry has an empty symbols list", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.mfa.enroll": { status: "implemented", symbols: [] }, + }, + }; + expect(checkDrift([], compliance)).toEqual([{ featureId: "auth.mfa.enroll" }]); + }); + + it("reports an unverifiable finding for the string status shorthand", () => { + const compliance = { + sdk: "javascript", + features: { + "auth.mfa.enroll": "implemented", + }, + }; + expect(checkDrift([], compliance)).toEqual([{ featureId: "auth.mfa.enroll" }]); + }); +}); + +describe("formatDriftSummary", () => { + it("includes the marker and the symbol-not-found section", () => { + const msg = formatDriftSummary( + [{ featureId: "auth.mfa.enroll", symbol: "MFAApi.enroll" }], + "supabase-flutter", + ); + expect(msg).toContain(""); + expect(msg).toContain("⚠️ Capability matrix drift detected"); + expect(msg).toContain("could not be found in supabase-flutter"); + expect(msg).toContain("auth.mfa.enroll → expected symbol: MFAApi.enroll"); + }); + + it("includes the unverifiable section for symbol-less findings", () => { + const msg = formatDriftSummary([{ featureId: "auth.mfa.enroll" }], "supabase-flutter"); + expect(msg).toContain("no registered symbols to verify"); + expect(msg).toContain( + "auth.mfa.enroll (no `symbols` list — cannot confirm implementation exists)", + ); + }); + + it("omits the symbol-not-found section when there are no such findings", () => { + const msg = formatDriftSummary([{ featureId: "auth.mfa.enroll" }], "javascript"); + expect(msg).not.toContain("could not be found in javascript"); + }); + + it("omits the unverifiable section when there are no such findings", () => { + const msg = formatDriftSummary( + [{ featureId: "auth.mfa.enroll", symbol: "MFAApi.enroll" }], + "javascript", + ); + expect(msg).not.toContain("no registered symbols to verify"); + }); +});