diff --git a/.github/feed-health.json b/.github/feed-health.json index 3abb200..0967ef4 100644 --- a/.github/feed-health.json +++ b/.github/feed-health.json @@ -1,12 +1 @@ -{ - "https://communitybuilders.dev/events/feed.json": { - "failures": 3, - "since": "2026-08-28", - "lastError": "schema errors:" - }, - "https://eventos.wiki/events.json": { - "failures": 4, - "since": "2026-08-27", - "lastError": "fetch failed: The operation was aborted due to timeout" - } -} +{} diff --git a/.github/workflows/check-feeds.yml b/.github/workflows/check-feeds.yml index fdfc7a8..4d32fcf 100644 --- a/.github/workflows/check-feeds.yml +++ b/.github/workflows/check-feeds.yml @@ -1,6 +1,9 @@ # Daily health check of the registered adopter feeds. scripts/check-feeds.mjs does -# the work (fetch + validate + issue management); this workflow only persists the -# consecutive-failure state between runs by committing .github/feed-health.json. +# the work (fetch + validate + issue management); this workflow only persists what +# has to survive between runs: the private failure window (.github/feed-health.json) +# and the public status the site serves (docs/data/feed-health.json). Both are +# written to only when something actually changes, so a healthy registry produces +# no daily commit. name: Check adopter feeds on: @@ -26,18 +29,24 @@ jobs: - name: Check feeds env: GITHUB_TOKEN: ${{ github.token }} - FEED_FAILURE_THRESHOLD: "3" + # A schema error or a 404 is the publisher's to fix: report it after two + # runs. A timeout or a 5xx has to dominate the window before it is worth + # an issue — feeds are slow for a morning all the time. + FEED_HARD_THRESHOLD: "2" + FEED_TRANSIENT_THRESHOLD: "5" + FEED_WINDOW: "7" run: node scripts/check-feeds.mjs - - name: Persist failure counters + - name: Persist health state run: | - if git diff --quiet .github/feed-health.json 2>/dev/null && ! git status --porcelain .github/feed-health.json | grep -q .; then + PATHS=".github/feed-health.json docs/data/feed-health.json" + if [ -z "$(git status --porcelain -- $PATHS)" ]; then echo "Health state unchanged." exit 0 fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add .github/feed-health.json + git add $PATHS git commit -m "chore(health): update feed failure counters [skip ci]" git pull --rebase origin "${GITHUB_REF_NAME}" git push origin HEAD:"${GITHUB_REF_NAME}" diff --git a/docs/data/feed-health.json b/docs/data/feed-health.json new file mode 100644 index 0000000..7bd25c6 --- /dev/null +++ b/docs/data/feed-health.json @@ -0,0 +1,31 @@ +{ + "_comment": "Health of the registered adopter feeds, refreshed daily by scripts/check-feeds.mjs. Feeds are validated against the spec version they declare; `supported` false means older than the supported window, which is information, not a failure. `specVersion` is the last version seen, kept through an outage. `updated` is when this status last changed.", + "updated": "2026-08-30", + "latestSpecVersion": "0.4.0", + "feeds": [ + { + "name": "Community Builders Community Events", + "url": "https://communitybuilders.dev/#events", + "feed": "https://communitybuilders.dev/events/feed.json", + "status": "ok", + "specVersion": "0.3.0", + "supported": true, + "cors": true, + "failureKind": null, + "failingSince": null, + "error": null + }, + { + "name": "eventos.wiki", + "url": "https://eventos.wiki", + "feed": "https://eventos.wiki/events.json", + "status": "ok", + "specVersion": "0.4.0", + "supported": true, + "cors": true, + "failureKind": null, + "failingSince": null, + "error": null + } + ] +} diff --git a/scripts/check-feeds.mjs b/scripts/check-feeds.mjs index 145f12d..053ed71 100644 --- a/scripts/check-feeds.mjs +++ b/scripts/check-feeds.mjs @@ -1,30 +1,59 @@ #!/usr/bin/env node /** * Health check for registered adopter feeds: fetches every feed in - * docs/data/adopters.json and validates it against the schemas. + * docs/data/adopters.json and validates it against the schemas of the spec + * version the feed itself declares — not against the newest one. A feed that is + * valid 0.3 is a healthy feed; publishing a new spec version must never turn + * every adopter that hasn't migrated yet into a daily issue. * - * Consecutive failures are tracked in .github/feed-health.json (committed by the - * workflow, not published — GitHub Pages only serves docs/). When a feed reaches - * FEED_FAILURE_THRESHOLD consecutive failures an issue is opened; when it - * recovers, the counter resets and the issue is closed. Nothing is ever removed - * from the registry automatically — feeds die over a weekend all the time, and - * delisting a community is a human call (a PR). + * Three separate signals, deliberately not collapsed into one boolean: + * - reachable — answers with JSON at all + * - valid — conforms to the schemas of its own declared version + * - current — how its version compares to the newest one. Metadata only: + * an old but supported version never opens an issue, it is + * reported in docs/data/feed-health.json for consumers to see. + * + * Failures are also classed before they count. A timeout or a 5xx is transient + * (slow origin, a bad afternoon) and needs to persist across most of a window of + * runs; a 4xx, non-JSON body or schema error is deterministic and is reported + * sooner. Both are tracked in .github/feed-health.json (committed by the + * workflow, not published); the public status lives in docs/data/feed-health.json. + * + * Nothing is ever removed from the registry automatically — feeds die over a + * weekend all the time, and delisting a community is a human call (a PR). * * With GITHUB_TOKEN + GITHUB_REPOSITORY set it manages the issues itself via the * REST API; without them (local run) it prints what it would do. */ -import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync, readdirSync } from "node:fs"; import { join } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; import { annotationKeywords, customFormats, customKeywords } from "../index.js"; -const SPEC = "spec/v0.4"; const ADOPTERS = join("docs", "data", "adopters.json"); const STATE = join(".github", "feed-health.json"); -const THRESHOLD = Number(process.env.FEED_FAILURE_THRESHOLD || 3); +const PUBLIC_STATE = join("docs", "data", "feed-health.json"); const LABEL = "feed-health"; +/* A deterministic failure is the publisher's to fix and won't fix itself: report + it after two runs. A transient one has to dominate a window of runs before it + is worth anyone's attention — a single slow morning must not open an issue. */ +const HARD_THRESHOLD = Number(process.env.FEED_HARD_THRESHOLD || 2); +const TRANSIENT_THRESHOLD = Number(process.env.FEED_TRANSIENT_THRESHOLD || 5); +const WINDOW = Number(process.env.FEED_WINDOW || 7); + +/* Fetching: generous timeout and retries, because the alternative is calling a + working feed broken. eventos.wiki answers in ~50s with half a megabyte. */ +const TIMEOUT_MS = Number(process.env.FEED_TIMEOUT_MS || 60000); +const ATTEMPTS = Number(process.env.FEED_ATTEMPTS || 3); +const BACKOFF_MS = [2000, 6000]; + +/* How many minor versions back still validate as healthy. Older than this and + the feed is reported as unsupported in the public status — still not an issue, + because the schemas are still here and the feed still works for consumers. */ +const SUPPORT_WINDOW = Number(process.env.FEED_SUPPORT_WINDOW || 3); + const token = process.env.GITHUB_TOKEN; const repo = process.env.GITHUB_REPOSITORY; @@ -55,19 +84,41 @@ async function openIssues() { return gh("GET", `/issues?state=open&labels=${LABEL}&per_page=100`); } -/* ---------- validation ---------- */ +/* ---------- validators, one per published spec version ---------- */ -const ajv = new Ajv2020({ strict: true, strictRequired: false, allErrors: true }); -addFormats(ajv); -// The schemas carry annotations strict mode would refuse to compile; they constrain nothing. -for (const kw of annotationKeywords) ajv.addKeyword(kw); -// Same setup as scripts/register-adopter.mjs: without the custom formats and keywords -// (`ote-local-date-time`, `distinctTranslationLanguages`) strict mode refuses to compile -// the schema at all, and the health check dies before fetching a single feed. -for (const f of customFormats) ajv.addFormat(f.name, f.validate); -for (const kw of customKeywords) ajv.addKeyword(kw); -ajv.addSchema(JSON.parse(readFileSync(join(SPEC, "event.schema.json"), "utf8"))); -const validateFeed = ajv.compile(JSON.parse(readFileSync(join(SPEC, "feed.schema.json"), "utf8"))); +// spec/v0.3 → "0.3.0". Every published version keeps its schemas in the repo, so +// a feed can always be checked against the rules it was actually written to. +const versions = readdirSync("spec") + .filter((d) => /^v\d+\.\d+$/.test(d)) + .map((dir) => ({ dir, minor: dir.slice(1).split(".").map(Number) })) + .sort((a, b) => a.minor[0] - b.minor[0] || a.minor[1] - b.minor[1]); + +const byVersion = new Map(versions.map((v) => [`${v.minor[0]}.${v.minor[1]}.0`, v.dir])); +const LATEST = [...byVersion.keys()].at(-1); +const supported = new Set([...byVersion.keys()].slice(-SUPPORT_WINDOW)); + +const validators = new Map(); + +function validatorFor(version) { + if (validators.has(version)) return validators.get(version); + const dir = join("spec", byVersion.get(version)); + const ajv = new Ajv2020({ strict: true, strictRequired: false, allErrors: true }); + addFormats(ajv); + // The schemas carry annotations strict mode would refuse to compile; they constrain nothing. + for (const kw of annotationKeywords) ajv.addKeyword(kw); + // Same setup as scripts/register-adopter.mjs: without the custom formats and keywords + // (`ote-local-date-time`, `distinctTranslationLanguages`) strict mode refuses to compile + // the schema at all, and the health check dies before fetching a single feed. + for (const f of customFormats) ajv.addFormat(f.name, f.validate); + for (const kw of customKeywords) ajv.addKeyword(kw); + ajv.addSchema(JSON.parse(readFileSync(join(dir, "event.schema.json"), "utf8"))); + const validate = ajv.compile(JSON.parse(readFileSync(join(dir, "feed.schema.json"), "utf8"))); + const entry = { ajv, validate }; + validators.set(version, entry); + return entry; +} + +/* ---------- fetching ---------- */ /* A feed nobody can read from a browser is still a valid, healthy feed — this is a note, never a failure. `*` is what lets an arbitrary client read it; anything else @@ -83,89 +134,170 @@ function corsNote(res) { : "no `Access-Control-Allow-Origin`"; } -async function checkFeed(url) { +const ok = (extra) => ({ error: null, kind: null, ...extra }); +const hard = (error, extra) => ({ error, kind: "hard", ...extra }); +const transient = (error, extra) => ({ error, kind: "transient", ...extra }); + +// One attempt. Transport errors, 429 and 5xx are transient; everything the origin +// answers deliberately is not, and retrying it only wastes the publisher's bandwidth. +async function attempt(url) { let res; try { - res = await fetch(url, { headers: { Origin: ORIGIN }, signal: AbortSignal.timeout(20000) }); + res = await fetch(url, { headers: { Origin: ORIGIN }, signal: AbortSignal.timeout(TIMEOUT_MS) }); } catch (err) { - return { error: `fetch failed: ${err.message}` }; + return transient(`fetch failed: ${err.message}`); } const cors = corsNote(res); - if (!res.ok) return { error: `HTTP ${res.status}`, cors }; + if (res.status === 429 || res.status >= 500) return transient(`HTTP ${res.status}`, { cors }); + if (!res.ok) return hard(`HTTP ${res.status}`, { cors }); + let doc; try { doc = await res.json(); } catch { - return { error: "response is not JSON", cors }; + return hard("response is not JSON", { cors }); + } + + const version = typeof doc.specVersion === "string" ? doc.specVersion : null; + if (!version || !byVersion.has(version)) { + // Not a version this repo has ever published: there is no schema to judge it by. + return hard(`unknown specVersion: ${JSON.stringify(doc.specVersion ?? null)}`, { cors }); + } + + // Older than the support window: reachable, and that is all we claim. Judging it + // against rules we no longer support would report a failure nobody agreed to fix. + if (!supported.has(version)) return ok({ cors, version }); + + const { ajv, validate } = validatorFor(version); + if (!validate(doc)) { + return hard("schema errors:\n" + ajv.errorsText(validate.errors, { separator: "\n" }), { cors, version }); } - if (!validateFeed(doc)) { - return { error: "schema errors:\n" + ajv.errorsText(validateFeed.errors, { separator: "\n" }), cors }; + return ok({ cors, version }); +} + +async function checkFeed(url) { + for (let i = 0; i < ATTEMPTS; i++) { + const result = await attempt(url); + if (result.kind !== "transient" || i === ATTEMPTS - 1) return result; + await new Promise((r) => setTimeout(r, BACKOFF_MS[Math.min(i, BACKOFF_MS.length - 1)])); } - return { error: null, cors }; // healthy +} + +/* ---------- window bookkeeping ---------- */ + +// Last WINDOW runs, oldest first: "." healthy, "x" failed. +const push = (window, char) => (String(window || "") + char).slice(-WINDOW); +const count = (window, char) => [...String(window || "")].filter((c) => c === char).length; + +function shouldAlert(entry) { + return entry.kind === "hard" + ? entry.failures >= HARD_THRESHOLD + : count(entry.window, "x") >= TRANSIENT_THRESHOLD; } /* ---------- main ---------- */ const adopters = JSON.parse(readFileSync(ADOPTERS, "utf8")).adopters.filter((a) => a.feed); const state = existsSync(STATE) ? JSON.parse(readFileSync(STATE, "utf8")) : {}; +const previous = existsSync(PUBLIC_STATE) ? JSON.parse(readFileSync(PUBLIC_STATE, "utf8")) : {}; +// A feed that times out tells us nothing about its version: keep the last one we saw +// rather than blanking it, so the public status stays useful through an outage. +const lastKnown = new Map((previous.feeds || []).map((f) => [f.feed, f.specVersion])); const today = new Date().toISOString().slice(0, 10); const next = {}; -let failures = 0; +const status = []; +let failing = 0; const noCors = []; for (const adopter of adopters) { - const { error, cors } = await checkFeed(adopter.feed); + const { error, kind, cors, version } = await checkFeed(adopter.feed); const prev = state[adopter.feed]; + const stale = version && !supported.has(version); + const seen = version || lastKnown.get(adopter.feed) || null; + + status.push({ + name: adopter.name, + url: adopter.url || null, + feed: adopter.feed, + status: error ? "failing" : "ok", + specVersion: seen, + // Old but still checkable against its own schemas: information for consumers, + // never a health failure. See the support window at the top of this file. + supported: seen ? supported.has(seen) : null, + cors: error ? null : cors === null, + failureKind: kind, + failingSince: error ? (prev ? prev.since : today) : null, + error: error ? error.split("\n")[0] : null, + }); if (!error) { - console.log(` ok ${adopter.name} — ${adopter.feed}`); + const age = stale ? ` (spec ${version}, older than the supported window)` : ` (spec ${version})`; + console.log(` ok ${adopter.name} — ${adopter.feed}${age}`); // Only worth saying about a feed that otherwise works: a broken one has a bigger problem. if (cors) { noCors.push(adopter); console.log(` ${cors} — browser-based clients can't read this feed`); } - if (prev && prev.failures >= THRESHOLD) { + if (prev && prev.alerted) { // Recovered after having been reported: close the alert. const issues = await openIssues(); const open = issues.find((i) => i.title === issueTitle(adopter.name)); if (open) { await gh("POST", `/issues/${open.number}/comments`, { - body: `The feed answers and validates again as of ${today}. Closing.`, + body: `The feed answers and validates against OTE ${version} again as of ${today}. Closing.`, }); await gh("PATCH", `/issues/${open.number}`, { state: "closed" }); console.log(` recovered — closed #${open.number}`); } } - continue; // healthy feeds carry no state + // A window with no failure left in it carries no information: drop the entry + // so a healthy registry produces no daily commit. + const window = push(prev && prev.window, "."); + if (count(window, "x")) next[adopter.feed] = { failures: 0, since: null, window, kind: null, alerted: false }; + continue; } - failures++; - next[adopter.feed] = { + failing++; + const entry = { failures: (prev ? prev.failures : 0) + 1, - since: prev ? prev.since : today, + since: prev && prev.since ? prev.since : today, lastError: error.split("\n")[0], + kind, + window: push(prev && prev.window, "x"), + alerted: Boolean(prev && prev.alerted), }; - console.log(` FAIL ${adopter.name} — ${adopter.feed} (${next[adopter.feed].failures} consecutive): ${error.split("\n")[0]}`); + next[adopter.feed] = entry; + console.log( + ` FAIL ${adopter.name} — ${adopter.feed} (${kind}, ${entry.failures} consecutive, window ${entry.window}): ${entry.lastError}` + ); - // Exactly at the threshold: open the alert once. Beyond it, the issue exists - // (or was closed by a human who knows better — don't reopen it every day). - if (next[adopter.feed].failures === THRESHOLD) { + // Alert once. After that the issue exists (or a human who knows better closed + // it — don't reopen it every day); `alerted` clears only on recovery. + if (!entry.alerted && shouldAlert(entry)) { + entry.alerted = true; await gh("POST", "/labels", { name: LABEL, color: "d93f0b", description: "Registered adopter feed failing its health check" }); const issues = await openIssues(); if (!issues.some((i) => i.title === issueTitle(adopter.name))) { + const how = + kind === "hard" + ? `${entry.failures} consecutive daily checks` + : `${count(entry.window, "x")} of the last ${entry.window.length} daily checks (transient errors: timeouts, 5xx)`; await gh("POST", "/issues", { title: issueTitle(adopter.name), labels: [LABEL], body: [ - `The feed of **${adopter.name}** has failed its daily health check ${THRESHOLD} times in a row (since ${next[adopter.feed].since}).`, + `The feed of **${adopter.name}** has failed ${how}, since ${entry.since}.`, "", `- Feed: ${adopter.feed}`, adopter.url ? `- Website: ${adopter.url}` : null, + version ? `- Declared spec version: \`${version}\` (checked against \`spec/${byVersion.get(version)}\`)` : null, `- Last error:`, "```", error, "```", "", + "Feeds are validated against the spec version they declare, so this is not about being on an older version.", + "", "If the feed is gone for good, delist the community with a PR removing its entry from `docs/data/adopters.json`. If it's temporary, this issue closes itself when the feed recovers.", ].filter((l) => l !== null).join("\n"), }); @@ -175,7 +307,32 @@ for (const adopter of adopters) { } writeFileSync(STATE, JSON.stringify(next, null, 2) + "\n"); -console.log(`\n${adopters.length} feed(s), ${failures} failing.`); + +/* Public status: what a consumer wants before subscribing — is it up, and which + version does it speak. `updated` is the date this content last *changed*, not + the date of the last run: a daily timestamp would commit a new file every + morning for no new information. */ +const unchanged = JSON.stringify(previous.feeds) === JSON.stringify(status); +writeFileSync( + PUBLIC_STATE, + JSON.stringify( + { + _comment: + "Health of the registered adopter feeds, refreshed daily by scripts/check-feeds.mjs. Feeds are validated against the spec version they declare; `supported` false means older than the supported window, which is information, not a failure. `specVersion` is the last version seen, kept through an outage. `updated` is when this status last changed.", + updated: unchanged && previous.updated ? previous.updated : today, + latestSpecVersion: LATEST, + feeds: status, + }, + null, + 2 + ) + "\n" +); + +console.log(`\n${adopters.length} feed(s), ${failing} failing.`); +const outdated = status.filter((s) => s.supported === false); +if (outdated.length) { + console.log(`${outdated.length} on an unsupported spec version (reported, not an issue): ${outdated.map((s) => `${s.name} ${s.specVersion}`).join(", ")}`); +} if (noCors.length) { // Deliberately not an issue and not part of the failure count: these feeds are healthy. console.log(