Skip to content

R3 Failure Ledger

R3 Failure Ledger #196

name: R3 Failure Ledger
on:
workflow_dispatch:
inputs:
official_specs_ref:
description: "Commit/tag/branch in mongodb/specifications"
required: false
default: "99704fa8860777da1d919ef765af1e41e75f5859"
readiness_min_streak:
description: "Minimum consecutive zero-run streak required for readiness"
required: false
default: "3"
fail_on_readiness:
description: "Fail workflow when readiness streak threshold is not met (true|false)"
required: false
default: "false"
fail_on_failures:
description: "Fail workflow when the generated ledger contains failures (true|false)"
required: false
default: "false"
schedule:
- cron: "30 4 * * *"
permissions:
contents: read
actions: read
issues: write
jobs:
generate-ledger:
name: Generate failure ledger
runs-on: ubuntu-latest
timeout-minutes: 60
env:
OFFICIAL_SPECS_REPO: https://github.com/mongodb/specifications.git
OFFICIAL_SPECS_REF: ${{ github.event.inputs.official_specs_ref || '99704fa8860777da1d919ef765af1e41e75f5859' }}
READINESS_MIN_STREAK: ${{ github.event.inputs.readiness_min_streak || '3' }}
FAIL_ON_READINESS: ${{ github.event.inputs.fail_on_readiness || 'false' }}
FAIL_ON_FAILURES: ${{ github.event.inputs.fail_on_failures || 'false' }}
MONGO_CONTAINER_NAME: jongodb-replset
MONGO_REPLSET_NAME: rs0
MONGO_PORT: "27017"
REPLSET_WAIT_TIMEOUT_SECONDS: "120"
REPLSET_WAIT_INTERVAL_SECONDS: "2"
REPLSET_DIAGNOSTICS_DIR: build/reports/replset-diagnostics/r3-failure-ledger
R3_LEDGER_OUTPUT_DIR: build/reports/r3-failure-ledger
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Temurin 17
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "17"
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v4
with:
gradle-version: "8.11.1"
- name: Make scripts executable
run: chmod +x scripts/ci/*.sh
- name: Bootstrap replica set fixture
run: ./scripts/ci/bootstrap-replset.sh
- name: Prepare official specs checkout
run: |
./scripts/ci/prepare-official-specs.sh \
--repo "${OFFICIAL_SPECS_REPO}" \
--ref "${OFFICIAL_SPECS_REF}" \
--dest "${RUNNER_TEMP}/mongodb-specifications"
- name: Run failure ledger generation
run: |
gradle --no-daemon --stacktrace \
-Pr3SpecRepoRoot="${RUNNER_TEMP}/mongodb-specifications" \
-Pr3FailureLedgerOutputDir="${R3_LEDGER_OUTPUT_DIR}" \
-Pr3FailureLedgerMongoUri="${JONGODB_REAL_MONGOD_URI}" \
-Pr3FailureLedgerFailOnFailures="${FAIL_ON_FAILURES}" \
r3FailureLedger
- name: Build trend snapshot
run: |
python3 - <<'PY'
import json
from datetime import datetime, timezone
from pathlib import Path
report = Path("build/reports/r3-failure-ledger/r3-failure-ledger.json")
out = Path("build/reports/r3-failure-ledger/r3-failure-ledger-trend.md")
data = json.loads(report.read_text(encoding="utf-8"))
now = datetime.now(timezone.utc).isoformat()
lines = [
"# R3 Differential Trend Snapshot",
"",
f"- generatedAtUtc: {now}",
f"- reportGeneratedAt: {data.get('generatedAt')}",
f"- failureCount: {data.get('failureCount')}",
f"- suiteCount: {data.get('suiteCount')}",
"",
"## byTrack",
]
by_track = data.get("byTrack", {})
if by_track:
for key in sorted(by_track):
lines.append(f"- {key}: {by_track[key]}")
else:
lines.append("- none")
lines.append("")
lines.append("## byStatus")
by_status = data.get("byStatus", {})
if by_status:
for key in sorted(by_status):
lines.append(f"- {key}: {by_status[key]}")
else:
lines.append("- none")
lines.append("")
lines.append("## failureIds")
entries = data.get("entries", [])
if entries:
for entry in entries[:20]:
first_diff_path = entry.get("firstDiffPath") or "<none>"
lines.append(f"- {entry.get('failureId', '<missing-id>')} | firstDiffPath={first_diff_path}")
else:
lines.append("- none")
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(out.read_text(encoding="utf-8"))
PY
- name: Compare ledger failure IDs with latest official shard artifacts
env:
GH_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
run: |
python3 - <<'PY'
import io
import json
import os
import urllib.request
import zipfile
from pathlib import Path
repo = os.environ["GITHUB_REPOSITORY"]
token = os.environ.get("GH_TOKEN", "")
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
output_path = Path("build/reports/r3-failure-ledger/r3-ledger-official-consistency.md")
ledger_path = Path("build/reports/r3-failure-ledger/r3-failure-ledger.json")
def request_json(url: str):
req = urllib.request.Request(url, headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"User-Agent": "jongodb-r3-ledger-consistency-check",
})
with urllib.request.urlopen(req, timeout=30) as res:
return json.loads(res.read().decode("utf-8"))
def request_bytes(url: str) -> bytes:
req = urllib.request.Request(url, headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"User-Agent": "jongodb-r3-ledger-consistency-check",
})
with urllib.request.urlopen(req, timeout=60) as res:
return res.read()
lines = ["# R3 vs Official Shard Consistency", ""]
if not ledger_path.exists():
lines.append("- status: skipped")
lines.append(f"- reason: missing ledger report at {ledger_path}")
output_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(output_path.read_text(encoding="utf-8"))
raise SystemExit(0)
ledger_json = json.loads(ledger_path.read_text(encoding="utf-8"))
ledger_ids = sorted({
str(entry.get("failureId", "")).strip()
for entry in ledger_json.get("entries", [])
if str(entry.get("failureId", "")).strip()
})
lines.append(f"- ledgerFailureCount: {len(ledger_ids)}")
try:
runs = request_json(
f"https://api.github.com/repos/{repo}/actions/workflows/official-suite-sharded.yml/runs"
"?branch=main&status=completed&per_page=20"
)
target_run = None
for run in runs.get("workflow_runs", []):
if run.get("conclusion") in {"success", "failure"}:
target_run = run
break
if target_run is None:
lines.append("- status: skipped")
lines.append("- reason: no completed official-suite-sharded runs found on main")
else:
run_id = target_run.get("id")
run_url = target_run.get("html_url", "")
lines.append(f"- officialRunId: {run_id}")
lines.append(f"- officialRunUrl: {run_url}")
artifacts = request_json(
f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/artifacts?per_page=100"
)
official_ids = set()
for artifact in artifacts.get("artifacts", []):
name = artifact.get("name", "")
if artifact.get("expired"):
continue
if not name.startswith("utf-shard-") or name == "utf-shard-summary":
continue
archive_bytes = request_bytes(artifact["archive_download_url"])
with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
for member in archive.namelist():
if not member.endswith("failure-replay-bundles/manifest.json"):
continue
manifest = json.loads(archive.read(member).decode("utf-8"))
for failure in manifest.get("failures", []):
failure_id = str(failure.get("failureId", "")).strip()
if failure_id:
official_ids.add(failure_id)
official_ids = sorted(official_ids)
lines.append(f"- officialFailureCount: {len(official_ids)}")
missing_in_official = sorted(set(ledger_ids) - set(official_ids))
missing_in_ledger = sorted(set(official_ids) - set(ledger_ids))
if not missing_in_official and not missing_in_ledger:
lines.append("- consistency: aligned")
else:
lines.append("- consistency: diverged")
lines.append("- missingInOfficial:")
if missing_in_official:
for failure_id in missing_in_official[:20]:
lines.append(f" - {failure_id}")
else:
lines.append(" - none")
lines.append("- missingInLedger:")
if missing_in_ledger:
for failure_id in missing_in_ledger[:20]:
lines.append(f" - {failure_id}")
else:
lines.append(" - none")
print("::warning::R3 ledger and latest official shard failure IDs diverged")
except Exception as exc: # pragma: no cover - workflow-only best effort path
lines.append("- status: warning")
lines.append(f"- reason: consistency check failed ({exc})")
print(f"::warning::consistency check failed: {exc}")
output_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(output_path.read_text(encoding="utf-8"))
if summary_path:
with Path(summary_path).open("a", encoding="utf-8") as handle:
handle.write("\n")
handle.write(output_path.read_text(encoding="utf-8"))
PY
- name: Build release-readiness streak summary
env:
GH_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
run: |
python3 - <<'PY'
import io
import json
import os
import re
import urllib.request
import zipfile
from datetime import datetime, timezone
from pathlib import Path
WINDOW_SIZE = 10
repo = os.environ["GITHUB_REPOSITORY"]
token = os.environ.get("GH_TOKEN", "")
run_id = int(os.environ.get("GITHUB_RUN_ID", "0"))
run_url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{repo}/actions/runs/{run_id}"
min_streak = int(os.environ.get("READINESS_MIN_STREAK", "3"))
fail_on_readiness = os.environ.get("FAIL_ON_READINESS", "false").strip().lower() == "true"
step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
ledger_path = Path("build/reports/r3-failure-ledger/r3-failure-ledger.json")
output_json = Path("build/reports/r3-failure-ledger/r3-release-readiness-streak.json")
output_md = Path("build/reports/r3-failure-ledger/r3-release-readiness-streak.md")
if not ledger_path.exists():
raise SystemExit(f"missing ledger report: {ledger_path}")
current_ledger = json.loads(ledger_path.read_text(encoding="utf-8"))
current_failure_count = int(current_ledger.get("failureCount", 0))
aggregate_pattern = re.compile(
r"## aggregate\s+"
r"- baseline: total=(?P<b_total>\d+) match=(?P<b_match>\d+) mismatch=(?P<b_mismatch>\d+) error=(?P<b_error>\d+).*?\n"
r"- rerun: total=(?P<r_total>\d+) match=(?P<r_match>\d+) mismatch=(?P<r_mismatch>\d+) error=(?P<r_error>\d+)",
re.S,
)
def parse_official_summary(text: str):
match = aggregate_pattern.search(text)
if not match:
return None
metrics = {
"baseline": {
"total": int(match.group("b_total")),
"match": int(match.group("b_match")),
"mismatch": int(match.group("b_mismatch")),
"error": int(match.group("b_error")),
},
"rerun": {
"total": int(match.group("r_total")),
"match": int(match.group("r_match")),
"mismatch": int(match.group("r_mismatch")),
"error": int(match.group("r_error")),
},
}
metrics["zeroMismatchError"] = (
metrics["baseline"]["mismatch"] == 0
and metrics["baseline"]["error"] == 0
and metrics["rerun"]["mismatch"] == 0
and metrics["rerun"]["error"] == 0
)
return metrics
def request_json(url: str):
req = urllib.request.Request(
url,
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"User-Agent": "jongodb-r3-readiness-streak",
},
)
with urllib.request.urlopen(req, timeout=30) as res:
return json.loads(res.read().decode("utf-8"))
def request_bytes(url: str) -> bytes:
req = urllib.request.Request(
url,
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"User-Agent": "jongodb-r3-readiness-streak",
},
)
with urllib.request.urlopen(req, timeout=60) as res:
return res.read()
def read_artifact_file(run_id_value: int, artifact_name: str, suffix: str):
artifacts = request_json(
f"https://api.github.com/repos/{repo}/actions/runs/{run_id_value}/artifacts?per_page=100"
)
for artifact in artifacts.get("artifacts", []):
if artifact.get("expired"):
continue
if artifact.get("name") != artifact_name:
continue
with zipfile.ZipFile(io.BytesIO(request_bytes(artifact["archive_download_url"]))) as archive:
for member in archive.namelist():
if member.endswith(suffix):
return archive.read(member).decode("utf-8")
return None
official_history = []
official_runs = request_json(
f"https://api.github.com/repos/{repo}/actions/workflows/official-suite-sharded.yml/runs"
"?branch=main&event=schedule&status=completed&per_page=30"
).get("workflow_runs", [])
for run in official_runs:
if len(official_history) >= WINDOW_SIZE:
break
run_ref = int(run.get("id", 0))
entry = {
"runId": run_ref,
"event": run.get("event"),
"conclusion": run.get("conclusion"),
"url": run.get("html_url"),
"createdAt": run.get("created_at"),
"metrics": None,
"zeroMismatchError": False,
}
try:
summary_text = read_artifact_file(run_ref, "utf-shard-summary", "utf-shard-summary.md")
if summary_text is not None:
parsed = parse_official_summary(summary_text)
if parsed is not None:
entry["metrics"] = parsed
entry["zeroMismatchError"] = parsed["zeroMismatchError"]
else:
entry["note"] = "summary-parse-failed"
else:
entry["note"] = "utf-shard-summary-artifact-missing"
except Exception as exc:
entry["note"] = f"artifact-read-error: {exc}"
official_history.append(entry)
r3_history = [
{
"runId": run_id,
"event": os.environ.get("GITHUB_EVENT_NAME"),
"conclusion": "in_progress",
"url": run_url,
"createdAt": datetime.now(timezone.utc).isoformat(),
"failureCount": current_failure_count,
"zeroFailure": current_failure_count == 0,
"source": "current-run-artifact",
}
]
r3_runs = request_json(
f"https://api.github.com/repos/{repo}/actions/workflows/r3-failure-ledger.yml/runs"
"?branch=main&event=schedule&status=completed&per_page=30"
).get("workflow_runs", [])
for run in r3_runs:
if len(r3_history) >= WINDOW_SIZE:
break
run_ref = int(run.get("id", 0))
if run_ref == run_id:
continue
entry = {
"runId": run_ref,
"event": run.get("event"),
"conclusion": run.get("conclusion"),
"url": run.get("html_url"),
"createdAt": run.get("created_at"),
"failureCount": None,
"zeroFailure": False,
"source": "schedule-history",
}
try:
ledger_text = read_artifact_file(run_ref, "r3-failure-ledger", "r3-failure-ledger.json")
if ledger_text is not None:
parsed = json.loads(ledger_text)
entry["failureCount"] = int(parsed.get("failureCount", 0))
entry["zeroFailure"] = entry["failureCount"] == 0
else:
entry["note"] = "r3-failure-ledger-artifact-missing"
except Exception as exc:
entry["note"] = f"artifact-read-error: {exc}"
r3_history.append(entry)
def compute_streak(items, key):
streak_value = 0
for item in items:
if item.get(key):
streak_value += 1
continue
break
return streak_value
official_streak = compute_streak(official_history, "zeroMismatchError")
r3_streak = compute_streak(r3_history, "zeroFailure")
readiness_met = official_streak >= min_streak and r3_streak >= min_streak
result = {
"generatedAt": datetime.now(timezone.utc).isoformat(),
"windowSize": WINDOW_SIZE,
"thresholds": {"minStreak": min_streak},
"counters": {
"officialZeroMismatchStreak": official_streak,
"r3LedgerZeroFailureStreak": r3_streak,
},
"readiness": {
"satisfied": readiness_met,
"failOnReadiness": fail_on_readiness,
},
"officialHistory": official_history,
"r3History": r3_history,
}
output_json.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# R3 Release Readiness Streak Summary",
"",
f"- generatedAt: {result['generatedAt']}",
f"- minStreak: {min_streak}",
f"- officialZeroMismatchStreak: {official_streak}",
f"- r3LedgerZeroFailureStreak: {r3_streak}",
f"- readinessSatisfied: {readiness_met}",
f"- failOnReadiness: {fail_on_readiness}",
"",
"## official-schedule-history",
]
for entry in official_history:
metrics = entry.get("metrics") or {}
baseline = metrics.get("baseline", {})
rerun = metrics.get("rerun", {})
lines.append(
f"- runId={entry.get('runId')} conclusion={entry.get('conclusion')} "
f"zeroMismatchError={entry.get('zeroMismatchError')} "
f"baseline(mismatch={baseline.get('mismatch', 'n/a')},error={baseline.get('error', 'n/a')}) "
f"rerun(mismatch={rerun.get('mismatch', 'n/a')},error={rerun.get('error', 'n/a')})"
)
lines.append("")
lines.append("## r3-ledger-history")
for entry in r3_history:
lines.append(
f"- runId={entry.get('runId')} conclusion={entry.get('conclusion')} "
f"failureCount={entry.get('failureCount', 'n/a')} zeroFailure={entry.get('zeroFailure')}"
)
output_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(output_md.read_text(encoding="utf-8"))
if step_summary:
with Path(step_summary).open("a", encoding="utf-8") as handle:
handle.write("\n")
handle.write(output_md.read_text(encoding="utf-8"))
if fail_on_readiness and not readiness_met:
print(
"::error::readiness streak threshold not met "
f"(officialZeroMismatchStreak={official_streak}, r3LedgerZeroFailureStreak={r3_streak}, minStreak={min_streak})"
)
raise SystemExit(1)
PY
- name: Post triage breadcrumb on scheduled ledger findings
if: always() && github.event_name == 'schedule'
env:
GH_TOKEN: ${{ github.token }}
TRIAGE_ISSUE_NUMBER: "378"
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
python3 - <<'PY'
import json
import os
import subprocess
from pathlib import Path
ledger_path = Path("build/reports/r3-failure-ledger/r3-failure-ledger.json")
if not ledger_path.exists():
print("ledger report missing; skip triage comment")
raise SystemExit(0)
payload = json.loads(ledger_path.read_text(encoding="utf-8"))
failure_count = int(payload.get("failureCount", 0))
if failure_count == 0:
print("failureCount=0; skip triage comment")
raise SystemExit(0)
entries = payload.get("entries", [])
lines = [
"R3 Failure Ledger scheduled run detected non-zero failures.",
"",
f"- run: {os.environ.get('RUN_URL', '')}",
f"- failureCount: {failure_count}",
f"- byTrack: {json.dumps(payload.get('byTrack', {}), ensure_ascii=False)}",
f"- byStatus: {json.dumps(payload.get('byStatus', {}), ensure_ascii=False)}",
"",
"Top failure IDs (up to 10):",
]
for entry in entries[:10]:
failure_id = entry.get("failureId", "<missing-id>")
first_diff_path = entry.get("firstDiffPath") or "<none>"
lines.append(f"- `{failure_id}` (`firstDiffPath={first_diff_path}`)")
consistency_path = Path("build/reports/r3-failure-ledger/r3-ledger-official-consistency.md")
if consistency_path.exists():
lines.append("")
lines.append("Consistency snapshot:")
lines.append("")
lines.extend(consistency_path.read_text(encoding="utf-8").strip().splitlines()[:30])
body = "\n".join(lines)
issue = os.environ.get("TRIAGE_ISSUE_NUMBER", "378")
subprocess.run(["gh", "issue", "comment", issue, "--body", body], check=True)
print(f"posted triage comment to issue #{issue}")
PY
- name: Collect replica set diagnostics
if: always()
run: ./scripts/ci/collect-replset-diagnostics.sh "${REPLSET_DIAGNOSTICS_DIR}"
- name: Upload failure ledger artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: r3-failure-ledger
path: |
build/reports/r3-failure-ledger/**/*.json
build/reports/r3-failure-ledger/**/*.md
if-no-files-found: error
retention-days: 14
- name: Upload replica set diagnostics
if: always()
uses: actions/upload-artifact@v4
with:
name: r3-failure-ledger-diagnostics
path: build/reports/replset-diagnostics/r3-failure-ledger/**
if-no-files-found: warn
retention-days: 14
- name: Teardown replica set fixture
if: always()
run: ./scripts/ci/teardown-replset.sh