From c2950df6c01d960d030efcc07def938344571706 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 9 Jun 2026 15:35:23 -0500 Subject: [PATCH 01/13] Add PR test failure summary comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Posts a sticky comment on each PR listing which matrix test jobs failed, with direct links to their logs. Updates on re-run; clears to a pass message if all jobs succeed. Uses only actions/github-script (already in repo) and the built-in GITHUB_TOKEN with pull-requests: write — no S3, no AWS credentials, no new third-party actions. --- .github/workflows/pr.yaml | 21 ++++ ci/utils/pr_test_summary.py | 235 ++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 ci/utils/pr_test_summary.py diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8c05b5d1b0..7af98d9c9d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -620,3 +620,24 @@ jobs: with: build_type: pull-request script: ci/test_self_hosted_service.sh + pr-test-summary: + needs: + - conda-cpp-tests + - conda-python-tests + - wheel-tests-cuopt + - wheel-tests-cuopt-server + - test-self-hosted-server + if: always() + runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: ci/utils/pr_test_summary.py + sparse-checkout-cone-mode: false + - run: python3 ci/utils/pr_test_summary.py + env: + GH_TOKEN: ${{ github.token }} diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py new file mode 100644 index 0000000000..4129124add --- /dev/null +++ b/ci/utils/pr_test_summary.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Post or update a sticky PR comment summarizing CI test job failures. + +Reads GITHUB_REPOSITORY, GITHUB_RUN_ID, GITHUB_REF, and GH_TOKEN from the +environment. Finds every test job in the current workflow run, then posts (or +updates) a single comment on the pull request listing any failed jobs with +direct log links and a collapsible list of individual failed test names. + +Usage (called from a GitHub Actions step): + python3 ci/utils/pr_test_summary.py +""" + +import json +import os +import sys +import urllib.error +import urllib.request + +# Job name prefixes that are considered test jobs. +_TEST_PREFIXES = ( + "conda-cpp-tests", + "conda-python-tests", + "wheel-tests-cuopt", + "wheel-tests-cuopt-server", + "test-self-hosted-server", +) + +_MARKER = "" +# Maximum failed test names shown per job dropdown. +_MAX_TESTS = 50 + +# Ordered by specificity; first match wins. +_CRASH_PATTERNS = [ + ("Segmentation fault", "SIGSEGV (segfault)"), + ("SIGSEGV", "SIGSEGV (segfault)"), + ("signal 11", "SIGSEGV (signal 11)"), + ("Aborted (core dumped)", "SIGABRT"), + ("SIGABRT", "SIGABRT"), + ("signal 6", "SIGABRT (signal 6)"), + ("SIGKILL", "SIGKILL"), + ("signal 9", "SIGKILL (signal 9)"), + ("Out of memory", "OOM"), + ("oom-kill", "OOM"), + ("core dumped", "core dumped"), +] + + +def _headers(token): + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def _paginate(path, token): + """Yield all items from a paginated GitHub REST API GET endpoint.""" + url = f"https://api.github.com{path}?per_page=100" + while url: + req = urllib.request.Request(url, headers=_headers(token)) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + # Jobs endpoint wraps items in {"jobs": [...]}; comments is a bare list. + yield from (data["jobs"] if isinstance(data, dict) else data) + link = resp.headers.get("Link", "") + url = next( + ( + p.split(";")[0].strip().strip("<>") + for p in link.split(",") + if 'rel="next"' in p + ), + None, + ) + + +def _api(path, token, method, payload): + req = urllib.request.Request( + f"https://api.github.com{path}", + data=json.dumps(payload).encode(), + method=method, + headers={**_headers(token), "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +def _is_test_job(name): + return any(name == p or name.startswith(p + " (") for p in _TEST_PREFIXES) + + +def _analyze_job_log(job_id, repo, token): + """Return (failed_test_ids, crash_description_or_None) from a job's log.""" + req = urllib.request.Request( + f"https://api.github.com/repos/{repo}/actions/jobs/{job_id}/logs", + headers=_headers(token), + ) + try: + with urllib.request.urlopen(req) as resp: + # Stream the log, retaining only the last 512 KB so the pytest + # summary section at the end of the output is always captured. + chunks = [] + total = 0 + while chunk := resp.read(65536): + chunks.append(chunk) + total += len(chunk) + if total > 512 * 1024: + chunks = chunks[-8:] + total = sum(len(c) for c in chunks) + except (urllib.error.HTTPError, urllib.error.URLError): + return [], None + + text = b"".join(chunks).decode("utf-8", errors="replace") + + crash = next( + (desc for pattern, desc in _CRASH_PATTERNS if pattern in text), None + ) + + failed = [] + in_summary = False + for raw in text.splitlines(): + # Strip GHA timestamp prefix: "2024-01-15T10:30:45.1234567Z content" + parts = raw.split("Z ", 1) + line = parts[1] if len(parts) > 1 and len(parts[0]) < 35 else raw + + if "short test summary info" in line: + in_summary = True + elif in_summary: + if line.startswith("FAILED "): + test_id = line[7:].split(" - ")[0].strip() + if test_id: + failed.append(test_id) + elif line.startswith("=") and failed: + break + + return failed[:_MAX_TESTS], crash + + +def _build_body(failed, passed, skipped, job_analysis): + lines = [_MARKER, "## CI Test Summary", ""] + if not failed: + lines.append(f"✅ All {len(passed)} test job(s) passed.") + else: + lines.append( + f"**{len(failed)} failed** · {len(passed)} passed · {len(skipped)} skipped" + ) + lines += ["", "| Job | Logs |", "|-----|------|"] + for job in failed: + lines.append( + f"| ❌ `{job['name']}` | [View logs]({job['html_url']}) |" + ) + + for job in failed: + tests, crash = job_analysis.get(job["id"], ([], None)) + if not tests and not crash: + continue + if crash and not tests: + summary = f"💥 crashed ({crash})" + detail = "Process was terminated before pytest completed." + else: + n = len(tests) + noun = "test" if n == 1 else "tests" + summary = f"{n} failed {noun}" + ( + f" · 💥 {crash}" if crash else "" + ) + detail = "\n".join(f"- `{t}`" for t in tests) + lines += [ + "", + "
", + f"{job['name']} — {summary}", + "", + detail, + "", + "
", + ] + + return "\n".join(lines) + + +def main(): + token = os.environ["GH_TOKEN"] + repo = os.environ["GITHUB_REPOSITORY"] + run_id = os.environ["GITHUB_RUN_ID"] + ref = os.environ["GITHUB_REF"] # refs/heads/pull-request/NNN + + branch = ref.removeprefix("refs/heads/") + if not branch.startswith("pull-request/"): + print(f"Not a PR branch ({branch}), skipping.", file=sys.stderr) + return + pr_number = int(branch.removeprefix("pull-request/")) + + jobs = list(_paginate(f"/repos/{repo}/actions/runs/{run_id}/jobs", token)) + test_jobs = [j for j in jobs if _is_test_job(j["name"])] + if not test_jobs: + print("No test jobs found in this run, skipping.", file=sys.stderr) + return + + failed = [j for j in test_jobs if j["conclusion"] == "failure"] + passed = [j for j in test_jobs if j["conclusion"] == "success"] + skipped = [j for j in test_jobs if j["conclusion"] == "skipped"] + + job_analysis = { + job["id"]: _analyze_job_log(job["id"], repo, token) for job in failed + } + + body = _build_body(failed, passed, skipped, job_analysis) + + comments = list( + _paginate(f"/repos/{repo}/issues/{pr_number}/comments", token) + ) + existing = next( + (c for c in comments if c.get("body", "").startswith(_MARKER)), None + ) + + if existing: + _api( + f"/repos/{repo}/issues/comments/{existing['id']}", + token, + "PATCH", + {"body": body}, + ) + print(f"Updated comment {existing['id']} on PR #{pr_number}.") + else: + result = _api( + f"/repos/{repo}/issues/{pr_number}/comments", + token, + "POST", + {"body": body}, + ) + print(f"Posted comment {result['id']} on PR #{pr_number}.") + + +if __name__ == "__main__": + main() From ea62e02292065b82754b92196d8855b755b90d24 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 9 Jun 2026 16:56:55 -0500 Subject: [PATCH 02/13] Add intentional failing tests to exercise PR summary comment Two test failures (one Python, one C++) to verify the PR test summary comment correctly captures and displays failed test names per job. Remove before merging. --- cpp/tests/routing/unit_tests/breaks.cu | 6 ++++++ .../cuopt/cuopt/tests/test_ci_summary_demo.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 python/cuopt/cuopt/tests/test_ci_summary_demo.py diff --git a/cpp/tests/routing/unit_tests/breaks.cu b/cpp/tests/routing/unit_tests/breaks.cu index 0d8a578b6e..2ccb4f2a17 100644 --- a/cpp/tests/routing/unit_tests/breaks.cu +++ b/cpp/tests/routing/unit_tests/breaks.cu @@ -430,6 +430,12 @@ TEST(vehicle_breaks, non_uniform_breaks) check_route(data_model, h_routing_solution); } +// Intentional failure to exercise the PR test summary comment feature. +TEST(vehicle_breaks, ci_summary_demo_failure) +{ + EXPECT_TRUE(false) << "Intentional failure: remove after verifying PR summary comment."; +} + } // namespace test } // namespace routing } // namespace cuopt diff --git a/python/cuopt/cuopt/tests/test_ci_summary_demo.py b/python/cuopt/cuopt/tests/test_ci_summary_demo.py new file mode 100644 index 0000000000..dd3d16303e --- /dev/null +++ b/python/cuopt/cuopt/tests/test_ci_summary_demo.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Intentional failures to exercise the PR test summary comment feature. +# Remove after verifying the summary comment shows correct output. + + +def test_ci_summary_demo_failure(): + assert False, ( + "Intentional failure: remove after verifying PR summary comment." + ) + + +def test_ci_summary_demo_failure_2(): + raise RuntimeError( + "Intentional error: remove after verifying PR summary comment." + ) From efe1d66147da56da293bde374e99053c9e2d5b8e Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 9 Jun 2026 16:59:39 -0500 Subject: [PATCH 03/13] Address review comments in pr_test_summary.py - Add HTTP timeout to all urlopen calls - Treat timed_out/cancelled jobs as failures in counts - Capture ERROR lines alongside FAILED in pytest summary parsing --- ci/utils/pr_test_summary.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 4129124add..5e2775b4e2 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -28,6 +28,7 @@ ) _MARKER = "" +_HTTP_TIMEOUT_SEC = 30 # Maximum failed test names shown per job dropdown. _MAX_TESTS = 50 @@ -60,7 +61,7 @@ def _paginate(path, token): url = f"https://api.github.com{path}?per_page=100" while url: req = urllib.request.Request(url, headers=_headers(token)) - with urllib.request.urlopen(req) as resp: + with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_SEC) as resp: data = json.loads(resp.read()) # Jobs endpoint wraps items in {"jobs": [...]}; comments is a bare list. yield from (data["jobs"] if isinstance(data, dict) else data) @@ -97,7 +98,7 @@ def _analyze_job_log(job_id, repo, token): headers=_headers(token), ) try: - with urllib.request.urlopen(req) as resp: + with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_SEC) as resp: # Stream the log, retaining only the last 512 KB so the pytest # summary section at the end of the output is always captured. chunks = [] @@ -127,8 +128,8 @@ def _analyze_job_log(job_id, repo, token): if "short test summary info" in line: in_summary = True elif in_summary: - if line.startswith("FAILED "): - test_id = line[7:].split(" - ")[0].strip() + if line.startswith(("FAILED ", "ERROR ")): + test_id = line.split(" ", 1)[1].split(" - ")[0].strip() if test_id: failed.append(test_id) elif line.startswith("=") and failed: @@ -196,9 +197,9 @@ def main(): print("No test jobs found in this run, skipping.", file=sys.stderr) return - failed = [j for j in test_jobs if j["conclusion"] == "failure"] passed = [j for j in test_jobs if j["conclusion"] == "success"] skipped = [j for j in test_jobs if j["conclusion"] == "skipped"] + failed = [j for j in test_jobs if j not in passed and j not in skipped] job_analysis = { job["id"]: _analyze_job_log(job["id"], repo, token) for job in failed From 644300d90b0abdac2c8913cec8d11b9b698b79a0 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 11 Jun 2026 10:57:56 -0500 Subject: [PATCH 04/13] Fix other-checks failure and 0-job-count bug - Add pr-test-summary to pr-builder.needs (required by RAPIDS checks.yaml which enforces all jobs appear in pr-builder) - Add continue-on-error: true so a GitHub API failure in the summary job never blocks a merge - Fix _is_test_job to match matrix job names using both ' / ' and ' (' separators; the GHA job name format is 'job / matrix-params' not 'job (matrix-params)' causing all test jobs to be silently skipped --- .github/workflows/pr.yaml | 2 ++ ci/utils/pr_test_summary.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7af98d9c9d..7fb07d1582 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,6 +34,7 @@ jobs: - wheel-tests-cuopt-server - wheel-build-cuopt-sh-client - test-self-hosted-server + - pr-test-summary permissions: contents: read uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main @@ -628,6 +629,7 @@ jobs: - wheel-tests-cuopt-server - test-self-hosted-server if: always() + continue-on-error: true runs-on: ubuntu-latest permissions: actions: read diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 5e2775b4e2..790b957b3c 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -88,7 +88,8 @@ def _api(path, token, method, payload): def _is_test_job(name): - return any(name == p or name.startswith(p + " (") for p in _TEST_PREFIXES) + # Matrix jobs use " / " or " (" as separator depending on GHA version. + return any(name == p or name.startswith(p + " ") for p in _TEST_PREFIXES) def _analyze_job_log(job_id, repo, token): From 2f24068a5b61b10fa326aa4c2c211f3bc9babee7 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 22 Jun 2026 17:58:13 -0500 Subject: [PATCH 05/13] ci: fix pr-test-summary dropping failure details MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The log analyzer kept only the last 512 KB of each job log on the assumption the pytest summary sits at the end. It does not: a job runs several suites (cuopt then cuopt_server), so the 'short test summary info' section can be hundreds of KB before EOF and the tail window discards it — leaving the comment with a job table but no per-test detail. Stream the whole log line by line instead (bounded memory: only matched failures are retained), and parse gtest/ctest '[ FAILED ] Suite.Test' output too, so C++ test jobs also show failed-test names. Verified against the real run-27983423156 logs: python job now yields both demo failures, cpp job yields the gtest failure (both previously empty). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Ramakrishna Prabhu --- ci/utils/pr_test_summary.py | 92 +++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 790b957b3c..f1abd16bf2 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -12,8 +12,10 @@ python3 ci/utils/pr_test_summary.py """ +import io import json import os +import re import sys import urllib.error import urllib.request @@ -32,6 +34,11 @@ # Maximum failed test names shown per job dropdown. _MAX_TESTS = 50 +# gtest prints "[ FAILED ] Suite.Test (12 ms)" per failing test and again, +# without the timing suffix, in the end-of-run "listed below" block. Capture the +# "Suite.Test" name; the dedup set collapses the duplicate. +_GTEST_FAILED = re.compile(r"\[ FAILED \] (\S+\.\S+?)(?: \(\d+ ms\))?$") + # Ordered by specificity; first match wins. _CRASH_PATTERNS = [ ("Segmentation fault", "SIGSEGV (segfault)"), @@ -93,49 +100,64 @@ def _is_test_job(name): def _analyze_job_log(job_id, repo, token): - """Return (failed_test_ids, crash_description_or_None) from a job's log.""" + """Return (failed_test_ids, crash_description_or_None) from a job's log. + + Streams the whole log line by line (bounded memory: only matched failures + are retained). Truncating to a tail window does not work — the pytest + summary is not necessarily near the end (a job may run several test suites, + e.g. cuopt then cuopt_server), so a fixed window can drop it entirely. + Recognizes both pytest ("short test summary info") and gtest/ctest + ("[ FAILED ] Suite.Test") output. + """ req = urllib.request.Request( f"https://api.github.com/repos/{repo}/actions/jobs/{job_id}/logs", headers=_headers(token), ) + failed = [] + seen = set() + crash = None + in_pytest_summary = False + + def _add(test_id): + if test_id and test_id not in seen: + seen.add(test_id) + failed.append(test_id) + try: with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_SEC) as resp: - # Stream the log, retaining only the last 512 KB so the pytest - # summary section at the end of the output is always captured. - chunks = [] - total = 0 - while chunk := resp.read(65536): - chunks.append(chunk) - total += len(chunk) - if total > 512 * 1024: - chunks = chunks[-8:] - total = sum(len(c) for c in chunks) + for raw in io.TextIOWrapper( + resp, encoding="utf-8", errors="replace" + ): + # Strip GHA timestamp prefix: "2024-01-15T10:30:45.1234567Z text" + parts = raw.rstrip("\n").split("Z ", 1) + line = ( + parts[1] + if len(parts) > 1 and len(parts[0]) < 35 + else raw.rstrip("\n") + ) + + if crash is None: + crash = next( + (d for p, d in _CRASH_PATTERNS if p in line), None + ) + + # gtest / ctest failures (C++ test jobs). + m = _GTEST_FAILED.match(line) + if m: + _add(m.group(1)) + continue + + # pytest failures (Python test jobs). + if "short test summary info" in line: + in_pytest_summary = True + elif in_pytest_summary: + if line.startswith(("FAILED ", "ERROR ")): + _add(line.split(" ", 1)[1].split(" - ")[0].strip()) + elif line.startswith("="): + in_pytest_summary = False except (urllib.error.HTTPError, urllib.error.URLError): return [], None - text = b"".join(chunks).decode("utf-8", errors="replace") - - crash = next( - (desc for pattern, desc in _CRASH_PATTERNS if pattern in text), None - ) - - failed = [] - in_summary = False - for raw in text.splitlines(): - # Strip GHA timestamp prefix: "2024-01-15T10:30:45.1234567Z content" - parts = raw.split("Z ", 1) - line = parts[1] if len(parts) > 1 and len(parts[0]) < 35 else raw - - if "short test summary info" in line: - in_summary = True - elif in_summary: - if line.startswith(("FAILED ", "ERROR ")): - test_id = line.split(" ", 1)[1].split(" - ")[0].strip() - if test_id: - failed.append(test_id) - elif line.startswith("=") and failed: - break - return failed[:_MAX_TESTS], crash @@ -159,7 +181,7 @@ def _build_body(failed, passed, skipped, job_analysis): continue if crash and not tests: summary = f"💥 crashed ({crash})" - detail = "Process was terminated before pytest completed." + detail = "Process was terminated before the test run completed." else: n = len(tests) noun = "test" if n == 1 else "tests" From 7a7ef891f50af50fa2df69b43be39f55c4147f74 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 22 Jun 2026 18:00:27 -0500 Subject: [PATCH 06/13] ci: apply ruff-format to pr_test_summary.py Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Ramakrishna Prabhu --- ci/utils/pr_test_summary.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index f1abd16bf2..030ea7c35b 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -181,7 +181,9 @@ def _build_body(failed, passed, skipped, job_analysis): continue if crash and not tests: summary = f"💥 crashed ({crash})" - detail = "Process was terminated before the test run completed." + detail = ( + "Process was terminated before the test run completed." + ) else: n = len(tests) noun = "test" if n == 1 else "tests" From 99229dc448c0d29085b7e9d41af070ced284bd87 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 23 Jun 2026 15:04:46 -0500 Subject: [PATCH 07/13] ci: fix log fetch failing silently due to S3 auth conflict on redirect GitHub's job-log endpoint redirects to a presigned S3 URL. urllib was forwarding the Authorization header to S3, which rejected it with 400 ("Only one auth mechanism allowed"), causing _analyze_job_log to return empty results for every job and the per-job details blocks to be omitted from the PR comment. Fix by using a custom HTTPRedirectHandler that strips the Authorization and X-GitHub-Api-Version headers before following the redirect. Also surface fetch errors to stderr instead of swallowing them silently. Co-Authored-By: Claude Sonnet 4.6 --- ci/utils/pr_test_summary.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 030ea7c35b..4d31171de7 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -20,6 +20,23 @@ import urllib.error import urllib.request + +class _DropAuthOnRedirect(urllib.request.HTTPRedirectHandler): + """Strip auth headers before following redirects to presigned URLs (e.g. S3). + + GitHub's job-log endpoint issues a 302 to a presigned S3 URL. Forwarding + the Authorization header causes S3 to return 400 ("Only one auth mechanism + allowed"), so we strip it before following the redirect. + """ + + def redirect_request(self, req, fp, code, msg, headers, newurl): + new_req = super().redirect_request(req, fp, code, msg, headers, newurl) + if new_req is not None: + for key in list(new_req.headers): + if key.lower() in ("authorization", "x-github-api-version"): + del new_req.headers[key] + return new_req + # Job name prefixes that are considered test jobs. _TEST_PREFIXES = ( "conda-cpp-tests", @@ -124,7 +141,8 @@ def _add(test_id): failed.append(test_id) try: - with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_SEC) as resp: + opener = urllib.request.build_opener(_DropAuthOnRedirect) + with opener.open(req, timeout=_HTTP_TIMEOUT_SEC) as resp: for raw in io.TextIOWrapper( resp, encoding="utf-8", errors="replace" ): @@ -155,7 +173,8 @@ def _add(test_id): _add(line.split(" ", 1)[1].split(" - ")[0].strip()) elif line.startswith("="): in_pytest_summary = False - except (urllib.error.HTTPError, urllib.error.URLError): + except (urllib.error.HTTPError, urllib.error.URLError) as exc: + print(f"Warning: could not fetch logs for job {job_id}: {exc}", file=sys.stderr) return [], None return failed[:_MAX_TESTS], crash From af8d0e132db9d0b56fe7fad0996753951d160fd0 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 23 Jun 2026 15:22:13 -0500 Subject: [PATCH 08/13] ci: apply ruff-format to pr_test_summary.py Co-Authored-By: Claude Sonnet 4.6 --- ci/utils/pr_test_summary.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 4d31171de7..26187d6d6d 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -37,6 +37,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): del new_req.headers[key] return new_req + # Job name prefixes that are considered test jobs. _TEST_PREFIXES = ( "conda-cpp-tests", @@ -174,7 +175,10 @@ def _add(test_id): elif line.startswith("="): in_pytest_summary = False except (urllib.error.HTTPError, urllib.error.URLError) as exc: - print(f"Warning: could not fetch logs for job {job_id}: {exc}", file=sys.stderr) + print( + f"Warning: could not fetch logs for job {job_id}: {exc}", + file=sys.stderr, + ) return [], None return failed[:_MAX_TESTS], crash From 50ba45e3f02676ddbe72a611c07bede6258138d9 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 23 Jun 2026 17:26:59 -0500 Subject: [PATCH 09/13] ci: remove intentional demo failing tests Co-Authored-By: Claude Sonnet 4.6 --- cpp/tests/routing/unit_tests/breaks.cu | 6 ------ .../cuopt/cuopt/tests/test_ci_summary_demo.py | 17 ----------------- 2 files changed, 23 deletions(-) delete mode 100644 python/cuopt/cuopt/tests/test_ci_summary_demo.py diff --git a/cpp/tests/routing/unit_tests/breaks.cu b/cpp/tests/routing/unit_tests/breaks.cu index 2ccb4f2a17..0d8a578b6e 100644 --- a/cpp/tests/routing/unit_tests/breaks.cu +++ b/cpp/tests/routing/unit_tests/breaks.cu @@ -430,12 +430,6 @@ TEST(vehicle_breaks, non_uniform_breaks) check_route(data_model, h_routing_solution); } -// Intentional failure to exercise the PR test summary comment feature. -TEST(vehicle_breaks, ci_summary_demo_failure) -{ - EXPECT_TRUE(false) << "Intentional failure: remove after verifying PR summary comment."; -} - } // namespace test } // namespace routing } // namespace cuopt diff --git a/python/cuopt/cuopt/tests/test_ci_summary_demo.py b/python/cuopt/cuopt/tests/test_ci_summary_demo.py deleted file mode 100644 index dd3d16303e..0000000000 --- a/python/cuopt/cuopt/tests/test_ci_summary_demo.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Intentional failures to exercise the PR test summary comment feature. -# Remove after verifying the summary comment shows correct output. - - -def test_ci_summary_demo_failure(): - assert False, ( - "Intentional failure: remove after verifying PR summary comment." - ) - - -def test_ci_summary_demo_failure_2(): - raise RuntimeError( - "Intentional error: remove after verifying PR summary comment." - ) From 4da707aa4fc7cb97e73d0c8a3627025a73524641 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 23 Jun 2026 17:27:58 -0500 Subject: [PATCH 10/13] ci: add Python 3.14-only demo failing test to exercise PR summary comment Co-Authored-By: Claude Sonnet 4.6 --- python/cuopt/cuopt/tests/test_ci_summary_demo.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 python/cuopt/cuopt/tests/test_ci_summary_demo.py diff --git a/python/cuopt/cuopt/tests/test_ci_summary_demo.py b/python/cuopt/cuopt/tests/test_ci_summary_demo.py new file mode 100644 index 0000000000..10cdb7d37a --- /dev/null +++ b/python/cuopt/cuopt/tests/test_ci_summary_demo.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys + +import pytest + + +@pytest.mark.skipif(sys.version_info < (3, 14), reason="Python 3.14 only") +def test_ci_summary_demo(): + pytest.fail( + "Intentional failure: remove after verifying PR summary comment." + ) From e66e720d492154000ae11f799aebe81c47922d75 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Wed, 24 Jun 2026 16:11:55 -0500 Subject: [PATCH 11/13] =?UTF-8?q?ci:=20simplify=20pr=5Ftest=5Fsummary=20?= =?UTF-8?q?=E2=80=94=20remove=20crash=20detection,=20trim=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop _CRASH_PATTERNS and all crash-detection logic - Simplify _is_test_job to plain startswith (exact-match fallback unneeded) - _analyze_job_log returns a list directly instead of (list, crash) tuple - Remove verbose module/function/class docstrings Co-Authored-By: Claude Sonnet 4.6 --- ci/utils/pr_test_summary.py | 108 ++++++++---------------------------- 1 file changed, 22 insertions(+), 86 deletions(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 26187d6d6d..677db02f8d 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -1,16 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Post or update a sticky PR comment summarizing CI test job failures. - -Reads GITHUB_REPOSITORY, GITHUB_RUN_ID, GITHUB_REF, and GH_TOKEN from the -environment. Finds every test job in the current workflow run, then posts (or -updates) a single comment on the pull request listing any failed jobs with -direct log links and a collapsible list of individual failed test names. - -Usage (called from a GitHub Actions step): - python3 ci/utils/pr_test_summary.py -""" +"""Post or update a sticky PR comment summarizing CI test job failures.""" import io import json @@ -20,24 +11,6 @@ import urllib.error import urllib.request - -class _DropAuthOnRedirect(urllib.request.HTTPRedirectHandler): - """Strip auth headers before following redirects to presigned URLs (e.g. S3). - - GitHub's job-log endpoint issues a 302 to a presigned S3 URL. Forwarding - the Authorization header causes S3 to return 400 ("Only one auth mechanism - allowed"), so we strip it before following the redirect. - """ - - def redirect_request(self, req, fp, code, msg, headers, newurl): - new_req = super().redirect_request(req, fp, code, msg, headers, newurl) - if new_req is not None: - for key in list(new_req.headers): - if key.lower() in ("authorization", "x-github-api-version"): - del new_req.headers[key] - return new_req - - # Job name prefixes that are considered test jobs. _TEST_PREFIXES = ( "conda-cpp-tests", @@ -49,28 +22,23 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): _MARKER = "" _HTTP_TIMEOUT_SEC = 30 -# Maximum failed test names shown per job dropdown. _MAX_TESTS = 50 -# gtest prints "[ FAILED ] Suite.Test (12 ms)" per failing test and again, -# without the timing suffix, in the end-of-run "listed below" block. Capture the -# "Suite.Test" name; the dedup set collapses the duplicate. +# gtest prints "[ FAILED ] Suite.Test (12 ms)" per failing test and again +# without the timing suffix; the dedup set collapses the duplicate. _GTEST_FAILED = re.compile(r"\[ FAILED \] (\S+\.\S+?)(?: \(\d+ ms\))?$") -# Ordered by specificity; first match wins. -_CRASH_PATTERNS = [ - ("Segmentation fault", "SIGSEGV (segfault)"), - ("SIGSEGV", "SIGSEGV (segfault)"), - ("signal 11", "SIGSEGV (signal 11)"), - ("Aborted (core dumped)", "SIGABRT"), - ("SIGABRT", "SIGABRT"), - ("signal 6", "SIGABRT (signal 6)"), - ("SIGKILL", "SIGKILL"), - ("signal 9", "SIGKILL (signal 9)"), - ("Out of memory", "OOM"), - ("oom-kill", "OOM"), - ("core dumped", "core dumped"), -] + +class _DropAuthOnRedirect(urllib.request.HTTPRedirectHandler): + # GitHub's job-log endpoint redirects to a presigned S3 URL. Forwarding + # the Authorization header causes S3 to return 400, so strip it first. + def redirect_request(self, req, fp, code, msg, headers, newurl): + new_req = super().redirect_request(req, fp, code, msg, headers, newurl) + if new_req is not None: + for key in list(new_req.headers): + if key.lower() in ("authorization", "x-github-api-version"): + del new_req.headers[key] + return new_req def _headers(token): @@ -82,13 +50,11 @@ def _headers(token): def _paginate(path, token): - """Yield all items from a paginated GitHub REST API GET endpoint.""" url = f"https://api.github.com{path}?per_page=100" while url: req = urllib.request.Request(url, headers=_headers(token)) with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT_SEC) as resp: data = json.loads(resp.read()) - # Jobs endpoint wraps items in {"jobs": [...]}; comments is a bare list. yield from (data["jobs"] if isinstance(data, dict) else data) link = resp.headers.get("Link", "") url = next( @@ -113,27 +79,16 @@ def _api(path, token, method, payload): def _is_test_job(name): - # Matrix jobs use " / " or " (" as separator depending on GHA version. - return any(name == p or name.startswith(p + " ") for p in _TEST_PREFIXES) + return any(name.startswith(p) for p in _TEST_PREFIXES) def _analyze_job_log(job_id, repo, token): - """Return (failed_test_ids, crash_description_or_None) from a job's log. - - Streams the whole log line by line (bounded memory: only matched failures - are retained). Truncating to a tail window does not work — the pytest - summary is not necessarily near the end (a job may run several test suites, - e.g. cuopt then cuopt_server), so a fixed window can drop it entirely. - Recognizes both pytest ("short test summary info") and gtest/ctest - ("[ FAILED ] Suite.Test") output. - """ req = urllib.request.Request( f"https://api.github.com/repos/{repo}/actions/jobs/{job_id}/logs", headers=_headers(token), ) failed = [] seen = set() - crash = None in_pytest_summary = False def _add(test_id): @@ -155,18 +110,11 @@ def _add(test_id): else raw.rstrip("\n") ) - if crash is None: - crash = next( - (d for p, d in _CRASH_PATTERNS if p in line), None - ) - - # gtest / ctest failures (C++ test jobs). m = _GTEST_FAILED.match(line) if m: _add(m.group(1)) continue - # pytest failures (Python test jobs). if "short test summary info" in line: in_pytest_summary = True elif in_pytest_summary: @@ -179,9 +127,8 @@ def _add(test_id): f"Warning: could not fetch logs for job {job_id}: {exc}", file=sys.stderr, ) - return [], None - return failed[:_MAX_TESTS], crash + return failed[:_MAX_TESTS] def _build_body(failed, passed, skipped, job_analysis): @@ -199,27 +146,16 @@ def _build_body(failed, passed, skipped, job_analysis): ) for job in failed: - tests, crash = job_analysis.get(job["id"], ([], None)) - if not tests and not crash: + tests = job_analysis[job["id"]] + if not tests: continue - if crash and not tests: - summary = f"💥 crashed ({crash})" - detail = ( - "Process was terminated before the test run completed." - ) - else: - n = len(tests) - noun = "test" if n == 1 else "tests" - summary = f"{n} failed {noun}" + ( - f" · 💥 {crash}" if crash else "" - ) - detail = "\n".join(f"- `{t}`" for t in tests) + n = len(tests) lines += [ "", "
", - f"{job['name']} — {summary}", + f"{job['name']} — {n} failed {'test' if n == 1 else 'tests'}", "", - detail, + "\n".join(f"- `{t}`" for t in tests), "", "
", ] @@ -231,7 +167,7 @@ def main(): token = os.environ["GH_TOKEN"] repo = os.environ["GITHUB_REPOSITORY"] run_id = os.environ["GITHUB_RUN_ID"] - ref = os.environ["GITHUB_REF"] # refs/heads/pull-request/NNN + ref = os.environ["GITHUB_REF"] branch = ref.removeprefix("refs/heads/") if not branch.startswith("pull-request/"): From ec4565c4aee64f56cf4498439b0152485314fe9c Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 25 Jun 2026 14:24:32 -0500 Subject: [PATCH 12/13] ci: remove redundant job table from PR test summary comment The log links table duplicates what's already in the CI status box. Keep only the per-job details dropdowns with failing test names. Co-Authored-By: Claude Sonnet 4.6 --- ci/utils/pr_test_summary.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/utils/pr_test_summary.py b/ci/utils/pr_test_summary.py index 677db02f8d..6971dd83a3 100644 --- a/ci/utils/pr_test_summary.py +++ b/ci/utils/pr_test_summary.py @@ -139,12 +139,6 @@ def _build_body(failed, passed, skipped, job_analysis): lines.append( f"**{len(failed)} failed** · {len(passed)} passed · {len(skipped)} skipped" ) - lines += ["", "| Job | Logs |", "|-----|------|"] - for job in failed: - lines.append( - f"| ❌ `{job['name']}` | [View logs]({job['html_url']}) |" - ) - for job in failed: tests = job_analysis[job["id"]] if not tests: From d6d302f08514aa7ed8f27b61261c1208e6d4084d Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 25 Jun 2026 16:17:05 -0500 Subject: [PATCH 13/13] ci: remove demo failing test Co-Authored-By: Claude Sonnet 4.6 --- python/cuopt/cuopt/tests/test_ci_summary_demo.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 python/cuopt/cuopt/tests/test_ci_summary_demo.py diff --git a/python/cuopt/cuopt/tests/test_ci_summary_demo.py b/python/cuopt/cuopt/tests/test_ci_summary_demo.py deleted file mode 100644 index 10cdb7d37a..0000000000 --- a/python/cuopt/cuopt/tests/test_ci_summary_demo.py +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import sys - -import pytest - - -@pytest.mark.skipif(sys.version_info < (3, 14), reason="Python 3.14 only") -def test_ci_summary_demo(): - pytest.fail( - "Intentional failure: remove after verifying PR summary comment." - )