From 8abd72bf60f5ab78669f42973396cd7782b97657 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 24 Jun 2026 11:43:01 -0400 Subject: [PATCH 1/2] Make GitHubIssueTest a plain Assertion object Rename GitHubIssueTest -> Assertion (with a back-compat alias) and stop storing the live PyGitHub Issue: capture the few fields actually needed (issue_id, url, state) as plain strings at construction. This makes the object picklable and trivial to build in tests or other tools, and gives downstream consumers a stable, low-level-free surface to interact with. Also adds: - `name`/`is_known` convenience accessors. - `run()`, which yields (param_set, TestResult) pairs by evaluating one param_set at a time, so callers can attribute every result back to its source param_set (CachedNodeNorm/CachedNameRes dedupe the network calls, so it's no slower). Co-Authored-By: Claude Opus 4.8 --- .../github/github_issues_test_cases.py | 77 +++++++++++++++---- tests/github_issues/test_system.py | 2 +- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/src/babel_validation/sources/github/github_issues_test_cases.py b/src/babel_validation/sources/github/github_issues_test_cases.py index 51e0c82..5ad9516 100644 --- a/src/babel_validation/sources/github/github_issues_test_cases.py +++ b/src/babel_validation/sources/github/github_issues_test_cases.py @@ -23,6 +23,7 @@ ["MONDO:0005015", "DOID:9351"]] """ +import itertools import json import logging import re @@ -50,29 +51,52 @@ def _to_list(value, context: str) -> list: raise ValueError(f"{context}: expected str or list, got {type(value).__name__}") -class GitHubIssueTest: - """Represents one assertion extracted from a GitHub issue body — an assertion name paired with a list of param_sets to evaluate.""" +class Assertion: + """One assertion extracted from a GitHub issue body — an assertion name paired with a + list of param_sets to evaluate, plus plain metadata about the issue it came from. - def __init__(self, github_issue_id: str, github_issue: Issue.Issue, assertion: str, param_sets: list[list[str]] = None): + This is the high-level object downstream consumers interact with. It deliberately stores + only plain strings (no live PyGitHub ``Issue``) so it is picklable and trivial to build in + tests or other tools that fetched the issue some other way. Consumers that run their own + executor (e.g. against a local database) can read ``assertion``/``param_sets`` and the + handler's ``curie_params()`` directly; consumers that want to test against live services + can call :meth:`run` (or the per-service ``test_with_*`` methods). + """ + + def __init__(self, assertion: str, param_sets: list[list[str]] = None, *, + issue_id: str = "", issue_url: str = "", issue_state: str = ""): """ - :param github_issue_id: Human-readable issue identifier, e.g. "org/repo#42". - :param github_issue: The PyGitHub Issue object this test was extracted from. :param assertion: The assertion name (case-insensitive), e.g. "Resolves" or "HasLabel". - Must match a key in ASSERTION_HANDLERS. + Must match a key in ASSERTION_HANDLERS to be runnable. :param param_sets: A list of param_sets (list[list[str]]) to evaluate for this assertion. Each inner list is one param_set — see module docstring for details. + :param issue_id: Human-readable issue identifier, e.g. "org/repo#42". + :param issue_url: The issue's html_url, for linking back from reports. + :param issue_state: The issue's state, "open" or "closed". """ if not isinstance(param_sets, list) and param_sets is not None: - raise ValueError(f"param_sets must be a list when creating a GitHubIssueTest({github_issue}, {assertion}, {param_sets})") - self.github_issue = github_issue + raise ValueError(f"param_sets must be a list when creating an Assertion({assertion!r}, {param_sets!r})") self.assertion = assertion self.param_sets = param_sets if param_sets is not None else [] - self.github_issue_id = github_issue_id + self.issue_id = issue_id + self.issue_url = issue_url + self.issue_state = issue_state + + _logger.info("Creating Assertion for %s %s(%s)", issue_url or issue_id, assertion, param_sets) - _logger.info("Creating GitHubIssueTest for %s %s(%s)", github_issue.html_url, assertion, param_sets) + @property + def name(self) -> str: + """Alias for the assertion type name (reads more naturally than ``a.assertion``).""" + return self.assertion + + @property + def is_known(self) -> bool: + """True if this assertion name maps to a registered handler.""" + return self.assertion.lower() in ASSERTION_HANDLERS def __str__(self): - return f"{self.github_issue_id}: {self.assertion}({len(self.param_sets)} param sets: {json.dumps(self.param_sets)})" + prefix = f"{self.issue_id}: " if self.issue_id else "" + return f"{prefix}{self.assertion}({len(self.param_sets)} param sets: {json.dumps(self.param_sets)})" def _get_handler(self): handler = ASSERTION_HANDLERS.get(self.assertion.lower()) @@ -86,6 +110,26 @@ def test_with_nodenorm(self, nodenorm: CachedNodeNorm) -> Iterator[TestResult]: def test_with_nameres(self, nodenorm: CachedNodeNorm, nameres: CachedNameRes, pass_if_found_in_top=5) -> Iterator[TestResult]: return self._get_handler().test_with_nameres(self.param_sets, nodenorm, nameres, pass_if_found_in_top, label=str(self)) + def run(self, nodenorm: CachedNodeNorm, nameres: CachedNameRes, + pass_if_found_in_top: int = 5) -> Iterator[tuple[list[str], TestResult]]: + """Yield ``(param_set, TestResult)`` pairs, evaluating each param_set against whichever + service(s) this assertion applies to. Running one param_set at a time lets callers + attribute every result back to its source param_set; ``CachedNodeNorm``/``CachedNameRes`` + deduplicate network calls, so this is no more expensive than a batch run. + + Raises ValueError if the assertion name is unknown (check :attr:`is_known` first).""" + handler = self._get_handler() + for param_set in self.param_sets: + for result in itertools.chain( + handler.test_with_nodenorm([param_set], nodenorm, label=str(self)), + handler.test_with_nameres([param_set], nodenorm, nameres, pass_if_found_in_top, label=str(self)), + ): + yield param_set, result + + +# Backwards-compatible alias for the pre-1.0 name. +GitHubIssueTest = Assertion + class GitHubIssuesTestCases: """ @@ -123,7 +167,7 @@ def __init__(self, github_token: str, github_repositories): self.github_repositories = github_repositories self.logger.info("Configured GitHub repositories: %s", self.github_repositories) - def get_test_issues_from_issue(self, github_issue: Issue.Issue) -> list[GitHubIssueTest]: + def get_test_issues_from_issue(self, github_issue: Issue.Issue) -> list[Assertion]: """ Extract test rows from a single GitHub issue. @@ -149,6 +193,11 @@ def get_test_issues_from_issue(self, github_issue: Issue.Issue) -> list[GitHubIs self.logger.debug("Looking for tests in issue %s: %s (%s, %s)", github_issue_id, github_issue.title, github_issue.state, github_issue.html_url) + # Capture the plain issue fields once so the Assertion objects don't hold a live + # PyGitHub Issue (keeps them picklable and easy to build in tests/other tools). + issue_meta = dict(issue_id=github_issue_id, issue_url=github_issue.html_url, + issue_state=github_issue.state) + # Is there an issue body at all? if not github_issue.body or github_issue.body.strip() == '': return [] @@ -168,7 +217,7 @@ def get_test_issues_from_issue(self, github_issue: Issue.Issue) -> list[GitHubIs # Wiki syntax: params[0] is the assertion name; params[1:] form a single # param_set (may be empty for assertions like Needed), so param_sets is a # one-element list: [params[1:]]. - testrows.append(GitHubIssueTest(github_issue_id, github_issue, params[0], [params[1:]])) + testrows.append(Assertion(params[0], [params[1:]], **issue_meta)) babeltest_yaml_matches = re.findall(self._BABELTEST_YAML_RE, github_issue.body) if babeltest_yaml_matches: @@ -211,7 +260,7 @@ def get_test_issues_from_issue(self, github_issue: Issue.Issue) -> list[GitHubIs _to_list(ps, f"YAML block in issue {github_issue_id}: assertion '{assertion}' param_set") for ps in normalized ] - testrows.append(GitHubIssueTest(github_issue_id, github_issue, assertion, param_sets)) + testrows.append(Assertion(assertion, param_sets, **issue_meta)) return testrows diff --git a/tests/github_issues/test_system.py b/tests/github_issues/test_system.py index 7705d58..282a49e 100644 --- a/tests/github_issues/test_system.py +++ b/tests/github_issues/test_system.py @@ -194,7 +194,7 @@ def test_yaml_null_assertion_params_raises(self, github_issues_test_cases): github_issues_test_cases.get_test_issues_from_issue(mock) def test_yaml_empty_assertion_params(self, github_issues_test_cases): - # Resolves: [] → GitHubIssueTest with empty param_sets (no crash) + # Resolves: [] → Assertion with empty param_sets (no crash) mock = _mock_issue("```yaml\nbabel_tests:\n Resolves: []\n```") tests = github_issues_test_cases.get_test_issues_from_issue(mock) assert len(tests) == 1 From b361dea2b1a72bc9def672f01250e5c5c2673161 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 24 Jun 2026 11:43:14 -0400 Subject: [PATCH 2/2] Add run_issue_tests reporting API; rewire pytest as an adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the issue-test orchestration out of the pytest test into a reusable, pytest-free runner so other tools (e.g. Babel Explorer) can "run all issue tests against NodeNorm X / NameRes Y and report pass/fail + messages": - babel_validation/runner.py: run_issue_tests() / run_assertions() returning IssueReport + ResultRecord. The library now owns the expectation rules — open issue => expected-fail (all-pass => `closeable`), closed issue => expected-pass (any-fail => `reopened`) — instead of burying them in the test. - babel_validation/__init__.py: export the public surface (Assertion, GitHubIssuesTestCases, run_issue_tests, run_assertions, IssueReport, ResultRecord, CachedNodeNorm, CachedNameRes, TestResult, TestStatus). - tests/github_issues/test_github_issues.py: now a thin adapter that maps an IssueReport onto pytest outcomes (xfail/subtests/skip/fail); behavior unchanged. - tests/github_issues/test_runner.py: offline unit tests for the runner using fake services, also exercising the public import surface. - README: "Use as a library" section with the Explorer (run_issue_tests) and Babel (parse + custom executor) usage examples. Co-Authored-By: Claude Opus 4.8 --- README.md | 52 +++++++ src/babel_validation/__init__.py | 42 ++++++ src/babel_validation/runner.py | 163 ++++++++++++++++++++++ tests/github_issues/test_github_issues.py | 115 +++++++-------- tests/github_issues/test_runner.py | 136 ++++++++++++++++++ 5 files changed, 444 insertions(+), 64 deletions(-) create mode 100644 src/babel_validation/runner.py create mode 100644 tests/github_issues/test_runner.py diff --git a/README.md b/README.md index b59a874..4594ab6 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,58 @@ ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss ======================================================= 41 passed, 1965 skipped, 4 xfailed in 10.11s ======================================================== ``` +## Use as a library + +`babel_validation` is also installable as a library so other tools can reuse the +GitHub-issue test framework. Test definitions live in GitHub issues (using the +`{{BabelTest|...}}` or ```` ```yaml babel_tests: ```` syntax — see +[`assertions/README.md`](./src/babel_validation/assertions/README.md)), so the library +needs a GitHub token to fetch them. + +Install it from GitHub in a `uv`-based project: + +```shell +$ uv add "babel-validation @ git+https://github.com/TranslatorSRI/babel-validation" +``` + +**Run every issue test against a chosen NodeNorm/NameRes pair and report results** +(e.g. an interactive UI, or comparing environments): + +```python +from babel_validation import GitHubIssuesTestCases, CachedNodeNorm, CachedNameRes, run_issue_tests + +cases = GitHubIssuesTestCases(github_token, ["NCATSTranslator/Babel"]) +reports = run_issue_tests( + cases, + nodenorm=CachedNodeNorm.from_url("https://nodenorm.transltr.io/"), + nameres=CachedNameRes.from_url("https://name-lookup.transltr.io/"), +) +for r in reports: + if r.closeable: + print(f"{r.issue_id} now passes — consider closing ({r.url})") + elif r.reopened: + print(f"{r.issue_id} regressed — consider reopening ({r.url})") + for failure in r.failed_results: + print(f" {failure.assertion} {failure.param_set}: {failure.message}") +``` + +**Parse the assertions and run them against your own data** (e.g. a Babel pipeline +checking its local DuckDB/JSON output before the data reaches NodeNorm). Each +`Assertion` exposes the assertion name and its param sets so you can implement your +own executor: + +```python +from babel_validation import GitHubIssuesTestCases + +cases = GitHubIssuesTestCases(github_token, ["NCATSTranslator/Babel"]) +for issue in cases.get_issues_with_tests(): + for assertion in cases.get_test_issues_from_issue(issue): + # assertion.assertion -> e.g. "Resolves" / "HasLabel" / "ResolvesWithType" + # assertion.param_sets -> list[list[str]]; assertion.issue_state -> "open"|"closed" + for param_set in assertion.param_sets: + ... # check param_set against your local data +``` + ## Log Analysis The Jupyter Notebook in `log-analysis/` contains some basic analysis of the diff --git a/src/babel_validation/__init__.py b/src/babel_validation/__init__.py index e69de29..4a869f6 100644 --- a/src/babel_validation/__init__.py +++ b/src/babel_validation/__init__.py @@ -0,0 +1,42 @@ +"""babel_validation — validate Babel (NodeNorm / NameRes) outputs against tests embedded in +GitHub issues. + +Public API (stable surface for downstream consumers such as the Babel pipeline and Babel +Explorer): + +- :class:`GitHubIssuesTestCases` — fetch + parse BabelTest assertions from GitHub issues. +- :class:`Assertion` — one parsed assertion (name + param_sets + issue metadata). Run it against + live services with ``run()`` / ``test_with_*``, or read ``assertion``/``param_sets`` to drive + your own executor (e.g. against a local database). +- :func:`run_issue_tests` / :func:`run_assertions` — run assertions and get + :class:`IssueReport` objects (pass/fail messages + closeable/reopened classification). +- :class:`CachedNodeNorm` / :class:`CachedNameRes` — caching service clients. +- :class:`TestResult` / :class:`TestStatus` — the result value types. +""" + +from babel_validation.core.testrow import TestResult, TestStatus +from babel_validation.runner import ( + IssueReport, + ResultRecord, + run_assertions, + run_issue_tests, +) +from babel_validation.services.nameres import CachedNameRes +from babel_validation.services.nodenorm import CachedNodeNorm +from babel_validation.sources.github.github_issues_test_cases import ( + Assertion, + GitHubIssuesTestCases, +) + +__all__ = [ + "Assertion", + "GitHubIssuesTestCases", + "run_issue_tests", + "run_assertions", + "IssueReport", + "ResultRecord", + "CachedNodeNorm", + "CachedNameRes", + "TestResult", + "TestStatus", +] diff --git a/src/babel_validation/runner.py b/src/babel_validation/runner.py new file mode 100644 index 0000000..926d645 --- /dev/null +++ b/src/babel_validation/runner.py @@ -0,0 +1,163 @@ +"""Run GitHub-issue BabelTest assertions and report results — independent of pytest. + +This is the orchestration layer shared by Babel Explorer (run every issue test against a chosen +NodeNorm/NameRes pair and show which passed/failed and why) and this repo's own pytest suite. + +It owns the *expectation rules*: + +- An **open** issue is expected to fail. If its tests now all pass, the issue is **closeable**. +- A **closed** issue is expected to pass. If its tests now fail, the issue should be **reopened**. + +Typical use:: + + from babel_validation import ( + GitHubIssuesTestCases, CachedNodeNorm, CachedNameRes, run_issue_tests, + ) + + cases = GitHubIssuesTestCases(token, repos) + reports = run_issue_tests( + cases, + nodenorm=CachedNodeNorm.from_url("https://nodenorm.transltr.io/"), + nameres=CachedNameRes.from_url("https://name-lookup.transltr.io/"), + ) + for r in reports: + if r.closeable: + print(f"{r.issue_id} now passes — consider closing ({r.url})") + elif r.reopened: + print(f"{r.issue_id} regressed — consider reopening ({r.url})") + for failure in r.failed_results: + print(f" {failure.assertion} {failure.param_set}: {failure.message}") +""" + +from dataclasses import dataclass, field +from typing import Iterable + +from babel_validation.core.testrow import TestResult, TestStatus +from babel_validation.services.nameres import CachedNameRes +from babel_validation.services.nodenorm import CachedNodeNorm +from babel_validation.sources.github.github_issues_test_cases import ( + Assertion, + GitHubIssuesTestCases, +) + + +@dataclass(frozen=True) +class ResultRecord: + """One TestResult with provenance back to the assertion + param_set that produced it.""" + + assertion: str + param_set: list[str] + result: TestResult + + @property + def status(self) -> TestStatus: + return self.result.status + + @property + def message(self) -> str: + return self.result.message + + +@dataclass +class IssueReport: + """The outcome of running every assertion found in one GitHub issue.""" + + issue_id: str + url: str + state: str # "open" | "closed" + results: list[ResultRecord] = field(default_factory=list) + # Assertion names that don't map to a known handler — a hard configuration error. + unknown_assertions: list[str] = field(default_factory=list) + + @property + def is_open(self) -> bool: + return self.state == "open" + + @property + def failed_results(self) -> list[ResultRecord]: + return [r for r in self.results if r.status == TestStatus.Failed] + + @property + def has_failures(self) -> bool: + return any(r.status == TestStatus.Failed for r in self.results) + + @property + def all_passed(self) -> bool: + return bool(self.results) and not self.has_failures + + @property + def closeable(self) -> bool: + """Open issue whose tests now all pass (and parse cleanly) — a candidate to close.""" + return self.is_open and self.all_passed and not self.unknown_assertions + + @property + def reopened(self) -> bool: + """Closed issue whose tests now fail — a candidate to reopen.""" + return (not self.is_open) and self.has_failures + + +def run_assertions( + issue_id: str, + url: str, + state: str, + assertions: Iterable[Assertion], + *, + nodenorm: CachedNodeNorm, + nameres: CachedNameRes, + pass_if_found_in_top: int = 5, +) -> IssueReport: + """Run every assertion for a single issue and collect the results into an IssueReport. + + Unknown assertion names are recorded in ``unknown_assertions`` rather than raised, so the + caller can decide how to surface them (the pytest suite fails hard; Explorer can flag them). + """ + report = IssueReport(issue_id=issue_id, url=url, state=state) + for assertion in assertions: + if not assertion.is_known: + report.unknown_assertions.append(assertion.assertion) + continue + for param_set, result in assertion.run(nodenorm, nameres, pass_if_found_in_top): + report.results.append( + ResultRecord(assertion=assertion.assertion, param_set=param_set, result=result) + ) + return report + + +def run_issue_tests( + test_cases: GitHubIssuesTestCases, + repos=None, + *, + nodenorm: CachedNodeNorm, + nameres: CachedNameRes, + pass_if_found_in_top: int = 5, + issue_ids: list[str] | None = None, +) -> list[IssueReport]: + """Fetch issue tests and run them, returning one IssueReport per issue. + + :param test_cases: a configured ``GitHubIssuesTestCases`` (provides the GitHub auth + repos). + :param repos: optional repo subset; defaults to the repos the test_cases was built with. + :param issue_ids: optional explicit issue IDs ('org/repo#N', 'repo#N', or 'N'). When given, + only those issues are tested (and the GitHub search index lag is bypassed); + otherwise every issue containing BabelTest syntax is discovered and run. + """ + if issue_ids: + issues = test_cases.get_issues_by_ids(issue_ids) + else: + issues = test_cases.get_issues_with_tests(repos) + + reports = [] + for issue in issues: + issue_id = f"{issue.repository.full_name}#{issue.number}" + assertions = test_cases.get_test_issues_from_issue(issue) + reports.append( + run_assertions( + issue_id, + issue.html_url, + issue.state, + assertions, + nodenorm=nodenorm, + nameres=nameres, + pass_if_found_in_top=pass_if_found_in_top, + ) + ) + return reports diff --git a/tests/github_issues/test_github_issues.py b/tests/github_issues/test_github_issues.py index 4dd7e81..8eb099c 100644 --- a/tests/github_issues/test_github_issues.py +++ b/tests/github_issues/test_github_issues.py @@ -1,90 +1,77 @@ -import itertools -import json - import pytest from babel_validation.assertions import ASSERTION_HANDLERS +from babel_validation.runner import run_assertions from babel_validation.services.nameres import CachedNameRes from babel_validation.services.nodenorm import CachedNodeNorm -from babel_validation.core.testrow import TestResult, TestStatus +from babel_validation.core.testrow import TestStatus def test_github_issue(request, target_info, github_issue_id, github_issue, github_issues_test_cases, subtests): + """Thin adapter around babel_validation.runner.run_assertions. + + The library decides what each result *is* (pass/fail) and what the issue state *implies* + (open ⇒ expected-fail, closed ⇒ expected-pass); this function only maps the resulting + IssueReport onto pytest outcomes (xfail / subtests / skip / fail).""" nodenorm = CachedNodeNorm.from_url(target_info['NodeNormURL']) nameres = CachedNameRes.from_url(target_info['NameResURL']) # NameRes assertions pass if the expected CURIE is in the top N results; N # comes from targets.ini (NameResXFailIfInTop) rather than being hardcoded. pass_if_found_in_top = int(target_info.get('NameResXFailIfInTop', 5)) - tests = github_issues_test_cases.get_test_issues_from_issue(github_issue) - if not tests: - pytest.skip(f"No tests found in issue {github_issue}") - return - is_open = github_issue.state == "open" + assertions = github_issues_test_cases.get_test_issues_from_issue(github_issue) + if not assertions: + pytest.skip(f"No tests found in issue {github_issue_id}") + + report = run_assertions( + github_issue_id, github_issue.html_url, github_issue.state, assertions, + nodenorm=nodenorm, nameres=nameres, pass_if_found_in_top=pass_if_found_in_top, + ) # Unknown assertion types must fail hard, not XFAIL. - unknown = [f"'{t.assertion}'" for t in tests - if t.assertion.lower() not in ASSERTION_HANDLERS] - if unknown: + if report.unknown_assertions: + unknown = ", ".join(f"'{a}'" for a in report.unknown_assertions) pytest.fail( - f"Issue {github_issue_id} uses unknown assertion type(s): " - f"{', '.join(unknown)} with param sets {json.dumps([t.param_sets for t in tests])}. " + f"Issue {github_issue_id} uses unknown assertion type(s): {unknown}. " f"Valid types (case-insensitive): {sorted(ASSERTION_HANDLERS.keys())}" ) - # Open issues are assumed to fail, so we set an xfail marker (but we set it to strict so - # that XPASSes are reported loudly). - if is_open: + # Open issues are assumed to fail, so we set a strict xfail marker (so XPASSes — issues that + # now pass and are therefore closeable — are reported loudly). + if report.is_open: request.node.add_marker(pytest.mark.xfail( - reason=f"Issue {github_issue.html_url} is expected to fail because the issue is open.", + reason=f"Issue {report.url} is expected to fail because the issue is open.", strict=True, )) - count_subtests = 0 - failed_messages = [] - for test_issue in tests: - results_nodenorm = test_issue.test_with_nodenorm(nodenorm) - results_nameres = test_issue.test_with_nameres(nodenorm, nameres, pass_if_found_in_top) - - for result in itertools.chain(results_nodenorm, results_nameres): - count_subtests += 1 - - if is_open: - # Don't assert inside subtests for open issues: subtest failures - # don't respect the xfail marker on the parent test, so they would - # be reported as real failures. Collect them and xfail below instead. - if result.status == TestStatus.Failed: - failed_messages.append(f"{github_issue_id} ({github_issue.state}): {result.message}") - continue - + if not report.is_open: + # Closed issue: assert each result in its own subtest. + for r in report.results: with subtests.test(msg=github_issue_id): - match result: - case TestResult(status=TestStatus.Passed, message=message): - assert True, f"{github_issue_id} ({github_issue.state}): {message}" - - case TestResult(status=TestStatus.Failed, message=message): - assert False, f"{github_issue_id} ({github_issue.state}): {message}" - - case TestResult(status=TestStatus.Skipped, message=message): - pytest.skip(f"{github_issue_id} ({github_issue.state}): {message}") - - case _: - assert False, f"Unknown result from {github_issue_id}: {result}" + if r.status == TestStatus.Passed: + assert True, f"{github_issue_id} ({report.state}): {r.message}" + elif r.status == TestStatus.Failed: + assert False, f"{github_issue_id} ({report.state}): {r.message}" + elif r.status == TestStatus.Skipped: + pytest.skip(f"{github_issue_id} ({report.state}): {r.message}") + else: + assert False, f"Unknown result from {github_issue_id}: {r.result}" + return - # For open issues: xfail so the result stays in the xfail family. - # - Some assertions failed → xfail with a count summary (expected outcome). - # - All assertions passed → the xfail marker added above makes this XPASS, - # which (strict=True) reports as a failure and signals the issue is closeable. - # - No assertions ran → xfail as a configuration error. - if is_open: - if count_subtests == 0: - pytest.xfail(f"Open issue {github_issue_id} produced no test results — check assertion configuration") - elif failed_messages: - pct = len(failed_messages) / count_subtests - details = "\n".join(failed_messages[:5]) - if len(failed_messages) > 5: - details += f"\n... and {len(failed_messages) - 5} more" - pytest.xfail( - f"Open issue {github_issue_id} has {len(failed_messages):,} failing assertions " - f"out of {count_subtests:,} ({pct:.0%}):\n{details}" - ) + # Open issue: keep the result in the xfail family. + # - No results → xfail as a configuration error. + # - Some failures → xfail with a count summary (the expected outcome). + # - All passed → the strict xfail marker above makes this XPASS, reported as a + # failure, signaling the issue is closeable. + if not report.results: + pytest.xfail(f"Open issue {github_issue_id} produced no test results — check assertion configuration") + failed = report.failed_results + if failed: + details = "\n".join(f"{github_issue_id} ({report.state}): {r.message}" for r in failed[:5]) + if len(failed) > 5: + details += f"\n... and {len(failed) - 5} more" + pct = len(failed) / len(report.results) + pytest.xfail( + f"Open issue {github_issue_id} has {len(failed):,} failing assertions " + f"out of {len(report.results):,} ({pct:.0%}):\n{details}" + ) diff --git a/tests/github_issues/test_runner.py b/tests/github_issues/test_runner.py new file mode 100644 index 0000000..b5e6bf9 --- /dev/null +++ b/tests/github_issues/test_runner.py @@ -0,0 +1,136 @@ +"""Unit tests for babel_validation.runner — the pytest-independent report layer. + +These use fake NodeNorm/NameRes clients (no network) and construct Assertion objects directly, +which is now trivial because Assertion stores only plain fields. They also import everything +through the public ``babel_validation`` API, so they double as an import-surface smoke test. +""" + +import pytest + +from babel_validation import ( + Assertion, + IssueReport, + ResultRecord, + TestStatus, + run_assertions, +) + +pytestmark = pytest.mark.unit + + +def _node(identifier, label="label", types=("biolink:SmallMolecule",)): + return {"id": {"identifier": identifier, "label": label}, "type": list(types)} + + +class FakeNodeNorm: + """Minimal stand-in for CachedNodeNorm backed by a {curie: node|None} dict.""" + + def __init__(self, resolved=None): + self.resolved = resolved or {} + + def normalize_curies(self, curies, **kwargs): + return {c: self.resolved.get(c) for c in curies} + + def normalize_curie(self, curie, **kwargs): + return self.resolved.get(curie) + + def __str__(self): + return "FakeNodeNorm" + + +class FakeNameRes: + """Minimal stand-in for CachedNameRes returning a fixed lookup result list.""" + + def __init__(self, results=None): + self.results = results if results is not None else [] + + def lookup(self, query, **kwargs): + return self.results + + def __str__(self): + return "FakeNameRes" + + +def _run(state, assertions, *, nodenorm, nameres=None): + return run_assertions( + "org/repo#1", "https://example/1", state, assertions, + nodenorm=nodenorm, nameres=nameres or FakeNameRes(), + ) + + +# --- classification: open vs closed ----------------------------------------- + +def test_closed_all_pass_is_not_reopened(): + a = Assertion("Resolves", [["CHEBI:15365"]], issue_state="closed") + report = _run("closed", [a], nodenorm=FakeNodeNorm({"CHEBI:15365": _node("CHEBI:15365")})) + assert report.all_passed + assert not report.has_failures + assert not report.reopened + assert not report.closeable + + +def test_closed_failure_is_reopened(): + a = Assertion("Resolves", [["CHEBI:15365"]], issue_state="closed") + report = _run("closed", [a], nodenorm=FakeNodeNorm({})) # CHEBI unresolved + assert report.has_failures + assert report.reopened + assert not report.closeable + + +def test_open_all_pass_is_closeable(): + a = Assertion("Resolves", [["CHEBI:15365"]], issue_state="open") + report = _run("open", [a], nodenorm=FakeNodeNorm({"CHEBI:15365": _node("CHEBI:15365")})) + assert report.closeable + assert report.all_passed + assert not report.reopened + + +def test_open_failure_is_neither_closeable_nor_reopened(): + a = Assertion("Resolves", [["CHEBI:15365"]], issue_state="open") + report = _run("open", [a], nodenorm=FakeNodeNorm({})) + assert report.has_failures + assert not report.closeable + assert not report.reopened + + +# --- unknown assertions ------------------------------------------------------ + +def test_unknown_assertion_recorded_not_run(): + a = Assertion("NotARealAssertion", [["CHEBI:1"]], issue_state="open") + report = _run("open", [a], nodenorm=FakeNodeNorm()) + assert report.unknown_assertions == ["NotARealAssertion"] + assert report.results == [] + # An unknown assertion blocks "closeable" even on an open issue with no failures. + assert not report.closeable + + +# --- provenance: each result carries its param_set --------------------------- + +def test_results_attributed_to_param_sets(): + a = Assertion("Resolves", [["CHEBI:1"], ["CHEBI:2"]], issue_state="closed") + nn = FakeNodeNorm({"CHEBI:1": _node("CHEBI:1"), "CHEBI:2": _node("CHEBI:2")}) + report = _run("closed", [a], nodenorm=nn) + assert [r.param_set for r in report.results] == [["CHEBI:1"], ["CHEBI:2"]] + assert all(isinstance(r, ResultRecord) and r.assertion == "Resolves" for r in report.results) + assert all(r.status == TestStatus.Passed for r in report.results) + + +# --- NameRes path executes through run() ------------------------------------- + +def test_searchbyname_runs_against_nameres(): + a = Assertion("SearchByName", [["water", "CHEBI:15377"]], issue_state="closed") + nn = FakeNodeNorm({"CHEBI:15377": _node("CHEBI:15377", "water")}) + nr = FakeNameRes([{"curie": "CHEBI:15377"}]) + report = _run("closed", [a], nodenorm=nn, nameres=nr) + assert report.all_passed + assert report.results[0].param_set == ["water", "CHEBI:15377"] + + +def test_report_is_an_issue_report_with_metadata(): + report = _run("open", [], nodenorm=FakeNodeNorm()) + assert isinstance(report, IssueReport) + assert report.issue_id == "org/repo#1" + assert report.url == "https://example/1" + assert report.is_open + # No assertions ⇒ no results ⇒ not closeable (nothing actually passed). + assert not report.closeable