Library API: Assertion object model + run_issue_tests reporting API

README.md

@@ -61,6 +61,58 @@ ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss
 ======================================================= 41 passed, 1965 skipped, 4 xfailed in 10.11s ========================================================
 ```
 
+## Use as a library
+
+`babel_validation` is also installable as a library so other tools can reuse the
+GitHub-issue test framework. Test definitions live in GitHub issues (using the
+`{{BabelTest|...}}` or ```` ```yaml babel_tests: ```` syntax — see
+[`assertions/README.md`](./src/babel_validation/assertions/README.md)), so the library
+needs a GitHub token to fetch them.
+
+Install it from GitHub in a `uv`-based project:
+
+```shell
+$ uv add "babel-validation @ git+https://github.com/TranslatorSRI/babel-validation"
+```
+
+**Run every issue test against a chosen NodeNorm/NameRes pair and report results**
+(e.g. an interactive UI, or comparing environments):
+
+```python
+from babel_validation import GitHubIssuesTestCases, CachedNodeNorm, CachedNameRes, run_issue_tests
+
+cases = GitHubIssuesTestCases(github_token, ["NCATSTranslator/Babel"])
+reports = run_issue_tests(
+    cases,
+    nodenorm=CachedNodeNorm.from_url("https://nodenorm.transltr.io/"),
+    nameres=CachedNameRes.from_url("https://name-lookup.transltr.io/"),
+)
+for r in reports:
+    if r.closeable:
+        print(f"{r.issue_id} now passes — consider closing ({r.url})")
+    elif r.reopened:
+        print(f"{r.issue_id} regressed — consider reopening ({r.url})")
+        for failure in r.failed_results:
+            print(f"    {failure.assertion} {failure.param_set}: {failure.message}")
+```
+
+**Parse the assertions and run them against your own data** (e.g. a Babel pipeline
+checking its local DuckDB/JSON output before the data reaches NodeNorm). Each
+`Assertion` exposes the assertion name and its param sets so you can implement your
+own executor:
+
+```python
+from babel_validation import GitHubIssuesTestCases
+
+cases = GitHubIssuesTestCases(github_token, ["NCATSTranslator/Babel"])
+for issue in cases.get_issues_with_tests():
+    for assertion in cases.get_test_issues_from_issue(issue):
+        # assertion.assertion -> e.g. "Resolves" / "HasLabel" / "ResolvesWithType"
+        # assertion.param_sets -> list[list[str]]; assertion.issue_state -> "open"|"closed"
+        for param_set in assertion.param_sets:
+            ...  # check param_set against your local data
+```
+
 ## Log Analysis
 
 The Jupyter Notebook in `log-analysis/` contains some basic analysis of the

src/babel_validation/__init__.py

@@ -0,0 +1,42 @@
+"""babel_validation — validate Babel (NodeNorm / NameRes) outputs against tests embedded in
+GitHub issues.
+
+Public API (stable surface for downstream consumers such as the Babel pipeline and Babel
+Explorer):
+
+- :class:`GitHubIssuesTestCases` — fetch + parse BabelTest assertions from GitHub issues.
+- :class:`Assertion` — one parsed assertion (name + param_sets + issue metadata). Run it against
+  live services with ``run()`` / ``test_with_*``, or read ``assertion``/``param_sets`` to drive
+  your own executor (e.g. against a local database).
+- :func:`run_issue_tests` / :func:`run_assertions` — run assertions and get
+  :class:`IssueReport` objects (pass/fail messages + closeable/reopened classification).
+- :class:`CachedNodeNorm` / :class:`CachedNameRes` — caching service clients.
+- :class:`TestResult` / :class:`TestStatus` — the result value types.
+"""
+
+from babel_validation.core.testrow import TestResult, TestStatus
+from babel_validation.runner import (
+    IssueReport,
+    ResultRecord,
+    run_assertions,
+    run_issue_tests,
+)
+from babel_validation.services.nameres import CachedNameRes
+from babel_validation.services.nodenorm import CachedNodeNorm
+from babel_validation.sources.github.github_issues_test_cases import (
+    Assertion,
+    GitHubIssuesTestCases,
+)
+
+__all__ = [
+    "Assertion",
+    "GitHubIssuesTestCases",
+    "run_issue_tests",
+    "run_assertions",
+    "IssueReport",
+    "ResultRecord",
+    "CachedNodeNorm",
+    "CachedNameRes",
+    "TestResult",
+    "TestStatus",
+]

src/babel_validation/runner.py

@@ -0,0 +1,163 @@
+"""Run GitHub-issue BabelTest assertions and report results — independent of pytest.
+
+This is the orchestration layer shared by Babel Explorer (run every issue test against a chosen
+NodeNorm/NameRes pair and show which passed/failed and why) and this repo's own pytest suite.
+
+It owns the *expectation rules*:
+
+- An **open** issue is expected to fail. If its tests now all pass, the issue is **closeable**.
+- A **closed** issue is expected to pass. If its tests now fail, the issue should be **reopened**.
+
+Typical use::
+
+    from babel_validation import (
+        GitHubIssuesTestCases, CachedNodeNorm, CachedNameRes, run_issue_tests,
+    )
+
+    cases = GitHubIssuesTestCases(token, repos)
+    reports = run_issue_tests(
+        cases,
+        nodenorm=CachedNodeNorm.from_url("https://nodenorm.transltr.io/"),
+        nameres=CachedNameRes.from_url("https://name-lookup.transltr.io/"),
+    )
+    for r in reports:
+        if r.closeable:
+            print(f"{r.issue_id} now passes — consider closing ({r.url})")
+        elif r.reopened:
+            print(f"{r.issue_id} regressed — consider reopening ({r.url})")
+            for failure in r.failed_results:
+                print(f"    {failure.assertion} {failure.param_set}: {failure.message}")
+"""
+
+from dataclasses import dataclass, field
+from typing import Iterable
+
+from babel_validation.core.testrow import TestResult, TestStatus
+from babel_validation.services.nameres import CachedNameRes
+from babel_validation.services.nodenorm import CachedNodeNorm
+from babel_validation.sources.github.github_issues_test_cases import (
+    Assertion,
+    GitHubIssuesTestCases,
+)
+
+
+@dataclass(frozen=True)
+class ResultRecord:
+    """One TestResult with provenance back to the assertion + param_set that produced it."""
+
+    assertion: str
+    param_set: list[str]
+    result: TestResult
+
+    @property
+    def status(self) -> TestStatus:
+        return self.result.status
+
+    @property
+    def message(self) -> str:
+        return self.result.message
+
+
+@dataclass
+class IssueReport:
+    """The outcome of running every assertion found in one GitHub issue."""
+
+    issue_id: str
+    url: str
+    state: str  # "open" | "closed"
+    results: list[ResultRecord] = field(default_factory=list)
+    # Assertion names that don't map to a known handler — a hard configuration error.
+    unknown_assertions: list[str] = field(default_factory=list)
+
+    @property
+    def is_open(self) -> bool:
+        return self.state == "open"
+
+    @property
+    def failed_results(self) -> list[ResultRecord]:
+        return [r for r in self.results if r.status == TestStatus.Failed]
+
+    @property
+    def has_failures(self) -> bool:
+        return any(r.status == TestStatus.Failed for r in self.results)
+
+    @property
+    def all_passed(self) -> bool:
+        return bool(self.results) and not self.has_failures
+
+    @property
+    def closeable(self) -> bool:
+        """Open issue whose tests now all pass (and parse cleanly) — a candidate to close."""
+        return self.is_open and self.all_passed and not self.unknown_assertions
+
+    @property
+    def reopened(self) -> bool:
+        """Closed issue whose tests now fail — a candidate to reopen."""
+        return (not self.is_open) and self.has_failures
+
+
+def run_assertions(
+    issue_id: str,
+    url: str,
+    state: str,
+    assertions: Iterable[Assertion],
+    *,
+    nodenorm: CachedNodeNorm,
+    nameres: CachedNameRes,
+    pass_if_found_in_top: int = 5,
+) -> IssueReport:
+    """Run every assertion for a single issue and collect the results into an IssueReport.
+
+    Unknown assertion names are recorded in ``unknown_assertions`` rather than raised, so the
+    caller can decide how to surface them (the pytest suite fails hard; Explorer can flag them).
+    """
+    report = IssueReport(issue_id=issue_id, url=url, state=state)
+    for assertion in assertions:
+        if not assertion.is_known:
+            report.unknown_assertions.append(assertion.assertion)
+            continue
+        for param_set, result in assertion.run(nodenorm, nameres, pass_if_found_in_top):
+            report.results.append(
+                ResultRecord(assertion=assertion.assertion, param_set=param_set, result=result)
+            )
+    return report
+
+
+def run_issue_tests(
+    test_cases: GitHubIssuesTestCases,
+    repos=None,
+    *,
+    nodenorm: CachedNodeNorm,
+    nameres: CachedNameRes,
+    pass_if_found_in_top: int = 5,
+    issue_ids: list[str] | None = None,
+) -> list[IssueReport]:
+    """Fetch issue tests and run them, returning one IssueReport per issue.
+
+    :param test_cases: a configured ``GitHubIssuesTestCases`` (provides the GitHub auth + repos).
+    :param repos: optional repo subset; defaults to the repos the test_cases was built with.
+    :param issue_ids: optional explicit issue IDs ('org/repo#N', 'repo#N', or 'N'). When given,
+                      only those issues are tested (and the GitHub search index lag is bypassed);
+                      otherwise every issue containing BabelTest syntax is discovered and run.
+    """
+    if issue_ids:
+        issues = test_cases.get_issues_by_ids(issue_ids)
+    else:
+        issues = test_cases.get_issues_with_tests(repos)
+
+    reports = []
+    for issue in issues:
+        issue_id = f"{issue.repository.full_name}#{issue.number}"
+        assertions = test_cases.get_test_issues_from_issue(issue)
+        reports.append(
+            run_assertions(
+                issue_id,
+                issue.html_url,
+                issue.state,
+                assertions,
+                nodenorm=nodenorm,
+                nameres=nameres,
+                pass_if_found_in_top=pass_if_found_in_top,
+            )
+        )
+    return reports