feat: GO single source of truth (fix 2, #604)

download.yaml

@@ -51,14 +51,10 @@
   local_name: chebi.owl.gz
 
 #
-# GO
-#
--
-  url: http://purl.obolibrary.org/obo/go.json
-  local_name: go.json
-
-#
-# GO
+# GO — single source of truth (fix 2, #604): download only go.owl; the
+# transform derives go.json from it (ROBOT owl→json) and go.db is built from
+# it (semsql), so the aspect map and the transform output can never drift to
+# different GO releases. Do NOT re-add a standalone go.json download.
 #
 -
   url: http://purl.obolibrary.org/obo/go.owl

kg_microbe/transform_utils/ontologies/ontologies_transform.py

@@ -48,7 +48,7 @@
     UNIPROT_PREFIX,
     XREF_COLUMN,
 )
-from kg_microbe.utils.ontology_utils import replace_category_ontology
+from kg_microbe.utils.ontology_utils import _derived_json_is_stale, replace_category_ontology
 from kg_microbe.utils.pandas_utils import (
     drop_duplicates,
     establish_transitive_relationship,
@@ -73,7 +73,10 @@
     "ncbitaxon": "ncbitaxon.owl.gz",
     "chebi": "chebi.owl.gz",
     "envo": "envo.json",
-    "go": "go.json",
+    # GO is single-source (fix 2, #604): the transform derives go.json from
+    # go.owl (ROBOT owl→json), the same OWL that go.db is built from — so the
+    # aspect map (go.db) and the transform output (go.json) share one release.
+    "go": "go.owl",
     ## "rhea": "rhea.json.gz", # Redundant to RheaMappingsTransform
     "ec": "ec.json",
     "upa": "upa.owl",
@@ -202,6 +205,17 @@ def parse(self, name: str, data_file: Optional[Path], source: str) -> None:
                 data_file = json_path
             elif data_file.suffix == ".owl":
                 json_path = str(data_file).replace(".owl", ".json")
+                # Regenerate the derived JSON when it's missing OR its OBO
+                # release no longer matches the source OWL. For single-source
+                # ontologies (fix 2 #604: GO derives go.json from go.owl) this
+                # keeps the transform output locked to the same release the OWL
+                # (and go.db, built from it) carry, so a refreshed go.owl can't
+                # leave a stale go.json behind. convert_to_json is a no-op when
+                # the target already exists, so a stale JSON must be removed
+                # first (mirrors _ensure_go_db's rebuild) — otherwise the
+                # reconversion silently keeps the old release.
+                if _derived_json_is_stale(data_file, Path(json_path)):
+                    Path(json_path).unlink()
                 if not Path(json_path).is_file():
                     convert_to_json(str(self.input_base_dir), name)
                 data_file = json_path

kg_microbe/utils/ontology_utils.py

@@ -66,6 +66,23 @@ def _obo_release_from_head(path: Path, nbytes: int = 2_000_000) -> Optional[str]
     return m.group(1) if m else None
 
 
+def _derived_json_is_stale(owl_path: Path, json_path: Path) -> bool:
+    """
+    Return True when a derived OBO-JSON's release no longer matches its source OWL.
+
+    Used by the ontologies transform to decide whether to re-run the ROBOT
+    ``owl→json`` conversion: a single-source ontology (e.g. GO, whose go.json
+    is derived from go.owl rather than downloaded) must regenerate the JSON
+    when the OWL is refreshed to a new release. Conservative — only reports
+    stale when *both* release stamps are readable and differ; an unstamped or
+    unreadable pair yields False so unstamped ``.owl`` inputs keep the prior
+    "convert only if missing" behavior.
+    """
+    owl_release = _obo_release_from_head(owl_path)
+    json_release = _obo_release_from_head(json_path)
+    return bool(owl_release and json_release and owl_release != json_release)
+
+
 def _version_check_strict(env_var: str, strict: Optional[bool], default_strict: bool = True) -> bool:
     """
     Resolve a version-gate's strictness: explicit arg wins, else the env var.
@@ -87,15 +104,18 @@ def _version_check_strict(env_var: str, strict: Optional[bool], default_strict:
 
 def assert_go_version_alignment(strict: Optional[bool] = None) -> None:
     """
-    Guard that GO's aspect-map source (go.owl→go.db) matches the transform's (go.json).
-
-    The GO transform TSV is built from ``go.json`` while the MF/BP/CC aspect map
-    is read from ``go.db`` (built from ``go.owl``). If a ``kg download`` straddles
-    a GO release boundary the two diverge and MF/CC terms silently fall through
-    to the ``biological_process`` default. Compare the two releases and, on
-    mismatch, raise (``strict``) or warn loudly. No-op when either release stamp
-    can't be read (e.g. a source is absent), so a missed versionIRI never
-    false-alarms — only two readable-but-different stamps trip the gate.
+    Guard that GO's derived ``go.json`` matches its single source ``go.owl``.
+
+    Since fix 2 (#604) GO is single-source: ``go.owl`` is the only download, and
+    ``go.json`` (transform output) and ``go.db`` (MF/BP/CC aspect map) are both
+    derived from it — the transform regenerates a stale go.json and
+    ``_ensure_go_db`` rebuilds a drifted go.db. This gate is the belt-and-braces
+    check that the derived go.json actually tracks go.owl: a leftover pre-fix-2
+    go.json, or a conversion that didn't re-run, would make MF/CC terms silently
+    fall through to the ``biological_process`` default. Compare the two releases
+    and, on mismatch, raise (``strict``) or warn loudly. No-op when either
+    release stamp can't be read (e.g. a source is absent), so a missed versionIRI
+    never false-alarms — only two readable-but-different stamps trip the gate.
 
     ``strict`` defaults to fail-loud (raise). Since the verdict rests on a
     release-stamp heuristic, ``KG_GO_VERSION_CHECK=warn`` downgrades to a
@@ -149,6 +169,37 @@ def _ncbitaxon_db_release(db_path: str) -> Optional[str]:
     return m.group(1) if m else None
 
 
+def _go_db_release(db_path: str) -> Optional[str]:
+    """
+    Return the GO release (YYYY-MM-DD) recorded in a SemSQL ``go.db``.
+
+    SemSQL stamps the ontology node directly: ``obo:go.owl | owl:versionInfo |
+    2026-05-19``. Match the GO ontology subject across the encodings different
+    semsql/rdftab builds emit — the CURIE ``obo:go.owl`` / ``obo:go`` or the
+    full IRI ``.../obo/go.owl`` — while still excluding GO *term* subjects
+    (``GO:...``) that might carry a version-shaped value. Returns None on any
+    read error / missing stamp; a None here makes ``_ensure_go_db`` conservatively
+    reuse the existing db rather than force a spurious rebuild.
+    """
+    try:
+        conn = sqlite3.connect(db_path)
+        try:
+            row = conn.execute(
+                "SELECT value FROM statements WHERE predicate = 'owl:versionInfo' "
+                "AND value IS NOT NULL AND ("
+                "subject IN ('obo:go.owl', 'obo:go') OR subject LIKE '%/go.owl'"
+                ") LIMIT 1"
+            ).fetchone()
+        finally:
+            conn.close()
+    except sqlite3.Error:
+        return None
+    if not row or not row[0]:
+        return None
+    m = re.search(r"(\d{4}-\d{2}-\d{2})", str(row[0]))
+    return m.group(1) if m else None
+
+
 def assert_ncbitaxon_version_alignment(db_path: str, strict: Optional[bool] = None) -> None:
     """
     Guard that the NCBITaxon lookup DB matches the transform's OWL release.
@@ -203,7 +254,20 @@ def _ensure_go_db(go_db_path: str) -> bool:
     from kg_microbe.transform_utils.constants import GO_SOURCE
 
     if os.path.exists(go_db_path) and os.path.getsize(go_db_path) >= _GO_DB_MIN_SIZE:
-        return True
+        # An existing, non-stub go.db is reused — unless it has drifted from the
+        # source OWL's release (single-source invariant, fix 2 #604): a refreshed
+        # go.owl must rebuild go.db, else the aspect map lags the transform output
+        # and MF/CC terms silently fall through to BiologicalProcess. Only rebuild
+        # when both release stamps are readable and differ (an unreadable stamp
+        # never forces a costly spurious rebuild).
+        owl_release = _obo_release_from_head(Path(GO_SOURCE)) if GO_SOURCE else None
+        db_release = _go_db_release(go_db_path)
+        if not (owl_release and db_release and owl_release != db_release):
+            return True
+        print(
+            f"Rebuilding {go_db_path}: release {db_release} drifted from "
+            f"go.owl {owl_release} (single-source realign)..."
+        )
     if not (GO_SOURCE and Path(GO_SOURCE).exists()):
         print(f"Warning: cannot build {go_db_path} — GO OWL source {GO_SOURCE} is missing")
         return False

tests/test_go_version_alignment.py

@@ -1,5 +1,6 @@
 """Tests for the GO version-alignment gate and go.db build helper (#604)."""
 
+import sqlite3
 from pathlib import Path
 
 import pytest
@@ -17,6 +18,22 @@ def _write_go_pair(tmp_path: Path, owl_date: str, json_date: str) -> Path:
     return tmp_path / "go.owl"
 
 
+def _make_go_db(tmp_path: Path, release, name: str = "go.db") -> str:
+    """Write a minimal SemSQL-shaped go.db stamping obo:go.owl owl:versionInfo."""
+    path = str(tmp_path / name)
+    conn = sqlite3.connect(path)
+    conn.execute("CREATE TABLE statements (subject TEXT, predicate TEXT, object TEXT, value TEXT)")
+    if release is not None:
+        conn.execute(
+            "INSERT INTO statements (subject, predicate, object, value) "
+            "VALUES ('obo:go.owl', 'owl:versionInfo', NULL, ?)",
+            (release,),
+        )
+    conn.commit()
+    conn.close()
+    return path
+
+
 def test_release_parsed_from_owl_versioniri(tmp_path):
     """The YYYY-MM-DD release is read from an OWL versionIRI."""
     (tmp_path / "go.owl").write_text(_OWL.format(d="2026-05-19"), encoding="utf-8")
@@ -61,9 +78,7 @@ def test_mismatch_warns_when_not_strict(tmp_path, monkeypatch, capsys):
 
 def test_missing_source_is_a_noop(tmp_path, monkeypatch):
     """When a GO source can't be read, the gate is a no-op (can't judge)."""
-    monkeypatch.setattr(
-        "kg_microbe.transform_utils.constants.GO_SOURCE", tmp_path / "absent.owl"
-    )
+    monkeypatch.setattr("kg_microbe.transform_utils.constants.GO_SOURCE", tmp_path / "absent.owl")
     ou.assert_go_version_alignment(strict=True)  # must not raise
 
 
@@ -117,8 +132,151 @@ def test_ensure_go_db_skips_build_when_valid(tmp_path, monkeypatch):
 
 def test_ensure_go_db_cannot_build_without_owl(tmp_path, monkeypatch, capsys):
     """A missing/empty go.db with no OWL source degrades gracefully (warn, False)."""
-    monkeypatch.setattr(
-        "kg_microbe.transform_utils.constants.GO_SOURCE", tmp_path / "absent.owl"
-    )
+    monkeypatch.setattr("kg_microbe.transform_utils.constants.GO_SOURCE", tmp_path / "absent.owl")
     assert ou._ensure_go_db(str(tmp_path / "go.db")) is False
     assert "missing" in capsys.readouterr().out
+
+
+# --- fix 2 (#604): GO single source of truth -------------------------------
+
+
+def test_derived_json_stale_when_releases_differ(tmp_path):
+    """A go.json whose release differs from go.owl is reported stale."""
+    owl = _write_go_pair(tmp_path, "2026-05-19", "2026-04-01")
+    assert ou._derived_json_is_stale(owl, tmp_path / "go.json") is True
+
+
+def test_derived_json_not_stale_when_aligned(tmp_path):
+    """Matching releases are not stale (no needless reconversion)."""
+    owl = _write_go_pair(tmp_path, "2026-05-19", "2026-05-19")
+    assert ou._derived_json_is_stale(owl, tmp_path / "go.json") is False
+
+
+def test_derived_json_unstamped_is_not_stale(tmp_path):
+    """An unreadable/unstamped pair yields False (preserves convert-if-missing)."""
+    (tmp_path / "go.owl").write_text("no version here", encoding="utf-8")
+    assert ou._derived_json_is_stale(tmp_path / "go.owl", tmp_path / "missing.json") is False
+
+
+def test_go_db_release_read_from_versioninfo(tmp_path):
+    """_go_db_release reads owl:versionInfo off the obo:go.owl subject."""
+    assert ou._go_db_release(_make_go_db(tmp_path, "2026-05-19")) == "2026-05-19"
+
+
+def test_go_db_release_ignores_decoy_subject(tmp_path):
+    """A version-shaped value on a non-ontology subject is ignored."""
+    path = str(tmp_path / "go.db")
+    conn = sqlite3.connect(path)
+    conn.execute("CREATE TABLE statements (subject TEXT, predicate TEXT, object TEXT, value TEXT)")
+    conn.execute("INSERT INTO statements VALUES ('GO:0008150', 'owl:versionInfo', NULL, '1999-01-01')")
+    conn.execute("INSERT INTO statements VALUES ('obo:go.owl', 'owl:versionInfo', NULL, '2026-05-19')")
+    conn.commit()
+    conn.close()
+    assert ou._go_db_release(path) == "2026-05-19"
+
+
+def test_go_db_release_none_when_absent_or_corrupt(tmp_path):
+    """Missing stamp, missing file, and a non-sqlite file all yield None."""
+    assert ou._go_db_release(_make_go_db(tmp_path, None)) is None
+    assert ou._go_db_release(str(tmp_path / "nope.db")) is None
+    corrupt = tmp_path / "corrupt.db"
+    corrupt.write_text("not sqlite", encoding="utf-8")
+    assert ou._go_db_release(str(corrupt)) is None
+
+
+def test_ensure_go_db_keeps_aligned_db(tmp_path, monkeypatch):
+    """A valid, release-aligned go.db is reused (no rebuild)."""
+    monkeypatch.setattr(ou, "_GO_DB_MIN_SIZE", 8)
+    monkeypatch.setattr(
+        "kg_microbe.transform_utils.constants.GO_SOURCE",
+        _write_go_pair(tmp_path, "2026-05-19", "2026-05-19"),
+    )
+    db = _make_go_db(tmp_path, "2026-05-19")
+    assert ou._ensure_go_db(db) is True
+
+
+def test_ensure_go_db_rebuilds_on_release_drift(tmp_path, monkeypatch, capsys):
+    """
+    A valid-size but release-drifted go.db is NOT short-circuited as usable.
+
+    With ``semsql`` forced off, the drift path falls through to a build attempt
+    and returns False — proving the stale db was rejected rather than reused.
+    """
+    monkeypatch.setattr(ou, "_GO_DB_MIN_SIZE", 8)
+    monkeypatch.setattr(
+        "kg_microbe.transform_utils.constants.GO_SOURCE",
+        _write_go_pair(tmp_path, "2026-05-19", "2026-05-19"),
+    )
+    monkeypatch.setattr(ou.shutil, "which", lambda _cmd: None)  # no semsql → build can't run
+    db = _make_go_db(tmp_path, "2026-01-01")  # drifted from go.owl (2026-05-19)
+    assert ou._ensure_go_db(db) is False
+    assert "drifted" in capsys.readouterr().out
+
+
+def test_go_db_release_reads_full_iri_subject(tmp_path):
+    """The db-release read also matches the full-IRI ontology subject encoding."""
+    path = str(tmp_path / "go.db")
+    conn = sqlite3.connect(path)
+    conn.execute("CREATE TABLE statements (subject TEXT, predicate TEXT, object TEXT, value TEXT)")
+    conn.execute(
+        "INSERT INTO statements VALUES ('http://purl.obolibrary.org/obo/go.owl', 'owl:versionInfo', NULL, '2026-05-19')"
+    )
+    conn.commit()
+    conn.close()
+    assert ou._go_db_release(path) == "2026-05-19"
+
+
+def _stub_go_transform(tmp_path, monkeypatch):
+    """
+    Build an OntologiesTransform whose post-conversion steps are stubbed out.
+
+    Returns (transform, calls) where calls['convert'] counts convert_to_json
+    invocations. convert_to_json is faked to write a fresh 2026-05-19 go.json
+    (what ROBOT would produce from the new go.owl).
+    """
+    from kg_microbe.transform_utils.ontologies import ontologies_transform as tf_mod
+
+    calls = {"convert": 0}
+
+    def fake_convert(path, ont):
+        calls["convert"] += 1
+        (Path(path) / f"{ont}.json").write_text(_JSON.format(d="2026-05-19"), encoding="utf-8")
+
+    monkeypatch.setattr(tf_mod, "convert_to_json", fake_convert)
+    monkeypatch.setattr(tf_mod, "transform", lambda **k: None)
+    t = tf_mod.OntologiesTransform.__new__(tf_mod.OntologiesTransform)
+    t.input_base_dir = tmp_path
+    t.output_dir = tmp_path
+    monkeypatch.setattr(t, "_sanitize_obograph_synonyms", lambda p: None, raising=False)
+    monkeypatch.setattr(t, "_drop_deprecated_terms", lambda p: None, raising=False)
+    monkeypatch.setattr(t, "post_process", lambda n: None, raising=False)
+    return t, calls
+
+
+def test_stale_go_json_is_removed_and_reconverted(tmp_path, monkeypatch):
+    """
+    A stale go.json is unlinked then regenerated — convert_to_json won't overwrite it.
+
+    Regression for the review finding: without the unlink, convert_to_json is a
+    no-op on an existing (stale) file, so a refreshed go.owl would leave the old
+    go.json in place instead of self-healing.
+    """
+    (tmp_path / "go.owl").write_text(_OWL.format(d="2026-05-19"), encoding="utf-8")
+    (tmp_path / "go.json").write_text(_JSON.format(d="2026-04-01"), encoding="utf-8")  # stale
+    t, calls = _stub_go_transform(tmp_path, monkeypatch)
+
+    t.parse("go", tmp_path / "go.owl", "go")
+
+    assert calls["convert"] == 1
+    assert ou._obo_release_from_head(tmp_path / "go.json") == "2026-05-19"  # regenerated
+
+
+def test_aligned_go_json_is_not_reconverted(tmp_path, monkeypatch):
+    """An already-aligned go.json is reused (no needless ROBOT reconversion)."""
+    (tmp_path / "go.owl").write_text(_OWL.format(d="2026-05-19"), encoding="utf-8")
+    (tmp_path / "go.json").write_text(_JSON.format(d="2026-05-19"), encoding="utf-8")  # aligned
+    t, calls = _stub_go_transform(tmp_path, monkeypatch)
+
+    t.parse("go", tmp_path / "go.owl", "go")
+
+    assert calls["convert"] == 0