From d16e733d8355dd909de33a0aa61ec1b1af0a071b Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:21:53 -0700 Subject: [PATCH 1/2] ontologies: run metamodel-edge drop AFTER the biolink:->CURIE remap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up bug fix to #602. The metamodel-edge drop was placed in `_add_kgx_metadata_to_edges`, which runs early in `post_process` — before `_normalize_schema` remaps KGX's meta-predicates from their `biolink:` serialization (`biolink:subPropertyOf` / `biolink:inverseOf` / `biolink:type`) to `rdfs:subPropertyOf` / `owl:inverseOf` / `rdf:type`. So the CURIE filter matched nothing and the edges survived to the output (confirmed: a full `kg transform -s ontologies` still emitted 252 envo / 75 metpo metamodel edges, and the output edge files carried the knowledge_level column proving the early method ran but dropped nothing). Move the drop into `_normalize_schema`, immediately after the biolink:->rdfs/owl/rdf remap, where predicates are in their final CURIE form and `_drop_metamodel_edges` actually matches. `_add_kgx_metadata_to_edges` reverts to metadata-only (keeps the low_memory=False read). Still single read/write (the drop rides `_normalize_schema`'s existing edge read/write). Test retargeted to `_normalize_schema`, feeding `biolink:`-form predicates to verify the remap-then-drop sequence. Verified end-to-end on real data: the drop removes 2,179 metamodel edges across the 10 ontology edge outputs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ontologies/ontologies_transform.py | 35 +++++++------ tests/test_ontologies_metamodel_filter.py | 52 +++++++++++-------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/kg_microbe/transform_utils/ontologies/ontologies_transform.py b/kg_microbe/transform_utils/ontologies/ontologies_transform.py index af1797fb..10717f3f 100644 --- a/kg_microbe/transform_utils/ontologies/ontologies_transform.py +++ b/kg_microbe/transform_utils/ontologies/ontologies_transform.py @@ -365,24 +365,14 @@ def _drop_metamodel_edges(self, df: pd.DataFrame) -> tuple: def _add_kgx_metadata_to_edges(self, edges_file_path: Path): """ - Drop metamodel edges and add knowledge_level/agent_type to an edge file. + Add knowledge_level and agent_type columns to ontology edge files. - Single read/filter/write pass over the (potentially very large — chebi, - ncbitaxon) edge file: removes ontology metamodel axiom edges - (:meth:`_drop_metamodel_edges`), then stamps every remaining edge with - knowledge_assertion + manual_agent since ontologies are manually curated - by domain expert curators (GO, ChEBI, ENVO, …). + All ontology edges use knowledge_assertion + manual_agent since ontologies + are manually curated by domain expert curators (GO, ChEBI, ENVO, etc.). + The relationships represent definitional assertions made by experts. """ df = pd.read_csv(edges_file_path, sep="\t", low_memory=False) - # Drop ontology metamodel axiom edges (non-biolink property/typing rows). - df, dropped = self._drop_metamodel_edges(df) - if dropped: - print( - f" Dropped {dropped} ontology metamodel edge(s) " - f"(rdfs:subPropertyOf/owl:inverseOf/rdf:type) from {edges_file_path.name}" - ) - # Add columns if they don't exist if KNOWLEDGE_LEVEL_COLUMN not in df.columns: df[KNOWLEDGE_LEVEL_COLUMN] = KNOWLEDGE_ASSERTION @@ -517,8 +507,9 @@ def post_process(self, name: str): nodes_file = self.output_dir / f"{name}_nodes.tsv" edges_file = self.output_dir / f"{name}_edges.tsv" - # Drop ontology metamodel axiom edges (rdfs:subPropertyOf / owl:inverseOf - # / rdf:type) and stamp knowledge_level/agent_type — single read/write. + # Add knowledge_level/agent_type columns. (Metamodel-axiom edges are + # dropped later in _normalize_schema, after KGX's biolink:->rdfs/owl/rdf + # predicate remap, so the CURIE filter actually matches.) self._add_kgx_metadata_to_edges(edges_file) # Fix node categories: specialized handlers for go/chebi/uberon/ncbitaxon, @@ -930,7 +921,19 @@ def _normalize_schema(self, nodes_file: Path, edges_file: Path) -> None: # prefix too (belt-and-braces — same-row consistency). df["relation"] = df["relation"].replace(owl_meta_predicate_map) + # Drop ontology metamodel-axiom edges now that predicates are in + # their final CURIE form (post the biolink:->rdfs/owl/rdf remap + # above). These property-level / typing statements are not biolink + # entity relationships. Nodes are left untouched. + df, dropped_metamodel = self._drop_metamodel_edges(df) + df.to_csv(edges_file, sep="\t", index=False) + if dropped_metamodel: + print( + f" [_normalize_schema] {edges_file.name}: dropped " + f"{dropped_metamodel} metamodel edge(s) " + "(rdfs:subPropertyOf/owl:inverseOf/rdf:type)" + ) if dropped_edge_cols or added_edge_cols or renamed: rename_note = " rename(knowledge_source→primary_knowledge_source)" if renamed else "" print( diff --git a/tests/test_ontologies_metamodel_filter.py b/tests/test_ontologies_metamodel_filter.py index 8b4e4ab0..7c163a8a 100644 --- a/tests/test_ontologies_metamodel_filter.py +++ b/tests/test_ontologies_metamodel_filter.py @@ -60,24 +60,34 @@ def test_drop_helper_missing_predicate_column(self): self.assertEqual(dropped, 0) self.assertEqual(len(out), 1) - # ---- Integration through the single read/write path ---- - - def test_metadata_pass_drops_and_preserves_columns(self): - """_add_kgx_metadata_to_edges drops metamodel rows, adds metadata, keeps other cols.""" - path = self._write_edges(_ROWS) - self.transform._add_kgx_metadata_to_edges(path) - df = pd.read_csv(path, sep="\t") - # metamodel predicates gone - self.assertEqual(set(df["predicate"]), {"biolink:subclass_of", "biolink:related_to"}) - self.assertEqual(len(df), 2) - # kgx metadata columns added - self.assertIn("knowledge_level", df.columns) - self.assertIn("agent_type", df.columns) - self.assertTrue((df["knowledge_level"] == "knowledge_assertion").all()) - self.assertTrue((df["agent_type"] == "manual_agent").all()) - # non-predicate columns survive intact on the kept rows (keyed by predicate, - # since both surviving rows share the same subject) - by_pred = df.set_index("predicate") - self.assertEqual(by_pred.loc["biolink:subclass_of", "relation"], "rdfs:subClassOf") - self.assertEqual(by_pred.loc["biolink:related_to", "relation"], "RO:0002131") - self.assertEqual(by_pred.loc["biolink:subclass_of", "primary_knowledge_source"], "envo.json") + # ---- Integration through _normalize_schema (post biolink:->CURIE remap) ---- + + def test_normalize_schema_remaps_then_drops_metamodel(self): + """ + Remap KGX's biolink: meta-predicates in _normalize_schema, then drop them. + + KGX emits these axioms as ``biolink:subPropertyOf`` / ``biolink:inverseOf`` + / ``biolink:type``; _normalize_schema remaps to rdfs/owl/rdf CURIEs and + only then can the drop match — so the drop must run there, not earlier. + """ + header = ["subject", "predicate", "object", "relation", "primary_knowledge_source"] + self.transform.edge_header = header + rows = [ + # Entity edge — kept (biolink:subclass_of is not a meta-predicate). + ["ENVO:00000015", "biolink:subclass_of", "ENVO:00000012", "rdfs:subClassOf", "envo.json"], + # Metamodel edges in KGX's biolink: serialization — remapped then dropped. + ["METPO:2000002", "biolink:subPropertyOf", "METPO:2000001", "subPropertyOf", "metpo.json"], + ["RO:0002327", "biolink:inverseOf", "RO:0002333", "inverseOf", "envo.json"], + ["WD_Entity:Q715269", "biolink:type", "ENVO:00000015", "type", "envo.json"], + ] + tmp = Path(tempfile.mkdtemp()) + edges = tmp / "x_edges.tsv" + pd.DataFrame(rows, columns=header).to_csv(edges, sep="\t", index=False) + # nodes_file absent → node branch is skipped by its is_file() guard. + self.transform._normalize_schema(tmp / "x_nodes.tsv", edges) + df = pd.read_csv(edges, sep="\t") + self.assertEqual(set(df["predicate"]), {"biolink:subclass_of"}) + self.assertEqual(len(df), 1) + # The surviving entity edge keeps its other columns intact. + self.assertEqual(df.iloc[0]["relation"], "rdfs:subClassOf") + self.assertEqual(df.iloc[0]["primary_knowledge_source"], "envo.json") From 67581cfa2f1f905f36433d4c779291f74e914aba Mon Sep 17 00:00:00 2001 From: "marcin p. joachimiak" <4625870+realmarcin@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:26:38 -0700 Subject: [PATCH 2/2] ontologies: belt-and-braces bare-token predicate normalize + test coverage (review) Addresses PR #603 review (all LOW): - Also normalize bare local-name predicates (subPropertyOf/inverseOf/type) to CURIEs in _normalize_schema before the metamodel drop, so a bare token landing in `predicate` (not just `relation`) is still caught. - Restore end-to-end coverage of the reverted _add_kgx_metadata_to_edges: assert it adds knowledge_level/agent_type and drops nothing (metamodel rows are still biolink:-namespaced at that stage). - Assert the post-drop column set/order equals edge_header, and cover the bare-token predicate path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ontologies/ontologies_transform.py | 4 ++++ tests/test_ontologies_metamodel_filter.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/kg_microbe/transform_utils/ontologies/ontologies_transform.py b/kg_microbe/transform_utils/ontologies/ontologies_transform.py index 10717f3f..0a87bdfc 100644 --- a/kg_microbe/transform_utils/ontologies/ontologies_transform.py +++ b/kg_microbe/transform_utils/ontologies/ontologies_transform.py @@ -915,6 +915,10 @@ def _normalize_schema(self, nodes_file: Path, edges_file: Path) -> None: } if "predicate" in df.columns: df["predicate"] = df["predicate"].replace(owl_meta_predicate_map) + # Belt-and-braces: obograph can emit bare local names (e.g. + # ``subPropertyOf``) rather than the biolink:-prefixed form, so + # normalise those too before the metamodel drop below. + df["predicate"] = df["predicate"].replace(owl_meta_relation_map) if "relation" in df.columns: df["relation"] = df["relation"].replace(owl_meta_relation_map) # Also catch cases where `relation` mistakenly got the biolink diff --git a/tests/test_ontologies_metamodel_filter.py b/tests/test_ontologies_metamodel_filter.py index 7c163a8a..c37ed42a 100644 --- a/tests/test_ontologies_metamodel_filter.py +++ b/tests/test_ontologies_metamodel_filter.py @@ -79,6 +79,8 @@ def test_normalize_schema_remaps_then_drops_metamodel(self): ["METPO:2000002", "biolink:subPropertyOf", "METPO:2000001", "subPropertyOf", "metpo.json"], ["RO:0002327", "biolink:inverseOf", "RO:0002333", "inverseOf", "envo.json"], ["WD_Entity:Q715269", "biolink:type", "ENVO:00000015", "type", "envo.json"], + # Bare local-name predicate (belt-and-braces) — normalized then dropped. + ["GO:1", "subPropertyOf", "GO:2", "subPropertyOf", "go.json"], ] tmp = Path(tempfile.mkdtemp()) edges = tmp / "x_edges.tsv" @@ -88,6 +90,25 @@ def test_normalize_schema_remaps_then_drops_metamodel(self): df = pd.read_csv(edges, sep="\t") self.assertEqual(set(df["predicate"]), {"biolink:subclass_of"}) self.assertEqual(len(df), 1) + # Column set/order preserved to edge_header. + self.assertEqual(list(df.columns), header) # The surviving entity edge keeps its other columns intact. self.assertEqual(df.iloc[0]["relation"], "rdfs:subClassOf") self.assertEqual(df.iloc[0]["primary_knowledge_source"], "envo.json") + + def test_add_kgx_metadata_only_adds_columns_no_drop(self): + """The reverted _add_kgx_metadata_to_edges adds metadata and drops nothing.""" + # It runs before the biolink:->CURIE remap, so metamodel rows are still + # biolink:-namespaced here and must NOT be dropped by this method. + rows = [ + ["ENVO:1", "biolink:subclass_of", "ENVO:2", "rdfs:subClassOf", "envo.json"], + ["METPO:2", "biolink:subPropertyOf", "METPO:1", "subPropertyOf", "metpo.json"], + ] + path = self._write_edges(rows) + self.transform._add_kgx_metadata_to_edges(path) + df = pd.read_csv(path, sep="\t") + # No rows dropped by this method. + self.assertEqual(len(df), 2) + # Metadata columns added. + self.assertTrue((df["knowledge_level"] == "knowledge_assertion").all()) + self.assertTrue((df["agent_type"] == "manual_agent").all())