diff --git a/.gitignore b/.gitignore index ffd74aa5..cd89b345 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ /.idea *.iml validation_report/ + +__pycache__/ +*.pyc diff --git a/buildScripts/dataset_header/README.md b/buildScripts/dataset_header/README.md new file mode 100644 index 00000000..427bd379 --- /dev/null +++ b/buildScripts/dataset_header/README.md @@ -0,0 +1,61 @@ +# Dataset-header validation & fixing + +A reusable, branch/version-agnostic tool that checks (and fixes) the DCAT +dataset headers of ReliCapGrid instance files against the rules from Svein's +PR #322 review. The **validator** and the **fixer** share one rule set +(`rules.py`) so they can't drift apart. + +## Two tiers + +- **Tier A — structural / fixed-value.** Deterministic and auto-fixable: + required properties, fixed values (`accessRights`, `type`, `license`, + `rights`, `rightsHolder`), `xml:lang` tags, forbidden properties, + disallowed namespaces (`eumd`/`euvoc`), UUID and UTC-`Z` datetime formats, + and `TSO-`/`RCC-` publisher naming. +- **Tier B — reference-data membership.** `conformsTo` / `isVersionOf` / + `spatial` / `wasGeneratedBy` / publisher-party validated against the + schemes in `Instance/referenceData`. **Report-only** — fixing is blocked + until those schemes are extended (the parked PR #322 items). New scheme + entries become valid automatically, no code change. + +## Usage + +```bash +# Validate (report-only; exit 0). Add --strict to fail on Tier-A. +uv run python buildScripts/validate_dataset_header.py --scope networkcode \ + --details --json validation_report/dataset_header.json + +# Auto-fix the deterministic Tier-A subset (dry-run without --apply). +uv run python buildScripts/fix_dataset_header.py --scope networkcode +uv run python buildScripts/fix_dataset_header.py --scope networkcode --apply +``` + +`--scope` is `networkcode` (default) or `all`. The fixer is idempotent and +formatting-preserving (text edits, not graph re-serialization), so diffs stay +minimal. It never invents Tier-B values, real dates, titles, or spatial refs. + +## CI / pytest + +`tests/test_dataset_header.py` runs the validator over the in-scope files. +It is **report-only** (`xfail`) until a branch's headers are clean; once you +have run the fixer and resolved the manual gaps, flip the fixable-Tier-A check +to a hard failure so regressions (e.g. re-introducing `eumd` or a forbidden +property) fail the build. Fast unit tests cover the rule and fixer logic. + +## Layout + +| File | Role | +|---|---| +| `rules.py` | single source of truth: rule catalog + rdflib checks | +| `schemes.py` | loads referenceData schemes into valid-value sets (Tier B) | +| `discovery.py` | enumerates in-scope header files | +| `fixer.py` | formatting-preserving Tier-A auto-fixes | +| `report.py` | text / markdown / JSON reporting | +| `../validate_dataset_header.py` | validator CLI | +| `../fix_dataset_header.py` | fixer CLI | + +## Roadmap + +- Extend scope beyond NetworkCode cimxml (Grid, GridSituation, boundary). +- Emit DCAT-AP-CIM SHACL shapes and run `pyshacl` as an independent + cross-check of the Python rules (`pyshacl` is already a dependency). diff --git a/buildScripts/dataset_header/__init__.py b/buildScripts/dataset_header/__init__.py new file mode 100644 index 00000000..926c2823 --- /dev/null +++ b/buildScripts/dataset_header/__init__.py @@ -0,0 +1,15 @@ +"""Dataset-header validation and fixing for ReliCapGrid instance files. + +Single source of truth for the DCAT dataset-header rules (Svein's review of +PR #322), consumed by two modes: + +- ``validate_dataset_header.py`` — semantic validation via rdflib (report). +- ``fix_dataset_header.py`` — formatting-preserving auto-fix of the + deterministic (Tier-A) subset. + +Rules are split into two tiers: + +- **Tier A** — structural / fixed-value: deterministic, auto-fixable now. +- **Tier B** — reference-data membership: checkable now, but fixing is blocked + until the reference schemes are extended (see the parked PR #322 items). +""" diff --git a/buildScripts/dataset_header/discovery.py b/buildScripts/dataset_header/discovery.py new file mode 100644 index 00000000..406022fb --- /dev/null +++ b/buildScripts/dataset_header/discovery.py @@ -0,0 +1,29 @@ +"""Discover instance files that carry a dcat:Dataset header. + +Phase-1 scope is the NetworkCode cimxml surface (the PR #322 files). The +existing ``create_cgm_zip.collect_cgm_files`` helper is too narrow (it skips +the ``Dataset_version_dependency/`` subfolder and the Jotunheim NetworkCode +IAM/SAR files), so we glob directly here. Scope can be widened later. +""" +from __future__ import annotations + +from pathlib import Path + +# Directories that are not instance datasets. +_EXCLUDE_PARTS = {"referenceData"} + + +def discover_headers(instance_dir: Path, scope: str = "networkcode") -> list[Path]: + """Return sorted instance XML files in scope that may contain a header.""" + if scope == "networkcode": + pattern = "*/NetworkCode/**/*.xml" + elif scope == "all": + pattern = "**/*.xml" + else: + raise ValueError(f"unknown scope: {scope!r}") + + files = [ + p for p in instance_dir.glob(pattern) + if not (_EXCLUDE_PARTS & set(p.parts)) + ] + return sorted(set(files)) diff --git a/buildScripts/dataset_header/fixer.py b/buildScripts/dataset_header/fixer.py new file mode 100644 index 00000000..7c609737 --- /dev/null +++ b/buildScripts/dataset_header/fixer.py @@ -0,0 +1,187 @@ +"""Formatting-preserving auto-fixer for the Tier-A header rules. + +Operates on the raw file text (not a re-serialized graph) so diffs stay +minimal and reviewable. Only the deterministic Tier-A subset is fixed; +Tier-B (reference-data) values are never invented here. + +Each ``fix_text`` call is idempotent: running it again yields no change. +""" +from __future__ import annotations + +import re + +# Namespace prefixes that must be dropped entirely. +_DISALLOWED_PREFIXES = ("eumd", "euvoc") + +# Canonical publisher party names (bare -> prefixed). Extend as needed. +KNOWN_PARTY = { + "Belgovia": "TSO-Belgovia", + "Espheim": "TSO-Espheim", + "Galia": "TSO-Galia", + "Svedala": "TSO-Svedala", + "Jotunheim": "RCC-Jotunheim", + "JOTUNHEIM": "RCC-Jotunheim", +} + +# Fixed-value properties: label -> full element line body (value-forcing). +_FIXED_URI = { + "dcterms:accessRights": "https://energy.referencedata.eu/Confidentiality/Public", + "dcterms:type": "https://energy.referencedata.eu/type/CIM-PowerSystemModel", + "dcterms:license": "https://creativecommons.org/licenses/by/4.0/", +} +_FIXED_LITERAL = { + "dcterms:rights": "Copyright", + "dcterms:rightsHolder": "ENTSO-E", +} + +_FORBIDDEN = ( + "dcatcim:alternativeVersionOf", "dcterms:accrualPeriodicity", + "dcterms:hasPart", "dcat:temporalResolution", + "dcatcim:preferredVersionOf", "dcat:inSeries", +) + +_DATASET_OPEN = re.compile(r" str: + m = re.search(r"\n([ \t]+)<", block) + return m.group(1) if m else " " + + +def fix_text(text: str) -> tuple[str, list[str]]: + """Return (new_text, list_of_applied_fix_ids).""" + applied: list[str] = [] + + # repair a doubled closing quote in an attribute value (well-formedness), + # e.g. rdf:resource=".../2.4""/> -> .../2.4"/> + dq = re.sub(r'(="[^"]*)""', r'\1"', text) + if dq != text: + text = dq + applied.append("fix-doubled-quote") + + open_m = _DATASET_OPEN.search(text) + close_m = re.search(r"\n[ \t]*", text) + if not open_m or not close_m: + return text, applied # not a header file + + prolog, ds_start = text[:open_m.start()], open_m.start() + block = text[ds_start:close_m.start()] # up to the newline before + tail = text[close_m.start():] # newline + indent + close tag + rest + + # --- drop disallowed prefixes (declaration AND any usage) -------------- + for pfx in _DISALLOWED_PREFIXES: + p = pfx_esc(pfx) + new = re.sub(rf'\n[ \t]*xmlns:{p}="[^"]*"', "", prolog) + new = re.sub(rf'[ \t]*xmlns:{p}="[^"]*"', "", new) # inline + if new != prolog: + prolog = new + applied.append(f"drop-ns:{pfx}") + # remove usage elements of the dropped prefix so it stays well-formed + for scope_name in ("block", "tail"): + src = block if scope_name == "block" else tail + cleaned = re.sub(rf'\n[ \t]*<{p}:[^>]*>.*?]*>', "", src) + cleaned = re.sub(rf'\n[ \t]*<{p}:[^>]*/>', "", cleaned) + if cleaned != src: + if scope_name == "block": + block = cleaned + else: + tail = cleaned + applied.append(f"remove-usage:{pfx}") + + # --- prolog: fix dcterms '#' ------------------------------------------- + if 'xmlns:dcterms="http://purl.org/dc/terms/#"' in prolog: + prolog = prolog.replace( + 'xmlns:dcterms="http://purl.org/dc/terms/#"', + 'xmlns:dcterms="http://purl.org/dc/terms/"') + applied.append("fix-dcterms-ns") + + # --- block: remove forbidden properties -------------------------------- + for label in _FORBIDDEN: + new = re.sub(rf'\n[ \t]*<{re.escape(label)}\b[^>]*/>', "", block) + new = re.sub(rf'\n[ \t]*<{re.escape(label)}\b[^>]*>.*?', + "", new) + if new != block: + block = new + applied.append(f"remove:{label}") + + # --- block: force fixed values ----------------------------------------- + for label, uri in _FIXED_URI.items(): + pat = re.compile(rf'(<{re.escape(label)} rdf:resource=")[^"]*(")') + if pat.search(block) and f'"{uri}"' not in block: + block = pat.sub(rf'\g<1>{uri}\g<2>', block) + applied.append(f"force-value:{label}") + for label, lit in _FIXED_LITERAL.items(): + pat = re.compile(rf'(<{re.escape(label)}>)[^<]*()') + m = pat.search(block) + if m and m.group(0) != f'<{label}>{lit}': + block = pat.sub(rf'\g<1>{lit}\g<2>', block) + applied.append(f"force-value:{label}") + + # --- block: add xml:lang to description / versionNotes ----------------- + for label in ("dcterms:description", "adms:versionNotes"): + pat = re.compile(rf'<{re.escape(label)}>') + if pat.search(block): + block = pat.sub(f'<{label} xml:lang="en">', block) + applied.append(f"add-lang:{label}") + + # --- block: normalize publisher party naming --------------------------- + def _pub(m): + name = m.group(2) + canon = KNOWN_PARTY.get(name) + return f'{m.group(1)}test/party/{canon}"' if canon else m.group(0) + + new = re.sub( + r'(rdf:resource="https://energy\.referencedata\.eu/)[Tt]est/[Pp]arty/([^"/]+)"', + _pub, block) + if new != block: + block = new + applied.append("normalize-publisher") + + # --- block: insert missing fixed-value props + versionNotes ------------ + indent = _indent_of(block) + inserts: list[str] = [] + for label, uri in _FIXED_URI.items(): + if f"<{label}" not in block: + inserts.append(f'{indent}<{label} rdf:resource="{uri}"/>') + for label, lit in _FIXED_LITERAL.items(): + if f"<{label}" not in block: + inserts.append(f'{indent}<{label}>{lit}') + if "Initial version.') + applied.append("insert:adms:versionNotes(placeholder)") + if "xmlns:adms" not in prolog: + prolog = _add_adms_ns(prolog) + applied.append("add-ns:adms") + if inserts: + block = block.rstrip("\n") + "\n" + "\n".join(inserts) + for ins in inserts: + label = re.search(r"<([\w:]+)", ins).group(1) + if not label == "adms:versionNotes": + applied.append(f"insert:{label}") + + text = prolog + block + tail + + # --- whole file: drop dcatcim ns if now unused ------------------------- + if "dcatcim:" not in re.sub(r'xmlns:dcatcim="[^"]*"', "", text): + new = re.sub(r'\n[ \t]*xmlns:dcatcim="[^"]*"', "", text) + new = re.sub(r'[ \t]*xmlns:dcatcim="[^"]*"', "", new) + if new != text: + text = new + applied.append("drop-ns:dcatcim") + + return text, applied + + +def _add_adms_ns(prolog: str) -> str: + """Insert an xmlns:adms declaration into the rdf:RDF open tag.""" + decl = 'xmlns:adms="http://www.w3.org/ns/adms#"' + m = re.search(r"( str: + return re.escape(pfx) diff --git a/buildScripts/dataset_header/report.py b/buildScripts/dataset_header/report.py new file mode 100644 index 00000000..816c39b1 --- /dev/null +++ b/buildScripts/dataset_header/report.py @@ -0,0 +1,58 @@ +"""Reporting helpers shared by the validator CLI and the pytest integration.""" +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +from .rules import Violation + + +def to_dicts(violations: list[Violation]) -> list[dict]: + return [ + { + "file": v.file, + "rule_id": v.rule_id, + "tier": v.tier, + "review_ref": v.review_ref, + "message": v.message, + "fixable": v.fixable, + "detail": v.detail, + } + for v in violations + ] + + +def summarize(violations: list[Violation], files_scanned: int, headers: int) -> str: + tier_a = [v for v in violations if v.tier == "A"] + tier_b = [v for v in violations if v.tier == "B"] + fixable = [v for v in tier_a if v.fixable] + lines = [ + f"Scanned {files_scanned} files ({headers} with a dcat:Dataset header).", + f"Tier-A violations: {len(tier_a)} ({len(fixable)} auto-fixable)", + f"Tier-B violations: {len(tier_b)} (blocked on reference-scheme extensions)", + ] + return "\n".join(lines) + + +def by_rule_table(violations: list[Violation]) -> str: + """Markdown table of violation counts per rule (for CI/PR comments).""" + counts = Counter((v.tier, v.rule_id, v.review_ref) for v in violations) + if not counts: + return "_No violations._" + rows = ["| Tier | Rule | Review | Count |", "|---|---|---|---|"] + for (tier, rule, ref), n in sorted(counts.items(), key=lambda x: (-x[1], x[0])): + rows.append(f"| {tier} | {rule} | {ref} | {n} |") + return "\n".join(rows) + + +def details_text(violations: list[Violation], root: Path | None = None) -> str: + lines = [] + for v in sorted(violations, key=lambda x: (x.tier, x.file, x.rule_id)): + f = v.file + if root is not None: + try: + f = str(Path(v.file).relative_to(root)) + except ValueError: + pass + lines.append(f"{f}: {v}") + return "\n".join(lines) diff --git a/buildScripts/dataset_header/rules.py b/buildScripts/dataset_header/rules.py new file mode 100644 index 00000000..8b2cd6b4 --- /dev/null +++ b/buildScripts/dataset_header/rules.py @@ -0,0 +1,243 @@ +"""Declarative dataset-header rule catalog + rdflib-based validation. + +This module is the single source of truth for the header rules. The validator +evaluates them against an rdflib graph; the fixer (``fix_dataset_header.py``) +consumes the same constants to apply the Tier-A subset via text edits. + +Review-point references (e.g. ``#3``) point to Svein's numbered review of +PR #322. +""" +from __future__ import annotations + +import re +import uuid +from dataclasses import dataclass, field + +from rdflib import Graph, Namespace, URIRef +from rdflib.namespace import RDF +from rdflib.term import Literal, URIRef as URIRefT + +# ------------------------------------------------------------------ namespaces +DCAT = Namespace("http://www.w3.org/ns/dcat#") +DCTERMS = Namespace("http://purl.org/dc/terms/") +ADMS = Namespace("http://www.w3.org/ns/adms#") +PROV = Namespace("http://www.w3.org/ns/prov#") +DCATCIM = Namespace("https://cim4.eu/ns/dcatcim#") + +BASE = "https://energy.referencedata.eu/" + +# ------------------------------------------------------------------ rule data +# Required properties that must appear at least once (Tier A = presence). +REQUIRED_PROPERTIES: dict[str, URIRef] = { + "dcterms:accessRights": DCTERMS.accessRights, + "dcterms:conformsTo": DCTERMS.conformsTo, + "dcterms:description": DCTERMS.description, + "dcterms:identifier": DCTERMS.identifier, + "dcterms:issued": DCTERMS.issued, + "dcat:isVersionOf": DCAT.isVersionOf, + "dcat:keyword": DCAT.keyword, + "dcterms:license": DCTERMS.license, + "dcterms:publisher": DCTERMS.publisher, + "dcterms:rights": DCTERMS.rights, + "dcterms:rightsHolder": DCTERMS.rightsHolder, + "dcterms:spatial": DCTERMS.spatial, + "dcat:startDate": DCAT.startDate, + "dcterms:title": DCTERMS.title, + "dcterms:type": DCTERMS.type, + "dcat:version": DCAT.version, + "prov:generatedAtTime": PROV.generatedAtTime, + "prov:wasGeneratedBy": PROV.wasGeneratedBy, + "adms:versionNotes": ADMS.versionNotes, +} + +# Properties whose value is a fixed constant (Tier A). +FIXED_VALUES: dict[str, tuple[URIRef, object]] = { + "dcterms:accessRights": (DCTERMS.accessRights, URIRef(f"{BASE}Confidentiality/Public")), + "dcterms:type": (DCTERMS.type, URIRef(f"{BASE}type/CIM-PowerSystemModel")), + "dcterms:license": (DCTERMS.license, URIRef("https://creativecommons.org/licenses/by/4.0/")), + "dcterms:rights": (DCTERMS.rights, Literal("Copyright")), + "dcterms:rightsHolder": (DCTERMS.rightsHolder, Literal("ENTSO-E")), +} + +# Properties whose object must be an English-tagged literal (Tier A). +LANG_REQUIRED: dict[str, URIRef] = { + "dcterms:description": DCTERMS.description, + "adms:versionNotes": ADMS.versionNotes, +} + +# Properties that shall not appear in any header (Tier A). +FORBIDDEN_PROPERTIES: dict[str, URIRef] = { + "dcatcim:alternativeVersionOf": DCATCIM.alternativeVersionOf, + "dcterms:accrualPeriodicity": DCTERMS.accrualPeriodicity, + "dcterms:hasPart": DCTERMS.hasPart, + "dcat:temporalResolution": DCAT.temporalResolution, + "dcatcim:preferredVersionOf": DCATCIM.preferredVersionOf, + "dcat:inSeries": DCAT.inSeries, +} + +# Namespace URIs that must not be declared (Tier A). Maps URI -> prefix label. +DISALLOWED_NAMESPACES: dict[str, str] = { + "https://cim4.eu/ns/Metadata-European#": "eumd", + "http://entsoe.eu/ns/Metadata-European#": "eumd", + "http://publications.europa.eu/ontology/euvoc#": "euvoc", +} + +# The correct dcterms namespace; a trailing '#' is a bug (Tier A). +DCTERMS_URI = "http://purl.org/dc/terms/" + +# Datetime-valued properties that must be UTC with a trailing 'Z' (Tier A). +DATETIME_PROPERTIES: dict[str, URIRef] = { + "dcterms:issued": DCTERMS.issued, + "dcat:startDate": DCAT.startDate, + "dcat:endDate": DCAT.endDate, + "prov:generatedAtTime": PROV.generatedAtTime, +} + +PUBLISHER_PATTERN = re.compile(rf"^{re.escape(BASE)}test/party/(TSO|RCC)-.+$") +DATETIME_Z_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$") + +# Tier-B reference-data rules: property -> (scheme key used by schemes.py). +REFERENCE_RULES: dict[str, tuple[URIRef, str, str]] = { + "dcterms:conformsTo": (DCTERMS.conformsTo, "conformance", "#6"), + "dcat:isVersionOf": (DCAT.isVersionOf, "model", "#10"), + "dcterms:spatial": (DCTERMS.spatial, "frame", "#16"), + "prov:wasGeneratedBy": (PROV.wasGeneratedBy, "activity", "#21"), + "dcterms:publisher": (DCTERMS.publisher, "party", "#13"), +} + + +# ------------------------------------------------------------------ violations +@dataclass +class Violation: + file: str + rule_id: str + tier: str # "A" or "B" + review_ref: str + message: str + fixable: bool = False + detail: dict = field(default_factory=dict) + + def __str__(self) -> str: + return f"[{self.tier}] {self.rule_id} ({self.review_ref}): {self.message}" + + +# ------------------------------------------------------------------ validation +def find_dataset(graph: Graph) -> URIRef | None: + """Return the dcat:Dataset subject of a header graph, if any.""" + for s in graph.subjects(RDF.type, DCAT.Dataset): + if isinstance(s, URIRefT): + return s + return None + + +def _lang_ok(obj) -> bool: + return isinstance(obj, Literal) and (obj.language or "").lower().startswith("en") + + +def validate_graph( + graph: Graph, file: str, schemes: "object | None" = None +) -> list[Violation]: + """Run all header rules against ``graph`` and return the violations found.""" + out: list[Violation] = [] + ds = find_dataset(graph) + if ds is None: + # Not a dataset-header file; nothing to validate here. + return out + + # --- namespace / prefix checks (Tier A) ------------------------------- + for prefix, ns in graph.namespaces(): + uri = str(ns) + if uri in DISALLOWED_NAMESPACES: + out.append(Violation( + file, "disallowed-namespace", "A", "#1/#2", + f"disallowed namespace declared: {prefix or DISALLOWED_NAMESPACES[uri]}={uri}", + fixable=True, detail={"uri": uri})) + if prefix == "dcterms" and uri != DCTERMS_URI: + out.append(Violation( + file, "bad-dcterms-namespace", "A", "#1", + f"dcterms namespace should be {DCTERMS_URI!r}, found {uri!r}", + fixable=True, detail={"uri": uri})) + + # --- forbidden properties (Tier A) ------------------------------------ + for label, pred in FORBIDDEN_PROPERTIES.items(): + if (ds, pred, None) in graph: + out.append(Violation( + file, "forbidden-property", "A", "#3", + f"forbidden property present: {label}", fixable=True, + detail={"property": label})) + + # --- required properties present (Tier A) ----------------------------- + for label, pred in REQUIRED_PROPERTIES.items(): + if not any(graph.objects(ds, pred)): + fixable = label in FIXED_VALUES or label in ( + "dcterms:rights", "dcterms:rightsHolder") + out.append(Violation( + file, "missing-required", "A", "#4-#20", + f"missing required property: {label}", fixable=fixable, + detail={"property": label})) + + # --- fixed values (Tier A) -------------------------------------------- + for label, (pred, expected) in FIXED_VALUES.items(): + values = list(graph.objects(ds, pred)) + if values and expected not in values: + out.append(Violation( + file, "wrong-fixed-value", "A", "#5/#12/#15/#18", + f"{label} should be {expected}, found {values[0]}", fixable=True, + detail={"property": label, "expected": str(expected)})) + + # --- language tags (Tier A) ------------------------------------------- + for label, pred in LANG_REQUIRED.items(): + objs = list(graph.objects(ds, pred)) + if objs and not any(_lang_ok(o) for o in objs): + out.append(Violation( + file, "missing-lang-tag", "A", "#4/#7", + f'{label} must be tagged xml:lang="en"', fixable=True, + detail={"property": label})) + + # --- identifier is a UUID (Tier A) ------------------------------------ + for ident in graph.objects(ds, DCTERMS.identifier): + try: + uuid.UUID(str(ident)) + except (ValueError, AttributeError, TypeError): + out.append(Violation( + file, "identifier-not-uuid", "A", "#8", + f"dcterms:identifier is not a UUID: {ident!r}")) + + # --- datetimes are UTC with 'Z' (Tier A) ------------------------------ + for label, pred in DATETIME_PROPERTIES.items(): + for obj in graph.objects(ds, pred): + if not DATETIME_Z_PATTERN.match(str(obj)): + out.append(Violation( + file, "datetime-not-utc-z", "A", "#9", + f"{label} should be UTC ending in 'Z': {obj!r}", + detail={"property": label})) + + # --- publisher naming pattern (Tier A) -------------------------------- + for pub in graph.objects(ds, DCTERMS.publisher): + if not PUBLISHER_PATTERN.match(str(pub)): + out.append(Violation( + file, "publisher-naming", "A", "#13", + f"publisher should match .../test/party/(TSO|RCC)-: {pub}", + detail={"value": str(pub)})) + + # --- Tier-B reference-data membership --------------------------------- + if schemes is not None: + for label, (pred, scheme_key, ref) in REFERENCE_RULES.items(): + valid = schemes.values_for(scheme_key) + if not valid: + continue # scheme not loaded; skip silently + for obj in graph.objects(ds, pred): + if str(obj) not in valid: + out.append(Violation( + file, f"unknown-{scheme_key}", "B", ref, + f"{label} value not found in {scheme_key} scheme: {obj}", + fixable=False, detail={"value": str(obj)})) + + return out + + +def parse_header(path: str) -> Graph: + """Parse an RDF/XML instance file into a graph (namespaces preserved).""" + g = Graph() + g.parse(path, format="xml") + return g diff --git a/buildScripts/dataset_header/schemes.py b/buildScripts/dataset_header/schemes.py new file mode 100644 index 00000000..4e763c29 --- /dev/null +++ b/buildScripts/dataset_header/schemes.py @@ -0,0 +1,78 @@ +"""Load the reference-data schemes into valid-value sets for Tier-B checks. + +The schemes live under ``Instance/referenceData``. Because they only list a +subset of parties/frames/models/activities/profiles today, Tier-B checks are +report-only and best-effort: membership is derived from the URI conventions +used in the scheme files, so newly added entries become valid with no code +change. This is exactly the parked PR #322 reference-data work. +""" +from __future__ import annotations + +from pathlib import Path + +from rdflib import Graph +from rdflib.namespace import RDF +from rdflib.term import URIRef + +from .rules import PROV + +# Scheme key -> (glob under referenceData, URI substring that marks a member). +_SCHEME_FILES: dict[str, tuple[str, str | None]] = { + "model": ("NetworkCode/cimxml/Test-ModelScheme-NCP_RD.xml", "/model/"), + "frame": ("NetworkCode/cimxml/Test-FrameScheme-NCP_RD.xml", "/frame/"), + "activity": ("NetworkCode/cimxml/ActivityScheme-NCP_RD.xml", "/activity/"), + "party": ("NetworkCode/cimxml/Test-PartyScheme-NCP_RD.xml", "/party/"), + "conformance": ("NetworkCode/cimxml/ConformanceReleaseScheme-*_RD.xml", None), +} + + +class Schemes: + """Holds the valid-value set for each reference scheme.""" + + def __init__(self, reference_dir: Path): + self.reference_dir = reference_dir + self._values: dict[str, set[str]] = {} + self._load() + + def values_for(self, key: str) -> set[str]: + return self._values.get(key, set()) + + def _load(self) -> None: + for key, (rel, marker) in _SCHEME_FILES.items(): + matches = sorted(self.reference_dir.glob(rel)) + if not matches: + self._values[key] = set() + continue + g = Graph() + for path in matches: + try: + g.parse(str(path), format="xml") + except Exception: + continue + self._values[key] = self._extract(g, marker) + + @staticmethod + def _extract(g: Graph, marker: str | None) -> set[str]: + values: set[str] = set() + if marker is not None: + # Individuals whose URI matches the scheme's naming convention. + for s in set(g.subjects()): + if isinstance(s, URIRef) and marker in str(s): + values.add(str(s)) + else: + # Conformance: gather members of any prov:Collection plus the + # semantic-asset subjects (profile / vocabulary identifiers). + for o in g.objects(None, PROV.hadMember): + if isinstance(o, URIRef): + values.add(str(o)) + for s in set(g.subjects()): + if isinstance(s, URIRef) and ( + "ap.cim4.eu" in str(s) or "cim4.eu/ns/nc" in str(s) + ): + values.add(str(s)) + return values + + def summary(self) -> str: + return ", ".join( + f"{k}={len(v)}" for k, v in sorted(self._values.items()) + ) diff --git a/buildScripts/fix_dataset_header.py b/buildScripts/fix_dataset_header.py new file mode 100644 index 00000000..9bbb6226 --- /dev/null +++ b/buildScripts/fix_dataset_header.py @@ -0,0 +1,62 @@ +"""Auto-fix the deterministic (Tier-A) dataset-header violations. + +Dry-run by default; pass ``--apply`` to write changes. Formatting-preserving +(text edits, not graph re-serialization). Tier-B (reference-data) values are +never invented here — run ``validate_dataset_header.py`` to see what remains. + +Usage: + python buildScripts/fix_dataset_header.py [--scope networkcode|all] [--apply] +""" +from __future__ import annotations + +import argparse +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from dataset_header.discovery import discover_headers +from dataset_header.fixer import fix_text + +REPO_ROOT = Path(__file__).resolve().parent.parent +INSTANCE_DIR = REPO_ROOT / "Instance" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scope", default="networkcode", + choices=["networkcode", "all"]) + parser.add_argument("--apply", action="store_true", + help="write changes (default: dry-run)") + args = parser.parse_args(argv) + + files = discover_headers(INSTANCE_DIR, scope=args.scope) + changed = 0 + fix_counts: Counter = Counter() + for path in files: + try: + text = path.read_text(encoding="utf-8") + except Exception: + continue + new_text, applied = fix_text(text) + if not applied or new_text == text: + continue + changed += 1 + fix_counts.update(a.split(":")[0] for a in applied) + rel = path.relative_to(REPO_ROOT) + print(f"{'FIX ' if args.apply else 'WOULD FIX '}{rel}: {', '.join(applied)}") + if args.apply: + path.write_text(new_text, encoding="utf-8", newline="") + + print() + print(f"{'Fixed' if args.apply else 'Would fix'} {changed} files.") + for kind, n in fix_counts.most_common(): + print(f" {kind}: {n}") + if not args.apply and changed: + print("\nDry-run only. Re-run with --apply to write changes.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/buildScripts/validate_dataset_header.py b/buildScripts/validate_dataset_header.py new file mode 100644 index 00000000..bd1ac756 --- /dev/null +++ b/buildScripts/validate_dataset_header.py @@ -0,0 +1,92 @@ +"""Validate ReliCapGrid dataset headers against the DCAT header rules. + +Report-only by default (exit 0); pass ``--strict`` to exit non-zero when any +Tier-A violation is found. Tier-B (reference-scheme membership) violations are +always report-only, since fixing them is blocked on extending the schemes. + +Usage: + python buildScripts/validate_dataset_header.py [--scope networkcode|all] + [--strict] [--json validation_report/dataset_header.json] +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +# Allow running as a plain script: make the sibling package importable. +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from dataset_header.discovery import discover_headers +from dataset_header.report import by_rule_table, details_text, summarize, to_dicts +from dataset_header.rules import Violation, find_dataset, parse_header, validate_graph +from dataset_header.schemes import Schemes + +REPO_ROOT = Path(__file__).resolve().parent.parent +INSTANCE_DIR = REPO_ROOT / "Instance" +REFERENCE_DIR = INSTANCE_DIR / "referenceData" + + +def run(scope: str = "networkcode") -> tuple[list[Violation], int, int]: + """Validate all in-scope headers. Returns (violations, files, headers).""" + schemes = Schemes(REFERENCE_DIR) + files = discover_headers(INSTANCE_DIR, scope=scope) + violations: list[Violation] = [] + headers = 0 + for path in files: + try: + graph = parse_header(str(path)) + except Exception as exc: # malformed XML is itself a finding + violations.append(Violation( + str(path), "xml-parse-error", "A", "#1", + f"file is not well-formed / parseable: {exc}")) + continue + if find_dataset(graph) is None: + continue + headers += 1 + violations.extend(validate_graph(graph, str(path), schemes)) + return violations, len(files), headers + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scope", default="networkcode", + choices=["networkcode", "all"]) + parser.add_argument("--strict", action="store_true", + help="exit non-zero if Tier-A violations are found") + parser.add_argument("--json", type=Path, default=None, + help="write a JSON report to this path") + parser.add_argument("--details", action="store_true", + help="print one line per violation") + args = parser.parse_args(argv) + + violations, n_files, headers = run(args.scope) + schemes = Schemes(REFERENCE_DIR) + + print(summarize(violations, n_files, headers)) + print(f"Reference schemes loaded: {schemes.summary()}") + print() + print(by_rule_table(violations)) + if args.details: + print() + print(details_text(violations, root=REPO_ROOT)) + + if args.json: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps({ + "scope": args.scope, + "files_scanned": n_files, + "headers": headers, + "violations": to_dicts(violations), + }, indent=2), encoding="utf-8") + print(f"\nJSON report written to {args.json}") + + tier_a = [v for v in violations if v.tier == "A"] + if args.strict and tier_a: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 0f392951..cfc787e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ dependencies = [ "triplets", "pytest", "tabulate>=0.10.0", + "rdflib>=7.0.0", + "pyshacl", ] [project.urls] diff --git a/tests/test_dataset_header.py b/tests/test_dataset_header.py new file mode 100644 index 00000000..ae347e4a --- /dev/null +++ b/tests/test_dataset_header.py @@ -0,0 +1,155 @@ +"""Dataset-header validation tests. + +Report-only until the fixer (phase 2) cleans this branch: while ``ENFORCE`` is +False, Tier-A findings ``xfail`` (visible in CI, non-blocking). Flip ``ENFORCE`` +to True once headers are clean so regressions fail the build. Tier-B findings +(reference-scheme membership) are always report-only — blocked on extending the +schemes. +""" +import sys +from pathlib import Path + +import pytest +from rdflib import Graph + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.append(str(REPO_ROOT / "buildScripts")) + +from dataset_header.report import by_rule_table, details_text # noqa: E402 +from dataset_header.rules import validate_graph # noqa: E402 +from validate_dataset_header import run # noqa: E402 + +# Flip to True in phase 2, after the fixer brings Tier-A to zero. +ENFORCE = False + + +@pytest.fixture(scope="module") +def findings(): + violations, n_files, headers = run(scope="networkcode") + return violations + + +def test_tier_a_headers(findings): + tier_a = [v for v in findings if v.tier == "A"] + if not tier_a: + return + message = ( + f"**{len(tier_a)} Tier-A dataset-header violations** " + f"({sum(v.fixable for v in tier_a)} auto-fixable):\n\n" + f"{by_rule_table(tier_a)}\n\n" + f"{details_text(tier_a, root=REPO_ROOT)[:4000]}" + ) + if ENFORCE: + pytest.fail(message) + pytest.xfail(message) + + +def test_tier_b_reference_data(findings): + tier_b = [v for v in findings if v.tier == "B"] + if not tier_b: + return + pytest.xfail( + f"{len(tier_b)} Tier-B reference-data violations " + f"(blocked on scheme extensions):\n\n{by_rule_table(tier_b)}" + ) + + +# --------------------------------------------------------------- unit tests + +_CLEAN_HEADER = """ + + + + + desc + b2d1215a-a124-4b6e-9df5-85ecdce9793b + 2025-05-18T18:00:00Z + + CO + + + Copyright + ENTSO-E + + 2025-05-18T18:00:00Z + 20250520_Belgovia_CO + + 1.0.0 + 2025-05-15T07:06:25Z + + Initial version. + +""" + + +def _validate(xml): + g = Graph() + g.parse(data=xml, format="xml") + return validate_graph(g, "", schemes=None) + + +def test_clean_header_has_no_tier_a(): + tier_a = [v for v in _validate(_CLEAN_HEADER) if v.tier == "A"] + assert tier_a == [], "\n".join(str(v) for v in tier_a) + + +def test_forbidden_property_detected(): + xml = _CLEAN_HEADER.replace( + "", + '', + ).replace("", + " X\n ", + ) + new, applied = fix_text(dirty) + assert "eumd" not in new + Graph().parse(data=new, format="xml") # still well-formed + + +def test_fixer_forces_value_and_normalizes_publisher(): + dirty = (_CLEAN_HEADER + .replace("type/CIM-PowerSystemModel", "type/Activity") + .replace("test/party/TSO-Belgovia", "test/party/Belgovia")) + new, _ = fix_text(dirty) + assert "type/CIM-PowerSystemModel" in new + assert "test/party/TSO-Belgovia" in new + tier_a_fixable = [v for v in _validate(new) if v.tier == "A" and v.fixable] + assert tier_a_fixable == [], "\n".join(str(v) for v in tier_a_fixable)