diff --git a/docs/spec-reference.md b/docs/spec-reference.md index 61119ba..ef56b6c 100644 --- a/docs/spec-reference.md +++ b/docs/spec-reference.md @@ -50,8 +50,17 @@ Fields with alpha feature-state metadata are listed in `docs/support-matrix.md`. Run these before handing a spec to another operator: ```bash +poetry run netengine validate --format text --explain poetry run netengine diagnose NETENGINE_MOCK=true poetry run netengine up ``` +For CI, archive machine-readable support-matrix results and fail on active unsupported fields: + +```bash +poetry run netengine validate --format json > support-matrix-results.json +``` + +The JSON output contains `ok`, `spec`, and `feature_states`; each active feature-state field includes `path`, `state`, `stage`, `reason`, `current_value`, and `default_value`. The command exits non-zero when any active field is `unsupported`. + Prefer explicit values in committed examples so changes are reviewable. diff --git a/docs/support-matrix.md b/docs/support-matrix.md index 32498d1..8780d63 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -53,3 +53,18 @@ Feature states: rotation worker and can be updated through the operator API. It remains experimental because cert-type coverage and graceful cutover behavior are still evolving. + +## CI support-matrix validation + +`netengine validate` can emit machine-readable support-matrix results for CI: + +```bash +poetry run netengine validate --format json > support-matrix-results.json +``` + +The JSON payload includes `ok`, `spec`, and a `feature_states` array. Each active +feature-state entry reports the concrete `path`, `state`, `stage`, `reason`, +`current_value`, and `default_value`. Active `unsupported` entries make the +command exit non-zero, so CI can fail the build while still archiving the JSON +artifact for review. Use `--format text --explain` (the default format is `text`) +for operator-facing diagnostics. diff --git a/netengine/cli/main.py b/netengine/cli/main.py index a0cb26f..369f895 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -1,6 +1,7 @@ """NetEngine CLI — operator surface for world management.""" import asyncio +import json import os import sys from pathlib import Path @@ -81,16 +82,21 @@ def _load_spec_for_cli( *, environment: str | None = None, set_values: tuple[str, ...] = (), + validate_feature_states: bool = True, ): """Load a spec using the same composition semantics as ``up``.""" overrides = _parse_set_overrides(set_values) + feature_state_kwargs = {} if validate_feature_states else {"validate_feature_states": False} if environment: return load_spec_with_environment( - spec_file, environment=environment, overrides=overrides or None + spec_file, + environment=environment, + overrides=overrides or None, + **feature_state_kwargs, ) if overrides: - return load_spec_with_composition(spec_file, overrides=overrides) - return load_spec(spec_file) + return load_spec_with_composition(spec_file, overrides=overrides, **feature_state_kwargs) + return load_spec(spec_file, **feature_state_kwargs) def _feature_state_explanations(spec: Any) -> list[str]: @@ -106,6 +112,34 @@ def _feature_state_explanations(spec: Any) -> list[str]: return lines +def _jsonable_feature_value(value: Any) -> Any: + """Return a JSON-serializable representation for feature-state values.""" + if hasattr(value, "value"): + return value.value + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + return value + + +def _active_feature_state_results(spec: Any) -> list[dict[str, Any]]: + """Return machine-readable active feature-state validation results.""" + results: list[dict[str, Any]] = [] + for entry, path, value, default_value in _resolve_feature_state_paths(spec): + if not _is_active_feature_value(value, default_value): + continue + results.append( + { + "path": path, + "state": entry.state, + "stage": entry.stage, + "reason": entry.reason, + "current_value": _jsonable_feature_value(value), + "default_value": _jsonable_feature_value(default_value), + } + ) + return results + + async def _run_migrations(db_url: str) -> MigrationRunResult: """Run SQL migrations using the shared migration service.""" result = await run_migrations(db_url) @@ -478,6 +512,14 @@ async def _readiness( default=False, help="Print active experimental, reserved, unsupported, or otherwise noteworthy fields.", ) +@click.option( + "--format", + "output_format", + type=click.Choice(["text", "json"]), + default="text", + show_default=True, + help="Emit human-readable text or machine-readable JSON support-matrix results.", +) @click.option( "--env", "environment", @@ -493,28 +535,61 @@ async def _readiness( def validate( spec_file: str, explain: bool, + output_format: str, environment: str | None, set_values: tuple[str, ...], ) -> None: """Validate SPEC_FILE without booting a world.""" try: spec = _load_spec_for_cli( - spec_file, environment=environment, set_values=set_values + spec_file, + environment=environment, + set_values=set_values, + validate_feature_states=False, ) except SpecLoadError as exc: - click.echo(f"Spec validation failed: {exc}", err=True) + if output_format == "json": + click.echo(json.dumps({"ok": False, "error": str(exc), "feature_states": []}, indent=2)) + else: + click.echo(f"Spec validation failed: {exc}", err=True) sys.exit(1) - click.echo(f"Spec validation succeeded: {spec.metadata.name}") - if explain: - explanations = _feature_state_explanations(spec) - if explanations: - click.echo("Feature-state details:") - for line in explanations: - prefix = "WARNING: " if ": experimental " in line else "" - click.echo(f" - {prefix}{line}") + feature_states = _active_feature_state_results(spec) + unsupported = [item for item in feature_states if item["state"] == "unsupported"] + + if output_format == "json": + click.echo( + json.dumps( + { + "ok": not unsupported, + "spec": spec.metadata.name, + "feature_states": feature_states, + }, + indent=2, + ) + ) + else: + if unsupported: + click.echo("Spec validation failed: Unsupported spec features enabled:", err=True) + for item in unsupported: + click.echo( + f" - {item['path']} is {item['state']} in {item['stage']}: {item['reason']}", + err=True, + ) else: - click.echo("Feature-state details: no active noteworthy fields.") + click.echo(f"Spec validation succeeded: {spec.metadata.name}") + if explain: + explanations = _feature_state_explanations(spec) + if explanations: + click.echo("Feature-state details:") + for line in explanations: + prefix = "WARNING: " if ": experimental " in line else "" + click.echo(f" - {prefix}{line}") + else: + click.echo("Feature-state details: no active noteworthy fields.") + + if unsupported: + sys.exit(1) @cli.command() diff --git a/netengine/spec/loader.py b/netengine/spec/loader.py index baba3f1..3472c0c 100644 --- a/netengine/spec/loader.py +++ b/netengine/spec/loader.py @@ -161,7 +161,7 @@ def _cross_validate(spec: NetEngineSpec) -> None: ) -def load_spec(yaml_path: str | Path) -> NetEngineSpec: +def load_spec(yaml_path: str | Path, *, validate_feature_states: bool = True) -> NetEngineSpec: """Load and validate a NetEngine YAML specification. Args: @@ -207,7 +207,8 @@ def load_spec(yaml_path: str | Path) -> NetEngineSpec: raise SpecLoadError(f"Spec validation failed: {e}") _cross_validate(spec) - _validate_feature_states(spec) + if validate_feature_states: + _validate_feature_states(spec) return spec @@ -215,6 +216,7 @@ def load_spec_with_composition( yaml_path: str | Path, base_path: Optional[str | Path] = None, overrides: Optional[dict[str, Any]] = None, + validate_feature_states: bool = True, ) -> NetEngineSpec: """Load spec with optional base spec composition and overrides. @@ -279,7 +281,8 @@ def load_spec_with_composition( raise SpecLoadError(f"Spec validation failed: {e}") _cross_validate(spec) - _validate_feature_states(spec) + if validate_feature_states: + _validate_feature_states(spec) return spec @@ -287,6 +290,7 @@ def load_spec_with_environment( base_spec: str | Path, environment: str = "dev", overrides: Optional[dict[str, Any]] = None, + validate_feature_states: bool = True, ) -> NetEngineSpec: """Load base spec and merge with environment-specific overrides. @@ -349,5 +353,6 @@ def load_spec_with_environment( raise SpecLoadError(f"Spec validation failed: {e}") _cross_validate(spec) - _validate_feature_states(spec) + if validate_feature_states: + _validate_feature_states(spec) return spec diff --git a/tests/test_cli.py b/tests/test_cli.py index 5c9fe7a..5370f4c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -456,6 +456,43 @@ def test_validate_experimental_enabled_feature_exits_zero_with_explanation(tmp_p assert "pki.intermediate_ca_enabled: experimental" in result.output +def test_validate_json_reports_active_feature_states(tmp_path: Path) -> None: + """JSON format should expose machine-readable support-matrix results.""" + spec_file = _write_cli_validate_spec(tmp_path, pki__intermediate_ca_enabled=True) + + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--format", "json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["ok"] is True + assert payload["spec"] == "minimal-example" + assert payload["feature_states"] == [ + { + "path": "pki.intermediate_ca_enabled", + "state": "experimental", + "stage": "alpha", + "reason": "intermediate CA handling is available but still stabilizing", + "current_value": True, + "default_value": False, + } + ] + + +def test_validate_json_unsupported_active_feature_exits_nonzero(tmp_path: Path) -> None: + """Unsupported active fields should be machine-readable and fail CI.""" + spec_file = _write_cli_validate_spec(tmp_path, pki__crl_enabled=True) + + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--format", "json"]) + + assert result.exit_code == 1 + payload = json.loads(result.output) + assert payload["ok"] is False + assert payload["feature_states"][0]["path"] == "pki.crl_enabled" + assert payload["feature_states"][0]["state"] == "unsupported" + assert payload["feature_states"][0]["current_value"] is True + assert payload["feature_states"][0]["default_value"] is False + + def test_validate_explain_includes_dotted_field_paths_and_feature_states(tmp_path: Path) -> None: """--explain should show concrete dotted paths and their feature states.""" spec_file = _write_cli_validate_spec(tmp_path, pki__intermediate_ca_enabled=True)