From 76b303d634f624508324263e50efc5824510f6e4 Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 18:50:06 +0100 Subject: [PATCH 1/2] Promote gateway mirror support to experimental --- docs/spec-alpha-support.md | 4 ++-- docs/support-matrix.md | 4 ++-- netengine/spec/feature_state.py | 14 ++++++++++---- tests/test_cli.py | 19 +++++++++++-------- tests/test_spec_parsing.py | 17 +++++++++++------ 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/docs/spec-alpha-support.md b/docs/spec-alpha-support.md index 3cc54fa..c154034 100644 --- a/docs/spec-alpha-support.md +++ b/docs/spec-alpha-support.md @@ -26,8 +26,8 @@ Feature states: | `pki.crl_enabled` | `experimental` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | step-ca CRL generation is enabled in `ca.json` and the distribution URL is published in Phase 3 output; client-validation coverage is hardened in CI e2e. | | `pki.ocsp_enabled` | `experimental` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | step-ca OCSP config is injected and the responder URL is published in Phase 3 output; responder lifecycle/verification is hardened in CI e2e. | | `pki.rotation_policy` | `experimental` | `{enabled: true, default_interval_hours: 24, default_warning_days: 30, cert_type_overrides: {}}` | `netengine.handlers.phase_pki`, `netengine.workers.pki_cert_rotation_worker`, `netengine.api.routes` | Wired from the spec into worker registration and live-reloaded from runtime state; policy shape and cert-type semantics may change during alpha. | -| `gateway_portal.real_internet.mode` | `unsupported` | `isolated` | `netengine.spec.loader` | Real-internet gateway policies are not implemented. | -| `gateway_portal.real_internet.service_mirrors` | `unsupported` | `[]` | `netengine.spec.loader` | Service mirror provisioning is not implemented. | +| `gateway_portal.real_internet.mode` | `experimental` | `isolated` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Isolated, shadowed, mirrored, and exposed nftables policies are wired; real-host integration remains alpha. | +| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Mirrored-mode allowlists and operator output are wired; automatic DNS aliasing for real hostnames remains alpha. | | `gateway_portal.real_internet.upstream_resolver_enabled` | `unsupported` | `false` | `netengine.spec.loader` | Upstream resolver forwarding is not implemented. | | `gateway_portal.cross_world.mode` | `unsupported` | `none` | `netengine.spec.loader` | Cross-world federation is not implemented. | | `gateway_portal.cross_world.peers` | `unsupported` | `[]` | `netengine.spec.loader` | Cross-world peer provisioning is not implemented. | diff --git a/docs/support-matrix.md b/docs/support-matrix.md index 8780d63..2ab3181 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -26,8 +26,8 @@ Feature states: | `pki.crl_enabled` | `unsupported` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | Handler code can inject a step-ca CRL config and report a URL, but CRL publication/distribution-point plumbing is incomplete and active use is unsupported in alpha validation. | | `pki.ocsp_enabled` | `unsupported` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | Handler code can inject OCSP-related step-ca config and report a URL, but responder deployment/verification is incomplete and active use is unsupported in alpha validation. | | `pki.rotation_policy` | `experimental` | `{enabled: true, default_interval_hours: 24, default_warning_days: 30, cert_type_overrides: {}}` | `netengine.handlers.phase_pki`, `netengine.workers.pki_cert_rotation_worker`, `netengine.api.routes` | Wired from the spec into worker registration and live-reloaded from runtime state; policy shape and cert-type semantics may change during alpha. | -| `gateway_portal.real_internet.mode` | `unsupported` | `isolated` | `netengine.spec.loader` | Real-internet gateway policies are not implemented. | -| `gateway_portal.real_internet.service_mirrors` | `unsupported` | `[]` | `netengine.spec.loader` | Service mirror provisioning is not implemented. | +| `gateway_portal.real_internet.mode` | `experimental` | `isolated` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Isolated, shadowed, mirrored, and exposed nftables policies are wired; real-host integration remains alpha. | +| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Mirrored-mode allowlists and operator output are wired; automatic DNS aliasing for real hostnames remains alpha. | | `gateway_portal.real_internet.upstream_resolver_enabled` | `unsupported` | `false` | `netengine.spec.loader` | Upstream resolver forwarding is not implemented. | | `gateway_portal.cross_world.mode` | `unsupported` | `none` | `netengine.spec.loader` | Cross-world federation is not implemented. | | `gateway_portal.cross_world.peers` | `unsupported` | `[]` | `netengine.spec.loader` | Cross-world peer provisioning is not implemented. | diff --git a/netengine/spec/feature_state.py b/netengine/spec/feature_state.py index 87bbfde..0849510 100644 --- a/netengine/spec/feature_state.py +++ b/netengine/spec/feature_state.py @@ -47,15 +47,21 @@ class FeatureStateEntry: ), FeatureStateEntry( path="gateway_portal.real_internet.mode", - state="unsupported", + state="experimental", stage="alpha", - reason="real-internet gateway policies are not implemented", + reason=( + "gateway policies for isolated, shadowed, mirrored, and exposed modes " + "are wired through nftables, but real-host integration remains alpha" + ), ), FeatureStateEntry( path="gateway_portal.real_internet.service_mirrors", - state="unsupported", + state="experimental", stage="alpha", - reason="service mirror provisioning is not implemented", + reason=( + "mirrored-mode nftables allowlists and operator output are wired; " + "automatic DNS aliasing for real hostnames remains alpha" + ), ), FeatureStateEntry( path="gateway_portal.real_internet.upstream_resolver_enabled", diff --git a/tests/test_cli.py b/tests/test_cli.py index 5370f4c..4d4263f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -433,15 +433,16 @@ def test_validate_valid_spec_exits_zero() -> None: assert "Spec validation succeeded: minimal-example" in result.output -def test_validate_unsupported_enabled_feature_exits_nonzero(tmp_path: Path) -> None: - """Unsupported active feature gates should fail validation precisely.""" +def test_validate_experimental_gateway_mode_exits_zero_with_explanation(tmp_path: Path) -> None: + """Mirrored gateway mode is wired but remains experimental during alpha.""" spec_file = _write_cli_validate_spec(tmp_path, gateway_portal__real_internet__mode="mirrored") - result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file)]) + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--explain"]) - assert result.exit_code == 1 - assert "Unsupported spec features enabled" in result.output - assert "gateway_portal.real_internet.mode is unsupported" in result.output + assert result.exit_code == 0, result.output + assert "Spec validation succeeded" in result.output + assert "WARNING:" in result.output + assert "gateway_portal.real_internet.mode: experimental" in result.output def test_validate_experimental_enabled_feature_exits_zero_with_explanation(tmp_path: Path) -> None: @@ -480,14 +481,16 @@ def test_validate_json_reports_active_feature_states(tmp_path: Path) -> None: def test_validate_json_unsupported_active_feature_exits_nonzero(tmp_path: Path) -> None: """Unsupported active fields should be machine-readable and fail CI.""" - spec_file = _write_cli_validate_spec(tmp_path, pki__crl_enabled=True) + spec_file = _write_cli_validate_spec( + tmp_path, gateway_portal__real_internet__upstream_resolver_enabled=True + ) result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--format", "json"]) assert result.exit_code == 1 payload = json.loads(result.output) assert payload["ok"] is False - assert payload["feature_states"][0]["path"] == "pki.crl_enabled" + assert payload["feature_states"][0]["path"] == "gateway_portal.real_internet.upstream_resolver_enabled" assert payload["feature_states"][0]["state"] == "unsupported" assert payload["feature_states"][0]["current_value"] is True assert payload["feature_states"][0]["default_value"] is False diff --git a/tests/test_spec_parsing.py b/tests/test_spec_parsing.py index 537013e..c29abef 100644 --- a/tests/test_spec_parsing.py +++ b/tests/test_spec_parsing.py @@ -180,16 +180,21 @@ def test_experimental_pki_field_logs_warning_not_error(self, tmp_path, caplog) - "pki.dnssec_enabled is experimental in alpha" in r.message for r in caplog.records ) - def test_unsupported_non_default_active_field_raises_spec_load_error(self, tmp_path) -> None: + def test_experimental_gateway_mode_logs_warning_not_raises(self, tmp_path, caplog) -> None: + import logging + spec_file = self._write_spec( tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}} ) - with pytest.raises( - SpecLoadError, - match="gateway_portal\\.real_internet\\.mode is unsupported in alpha", - ): - load_spec(spec_file) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + + assert spec.gateway_portal.real_internet.mode.value == "mirrored" + assert any( + "gateway_portal.real_internet.mode is experimental in alpha" in r.message + for r in caplog.records + ) def test_experimental_profile_field_logs_warning_not_raises(self, tmp_path, caplog) -> None: import logging From 136bb36cb1fa7877ad0e275dc0e89cf0706300bf Mon Sep 17 00:00:00 2001 From: aidankhogg Date: Mon, 29 Jun 2026 18:58:59 +0100 Subject: [PATCH 2/2] Promote gateway real-internet mode & service_mirrors to experimental ### Motivation - The gateway nftables policies and mirrored-mode operator output are now wired, so `gateway_portal.real_internet.mode` and `gateway_portal.real_internet.service_mirrors` should be represented as `experimental` rather than `unsupported` in the feature-state registry and docs. - Tests and CLI validation must reflect that mirrored mode validates with a warning instead of failing alpha validation, while truly unsupported flags remain CI-failing. ### Description - Update feature-state registry in `netengine/spec/feature_state.py` to mark `gateway_portal.real_internet.mode` and `gateway_portal.real_internet.service_mirrors` as `experimental` and add explanatory caveats. - Update documentation rows in `docs/spec-alpha-support.md` and `docs/support-matrix.md` to document the promoted experimental state and caveats about remaining real-host integration and DNS aliasing. - Adjust CLI/tests in `tests/test_cli.py` so a spec with `gateway_portal.real_internet.mode: mirrored` validates successfully with `--explain` (shows a warning) and change the JSON CI-failure test to target `upstream_resolver_enabled` which remains `unsupported`. - Adjust spec parsing test in `tests/test_spec_parsing.py` to assert the loader logs an experimental warning for mirrored gateway mode instead of raising `SpecLoadError`. ### Testing - Ran `poetry run pytest tests/test_spec_parsing.py tests/test_cli.py tests/test_gateway_portal.py tests/test_feature_states.py` and those targeted tests passed. - Ran the full suite with `poetry run pytest`; the full run completed but showed 2 unrelated integration failures (`TestFullMVPLifecycle.test_full_mvp_lifecycle` returned `401` for reload and `TestHealthRoute.test_health_does_not_leak_last_error_detail` exposed `last_error`) that pre-existed this change. - No new regressions observed in gateway rule generation or unit tests covering gateway portal behavior. --- docs/netengines-spec-v1.yaml | 6 ++-- docs/spec-alpha-support.md | 2 +- docs/support-matrix.md | 2 +- netengine/spec/loader.py | 27 ++++++++++++++++ tests/test_cli.py | 8 ++++- tests/test_spec_parsing.py | 60 +++++++++++++++++++++++++++++++++++- 6 files changed, 99 insertions(+), 6 deletions(-) diff --git a/docs/netengines-spec-v1.yaml b/docs/netengines-spec-v1.yaml index c6b9910..f2c1ed1 100644 --- a/docs/netengines-spec-v1.yaml +++ b/docs/netengines-spec-v1.yaml @@ -447,11 +447,13 @@ gateway_portal: # custom — explicit rule list (use when presets don't fit) # Required if mode: mirrored + # Alpha: in_world_service must be an IPv4 address because gateway rules + # render nftables ip daddr allowlists; DNS aliasing for names is planned. # service_mirrors: # - real_hostname: s3.amazonaws.com - # in_world_service: storage.platform.internal + # in_world_service: 10.0.0.11 # - real_hostname: github.com - # in_world_service: gitea.acme.internal + # in_world_service: 10.0.1.20 # Required if mode: custom # rules: [] diff --git a/docs/spec-alpha-support.md b/docs/spec-alpha-support.md index c154034..db5ffb4 100644 --- a/docs/spec-alpha-support.md +++ b/docs/spec-alpha-support.md @@ -27,7 +27,7 @@ Feature states: | `pki.ocsp_enabled` | `experimental` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | step-ca OCSP config is injected and the responder URL is published in Phase 3 output; responder lifecycle/verification is hardened in CI e2e. | | `pki.rotation_policy` | `experimental` | `{enabled: true, default_interval_hours: 24, default_warning_days: 30, cert_type_overrides: {}}` | `netengine.handlers.phase_pki`, `netengine.workers.pki_cert_rotation_worker`, `netengine.api.routes` | Wired from the spec into worker registration and live-reloaded from runtime state; policy shape and cert-type semantics may change during alpha. | | `gateway_portal.real_internet.mode` | `experimental` | `isolated` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Isolated, shadowed, mirrored, and exposed nftables policies are wired; real-host integration remains alpha. | -| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Mirrored-mode allowlists and operator output are wired; automatic DNS aliasing for real hostnames remains alpha. | +| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Mirrored-mode allowlists and operator output are wired; `in_world_service` must be an IPv4 address until automatic DNS aliasing for real hostnames lands. | | `gateway_portal.real_internet.upstream_resolver_enabled` | `unsupported` | `false` | `netengine.spec.loader` | Upstream resolver forwarding is not implemented. | | `gateway_portal.cross_world.mode` | `unsupported` | `none` | `netengine.spec.loader` | Cross-world federation is not implemented. | | `gateway_portal.cross_world.peers` | `unsupported` | `[]` | `netengine.spec.loader` | Cross-world peer provisioning is not implemented. | diff --git a/docs/support-matrix.md b/docs/support-matrix.md index 2ab3181..9460346 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -27,7 +27,7 @@ Feature states: | `pki.ocsp_enabled` | `unsupported` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | Handler code can inject OCSP-related step-ca config and report a URL, but responder deployment/verification is incomplete and active use is unsupported in alpha validation. | | `pki.rotation_policy` | `experimental` | `{enabled: true, default_interval_hours: 24, default_warning_days: 30, cert_type_overrides: {}}` | `netengine.handlers.phase_pki`, `netengine.workers.pki_cert_rotation_worker`, `netengine.api.routes` | Wired from the spec into worker registration and live-reloaded from runtime state; policy shape and cert-type semantics may change during alpha. | | `gateway_portal.real_internet.mode` | `experimental` | `isolated` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Isolated, shadowed, mirrored, and exposed nftables policies are wired; real-host integration remains alpha. | -| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Mirrored-mode allowlists and operator output are wired; automatic DNS aliasing for real hostnames remains alpha. | +| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.handlers.gateway_handler`, `netengine.handlers.gateway_portal_handler`, `netengine.spec.loader` | Mirrored-mode allowlists and operator output are wired; `in_world_service` must be an IPv4 address until automatic DNS aliasing for real hostnames lands. | | `gateway_portal.real_internet.upstream_resolver_enabled` | `unsupported` | `false` | `netengine.spec.loader` | Upstream resolver forwarding is not implemented. | | `gateway_portal.cross_world.mode` | `unsupported` | `none` | `netengine.spec.loader` | Cross-world federation is not implemented. | | `gateway_portal.cross_world.peers` | `unsupported` | `[]` | `netengine.spec.loader` | Cross-world peer provisioning is not implemented. | diff --git a/netengine/spec/loader.py b/netengine/spec/loader.py index 3472c0c..3d9e8c2 100644 --- a/netengine/spec/loader.py +++ b/netengine/spec/loader.py @@ -11,6 +11,7 @@ from netengine.config.loader import ConfigLoader from netengine.spec.models import SUPPORTED_SPEC_SCHEMA_VERSIONS, NetEngineSpec +from netengine.spec.types import GatewayRealInternetMode logger = logging.getLogger(__name__) @@ -155,6 +156,32 @@ def _cross_validate(spec: NetEngineSpec) -> None: if net_a.overlaps(net_b): errors.append(f"subnet overlap: {name_a} ({net_a}) overlaps {name_b} ({net_b})") + # 5. Mirrored real-internet mode needs explicit IP allowlist targets. + # The alpha gateway implementation renders nftables `ip daddr` rules from + # service_mirrors, so hostnames would generate invalid nftables policy until + # DNS aliasing/resolution is implemented. + real_internet = spec.gateway_portal.real_internet + if real_internet.mode == GatewayRealInternetMode.MIRRORED: + if not real_internet.service_mirrors: + errors.append( + "gateway_portal.real_internet.service_mirrors: " + "at least one service mirror is required when mode is 'mirrored'" + ) + elif real_internet.service_mirrors: + errors.append( + "gateway_portal.real_internet.service_mirrors: " + "service mirrors require real_internet.mode: 'mirrored'" + ) + + for idx, mirror in enumerate(real_internet.service_mirrors): + try: + ipaddress.IPv4Address(mirror.in_world_service) + except ValueError: + errors.append( + f"gateway_portal.real_internet.service_mirrors[{idx}].in_world_service: " + "must be an IPv4 address for alpha mirrored gateway nftables rules" + ) + if errors: raise SpecLoadError( "Spec cross-validation failed:\n" + "\n".join(f" - {e}" for e in errors) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4d4263f..7d8c307 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -435,7 +435,13 @@ def test_validate_valid_spec_exits_zero() -> None: def test_validate_experimental_gateway_mode_exits_zero_with_explanation(tmp_path: Path) -> None: """Mirrored gateway mode is wired but remains experimental during alpha.""" - spec_file = _write_cli_validate_spec(tmp_path, gateway_portal__real_internet__mode="mirrored") + spec_file = _write_cli_validate_spec( + tmp_path, + gateway_portal__real_internet__mode="mirrored", + gateway_portal__real_internet__service_mirrors=[ + {"real_hostname": "github.com", "in_world_service": "10.0.1.20"} + ], + ) result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--explain"]) diff --git a/tests/test_spec_parsing.py b/tests/test_spec_parsing.py index c29abef..992253a 100644 --- a/tests/test_spec_parsing.py +++ b/tests/test_spec_parsing.py @@ -184,18 +184,76 @@ def test_experimental_gateway_mode_logs_warning_not_raises(self, tmp_path, caplo import logging spec_file = self._write_spec( - tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}} + tmp_path, + { + "gateway_portal": { + "real_internet": { + "mode": "mirrored", + "service_mirrors": [ + {"real_hostname": "github.com", "in_world_service": "10.0.1.20"} + ], + } + } + }, ) with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): spec = load_spec(spec_file) assert spec.gateway_portal.real_internet.mode.value == "mirrored" + assert spec.gateway_portal.real_internet.service_mirrors[0].in_world_service == "10.0.1.20" assert any( "gateway_portal.real_internet.mode is experimental in alpha" in r.message for r in caplog.records ) + def test_mirrored_gateway_requires_service_mirrors(self, tmp_path) -> None: + spec_file = self._write_spec( + tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}} + ) + + with pytest.raises(SpecLoadError, match="at least one service mirror is required"): + load_spec(spec_file) + + def test_service_mirrors_require_mirrored_gateway_mode(self, tmp_path) -> None: + spec_file = self._write_spec( + tmp_path, + { + "gateway_portal": { + "real_internet": { + "service_mirrors": [ + {"real_hostname": "github.com", "in_world_service": "10.0.1.20"} + ] + } + } + }, + ) + + with pytest.raises(SpecLoadError, match="service mirrors require"): + load_spec(spec_file) + + def test_service_mirror_targets_must_be_ipv4_for_alpha_nftables(self, tmp_path) -> None: + spec_file = self._write_spec( + tmp_path, + { + "gateway_portal": { + "real_internet": { + "mode": "mirrored", + "service_mirrors": [ + { + "real_hostname": "github.com", + "in_world_service": "gitea.acme.internal", + } + ], + } + } + }, + ) + + with pytest.raises(SpecLoadError, match="must be an IPv4 address"): + load_spec(spec_file) + + def test_experimental_profile_field_logs_warning_not_raises(self, tmp_path, caplog) -> None: import logging