diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ddda65f..6ee55697 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + # Bench evidence-validation tests verify repository-history invariants + # (retained-run ancestry / reviewed squash relations) that need the + # real commit graph, not a depth-1 checkout. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 @@ -34,8 +39,14 @@ jobs: - name: Run repository checks run: python scripts/check_repo.py + - name: Check documentation local links + run: python tools/check_markdown_links.py + - name: Run full project test suite run: python -m pytest tests -q + - name: Run bench unit/contract suites (isolated per bench) + run: python tools/run_bench_unit_tests.py + - name: Run smoke tests run: python scripts/smoke_test.py diff --git a/docs/architecture-decisions/0014-ratify-initial-aebs-product-line-scope.md b/docs/architecture-decisions/0014-ratify-initial-aebs-product-line-scope.md index 1269d8b5..08935f50 100644 --- a/docs/architecture-decisions/0014-ratify-initial-aebs-product-line-scope.md +++ b/docs/architecture-decisions/0014-ratify-initial-aebs-product-line-scope.md @@ -168,4 +168,4 @@ This ADR does not: - [Gate C review](../product-line-engineering/gate-c-product-line-scope-review.md) - [Product-line semantic kernel](../../textual-notation-of-model/packages/methods/de4sdv/de4sdv_product_line.sysml) - [Standalone reference product](../../model-based-product-line-engineering/product-models/aebs_autoware_reference_product.sysml) -- [AAOS-integrated configured member](../../textual-notation-of-model/packages/features/middleware/mw_variability_configuration.sysml) +- [AAOS-integrated configured member](../../textual-notation-of-model/packages/features/middleware/middleware_variability_configuration.sysml) diff --git a/docs/naming/naming-conventions.md b/docs/naming/naming-conventions.md index 14ae5419..5d99c59a 100644 --- a/docs/naming/naming-conventions.md +++ b/docs/naming/naming-conventions.md @@ -460,6 +460,14 @@ Artifact-aware rules (small and explicit, no heuristic parser): templates), the historical 009C–009I implementation-plan record, and the naming QA/manifest docs (which quote unregistered forms as counterexamples). The conventions doc itself stays fully governed. +- **Git-ignored bench runtime workspaces** + (`implementation/*/workspace/...` that `git check-ignore` confirms is + ignored): vendored upstream source checkouts and build outputs produced + by bench prepare/build scripts. Their identifiers follow upstream naming + and are runtime material, not committed governed data. The exemption is + behavioral, not a blanket path rule — any tracked file under such a + directory stays governed — so a clean CI checkout and a developer + checkout with a built workspace behave identically. - **Non-governed look-alikes**: hash-algorithm names (`SHA-256`, `SHA-1`), GitHub line anchors (`#L743-L754`), charset fragments, mixed-case prose (`SERVER-IPv4`, `AI-Ready`), external project names (`S-CORE`, diff --git a/docs/product-line-engineering/gate-c-product-line-scope-review.md b/docs/product-line-engineering/gate-c-product-line-scope-review.md index bfbccdbc..80b9d59d 100644 --- a/docs/product-line-engineering/gate-c-product-line-scope-review.md +++ b/docs/product-line-engineering/gate-c-product-line-scope-review.md @@ -44,7 +44,7 @@ runtime evidence. | Planned reference member | Vehicle Platform Integration Mode | Existing supporting model evidence | |---|---|---| | Standalone Autoware AEBS Reference Member | Standalone | [Standalone reference product](../../model-based-product-line-engineering/product-models/aebs_autoware_reference_product.sysml) | -| AAOS-Integrated Autoware AEBS Reference Member | AAOS Integrated | [Middleware configured member](../../textual-notation-of-model/packages/features/middleware/mw_variability_configuration.sysml) | +| AAOS-Integrated Autoware AEBS Reference Member | AAOS Integrated | [Middleware configured member](../../textual-notation-of-model/packages/features/middleware/middleware_variability_configuration.sysml) | The supporting artifacts remain evidence of local definitions and maturity. They do not replace the governed membership decision. diff --git a/implementation/aebs-aaos-sdv-visualization-bench/evidence/010/warning-lead/p188-exact-head-108bfa9/README.md b/implementation/aebs-aaos-sdv-visualization-bench/evidence/010/warning-lead/p188-exact-head-108bfa9/README.md index 1cc0ce61..961dbb43 100644 --- a/implementation/aebs-aaos-sdv-visualization-bench/evidence/010/warning-lead/p188-exact-head-108bfa9/README.md +++ b/implementation/aebs-aaos-sdv-visualization-bench/evidence/010/warning-lead/p188-exact-head-108bfa9/README.md @@ -58,7 +58,7 @@ compliance, or homologation claim is made. Video bytes are not tracked in Git. The three continuous raw recordings are held in the maintainer archive under `2026-09-06/p188-exact-head-108bfa9/`; their checksums are recorded in -[`external-media.yaml`](../../../external-media.yaml). Per-state stills are +[`external-media.yaml`](../../external-media.yaml). Per-state stills are retained here as frame-level evidence. ## Per-run record contents diff --git a/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_evidence_validation.py b/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_evidence_validation.py index d7683e8a..a17a96c7 100644 --- a/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_evidence_validation.py +++ b/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_evidence_validation.py @@ -94,6 +94,25 @@ def test_squash_delivered_reviewed_head_accepts_retained_run_ancestor(self) -> N "reviewed_head": "871ef95bbdf3b865d5761d692065674fc0b4e196", "delivery_commit": "81e043386251118b302bafbed91922f8fa821522", } + missing = [ + name + for name, revision in ( + ("retained_run_head", relation["retained_run_head"]), + ("reviewed_head", relation["reviewed_head"]), + ) + if subprocess.run( + ["git", "-C", repository, "cat-file", "-e", f"{revision}^{{commit}}"], + capture_output=True, + ).returncode + != 0 + ] + if missing: + self.skipTest( + "requires the squashed PR #66 commit objects " + f"({', '.join(missing)}) which exist only in clones that " + "fetched the pull refs; run this test from a full developer " + "clone or fetch refs/pull/66/head" + ) live_head = subprocess.check_output( ["git", "-C", repository, "rev-parse", "HEAD"], text=True ).strip() diff --git a/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_runtime_controls.py b/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_runtime_controls.py index 8a4ac9f0..515f15ba 100644 --- a/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_runtime_controls.py +++ b/implementation/aebs-autoware-stationary-target-bench/src/de4sdv_aebs_009c_bench/test/test_runtime_controls.py @@ -173,6 +173,13 @@ def test_identity_changes_for_authoritative_input_and_ignores_pycache(tmp_path: def test_runtime_verifier_accepts_live_inheritance_and_rejects_mismatch( tmp_path: Path, ) -> None: + inherited_setup = INHERITED / "workspace/install/setup.bash" + if not inherited_setup.is_file(): + pytest.skip( + "requires the git-ignored inherited 009A workspace build output " + f"({inherited_setup}); built by the bench prepare/build flow, " + "not by a source checkout" + ) command = [ sys.executable, str(BENCH / "scripts/verify_runtime.py"), diff --git a/scripts/check_naming.py b/scripts/check_naming.py index aa985359..18bfb896 100644 --- a/scripts/check_naming.py +++ b/scripts/check_naming.py @@ -42,6 +42,7 @@ from pathlib import Path import re +import subprocess import sys try: @@ -137,6 +138,17 @@ # section 11 documents this narrowing). _GOVERNED_SUFFIXES = {".sysml", ".yaml", ".yml", ".md"} +# Ignored upstream/runtime workspaces under implementation benches: these +# directories hold vendored upstream source checkouts (Autoware) and build +# outputs produced by prepare/build scripts. They are git-ignored runtime +# material whose identifiers follow upstream naming, not the DE4SDV +# registry. Scanning them would fail CI whenever a developer has a local +# workspace checkout, regardless of committed content. The committed +# repository surface is still fully governed. +_IGNORABLE_RUNTIME_PARTS = { + "workspace", # bench runtime checkouts/build trees (git-ignored) +} + # --------------------------------------------------------------------------- # Identifier registry (docs/naming/naming-conventions.md section 5). # @@ -359,9 +371,32 @@ def _is_exempt_from_id_scan(path: Path) -> bool: return True if any(part in _EXEMPT_ID_PATH_PARTS for part in parts): return True + # Git-ignored bench runtime workspaces (vendored upstream checkouts and + # build outputs) are outside the governed committed surface. Any tracked + # file under a bench workspace remains scanned. + if any(part in _IGNORABLE_RUNTIME_PARTS for part in parts): + return _is_ignored_runtime_path(path) return any(rel.startswith(prefix) for prefix in _EXEMPT_PATH_PREFIXES) +def _is_ignored_runtime_path(path: Path, repository: Path | None = None) -> bool: + """True only when git actually ignores the path (tracked files stay governed). + + ``repository`` defaults to the DE4SDV root; tests may pass an isolated + fixture repository to exercise both sides of the rule deterministically. + """ + repository = ROOT if repository is None else repository + try: + result = subprocess.run( + ["git", "check-ignore", "-q", str(path.relative_to(repository))], + cwd=repository, + capture_output=True, + ) + except (OSError, ValueError): + return False + return result.returncode == 0 + + # Technical tokens that merely look like ID-shaped but are external # vocabulary or URL fragments (never governed identifiers). The URL-pattern # guard strips GitHub line-anchor suffixes such as `#L743-L754` before diff --git a/tests/test_check_markdown_links.py b/tests/test_check_markdown_links.py new file mode 100644 index 00000000..b4fdda05 --- /dev/null +++ b/tests/test_check_markdown_links.py @@ -0,0 +1,224 @@ +"""Behavioral tests for documentation local-link validation.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / "tools" / "check_markdown_links.py" + + +def _load_tool(): + import importlib.util + + spec = importlib.util.spec_from_file_location("check_markdown_links", TOOL) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_committed_markdown_local_links_resolve() -> None: + """The committed corpus passes against the real tracked surface.""" + completed = subprocess.run( + [sys.executable, str(TOOL)], cwd=ROOT, capture_output=True, text=True + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "Markdown local-link check passed" in completed.stdout + + +import contextlib + + +@contextlib.contextmanager +def _scenario(tmp_path: Path, files: dict[str, str], tracked: set[str]): + module = _load_tool() + for relative, content in files.items(): + destination = tmp_path / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + original_root = module.ROOT + try: + module.ROOT = tmp_path + errors = module.broken_links( + [name for name in files if name.endswith(".md")], + tracked=frozenset(tracked), + ) + yield errors + finally: + module.ROOT = original_root + + +def test_tracked_relative_file_passes(tmp_path: Path) -> None: + with _scenario( + tmp_path, + { + "docs/doc.md": "[target](./target.md)\n", + "docs/target.md": "exists\n", + }, + {"docs/doc.md", "docs/target.md"}, + ) as errors: + assert errors == [], errors + + +def test_missing_file_fails(tmp_path: Path) -> None: + with _scenario( + tmp_path, + {"docs/doc.md": "[missing](./no-such-file.md)\n"}, + {"docs/doc.md"}, + ) as errors: + assert errors == ["docs/doc.md:1: broken local link -> ./no-such-file.md (resolved docs/no-such-file.md is not tracked repository content)"], errors + + +def test_existing_but_untracked_file_fails(tmp_path: Path) -> None: + """A local file that exists but is not tracked must not satisfy a link.""" + with _scenario( + tmp_path, + { + "docs/doc.md": "[target](./untracked.md)\n", + "docs/untracked.md": "exists locally, not tracked\n", + }, + {"docs/doc.md"}, # untracked.md deliberately absent from the surface + ) as errors: + assert len(errors) == 1, errors + assert "untracked.md" in errors[0] + assert "not tracked repository content" in errors[0] + + +def test_repository_root_escape_fails(tmp_path: Path) -> None: + """A link escaping the repo fails even when the outside file exists.""" + outside = tmp_path / "outside-repository-file.md" + outside.write_text("beyond the root\n", encoding="utf-8") + with _scenario( + tmp_path, + { + "docs/doc.md": "[escape](../../../../outside-repository-file.md)\n", + }, + {"docs/doc.md", "outside-repository-file.md"}, + ) as errors: + assert len(errors) == 1, errors + assert "escapes repository root" in errors[0] + + +def test_url_decoded_paths_continue_to_work(tmp_path: Path) -> None: + with _scenario( + tmp_path, + { + "docs/my doc.md": "exists\n", + "docs/doc.md": "[decoded](my%20doc.md)\n", + }, + {"docs/my doc.md", "docs/doc.md"}, + ) as errors: + assert errors == [], errors + + +def test_fragment_and_query_stripping(tmp_path: Path) -> None: + with _scenario( + tmp_path, + { + "docs/target.md": "exists\n", + "docs/doc.md": ( + "[fragment](./target.md#section)\n" + "[query](./target.md?download=1)\n" + "[both](./target.md?download=1#section)\n" + ), + }, + {"docs/target.md", "docs/doc.md"}, + ) as errors: + assert errors == [], errors + + +def test_tracked_directory_link_passes_and_empty_directory_fails( + tmp_path: Path, +) -> None: + """Directory links resolve when tracked content lies beneath them.""" + with _scenario( + tmp_path, + { + "docs/doc.md": "[dir](./sub/)\n", + "docs/sub/file.md": "tracked content beneath sub\n", + }, + {"docs/doc.md", "docs/sub/file.md"}, + ) as errors: + assert errors == [], errors + with _scenario( + tmp_path, + {"docs/doc.md": "[dir](./empty/)\n"}, + {"docs/doc.md"}, + ) as errors: + assert len(errors) == 1, errors + assert "no tracked content beneath" in errors[0] + + +def test_tracked_but_deleted_file_fails(tmp_path: Path) -> None: + """An indexed-but-deleted (unstaged deletion) target must fail.""" + module = _load_tool() + doc = tmp_path / "docs/doc.md" + target = tmp_path / "docs/target.md" + doc.parent.mkdir(parents=True) + doc.write_text("[target](./target.md)\n", encoding="utf-8") + target.write_text("exists\n", encoding="utf-8") + original_root = module.ROOT + try: + module.ROOT = tmp_path + errors = module.broken_links( + ["docs/doc.md"], tracked=frozenset({"docs/doc.md", "docs/target.md"}) + ) + assert errors == [], errors + target.unlink() # unstaged deletion of the tracked target + errors = module.broken_links( + ["docs/doc.md"], tracked=frozenset({"docs/doc.md", "docs/target.md"}) + ) + finally: + module.ROOT = original_root + assert len(errors) == 1, errors + assert "missing from the working tree" in errors[0] + + +def test_single_quoted_title_target_still_checked(tmp_path: Path) -> None: + """Single-quoted titles are inline links, not extractor escapes.""" + with _scenario( + tmp_path, + {"docs/doc.md": "[missing](./no-such.md \'optional title\')\n"}, + {"docs/doc.md"}, + ) as errors: + assert len(errors) == 1, errors + assert "no-such.md" in errors[0] + + +def test_angle_destination_target_still_checked(tmp_path: Path) -> None: + """Angle-bracket destinations are inline links too.""" + with _scenario( + tmp_path, + {"docs/doc.md": "[missing](<./no-such.md>)\n"}, + {"docs/doc.md"}, + ) as errors: + assert len(errors) == 1, errors + assert "no-such.md" in errors[0] + + +def test_git_inventory_failure_fails_closed(tmp_path: Path, capsys) -> None: + """A failed git inventory must fail the check, not pass vacuously.""" + module = _load_tool() + with mock.patch.object( + module.subprocess, + "run", + return_value=subprocess.CompletedProcess( + ["git"], 128, b"", b"fatal: not a git repository" + ), + ), mock.patch.object(sys, "argv", ["check_markdown_links.py"]): + assert module.main() == 1 + assert "git ls-files failed" in capsys.readouterr().err + + +def test_root_directory_link_counts_all_tracked_files(tmp_path: Path) -> None: + """./ must not depend on tracked dotfiles existing.""" + with _scenario( + tmp_path, + {"README.md": "[root](./)\n", "src/code.py": "x\n"}, + {"README.md", "src/code.py"}, + ) as errors: + assert errors == [], errors diff --git a/tests/test_check_naming.py b/tests/test_check_naming.py index c6eb6d61..fc125af5 100644 --- a/tests/test_check_naming.py +++ b/tests/test_check_naming.py @@ -231,4 +231,209 @@ def test_subject_registry_grammar_is_consistent(): def test_sha_names_are_documented_non_governed(): assert "SHA-256" in check_naming._EXTERNAL_ID_NAMES - assert "SHA-1" in check_naming._EXTERNAL_ID_NAMES \ No newline at end of file + assert "SHA-1" in check_naming._EXTERNAL_ID_NAMES + + +def _ignored_runtime_fixture(tmp_path: Path): + """Create an isolated git repository mirroring the bench ignore rules. + + Returns the fixture repo path. Contains: + - an ignored runtime workspace path with an ID-shaped upstream token + (simulates the vendored Autoware checkout), + - a tracked file under the same workspace directory tree (proves tracked + files stay governed even below an ignorable directory name), + - a tracked file elsewhere with the same token (control: governed). + """ + import subprocess as sp + + repo = tmp_path / "fixture-repo" + repo.mkdir() + def git(*args, check=True): + return sp.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=check) + git("init", "-q") + git("config", "user.name", "fixture") + git("config", "user.email", "fixture@example.org") + (repo / ".gitignore").write_text( + "implementation/aebs-bench/workspace/*\n" + "!implementation/aebs-bench/workspace/tracked.sysml\n", + encoding="utf-8", + ) + ignored_dir = repo / "implementation/aebs-bench/workspace/src/upstream/config" + ignored_dir.mkdir(parents=True) + (ignored_dir / "vendor.param.yaml").write_text( + "channel: LIDAR-01\n", encoding="utf-8" + ) + tracked_in_workspace = repo / "implementation/aebs-bench/workspace/tracked.sysml" + tracked_in_workspace.write_text("part def TrackedInWorkspace\n", encoding="utf-8") + governed = repo / "implementation" + governed.mkdir(exist_ok=True) + (governed / "governed.param.yaml").write_text( + "subject: REQ-FIXTURE-001\n", encoding="utf-8" + ) + git("add", "-A") + git("commit", "-q", "-m", "fixture") + return repo + + +def _probe_fixture(repo: Path, relative: str) -> bool: + import subprocess as sp + + return ( + sp.run( + ["git", "check-ignore", "-q", relative], + cwd=repo, + capture_output=True, + ).returncode + == 0 + ) + + +def test_ignored_runtime_path_excluded_and_tracked_path_governed(tmp_path: Path): + """Deterministic proof of both sides of the runtime-workspace rule. + + Uses an isolated git fixture, so clean CI proves the contract without a + built developer workspace: + - a git-ignored untracked runtime path is outside the governed surface; + - a tracked file under the same ignorable directory stays governed; + - the ignore decision comes from git, not from a directory-name blanket. + """ + repo = _ignored_runtime_fixture(tmp_path) + + ignored_file = repo / "implementation/aebs-bench/workspace/src/upstream/config/vendor.param.yaml" + tracked_file = repo / "implementation/aebs-bench/workspace/tracked.sysml" + + # Git classifies: ignored path ignored; tracked path (negated) not ignored. + assert _probe_fixture(repo, "implementation/aebs-bench/workspace/src/upstream/config/vendor.param.yaml") + assert not _probe_fixture(repo, "implementation/aebs-bench/workspace/tracked.sysml") + + # Production seam honors both classifications. + assert check_naming._is_ignored_runtime_path(ignored_file, repository=repo) + assert not check_naming._is_ignored_runtime_path(tracked_file, repository=repo) + + +def test_tracked_file_under_workspace_area_remains_governed(tmp_path: Path): + """A tracked file below an ignorable workspace directory keeps ID checks.""" + repo = _ignored_runtime_fixture(tmp_path) + import tempfile + + with tempfile.TemporaryDirectory() as td: + # Simulate the governed scan by scanning a copy of the tracked file + # content through the public behavioral seam. + tracked = repo / "implementation/aebs-bench/workspace/tracked.sysml" + tracked.write_text( + "part def TrackedInWorkspace {\n" + " doc /* REQ-FIXTURE-001 anchor; BAD-TOK lives in code below */\n" + "}\n" + "part untrackedStyle : BAD-TOK;\n", + encoding="utf-8", + ) + errors = check_naming.check_identifier_tokens_in_text( + tracked.read_text(encoding="utf-8"), + display_path="implementation/aebs-bench/workspace/tracked.sysml", + suffix=".sysml", + ) + assert any("BAD-" in error for error in errors), errors + + +def test_real_bench_workspace_ignored_when_present(tmp_path: Path): + """Behavioral probe follows git on the real bench workspace, if present. + + Skipped on clean checkouts without a built workspace; the deterministic + fixture tests above already cover both classifications there. + """ + candidate = ( + ROOT + / "implementation/aebs-autoware-executable-bench/workspace/install/setup.bash" + ) + if not candidate.exists(): + import pytest + + pytest.skip( + "no built bench workspace in this checkout; the isolated fixture " + "tests cover the ignored/tracked classifications deterministically" + ) + expected = _probe_fixture(ROOT, str(candidate.relative_to(ROOT))) + assert check_naming._is_ignored_runtime_path(candidate) is expected + + +def _force_tracked_under_ignored_fixture(tmp_path: Path) -> Path: + """Fixture repo where a bad-ID file is tracked WITHOUT a gitignore negation. + + Gitignore says `implementation/aebs-bench/workspace/*`; the file is added + with `git add -f`, so it is tracked while still matching the ignore rule. + This is the case a directory-name blanket exemption would silently skip. + """ + import subprocess as sp + + repo = tmp_path / "aggregate-fixture" + repo.mkdir() + def git(*args, check=True): + return sp.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=check) + git("init", "-q") + git("config", "user.name", "fixture") + git("config", "user.email", "fixture@example.org") + (repo / ".gitignore").write_text( + "implementation/aebs-bench/workspace/*\n", encoding="utf-8" + ) + nested = repo / "implementation/aebs-bench/workspace/build" + nested.mkdir(parents=True) + (nested / "tracked_bad.sysml").write_text( + "part def TrackedUnderIgnored {\n" + " part brokenStyle : BAD-TOK;\n" + "}\n", + encoding="utf-8", + ) + governed = repo / "implementation" + governed.mkdir(exist_ok=True) + (governed / "governed.param.yaml").write_text( + "subject: REQ-FIXTURE-001\n", encoding="utf-8" + ) + git("add", "implementation/governed.param.yaml") + git("add", "-f", "implementation/aebs-bench/workspace/build/tracked_bad.sysml") + git("add", ".gitignore") + git("commit", "-q", "-m", "fixture") + return repo + + +def test_aggregate_scanner_governs_tracked_file_under_ignored_dir( + tmp_path: Path, +): + """The full governed scan still checks a tracked file under an ignored dir. + + End-to-end seam: the path matches a gitignore rule (so a directory-name + blanket exemption would drop it), yet git tracks it, so its invalid + identifier must reach `check_identifier_tokens()` results. The ignore + match is proven with `git check-ignore --no-index`, because a tracked + path is by definition not ignored in index mode. + """ + import subprocess as sp + + repo = _force_tracked_under_ignored_fixture(tmp_path) + relative = "implementation/aebs-bench/workspace/build/tracked_bad.sysml" + # Preconditions: matches the gitignore rule in no-index mode, yet tracked. + assert ( + sp.run( + ["git", "check-ignore", "--no-index", "-q", relative], + cwd=repo, + capture_output=True, + ).returncode + == 0 + ) + assert ( + sp.run( + ["git", "ls-files", "--", relative], + cwd=repo, + capture_output=True, + text=True, + ).stdout.strip() + ) + + original_root = check_naming.ROOT + try: + check_naming.ROOT = repo + errors = check_naming.check_identifier_tokens() + finally: + check_naming.ROOT = original_root + assert any( + "tracked_bad.sysml" in error and "BAD-" in error for error in errors + ), errors diff --git a/tests/test_ci_integrity.py b/tests/test_ci_integrity.py index aafa972b..3a4ee6cc 100644 --- a/tests/test_ci_integrity.py +++ b/tests/test_ci_integrity.py @@ -14,6 +14,16 @@ def test_required_ci_runs_complete_repository_test_suite() -> None: assert "pytest tests/test_semantic_mcp.py" not in workflow +def test_ci_runs_bench_unit_contract_suites() -> None: + """Implementation benches are not in the root suite; CI must run them. + + The benches use same-named bare imports, so they run through the + isolated per-bench runner instead of a single pytest invocation. + """ + workflow = CI.read_text(encoding="utf-8") + assert "python tools/run_bench_unit_tests.py" in workflow + + def test_root_pytest_collection_is_scoped_to_project_tests() -> None: parser = ConfigParser() assert parser.read(PYTEST_CONFIG, encoding="utf-8") == [str(PYTEST_CONFIG)] @@ -21,3 +31,25 @@ def test_root_pytest_collection_is_scoped_to_project_tests() -> None: assert pytest.get("testpaths", "").split() == ["tests"] assert "implementation" in pytest.get("norecursedirs", "").split() assert ".sysand" in pytest.get("norecursedirs", "").split() + + +def test_bench_runner_fails_closed_on_git_inventory_failure(tmp_path) -> None: + """A failed git inventory must fail the bench runner, not pass vacuously.""" + import importlib.util + import subprocess + import sys + from unittest import mock + + tool = ROOT / "tools" / "run_bench_unit_tests.py" + spec = importlib.util.spec_from_file_location("run_bench_unit_tests", tool) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + failing = subprocess.CompletedProcess( + ["git"], 128, "", "fatal: inventory failed" + ) + with mock.patch.object( + module.subprocess, "run", return_value=failing + ), mock.patch.object(sys, "argv", ["run_bench_unit_tests.py"]): + assert module.main() == 1 diff --git a/textual-notation-of-model/sysml-v2-semantic-audit.md b/textual-notation-of-model/sysml-v2-semantic-audit.md index 401bb6a0..ea9a57ee 100644 --- a/textual-notation-of-model/sysml-v2-semantic-audit.md +++ b/textual-notation-of-model/sysml-v2-semantic-audit.md @@ -2,7 +2,15 @@ ## Status -Draft semantic audit for the current DE4SDV SysML v2 textual notation baseline. +Historical record — not a current inventory. This audit was written while the +early AEBS slices were under open review; some rows describe per-PR states +("added in this PR"), reference a since-merged separate functional-interface +slice, and predate later consolidations. Use it to understand repair history, +not as evidence of the present model shape. The current baseline is the +SysML model itself, validated through the repository SysML validation paths; +view inventories are generated from model sources. + +This audit separates three different claims: This audit separates three different claims: diff --git a/tools/check_markdown_links.py b/tools/check_markdown_links.py new file mode 100644 index 00000000..74c34476 --- /dev/null +++ b/tools/check_markdown_links.py @@ -0,0 +1,271 @@ +"""Validate inline local links in tracked Markdown documentation. + +A relative Markdown link must resolve, after URL decoding, fragment and +query stripping, and path normalization, to a file that is BOTH present in +the working tree AND tracked repository content (``git ls-files``). A file +that merely exists locally but is not tracked does not make a link valid, +and a tracked-but-deleted (unstaged deletion) target does not either. +A directory link (trailing slash, or a target that resolves to a directory) +is valid only when at least one tracked file lives beneath that directory +AND that content is actually present; the repository root is valid when any +tracked file exists and the root directory itself is present. + +The link must stay inside the repository. A relative path that escapes the +repository root is rejected even if a file exists at the escaped location. + +Destinations are extracted by a scanner (not a single regex) so every +accepted form is handled exactly: angle-bracket destinations, bare +destinations up to whitespace-then-title-then-``)`` or an unescaped close +paren (balanced ``(...)`` groups allowed), and double- or single-quoted +titles with surrounding whitespace. A link whose destination cannot be +scanned cleanly is reported as unparsable rather than silently skipped. + +Explicitly out of scope: remote URLs, same-page anchors, generated SVG +targets (guarded by the naming checks), and link text. + +Usage: + python tools/check_markdown_links.py # fail on broken links + python tools/check_markdown_links.py --list # print tracked Markdown +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import urllib.parse +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parents[1] + +_FENCE = re.compile(r"```.*?```", re.DOTALL) +_LINK_TEXT_RE = re.compile(r"\[[^\]\n]*\]\(") +_QUOTED_TITLE_RE = re.compile(r"""[ \t]+(?:"[^"\n]*"|'[^'\n]*')""") + +# Generated artifacts are guarded by the naming checks, not this tool. +_SKIP_SUFFIXES = {".svg"} + + +class GitInventoryError(RuntimeError): + """The git file inventory could not be read (fail closed).""" + + +def _scan_inline_destination(text: str, open_paren: int) -> tuple[str, int] | None: + """Return ``(destination, end)`` for the inline link opened at ``open_paren``. + + ``open_paren`` is the index of the ``(`` following the link text. + Accepts an angle-bracket destination, or a bare destination terminated + by whitespace-then-title-then-``)``, or by an unescaped ``)``. A bare + destination may contain balanced ``(...)`` groups; the scanner tracks + depth and backslash escapes. Returns ``None`` when no clean destination + can be scanned — callers must report that, never silently skip it. + """ + index = open_paren + 1 + if index < len(text) and text[index] == "<": + close = text.find(">", index + 1) + if close == -1 or "\n" in text[index + 1 : close]: + return None + return text[index + 1 : close], close + 1 + destination: list[str] = [] + depth = 0 + while index < len(text): + char = text[index] + if char == "\\" and index + 1 < len(text): + destination.append(text[index + 1]) + index += 2 + continue + if char == "\n": + return None + if char == "(": + depth += 1 + destination.append(char) + elif char == ")": + if depth == 0: + candidate = "".join(destination) + if not candidate.strip(): + return None + title = _QUOTED_TITLE_RE.match(text[index + 1 :]) + end = index + 1 + (title.end() if title else 0) + if end < len(text) and text[end] == ")": + return candidate, end + 1 + return candidate, index + 1 + depth -= 1 + destination.append(char) + elif char in " \t": + title = _QUOTED_TITLE_RE.match(text[index:]) + if title: + after = index + title.end() + if after < len(text) and text[after] == ")": + return "".join(destination), after + 1 + return None + if "".join(destination).strip(): + # Whitespace inside a bare destination without a title is + # invalid Markdown; treat as unparsed. + return None + # Leading whitespace before the destination is tolerated. + else: + destination.append(char) + index += 1 + return None + + +def inline_links(text: str) -> list[tuple[str, int]]: + """Extract ``(destination, line)`` pairs for every inline link in ``text``. + + Links whose destination cannot be scanned cleanly yield the synthetic + destination ``"\\x00unparsable"`` so callers report them instead of + silently skipping valid-but-unrecognized Markdown. + """ + scrubbed = _FENCE.sub(lambda match: "\n" * match.group().count("\n"), text) + results: list[tuple[str, int]] = [] + search_from = 0 + while match := _LINK_TEXT_RE.search(scrubbed, search_from): + line = scrubbed[: match.start()].count("\n") + 1 + scanned = _scan_inline_destination(scrubbed, match.end() - 1) + if scanned is None: + results.append(("\x00unparsable", line)) + search_from = match.end() + else: + destination, end = scanned + results.append((destination, line)) + search_from = end + return results + + +def tracked_files() -> list[str]: + completed = subprocess.run( + ["git", "ls-files", "-z"], cwd=ROOT, capture_output=True + ) + if completed.returncode != 0: + raise GitInventoryError( + "git ls-files failed: " + + completed.stderr.decode("utf-8", errors="replace").strip() + ) + stdout = completed.stdout.decode("utf-8", errors="replace") + return [path for path in stdout.split("\0") if path] + + +def tracked_surface() -> frozenset[str]: + """Normalized repository-relative paths of every tracked file.""" + return frozenset( + str(PurePosixPath(name)) for name in tracked_files() + ) + + +def broken_links( + markdown_files: list[str], + tracked: frozenset[str] | None = None, +) -> list[str]: + """Return sorted, de-duplicated broken-link reports for the given files. + + ``tracked`` defaults to the real repository surface; tests may inject a + synthetic surface. + """ + if tracked is None: + tracked = tracked_surface() + errors: list[str] = [] + for name in markdown_files: + absolute = ROOT / name + text = absolute.read_text(encoding="utf-8", errors="replace") + for destination, line in inline_links(text): + if destination == "\x00unparsable": + errors.append( + f"{name}:{line}: inline link destination could not be " + "parsed; fix the link syntax so it is checkable" + ) + continue + raw = destination.strip().strip("<>") + if re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*:", raw) or raw.startswith("#"): + continue # remote or same-page anchor + path_part = urllib.parse.unquote(raw.split("#")[0].split("?")[0]) + if not path_part: + continue # pure fragment/query target + if Path(path_part).suffix.lower() in _SKIP_SUFFIXES: + continue + # Resolve relative to the Markdown file's directory, then make + # the result repository-relative and canonical. + resolved = (absolute.parent / path_part).resolve() + try: + repo_relative = resolved.relative_to(ROOT.resolve()) + except ValueError: + errors.append( + f"{name}:{line}: link escapes repository root -> {raw}" + ) + continue + normalized = PurePosixPath(repo_relative).as_posix() + target_is_directory = ( + path_part.endswith("/") + or (resolved.is_dir() and not resolved.is_file()) + ) + if target_is_directory: + # Git tracks files, not directories: a directory link is + # valid when tracked content lives beneath the directory + # AND that content is present in the working tree. The + # index alone would let a deleted directory satisfy a link. + if normalized == ".": + tracked_inside = bool(tracked) + else: + tracked_prefix = normalized + "/" + tracked_inside = any( + tracked_path.startswith(tracked_prefix) + for tracked_path in tracked + ) + if not tracked_inside: + errors.append( + f"{name}:{line}: broken local link -> {raw} " + f"(no tracked content beneath {normalized})" + ) + elif normalized != "." and not resolved.is_dir(): + errors.append( + f"{name}:{line}: broken local link -> {raw} " + f"(tracked {normalized} is missing from the working tree)" + ) + continue + # A link target must be BOTH tracked and present in the tree. + # Membership alone would let a tracked-but-deleted (unstaged + # deletion) file satisfy the link. + if normalized not in tracked: + errors.append( + f"{name}:{line}: broken local link -> {raw} " + f"(resolved {normalized} is not tracked repository content)" + ) + elif not resolved.is_file(): + errors.append( + f"{name}:{line}: broken local link -> {raw} " + f"(tracked {normalized} is missing from the working tree)" + ) + return sorted(set(errors)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--list", action="store_true", help="list tracked Markdown files" + ) + arguments = parser.parse_args() + + try: + markdown_files = [ + name for name in tracked_files() if name.lower().endswith(".md") + ] + except GitInventoryError as error: + print(f"Markdown local-link check failed: {error}", file=sys.stderr) + return 1 + if arguments.list: + for name in markdown_files: + print(name) + return 0 + + errors = broken_links(markdown_files) + if errors: + print("Markdown local-link check failed:") + for error in errors: + print(f"- {error}") + return 1 + print(f"Markdown local-link check passed ({len(markdown_files)} files).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/run_bench_unit_tests.py b/tools/run_bench_unit_tests.py new file mode 100644 index 00000000..c2177245 --- /dev/null +++ b/tools/run_bench_unit_tests.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Run tracked implementation-bench unit/contract suites in isolated processes. + +The benches use same-named bare imports (e.g. ``evidence_document``) with +per-bench module shapes, so their test directories must not share one Python +process. This runner executes each suite separately, records required import +paths, and fails on any suite failure. It covers only repository-tracked +unit/contract tests; it does not build containers, run ROS, or produce +runtime evidence. + +Usage: + python tools/run_bench_unit_tests.py # run all suites + python tools/run_bench_unit_tests.py --dry-run # list suites only +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class GitInventoryError(RuntimeError): + """The git file inventory could not be read (fail closed).""" + + +# One entry per isolated suite: test paths plus required PYTHONPATH entries +# (bench packages imported as bare modules by their tests). +SUITES: list[dict[str, object]] = [ + { + "name": "aaos-sdv-reference-interop-bench", + "tests": [ + "implementation/aaos-sdv-reference-interop-bench/tests", + "implementation/aaos-sdv-reference-interop-bench/ros2" + "/vehicle_speed_tcp_bridge/tests", + ], + "pythonpath": [], + }, + { + "name": "aebs-aaos-sdv-visualization-bench (010 bridge)", + "tests": [ + "implementation/aebs-aaos-sdv-visualization-bench/src" + "/de4sdv_aebs_010_bridge/test", + ], + "pythonpath": [], + }, + { + "name": "aebs-autoware-nominal-vehicle-target-bench (009B/009D)", + "tests": [ + "implementation/aebs-autoware-nominal-vehicle-target-bench/src" + "/de4sdv_aebs_009b_bench/test", + ], + "pythonpath": [ + "implementation/aebs-autoware-nominal-vehicle-target-bench/src" + "/de4sdv_aebs_009b_bench", + "implementation/aebs-bench-framework", + ], + }, + { + "name": "aebs-autoware-stationary-target-bench (009C)", + "tests": [ + "implementation/aebs-autoware-stationary-target-bench/src" + "/de4sdv_aebs_009c_bench/test", + ], + "pythonpath": ["implementation/aebs-bench-framework"], + }, + { + "name": "vss-vehicle-speed-adapter", + "tests": ["implementation/vss-vehicle-speed-adapter/tests"], + "pythonpath": [], + }, +] + + +def _tracked_test_paths(paths: list[str]) -> list[str]: + """Keep only test paths that contain tracked test files. + + Raises :class:`GitInventoryError` when git itself fails: an unreadable + inventory must fail the run, never shrink it to zero suites. + """ + existing: list[str] = [] + for rel in paths: + absolute = ROOT / rel + if not absolute.is_dir(): + continue + completed = subprocess.run( + ["git", "ls-files", "--", rel], + cwd=ROOT, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise GitInventoryError( + f"git ls-files -- {rel} failed: {completed.stderr.strip()}" + ) + if not completed.stdout.strip(): + continue + existing.append(rel) + return existing + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", action="store_true", help="list suites without running" + ) + arguments = parser.parse_args() + + if not arguments.dry_run: + # Verify the git inventory is readable BEFORE running any suite: a + # broken inventory would otherwise silently shrink the run to zero + # suites and report success. + try: + subprocess.run( + ["git", "rev-parse", "--git-dir"], + cwd=ROOT, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as error: + print( + "Bench unit/contract runner failed: git inventory is " + f"unreadable ({error})", + file=sys.stderr, + ) + return 1 + + failures: list[str] = [] + for suite in SUITES: + try: + tests = _tracked_test_paths(suite["tests"]) # type: ignore[arg-type] + except GitInventoryError as error: + print( + "Bench unit/contract runner failed: unreadable git inventory " + f"({error})", + file=sys.stderr, + ) + return 1 + if not tests: + print(f"[skip] {suite['name']}: no tracked test paths") + continue + if arguments.dry_run: + print(f"[suite] {suite['name']}: {', '.join(tests)}") + continue + environment = os.environ.copy() + pythonpath = [str(ROOT / entry) for entry in suite["pythonpath"]] # type: ignore[arg-type] + if pythonpath: + existing = environment.get("PYTHONPATH", "") + environment["PYTHONPATH"] = os.pathsep.join([*pythonpath, existing]).rstrip( + os.pathsep + ) + command = [sys.executable, "-m", "pytest", "-q", "--tb=short", *tests] + print(f"[run] {suite['name']}") + completed = subprocess.run(command, cwd=ROOT, env=environment) + if completed.returncode != 0: + failures.append(str(suite["name"])) + + if arguments.dry_run: + return 0 + if failures: + print(f"\nBench unit/contract suites FAILED: {', '.join(failures)}") + return 1 + print("\nAll bench unit/contract suites passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())