From 2f6bf3166cec64c52359d180b499627cb62b3bbc Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Tue, 6 Oct 2026 20:22:19 -0400 Subject: [PATCH 01/68] fix(agent): adopt, update and roll back stacks that Portainer runs Portainer runs compose inside its own container, so the paths compose writes into the labels (/data/compose/49/docker-compose.yml) are Portainer's, not the host's, and adopting such a stack failed with "No such file or directory". A label path that does not exist on the host is now mapped through the mount of the container that holds it (longest destination wins), and used only if the mapped file exists. The env file compose recorded (Portainer's stack.env) is passed with --env-file, so a re-up from the host interpolates the same values instead of blanks, and adopt reads the env from it. --- agent/tests/test_compose_host_paths.py | 210 ++++++++++++++++++++++++ agent/tests/test_stack_actions.py | 4 +- agent/tests/test_stack_read.py | 13 ++ agent/tests/test_update_container.py | 4 +- agent/vigil_agent/actions/containers.py | 83 +++++++++- agent/vigil_agent/actions/stacks.py | 3 +- 6 files changed, 306 insertions(+), 11 deletions(-) create mode 100644 agent/tests/test_compose_host_paths.py diff --git a/agent/tests/test_compose_host_paths.py b/agent/tests/test_compose_host_paths.py new file mode 100644 index 00000000..dbab4391 --- /dev/null +++ b/agent/tests/test_compose_host_paths.py @@ -0,0 +1,210 @@ +"""QA-07: compose paths written by a manager that runs compose in its own +container (Portainer) are the container's, not the host's — map them through +the mount that holds them before compose or Vigil reads them.""" +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import tests._safety_net # noqa: F401 — the guard, even when this file is run or imported on its own + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +# isort: split +from vigil_agent import engine, executor +from vigil_agent.actions import containers +from vigil_agent.config import AgentConfig + +from tests.fake_engine import FakeEngine + +_CFG = AgentConfig(server_url="https://v", agent_token="t", mode="full_control") +PORTAINER_COMPOSE = "/data/compose/49/docker-compose.yml" +PORTAINER_ENV = "/data/compose/49/stack.env" +PORTAINER_WD = "/data/compose/49" + + +def _mount(source: str, destination: str) -> dict: + return {"Type": "bind", "Source": source, "Destination": destination} + + +class HostPathTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + p = patch.object(executor, "_validate_path", side_effect=lambda v, label: Path(v)) + p.start() + self.addCleanup(p.stop) + + def _engine(self, containers_): + fake = FakeEngine(routes={("GET", "/containers/json"): (200, containers_)}) + self.addCleanup(fake.close) + p = patch.object(executor, "_engine", return_value=engine.EngineClient(fake.path)) + p.start() + self.addCleanup(p.stop) + return fake + + def test_existing_path_is_unchanged(self): + existing = self.root / "docker-compose.yml" + existing.write_text("services: {}\n") + fake = self._engine([{"Id": "p", "Mounts": [_mount(str(self.root / "host"), "/data")]}]) + self.assertEqual(containers._host_path(str(existing)), str(existing)) + self.assertFalse([r for r in fake.requests if "/containers/json" in r[1]]) + + def test_maps_through_the_containers_mount(self): + host = self.root / "host" / "compose" / "49" + host.mkdir(parents=True) + compose = host / "docker-compose.yml" + compose.write_text("services:\n searxng:\n image: searxng/searxng\n") + self._engine([{"Id": "p", "Mounts": [_mount(str(self.root / "host"), "/data"), + _mount("/var/run/docker.sock", "/var/run/docker.sock")]}]) + self.assertEqual(containers._host_path(PORTAINER_COMPOSE), str(compose)) + + def test_longest_destination_wins(self): + a = self.root / "a" / "compose" / "49" + b = self.root / "b" / "49" + a.mkdir(parents=True) + b.mkdir(parents=True) + (b / "docker-compose.yml").write_text("services: {}\n") + self._engine([{"Id": "p", "Mounts": [_mount(str(self.root / "b"), "/data/compose"), + _mount(str(self.root / "a"), "/data")]}]) + self.assertEqual(containers._host_path(PORTAINER_COMPOSE), str(b / "docker-compose.yml")) + + def test_unexplained_path_is_unchanged(self): + self._engine([{"Id": "p", "Mounts": [_mount(str(self.root / "elsewhere"), "/srv")]}]) + self.assertEqual(containers._host_path(PORTAINER_COMPOSE), PORTAINER_COMPOSE) + + def test_a_mount_that_maps_to_nothing_keeps_the_label_path(self): + self._engine([{"Id": "p", "Mounts": [_mount(str(self.root / "host"), "/data")]}]) + self.assertEqual(containers._host_path(PORTAINER_COMPOSE), PORTAINER_COMPOSE) + + def test_engine_error_is_unchanged(self): + """An agent with no answering engine keeps the label path — the + caller's error then names it. _mount_lookup swallows nothing else.""" + from vigil_agent.engine import EngineError + for boom in (RuntimeError("No container engine found (Docker or Podman socket)"), + EngineError("engine refused", status=500), + OSError("socket gone"), + ValueError("not json")): + p = patch.object(executor, "_engine", side_effect=boom) + p.start() + self.addCleanup(p.stop) + self.assertEqual(containers._host_path(PORTAINER_COMPOSE), PORTAINER_COMPOSE) + + def test_the_labelled_path_is_what_an_unexplained_failure_names(self): + """The fallback of last resort is the label itself, so an admin who + hits it sees the path Portainer's own stack entry shows.""" + self._engine([{"Id": "p", "Mounts": []}]) + self.assertEqual(containers._host_path("/data/compose/49/.env"), + "/data/compose/49/.env") + self.assertEqual(containers._label_env_files( + {"com.docker.compose.project.environment_file": "/data/compose/49/stack.env"}), []) + + +class PortainerActionTests(unittest.TestCase): + """The actions the cphmserv report hit, end to end through the seams: the + argv that reaches compose carries host paths, never a container's.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.host = Path(self.tmp.name) / "portainer_data" + stack = self.host / "compose" / "49" + stack.mkdir(parents=True) + (stack / "docker-compose.yml").write_text("services:\n searxng:\n image: searxng/searxng\n") + (stack / "stack.env").write_text("SEARXNG_SECRET=abc\n") + self.compose = stack / "docker-compose.yml" + self.env = stack / "stack.env" + self.fake = FakeEngine(routes={("GET", "/containers/json"): (200, [ + {"Id": "p", "Mounts": [_mount(str(self.host), "/data"), + _mount("/var/run/docker.sock", "/var/run/docker.sock")]}])}) + self.addCleanup(self.fake.close) + self.calls = [] + for name, value in (("_engine", engine.EngineClient(self.fake.path)), + ("_compose_cmd", ["docker", "compose"]), + ("_compose_env", {})): + p = patch.object(executor, name, return_value=value) + p.start() + self.addCleanup(p.stop) + p = patch.object(executor, "_run", side_effect=lambda cmd, timeout=60, extra_env=None: + self.calls.append(cmd) or "ok") + p.start() + self.addCleanup(p.stop) + p = patch.object(executor, "_validate_path", side_effect=lambda v, label: Path(v)) + p.start() + self.addCleanup(p.stop) + p = patch.object(executor.collector, "request_docker_recheck") + p.start() + self.addCleanup(p.stop) + + def test_portainer_container_rollback(self): + spec = {"Config": { + "Image": "searxng/searxng:latest", + "Labels": {"com.docker.compose.project": "searxng", + "com.docker.compose.service": "searxng", + "com.docker.compose.project.config_files": PORTAINER_COMPOSE, + "com.docker.compose.project.working_dir": PORTAINER_WD, + "com.docker.compose.project.environment_file": PORTAINER_ENV}}} + with patch.object(executor, "_docker_inspect", return_value=spec): + executor._container_rollback({"container_name": "searxng", + "image": "searxng/searxng:2025-09-17"}, _CFG) + + cmd = self.calls[0] + override = str(self.host / "compose" / "49" / containers.ROLLBACK_OVERRIDE) + self.assertEqual( + cmd, + ["docker", "compose", "-p", "searxng", "--project-directory", + str(self.host / "compose" / "49"), "--env-file", str(self.env), + "-f", str(self.compose), "-f", override, "up", "-d", "--no-deps", "searxng"]) + self.assertFalse([a for a in cmd if a.startswith("/data/")], + f"a container path leaked into the host's argv: {cmd!r}") + self.assertEqual((self.host / "compose" / "49" / containers.ROLLBACK_OVERRIDE).read_text(), + "# Written by Vigil: searxng rolled back. The next update removes it.\n" + "services:\n searxng:\n image: \"searxng/searxng:2025-09-17\"\n") + + def test_portainer_stack_compose_args(self): + self.fake.routes[("GET", "/containers/json")] = (200, [{ + "Id": "s", + "Labels": {"com.docker.compose.project": "searxng", + "com.docker.compose.project.config_files": PORTAINER_COMPOSE, + "com.docker.compose.project.working_dir": PORTAINER_WD, + "com.docker.compose.project.environment_file": PORTAINER_ENV}, + "Mounts": [_mount(str(self.host), "/data")]}]) + self.assertEqual( + containers._stack_compose_args("searxng"), + ["-p", "searxng", "--project-directory", str(self.host / "compose" / "49"), + "--env-file", str(self.env), "-f", str(self.compose)]) + + def test_portainer_update_container(self): + spec = {"Image": "sha256:old", "Config": { + "Image": "searxng/searxng:latest", + "Labels": {"com.docker.compose.project": "searxng", + "com.docker.compose.service": "searxng", + "com.docker.compose.project.config_files": PORTAINER_COMPOSE, + "com.docker.compose.project.working_dir": PORTAINER_WD, + "com.docker.compose.project.environment_file": PORTAINER_ENV}}} + with patch.object(executor, "_docker_inspect", return_value=spec), \ + patch.object(executor, "_pull", return_value=""), \ + patch.object(executor, "_image_id", return_value="sha256:new"), \ + patch.object(executor, "_container_image_id", return_value="sha256:new"): + executor._update_container({"container_name": "searxng"}, _CFG) + + self.assertEqual(len(self.calls), 2) + tail = ["--project-directory", str(self.host / "compose" / "49"), + "--env-file", str(self.env), "-f", str(self.compose)] + self.assertEqual(self.calls[0], ["docker", "compose", *tail, "pull", "searxng"]) + self.assertEqual(self.calls[1], ["docker", "compose", *tail, + "up", "-d", "--no-deps", "searxng"]) + for cmd in self.calls: # both compose runs carry the host's argv + self.assertFalse([a for a in cmd if a.startswith("/data/")], + f"a container path leaked into the host's argv: {cmd!r}") + # compose writes the labels from the paths it is handed, so the same + # re-up has to carry the env file — otherwise it interpolates blanks + # and recreates every service of the stack. + self.assertEqual(tail.count("--env-file"), 1) + self.assertEqual(tail[tail.index("--env-file") + 1], str(self.env)) + + +if __name__ == "__main__": + unittest.main() diff --git a/agent/tests/test_stack_actions.py b/agent/tests/test_stack_actions.py index 8e9923d7..95bfb17a 100644 --- a/agent/tests/test_stack_actions.py +++ b/agent/tests/test_stack_actions.py @@ -48,7 +48,9 @@ def test_restart_uses_the_labels(self): "-f", "/opt/media/compose.yaml", "-f", "/opt/media/extra.yaml", "restart"]) self.assertEqual(env, {"DOCKER_HOST": f"unix://{self.fake.path}"}) self.assertEqual(out.data, {"project": "media"}) - query = self.fake.requests[-1][1] + # The mount lookup QA-07 adds is also a /containers/json call, so the + # filtered request this asserts on is found by name, not by position. + query = [q for _, q, _ in self.fake.requests if "filters=" in (q or "")][-1] self.assertIn("filters=", query) self.assertIn("com.docker.compose.project%3Dmedia", query) diff --git a/agent/tests/test_stack_read.py b/agent/tests/test_stack_read.py index 2ef13714..2e04325f 100644 --- a/agent/tests/test_stack_read.py +++ b/agent/tests/test_stack_read.py @@ -31,6 +31,7 @@ def setUp(self): "com.docker.compose.config-hash": h, "com.docker.compose.project.config_files": str(self.dir / "docker-compose.yml"), "com.docker.compose.project.working_dir": str(self.dir)} + self.labels = labels("web", "h-web") self.fake = FakeEngine(routes={("GET", "/containers/json"): (200, [ {"Id": "a", "Labels": labels("web", "h-web")}, {"Id": "b", "Labels": labels("db", "h-old")}])}) self.addCleanup(self.fake.close) @@ -59,6 +60,18 @@ def test_reads_posts_and_keeps_secrets_out_of_the_result(self): self.assertEqual(out.data, {"project": "shop", "would_recreate": 1}) self.assertEqual(self.run_mock.call_args.args[0][-3:], ["config", "--hash", "*"]) + def test_reads_env_from_the_recorded_env_file(self): + """Portainer passes --env-file stack.env; adopt reads that, not .env.""" + (self.dir / "stack.env").write_text("A=1\n") + self.fake.routes[("GET", "/containers/json")] = (200, [ + {"Id": "a", "Labels": {**self.labels, + "com.docker.compose.project.environment_file": + str(self.dir / "stack.env")}}]) + (self.dir / ".env").unlink() + with patch.object(client, "post_stack_read") as post: + stacks._stack_read({"project": "shop", "adopt_ticket": TICKET}, _CFG) + self.assertEqual(post.call_args.args[2]["env"], "A=1\n") + def test_refusals(self): with patch.object(client, "post_stack_read") as post: for params in ({"project": "shop", "adopt_ticket": "nope"}, {"project": "Shop", "adopt_ticket": TICKET}): diff --git a/agent/tests/test_update_container.py b/agent/tests/test_update_container.py index 121d16f1..5359d9ed 100644 --- a/agent/tests/test_update_container.py +++ b/agent/tests/test_update_container.py @@ -18,7 +18,7 @@ import tempfile import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -68,6 +68,8 @@ def fake_run(cmd, timeout=60, extra_env=None): stack.enter_context(patch.object(executor, "_container_image_id", return_value=after)) stack.enter_context(patch.object(executor, "_compose_cmd", return_value=["docker", "compose"])) stack.enter_context(patch.object(executor, "_compose_env", return_value={})) + stack.enter_context(patch.object( + executor, "_engine", return_value=MagicMock(get=MagicMock(return_value=[])))) mock = stack.enter_context(patch.object(executor, "_recreate_container", return_value="recreated")) if recreate else None yield mock diff --git a/agent/vigil_agent/actions/containers.py b/agent/vigil_agent/actions/containers.py index cf1345ab..44e85c36 100644 --- a/agent/vigil_agent/actions/containers.py +++ b/agent/vigil_agent/actions/containers.py @@ -28,6 +28,22 @@ def _engine(): return ex._engine() +def _mount_lookup(): + """Every running container, for the mount lookup ``_host_path`` needs. + Unfiltered: the container holding the mount (Portainer itself) carries no + label of this project, so a project filter could never find it. None when + there is no engine to ask — the caller's error then names the label path, + which is what the admin can match. One lookup serves every path of an + action; ``_host_path`` and ``_label_env_files`` take it as ``known``.""" + try: + return _engine().get("/containers/json") or [] + except (RuntimeError, OSError, ValueError): + # no engine at all, or one that won't answer (EngineError is a + # RuntimeError): no mapping. The caller's error still names the + # labelled path, which is what the admin can see. + return None + + def _lifecycle(params: dict, verb: str) -> ActionOutput: name = _validate_name( params.get("container_name") or params.get("container_id", ""), @@ -368,15 +384,16 @@ def _update_container(params: dict, _config: AgentConfig) -> str: f"compose labels are missing config files — use " f"docker_compose_up with an explicit compose_file" ) - compose_file = str(ex._validate_path(compose_file, "compose_file")) + known = _mount_lookup() + compose_file = str(ex._validate_path(_host_path(compose_file, known), "compose_file")) - project_dir = labels.get("com.docker.compose.project.working_dir") or "" + project_dir = _host_path(labels.get("com.docker.compose.project.working_dir") or "", known) dir_args = [] if project_dir: dir_args = ["--project-directory", str(ex._validate_path(project_dir, "project directory"))] _clear_rollback(project_dir) # an update ends a rollback - cmd = [*ex._compose_cmd(), *dir_args, "-f", compose_file] + cmd = [*ex._compose_cmd(), *dir_args, *_label_env_files(labels, known), "-f", compose_file] ex._run(cmd + ["pull", service], timeout=600, extra_env=ex._compose_env()) ex._run(cmd + ["up", "-d", "--no-deps", service], timeout=300, extra_env=ex._compose_env()) @@ -480,16 +497,64 @@ def _stack_compose_args(project: str) -> list[str]: if not files: continue args = ["-p", project] - workdir = labels.get("com.docker.compose.project.working_dir") or "" + # One engine lookup serves the workdir, the env files and the compose files. + known = _mount_lookup() + workdir = _host_path(labels.get("com.docker.compose.project.working_dir") or "", known) if workdir: args += ["--project-directory", str(ex._validate_path(workdir, "project directory"))] + args += _label_env_files(labels, known) for f in files: - args += ["-f", str(ex._validate_path(f, "compose file"))] + args += ["-f", str(ex._validate_path(_host_path(f, known), "compose file"))] return args raise ValueError(f"No containers of stack {project!r} carry compose file labels — " f"deploy it with docker_compose_up and an explicit compose_file") +def _host_path(raw: str, known=None) -> str: + """The host's path for a path compose wrote into a label. + + A stack started by a manager that runs compose in its own container + (Portainer: /data/compose//docker-compose.yml) carries that container's + paths. Map them through the mount that holds them; a path that already + exists, or that no mount explains, comes back unchanged. ``known`` is the + caller's ``_mount_lookup()``, so one action asks the engine once. + """ + if not raw or Path(raw).exists(): + return raw + target = Path(raw) + if not target.is_absolute() or ".." in target.parts: + # a relative label can be read as escaping the mount it maps through; + # leave it to the caller, whose error names the labelled path + return raw + if known is None: + known = _mount_lookup() + if known is None: + return raw + best = None + for c in known: + for m in c.get("Mounts") or []: + dest, src = str(m.get("Destination") or ""), str(m.get("Source") or "") + if not dest.startswith("/") or not src: + continue + d = Path(dest) + if (target == d or d in target.parents) and (best is None or len(d.parts) > len(best[0].parts)): + best = (d, Path(src)) + if best is None: + return raw + mapped = best[1] / target.relative_to(best[0]) + return str(mapped) if mapped.exists() else raw + + +def _label_env_files(labels: dict, known=None) -> list[str]: + """``--env-file`` args for the env files compose recorded, mapped to the host.""" + args = [] + for f in str(labels.get("com.docker.compose.project.environment_file") or "").split(","): + f = _host_path(f.strip(), known) + if f and Path(f).is_file(): + args += ["--env-file", str(ex._validate_path(f, "env file"))] + return args + + def _stack_restart(params: dict, _config: AgentConfig) -> str: project = str(params.get("project") or "") args = _stack_compose_args(project) @@ -548,7 +613,8 @@ def _container_rollback(params: dict, _config: AgentConfig) -> str: if project: service = labels.get("com.docker.compose.service") or name _validate_name(service, "service name") - workdir = labels.get("com.docker.compose.project.working_dir") or "" + known = _mount_lookup() + workdir = _host_path(labels.get("com.docker.compose.project.working_dir") or "", known) files = [f.strip() for f in str(labels.get("com.docker.compose.project.config_files") or "") .split(",") if f.strip() and not f.strip().endswith(ROLLBACK_OVERRIDE)] if not files or not workdir: @@ -557,9 +623,10 @@ def _container_rollback(params: dict, _config: AgentConfig) -> str: override = wd / ROLLBACK_OVERRIDE override.write_text(f"# Written by Vigil: {service} rolled back. The next update removes it.\n" f"services:\n {service}:\n image: {json.dumps(image)}\n") - cmd = [*ex._compose_cmd(), "-p", project, "--project-directory", str(wd)] + cmd = [*ex._compose_cmd(), "-p", project, "--project-directory", str(wd), + *_label_env_files(labels, known)] for f in files: - cmd += ["-f", str(ex._validate_path(f, "compose file"))] + cmd += ["-f", str(ex._validate_path(_host_path(f, known), "compose file"))] cmd += ["-f", str(override), "up", "-d", "--no-deps", service] ex._run(cmd, timeout=300, extra_env=ex._compose_env()) via = "compose override" diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 680850b7..4a5d8803 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -159,7 +159,8 @@ def _stack_read(params: dict, config: AgentConfig) -> str: compose_path = Path(files[0]) workdir = Path(args[args.index("--project-directory") + 1]) if "--project-directory" in args \ else compose_path.parent - env_path = workdir / ".env" + env_path = Path(args[args.index("--env-file") + 1]) if "--env-file" in args \ + else workdir / ".env" payload = { "project": project, "compose_file": compose_path.name, From 4e6f2d9e5aee2a5820f68f74940f8da9c75e4186 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Tue, 6 Oct 2026 20:54:20 -0400 Subject: [PATCH 02/68] fix(agent): a managed agent left on the monitor-mode service says so instead of failing every task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host installed in monitor mode and switched to managed afterwards keeps the monitor unit: User=vigil-agent under ProtectSystem=strict. Every task it accepted then died on its first write — Update Agent with "[Errno 30] Read-only file system: '/usr/local/bin/...'". The agent now rejects each task up front with the fix (re-run the installer), reports runs_as_root at check-in, and the server counts such a host as refusing every action, so the deploy dialog's "will refuse" chip and skip logic catch it before sending. --- agent/tests/test_checkin_allowlist.py | 13 ++++++++ agent/tests/test_privilege_mismatch_advice.py | 29 ++++++++++++++++- agent/vigil_agent/__main__.py | 32 +++++++++++++++---- agent/vigil_agent/client.py | 7 ++++ .../0025_host_agent_runs_as_root.py | 18 +++++++++++ server/apps/hosts/models.py | 3 ++ server/apps/hosts/serializers.py | 1 + server/apps/hosts/views.py | 3 ++ server/apps/tasks/refusals.py | 3 ++ server/apps/tasks/test_refusals.py | 32 +++++++++++++++++++ 10 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 server/apps/hosts/migrations/0025_host_agent_runs_as_root.py diff --git a/agent/tests/test_checkin_allowlist.py b/agent/tests/test_checkin_allowlist.py index 3439cb10..7d0d389e 100644 --- a/agent/tests/test_checkin_allowlist.py +++ b/agent/tests/test_checkin_allowlist.py @@ -54,6 +54,19 @@ def test_empty_allowlist_is_sent(self): client.checkin(cfg, metrics=[]) self.assertEqual(self.sent["allowlist"], []) + def test_payload_says_whether_it_runs_as_root(self): + """QA-08: a managed agent on the monitor-mode unit runs unprivileged, + and the server needs that fact to warn before a deploy.""" + with tempfile.TemporaryDirectory() as tmp: + cfg = AgentConfig(server_url="http://127.0.0.1:1", agent_token="t" * 40, + data_dir=Path(tmp), mode="managed") + with patch.object(client.os, "geteuid", return_value=0): + client.checkin(cfg, metrics=[]) + self.assertIs(self.sent["runs_as_root"], True) + with patch.object(client.os, "geteuid", return_value=1000): + client.checkin(cfg, metrics=[]) + self.assertIs(self.sent["runs_as_root"], False) + if __name__ == "__main__": unittest.main() diff --git a/agent/tests/test_privilege_mismatch_advice.py b/agent/tests/test_privilege_mismatch_advice.py index 1d6cc058..2906f6de 100644 --- a/agent/tests/test_privilege_mismatch_advice.py +++ b/agent/tests/test_privilege_mismatch_advice.py @@ -24,7 +24,7 @@ import unittest from unittest.mock import patch -from vigil_agent.__main__ import _warn_on_privilege_mismatch +from vigil_agent.__main__ import _privilege_mismatch, _process_tasks, _warn_on_privilege_mismatch class _Cfg: @@ -73,5 +73,32 @@ def test_the_advice_does_not_stop_at_removing_user(self): ) +class PrivilegeMismatchRejection(unittest.TestCase): + """QA-08: the warning is invisible from the UI, and the task it warns about + dies on '[Errno 30] Read-only file system' one Errno at a time.""" + + def test_managed_as_non_root_rejects_every_task(self): + tasks = [{"id": "1", "action": "update_agent"}, {"id": "2", "action": "check_service"}] + with patch("vigil_agent.__main__.os.geteuid", return_value=1000): + with patch("vigil_agent.__main__.execute_action") as run, \ + patch("vigil_agent.__main__._report_rejected") as rejected: + _process_tasks(tasks, _Cfg("managed"), None, object()) + self.assertEqual(rejected.call_count, 2, + "every task the agent cannot run must be rejected, not run") + run.assert_not_called() + for call in rejected.call_args_list: + message = call.args[2] + self.assertIn("install.sh", message, + "the only complete fix is re-running the installer") + self.assertIn("managed", message) + + def test_root_managed_is_not_rejected_for_privilege(self): + with patch("vigil_agent.__main__.os.geteuid", return_value=0): + self.assertFalse(_privilege_mismatch(_Cfg("managed"))) + with patch("vigil_agent.__main__.os.geteuid", return_value=1000): + self.assertFalse(_privilege_mismatch(_Cfg("monitor")), + "monitor mode needs no root — it must keep monitoring") + + if __name__ == "__main__": unittest.main() diff --git a/agent/vigil_agent/__main__.py b/agent/vigil_agent/__main__.py index 752604ea..f7bb181a 100644 --- a/agent/vigil_agent/__main__.py +++ b/agent/vigil_agent/__main__.py @@ -61,6 +61,13 @@ def _process_tasks(tasks: list[dict], config, nonce_store: NonceStore, verify_ke ) return + if _privilege_mismatch(config): + # Better one clear rejection now than the task dying on Errno 30 + # halfway through a write (QA-08). + for task in tasks: + _report_rejected(config, task, PRIVILEGE_MISMATCH.format(mode=config.mode)) + return + if verify_key is None: if tasks: logger.warning( @@ -578,6 +585,24 @@ def _report_skipped(config, task: dict, output: str, steps=None) -> None: logger.exception("Failed to report task %s skip", task.get("id")) +#: What a task gets back when this agent cannot run it. +PRIVILEGE_MISMATCH = ( + "This agent's mode ({mode}) runs tasks as root, but its service still runs it as the " + "unprivileged 'vigil-agent' user under the monitor-mode sandbox (read-only /usr and /etc), " + "so every task would fail. Re-run the installer on this host to regenerate the service for " + "its mode: curl -fsSL /agent/install.sh | sudo bash") + + +def _privilege_mismatch(config) -> bool: + """True when the mode needs root and this process is not root (POSIX only).""" + if config.mode == "monitor": + return False + try: + return os.geteuid() != 0 + except AttributeError: # Windows has no geteuid + return False + + def _warn_on_privilege_mismatch(config) -> None: """Say so when the mode needs root and this process does not have it. @@ -604,12 +629,7 @@ def _warn_on_privilege_mismatch(config) -> None: A warning, not a refusal: an agent that stops monitoring because it cannot execute is worse than one that monitors and says it cannot execute. """ - if config.mode == "monitor": - return - try: - if os.geteuid() == 0: - return - except AttributeError: # Windows has no geteuid + if not _privilege_mismatch(config): return logger.warning( diff --git a/agent/vigil_agent/client.py b/agent/vigil_agent/client.py index f23ab0e5..78ab2b39 100644 --- a/agent/vigil_agent/client.py +++ b/agent/vigil_agent/client.py @@ -4,6 +4,7 @@ """ import logging +import os import platform import socket @@ -72,6 +73,12 @@ def checkin( "allowlist": sorted(config.allowlist), "allow_reprovision": bool(config.allow_reprovision), } + # A managed agent left on the monitor-mode unit runs unprivileged and fails + # every task; the server marks such a host as refusing them (QA-08). + try: + payload["runs_as_root"] = os.geteuid() == 0 + except AttributeError: # Windows + pass # The ingest distinguishes an absent key (agent too old to report it — # stored value left alone) from an explicit False, so the key is only # sent when the probe produced a value. diff --git a/server/apps/hosts/migrations/0025_host_agent_runs_as_root.py b/server/apps/hosts/migrations/0025_host_agent_runs_as_root.py new file mode 100644 index 00000000..1476191a --- /dev/null +++ b/server/apps/hosts/migrations/0025_host_agent_runs_as_root.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-10-07 00:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('hosts', '0024_merge_0018_host_machine_id_0023_container_rollback'), + ] + + operations = [ + migrations.AddField( + model_name='host', + name='agent_runs_as_root', + field=models.BooleanField(blank=True, null=True), + ), + ] diff --git a/server/apps/hosts/models.py b/server/apps/hosts/models.py index a725d440..60a7fdfc 100644 --- a/server/apps/hosts/models.py +++ b/server/apps/hosts/models.py @@ -53,6 +53,9 @@ def save(self, *args, **kwargs): # agent new enough to report it checks in — unknown, which is not "refuses". agent_allowlist = models.JSONField(null=True, blank=True) agent_allow_reprovision = models.BooleanField(default=False) + # Whether the agent's process runs as root, as it last reported it (QA-08): + # None until an agent new enough to report it checks in. + agent_runs_as_root = models.BooleanField(null=True, blank=True) last_checkin = models.DateTimeField(null=True, blank=True) #: What this machine is missing, as the agent last counted it: #: {"pending", "critical", "important", "reboot_required"}. Null means no diff --git a/server/apps/hosts/serializers.py b/server/apps/hosts/serializers.py index 6d9ffe52..e54e4a78 100644 --- a/server/apps/hosts/serializers.py +++ b/server/apps/hosts/serializers.py @@ -27,6 +27,7 @@ class Meta: "agent_version", "agent_allowlist", "agent_allow_reprovision", + "agent_runs_as_root", "reboot_required", "last_checkin", "created_at", diff --git a/server/apps/hosts/views.py b/server/apps/hosts/views.py index b9cb7f7a..9ef2bdb0 100644 --- a/server/apps/hosts/views.py +++ b/server/apps/hosts/views.py @@ -432,6 +432,9 @@ def checkin(request): host.agent_allowlist = sorted(set(raw_allow)) if isinstance(data.get("allow_reprovision"), bool): host.agent_allow_reprovision = data["allow_reprovision"] + # Absent means an agent too old to report it (or Windows) — keep what we knew. + if isinstance(data.get("runs_as_root"), bool): + host.agent_runs_as_root = data["runs_as_root"] if data.get("features") is not None: # A list of short strings the agent understands (task-language diff --git a/server/apps/tasks/refusals.py b/server/apps/tasks/refusals.py index 93513391..424b6e22 100644 --- a/server/apps/tasks/refusals.py +++ b/server/apps/tasks/refusals.py @@ -103,6 +103,9 @@ def refused_actions(host, action_types: set[str]) -> list[str]: ``[]`` when the agent has never reported an allowlist: unknown is not the same as refused, and an old agent must not look broken. """ + if host.agent_runs_as_root is False and host.mode != "monitor": + # Its service is still the monitor-mode unit: it can run nothing (QA-08). + return sorted(set(action_types) & AGENT_ACTIONS) if host.agent_allowlist is None: return [] mode = host.mode diff --git a/server/apps/tasks/test_refusals.py b/server/apps/tasks/test_refusals.py index e3737f6c..f927772a 100644 --- a/server/apps/tasks/test_refusals.py +++ b/server/apps/tasks/test_refusals.py @@ -59,6 +59,26 @@ def test_refused_actions_per_mode(self): # An agent that never reported an allowlist is unknown, not refusing. self.assertEqual(refused_actions(_host("m6", allowlist=None), wanted), []) + def test_unprivileged_managed_host_refuses_everything(self): + """QA-08: an agent still running under the monitor-mode unit's sandbox + fails every task it accepts, so it refuses all of them up front.""" + wanted = {"check_service", "hunt_service", "run_command", "reprovision_stage", "playbook"} + every = sorted(wanted & AGENT_ACTIONS) + + host = _host("p1", allowlist=("check_service", "hunt_service"), reprovision=True) + host.agent_runs_as_root = False + host.save(update_fields=["agent_runs_as_root"]) + self.assertEqual(refused_actions(host, wanted), every) + + host.mode = Host.Mode.FULL_CONTROL + host.save(update_fields=["mode"]) + self.assertEqual(refused_actions(host, wanted), every) + + # Unknown (None) is not "refuses" — an old agent must not look broken. + host.agent_runs_as_root = None + host.save(update_fields=["agent_runs_as_root"]) + self.assertEqual(refused_actions(host, wanted), []) + def test_task_action_types_includes_relevant_probes(self): self.assertEqual(task_action_types(parse_and_validate(RELEVANT)), {"check_service", "hunt_service"}) @@ -126,3 +146,15 @@ def test_checkin_stores_allowlist(self): self._checkin(host) # absent: kept host.refresh_from_db() self.assertEqual(host.agent_allowlist, ["check_service", "hunt_file"]) + + def test_checkin_stores_runs_as_root(self): + """QA-08: the server needs the agent's privilege to warn about a host + whose service is still the monitor-mode unit.""" + host = _host("rr") + self._checkin(host, runs_as_root=False) + host.refresh_from_db() + self.assertIs(host.agent_runs_as_root, False) + self._checkin(host, runs_as_root="no") # malformed: kept + self._checkin(host) # absent: kept + host.refresh_from_db() + self.assertIs(host.agent_runs_as_root, False) From 3c4179b83c07675d6bf27e093f33b3476ec55a88 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Tue, 6 Oct 2026 23:37:18 -0400 Subject: [PATCH 03/68] feat(apps): the Apps page explains itself; "By host" opens rows in place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page assumed an operator already knew that "unmanaged" means "seen only in Windows' Apps & features, reported by no package manager" — which is exactly the thing that needs explaining, because it is why such an item can never show as outdated. Each tab now opens with one line saying what it lists, and the unmanaged, outdated and decision columns carry the existing .field-hint markers, so the copy is one hover away instead of a footnote at the bottom of one tab. "By host" jumped to the Monitor page, which read as a broken link: the tab promises a per-host software list, so the row expands in place the way a "By app" row does, fetching /software/hosts// once and caching it. The jump to Monitor is still there, now as an explicit link inside the row. MUTATION RED: removing data-app-host-row from the row template turns test_by_host_expands_in_place red; restored and green. End-to-end (rendered template, not source scan): field-hint markers: 8 apps-intro: 3 upd-note: 0 hints: 3x unmanaged (Apps & features / cannot upgrade or uninstall, paragraphs intact), 1 outdated, 3 bulk decisions, 1 decision column (Approved / Declined / Undecided). Ran 2182 tests in 112.371s — OK (skipped=1); manage.py check: no issues. --- server/apps/software/test_apps_page.py | 68 ++++++++++++++- server/static/css/vigil.css | 11 ++- server/static/js/vigil-apps.js | 110 +++++++++++++++++++++++-- server/templates/pages/_apps.html | 22 ++--- 4 files changed, 190 insertions(+), 21 deletions(-) diff --git a/server/apps/software/test_apps_page.py b/server/apps/software/test_apps_page.py index a950c907..b31f3ebb 100644 --- a/server/apps/software/test_apps_page.py +++ b/server/apps/software/test_apps_page.py @@ -3,10 +3,12 @@ The UI has no JS runner (the architect drives it in Chromium), so its wiring is pinned by source scan: the ids and calls the page depends on. """ +import html +import re from pathlib import Path from django.contrib.auth import get_user_model -from django.test import TestCase +from django.test import SimpleTestCase, TestCase from rest_framework.test import APIClient from apps.accounts.models import Role, UserProfile @@ -110,3 +112,67 @@ def test_apps_js_escapes_api_values(self): f"{field} reaches innerHTML unescaped") self.assertRegex(js, r"/software/apps/\$\{encodeURIComponent\(", "name_key must be URL-encoded in the app detail URL") + + +class AppsExplainedTests(SimpleTestCase): + """The page explains itself, and "By host" expands instead of navigating away.""" + + def _rendered_hints(self): + """The Apps page as the browser receives it, plus its decoded hints.""" + from django.template.loader import render_to_string + + rendered = render_to_string("pages/_apps.html", {}) + # The hint names the Windows "Apps & features" setting; the attribute + # carries it escaped, so decode it the way the browser would. + return rendered, [html.unescape(h) + for h in re.findall(r'data-hint="([^"]*)"', rendered)] + + def test_unmanaged_is_explained(self): + _rendered, hints = self._rendered_hints() + with_manager = [h for h in hints if "package manager" in h] + self.assertGreaterEqual(len(with_manager), 3, + "the unmanaged filter and both Unmanaged columns explain it") + self.assertTrue(any("cannot upgrade or uninstall" in h for h in hints), + "unmanaged must say what Vigil cannot do") + + def test_decisions_are_explained(self): + _rendered, hints = self._rendered_hints() + for phrase in ("without waiting out", "exclude list", "goes back to its own rules"): + self.assertTrue(any(phrase in h for h in hints), phrase) + self.assertNotIn("upd-note", _source("templates", "pages", "_apps.html"), + "the bottom note is replaced by the hints") + + def test_each_tab_has_an_intro(self): + rendered, _hints = self._rendered_hints() + self.assertEqual(rendered.count('class="apps-intro"'), 3, + "by app, by host and by update each open with one line") + + def test_by_host_expands_in_place(self): + js = _source("static", "js", "vigil-apps.js") + for literal in ('data-app-host-row="${', "function toggleHostDetail", + "/api/v1/software/hosts/", "data-host-show-all="): + self.assertIn(literal, js, literal) + row_handler = re.search(r"delegateClick\('\[data-app-host-row\]',[\s\S]*?\n\}\);", js) + self.assertIsNotNone(row_handler, "the host row is not wired through delegateClick") + self.assertNotIn("navigateTo", row_handler.group(0), + "clicking a host row must expand it, not leave the page") + + def test_the_page_renders_its_new_markup(self): + """End-to-end: what the browser gets is what the hints promise. + + A bare ampersand inside an attribute would silently truncate the + sentence it appears in, so this counts the markers and reads their + decoded values rather than trusting the template source. + """ + from django.template.loader import render_to_string + + rendered = render_to_string("pages/_apps.html", {}) + self.assertEqual(rendered.count("upd-note"), 0) + self.assertEqual(rendered.count('class="field-hint"'), 8, + "every hint the page promises renders") + _rendered, hints = self._rendered_hints() + unmanaged = [h for h in hints if "Apps & features" in h] + self.assertEqual(len(unmanaged), 3, "the filter and both Unmanaged columns") + self.assertTrue(all("cannot upgrade or uninstall" in h for h in unmanaged)) + self.assertTrue(all("\n\n" in h for h in unmanaged), + "the unmanaged hint's paragraphs must survive into the attribute") diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index efc0275c..8c3ac1f7 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -3765,13 +3765,22 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } .apps-host-toolbar { margin-bottom: 10px; } .apps-host-toolbar .form-control { max-width: 320px; font-size: 12px; } .apps-show-all { margin-top: 10px; } +.apps-intro { font-size: 13px; color: var(--text-3); margin: 4px 0 14px; max-width: 72ch; } +.apps-chev { display: inline-block; color: var(--text-3); margin-right: 2px; + transition: transform 0.2s var(--ease-out-expo); } +.apps-row.open .apps-chev { transform: rotate(90deg); } +.apps-host-detail { margin: 0; } +.apps-host-detail th, .apps-host-detail td { font-size: 12.5px; } +.apps-host-open { display: inline-block; margin-top: 8px; font-size: 12.5px; + color: var(--sky-ink); text-decoration: none; } +.apps-host-open:hover { text-decoration: underline; } /* ── Apps: by update (M8) ────────────────────────────────────────────── */ .apps-table th.upd-check, .apps-table td.upd-check { width: 1%; padding-right: 4px; } .upd-title { font-size: 11px; color: var(--text-3); } .upd-bulk { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin: 0 0 10px; font-size: 12px; color: var(--text-2); } -.upd-note { margin-top: 10px; font-size: 12px; } +.upd-bulk .field-hint { margin-left: -2px; } .chip-mint { background: rgba(var(--rgb-mint), 0.14); color: var(--mint-ink); } .chip-lemon { background: rgba(var(--rgb-lemon), 0.16); color: var(--lemon-ink); } diff --git a/server/static/js/vigil-apps.js b/server/static/js/vigil-apps.js index 29666231..b364746e 100644 --- a/server/static/js/vigil-apps.js +++ b/server/static/js/vigil-apps.js @@ -4,7 +4,7 @@ // HTML: templates/pages/_apps.html, Software section in templates/pages/_monitor.html // Depends on: vigil-utils.js (escHtml, escAttr, timeAgo, delegateClick), // vigil-nav.js (navigateTo — wrapped below), vigil-monitor.js -// (selectMonitorHost, reached from a hostname link). +// (selectMonitorHost, reached from the "Open this host's monitor" link). // API: GET /api/v1/software/apps/, /api/v1/software/apps//, // /api/v1/software/hosts/, /api/v1/software/hosts//, // /api/v1/software/updates/, /api/v1/software/updates/hosts/; @@ -26,6 +26,9 @@ const appsState = { hostSeq: 0, details: {}, // name_key → host rows, fetched once per app; null = in flight expanded: {}, // name_key → true while its detail row is open + hostDetails: {}, // host_id → software items, fetched once per host; null = in flight + hostExpanded: {}, // host_id → true while its detail row is open + hostShowAll: {}, // host_id → true once "Show all" was clicked monitorItems: [], monitorSnapshot: null, monitorSeq: 0, @@ -209,19 +212,95 @@ async function fetchAppHosts() { function renderAppsHostsTable() { const tbody = document.getElementById('apps-hosts-body'); if (!tbody) return; - const html = appsState.hosts.map(row => ` - ${escHtml(row.hostname)} - ${row.item_count} - ${_countChip(row.outdated, 'chip-rose')} - ${_countChip(row.unmanaged, 'apps-chip-warn')} - ${escHtml(timeAgo(row.received_at) || 'never')} - ${_errorChips(row.errors) || '—'} - `).join(''); + const html = appsState.hosts.map(row => { + const open = appsState.hostExpanded[row.host_id] ? ' open' : ''; + let out = ` + ▸ ${escHtml(row.hostname)} + ${row.item_count} + ${_countChip(row.outdated, 'chip-rose')} + ${_countChip(row.unmanaged, 'apps-chip-warn')} + ${escHtml(timeAgo(row.received_at) || 'never')} + ${_errorChips(row.errors) || '—'} + `; + if (appsState.hostExpanded[row.host_id]) out += _hostDetailHtml(row.host_id); + return out; + }).join(''); tbody.innerHTML = html || _emptyRow('No hosts in your scope.'); renderAppsTable(); } +function _hostSoftwareSorted(items) { + // The Monitor page's Software list orders this way; By host's expanded rows match it. + return [...items].sort((a, b) => { + if (!!b.outdated !== !!a.outdated) return b.outdated ? 1 : -1; + if (!!b.managed !== !!a.managed) return b.managed ? 1 : -1; + return String(a.name || '').localeCompare(String(b.name || '')); + }); +} + +function _hostDetailHtml(id) { + const items = appsState.hostDetails[id]; + if (items === null || items === undefined) { + return ` +
Loading software…
`; + } + if (!items.length) { + return ` +
No software reported for this host yet.
`; + } + const sorted = _hostSoftwareSorted(items); + const showAll = !!appsState.hostShowAll[id]; + const shown = showAll ? sorted : sorted.slice(0, APPS_HOST_DETAIL_LIMIT); + const rows = shown.map(it => { + const latest = it.outdated + ? ` ${escHtml(it.latest_version)}` : ''; + const unmanaged = it.managed ? '' : ' unmanaged'; + return ` + ${escHtml(it.name)}${latest}${unmanaged} + ${escHtml(it.version)} + ${escHtml(it.latest_version)} + ${escHtml(it.source)} + ${escHtml(_scopeCell(it))} + `; + }).join(''); + const more = (!showAll && sorted.length > APPS_HOST_DETAIL_LIMIT) + ? `
+ +
` : ''; + return ` + + + ${rows} +
NameVersionLatestSourceScope
+ ${more} + Open this host's monitor → + `; +} + +async function toggleHostDetail(id) { + if (appsState.hostExpanded[id]) { + delete appsState.hostExpanded[id]; + renderAppsHostsTable(); + return; + } + appsState.hostExpanded[id] = true; + renderAppsHostsTable(); + if (appsState.hostDetails[id] !== undefined) return; + appsState.hostDetails[id] = null; + let items = []; + try { + const resp = await fetch(`/api/v1/software/hosts/${id}/`, { credentials: 'same-origin' }); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + items = (await resp.json()).items || []; + } catch { items = []; } + appsState.hostDetails[id] = items; + if (appsState.hostExpanded[id]) renderAppsHostsTable(); +} + function refreshApps() { + appsState.hostDetails = {}; + appsState.hostExpanded = {}; + appsState.hostShowAll = {}; fetchApps(true); fetchAppHosts(); fetchUpdates(); @@ -430,6 +509,9 @@ function renderHostSoftwareList() { const SOFTWARE_ROWS = 100; let _softwareShowAll = false; +// An expanded "By host" row shows this many packages; a Linux host reports ~1500. +const APPS_HOST_DETAIL_LIMIT = 50; + /* ── Wiring ──────────────────────────────────────────────────────────── */ let _appsSearchTimer = null; @@ -485,6 +567,16 @@ delegateClick('[data-upd-row]', (el, ev) => { delegateClick('[data-upd-decide]', (el) => decideUpdates(el.dataset.updDecide)); +delegateClick('[data-app-host-row]', (el, ev) => { + if (ev.target.closest('a, button')) return; + toggleHostDetail(el.dataset.appHostRow); +}); + +delegateClick('[data-host-show-all]', (el) => { + appsState.hostShowAll[el.dataset.hostShowAll] = true; + renderAppsHostsTable(); +}); + delegateClick('[data-app-host]', (el, ev) => { ev.preventDefault(); navigateTo('monitor'); diff --git a/server/templates/pages/_apps.html b/server/templates/pages/_apps.html index d1e45c03..d8748048 100644 --- a/server/templates/pages/_apps.html +++ b/server/templates/pages/_apps.html @@ -4,7 +4,7 @@
Apps.
-
Installed software across your hosts
+
Every piece of software your hosts report — by app, by host, or by the update it is missing
+

Each app once, with how many hosts have it and at which versions. Click a row to see the hosts.

- + +
+
By fix
+
By host
+
+ + {# By fix (M9) — findings grouped by the action that resolves them. #} +
+
+
+

One row per fix: deploying it resolves every finding listed under it, on every host.

+ +
+
-
-
-
- Loading vulnerability data… + {# By host — each host's score and open findings. #} +
+
+
+ Loading vulnerability data… +
From 43bed4a166418e4a28ef89c914cec0177fa12e1b Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Wed, 7 Oct 2026 10:35:37 -0400 Subject: [PATCH 06/68] =?UTF-8?q?fix(firewall):=20the=20page=20opens=20on?= =?UTF-8?q?=20a=20host=20instead=20of=20"Choose=20a=20host=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host control became a picker button plus a hidden , but _fwPopulateHosts still appended
+ + + + - + `; @@ -170,7 +175,7 @@ async function renderDockerContainers(hostId) { openContainerLogs(btn.dataset.host, btn.dataset.name); })); wrap.querySelectorAll('[data-stack-adopt]').forEach(btn => btn.addEventListener('click', async () => { - const totp = window.prompt(`Adopting ${btn.dataset.project} reads its compose file and .env into Vigil. Your TOTP code:`); + const totp = await totpPrompt(`Adopting ${btn.dataset.project} reads its compose file and .env into Vigil.`); if (!totp) return; try { await apiJson('/api/v1/stacks/adopt/', { method: 'POST', body: JSON.stringify( @@ -219,7 +224,7 @@ function _logEls() { } async function openContainerLogs(hostId, name) { - const totp = window.prompt(`Reading the logs of ${name} needs your TOTP code:`); + const totp = await totpPrompt(`Reading the logs of ${name} shows everything those containers printed.`); if (!totp) return; let opened; try { diff --git a/server/static/js/vigil-firewall.js b/server/static/js/vigil-firewall.js index b0a0b145..c2c1e354 100644 --- a/server/static/js/vigil-firewall.js +++ b/server/static/js/vigil-firewall.js @@ -510,7 +510,7 @@ function _fwRenderRules(hostId, data) { /* ── Writes ──────────────────────────────────────────────────────────── */ async function applyFirewallChange(hostId, action, params, btn) { - const totp = (window.prompt('Enter your TOTP code to apply this firewall change:') || '').trim(); + const totp = await totpPrompt('Apply this firewall change.'); if (!totp) return; const prevOpacity = btn ? btn.style.opacity : ''; if (btn) { btn.disabled = true; btn.style.opacity = '0.5'; } diff --git a/server/static/js/vigil-host-cards.js b/server/static/js/vigil-host-cards.js index f6e4e496..06b770dd 100644 --- a/server/static/js/vigil-host-cards.js +++ b/server/static/js/vigil-host-cards.js @@ -97,10 +97,7 @@ function openHostDetail(card) { async function forceUpdateAgent() { const hostId = panel.dataset.hostId; if (!hostId) return; - const totp = (window.prompt( - 'Force agent self-update on this host.\n\n' + - 'Enter your 6-digit 2FA code to authorize:' - ) || '').trim(); + const totp = await totpPrompt('Force the agent to self-update on this host. It applies on the next check-in.'); if (!totp) return; try { const resp = await fetch(`/api/v1/hosts/${hostId}/update-agent/`, { @@ -404,7 +401,7 @@ document.addEventListener('keydown', e => { /* ── Approve / reject / delete actions ───────────────────────────────── */ async function approveHost(hostId, btn) { - const totp = (window.prompt('Enter your 6-digit TOTP code to approve this host:') || '').trim(); + const totp = await totpPrompt('Approve this host.'); if (!totp) return; try { btn.disabled = true; diff --git a/server/static/js/vigil-policies.js b/server/static/js/vigil-policies.js index 7313bd18..732a4382 100644 --- a/server/static/js/vigil-policies.js +++ b/server/static/js/vigil-policies.js @@ -256,7 +256,7 @@ async function savePolicy() { if (wantHighRisk !== (current ? current.allow_high_risk : false)) { body.allow_high_risk = wantHighRisk; if (wantHighRisk) { - const code = window.prompt('Allowing unattended restarts needs your TOTP code:'); + const code = await totpPrompt('Allow unattended restarts from this policy.'); if (!code) return; body.totp = code.trim(); } diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index b0cc3df9..2b1987b6 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -155,7 +155,7 @@ async function saveStack() { } async function _stackTotpAction(path, payload, done) { - const code = window.prompt('This needs your TOTP code:'); + const code = await totpPrompt('This change runs on the host.'); if (!code) return null; try { return await apiJson(path, { method: 'POST', body: JSON.stringify({ ...payload, totp: code.trim() }) }); diff --git a/server/static/js/vigil-utils.js b/server/static/js/vigil-utils.js index 518f7ee7..2c084871 100644 --- a/server/static/js/vigil-utils.js +++ b/server/static/js/vigil-utils.js @@ -365,6 +365,58 @@ function promptModal(message, opts) { }); } +/* ── TOTP dialog (replaces the bare window.prompt the writes used) ────── */ +//: The reason is operator copy, not untrusted data: the dialog renders the +//: `reason` argument as plain text via textContent, so callers interpolate +//: freely without escaping. +function totpPrompt(reason, opts) { + opts = opts || {}; + return new Promise((resolve) => { + const m = mountModal('totp', { variant: 'm-pop' }); + m.setBody(` +
+ Confirm with your authenticator + +
+
+
+ +
+
+ + +
`); + m.modal.querySelector('#totp-reason').textContent = reason || ''; + const input = m.modal.querySelector('#totp-input'); + const okBtn = m.modal.querySelector('#totp-ok'); + okBtn.textContent = opts.confirmText || 'Confirm'; + okBtn.disabled = true; + + // Digits only, and never more than six: the pattern in the markup is a + // hint, this is the enforcement (a pasted "12 34 56" still yields 123456). + input.oninput = () => { + input.value = input.value.replace(/\D/g, '').slice(0, 6); + okBtn.disabled = input.value.length !== 6; + }; + + const done = (val) => { m.close(); setTimeout(() => resolve(val), 200); }; + const submit = () => { if (/^\d{6}$/.test(input.value)) done(input.value); }; + okBtn.onclick = submit; + m.modal.querySelector('#totp-cancel').onclick = () => done(null); + m.modal.querySelector('#totp-x').onclick = () => done(null); + m.overlay.onclick = () => done(null); + input.onkeydown = (ev) => { + if (ev.key === 'Enter') submit(); + if (ev.key === 'Escape') done(null); + }; + requestAnimationFrame(() => { m.open(); input.focus(); }); + }); +} + /* ── Lightweight YAML syntax coloring (display only) ─────────────────── */ function yamlToHtml(src) { const esc = (s) => { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; }; diff --git a/server/static/js/vigil-vulns.js b/server/static/js/vigil-vulns.js index 622441de..ae84f3f7 100644 --- a/server/static/js/vigil-vulns.js +++ b/server/static/js/vigil-vulns.js @@ -253,7 +253,7 @@ async function refreshVulnScans() { } async function startVulnScan(hostId, btn) { - const totp = (window.prompt('Enter your TOTP code to launch a vulnerability scan:') || '').trim(); + const totp = await totpPrompt('Launch a vulnerability scan.'); if (!totp) return; try { btn.disabled = true; btn.style.opacity = '0.5'; diff --git a/server/templates/pages/_containers.html b/server/templates/pages/_containers.html index db1d4b91..617f838b 100644 --- a/server/templates/pages/_containers.html +++ b/server/templates/pages/_containers.html @@ -15,8 +15,9 @@ -
- {% for host in hosts %} From 8a5a0223404c9cf1347cda71b9a2ada4fcb49d4b Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:35:18 -0400 Subject: [PATCH 27/68] fix(security): agent endpoints answer only approved hosts Registering a host is open by design, so an agent token proves nothing until an admin approves the host. authenticate_agent accepted any token on file, pending and rejected hosts included. Anyone who could reach the server could register a host and at once fetch every registry credential scoped to all hosts, without anyone approving anything. Every agent endpoint now refuses a pending or rejected host with 403. Check-in alone opts out, because a pending host has to check in to reach the enrolment queue, and it already answers rejected hosts itself. --- server/apps/hosts/authentication.py | 15 ++++-- server/apps/hosts/test_agent_auth_approval.py | 47 +++++++++++++++++++ server/apps/hosts/views.py | 4 +- 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 server/apps/hosts/test_agent_auth_approval.py diff --git a/server/apps/hosts/authentication.py b/server/apps/hosts/authentication.py index 0607465d..17cea9bd 100644 --- a/server/apps/hosts/authentication.py +++ b/server/apps/hosts/authentication.py @@ -3,16 +3,25 @@ from .models import Host -def authenticate_agent(request): +def authenticate_agent(request, allow_unapproved: bool = False): """Validate a Bearer agent token. Returns (Host, None) on success or (None, error Response) on failure. + + Only an approved host passes unless *allow_unapproved*. Registering is open + by design, so a token alone proves nothing until an admin approves the host: + without this, a host nobody approved could fetch registry credentials and + use every other agent endpoint. Check-in is the one caller that opts out, + since a pending host has to check in to reach the enrolment queue. """ header = request.META.get("HTTP_AUTHORIZATION", "") if not header.startswith("Bearer "): return None, Response({"error": "Agent token required"}, status=401) token = header[7:].strip() try: - return Host.objects.get(agent_token=token), None + host = Host.objects.get(agent_token=token) except Host.DoesNotExist: - return None, Response({"error": "Invalid agent token"}, status=401) \ No newline at end of file + return None, Response({"error": "Invalid agent token"}, status=401) + if not allow_unapproved and host.status in (Host.Status.PENDING, Host.Status.REJECTED): + return None, Response({"error": "This host has not been approved"}, status=403) + return host, None \ No newline at end of file diff --git a/server/apps/hosts/test_agent_auth_approval.py b/server/apps/hosts/test_agent_auth_approval.py new file mode 100644 index 00000000..cb9064e9 --- /dev/null +++ b/server/apps/hosts/test_agent_auth_approval.py @@ -0,0 +1,47 @@ +"""Only an approved host may use the agent endpoints that hand out secrets or +change state (architect review, 2026-10-08). Registering is open by design, so +before this a host nobody had approved could fetch every registry credential +scoped to "all hosts" just by registering. Check-in alone still accepts a +pending host: that is how it reaches the enrolment queue.""" +from django.test import TestCase + +from apps.hosts.models import Host +from apps.stacks.models import RegistryCredential +from vigil.signing import get_signing_key # noqa: F401 (settings import order) + + +def _cred(): + from apps.hosts.crypto import encrypt_secret + return RegistryCredential.objects.create(registry="ghcr.io", username="bot", + password_encrypted=encrypt_secret("s3cret"), host_tags=[]) + + +class AgentAuthApprovalTests(TestCase): + def _get(self, token): + return self.client.get("/api/v1/agent/registry-auth/?registry=ghcr.io", + HTTP_AUTHORIZATION=f"Bearer {token}") + + def test_a_pending_host_gets_no_credentials(self): + _cred() + Host.objects.create(hostname="new", agent_token="tok-new", status=Host.Status.PENDING) + resp = self._get("tok-new") + self.assertEqual(resp.status_code, 403) + self.assertNotIn("s3cret", resp.content.decode()) + + def test_a_rejected_host_gets_no_credentials(self): + _cred() + Host.objects.create(hostname="bad", agent_token="tok-bad", status=Host.Status.REJECTED) + self.assertEqual(self._get("tok-bad").status_code, 403) + + def test_an_approved_host_still_does(self): + _cred() + Host.objects.create(hostname="ok", agent_token="tok-ok", status=Host.Status.ONLINE) + resp = self._get("tok-ok") + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json()["password"], "s3cret") + + def test_a_pending_host_can_still_check_in(self): + Host.objects.create(hostname="new", agent_token="tok-new", status=Host.Status.PENDING) + resp = self.client.post("/api/v1/checkin", {"hostname": "new", "metrics": {}}, + content_type="application/json", HTTP_AUTHORIZATION="Bearer tok-new") + self.assertNotEqual(resp.status_code, 403) diff --git a/server/apps/hosts/views.py b/server/apps/hosts/views.py index 82c014f7..80cae0fa 100644 --- a/server/apps/hosts/views.py +++ b/server/apps/hosts/views.py @@ -372,7 +372,9 @@ def _refuse_unsupported_features(host: Host) -> None: @api_view(["POST"]) @permission_classes([AllowAny]) def checkin(request): - host, err = authenticate_agent(request) + # A pending host checks in to reach the enrolment queue; rejected hosts + # are answered just below. + host, err = authenticate_agent(request, allow_unapproved=True) if err: return err From 978d28e420c2b3aaf4e6b99aaed0bcb35c0ee9ae Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:38:40 -0400 Subject: [PATCH 28/68] fix(security): TOTP-gated actions stop after five wrong codes consume_totp blocked reuse of a code but not repeated wrong ones. The TOTP gate on deploys, approvals and firewall changes exists for the case where someone holds a session without the device, and with no limit a six-digit code falls to guessing. After five wrong codes in ten minutes every TOTP-gated action refuses that user for the rest of the window; a right code before the limit clears the count. Kept in the LoginAttempt table, so the limit holds across workers and restarts. --- server/apps/accounts/test_totp_rate_limit.py | 38 ++++++++++++++++++++ server/apps/accounts/totp.py | 38 ++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 server/apps/accounts/test_totp_rate_limit.py diff --git a/server/apps/accounts/test_totp_rate_limit.py b/server/apps/accounts/test_totp_rate_limit.py new file mode 100644 index 00000000..e62a388f --- /dev/null +++ b/server/apps/accounts/test_totp_rate_limit.py @@ -0,0 +1,38 @@ +"""TOTP-gated actions stop accepting codes after repeated wrong ones (architect +review, 2026-10-08). The gate exists for a session that someone holds without +the device; without a limit, a six-digit code falls to guessing.""" +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.utils.timezone import now + +from apps.accounts.models import UserProfile +from apps.accounts.totp import MAX_TOTP_FAILURES, consume_totp, generate_secret, generate_totp + + +class TotpRateLimitTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user("u", password="pw") + profile = UserProfile.objects.create(user=self.user) + self.secret = generate_secret() + profile.totp_secret = self.secret + profile.totp_confirmed_at = now() + profile.save() + + def _wrong(self): + good = generate_totp(self.secret) + return "000000" if good != "000000" else "111111" + + def test_after_the_limit_even_the_right_code_is_refused(self): + for _ in range(MAX_TOTP_FAILURES): + self.assertFalse(consume_totp(self.user, self._wrong())[0]) + ok, err = consume_totp(self.user, generate_totp(self.secret)) + self.assertFalse(ok) + self.assertIn("Too many wrong codes", err) + + def test_a_right_code_before_the_limit_clears_the_count(self): + for _ in range(MAX_TOTP_FAILURES - 1): + consume_totp(self.user, self._wrong()) + self.assertTrue(consume_totp(self.user, generate_totp(self.secret))[0]) + for _ in range(MAX_TOTP_FAILURES - 1): + consume_totp(self.user, self._wrong()) + self.assertNotIn("Too many", consume_totp(self.user, self._wrong())[1]) diff --git a/server/apps/accounts/totp.py b/server/apps/accounts/totp.py index b16e1dae..7177df12 100644 --- a/server/apps/accounts/totp.py +++ b/server/apps/accounts/totp.py @@ -74,6 +74,38 @@ def otpauth_uri(secret_b32: str, account_name: str, issuer: str = "Vigil") -> st _REPLAY_WINDOW_SECONDS = (2 * 1 + 1) * _STEP_SECONDS +#: Wrong codes allowed per user inside the window before TOTP-gated actions +#: are refused for the rest of it (architect review, 2026-10-08). +MAX_TOTP_FAILURES = 5 +TOTP_FAILURE_WINDOW_MINUTES = 10 + + +def _totp_key(user) -> str: + # LoginAttempt.username doubles as the key; the prefix keeps these apart + # from sign-in failures for a username. + return f"totp:{user.pk}" + + +def _totp_blocked(user) -> bool: + from datetime import timedelta + + from django.utils.timezone import now + + from .models import LoginAttempt + since = now() - timedelta(minutes=TOTP_FAILURE_WINDOW_MINUTES) + return LoginAttempt.objects.filter(username=_totp_key(user), created_at__gte=since).count() >= MAX_TOTP_FAILURES + + +def _record_totp_failure(user) -> None: + from .models import LoginAttempt + LoginAttempt.objects.create(username=_totp_key(user), ip=None) + + +def _clear_totp_failures(user) -> None: + from .models import LoginAttempt + LoginAttempt.objects.filter(username=_totp_key(user)).delete() + + def consume_totp(user, code: str) -> tuple[bool, str | None]: """Verify a TOTP code for ``user`` and mark it consumed. @@ -93,8 +125,14 @@ def consume_totp(user, code: str) -> tuple[bool, str | None]: code = (code or "").strip().replace(" ", "") if not code: return False, "TOTP code required" + # A six-digit code falls to guessing without a limit, and the gate exists + # for exactly the case where someone holds a session but not the device. + if _totp_blocked(user): + return False, "Too many wrong codes — wait a few minutes and try again" if not verify_totp(secret, code): + _record_totp_failure(user) return False, "Invalid TOTP code" + _clear_totp_failures(user) if profile.last_totp_code == code and profile.last_totp_used_at: elapsed = (now() - profile.last_totp_used_at).total_seconds() From 29bf48b16faa5b98ed361f10f39f0e57835ec4f1 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:41:08 -0400 Subject: [PATCH 29/68] fix(security): the TOTP limit has its own table and counts an attempt before checking it The limit added in the previous commit kept its rows in LoginAttempt under "totp:". A failed sign-in records whatever username was typed, so anyone could type that key a few times and lock the user out of every TOTP-gated action. It also checked the count before recording the failure, so parallel requests could all pass a check none of them had incremented. Attempts now go in a TotpAttempt table keyed by the user, which sign-in never writes to. Each attempt is recorded first and the count is checked after, so racing requests see each other. A right code within the limit clears the user's attempts. --- .../accounts/migrations/0006_totp_attempt.py | 24 +++++++++++++ server/apps/accounts/models.py | 13 +++++++ server/apps/accounts/test_totp_rate_limit.py | 22 ++++++++++++ server/apps/accounts/totp.py | 34 ++++++++----------- 4 files changed, 74 insertions(+), 19 deletions(-) create mode 100644 server/apps/accounts/migrations/0006_totp_attempt.py diff --git a/server/apps/accounts/migrations/0006_totp_attempt.py b/server/apps/accounts/migrations/0006_totp_attempt.py new file mode 100644 index 00000000..1d2b6389 --- /dev/null +++ b/server/apps/accounts/migrations/0006_totp_attempt.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.17 on 2026-10-08 18:40 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0005_userprofile_role'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='TotpAttempt', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/server/apps/accounts/models.py b/server/apps/accounts/models.py index ec0237cb..5331029f 100644 --- a/server/apps/accounts/models.py +++ b/server/apps/accounts/models.py @@ -51,6 +51,19 @@ def __str__(self) -> str: return f"profile<{self.user_id}>" +class TotpAttempt(models.Model): + """One TOTP code tried for a user, kept briefly to rate-limit guessing. + + A table of its own, keyed by the user: sharing LoginAttempt let a failed + *login* typed with the right username land in the same count, so anyone + could lock a user out of TOTP-gated actions (architect review, 2026-10-08). + """ + + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, + related_name="+") + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + + class LoginAttempt(models.Model): """A failed console login, kept briefly to rate-limit brute forcing. diff --git a/server/apps/accounts/test_totp_rate_limit.py b/server/apps/accounts/test_totp_rate_limit.py index e62a388f..469d9a98 100644 --- a/server/apps/accounts/test_totp_rate_limit.py +++ b/server/apps/accounts/test_totp_rate_limit.py @@ -36,3 +36,25 @@ def test_a_right_code_before_the_limit_clears_the_count(self): for _ in range(MAX_TOTP_FAILURES - 1): consume_totp(self.user, self._wrong()) self.assertNotIn("Too many", consume_totp(self.user, self._wrong())[1]) + + def test_failed_logins_cannot_lock_someone_out(self): + # Failed sign-ins with a crafted username used to share the count. + from apps.accounts.models import LoginAttempt + for _ in range(20): + LoginAttempt.objects.create(username=f"totp:{self.user.pk}", ip="203.0.113.9") + self.assertTrue(consume_totp(self.user, generate_totp(self.secret))[0]) + + def test_an_attempt_is_recorded_before_the_code_is_checked(self): + # Record-then-count is what stops parallel guesses all passing a check + # none of them had incremented yet. + from unittest.mock import patch + + from apps.accounts import totp + from apps.accounts.models import TotpAttempt + seen = [] + real = totp.verify_totp + with patch.object(totp, "verify_totp", + side_effect=lambda s, c: seen.append(TotpAttempt.objects.filter(user=self.user).count()) + or real(s, c)): + consume_totp(self.user, self._wrong()) + self.assertEqual(seen, [1]) diff --git a/server/apps/accounts/totp.py b/server/apps/accounts/totp.py index 7177df12..14807cc3 100644 --- a/server/apps/accounts/totp.py +++ b/server/apps/accounts/totp.py @@ -80,30 +80,27 @@ def otpauth_uri(secret_b32: str, account_name: str, issuer: str = "Vigil") -> st TOTP_FAILURE_WINDOW_MINUTES = 10 -def _totp_key(user) -> str: - # LoginAttempt.username doubles as the key; the prefix keeps these apart - # from sign-in failures for a username. - return f"totp:{user.pk}" +def _totp_attempt_allowed(user) -> bool: + """Record this attempt, then say whether it is within the limit. - -def _totp_blocked(user) -> bool: + Recorded first and counted after, so requests racing each other each see + the others' rows: checking and recording in the other order let a burst of + parallel guesses all pass a check that none of them had yet incremented. + """ from datetime import timedelta from django.utils.timezone import now - from .models import LoginAttempt + from .models import TotpAttempt + TotpAttempt.objects.create(user=user) since = now() - timedelta(minutes=TOTP_FAILURE_WINDOW_MINUTES) - return LoginAttempt.objects.filter(username=_totp_key(user), created_at__gte=since).count() >= MAX_TOTP_FAILURES - - -def _record_totp_failure(user) -> None: - from .models import LoginAttempt - LoginAttempt.objects.create(username=_totp_key(user), ip=None) + TotpAttempt.objects.filter(created_at__lt=now() - timedelta(hours=24)).delete() + return TotpAttempt.objects.filter(user=user, created_at__gte=since).count() <= MAX_TOTP_FAILURES -def _clear_totp_failures(user) -> None: - from .models import LoginAttempt - LoginAttempt.objects.filter(username=_totp_key(user)).delete() +def _clear_totp_attempts(user) -> None: + from .models import TotpAttempt + TotpAttempt.objects.filter(user=user).delete() def consume_totp(user, code: str) -> tuple[bool, str | None]: @@ -127,12 +124,11 @@ def consume_totp(user, code: str) -> tuple[bool, str | None]: return False, "TOTP code required" # A six-digit code falls to guessing without a limit, and the gate exists # for exactly the case where someone holds a session but not the device. - if _totp_blocked(user): + if not _totp_attempt_allowed(user): return False, "Too many wrong codes — wait a few minutes and try again" if not verify_totp(secret, code): - _record_totp_failure(user) return False, "Invalid TOTP code" - _clear_totp_failures(user) + _clear_totp_attempts(user) if profile.last_totp_code == code and profile.last_totp_used_at: elapsed = (now() - profile.last_totp_used_at).total_seconds() From 35978786be340bc124c02964ef8f3c53693e38f2 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:44:55 -0400 Subject: [PATCH 30/68] fix(security): turning TOTP off goes through the same attempt limit as every other gate totp_disable checked the code with verify_totp directly, outside the attempt limit and replay check in consume_totp. Disabling is the strongest thing a code unlocks, since afterwards a different device can be enrolled, so a stolen session could guess its way out of every TOTP gate. It now uses consume_totp. --- server/apps/accounts/test_totp_rate_limit.py | 24 ++++++++++++++++++++ server/apps/accounts/views.py | 10 ++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/server/apps/accounts/test_totp_rate_limit.py b/server/apps/accounts/test_totp_rate_limit.py index 469d9a98..7e40eb4a 100644 --- a/server/apps/accounts/test_totp_rate_limit.py +++ b/server/apps/accounts/test_totp_rate_limit.py @@ -58,3 +58,27 @@ def test_an_attempt_is_recorded_before_the_code_is_checked(self): or real(s, c)): consume_totp(self.user, self._wrong()) self.assertEqual(seen, [1]) + + +class TotpDisableLimitTests(TestCase): + """Disabling TOTP used verify_totp directly: unlimited guesses at the one + action that lets a new device be enrolled afterwards.""" + + def setUp(self): + self.user = get_user_model().objects.create_user("d", password="pw") + self.profile = UserProfile.objects.create(user=self.user) + self.secret = generate_secret() + self.profile.totp_secret = self.secret + self.profile.totp_confirmed_at = now() + self.profile.save() + self.client.force_login(self.user) + + def test_disable_is_rate_limited(self): + good = generate_totp(self.secret) + wrong = "000000" if good != "000000" else "111111" + for _ in range(MAX_TOTP_FAILURES): + self.client.post("/api/v1/accounts/totp/disable/", {"code": wrong}, content_type="application/json") + resp = self.client.post("/api/v1/accounts/totp/disable/", {"code": good}, content_type="application/json") + self.assertEqual(resp.status_code, 400) + self.profile.refresh_from_db() + self.assertIsNotNone(self.profile.totp_confirmed_at, "TOTP must still be enrolled") diff --git a/server/apps/accounts/views.py b/server/apps/accounts/views.py index 386f2913..9c218c19 100644 --- a/server/apps/accounts/views.py +++ b/server/apps/accounts/views.py @@ -107,8 +107,14 @@ def totp_disable(request): code = (request.data.get("code") or "").strip() if not profile.totp_confirmed_at or not profile.totp_secret: return Response({"error": "TOTP is not enrolled"}, status=400) - if not verify_totp(profile.totp_secret, code): - return Response({"error": "Invalid code"}, status=400) + # Through consume_totp, not verify_totp: turning TOTP off is the strongest + # thing a code unlocks (afterwards a new device can be enrolled), so it + # gets the same attempt limit and replay check as every other gate. + from .totp import consume_totp + ok, err = consume_totp(request.user, code) + if not ok: + return Response({"error": err or "Invalid code"}, status=400) + profile.refresh_from_db() profile.totp_secret = "" profile.totp_confirmed_at = None profile.save(update_fields=["totp_secret_encrypted", "totp_confirmed_at"]) From c69f76da21643f077458e06376ec61f600d08315 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:50:35 -0400 Subject: [PATCH 31/68] fix(security): agent tokens are stored as a SHA-256, not in the clear Every host's agent token sat in the database in plaintext, so a database dump or backup handed out a working credential for every machine. Tokens are now stored as "sha256$". The model hashes a plaintext token on save, and a lookup always hashes what the agent presents, so presenting a stored hash does not authenticate. A migration hashes existing rows in place, and running agents are unaffected because they keep sending their plaintext token. The hex part is exactly the fingerprint v2 task signatures bind to, which the agent computes from its own token. Registration and reprovision enrolment now refuse a token that is not 16-128 letters, digits, '-' or '_'. A revoked host's token used to be "revoked-", which anyone who could see the job could present. It is random now. Live on the test VMs: after the migration both agents kept checking in, and a v2-signed task deployed through the API ran on the Ubuntu agent. --- server/apps/accounts/test_session_timeout.py | 2 +- server/apps/automations/tests.py | 2 +- server/apps/hosts/authentication.py | 2 +- .../migrations/0026_hash_agent_tokens.py | 29 ++++++++++++ server/apps/hosts/models.py | 37 +++++++++++++++ server/apps/hosts/test_docker.py | 2 +- .../apps/hosts/test_enrollment_replacement.py | 4 +- server/apps/hosts/test_machine_id.py | 12 ++--- server/apps/hosts/test_reboot_gate.py | 2 +- server/apps/hosts/test_token_hashing.py | 47 +++++++++++++++++++ server/apps/hosts/tests.py | 12 ++--- server/apps/hosts/views.py | 12 +++-- .../apps/playbooks/test_playbook_branches.py | 2 +- server/apps/playbooks/test_playbook_chains.py | 2 +- server/apps/reprovision/installer_views.py | 9 +++- .../reprovision/test_installer_endpoints.py | 22 ++++++--- server/apps/tasks/test_branches_spec.py | 2 +- server/apps/tasks/test_hunt_results.py | 2 +- server/apps/tasks/test_hunt_stays_open.py | 4 +- server/apps/tasks/test_not_applicable.py | 2 +- server/apps/tasks/test_refusals.py | 2 +- server/apps/tasks/test_relevant_gate.py | 4 +- server/apps/tasks/test_step_result_storage.py | 2 +- server/apps/vulns/test_detection.py | 2 +- server/vigil/signing.py | 6 ++- 25 files changed, 180 insertions(+), 44 deletions(-) create mode 100644 server/apps/hosts/migrations/0026_hash_agent_tokens.py create mode 100644 server/apps/hosts/test_token_hashing.py diff --git a/server/apps/accounts/test_session_timeout.py b/server/apps/accounts/test_session_timeout.py index ebbecdca..77c198f6 100644 --- a/server/apps/accounts/test_session_timeout.py +++ b/server/apps/accounts/test_session_timeout.py @@ -143,7 +143,7 @@ def test_agents_unaffected(self): "/api/v1/checkin", {}, format="json", - HTTP_AUTHORIZATION=f"Bearer {host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200, getattr(resp, "data", None)) self.assertNotIn("session_", resp.content.decode("utf-8", "ignore")) diff --git a/server/apps/automations/tests.py b/server/apps/automations/tests.py index e0b14789..8cc56c40 100644 --- a/server/apps/automations/tests.py +++ b/server/apps/automations/tests.py @@ -347,7 +347,7 @@ def test_the_run_resolves_when_the_task_reports_back(self): data=json.dumps({"task_id": str(task.id), "state": "completed", "output": "done"}), content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {host.agent_token}") + HTTP_AUTHORIZATION=f"Bearer {host.raw_agent_token}") self.assertIn(resp.status_code, (200, 201), resp.content) run = TaskRun.objects.get() self.assertEqual(run.state, TaskRun.State.COMPLETED) diff --git a/server/apps/hosts/authentication.py b/server/apps/hosts/authentication.py index 17cea9bd..41b94a2c 100644 --- a/server/apps/hosts/authentication.py +++ b/server/apps/hosts/authentication.py @@ -19,7 +19,7 @@ def authenticate_agent(request, allow_unapproved: bool = False): return None, Response({"error": "Agent token required"}, status=401) token = header[7:].strip() try: - host = Host.objects.get(agent_token=token) + host = Host.objects.by_token(token).get() except Host.DoesNotExist: return None, Response({"error": "Invalid agent token"}, status=401) if not allow_unapproved and host.status in (Host.Status.PENDING, Host.Status.REJECTED): diff --git a/server/apps/hosts/migrations/0026_hash_agent_tokens.py b/server/apps/hosts/migrations/0026_hash_agent_tokens.py new file mode 100644 index 00000000..ca71ece2 --- /dev/null +++ b/server/apps/hosts/migrations/0026_hash_agent_tokens.py @@ -0,0 +1,29 @@ +# Generated by Django 5.2.17 on 2026-10-08 18:39 + +import hashlib + +from django.db import migrations + +PREFIX = "sha256$" + + +def hash_tokens(apps, schema_editor): + """Store every agent token as a SHA-256 (architect review, 2026-10-08). + Agents keep presenting their plaintext token; the server hashes it to look + the host up, so nothing changes for a running agent.""" + Host = apps.get_model("hosts", "Host") + for host in Host.objects.exclude(agent_token__startswith=PREFIX).only("id", "agent_token"): + host.agent_token = PREFIX + hashlib.sha256(host.agent_token.encode()).hexdigest() + host.save(update_fields=["agent_token"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('hosts', '0025_host_agent_runs_as_root'), + ] + + operations = [ + # One way: a hash cannot be turned back into the token. + migrations.RunPython(hash_tokens, migrations.RunPython.noop), + ] diff --git a/server/apps/hosts/models.py b/server/apps/hosts/models.py index 60a7fdfc..5368c23f 100644 --- a/server/apps/hosts/models.py +++ b/server/apps/hosts/models.py @@ -1,10 +1,34 @@ +import hashlib +import re import uuid from django.conf import settings from django.db import models +#: Agent tokens are kept as a SHA-256, never in the clear: a database dump must +#: not hand out a working credential for every machine (architect review, +#: 2026-10-08). The hex part doubles as the agent fingerprint that v2 task +#: signatures bind to (vigil/signing.py), since the agent hashes its own token. +TOKEN_PREFIX = "sha256$" +#: The shape a token presented at registration or enrolment must have. +TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{16,128}$") + + +def hash_agent_token(raw: str) -> str: + """The stored form of a plaintext token. Always hashes: a caller holding a + stored value must not be able to present it as if it were the token.""" + return TOKEN_PREFIX + hashlib.sha256(str(raw).encode()).hexdigest() + + +class HostManager(models.Manager): + def by_token(self, raw: str): + """Hosts whose token is *raw* (the plaintext an agent presents).""" + return self.filter(agent_token=hash_agent_token(raw)) + class Host(models.Model): + objects = HostManager() + class Status(models.TextChoices): PENDING = "pending", "Pending Enrollment" ONLINE = "online", "Online" @@ -39,10 +63,23 @@ class Mode(models.TextChoices): tag_sync_fields = [("tags", "tag_rows")] def save(self, *args, **kwargs): + # A plaintext token set anywhere (registration, enrolment, tests) is + # stored hashed; one that already is stays as it is. + if self.agent_token and not self.agent_token.startswith(TOKEN_PREFIX): + # The plaintext stays on this in-memory instance only (never stored, + # never read by the server) so whoever just set it, a test or an + # enrolment flow, can still hand it to the agent. + self.raw_agent_token = self.agent_token + self.agent_token = hash_agent_token(self.agent_token) super().save(*args, **kwargs) # Defined below this class, hence the local import. for string_field, relation in self.tag_sync_fields: sync_tag_rows(self, string_field, relation) + @property + def token_fingerprint(self) -> str: + """SHA-256 hex of the agent's token: what a v2 signature binds to.""" + return self.agent_token[len(TOKEN_PREFIX):] if self.agent_token.startswith(TOKEN_PREFIX) else "" + agent_version = models.CharField(max_length=50, blank=True, default="") #: Task-language features the agent said it understands at check-in. #: The server refuses to hand a task to a host that lacks a feature the diff --git a/server/apps/hosts/test_docker.py b/server/apps/hosts/test_docker.py index 1abb7f92..3212b350 100644 --- a/server/apps/hosts/test_docker.py +++ b/server/apps/hosts/test_docker.py @@ -32,7 +32,7 @@ def setUp(self): def _checkin(self, body): return self.client.post( "/api/v1/checkin", body, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) def test_checkin_ingests_docker_containers(self): diff --git a/server/apps/hosts/test_enrollment_replacement.py b/server/apps/hosts/test_enrollment_replacement.py index 1ef9528f..f20b8cf8 100644 --- a/server/apps/hosts/test_enrollment_replacement.py +++ b/server/apps/hosts/test_enrollment_replacement.py @@ -131,7 +131,7 @@ def test_a_second_pending_row_for_the_same_machine_is_cleared(self): def test_check_pending_reports_what_it_replaces(self): resp = self.client.post( "/api/v1/hosts/check-pending/", - {"token": self.host_b.agent_token}, + {"token": self.host_b.raw_agent_token}, format="json", ) self.assertEqual(resp.status_code, 200) @@ -149,7 +149,7 @@ def test_check_pending_reports_null_when_nothing_is_replaced(self): ) resp = self.client.post( "/api/v1/hosts/check-pending/", - {"token": solo.agent_token}, + {"token": solo.raw_agent_token}, format="json", ) self.assertEqual(resp.status_code, 200) diff --git a/server/apps/hosts/test_machine_id.py b/server/apps/hosts/test_machine_id.py index e8774c98..4276c865 100644 --- a/server/apps/hosts/test_machine_id.py +++ b/server/apps/hosts/test_machine_id.py @@ -27,21 +27,21 @@ def _register(self, token, hostname="web-01", **extra): def test_register_stores_the_machine_id(self): resp = self._register("tok-" + "a" * 32, machine_id="mid-1") self.assertEqual(resp.status_code, 201, getattr(resp, "data", None)) - host = Host.objects.get(agent_token="tok-" + "a" * 32) + host = Host.objects.by_token("tok-" + "a" * 32).get() self.assertEqual(host.machine_id, "mid-1") def test_register_backfills_machine_id_for_an_existing_token(self): self._register("tok-" + "b" * 32) resp = self._register("tok-" + "b" * 32, machine_id="mid-2") self.assertEqual(resp.status_code, 200) - self.assertEqual(Host.objects.filter(agent_token="tok-" + "b" * 32).count(), 1) - host = Host.objects.get(agent_token="tok-" + "b" * 32) + self.assertEqual(Host.objects.by_token("tok-" + "b" * 32).count(), 1) + host = Host.objects.by_token("tok-" + "b" * 32).get() self.assertEqual(host.machine_id, "mid-2") def test_register_without_a_machine_id_still_works(self): resp = self._register("tok-" + "c" * 32) self.assertEqual(resp.status_code, 201) - host = Host.objects.get(agent_token="tok-" + "c" * 32) + host = Host.objects.by_token("tok-" + "c" * 32).get() self.assertEqual(host.machine_id, "") def test_a_second_token_still_creates_a_second_host(self): @@ -54,7 +54,7 @@ def test_a_second_token_still_creates_a_second_host(self): resp = self._register("tok-" + "e" * 32, hostname="same-box", machine_id="mid-x") self.assertEqual(resp.status_code, 201) self.assertEqual(Host.objects.filter(hostname="same-box").count(), 2) - new_host = Host.objects.get(agent_token="tok-" + "e" * 32) + new_host = Host.objects.by_token("tok-" + "e" * 32).get() self.assertEqual(new_host.status, Host.Status.PENDING) self.assertEqual(new_host.machine_id, "mid-x") @@ -74,7 +74,7 @@ def test_checkin_backfills_the_machine_id(self): "/api/v1/checkin", {"hostname": "web-01", "machine_id": "mid-3"}, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200) self.host.refresh_from_db() diff --git a/server/apps/hosts/test_reboot_gate.py b/server/apps/hosts/test_reboot_gate.py index 2cd411c8..f9246023 100644 --- a/server/apps/hosts/test_reboot_gate.py +++ b/server/apps/hosts/test_reboot_gate.py @@ -34,7 +34,7 @@ def _checkin(client, host, body): _CHECKIN, body, format="json", - HTTP_AUTHORIZATION=f"Bearer {host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {host.raw_agent_token}", ) diff --git a/server/apps/hosts/test_token_hashing.py b/server/apps/hosts/test_token_hashing.py new file mode 100644 index 00000000..871a3c52 --- /dev/null +++ b/server/apps/hosts/test_token_hashing.py @@ -0,0 +1,47 @@ +"""Agent tokens are stored as a SHA-256 (architect review, 2026-10-08): a database +dump must not hand out a working credential for every machine.""" +import hashlib + +from django.test import TestCase + +from apps.hosts.models import Host, hash_agent_token + +TOKEN = "plain-token-0123456789abcdef" + + +class TokenHashingTests(TestCase): + def setUp(self): + self.host = Host.objects.create(hostname="h", agent_token=TOKEN, status=Host.Status.ONLINE) + + def test_the_token_is_never_stored_in_the_clear(self): + self.host.refresh_from_db() + self.assertNotIn(TOKEN, self.host.agent_token) + self.assertEqual(self.host.agent_token, hash_agent_token(TOKEN)) + + def test_the_plaintext_still_authenticates(self): + resp = self.client.post("/api/v1/checkin", {"hostname": "h", "metrics": {}}, + content_type="application/json", HTTP_AUTHORIZATION=f"Bearer {TOKEN}") + self.assertEqual(resp.status_code, 200, resp.content) + + def test_the_stored_hash_is_not_a_token(self): + self.host.refresh_from_db() + resp = self.client.post("/api/v1/checkin", {"hostname": "h", "metrics": {}}, + content_type="application/json", + HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}") + self.assertEqual(resp.status_code, 401) + + def test_registration_refuses_a_hash_shaped_token(self): + resp = self.client.post("/api/v1/register", {"agent_token": "sha256$" + "a" * 64, "hostname": "x"}, + content_type="application/json") + self.assertEqual(resp.status_code, 400) + + def test_fingerprint_is_what_the_agent_computes(self): + self.host.refresh_from_db() + self.assertEqual(self.host.token_fingerprint, hashlib.sha256(TOKEN.encode()).hexdigest()) + + def test_saving_twice_does_not_hash_twice(self): + self.host.refresh_from_db() + stored = self.host.agent_token + self.host.save() + self.host.refresh_from_db() + self.assertEqual(self.host.agent_token, stored) diff --git a/server/apps/hosts/tests.py b/server/apps/hosts/tests.py index 3ccbaae0..f943037c 100644 --- a/server/apps/hosts/tests.py +++ b/server/apps/hosts/tests.py @@ -18,7 +18,7 @@ def test_register_creates_pending_host(self): format="json", ) self.assertEqual(resp.status_code, 201) - host = Host.objects.get(agent_token="tok-" + "a" * 32) + host = Host.objects.by_token("tok-" + "a" * 32).get() self.assertEqual(host.status, Host.Status.PENDING) self.assertEqual(host.hostname, "web-01") @@ -44,7 +44,7 @@ def test_checkin_ignores_agent_supplied_ip(self): "/api/v1/checkin", {"ip_address": "10.9.9.9", "hostname": "web-01"}, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", REMOTE_ADDR="203.0.113.7", ) self.assertEqual(resp.status_code, 200) @@ -58,7 +58,7 @@ def test_checkin_syncs_mode_from_agent(self): "/api/v1/checkin", {"hostname": "web-01", "mode": "managed"}, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200) self.host.refresh_from_db() @@ -69,7 +69,7 @@ def test_checkin_ignores_invalid_mode(self): "/api/v1/checkin", {"hostname": "web-01", "mode": "root_me_please"}, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200) self.host.refresh_from_db() @@ -98,7 +98,7 @@ def test_register_namespaces_agent_tags(self): {"agent_token": "tok-" + "c" * 32, "hostname": "h", "tags": ["prod"]}, format="json", ) - host = Host.objects.get(agent_token="tok-" + "c" * 32) + host = Host.objects.by_token("tok-" + "c" * 32).get() self.assertIn("agent:prod", host.tags) self.assertNotIn("prod", host.tags) @@ -111,7 +111,7 @@ def test_checkin_namespaces_agent_tags(self): "/api/v1/checkin", {"hostname": "h", "tags": ["office"]}, format="json", - HTTP_AUTHORIZATION=f"Bearer {host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {host.raw_agent_token}", ) host.refresh_from_db() self.assertIn("agent:office", host.tags) diff --git a/server/apps/hosts/views.py b/server/apps/hosts/views.py index 80cae0fa..db5b373e 100644 --- a/server/apps/hosts/views.py +++ b/server/apps/hosts/views.py @@ -211,6 +211,12 @@ def register(request): {"error": "Field value too long"}, status=status.HTTP_400_BAD_REQUEST, ) + from .models import TOKEN_RE + if not TOKEN_RE.match(token): + # Tokens are stored hashed; one shaped like anything else (a stored + # hash included) is refused rather than guessed at. + return Response({"error": "agent_token must be 16-128 letters, digits, '-' or '_'"}, + status=status.HTTP_400_BAD_REQUEST) machine_id = str(request.data.get("machine_id") or "").strip()[:200] @@ -243,7 +249,7 @@ def register(request): ) # Idempotent: if the token already exists, return current status - existing = Host.objects.filter(agent_token=token).first() + existing = Host.objects.by_token(token).first() if existing: # An agent upgraded into this release backfills its own row on next # start; a changed machine_id means this token now belongs to another @@ -696,7 +702,7 @@ def checkin(request): signed_v2 = "signed_v2" in set(host.agent_features or []) for task in eligible: if signed_v2: - signature = sign_task_v2(task, dispatch_iso, host.agent_token) + signature = sign_task_v2(task, dispatch_iso, host.token_fingerprint) else: if not task.signature: task.signature = sign_task(task) @@ -1242,7 +1248,7 @@ def check_pending(request): if not token: return Response({"error": "token is required"}, status=status.HTTP_400_BAD_REQUEST) - host = Host.objects.filter(agent_token=token).first() + host = Host.objects.by_token(token).first() if host is None: return Response({"status": "waiting"}) diff --git a/server/apps/playbooks/test_playbook_branches.py b/server/apps/playbooks/test_playbook_branches.py index c74e04ae..07c00948 100644 --- a/server/apps/playbooks/test_playbook_branches.py +++ b/server/apps/playbooks/test_playbook_branches.py @@ -79,7 +79,7 @@ def report(self, task, state="completed", results=None): data=json.dumps({"task_id": str(task.id), "state": state, "output": "done", "steps": steps}), content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}") + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}") self.assertEqual(resp.status_code, 200, resp.content) def states(self, run): diff --git a/server/apps/playbooks/test_playbook_chains.py b/server/apps/playbooks/test_playbook_chains.py index 3c87fee0..ffd40ff2 100644 --- a/server/apps/playbooks/test_playbook_chains.py +++ b/server/apps/playbooks/test_playbook_chains.py @@ -157,7 +157,7 @@ def _drive(self, states): {"task_id": str(task.id), "state": state, "output": "ok", "result_output": "ok"}, content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200, resp.content) task.refresh_from_db() diff --git a/server/apps/reprovision/installer_views.py b/server/apps/reprovision/installer_views.py index cc51f429..c49ab756 100644 --- a/server/apps/reprovision/installer_views.py +++ b/server/apps/reprovision/installer_views.py @@ -101,8 +101,12 @@ def _revoke_old_agent_token(job) -> None: Without this, a token recovered from a backup or forensic image of the wiped disk authenticates as this host forever. """ + import secrets + host = job.host - host.agent_token = f"revoked-{job.id}" + # Random, not derived from the job: "revoked-" was itself a + # working token for anyone who could see the job's id. + host.agent_token = f"revoked-{secrets.token_urlsafe(32)}" host.save(update_fields=["agent_token"]) @@ -135,6 +139,9 @@ def enroll(request): if not token or not agent_token: return Response({"error": "enroll_token and agent_token are required"}, status=400) + from apps.hosts.models import TOKEN_RE + if not TOKEN_RE.match(agent_token): + return Response({"error": "agent_token must be 16-128 letters, digits, '-' or '_'"}, status=400) token_hash = jobs.hash_token(token) with transaction.atomic(): diff --git a/server/apps/reprovision/test_installer_endpoints.py b/server/apps/reprovision/test_installer_endpoints.py index 6b11e9a2..c743f3b9 100644 --- a/server/apps/reprovision/test_installer_endpoints.py +++ b/server/apps/reprovision/test_installer_endpoints.py @@ -5,7 +5,7 @@ from django.test import TestCase from django.utils.timezone import now -from apps.hosts.models import Host +from apps.hosts.models import Host, hash_agent_token from apps.reprovision import jobs from apps.reprovision.models import InstallProfile, OSImage, RebuildJob @@ -90,7 +90,7 @@ def test_old_agent_token_is_revoked_at_the_point_of_no_return(self): # A token recovered from a backup of the wiped disk must not keep # authenticating as this host. self.client.get(f"/reprovision/answer/{self.answer}/user-data") - self.assertFalse(Host.objects.filter(agent_token="tok-h1").exists()) + self.assertFalse(Host.objects.by_token("tok-h1").exists()) # ── Enrolment ──────────────────────────────────────────────────────── def _installing(self): @@ -98,7 +98,7 @@ def _installing(self): self.job.save(update_fields=["state"]) def _enrol(self, **over): - body = {"enroll_token": self.enroll, "agent_token": "brand-new-token", + body = {"enroll_token": self.enroll, "agent_token": "brand-new-token-0001", "hostname": "h1"} body.update(over) return self.client.post("/api/v1/reprovision/enroll", body, @@ -109,7 +109,7 @@ def test_enrol_rotates_the_token_on_the_same_host_row(self): resp = self._enrol() self.assertEqual(resp.status_code, 200) self.host.refresh_from_db() - self.assertEqual(self.host.agent_token, "brand-new-token") + self.assertEqual(self.host.agent_token, hash_agent_token("brand-new-token-0001")) self.assertEqual(self.host.status, Host.Status.ONLINE) def test_enrol_keeps_the_same_host_id(self): @@ -121,15 +121,23 @@ def test_enrol_keeps_the_same_host_id(self): def test_enrol_token_cannot_be_redeemed_twice(self): self._installing() - self.assertEqual(self._enrol(agent_token="first").status_code, 200) - second = self._enrol(agent_token="second") + self.assertEqual(self._enrol(agent_token="first-token-000001").status_code, 200) + second = self._enrol(agent_token="second-token-00002") self.assertEqual(second.status_code, 403) self.host.refresh_from_db() - self.assertEqual(self.host.agent_token, "first") + self.assertEqual(self.host.agent_token, hash_agent_token("first-token-000001")) def test_enrol_rejected_before_installing(self): self.assertEqual(self._enrol().status_code, 403) + def test_enrol_refuses_a_token_that_is_not_token_shaped(self): + self._installing() + self.assertEqual(self._enrol(agent_token="sha256$" + "a" * 64).status_code, 400) + + def test_revoked_token_is_not_derived_from_the_job(self): + self.client.get(f"/reprovision/answer/{self.answer}/user-data") + self.assertFalse(Host.objects.by_token(f"revoked-{self.job.id}").exists()) + def test_enrol_requires_both_fields(self): self._installing() self.assertEqual(self._enrol(agent_token="").status_code, 400) diff --git a/server/apps/tasks/test_branches_spec.py b/server/apps/tasks/test_branches_spec.py index 6f5c101f..af9478df 100644 --- a/server/apps/tasks/test_branches_spec.py +++ b/server/apps/tasks/test_branches_spec.py @@ -372,7 +372,7 @@ def setUp(self): def _checkin(self, **extra): return self.client.post( _CHECKIN, {"hostname": self.host.hostname, **extra}, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) def test_branch_task_refused_without_feature(self): diff --git a/server/apps/tasks/test_hunt_results.py b/server/apps/tasks/test_hunt_results.py index 5f817c4f..bb7ccebb 100644 --- a/server/apps/tasks/test_hunt_results.py +++ b/server/apps/tasks/test_hunt_results.py @@ -83,7 +83,7 @@ def _post(self, task, steps): "steps": steps, }), content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {task.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {'t' * 31}{task.host.hostname[1:]}", # _host(n): ("t" * 31) + str(n) ) assert resp.status_code == 200, resp.content task.refresh_from_db() diff --git a/server/apps/tasks/test_hunt_stays_open.py b/server/apps/tasks/test_hunt_stays_open.py index 03edaade..ac8f78f5 100644 --- a/server/apps/tasks/test_hunt_stays_open.py +++ b/server/apps/tasks/test_hunt_stays_open.py @@ -177,7 +177,7 @@ def test_checkin_skips_expired_pending(self): resp = self.client.post( "/api/v1/checkin", {"hostname": self.host.hostname}, content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200, resp.content) self.assertEqual(resp.json().get("tasks", []), [], @@ -190,7 +190,7 @@ def test_checkin_still_dispatches_unexpired_pending(self): resp = self.client.post( "/api/v1/checkin", {"hostname": self.host.hostname}, content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200, resp.content) task.refresh_from_db() diff --git a/server/apps/tasks/test_not_applicable.py b/server/apps/tasks/test_not_applicable.py index 0bfbf0a9..2adb6a7b 100644 --- a/server/apps/tasks/test_not_applicable.py +++ b/server/apps/tasks/test_not_applicable.py @@ -74,7 +74,7 @@ def _post(self, task, state="not_applicable", **extra): "/api/v1/tasks/result/", data=json.dumps(payload), content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) def test_agent_can_report_not_applicable(self): diff --git a/server/apps/tasks/test_refusals.py b/server/apps/tasks/test_refusals.py index f927772a..fe28afb2 100644 --- a/server/apps/tasks/test_refusals.py +++ b/server/apps/tasks/test_refusals.py @@ -134,7 +134,7 @@ class CheckinStoresAllowlistTests(TestCase): def _checkin(self, host, **payload): return self.client.post("/api/v1/checkin", {"hostname": host.hostname, "metrics": {}, **payload}, content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {host.agent_token}") + HTTP_AUTHORIZATION=f"Bearer {host.raw_agent_token}") def test_checkin_stores_allowlist(self): host = _host("ci", allowlist=None) diff --git a/server/apps/tasks/test_relevant_gate.py b/server/apps/tasks/test_relevant_gate.py index fac69916..69a43bf9 100644 --- a/server/apps/tasks/test_relevant_gate.py +++ b/server/apps/tasks/test_relevant_gate.py @@ -45,7 +45,7 @@ def setUp(self): def _checkin(self, **extra): return self.client.post( _CHECKIN, {"hostname": self.host.hostname, **extra}, format="json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) def test_old_agent_is_refused(self): @@ -103,7 +103,7 @@ def test_probe_evidence_is_stored(self): data=json.dumps({"task_id": str(task.id), "state": "not_applicable", "output": "[NOT APPLICABLE]", "steps": [probe]}), content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) self.assertEqual(resp.status_code, 200, resp.content) row = HuntMatch.objects.get(task=task) diff --git a/server/apps/tasks/test_step_result_storage.py b/server/apps/tasks/test_step_result_storage.py index 0288654c..3c8e4926 100644 --- a/server/apps/tasks/test_step_result_storage.py +++ b/server/apps/tasks/test_step_result_storage.py @@ -51,7 +51,7 @@ def _post(self, task, **extra): "/api/v1/tasks/result/", data=json.dumps(payload), content_type="application/json", - HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}", + HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}", ) def test_step_results_are_stored(self): diff --git a/server/apps/vulns/test_detection.py b/server/apps/vulns/test_detection.py index e63f0ebc..05516aee 100644 --- a/server/apps/vulns/test_detection.py +++ b/server/apps/vulns/test_detection.py @@ -29,7 +29,7 @@ def _report(self, state, steps): nonce=f"{self.n}" * 64, state=Task.State.DISPATCHED) resp = self.client.post("/api/v1/tasks/result/", data=json.dumps( {"task_id": str(task.id), "state": state, "output": "x", "steps": steps}), - content_type="application/json", HTTP_AUTHORIZATION=f"Bearer {self.host.agent_token}") + content_type="application/json", HTTP_AUTHORIZATION=f"Bearer {self.host.raw_agent_token}") self.assertEqual(resp.status_code, 200, resp.content) def _matched_steps(self, fix_status="ok", boost=True): diff --git a/server/vigil/signing.py b/server/vigil/signing.py index 1366e245..3920180d 100644 --- a/server/vigil/signing.py +++ b/server/vigil/signing.py @@ -83,7 +83,7 @@ def agent_fingerprint(agent_token: str) -> str: return hashlib.sha256(agent_token.encode()).hexdigest() -def sign_task_v2(task, dispatched_at: str, agent_token: str) -> str: +def sign_task_v2(task, dispatched_at: str, agent_fingerprint_hex: str) -> str: """Signature over the v1 fields plus when the task was sent and to which agent (SEC-4). v1 left the dispatch time unsigned, so the TTL could be restarted by whoever relayed the task, and it named the target host @@ -97,7 +97,9 @@ def sign_task_v2(task, dispatched_at: str, agent_token: str) -> str: "nonce": task.nonce, "ttl_seconds": task.ttl_seconds, "dispatched_at": dispatched_at, - "agent": agent_fingerprint(agent_token), + # The SHA-256 of the agent's token, which is exactly how the token + # is stored (Host.token_fingerprint); the agent hashes its own. + "agent": agent_fingerprint_hex, "v": 2, }, sort_keys=True, From 48665b5a26ee8d6223c2ceae5994a136ae115cea Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:51:35 -0400 Subject: [PATCH 32/68] fix(security): the public status page keys its cards by an opaque id, not the host's The page and its JSON published each host's internal UUID only so the live refresh could find each card again. A per-page HMAC of the host id does the same job and means nothing outside that page. --- server/apps/statuspage/tests.py | 17 +++++++++++++++++ server/apps/statuspage/views.py | 11 ++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/apps/statuspage/tests.py b/server/apps/statuspage/tests.py index 3de32411..25f283ac 100644 --- a/server/apps/statuspage/tests.py +++ b/server/apps/statuspage/tests.py @@ -302,3 +302,20 @@ def test_a_host_outside_your_sites_is_not_readable_by_id(self): def test_a_host_inside_your_sites_is_readable(self): self._sample(True, 1) self.assertEqual(self._get().status_code, 200) + + +class PublicPageDoesNotPublishHostIdsTests(TestCase): + """The public page keyed its cards by the host's internal id; an opaque + per-page key does the same job (architect review, 2026-10-08).""" + + def test_no_host_id_on_the_public_page_or_its_data(self): + from apps.hosts.models import Host + from apps.statuspage.models import StatusPage + host = Host.objects.create(hostname="web-1", agent_token="tok-web-1-000000000000", status=Host.Status.ONLINE) + page = StatusPage.objects.create(enabled=True, host_ids=[str(host.id)]) + for url in (f"/status/{page.token}/", f"/status/{page.token}/data/"): + resp = self.client.get(url) + self.assertEqual(resp.status_code, 200, url) + body = resp.content.decode() + self.assertIn("web-1", body) + self.assertNotIn(str(host.id), body) diff --git a/server/apps/statuspage/views.py b/server/apps/statuspage/views.py index 4447ccae..819b0180 100644 --- a/server/apps/statuspage/views.py +++ b/server/apps/statuspage/views.py @@ -70,6 +70,12 @@ def _uptime_history(host_objs): return history +def _public_key(page, host) -> str: + import hashlib + import hmac + return hmac.new(page.token.encode(), str(host.id).encode(), hashlib.sha256).hexdigest()[:16] + + def _page_hosts(page): """The host rows the page renders: current up/down, uptime bars, and the overall percent. Shared by the HTML page and its polling JSON endpoint.""" @@ -81,7 +87,10 @@ def _page_hosts(page): hist = history.get(str(h.id), {}) pct = hist.get("pct") hosts.append({ - "id": str(h.id), + # Opaque and per page: the live refresh only needs a stable key to + # find each card, and the public page should not publish the + # host's internal id. + "id": _public_key(page, h), "hostname": labels.get(str(h.id)) or h.hostname, "up": h.status == Host.Status.ONLINE, "bars": hist.get("bars", []), From c93fc39cd0aed5a45d2a264f6c09be2684949b5c Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:54:05 -0400 Subject: [PATCH 33/68] fix(security): the installer file tree serves only files that really live in it install_tree needs no login and normalised '..' out of the requested path, but followed symlinks. Extraction declines links whose text escapes the tree; the endpoint now also checks where the request really lands, links resolved, so one link can never make it serve a file from elsewhere on the server. --- server/apps/reprovision/installer_views.py | 11 +++++++++-- .../apps/reprovision/test_installer_endpoints.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/server/apps/reprovision/installer_views.py b/server/apps/reprovision/installer_views.py index c49ab756..8e409abf 100644 --- a/server/apps/reprovision/installer_views.py +++ b/server/apps/reprovision/installer_views.py @@ -124,9 +124,16 @@ def install_tree(request, image_id, path: str = ""): target = Path(os.path.normpath(root / path.lstrip("/"))) if target != root and root not in target.parents: raise Http404 - if not target.is_file(): + # And where it really lands, links resolved. Extraction declines symlinks + # whose text escapes the tree, but this endpoint needs no login, so it does + # not rest on that alone: a link must not turn it into a reader for + # anything the server can open (/proc/self/environ holds the signing seed). + real = target.resolve() + if real != root and root not in real.parents: raise Http404 - return FileResponse(target.open("rb")) + if not real.is_file(): + raise Http404 + return FileResponse(real.open("rb")) @api_view(["POST"]) diff --git a/server/apps/reprovision/test_installer_endpoints.py b/server/apps/reprovision/test_installer_endpoints.py index c743f3b9..3702844a 100644 --- a/server/apps/reprovision/test_installer_endpoints.py +++ b/server/apps/reprovision/test_installer_endpoints.py @@ -188,6 +188,22 @@ def test_traversal_out_of_the_tree_is_refused(self): f"/reprovision/tree/{self.image.id}/../secret.txt") self.assertEqual(resp.status_code, 404) + def test_a_symlink_out_of_the_tree_is_not_followed(self): + # The endpoint needs no login: a link in the tree must not make it a + # reader for files outside it. + from pathlib import Path + tree = Path(self.image.tree_path) + (tree / "casper" / "leak").symlink_to(Path(self.tmp.name) / "secret.txt") + resp = self.client.get(f"/reprovision/tree/{self.image.id}/casper/leak") + self.assertEqual(resp.status_code, 404) + + def test_a_symlink_inside_the_tree_still_serves(self): + from pathlib import Path + tree = Path(self.image.tree_path) + (tree / "alias").symlink_to(tree / "casper") + resp = self.client.get(f"/reprovision/tree/{self.image.id}/alias/filesystem.squashfs") + self.assertEqual(resp.status_code, 200) + def test_unready_image_is_not_served(self): self.image.status = OSImage.Status.IMPORTING self.image.save(update_fields=["status"]) From cbcfa0d0939416af55439204b4fb22ffff4d824b Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 14:58:20 -0400 Subject: [PATCH 34/68] fix(ui): line up rollout wave columns, and let the keyboard open a card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each wave row was a flex row where only the name had a min-width, so the host/report counts, the failed chip, the tags and the Machines button landed at a different x in every card and every row. One grid template per row gives the whole page the same column positions; empty slots are filled cells rather than nothing, which is what made the remaining columns drift. The card is role="button" tabindex="0" but its onkeydown attribute was dead markup — CSP has no 'unsafe-inline' for script-src, so pressing Enter on a focused card did nothing. The page already delegates clicks off [data-rlt-open], so keydown delegates off the same attribute instead of binding per card, which would not survive the 5-second re-render. The card's inline styles moved into .rlt-* classes with the same values. The state colours stay inline because they are data-driven. Phase QA-18. --- .../apps/tasks/test_qa18_rollout_columns.py | 127 ++++++++++++++++++ server/static/css/vigil.css | 38 ++++++ server/static/js/vigil-rollout.js | 62 +++++---- 3 files changed, 202 insertions(+), 25 deletions(-) create mode 100644 server/apps/tasks/test_qa18_rollout_columns.py diff --git a/server/apps/tasks/test_qa18_rollout_columns.py b/server/apps/tasks/test_qa18_rollout_columns.py new file mode 100644 index 00000000..89fb6dca --- /dev/null +++ b/server/apps/tasks/test_qa18_rollout_columns.py @@ -0,0 +1,127 @@ +"""Rollout cards: wave columns line up, and the card opens from the keyboard. + +The waves page re-renders every five seconds, so both of these are text +properties of the generated markup — the page can only be checked by reading +the template strings the browser will run. +""" + +import re +from pathlib import Path + +from django.test import SimpleTestCase + +ROOT = Path(__file__).resolve().parents[2] +JS = ROOT / "static/js/vigil-rollout.js" +CSS = ROOT / "static/css/vigil.css" + + +def _code_only(src: str) -> str: + """JS with comments blanked, so a comment that *names* a banned attribute + isn't reported as the offence. Quotes are kept intact — the banned + attributes are searched for inside template strings.""" + out = [] + i, n = 0, len(src) + in_line = in_block = in_str = False + quote = "" + while i < n: + ch = src[i] + nxt = src[i + 1] if i + 1 < n else "" + if in_line: + if ch == "\n": + in_line = False + out.append(ch) + elif in_block: + if ch == "*" and nxt == "/": + in_block = False + out.append(" ") + i += 2 + continue + out.append("\n" if ch == "\n" else " ") + elif in_str: + out.append(ch) + if ch == "\\": + out.append(nxt) + i += 2 + continue + if ch == quote: + in_str = False + elif ch == "/" and nxt == "/": + in_line = True + out.append(" ") + i += 2 + continue + elif ch == "/" and nxt == "*": + in_block = True + out.append(" ") + i += 2 + continue + else: + out.append(ch) + if ch in "\"'`": + in_str = True + quote = ch + i += 1 + return "".join(out) + + +def _wave_progress_src(code: str) -> str: + """The body of _waveProgress, up to the next top-level function.""" + start = code.index("function _waveProgress") + end = code.index("\nfunction ", start + len("function _waveProgress")) + return code[start:end] + + +class WaveColumnTests(SimpleTestCase): + + def test_wave_rows_are_a_grid(self): + wave = _wave_progress_src(_code_only(JS.read_text())) + for needed in ('class="rlt-wave"', "rlt-wave-tags", "rlt-wave-act"): + self.assertIn(needed, wave) + self.assertNotIn("display:flex", wave) + + css = CSS.read_text() + block = re.search(r"\.rlt-wave \{([^}]*)\}", css) + self.assertIsNotNone(block, "vigil.css has no .rlt-wave rule") + self.assertIn( + "grid-template-columns: 14px minmax(110px, 160px) minmax(0, 1fr) " + "96px minmax(0, 180px) 92px", + block.group(1), + ) + + collapse = re.search( + r"@media \(max-width: 640px\) \{(.*?)\n\}", css, re.DOTALL + ) + self.assertIsNotNone(collapse, "no 640px collapse for the wave grid") + self.assertIn(".rlt-wave { grid-template-columns: 14px 1fr auto; }", + collapse.group(1)) + self.assertIn(".rlt-wave-tags { display: none; }", collapse.group(1)) + + def test_six_cells_always(self): + """An empty slot collapses the grid to five columns, and everything + after the gap shifts — the drift this phase exists to stop.""" + wave = _wave_progress_src(_code_only(JS.read_text())) + self.assertIn(": ''", wave) + self.assertNotIn(": ''}", wave) + + +class CardKeyboardTests(SimpleTestCase): + + def test_no_inline_handler(self): + js = _code_only(JS.read_text()) + for forbidden in ("onkeydown=", "onclick="): + self.assertNotIn(forbidden, js) + + listeners = re.findall( + r"addEventListener\('keydown'.*?\n\}\);", js, re.DOTALL + ) + self.assertTrue(listeners, "vigil-rollout.js has no keydown listener") + # There is more than one keydown listener on this page (the modals' + # Escape); the card's is the one keyed off the open attribute. + body = next((b for b in listeners if "[data-rlt-open]" in b), listeners[0]) + self.assertIn("[data-rlt-open]", body) + self.assertIn("openRolloutDetail(", body) + + def test_card_keeps_its_openable_attributes(self): + js = JS.read_text() + self.assertIn('role="button"', js) + self.assertIn('tabindex="0"', js) diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index ee3c2745..a89356e3 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -3926,3 +3926,41 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } letter-spacing: .4em; text-align: center; } + +/* QA-18: rollout cards. Wave rows were flex, so the counts, the failed chip, + the tags and the Machines button landed at a different x in every card. One + grid template per row lines the columns up down the whole page. */ +.rlt-card { padding: 14px 16px; cursor: pointer; } +.rlt-head { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; } +.rlt-body { flex: 1; min-width: 0; } +.rlt-head-meta { margin-left: auto; color: var(--text-3); font-size: 11px; } +.rlt-state { font-weight: 600; font-size: 12px; text-transform: uppercase; letter-spacing: 0.04em; } +.rlt-wave { + display: grid; + grid-template-columns: 14px minmax(110px, 160px) minmax(0, 1fr) 96px minmax(0, 180px) 92px; + align-items: center; + gap: 8px; + padding: 5px 0; + border-bottom: 1px solid var(--s2); +} +.rlt-wave-name { font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.rlt-wave-count { color: var(--text-3); font-size: 12px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.rlt-wave-failed { display: flex; justify-content: flex-start; } +.rlt-wave-tags { color: var(--text-3); font-size: 11px; text-align: right; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.rlt-wave-act { text-align: right; } +.rlt-halted { + margin: 8px 0 2px; + padding: 8px 10px; + border: 1px solid var(--rose); + border-radius: 6px; + color: var(--rose-ink); + font-size: 12px; +} +.rlt-waves { margin-top: 10px; } +.rlt-gate { display: flex; align-items: center; gap: 10px; margin-top: 10px; flex-wrap: wrap; } +.rlt-gate-text { color: var(--text-3); font-size: 11px; } + +@media (max-width: 640px) { + .rlt-wave { grid-template-columns: 14px 1fr auto; } + .rlt-wave-tags { display: none; } +} diff --git a/server/static/js/vigil-rollout.js b/server/static/js/vigil-rollout.js index 3075e283..077c0b9b 100644 --- a/server/static/js/vigil-rollout.js +++ b/server/static/js/vigil-rollout.js @@ -53,14 +53,14 @@ function _waveProgress(r) { const openable = wave.tasks_total > 0; const failed = wave.tasks_failed ? `${wave.tasks_failed} failed` - : ''; - return `
+ : ''; + return `
${_waveDot(wave.status)} - ${escHtml(wave.name)} - ${wave.hosts} host${wave.hosts === 1 ? '' : 's'} · ${count}${validation} - ${failed} - ${(wave.tags || []).map(escHtml).join(', ')} - ${openable ? `` : ''} + ${escHtml(wave.name)} + ${wave.hosts} host${wave.hosts === 1 ? '' : 's'} · ${count}${validation} + ${failed} + ${(wave.tags || []).map(escHtml).join(', ')} + ${openable ? `` : ''}
`; }).join(''); } @@ -128,28 +128,29 @@ function _rolloutCard(r) { actions = ``; } const reason = r.halted_reason - ? `
+ ? `
Halted: ${escHtml(r.halted_reason)} ${r.halted_by_name ? `by ${escHtml(r.halted_by_name)}` : ''}
` : ''; - return `
-
- ${escHtml(r.target_name || r.definition_name || r.playbook_name || '(deleted)')}${r.action_kind === 'playbook' ? ' playbook' : ''} - ${escHtml(r.state)} - ${r.current_wave_name ? `wave: ${escHtml(r.current_wave_name)}` : ''} - ${escHtml(r.created_by_name || '')} · started ${_fmtTs(r.started_at)} -
- ${reason} -
${_waveProgress(r)}
-
- - gate: halt above ${r.failure_threshold_pct}% · min ${r.min_results_before_halt} results - ${r.resumed_by_name ? ` · resumed by ${escHtml(r.resumed_by_name)}` : ''} - - ${actions} + return `
+
+
+ ${escHtml(r.target_name || r.definition_name || r.playbook_name || '(deleted)')}${r.action_kind === 'playbook' ? ' playbook' : ''} + ${escHtml(r.state)} + ${r.current_wave_name ? `wave: ${escHtml(r.current_wave_name)}` : ''} + ${escHtml(r.created_by_name || '')} · started ${_fmtTs(r.started_at)} +
+ ${reason} +
${_waveProgress(r)}
+
+ + gate: halt above ${r.failure_threshold_pct}% · min ${r.min_results_before_halt} results + ${r.resumed_by_name ? ` · resumed by ${escHtml(r.resumed_by_name)}` : ''} + + ${actions} +
`; } @@ -534,3 +535,14 @@ delegateClick('[data-rlt-open]', (el, ev) => { if (ev.target.closest('button')) return; openRolloutDetail(el.dataset.rltOpen); }); +/* The card is role="button" tabindex="0", so the keyboard has to open it too. + An onkeydown attribute would do it but CSP forbids inline handlers, and a + listener per card would not survive the 5-second re-render — so one + delegated listener, keyed off the same attribute the click uses. */ +document.addEventListener('keydown', (ev) => { + const card = ev.target.closest && ev.target.closest('[data-rlt-open]'); + if (!card || (ev.key !== 'Enter' && ev.key !== ' ')) return; + if (ev.target.closest('button')) return; + ev.preventDefault(); + openRolloutDetail(card.dataset.rltOpen); +}); From 4eccbce988557015a2d83a280528cc12b9d387c1 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:21:31 -0400 Subject: [PATCH 35/68] fix(security): a compose file cannot reach the host through capabilities, devices or host paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_compose is the sandbox that keeps a stack from owning its host where stack_deploy is allowed but scripts are not. It refused privileged, pid: host, cap_add: ALL and binds of system paths, but each of these got through: host-escaping capabilities named one at a time (SYS_ADMIN, SYS_MODULE, SYS_PTRACE, …), raw device passthrough, security_opt unconfined or label disable, ipc/userns/cgroup/uts host, a bind of ~ (root's home), a named volume whose driver_opts bind a host path, and env_file, build contexts, secrets and configs that read files from anywhere on the host. All of them are refused now. GPU, TUN and FUSE devices, NET_ADMIN, network_mode: host, and paths inside the stack's own folder are still fine. --- server/apps/stacks/tests.py | 49 +++++++++++++++++++++++++ server/apps/stacks/validation.py | 61 ++++++++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 24d9f04d..93df9a4a 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -337,3 +337,52 @@ def test_untagged_credential_covers_every_host_and_refusals(self): with self.subTest(body=body): self.assertEqual(self.api.post("/api/v1/stacks/registries/", body, format="json").status_code, 400) self.assertEqual(self.client.get("/api/v1/agent/registry-auth/?registry=ghcr.io").status_code, 401) + + +class SandboxEscapeTests(SimpleTestCase): + """validate_compose is what keeps a stack from owning its host where + stack_deploy is allowlisted but scripts are not (architect review, + 2026-10-08). Each of these reached the host another way.""" + + S = "services:\n a:\n image: x\n" + + def test_more_escapes_are_refused(self): + bad = { + "cap sys_admin": self.S + " cap_add: [SYS_ADMIN]\n", + "cap sys_module lowercase": self.S + " cap_add: [cap_sys_module]\n", + "cap sys_ptrace": self.S + " cap_add: [SYS_PTRACE]\n", + "raw disk device": self.S + " devices: ['/dev/sda:/dev/sda']\n", + "device long form": self.S + " devices:\n - /dev/mem\n", + "apparmor unconfined": self.S + " security_opt: ['apparmor:unconfined']\n", + "seccomp unconfined": self.S + " security_opt: ['seccomp=unconfined']\n", + "label disable": self.S + " security_opt: ['label:disable']\n", + "ipc host": self.S + " ipc: host\n", + "userns host": self.S + " userns_mode: host\n", + "cgroup host": self.S + " cgroup: host\n", + "home bind": self.S + " volumes: ['~/.ssh:/keys']\n", + "env_file absolute": self.S + " env_file: /etc/shadow\n", + "env_file climb": self.S + " env_file: ['../../etc/x']\n", + "build context absolute": "services:\n a:\n build: /\n", + "build context climb": "services:\n a:\n build: {context: ../..}\n", + "named volume binds etc": self.S + " volumes: ['data:/d']\nvolumes:\n data:\n driver_opts: {type: none, o: bind, device: /etc}\n", + "secret from host file": self.S + "secrets:\n s:\n file: /etc/shadow\n", + "config from host file": self.S + "configs:\n c:\n file: /root/.ssh/id_rsa\n", + } + for label, text in bad.items(): + with self.subTest(label): + with self.assertRaises(StackError): + validate_compose(text) + + def test_ordinary_setups_still_pass(self): + good = [ + self.S + " devices: ['/dev/dri:/dev/dri']\n", + self.S + " devices: ['/dev/net/tun:/dev/net/tun']\n cap_add: [NET_ADMIN]\n", + self.S + " network_mode: host\n", + self.S + " env_file: .env\n", + "services:\n a:\n build: ./app\n", + self.S + " volumes: ['data:/d']\nvolumes:\n data: {}\n", + self.S + "secrets:\n s:\n file: ./secret.txt\n", + ] + for text in good: + with self.subTest(text=text): + validate_compose(text) diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index 66974617..cbfbb328 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -29,6 +29,28 @@ class StackError(ValueError): pass +#: Capabilities that hand a container the host (kernel modules, mounts, other +#: processes' memory, raw devices, LSM policy). NET_ADMIN and friends stay +#: allowed: VPN and network containers need them and they do not escape. +_FORBIDDEN_CAPS = {"ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_PTRACE", "SYS_RAWIO", "SYS_BOOT", + "DAC_READ_SEARCH", "BPF", "PERFMON", "MAC_ADMIN", "MAC_OVERRIDE"} +#: Host devices a stack may pass through: GPUs, a TUN device for VPNs, FUSE. +_ALLOWED_DEVICE_PREFIXES = ("/dev/dri", "/dev/nvidia", "/dev/net/tun", "/dev/fuse", "/dev/kfd") +#: Namespaces a stack may not share with the host. network_mode: host stays +#: allowed; it shares ports, not the filesystem or other processes. +_HOST_NAMESPACE_KEYS = ("pid", "ipc", "userns_mode", "cgroup", "uts") + + +def _local_path_problem(value, what: str, name: str) -> None: + """A path compose reads on the host must stay inside the stack's folder.""" + text = str(value or "").strip() + if not text: + return + parts = text.replace("\\", "/").split("/") + if text.startswith(("/", "~")) or ".." in parts: + raise StackError(f"{name}: {what} must be a path inside the stack's folder, not {text}") + + def _bind_source(volume) -> str: if isinstance(volume, dict): return str(volume.get("source") or "") if volume.get("type") == "bind" else "" @@ -57,18 +79,53 @@ def validate_compose(text: str) -> dict: raise StackError(f"service {name!r}: privileged containers are not allowed") if str(svc.get("pid") or "") == "host": raise StackError(f"service {name!r}: pid: host is not allowed") - if any(str(c).upper() == "ALL" for c in svc.get("cap_add") or []): - raise StackError(f"service {name!r}: cap_add: ALL is not allowed") + for cap in svc.get("cap_add") or []: + normal = str(cap).upper().removeprefix("CAP_") + if normal in _FORBIDDEN_CAPS: + raise StackError(f"service {name!r}: cap_add: {normal} is not allowed") + for key in _HOST_NAMESPACE_KEYS: + if str(svc.get(key) or "") == "host": + raise StackError(f"service {name!r}: {key}: host is not allowed") + for opt in svc.get("security_opt") or []: + low = str(opt).lower().replace("=", ":") + if "unconfined" in low or low in ("label:disable", "no-new-privileges:false"): + raise StackError(f"service {name!r}: security_opt {opt} is not allowed") + for device in svc.get("devices") or []: + source = (str(device.get("source") or "") if isinstance(device, dict) + else str(device).split(":", 1)[0]) + if not source.startswith(_ALLOWED_DEVICE_PREFIXES): + raise StackError(f"service {name!r}: passing through the host device {source} is not allowed") + env_files = svc.get("env_file") or [] + for env_file in env_files if isinstance(env_files, list) else [env_files]: + path = env_file.get("path") if isinstance(env_file, dict) else env_file + _local_path_problem(path, "env_file", f"service {name!r}") + build = svc.get("build") + if build is not None: + _local_path_problem(build.get("context") if isinstance(build, dict) else build, + "the build context", f"service {name!r}") for volume in svc.get("volumes") or []: source = _bind_source(volume) if not source: continue + if source.startswith("~"): + raise StackError(f"service {name!r}: binding a home directory ({source}) is not allowed") parts = source.replace("\\", "/").split("/") if ".." in parts: raise StackError(f"service {name!r}: a bind source may not climb out with '..' ({source})") if source.startswith("/") and (source.rstrip("/") == "" or any( source == p or source.startswith(p + "/") for p in _FORBIDDEN_BINDS)): raise StackError(f"service {name!r}: binding {source} from the host is not allowed") + # Top-level named volumes can bind a host path through driver_opts, and + # secrets/configs can read a host file into the container. + for vname, vol in (doc.get("volumes") or {}).items(): + opts = (vol or {}).get("driver_opts") if isinstance(vol, dict) else None + device = str((opts or {}).get("device") or "") + if device and (device.startswith(("/", "~")) or ".." in device.split("/")): + raise StackError(f"volume {vname!r}: binding the host path {device} is not allowed") + for section in ("secrets", "configs"): + for item, spec in (doc.get(section) or {}).items(): + if isinstance(spec, dict) and "file" in spec: + _local_path_problem(spec["file"], "file", f"{section[:-1]} {item!r}") return doc From 6806b215ceb66cc77b6a51adf458ab77ee199f66 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:26:52 -0400 Subject: [PATCH 36/68] fix(security): the compose sandbox allows only keys it knows, and no variables where they decide access The previous commit extended a denylist, and a denylist only knows the routes someone already thought of. A review found more: volumes_from a container that holds the engine socket, include and extends pulling unchecked compose files from the host, device_cgroup_rules, use_api_socket, and privileged written as the string "true" or yes, which compose reads as true. And compose substitutes $-variables at deploy time, after this check: '${HOSTDIR}:/data' passed as not a host path and became a bind of whatever the .env said. validate_compose now allows only top-level and service keys it knows (x-* extensions too) and refuses the rest. Fields that decide what a container can reach (volumes, devices, capabilities, privileged, namespaces, env_file, build, driver_opts, secrets and configs) may not contain variables. volumes_from may name only services in the same stack, sysctls are limited to net.*, and true is read the way compose reads it. Variables elsewhere (image tags, ports, environment) still work. --- server/apps/stacks/tests.py | 48 ++++++++++++ server/apps/stacks/validation.py | 129 ++++++++++++++++++++++++++----- 2 files changed, 158 insertions(+), 19 deletions(-) diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 93df9a4a..5dec1854 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -386,3 +386,51 @@ def test_ordinary_setups_still_pass(self): for text in good: with self.subTest(text=text): validate_compose(text) + + +class SandboxAllowlistTests(SimpleTestCase): + """The denylist missed routes compose offers; the sandbox now allows only + keys it knows, refuses $ variables where a value decides what the + container can reach, and reads YAML booleans the way compose does.""" + + S = "services:\n a:\n image: x\n" + + def test_more_routes_are_refused(self): + bad = { + "volumes_from another container": self.S + " volumes_from: ['container:portainer']\n", + "include other compose files": "include: ['/opt/other/compose.yaml']\n" + self.S, + "extends a file": "services:\n a:\n extends: {file: /opt/x.yaml, service: b}\n", + "device cgroup rules": self.S + " device_cgroup_rules: ['b 8:* rmw']\n", + "api socket": self.S + " use_api_socket: true\n", + "privileged as string": self.S + " privileged: 'true'\n", + "privileged as yes": self.S + " privileged: yes\n", + "variable bind source": self.S + " volumes: ['${HOSTDIR}:/data']\n", + "variable long bind": self.S + " volumes:\n - {type: bind, source: '${D}', target: /d}\n", + "variable cap": self.S + " cap_add: ['${CAP}']\n", + "variable device": self.S + " devices: ['${DEV}:/dev/x']\n", + "variable privileged": self.S + " privileged: ${P}\n", + "variable env_file": self.S + " env_file: ${F}\n", + "variable driver device": self.S + " volumes: ['d:/d']\nvolumes:\n d:\n driver_opts: {o: bind, device: '${X}'}\n", + "unknown service key": self.S + " some_future_key: true\n", + "unknown top-level key": "something: else\n" + self.S, + "pid of another container": self.S + " pid: 'container:other'\n", + "host sysctl": self.S + " sysctls: {kernel.core_pattern: '|/tmp/x'}\n", + } + for label, text in bad.items(): + with self.subTest(label): + with self.assertRaises(StackError): + validate_compose(text) + + def test_ordinary_variables_and_keys_still_pass(self): + good = [ + self.S + " environment: {TZ: '${TZ:-UTC}', DB: '${DB_PASSWORD}'}\n", + "services:\n a:\n image: 'nginx:${TAG:-stable}'\n ports: ['${PORT:-80}:80']\n", + self.S + " volumes_from: [b]\n b:\n image: y\n", + self.S + " sysctls: {net.core.somaxconn: 1024}\n", + self.S + " deploy:\n resources:\n limits: {memory: 512M}\n", + "x-common: &c {restart: always}\n" + self.S + " <<: *c\n", + self.S + " healthcheck: {test: ['CMD', 'true']}\n labels: {a: b}\n logging: {driver: json-file}\n", + ] + for text in good: + with self.subTest(text=text): + validate_compose(text) diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index cbfbb328..c5a9077a 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -26,7 +26,8 @@ class StackError(ValueError): - pass + #: 1-based line of a YAML error, when the parser reported one. + line = None #: Capabilities that hand a container the host (kernel modules, mounts, other @@ -61,6 +62,48 @@ def _bind_source(volume) -> str: return source if source.startswith(("/", ".", "~")) else "" +#: Top-level keys a stack file may use. ``x-*`` extension fields are allowed +#: too. Everything else is refused: ``include`` pulls in other compose files +#: from the host, unchecked, and a key added to compose later is unknown here +#: until someone has looked at what it can reach. +_TOP_LEVEL_KEYS = {"version", "name", "services", "networks", "volumes", "secrets", "configs"} + +#: Service keys a stack may use. An allowlist, not a denylist: compose keeps +#: adding ways to reach the host (use_api_socket, device_cgroup_rules, +#: volumes_from a container that holds the engine socket), and a denylist only +#: knows the ones someone already thought of. Keys checked further below are +#: marked; the rest cannot reach outside the container's own namespaces. +_SERVICE_KEYS = { + "image", "command", "entrypoint", "environment", "ports", "expose", "restart", + "depends_on", "networks", "healthcheck", "labels", "logging", "container_name", "hostname", + "domainname", "user", "working_dir", "stop_signal", "stop_grace_period", "deploy", "cap_drop", + "dns", "dns_search", "dns_opt", "extra_hosts", "tmpfs", "ulimits", "shm_size", "mem_limit", + "memswap_limit", "mem_reservation", "cpus", "cpu_shares", "cpuset", "cpu_count", "cpu_percent", + "pids_limit", "read_only", "init", "tty", "stdin_open", "platform", "pull_policy", "profiles", + "group_add", "oom_score_adj", "oom_kill_disable", "mac_address", "links", "attach", "scale", + "annotations", "gpus", "runtime", "stop_grace", "post_start", "pre_stop", "develop", + # checked below + "build", "env_file", "volumes", "volumes_from", "devices", "cap_add", "security_opt", + "privileged", "pid", "ipc", "userns_mode", "cgroup", "uts", "network_mode", "sysctls", + "secrets", "configs", +} + +#: Values compose reads as true, whatever YAML parser read them first. +_TRUTHY = {"true", "yes", "y", "on", "1"} + + +def _no_variables(value, where: str) -> None: + """A $-variable is substituted by compose at deploy time, from the .env or + the agent's environment, after this check has run: in a field that decides + what the container can reach, it would let a later value undo the check.""" + if "$" in str(value): + raise StackError(f"{where}: variables are not allowed here ({value})") + + +def _flag(value) -> bool: + return value is True or str(value).strip().lower() in _TRUTHY + + def validate_compose(text: str) -> dict: if not text or not text.strip(): raise StackError("the compose file is empty") @@ -69,62 +112,110 @@ def validate_compose(text: str) -> dict: try: doc = yaml.safe_load(text) except yaml.YAMLError as exc: - raise StackError(f"the compose file is not valid YAML: {exc}") from exc + err = StackError(f"the compose file is not valid YAML: {exc}") + mark = getattr(exc, "problem_mark", None) + err.line = mark.line + 1 if mark is not None else None + raise err from exc if not isinstance(doc, dict) or not isinstance(doc.get("services"), dict) or not doc["services"]: raise StackError("the compose file needs a services: mapping with at least one service") + for key in doc: + if key not in _TOP_LEVEL_KEYS and not str(key).startswith("x-"): + raise StackError(f"top-level key {key!r} is not allowed in a Vigil-managed stack") + + service_names = set(doc["services"]) for name, svc in doc["services"].items(): + where = f"service {name!r}" if not isinstance(svc, dict): - raise StackError(f"service {name!r} must be a mapping") - if svc.get("privileged") is True: - raise StackError(f"service {name!r}: privileged containers are not allowed") - if str(svc.get("pid") or "") == "host": - raise StackError(f"service {name!r}: pid: host is not allowed") + raise StackError(f"{where} must be a mapping") + for key in svc: + if key not in _SERVICE_KEYS and not str(key).startswith("x-"): + raise StackError(f"{where}: {key!r} is not allowed in a Vigil-managed stack") + + if "privileged" in svc: + _no_variables(svc["privileged"], f"{where}: privileged") + if _flag(svc["privileged"]): + raise StackError(f"{where}: privileged containers are not allowed") + for key in ("pid", "ipc", "userns_mode", "cgroup", "uts"): + if key in svc: + value = str(svc[key] or "") + _no_variables(value, f"{where}: {key}") + if value == "host" or value.startswith("container:"): + raise StackError(f"{where}: {key}: {value} is not allowed") + if "network_mode" in svc: + mode = str(svc["network_mode"] or "") + _no_variables(mode, f"{where}: network_mode") + if mode.startswith("service:") and mode.split(":", 1)[1] not in service_names: + raise StackError(f"{where}: network_mode {mode} names a service outside this stack") for cap in svc.get("cap_add") or []: + _no_variables(cap, f"{where}: cap_add") normal = str(cap).upper().removeprefix("CAP_") if normal in _FORBIDDEN_CAPS: - raise StackError(f"service {name!r}: cap_add: {normal} is not allowed") - for key in _HOST_NAMESPACE_KEYS: - if str(svc.get(key) or "") == "host": - raise StackError(f"service {name!r}: {key}: host is not allowed") + raise StackError(f"{where}: cap_add: {normal} is not allowed") for opt in svc.get("security_opt") or []: + _no_variables(opt, f"{where}: security_opt") low = str(opt).lower().replace("=", ":") if "unconfined" in low or low in ("label:disable", "no-new-privileges:false"): - raise StackError(f"service {name!r}: security_opt {opt} is not allowed") + raise StackError(f"{where}: security_opt {opt} is not allowed") for device in svc.get("devices") or []: source = (str(device.get("source") or "") if isinstance(device, dict) else str(device).split(":", 1)[0]) + _no_variables(source, f"{where}: devices") if not source.startswith(_ALLOWED_DEVICE_PREFIXES): - raise StackError(f"service {name!r}: passing through the host device {source} is not allowed") + raise StackError(f"{where}: passing through the host device {source} is not allowed") + for key, value in (svc.get("sysctls") or {}).items() if isinstance(svc.get("sysctls"), dict) else []: + if not str(key).startswith("net."): + raise StackError(f"{where}: sysctl {key} is not allowed (only net.*)") + for entry in svc.get("sysctls") or [] if isinstance(svc.get("sysctls"), list) else []: + if not str(entry).startswith("net."): + raise StackError(f"{where}: sysctl {entry} is not allowed (only net.*)") + for source in svc.get("volumes_from") or []: + _no_variables(source, f"{where}: volumes_from") + target = str(source).split(":", 1)[0] + if str(source).startswith("container:") or target not in service_names: + raise StackError(f"{where}: volumes_from may only name a service in this stack ({source})") + env_files = svc.get("env_file") or [] for env_file in env_files if isinstance(env_files, list) else [env_files]: path = env_file.get("path") if isinstance(env_file, dict) else env_file - _local_path_problem(path, "env_file", f"service {name!r}") + _no_variables(path, f"{where}: env_file") + _local_path_problem(path, "env_file", where) build = svc.get("build") if build is not None: - _local_path_problem(build.get("context") if isinstance(build, dict) else build, - "the build context", f"service {name!r}") + for value in ((build.get("context"), build.get("dockerfile")) if isinstance(build, dict) else (build,)): + if value is not None: + _no_variables(value, f"{where}: build") + _local_path_problem(value, "the build context", where) + for volume in svc.get("volumes") or []: + if isinstance(volume, dict): + _no_variables(volume.get("source") or "", f"{where}: volumes") + else: + _no_variables(str(volume).split(":", 1)[0], f"{where}: volumes") source = _bind_source(volume) if not source: continue if source.startswith("~"): - raise StackError(f"service {name!r}: binding a home directory ({source}) is not allowed") + raise StackError(f"{where}: binding a home directory ({source}) is not allowed") parts = source.replace("\\", "/").split("/") if ".." in parts: - raise StackError(f"service {name!r}: a bind source may not climb out with '..' ({source})") + raise StackError(f"{where}: a bind source may not climb out with '..' ({source})") if source.startswith("/") and (source.rstrip("/") == "" or any( source == p or source.startswith(p + "/") for p in _FORBIDDEN_BINDS)): - raise StackError(f"service {name!r}: binding {source} from the host is not allowed") + raise StackError(f"{where}: binding {source} from the host is not allowed") + # Top-level named volumes can bind a host path through driver_opts, and # secrets/configs can read a host file into the container. for vname, vol in (doc.get("volumes") or {}).items(): opts = (vol or {}).get("driver_opts") if isinstance(vol, dict) else None + for opt_value in (opts or {}).values(): + _no_variables(opt_value, f"volume {vname!r}: driver_opts") device = str((opts or {}).get("device") or "") if device and (device.startswith(("/", "~")) or ".." in device.split("/")): raise StackError(f"volume {vname!r}: binding the host path {device} is not allowed") for section in ("secrets", "configs"): for item, spec in (doc.get(section) or {}).items(): if isinstance(spec, dict) and "file" in spec: + _no_variables(spec["file"], f"{section[:-1]} {item!r}") _local_path_problem(spec["file"], "file", f"{section[:-1]} {item!r}") return doc From 885cbfcd8fcbaecd8a056db8909d091c2902a098 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:30:46 -0400 Subject: [PATCH 37/68] fix(security): no lifecycle hooks, build options or device paths around the compose sandbox The allowlist still let three things through. post_start and pre_stop hooks can run privileged, develop watches host paths, and runtime can name any runtime the host has configured; all four are refused. build: had its own routes to the host (ssh, entitlements, privileged builds, network host, additional_contexts and local cache import or export), and only context and dockerfile were checked. build now has an allowlist of its own, local cache sources are refused, and additional contexts must be inside the stack or an image, service or git reference. Devices were matched by string prefix, so /dev/dri/../sda and /dev/fuse-anything passed. They are normalised and matched exactly now: DRI card and render nodes, the NVIDIA devices, /dev/kfd, /dev/net/tun and /dev/fuse. --- server/apps/stacks/tests.py | 39 ++++++++++++++++++++++++ server/apps/stacks/validation.py | 52 +++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 5dec1854..0a92e6e5 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -434,3 +434,42 @@ def test_ordinary_variables_and_keys_still_pass(self): for text in good: with self.subTest(text=text): validate_compose(text) + + +class SandboxReviewTests(SimpleTestCase): + """Second review of the allowlist: hooks, build options and device paths.""" + + S = "services:\n a:\n image: x\n" + + def test_refused(self): + bad = { + "privileged post_start hook": self.S + " post_start: [{command: id, privileged: true}]\n", + "pre_stop hook": self.S + " pre_stop: [{command: id}]\n", + "develop watch": self.S + " develop: {watch: [{path: /etc, action: sync, target: /x}]}\n", + "runtime": self.S + " runtime: runc-unsafe\n", + "build additional context": "services:\n a:\n build: {context: ., additional_contexts: {h: /etc}}\n", + "build ssh": "services:\n a:\n build: {context: ., ssh: [default]}\n", + "build entitlements": "services:\n a:\n build: {context: ., entitlements: [network.host]}\n", + "build network host": "services:\n a:\n build: {context: ., network: host}\n", + "build cache_from local": "services:\n a:\n build: {context: ., cache_from: ['type=local,src=/etc']}\n", + "build cache_to local": "services:\n a:\n build: {context: ., cache_to: ['type=local,dest=/etc']}\n", + "build privileged": "services:\n a:\n build: {context: ., privileged: true}\n", + "device climbs out": self.S + " devices: ['/dev/dri/../sda:/dev/sda']\n", + "device prefix trick": self.S + " devices: ['/dev/fuse-not-really:/x']\n", + "nvidia prefix trick": self.S + " devices: ['/dev/nvidiaXYZ:/x']\n", + } + for label, text in bad.items(): + with self.subTest(label): + with self.assertRaises(StackError): + validate_compose(text) + + def test_still_allowed(self): + good = [ + "services:\n a:\n build: {context: ./app, dockerfile: Dockerfile, args: {V: 1}, target: prod}\n", + "services:\n a:\n build: {context: ., additional_contexts: {base: 'docker-image://alpine:3', b: 'service:b'}}\n b:\n image: y\n", + "services:\n a:\n build: {context: ., cache_from: ['type=registry,ref=ghcr.io/x/y:cache']}\n", + self.S + " devices: ['/dev/dri/renderD128:/dev/dri/renderD128', '/dev/nvidia0', '/dev/nvidiactl', '/dev/net/tun', '/dev/fuse']\n", + ] + for text in good: + with self.subTest(text=text): + validate_compose(text) diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index c5a9077a..192c67d2 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -36,7 +36,6 @@ class StackError(ValueError): _FORBIDDEN_CAPS = {"ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_PTRACE", "SYS_RAWIO", "SYS_BOOT", "DAC_READ_SEARCH", "BPF", "PERFMON", "MAC_ADMIN", "MAC_OVERRIDE"} #: Host devices a stack may pass through: GPUs, a TUN device for VPNs, FUSE. -_ALLOWED_DEVICE_PREFIXES = ("/dev/dri", "/dev/nvidia", "/dev/net/tun", "/dev/fuse", "/dev/kfd") #: Namespaces a stack may not share with the host. network_mode: host stays #: allowed; it shares ports, not the filesystem or other processes. _HOST_NAMESPACE_KEYS = ("pid", "ipc", "userns_mode", "cgroup", "uts") @@ -81,13 +80,33 @@ def _bind_source(volume) -> str: "memswap_limit", "mem_reservation", "cpus", "cpu_shares", "cpuset", "cpu_count", "cpu_percent", "pids_limit", "read_only", "init", "tty", "stdin_open", "platform", "pull_policy", "profiles", "group_add", "oom_score_adj", "oom_kill_disable", "mac_address", "links", "attach", "scale", - "annotations", "gpus", "runtime", "stop_grace", "post_start", "pre_stop", "develop", + "annotations", "gpus", # checked below "build", "env_file", "volumes", "volumes_from", "devices", "cap_add", "security_opt", "privileged", "pid", "ipc", "userns_mode", "cgroup", "uts", "network_mode", "sysctls", "secrets", "configs", } +#: Build keys a stack may use. ssh, entitlements, privileged builds, host +#: networking and local cache import/export all reach the host. +_BUILD_KEYS = {"context", "dockerfile", "dockerfile_inline", "args", "target", "labels", + "cache_from", "tags", "platforms", "shm_size", "extra_hosts", "no_cache", + "pull", "additional_contexts", "network"} + +#: Devices a stack may pass through, matched on the normalised path: GPUs (DRI +#: render/card nodes, NVIDIA, AMD KFD), TUN for VPNs, and FUSE. +_DEVICE_RE = re.compile( + r"/dev/dri(/(card|renderD)\d+)?|/dev/nvidia(\d+|ctl|-uvm|-uvm-tools|-modeset)" + r"|/dev/net/tun|/dev/fuse|/dev/kfd") + + +def _device_allowed(source: str) -> bool: + import posixpath + if ".." in source.split("/") or posixpath.normpath(source) != source.rstrip("/"): + return False + return bool(_DEVICE_RE.fullmatch(posixpath.normpath(source))) + + #: Values compose reads as true, whatever YAML parser read them first. _TRUTHY = {"true", "yes", "y", "on", "1"} @@ -160,7 +179,7 @@ def validate_compose(text: str) -> dict: source = (str(device.get("source") or "") if isinstance(device, dict) else str(device).split(":", 1)[0]) _no_variables(source, f"{where}: devices") - if not source.startswith(_ALLOWED_DEVICE_PREFIXES): + if not _device_allowed(source): raise StackError(f"{where}: passing through the host device {source} is not allowed") for key, value in (svc.get("sysctls") or {}).items() if isinstance(svc.get("sysctls"), dict) else []: if not str(key).startswith("net."): @@ -181,10 +200,29 @@ def validate_compose(text: str) -> dict: _local_path_problem(path, "env_file", where) build = svc.get("build") if build is not None: - for value in ((build.get("context"), build.get("dockerfile")) if isinstance(build, dict) else (build,)): - if value is not None: - _no_variables(value, f"{where}: build") - _local_path_problem(value, "the build context", where) + if not isinstance(build, dict): + build = {"context": build} + for key in build: + if key not in _BUILD_KEYS: + raise StackError(f"{where}: build.{key} is not allowed in a Vigil-managed stack") + for key in ("context", "dockerfile"): + if build.get(key) is not None: + _no_variables(build[key], f"{where}: build.{key}") + _local_path_problem(build[key], f"build.{key}", where) + if str(build.get("network") or "") not in ("", "default", "none"): + raise StackError(f"{where}: build.network {build['network']} is not allowed") + for ref in build.get("cache_from") or []: + text_ref = str(ref) + _no_variables(text_ref, f"{where}: build.cache_from") + if "type=local" in text_ref.replace(" ", ""): + raise StackError(f"{where}: build.cache_from may not read a local path ({ref})") + contexts = build.get("additional_contexts") or {} + for ctx in (contexts.values() if isinstance(contexts, dict) else contexts): + text_ctx = str(ctx).split("=", 1)[-1] if not isinstance(contexts, dict) else str(ctx) + _no_variables(text_ctx, f"{where}: build.additional_contexts") + if text_ctx.startswith(("docker-image://", "service:", "https://", "git@")): + continue + _local_path_problem(text_ctx, "build.additional_contexts", where) for volume in svc.get("volumes") or []: if isinstance(volume, dict): From 198f47f7bf6176e32b40463f0b3062dc1c521af4 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:34:00 -0400 Subject: [PATCH 38/68] fix(security): a stack cannot bind a directory that contains the engine socket Binds were refused when they were a forbidden path or inside one, but not when they contained one: /run and /var/run hold the Docker, containerd and systemd sockets, /var holds /var/lib/docker, and binding any of them reached what the list meant to protect. A bind is now refused if it is, is inside, or contains a protected path, and the list adds /run, /var/run, containerd's state, the agent's own data directory and cron's spool. Data paths such as /srv, /mnt, /home/ and /var/log/ are still fine. Keys may not contain variables anywhere in the file, since the checks look keys up by name and compose fills them in later, and annotations, which some runtimes read as instructions, are dropped from the allowlist. --- server/apps/stacks/tests.py | 27 +++++++++++++++++++++++ server/apps/stacks/validation.py | 37 ++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 0a92e6e5..69fa8777 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -473,3 +473,30 @@ def test_still_allowed(self): for text in good: with self.subTest(text=text): validate_compose(text) + + +class SandboxAncestorTests(SimpleTestCase): + """A bind of a directory that *contains* a forbidden path reaches it too.""" + + S = "services:\n a:\n image: x\n" + + def test_refused(self): + bad = { + "/run holds the engine socket": self.S + " volumes: ['/run:/host-run']\n", + "/var/run (a link to /run)": self.S + " volumes: ['/var/run:/r']\n", + "/var holds /var/lib/docker": self.S + " volumes: ['/var:/v']\n", + "/var/lib": self.S + " volumes: ['/var/lib:/l']\n", + "containerd socket dir": self.S + " volumes: ['/run/containerd:/c']\n", + "agent data": self.S + " volumes: ['/var/lib/vigil-agent:/a']\n", + "variable in a key": self.S + " volumes: ['d:/d']\nvolumes:\n d:\n driver_opts: {'${K}': /etc}\n", + "annotations": self.S + " annotations: {run.oci.keep_original_groups: '1'}\n", + } + for label, text in bad.items(): + with self.subTest(label): + with self.assertRaises(StackError): + validate_compose(text) + + def test_data_paths_still_allowed(self): + for path in ("/srv/media", "/mnt/storage", "/home/alice/music", "/var/log/app", "/opt/app-data"): + with self.subTest(path): + validate_compose(self.S + f" volumes: ['{path}:/data']\n") diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index 192c67d2..db212a14 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -20,9 +20,21 @@ MAX_ENV = 64_000 #: Host paths a stack may not bind — the engine's socket and the system. -_FORBIDDEN_BINDS = ("/var/run/docker.sock", "/run/docker.sock", "/run/podman", "/var/run/podman", - "/etc", "/root", "/boot", "/proc", "/sys", "/dev", "/usr", "/bin", "/sbin", - "/lib", "/var/lib/docker", "/var/lib/containers", "/opt/vigil") +_FORBIDDEN_BINDS = ("/run", "/var/run", "/etc", "/root", "/boot", "/proc", "/sys", "/dev", "/usr", + "/bin", "/sbin", "/lib", "/lib64", "/var/lib/docker", "/var/lib/containers", + "/var/lib/containerd", "/var/lib/vigil-agent", "/var/spool/cron", "/opt/vigil") + + +def _bind_forbidden(source: str) -> bool: + """A bind reaches a forbidden path if it is one, is inside one, or *contains* + one: binding /var reaches /var/lib/docker, and /run or /var/run reach every + engine and systemd socket on the host.""" + import posixpath + path = posixpath.normpath(source) + if path == "/": + return True + return any(path == p or path.startswith(p + "/") or p.startswith(path + "/") + for p in _FORBIDDEN_BINDS) class StackError(ValueError): @@ -80,7 +92,7 @@ def _bind_source(volume) -> str: "memswap_limit", "mem_reservation", "cpus", "cpu_shares", "cpuset", "cpu_count", "cpu_percent", "pids_limit", "read_only", "init", "tty", "stdin_open", "platform", "pull_policy", "profiles", "group_add", "oom_score_adj", "oom_kill_disable", "mac_address", "links", "attach", "scale", - "annotations", "gpus", + "gpus", # checked below "build", "env_file", "volumes", "volumes_from", "devices", "cap_add", "security_opt", "privileged", "pid", "ipc", "userns_mode", "cgroup", "uts", "network_mode", "sysctls", @@ -119,6 +131,19 @@ def _no_variables(value, where: str) -> None: raise StackError(f"{where}: variables are not allowed here ({value})") +def _no_variable_keys(node, where: str = "compose file") -> None: + """No $ in any mapping key, anywhere. The checks below look keys up by + name, and a key compose would fill in at deploy time is one they cannot.""" + if isinstance(node, dict): + for key, value in node.items(): + if "$" in str(key): + raise StackError(f"{where}: variables are not allowed in keys ({key})") + _no_variable_keys(value, f"{where}.{key}") + elif isinstance(node, list): + for value in node: + _no_variable_keys(value, where) + + def _flag(value) -> bool: return value is True or str(value).strip().lower() in _TRUTHY @@ -141,6 +166,7 @@ def validate_compose(text: str) -> dict: if key not in _TOP_LEVEL_KEYS and not str(key).startswith("x-"): raise StackError(f"top-level key {key!r} is not allowed in a Vigil-managed stack") + _no_variable_keys(doc) service_names = set(doc["services"]) for name, svc in doc["services"].items(): where = f"service {name!r}" @@ -237,8 +263,7 @@ def validate_compose(text: str) -> dict: parts = source.replace("\\", "/").split("/") if ".." in parts: raise StackError(f"{where}: a bind source may not climb out with '..' ({source})") - if source.startswith("/") and (source.rstrip("/") == "" or any( - source == p or source.startswith(p + "/") for p in _FORBIDDEN_BINDS)): + if source.startswith("/") and _bind_forbidden(source): raise StackError(f"{where}: binding {source} from the host is not allowed") # Top-level named volumes can bind a host path through driver_opts, and From d670916623caff8e3d28ef922008674860497990 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:34:37 -0400 Subject: [PATCH 39/68] fix(security): //run is /run, and pid: Host is host posixpath.normpath keeps a leading '//', which Linux reads as '/', so a bind of //run or ///etc passed the protected-path check. Leading slashes are collapsed before comparing, and namespace values (pid, ipc, userns_mode, cgroup, uts) are compared case-insensitively. --- server/apps/stacks/tests.py | 4 ++++ server/apps/stacks/validation.py | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 69fa8777..46ff6690 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -490,6 +490,10 @@ def test_refused(self): "agent data": self.S + " volumes: ['/var/lib/vigil-agent:/a']\n", "variable in a key": self.S + " volumes: ['d:/d']\nvolumes:\n d:\n driver_opts: {'${K}': /etc}\n", "annotations": self.S + " annotations: {run.oci.keep_original_groups: '1'}\n", + "double leading slash": self.S + " volumes: ['//run:/r']\n", + "triple slash etc": self.S + " volumes: ['///etc/:/e']\n", + "pid Host": self.S + " pid: Host\n", + "ipc HOST": self.S + " ipc: ' HOST '\n", } for label, text in bad.items(): with self.subTest(label): diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index db212a14..e18cd3ce 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -30,7 +30,9 @@ def _bind_forbidden(source: str) -> bool: one: binding /var reaches /var/lib/docker, and /run or /var/run reach every engine and systemd socket on the host.""" import posixpath - path = posixpath.normpath(source) + # Linux reads //run as /run, but normpath keeps a leading "//" (POSIX + # leaves it implementation-defined), so collapse it before comparing. + path = "/" + posixpath.normpath(source).lstrip("/") if path == "/": return True return any(path == p or path.startswith(p + "/") or p.startswith(path + "/") @@ -182,7 +184,7 @@ def validate_compose(text: str) -> dict: raise StackError(f"{where}: privileged containers are not allowed") for key in ("pid", "ipc", "userns_mode", "cgroup", "uts"): if key in svc: - value = str(svc[key] or "") + value = str(svc[key] or "").strip().lower() _no_variables(value, f"{where}: {key}") if value == "host" or value.startswith("container:"): raise StackError(f"{where}: {key}: {value} is not allowed") From e39912c421421e77f9d68faa851be3d1ca2b5f9f Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:39:24 -0400 Subject: [PATCH 40/68] fix(security): the agent checks compose's resolved configuration before it starts a stack Every round of review of the server's compose check found another way compose reads the same text differently: variables substituted from the .env after the check, include and extends merging other files, "true" strings read as booleans, //run read as /run. Checking the text can't win that race. So before `up`, the agent now asks compose for its resolved configuration (`config --format json`, or YAML from podman-compose) and applies the same rules to exactly what will run: protected binds, including their ancestors, privileged, host namespaces, dangerous capabilities, raw devices, sockets, volumes_from other containers, and build, secret and config paths outside the stack. If the configuration can't be read, nothing runs. Checked against real compose on the Ubuntu VM: a file hiding /etc in ${HOSTDIR} and privileged in a "true" string came back resolved, and the check named both. The server's check stays as early feedback for the editor. Its key walk now visits each YAML anchor's node once and caps nesting, so an anchor bomb cannot pin a worker. --- agent/tests/test_compose_check.py | 72 +++++++++++++++++++++ agent/tests/test_stack_deploy.py | 3 +- agent/tests/test_stack_read.py | 3 +- agent/vigil_agent/actions/stacks.py | 32 ++++++++++ agent/vigil_agent/composecheck.py | 98 +++++++++++++++++++++++++++++ server/apps/stacks/tests.py | 12 ++++ server/apps/stacks/validation.py | 19 ++++-- 7 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 agent/tests/test_compose_check.py create mode 100644 agent/vigil_agent/composecheck.py diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py new file mode 100644 index 00000000..ec2735ed --- /dev/null +++ b/agent/tests/test_compose_check.py @@ -0,0 +1,72 @@ +"""The agent checks compose's *resolved* configuration before `up` (2026-10-08). +The server checks the text; compose re-reads it (variables, include/extends, +path and boolean normalisation), so the check that counts is on what will run.""" +import tests._safety_net # noqa: F401 — the guard, even when this file is run or imported on its own + +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +# isort: split +from vigil_agent import composecheck +from vigil_agent.actions import stacks + +WD = "/opt/vigil/stacks/web" + + +def _cfg(**svc): + return {"services": {"a": {"image": "x", **svc}}} + + +class ProblemsTests(unittest.TestCase): + def test_a_plain_service_is_fine(self): + self.assertEqual(composecheck.problems(_cfg(volumes=[{"type": "bind", "source": "/srv/data", "target": "/d"}]), WD), []) + + def test_resolved_escapes_are_found(self): + cases = { + "interpolated bind": _cfg(volumes=[{"type": "bind", "source": "/etc", "target": "/e"}]), + "double slash": _cfg(volumes=[{"type": "bind", "source": "//run", "target": "/r"}]), + "ancestor of socket": _cfg(volumes=[{"type": "bind", "source": "/var", "target": "/v"}]), + "privileged": _cfg(privileged=True), + "pid host": _cfg(pid="host"), + "cap": _cfg(cap_add=["CAP_SYS_ADMIN"]), + "device": _cfg(devices=[{"source": "/dev/sda", "target": "/dev/sda"}]), + "api socket": _cfg(use_api_socket=True), + "volumes_from container": _cfg(volumes_from=["container:portainer"]), + "build outside": _cfg(build={"context": "/"}), + "secret outside": {**_cfg(), "secrets": {"s": {"file": "/etc/shadow"}}}, + "driver_opts device": {**_cfg(), "volumes": {"v": {"driver_opts": {"device": "/etc"}}}}, + } + for label, cfg in cases.items(): + with self.subTest(label): + self.assertTrue(composecheck.problems(cfg, WD)) + + +class DeployRefusesTests(unittest.TestCase): + def test_deploy_stops_before_up_when_the_resolved_config_escapes(self): + resolved = json.dumps(_cfg(volumes=[{"type": "bind", "source": "/etc", "target": "/e"}])) + calls = [] + + def fake_compose(workdir, project, *args, **kw): + calls.append(args) + return resolved if args[:1] == ("config",) else "up ok" + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(stacks, "_compose", side_effect=fake_compose), \ + patch.object(stacks.client if hasattr(stacks, "client") else stacks, "fetch_stack_env", return_value="", create=True): + with self.assertRaises(ValueError): + stacks._check_resolved(Path(tmp), "web", "compose.yaml") + self.assertFalse([c for c in calls if c[:1] == ("up",)]) + + def test_unreadable_config_refuses(self): + with patch.object(stacks, "_compose", side_effect=RuntimeError("compose missing")): + with self.assertRaises(ValueError): + stacks._check_resolved(Path("/tmp"), "web", "compose.yaml") + + +if __name__ == "__main__": + unittest.main() diff --git a/agent/tests/test_stack_deploy.py b/agent/tests/test_stack_deploy.py index bb49a922..3badd5a3 100644 --- a/agent/tests/test_stack_deploy.py +++ b/agent/tests/test_stack_deploy.py @@ -49,7 +49,8 @@ def _params(self, **extra): "env_ticket": TICKET, "revision": 3, **extra} def test_deploy_writes_files_and_fetches_env_once(self): - with patch.object(client, "fetch_stack_env", return_value="API_KEY=hunter2\n") as fetch: + with patch.object(client, "fetch_stack_env", return_value="API_KEY=hunter2\n") as fetch, \ + patch.object(stacks, "_check_resolved"): # tests/test_compose_check.py covers it out = stacks._stack_deploy(self._params(), _CFG) fetch.assert_called_once_with(_CFG, TICKET) workdir = self.root / "media" diff --git a/agent/tests/test_stack_read.py b/agent/tests/test_stack_read.py index 2e04325f..5debc550 100644 --- a/agent/tests/test_stack_read.py +++ b/agent/tests/test_stack_read.py @@ -81,7 +81,8 @@ def test_refusals(self): def test_deploy_writes_an_adopted_stacks_own_file(self): with patch.object(client, "fetch_stack_env", return_value=""), \ - patch.object(executor.collector, "request_docker_recheck"): + patch.object(executor.collector, "request_docker_recheck"), \ + patch.object(stacks, "_check_resolved"): # tests/test_compose_check.py covers it stacks._stack_deploy({"project": "shop", "compose": "services:\n web:\n image: nginx\n", "working_dir": str(self.dir), "compose_file": "docker-compose.yml"}, _CFG) self.assertIn(str(self.dir / "docker-compose.yml"), self.run_mock.call_args.args[0]) diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 4a5d8803..77c974bd 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -68,6 +68,37 @@ def _compose(workdir: Path, project: str, *args: str, timeout: int = 600, timeout=timeout, extra_env=ex._compose_env()) +def _resolved_config(workdir: Path, project: str, compose_file: str) -> dict: + """Compose's own resolved configuration: variables substituted from the + .env, include/extends merged, paths and booleans normalised.""" + import json + + import yaml + try: + return json.loads(_compose(workdir, project, "config", "--format", "json", + timeout=120, compose_file=compose_file)) + except (RuntimeError, ValueError): + # podman-compose has no --format json; its plain output is YAML. + doc = yaml.safe_load(_compose(workdir, project, "config", timeout=120, compose_file=compose_file)) + if not isinstance(doc, dict): + raise ValueError("compose config did not produce a configuration") from None + return doc + + +def _check_resolved(workdir: Path, project: str, compose_file: str) -> None: + """Refuse to start anything the resolved configuration would let reach the + host (SEC, 2026-10-08). The server checks the text it was sent; this checks + what compose will actually run, so no reading of the text can differ.""" + from ..composecheck import problems + try: + resolved = _resolved_config(workdir, project, compose_file) + except Exception as exc: # noqa: BLE001 — cannot check it, so do not run it + raise ValueError(f"could not read the resolved compose configuration: {exc}") from exc + found = problems(resolved, str(workdir)) + if found: + raise ValueError("refusing to deploy " + project + ": " + "; ".join(found[:10])) + + def _stack_deploy(params: dict, config: AgentConfig) -> str: from .. import client @@ -88,6 +119,7 @@ def _stack_deploy(params: dict, config: AgentConfig) -> str: (workdir / compose_file).write_text(compose) if env_text is not None: _write_private(workdir / ".env", env_text) + _check_resolved(workdir, project, compose_file) output = _compose(workdir, project, "up", "-d", "--remove-orphans", compose_file=compose_file) collector.request_docker_recheck() revision = params.get("revision") diff --git a/agent/vigil_agent/composecheck.py b/agent/vigil_agent/composecheck.py new file mode 100644 index 00000000..eddf2257 --- /dev/null +++ b/agent/vigil_agent/composecheck.py @@ -0,0 +1,98 @@ +"""Check what compose will actually run, just before it runs it. + +The server validates the compose text it is sent (apps/stacks/validation.py), +but compose re-reads that text: it substitutes variables from the .env and +the environment, merges include/extends, normalises paths and booleans. Any +difference between the two readings is a way past the server's check. So the +agent asks compose for its resolved configuration (``config --format json``) +and checks that, the same rules applied to exactly what ``up`` will start. +""" +from __future__ import annotations + +import posixpath +import re + +FORBIDDEN_BINDS = ("/run", "/var/run", "/etc", "/root", "/boot", "/proc", "/sys", "/dev", "/usr", + "/bin", "/sbin", "/lib", "/lib64", "/var/lib/docker", "/var/lib/containers", + "/var/lib/containerd", "/var/lib/vigil-agent", "/var/spool/cron", "/opt/vigil") +FORBIDDEN_CAPS = {"ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_PTRACE", "SYS_RAWIO", "SYS_BOOT", + "DAC_READ_SEARCH", "BPF", "PERFMON", "MAC_ADMIN", "MAC_OVERRIDE"} +DEVICE_RE = re.compile(r"/dev/dri(/(card|renderD)\d+)?|/dev/nvidia(\d+|ctl|-uvm|-uvm-tools|-modeset)" + r"|/dev/net/tun|/dev/fuse|/dev/kfd") +REFUSED_SERVICE_KEYS = ("use_api_socket", "device_cgroup_rules", "post_start", "pre_stop", "develop", + "runtime", "annotations", "cgroup_parent") + + +def _norm(path: str) -> str: + return "/" + posixpath.normpath(str(path)).lstrip("/") + + +def bind_forbidden(source: str) -> bool: + """Is, is inside, or contains a protected path.""" + path = _norm(source) + if path == "/": + return True + return any(path == p or path.startswith(p + "/") or p.startswith(path + "/") for p in FORBIDDEN_BINDS) + + +def _inside(path: str, root: str) -> bool: + path, root = _norm(path), _norm(root) + return path == root or path.startswith(root + "/") + + +def problems(config: dict, workdir: str) -> list[str]: + """Every reason not to start this resolved configuration ([] = fine).""" + out: list[str] = [] + services = config.get("services") or {} + names = set(services) + for name, svc in services.items(): + where = f"service {name!r}" + for key in REFUSED_SERVICE_KEYS: + if svc.get(key): + out.append(f"{where}: {key} is not allowed") + if svc.get("privileged") is True: + out.append(f"{where}: privileged") + for key in ("pid", "ipc", "userns_mode", "cgroup", "uts"): + value = str(svc.get(key) or "").strip().lower() + if value == "host" or value.startswith("container:"): + out.append(f"{where}: {key}: {value}") + for cap in svc.get("cap_add") or []: + if str(cap).upper().removeprefix("CAP_") in FORBIDDEN_CAPS: + out.append(f"{where}: cap_add {cap}") + for opt in svc.get("security_opt") or []: + low = str(opt).lower().replace("=", ":") + if "unconfined" in low or low in ("label:disable", "no-new-privileges:false"): + out.append(f"{where}: security_opt {opt}") + for device in svc.get("devices") or []: + source = device.get("source", "") if isinstance(device, dict) else str(device).split(":", 1)[0] + if not DEVICE_RE.fullmatch(_norm(source)) or ".." in str(source).split("/"): + out.append(f"{where}: device {source}") + for key in (svc.get("sysctls") or {}): + if not str(key).startswith("net."): + out.append(f"{where}: sysctl {key}") + for ref in svc.get("volumes_from") or []: + if str(ref).startswith("container:") or str(ref).split(":", 1)[0] not in names: + out.append(f"{where}: volumes_from {ref}") + for vol in svc.get("volumes") or []: + if isinstance(vol, dict) and vol.get("type") == "bind" and bind_forbidden(vol.get("source", "")): + out.append(f"{where}: bind of {vol.get('source')}") + build = svc.get("build") or {} + if isinstance(build, dict): + if build.get("context") and not str(build["context"]).startswith(("http", "git@")) \ + and not _inside(build["context"], workdir): + out.append(f"{where}: build context {build['context']} is outside the stack") + for key in ("ssh", "entitlements", "privileged"): + if build.get(key): + out.append(f"{where}: build.{key}") + if str(build.get("network") or "") not in ("", "default", "none"): + out.append(f"{where}: build.network {build['network']}") + for vname, vol in (config.get("volumes") or {}).items(): + device = str(((vol or {}).get("driver_opts") or {}).get("device") or "") + if device.startswith("/") or ".." in device.split("/"): + out.append(f"volume {vname!r}: binds the host path {device}") + for section in ("secrets", "configs"): + for item, spec in (config.get(section) or {}).items(): + path = (spec or {}).get("file") + if path and not _inside(path, workdir): + out.append(f"{section[:-1]} {item!r}: {path} is outside the stack") + return out diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 46ff6690..d2a68c91 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -504,3 +504,15 @@ def test_data_paths_still_allowed(self): for path in ("/srv/media", "/mnt/storage", "/home/alice/music", "/var/log/app", "/opt/app-data"): with self.subTest(path): validate_compose(self.S + f" volumes: ['{path}:/data']\n") + + +class SandboxDosTests(SimpleTestCase): + def test_an_anchor_bomb_is_checked_quickly(self): + import time + lines = ["x-a0: &a0 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]"] + for i in range(1, 25): + lines.append(f"x-a{i}: &a{i} [" + ", ".join([f"*a{i-1}"] * 10) + "]") + text = "\n".join(lines) + "\nservices:\n a:\n image: x\n" + start = time.monotonic() + validate_compose(text) + self.assertLess(time.monotonic() - start, 2.0) diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index e18cd3ce..cd36098b 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -133,17 +133,28 @@ def _no_variables(value, where: str) -> None: raise StackError(f"{where}: variables are not allowed here ({value})") -def _no_variable_keys(node, where: str = "compose file") -> None: +def _no_variable_keys(node, where: str = "compose file", _seen=None, _depth: int = 0) -> None: """No $ in any mapping key, anywhere. The checks below look keys up by - name, and a key compose would fill in at deploy time is one they cannot.""" + name, and a key compose would fill in at deploy time is one they cannot. + + YAML anchors let a small file reference one node many times over; each + node is walked once (by identity) and nesting is capped, so a crafted + file cannot make this walk run for ever.""" + _seen = set() if _seen is None else _seen + if id(node) in _seen: + return + if _depth > 64: + raise StackError(f"{where}: nested too deeply") if isinstance(node, dict): + _seen.add(id(node)) for key, value in node.items(): if "$" in str(key): raise StackError(f"{where}: variables are not allowed in keys ({key})") - _no_variable_keys(value, f"{where}.{key}") + _no_variable_keys(value, f"{where}.{key}", _seen, _depth + 1) elif isinstance(node, list): + _seen.add(id(node)) for value in node: - _no_variable_keys(value, where) + _no_variable_keys(value, where, _seen, _depth + 1) def _flag(value) -> bool: From ec0809f7baba4a7e4885e960f6927dad907d122d Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:50:54 -0400 Subject: [PATCH 41/68] fix(security): the agent starts the compose file it checked, in every shape compose accepts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolved-config check only understood docker's normal form, and `up` then re-read the original files. podman-compose's `config` hands back the file much as written, so short volume strings ("/etc:/e"), "true" strings, list sysctls and a lone cap_add string went unchecked, and anything that changed between `config` and `up` (a .env, a file a container bound to the stack folder could rewrite) was never checked at all. - composecheck: an allowlist of service and top-level keys, like the server's; every volume, device, sysctl, cap and security_opt shape; build additional_contexts and dockerfile; env_file; ~ refused; a $ in any field that decides access refused; security_opt allowlisted (custom seccomp profiles and label:type are out); bind sources and devices checked again after following symlinks. - Binds inside the stack's own folder are allowed again — ./data resolves under /opt/vigil and was refused by the previous commit. - _stack_deploy writes the checked configuration to /stacks/.json (0600, a path no stack may bind) and runs `up -f` on it. Compose labels the containers with that file, so stack_update re-runs exactly what was checked. stack_remove deletes it. Verified on the Ubuntu test VM: up from the resolved JSON keeps $$ literal, substitutes .env values once and binds ./data where expected. --- agent/tests/test_compose_check.py | 64 ++++++- agent/tests/test_stack_deploy.py | 31 +++- agent/tests/test_stack_read.py | 8 +- agent/vigil_agent/actions/stacks.py | 31 +++- agent/vigil_agent/composecheck.py | 263 ++++++++++++++++++++++------ 5 files changed, 326 insertions(+), 71 deletions(-) diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py index ec2735ed..266e7ccc 100644 --- a/agent/tests/test_compose_check.py +++ b/agent/tests/test_compose_check.py @@ -4,6 +4,7 @@ import tests._safety_net # noqa: F401 — the guard, even when this file is run or imported on its own import json +import os import sys import tempfile import unittest @@ -13,7 +14,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # isort: split -from vigil_agent import composecheck +from vigil_agent import composecheck, executor # noqa: F401 — executor first, it imports stacks from vigil_agent.actions import stacks WD = "/opt/vigil/stacks/web" @@ -46,6 +47,67 @@ def test_resolved_escapes_are_found(self): with self.subTest(label): self.assertTrue(composecheck.problems(cfg, WD)) + def test_podman_shapes_are_checked_too(self): + """podman-compose's `config` returns the file much as written, so the + short and string forms must be caught as well as docker's normal form.""" + cases = { + "short bind": _cfg(volumes=["/etc:/e"]), + "short relative escape": _cfg(volumes=["../../../../etc:/e"]), + "short home": _cfg(volumes=["~:/h"]), + "dict bind without type": _cfg(volumes=[{"source": "/root", "target": "/r"}]), + "odd volume type": _cfg(volumes=[{"type": "npipe", "source": "x", "target": "/x"}]), + "privileged string": _cfg(privileged="true"), + "privileged yes": _cfg(privileged="Yes"), + "cap string": _cfg(cap_add="SYS_ADMIN"), + "sysctl list": _cfg(sysctls=["kernel.core_pattern=|/tmp/x"]), + "device short": _cfg(devices=["/dev/sda:/dev/sda:rwm"]), + "device escape": _cfg(devices=["/dev/dri/../sda"]), + "unknown key": _cfg(storage_opt={"size": "1G"}), + "include left over": {**_cfg(), "include": ["/etc/other.yml"]}, + "network container": _cfg(network_mode="container:portainer"), + "seccomp profile": _cfg(security_opt=["seccomp=/tmp/allow-all.json"]), + "selinux spc": _cfg(security_opt=["label=type:spc_t"]), + "additional context": _cfg(build={"context": WD, "additional_contexts": {"x": "/etc"}}), + "additional context list": _cfg(build={"context": WD, "additional_contexts": ["x=/"]}), + "dockerfile outside": _cfg(build={"context": WD, "dockerfile": "/etc/shadow"}), + "env_file outside": _cfg(env_file=["/etc/shadow"]), + "env_file in home": _cfg(env_file=["~/.ssh/id_rsa"]), + "build in home": _cfg(build="~/src"), + "variable left in a bind": _cfg(volumes=["$$HOME:/h"]), + "variable privileged": _cfg(privileged="${P}"), + "relative device driver": {**_cfg(), "volumes": {"v": {"driver_opts": {"type": "none", "o": "bind", "device": "etc"}}}}, + } + for label, cfg in cases.items(): + with self.subTest(label): + self.assertTrue(composecheck.problems(cfg, WD), label) + + def test_ordinary_stacks_still_pass(self): + # docker's resolved JSON for a normal stack, captured on the test VM 2026-10-08 + cfg = {"name": "cc", "networks": {"default": {"name": "cc_default", "ipam": {}}}, + "services": {"a": {"cap_add": ["NET_ADMIN"], "command": None, "entrypoint": None, + "environment": {"A": "1"}, "image": "nginx", "networks": {"default": None}, + "ports": [{"mode": "ingress", "target": 80, "published": "8080", "protocol": "tcp"}], + "restart": "unless-stopped", "sysctls": {"net.core.somaxconn": "1024"}, + "volumes": [{"type": "bind", "source": WD + "/data", "target": "/d", + "bind": {"create_host_path": True}}, + {"type": "bind", "source": "/srv/x", "target": "/x", "read_only": True}, + {"type": "volume", "source": "named", "target": "/n", "volume": {}}]}, + "b": {"build": {"context": WD + "/sub", "dockerfile": "Dockerfile"}, + "depends_on": {"a": {"condition": "service_started", "required": True}}, + "devices": [{"source": "/dev/fuse", "target": "/dev/fuse", "permissions": "rwm"}], + "volumes_from": ["a"], "privileged": False, + "security_opt": ["no-new-privileges:true"]}}, + "volumes": {"named": {"name": "cc_named"}}} + self.assertEqual(composecheck.problems(cfg, WD), []) + self.assertEqual(composecheck.problems(_cfg(volumes=["./data:/d", "named:/n", "/anon"], + network_mode="host", sysctls=["net.ipv4.ip_forward=1"]), WD), []) + + def test_a_symlink_planted_in_the_stack_folder_is_followed(self): + with tempfile.TemporaryDirectory() as tmp: + os.symlink("/etc", os.path.join(tmp, "data")) + self.assertTrue(composecheck.problems(_cfg(volumes=["./data:/d"]), tmp)) + self.assertTrue(composecheck.problems(_cfg(build={"context": tmp + "/data"}), tmp)) + class DeployRefusesTests(unittest.TestCase): def test_deploy_stops_before_up_when_the_resolved_config_escapes(self): diff --git a/agent/tests/test_stack_deploy.py b/agent/tests/test_stack_deploy.py index 3badd5a3..726b9309 100644 --- a/agent/tests/test_stack_deploy.py +++ b/agent/tests/test_stack_deploy.py @@ -1,4 +1,6 @@ """M11 09: stack_deploy writes the stack, fetches the .env once, ups it.""" +import dataclasses +import json import os import stat import sys @@ -50,17 +52,26 @@ def _params(self, **extra): def test_deploy_writes_files_and_fetches_env_once(self): with patch.object(client, "fetch_stack_env", return_value="API_KEY=hunter2\n") as fetch, \ - patch.object(stacks, "_check_resolved"): # tests/test_compose_check.py covers it - out = stacks._stack_deploy(self._params(), _CFG) - fetch.assert_called_once_with(_CFG, TICKET) + patch.object(stacks, "_check_resolved", return_value={"services": {}}): # test_compose_check.py + cfg = dataclasses.replace(_CFG, data_dir=self.root / "agent-data") + out = stacks._stack_deploy(self._params(), cfg) + fetch.assert_called_once_with(cfg, TICKET) workdir = self.root / "media" self.assertEqual((workdir / "compose.yaml").read_text(), COMPOSE) env = workdir / ".env" self.assertEqual(env.read_text(), "API_KEY=hunter2\n") self.assertEqual(stat.S_IMODE(os.stat(env).st_mode), 0o600) - self.assertEqual(self.calls[0], ["docker", "compose", "-p", "media", "--project-directory", - str(workdir), "-f", str(workdir / "compose.yaml"), - "up", "-d", "--remove-orphans"]) + up = self.calls[0] + self.assertEqual(up[:5], ["docker", "compose", "-p", "media", "--project-directory"]) + self.assertEqual(up[5], str(workdir)) + self.assertEqual(up[-3:], ["up", "-d", "--remove-orphans"]) + # up reads the checked configuration, not compose.yaml again: kept in the agent's + # data dir (outside the stack folder, owner-only) for the containers' labels to name + checked = self.root / "agent-data" / "stacks" / "media.json" + self.assertEqual(up[6:8], ["-f", str(checked)]) + self.assertEqual(json.loads(checked.read_text()), {"services": {}}) + self.assertEqual(stat.S_IMODE(os.stat(checked).st_mode), 0o600) + self.assertEqual(stat.S_IMODE(os.stat(checked.parent).st_mode), 0o700) self.assertEqual(out.data, {"project": "media", "revision": 3}) self.assertNotIn("hunter2", str(out)) @@ -82,8 +93,12 @@ def test_remove_downs_and_only_deletes_vigils_own_folder(self): workdir = self.root / "media" workdir.mkdir(parents=True) (workdir / "compose.yaml").write_text(COMPOSE) - out = stacks._stack_remove({"project": "media", "working_dir": str(workdir), - "delete_files": True}, _CFG) + checked = self.root / "agent-data" / "stacks" / "media.json" + checked.parent.mkdir(parents=True) + checked.write_text("{}") + out = stacks._stack_remove({"project": "media", "working_dir": str(workdir), "delete_files": True}, + dataclasses.replace(_CFG, data_dir=self.root / "agent-data")) + self.assertFalse(checked.exists(), "the checked configuration goes with the stack") self.assertEqual(self.calls[0][-1], "down") self.assertEqual(out.data, {"project": "media", "files_deleted": True}) self.assertFalse(workdir.exists()) diff --git a/agent/tests/test_stack_read.py b/agent/tests/test_stack_read.py index 5debc550..b2eb777f 100644 --- a/agent/tests/test_stack_read.py +++ b/agent/tests/test_stack_read.py @@ -1,4 +1,5 @@ """M11 12: stack_read — adopt a stack in place, secrets off the task result.""" +import dataclasses import sys import tempfile import unittest @@ -82,10 +83,11 @@ def test_refusals(self): def test_deploy_writes_an_adopted_stacks_own_file(self): with patch.object(client, "fetch_stack_env", return_value=""), \ patch.object(executor.collector, "request_docker_recheck"), \ - patch.object(stacks, "_check_resolved"): # tests/test_compose_check.py covers it + patch.object(stacks, "_check_resolved", return_value={"services": {}}) as check: stacks._stack_deploy({"project": "shop", "compose": "services:\n web:\n image: nginx\n", - "working_dir": str(self.dir), "compose_file": "docker-compose.yml"}, _CFG) - self.assertIn(str(self.dir / "docker-compose.yml"), self.run_mock.call_args.args[0]) + "working_dir": str(self.dir), "compose_file": "docker-compose.yml"}, + dataclasses.replace(_CFG, data_dir=self.dir / "agent-data")) + self.assertEqual(check.call_args.args[2], "docker-compose.yml") with self.assertRaises(ValueError): stacks._stack_deploy({"project": "shop", "compose": "services: {}", "working_dir": str(self.dir), "compose_file": "../etc/x.yml"}, _CFG) diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 77c974bd..2eb46300 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -85,10 +85,11 @@ def _resolved_config(workdir: Path, project: str, compose_file: str) -> dict: return doc -def _check_resolved(workdir: Path, project: str, compose_file: str) -> None: +def _check_resolved(workdir: Path, project: str, compose_file: str) -> dict: """Refuse to start anything the resolved configuration would let reach the host (SEC, 2026-10-08). The server checks the text it was sent; this checks - what compose will actually run, so no reading of the text can differ.""" + what compose resolved it to, and _up starts exactly that, so no second + reading of the text (or of a .env changed in between) can differ.""" from ..composecheck import problems try: resolved = _resolved_config(workdir, project, compose_file) @@ -97,6 +98,24 @@ def _check_resolved(workdir: Path, project: str, compose_file: str) -> None: found = problems(resolved, str(workdir)) if found: raise ValueError("refusing to deploy " + project + ": " + "; ".join(found[:10])) + return resolved + + +def _up(workdir: Path, project: str, resolved: dict, data_dir: Path) -> str: + """``up`` on the checked configuration, kept in the agent's own data dir: + outside the stack folder (a container bound there could swap a file) and a + path no stack may bind. It stays, because compose records it in the + containers' config_files label and stack_update re-runs ``up`` from it.""" + import json + + resolved_dir = Path(data_dir) / "stacks" + resolved_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(resolved_dir, 0o700) + path = resolved_dir / f"{project}.json" + _write_private(path, json.dumps(resolved)) + return ex._run([*ex._compose_cmd(), "-p", project, "--project-directory", str(workdir), + "-f", str(path), "up", "-d", "--remove-orphans"], + timeout=600, extra_env=ex._compose_env()) def _stack_deploy(params: dict, config: AgentConfig) -> str: @@ -119,8 +138,8 @@ def _stack_deploy(params: dict, config: AgentConfig) -> str: (workdir / compose_file).write_text(compose) if env_text is not None: _write_private(workdir / ".env", env_text) - _check_resolved(workdir, project, compose_file) - output = _compose(workdir, project, "up", "-d", "--remove-orphans", compose_file=compose_file) + resolved = _check_resolved(workdir, project, compose_file) + output = _up(workdir, project, resolved, config.data_dir) collector.request_docker_recheck() revision = params.get("revision") return ActionOutput(output or f"deployed {project}", @@ -128,7 +147,7 @@ def _stack_deploy(params: dict, config: AgentConfig) -> str: "revision": revision if isinstance(revision, int) else 0}) -def _stack_remove(params: dict, _config: AgentConfig) -> str: +def _stack_remove(params: dict, config: AgentConfig) -> str: project = _project(params) workdir = _workdir(params) delete = params.get("delete_files") @@ -139,6 +158,8 @@ def _stack_remove(params: dict, _config: AgentConfig) -> str: raise ValueError(f"refusing to delete {workdir}: only {STACKS_ROOT}/ is Vigil's") output = _compose(workdir, project, "down", timeout=300) if (workdir / COMPOSE_NAME).exists() \ else f"{project}: no compose file at {workdir} — nothing to take down" + # the configuration _up checked and started, kept for the containers' labels + (Path(config.data_dir) / "stacks" / f"{project}.json").unlink(missing_ok=True) deleted = False if delete: if workdir.exists(): diff --git a/agent/vigil_agent/composecheck.py b/agent/vigil_agent/composecheck.py index eddf2257..297163ca 100644 --- a/agent/vigil_agent/composecheck.py +++ b/agent/vigil_agent/composecheck.py @@ -4,11 +4,18 @@ but compose re-reads that text: it substitutes variables from the .env and the environment, merges include/extends, normalises paths and booleans. Any difference between the two readings is a way past the server's check. So the -agent asks compose for its resolved configuration (``config --format json``) -and checks that, the same rules applied to exactly what ``up`` will start. +agent asks compose for its resolved configuration (``config --format json``), +checks that, and then starts *that file* — the checked bytes are the ones +``up`` reads (stacks._stack_deploy). + +The rules accept every shape compose allows, not just the one docker prints: +podman-compose's ``config`` hands back the file much as it was written (short +volume strings, list sysctls, "true" strings). A shape not understood here is +refused, and so is any service or top-level key not on the allowlist. """ from __future__ import annotations +import os import posixpath import re @@ -19,80 +26,228 @@ "DAC_READ_SEARCH", "BPF", "PERFMON", "MAC_ADMIN", "MAC_OVERRIDE"} DEVICE_RE = re.compile(r"/dev/dri(/(card|renderD)\d+)?|/dev/nvidia(\d+|ctl|-uvm|-uvm-tools|-modeset)" r"|/dev/net/tun|/dev/fuse|/dev/kfd") -REFUSED_SERVICE_KEYS = ("use_api_socket", "device_cgroup_rules", "post_start", "pre_stop", "develop", - "runtime", "annotations", "cgroup_parent") + +#: The same allowlist as the server's validation.py, less what it refuses. +#: A key compose adds later is refused here until someone has looked at it. +SERVICE_KEYS = { + "image", "command", "entrypoint", "environment", "ports", "expose", "restart", + "depends_on", "networks", "healthcheck", "labels", "logging", "container_name", "hostname", + "domainname", "user", "working_dir", "stop_signal", "stop_grace_period", "deploy", "cap_drop", + "dns", "dns_search", "dns_opt", "extra_hosts", "tmpfs", "ulimits", "shm_size", "mem_limit", + "memswap_limit", "mem_reservation", "cpus", "cpu_shares", "cpuset", "cpu_count", "cpu_percent", + "pids_limit", "read_only", "init", "tty", "stdin_open", "platform", "pull_policy", "profiles", + "group_add", "oom_score_adj", "oom_kill_disable", "mac_address", "links", "attach", "scale", "gpus", + # checked below + "build", "env_file", "volumes", "volumes_from", "devices", "cap_add", "security_opt", + "privileged", "pid", "ipc", "userns_mode", "cgroup", "uts", "network_mode", "sysctls", + "secrets", "configs", +} +TOP_LEVEL_KEYS = {"version", "name", "services", "networks", "volumes", "secrets", "configs"} +_TRUTHY = {"true", "yes", "y", "on", "1"} +_REMOTE_CONTEXT = ("http://", "https://", "git://", "git@", "ssh://") def _norm(path: str) -> str: return "/" + posixpath.normpath(str(path)).lstrip("/") -def bind_forbidden(source: str) -> bool: - """Is, is inside, or contains a protected path.""" - path = _norm(source) +def _forbidden(path: str) -> bool: + path = _norm(path) if path == "/": return True return any(path == p or path.startswith(p + "/") or p.startswith(path + "/") for p in FORBIDDEN_BINDS) +def bind_forbidden(source: str) -> bool: + """Is, is inside, or contains a protected path — as written, and once + symlinks on the host are followed (a container bound to its own stack + folder can plant ``data -> /`` there before the next deploy).""" + return _forbidden(source) or _forbidden(os.path.realpath(_norm(source))) + + def _inside(path: str, root: str) -> bool: - path, root = _norm(path), _norm(root) - return path == root or path.startswith(root + "/") + root = _norm(root) + return all(p == root or p.startswith(root + "/") + for p in (_norm(path), os.path.realpath(_norm(path)))) + + +def _flag(value) -> bool: + return value is True or str(value).strip().lower() in _TRUTHY + + +def _items(value) -> list: + """A list as given, a lone string as a one-item list, nothing as [].""" + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +def _has_var(value) -> bool: + # In docker's output a literal $ is escaped as $$; podman-compose may hand + # back an unsubstituted one. Either way the file is read again by `up`, so + # a $ in a field that decides access could still change after this check. + return "$" in str(value) + + +def _abs(path: str, workdir: str) -> str: + path = str(path) + # "~" is the agent's home — never inside a stack, so leave it to be refused + return path if path.startswith(("/", "~")) else posixpath.join(workdir, path) + + +def _bind_problem(source: str, workdir: str) -> str | None: + if source.startswith("~"): + return f"bind of a home directory ({source})" + path = _abs(source, workdir) + if _inside(path, workdir): # the stack's own folder, symlinks followed + return None + return f"bind of {source}" if bind_forbidden(path) else None + + +def _volume_problem(vol, workdir: str) -> str | None: + if isinstance(vol, str): + parts = vol.split(":") + if len(parts) == 1: + return None # anonymous volume: "/data" + source = parts[0] + if _has_var(source): + return f"volume source {source} has a variable" + if source.startswith(("/", ".", "~")): + return _bind_problem(source, workdir) + return None # a named volume + if not isinstance(vol, dict): + return f"volume {vol!r} is not understood" + kind = vol.get("type") or "volume" + source = str(vol.get("source") or "") + if _has_var(source): + return f"volume source {source} has a variable" + if kind == "bind" or source.startswith(("/", ".", "~")): + if not source: + return "bind without a source" + return _bind_problem(source, workdir) + if kind not in ("volume", "tmpfs", "image"): + return f"volume type {kind}" + return None + + +def _service(name: str, svc, names: set, workdir: str) -> list[str]: + where = f"service {name!r}" + if not isinstance(svc, dict): + return [f"{where} is not a mapping"] + out: list[str] = [] + for key, value in svc.items(): + if key not in SERVICE_KEYS and not str(key).startswith("x-") and value not in (None, "", [], {}): + out.append(f"{where}: {key} is not allowed") + if "privileged" in svc and (_has_var(svc["privileged"]) or _flag(svc["privileged"])): + out.append(f"{where}: privileged") + for key in ("pid", "ipc", "userns_mode", "cgroup", "uts", "network_mode"): + value = str(svc.get(key) or "").strip().lower() + if _has_var(value) or value.startswith("container:") or (value == "host" and key != "network_mode"): + out.append(f"{where}: {key}: {value}") + if key == "network_mode" and value.startswith("service:") and value.split(":", 1)[1] not in names: + out.append(f"{where}: network_mode {value}") + for cap in _items(svc.get("cap_add")): + if _has_var(cap) or str(cap).strip().upper().removeprefix("CAP_") in FORBIDDEN_CAPS: + out.append(f"{where}: cap_add {cap}") + for opt in _items(svc.get("security_opt")): + low = str(opt).strip().lower().replace("=", ":") + allowed = (low in ("no-new-privileges", "no-new-privileges:true") + or (low.startswith("apparmor:") and "unconfined" not in low) + or low.startswith("label:level:")) + if _has_var(opt) or not allowed: + out.append(f"{where}: security_opt {opt}") + for device in _items(svc.get("devices")): + source = str(device.get("source") or "") if isinstance(device, dict) else str(device).split(":", 1)[0] + if _has_var(source) or ".." in source.split("/") or not DEVICE_RE.fullmatch(_norm(source)) \ + or not DEVICE_RE.fullmatch(os.path.realpath(_norm(source))): + out.append(f"{where}: device {source}") + sysctls = svc.get("sysctls") or {} + keys = sysctls.keys() if isinstance(sysctls, dict) else [str(s).split("=", 1)[0] for s in _items(sysctls)] + for key in keys: + if _has_var(key) or not str(key).strip().startswith("net."): + out.append(f"{where}: sysctl {key}") + for ref in _items(svc.get("volumes_from")): + target = str(ref).removeprefix("service:").split(":", 1)[0] + if str(ref).startswith("container:") or target not in names: + out.append(f"{where}: volumes_from {ref}") + for vol in _items(svc.get("volumes")): + problem = _volume_problem(vol, workdir) + if problem: + out.append(f"{where}: {problem}") + for env_file in _items(svc.get("env_file")): + path = env_file.get("path") if isinstance(env_file, dict) else env_file + if path and not _inside(_abs(path, workdir), workdir): + out.append(f"{where}: env_file {path} is outside the stack") + out += _build(where, svc.get("build"), workdir) + return out + + +def _local_context(value) -> bool: + """A path on this host, not a remote repository, an image or another service.""" + return not str(value).startswith((*_REMOTE_CONTEXT, "docker-image://", "service:")) + + +def _build(where: str, build, workdir: str) -> list[str]: + if build is None: + return [] + if isinstance(build, str): + build = {"context": build} + if not isinstance(build, dict): + return [f"{where}: build is not understood"] + out = [] + context = build.get("context") or "." + extra = build.get("additional_contexts") or {} + if isinstance(extra, list): # ["name=path", ...] + extra = dict(str(e).split("=", 1) if "=" in str(e) else (str(e), "/") for e in extra) + if not isinstance(extra, dict): + return [f"{where}: build.additional_contexts is not understood"] + paths = [("context", context), *((f"additional_contexts.{k}", v) for k, v in extra.items())] + for label, value in paths: + if _has_var(value): + out.append(f"{where}: build.{label} has a variable") + elif _local_context(value) and not _inside(_abs(value, workdir), workdir): + out.append(f"{where}: build.{label} {value} is outside the stack") + dockerfile = build.get("dockerfile") + if dockerfile and _local_context(context): + base = _abs(context, workdir) + if _has_var(dockerfile) or not _inside(_abs(dockerfile, base), workdir): + out.append(f"{where}: build.dockerfile {dockerfile} is outside the stack") + for key in ("ssh", "entitlements", "privileged"): + if build.get(key): + out.append(f"{where}: build.{key}") + if str(build.get("network") or "") not in ("", "default", "none"): + out.append(f"{where}: build.network {build['network']}") + return out def problems(config: dict, workdir: str) -> list[str]: """Every reason not to start this resolved configuration ([] = fine).""" + if not isinstance(config, dict): + return ["the configuration is not a mapping"] out: list[str] = [] + for key, value in config.items(): + if key not in TOP_LEVEL_KEYS and not str(key).startswith("x-") and value not in (None, "", [], {}): + out.append(f"top-level {key} is not allowed") services = config.get("services") or {} + if not isinstance(services, dict): + return out + ["services is not a mapping"] names = set(services) for name, svc in services.items(): - where = f"service {name!r}" - for key in REFUSED_SERVICE_KEYS: - if svc.get(key): - out.append(f"{where}: {key} is not allowed") - if svc.get("privileged") is True: - out.append(f"{where}: privileged") - for key in ("pid", "ipc", "userns_mode", "cgroup", "uts"): - value = str(svc.get(key) or "").strip().lower() - if value == "host" or value.startswith("container:"): - out.append(f"{where}: {key}: {value}") - for cap in svc.get("cap_add") or []: - if str(cap).upper().removeprefix("CAP_") in FORBIDDEN_CAPS: - out.append(f"{where}: cap_add {cap}") - for opt in svc.get("security_opt") or []: - low = str(opt).lower().replace("=", ":") - if "unconfined" in low or low in ("label:disable", "no-new-privileges:false"): - out.append(f"{where}: security_opt {opt}") - for device in svc.get("devices") or []: - source = device.get("source", "") if isinstance(device, dict) else str(device).split(":", 1)[0] - if not DEVICE_RE.fullmatch(_norm(source)) or ".." in str(source).split("/"): - out.append(f"{where}: device {source}") - for key in (svc.get("sysctls") or {}): - if not str(key).startswith("net."): - out.append(f"{where}: sysctl {key}") - for ref in svc.get("volumes_from") or []: - if str(ref).startswith("container:") or str(ref).split(":", 1)[0] not in names: - out.append(f"{where}: volumes_from {ref}") - for vol in svc.get("volumes") or []: - if isinstance(vol, dict) and vol.get("type") == "bind" and bind_forbidden(vol.get("source", "")): - out.append(f"{where}: bind of {vol.get('source')}") - build = svc.get("build") or {} - if isinstance(build, dict): - if build.get("context") and not str(build["context"]).startswith(("http", "git@")) \ - and not _inside(build["context"], workdir): - out.append(f"{where}: build context {build['context']} is outside the stack") - for key in ("ssh", "entitlements", "privileged"): - if build.get(key): - out.append(f"{where}: build.{key}") - if str(build.get("network") or "") not in ("", "default", "none"): - out.append(f"{where}: build.network {build['network']}") + out += _service(name, svc, names, workdir) for vname, vol in (config.get("volumes") or {}).items(): - device = str(((vol or {}).get("driver_opts") or {}).get("device") or "") - if device.startswith("/") or ".." in device.split("/"): - out.append(f"volume {vname!r}: binds the host path {device}") + opts = (vol or {}).get("driver_opts") if isinstance(vol, dict) else None + if not isinstance(opts, dict): + continue + device = str(opts.get("device") or "") + kind = str(opts.get("type") or "").lower() + if any(_has_var(v) for v in opts.values()) or "bind" in str(opts.get("o") or "").lower() \ + or (device and kind not in ("tmpfs", "nfs", "nfs4", "cifs")): + out.append(f"volume {vname!r}: driver_opts {opts}") for section in ("secrets", "configs"): for item, spec in (config.get(section) or {}).items(): - path = (spec or {}).get("file") - if path and not _inside(path, workdir): + path = (spec or {}).get("file") if isinstance(spec, dict) else None + if path and (_has_var(path) or not _inside(_abs(path, workdir), workdir)): out.append(f"{section[:-1]} {item!r}: {path} is outside the stack") return out From 3385408622a6cb44ce68e65ae2c2da0b7b3255f6 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:55:08 -0400 Subject: [PATCH 42/68] fix(security): pin compose paths to the stack folder, and refuse binds a container could re-link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps in ec0809f's deploy-what-you-checked: - Paths: `up` reads the checked file from /stacks, so a relative path left in it (podman-compose's config does not absolutise) would resolve against the agent's data dir, while the check resolved it against the stack folder — "../" would bind the agent's own state. composecheck.pin makes every host path absolute against the stack folder; the pinned copy is what is checked and what is started. - Symlinks: following links at check time cannot stop a container that can write a folder from swapping one in before the engine mounts. A bind is refused when a component under the stack folder is already a symlink, or when it lies inside a folder that this config or a running Vigil-stack container mounts writable. - Variables: only some fields refused a $. Now every string does, except in fields that only reach inside the container (environment, command, entrypoint, healthcheck, labels, content), where docker's $$ re-reads as a literal $. Checked on the Ubuntu test VM against docker's real resolved output. --- agent/tests/test_compose_check.py | 56 ++++++++++- agent/vigil_agent/actions/stacks.py | 25 ++++- agent/vigil_agent/composecheck.py | 140 +++++++++++++++++++++++++++- 3 files changed, 211 insertions(+), 10 deletions(-) diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py index 266e7ccc..29a964c5 100644 --- a/agent/tests/test_compose_check.py +++ b/agent/tests/test_compose_check.py @@ -8,6 +8,7 @@ import sys import tempfile import unittest +import unittest.mock from pathlib import Path from unittest.mock import patch @@ -102,6 +103,42 @@ def test_ordinary_stacks_still_pass(self): self.assertEqual(composecheck.problems(_cfg(volumes=["./data:/d", "named:/n", "/anon"], network_mode="host", sysctls=["net.ipv4.ip_forward=1"]), WD), []) + def test_relative_paths_are_pinned_to_the_stack_folder(self): + """`up` reads the checked file from the agent's data dir; a relative + path left in it would resolve there (../ = the agent's own state).""" + cfg = composecheck.pin(_cfg(volumes=["./data:/d", {"type": "bind", "source": "../x", "target": "/x"}], + env_file=".env", build="./src", + ), WD) + svc = cfg["services"]["a"] + self.assertEqual(svc["volumes"][0], WD + "/data:/d") + self.assertEqual(svc["volumes"][1]["source"], "/opt/vigil/stacks/x") + self.assertEqual(svc["env_file"], [WD + "/.env"]) + self.assertEqual(svc["build"]["context"], WD + "/src") + + def test_a_variable_anywhere_that_matters_is_refused(self): + self.assertTrue(composecheck.problems(_cfg(tmpfs=["/run/${X}"]), WD)) + self.assertTrue(composecheck.problems(_cfg(networks={"${NET}": None}), WD)) + self.assertTrue(composecheck.problems(_cfg(ulimits={"nofile": "$N"}), WD)) + # where it can only reach inside the container, docker's $$ stays fine + self.assertEqual(composecheck.problems(_cfg(environment={"A": "x$$y"}, command=["sh", "-c", "echo $$HOME"], + labels={"l": "$$"}), WD), []) + + def test_a_bind_inside_a_writable_bind_is_refused(self): + """A container that can write a folder can swap a symlink into it + between the check and the mount.""" + self.assertTrue(composecheck.problems(_cfg(volumes=["./:/stack", "./data:/d"]), WD)) + self.assertTrue(composecheck.problems(_cfg(volumes=["./data:/d"]), WD, live_rw=(WD,))) + self.assertTrue(composecheck.problems(_cfg(volumes=["/srv/a/b:/d"]), WD, live_rw=("/srv/a",))) + # read-only, or the same folder, cannot plant anything + self.assertEqual(composecheck.problems(_cfg(volumes=["./:/stack:ro", "./data:/d"]), WD), []) + self.assertEqual(composecheck.problems(_cfg(volumes=["./data:/d"]), WD, live_rw=(WD + "/data",)), []) + + def test_a_symlink_already_in_the_stack_folder_is_refused(self): + with tempfile.TemporaryDirectory() as tmp: + os.mkdir(os.path.join(tmp, "real")) + os.symlink(os.path.join(tmp, "real"), os.path.join(tmp, "data")) + self.assertTrue(composecheck.problems(_cfg(volumes=["./data/sub:/d"]), tmp)) + def test_a_symlink_planted_in_the_stack_folder_is_followed(self): with tempfile.TemporaryDirectory() as tmp: os.symlink("/etc", os.path.join(tmp, "data")) @@ -119,15 +156,30 @@ def fake_compose(workdir, project, *args, **kw): return resolved if args[:1] == ("config",) else "up ok" with tempfile.TemporaryDirectory() as tmp, \ patch.object(stacks, "_compose", side_effect=fake_compose), \ + patch.object(stacks, "_live_rw_binds", return_value=()), \ patch.object(stacks.client if hasattr(stacks, "client") else stacks, "fetch_stack_env", return_value="", create=True): with self.assertRaises(ValueError): - stacks._check_resolved(Path(tmp), "web", "compose.yaml") + stacks._check_resolved(Path(tmp), "web", "compose.yaml", Path(tmp)) self.assertFalse([c for c in calls if c[:1] == ("up",)]) + def test_live_writable_binds_come_from_vigil_stacks_only(self): + engine = unittest.mock.Mock() + engine.get.return_value = [ + {"Labels": {"com.docker.compose.project.config_files": "/data/stacks/web.json"}, + "Mounts": [{"Type": "bind", "Source": "/srv/web", "RW": True}, + {"Type": "bind", "Source": "/srv/ro", "RW": False}, + {"Type": "volume", "Source": "/var/lib/docker/volumes/x", "RW": True}]}, + {"Labels": {"com.docker.compose.project.config_files": "/home/admin/compose.yml"}, + "Mounts": [{"Type": "bind", "Source": "/home", "RW": True}]}, + {"Labels": {}, "Mounts": [{"Type": "bind", "Source": "/", "RW": True}]}, + ] + with patch.object(stacks.ex, "_engine", return_value=engine): + self.assertEqual(stacks._live_rw_binds(Path("/data")), ("/srv/web",)) + def test_unreadable_config_refuses(self): with patch.object(stacks, "_compose", side_effect=RuntimeError("compose missing")): with self.assertRaises(ValueError): - stacks._check_resolved(Path("/tmp"), "web", "compose.yaml") + stacks._check_resolved(Path("/tmp"), "web", "compose.yaml", Path("/tmp")) if __name__ == "__main__": diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 2eb46300..eb72bbab 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -85,17 +85,32 @@ def _resolved_config(workdir: Path, project: str, compose_file: str) -> dict: return doc -def _check_resolved(workdir: Path, project: str, compose_file: str) -> dict: +def _live_rw_binds(data_dir: Path) -> tuple: + """Host paths that running containers of Vigil-deployed stacks (those + started from a checked file in /stacks) mount writable.""" + checked = str(Path(data_dir) / "stacks") + "/" + out = [] + for c in ex._engine().get("/containers/json", query={"all": "1"}) or []: + files = str((c.get("Labels") or {}).get("com.docker.compose.project.config_files") or "") + if not any(f.strip().startswith(checked) for f in files.split(",")): + continue + out += [m.get("Source") for m in c.get("Mounts") or [] + if m.get("Type") == "bind" and m.get("RW") and m.get("Source")] + return tuple(out) + + +def _check_resolved(workdir: Path, project: str, compose_file: str, data_dir: Path) -> dict: """Refuse to start anything the resolved configuration would let reach the host (SEC, 2026-10-08). The server checks the text it was sent; this checks what compose resolved it to, and _up starts exactly that, so no second reading of the text (or of a .env changed in between) can differ.""" - from ..composecheck import problems + from ..composecheck import pin, problems try: - resolved = _resolved_config(workdir, project, compose_file) + resolved = pin(_resolved_config(workdir, project, compose_file), str(workdir)) + live_rw = _live_rw_binds(data_dir) except Exception as exc: # noqa: BLE001 — cannot check it, so do not run it raise ValueError(f"could not read the resolved compose configuration: {exc}") from exc - found = problems(resolved, str(workdir)) + found = problems(resolved, str(workdir), live_rw) if found: raise ValueError("refusing to deploy " + project + ": " + "; ".join(found[:10])) return resolved @@ -138,7 +153,7 @@ def _stack_deploy(params: dict, config: AgentConfig) -> str: (workdir / compose_file).write_text(compose) if env_text is not None: _write_private(workdir / ".env", env_text) - resolved = _check_resolved(workdir, project, compose_file) + resolved = _check_resolved(workdir, project, compose_file, config.data_dir) output = _up(workdir, project, resolved, config.data_dir) collector.request_docker_recheck() revision = params.get("revision") diff --git a/agent/vigil_agent/composecheck.py b/agent/vigil_agent/composecheck.py index 297163ca..0c32457b 100644 --- a/agent/vigil_agent/composecheck.py +++ b/agent/vigil_agent/composecheck.py @@ -94,18 +94,37 @@ def _has_var(value) -> bool: def _abs(path: str, workdir: str) -> str: path = str(path) # "~" is the agent's home — never inside a stack, so leave it to be refused - return path if path.startswith(("/", "~")) else posixpath.join(workdir, path) + return path if path.startswith(("/", "~")) else posixpath.normpath(posixpath.join(workdir, path)) def _bind_problem(source: str, workdir: str) -> str | None: if source.startswith("~"): return f"bind of a home directory ({source})" path = _abs(source, workdir) + link = _symlink_in(path, workdir) + if link: + return f"bind of {source}: {link} is a symlink" if _inside(path, workdir): # the stack's own folder, symlinks followed return None return f"bind of {source}" if bind_forbidden(path) else None +def _symlink_in(path: str, workdir: str) -> str | None: + """The first symlink on the way from the stack folder down to *path*. A + container that can write the stack folder can plant one; the engine follows + it when it mounts, after any check here — so a link already there is + refused, and _covered() refuses a bind a running container could re-link.""" + path, root = _norm(path), _norm(workdir) + if not path.startswith(root + "/"): + return None + current = root + for part in path[len(root) + 1:].split("/"): + current = posixpath.join(current, part) + if os.path.islink(current): + return current + return None + + def _volume_problem(vol, workdir: str) -> str | None: if isinstance(vol, str): parts = vol.split(":") @@ -222,11 +241,126 @@ def _build(where: str, build, workdir: str) -> list[str]: return out -def problems(config: dict, workdir: str) -> list[str]: - """Every reason not to start this resolved configuration ([] = fine).""" +#: Fields whose strings only reach inside the container: a $ there may stay +#: ($$ in docker's output re-reads as a literal $). Anywhere else a $ is +#: refused — compose reads the checked file again, and could substitute it. +INERT_KEYS = {"environment", "command", "entrypoint", "healthcheck", "labels", "content"} + + +def _variables(node, where: str, out: list[str], depth: int = 0) -> None: + if depth > 64: + out.append(f"{where}: nested too deeply") + elif isinstance(node, dict): + for key, value in node.items(): + if _has_var(key): + out.append(f"{where}: key {key!r} has a variable") + elif key not in INERT_KEYS and not str(key).startswith("x-"): + _variables(value, f"{where}.{key}", out, depth + 1) + elif isinstance(node, list): + for value in node: + _variables(value, where, out, depth + 1) + elif _has_var(node): + out.append(f"{where}: {node!r} has a variable") + + +def _pin_volume(vol, workdir: str): + if isinstance(vol, str): + parts = vol.split(":") + if len(parts) > 1 and parts[0].startswith("."): + return ":".join([_abs(parts[0], workdir), *parts[1:]]) + return vol + if isinstance(vol, dict) and str(vol.get("source") or "").startswith("."): + return {**vol, "source": _abs(vol["source"], workdir)} + return vol + + +def pin(config: dict, workdir: str) -> dict: + """The configuration with every relative host path made absolute against + the stack folder. docker's output already is; podman-compose's may not be, + and ``up`` reads the checked file from the agent's data dir, where compose + would resolve "./" and "../" against *that* folder instead. What is + checked and what is started is this pinned copy.""" + import copy + + config = copy.deepcopy(config) + if not isinstance(config, dict): + return config + for svc in (config.get("services") or {}).values() if isinstance(config.get("services"), dict) else (): + if not isinstance(svc, dict): + continue + if isinstance(svc.get("volumes"), list): + svc["volumes"] = [_pin_volume(v, workdir) for v in svc["volumes"]] + if "env_file" in svc: + svc["env_file"] = [{**e, "path": _abs(e["path"], workdir)} if isinstance(e, dict) and e.get("path") + else _abs(e, workdir) if isinstance(e, str) else e + for e in _items(svc["env_file"])] + build = svc.get("build") + if isinstance(build, str): + build = svc["build"] = {"context": build} + if isinstance(build, dict): + context = build.get("context") or "." + if _local_context(context): + build["context"] = _abs(context, workdir) + extra = build.get("additional_contexts") + if isinstance(extra, dict): + build["additional_contexts"] = {k: _abs(v, workdir) if _local_context(v) else v + for k, v in extra.items()} + for section in ("secrets", "configs"): + for spec in (config.get(section) or {}).values() if isinstance(config.get(section), dict) else (): + if isinstance(spec, dict) and spec.get("file"): + spec["file"] = _abs(spec["file"], workdir) + return config + + +def rw_binds(config: dict, workdir: str) -> list[str]: + """Host paths this configuration mounts writable.""" + out = [] + for svc in (config.get("services") or {}).values(): + for vol in _items((svc or {}).get("volumes")) if isinstance(svc, dict) else (): + if isinstance(vol, str): + parts = vol.split(":") # source:target[:options] + options = parts[2].split(",") if len(parts) > 2 else [] + if len(parts) > 1 and parts[0].startswith(("/", ".")) and "ro" not in options: + out.append(_norm(_abs(parts[0], workdir))) + elif isinstance(vol, dict) and str(vol.get("source") or "").startswith(("/", ".")) \ + and not _flag(vol.get("read_only")): + out.append(_norm(_abs(vol["source"], workdir))) + return out + + +def _sources(config: dict, workdir: str) -> list[str]: + out = [] + for svc in (config.get("services") or {}).values(): + for vol in _items((svc or {}).get("volumes")) if isinstance(svc, dict) else (): + source = vol.split(":")[0] if isinstance(vol, str) and ":" in vol \ + else str(vol.get("source") or "") if isinstance(vol, dict) else "" + if source.startswith(("/", ".")): + out.append(_norm(_abs(source, workdir))) + return out + + +def _covered(config: dict, workdir: str, live_rw: tuple) -> list[str]: + """A bind inside a folder some Vigil stack's container can write: that + container can swap in a symlink between this check and the mount.""" + writable = set(rw_binds(config, workdir)) | {_norm(p) for p in live_rw} + out = [] + for source in sorted(set(_sources(config, workdir))): + for real in {source, os.path.realpath(source)}: + above = [w for w in writable if real.startswith(w.rstrip("/") + "/")] + if above: + out.append(f"bind of {source} is inside {above[0]}, which a container can write") + break + return out + + +def problems(config: dict, workdir: str, live_rw: tuple = ()) -> list[str]: + """Every reason not to start this (pinned) resolved configuration ([] = + fine). *live_rw*: host paths running Vigil-stack containers mount writable.""" if not isinstance(config, dict): return ["the configuration is not a mapping"] out: list[str] = [] + _variables(config, "config", out) + out += _covered(config, workdir, live_rw) for key, value in config.items(): if key not in TOP_LEVEL_KEYS and not str(key).startswith("x-") and value not in (None, "", [], {}): out.append(f"top-level {key} is not allowed") From 2034599406f15051c2d292fa7f953932c344c5fc Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 22:59:47 -0400 Subject: [PATCH 43/68] fix(security): no stack path a non-root user could re-point, no writes through planted links, no x-podman MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Binds: every folder above a bind source (as written and once links are followed) must be root-owned and not group/other-writable. Whoever can write one can swap a path component for a symlink after the check and have the engine mount, say, /etc into a root container — the "owns the machine but is not admin" case. Stack data belongs in the stack folder or a root-owned path such as /srv; a bind under /home/ is refused. - The stack folder: the agent writes compose.yaml and .env there as root. It is refused unless root alone can change the folders above it; the files are replaced (unlink + O_EXCL|O_NOFOLLOW), never written through a link; and no container may mount the stack folder, or a folder holding it, writable — in the new config or already running. - x-podman: an inert extension field to docker, but podman-compose acts on it (podman_args, pod_args, uidmaps). Refused at any depth by the server and by the agent. The agent's variable walk visits shared YAML nodes once. Checked on the Ubuntu test VM: /srv and not-yet-created /opt/vigil paths pass, /home/vboxuser and /tmp are refused. --- agent/tests/test_compose_check.py | 46 +++++++++++++++++++++++++++++ agent/tests/test_stack_deploy.py | 1 + agent/tests/test_stack_read.py | 1 + agent/vigil_agent/actions/stacks.py | 33 +++++++++++++++++---- agent/vigil_agent/composecheck.py | 45 +++++++++++++++++++++++++--- server/apps/stacks/tests.py | 14 +++++++++ server/apps/stacks/validation.py | 3 ++ 7 files changed, 134 insertions(+), 9 deletions(-) diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py index 29a964c5..6b8bc531 100644 --- a/agent/tests/test_compose_check.py +++ b/agent/tests/test_compose_check.py @@ -146,6 +146,52 @@ def test_a_symlink_planted_in_the_stack_folder_is_followed(self): self.assertTrue(composecheck.problems(_cfg(build={"context": tmp + "/data"}), tmp)) +class OwnershipTests(unittest.TestCase): + """A folder a non-root user can write lets that user swap a path + component for a symlink after the check — the engine, or the agent + writing as root, follows it.""" + + def test_a_bind_under_a_user_owned_folder_is_refused(self): + with tempfile.TemporaryDirectory() as tmp: # owned by the test user, under sticky /tmp + data = os.path.join(tmp, "data") + problems = composecheck.problems(_cfg(volumes=[f"{data}:/d"]), WD) + self.assertTrue(any("other than root" in p for p in problems), problems) + self.assertIsNone(composecheck.unsafe_dir("/srv/x/y")) # root-owned all the way (or not there yet) + + def test_the_stack_folder_itself_may_not_be_mounted_writable(self): + self.assertTrue(composecheck.problems(_cfg(volumes=["./:/stack"]), WD)) + self.assertTrue(composecheck.problems(_cfg(volumes=["/opt/vigil/stacks:/s"]), WD)) + self.assertTrue(composecheck.problems(_cfg(), WD, live_rw=(WD,))) + self.assertEqual(composecheck.problems(_cfg(volumes=["./:/stack:ro"]), WD), []) + + def test_x_podman_is_refused_anywhere(self): + for cfg in ({**_cfg(), "x-podman": {"in_pod": False}}, + _cfg(**{"x-podman": {"podman_args": ["--privileged"]}}), + _cfg(**{"X-Podman.uidmaps": ["0:0:1"]})): + with self.subTest(cfg=cfg): + self.assertTrue(composecheck.problems(cfg, WD)) + self.assertEqual(composecheck.problems(_cfg(**{"x-common": {"a": 1}}), WD), []) + + def test_agent_writes_never_follow_a_planted_link(self): + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "elsewhere" + target.write_text("untouched") + link = Path(tmp) / "compose.yaml" + link.symlink_to(target) + stacks._write_new(link, "services: {}", 0o644) + self.assertFalse(link.is_symlink()) + self.assertEqual(link.read_text(), "services: {}") + self.assertEqual(target.read_text(), "untouched") + + def test_deploy_refuses_a_user_owned_stack_folder(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError): + stacks._safe_workdir(Path(tmp) / "web") + os.symlink(tmp, Path(tmp) / "link") + with patch.object(composecheck, "unsafe_dir", return_value=None), self.assertRaises(ValueError): + stacks._safe_workdir(Path(tmp) / "link") + + class DeployRefusesTests(unittest.TestCase): def test_deploy_stops_before_up_when_the_resolved_config_escapes(self): resolved = json.dumps(_cfg(volumes=[{"type": "bind", "source": "/etc", "target": "/e"}])) diff --git a/agent/tests/test_stack_deploy.py b/agent/tests/test_stack_deploy.py index 726b9309..ad97138a 100644 --- a/agent/tests/test_stack_deploy.py +++ b/agent/tests/test_stack_deploy.py @@ -52,6 +52,7 @@ def _params(self, **extra): def test_deploy_writes_files_and_fetches_env_once(self): with patch.object(client, "fetch_stack_env", return_value="API_KEY=hunter2\n") as fetch, \ + patch.object(stacks, "_safe_workdir"), \ patch.object(stacks, "_check_resolved", return_value={"services": {}}): # test_compose_check.py cfg = dataclasses.replace(_CFG, data_dir=self.root / "agent-data") out = stacks._stack_deploy(self._params(), cfg) diff --git a/agent/tests/test_stack_read.py b/agent/tests/test_stack_read.py index b2eb777f..6f3781da 100644 --- a/agent/tests/test_stack_read.py +++ b/agent/tests/test_stack_read.py @@ -83,6 +83,7 @@ def test_refusals(self): def test_deploy_writes_an_adopted_stacks_own_file(self): with patch.object(client, "fetch_stack_env", return_value=""), \ patch.object(executor.collector, "request_docker_recheck"), \ + patch.object(stacks, "_safe_workdir"), \ patch.object(stacks, "_check_resolved", return_value={"services": {}}) as check: stacks._stack_deploy({"project": "shop", "compose": "services:\n web:\n image: nginx\n", "working_dir": str(self.dir), "compose_file": "docker-compose.yml"}, diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index eb72bbab..75183178 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -39,14 +39,36 @@ def _workdir(params: dict) -> Path: return path -def _write_private(path: Path, text: str) -> None: - """Create or replace *path* readable by its owner only.""" - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) +def _write_new(path: Path, text: str, mode: int) -> None: + """Replace *path* with a fresh file, never following a link: whatever is + there (a symlink a container planted, say) is removed, not written + through, and O_EXCL fails if one reappears in between.""" + try: + os.unlink(path) + except FileNotFoundError: + pass + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), mode) try: os.write(fd, text.encode()) + os.fchmod(fd, mode) finally: os.close(fd) - os.chmod(path, 0o600) + + +def _write_private(path: Path, text: str) -> None: + """Create or replace *path* readable by its owner only.""" + _write_new(path, text, 0o600) + + +def _safe_workdir(workdir: Path) -> None: + """The agent writes here as root, so no one but root may be able to + re-point it: every folder above it root-owned and not group/other-writable.""" + from ..composecheck import unsafe_dir + loose = unsafe_dir(str(workdir / "x")) + if loose: + raise ValueError(f"refusing to deploy into {workdir}: {loose} can be changed by a user other than root") + if workdir.is_symlink(): + raise ValueError(f"refusing to deploy into {workdir}: it is a symlink") _FILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}\.ya?ml$") @@ -149,8 +171,9 @@ def _stack_deploy(params: dict, config: AgentConfig) -> str: # half-configured containers. env_text = client.fetch_stack_env(config, ticket) if ticket else None + _safe_workdir(workdir) workdir.mkdir(parents=True, exist_ok=True) - (workdir / compose_file).write_text(compose) + _write_new(workdir / compose_file, compose, 0o644) if env_text is not None: _write_private(workdir / ".env", env_text) resolved = _check_resolved(workdir, project, compose_file, config.data_dir) diff --git a/agent/vigil_agent/composecheck.py b/agent/vigil_agent/composecheck.py index 0c32457b..e2bba2a8 100644 --- a/agent/vigil_agent/composecheck.py +++ b/agent/vigil_agent/composecheck.py @@ -104,11 +104,34 @@ def _bind_problem(source: str, workdir: str) -> str | None: link = _symlink_in(path, workdir) if link: return f"bind of {source}: {link} is a symlink" + loose = unsafe_dir(path) + if loose: + return (f"bind of {source}: {loose} can be changed by a user other than root " + f"(keep stack data in the stack folder or a root-owned path such as /srv)") if _inside(path, workdir): # the stack's own folder, symlinks followed return None return f"bind of {source}" if bind_forbidden(path) else None +def unsafe_dir(path: str) -> str | None: + """The first folder above *path* (as written, and once links are + followed) that someone other than root could change: not root-owned, or + writable by group/others (sticky /tmp too). Whoever can write it can swap + a component of the path for a symlink after any check here, and the + engine — or the agent writing as root — would follow it.""" + for candidate in {_norm(path), os.path.realpath(_norm(path))}: + current = "/" + for part in candidate.strip("/").split("/")[:-1]: + current = posixpath.join(current, part) + try: + st = os.stat(current) + except FileNotFoundError: + break # created later by root + if st.st_uid != 0 or st.st_mode & 0o022: + return current + return None + + def _symlink_in(path: str, workdir: str) -> str | None: """The first symlink on the way from the stack folder down to *path*. A container that can write the stack folder can plant one; the engine follows @@ -247,18 +270,27 @@ def _build(where: str, build, workdir: str) -> list[str]: INERT_KEYS = {"environment", "command", "entrypoint", "healthcheck", "labels", "content"} -def _variables(node, where: str, out: list[str], depth: int = 0) -> None: +def _variables(node, where: str, out: list[str], depth: int = 0, seen=None) -> None: + # podman-compose's YAML can share nodes through anchors: walk each once + seen = set() if seen is None else seen + if isinstance(node, (dict, list)): + if id(node) in seen: + return + seen.add(id(node)) if depth > 64: out.append(f"{where}: nested too deeply") elif isinstance(node, dict): for key, value in node.items(): - if _has_var(key): + if str(key).lower().startswith("x-podman"): + # inert to docker, but podman-compose acts on it (podman_args, pod_args, …) + out.append(f"{where}: {key} is not allowed") + elif _has_var(key): out.append(f"{where}: key {key!r} has a variable") elif key not in INERT_KEYS and not str(key).startswith("x-"): - _variables(value, f"{where}.{key}", out, depth + 1) + _variables(value, f"{where}.{key}", out, depth + 1, seen) elif isinstance(node, list): for value in node: - _variables(value, where, out, depth + 1) + _variables(value, where, out, depth + 1, seen) elif _has_var(node): out.append(f"{where}: {node!r} has a variable") @@ -344,6 +376,11 @@ def _covered(config: dict, workdir: str, live_rw: tuple) -> list[str]: container can swap in a symlink between this check and the mount.""" writable = set(rw_binds(config, workdir)) | {_norm(p) for p in live_rw} out = [] + root = _norm(workdir) + for w in sorted(writable): + if root == w or root.startswith(w.rstrip("/") + "/"): + # the agent writes compose.yaml and .env here as root + out.append(f"{w} is mounted writable and holds the stack folder; bind a subfolder instead") for source in sorted(set(_sources(config, workdir))): for real in {source, os.path.realpath(source)}: above = [w for w in writable if real.startswith(w.rstrip("/") + "/")] diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index d2a68c91..795fbd80 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -506,6 +506,20 @@ def test_data_paths_still_allowed(self): validate_compose(self.S + f" volumes: ['{path}:/data']\n") +class SandboxPodmanTests(SimpleTestCase): + """x-* is an inert extension field to docker, but podman-compose acts on + x-podman (podman_args, pod_args, uidmaps): refused at any depth.""" + + def test_x_podman_is_refused(self): + S = "services:\n a:\n image: x\n" + for text in (S + "x-podman:\n in_pod: false\n", + S + " x-podman:\n podman_args: [--privileged]\n", + S + " X-Podman.uidmaps: ['0:0:1']\n"): + with self.subTest(text=text), self.assertRaises(StackError): + validate_compose(text) + validate_compose("x-common: &c {restart: always}\n" + S) + + class SandboxDosTests(SimpleTestCase): def test_an_anchor_bomb_is_checked_quickly(self): import time diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index cd36098b..f1b643b8 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -150,6 +150,9 @@ def _no_variable_keys(node, where: str = "compose file", _seen=None, _depth: int for key, value in node.items(): if "$" in str(key): raise StackError(f"{where}: variables are not allowed in keys ({key})") + if str(key).lower().startswith("x-podman"): + # an extension field to docker, but podman-compose acts on it (podman_args, pod_args, …) + raise StackError(f"{where}: {key} is not allowed in a Vigil-managed stack") _no_variable_keys(value, f"{where}.{key}", _seen, _depth + 1) elif isinstance(node, list): _seen.add(id(node)) From ba37f73a0d2ad7bfd82bcad839fd3d250f57734e Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 23:03:02 -0400 Subject: [PATCH 44/68] fix(security): adopting a stack reads its files without following a symlink _stack_read runs as root and reads compose and .env from paths in container labels, which may sit in a folder a non-root user owns. A planted .env -> /etc/shadow (or a re-pointed folder) would have been read and posted to the server as the stack's env. Reads now open each folder relative to the last with O_NOFOLLOW, accept regular files only, and stop at 1 MB. _up also refuses a symlinked /stacks. --- agent/tests/test_compose_check.py | 19 ++++++++++ agent/vigil_agent/actions/stacks.py | 54 +++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py index 6b8bc531..8059c5b6 100644 --- a/agent/tests/test_compose_check.py +++ b/agent/tests/test_compose_check.py @@ -183,6 +183,25 @@ def test_agent_writes_never_follow_a_planted_link(self): self.assertEqual(link.read_text(), "services: {}") self.assertEqual(target.read_text(), "untouched") + def test_adopt_reads_never_follow_a_planted_link(self): + """_stack_read runs as root on paths from container labels, possibly in + a folder a non-root user owns: .env -> /etc/shadow must not be posted.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "secret").write_text("root:$6$hash") + (root / "stack").mkdir() + (root / "stack" / "compose.yaml").write_text("services: {}") + self.assertEqual(stacks._read_plain(root / "stack" / "compose.yaml"), "services: {}") + self.assertIsNone(stacks._read_plain(root / "stack" / ".env")) + (root / "stack" / ".env").symlink_to(root / "secret") + with self.assertRaises(ValueError): + stacks._read_plain(root / "stack" / ".env") + (root / "linked").symlink_to(root / "stack") + with self.assertRaises(ValueError): + stacks._read_plain(root / "linked" / "compose.yaml") + with self.assertRaises(ValueError): + stacks._read_plain(root / "stack") # not a regular file + def test_deploy_refuses_a_user_owned_stack_folder(self): with tempfile.TemporaryDirectory() as tmp: with self.assertRaises(ValueError): diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 75183178..25c9524b 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -11,6 +11,7 @@ import os import re import shutil +import stat from pathlib import Path from .. import collector @@ -55,6 +56,50 @@ def _write_new(path: Path, text: str, mode: int) -> None: os.close(fd) +MAX_READ = 1_000_000 + + +def _read_plain(path: Path) -> str | None: + """A regular file's text, read as root without following a symlink + anywhere on the way (each folder opened relative to the last, with + O_NOFOLLOW). The paths come from container labels and may sit in a folder a + non-root user owns, who could otherwise point .env at /etc/shadow and + have it posted to the server. None when there is no such file.""" + path = Path(path) + if not path.is_absolute() or ".." in path.parts: + raise ValueError(f"{path} must be an absolute path without '..'") + nofollow = getattr(os, "O_NOFOLLOW", 0) + fd = os.open("/", os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + for part in path.parts[1:-1]: + try: + nxt = os.open(part, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | nofollow, dir_fd=fd) + except FileNotFoundError: + return None + except OSError as exc: # ELOOP / ENOTDIR: a symlink or not a folder + raise ValueError(f"refusing to read {path}: {part} is not a plain folder") from exc + os.close(fd) + fd = nxt + try: + leaf = os.open(path.name, os.O_RDONLY | nofollow | getattr(os, "O_NONBLOCK", 0), dir_fd=fd) + except FileNotFoundError: + return None + except OSError as exc: + raise ValueError(f"refusing to read {path}: it is a symlink or unreadable") from exc + try: + st = os.fstat(leaf) + if not stat.S_ISREG(st.st_mode): + raise ValueError(f"refusing to read {path}: not a regular file") + if st.st_size > MAX_READ: + raise ValueError(f"refusing to read {path}: larger than {MAX_READ} bytes") + with os.fdopen(leaf, "rb", closefd=False) as fh: + return fh.read(MAX_READ + 1).decode() + finally: + os.close(leaf) + finally: + os.close(fd) + + def _write_private(path: Path, text: str) -> None: """Create or replace *path* readable by its owner only.""" _write_new(path, text, 0o600) @@ -146,6 +191,8 @@ def _up(workdir: Path, project: str, resolved: dict, data_dir: Path) -> str: import json resolved_dir = Path(data_dir) / "stacks" + if resolved_dir.is_symlink(): + raise ValueError(f"refusing to deploy: {resolved_dir} is a symlink") resolved_dir.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(resolved_dir, 0o700) path = resolved_dir / f"{project}.json" @@ -252,12 +299,15 @@ def _stack_read(params: dict, config: AgentConfig) -> str: else compose_path.parent env_path = Path(args[args.index("--env-file") + 1]) if "--env-file" in args \ else workdir / ".env" + compose = _read_plain(compose_path) + if compose is None: + raise ValueError(f"{compose_path} does not exist") payload = { "project": project, "compose_file": compose_path.name, "working_dir": str(workdir), - "compose": compose_path.read_text(), - "env": env_path.read_text() if env_path.exists() else "", + "compose": compose, + "env": _read_plain(env_path) or "", "hashes": _hash_report(project, args), } client.post_stack_read(config, ticket, payload) From ff4808398eaafa6791eee1eab25462968321e2ca Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 23:05:47 -0400 Subject: [PATCH 45/68] fix(security): adopt hashes private copies, keeps compose's errors to itself, and refuses hard links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _read_plain refuses a file with more than one hard link: a link to /etc/shadow is no symlink, and gets past O_NOFOLLOW on a host without fs.protected_hardlinks. - _hash_report ran `compose config --hash` on the labelled paths — a second, link-following read as root after the safe one — and passed compose's error text on, which quotes the line it could not parse. It now runs on private copies of the files already read (same --project-directory, so the hashes match: checked on the Ubuntu test VM) and reports only that compose could not resolve the stack. --- agent/tests/test_compose_check.py | 20 +++++++++++++++++ agent/vigil_agent/actions/stacks.py | 33 ++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py index 8059c5b6..345ab5d0 100644 --- a/agent/tests/test_compose_check.py +++ b/agent/tests/test_compose_check.py @@ -201,6 +201,26 @@ def test_adopt_reads_never_follow_a_planted_link(self): stacks._read_plain(root / "linked" / "compose.yaml") with self.assertRaises(ValueError): stacks._read_plain(root / "stack") # not a regular file + (root / "stack" / ".env").unlink() + os.link(root / "secret", root / "stack" / ".env") # a hard link is no symlink + with self.assertRaises(ValueError): + stacks._read_plain(root / "stack" / ".env") + + def test_hash_report_uses_private_copies_and_hides_compose_errors(self): + seen = [] + + def fake_run(cmd, timeout=60, extra_env=None): + seen.append(cmd) + raise RuntimeError("unexpected character in variable name 'root:$6$hash'") + with patch.object(stacks.ex, "_run", side_effect=fake_run), \ + patch.object(stacks.ex, "_compose_cmd", return_value=["docker", "compose"]), \ + patch.object(stacks.ex, "_compose_env", return_value={}): + report = stacks._hash_report("shop", Path("/srv/shop"), "services: {}", "A=1\n") + self.assertNotIn("root:", str(report)) + cmd = seen[0] + self.assertTrue(cmd[cmd.index("-f") + 1].startswith(tempfile.gettempdir())) + self.assertTrue(cmd[cmd.index("--env-file") + 1].startswith(tempfile.gettempdir())) + self.assertEqual(cmd[cmd.index("--project-directory") + 1], "/srv/shop") def test_deploy_refuses_a_user_owned_stack_folder(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 25c9524b..7f50f191 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -90,6 +90,10 @@ def _read_plain(path: Path) -> str | None: st = os.fstat(leaf) if not stat.S_ISREG(st.st_mode): raise ValueError(f"refusing to read {path}: not a regular file") + if st.st_nlink > 1: + # a hard link to /etc/shadow is no symlink, and needs no + # O_NOFOLLOW to get through (where fs.protected_hardlinks is off) + raise ValueError(f"refusing to read {path}: it has {st.st_nlink} hard links") if st.st_size > MAX_READ: raise ValueError(f"refusing to read {path}: larger than {MAX_READ} bytes") with os.fdopen(leaf, "rb", closefd=False) as fh: @@ -254,16 +258,28 @@ def _stack_remove(params: dict, config: AgentConfig) -> str: return ActionOutput(output, {"project": project, "files_deleted": deleted}) -def _hash_report(project: str, args: list[str]) -> dict: +def _hash_report(project: str, workdir: Path, compose: str, env: str) -> dict: """Which services would compose recreate? ``config --hash '*'`` against - the config-hash label on each running container. Matching = untouched.""" + the config-hash label on each running container. Matching = untouched. + + Run on private copies of the files _stack_read already read safely, not on + the labelled paths (a second, link-following read as root); and compose's + error text is never passed on — its parse errors quote the offending line, + of whatever file it was.""" + import tempfile + from .containers import _COMPOSE_PROJECT_LABEL - try: - out = ex._run([*ex._compose_cmd(), *args, "config", "--hash", "*"], timeout=120, - extra_env=ex._compose_env()) - except RuntimeError as exc: - return {"error": str(exc)[:300]} + with tempfile.TemporaryDirectory(prefix="vigil-adopt-") as tmp: + compose_copy, env_copy = Path(tmp) / "compose.yaml", Path(tmp) / "stack.env" + _write_private(compose_copy, compose) + _write_private(env_copy, env) + try: + out = ex._run([*ex._compose_cmd(), "-p", project, "--project-directory", str(workdir), + "--env-file", str(env_copy), "-f", str(compose_copy), + "config", "--hash", "*"], timeout=120, extra_env=ex._compose_env()) + except RuntimeError: + return {"error": "compose could not resolve this stack as it is on the host"} wanted = {} for line in out.splitlines(): parts = line.split() @@ -308,8 +324,9 @@ def _stack_read(params: dict, config: AgentConfig) -> str: "working_dir": str(workdir), "compose": compose, "env": _read_plain(env_path) or "", - "hashes": _hash_report(project, args), + "hashes": None, } + payload["hashes"] = _hash_report(project, workdir, compose, payload["env"]) client.post_stack_read(config, ticket, payload) report = payload["hashes"] return ActionOutput( From f50d5cf835ea0f090d8ae86b2dc52f8f909e7b63 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 23:30:43 -0400 Subject: [PATCH 46/68] feat(stacks): an endpoint that tells the editor what a compose file says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-screen stack editor (QA-20) validates as the person types, which needs an answer for every keystroke rather than a status code: bad YAML, a sandbox refusal and a good file are all 200 with the reason in the body, so the editor never renders a failed request as a broken file. describe_compose calls validate_compose and reports its refusal verbatim instead of re-implementing a lighter copy of the rules — a validation endpoint that skipped the sandbox would be a way to develop a file that could reach the host, and the editor would look authoritative about it. StackError now carries the parser's line (1-based) so the editor can point the gutter at the place PyYAML actually blamed. env_refs/missing_env are the side panel's "these need a value" list. A reference counts as missing only when nothing can settle it: a ${VAR…} clause of any kind does, and a bare $VAR/${VAR} is settled by the stack's .env, which is what env_keys is the editor's view of. $$ is an escaped dollar sign and never a reference. Phase QA-19. --- server/apps/stacks/test_validate_endpoint.py | 78 ++++++++++++++++++++ server/apps/stacks/urls.py | 1 + server/apps/stacks/validation.py | 56 ++++++++++++++ server/apps/stacks/views.py | 20 ++++- 4 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 server/apps/stacks/test_validate_endpoint.py diff --git a/server/apps/stacks/test_validate_endpoint.py b/server/apps/stacks/test_validate_endpoint.py new file mode 100644 index 00000000..f458f863 --- /dev/null +++ b/server/apps/stacks/test_validate_endpoint.py @@ -0,0 +1,78 @@ +"""QA-19: POST /api/v1/stacks/validate/ — the stack editor's live validator.""" +from django.contrib.auth import get_user_model +from django.test import TestCase + +from apps.accounts.models import Role, UserProfile + +GOOD = """services: + web: + image: nginx + ports: + - "80:80" + environment: + - DB_PASSWORD=${DB_PASSWORD} + - TZ=${TZ:-UTC} +""" + + +class ValidateEndpointTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user("a", password="x") + UserProfile.objects.create(user=self.user, role=Role.ADMIN) + self.client.force_login(self.user) + + def _post(self, compose_yaml, env_keys=None): + body = {"compose_yaml": compose_yaml} + if env_keys is not None: + body["env_keys"] = env_keys + return self.client.post("/api/v1/stacks/validate/", body, + content_type="application/json") + + def test_a_good_file_lists_services_and_env(self): + resp = self._post(GOOD, []) + self.assertEqual(resp.status_code, 200) + body = resp.json() + self.assertTrue(body["ok"]) + self.assertEqual(body["error"], "") + self.assertIsNone(body["line"]) + self.assertEqual( + body["services"], + [{"name": "web", "image": "nginx", "ports": ["80:80"], "build": False}]) + self.assertEqual(body["env_refs"], ["DB_PASSWORD", "TZ"]) + self.assertEqual(body["missing_env"], ["DB_PASSWORD"]) + + def test_env_keys_satisfy_refs(self): + body = self._post(GOOD, ["DB_PASSWORD"]).json() + self.assertEqual(body["missing_env"], []) + self.assertEqual(body["env_refs"], ["DB_PASSWORD", "TZ"]) + + def test_bad_yaml_reports_a_line(self): + resp = self._post("services:\n web:\n image: [\n") + self.assertEqual(resp.status_code, 200) + body = resp.json() + self.assertFalse(body["ok"]) + self.assertIn("not valid YAML", body["error"]) + self.assertIsInstance(body["line"], int) + self.assertGreaterEqual(body["line"], 2) + self.assertEqual(body["services"], []) + + def test_sandbox_refusals_come_through(self): + body = self._post("services:\n web:\n image: nginx\n privileged: true\n") + self.assertEqual(body.status_code, 200) + body = body.json() + self.assertFalse(body["ok"]) + self.assertIn("privileged", body["error"]) + self.assertEqual(body["services"], []) + + def test_admin_only(self): + viewer = get_user_model().objects.create_user("v", password="x") + UserProfile.objects.create(user=viewer, role=Role.VIEWER) + self.client.force_login(viewer) + self.assertEqual(self._post(GOOD).status_code, 403) + + def test_double_dollar_is_not_a_ref(self): + body = self._post("services:\n web:\n image: nginx\n" + " command: echo $$HOME\n").json() + self.assertTrue(body["ok"]) + self.assertEqual(body["env_refs"], []) + self.assertEqual(body["missing_env"], []) diff --git a/server/apps/stacks/urls.py b/server/apps/stacks/urls.py index 5e4994e1..2c097e87 100644 --- a/server/apps/stacks/urls.py +++ b/server/apps/stacks/urls.py @@ -5,6 +5,7 @@ urlpatterns = [ path("", views.stack_index, name="stack-index"), path("adopt/", views.stack_adopt, name="stack-adopt"), + path("validate/", views.stack_validate, name="stack-validate"), path("registries/", registries.registry_index, name="stack-registries"), path("registries//", registries.registry_detail, name="stack-registry-detail"), path("/", views.stack_detail, name="stack-detail"), diff --git a/server/apps/stacks/validation.py b/server/apps/stacks/validation.py index f1b643b8..7f82f9c6 100644 --- a/server/apps/stacks/validation.py +++ b/server/apps/stacks/validation.py @@ -299,6 +299,62 @@ def validate_compose(text: str) -> dict: return doc +#: ``${VAR}`` / ``${VAR:-default}`` / ``$VAR``. Group 2 is a clause — a +#: default or an error form — and either one settles the reference. +_ENV_REF = re.compile( + r"\$\{([A-Za-z_][A-Za-z0-9_]*)(:?[-?][^}]*)?\}|\$([A-Za-z_][A-Za-z0-9_]*)" +) + + +def _env_refs(text: str) -> tuple[list[str], list[str]]: + """(every reference, the ones that need a value) — sorted, unique. + + A reference whose match starts just after the first ``$`` of a ``$$`` is + an escaped dollar sign — ``$$FOO`` is a literal ``$`` followed by text, + and never something to satisfy. Only the bare ``$VAR`` / ``${VAR}`` forms + land in the second list; see ``describe_compose`` for why a clause of any + kind settles a reference.""" + all_names, needed = set(), set() + for match in _ENV_REF.finditer(text): + if text[match.start() - 1:match.start()] == "$": + continue + name = match.group(1) or match.group(3) + all_names.add(name) + if match.group(1) is None or match.group(2) is None: + needed.add(name) + return sorted(all_names), sorted(needed) + + +def describe_compose(text: str, env_keys=()) -> dict: + """What the editor's side panel shows for this text, valid or not. + + Never raises: a refusal from ``validate_compose`` is the answer, not an + exception, so the editor can show it without treating a bad file as a + failed request. The full refusal is reused rather than a lighter copy of + the rules, so the sandbox in ``validate_compose`` stays the one gate. + + A reference counts as missing only when nothing can resolve it: a + ``${VAR…}`` clause of any kind settles it (a default supplies a value, the + ``?`` forms fail in compose itself), and a bare ``$VAR``/``${VAR}`` is + resolved by the stack's ``.env`` — which is what ``env_keys`` is the + editor's view of, and which compose consults for exactly this form through + a service's ``env_file: .env``. Unset and un-defaulted, compose would + substitute an empty string, and that is what to warn about.""" + try: + doc = validate_compose(text) + except StackError as exc: + return {"ok": False, "error": str(exc), "line": exc.line, "services": [], + "env_refs": _env_refs(text)[0], "missing_env": []} + services = [{"name": str(name), "image": str(svc.get("image") or ""), + "ports": [str(p) for p in (svc.get("ports") or [])], + "build": svc.get("build") is not None} + for name, svc in doc["services"].items()] + refs, needed = _env_refs(text) + known = {str(k) for k in env_keys if isinstance(k, str)} + return {"ok": True, "error": "", "line": None, "services": services, + "env_refs": refs, "missing_env": [r for r in needed if r not in known]} + + def parse_env(text: str) -> list[tuple[str, str]]: """``KEY=value`` lines (comments and blanks allowed), refused otherwise.""" if len(text or "") > MAX_ENV: diff --git a/server/apps/stacks/views.py b/server/apps/stacks/views.py index 7f5d7036..3bc809f9 100644 --- a/server/apps/stacks/views.py +++ b/server/apps/stacks/views.py @@ -18,7 +18,14 @@ from vigil import hooks, scoping from .models import ManagedStack, StackRevision -from .validation import NAME_RE, StackError, parse_env, render_env, validate_compose +from .validation import ( + NAME_RE, + StackError, + describe_compose, + parse_env, + render_env, + validate_compose, +) STACKS_ROOT = "/opt/vigil/stacks" @@ -115,6 +122,17 @@ def stack_index(request): return Response(_row(stack), status=status.HTTP_201_CREATED) +@api_view(["POST"]) +@permission_classes([IsAuthenticated, IsAdmin]) +def stack_validate(request): + """Tell the editor what this compose text says — 200 whether it's valid + or not, so a bad file is an answer and never a failed request.""" + text = str(request.data.get("compose_yaml") or "") + keys = request.data.get("env_keys") + env_keys = [k for k in keys if isinstance(k, str)] if isinstance(keys, list) else [] + return Response(describe_compose(text, env_keys)) + + def _stack_or_404(request, stack_id): stack = get_object_or_404(ManagedStack.objects.select_related("host"), pk=stack_id) if not scoping.host_in_scope(request.user, stack.host): From bce6a2c0b1d9f09e3dbea148784893b64261c345 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Thu, 8 Oct 2026 23:40:29 -0400 Subject: [PATCH 47/68] fix(security): a deploy's compose config error stays in the host's log compose's parse errors quote the line they could not read, of whatever file that was. _check_resolved passed that text back as the task result; it now logs it on the host and returns a generic refusal, as _hash_report does. --- agent/tests/test_compose_check.py | 7 +++++-- agent/vigil_agent/actions/stacks.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/agent/tests/test_compose_check.py b/agent/tests/test_compose_check.py index 345ab5d0..a820f83d 100644 --- a/agent/tests/test_compose_check.py +++ b/agent/tests/test_compose_check.py @@ -262,9 +262,12 @@ def test_live_writable_binds_come_from_vigil_stacks_only(self): self.assertEqual(stacks._live_rw_binds(Path("/data")), ("/srv/web",)) def test_unreadable_config_refuses(self): - with patch.object(stacks, "_compose", side_effect=RuntimeError("compose missing")): - with self.assertRaises(ValueError): + leak = "line 1: unexpected character in 'root:$6$hash'" + with patch.object(stacks, "_compose", side_effect=RuntimeError(leak)), \ + self.assertLogs("vigil_agent.actions.stacks", "WARNING"): + with self.assertRaises(ValueError) as caught: stacks._check_resolved(Path("/tmp"), "web", "compose.yaml", Path("/tmp")) + self.assertNotIn("root:", str(caught.exception), "compose's error quotes host files: log it, don't return it") if __name__ == "__main__": diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py index 7f50f191..75b1c25c 100644 --- a/agent/vigil_agent/actions/stacks.py +++ b/agent/vigil_agent/actions/stacks.py @@ -8,6 +8,7 @@ from __future__ import annotations +import logging import os import re import shutil @@ -19,6 +20,8 @@ from ..config import AgentConfig from ..executor import ActionOutput +log = logging.getLogger(__name__) + STACKS_ROOT = Path("/opt/vigil/stacks") COMPOSE_NAME = "compose.yaml" _PROJECT = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$") @@ -179,8 +182,12 @@ def _check_resolved(workdir: Path, project: str, compose_file: str, data_dir: Pa try: resolved = pin(_resolved_config(workdir, project, compose_file), str(workdir)) live_rw = _live_rw_binds(data_dir) - except Exception as exc: # noqa: BLE001 — cannot check it, so do not run it - raise ValueError(f"could not read the resolved compose configuration: {exc}") from exc + except Exception: # noqa: BLE001 — cannot check it, so do not run it + # compose's errors quote the line they could not parse, of whatever + # file that was: the detail stays in the host's log, not the result + log.warning("compose config failed for stack %s", project, exc_info=True) + raise ValueError("could not read the resolved compose configuration " + "(the agent's log on the host has compose's error)") from None found = problems(resolved, str(workdir), live_rw) if found: raise ValueError("refusing to deploy " + project + ": " + "; ".join(found[:10])) From 8c78a2af9ba18bd46a7a730706258a07854a5460 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 00:27:13 -0400 Subject: [PATCH 48/68] feat: make the stack editor a full-screen page (QA-20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creating or editing a compose stack happened in a 760px modal, which is a cramped place to write a YAML file. The editor is now a page of its own, in the layout the user picked: a sticky bar (breadcrumb, status pill, take down/deploy/save), a highlighted YAML editor with a line-number gutter, and a side column for the masked .env and the revisions. Live validation is the reason for the page: 400ms after the last keystroke the file goes to QA-19's /api/v1/stacks/validate/, and the answer becomes the status pill, the service chips, the gutter's red line, and an "Add VAR" button per variable the file references but the .env does not carry. Responses are sequenced so a stale answer cannot overwrite a newer one. Because the page can now be left — it is navigation, not a dialog — unsaved edits are tracked and Back, Escape and any navigateTo out of the page ask first, and Deploy is disabled until a save: deploy ships a revision, not the bytes on screen. Take down asks twice, the second time about the folder on the host, matching the server's rule that adopted stacks keep their files. --- server/apps/stacks/test_stack_editor_page.py | 51 +++ server/apps/stacks/tests.py | 8 +- server/static/css/vigil.css | 55 +++ server/static/js/vigil-stacks.js | 392 ++++++++++++++----- server/templates/dashboard.html | 4 + server/templates/pages/_stack_editor.html | 58 +++ 6 files changed, 477 insertions(+), 91 deletions(-) create mode 100644 server/apps/stacks/test_stack_editor_page.py create mode 100644 server/templates/pages/_stack_editor.html diff --git a/server/apps/stacks/test_stack_editor_page.py b/server/apps/stacks/test_stack_editor_page.py new file mode 100644 index 00000000..54d5cf7f --- /dev/null +++ b/server/apps/stacks/test_stack_editor_page.py @@ -0,0 +1,51 @@ +"""QA-20: the stack editor is a page of its own, not a modal.""" +from pathlib import Path + +from django.test import SimpleTestCase + +ROOT = Path(__file__).resolve().parents[2] + + +def _text(relpath: str) -> str: + return (ROOT / relpath).read_text(encoding="utf-8") + + +class StackEditorPageTests(SimpleTestCase): + def test_page_exists_and_is_included(self): + page = _text("templates/pages/_stack_editor.html") + for marker in ('id="page-stack-editor"', 'id="stack-compose"', 'id="sted-gutter"', + 'id="stack-env"', 'id="stack-revisions"'): + self.assertIn(marker, page) + self.assertIn("pages/_stack_editor.html", _text("templates/dashboard.html")) + + def test_no_modal_left(self): + js = _text("static/js/vigil-stacks.js") + self.assertNotIn("stack-modal", js) + self.assertNotIn("stack-overlay", js) + self.assertIn("navigateTo('stack-editor')", js) + + def test_live_validation(self): + js = _text("static/js/vigil-stacks.js") + for needle in ("/api/v1/stacks/validate/", "env_keys", "setTimeout"): + self.assertIn(needle, js) + # The POST itself, not merely its path: a call that quietly disappears + # is the regression this test exists to catch. + source = js[js.index("async function _validateStack"):] + source = source[:source.index("\nasync function ")] + for needle in ("apiJson('/api/v1/stacks/validate/'", "method: 'POST'", + "compose_yaml: ta.value"): + self.assertIn(needle, source) + + def test_dirty_guard(self): + js = _text("static/js/vigil-stacks.js") + self.assertIn("dirty", js) + self.assertIn("confirmModal(", js) + self.assertNotIn("window.confirm(", js) + + def test_env_values_never_in_markup(self): + js = _text("static/js/vigil-stacks.js") + source = js[js.index("function _renderStackEnv"):] + source = source[:source.index("\nfunction ")] + self.assertIn(".value =", source) + self.assertNotIn("${e.value}", source) + self.assertNotIn("${escAttr(e.value)}", source) diff --git a/server/apps/stacks/tests.py b/server/apps/stacks/tests.py index 795fbd80..9bfaccab 100644 --- a/server/apps/stacks/tests.py +++ b/server/apps/stacks/tests.py @@ -226,10 +226,14 @@ def test_editor_is_wired(self): from pathlib import Path root = Path(__file__).resolve().parents[2] js = (root / "static/js/vigil-stacks.js").read_text(encoding="utf-8") - page = (root / "templates/pages/_containers.html").read_text(encoding="utf-8") + bar = (root / "templates/pages/_containers.html").read_text(encoding="utf-8") + page = (root / "templates/pages/_stack_editor.html").read_text(encoding="utf-8") base = (root / "templates/base.html").read_text(encoding="utf-8") - self.assertIn('id="managed-stacks"', page) + self.assertIn('id="managed-stacks"', bar) self.assertIn("js/vigil-stacks.js", base) + for marker in ('id="page-stack-editor"', 'id="stack-compose"', 'data-stack-save', + 'data-stack-deploy', 'data-stack-remove', 'data-stack-env-reveal'): + self.assertIn(marker, page) for needle in ("/env/reveal/", "/deploy/", "/remove/", "/revisions/", "keep: true", "type=\"${e.revealed ? 'text' : 'password'}\"", "async function renderManagedStacks"): self.assertIn(needle, js) diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index a89356e3..d8c0685d 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -3964,3 +3964,58 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } .rlt-wave { grid-template-columns: 14px 1fr auto; } .rlt-wave-tags { display: none; } } + +/* ── Stack editor page (QA-20) ───────────────────────────────────────── */ +.sted { width: 100%; } +.sted-bar { + position: sticky; top: 0; z-index: 20; + display: flex; align-items: center; gap: 14px; + margin: -12px 0 18px; padding: 12px 0; + background: var(--bg); border-bottom: 1px solid var(--border); +} +.sted-crumbs { display: flex; align-items: center; gap: 7px; min-width: 0; font-size: 13px; color: var(--text-2); } +.sted-crumb { background: none; border: 0; padding: 0; cursor: pointer; font-family: inherit; + font-size: 13px; color: var(--text-3); } +.sted-crumb:hover { color: var(--sky-ink); } +.sted-crumb-now { color: var(--text-1); font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.sted-sep { color: var(--text-3); } +.sted-status { margin-left: auto; font-size: 11px; font-weight: 600; padding: 3px 11px; + border-radius: 20px; background: var(--s3); color: var(--text-3); white-space: nowrap; } +.sted-status.is-good { background: rgba(var(--rgb-mint), 0.14); color: var(--mint-ink); } +.sted-status.is-bad { background: rgba(var(--rgb-rose), 0.14); color: var(--rose-ink); } +.sted-actions { display: flex; align-items: center; gap: 8px; } + +.sted-grid { display: grid; grid-template-columns: minmax(0, 1fr) 340px; gap: 18px; align-items: start; } +.sted-main { display: flex; flex-direction: column; gap: 10px; min-width: 0; } +.sted-label { font-size: 11px; font-weight: 700; letter-spacing: 0.6px; text-transform: uppercase; + color: var(--text-3); } +.sted-field { display: flex; flex-direction: column; gap: 6px; } +.sted-editor { min-height: 60vh; } + +.sted-gutter { position: absolute; top: 0; left: 0; bottom: 0; z-index: 3; width: 52px; + /* line-height here must stay the 1.6 the .te-editor rule sets for the text + itself, or a gutter number stops lining up with its line. */ + padding: 14px 10px 14px 0; text-align: right; overflow: hidden; pointer-events: none; + font-family: var(--font-mono); font-size: 13px; line-height: 1.6; color: var(--text-3); + border-right: 1px solid var(--border); } +.sted-gutter .sted-ln.is-error { color: var(--rose-ink); font-weight: 700; } +.sted-editor .te-highlight, .sted-editor .te-yaml { padding-left: 66px; } + +.sted-check { display: flex; flex-direction: column; gap: 6px; font-size: 12px; } +.sted-check-ok { color: var(--mint-ink); font-weight: 600; } +.sted-check-bad { color: var(--rose-ink); font-family: var(--font-mono); word-break: break-word; } +.sted-check-line { color: var(--text-3); margin-left: 8px; } +.sted-services { display: flex; flex-wrap: wrap; gap: 6px; } +.sted-service { gap: 7px; } +.sted-service-img { color: var(--text-3); font-family: var(--font-mono); font-weight: 400; } +.sted-missing { color: var(--text-2); display: flex; align-items: center; gap: 8px; } + +.sted-side { display: flex; flex-direction: column; gap: 14px; min-width: 0; } +.sted-card { background: var(--s1); border-radius: var(--r-lg); padding: 16px; } +.sted-card-title { font-size: 12px; font-weight: 600; color: var(--text-1); margin-bottom: 10px; } +.sted-card-sub { font-weight: 400; color: var(--text-3); } +.sted-side .stack-env { gap: 8px; } + +@media (max-width: 980px) { + .sted-grid { grid-template-columns: minmax(0, 1fr); } +} diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index 2b1987b6..faab79a9 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -1,14 +1,24 @@ // vigil-stacks.js // Owns: Vigil-managed compose stacks on the Containers page -// (M11) — the list, the editor (compose file + masked .env), revisions, -// deploy and take-down. Free, one host at a time. -// HTML: #managed-stacks in templates/pages/_containers.html; the editor modal is -// built here on first use. -// Depends on: vigil-utils.js (escHtml, escAttr, apiJson, showToast, getCsrf). -// API: /api/v1/stacks/ (CRUD), //revisions/, //env/reveal/ (TOTP), -// //deploy/ (TOTP), //remove/ (TOTP) +// (M11) — the list, and the full-screen stack editor page +// (pages/_stack_editor.html): the compose file with a highlighted editor, +// live validation, the masked .env, revisions, deploy and take-down. +// Free, one host at a time. +// HTML: #managed-stacks in templates/pages/_containers.html; +// #page-stack-editor in templates/pages/_stack_editor.html. +// Depends on: vigil-utils.js (escHtml, escAttr, apiJson, showToast, getCsrf, +// confirmModal, totpPrompt, yamlToHtml, timeAgo), vigil-nav.js (navigateTo — +// wrapped below, and it needs every .page section to exist, so this file +// loads after base.html's content block is parsed). +// API: /api/v1/stacks/ (CRUD), /validate/, //revisions/, //env/reveal/ +// (TOTP), //deploy/ (TOTP), //remove/ (TOTP) -const stackEd = { hostId: null, stack: null, env: [], stacks: [] }; +const stackEd = { hostId: null, stack: null, env: [], stacks: [], revisions: [], dirty: false }; +let _stackCheckSeq = 0; +let _stackCheckTimer = null; + +const _NEW_COMPOSE = 'services:\n app:\n image: nginx:stable\n restart: unless-stopped\n'; +const DEPLOY_SAVE_HINT = 'Save first: deploy ships a saved revision'; async function renderManagedStacks(hostId) { const wrap = document.getElementById('managed-stacks'); @@ -26,108 +36,205 @@ async function renderManagedStacks(hostId) {
`; } -function _stackEls() { - let modal = document.getElementById('stack-modal'); - if (!modal) { - const overlay = document.createElement('div'); - overlay.className = 'modal-overlay'; - overlay.id = 'stack-overlay'; - modal = document.createElement('div'); - modal.className = 'modal modal-wide'; - modal.id = 'stack-modal'; - modal.setAttribute('role', 'dialog'); - modal.setAttribute('aria-modal', 'true'); - modal.setAttribute('aria-labelledby', 'stack-title'); - modal.innerHTML = `
New stack -
-
-
- -
Lowercase letters, digits, _ and -. It lives in /opt/vigil/stacks/<name> on the host.
-
-
-
-
-
- - -
- -
-
-
- - - - -
`; - document.body.append(overlay, modal); - overlay.addEventListener('click', closeStackEditor); - document.addEventListener('keydown', (ev) => { - if (ev.key === 'Escape' && modal.classList.contains('open')) closeStackEditor(); - }); - } - return modal; -} - function _renderStackEnv() { const wrap = document.getElementById('stack-env'); + // Values go in as .value after the markup is built: never interpolate one + // into an attribute — a revealed secret must not become markup. wrap.innerHTML = stackEd.env.map((e, i) => `
-
`).join('') || '
No variables.
'; + wrap.querySelectorAll('[data-env-row]').forEach((row) => { + const entry = stackEd.env[parseInt(row.dataset.envRow, 10)]; + if (!entry) return; + row.querySelector('[data-env-field="key"]').value = entry.key || ''; + row.querySelector('[data-env-field="value"]').value = entry.value || ''; + }); +} + +function _stedEl(id) { return document.getElementById(id); } + +function _stedHostLabel(stack) { + if (stack && stack.hostname) return stack.hostname; + if (stackEd.hostId) { + const opt = document.querySelector(`#containers-host option[value="${CSS.escape(stackEd.hostId)}"]`); + if (opt) return opt.textContent.trim(); + } + return ''; +} + +function _stedGutter(lines) { + let out = ''; + for (let i = 1; i <= lines; i += 1) { + out += `
${i}
`; + } + return out; +} + +function _syncStackEditor() { + const ta = _stedEl('stack-compose'); + const hl = _stedEl('sted-hl'); + const gutter = _stedEl('sted-gutter'); + if (!ta || !hl || !gutter) return; + hl.innerHTML = yamlToHtml(ta.value) + '\n'; // trailing NL keeps last line visible + gutter.innerHTML = _stedGutter(ta.value.split('\n').length); + _mirrorStackScroll(); +} + +function _mirrorStackScroll() { + const ta = _stedEl('stack-compose'); + const hl = _stedEl('sted-hl'); + const gutter = _stedEl('sted-gutter'); + if (!ta || !hl || !gutter) return; + hl.scrollTop = ta.scrollTop; hl.scrollLeft = ta.scrollLeft; + gutter.scrollTop = ta.scrollTop; +} + +function _stedButtons() { + const editing = !!stackEd.stack; + document.querySelector('.sted [data-stack-deploy]').hidden = !editing; + document.querySelector('.sted [data-stack-remove]').hidden = !editing; + _stedEl('stack-revisions-field').hidden = !editing; + _stedEl('stack-name-field').hidden = editing; + _updateDeployButton(); +} + +function _updateDeployButton() { + const btn = document.querySelector('.sted [data-stack-deploy]'); + if (!btn || btn.hidden) return; // a new stack has no Deploy to disable + btn.disabled = stackEd.dirty; + btn.title = stackEd.dirty ? DEPLOY_SAVE_HINT : ''; +} + +function _markStackDirty() { + stackEd.dirty = true; + _updateDeployButton(); +} + +function _clearStackDirty() { + stackEd.dirty = false; + _updateDeployButton(); } async function openStackEditor(stackId) { - const modal = _stackEls(); const stack = stackId ? stackEd.stacks.find(s => s.id === stackId) : null; stackEd.stack = stack; stackEd.env = stack ? stack.env.map(e => ({ key: e.key, value: '', stored: true })) : []; - document.getElementById('stack-title').textContent = stack ? `Stack ${stack.name}` : 'New stack'; - const name = document.getElementById('stack-name'); + stackEd.revisions = []; + stackEd.errorLine = null; + _clearStackDirty(); + _stedEl('sted-host').textContent = _stedHostLabel(stack); + _stedEl('sted-name-label').textContent = stack ? stack.name : 'New stack'; + const name = _stedEl('stack-name'); name.value = stack ? stack.name : ''; name.disabled = !!stack; - document.getElementById('stack-compose').value = stack ? stack.compose_yaml - : 'services:\n app:\n image: nginx:stable\n restart: unless-stopped\n'; - document.getElementById('stack-error').classList.remove('show'); - modal.querySelector('[data-stack-deploy]').hidden = !stack; - modal.querySelector('[data-stack-remove]').hidden = !stack; - document.getElementById('stack-revisions-field').hidden = !stack; + _stedEl('stack-compose').value = stack ? stack.compose_yaml : _NEW_COMPOSE; + const err = _stedEl('stack-error'); + err.textContent = ''; + err.classList.remove('show'); const report = (stack && stack.adopt_report) || {}; const recreate = report.recreate || []; if (stack && stack.adopted) { - const err = document.getElementById('stack-error'); err.textContent = recreate.length ? `Adopted from ${stack.working_dir}/${stack.compose_file}. Deploying this file would recreate: ${recreate.join(', ')} — their running config differs from the file.` : `Adopted from ${stack.working_dir}/${stack.compose_file}. Deploying it unchanged recreates nothing.`; err.classList.add('show'); } _renderStackEnv(); - document.getElementById('stack-overlay').classList.add('open'); - modal.classList.add('open'); - (stack ? document.getElementById('stack-compose') : name).focus(); + _stedButtons(); + _syncStackEditor(); + _setStackStatus(null); + navigateTo('stack-editor'); + (stack ? _stedEl('stack-compose') : name).focus(); if (stack) _loadStackRevisions(); + _validateStack(); } -function closeStackEditor() { - document.getElementById('stack-overlay')?.classList.remove('open'); - document.getElementById('stack-modal')?.classList.remove('open'); +async function closeStackEditor() { + if (stackEd.dirty) { + const discard = await confirmModal('Discard unsaved changes to this stack?', + { danger: true, confirmText: 'Discard' }); + if (!discard) return; + } stackEd.env = []; // forget any revealed values + _clearStackDirty(); + navigateTo('containers'); + if (stackEd.hostId) renderManagedStacks(stackEd.hostId); } function _stackError(message) { - const el = document.getElementById('stack-error'); + const el = _stedEl('stack-error'); el.textContent = message; el.classList.add('show'); } +/* ── Live validation (QA-19: POST /api/v1/stacks/validate/) ─────────── */ +function _scheduleStackCheck() { + clearTimeout(_stackCheckTimer); + _stackCheckTimer = setTimeout(_validateStack, 400); +} + +function _setStackStatus(problems) { + const pill = _stedEl('sted-status'); + if (!pill) return; + if (problems === null) { + pill.className = 'sted-status'; + pill.textContent = 'checking…'; + return; + } + pill.className = 'sted-status ' + (problems ? 'is-bad' : 'is-good'); + pill.textContent = problems ? `${problems} problem(s)` : 'valid'; +} + +function _renderStackCheck(r) { + const wrap = _stedEl('sted-check'); + if (!wrap) return; + const problems = (r.ok ? 0 : 1) + (r.missing_env || []).length; + let html = ''; + if (r.ok) { + const services = r.services || []; + html += `
✓ valid · ${services.length} service(s)
`; + if (services.length) { + html += `
${services.map(s => `${escHtml(s.name)}` + + `${escHtml(s.build ? 'build' : (s.image || ''))}`).join('')}
`; + } + } else { + html += `
✕ ${escHtml(r.error || 'This file does not validate.')}` + + (r.line ? `line ${r.line}` : '') + '
'; + } + (r.missing_env || []).forEach((v) => { + html += `
⚠ ${escHtml(v)} is used but not set ` + + `
`; + }); + wrap.innerHTML = html; + stackEd.errorLine = r.ok ? null : (r.line || null); + _syncStackEditor(); + _setStackStatus(problems); +} + +async function _validateStack() { + clearTimeout(_stackCheckTimer); + const ta = _stedEl('stack-compose'); + if (!ta) return; + const seq = ++_stackCheckSeq; + _setStackStatus(null); + let r; + try { + r = await apiJson('/api/v1/stacks/validate/', { + method: 'POST', + body: JSON.stringify({ compose_yaml: ta.value, env_keys: stackEd.env.map(e => e.key.trim()).filter(Boolean) }), + }); + } catch { return; } + if (seq !== _stackCheckSeq) return; // a newer keystroke already answered + if (!_stedEl('sted-check')) return; + _renderStackCheck(r); +} + async function _loadStackRevisions() { - const wrap = document.getElementById('stack-revisions'); + const wrap = _stedEl('stack-revisions'); try { const revs = (await apiJson(`/api/v1/stacks/${stackEd.stack.id}/revisions/`)).results || []; wrap.innerHTML = revs.map(r => `
@@ -139,22 +246,25 @@ async function _loadStackRevisions() { } async function saveStack() { - const compose = document.getElementById('stack-compose').value; + const compose = _stedEl('stack-compose').value; const env = stackEd.env.filter(e => e.key.trim()).map(e => (e.stored && !e.value && !e.revealed) ? { key: e.key.trim(), keep: true } : { key: e.key.trim(), value: e.value || '' }); const editing = stackEd.stack; const body = editing ? { compose_yaml: compose, env, note: 'edited' } - : { host_id: stackEd.hostId, name: document.getElementById('stack-name').value.trim(), compose_yaml: compose, env }; + : { host_id: stackEd.hostId, name: _stedEl('stack-name').value.trim(), compose_yaml: compose, env }; try { const saved = await apiJson(editing ? `/api/v1/stacks/${editing.id}/` : '/api/v1/stacks/', { method: editing ? 'PUT' : 'POST', body: JSON.stringify(body) }); showToast(`Saved ${saved.name} as revision ${saved.revision} — deploy it to apply`, 'success'); await renderManagedStacks(stackEd.hostId); - openStackEditor(saved.id); + // Reopen the saved row — but only if the editor is still on screen. An + // async Ctrl+S can land after the operator has moved on, and dragging + // them back to the page they left would be worse than a stale view. + if (document.getElementById('page-stack-editor')?.classList.contains('active')) openStackEditor(saved.id); } catch (e) { _stackError(e.message); } } -async function _stackTotpAction(path, payload, done) { +async function _stackTotpAction(path, payload) { const code = await totpPrompt('This change runs on the host.'); if (!code) return null; try { @@ -177,15 +287,32 @@ async function deployStack() { } async function removeStack() { - const wipe = window.confirm(`Take down ${stackEd.stack.name}? OK also deletes its folder on the host; Cancel keeps the files.`); - const body = await _stackTotpAction(`/api/v1/stacks/${stackEd.stack.id}/remove/`, { delete_files: wipe }); - if (body) showToast(`Taking down ${stackEd.stack.name}`, 'success'); + const stack = stackEd.stack; + const sure = await confirmModal(`Take down ${stack.name}?`, + { title: 'Take down this stack?', danger: true, confirmText: 'Take down' }); + if (!sure) return; + // An adopted stack's folder belongs to someone else — matching the server, + // there is nothing to delete, so the editor never offers to. + let wipe = false; + if (!stack.adopted) { + // Cancel means keep the files; the server refuses to delete an adopted + // stack's folder anyway, so adopted stacks are never asked. + wipe = await confirmModal(`Also delete ${stack.name}'s folder on ${stack.hostname}?`, + { title: 'Delete its files too?', danger: true, confirmText: 'Delete the folder' }); + } + const body = await _stackTotpAction(`/api/v1/stacks/${stack.id}/remove/`, { delete_files: wipe }); + if (!body) return; + showToast(`Taking down ${stack.name}`, 'success'); + stackEd.env = []; + _clearStackDirty(); + navigateTo('containers'); + if (stackEd.hostId) renderManagedStacks(stackEd.hostId); } /* ── Wiring ──────────────────────────────────────────────────────────── */ delegateClick('[data-stack-new]', () => openStackEditor(null)); delegateClick('[data-stack-edit]', (el) => openStackEditor(el.dataset.stackEdit)); -delegateClick('[data-stack-close]', () => closeStackEditor()); +delegateClick('[data-sted-back]', () => closeStackEditor()); delegateClick('[data-stack-save]', () => saveStack()); delegateClick('[data-stack-deploy]', () => deployStack()); delegateClick('[data-stack-remove]', () => removeStack()); @@ -193,21 +320,108 @@ delegateClick('[data-stack-env-reveal]', () => revealStackEnv()); delegateClick('[data-stack-env-add]', () => { stackEd.env.push({ key: '', value: '', stored: false }); _renderStackEnv(); + _markStackDirty(); + _scheduleStackCheck(); }); delegateClick('[data-env-remove]', (el) => { stackEd.env.splice(parseInt(el.dataset.envRemove, 10), 1); _renderStackEnv(); + _markStackDirty(); + _scheduleStackCheck(); +}); +delegateClick('[data-sted-add-env]', (el) => { + stackEd.env.push({ key: el.dataset.stedAddEnv, value: '', stored: false }); + _renderStackEnv(); + _markStackDirty(); + _scheduleStackCheck(); }); delegateClick('[data-stack-load-rev]', (el) => { const rev = (stackEd.revisions || []).find(r => r.number === parseInt(el.dataset.stackLoadRev, 10)); - if (rev) { - document.getElementById('stack-compose').value = rev.compose_yaml; - showToast(`Loaded r${rev.number}'s compose file — Save makes it the next revision`, 'success'); - } + if (!rev) return; + _stedEl('stack-compose').value = rev.compose_yaml; + _syncStackEditor(); + _markStackDirty(); + _scheduleStackCheck(); + showToast(`Loaded r${rev.number}'s compose file — Save makes it the next revision`, 'success'); }); document.addEventListener('input', (ev) => { const row = ev.target.closest && ev.target.closest('[data-env-row]'); - if (!row) return; - const entry = stackEd.env[parseInt(row.dataset.envRow, 10)]; - if (entry) entry[ev.target.dataset.envField] = ev.target.value; + if (row) { + const entry = stackEd.env[parseInt(row.dataset.envRow, 10)]; + if (!entry) return; + entry[ev.target.dataset.envField] = ev.target.value; + _markStackDirty(); + if (ev.target.dataset.envField === 'key') _scheduleStackCheck(); + return; + } + if (ev.target.id === 'stack-compose') { + _syncStackEditor(); + _markStackDirty(); + _scheduleStackCheck(); + } else if (ev.target.id === 'stack-name') { + _markStackDirty(); + } +}); +document.addEventListener('scroll', (ev) => { + if (ev.target && ev.target.id === 'stack-compose') _mirrorStackScroll(); +}, true); +document.addEventListener('keydown', (ev) => { + const ta = _stedEl('stack-compose'); + const inEditor = ev.target === ta; + if ((ev.metaKey || ev.ctrlKey) && inEditor && ev.key.toLowerCase() === 's') { + ev.preventDefault(); + saveStack(); + return; + } + if (inEditor && ev.key === 'Tab') { + // Two-space soft tab, and Shift+Tab takes up to two back — the editor is + // the page's only place to type YAML, so the Tab key must not leave it. + ev.preventDefault(); + const start = ta.selectionStart, end = ta.selectionEnd; + if (ev.shiftKey) { + const lineStart = ta.value.lastIndexOf('\n', start - 1) + 1; + let cut = 0; + while (cut < 2 && ta.value[lineStart + cut] === ' ') cut += 1; + if (cut) { + ta.value = ta.value.slice(0, lineStart) + ta.value.slice(lineStart + cut); + ta.selectionStart = ta.selectionEnd = Math.max(lineStart, start - cut); + } + } else { + ta.value = ta.value.slice(0, start) + ' ' + ta.value.slice(end); + ta.selectionStart = ta.selectionEnd = start + 2; + } + _syncStackEditor(); + _markStackDirty(); + _scheduleStackCheck(); + return; + } + if (ev.key === 'Escape' && document.getElementById('page-stack-editor')?.classList.contains('active')) { + // vigil-modal.js stops the event while a dialog is open, so Escape + // reaching here means the editor is the topmost thing on screen. + closeStackEditor(); + } }); + +/* ── navigateTo wrapper: leaving the editor with unsaved edits asks first ─ */ +(function () { + const previous = window.navigateTo; + if (typeof previous !== 'function') return; + let navigating = false; + window.navigateTo = function (pageName) { + if (!navigating && pageName !== 'stack-editor' + && document.getElementById('page-stack-editor')?.classList.contains('active') + && stackEd.dirty) { + navigating = true; + confirmModal('Discard unsaved changes to this stack?', { danger: true, confirmText: 'Discard' }) + .then((go) => { + navigating = false; + if (!go) return; + _clearStackDirty(); + stackEd.env = []; + previous.apply(window, arguments); + }); + return; + } + previous.apply(this, arguments); + }; +})(); diff --git a/server/templates/dashboard.html b/server/templates/dashboard.html index 7cb07c97..31be23f6 100644 --- a/server/templates/dashboard.html +++ b/server/templates/dashboard.html @@ -30,6 +30,10 @@ {# Logic: static/js/vigil-containers.js + vigil-stacks.js | Data: client-side fetch /api/v1/hosts/{id}/containers/, /api/v1/stacks/ #} {% include "pages/_containers.html" %} + {# Stack editor page — full-screen compose editor (YAML + gutter, live validation, masked .env, revisions) #} + {# Logic: static/js/vigil-stacks.js | Data: client-side fetch /api/v1/stacks/ (+ //revisions/, POST /api/v1/stacks/validate/) #} + {% include "pages/_stack_editor.html" %} + {# Playbooks page — sequence editor for auto-enroll task chains #} {# Logic: static/js/vigil-playbooks.js | Data: /api/v1/playbooks/ + /api/v1/tasks/definitions/ #} {% include "pages/_playbooks.html" %} diff --git a/server/templates/pages/_stack_editor.html b/server/templates/pages/_stack_editor.html new file mode 100644 index 00000000..82488ec6 --- /dev/null +++ b/server/templates/pages/_stack_editor.html @@ -0,0 +1,58 @@ +{# Stack editor page section — a full-screen compose editor: highlighted YAML with a line gutter, live validation, masked .env, revisions #} +{# Logic: static/js/vigil-stacks.js (openStackEditor / saveStack / validateStack / deployStack / removeStack) | Styles: vigil.css .sted-* (+ .te-editor/.te-highlight/.te-yaml from the task editor) | Data: client-side fetch /api/v1/stacks/ (CRUD), //revisions/, //env/reveal/, //deploy/, //remove/ (TOTP), POST /api/v1/stacks/validate/ #} +
+ +
+
+ + › + + › + +
+ +
+ + + +
+
+ +
+
+
+ + +
Lowercase letters, digits, _ and -. It lives in /opt/vigil/stacks/<name> on the host.
+
+ +
compose.yaml
+
+ + + +
+ +
+
+
+ + +
+
From 38ad3004b2ce1f45548e0d51162e0f003b3637d0 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 01:53:15 -0400 Subject: [PATCH 49/68] feat(rollouts): start several wave groups in parallel as one batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rollout walked one ladder. Starting Servers, Desktops and Labs side by side took three requests and three TOTP codes, and a machine tagged into two ladders (a shared Canary wave) was patched once per ladder. POST /api/v1/rollouts/ with wave_group_tags (2-12) starts one ordinary rollout per group under a shared PatchRollout.batch id, all at once, inside one transaction. Every machine goes to exactly one of them — the first group listed that reaches it — so parallel ladders never send the same task twice. An unknown group refuses the whole batch before anything starts; a group the split leaves empty is skipped. Each rollout then runs, gates and halts exactly as a single one does. POST /api/v1/rollouts/batch//halt/ (admin + one TOTP code) halts every active rollout in the batch: codes are single-use, so halting each group separately would need a fresh code per group. Phase QA-21. --- .../tasks/migrations/0038_rollout_batch.py | 18 + server/apps/tasks/models.py | 3 + server/apps/tasks/rollout.py | 117 +++++++ server/apps/tasks/rollout_serializers.py | 2 +- server/apps/tasks/rollout_urls.py | 2 + server/apps/tasks/test_rollout_batch.py | 326 ++++++++++++++++++ server/apps/tasks/views.py | 49 +++ 7 files changed, 516 insertions(+), 1 deletion(-) create mode 100644 server/apps/tasks/migrations/0038_rollout_batch.py create mode 100644 server/apps/tasks/test_rollout_batch.py diff --git a/server/apps/tasks/migrations/0038_rollout_batch.py b/server/apps/tasks/migrations/0038_rollout_batch.py new file mode 100644 index 00000000..8a5f2ccb --- /dev/null +++ b/server/apps/tasks/migrations/0038_rollout_batch.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-10-09 04:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tasks', '0037_seed_container_rollback'), + ] + + operations = [ + migrations.AddField( + model_name='patchrollout', + name='batch', + field=models.UUIDField(blank=True, db_index=True, null=True), + ), + ] diff --git a/server/apps/tasks/models.py b/server/apps/tasks/models.py index e6c2a9aa..d24b9373 100644 --- a/server/apps/tasks/models.py +++ b/server/apps/tasks/models.py @@ -391,6 +391,9 @@ class ActionKind(models.TextChoices): #: the hosts that drifted, not every host its waves carry. Null = no #: restriction, which is what every other rollout is. host_ids = models.JSONField(null=True, blank=True) + #: Rollouts started together by one request share this id — a parallel + #: batch, one rollout per wave group. Null = a rollout on its own. + batch = models.UUIDField(null=True, blank=True, db_index=True) playbook = models.ForeignKey( "baselines.Playbook", on_delete=models.CASCADE, related_name="rollouts", null=True, blank=True, diff --git a/server/apps/tasks/rollout.py b/server/apps/tasks/rollout.py index fb958a7c..36076764 100644 --- a/server/apps/tasks/rollout.py +++ b/server/apps/tasks/rollout.py @@ -7,6 +7,7 @@ import logging import secrets +import uuid from datetime import timedelta from django.db import transaction @@ -192,6 +193,100 @@ def start_rollout( return rollout +def _batch_group_tags(group_tags) -> list[str]: + """The request's ladders, cleaned: stripped, non-empty, de-duplicated + case-insensitively, first spelling kept and order preserved.""" + tags: list[str] = [] + seen: set[str] = set() + for raw in group_tags or (): + tag = str(raw).strip() + if not tag or tag.lower() in seen: + continue + seen.add(tag.lower()) + tags.append(tag) + if len(tags) < 2: + raise ValueError("a parallel rollout needs at least two wave groups") + if len(tags) > 12: + raise ValueError("a parallel rollout takes at most 12 wave groups") + return tags + + +def _batch_host_plan(tags: list[str], host_ids) -> "dict[str, list[str]]": + """Hosts per ladder, each host on exactly one. + + The plan is the whole fleet's — the same one a wave walks — so a host a + higher wave already claimed never shows up here twice. Within that, a host + tagged into two ladders goes to the **first listed** group only. + """ + plan = rollout_wave_plan( + list(PatchWave.objects.filter(enabled=True).order_by("order", "id")) + ) + wanted = ({str(h).strip().lower() for h in host_ids} + if host_ids is not None else None) + taken: set[str] = set() + per_tag: dict[str, list[str]] = {} + for tag in tags: + hosts: list[str] = [] + for wave in _waves_in_group(tag): + for host_id in plan.get(wave.id, []): + host = str(host_id) + low = host.lower() + if low in taken or (wanted is not None and low not in wanted): + continue + taken.add(low) + hosts.append(host) + per_tag[tag] = hosts + return per_tag + + +def start_rollout_batch( + definition=None, + user=None, + failure_threshold_pct: int = 10, + min_results_before_halt: int = 3, + playbook=None, + group_tags=(), + host_ids=None, +) -> list[PatchRollout]: + """Start one rollout per wave group, all at once, sharing a ``batch`` id. + + Every host the request could reach ends up in exactly one of them: a + machine tagged into two ladders belongs to the first listed group, so + parallel ladders never send the same task twice. + + Raises ``ValueError`` before anything is created when fewer than two (or + more than twelve) distinct groups are named, when one of them has no + enabled wave, or when none of them has a machine to send this to. + """ + tags = _batch_group_tags(group_tags) + for tag in tags: + if _first_enabled_wave(tag) is None: + raise ValueError(f"no enabled patch waves tagged {tag!r}") + + per_tag = _batch_host_plan(tags, host_ids) + kept = [(tag, hosts) for tag, hosts in per_tag.items() if hosts] + if not kept: + raise ValueError("none of those groups has a machine to send this to") + + batch = uuid.uuid4() + with transaction.atomic(): + rollouts = [] + for tag, hosts in kept: + rollout = start_rollout( + definition, + user=user, + failure_threshold_pct=failure_threshold_pct, + min_results_before_halt=min_results_before_halt, + playbook=playbook, + wave_group_tag=tag, + host_ids=hosts, + ) + rollout.batch = batch + rollout.save(update_fields=["batch"]) + rollouts.append(rollout) + return rollouts + + def _dispatch_wave(rollout: PatchRollout, spec: dict) -> int: """Dispatch one task per wave host and open a TaskRun for them. @@ -408,6 +503,28 @@ def halt_rollout(rollout: PatchRollout, user=None, reason: str = "") -> None: rollout.save(update_fields=["state", "halted_reason", "halted_by"]) +def halt_batch(batch, user=None, reason: str = "") -> int: + """Halt every still-active rollout in one batch. Returns how many. + + TOTP codes are single-use, so halting a batch of rollouts has to be one + request — that is the whole reason this exists. + """ + with transaction.atomic(): + rollouts = [ + r for r in PatchRollout.objects.filter(batch=batch) + if r.state in ( + PatchRollout.State.PENDING, + PatchRollout.State.RUNNING, + PatchRollout.State.VALIDATING, + ) + ] + for rollout in rollouts: + halt_rollout(rollout, user=user, reason=reason) + if not rollouts: + raise ValueError("nothing in that batch is running") + return len(rollouts) + + def skip_validation(rollout: PatchRollout, user=None) -> None: """End the current wave's validation window now and advance. diff --git a/server/apps/tasks/rollout_serializers.py b/server/apps/tasks/rollout_serializers.py index 55b5f49b..dea964d3 100644 --- a/server/apps/tasks/rollout_serializers.py +++ b/server/apps/tasks/rollout_serializers.py @@ -29,7 +29,7 @@ class Meta: "id", "action_kind", "definition", "definition_name", "playbook", "playbook_name", "target_name", "state", "current_wave", "current_wave_name", "current_wave_order", - "wave_group_tag", + "wave_group_tag", "batch", "failure_threshold_pct", "min_results_before_halt", "halted_reason", "halted_by_name", "resumed_by_name", "started_at", "wave_started_at", "finished_at", diff --git a/server/apps/tasks/rollout_urls.py b/server/apps/tasks/rollout_urls.py index c7fe62e2..3ad67034 100644 --- a/server/apps/tasks/rollout_urls.py +++ b/server/apps/tasks/rollout_urls.py @@ -9,6 +9,8 @@ urlpatterns = [ path("rollouts/", views.rollout_collection, name="rollout-collection"), + path("rollouts/batch//halt/", views.rollout_batch_halt, + name="rollout-batch-halt"), path("rollouts//", views.rollout_detail, name="rollout-detail"), path("rollouts//halt/", views.rollout_halt, name="rollout-halt"), path("rollouts//resume/", views.rollout_resume, name="rollout-resume"), diff --git a/server/apps/tasks/test_rollout_batch.py b/server/apps/tasks/test_rollout_batch.py new file mode 100644 index 00000000..a4356918 --- /dev/null +++ b/server/apps/tasks/test_rollout_batch.py @@ -0,0 +1,326 @@ +"""Parallel wave groups: one request, one rollout per ladder, one batch id. + +Starting "Servers" and "Desktops" side by side used to mean two rollouts and +two TOTP codes, and a host tagged into both ladders got the task twice. A +batch is one request that starts them together and hands every host to exactly +one of them. +""" + +import time +import uuid +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.utils.timezone import now +from rest_framework.test import APIClient + +from apps.accounts.models import Role, UserProfile +from apps.accounts.totp import generate_secret, generate_totp +from apps.hosts.models import Host +from apps.tasks import rollout as rollout_module +from apps.tasks.models import PatchRollout, PatchWave, Task, TaskDefinition +from apps.tasks.rollout import halt_batch, start_rollout_batch +from apps.tasks.spec import parse_and_validate + +PATCH_YAML = ( + "name: Patch OS\nrisk: low\nactions:\n" + " - type: run_command\n params:\n" + " command: apt-get upgrade\n" +) + + +def _definition(owner=None, name="Patch OS"): + return TaskDefinition.objects.create( + owner=owner, name=name, yaml_source=PATCH_YAML, + parsed_spec=parse_and_validate(PATCH_YAML), + ) + + +def _host(name, tags): + return Host.objects.create( + hostname=name, ip_address=f"10.70.0.{abs(hash(name)) % 250 + 1}", + agent_token=f"t-{name}", tags=tags, status=Host.Status.ONLINE, + mode="managed") + + +def _wave(name, order, tags, groups=(), **kw): + return PatchWave.objects.create( + name=name, order=order, tags=tags, group_tags=list(groups), **kw) + + +_real_host_plan = rollout_module._batch_host_plan + + +def _batch(case, tags, host_ids=None): + """``start_rollout_batch``, reporting the host partition computed for + **every** named group in tag order — an emptied group included, which is + what a ladder whose every host went to an earlier group looks like. + + The real call skips a group left without machines before starting it, so a + batch's rollouts alone cannot show which machines it withheld from a later + group: ``["Servers"]`` looks the same whether Desktops was emptied by the + split or simply never matched a host. The partition is what the split + returns, so this reads it where it is decided. + + Returns ``(started_tags, hosts_by_tag)``. + """ + def keep_emptied_groups(tags_in, host_ids_in): + per_tag = _real_host_plan(tags_in, host_ids_in) + for tag in tags_in: + hosts_by_tag[tag] = [str(h) for h in per_tag.get(tag, [])] + per_tag[tag] = hosts_by_tag[tag] + return per_tag + + hosts_by_tag: dict[str, list[str]] = {} + with patch.object(rollout_module, "_batch_host_plan", keep_emptied_groups): + rollouts = rollout_module.start_rollout_batch( + case.definition, user=case.user, group_tags=tags, + host_ids=host_ids) + assert rollouts + return [r.wave_group_tag for r in rollouts], hosts_by_tag + + +class BatchStartTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user("op", password="pw") + self.definition = _definition(self.user) + + def test_each_group_gets_its_own_rollout_in_one_batch(self): + _wave("srv-1", 1, ["srv1"], groups=["Servers"], validation_hours=0) + _wave("srv-2", 2, ["srv2"], groups=["Servers"], validation_hours=0) + _wave("desk-1", 1, ["desk"], groups=["Desktops"], validation_hours=0) + _host("srv-a", ["srv1"]) + _host("srv-b", ["srv2"]) + _host("desk-a", ["desk"]) + rollouts = start_rollout_batch( + self.definition, user=self.user, group_tags=["Servers", "Desktops"]) + self.assertEqual([r.wave_group_tag for r in rollouts], + ["Servers", "Desktops"]) + self.assertEqual(len({r.batch for r in rollouts}), 1) + self.assertIsNotNone(rollouts[0].batch) + for r in rollouts: + self.assertEqual(r.state, PatchRollout.State.RUNNING) + + def test_a_host_on_two_ladders_goes_to_the_first_listed_group(self): + # One Canary wave is the first rung of **both** ladders (the case the + # model's group_tags note describes), so the fleet-wide plan hands its + # machine to that one wave — and both group rollouts walk it. Only the + # batch's split stops the machine being patched twice. + canary = _host("canary-1", ["canary"]) + srv = _host("srv-a", ["srv"]) + desk = _host("desk-a", ["desk"]) + _wave("canary", 1, ["canary"], groups=["Servers", "Desktops"], validation_hours=0) + _wave("srv-2", 2, ["srv"], groups=["Servers"], validation_hours=0) + _wave("desk-2", 2, ["desk"], groups=["Desktops"], validation_hours=0) + started, hosts = _batch(self, ["Desktops", "Servers"]) + self.assertEqual(started, ["Desktops", "Servers"]) + self.assertEqual(hosts["Desktops"], [str(canary.id), str(desk.id)]) + self.assertEqual(hosts["Servers"], [str(srv.id)]) + self.assertEqual(Task.objects.filter(host=canary).count(), 1) + def test_unknown_group_refuses_the_whole_batch(self): + _wave("srv-1", 1, ["srv1"], groups=["Servers"], validation_hours=0) + _host("srv-a", ["srv1"]) + with self.assertRaises(ValueError) as ctx: + start_rollout_batch(self.definition, user=self.user, + group_tags=["Servers", "Nope"]) + self.assertIn("Nope", str(ctx.exception)) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_a_group_with_no_machines_left_is_skipped(self): + # desk-canary is the only rung the Desktops ladder has, and the one + # machine matching it goes to Servers, which is listed first. Desktops + # is left with no machine at all — no rollout for it, nothing dispatched + # from it — and the batch is Servers walking one ladder. + _host("canary-1", ["canary"]) + _wave("srv-canary", 1, ["canary"], groups=["Servers"], + validation_hours=0) + _wave("desk-canary", 1, ["canary"], groups=["Desktops"], + validation_hours=0) + started, hosts = _batch(self, ["Servers", "Desktops"]) + self.assertEqual(started, ["Servers"]) + self.assertEqual(hosts["Servers"], [str(Host.objects.get(hostname="canary-1").id)]) + self.assertEqual(hosts["Desktops"], []) + self.assertEqual(PatchRollout.objects.count(), 1) + self.assertEqual(Task.objects.count(), 1) + + def test_no_group_left_with_a_machine_is_refused(self): + # One machine sits on both ladders, and the request narrowed to a host + # nobody has: no group is left with anything to send this to. + _host("dual-1", ["dual"]) + _wave("dual", 1, ["dual"], groups=["Servers", "Desktops"], + validation_hours=0) + with self.assertRaises(ValueError) as ctx: + start_rollout_batch(self.definition, user=self.user, + group_tags=["Servers", "Desktops"], + host_ids=[str(uuid.uuid4())]) + self.assertIn("none of those groups", str(ctx.exception)) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_fewer_than_two_groups_is_refused(self): + _wave("srv-1", 1, ["srv1"], groups=["Servers"], validation_hours=0) + _host("srv-a", ["srv1"]) + with self.assertRaises(ValueError) as ctx: + start_rollout_batch(self.definition, user=self.user, + group_tags=["Servers"]) + self.assertIn("at least two", str(ctx.exception)) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_the_same_group_listed_twice_counts_once(self): + _wave("srv-1", 1, ["srv1"], groups=["Servers"], validation_hours=0) + _wave("desk-1", 1, ["desk"], groups=["Desktops"], validation_hours=0) + _host("srv-a", ["srv1"]) + _host("desk-a", ["desk"]) + with self.assertRaises(ValueError): + start_rollout_batch(self.definition, user=self.user, + group_tags=["Servers", " servers ", "Servers"]) + + def test_the_batch_halt_stops_every_rollout(self): + _wave("srv-1", 1, ["srv1"], groups=["Servers"], validation_hours=0) + _wave("desk-1", 1, ["desk"], groups=["Desktops"], validation_hours=0) + _host("srv-a", ["srv1"]) + _host("desk-a", ["desk"]) + rollouts = start_rollout_batch( + self.definition, user=self.user, group_tags=["Servers", "Desktops"]) + self.assertEqual(halt_batch(rollouts[0].batch, user=self.user, + reason="stop"), 2) + for r in rollouts: + r.refresh_from_db() + self.assertEqual(r.state, PatchRollout.State.HALTED) + with self.assertRaises(ValueError): + halt_batch(rollouts[0].batch, user=self.user) + + +class BatchApiTests(TestCase): + def setUp(self): + self.client = APIClient() + self.operator = get_user_model().objects.create_user("op", password="pw") + self.secret = generate_secret() + profile = UserProfile.objects.create(user=self.operator, role=Role.OPERATOR) + profile.totp_secret = self.secret + profile.totp_confirmed_at = now() + profile.save() + self.client.force_authenticate(self.operator) + + self.definition = _definition(self.operator) + _wave("srv-canary", 1, ["srv1"], groups=["Servers"], validation_hours=0) + _wave("desk-canary", 1, ["desk"], groups=["Desktops"], validation_hours=0) + _host("srv-1", ["srv1"]) + _host("desk-1", ["desk"]) + + def _totp(self): + return generate_totp(self.secret) + + def _post_batch(self, tags, **extra): + return self.client.post( + "/api/v1/rollouts/", + {"definition_id": str(self.definition.id), + "wave_group_tags": tags, "totp": self._totp(), **extra}, + format="json", + ) + + def test_api_starts_a_batch(self): + resp = self._post_batch(["Servers", "Desktops"]) + self.assertEqual(resp.status_code, 201, getattr(resp, "data", None)) + body = resp.json() + self.assertEqual(len(body["rollouts"]), 2) + self.assertEqual({r["batch"] for r in body["rollouts"]}, + {body["batch"]}) + + def test_api_refuses_a_batch_without_totp(self): + resp = self.client.post( + "/api/v1/rollouts/", + {"definition_id": str(self.definition.id), + "wave_group_tags": ["Servers", "Desktops"]}, + format="json", + ) + self.assertEqual(resp.status_code, 401) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_api_refuses_a_viewer(self): + viewer = get_user_model().objects.create_user("view", password="pw") + UserProfile.objects.create(user=viewer, role=Role.VIEWER) + self.client.force_authenticate(viewer) + resp = self.client.post( + "/api/v1/rollouts/", + {"definition_id": str(self.definition.id), + "wave_group_tags": ["Servers", "Desktops"], "totp": "123456"}, + format="json", + ) + self.assertEqual(resp.status_code, 403) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_api_refuses_wave_group_tags_that_are_not_a_list_of_strings(self): + resp = self._post_batch("Servers") + self.assertEqual(resp.status_code, 400) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_api_refuses_both_group_fields_at_once(self): + resp = self._post_batch(["Servers", "Desktops"], wave_group_tag="Servers") + self.assertEqual(resp.status_code, 400) + self.assertEqual(PatchRollout.objects.count(), 0) + + def test_single_group_start_is_unchanged(self): + resp = self.client.post( + "/api/v1/rollouts/", + {"definition_id": str(self.definition.id), + "wave_group_tag": "Servers", "totp": self._totp()}, + format="json", + ) + self.assertEqual(resp.status_code, 201, getattr(resp, "data", None)) + body = resp.json() + self.assertIn("id", body) + self.assertNotIn("rollouts", body) + self.assertIsNone(body["batch"]) + + def test_batch_halt_halts_every_running_rollout_with_one_code(self): + resp = self._post_batch(["Servers", "Desktops"]) + self.assertEqual(resp.status_code, 201, getattr(resp, "data", None)) + batch = resp.json()["batch"] + + admin = get_user_model().objects.create_user( + "root", password="x", is_staff=True, is_superuser=True) + a_secret = generate_secret() + prof = UserProfile.objects.create(user=admin) + prof.totp_secret = a_secret + prof.totp_confirmed_at = now() + prof.save() + self.client.force_authenticate(admin) + + # The replay guard burns a code for its whole window, so the halt uses + # a code from a different step than the one that started the batch. + code = generate_totp(a_secret, at=time.time() + 30) + halted = self.client.post( + f"/api/v1/rollouts/batch/{batch}/halt/", + {"reason": "recall", "totp": code}, format="json", + ) + self.assertEqual(halted.status_code, 200, getattr(halted, "data", None)) + self.assertEqual(halted.json(), {"halted": 2}) + for rollout in PatchRollout.objects.filter(batch=batch): + self.assertEqual(rollout.state, PatchRollout.State.HALTED) + self.assertEqual(rollout.halted_reason, "recall") + + def test_batch_halt_needs_admin(self): + resp = self._post_batch(["Servers", "Desktops"]) + batch = resp.json()["batch"] + denied = self.client.post( + f"/api/v1/rollouts/batch/{batch}/halt/", + {"totp": generate_totp(self.secret, at=time.time() + 30)}, + format="json", + ) + self.assertEqual(denied.status_code, 403) + + def test_batch_halt_404_for_an_unknown_batch(self): + admin = get_user_model().objects.create_user( + "root", password="x", is_staff=True, is_superuser=True) + a_secret = generate_secret() + prof = UserProfile.objects.create(user=admin) + prof.totp_secret = a_secret + prof.totp_confirmed_at = now() + prof.save() + self.client.force_authenticate(admin) + resp = self.client.post( + f"/api/v1/rollouts/batch/{uuid.uuid4()}/halt/", + {"totp": generate_totp(a_secret)}, format="json", + ) + self.assertEqual(resp.status_code, 404) diff --git a/server/apps/tasks/views.py b/server/apps/tasks/views.py index 6bcc3036..6871a297 100644 --- a/server/apps/tasks/views.py +++ b/server/apps/tasks/views.py @@ -32,9 +32,11 @@ FAILURE_STATES, _emit_hunt_text_requested, _hunt_text_step_ids, + halt_batch, halt_rollout, resume_rollout, start_rollout, + start_rollout_batch, ) from .rollout_serializers import PatchRolloutSerializer from .serializers import ( @@ -1864,6 +1866,18 @@ def rollout_collection(request): if request.method == "POST": from apps.playbooks.models import Playbook + raw_tags = request.data.get("wave_group_tags") + if raw_tags is not None and not ( + isinstance(raw_tags, list) and all(isinstance(t, str) for t in raw_tags) + ): + return Response({"detail": "wave_group_tags must be a list of strings"}, + status=400) + if raw_tags is not None and (request.data.get("wave_group_tag") or "").strip(): + return Response( + {"detail": "send either wave_group_tag or wave_group_tags, not both"}, + status=400, + ) + definition_id = request.data.get("definition_id") playbook_id = request.data.get("playbook_id") if bool(definition_id) == bool(playbook_id): @@ -1886,6 +1900,23 @@ def rollout_collection(request): if error: return Response({"detail": error}, status=401) + if raw_tags is not None: + try: + rollouts = start_rollout_batch( + definition, + playbook=playbook, + user=request.user, + group_tags=raw_tags, + failure_threshold_pct=int(request.data.get("failure_threshold_pct", 10)), + min_results_before_halt=int(request.data.get("min_results_before_halt", 3)), + ) + except (ValueError, TypeError) as exc: + return Response({"detail": str(exc)}, status=400) + return Response({ + "batch": str(rollouts[0].batch), + "rollouts": [_rollout_response(r) for r in rollouts], + }, status=201) + try: rollout = start_rollout( definition, @@ -1944,6 +1975,24 @@ def rollout_halt(request, rollout_id): return Response(_rollout_response(rollout)) +@api_view(["POST"]) +@permission_classes([IsAuthenticated, IsAdmin]) +def rollout_batch_halt(request, batch): + """Halt every running rollout in one batch. One request because one TOTP + code is single-use — three rollouts would otherwise need three codes.""" + if not PatchRollout.objects.filter(batch=batch).exists(): + return Response({"detail": "Not found"}, status=404) + error = _verify_confirmation(request.user, request.data) + if error: + return Response({"detail": error}, status=401) + reason = str(request.data.get("reason") or "").strip() + try: + halted = halt_batch(batch, user=request.user, reason=reason) + except ValueError as exc: + return Response({"detail": str(exc)}, status=400) + return Response({"halted": halted}) + + @api_view(["POST"]) @permission_classes([IsAuthenticated]) def rollout_resume(request, rollout_id): From c38c0f32220190b2659914eaaf664d8d526ab4c4 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 16:36:13 -0400 Subject: [PATCH 50/68] feat(rollouts): pick several wave groups, and see a parallel batch as lanes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The start dialog picks wave groups from chips instead of a typed tag: none is every enabled wave, one is that ladder, two or more start a parallel batch. A picked chip shows its position, because pick order decides which group gets a machine that sits on two ladders. Rollouts that share a batch are one card with a lane per group, side by side — 1 2 | 1 2 | 1 2 3 — each lane listing its waves with its own Halt/Resume, and a Halt all that stops the batch with one TOTP code. Lanes are a tinted surface, not a striped edge, and stack under 640px. Phase QA-22. --- .../tasks/test_qa22_parallel_groups_ui.py | 141 +++++++++++ server/static/css/vigil.css | 32 +++ server/static/js/vigil-rollout.js | 233 ++++++++++++++---- server/static/js/vigil-tags.js | 3 +- server/templates/base.html | 6 +- 5 files changed, 369 insertions(+), 46 deletions(-) create mode 100644 server/apps/tasks/test_qa22_parallel_groups_ui.py diff --git a/server/apps/tasks/test_qa22_parallel_groups_ui.py b/server/apps/tasks/test_qa22_parallel_groups_ui.py new file mode 100644 index 00000000..a49d1ba3 --- /dev/null +++ b/server/apps/tasks/test_qa22_parallel_groups_ui.py @@ -0,0 +1,141 @@ +"""Rollout start dialog and batch lanes: what the browser will run. + +The start dialog picks several ladders and the list draws the batch that start +returns as one card with a lane per group. The page re-renders from template +strings every five seconds, so the checkable properties are the markup those +strings produce and the payload the dialog POSTs. +""" + +import re +from pathlib import Path + +from django.test import SimpleTestCase + +ROOT = Path(__file__).resolve().parents[2] +JS = ROOT / "static/js/vigil-rollout.js" +CSS = ROOT / "static/css/vigil.css" +BASE = ROOT / "templates/base.html" + + +def _code_only(src: str) -> str: + """JS with comments blanked, so a comment that *names* a banned attribute + isn't reported as the offence. Quotes are kept intact — the tokens are + searched for inside template strings.""" + out = [] + i, n = 0, len(src) + in_line = in_block = in_str = False + quote = "" + while i < n: + ch = src[i] + nxt = src[i + 1] if i + 1 < n else "" + if in_line: + if ch == "\n": + in_line = False + out.append(ch) + elif in_block: + if ch == "*" and nxt == "/": + in_block = False + out.append(" ") + i += 2 + continue + out.append("\n" if ch == "\n" else " ") + elif in_str: + out.append(ch) + if ch == "\\": + out.append(nxt) + i += 2 + continue + if ch == quote: + in_str = False + elif ch == "/" and nxt == "/": + in_line = True + out.append(" ") + i += 2 + continue + elif ch == "/" and nxt == "*": + in_block = True + out.append(" ") + i += 2 + continue + else: + out.append(ch) + if ch in "\"'`": + in_str = True + quote = ch + i += 1 + return "".join(out) + + +def _function(code: str, name: str) -> str: + """The named function's source, up to the next top-level ``function``.""" + start = code.index(f"function {name}") + end = code.find("\nfunction ", start + len(f"function {name}")) + return code[start:] if end == -1 else code[start:end] + + +class StartDialogTests(SimpleTestCase): + + def test_start_picks_several_groups(self): + code = _code_only(JS.read_text()) + for needed in ("/api/v1/wave-groups/", "data-rlt-group", "aria-pressed", + "wave_group_tags"): + self.assertIn(needed, code) + self.assertNotIn("getElementById('rollout-start-group')", code) + + base = BASE.read_text() + self.assertIn('id="rollout-start-groups"', base) + self.assertNotIn('id="rollout-start-group"', base) + + def test_payload_follows_the_number_of_picks(self): + """No pick = every enabled wave, one = that ladder, two or more = a + parallel batch. Always sending wave_group_tag is the bug: the second + group on would be dropped and one ladder started alone.""" + submit = _function(_code_only(JS.read_text()), "submitRolloutStart") + self.assertIn("wave_group_tag: picked[0]", submit) + self.assertIn("wave_group_tags: picked.slice()", submit) + + +class BatchLaneTests(SimpleTestCase): + + def test_batches_draw_as_lanes(self): + code = _code_only(JS.read_text()) + self.assertIn("function _batchCard(", code) + card = _function(code, "_batchCard") + self.assertIn("rlt-lanes", card) + self.assertIn("rlt-lane", card) + self.assertIn("r.batch", _function(code, "_groupRollouts")) + # The card itself must not open a rollout — a batch has N of them to + # open, and each lane head carries its own data-rlt-open. + self.assertNotIn("data-rlt-open", card) + self.assertIn("data-rlt-open", _function(code, "_batchLane")) + + def test_halt_all_uses_the_batch_endpoint_once(self): + code = _code_only(JS.read_text()) + self.assertIn("/api/v1/rollouts/batch/", code) + self.assertIn("halt-batch", code) + self.assertEqual(1, _function(code, "confirmRolloutAction") + .count("/api/v1/rollouts/batch/")) + + +class LaneStyleTests(SimpleTestCase): + + def _lane_rules(self, css: str) -> str: + return "\n".join(re.findall(r"\.rlt-lane[\w-]*\s*\{[^}]*\}", css)) + + def test_lanes_have_no_side_stripe(self): + rules = self._lane_rules(CSS.read_text()) + # tinted as a whole surface (--s2: the card itself is --s1, so a --s1 + # lane would not show against it) + self.assertRegex(rules, r"background: var\(--s\d\)") + for banned in ("border-left", "border-right"): + self.assertNotIn(banned, rules) + + def test_lanes_stack_under_640px(self): + css = CSS.read_text() + self.assertIn(".rlt-lanes { display: grid; grid-template-columns: " + "repeat(auto-fit, minmax(260px, 1fr));", css) + self.assertRegex( + css, + r"@media \(max-width: 640px\) \{[^}]*\.rlt-lanes \{ " + r"grid-template-columns: minmax\(0, 1fr\); \}", + ) diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index d8c0685d..898e5312 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -4019,3 +4019,35 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } @media (max-width: 980px) { .sted-grid { grid-template-columns: minmax(0, 1fr); } } + +/* ── Parallel wave groups (QA-22) ─────────────────────────────────────── */ +/* The start dialog offers the ladders as toggle chips. Pick order matters — it + is the tie-break for a machine tagged into two of them — so a picked chip + carries its position. */ +.rlt-groups { display: flex; flex-wrap: wrap; gap: 6px; } +.rlt-group-chip { border: 0; cursor: pointer; font-family: inherit; gap: 6px; } +.rlt-group-chip-count { color: var(--text-3); font-weight: 400; } +.rlt-group-chip[aria-pressed="true"] { background: var(--lavender); color: var(--lavender-ink); } +.rlt-group-chip[aria-pressed="true"] .rlt-group-chip-count { color: var(--lavender-ink); opacity: .75; } +.rlt-group-chip-order { display: none; font-size: 10px; font-weight: 700; + background: var(--on-accent); color: var(--lavender-ink); border-radius: 50%; + width: 16px; height: 16px; align-items: center; justify-content: center; } +.rlt-group-chip[aria-pressed="true"] .rlt-group-chip-order { display: inline-flex; } + +/* A batch is one card with a column per group. The lane is a tinted surface, + never a striped edge. */ +.rlt-lanes { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); + gap: 12px; margin-top: 10px; } +.rlt-lane { background: var(--s2); border-radius: var(--r-lg); padding: 10px 12px; min-width: 0; } +.rlt-lane-head { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; cursor: pointer; } +.rlt-lane-head strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.rlt-lane-head .rlt-wave-count { flex: 1; } +/* A lane is ~260px: the counts move under the wave name, so the dot, name, + failed chip and Machines button keep one row. */ +.rlt-lane-body .rlt-wave { grid-template-columns: 14px minmax(0, 1fr) auto auto; row-gap: 0; } +.rlt-lane-body .rlt-wave-count { grid-column: 2 / -1; grid-row: 2; } +.rlt-lane-body .rlt-wave-tags { display: none; } + +@media (max-width: 640px) { + .rlt-lanes { grid-template-columns: minmax(0, 1fr); } +} diff --git a/server/static/js/vigil-rollout.js b/server/static/js/vigil-rollout.js index 077c0b9b..05d8fac0 100644 --- a/server/static/js/vigil-rollout.js +++ b/server/static/js/vigil-rollout.js @@ -4,7 +4,8 @@ // Depends on: vigil-utils.js (apiJson, showToast, navigateTo, el) // API: /api/v1/rollouts/ (GET list, POST start), // /api/v1/rollouts// (GET), /halt/, /resume/, -// /skip-validation/ (POST, TOTP). +// /skip-validation/ (POST, TOTP), /api/v1/rollouts/batch//halt/ +// (POST, TOTP), /api/v1/wave-groups/ (GET). // The server's beat task (tasks.advance_rollouts, every 5 min) advances the // state machine; this page only reads and sends operator actions. @@ -29,8 +30,12 @@ const _rolloutState = { items: [], pollTimer: null, pendingAction: null, // { title, message, fn(totp) } for the confirm modal + groups: [], // picked wave-group tags, in pick order — order decides where a host on two ladders goes + groupChoices: [], // the enabled groups GET /api/v1/wave-groups/ offered on open }; +const RLT_ACTIVE = ['pending', 'running', 'validating']; + function _waveDot(status) { const color = WAVE_STATUS_COLOR[status] || 'var(--text-3)'; return ``; @@ -41,6 +46,9 @@ function _fmtTs(ts) { return new Date(ts).toLocaleString([], { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }); } +/* The wave rows: six cells always, so the columns line up with the next card. + An empty slot renders an empty ; `''` collapsed to nothing and + everything after the gap shifted. */ function _waveProgress(r) { return (r.waves || []).map(wave => { const color = WAVE_STATUS_COLOR[wave.status] || 'var(--text-3)'; @@ -181,19 +189,80 @@ function _filterRollouts() { (w.tags || []).some(t => (t || '').toLowerCase().includes(q)))); } +// Rollouts started together share a batch id, so they are one entry in the +// list — a card with a lane per group, `1 2 | 1 2 | 1 2 3`. Members are taken +// from the unfiltered list: a search that matches one lane shows the whole +// batch, because one lane alone would hide what that group ran beside. +function _groupRollouts(items) { + const members = new Map(); + for (const r of _rolloutState.items) { + if (!r.batch) continue; + if (!members.has(r.batch)) members.set(r.batch, []); + members.get(r.batch).push(r); + } + const entries = new Map(); + for (const r of items) { + if (!r.batch) { entries.set(r.id, [r]); continue; } + if (!entries.has(r.batch)) entries.set(r.batch, members.get(r.batch)); + } + return [...entries.values()]; +} + +// One card, a column per group. The card itself is not openable — a batch has +// N rollouts to open, so each lane head carries its own data-rlt-open. +function _batchCard(rs) { + const batch = rs[0].batch; + const anyActive = rs.some(r => RLT_ACTIVE.includes(r.state)); + const overall = anyActive + ? (rs.find(r => r.state === 'running' || r.state === 'validating') || rs[0]).state + : (rs.some(r => r.state === 'halted') ? 'halted' + : (rs.every(r => r.state === 'completed') ? 'completed' : rs[0].state)); + // Same skeleton as _rolloutCard (def-card > rlt-body > head, then content): + // def-card is a flex row, so anything outside rlt-body sits beside it. + return `
+
+
+ ${escHtml(rs[0].target_name || rs[0].definition_name || rs[0].playbook_name || '(deleted)')}${rs[0].action_kind === 'playbook' ? ' playbook' : ''} + parallel · ${rs.length} groups + ${escHtml(overall)} + ${escHtml(rs[0].created_by_name || '')} · started ${_fmtTs(rs[0].started_at)} + ${anyActive ? `` : ''} +
+
${rs.map(_batchLane).join('')}
+
+
`; +} + +function _batchLane(r) { + const actions = r.state === 'halted' + ? `` + : (RLT_ACTIVE.includes(r.state) + ? `` + : ''); + return `
+
+ ${escHtml(r.wave_group_tag || 'Ungrouped')} + ${escHtml(r.state)} + ${r.current_wave_name ? `${escHtml(r.current_wave_name)}` : ''} + ${actions} +
+
${_waveProgress(r)}
+
`; +} + function _renderRollouts() { const box = document.getElementById('rollout-list'); if (!box) return; const items = _filterRollouts(); - { - if (!items.length) { - box.innerHTML = `
-
${_rolloutState.items.length ? 'No rollouts match that search' : 'No rollouts yet'}
-
${_rolloutState.items.length ? 'Clear the search to see them all.' : 'Start one from a task definition — it fans out wave by wave and halts itself on failure.'}
-
`; - } else { - box.innerHTML = items.map(_rolloutCard).join(''); - } + if (!items.length) { + box.innerHTML = `
+
${_rolloutState.items.length ? 'No rollouts match that search' : 'No rollouts yet'}
+
${_rolloutState.items.length ? 'Clear the search to see them all.' : 'Start one from a task definition — it fans out wave by wave and halts itself on failure.'}
+
`; + } else { + box.innerHTML = _groupRollouts(items) + .map(group => (group.length > 1 ? _batchCard(group) : _rolloutCard(group[0]))) + .join(''); } _bindWaveHostButtons(box); } @@ -240,20 +309,61 @@ async function openRolloutStart() { document.getElementById('rollout-start-def').value = ''; document.getElementById('rollout-start-def-label').textContent = 'Choose a task or playbook…'; document.getElementById('rollout-start-totp').value = ''; - const groupInput = document.getElementById('rollout-start-group'); - if (groupInput) { - groupInput.value = ''; - if (!groupInput.dataset.hintBound) { - groupInput.dataset.hintBound = '1'; - groupInput.addEventListener('input', _refreshRolloutGroupHint); - groupInput.addEventListener('change', _refreshRolloutGroupHint); - } - } + // The chips need no reset beyond clearing the pick: they are re-fetched and + // repainted on every open. + _rolloutState.groups = []; + await _renderRolloutGroupChips(); _refreshRolloutGroupHint(); document.getElementById('rollout-start-overlay').classList.add('open'); modal.classList.add('open'); } +/* The ladders to pick from, fetched on open so a wave added in the waves tab + shows up without a reload. A group with no enabled wave cannot start a + rollout, so it isn't offered. */ +async function _renderRolloutGroupChips() { + const box = document.getElementById('rollout-start-groups'); + if (!box) return; + _rolloutState.groupChoices = []; + box.innerHTML = 'Loading wave groups…'; + try { + const data = await apiJson('/api/v1/wave-groups/'); + _rolloutState.groupChoices = (data.groups || []).filter(g => g.enabled_waves > 0); + _paintRolloutGroupChips(); + } catch (e) { + box.innerHTML = `Could not load wave groups: ${escHtml(e.message || 'Request failed')}`; + } +} + +function _paintRolloutGroupChips() { + const box = document.getElementById('rollout-start-groups'); + if (!box) return; + const groups = _rolloutState.groupChoices || []; + if (!groups.length) { + box.innerHTML = 'No wave groups defined — every enabled wave is one ladder.'; + return; + } + box.innerHTML = groups.map(g => { + const at = _rolloutState.groups.indexOf(g.tag); + const waves = g.enabled_waves === 1 ? '1 wave' : `${g.enabled_waves} waves`; + return ``; + }).join(''); +} + +// Pick order is the tie-break for a machine tagged into two of the picked +// groups, so a picked chip shows its position. +delegateClick('[data-rlt-group]', (el) => { + const tag = el.dataset.rltGroup; + const at = _rolloutState.groups.indexOf(tag); + if (at >= 0) _rolloutState.groups.splice(at, 1); + else _rolloutState.groups.push(tag); + _paintRolloutGroupChips(); + _refreshRolloutGroupHint(); +}); + function pickRolloutTarget() { openPicker({ type: 'rollout_target', @@ -279,24 +389,33 @@ async function submitRolloutStart() { const btn = document.getElementById('rollout-start-submit'); if (!sel.value) { showToast('Pick something to roll out first', 'error'); return; } if (!/^\d{6}$/.test(totp)) { showToast('Enter the 6-digit TOTP code', 'error'); return; } + const picked = _rolloutState.groups; btn.disabled = true; try { const r = await apiJson('/api/v1/rollouts/', { method: 'POST', // The option value is "task:" or "playbook:"; the API wants // exactly one of the two id fields and rejects both or neither. + // Likewise exactly one of wave_group_tag / wave_group_tags: one group is + // an ordinary rollout, two or more are a parallel batch. body: JSON.stringify({ ...(sel.value.startsWith('playbook:') ? { playbook_id: sel.value.slice('playbook:'.length) } : { definition_id: sel.value.replace(/^task:/, '') }), failure_threshold_pct: threshold, min_results_before_halt: minResults, - wave_group_tag: (document.getElementById('rollout-start-group')?.value || '').trim(), + ...(picked.length === 1 ? { wave_group_tag: picked[0] } + : picked.length > 1 ? { wave_group_tags: picked.slice() } + : {}), totp, }), }); closeRolloutStart(); - showToast(`Rollout started — wave 1 dispatched (${r.waves?.[0]?.tasks_total ?? 0} host(s))`, 'success'); + if (r && r.batch) { + showToast(`Started ${r.rollouts.length} rollouts in parallel`, 'success'); + } else { + showToast(`Rollout started — wave 1 dispatched (${r.waves?.[0]?.tasks_total ?? 0} host(s))`, 'success'); + } refreshRollouts(); } catch (e) { const msg = e.message || 'Request failed'; @@ -319,19 +438,30 @@ function promptRolloutAction(btn, kind) { const overlay = document.getElementById('rollout-action-overlay'); if (!modal || !overlay) return; const rolloutId = btn ? btn.dataset.rlt : null; + const batch = btn ? btn.dataset.rltBatch : null; + const card = btn ? btn.closest('.rlt-batch') : null; + const lane = btn ? btn.closest('.rlt-lane') : null; const r = _rolloutState.items.find(x => x.id === rolloutId); - const name = r ? (r.target_name || r.definition_name || r.playbook_name) : 'the rollout'; + // Inside a lane the group's own name reads better than the shared target. + const name = lane && lane.querySelector('strong') + ? lane.querySelector('strong').textContent + : (r ? (r.target_name || r.definition_name || r.playbook_name) : 'the rollout'); + const lanes = card ? card.querySelectorAll('.rlt-lane').length : 0; + const batchName = (card && card.querySelector('strong')?.textContent) || 'the rollout'; const title = kind === 'halt' ? 'Halt rollout' : kind === 'resume' ? 'Resume rollout' + : kind === 'halt-batch' ? 'Halt every group' : 'Skip the validation window'; const message = kind === 'halt' ? `Stop ${name} now? The current wave keeps running; nothing new is dispatched. An admin TOTP is required.` : kind === 'resume' ? `Restart ${name} at its current wave? Failed tasks on the wave are re-queued and the gate re-evaluates.` - : `End this wave's validation window for ${name} and move to the next wave now? ` - + `The window exists so a slow failure has time to show up, so only do this if you have ` - + `checked the wave yourself. The failure gate still applies.`; - _rolloutState.pendingAction = { title, message, rolloutId, kind }; + : kind === 'halt-batch' + ? `Stop all ${lanes} groups of ${batchName} now? Every group keeps its current wave; nothing new is dispatched. An admin TOTP is required.` + : `End this wave's validation window for ${name} and move to the next wave now? ` + + `The window exists so a slow failure has time to show up, so only do this if you have ` + + `checked the wave yourself. The failure gate still applies.`; + _rolloutState.pendingAction = { title, message, rolloutId, batch, kind }; document.getElementById('rollout-action-title').textContent = title; document.getElementById('rollout-action-message').textContent = message; document.getElementById('rollout-action-totp').value = ''; @@ -356,15 +486,27 @@ async function confirmRolloutAction() { if (!/^\d{6}$/.test(totp)) { showToast('Enter the 6-digit TOTP code', 'error'); return; } btn.disabled = true; try { - await apiJson(`/api/v1/rollouts/${pending.rolloutId}/${pending.kind}/`, { - method: 'POST', - body: JSON.stringify({ totp }), - }); + let note; + if (pending.kind === 'halt-batch') { + // One request, one code: TOTP codes are single-use, so halting a batch + // group by group would need a fresh code per group. + const res = await apiJson(`/api/v1/rollouts/batch/${encodeURIComponent(pending.batch)}/halt/`, { + method: 'POST', + body: JSON.stringify({ totp }), + }); + note = `Halted ${res.halted} rollouts`; + } else { + await apiJson(`/api/v1/rollouts/${pending.rolloutId}/${pending.kind}/`, { + method: 'POST', + body: JSON.stringify({ totp }), + }); + note = pending.kind === 'halt' ? 'Rollout halted' + : pending.kind === 'resume' ? 'Rollout resumed' + : 'Validation window skipped — moving to the next wave'; + } closeRolloutAction(); closeRolloutDetail(); - showToast(pending.kind === 'halt' ? 'Rollout halted' - : pending.kind === 'resume' ? 'Rollout resumed' - : 'Validation window skipped — moving to the next wave', 'success'); + showToast(note, 'success'); refreshRollouts(); } catch (e) { const msg = e.message || 'Request failed'; @@ -443,6 +585,7 @@ async function openRolloutDetail(rolloutId) {
${_waveProgress(r)}
${_detailRow('What is rolling out', (r.target_name || '') + (r.action_kind === 'playbook' ? ' (playbook)' : ' (task)'))} ${_detailRow('State', r.state)} + ${r.batch && r.wave_group_tag ? _detailRow('Wave group', r.wave_group_tag) : ''} ${_detailRow('Current wave', r.current_wave_name)} ${_detailRow('Failure gate', gate)} ${_detailRow('Started', _fmtTs(r.started_at))} @@ -494,23 +637,29 @@ document.addEventListener('keydown', (e) => { }); -/* The rollout picker is the one place the blast radius is chosen, so it says - how many waves and machines the named group covers before you start it. */ +/* The picker is the one place the blast radius is chosen, so it says what the + pick covers before you start it — and warns that a machine on two of the + picked ladders goes with the first group you picked. */ async function _refreshRolloutGroupHint() { - const input = document.getElementById('rollout-start-group'); const hint = document.getElementById('rollout-start-group-hint'); - if (!input || !hint) return; - const tag = (input.value || '').trim(); - if (!tag) { - hint.textContent = 'Blank sends it to every enabled wave, in order.'; + if (!hint) return; + const picked = _rolloutState.groups; + if (!picked.length) { + hint.textContent = 'Every enabled wave, in order.'; + return; + } + if (picked.length > 1) { + hint.textContent = `${picked.length} groups in parallel — each walks its own waves; ` + + 'nothing waits for another group. A machine on more than one picked group ' + + 'goes with the first one you picked.'; return; } try { - const waves = await apiJson(`/api/v1/waves/?group=${encodeURIComponent(tag)}`); + const waves = await apiJson(`/api/v1/waves/?group=${encodeURIComponent(picked[0])}`); const live = waves.filter(w => w.enabled); if (!live.length) { hint.innerHTML = `No enabled wave is tagged ` - + `${escHtml(tag)} — the rollout would be refused.`; + + `${escHtml(picked[0])} — the rollout would be refused.`; return; } const machines = live.reduce((n, w) => n + (w.exclusive_host_count || 0), 0); diff --git a/server/static/js/vigil-tags.js b/server/static/js/vigil-tags.js index e76b36a1..58071d8b 100644 --- a/server/static/js/vigil-tags.js +++ b/server/static/js/vigil-tags.js @@ -212,9 +212,10 @@ document.addEventListener('DOMContentLoaded', () => { }); // Every tag field in the app, in one place so none gets forgotten. + // (The rollout start dialog picks wave groups from chips in vigil-rollout.js, + // so it has no tag field to wire.) attachTagPicker('wave-tags', { title: 'Machines in this wave are tagged' }); attachTagPicker('wave-group-tags', { title: 'Ladders this wave belongs to' }); - attachTagPicker('rollout-start-group', { title: 'Send this to the wave group' }); attachTagPicker('bl-tags', { title: 'Machines this playbook targets' }); attachTagPicker('auto-event-tags', { title: 'Only fire for hosts tagged' }); attachTagPicker('auto-target-tags', { title: 'Run on hosts tagged' }); diff --git a/server/templates/base.html b/server/templates/base.html index 89ec9872..c13b9c5b 100644 --- a/server/templates/base.html +++ b/server/templates/base.html @@ -498,9 +498,9 @@
Tasks that need someone to fill in an input cannot be rolled out unattended — a rollout runs with nobody watching.
- - -
Send this to one ladder only. The rollout walks that group's waves in order and never touches a machine outside it.
+ +
+
Pick one ladder, or several to run them in parallel. The rollout walks a group's waves in order and never touches a machine outside it.
From 5de25492c8cd61f6c8498de9658364d704ed0807 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 16:51:11 -0400 Subject: [PATCH 51/68] test: six tests no longer depend on which tests ran before them in the process The parallel runner failed now and then, and hid which test failed behind "cannot pickle 'traceback' object". Two pieces of per-process state leaked between tests in a worker: - SetupRedirectMiddleware sends every page to /setup/ until it has seen an account, then remembers that for the process. Tests that created no user passed only when an earlier test had already set the flag. Found all of them by running the suite with the flag uncached: the Civil SSO disabled and login-start tests and three public status page tests. Each now creates an account, as any installed Vigil has. - The license state is cached in-process; the lapse test now reloads it instead of inheriting a license an earlier test installed. Six consecutive parallel runs pass. --- server/apps/civilsso/tests.py | 12 ++++++++++++ server/apps/statuspage/tests.py | 12 ++++++++++++ server/vigil/tests_scoping.py | 5 +++++ 3 files changed, 29 insertions(+) diff --git a/server/apps/civilsso/tests.py b/server/apps/civilsso/tests.py index d92544d3..ac92605a 100644 --- a/server/apps/civilsso/tests.py +++ b/server/apps/civilsso/tests.py @@ -117,6 +117,12 @@ def test_inactive_local_user_cannot_enter(self): class DisabledTests(TestCase): + def setUp(self): + # With no account at all, SetupRedirectMiddleware sends every page to + # /setup/ — unless an earlier test in the same process already set its + # once-per-process flag. A user makes this test about Civil, not order. + get_user_model().objects.create_user("someone", password="x") + def test_unconfigured_means_404(self): # No CIVIL_URL: the routes simply do not exist for this install. self.assertEqual( @@ -127,6 +133,12 @@ def test_unconfigured_means_404(self): @override_settings(CIVIL_URL="http://civil.test") class LoginStartTests(TestCase): + def setUp(self): + # An account exists, as on any installed Vigil: otherwise + # SetupRedirectMiddleware sends the page to /setup/, unless an earlier + # test in the same process already set its once-per-process flag. + get_user_model().objects.create_user("someone", password="x") + def test_redirects_to_civil_with_state_and_callback(self): resp = self.client.get("/accounts/civil/login/", {"next": "/"}) self.assertEqual(resp.status_code, 302) diff --git a/server/apps/statuspage/tests.py b/server/apps/statuspage/tests.py index 25f283ac..357c5d07 100644 --- a/server/apps/statuspage/tests.py +++ b/server/apps/statuspage/tests.py @@ -151,6 +151,12 @@ def test_labels_saved_via_api(self): class UptimeHistoryTests(TestCase): + def setUp(self): + # An account exists, as on any installed Vigil: otherwise + # SetupRedirectMiddleware sends the page to /setup/, unless an earlier + # test in the same process already set its once-per-process flag. + get_user_model().objects.create_user("someone", password="x") + def test_sampler_records_one_reading_per_non_pending_host(self): from apps.statuspage.models import HostUptimeSample from apps.statuspage.tasks import sample_uptime @@ -308,6 +314,12 @@ class PublicPageDoesNotPublishHostIdsTests(TestCase): """The public page keyed its cards by the host's internal id; an opaque per-page key does the same job (architect review, 2026-10-08).""" + def setUp(self): + # An account exists, as on any installed Vigil: otherwise + # SetupRedirectMiddleware sends the page to /setup/, unless an earlier + # test in the same process already set its once-per-process flag. + get_user_model().objects.create_user("someone", password="x") + def test_no_host_id_on_the_public_page_or_its_data(self): from apps.hosts.models import Host from apps.statuspage.models import StatusPage diff --git a/server/vigil/tests_scoping.py b/server/vigil/tests_scoping.py index 7b07b9b3..1907c1cd 100644 --- a/server/vigil/tests_scoping.py +++ b/server/vigil/tests_scoping.py @@ -68,6 +68,11 @@ class LapseBehaviorTests(TestCase): """On lapse we stop fixing it for you; we never stop telling you.""" def setUp(self): + # The license state is cached in-process; a test that installed one + # earlier in the same worker would otherwise make this one licensed. + from vigil import licensing + licensing.reload() + self.addCleanup(licensing.reload) self.west = Site.objects.create(name="West Campus", slug="west-campus") self.scoped = Playbook.objects.create(name="Edge hardening") PlaybookSiteAssignment.objects.create(playbook=self.scoped, site=self.west) From fed266aadaa167244154c777409df2ee4b6caecc Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 20:11:35 -0400 Subject: [PATCH 52/68] fix(containers): a stopped or crashed stack shows as stopped, with its exit code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Containers page showed only "exited", so a clean stop and a crash looked the same, stacks had no state of their own, and a stack Vigil manages vanished from the list once it had no containers. - A container reads "stopped (exit 0)" in neutral grey or "crashed (exit 137)" in rose, from Docker's status text, which is also its tooltip. - Each stack header carries one state: running, stopped (exit 0), crashed (exit N), or "1/2 running · 1 crashed (exit 137)". - A managed or adopted stack with no containers left stays listed as down. - The State column is wide enough to hold those on one line. --- server/apps/hosts/test_container_states.py | 71 ++++++++++++++++++++++ server/static/css/vigil.css | 17 ++++-- server/static/js/vigil-containers.js | 61 ++++++++++++++++++- 3 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 server/apps/hosts/test_container_states.py diff --git a/server/apps/hosts/test_container_states.py b/server/apps/hosts/test_container_states.py new file mode 100644 index 00000000..ef01da8f --- /dev/null +++ b/server/apps/hosts/test_container_states.py @@ -0,0 +1,71 @@ +"""Stopped and crashed containers and stacks stay visible, with their exit code. + +User, 2026-10-09: "make sure stack that crashed or stop are visible as stopped +or exited code 0". Docker's status text carries the code ("Exited (137) …"); +the page showed only "exited", so a clean stop and a crash looked the same, +and a managed stack with no containers left vanished from the list. +""" + +import re +import shutil +import subprocess +from pathlib import Path +from unittest import skipUnless + +from django.test import SimpleTestCase + +ROOT = Path(__file__).resolve().parents[2] +JS = ROOT / "static/js/vigil-containers.js" + + +def _function(src: str, name: str) -> str: + start = src.index(f"function {name}(") + depth, i = 0, src.index("{", start) + while True: + depth += {"{": 1, "}": -1}.get(src[i], 0) + if depth == 0: + return src[start:i + 1] + i += 1 + + +class ContainerStateTests(SimpleTestCase): + def setUp(self): + self.src = JS.read_text() + + def test_rows_and_stacks_use_the_exit_aware_badges(self): + self.assertIn("${_ctrStateBadge(c)}", self.src) + self.assertIn("_stackStateBadge(rows)", self.src) + + def test_a_down_managed_stack_stays_listed(self): + self.assertRegex(self.src, r"downStacks = \(stacks \|\| \[\]\)\.filter\(s => s\.ownership !== 'external'") + self.assertIn("for (const s of downStacks)", self.src) + + @skipUnless(shutil.which("node"), "node is not installed") + def test_badges_tell_a_clean_stop_from_a_crash(self): + """Run the real functions from the file, not a copy of them.""" + code = "\n".join([ + "function escHtml(s){return String(s);} function escAttr(s){return String(s);}", + re.search(r"const CTR_STATES = \[[^\]]*\];", self.src).group(0), + _function(self.src, "_ctrExitCode"), + _function(self.src, "_ctrStateBadge"), + _function(self.src, "_stackStateBadge"), + "const C=(state,status)=>({state,status});", + "console.log(JSON.stringify([", + " _ctrStateBadge(C('exited','Exited (0) 2 hours ago')),", + " _ctrStateBadge(C('exited','Exited (137) 5 minutes ago')),", + " _ctrStateBadge(C('running','Up 3 hours')),", + " _stackStateBadge([C('exited','Exited (0) 1 hour ago')]),", + " _stackStateBadge([C('running','Up'), C('exited','Exited (1) now')]),", + " _stackStateBadge([C('dead','Dead')]),", + "]));", + ]) + out = subprocess.run(["node", "-e", code], capture_output=True, text=True, timeout=30, check=True) + clean, crash, running, stack_stopped, stack_partial, stack_dead = __import__("json").loads(out.stdout) + self.assertIn("stopped (exit 0)", clean) + self.assertIn('class="ctr-state stopped"', clean) + self.assertIn("crashed (exit 137)", crash) + self.assertIn('class="ctr-state exited"', crash) + self.assertIn(">running<", running) + self.assertIn("stopped (exit 0)", stack_stopped) + self.assertIn("1/2 running · 1 crashed (exit 1)", stack_partial) + self.assertIn("crashed", stack_dead) diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index 898e5312..da734868 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -2095,7 +2095,16 @@ textarea.form-control { min-height: 110px; resize: vertical; font-family: 'IBM P .ctr-state.exited, .ctr-state.dead { background: rgba(var(--rgb-rose), var(--tint-a)); color: var(--rose-ink); } .ctr-state.paused, -.ctr-state.restarting { background: rgba(var(--rgb-lemon), var(--tint-a)); color: var(--lemon-ink); } +.ctr-state.restarting, +.ctr-state.partial { background: rgba(var(--rgb-lemon), var(--tint-a)); color: var(--lemon-ink); } +/* A clean stop (exit 0) is not a crash: neutral, where a crash is rose. */ +.ctr-state.stopped { background: rgba(110,107,118,0.18); color: var(--text-2); text-transform: none; } +.ctr-state.exited, +.ctr-state.partial { text-transform: none; } +.ctr-state.stopped, +.ctr-state.exited, +.ctr-state.partial { white-space: nowrap; } +.docker-stack-header .ctr-state { margin-left: 4px; } /* Unmanaged devices (inventory page) */ .dev-sub { @@ -3906,9 +3915,9 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } /* Fixed columns so every stack card's Name/Image/State/CPU/Memory/buttons sit at the same x down the page. */ -.ctr-c-name { width: 22%; } -.ctr-c-image { width: 28%; } -.ctr-c-state { width: 11%; } +.ctr-c-name { width: 20%; } +.ctr-c-image { width: 24%; } +.ctr-c-state { width: 17%; } /* fits "crashed (exit 137)" on one line */ .ctr-c-cpu { width: 8%; } .ctr-c-mem { width: 9%; } .ctr-c-acts { width: 22%; } diff --git a/server/static/js/vigil-containers.js b/server/static/js/vigil-containers.js index d3663538..00cec937 100644 --- a/server/static/js/vigil-containers.js +++ b/server/static/js/vigil-containers.js @@ -64,6 +64,46 @@ navigateTo = function(pageName) { /* ── Docker containers ───────────────────────────────────────────────── */ const CTR_STATES = ['running', 'exited', 'dead', 'paused', 'restarting', 'created']; +/* Docker's status text carries the exit code ("Exited (137) 2 hours ago"). + Code 0 is a clean stop; anything else (or "dead") is a crash, and the two + must not look the same. */ +function _ctrExitCode(c) { + const m = /exited \((-?\d+)\)/i.exec(c.status || ''); + return m ? parseInt(m[1], 10) : null; +} + +function _ctrStateBadge(c) { + const state = (c.state || '').toLowerCase(); + const code = _ctrExitCode(c); + let text = state || 'unknown'; + let cls = CTR_STATES.includes(state) ? state : ''; + if (state === 'exited' && code !== null) { + text = code === 0 ? 'stopped (exit 0)' : `crashed (exit ${code})`; + cls = code === 0 ? 'stopped' : 'exited'; + } + return `${escHtml(text)}`; +} + +/* One state for a whole stack: running, partly running, stopped or crashed — + so a stack whose containers all exited still reads as down at a glance. */ +function _stackStateBadge(rows) { + const running = rows.filter(c => (c.state || '').toLowerCase() === 'running').length; + const crashed = rows.filter(c => (c.state || '').toLowerCase() === 'dead' + || ((c.state || '').toLowerCase() === 'exited' && (_ctrExitCode(c) || 0) !== 0)); + const codes = [...new Set(crashed.map(_ctrExitCode).filter(x => x !== null))]; + const exitNote = codes.length ? ` (exit ${codes.join(', ')})` : ''; + let text, cls; + if (running === rows.length) { text = 'running'; cls = 'running'; } + else if (running === 0) { + text = crashed.length ? `crashed${exitNote}` : 'stopped (exit 0)'; + cls = crashed.length ? 'exited' : 'stopped'; + } else { + text = `${running}/${rows.length} running` + (crashed.length ? ` · ${crashed.length} crashed${exitNote}` : ''); + cls = 'partial'; + } + return `${escHtml(text)}`; +} + async function renderDockerContainers(hostId) { const wrap = document.getElementById('docker-stacks'); const countEl = document.getElementById('docker-count'); @@ -82,7 +122,11 @@ async function renderDockerContainers(hostId) { } catch { containers = []; } const stackInfo = Object.fromEntries((stacks || []).map(s => [s.project, s])); - if (!containers.length) { + // A stack Vigil deployed or adopted keeps its row after `down` or after its + // containers are removed: it is still there, just down, so it stays listed. + const downStacks = (stacks || []).filter(s => s.ownership !== 'external' + && !containers.some(c => c.stack === s.project)); + if (!containers.length && !downStacks.length) { countEl.textContent = ''; wrap.innerHTML = '
No containers reported for this host.
'; return; @@ -117,6 +161,7 @@ async function renderDockerContainers(hostId) {
${escHtml(label)} ${rows.length} ${noun} + ${stack ? _stackStateBadge(rows) : ''} ${stackMeta}${stackActs}
ContainerImageStateNameImageState CPUMemory
@@ -131,7 +176,6 @@ async function renderDockerContainers(hostId) { `; for (const c of rows) { const state = (c.state || '').toLowerCase(); - const stateClass = CTR_STATES.includes(state) ? state : ''; const cpu = (c.cpu_percent === null || c.cpu_percent === undefined) ? '—' : c.cpu_percent.toFixed(1) + '%'; let mem = '—'; @@ -143,7 +187,7 @@ async function renderDockerContainers(hostId) { html += ` - +
${escHtml(c.name || '')}
${svc}		 ${escHtml(c.image || '')}${escHtml(state || 'unknown')}${_ctrStateBadge(c)} ${cpu} ${mem} ${c.outdated ? `` : ''} @@ -158,6 +202,17 @@ async function renderDockerContainers(hostId) { } html += `
`; } + for (const s of downStacks) { + html += `
+
+ ${escHtml(s.project)} + 0 containers + down + ${escHtml(s.ownership)} +
+
No containers — it was taken down or its containers were removed. Deploy it again from Managed by Vigil above.
+
`; + } wrap.innerHTML = html; wrap.querySelectorAll('[data-ctr-update]').forEach(btn => btn.addEventListener('click', () => { if (typeof openUpdateContainer === 'function') { From 2f6b00ade36a5e5587a25f364211979d7bf44526 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 20:24:06 -0400 Subject: [PATCH 53/68] feat(stacks): create a stack from a Git repository (server) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For teams that build their images rather than publish them. A stack can now take its compose file from https or ssh Git: track a branch (each deploy takes its latest commit, and every revision records the commit it came from) or pin an exact commit or tag. The repository is the source of truth for the compose file; the .env is still edited in Vigil. The server, never a host, talks to the Git server, so a repository credential stays in Vigil (GitCredential: HTTPS token or SSH deploy key, encrypted, write-only, TOTP to add). apps/stacks/gitsource.py: - resolves the branch/pin to one commit, fetches only that commit into a bare repo (no checkout, hooks or filters) and takes `git archive` of the compose file's folder, repacked as a deterministic tar.gz of regular files only — a symlink or special file is refused, not skipped; - allows https and ssh URLs only (no file://, ext::, local paths, embedded passwords or leading '-'), sets GIT_ALLOW_PROTOCOL, ignores user and system git config, follows no HTTP redirects, refuses hosts resolving to loopback / link-local / unspecified addresses (a LAN Git server is fine); - sends a token through GIT_ASKPASS from the environment — never a command line, which every local user can read — and uses an SSH key only with pinned host keys (StrictHostKeyChecking=yes, no trust on first use); - caps time, files and size. Every fetched compose file passes validate_compose like any other. A deploy carries a one-time SourceTicket and the archive's sha256 in the signed task (stack_deploy source_ticket / source_sha256 / source_commit), and is refused for an agent that does not advertise `stack_source`. The server image gains git and openssh-client. Fetched for real from github.com/docker/awesome-compose in testing. --- server/Dockerfile | 4 +- server/apps/stacks/git_views.py | 206 +++++++++++ server/apps/stacks/gitsource.py | 322 ++++++++++++++++++ .../apps/stacks/migrations/0005_git_source.py | 91 +++++ server/apps/stacks/models.py | 88 +++++ server/apps/stacks/test_git_source.py | 301 ++++++++++++++++ server/apps/stacks/urls.py | 6 +- server/apps/stacks/views.py | 59 +++- .../commands/render_wiki_actions.py | 3 + server/apps/tasks/registry.py | 3 +- server/apps/tasks/spec.py | 11 + server/vigil/hooks.py | 1 + server/vigil/urls.py | 2 + wiki/vigil-wiki.html | 4 +- 14 files changed, 1093 insertions(+), 8 deletions(-) create mode 100644 server/apps/stacks/git_views.py create mode 100644 server/apps/stacks/gitsource.py create mode 100644 server/apps/stacks/migrations/0005_git_source.py create mode 100644 server/apps/stacks/test_git_source.py diff --git a/server/Dockerfile b/server/Dockerfile index 140a8219..7f8d4821 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -31,8 +31,10 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +# git + openssh-client: stacks from Git (2026.14.1) are fetched by the server, +# never by a host, so a repository credential stays in Vigil. RUN apt-get update && \ - apt-get install -y --no-install-recommends libpq-dev gosu && \ + apt-get install -y --no-install-recommends libpq-dev gosu git openssh-client && \ rm -rf /var/lib/apt/lists/* # Gunicorn and every request it serves ran as UID 0, so a code-execution bug in diff --git a/server/apps/stacks/git_views.py b/server/apps/stacks/git_views.py new file mode 100644 index 00000000..08dbb68e --- /dev/null +++ b/server/apps/stacks/git_views.py @@ -0,0 +1,206 @@ +"""Stacks from Git (2026.14.1): credentials, pulling a repository into a +revision, and the agent's one-time download of a revision's source folder. + +The repository is the source of truth for a Git stack's compose file — it is +not edited in Vigil — while its .env still is. See gitsource.py for how the +server talks to the Git server. +""" + +from datetime import timedelta + +from django.db import transaction +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from django.utils.timezone import now +from rest_framework import status +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import AllowAny, IsAuthenticated +from rest_framework.response import Response + +from apps.accounts.permissions import IsAdmin +from apps.hosts.crypto import decrypt_secret, encrypt_secret +from vigil import hooks + +from . import gitsource +from .models import GitCredential, ManagedStack, SourceSnapshot, SourceTicket, StackRevision +from .validation import StackError, validate_compose + +#: Snapshots kept per stack; older ones (and their tickets) are pruned. +KEEP_SNAPSHOTS = 5 + + +# ── Credentials ────────────────────────────────────────────────────────── + + +def _cred_row(c: GitCredential) -> dict: + return {"id": str(c.id), "name": c.name, "kind": c.kind, "username": c.username, + "has_known_hosts": bool(c.known_hosts.strip()), + "created_at": c.created_at.isoformat()} + + +@api_view(["GET", "POST"]) +@permission_classes([IsAuthenticated, IsAdmin]) +def git_credential_index(request): + """List (never the secret) or add a Git credential. Adding one asks for + TOTP: it is a key to a repository whose contents run as root on hosts.""" + if request.method == "GET": + return Response({"results": [_cred_row(c) for c in GitCredential.objects.all()]}) + from apps.accounts.totp import require_totp_confirmation + if error := require_totp_confirmation(request.user, request.data): + return Response({"detail": error, "needs_totp": True}, status=status.HTTP_401_UNAUTHORIZED) + name = str(request.data.get("name") or "").strip()[:120] + kind = str(request.data.get("kind") or "") + secret = str(request.data.get("secret") or "") + known_hosts = str(request.data.get("known_hosts") or "") + if not name or kind not in GitCredential.Kind.values or not secret.strip(): + return Response({"detail": "name, kind (token or ssh_key) and secret are required"}, + status=status.HTTP_400_BAD_REQUEST) + if kind == GitCredential.Kind.SSH_KEY: + if "PRIVATE KEY" not in secret: + return Response({"detail": "paste the private key (-----BEGIN … PRIVATE KEY-----)"}, + status=status.HTTP_400_BAD_REQUEST) + if not known_hosts.strip(): + return Response({"detail": "an SSH key needs the Git server's host key (a known_hosts " + "line, e.g. from ssh-keyscan) — Vigil does not trust on first use"}, + status=status.HTTP_400_BAD_REQUEST) + if len(secret) > 20_000 or len(known_hosts) > 20_000: + return Response({"detail": "that is too long for a credential"}, + status=status.HTTP_400_BAD_REQUEST) + cred = GitCredential.objects.create( + name=name, kind=kind, username=str(request.data.get("username") or "").strip()[:255], + secret_encrypted=encrypt_secret(secret), known_hosts=known_hosts.strip(), + created_by=request.user) + hooks.emit("git_credential_added", credential=cred, user=request.user) + return Response(_cred_row(cred), status=status.HTTP_201_CREATED) + + +@api_view(["DELETE"]) +@permission_classes([IsAuthenticated, IsAdmin]) +def git_credential_detail(request, cred_id): + get_object_or_404(GitCredential, pk=cred_id).delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + +# ── Pulling ────────────────────────────────────────────────────────────── + + +def source_for(stack: ManagedStack) -> gitsource.Source: + cred = None + if stack.git_credential_id: + c = stack.git_credential + cred = (c.kind, c.username, decrypt_secret(bytes(c.secret_encrypted)), c.known_hosts) + return gitsource.Source(url=stack.git_url, branch=stack.git_branch or "main", + pin=stack.git_pin, path=stack.git_path or "compose.yaml", + credential=cred) + + +def apply_git_settings(stack: ManagedStack, data) -> None: + """Copy a request's ``git`` block onto *stack*, checked. Raises + GitSourceError.""" + git = data.get("git") + if not isinstance(git, dict): + raise gitsource.GitSourceError("git must be an object with url, branch, pin, path") + url = str(git.get("url") or "").strip() + gitsource.parse_url(url) + branch = gitsource.check_ref(str(git.get("branch") or "main"), "branch") + pin = str(git.get("pin") or "").strip() + if pin and not gitsource._SHA_RE.match(pin.lower()): + gitsource.check_ref(pin, "pin") + path = str(git.get("path") or "compose.yaml").strip() + gitsource.check_path(path) + cred_id = git.get("credential_id") or None + cred = None + if cred_id: + cred = GitCredential.objects.filter(pk=cred_id).first() + if cred is None: + raise gitsource.GitSourceError("that Git credential does not exist") + stack.git_url, stack.git_branch, stack.git_pin, stack.git_path = url, branch, pin, path + stack.git_credential = cred + + +def pull(stack: ManagedStack, user, *, note: str = "") -> bool: + """Fetch the stack's repository; when it brings a new commit (or a + changed compose file), make it a new revision with its source snapshot. + Returns whether a revision was made. Raises GitSourceError / StackError.""" + fetched = gitsource.fetch(source_for(stack)) + validate_compose(fetched.compose_yaml) + current = stack.revisions.filter(number=stack.revision).first() if not stack._state.adding else None + if (current is not None and current.git_commit == fetched.commit + and current.compose_yaml == fetched.compose_yaml + and stack.snapshots.filter(revision=stack.revision, sha256=fetched.sha256).exists()): + return False + with transaction.atomic(): + if not stack._state.adding: + stack.revision += 1 + stack.compose_yaml = fetched.compose_yaml + stack.compose_file = fetched.compose_file + stack.save() + StackRevision.objects.create( + stack=stack, number=stack.revision, compose_yaml=stack.compose_yaml, + env_encrypted=stack.env_encrypted, git_commit=fetched.commit, + note=(note or f"from Git {fetched.commit[:12]}")[:200], created_by=user) + SourceSnapshot.objects.create(stack=stack, revision=stack.revision, commit=fetched.commit, + sha256=fetched.sha256, size=len(fetched.archive), + data=fetched.archive) + stale = list(stack.snapshots.order_by("-created_at").values_list("id", flat=True)[KEEP_SNAPSHOTS:]) + SourceSnapshot.objects.filter(id__in=stale).delete() + hooks.emit("stack_saved", stack=stack, user=user, revision=stack.revision) + return True + + +@api_view(["POST"]) +@permission_classes([IsAuthenticated, IsAdmin]) +def stack_pull(request, stack_id): + """Fetch the repository now and make a revision if it changed.""" + from .views import _row, _stack_or_404 + + stack = _stack_or_404(request, stack_id) + if stack is None: + return Response(status=status.HTTP_404_NOT_FOUND) + if not stack.git_url: + return Response({"detail": "this stack's compose file is edited in Vigil, not pulled from Git"}, + status=status.HTTP_400_BAD_REQUEST) + try: + changed = pull(stack, request.user) + except (gitsource.GitSourceError, StackError) as exc: + return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST) + return Response({"changed": changed, "stack": _row(stack)}) + + +def deploy_source_params(stack: ManagedStack) -> dict: + """Task params that let the host fetch this revision's folder once: + a ticket, and the sha256 the archive must have.""" + snap = stack.snapshots.filter(revision=stack.revision).order_by("-created_at").first() + if snap is None: + raise gitsource.GitSourceError("this revision has no source from Git yet — pull it first") + ticket = SourceTicket.objects.create(snapshot=snap, host=stack.host, + expires_at=now() + timedelta(hours=SourceTicket.TTL_HOURS)) + return {"source_ticket": str(ticket.id), "source_sha256": snap.sha256, + "source_commit": snap.commit} + + +# ── The agent's download ───────────────────────────────────────────────── + + +@api_view(["GET"]) +@permission_classes([AllowAny]) +def agent_stack_source(request, ticket_id): + """The agent redeems a deploy's source ticket — once, before it expires, + and only for its own host. The archive's sha256 is in the signed task.""" + from apps.hosts.authentication import authenticate_agent + + host, err = authenticate_agent(request) + if err: + return err + with transaction.atomic(): + ticket = (SourceTicket.objects.select_for_update().select_related("snapshot") + .filter(pk=ticket_id, host=host).first()) + if ticket is None or ticket.used_at is not None or ticket.expires_at <= now(): + return Response({"detail": "ticket unknown, used or expired"}, status=status.HTTP_404_NOT_FOUND) + ticket.used_at = now() + ticket.save(update_fields=["used_at"]) + data = bytes(ticket.snapshot.data) + response = HttpResponse(data, content_type="application/gzip") + response["Content-Length"] = str(len(data)) + response["Cache-Control"] = "no-store" + return response diff --git a/server/apps/stacks/gitsource.py b/server/apps/stacks/gitsource.py new file mode 100644 index 00000000..1e91f19d --- /dev/null +++ b/server/apps/stacks/gitsource.py @@ -0,0 +1,322 @@ +"""Fetch a stack's compose folder from a Git repository (2026.14.1). + +The server — never a host — talks to the Git server, so a repository +credential stays in Vigil. A fetch resolves the branch (or the pinned tag or +commit) to one commit, fetches only that commit into a bare repository (so no +checkout, no hooks, no filters run), takes ``git archive`` of the folder that +holds the compose file, and repacks it as a tar.gz of **regular files only**. +The deploy task carries that archive's sha256; the agent checks it before it +places anything (agent/vigil_agent/actions/stacks.py). + +What reaches git is narrow on purpose: + +- URLs: ``https://host/path`` or SSH (``user@host:path`` / ``ssh://user@host/path``). + No ``file://``, ``ext::``, local paths, embedded passwords, or anything + starting with ``-``. ``GIT_ALLOW_PROTOCOL`` holds git itself to https and ssh, + and HTTP redirects are not followed (a redirect could point anywhere). +- A host that resolves to loopback, link-local (cloud metadata) or an + unspecified address is refused. A LAN Git server is fine. +- A token goes through GIT_ASKPASS from the environment — never a command line, + which every user on the machine can read in /proc. An SSH key is used only + with pinned host keys (StrictHostKeyChecking=yes, no global known_hosts). +- No user or system git config is read; time and size are capped. +""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import ipaddress +import os +import re +import shlex +import socket +import subprocess +import tarfile +import tempfile +from dataclasses import dataclass + +#: The packed archive a host downloads. +MAX_ARCHIVE_BYTES = 20 * 1024 * 1024 +#: What `git archive` may produce before repacking (uncompressed). +MAX_TREE_BYTES = 100 * 1024 * 1024 +MAX_FILES = 5000 +TIMEOUT_SECONDS = 120 + +_HOST = r"[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?" +_PATH = r"[A-Za-z0-9._~/-]{1,400}" +_HTTPS_RE = re.compile(rf"^https://(?P{_HOST})(?::(?P[0-9]{{1,5}}))?/(?P{_PATH})$") +_SCP_RE = re.compile(rf"^(?P[A-Za-z0-9._-]{{1,64}})@(?P{_HOST}):(?P{_PATH})$") +_SSH_RE = re.compile(rf"^ssh://(?P[A-Za-z0-9._-]{{1,64}})@(?P{_HOST})" + rf"(?::(?P[0-9]{{1,5}}))?/(?P{_PATH})$") +_SHA_RE = re.compile(r"^[0-9a-f]{40}$|^[0-9a-f]{64}$") +_REF_RE = re.compile(r"^[A-Za-z0-9._/-]{1,200}$") +_COMPOSE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}\.ya?ml$") + + +class GitSourceError(ValueError): + """Why a repository could not be used — safe to show the admin.""" + + +@dataclass +class Source: + url: str + branch: str = "main" + pin: str = "" + path: str = "compose.yaml" + #: (kind, username, secret, known_hosts) or None. + credential: tuple[str, str, str, str] | None = None + + +@dataclass +class Fetched: + commit: str + compose_yaml: str + compose_file: str + archive: bytes + sha256: str + + +# ── Input checks ───────────────────────────────────────────────────────── + + +def parse_url(url: str) -> tuple[str, str]: + """(kind, host) for an allowed URL — kind is "https" or "ssh".""" + url = (url or "").strip() + for kind, regex in (("https", _HTTPS_RE), ("ssh", _SCP_RE), ("ssh", _SSH_RE)): + m = regex.match(url) + if m: + if ".." in m.group("path").split("/") or "//" in m.group("path"): + break + return kind, m.group("host").lower() + raise GitSourceError( + "the repository URL must be https://host/owner/repo or git@host:owner/repo " + "(no passwords in the URL — use a Git credential)") + + +def check_ref(name: str, what: str) -> str: + name = (name or "").strip() + if (not _REF_RE.match(name) or name.startswith(("-", "/")) or name.endswith(("/", ".lock")) + or ".." in name or "//" in name or "@{" in name): + raise GitSourceError(f"{what} {name!r} is not a valid Git branch or tag name") + return name + + +def check_path(path: str) -> tuple[str, str]: + """(folder, compose file name) for the compose file's path in the repo.""" + path = (path or "").strip() + if path.startswith("/"): + raise GitSourceError("the compose path must be relative to the repository, e.g. deploy/compose.yaml") + parts = path.split("/") if path else [] + if not parts or any(p in ("", ".", "..") for p in parts) or len(path) > 300: + raise GitSourceError("the compose path must be a relative path such as deploy/compose.yaml") + if not _COMPOSE_NAME_RE.match(parts[-1]): + raise GitSourceError("the compose path must end in a .yaml or .yml file name") + if not all(re.match(r"^[A-Za-z0-9._ -]{1,100}$", p) for p in parts): + raise GitSourceError("the compose path may use letters, digits, '.', '_', '-' and spaces") + return "/".join(parts[:-1]), parts[-1] + + +def _check_host_address(host: str, port: int) -> None: + """Loopback, link-local (cloud metadata) and unspecified are refused — + the server must not be talked into fetching from itself. Every address the + name resolves to is checked.""" + try: + infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) + except socket.gaierror as exc: + raise GitSourceError(f"cannot resolve {host}") from exc + for info in infos: + addr = ipaddress.ip_address(info[4][0].split("%", 1)[0]) + mapped = getattr(addr, "ipv4_mapped", None) + for a in (addr, mapped) if mapped else (addr,): + if a.is_loopback or a.is_link_local or a.is_unspecified or a.is_multicast: + raise GitSourceError(f"{host} resolves to {a}, which Vigil will not fetch from") + + +# ── Running git ────────────────────────────────────────────────────────── + +_GIT_CONFIG = ( + "-c", "protocol.allow=never", "-c", "protocol.https.allow=always", + "-c", "protocol.ssh.allow=always", "-c", "http.followRedirects=false", + "-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", + "-c", "credential.helper=", "-c", "submodule.recurse=false", +) + + +def _env(tmp: str, source: Source, kind: str) -> dict: + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": tmp, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + "GIT_ALLOW_PROTOCOL": "https:ssh", + "GIT_ASKPASS": "", + "SSH_ASKPASS": "", + "LC_ALL": "C", + } + cred = source.credential + if kind == "https" and cred and cred[0] == "token": + askpass = os.path.join(tmp, "askpass.sh") + # Reads the values from its own environment: nothing secret is in a + # file or on a command line. + with open(askpass, "w") as fh: + fh.write('#!/bin/sh\ncase "$1" in Username*) printf "%s" "$VIGIL_GIT_USER";; ' + '*) printf "%s" "$VIGIL_GIT_TOKEN";; esac\n') + os.chmod(askpass, 0o700) + env.update(GIT_ASKPASS=askpass, VIGIL_GIT_USER=cred[1] or "x-access-token", + VIGIL_GIT_TOKEN=cred[2]) + if kind == "ssh": + if not cred or cred[0] != "ssh_key": + raise GitSourceError("an SSH repository needs an SSH deploy key credential") + if not cred[3].strip(): + raise GitSourceError("the SSH credential has no known_hosts entry for the Git server") + key, known = os.path.join(tmp, "id_key"), os.path.join(tmp, "known_hosts") + fd = os.open(key, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as fh: + fh.write(cred[2].strip() + "\n") + with open(known, "w") as fh: + fh.write(cred[3].strip() + "\n") + env["GIT_SSH_COMMAND"] = " ".join([ + "ssh", "-F", "/dev/null", "-i", shlex.quote(key), "-o", "IdentitiesOnly=yes", + "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", + "-o", f"UserKnownHostsFile={shlex.quote(known)}", "-o", "GlobalKnownHostsFile=/dev/null", + "-o", "ConnectTimeout=15"]) + return env + + +def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_000_000) -> bytes: + try: + proc = subprocess.Popen(["git", *_GIT_CONFIG, *args], cwd=cwd, env=env, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except FileNotFoundError as exc: + raise GitSourceError("git is not installed on the Vigil server") from exc + try: + out, err = proc.communicate(timeout=TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + raise GitSourceError("the Git server did not answer in time") from None + if len(out) > limit: + raise GitSourceError("the repository folder is too large for a stack") + if proc.returncode != 0: + # The last stderr line says what went wrong; the token is never in it + # (it never was on a command line or in the URL). + lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()] + raise GitSourceError(f"git failed: {lines[-1][:300] if lines else 'unknown error'}") + return out + + +def _resolve(url: str, source: Source, env: dict) -> str: + """The commit a fetch should take: the pin if it is a commit, else what + the pinned tag or tracked branch points at now.""" + pin = (source.pin or "").strip() + if pin and _SHA_RE.match(pin.lower()): + return pin.lower() + if pin: + name = check_ref(pin, "pin") + refs = [f"refs/tags/{name}", f"refs/tags/{name}^{{}}"] + else: + name = check_ref(source.branch or "main", "branch") + refs = [f"refs/heads/{name}"] + out = _git(["ls-remote", "--", url, *refs], env).decode(errors="replace") + found = {} + for line in out.splitlines(): + sha, _, ref = line.partition("\t") + found[ref.strip()] = sha.strip().lower() + # An annotated tag's ^{} line is the commit; a lightweight tag has none. + for ref in reversed(refs): + if _SHA_RE.match(found.get(ref, "")): + return found[ref] + raise GitSourceError(f"{'tag' if pin else 'branch'} {name!r} was not found in the repository") + + +# ── Repacking ──────────────────────────────────────────────────────────── + + +def repack(raw_tar: bytes, folder: str) -> tuple[bytes, dict[str, bytes]]: + """``git archive`` output → a deterministic tar.gz of *folder*'s regular + files, paths relative to it. Symlinks, hard links, devices and anything + else are refused rather than skipped: a stack that silently lost a file + would fail in a confusing way on the host. Returns (archive, files).""" + prefix = f"{folder}/" if folder else "" + files: dict[str, bytes] = {} + executable: set[str] = set() + total = 0 + with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as tar: + for member in tar: + name = member.name + if prefix and not (name + "/").startswith(prefix): + continue + rel = name[len(prefix):].strip("/") + if not rel: + continue + parts = rel.split("/") + if name.startswith("/") or any(p in ("", ".", "..") for p in parts) or "\\" in rel: + raise GitSourceError(f"the repository has an unusable path: {name!r}") + if member.isdir(): + continue + if not member.isreg(): + raise GitSourceError( + f"{rel} is a symlink or special file — a Git stack may contain regular files only") + total += member.size + if len(files) >= MAX_FILES or total > MAX_TREE_BYTES: + raise GitSourceError("the repository folder is too large for a stack") + files[rel] = tar.extractfile(member).read() + if member.mode & 0o111: + executable.add(rel) + out = io.BytesIO() + with gzip.GzipFile(fileobj=out, mode="wb", mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar: + for rel in sorted(files): + info = tarfile.TarInfo(rel) + info.size = len(files[rel]) + info.mode = 0o755 if rel in executable else 0o644 + info.mtime = 0 + tar.addfile(info, io.BytesIO(files[rel])) + archive = out.getvalue() + if len(archive) > MAX_ARCHIVE_BYTES: + raise GitSourceError("the repository folder is too large for a stack") + return archive, files + + +# ── The fetch ──────────────────────────────────────────────────────────── + + +def fetch(source: Source) -> Fetched: + """Fetch, check and pack one commit's compose folder. Raises + GitSourceError with a message fit for the admin.""" + url = (source.url or "").strip() + kind, host = parse_url(url) + folder, compose_file = check_path(source.path) + port = 443 if kind == "https" else 22 + if (m := (_HTTPS_RE.match(url) or _SSH_RE.match(url))) and m.groupdict().get("port"): + port = int(m.group("port")) + _check_host_address(host, port) + + with tempfile.TemporaryDirectory(prefix="vigil-git-") as tmp: + os.chmod(tmp, 0o700) + env = _env(tmp, source, kind) + commit = _resolve(url, source, env) + repo = os.path.join(tmp, "repo.git") + _git(["init", "-q", "--bare", repo], env) + _git(["fetch", "-q", "--depth", "1", "--no-tags", "--no-recurse-submodules", + "--", url, commit], env, cwd=repo) + got = _git(["rev-parse", "--verify", "--end-of-options", f"{commit}^{{commit}}"], + env, cwd=repo).decode().strip().lower() + if got != commit: + raise GitSourceError(f"the Git server returned {got[:12]} for {commit[:12]}") + raw = _git(["archive", "--format=tar", commit, "--", folder or "."], env, cwd=repo, + limit=MAX_TREE_BYTES + MAX_FILES * 1024) + + archive, files = repack(raw, folder) + if compose_file not in files: + raise GitSourceError(f"{source.path} is not in the repository at {commit[:12]}") + try: + compose_yaml = files[compose_file].decode("utf-8") + except UnicodeDecodeError as exc: + raise GitSourceError(f"{source.path} is not UTF-8 text") from exc + return Fetched(commit=commit, compose_yaml=compose_yaml, compose_file=compose_file, + archive=archive, sha256=hashlib.sha256(archive).hexdigest()) diff --git a/server/apps/stacks/migrations/0005_git_source.py b/server/apps/stacks/migrations/0005_git_source.py new file mode 100644 index 00000000..53e53045 --- /dev/null +++ b/server/apps/stacks/migrations/0005_git_source.py @@ -0,0 +1,91 @@ +# Generated by Django 5.2.17 on 2026-10-10 00:14 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('hosts', '0026_hash_agent_tokens'), + ('stacks', '0004_registry_credential'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='managedstack', + name='git_branch', + field=models.CharField(blank=True, default='main', max_length=200), + ), + migrations.AddField( + model_name='managedstack', + name='git_path', + field=models.CharField(blank=True, default='compose.yaml', max_length=300), + ), + migrations.AddField( + model_name='managedstack', + name='git_pin', + field=models.CharField(blank=True, max_length=200), + ), + migrations.AddField( + model_name='managedstack', + name='git_url', + field=models.CharField(blank=True, max_length=500), + ), + migrations.AddField( + model_name='stackrevision', + name='git_commit', + field=models.CharField(blank=True, max_length=64), + ), + migrations.CreateModel( + name='GitCredential', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=120)), + ('kind', models.CharField(choices=[('token', 'HTTPS token'), ('ssh_key', 'SSH deploy key')], max_length=10)), + ('username', models.CharField(blank=True, max_length=255)), + ('secret_encrypted', models.BinaryField()), + ('known_hosts', models.TextField(blank=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ('name',), + }, + ), + migrations.AddField( + model_name='managedstack', + name='git_credential', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='stacks', to='stacks.gitcredential'), + ), + migrations.CreateModel( + name='SourceSnapshot', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('revision', models.PositiveIntegerField()), + ('commit', models.CharField(max_length=64)), + ('sha256', models.CharField(max_length=64)), + ('size', models.PositiveIntegerField()), + ('data', models.BinaryField()), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('stack', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='snapshots', to='stacks.managedstack')), + ], + options={ + 'ordering': ('-created_at',), + }, + ), + migrations.CreateModel( + name='SourceTicket', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('expires_at', models.DateTimeField()), + ('used_at', models.DateTimeField(blank=True, null=True)), + ('host', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='hosts.host')), + ('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tickets', to='stacks.sourcesnapshot')), + ], + ), + ] diff --git a/server/apps/stacks/models.py b/server/apps/stacks/models.py index 8fbc743e..5cdd4974 100644 --- a/server/apps/stacks/models.py +++ b/server/apps/stacks/models.py @@ -32,6 +32,23 @@ class ManagedStack(models.Model): #: services a deploy of this file would leave alone, and those it would #: recreate (their running config differs from the file's). adopt_report = models.JSONField(default=dict, blank=True) + #: A stack whose compose file comes from a Git repository (2026.14.1). + #: Blank = the compose file is edited in Vigil. Otherwise the repo is the + #: source of truth: each pull that finds a new commit makes a revision + #: recording that commit (StackRevision.git_commit), and the folder that + #: holds the compose file is shipped to the host (SourceSnapshot) so + #: ``build:`` contexts have their source. https:// or ssh only. + git_url = models.CharField(max_length=500, blank=True) + #: The branch to track; each deploy takes its latest commit. + git_branch = models.CharField(max_length=200, blank=True, default="main") + #: An exact commit (40 hex) or tag to hold the stack at instead. Blank = + #: follow git_branch. + git_pin = models.CharField(max_length=200, blank=True) + #: The compose file's path inside the repo, e.g. deploy/compose.yaml. Its + #: folder becomes the stack folder on the host. + git_path = models.CharField(max_length=300, blank=True, default="compose.yaml") + git_credential = models.ForeignKey("GitCredential", null=True, blank=True, + on_delete=models.SET_NULL, related_name="stacks") created_by = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL, related_name="+") created_at = models.DateTimeField(auto_now_add=True) @@ -51,6 +68,8 @@ class StackRevision(models.Model): compose_yaml = models.TextField() env_encrypted = models.BinaryField(blank=True, default=b"") note = models.CharField(max_length=200, blank=True) + #: The commit this revision's compose file came from (Git stacks only). + git_commit = models.CharField(max_length=64, blank=True) created_by = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL, related_name="+") created_at = models.DateTimeField(auto_now_add=True) @@ -121,3 +140,72 @@ class RegistryCredential(models.Model): class Meta: ordering = ("registry",) + + +class GitCredential(models.Model): + """How Vigil reaches a private Git repository (2026.14.1), encrypted at rest. + + Only the server ever uses it: the server fetches the repo and hands the + host a packed copy, so no host holds a repository credential. ``token`` is + an HTTPS access token (GitHub, GitLab, Gitea …) sent through GIT_ASKPASS, + never on a command line; ``ssh_key`` is a deploy key, used only with the + host keys pinned in ``known_hosts`` — an unknown or changed host key is + refused, not trusted on first use. + """ + + class Kind(models.TextChoices): + TOKEN = "token", "HTTPS token" + SSH_KEY = "ssh_key", "SSH deploy key" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=120) + kind = models.CharField(max_length=10, choices=Kind.choices) + #: For a token: the user name the host expects (x-access-token, oauth2 …). + username = models.CharField(max_length=255, blank=True) + secret_encrypted = models.BinaryField() + #: For an SSH key: known_hosts lines for the Git server(s). + known_hosts = models.TextField(blank=True) + created_by = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL, + related_name="+") + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ("name",) + + def __str__(self): + return f"git-credential:{self.name} ({self.kind})" + + +class SourceSnapshot(models.Model): + """The packed folder of one Git revision (tar.gz of regular files only). + + Built once by the server at pull time; a deploy's signed task carries its + sha256, so the host can tell it got exactly this archive. Kept small + (gitsource.MAX_ARCHIVE_BYTES) and pruned to the last few per stack. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + stack = models.ForeignKey(ManagedStack, on_delete=models.CASCADE, related_name="snapshots") + revision = models.PositiveIntegerField() + commit = models.CharField(max_length=64) + sha256 = models.CharField(max_length=64) + size = models.PositiveIntegerField() + data = models.BinaryField() + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ("-created_at",) + + +class SourceTicket(models.Model): + """A one-time reference to a SourceSnapshot for one deploy, like EnvTicket: + redeemable once, by the stack's own host, before it expires.""" + + TTL_HOURS = 24 + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + snapshot = models.ForeignKey(SourceSnapshot, on_delete=models.CASCADE, related_name="tickets") + host = models.ForeignKey("hosts.Host", on_delete=models.CASCADE, related_name="+") + created_at = models.DateTimeField(auto_now_add=True) + expires_at = models.DateTimeField() + used_at = models.DateTimeField(null=True, blank=True) diff --git a/server/apps/stacks/test_git_source.py b/server/apps/stacks/test_git_source.py new file mode 100644 index 00000000..752fb2e5 --- /dev/null +++ b/server/apps/stacks/test_git_source.py @@ -0,0 +1,301 @@ +"""Stacks from Git (2026.14.1): the fetcher's input checks, a real fetch from +a local repository, and the API around it. + +Production git may speak only https and ssh, so the fetch tests open the +``file`` protocol for one local repository through ``_local_git`` — nothing +else in the module changes. +""" + +import gzip +import hashlib +import io +import os +import socket +import subprocess +import tarfile +import tempfile +from contextlib import contextmanager +from pathlib import Path +from unittest import mock + +from django.contrib.auth import get_user_model +from django.test import SimpleTestCase, TestCase +from rest_framework.test import APIClient + +from apps.accounts.models import Role, UserProfile +from apps.hosts.models import Host + +from . import gitsource +from .gitsource import GitSourceError, Source + +_TOTP = "apps.accounts.totp.require_totp_confirmation" + +COMPOSE = "services:\n app:\n build: ./app\n ports: ['8080:80']\n" + + +def _run(repo, *args): + env = {**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t"} + return subprocess.run(["git", *args], cwd=repo, env=env, check=True, + capture_output=True, text=True).stdout.strip() + + +def _make_repo(root: Path) -> tuple[str, str]: + """A repo with deploy/compose.yaml, its build folder, an executable and a + file outside the folder. Returns (path, head commit).""" + repo = root / "repo" + (repo / "deploy" / "app").mkdir(parents=True) + (repo / "deploy" / "compose.yaml").write_text(COMPOSE) + (repo / "deploy" / "app" / "Dockerfile").write_text("FROM nginx\n") + (repo / "deploy" / "run.sh").write_text("#!/bin/sh\necho hi\n") + (repo / "deploy" / "run.sh").chmod(0o755) + (repo / "README.md").write_text("not part of the stack\n") + _run(repo, "init", "-q", "-b", "main") + _run(repo, "add", ".") + _run(repo, "commit", "-q", "-m", "first") + _run(repo, "tag", "-a", "v1", "-m", "release 1") + return str(repo), _run(repo, "rev-parse", "HEAD") + + +@contextmanager +def _local_git(): + """Let the fetcher reach a local repository: allow git's file protocol and + skip the URL/host checks (which are tested on their own below).""" + real_env = gitsource._env + + def env(tmp, source, kind): + e = real_env(tmp, source, kind) + e["GIT_ALLOW_PROTOCOL"] = "https:ssh:file" + return e + with mock.patch.object(gitsource, "parse_url", return_value=("https", "git.test")), \ + mock.patch.object(gitsource, "_check_host_address"), \ + mock.patch.object(gitsource, "_env", side_effect=env), \ + mock.patch.object(gitsource, "_GIT_CONFIG", gitsource._GIT_CONFIG + ("-c", "protocol.file.allow=always")): + yield + + +def _members(archive: bytes) -> dict: + with tarfile.open(fileobj=io.BytesIO(gzip.decompress(archive)), mode="r:") as tar: + return {m.name: (m.mode, tar.extractfile(m).read()) for m in tar if m.isreg()} + + +class InputCheckTests(SimpleTestCase): + def test_only_https_and_ssh_urls(self): + for good in ("https://github.com/acme/app.git", "https://git.lan:3000/acme/app", + "git@github.com:acme/app.git", "ssh://git@git.lan:2222/acme/app.git"): + with self.subTest(good=good): + gitsource.parse_url(good) + for bad in ("file:///etc", "/srv/repo", "ext::sh -c id", "https://user:pw@github.com/a/b", + "-uhttps://x/y", "http://github.com/a/b", "git://github.com/a/b", + "https://github.com/a/../b", "https://github.com/a/b?x=1", "--upload-pack=id", + "git@github.com:-oProxyCommand=id", ""): + with self.subTest(bad=bad), self.assertRaises(GitSourceError): + gitsource.parse_url(bad) + + def test_ref_and_path_checks(self): + for bad in ("-x", "a..b", "a@{1}", "x.lock", "/main", "main/", ""): + with self.subTest(ref=bad), self.assertRaises(GitSourceError): + gitsource.check_ref(bad, "branch") + self.assertEqual(gitsource.check_path("deploy/compose.yaml"), ("deploy", "compose.yaml")) + self.assertEqual(gitsource.check_path("docker-compose.yml"), ("", "docker-compose.yml")) + for bad in ("../x.yaml", "/abs.yaml", "deploy/notes.txt", "", "a/./b.yaml"): + with self.subTest(path=bad), self.assertRaises(GitSourceError): + gitsource.check_path(bad) + + def test_the_server_will_not_fetch_from_itself_or_metadata(self): + def resolving(ip): + return mock.patch.object(socket, "getaddrinfo", + return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 443))]) + for ip in ("127.0.0.1", "169.254.169.254", "0.0.0.0"): + with self.subTest(ip=ip), resolving(ip), self.assertRaises(GitSourceError): + gitsource._check_host_address("git.example", 443) + with resolving("10.0.0.5"): + gitsource._check_host_address("gitea.lan", 443) # a LAN Git server is fine + + +class FetchTests(SimpleTestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.repo, self.head = _make_repo(self.root) + + def tearDown(self): + self.tmp.cleanup() + + def _fetch(self, **kw): + with _local_git(): + return gitsource.fetch(Source(url=self.repo, path="deploy/compose.yaml", **kw)) + + def test_a_branch_fetch_packs_only_the_compose_folder(self): + got = self._fetch(branch="main") + self.assertEqual(got.commit, self.head) + self.assertEqual(got.compose_yaml, COMPOSE) + self.assertEqual(got.compose_file, "compose.yaml") + self.assertEqual(got.sha256, hashlib.sha256(got.archive).hexdigest()) + files = _members(got.archive) + self.assertEqual(sorted(files), ["app/Dockerfile", "compose.yaml", "run.sh"]) + self.assertEqual(files["run.sh"][0], 0o755) + self.assertEqual(files["compose.yaml"][0], 0o644) + self.assertEqual(self._fetch(branch="main").sha256, got.sha256, "the archive is deterministic") + + def test_pins_hold_while_the_branch_moves(self): + Path(self.repo, "deploy", "compose.yaml").write_text(COMPOSE + " # changed\n") + _run(self.repo, "commit", "-q", "-am", "second") + new_head = _run(self.repo, "rev-parse", "HEAD") + self.assertEqual(self._fetch(branch="main").commit, new_head) + self.assertEqual(self._fetch(pin="v1").commit, self.head) # annotated tag → its commit + self.assertEqual(self._fetch(pin=self.head).commit, self.head) # an exact commit + self.assertEqual(self._fetch(pin="v1").compose_yaml, COMPOSE) + + def test_a_symlink_in_the_folder_is_refused(self): + os.symlink("/etc/passwd", Path(self.repo, "deploy", "passwd")) + _run(self.repo, "add", ".") + _run(self.repo, "commit", "-q", "-m", "link") + with self.assertRaisesRegex(GitSourceError, "regular files only"): + self._fetch(branch="main") + + def test_missing_branch_and_missing_compose_file(self): + with self.assertRaisesRegex(GitSourceError, "not found"): + self._fetch(branch="nope") + with _local_git(), self.assertRaisesRegex(GitSourceError, "not in the repository"): + gitsource.fetch(Source(url=self.repo, path="deploy/other.yaml")) + + def test_a_token_is_never_on_a_command_line(self): + seen = [] + real = subprocess.Popen + + def spy(args, **kw): + seen.append((args, kw.get("env") or {})) + return real(args, **kw) + with mock.patch.object(gitsource.subprocess, "Popen", side_effect=spy): + self._fetch(branch="main", credential=("token", "bot", "s3cr3t-token", "")) + self.assertTrue(seen) + for args, env in seen: + self.assertNotIn("s3cr3t-token", " ".join(args)) + self.assertEqual(env["VIGIL_GIT_TOKEN"], "s3cr3t-token") + self.assertTrue(env["GIT_ASKPASS"].endswith("askpass.sh")) + self.assertEqual(env["GIT_ALLOW_PROTOCOL"], "https:ssh:file") # file only in this test + + def test_ssh_needs_a_key_and_pinned_host_keys(self): + with tempfile.TemporaryDirectory() as tmp: + src = Source(url="git@github.com:a/b.git") + with self.assertRaises(GitSourceError): + gitsource._env(tmp, src, "ssh") + src.credential = ("ssh_key", "", "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n", "") + with self.assertRaisesRegex(GitSourceError, "known_hosts"): + gitsource._env(tmp, src, "ssh") + src.credential = ("ssh_key", "", "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n", + "github.com ssh-ed25519 AAAA") + env = gitsource._env(tmp, src, "ssh") + self.assertIn("StrictHostKeyChecking=yes", env["GIT_SSH_COMMAND"]) + self.assertIn("GlobalKnownHostsFile=/dev/null", env["GIT_SSH_COMMAND"]) + self.assertEqual(os.stat(Path(tmp, "id_key")).st_mode & 0o777, 0o600) + + +class GitStackApiTests(TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.repo, self.head = _make_repo(Path(self.tmp.name)) + self.host = Host.objects.create(hostname="nas", agent_token="nastok", status=Host.Status.ONLINE, + mode="managed", agent_features=["stack_source"]) + self.other = Host.objects.create(hostname="other", agent_token="othertok", + status=Host.Status.ONLINE, mode="managed") + self.user = get_user_model().objects.create_user("a", password="x") + UserProfile.objects.create(user=self.user, role=Role.ADMIN) + self.api = APIClient() + self.api.force_authenticate(self.user) + self.local = _local_git() + self.local.__enter__() + self.addCleanup(self.local.__exit__, None, None, None) + + def tearDown(self): + self.tmp.cleanup() + + def _create(self): + return self.api.post("/api/v1/stacks/", { + "host_id": str(self.host.id), "name": "web", + "git": {"url": self.repo, "branch": "main", "path": "deploy/compose.yaml"}, + "env_text": "TOKEN=abc\n"}, format="json") + + def test_create_pull_and_no_edits_in_vigil(self): + resp = self._create() + self.assertEqual(resp.status_code, 201, resp.content) + body = resp.json() + self.assertEqual(body["git"]["commit"], self.head) + self.assertEqual(body["compose_yaml"], COMPOSE) + sid = body["id"] + edit = self.api.put(f"/api/v1/stacks/{sid}/", {"compose_yaml": COMPOSE}, format="json") + self.assertEqual(edit.status_code, 400) + self.assertIn("comes from Git", edit.json()["detail"]) + same = self.api.post(f"/api/v1/stacks/{sid}/pull/", format="json").json() + self.assertFalse(same["changed"]) + Path(self.repo, "deploy", "compose.yaml").write_text(COMPOSE + " # v2\n") + _run(self.repo, "commit", "-q", "-am", "second") + moved = self.api.post(f"/api/v1/stacks/{sid}/pull/", format="json").json() + self.assertTrue(moved["changed"]) + self.assertEqual(moved["stack"]["revision"], 2) + self.assertEqual(moved["stack"]["git"]["commit"], _run(self.repo, "rev-parse", "HEAD")) + + def test_a_refused_compose_file_makes_no_stack(self): + Path(self.repo, "deploy", "compose.yaml").write_text( + "services:\n a:\n image: x\n privileged: true\n") + _run(self.repo, "commit", "-q", "-am", "bad") + resp = self._create() + self.assertEqual(resp.status_code, 400) + self.assertIn("privileged", resp.json()["detail"]) + from .models import ManagedStack + self.assertFalse(ManagedStack.objects.exists()) + + def test_deploy_ships_a_ticket_and_the_archive_hash(self): + from apps.tasks.models import Task + + from .models import SourceSnapshot + sid = self._create().json()["id"] + with mock.patch(_TOTP, return_value=None): + resp = self.api.post(f"/api/v1/stacks/{sid}/deploy/", {"totp": "1"}, format="json") + self.assertEqual(resp.status_code, 201, resp.content) + params = Task.objects.get(pk=resp.json()["task"]).params["steps"][0]["params"] + snap = SourceSnapshot.objects.get() + self.assertEqual(params["source_sha256"], snap.sha256) + self.assertEqual(params["source_commit"], self.head) + self.assertNotIn("TOKEN=abc", str(params)) + + ticket = params["source_ticket"] + url = f"/api/v1/agent/stack-source/{ticket}/" + self.assertEqual(self.client.get(url, HTTP_AUTHORIZATION="Bearer othertok").status_code, 404) + got = self.client.get(url, HTTP_AUTHORIZATION="Bearer nastok") + self.assertEqual(got.status_code, 200) + self.assertEqual(hashlib.sha256(got.content).hexdigest(), snap.sha256) + self.assertEqual(self.client.get(url, HTTP_AUTHORIZATION="Bearer nastok").status_code, 404, + "a ticket is good once") + + def test_an_agent_without_stack_source_is_refused(self): + sid = self._create().json()["id"] + self.host.agent_features = [] + self.host.save(update_fields=["agent_features"]) + with mock.patch(_TOTP, return_value=None): + resp = self.api.post(f"/api/v1/stacks/{sid}/deploy/", {"totp": "1"}, format="json") + self.assertEqual(resp.status_code, 400) + self.assertIn("too old", resp.json()["detail"]) + + def test_credentials_are_write_only_and_need_totp(self): + url = "/api/v1/stacks/git-credentials/" + body = {"name": "gh", "kind": "token", "secret": "ghp_secret", "username": "bot"} + self.assertEqual(self.api.post(url, body, format="json").status_code, 401) + with mock.patch(_TOTP, return_value=None): + self.assertEqual(self.api.post(url, body, format="json").status_code, 201) + ssh = {"name": "deploy", "kind": "ssh_key", + "secret": "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n-----END OPENSSH PRIVATE KEY-----"} + self.assertEqual(self.api.post(url, ssh, format="json").status_code, 400, "no host key pinned") + listed = self.api.get(url).json()["results"] + self.assertEqual([c["name"] for c in listed], ["gh"]) + self.assertNotIn("ghp_secret", str(listed)) + + def test_viewers_cannot_touch_git_stacks(self): + viewer = get_user_model().objects.create_user("v", password="x") + api = APIClient() + api.force_authenticate(viewer) + self.assertEqual(api.get("/api/v1/stacks/git-credentials/").status_code, 403) + sid = self._create().json()["id"] + self.assertEqual(api.post(f"/api/v1/stacks/{sid}/pull/").status_code, 403) diff --git a/server/apps/stacks/urls.py b/server/apps/stacks/urls.py index 2c097e87..b3bdd789 100644 --- a/server/apps/stacks/urls.py +++ b/server/apps/stacks/urls.py @@ -1,16 +1,20 @@ from django.urls import path -from . import registries, views +from . import git_views, registries, views urlpatterns = [ path("", views.stack_index, name="stack-index"), path("adopt/", views.stack_adopt, name="stack-adopt"), path("validate/", views.stack_validate, name="stack-validate"), + path("git-credentials/", git_views.git_credential_index, name="stack-git-credentials"), + path("git-credentials//", git_views.git_credential_detail, + name="stack-git-credential-detail"), path("registries/", registries.registry_index, name="stack-registries"), path("registries//", registries.registry_detail, name="stack-registry-detail"), path("/", views.stack_detail, name="stack-detail"), path("/revisions/", views.stack_revisions, name="stack-revisions"), path("/env/reveal/", views.stack_env_reveal, name="stack-env-reveal"), path("/deploy/", views.stack_deploy, name="stack-deploy"), + path("/pull/", git_views.stack_pull, name="stack-pull"), path("/remove/", views.stack_remove, name="stack-remove"), ] diff --git a/server/apps/stacks/views.py b/server/apps/stacks/views.py index 3bc809f9..98585a73 100644 --- a/server/apps/stacks/views.py +++ b/server/apps/stacks/views.py @@ -40,6 +40,11 @@ def _row(s: ManagedStack) -> dict: "name": s.name, "compose_yaml": s.compose_yaml, "revision": s.revision, "working_dir": s.working_dir, "adopted": s.adopted, "compose_file": s.compose_file, "adopt_report": s.adopt_report or {}, + "git": ({"url": s.git_url, "branch": s.git_branch, "pin": s.git_pin, "path": s.git_path, + "credential_id": str(s.git_credential_id) if s.git_credential_id else None, + "commit": (s.revisions.filter(number=s.revision).values_list("git_commit", flat=True) + .first() or "")} + if s.git_url else None), "env": [{"key": k, "set": bool(v)} for k, v in _env_pairs(s)], "updated_at": s.updated_at.isoformat() if s.updated_at else None} @@ -73,6 +78,9 @@ def _merge_env(stack: ManagedStack | None, data) -> list[tuple[str, str]]: def _save(request, stack: ManagedStack, data, *, note: str) -> Response | None: + if stack.git_url and "compose_yaml" in data: + return Response({"detail": "this stack's compose file comes from Git: change it in the " + "repository and pull"}, status=status.HTTP_400_BAD_REQUEST) try: if "compose_yaml" in data: validate_compose(str(data["compose_yaml"])) @@ -113,15 +121,33 @@ def stack_index(request): if ManagedStack.objects.filter(host=host, name=name).exists(): return Response({"detail": f"{host.hostname} already has a stack called {name}"}, status=status.HTTP_400_BAD_REQUEST) - if "compose_yaml" not in request.data: - return Response({"detail": "compose_yaml is required"}, status=status.HTTP_400_BAD_REQUEST) stack = ManagedStack(host=host, name=name, created_by=request.user, working_dir=f"{STACKS_ROOT}/{name}") + if "git" in request.data: + return _create_from_git(request, stack) + if "compose_yaml" not in request.data: + return Response({"detail": "compose_yaml (or git) is required"}, + status=status.HTTP_400_BAD_REQUEST) if error := _save(request, stack, request.data, note="created"): return error return Response(_row(stack), status=status.HTTP_201_CREATED) +def _create_from_git(request, stack: ManagedStack) -> Response: + """A new stack whose compose file comes from a repository: fetch it, + check it like any compose file, and store revision 1 with its source.""" + from . import git_views, gitsource + + try: + git_views.apply_git_settings(stack, request.data) + if "env_text" in request.data or "env" in request.data: + stack.env_encrypted = encrypt_secret(render_env(_merge_env(None, request.data))) + git_views.pull(stack, request.user, note="created from Git") + except (gitsource.GitSourceError, StackError) as exc: + return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST) + return Response(_row(stack), status=status.HTTP_201_CREATED) + + @api_view(["POST"]) @permission_classes([IsAuthenticated, IsAdmin]) def stack_validate(request): @@ -153,6 +179,19 @@ def stack_detail(request, stack_id): # stack_remove task (phase 09). This only forgets Vigil's copy. stack.delete() return Response(status=status.HTTP_204_NO_CONTENT) + if "git" in request.data: + # New repository settings take effect by pulling them. + from . import git_views, gitsource + if not stack.git_url: + return Response({"detail": "a stack edited in Vigil cannot be switched to Git; " + "create a new stack from the repository"}, + status=status.HTTP_400_BAD_REQUEST) + try: + git_views.apply_git_settings(stack, request.data) + git_views.pull(stack, request.user, note="Git settings changed") + except (gitsource.GitSourceError, StackError) as exc: + return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST) + return Response(_row(stack)) if error := _save(request, stack, request.data, note=str(request.data.get("note") or "edited")): return error return Response(_row(stack)) @@ -236,8 +275,22 @@ def stack_deploy(request, stack_id): return refused if denied := _confirmed(request): return denied + source = {} + if stack.git_url and "stack_source" not in set(stack.host.agent_features or []): + # An older agent would ignore the source ticket and build with no source. + return Response({"detail": f"{stack.host.hostname}'s agent is too old for stacks from Git — " + "update the agent first"}, status=status.HTTP_400_BAD_REQUEST) + if stack.git_url: + # Tracking a branch means a deploy takes its latest commit; a pinned + # stack re-fetches the same one (and makes no revision). + from . import git_views, gitsource + try: + git_views.pull(stack, request.user) + source = git_views.deploy_source_params(stack) + except (gitsource.GitSourceError, StackError) as exc: + return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST) params = {"project": stack.name, "compose": stack.compose_yaml, - "working_dir": stack.working_dir, "revision": stack.revision} + "working_dir": stack.working_dir, "revision": stack.revision, **source} if stack.compose_file != "compose.yaml": params["compose_file"] = stack.compose_file if bytes(stack.env_encrypted or b""): diff --git a/server/apps/tasks/management/commands/render_wiki_actions.py b/server/apps/tasks/management/commands/render_wiki_actions.py index aeb28537..7179603b 100644 --- a/server/apps/tasks/management/commands/render_wiki_actions.py +++ b/server/apps/tasks/management/commands/render_wiki_actions.py @@ -416,6 +416,9 @@ ("stack_deploy", "working_dir"): "Where the stack lives on the host — /opt/vigil/stacks/ for a new one.", ("stack_deploy", "env_ticket"): "Set by Vigil, not by hand: a one-time id the agent redeems for the .env just before deploying. The secrets never ride in the task.", ("stack_deploy", "revision"): "The stack revision being deployed, for the record.", + ("stack_deploy", "source_ticket"): "Set by Vigil for a stack from Git: a one-time id the agent redeems for the revision's packed source folder. The repository credential never leaves the server.", + ("stack_deploy", "source_sha256"): "Set by Vigil for a stack from Git: the sha256 the source archive must have. The agent refuses any other archive before placing a file.", + ("stack_deploy", "source_commit"): "Set by Vigil for a stack from Git: the commit the source folder was taken from, for the record.", ("stack_deploy", "compose_file"): "The compose file's name in working_dir — compose.yaml unless the stack was adopted with its own.", ("container_rollback", "container_name"): "The container to roll back.", ("container_rollback", "image"): "The image to go back to — a repo digest (repo@sha256:…) or an image id. A container's page offers the one it ran before its last update.", diff --git a/server/apps/tasks/registry.py b/server/apps/tasks/registry.py index 785359cf..58423fc9 100644 --- a/server/apps/tasks/registry.py +++ b/server/apps/tasks/registry.py @@ -174,7 +174,8 @@ "label": "Deploy a Vigil-managed compose stack", "risk": "high", "required": ["project", "compose", "working_dir"], - "optional": ["env_ticket", "revision", "compose_file"], + "optional": ["env_ticket", "revision", "compose_file", + "source_ticket", "source_sha256", "source_commit"], "outputs": {"project": "str", "revision": "int"}, }, "stack_remove": { diff --git a/server/apps/tasks/spec.py b/server/apps/tasks/spec.py index 2f229314..dd2921e0 100644 --- a/server/apps/tasks/spec.py +++ b/server/apps/tasks/spec.py @@ -529,6 +529,17 @@ def _validate_stack_params(params: dict[str, Any], position: int, action_type: s ticket = params.get("env_ticket") if ticket not in (None, "") and not (isinstance(ticket, str) and _UUID_RE.match(ticket)): raise SpecError(f"{where}: 'env_ticket' must be a ticket id") + # A Git stack's source folder: a one-time ticket, the sha256 the + # archive must have (the agent refuses anything else), its commit. + source = [params.get(k) for k in ("source_ticket", "source_sha256", "source_commit")] + if any(v not in (None, "") for v in source): + ticket, sha, commit = source + if not (isinstance(ticket, str) and _UUID_RE.match(ticket)): + raise SpecError(f"{where}: 'source_ticket' must be a ticket id") + if not (isinstance(sha, str) and re.fullmatch(r"[0-9a-f]{64}", sha)): + raise SpecError(f"{where}: 'source_sha256' must be 64 hex characters") + if not (isinstance(commit, str) and re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", commit)): + raise SpecError(f"{where}: 'source_commit' must be a commit id") else: delete = params.get("delete_files") if delete is not None and not isinstance(delete, bool): diff --git a/server/vigil/hooks.py b/server/vigil/hooks.py index c78c1814..db9db575 100644 --- a/server/vigil/hooks.py +++ b/server/vigil/hooks.py @@ -49,6 +49,7 @@ "task_completed", # payload: task # Managed stacks (M11). Business audit subscribes; Free has no listener. "stack_saved", # payload: stack, user, revision + "git_credential_added", # payload: credential, user "stack_env_revealed", # payload: stack, user "stack_deployed", # payload: stack, user, task # Reprovisioning (docs/reprovisioning.md §4.5). Audit subscribes to these diff --git a/server/vigil/urls.py b/server/vigil/urls.py index 0eaf8986..ed4acde1 100644 --- a/server/vigil/urls.py +++ b/server/vigil/urls.py @@ -17,6 +17,7 @@ from apps.hosts.enrollment import replacement_candidate from apps.hosts.logtail import agent_log_lines from apps.stacks.registries import agent_registry_auth +from apps.stacks.git_views import agent_stack_source from apps.stacks.views import agent_stack_adopt, agent_stack_env from apps.hosts.views import checkin, register from apps.playbooks.urls import legacy_urlpatterns as _legacy_playbook_urls @@ -108,6 +109,7 @@ def dashboard(request): path("api/v1/checkin", checkin, name="checkin"), path("api/v1/agent/log-tail//", agent_log_lines, name="agent-log-tail"), path("api/v1/agent/stack-env//", agent_stack_env, name="agent-stack-env"), + path("api/v1/agent/stack-source//", agent_stack_source, name="agent-stack-source"), path("api/v1/agent/stack-adopt//", agent_stack_adopt, name="agent-stack-adopt"), path("api/v1/agent/registry-auth/", agent_registry_auth, name="agent-registry-auth"), path("api/v1/hosts/", include("apps.hosts.urls")), diff --git a/wiki/vigil-wiki.html b/wiki/vigil-wiki.html index 0cd5b41e..ceb06947 100644 --- a/wiki/vigil-wiki.html +++ b/wiki/vigil-wiki.html @@ -3251,8 +3251,8 @@

Container management

params: container_name: nextcloud
-
stack_deployhighDeploy a Vigil-managed compose stackproject, compose, working_dir
-
ParamMeaning
projectrequiredThe compose project name.
composerequiredThe compose file itself, as Vigil stores it. Checked like the stack editor checks it.
working_dirrequiredWhere the stack lives on the host — /opt/vigil/stacks/<name> for a new one.
env_ticketoptionalSet by Vigil, not by hand: a one-time id the agent redeems for the .env just before deploying. The secrets never ride in the task.
revisionoptionalThe stack revision being deployed, for the record.
compose_fileoptionalThe compose file's name in working_dir — compose.yaml unless the stack was adopted with its own.
+
stack_deployhighDeploy a Vigil-managed compose stackproject, compose, working_dir
+
ParamMeaning
projectrequiredThe compose project name.
composerequiredThe compose file itself, as Vigil stores it. Checked like the stack editor checks it.
working_dirrequiredWhere the stack lives on the host — /opt/vigil/stacks/<name> for a new one.
env_ticketoptionalSet by Vigil, not by hand: a one-time id the agent redeems for the .env just before deploying. The secrets never ride in the task.
revisionoptionalThe stack revision being deployed, for the record.
compose_fileoptionalThe compose file's name in working_dir — compose.yaml unless the stack was adopted with its own.
source_ticketoptionalSet by Vigil for a stack from Git: a one-time id the agent redeems for the revision's packed source folder. The repository credential never leaves the server.
source_sha256optionalSet by Vigil for a stack from Git: the sha256 the source archive must have. The agent refuses any other archive before placing a file.
source_commitoptionalSet by Vigil for a stack from Git: the commit the source folder was taken from, for the record.
OutputTypeMeaning
statusstrok or skipped, as a later step sees it (a failed step stops the task).
projectstrThe stack deployed.
revisionintThe revision deployed (0 when not given).

Read one later as ${{ steps.<id>.result.<field> }}.

YAML
name: Deploy media
 description: "What Vigil sends when you deploy the media stack."

From 1da23f5255063dab64648e9a9edede6832d02ff0 Mon Sep 17 00:00:00 2001
From: Connor Haggerty 
Date: Fri, 9 Oct 2026 20:24:06 -0400
Subject: [PATCH 54/68] feat(agent): place a Git stack's source folder, checked
 and without following links
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

stack_deploy with source_ticket / source_sha256 (advertised as the
`stack_source` feature) redeems the ticket for the packed folder, refuses it
unless its sha256 is the one in the signed task, and accepts regular files
at plain relative paths only.

The files are written as root into the stack folder through directory file
descriptors: each folder opened relative to the last with O_NOFOLLOW, each
file replaced with unlink + O_EXCL|O_NOFOLLOW — a link planted by a
container or a user cannot redirect a write. Files an earlier commit placed
and this one dropped are removed, known from a manifest in the agent's data
dir; data folders and the .env are never touched. The checked compose text
is then written over the repo's copy, and the resolved-config check and `up`
run as for any stack.

Verified on the Ubuntu test VM: docker/awesome-compose's nginx-golang placed,
checked, built from its source and started; a wrong hash refused.
---
 agent/tests/test_branches_agent.py  |   4 +-
 agent/tests/test_relevant_eval.py   |   2 +-
 agent/tests/test_stack_source.py    | 176 ++++++++++++++++++++++++++++
 agent/vigil_agent/actions/stacks.py | 136 ++++++++++++++++++++-
 agent/vigil_agent/client.py         |  20 ++++
 agent/vigil_agent/features.py       |   4 +-
 6 files changed, 334 insertions(+), 8 deletions(-)
 create mode 100644 agent/tests/test_stack_source.py

diff --git a/agent/tests/test_branches_agent.py b/agent/tests/test_branches_agent.py
index 04b5404c..1c2114a0 100644
--- a/agent/tests/test_branches_agent.py
+++ b/agent/tests/test_branches_agent.py
@@ -313,7 +313,7 @@ def setUp(self):
 
     def test_features_include_branches(self):
         self.assertIn("branches", features.FEATURES)
-        self.assertEqual(features.FEATURES, ("relevant", "branches", "boost", "signed_v2"))
+        self.assertEqual(features.FEATURES, ("relevant", "branches", "boost", "signed_v2", "stack_source"))
 
     def test_checkin_sends_branches(self):
         resp = MagicMock()
@@ -321,7 +321,7 @@ def test_checkin_sends_branches(self):
         with patch.object(client.requests, "post", return_value=resp) as post:
             client.checkin(self.config, {})
         payload = post.call_args.kwargs["json"]
-        self.assertEqual(payload["features"], ["relevant", "branches", "boost", "signed_v2"])
+        self.assertEqual(payload["features"], ["relevant", "branches", "boost", "signed_v2", "stack_source"])
 
 
 if __name__ == "__main__":
diff --git a/agent/tests/test_relevant_eval.py b/agent/tests/test_relevant_eval.py
index b3bf3762..dae4d619 100644
--- a/agent/tests/test_relevant_eval.py
+++ b/agent/tests/test_relevant_eval.py
@@ -248,7 +248,7 @@ def test_checkin_sends_features(self):
         with patch.object(client.requests, "post", return_value=resp) as post:
             client.checkin(self.config, {})
         payload = post.call_args.kwargs["json"]
-        self.assertEqual(payload["features"], ["relevant", "branches", "boost", "signed_v2"])
+        self.assertEqual(payload["features"], ["relevant", "branches", "boost", "signed_v2", "stack_source"])
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/agent/tests/test_stack_source.py b/agent/tests/test_stack_source.py
new file mode 100644
index 00000000..a7161c17
--- /dev/null
+++ b/agent/tests/test_stack_source.py
@@ -0,0 +1,176 @@
+"""A Git stack's source folder on the host (2026.14.1): the archive must be
+the one the signed task names, may hold regular files only, and is written as
+root without following any link a container or user planted in the folder."""
+import tests._safety_net  # noqa: F401 — the guard, even when this file is run or imported on its own
+
+import dataclasses
+import gzip
+import hashlib
+import io
+import json
+import os
+import stat
+import sys
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+# isort: split
+from vigil_agent import client, executor  # noqa: F401 — executor first, it imports stacks
+from vigil_agent.actions import stacks
+from vigil_agent.config import AgentConfig
+
+TICKET = "6f1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"
+COMPOSE = "services:\n  app:\n    build: ./app\n"
+
+
+def _archive(entries) -> bytes:
+    """entries: (name, bytes|None for a dir, mode) or ("link", name, target)."""
+    raw = io.BytesIO()
+    with tarfile.open(fileobj=raw, mode="w") as tar:
+        for entry in entries:
+            if entry[0] == "link":
+                info = tarfile.TarInfo(entry[1])
+                info.type, info.linkname = tarfile.SYMTYPE, entry[2]
+                tar.addfile(info)
+                continue
+            name, data, mode = entry
+            info = tarfile.TarInfo(name)
+            if data is None:
+                info.type = tarfile.DIRTYPE
+                tar.addfile(info)
+            else:
+                info.size, info.mode = len(data), mode
+                tar.addfile(info, io.BytesIO(data))
+    return gzip.compress(raw.getvalue())
+
+
+GOOD = _archive([("compose.yaml", COMPOSE.encode(), 0o644), ("app", None, 0o755),
+                 ("app/Dockerfile", b"FROM nginx\n", 0o644), ("run.sh", b"#!/bin/sh\n", 0o755)])
+
+
+def _sha(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+class SourceFilesTests(unittest.TestCase):
+    def test_the_archive_must_be_the_signed_one(self):
+        files = stacks._source_files(GOOD, _sha(GOOD))
+        self.assertEqual(sorted(files), ["app/Dockerfile", "compose.yaml", "run.sh"])
+        self.assertEqual(files["run.sh"][1], 0o755)
+        with self.assertRaisesRegex(ValueError, "does not match"):
+            stacks._source_files(GOOD, "0" * 64)
+
+    def test_only_regular_files_at_plain_paths(self):
+        for bad in (_archive([("link", "etc", "/etc")]), _archive([("../x", b"x", 0o644)]),
+                    _archive([("/abs", b"x", 0o644)]), _archive([("a//b", b"x", 0o644)])):
+            with self.subTest(), self.assertRaises(ValueError):
+                stacks._source_files(bad, _sha(bad))
+
+
+class PlaceSourceTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        self.work = self.root / "web"
+        self.work.mkdir()
+        self.manifest = self.root / "data" / "web.source.json"
+        self.outside = self.root / "outside"
+        self.outside.mkdir()
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def _place(self, archive=GOOD):
+        stacks._place_source(self.work, stacks._source_files(archive, _sha(archive)), self.manifest)
+
+    def test_files_land_with_their_modes_and_a_private_manifest(self):
+        (self.work / "data").mkdir()
+        (self.work / "data" / "db.sqlite").write_text("keep me")
+        (self.work / ".env").write_text("A=1\n")
+        self._place()
+        self.assertEqual((self.work / "app" / "Dockerfile").read_text(), "FROM nginx\n")
+        self.assertEqual(stat.S_IMODE(os.stat(self.work / "run.sh").st_mode), 0o755)
+        self.assertEqual((self.work / "data" / "db.sqlite").read_text(), "keep me")
+        self.assertEqual((self.work / ".env").read_text(), "A=1\n")
+        self.assertEqual(json.loads(self.manifest.read_text()), ["app/Dockerfile", "compose.yaml", "run.sh"])
+        self.assertEqual(stat.S_IMODE(os.stat(self.manifest).st_mode), 0o600)
+
+    def test_a_planted_folder_link_is_refused_and_nothing_escapes(self):
+        os.symlink(self.outside, self.work / "app")
+        with self.assertRaisesRegex(ValueError, "not a plain folder"):
+            self._place()
+        self.assertEqual(list(self.outside.iterdir()), [])
+
+    def test_a_planted_file_link_is_replaced_not_written_through(self):
+        target = self.outside / "cron"
+        target.write_text("untouched")
+        os.symlink(target, self.work / "run.sh")
+        self._place()
+        self.assertFalse((self.work / "run.sh").is_symlink())
+        self.assertEqual(target.read_text(), "untouched")
+
+    def test_files_a_newer_commit_dropped_are_removed_and_only_those(self):
+        self._place()
+        (self.work / "mine.txt").write_text("not from the repo")
+        smaller = _archive([("compose.yaml", COMPOSE.encode(), 0o644)])
+        self._place(smaller)
+        self.assertFalse((self.work / "run.sh").exists())
+        self.assertFalse((self.work / "app" / "Dockerfile").exists())
+        self.assertTrue((self.work / "mine.txt").exists())
+
+    def test_a_forged_manifest_entry_cannot_reach_outside(self):
+        victim = self.outside / "victim"
+        victim.write_text("x")
+        self.manifest.parent.mkdir(parents=True)
+        self.manifest.write_text(json.dumps(["../outside/victim", "/etc/passwd"]))
+        os.symlink(self.outside, self.work / "lnk")
+        self.manifest.write_text(json.dumps(["../outside/victim", "lnk/victim"]))
+        self._place()
+        self.assertTrue(victim.exists())
+
+
+class DeployWithSourceTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        self.cfg = dataclasses.replace(AgentConfig(server_url="https://v", agent_token="t", mode="full_control"),
+                                       data_dir=self.root / "agent-data")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def _deploy(self, archive, sha):
+        params = {"project": "web", "compose": COMPOSE, "working_dir": str(self.root / "web"),
+                  "source_ticket": TICKET, "source_sha256": sha, "source_commit": "a" * 40}
+        with patch.object(client, "fetch_stack_source", return_value=archive) as fetch, \
+                patch.object(stacks, "_safe_workdir"), \
+                patch.object(stacks, "_check_resolved", return_value={"services": {}}), \
+                patch.object(stacks, "_up", return_value="up"), \
+                patch.object(executor.collector, "request_docker_recheck"):
+            stacks._stack_deploy(params, self.cfg)
+        fetch.assert_called_once_with(self.cfg, TICKET)
+
+    def test_a_git_stack_gets_its_source_then_the_checked_compose(self):
+        self._deploy(GOOD, _sha(GOOD))
+        self.assertEqual((self.root / "web" / "app" / "Dockerfile").read_text(), "FROM nginx\n")
+        self.assertEqual((self.root / "web" / "compose.yaml").read_text(), COMPOSE)
+        self.assertTrue((self.root / "agent-data" / "stacks" / "web.source.json").exists())
+
+    def test_a_tampered_archive_writes_nothing(self):
+        with self.assertRaisesRegex(ValueError, "does not match"):
+            self._deploy(GOOD, "0" * 64)
+        self.assertFalse((self.root / "web").exists())
+
+    def test_half_a_source_param_is_refused(self):
+        with self.assertRaises(ValueError):
+            stacks._source_params({"source_ticket": TICKET})
+        self.assertIsNone(stacks._source_params({}))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/agent/vigil_agent/actions/stacks.py b/agent/vigil_agent/actions/stacks.py
index 75b1c25c..7a218e3d 100644
--- a/agent/vigil_agent/actions/stacks.py
+++ b/agent/vigil_agent/actions/stacks.py
@@ -107,6 +107,126 @@ def _read_plain(path: Path) -> str | None:
         os.close(fd)
 
 
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+MAX_SOURCE_TREE = 100 * 1024 * 1024
+MAX_SOURCE_FILES = 5000
+
+
+def _source_files(archive: bytes, sha256: str) -> dict[str, tuple[bytes, int]]:
+    """A Git stack's source folder, from the archive the server packed: the
+    hash must be the one in the signed task, and it may hold regular files
+    only, at plain relative paths. {path: (bytes, mode)}."""
+    import gzip
+    import hashlib
+    import io
+    import tarfile
+
+    if hashlib.sha256(archive).hexdigest() != sha256:
+        raise ValueError("the stack's source does not match the signed task — refusing it")
+    files: dict[str, tuple[bytes, int]] = {}
+    total = 0
+    with tarfile.open(fileobj=io.BytesIO(gzip.decompress(archive)), mode="r:") as tar:
+        for member in tar:
+            parts = member.name.split("/")
+            if (member.name.startswith("/") or "\\" in member.name
+                    or any(p in ("", ".", "..") for p in parts)):
+                raise ValueError(f"the stack's source has an unusable path: {member.name!r}")
+            if member.isdir():
+                continue
+            if not member.isreg():
+                raise ValueError(f"the stack's source holds a non-regular file: {member.name!r}")
+            total += member.size
+            if len(files) >= MAX_SOURCE_FILES or total > MAX_SOURCE_TREE:
+                raise ValueError("the stack's source is too large")
+            files[member.name] = (tar.extractfile(member).read(), 0o755 if member.mode & 0o111 else 0o644)
+    return files
+
+
+def _open_dir(parent_fd: int, name: str, *, create: bool) -> int:
+    """A folder under *parent_fd*, never through a symlink."""
+    if create:
+        try:
+            os.mkdir(name, 0o755, dir_fd=parent_fd)
+        except FileExistsError:
+            pass
+    try:
+        return os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd)
+    except OSError as exc:      # ELOOP / ENOTDIR: a symlink or a file where a folder belongs
+        raise ValueError(f"refusing to place the stack's source: {name} is not a plain folder") from exc
+
+
+def _place_source(workdir: Path, files: dict[str, tuple[bytes, int]], manifest: Path) -> None:
+    """Write the source folder into the stack folder as root without following
+    a link anywhere: each folder is opened relative to the last with
+    O_NOFOLLOW, each file replaced (unlink + O_EXCL|O_NOFOLLOW). Files a
+    previous deploy placed and this source no longer has are removed — known
+    from a manifest in the agent's own data dir, which no container can
+    reach — and nothing else in the folder (data, the .env) is touched."""
+    import json
+
+    root_fd = os.open(workdir, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        for rel, (data, mode) in sorted(files.items()):
+            *folders, name = rel.split("/")
+            fd = os.dup(root_fd)
+            try:
+                for folder in folders:
+                    nxt = _open_dir(fd, folder, create=True)
+                    os.close(fd)
+                    fd = nxt
+                try:
+                    os.unlink(name, dir_fd=fd)
+                except FileNotFoundError:
+                    pass
+                except IsADirectoryError as exc:
+                    raise ValueError(f"refusing to place {rel}: a folder of that name is in the way") from exc
+                out = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode, dir_fd=fd)
+                try:
+                    os.write(out, data)
+                    os.fchmod(out, mode)
+                finally:
+                    os.close(out)
+            finally:
+                os.close(fd)
+
+        try:
+            previous = set(json.loads(manifest.read_text()))
+        except (OSError, ValueError):
+            previous = set()
+        for rel in sorted(previous - set(files)):
+            *folders, name = str(rel).split("/")
+            if any(p in ("", ".", "..") for p in [*folders, name]):
+                continue
+            fd = os.dup(root_fd)
+            try:
+                for folder in folders:
+                    nxt = _open_dir(fd, folder, create=False)
+                    os.close(fd)
+                    fd = nxt
+                st = os.stat(name, dir_fd=fd, follow_symlinks=False)
+                if stat.S_ISREG(st.st_mode):
+                    os.unlink(name, dir_fd=fd)
+            except (OSError, ValueError):
+                pass        # already gone, or no longer ours to remove
+            finally:
+                os.close(fd)
+    finally:
+        os.close(root_fd)
+    manifest.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    _write_private(manifest, json.dumps(sorted(files)))
+
+
+def _source_params(params: dict) -> tuple[str, str] | None:
+    """(ticket, sha256) for a Git stack's deploy, or None for a stack edited in Vigil."""
+    ticket = str(params.get("source_ticket") or "")
+    sha = str(params.get("source_sha256") or "")
+    if not ticket and not sha:
+        return None
+    if not _TICKET.match(ticket) or not _SHA256.match(sha):
+        raise ValueError("source_ticket must be a ticket id and source_sha256 64 hex characters")
+    return ticket, sha
+
+
 def _write_private(path: Path, text: str) -> None:
     """Create or replace *path* readable by its owner only."""
     _write_new(path, text, 0o600)
@@ -225,12 +345,18 @@ def _stack_deploy(params: dict, config: AgentConfig) -> str:
     if ticket and not _TICKET.match(ticket):
         raise ValueError("env_ticket must be a ticket id")
     compose_file = _compose_file(params)
-    # Redeemed first: a deploy that cannot have its secrets must not start
-    # half-configured containers.
+    source = _source_params(params)
+    # Redeemed first: a deploy that cannot have its secrets (or its source)
+    # must not start half-configured containers.
     env_text = client.fetch_stack_env(config, ticket) if ticket else None
+    files = _source_files(client.fetch_stack_source(config, source[0]), source[1]) if source else None
 
     _safe_workdir(workdir)
     workdir.mkdir(parents=True, exist_ok=True)
+    if files is not None:
+        # The repo's folder first; the compose file and .env below are then
+        # written over it — the compose text is the one the server checked.
+        _place_source(workdir, files, Path(config.data_dir) / "stacks" / f"{project}.source.json")
     _write_new(workdir / compose_file, compose, 0o644)
     if env_text is not None:
         _write_private(workdir / ".env", env_text)
@@ -254,8 +380,10 @@ def _stack_remove(params: dict, config: AgentConfig) -> str:
         raise ValueError(f"refusing to delete {workdir}: only {STACKS_ROOT}/ is Vigil's")
     output = _compose(workdir, project, "down", timeout=300) if (workdir / COMPOSE_NAME).exists() \
         else f"{project}: no compose file at {workdir} — nothing to take down"
-    # the configuration _up checked and started, kept for the containers' labels
-    (Path(config.data_dir) / "stacks" / f"{project}.json").unlink(missing_ok=True)
+    # the configuration _up checked and started, kept for the containers' labels,
+    # and the list of source files a Git stack's deploys placed
+    for kept in (f"{project}.json", f"{project}.source.json"):
+        (Path(config.data_dir) / "stacks" / kept).unlink(missing_ok=True)
     deleted = False
     if delete:
         if workdir.exists():
diff --git a/agent/vigil_agent/client.py b/agent/vigil_agent/client.py
index 78ab2b39..8dd5a772 100644
--- a/agent/vigil_agent/client.py
+++ b/agent/vigil_agent/client.py
@@ -194,6 +194,26 @@ def fetch_stack_env(config: AgentConfig, ticket: str) -> str:
     return str(resp.json().get("env") or "")
 
 
+#: A Git stack's packed source folder (the server caps it at 20 MB).
+MAX_STACK_SOURCE_BYTES = 20 * 1024 * 1024
+
+
+def fetch_stack_source(config: AgentConfig, ticket: str) -> bytes:
+    """Redeem a deploy's one-time source ticket (2026.14.1) for the packed
+    folder of a Git stack. The caller checks it against the signed sha256."""
+    url = f"{config.server_url}/api/v1/agent/stack-source/{ticket}/"
+    with requests.get(url, headers=_headers(config), timeout=_TIMEOUT, stream=True) as resp:
+        if resp.status_code != 200:
+            raise RuntimeError(f"the server would not hand over this deploy's source "
+                               f"({resp.status_code}) — the ticket is used or expired; deploy again")
+        body = bytearray()
+        for chunk in resp.iter_content(64 * 1024):
+            body += chunk
+            if len(body) > MAX_STACK_SOURCE_BYTES:
+                raise RuntimeError("the stack's source is larger than an agent accepts")
+    return bytes(body)
+
+
 def post_stack_read(config: AgentConfig, ticket: str, payload: dict) -> None:
     """Hand an adopted stack's files to the server (M11) — over the agent's
     own connection, never in the task result, because .env holds secrets."""
diff --git a/agent/vigil_agent/features.py b/agent/vigil_agent/features.py
index 079a9b6f..9cbf4139 100644
--- a/agent/vigil_agent/features.py
+++ b/agent/vigil_agent/features.py
@@ -8,4 +8,6 @@
 
 # signed_v2 (SEC-4): the server signs when it sent a task and to which agent,
 # and this agent refuses any task that is not signed that way.
-FEATURES = ("relevant", "branches", "boost", "signed_v2")
+# stack_source (2026.14.1): stack_deploy places a Git stack's source folder
+# from a ticketed, sha256-checked archive.
+FEATURES = ("relevant", "branches", "boost", "signed_v2", "stack_source")

From 628270c88a3547d8e60ec233bdc83819cb829b60 Mon Sep 17 00:00:00 2001
From: Connor Haggerty 
Date: Fri, 9 Oct 2026 20:24:31 -0400
Subject: [PATCH 55/68] feat(stacks): a stack revision lists the Git commit it
 came from

---
 server/apps/stacks/test_git_source.py | 2 ++
 server/apps/stacks/views.py           | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/apps/stacks/test_git_source.py b/server/apps/stacks/test_git_source.py
index 752fb2e5..9e60369e 100644
--- a/server/apps/stacks/test_git_source.py
+++ b/server/apps/stacks/test_git_source.py
@@ -236,6 +236,8 @@ def test_create_pull_and_no_edits_in_vigil(self):
         self.assertTrue(moved["changed"])
         self.assertEqual(moved["stack"]["revision"], 2)
         self.assertEqual(moved["stack"]["git"]["commit"], _run(self.repo, "rev-parse", "HEAD"))
+        revs = self.api.get(f"/api/v1/stacks/{sid}/revisions/").json()["results"]
+        self.assertEqual([r["git_commit"] for r in revs], [moved["stack"]["git"]["commit"], self.head])
 
     def test_a_refused_compose_file_makes_no_stack(self):
         Path(self.repo, "deploy", "compose.yaml").write_text(
diff --git a/server/apps/stacks/views.py b/server/apps/stacks/views.py
index 98585a73..35d9e0be 100644
--- a/server/apps/stacks/views.py
+++ b/server/apps/stacks/views.py
@@ -204,7 +204,7 @@ def stack_revisions(request, stack_id):
     if stack is None:
         return Response(status=status.HTTP_404_NOT_FOUND)
     return Response({"results": [{
-        "number": r.number, "note": r.note, "compose_yaml": r.compose_yaml,
+        "number": r.number, "note": r.note, "compose_yaml": r.compose_yaml, "git_commit": r.git_commit,
         "env_keys": [k for k, _v in _env_pairs(r)],
         "created_by": r.created_by.username if r.created_by else None,
         "created_at": r.created_at.isoformat()} for r in stack.revisions.all()[:50]]})

From b9a1035a597adc6d00ffae50d63dbeb0ee07c0a6 Mon Sep 17 00:00:00 2001
From: Connor Haggerty 
Date: Fri, 9 Oct 2026 20:28:46 -0400
Subject: [PATCH 56/68] fix(security): a Git credential goes to its own server
 only, git keeps to the checked address, and fetches are capped on disk
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review of the Git stack fetcher:

- Exfiltration: any admin session could point a stack at a server it runs
  and pick an existing token, and git would send it there. A credential now
  names its Git server (git_host, required) and is refused for any other —
  checked when it is attached and again at every fetch — and attaching one
  to a stack asks for TOTP.
- DNS rebinding: the host's addresses were checked, then git looked the name
  up again. git is now held to the checked address (http.curloptResolve for
  https; HostName + HostKeyAlias for ssh, so the TLS certificate and the
  pinned host key are still verified for the real name).
- Resource caps: git's output was read whole into memory before the size
  check, and nothing bounded what a fetch wrote to disk. git now runs under
  `prlimit --fsize` (512 MB per file: its pack and its captured output),
  output goes to files and is read back only within the limit. The server
  image installs util-linux for prlimit.

Real fetch from github.com still works with the address pinned.
---
 server/Dockerfile                             |   4 +-
 server/apps/stacks/git_views.py               |  29 ++++-
 server/apps/stacks/gitsource.py               | 110 ++++++++++++------
 .../migrations/0006_git_credential_host.py    |  18 +++
 server/apps/stacks/models.py                  |   4 +
 server/apps/stacks/test_git_source.py         |  64 ++++++++--
 server/apps/stacks/views.py                   |   4 +
 7 files changed, 184 insertions(+), 49 deletions(-)
 create mode 100644 server/apps/stacks/migrations/0006_git_credential_host.py

diff --git a/server/Dockerfile b/server/Dockerfile
index 7f8d4821..90894d19 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -31,10 +31,10 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 
 WORKDIR /app
 
-# git + openssh-client: stacks from Git (2026.14.1) are fetched by the server,
+# git + openssh-client (+ util-linux for prlimit): stacks from Git (2026.14.1) are fetched by the server,
 # never by a host, so a repository credential stays in Vigil.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends libpq-dev gosu git openssh-client && \
+    apt-get install -y --no-install-recommends libpq-dev gosu git openssh-client util-linux && \
     rm -rf /var/lib/apt/lists/*
 
 # Gunicorn and every request it serves ran as UID 0, so a code-execution bug in
diff --git a/server/apps/stacks/git_views.py b/server/apps/stacks/git_views.py
index 08dbb68e..9d678f2c 100644
--- a/server/apps/stacks/git_views.py
+++ b/server/apps/stacks/git_views.py
@@ -6,6 +6,7 @@
 server talks to the Git server.
 """
 
+import re
 from datetime import timedelta
 
 from django.db import transaction
@@ -33,7 +34,7 @@
 
 
 def _cred_row(c: GitCredential) -> dict:
-    return {"id": str(c.id), "name": c.name, "kind": c.kind, "username": c.username,
+    return {"id": str(c.id), "name": c.name, "kind": c.kind, "git_host": c.git_host, "username": c.username,
             "has_known_hosts": bool(c.known_hosts.strip()),
             "created_at": c.created_at.isoformat()}
 
@@ -52,9 +53,14 @@ def git_credential_index(request):
     kind = str(request.data.get("kind") or "")
     secret = str(request.data.get("secret") or "")
     known_hosts = str(request.data.get("known_hosts") or "")
+    git_host = str(request.data.get("git_host") or "").strip().lower()
     if not name or kind not in GitCredential.Kind.values or not secret.strip():
         return Response({"detail": "name, kind (token or ssh_key) and secret are required"},
                         status=status.HTTP_400_BAD_REQUEST)
+    if not re.fullmatch(gitsource._HOST, git_host):
+        return Response({"detail": "git_host must be the Git server's host name, e.g. github.com — "
+                                   "the credential is used with that server only"},
+                        status=status.HTTP_400_BAD_REQUEST)
     if kind == GitCredential.Kind.SSH_KEY:
         if "PRIVATE KEY" not in secret:
             return Response({"detail": "paste the private key (-----BEGIN … PRIVATE KEY-----)"},
@@ -67,7 +73,7 @@ def git_credential_index(request):
         return Response({"detail": "that is too long for a credential"},
                         status=status.HTTP_400_BAD_REQUEST)
     cred = GitCredential.objects.create(
-        name=name, kind=kind, username=str(request.data.get("username") or "").strip()[:255],
+        name=name, kind=kind, git_host=git_host, username=str(request.data.get("username") or "").strip()[:255],
         secret_encrypted=encrypt_secret(secret), known_hosts=known_hosts.strip(),
         created_by=request.user)
     hooks.emit("git_credential_added", credential=cred, user=request.user)
@@ -88,7 +94,8 @@ def source_for(stack: ManagedStack) -> gitsource.Source:
     cred = None
     if stack.git_credential_id:
         c = stack.git_credential
-        cred = (c.kind, c.username, decrypt_secret(bytes(c.secret_encrypted)), c.known_hosts)
+        cred = (c.kind, c.username, decrypt_secret(bytes(c.secret_encrypted)), c.known_hosts,
+                c.git_host)
     return gitsource.Source(url=stack.git_url, branch=stack.git_branch or "main",
                             pin=stack.git_pin, path=stack.git_path or "compose.yaml",
                             credential=cred)
@@ -114,6 +121,9 @@ def apply_git_settings(stack: ManagedStack, data) -> None:
         cred = GitCredential.objects.filter(pk=cred_id).first()
         if cred is None:
             raise gitsource.GitSourceError("that Git credential does not exist")
+        _kind, host = gitsource.parse_url(url)
+        if cred.git_host != host:
+            raise gitsource.GitSourceError(f"the credential {cred.name!r} is for {cred.git_host}, not {host}")
     stack.git_url, stack.git_branch, stack.git_pin, stack.git_path = url, branch, pin, path
     stack.git_credential = cred
 
@@ -204,3 +214,16 @@ def agent_stack_source(request, ticket_id):
     response["Content-Length"] = str(len(data))
     response["Cache-Control"] = "no-store"
     return response
+
+
+def credential_step_up(request) -> Response | None:
+    """Attaching a credential to a stack decides which server a secret is
+    sent to: it asks for TOTP, like every other action that releases one."""
+    git = request.data.get("git")
+    if not (isinstance(git, dict) and git.get("credential_id")):
+        return None
+    from apps.accounts.totp import require_totp_confirmation
+    if error := require_totp_confirmation(request.user, request.data):
+        return Response({"detail": f"Using a Git credential needs confirmation: {error}", "needs_totp": True},
+                        status=status.HTTP_401_UNAUTHORIZED)
+    return None
diff --git a/server/apps/stacks/gitsource.py b/server/apps/stacks/gitsource.py
index 1e91f19d..b4591ccd 100644
--- a/server/apps/stacks/gitsource.py
+++ b/server/apps/stacks/gitsource.py
@@ -65,8 +65,8 @@ class Source:
     branch: str = "main"
     pin: str = ""
     path: str = "compose.yaml"
-    #: (kind, username, secret, known_hosts) or None.
-    credential: tuple[str, str, str, str] | None = None
+    #: (kind, username, secret, known_hosts, git_host) or None.
+    credential: tuple[str, str, str, str, str] | None = None
 
 
 @dataclass
@@ -118,20 +118,34 @@ def check_path(path: str) -> tuple[str, str]:
     return "/".join(parts[:-1]), parts[-1]
 
 
-def _check_host_address(host: str, port: int) -> None:
+def _check_host_address(host: str, port: int) -> str:
     """Loopback, link-local (cloud metadata) and unspecified are refused —
     the server must not be talked into fetching from itself. Every address the
-    name resolves to is checked."""
+    name resolves to is checked, and the first is returned: git is then held
+    to that address (``_pin_args``), so a DNS answer that changes between this
+    check and git's own lookup (rebinding) cannot move it somewhere else."""
     try:
         infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
     except socket.gaierror as exc:
         raise GitSourceError(f"cannot resolve {host}") from exc
+    if not infos:
+        raise GitSourceError(f"cannot resolve {host}")
     for info in infos:
         addr = ipaddress.ip_address(info[4][0].split("%", 1)[0])
         mapped = getattr(addr, "ipv4_mapped", None)
         for a in (addr, mapped) if mapped else (addr,):
             if a.is_loopback or a.is_link_local or a.is_unspecified or a.is_multicast:
                 raise GitSourceError(f"{host} resolves to {a}, which Vigil will not fetch from")
+    return infos[0][4][0].split("%", 1)[0]
+
+
+def _pin_args(kind: str, host: str, port: int, ip: str) -> tuple[list[str], list[str]]:
+    """(git -c args, ssh options) that make git connect to the checked *ip*
+    while still verifying *host*'s TLS certificate or SSH host key."""
+    if kind == "https":
+        literal = f"[{ip}]" if ":" in ip else ip
+        return ["-c", f"http.curloptResolve={host}:{port}:{literal}"], []
+    return [], ["-o", f"HostName={ip}", "-o", f"HostKeyAlias={host}", "-o", f"Port={port}"]
 
 
 # ── Running git ──────────────────────────────────────────────────────────
@@ -144,7 +158,7 @@ def _check_host_address(host: str, port: int) -> None:
 )
 
 
-def _env(tmp: str, source: Source, kind: str) -> dict:
+def _env(tmp: str, source: Source, kind: str, ssh_pin: list[str] | None = None) -> dict:
     env = {
         "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
         "HOME": tmp,
@@ -182,34 +196,55 @@ def _env(tmp: str, source: Source, kind: str) -> dict:
             "ssh", "-F", "/dev/null", "-i", shlex.quote(key), "-o", "IdentitiesOnly=yes",
             "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
             "-o", f"UserKnownHostsFile={shlex.quote(known)}", "-o", "GlobalKnownHostsFile=/dev/null",
-            "-o", "ConnectTimeout=15"])
+            "-o", "ConnectTimeout=15", *(shlex.quote(a) for a in ssh_pin or ())])
     return env
 
 
-def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_000_000) -> bytes:
-    try:
-        proc = subprocess.Popen(["git", *_GIT_CONFIG, *args], cwd=cwd, env=env,
-                                stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-    except FileNotFoundError as exc:
-        raise GitSourceError("git is not installed on the Vigil server") from exc
-    try:
-        out, err = proc.communicate(timeout=TIMEOUT_SECONDS)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        proc.communicate()
-        raise GitSourceError("the Git server did not answer in time") from None
-    if len(out) > limit:
-        raise GitSourceError("the repository folder is too large for a stack")
-    if proc.returncode != 0:
-        # The last stderr line says what went wrong; the token is never in it
-        # (it never was on a command line or in the URL).
-        lines = [ln for ln in err.decode(errors="replace").splitlines() if ln.strip()]
-        raise GitSourceError(f"git failed: {lines[-1][:300] if lines else 'unknown error'}")
-    return out
-
-
-def _resolve(url: str, source: Source, env: dict) -> str:
+#: The most any file git writes may grow to — the fetched pack, and the
+#: captured output below. Past it the kernel stops git (SIGXFSZ), so a huge
+#: repository cannot fill the server's disk or memory.
+MAX_FETCH_BYTES = 512 * 1024 * 1024
+
+
+def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_000_000,
+         pin: list[str] | None = None) -> bytes:
+    """Run git with output going to files, never into memory unbounded:
+    ``prlimit --fsize`` caps every file the process writes (its pack and its
+    stdout here), then stdout is read back only if it is within *limit*."""
+    cmd = ["prlimit", f"--fsize={MAX_FETCH_BYTES}", "--", "git", *_GIT_CONFIG, *(pin or ()), *args]
+    with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
+        try:
+            proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdin=subprocess.DEVNULL,
+                                    stdout=out, stderr=err)
+        except FileNotFoundError as exc:
+            raise GitSourceError("git (and prlimit) must be installed on the Vigil server") from exc
+        try:
+            proc.wait(timeout=TIMEOUT_SECONDS)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+            raise GitSourceError("the Git server did not answer in time") from None
+        if proc.returncode in (-25, 128 + 25, 153):      # SIGXFSZ, however it is reported
+            raise GitSourceError("the repository is too large for a stack")
+        size = out.seek(0, os.SEEK_END)
+        if size > limit:
+            raise GitSourceError("the repository folder is too large for a stack")
+        if proc.returncode != 0:
+            # The last stderr line says what went wrong; the token is never in it
+            # (it never was on a command line or in the URL).
+            err.seek(max(0, err.seek(0, os.SEEK_END) - 4096))
+            text = err.read().decode(errors="replace")
+            # The size cap usually stops git's unpacking child, and the parent
+            # then reports only that unpacking failed.
+            if re.search(r"(unpack-objects|index-pack) failed|File size limit", text):
+                raise GitSourceError("the repository is too large for a stack (or its data could not be unpacked)")
+            lines = [ln for ln in text.splitlines() if ln.strip()]
+            raise GitSourceError(f"git failed: {lines[-1][:300] if lines else 'unknown error'}")
+        out.seek(0)
+        return out.read(limit + 1)
+
+
+def _resolve(url: str, source: Source, env: dict, pin_args: list[str]) -> str:
     """The commit a fetch should take: the pin if it is a commit, else what
     the pinned tag or tracked branch points at now."""
     pin = (source.pin or "").strip()
@@ -221,7 +256,7 @@ def _resolve(url: str, source: Source, env: dict) -> str:
     else:
         name = check_ref(source.branch or "main", "branch")
         refs = [f"refs/heads/{name}"]
-    out = _git(["ls-remote", "--", url, *refs], env).decode(errors="replace")
+    out = _git(["ls-remote", "--", url, *refs], env, pin=pin_args).decode(errors="replace")
     found = {}
     for line in out.splitlines():
         sha, _, ref = line.partition("\t")
@@ -294,16 +329,21 @@ def fetch(source: Source) -> Fetched:
     port = 443 if kind == "https" else 22
     if (m := (_HTTPS_RE.match(url) or _SSH_RE.match(url))) and m.groupdict().get("port"):
         port = int(m.group("port"))
-    _check_host_address(host, port)
+    if source.credential and source.credential[4].lower() != host:
+        # A credential is for one Git server. Using it with any other would
+        # hand a token to whoever runs that server.
+        raise GitSourceError(f"that credential is for {source.credential[4]}, not {host}")
+    ip = _check_host_address(host, port)
+    git_pin, ssh_pin = _pin_args(kind, host, port, ip)
 
     with tempfile.TemporaryDirectory(prefix="vigil-git-") as tmp:
         os.chmod(tmp, 0o700)
-        env = _env(tmp, source, kind)
-        commit = _resolve(url, source, env)
+        env = _env(tmp, source, kind, ssh_pin)
+        commit = _resolve(url, source, env, git_pin)
         repo = os.path.join(tmp, "repo.git")
         _git(["init", "-q", "--bare", repo], env)
         _git(["fetch", "-q", "--depth", "1", "--no-tags", "--no-recurse-submodules",
-              "--", url, commit], env, cwd=repo)
+              "--", url, commit], env, cwd=repo, pin=git_pin)
         got = _git(["rev-parse", "--verify", "--end-of-options", f"{commit}^{{commit}}"],
                    env, cwd=repo).decode().strip().lower()
         if got != commit:
diff --git a/server/apps/stacks/migrations/0006_git_credential_host.py b/server/apps/stacks/migrations/0006_git_credential_host.py
new file mode 100644
index 00000000..edc56344
--- /dev/null
+++ b/server/apps/stacks/migrations/0006_git_credential_host.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.2.17 on 2026-10-10 00:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stacks', '0005_git_source'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='gitcredential',
+            name='git_host',
+            field=models.CharField(default='', max_length=255),
+        ),
+    ]
diff --git a/server/apps/stacks/models.py b/server/apps/stacks/models.py
index 5cdd4974..f7a11a00 100644
--- a/server/apps/stacks/models.py
+++ b/server/apps/stacks/models.py
@@ -160,6 +160,10 @@ class Kind(models.TextChoices):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=120)
     kind = models.CharField(max_length=10, choices=Kind.choices)
+    #: The one Git server this credential may be used with (github.com,
+    #: gitea.lan). A stack pointed anywhere else is refused, so a credential
+    #: cannot be handed to a server someone else runs.
+    git_host = models.CharField(max_length=255, default="")
     #: For a token: the user name the host expects (x-access-token, oauth2 …).
     username = models.CharField(max_length=255, blank=True)
     secret_encrypted = models.BinaryField()
diff --git a/server/apps/stacks/test_git_source.py b/server/apps/stacks/test_git_source.py
index 9e60369e..254c78db 100644
--- a/server/apps/stacks/test_git_source.py
+++ b/server/apps/stacks/test_git_source.py
@@ -64,12 +64,12 @@ def _local_git():
     skip the URL/host checks (which are tested on their own below)."""
     real_env = gitsource._env
 
-    def env(tmp, source, kind):
-        e = real_env(tmp, source, kind)
+    def env(tmp, source, kind, ssh_pin=None):
+        e = real_env(tmp, source, kind, ssh_pin)
         e["GIT_ALLOW_PROTOCOL"] = "https:ssh:file"
         return e
     with mock.patch.object(gitsource, "parse_url", return_value=("https", "git.test")), \
-            mock.patch.object(gitsource, "_check_host_address"), \
+            mock.patch.object(gitsource, "_check_host_address", return_value="192.0.2.10"), \
             mock.patch.object(gitsource, "_env", side_effect=env), \
             mock.patch.object(gitsource, "_GIT_CONFIG", gitsource._GIT_CONFIG + ("-c", "protocol.file.allow=always")):
         yield
@@ -161,6 +161,35 @@ def test_missing_branch_and_missing_compose_file(self):
         with _local_git(), self.assertRaisesRegex(GitSourceError, "not in the repository"):
             gitsource.fetch(Source(url=self.repo, path="deploy/other.yaml"))
 
+    def test_git_is_held_to_the_checked_address(self):
+        """No second DNS lookup: the address that passed the check is the one git uses."""
+        seen = []
+        real = subprocess.Popen
+
+        def spy(args, **kw):
+            seen.append(args)
+            return real(args, **kw)
+        with mock.patch.object(gitsource.subprocess, "Popen", side_effect=spy):
+            self._fetch(branch="main")
+        network = [a for a in seen if "ls-remote" in a or "fetch" in a]
+        self.assertTrue(network)
+        for args in network:
+            self.assertIn("http.curloptResolve=git.test:443:192.0.2.10", args)
+        self.assertEqual(gitsource._pin_args("https", "h", 443, "2001:db8::1")[0][1],
+                         "http.curloptResolve=h:443:[2001:db8::1]")
+
+    def test_a_credential_only_goes_to_its_own_server(self):
+        with self.assertRaisesRegex(GitSourceError, "is for github.com"):
+            self._fetch(branch="main", credential=("token", "bot", "tok", "", "github.com"))
+
+    def test_a_huge_repository_is_stopped_not_buffered(self):
+        Path(self.repo, "deploy", "big.bin").write_bytes(os.urandom(256 * 1024))
+        _run(self.repo, "add", ".")
+        _run(self.repo, "commit", "-q", "-m", "big")
+        with mock.patch.object(gitsource, "MAX_FETCH_BYTES", 64 * 1024), \
+                self.assertRaisesRegex(GitSourceError, "too large"):
+            self._fetch(branch="main")
+
     def test_a_token_is_never_on_a_command_line(self):
         seen = []
         real = subprocess.Popen
@@ -169,7 +198,7 @@ def spy(args, **kw):
             seen.append((args, kw.get("env") or {}))
             return real(args, **kw)
         with mock.patch.object(gitsource.subprocess, "Popen", side_effect=spy):
-            self._fetch(branch="main", credential=("token", "bot", "s3cr3t-token", ""))
+            self._fetch(branch="main", credential=("token", "bot", "s3cr3t-token", "", "git.test"))
         self.assertTrue(seen)
         for args, env in seen:
             self.assertNotIn("s3cr3t-token", " ".join(args))
@@ -182,12 +211,14 @@ def test_ssh_needs_a_key_and_pinned_host_keys(self):
             src = Source(url="git@github.com:a/b.git")
             with self.assertRaises(GitSourceError):
                 gitsource._env(tmp, src, "ssh")
-            src.credential = ("ssh_key", "", "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n", "")
+            src.credential = ("ssh_key", "", "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n", "", "github.com")
             with self.assertRaisesRegex(GitSourceError, "known_hosts"):
                 gitsource._env(tmp, src, "ssh")
             src.credential = ("ssh_key", "", "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n",
-                              "github.com ssh-ed25519 AAAA")
-            env = gitsource._env(tmp, src, "ssh")
+                              "github.com ssh-ed25519 AAAA", "github.com")
+            env = gitsource._env(tmp, src, "ssh", ["-o", "HostName=140.82.112.3", "-o", "HostKeyAlias=github.com"])
+            self.assertIn("HostName=140.82.112.3", env["GIT_SSH_COMMAND"])
+            self.assertIn("HostKeyAlias=github.com", env["GIT_SSH_COMMAND"])
             self.assertIn("StrictHostKeyChecking=yes", env["GIT_SSH_COMMAND"])
             self.assertIn("GlobalKnownHostsFile=/dev/null", env["GIT_SSH_COMMAND"])
             self.assertEqual(os.stat(Path(tmp, "id_key")).st_mode & 0o777, 0o600)
@@ -283,17 +314,32 @@ def test_an_agent_without_stack_source_is_refused(self):
 
     def test_credentials_are_write_only_and_need_totp(self):
         url = "/api/v1/stacks/git-credentials/"
-        body = {"name": "gh", "kind": "token", "secret": "ghp_secret", "username": "bot"}
+        body = {"name": "gh", "kind": "token", "secret": "ghp_secret", "username": "bot", "git_host": "github.com"}
         self.assertEqual(self.api.post(url, body, format="json").status_code, 401)
         with mock.patch(_TOTP, return_value=None):
             self.assertEqual(self.api.post(url, body, format="json").status_code, 201)
-            ssh = {"name": "deploy", "kind": "ssh_key",
+            nohost = {**body, "git_host": ""}
+            self.assertEqual(self.api.post(url, nohost, format="json").status_code, 400, "a credential names its server")
+            ssh = {"name": "deploy", "kind": "ssh_key", "git_host": "github.com",
                    "secret": "-----BEGIN OPENSSH PRIVATE KEY-----\nx\n-----END OPENSSH PRIVATE KEY-----"}
             self.assertEqual(self.api.post(url, ssh, format="json").status_code, 400, "no host key pinned")
         listed = self.api.get(url).json()["results"]
         self.assertEqual([c["name"] for c in listed], ["gh"])
         self.assertNotIn("ghp_secret", str(listed))
 
+    def test_attaching_a_credential_needs_totp_and_the_right_server(self):
+        from .models import GitCredential
+        cred = GitCredential.objects.create(name="gh", kind="token", git_host="github.com",
+                                            secret_encrypted=b"x")
+        body = {"host_id": str(self.host.id), "name": "web",
+                "git": {"url": "https://github.com/acme/app.git", "credential_id": str(cred.id)}}
+        self.assertEqual(self.api.post("/api/v1/stacks/", body, format="json").status_code, 401)
+        body["git"]["url"] = "https://evil.example/acme/app.git"
+        with mock.patch(_TOTP, return_value=None):
+            resp = self.api.post("/api/v1/stacks/", body, format="json")
+        self.assertEqual(resp.status_code, 400)
+        self.assertIn("is for github.com", resp.json()["detail"])
+
     def test_viewers_cannot_touch_git_stacks(self):
         viewer = get_user_model().objects.create_user("v", password="x")
         api = APIClient()
diff --git a/server/apps/stacks/views.py b/server/apps/stacks/views.py
index 35d9e0be..1d06c16d 100644
--- a/server/apps/stacks/views.py
+++ b/server/apps/stacks/views.py
@@ -138,6 +138,8 @@ def _create_from_git(request, stack: ManagedStack) -> Response:
     check it like any compose file, and store revision 1 with its source."""
     from . import git_views, gitsource
 
+    if denied := git_views.credential_step_up(request):
+        return denied
     try:
         git_views.apply_git_settings(stack, request.data)
         if "env_text" in request.data or "env" in request.data:
@@ -186,6 +188,8 @@ def stack_detail(request, stack_id):
             return Response({"detail": "a stack edited in Vigil cannot be switched to Git; "
                                        "create a new stack from the repository"},
                             status=status.HTTP_400_BAD_REQUEST)
+        if denied := git_views.credential_step_up(request):
+            return denied
         try:
             git_views.apply_git_settings(stack, request.data)
             git_views.pull(stack, request.user, note="Git settings changed")

From 39266720deedef91b61cb610a459cacf464d63d5 Mon Sep 17 00:00:00 2001
From: Connor Haggerty 
Date: Fri, 9 Oct 2026 20:31:38 -0400
Subject: [PATCH 57/68] fix(security): a Git fetch keeps one pack file, and
 fetches run one at a time
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The per-file size cap (prlimit --fsize) did not bound a fetch that git
unpacked into loose objects — one small file per object, each under the cap,
as many as the commit holds. fetch.unpackLimit / transfer.unpackLimit = 1
keep it a single pack, which the cap does bound (tested with 120 small
objects under a 64 KB cap). A file lock also serialises fetches across
server workers, so a burst of requests cannot multiply the disk they use;
a second fetch waits up to 30 s, then says another is running.
---
 server/apps/stacks/gitsource.py       | 37 +++++++++++++++++++++++++--
 server/apps/stacks/test_git_source.py | 23 +++++++++++++++++
 2 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/server/apps/stacks/gitsource.py b/server/apps/stacks/gitsource.py
index b4591ccd..1270e183 100644
--- a/server/apps/stacks/gitsource.py
+++ b/server/apps/stacks/gitsource.py
@@ -35,6 +35,7 @@
 import subprocess
 import tarfile
 import tempfile
+from contextlib import contextmanager
 from dataclasses import dataclass
 
 #: The packed archive a host downloads.
@@ -155,6 +156,10 @@ def _pin_args(kind: str, host: str, port: int, ip: str) -> tuple[list[str], list
     "-c", "protocol.ssh.allow=always", "-c", "http.followRedirects=false",
     "-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false",
     "-c", "credential.helper=", "-c", "submodule.recurse=false",
+    # Keep what is fetched as one pack file, never unpacked into one loose
+    # file per object: the per-file size cap (MAX_FETCH_BYTES) then bounds the
+    # whole fetch, where a million small loose objects would each pass it.
+    "-c", "fetch.unpackLimit=1", "-c", "transfer.unpackLimit=1",
 )
 
 
@@ -236,7 +241,7 @@ def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_0
             text = err.read().decode(errors="replace")
             # The size cap usually stops git's unpacking child, and the parent
             # then reports only that unpacking failed.
-            if re.search(r"(unpack-objects|index-pack) failed|File size limit", text):
+            if re.search(r"(unpack-objects|index-pack) failed|invalid index-pack output|File size limit", text):
                 raise GitSourceError("the repository is too large for a stack (or its data could not be unpacked)")
             lines = [ln for ln in text.splitlines() if ln.strip()]
             raise GitSourceError(f"git failed: {lines[-1][:300] if lines else 'unknown error'}")
@@ -319,6 +324,34 @@ def repack(raw_tar: bytes, folder: str) -> tuple[bytes, dict[str, bytes]]:
 
 # ── The fetch ────────────────────────────────────────────────────────────
 
+#: How long a fetch waits for another to finish before giving up.
+LOCK_WAIT_SECONDS = 30
+
+
+@contextmanager
+def _one_fetch_at_a_time():
+    """One fetch on this server at a time, across every worker process — so
+    the size caps bound the disk a burst of requests can use, rather than
+    multiplying by however many arrive together."""
+    import fcntl
+    import time
+
+    path = os.path.join(tempfile.gettempdir(), "vigil-git-fetch.lock")
+    fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600)
+    try:
+        deadline = time.monotonic() + LOCK_WAIT_SECONDS
+        while True:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.monotonic() > deadline:
+                    raise GitSourceError("another Git fetch is running — try again in a moment") from None
+                time.sleep(0.25)
+        yield
+    finally:
+        os.close(fd)        # releases the lock
+
 
 def fetch(source: Source) -> Fetched:
     """Fetch, check and pack one commit's compose folder. Raises
@@ -336,7 +369,7 @@ def fetch(source: Source) -> Fetched:
     ip = _check_host_address(host, port)
     git_pin, ssh_pin = _pin_args(kind, host, port, ip)
 
-    with tempfile.TemporaryDirectory(prefix="vigil-git-") as tmp:
+    with _one_fetch_at_a_time(), tempfile.TemporaryDirectory(prefix="vigil-git-") as tmp:
         os.chmod(tmp, 0o700)
         env = _env(tmp, source, kind, ssh_pin)
         commit = _resolve(url, source, env, git_pin)
diff --git a/server/apps/stacks/test_git_source.py b/server/apps/stacks/test_git_source.py
index 254c78db..d3040d84 100644
--- a/server/apps/stacks/test_git_source.py
+++ b/server/apps/stacks/test_git_source.py
@@ -190,6 +190,29 @@ def test_a_huge_repository_is_stopped_not_buffered(self):
                 self.assertRaisesRegex(GitSourceError, "too large"):
             self._fetch(branch="main")
 
+    def test_many_small_objects_cannot_get_past_the_cap(self):
+        """Loose objects would each be under a per-file cap; one pack is not."""
+        for i in range(120):
+            Path(self.repo, "deploy", f"f{i}.bin").write_bytes(os.urandom(2048))
+        _run(self.repo, "add", ".")
+        _run(self.repo, "commit", "-q", "-m", "many")
+        with mock.patch.object(gitsource, "MAX_FETCH_BYTES", 64 * 1024), \
+                self.assertRaisesRegex(GitSourceError, "too large"):
+            self._fetch(branch="main")
+
+    def test_fetches_wait_for_each_other(self):
+        import fcntl
+        path = os.path.join(tempfile.gettempdir(), "vigil-git-fetch.lock")
+        fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            with mock.patch.object(gitsource, "LOCK_WAIT_SECONDS", 0.3), \
+                    self.assertRaisesRegex(GitSourceError, "another Git fetch"):
+                self._fetch(branch="main")
+        finally:
+            os.close(fd)
+        self._fetch(branch="main")      # free again
+
     def test_a_token_is_never_on_a_command_line(self):
         seen = []
         real = subprocess.Popen

From 6a282351ff248ef985e7def5ee931e0bcec223fc Mon Sep 17 00:00:00 2001
From: Connor Haggerty 
Date: Fri, 9 Oct 2026 20:33:31 -0400
Subject: [PATCH 58/68] fix(security): a slow Git server cannot hold every Git
 fetch up
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The one-at-a-time lock from 3926672 let a deliberately slow repository hold
it for up to 120 s per git command — several minutes per fetch — blocking
every other Git stack's pull and deploy; and its lock file sat in the shared
/tmp, where anyone could create it first and hold it for ever.

- A whole fetch now has one 180 s deadline; each git command gets only what
  is left of it, and none starts after it has passed.
- Two fetch slots instead of one: a slow repository occupies one, the other
  keeps working. Still few, so the size caps bound the disk.
- The slot lock files live in a 0700 folder owned by the server's user,
  checked before use.
---
 server/apps/stacks/gitsource.py       | 82 ++++++++++++++++++---------
 server/apps/stacks/test_git_source.py | 25 +++++---
 2 files changed, 74 insertions(+), 33 deletions(-)

diff --git a/server/apps/stacks/gitsource.py b/server/apps/stacks/gitsource.py
index 1270e183..814e298a 100644
--- a/server/apps/stacks/gitsource.py
+++ b/server/apps/stacks/gitsource.py
@@ -212,10 +212,13 @@ def _env(tmp: str, source: Source, kind: str, ssh_pin: list[str] | None = None)
 
 
 def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_000_000,
-         pin: list[str] | None = None) -> bytes:
+         pin: list[str] | None = None, deadline: float | None = None) -> bytes:
     """Run git with output going to files, never into memory unbounded:
     ``prlimit --fsize`` caps every file the process writes (its pack and its
     stdout here), then stdout is read back only if it is within *limit*."""
+    import time
+    if deadline is not None and deadline - time.monotonic() <= 0:
+        raise GitSourceError("the Git server did not answer in time")
     cmd = ["prlimit", f"--fsize={MAX_FETCH_BYTES}", "--", "git", *_GIT_CONFIG, *(pin or ()), *args]
     with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
         try:
@@ -223,8 +226,9 @@ def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_0
                                     stdout=out, stderr=err)
         except FileNotFoundError as exc:
             raise GitSourceError("git (and prlimit) must be installed on the Vigil server") from exc
+        budget = TIMEOUT_SECONDS if deadline is None else min(TIMEOUT_SECONDS, deadline - time.monotonic())
         try:
-            proc.wait(timeout=TIMEOUT_SECONDS)
+            proc.wait(timeout=max(budget, 0.1))
         except subprocess.TimeoutExpired:
             proc.kill()
             proc.wait()
@@ -249,7 +253,7 @@ def _git(args: list[str], env: dict, cwd: str | None = None, *, limit: int = 1_0
         return out.read(limit + 1)
 
 
-def _resolve(url: str, source: Source, env: dict, pin_args: list[str]) -> str:
+def _resolve(url: str, source: Source, env: dict, pin_args: list[str], deadline: float | None = None) -> str:
     """The commit a fetch should take: the pin if it is a commit, else what
     the pinned tag or tracked branch points at now."""
     pin = (source.pin or "").strip()
@@ -261,7 +265,7 @@ def _resolve(url: str, source: Source, env: dict, pin_args: list[str]) -> str:
     else:
         name = check_ref(source.branch or "main", "branch")
         refs = [f"refs/heads/{name}"]
-    out = _git(["ls-remote", "--", url, *refs], env, pin=pin_args).decode(errors="replace")
+    out = _git(["ls-remote", "--", url, *refs], env, pin=pin_args, deadline=deadline).decode(errors="replace")
     found = {}
     for line in out.splitlines():
         sha, _, ref = line.partition("\t")
@@ -324,33 +328,57 @@ def repack(raw_tar: bytes, folder: str) -> tuple[bytes, dict[str, bytes]]:
 
 # ── The fetch ────────────────────────────────────────────────────────────
 
-#: How long a fetch waits for another to finish before giving up.
+#: How long a fetch waits for a free slot before giving up.
 LOCK_WAIT_SECONDS = 30
+#: Fetches allowed at once on this server, across every worker. More than one,
+#: so a single slow repository cannot hold up every Git stack; few, so the
+#: size caps bound the disk a burst of requests can use.
+FETCH_SLOTS = 2
+#: The whole fetch — every git command in it — must finish within this.
+FETCH_DEADLINE_SECONDS = 180
+
+
+def _lock_dir() -> str:
+    """A folder only this server's user can write: a lock file in the shared
+    /tmp could be created first by anyone and held for ever."""
+    path = os.path.join(tempfile.gettempdir(), f"vigil-git-locks-{os.getuid()}")
+    try:
+        os.mkdir(path, 0o700)
+    except FileExistsError:
+        pass
+    st = os.lstat(path)
+    import stat as _stat
+    if not _stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or st.st_mode & 0o077:
+        raise GitSourceError(f"{path} is not a private folder of the Vigil server's user — refusing to use it")
+    return path
 
 
 @contextmanager
-def _one_fetch_at_a_time():
-    """One fetch on this server at a time, across every worker process — so
-    the size caps bound the disk a burst of requests can use, rather than
-    multiplying by however many arrive together."""
+def _fetch_slot():
+    """One of FETCH_SLOTS fetches at a time, across worker processes."""
     import fcntl
     import time
 
-    path = os.path.join(tempfile.gettempdir(), "vigil-git-fetch.lock")
-    fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600)
+    folder = _lock_dir()
+    fds = [os.open(os.path.join(folder, f"slot{i}.lock"),
+                   os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600)
+           for i in range(FETCH_SLOTS)]
     try:
         deadline = time.monotonic() + LOCK_WAIT_SECONDS
         while True:
-            try:
-                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                break
-            except BlockingIOError:
-                if time.monotonic() > deadline:
-                    raise GitSourceError("another Git fetch is running — try again in a moment") from None
-                time.sleep(0.25)
-        yield
+            for fd in fds:
+                try:
+                    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    yield
+                    return
+                except BlockingIOError:
+                    continue
+            if time.monotonic() > deadline:
+                raise GitSourceError("other Git fetches are running — try again in a moment")
+            time.sleep(0.25)
     finally:
-        os.close(fd)        # releases the lock
+        for fd in fds:
+            os.close(fd)        # releases whichever lock was held
 
 
 def fetch(source: Source) -> Fetched:
@@ -369,19 +397,21 @@ def fetch(source: Source) -> Fetched:
     ip = _check_host_address(host, port)
     git_pin, ssh_pin = _pin_args(kind, host, port, ip)
 
-    with _one_fetch_at_a_time(), tempfile.TemporaryDirectory(prefix="vigil-git-") as tmp:
+    import time
+    deadline = time.monotonic() + FETCH_DEADLINE_SECONDS
+    with _fetch_slot(), tempfile.TemporaryDirectory(prefix="vigil-git-") as tmp:
         os.chmod(tmp, 0o700)
         env = _env(tmp, source, kind, ssh_pin)
-        commit = _resolve(url, source, env, git_pin)
+        commit = _resolve(url, source, env, git_pin, deadline)
         repo = os.path.join(tmp, "repo.git")
-        _git(["init", "-q", "--bare", repo], env)
+        _git(["init", "-q", "--bare", repo], env, deadline=deadline)
         _git(["fetch", "-q", "--depth", "1", "--no-tags", "--no-recurse-submodules",
-              "--", url, commit], env, cwd=repo, pin=git_pin)
+              "--", url, commit], env, cwd=repo, pin=git_pin, deadline=deadline)
         got = _git(["rev-parse", "--verify", "--end-of-options", f"{commit}^{{commit}}"],
-                   env, cwd=repo).decode().strip().lower()
+                   env, cwd=repo, deadline=deadline).decode().strip().lower()
         if got != commit:
             raise GitSourceError(f"the Git server returned {got[:12]} for {commit[:12]}")
-        raw = _git(["archive", "--format=tar", commit, "--", folder or "."], env, cwd=repo,
+        raw = _git(["archive", "--format=tar", commit, "--", folder or "."], env, cwd=repo, deadline=deadline,
                    limit=MAX_TREE_BYTES + MAX_FILES * 1024)
 
     archive, files = repack(raw, folder)
diff --git a/server/apps/stacks/test_git_source.py b/server/apps/stacks/test_git_source.py
index d3040d84..7d12b044 100644
--- a/server/apps/stacks/test_git_source.py
+++ b/server/apps/stacks/test_git_source.py
@@ -200,18 +200,29 @@ def test_many_small_objects_cannot_get_past_the_cap(self):
                 self.assertRaisesRegex(GitSourceError, "too large"):
             self._fetch(branch="main")
 
-    def test_fetches_wait_for_each_other(self):
+    def test_fetch_slots_are_shared_and_private(self):
         import fcntl
-        path = os.path.join(tempfile.gettempdir(), "vigil-git-fetch.lock")
-        fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600)
+        folder = gitsource._lock_dir()
+        self.assertEqual(os.stat(folder).st_mode & 0o777, 0o700)
+        held = [os.open(os.path.join(folder, f"slot{i}.lock"), os.O_RDWR | os.O_CREAT, 0o600)
+                for i in range(gitsource.FETCH_SLOTS)]
         try:
-            fcntl.flock(fd, fcntl.LOCK_EX)
+            for fd in held[:-1]:
+                fcntl.flock(fd, fcntl.LOCK_EX)
+            self._fetch(branch="main")                 # one slot still free: no waiting
+            fcntl.flock(held[-1], fcntl.LOCK_EX)
             with mock.patch.object(gitsource, "LOCK_WAIT_SECONDS", 0.3), \
-                    self.assertRaisesRegex(GitSourceError, "another Git fetch"):
+                    self.assertRaisesRegex(GitSourceError, "other Git fetches"):
                 self._fetch(branch="main")
         finally:
-            os.close(fd)
-        self._fetch(branch="main")      # free again
+            for fd in held:
+                os.close(fd)
+        self._fetch(branch="main")
+
+    def test_a_slow_git_server_is_cut_off_by_the_deadline(self):
+        with mock.patch.object(gitsource, "FETCH_DEADLINE_SECONDS", 0), \
+                self.assertRaisesRegex(GitSourceError, "did not answer in time"):
+            self._fetch(branch="main")
 
     def test_a_token_is_never_on_a_command_line(self):
         seen = []

From 49569629b84fcd20127998086845da33275a500a Mon Sep 17 00:00:00 2001
From: Connor Haggerty 
Date: Fri, 9 Oct 2026 21:20:51 -0400
Subject: [PATCH 59/68] =?UTF-8?q?feat(stacks):=20pick=20a=20stack's=20sour?=
 =?UTF-8?q?ce=20in=20the=20editor=20=E2=80=94=20write=20it=20here,=20or=20?=
 =?UTF-8?q?from=20Git?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

QA-23. The server has taken a compose file from a repository since 2f6b00a;
the editor had no way to say "this one comes from Git", so the merged half was
unreachable from the UI.

A new stack chooses its source up front. For a Git stack the compose file is
read-only — the repository is the source of truth — and the fields that aim the
stack at a repo (url, branch, pin, path, credential) are editable afterwards,
a change taking effect by pulling. The .env panel is unchanged: that stays
Vigil's, which is why a Git stack's save still PUTs {env, note}.

A credential is added from the same select, in a modal, and its secret is only
ever sent — never interpolated into markup, cleared on close and after save
either way. Adding one asks for TOTP because it is a key to a repository whose
contents run as root on a host.
---
 server/apps/stacks/test_git_stack_ui.py   |  42 ++++
 server/static/css/vigil.css               |  31 +++
 server/static/js/vigil-stacks.js          | 292 +++++++++++++++++++++-
 server/templates/pages/_stack_editor.html |  42 +++-
 4 files changed, 394 insertions(+), 13 deletions(-)
 create mode 100644 server/apps/stacks/test_git_stack_ui.py

diff --git a/server/apps/stacks/test_git_stack_ui.py b/server/apps/stacks/test_git_stack_ui.py
new file mode 100644
index 00000000..b3eb03b8
--- /dev/null
+++ b/server/apps/stacks/test_git_stack_ui.py
@@ -0,0 +1,42 @@
+"""QA-23: the stack editor can take its compose file from a Git repository.
+
+The repository is the source of truth for such a stack, so the editor's job is
+to show it read-only, offer Pull, and carry the .env side. These read the
+shipped files as text: the points worth guarding (read-only, no secret in
+markup, the Git endpoints actually called) are all visible in the source.
+"""
+import re
+from pathlib import Path
+
+from django.test import SimpleTestCase
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def _text(relpath: str) -> str:
+    return (ROOT / relpath).read_text(encoding="utf-8")
+
+
+class GitSourceInEditorTests(SimpleTestCase):
+    def test_template_offers_a_git_source(self):
+        page = _text("templates/pages/_stack_editor.html")
+        for marker in ('data-sted-source="git"', 'id="sted-git-url"', 'id="sted-git-branch"',
+                       'id="sted-git-pin"', 'id="sted-git-path"', 'id="sted-git-cred"',
+                       'data-sted-git-pull'):
+            self.assertIn(marker, page)
+
+    def test_save_and_pull_use_the_git_api(self):
+        js = _text("static/js/vigil-stacks.js")
+        for needle in ("git: {", "/pull/", "/api/v1/stacks/git-credentials/", "readOnly"):
+            self.assertIn(needle, js)
+
+    def test_secret_never_enters_markup(self):
+        js = _text("static/js/vigil-stacks.js")
+        offenders = re.findall(r"\$\{[^}]*secret[^}]*\}", js)
+        self.assertEqual(offenders, [],
+                         f"a credential secret is interpolated into markup: {offenders}")
+        self.assertIn(".value = ''", js)
+
+    def test_git_commit_shown_in_revisions(self):
+        js = _text("static/js/vigil-stacks.js")
+        self.assertIn("git_commit", js)
diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css
index da734868..2b3c4c13 100644
--- a/server/static/css/vigil.css
+++ b/server/static/css/vigil.css
@@ -4060,3 +4060,34 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; }
 @media (max-width: 640px) {
   .rlt-lanes { grid-template-columns: minmax(0, 1fr); }
 }
+
+/* ── Stacks from Git in the editor (QA-23) ────────────────────────────── */
+/* Two choices, one control: the picked half carries the accent across its
+   whole face, the way the wave-group chips do. */
+.sted-source { display: inline-flex; align-self: flex-start; gap: 2px; padding: 3px;
+  background: var(--s1); border-radius: var(--r-lg); }
+.sted-source-btn { border: 0; cursor: pointer; font-family: inherit; font-size: 12px;
+  font-weight: 600; padding: 6px 16px; border-radius: 16px;
+  background: transparent; color: var(--text-2); }
+.sted-source-btn:hover { color: var(--text-1); }
+.sted-source-btn[aria-checked="true"] { background: var(--lavender); color: var(--lavender-ink); }
+
+.sted-git { background: var(--s1); border-radius: var(--r-lg); padding: 16px;
+  display: flex; flex-direction: column; gap: 12px; }
+.sted-git-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; }
+.sted-git-wide { grid-column: 1 / -1; }
+.sted-git-state { display: flex; align-items: center; gap: 10px; font-size: 12px;
+  color: var(--text-2); flex-wrap: wrap; }
+.sted-git-state code { font-family: var(--font-mono); font-size: 11px; color: var(--text-1);
+  background: var(--s2); border-radius: var(--r-sm); padding: 2px 7px; }
+.sted-git-chip { font-family: var(--font-mono); font-size: 10px; font-weight: 400;
+  color: var(--text-3); background: var(--s2); border-radius: var(--r-sm); padding: 1px 5px; }
+.sted-git-kinds { display: flex; gap: 6px; margin-bottom: 12px; }
+.sted-git-kinds .btn[aria-pressed="true"] { background: var(--lavender); color: var(--lavender-ink); }
+
+#stack-compose.is-readonly { caret-color: transparent; cursor: default; }
+#stack-compose.is-readonly::selection { background: rgba(var(--rgb-lavender), var(--tint-a)); }
+
+@media (max-width: 700px) {
+  .sted-git-grid { grid-template-columns: minmax(0, 1fr); }
+}
diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js
index faab79a9..edeae0b1 100644
--- a/server/static/js/vigil-stacks.js
+++ b/server/static/js/vigil-stacks.js
@@ -7,13 +7,17 @@
 // HTML: #managed-stacks in templates/pages/_containers.html;
 //   #page-stack-editor in templates/pages/_stack_editor.html.
 // Depends on: vigil-utils.js (escHtml, escAttr, apiJson, showToast, getCsrf,
-//   confirmModal, totpPrompt, yamlToHtml, timeAgo), vigil-nav.js (navigateTo —
-//   wrapped below, and it needs every .page section to exist, so this file
-//   loads after base.html's content block is parsed).
+//   confirmModal, totpPrompt, yamlToHtml, timeAgo, mountModal), vigil-nav.js
+//   (navigateTo — wrapped below, and it needs every .page section to exist, so
+//   this file loads after base.html's content block is parsed).
 // API: /api/v1/stacks/ (CRUD), /validate/, //revisions/, //env/reveal/
-//      (TOTP), //deploy/ (TOTP), //remove/ (TOTP)
+//      (TOTP), //deploy/ (TOTP), //remove/ (TOTP), //pull/ (Git),
+//      /git-credentials/ (list, add with TOTP, delete)
 
-const stackEd = { hostId: null, stack: null, env: [], stacks: [], revisions: [], dirty: false };
+const stackEd = {
+  hostId: null, stack: null, env: [], stacks: [], revisions: [], dirty: false,
+  source: 'editor', gitCreds: null, gitSnapshot: null,
+};
 let _stackCheckSeq = 0;
 let _stackCheckTimer = null;
 
@@ -123,7 +127,9 @@ async function openStackEditor(stackId) {
   stackEd.stack = stack;
   stackEd.env = stack ? stack.env.map(e => ({ key: e.key, value: '', stored: true })) : [];
   stackEd.revisions = [];
+  stackEd.gitSnapshot = null;
   stackEd.errorLine = null;
+  stackEd.source = (stack && stack.git) ? 'git' : 'editor';
   _clearStackDirty();
   _stedEl('sted-host').textContent = _stedHostLabel(stack);
   _stedEl('sted-name-label').textContent = stack ? stack.name : 'New stack';
@@ -131,6 +137,8 @@ async function openStackEditor(stackId) {
   name.value = stack ? stack.name : '';
   name.disabled = !!stack;
   _stedEl('stack-compose').value = stack ? stack.compose_yaml : _NEW_COMPOSE;
+  _applyStackSource();
+  if (stack && stack.git) _fillGitFields(stack.git);
   const err = _stedEl('stack-error');
   err.textContent = '';
   err.classList.remove('show');
@@ -149,6 +157,7 @@ async function openStackEditor(stackId) {
   navigateTo('stack-editor');
   (stack ? _stedEl('stack-compose') : name).focus();
   if (stack) _loadStackRevisions();
+  _loadGitCredentials();
   _validateStack();
 }
 
@@ -219,6 +228,7 @@ async function _validateStack() {
   clearTimeout(_stackCheckTimer);
   const ta = _stedEl('stack-compose');
   if (!ta) return;
+  if (ta.closest('.sted-editor').hidden) { _setStackStatus(null); return; }
   const seq = ++_stackCheckSeq;
   _setStackStatus(null);
   let r;
@@ -235,28 +245,272 @@ async function _validateStack() {
 
 async function _loadStackRevisions() {
   const wrap = _stedEl('stack-revisions');
+  if (!stackEd.stack) { wrap.innerHTML = ''; return; }   // saved-and-closed beats the in-flight fetch
   try {
     const revs = (await apiJson(`/api/v1/stacks/${stackEd.stack.id}/revisions/`)).results || [];
+    const isGit = !!stackEd.stack.git;
     wrap.innerHTML = revs.map(r => `

-        r${r.number}${escHtml(r.note || '')} · ${escHtml(r.created_by || '')} · ${escHtml(timeAgo(r.created_at) || '')}
-        ${r.number !== stackEd.stack.revision ? `` : 'current'}
+        r${r.number}${r.git_commit ? ` ${escHtml(r.git_commit.slice(0, 12))}` : ''}${escHtml(r.note || '')} · ${escHtml(r.created_by || '')} · ${escHtml(timeAgo(r.created_at) || '')}
+        ${!isGit && r.number !== stackEd.stack.revision ? `` : 'current'}
       
`).join('');
     stackEd.revisions = revs;
   } catch { wrap.innerHTML = ''; }
 }
 
+/* ── Source: written here, or a Git repository ───────────────────────── */
+
+/* The five Git fields a save sends, read in the one place they are touched. */
+function _gitFields() {
+  const pick = (id) => _stedEl(id).value.trim();
+  const credential = _stedEl('sted-git-cred').value;
+  return {
+    url: pick('sted-git-url'),
+    branch: pick('sted-git-branch') || 'main',
+    pin: pick('sted-git-pin'),
+    path: pick('sted-git-path') || 'compose.yaml',
+    credential_id: credential || null,
+  };
+}
+
+/* A new Git stack POSTs this instead of compose_yaml: git: { url, branch,
+   pin, path, credential_id }. The .env is still Vigil's own — only the compose
+   file belongs to the repository. */
+function _newGitStackBody(name, env) {
+  return { host_id: stackEd.hostId, name, git: _gitFields(), env };
+}
+
+/* An existing Git stack PUTs nothing but its .env, unless the repository
+   settings changed — then git: { … } takes effect by pulling it. */
+function _gitSettingsBody() {
+  return { git: _gitFields(), note: 'Git settings changed' };
+}
+
+function _applyStackSource() {
+  const git = stackEd.source === 'git';
+  const stack = stackEd.stack;
+  document.querySelectorAll('[data-sted-source]').forEach(
+    b => b.setAttribute('aria-checked', String(b.dataset.stedSource === stackEd.source)));
+  _stedEl('sted-source').hidden = !!stack;      // an existing stack's source is fixed
+  _stedEl('sted-git').hidden = !git;
+  // A Git stack shows the repository's compose file so its folders can be
+  // understood here; a new one has no file to show until the first fetch.
+  const showEditor = !git || !!stack;
+  const editor = _stedEl('stack-compose').closest('.sted-editor');
+  editor.hidden = !showEditor;
+  editor.previousElementSibling.hidden = !showEditor;
+  _stedEl('sted-check').hidden = !showEditor;
+  const ta = _stedEl('stack-compose');
+  ta.readOnly = git;                            // the repository is the source of truth
+  ta.classList.toggle('is-readonly', git);
+  _renderGitState();
+}
+
+function _renderGitState() {
+  const git = stackEd.stack && stackEd.stack.git;
+  const state = _stedEl('sted-git-state');
+  const pull = document.querySelector('.sted [data-sted-git-pull]');
+  state.hidden = pull.hidden = !git;
+  if (!git) return;
+  // "Commit abc123def456 · tracking main" / "· pinned to v1.2". The commit is
+  // the span's only , so a repository string never becomes markup.
+  state.querySelector('code').textContent = (git.commit || '').slice(0, 12);
+  state.lastChild.textContent = git.pin ? ` · pinned to ${git.pin}` : ` · tracking ${git.branch || 'main'}`;
+}
+
+function _fillGitFields(git) {
+  _stedEl('sted-git-url').value = git.url || '';
+  _stedEl('sted-git-branch').value = git.branch || 'main';
+  _stedEl('sted-git-pin').value = git.pin || '';
+  _stedEl('sted-git-path').value = git.path || 'compose.yaml';
+  stackEd.gitSnapshot = { url: git.url || '', branch: git.branch || 'main', pin: git.pin || '',
+                          path: git.path || 'compose.yaml', credential_id: git.credential_id || '' };
+}
+
+function _gitSettingsChanged() {
+  const was = stackEd.gitSnapshot, fresh = _gitFields();
+  if (!was) return true;
+  return ['url', 'branch', 'pin', 'path', 'credential_id'].some(k => (was[k] || '') !== (fresh[k] || ''));
+}
+
+function _setStackSource(source) {
+  if (stackEd.stack || stackEd.source === source) return;   // fixed once the stack exists
+  stackEd.source = source;
+  _applyStackSource();
+  _validateStack();
+}
+
+async function _loadGitCredentials() {
+  if (stackEd.gitCreds === null) {
+    stackEd.gitCreds = [];
+    try {
+      stackEd.gitCreds = (await apiJson('/api/v1/stacks/git-credentials/')).results || [];
+    } catch (e) {
+      _stackError(e.message);
+    }
+  }
+  _renderGitCredentials();
+}
+
+function _renderGitCredentials() {
+  const sel = _stedEl('sted-git-cred');
+  const wanted = stackEd.gitSnapshot ? stackEd.gitSnapshot.credential_id : '';
+  sel.innerHTML = ''
+    + stackEd.gitCreds.map(c => ``).join('')
+    + '';
+  // A credential that vanishes between the list and now (another tab deleted
+  // it) leaves the field empty rather than throwing.
+  sel.value = wanted && stackEd.gitCreds.some(c => c.id === wanted) ? wanted : '';
+}
+
+function _pickGitCredential() {
+  const sel = _stedEl('sted-git-cred');
+  if (sel.value !== '__add__') return;
+  sel.value = stackEd.gitSnapshot ? stackEd.gitSnapshot.credential_id || '' : '';
+  addGitCredential();
+}
+
+function addGitCredential() {
+  const m = mountModal('git-cred', { variant: 'm-pop' });
+  let kind = 'token';
+
+  const close = () => {
+    // The secret leaves the page on close, saved or not: it is only ever sent.
+    const field = m.modal.querySelector('#git-cred-secret');
+    if (field) field.value = '';      // the secret is only ever sent, never kept on screen
+    m.close();
+  };
+
+  const save = async () => {
+    const pick = (id) => { const el = m.modal.querySelector(id); return el ? el.value : ''; };
+    const name = pick('#git-cred-name').trim();
+    const username = pick('#git-cred-user').trim();
+    const pass = pick('#git-cred-secret');
+    const hosts = pick('#git-cred-hosts');
+    if (!name) { showToast('Give the credential a name', 'warn'); return; }
+    if (!pass.trim()) { showToast('The secret is empty', 'warn'); return; }
+    if (kind === 'ssh_key' && !hosts.trim()) {
+      showToast('An SSH deploy key needs the host key', 'warn');
+      return;
+    }
+    const code = await totpPrompt('Adding a Git credential');
+    if (!code) { close(); return; }
+    try {
+      const created = await apiJson('/api/v1/stacks/git-credentials/', {
+        method: 'POST',
+        body: JSON.stringify({ name, kind, username, secret: pass, known_hosts: hosts,
+                               totp: code.trim() }),
+      });
+      stackEd.gitCreds.push(created);
+      _renderGitCredentials();
+      stackEd.gitSnapshot = { ...(stackEd.gitSnapshot || {}), credential_id: created.id };
+      _pickCredentialOption(created.id);
+      showToast(`Added ${created.name}`, 'success');
+    } catch (e) {
+      _stackError(e.message);
+    }
+    close();   // clears the secret whether the save landed or not
+  };
+
+  const render = () => {
+    const ssh = kind === 'ssh_key';
+    // The fields a kind needs, then the one that holds its key. Field ids in
+    // markup strings; only the kind buttons interpolate.
+    const extra = ssh ? `
+      

+        
+        
+        
The output of ssh-keyscan github.com — Vigil does not
+          trust an unknown host key.

+      
` : `
+      

+        
+        
+      
`;
+    const holder = ssh
+      ? ''
+      : '';
+    const holderHint = ssh ? 'Paste the private key Vigil should authenticate with.'
+                           : 'The token Vigil should authenticate with.';
+    m.setBody(`
+      

+        A Git credential
+        
+      

+      

+        
+        
+      

+      

+        
+        
+      

+      ${extra}
+      

+        
+        ${holder}
+        
${holderHint}

+      

+      

+        
+        
+      
`);
+    m.overlay.onclick = close;
+    m.modal.querySelectorAll('[data-cred-x]').forEach(b => { b.onclick = close; });
+    m.modal.querySelectorAll('[data-cred-kind]').forEach(b => {
+      b.onclick = () => { if (kind === b.dataset.credKind) return; kind = b.dataset.credKind; render(); };
+    });
+    m.modal.querySelector('[data-cred-save]').onclick = save;
+  };
+
+  render();
+  requestAnimationFrame(() => { m.open(); m.modal.querySelector('#git-cred-name').focus(); });
+}
+
+async function pullStackFromGit() {
+  const stack = stackEd.stack;
+  if (!stack) return;
+  try {
+    const res = await apiJson(`/api/v1/stacks/${stack.id}/pull/`, { method: 'POST', body: '{}' });
+    if (!res.changed) {
+      showToast('Already at the latest commit', 'info');
+      return;
+    }
+    const commit = ((res.stack.git || {}).commit || '').slice(0, 12);
+    await renderManagedStacks(stackEd.hostId);   // the chip's r-number changed too
+    showToast(`New commit ${commit} is revision ${res.stack.revision} — deploy it to apply`, 'info');
+    openStackEditor(res.stack.id);
+  } catch (e) {
+    _stackError(e.message);
+  }
+}
+
+function _gitFetchedToast(saved) {
+  const git = saved.git || {};
+  return `Fetched ${(git.commit || '').slice(0, 12)} from ${git.pin || git.branch || 'main'} — deploy it to run`;
+}
+
 async function saveStack() {
-  const compose = _stedEl('stack-compose').value;
+  const editing = stackEd.stack;
+  const gitStack = !!(editing && editing.git);
+  const gitChanged = gitStack && _gitSettingsChanged();
   const env = stackEd.env.filter(e => e.key.trim()).map(e => (e.stored && !e.value && !e.revealed)
     ? { key: e.key.trim(), keep: true } : { key: e.key.trim(), value: e.value || '' });
-  const editing = stackEd.stack;
-  const body = editing ? { compose_yaml: compose, env, note: 'edited' }
-    : { host_id: stackEd.hostId, name: _stedEl('stack-name').value.trim(), compose_yaml: compose, env };
+  let body;
+  if (gitStack && gitChanged) body = _gitSettingsBody();
+  else if (gitStack) body = { env, note: 'edited' };   // the compose file comes from Git, not from here
+  else if (stackEd.source === 'git') body = _newGitStackBody(_stedEl('stack-name').value.trim(), env);
+  else body = { compose_yaml: _stedEl('stack-compose').value, env, note: 'edited' };
+  const wasNew = !editing;
   try {
     const saved = await apiJson(editing ? `/api/v1/stacks/${editing.id}/` : '/api/v1/stacks/',
       { method: editing ? 'PUT' : 'POST', body: JSON.stringify(body) });
-    showToast(`Saved ${saved.name} as revision ${saved.revision} — deploy it to apply`, 'success');
     await renderManagedStacks(stackEd.hostId);
+    if (wasNew && stackEd.source === 'git') showToast(_gitFetchedToast(saved), 'success');
+    else showToast(gitChanged ? 'Repository settings updated — pulled what it holds now'
+                              : `Saved ${saved.name} as revision ${saved.revision} — deploy it to apply`, 'success');
     // Reopen the saved row — but only if the editor is still on screen. An
     // async Ctrl+S can land after the operator has moved on, and dragging
     // them back to the page they left would be worse than a stale view.
@@ -316,6 +570,8 @@ delegateClick('[data-sted-back]', () => closeStackEditor());
 delegateClick('[data-stack-save]', () => saveStack());
 delegateClick('[data-stack-deploy]', () => deployStack());
 delegateClick('[data-stack-remove]', () => removeStack());
+delegateClick('[data-sted-source]', (el) => _setStackSource(el.dataset.stedSource));
+delegateClick('[data-sted-git-pull]', () => pullStackFromGit());
 delegateClick('[data-stack-env-reveal]', () => revealStackEnv());
 delegateClick('[data-stack-env-add]', () => {
   stackEd.env.push({ key: '', value: '', stored: false });
@@ -402,6 +658,18 @@ document.addEventListener('keydown', (ev) => {
   }
 });
 
+/* A 
+   can select an option too, which is why the handler stops at the sentinel. */
+(function () {
+  const wire = () => {
+    const sel = _stedEl('sted-git-cred');
+    if (sel) sel.addEventListener('change', () => _pickGitCredential());
+  };
+  document.addEventListener('DOMContentLoaded', wire);
+  if (document.readyState !== 'loading') wire();
+})();
+
 /* ── navigateTo wrapper: leaving the editor with unsaved edits asks first ─ */
 (function () {
   const previous = window.navigateTo;
diff --git a/server/templates/pages/_stack_editor.html b/server/templates/pages/_stack_editor.html
index 82488ec6..96db328e 100644
--- a/server/templates/pages/_stack_editor.html
+++ b/server/templates/pages/_stack_editor.html
@@ -19,7 +19,12 @@
-
+
+ +
Lowercase letters, digits, _ and -. It lives in /opt/vigil/stacks/<name> on the host.
+ +
compose.yaml
From d0ab36ef40a8fb976a533147f621d8682eebfa94 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 21:23:50 -0400 Subject: [PATCH 60/68] docs(stacks): say where the credential select gets its options --- server/static/js/vigil-stacks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index edeae0b1..f54f56a0 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -157,7 +157,7 @@ async function openStackEditor(stackId) { navigateTo('stack-editor'); (stack ? _stedEl('stack-compose') : name).focus(); if (stack) _loadStackRevisions(); - _loadGitCredentials(); + _loadGitCredentials(); // fills #sted-git-cred; _renderGitCredentials() re-selects from gitSnapshot _validateStack(); } From 5efe4945b910da41a41fed5600c82533f1bc06d3 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 21:24:36 -0400 Subject: [PATCH 61/68] fix(stacks): no autofill on a pasted deploy key either autocomplete=off on the SSH private-key textarea, matching the new-password token field: a password manager offering to fill a pasted key into a form it cannot read back is only a way to leak it. --- server/static/js/vigil-stacks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index f54f56a0..5c09f31c 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -428,7 +428,7 @@ function addGitCredential() { autocomplete="off" autocapitalize="off" spellcheck="false">
`; const holder = ssh - ? '' + ? '' : ''; const holderHint = ssh ? 'Paste the private key Vigil should authenticate with.' : 'The token Vigil should authenticate with.'; From d3504010ee65d5a2a062d7a6f0e9a77c769d1a23 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 21:34:20 -0400 Subject: [PATCH 62/68] fix(stacks): new Git settings stick when they land on the same commit, with the .env sent alongside --- server/apps/stacks/test_git_source.py | 19 +++++++++++++++++++ server/apps/stacks/views.py | 16 +++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/server/apps/stacks/test_git_source.py b/server/apps/stacks/test_git_source.py index 7d12b044..0098dcfc 100644 --- a/server/apps/stacks/test_git_source.py +++ b/server/apps/stacks/test_git_source.py @@ -23,6 +23,7 @@ from rest_framework.test import APIClient from apps.accounts.models import Role, UserProfile +from apps.hosts.crypto import decrypt_secret from apps.hosts.models import Host from . import gitsource @@ -304,6 +305,24 @@ def test_create_pull_and_no_edits_in_vigil(self): revs = self.api.get(f"/api/v1/stacks/{sid}/revisions/").json()["results"] self.assertEqual([r["git_commit"] for r in revs], [moved["stack"]["git"]["commit"], self.head]) + def test_new_settings_at_the_same_commit_still_stick_with_the_env(self): + from .models import ManagedStack + sid = self._create().json()["id"] + _run(self.repo, "branch", "release") + git = {"url": self.repo, "branch": "release", "path": "deploy/compose.yaml"} + resp = self.api.put(f"/api/v1/stacks/{sid}/", {"git": git}, format="json") + self.assertEqual(resp.status_code, 200, resp.content) + stack = ManagedStack.objects.get(pk=sid) + self.assertEqual((stack.git_branch, stack.revision), ("release", 1), "same commit: no revision") + git["pin"] = self.head + resp = self.api.put(f"/api/v1/stacks/{sid}/", { + "git": git, "env": [{"key": "TOKEN", "value": "rotated"}]}, format="json") + self.assertEqual(resp.status_code, 200, resp.content) + stack.refresh_from_db() + self.assertEqual((stack.git_pin, stack.revision), (self.head, 2)) + self.assertIn("TOKEN=rotated", decrypt_secret(bytes(stack.env_encrypted))) + self.assertEqual(stack.revisions.get(number=2).compose_yaml, COMPOSE) + def test_a_refused_compose_file_makes_no_stack(self): Path(self.repo, "deploy", "compose.yaml").write_text( "services:\n a:\n image: x\n privileged: true\n") diff --git a/server/apps/stacks/views.py b/server/apps/stacks/views.py index 1d06c16d..9185adf7 100644 --- a/server/apps/stacks/views.py +++ b/server/apps/stacks/views.py @@ -190,11 +190,25 @@ def stack_detail(request, stack_id): status=status.HTTP_400_BAD_REQUEST) if denied := git_views.credential_step_up(request): return denied + env_given = "env" in request.data or "env_text" in request.data try: git_views.apply_git_settings(stack, request.data) - git_views.pull(stack, request.user, note="Git settings changed") + if env_given: + # The .env edited alongside goes into the revision the pull makes. + stack.env_encrypted = encrypt_secret(render_env(_merge_env(stack, request.data))) + changed = git_views.pull(stack, request.user, note="Git settings changed") except (gitsource.GitSourceError, StackError) as exc: return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST) + if not changed: + # Same commit: the settings (a branch at that commit, another + # credential) still stick, and a .env edit is its own revision. + if env_given: + if error := _save(request, stack, request.data, + note=str(request.data.get("note") or "edited")): + return error + else: + stack.save(update_fields=["git_url", "git_branch", "git_pin", "git_path", + "git_credential"]) return Response(_row(stack)) if error := _save(request, stack, request.data, note=str(request.data.get("note") or "edited")): return error From 5d10934d924a75810ddcc6f6719d65e3d42c1450 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Fri, 9 Oct 2026 21:34:20 -0400 Subject: [PATCH 63/68] fix(stacks): the Git editor names a credential's server, asks TOTP to use one, and reads right in dark mode The credential form sends git_host (prefilled from the repository URL) and keeps typed fields across a kind switch; saving with a credential asks for TOTP; the commit line no longer overwrites its own commit; only the current revision is marked current; selected chips use on-accent text, which also fixes the rollout group chips; an empty status pill is hidden. --- server/apps/stacks/test_git_stack_ui.py | 6 +++ server/static/css/vigil.css | 9 ++-- server/static/js/vigil-stacks.js | 58 ++++++++++++++++++----- server/templates/pages/_stack_editor.html | 2 +- 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/server/apps/stacks/test_git_stack_ui.py b/server/apps/stacks/test_git_stack_ui.py index b3eb03b8..b377e869 100644 --- a/server/apps/stacks/test_git_stack_ui.py +++ b/server/apps/stacks/test_git_stack_ui.py @@ -40,3 +40,9 @@ def test_secret_never_enters_markup(self): def test_git_commit_shown_in_revisions(self): js = _text("static/js/vigil-stacks.js") self.assertIn("git_commit", js) + + def test_credential_names_its_server_and_attaching_asks_totp(self): + js = _text("static/js/vigil-stacks.js") + self.assertIn("git_host: gitHost", js) + self.assertIn("body.git && body.git.credential_id", js) + self.assertIn("data-sted-git-ref", _text("templates/pages/_stack_editor.html")) diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index 2b3c4c13..391ded4e 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -4036,8 +4036,8 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } .rlt-groups { display: flex; flex-wrap: wrap; gap: 6px; } .rlt-group-chip { border: 0; cursor: pointer; font-family: inherit; gap: 6px; } .rlt-group-chip-count { color: var(--text-3); font-weight: 400; } -.rlt-group-chip[aria-pressed="true"] { background: var(--lavender); color: var(--lavender-ink); } -.rlt-group-chip[aria-pressed="true"] .rlt-group-chip-count { color: var(--lavender-ink); opacity: .75; } +.rlt-group-chip[aria-pressed="true"] { background: var(--lavender); color: var(--on-accent); } +.rlt-group-chip[aria-pressed="true"] .rlt-group-chip-count { color: var(--on-accent); opacity: .75; } .rlt-group-chip-order { display: none; font-size: 10px; font-weight: 700; background: var(--on-accent); color: var(--lavender-ink); border-radius: 50%; width: 16px; height: 16px; align-items: center; justify-content: center; } @@ -4070,7 +4070,7 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } font-weight: 600; padding: 6px 16px; border-radius: 16px; background: transparent; color: var(--text-2); } .sted-source-btn:hover { color: var(--text-1); } -.sted-source-btn[aria-checked="true"] { background: var(--lavender); color: var(--lavender-ink); } +.sted-source-btn[aria-checked="true"] { background: var(--lavender); color: var(--on-accent); } .sted-git { background: var(--s1); border-radius: var(--r-lg); padding: 16px; display: flex; flex-direction: column; gap: 12px; } @@ -4083,7 +4083,7 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } .sted-git-chip { font-family: var(--font-mono); font-size: 10px; font-weight: 400; color: var(--text-3); background: var(--s2); border-radius: var(--r-sm); padding: 1px 5px; } .sted-git-kinds { display: flex; gap: 6px; margin-bottom: 12px; } -.sted-git-kinds .btn[aria-pressed="true"] { background: var(--lavender); color: var(--lavender-ink); } +.sted-git-kinds .btn[aria-pressed="true"] { background: var(--lavender); color: var(--on-accent); } #stack-compose.is-readonly { caret-color: transparent; cursor: default; } #stack-compose.is-readonly::selection { background: rgba(var(--rgb-lavender), var(--tint-a)); } @@ -4091,3 +4091,4 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } @media (max-width: 700px) { .sted-git-grid { grid-template-columns: minmax(0, 1fr); } } +.sted-status:empty { display: none; } diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index 5c09f31c..d54f4a2d 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -228,7 +228,12 @@ async function _validateStack() { clearTimeout(_stackCheckTimer); const ta = _stedEl('stack-compose'); if (!ta) return; - if (ta.closest('.sted-editor').hidden) { _setStackStatus(null); return; } + const pill = _stedEl('sted-status'); + if (ta.closest('.sted-editor').hidden) { // a new Git stack: nothing to check until it is fetched + ++_stackCheckSeq; + if (pill) { pill.className = 'sted-status'; pill.textContent = ''; } + return; + } const seq = ++_stackCheckSeq; _setStackStatus(null); let r; @@ -251,7 +256,8 @@ async function _loadStackRevisions() { const isGit = !!stackEd.stack.git; wrap.innerHTML = revs.map(r => `
r${r.number}${r.git_commit ? ` ${escHtml(r.git_commit.slice(0, 12))}` : ''}${escHtml(r.note || '')} · ${escHtml(r.created_by || '')} · ${escHtml(timeAgo(r.created_at) || '')} - ${!isGit && r.number !== stackEd.stack.revision ? `` : 'current'} + ${r.number === stackEd.stack.revision ? 'current' + : isGit ? '' : ``}
`).join(''); stackEd.revisions = revs; } catch { wrap.innerHTML = ''; } @@ -280,9 +286,9 @@ function _newGitStackBody(name, env) { } /* An existing Git stack PUTs nothing but its .env, unless the repository - settings changed — then git: { … } takes effect by pulling it. */ -function _gitSettingsBody() { - return { git: _gitFields(), note: 'Git settings changed' }; + settings changed — then git: { … } (with the .env) takes effect by pulling it. */ +function _gitSettingsBody(env) { + return { git: _gitFields(), env, note: 'Git settings changed' }; } function _applyStackSource() { @@ -314,7 +320,7 @@ function _renderGitState() { // "Commit abc123def456 · tracking main" / "· pinned to v1.2". The commit is // the span's only , so a repository string never becomes markup. state.querySelector('code').textContent = (git.commit || '').slice(0, 12); - state.lastChild.textContent = git.pin ? ` · pinned to ${git.pin}` : ` · tracking ${git.branch || 'main'}`; + state.querySelector('[data-sted-git-ref]').textContent = git.pin ? ` · pinned to ${git.pin}` : ` · tracking ${git.branch || 'main'}`; } function _fillGitFields(git) { @@ -386,7 +392,9 @@ function addGitCredential() { const username = pick('#git-cred-user').trim(); const pass = pick('#git-cred-secret'); const hosts = pick('#git-cred-hosts'); + const gitHost = pick('#git-cred-host').trim().toLowerCase(); if (!name) { showToast('Give the credential a name', 'warn'); return; } + if (!gitHost) { showToast('Name the Git server the credential is for', 'warn'); return; } if (!pass.trim()) { showToast('The secret is empty', 'warn'); return; } if (kind === 'ssh_key' && !hosts.trim()) { showToast('An SSH deploy key needs the host key', 'warn'); @@ -397,13 +405,13 @@ function addGitCredential() { try { const created = await apiJson('/api/v1/stacks/git-credentials/', { method: 'POST', - body: JSON.stringify({ name, kind, username, secret: pass, known_hosts: hosts, + body: JSON.stringify({ name, kind, git_host: gitHost, username, secret: pass, known_hosts: hosts, totp: code.trim() }), }); stackEd.gitCreds.push(created); _renderGitCredentials(); stackEd.gitSnapshot = { ...(stackEd.gitSnapshot || {}), credential_id: created.id }; - _pickCredentialOption(created.id); + _stedEl('sted-git-cred').value = created.id; showToast(`Added ${created.name}`, 'success'); } catch (e) { _stackError(e.message); @@ -411,19 +419,29 @@ function addGitCredential() { close(); // clears the secret whether the save landed or not }; + // The host the repository URL names, so the usual case needs no typing. + const urlHost = (() => { + const url = _stedEl('sted-git-url').value.trim(); + const m2 = url.match(/^https:\/\/([^/:@]+)/i) || url.match(/^[^@\s]+@([^:/\s]+):/); + return m2 ? m2[1].toLowerCase() : ''; + })(); + const render = () => { const ssh = kind === 'ssh_key'; + // Switching kind re-renders: keep the name and server already typed. + const keep = (id, fallback) => { const el = m.modal.querySelector(id); return el ? el.value : fallback; }; + const keptName = keep('#git-cred-name', ''), keptHost = keep('#git-cred-host', urlHost); // The fields a kind needs, then the one that holds its key. Field ids in // markup strings; only the kind buttons interpolate. const extra = ssh ? `
- +
The output of ssh-keyscan github.com — Vigil does not trust an unknown host key.
` : `
- +
`; @@ -444,12 +462,18 @@ function addGitCredential() {
- +
+
+ + +
The credential is only ever sent to this server.
+
${extra}
- + ${holder}
${holderHint}
@@ -463,6 +487,8 @@ function addGitCredential() { b.onclick = () => { if (kind === b.dataset.credKind) return; kind = b.dataset.credKind; render(); }; }); m.modal.querySelector('[data-cred-save]').onclick = save; + m.modal.querySelector('#git-cred-name').value = keptName; + m.modal.querySelector('#git-cred-host').value = keptHost; }; render(); @@ -499,11 +525,17 @@ async function saveStack() { const env = stackEd.env.filter(e => e.key.trim()).map(e => (e.stored && !e.value && !e.revealed) ? { key: e.key.trim(), keep: true } : { key: e.key.trim(), value: e.value || '' }); let body; - if (gitStack && gitChanged) body = _gitSettingsBody(); + if (gitStack && gitChanged) body = _gitSettingsBody(env); else if (gitStack) body = { env, note: 'edited' }; // the compose file comes from Git, not from here else if (stackEd.source === 'git') body = _newGitStackBody(_stedEl('stack-name').value.trim(), env); else body = { compose_yaml: _stedEl('stack-compose').value, env, note: 'edited' }; const wasNew = !editing; + if (body.git && body.git.credential_id) { + // Using a credential decides which server a secret goes to: the server asks for TOTP. + const code = await totpPrompt('Fetching with a Git credential'); + if (!code) return; + body.totp = code.trim(); + } try { const saved = await apiJson(editing ? `/api/v1/stacks/${editing.id}/` : '/api/v1/stacks/', { method: editing ? 'PUT' : 'POST', body: JSON.stringify(body) }); diff --git a/server/templates/pages/_stack_editor.html b/server/templates/pages/_stack_editor.html index 96db328e..a52f67b1 100644 --- a/server/templates/pages/_stack_editor.html +++ b/server/templates/pages/_stack_editor.html @@ -60,7 +60,7 @@
- +
Vigil fetches the folder that holds this compose file and ships it to the host, so From 57c90b606210fbbe8c2fb887f30453a3a89f93fd Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Sat, 10 Oct 2026 11:50:26 -0400 Subject: [PATCH 64/68] fix(rollouts): a ladder claims hosts from its own waves only, and its lane lists its own waves Dispatch and the batch split planned hosts across every ladder's waves, so a host matching two ladders' same-order waves was claimed by whichever sorted first: a Desktops-only rollout could skip a desktop tagged canary because the Servers ladder had a Canary rung too. Each ladder now plans from its own waves, as the wave serializer already counted them; the batch split still gives a host on two ladders to the first listed group. --- server/apps/tasks/rollout.py | 26 ++++++++++++------------- server/apps/tasks/test_rollout_batch.py | 19 ++++++++++++++++++ server/apps/tasks/views.py | 6 +++++- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/server/apps/tasks/rollout.py b/server/apps/tasks/rollout.py index 36076764..b678102a 100644 --- a/server/apps/tasks/rollout.py +++ b/server/apps/tasks/rollout.py @@ -50,7 +50,7 @@ def _waves_in_group(group_tag): qs = PatchWave.objects.filter(enabled=True) tag = (group_tag or "").strip() if tag: - qs = qs.filter(group_tag_rows__key=tag.lower()) + qs = qs.filter(group_tag_rows__key=tag.lower()).distinct() return qs @@ -214,20 +214,20 @@ def _batch_group_tags(group_tags) -> list[str]: def _batch_host_plan(tags: list[str], host_ids) -> "dict[str, list[str]]": """Hosts per ladder, each host on exactly one. - The plan is the whole fleet's — the same one a wave walks — so a host a - higher wave already claimed never shows up here twice. Within that, a host - tagged into two ladders goes to the **first listed** group only. + Each ladder's plan is its own — the same one its waves walk — so a host + an earlier wave of the ladder claimed never shows up twice, and a wave of + another ladder claims nothing from it. A host tagged into two ladders + goes to the **first listed** group only. """ - plan = rollout_wave_plan( - list(PatchWave.objects.filter(enabled=True).order_by("order", "id")) - ) wanted = ({str(h).strip().lower() for h in host_ids} if host_ids is not None else None) taken: set[str] = set() per_tag: dict[str, list[str]] = {} for tag in tags: + waves = list(_waves_in_group(tag).order_by("order", "id")) + plan = rollout_wave_plan(waves) hosts: list[str] = [] - for wave in _waves_in_group(tag): + for wave in waves: for host_id in plan.get(wave.id, []): host = str(host_id) low = host.lower() @@ -290,8 +290,8 @@ def start_rollout_batch( def _dispatch_wave(rollout: PatchRollout, spec: dict) -> int: """Dispatch one task per wave host and open a TaskRun for them. - Hosts come from the cross-wave plan, not the raw wave membership: a - host tagged into two waves belongs to the earliest wave only, so this + Hosts come from the ladder's cross-wave plan, not the raw wave membership: + a host tagged into two of its waves belongs to the earliest only, so this wave must not re-claim it. Must run inside a transaction that holds the rollout row lock (see ``evaluate_rollout``) — otherwise two beat ticks can both pass the "no run yet for this wave" check and dispatch @@ -301,10 +301,10 @@ def _dispatch_wave(rollout: PatchRollout, spec: dict) -> int: """ from apps.hosts.models import Host - enabled = list( - PatchWave.objects.filter(enabled=True).order_by("order", "id") + # The ladder's own plan: a wave of another ladder claims no host from it. + plan = rollout_wave_plan( + list(_waves_in_group(rollout.wave_group_tag).order_by("order", "id")) ) - plan = rollout_wave_plan(enabled) host_ids = plan.get(rollout.current_wave.id, []) if rollout.host_ids is not None: allowed = set(rollout.host_ids) diff --git a/server/apps/tasks/test_rollout_batch.py b/server/apps/tasks/test_rollout_batch.py index a4356918..e71d0f85 100644 --- a/server/apps/tasks/test_rollout_batch.py +++ b/server/apps/tasks/test_rollout_batch.py @@ -118,6 +118,25 @@ def test_a_host_on_two_ladders_goes_to_the_first_listed_group(self): self.assertEqual(hosts["Desktops"], [str(canary.id), str(desk.id)]) self.assertEqual(hosts["Servers"], [str(srv.id)]) self.assertEqual(Task.objects.filter(host=canary).count(), 1) + + def test_each_ladder_claims_hosts_from_its_own_waves_only(self): + # Each ladder has its own Canary rung matching the same tag. A wave of + # one ladder claims nothing from the other: a desktop tagged canary is + # still patched by a Desktops-only rollout, and once — not left out + # because the Servers ladder's Canary sorted first. + from apps.tasks.rollout import start_rollout + from apps.tasks.views import _rollout_wave_progress + _wave("srv-canary", 1, ["canary"], groups=["Servers"], validation_hours=0) + _wave("srv-broad", 2, ["srv"], groups=["Servers"], validation_hours=0) + _wave("desk-canary", 1, ["canary"], groups=["Desktops"], validation_hours=0) + _wave("desk-broad", 2, ["desk"], groups=["Desktops"], validation_hours=0) + desk = _host("desk-a", ["desk", "canary"]) + rollout = start_rollout(self.definition, user=self.user, wave_group_tag="Desktops") + self.assertEqual(list(Task.objects.filter(host=desk).values_list("run__wave__name", flat=True)), + ["desk-canary"]) + self.assertEqual([w["name"] for w in _rollout_wave_progress(rollout)], + ["desk-canary", "desk-broad"], "a lane shows its own ladder's waves") + def test_unknown_group_refuses_the_whole_batch(self): _wave("srv-1", 1, ["srv1"], groups=["Servers"], validation_hours=0) _host("srv-a", ["srv1"]) diff --git a/server/apps/tasks/views.py b/server/apps/tasks/views.py index 6871a297..75ec08e7 100644 --- a/server/apps/tasks/views.py +++ b/server/apps/tasks/views.py @@ -1788,7 +1788,11 @@ def _rollout_wave_progress(rollout: PatchRollout) -> list: from .models import PatchWave SUCCESS_STATES = (Task.State.COMPLETED, Task.State.SKIPPED) - waves = list(PatchWave.objects.order_by("order", "id")) + waves = PatchWave.objects.order_by("order", "id") + if (rollout.wave_group_tag or "").strip(): + # A rollout walks one ladder: show its waves, not every ladder's. + waves = waves.filter(group_tag_rows__key=rollout.wave_group_tag.strip().lower()).distinct() + waves = list(waves) per_wave: dict = {} for run in rollout.runs.all(): if run.wave is None: From b23ca82bdf09cf496640a0ef720f160eb9a61842 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Sat, 10 Oct 2026 11:50:26 -0400 Subject: [PATCH 65/68] fix(stacks): a new stack starts with blank Git fields, and the editor bar keeps its layout The Git fields kept the last stack's repository; an empty status pill collapsed and pulled Save to the left; a managed stack chip's revision sat flush against its name. --- server/static/css/vigil.css | 3 ++- server/static/js/vigil-stacks.js | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/static/css/vigil.css b/server/static/css/vigil.css index 391ded4e..4f8deb9d 100644 --- a/server/static/css/vigil.css +++ b/server/static/css/vigil.css @@ -4091,4 +4091,5 @@ button.fcn:focus-visible { outline: 2px solid var(--sky); outline-offset: 3px; } @media (max-width: 700px) { .sted-git-grid { grid-template-columns: minmax(0, 1fr); } } -.sted-status:empty { display: none; } +.sted-status:empty { visibility: hidden; } +.managed-stack-chip .apps-zero { margin-left: 4px; } diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index d54f4a2d..da3b47de 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -138,7 +138,9 @@ async function openStackEditor(stackId) { name.disabled = !!stack; _stedEl('stack-compose').value = stack ? stack.compose_yaml : _NEW_COMPOSE; _applyStackSource(); - if (stack && stack.git) _fillGitFields(stack.git); + // A new stack starts from blank Git fields, not the last stack opened. + _fillGitFields((stack && stack.git) || {}); + if (!(stack && stack.git)) stackEd.gitSnapshot = null; const err = _stedEl('stack-error'); err.textContent = ''; err.classList.remove('show'); From f99e091ebe067ff7f735182189fa296d2ee940fb Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Sat, 10 Oct 2026 15:17:56 -0400 Subject: [PATCH 66/68] =?UTF-8?q?docs(wiki):=202026.14.1=20=E2=80=94=20sta?= =?UTF-8?q?cks=20from=20Git,=20the=20stack=20editor,=20parallel=20wave=20g?= =?UTF-8?q?roups,=20sessions=20and=20the=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything the release added was missing from the wiki. Written up from the code's facts only: container states and the stack editor, stacks sourced from Git (with the guarantee stated where it lives — the host's resolved-config check, not the server's editor check), several wave groups at once, sign-in session limits, the agent/task-signing and installer hardening, the two session settings, validate_community, and the stacks/rollouts/session endpoints. --- server/apps/stacks/test_wiki_14_1.py | 86 ++++++++++++++++++++ wiki/vigil-wiki.html | 112 ++++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 server/apps/stacks/test_wiki_14_1.py diff --git a/server/apps/stacks/test_wiki_14_1.py b/server/apps/stacks/test_wiki_14_1.py new file mode 100644 index 00000000..4d348305 --- /dev/null +++ b/server/apps/stacks/test_wiki_14_1.py @@ -0,0 +1,86 @@ +"""QA-24: 2026.14.1 must be written up in the wiki. + +The section placement is checked because a subsection that landed in the wrong +section reads as a promise the feature never made, and the Git subsection's +facts are checked verbatim because they are the security story a reader relies +on ("Only the Vigil server talks to Git"). +""" +import re +from pathlib import Path + +from django.conf import settings +from django.test import SimpleTestCase + +NEW_IDS = { + "containers-states": "docker", + "containers-stack-editor": "docker", + "containers-git-stacks": "docker", + "containers-sandbox": "docker", + "parallel-groups": "deployments", + "security-sessions": "security", + "security-agents": "security", + "security-installer": "security", + "community-validate": "community", + "api-stacks": "api", +} + + +def _wiki() -> str: + return (Path(settings.BASE_DIR).parent / "wiki" / "vigil-wiki.html").read_text(encoding="utf-8") + + +def _section(text: str, name: str) -> str: + start = text.index(f'
') + return text[start:text.index(" str: + """The subsection's own markup: its

up to the next heading of any + level, or (for the last subsection of a section) the section's end.""" + start = text.index(f'

")) if e != -1] + return text[start:min(ends)] if ends else text[start:] + +class Wiki141Tests(SimpleTestCase): + def test_new_subsections_exist(self): + text = _wiki() + for anchor_id in NEW_IDS: + self.assertEqual( + text.count(f'id="{anchor_id}"'), 1, + f'{anchor_id} must appear exactly once', + ) + + def test_subsections_sit_in_their_sections(self): + text = _wiki() + for anchor_id, section in NEW_IDS.items(): + pos = text.index(f'

Sending a rollout to one group

The field tells you the blast radius before you commit, meaning how many waves the group holds and how many machines those waves actually patch. It says so plainly when no enabled wave carries the tag, rather than letting you start a rollout that gets refused.

The group is pinned when the rollout starts. Re-tagging a wave afterwards cannot reshape a rollout that is already in flight.

+

Several groups at once

+
+ + + + + + + + +
BehaviorWhat you see
Starting a rolloutThe start dialog's group chips can pick more than one wave group. The groups start together, each walking its own ladder.
Watching itThe rollout card shows one lane per group, side by side. Halt all stops every lane with one TOTP code; each lane also has its own Halt.
A machine on two laddersIt goes to the first listed group only, so the task never reaches it twice. A group left with no machines is skipped.
Which machines a wave takesEach ladder claims machines from its own waves only: a wave on another ladder never takes a machine away from this one, even when both ladders have a wave matching the same tag.
+
+

API: POST /api/v1/rollouts/ with wave_group_tags (a list, 2–12 names) instead of wave_group_tag.

+

Rolling out a playbook

A rollout can carry a single task or a whole playbook. Picking a playbook runs all of its steps on each machine, wave by wave. It is the same composition the task editor uses, so a playbook that works when deployed directly works the same way in a rollout.

One restriction: a task that needs someone to fill in an input cannot be rolled out. A rollout runs unattended across waves and has no value to supply, so it is refused when you start it rather than discovered halfway through.

@@ -2072,10 +2086,52 @@

Containers and stacks on a host's page

Roll back puts a container on the image it ran before its last update (by digest when known). A compose container is pinned with vigil-rollback.override.yaml next to its compose file, which stays untouched; a standalone one is recreated on the image. It shows rolled back until its next update clears the pin.

Stacks Vigil manages

-

New stack under Managed by Vigil stores a compose file and its .env in Vigil. Every save is a revision. The file is checked first: it must have services, and may not climb out with .., bind the engine socket or system directories, run privileged, share the host's PID namespace or add every capability. The .env is encrypted at rest and never shown unmasked without a TOTP code.

+

New stack under Managed by Vigil stores a compose file and its .env in Vigil. Every save is a revision. The file is checked as you type and again when saved, and the host checks it once more before it runs anything (see What a stack may do). The .env is encrypted at rest and never shown unmasked without a TOTP code.

Deploy (high risk, TOTP) writes the stack to /opt/vigil/stacks/<name> and runs compose up -d. The task carries the compose file but never the secrets: it carries a one-time ticket the agent redeems over its own connection for the .env, written mode 0600. A ticket works once, for its host, within 24 hours. Take down runs compose down and can delete the folder (only ever one under /opt/vigil/stacks).

Adopt takes over an external stack where it stands. The agent reads its compose file and .env into Vigil (again by ticket, never in the task result) and compares compose config --hash with what each container runs. The editor then says whether deploying the file unchanged would recreate anything. A stack with more than one compose file is refused until the override is merged in.

+

Stopped, crashed and down

+

A container that exited with code 0 shows stopped (exit 0); one that exited with any other code shows crashed (exit N) with its code, e.g. crashed (exit 137). A stack's header counts them: e.g. 1/2 running · 1 crashed (exit 137).

+

A stack Vigil manages that has no containers on the host at all — taken down, or its containers removed — still appears, marked down, with a note that it can be deployed again from Managed by Vigil.

+ +

The stack editor

+

New stack or a stack's chip opens a full-screen editor: the compose file with line numbers and highlighting on the left; the .env and the revision list on the right.

+

The file is checked as you type; a pill in the bar reads valid or N problem(s), and each problem names its line. Save revision stores a new revision; Load this file on an older revision puts it back in the editor. Deploy and Take down… ask for a TOTP code. Leaving with unsaved changes asks first. Ctrl+S saves.

+

.env values stay hidden; leave a value blank to keep the stored one. Show values asks for a TOTP code.

+ +

Stacks from Git

+

For teams that build their own images instead of publishing them: New stack → From Git takes the compose file from a repository. Repository URL (https://host/owner/repo.git or git@host:owner/repo.git), Branch (default main), Pin to a tag or commit (optional), Compose file in the repo (default compose.yaml, e.g. deploy/compose.yaml) and Credential (none for a public repository) are the fields.

+

The repository is the source of truth: the compose file is shown read-only in Vigil and changed in the repository. The .env is still kept and edited in Vigil.

+

A stack tracks its branch: each deploy fetches the branch's latest commit first and records it. A pin (a tag or a 40-character commit) holds it still. Pull latest fetches now; a new commit becomes a new revision, and each revision lists the commit it came from. Deploy to apply it.

+

Vigil ships the folder that holds the compose file to the host, so build: contexts have their source, and the host builds the images itself.

+

Only the Vigil server talks to Git. Hosts never get repository access or a credential. A deploy hands the host a one-time ticket for that revision's files plus the files' SHA-256 inside the signed task; the host refuses files whose hash differs, takes regular files only (no links), and does not follow links when writing them.

+
    +
  • Fetching is limited: https and ssh only, no redirects, no addresses on the server's own machine or link-local ranges, a size and time limit on every fetch.
    • +
  • Credentials: an HTTPS token (with a username, e.g. x-access-token) or an SSH deploy key. Each credential names the one Git server it may be sent to (e.g. github.com), is encrypted at rest and is never shown again.
    • +
  • An SSH key needs the server's host key pasted in (ssh-keyscan github.com output) — Vigil does not trust an unknown host key. Adding a credential, and using one on a stack, asks for a TOTP code.
    • +
  • The host's agent must be 2026.14.1 or newer; an older agent is refused with a message saying so.
    • +
+ +

What a stack may do

+
+ + + + + + + + + +
CheckWhat Vigil refuses
Host namespacesSharing the host's PID, IPC, user, cgroup or UTS namespace, or another container's (network_mode: host is allowed)
PrivilegesPrivileged containers; adding dangerous capabilities
DevicesDevices other than GPUs (/dev/dri, /dev/nvidia*)
Mounts and socketsBinding the engine socket or system directories
Engine settingsx-podman settings; build options that reach outside the stack
+
+

Vigil checks a compose file in two places. The server's check gives quick feedback while you edit. The guarantee is the host's check: before up, the agent has the engine resolve the file (variables, include, extends) and checks the result; then it runs that checked result, not the original text.

+

A bind mount must sit inside the stack's own folder, or in a folder where every parent is owned by root and writable only by root. So binds under a user's home directory are refused on purpose — a user who could rename a parent folder could redirect the mount.

+
+
i
+
Move the data under a root-owned path such as /srv, or inside the stack folder.
+
+

Private registries

Store a registry login at POST /api/v1/stacks/registries/ (registry, username, password, optional host_tags). The password is encrypted and write-only. Pulls through the engine send it as X-Registry-Auth, but only from hosts the login covers. Compose pulls still use the host's own docker login.

@@ -2145,6 +2201,27 @@

Security Model

Agents never open a listening port. They connect to the server; the server never connects to them. Safe behind NAT and enterprise firewalls.
+ +

Sign-in sessions

+

A sign-in ends after 15 minutes without use and after 12 hours in any case (settings VIGIL_SESSION_IDLE_MINUTES, minimum 5, and VIGIL_SESSION_MAX_HOURS, minimum 1). Only real use counts: a page load, a change, or the page's heartbeat when someone touches it — background refreshes never keep a session alive, so an open Monitor tab does not.

+

Two minutes before the end a dialog counts down: Stay signed in (idle limit only — the 12-hour cap cannot be extended) or Sign out now. After signing in again you return to the page you were on.

+ +

Agents, tokens and signed tasks

+
    +
  • Agent endpoints answer only approved hosts; a pending or rejected host can check in and nothing else.
    • +
  • Agent tokens are stored hashed on the server, so a database copy does not hand out working tokens.
    • +
  • A signed task also covers when it was dispatched and which agent it is for; the agent accepts only this signature format and remembers each task's nonce for the task's lifetime, so a task cannot be replayed or sent to another host.
    • +
  • Who may run a task is checked on every path that creates one, rollouts included, with sites and per-host permissions applied.
    • +
  • TOTP codes are limited to 5 attempts per 10 minutes per user.
    • +
+ +

What the installers lock down

+

The installer writes the server's public key into the agent's config; the agent does not learn it on first contact. Linux: the agent config is rebuilt if it is not root-owned and private. Windows: C:\ProgramData\Vigil is locked so only SYSTEM and Administrators can change it.

+

The agent runs a script only when the file and every folder above it are owned by root (Windows: SYSTEM, Administrators or TrustedInstaller) and writable by no one else. This run-time check is the guarantee; the installer only narrows the window.

+
+
i
+
These are aimed at a user who has an account on a managed machine but is not an administrator: they cannot change the agent's config, swap a script it runs, or feed a stack a folder they control.
+
@@ -2745,6 +2822,10 @@

The three content types

The slug reference rule: a playbook does not copy its tasks, it references them by slug, meaning the filename without its extension. A step with task: remove-pycharm-process-lock refers to tasks/remove-pycharm-process-lock.yaml, and an automation that runs a playbook names it by the stem of the file in playbooks/. Every reference must resolve to a file that exists in the repo; the validators resolve each slug against the filesystem and reject a file that points at nothing, because a playbook that silently does nothing is worse than a rejected file.

target: host is not supported in community automations. An automation can target event_host (the host that raised the event), tags (hosts carrying one of target_tags), or all, but never a single host by id. Host ids are not portable across installations: a file that names "host 42" is right on the author's Vigil and meaningless on yours, so the community schema does not accept it and the validator rejects it. If you need "the machines that are my web servers," that is a tag.

+

Checking content before you submit

+

python manage.py validate_community <checkout> checks a Vigil-Approved-Scripts checkout with Vigil's own parsers — the same ones the Community tab uses — so a file that passes is a file the server can read. --strict also fails tasks that parse with warnings (deprecated syntax).

+

It needs no database; references resolve against the checkout's own files. The community repository's CI runs it.

+

Submitting your work

Every editor has a Submit to Community button: the task editor's toolbar, and the Edit as YAML view for playbooks and automations. Each one targets its own directory in the catalog. Clicking it:

    @@ -2890,6 +2971,33 @@

    Diagnostics

+

Stacks, rollouts and sessions

+
+ + + + + + + + + + + + + + + + + + + + + + +
MethodPathDescription
GET/api/v1/stacks/?host={id}A host's managed stacks
POST/api/v1/stacks/Create a stack: a compose_yaml, or git: {url, branch, pin, path, credential_id}
POST/api/v1/stacks/validate/Check a compose file without saving
GET/api/v1/stacks/{id}/Stack detail
PUT/api/v1/stacks/{id}/Update a stack
DELETE/api/v1/stacks/{id}/Delete a stack
GET/api/v1/stacks/{id}/revisions/The stack's revision list
POST/api/v1/stacks/{id}/deploy/Deploy the stack (TOTP)
POST/api/v1/stacks/{id}/remove/Take the stack down (TOTP)
POST/api/v1/stacks/{id}/env/reveal/Show the stored .env values (TOTP)
POST/api/v1/stacks/{id}/pull/Fetch the repository now (Git stacks)
GET/api/v1/stacks/git-credentials/List Git credentials
POST/api/v1/stacks/git-credentials/Add a Git credential (needs TOTP; the secret is never returned)
DELETE/api/v1/stacks/git-credentials/{id}/Remove a Git credential
POST/api/v1/rollouts/Start a rollout (wave_group_tags for several groups)
POST/api/v1/rollouts/batch/{batch}/halt/Halt a rollout batch (TOTP)
GET/api/v1/accounts/session/Time left on this sign-in
POST/api/v1/accounts/session/activity/The heartbeat
+
+

Task definitions and history

@@ -4221,6 +4329,8 @@

Core

+ +
VIGIL_TIMEZONEUTCIANA timezone for display and scheduling (e.g. America/Chicago).
VIGIL_TIME_FORMAT12h12h or 24h clock display.
VIGIL_SKIP_STARTUP_CHECKSunsetSet 1 to bypass startup validation. Escape hatch for tooling/CI, not for production.
VIGIL_SESSION_IDLE_MINUTES15Minutes without use before a sign-in ends (minimum 5).
VIGIL_SESSION_MAX_HOURS12Hours after which a sign-in ends regardless of use (minimum 1).
From 5e60b0321adac10a26a847990a45ce79638ff977 Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Sat, 10 Oct 2026 15:27:45 -0400 Subject: [PATCH 67/68] docs(wiki): style the PUT method badge the API table now uses --- wiki/vigil-wiki.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/vigil-wiki.html b/wiki/vigil-wiki.html index 03e16d65..88bcf557 100644 --- a/wiki/vigil-wiki.html +++ b/wiki/vigil-wiki.html @@ -173,7 +173,7 @@ .api-method { display: inline-block; font-family: 'IBM Plex Mono', monospace; font-size: 10px; font-weight: 600; padding: 2px 8px; border-radius: 4px; letter-spacing: 0.5px; min-width: 50px; text-align: center; } .m-get { background: rgba(130,196,238,0.15); color: var(--sky-ink); } .m-post { background: rgba(126,221,181,0.15); color: var(--mint-ink); } - .m-patch { background: rgba(240,184,136,0.15); color: var(--peach-ink); } + .m-patch, .m-put { background: rgba(240,184,136,0.15); color: var(--peach-ink); } .m-delete { background: rgba(242,160,184,0.15); color: var(--rose-ink); } .steps { counter-reset: step; margin: 16px 0 20px; } From 463618dcff3bfc1e09ecde7acd9b6bb0b18eb09c Mon Sep 17 00:00:00 2001 From: Connor Haggerty Date: Sat, 10 Oct 2026 15:41:04 -0400 Subject: [PATCH 68/68] docs: no emoji in the wiki, the app, the README or the editions table Words instead: Yes / Partly in the Podman and editions tables; the warnings read as plain sentences. --- README.md | 2 +- docs/EDITIONS.md | 38 +++++++++++++-------------- server/static/js/vigil-stacks.js | 2 +- server/templates/pages/_settings.html | 2 +- server/templates/pages/_vulns.html | 1 - wiki/vigil-wiki.html | 10 +++---- 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a6bde84e..566f5632 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ python3 -c "import os, base64; print(base64.b64encode(os.urandom(32)).decode())" VIGIL_SIGNING_KEY_SEED=Hk3PtP...P0c= ``` -> ⚠️ **If you're pasting the compose stack into Portainer** (or any other UI that round-trips through YAML), set the value in the **Environment variables** tab — never inline it into the YAML. If you must inline it, **wrap it in double quotes** (`"Hk3...P0c="`) — the trailing `=` is base64 padding and unquoted YAML can strip or mangle it. A malformed seed produces `binascii.Error: Incorrect padding` and breaks every agent checkin in the fleet. +> **If you're pasting the compose stack into Portainer** (or any other UI that round-trips through YAML), set the value in the **Environment variables** tab — never inline it into the YAML. If you must inline it, **wrap it in double quotes** (`"Hk3...P0c="`) — the trailing `=` is base64 padding and unquoted YAML can strip or mangle it. A malformed seed produces `binascii.Error: Incorrect padding` and breaks every agent checkin in the fleet. **3. Verify the seed is good before bringing the stack up:** diff --git a/docs/EDITIONS.md b/docs/EDITIONS.md index de545482..d90de92a 100644 --- a/docs/EDITIONS.md +++ b/docs/EDITIONS.md @@ -20,28 +20,28 @@ with `seats: 9999`). SSO/SAML is "available — talk to us," not a tier. | Feature | Free | Business | |---|:--:|:--:| -| Monitoring, alerting (smart defaults), unlimited agents/hosts/retention | ✅ | ✅ | -| Remote task execution + agent (agent is 100% OSS, no tier awareness) | ✅ | ✅ | -| Vuln scanning (Trivy / Nessus / Greenbone) | ✅ | ✅ | -| Vigil's own vulnerability matcher (OSV / KEV / EPSS, offline bundle), evidence and fix groups | ✅ | ✅ | -| Remediation by fix (deploy a fix to every affected host), AI mitigations when there is no fix | ✅ | ✅ | -| Detection tasks (severity / cves / boost), Organization · Community · Vendor tags, Anvil SARIF import | ✅ | ✅ | -| Containers & stacks (Docker / Podman): lifecycle, logs, rollback, single-host stack editor and .env, adopt | ✅ | ✅ | -| Audit trail of stack and .env events (viewer) | — | ✅ | -| **Baselines** (auto-dispatch on host approval) | ✅ | ✅ | -| **AI suggestions** — BYO endpoint, any OpenAI-compatible or Anthropic | ✅ | ✅ | -| **Status page** (public token URL, Powered-by-Vigil badge) | ✅ | ✅ | -| Jackil integration (alert → ticket) | ✅ | ✅ | -| **App & patch policies** (present / latest / pinned / absent, patching windows, deferral, approve each change) | ✅ | ✅ | -| Fleet view by update, fleet-wide approve / decline | ✅ | ✅ | -| Patch compliance numbers (% of hosts patched within SLA) | ✅ | ✅ | -| **Compliance report** — pass/fail per site, CSV export, branded printable report with "Licensed to" | — | ✅ | +| Monitoring, alerting (smart defaults), unlimited agents/hosts/retention | Yes | Yes | +| Remote task execution + agent (agent is 100% OSS, no tier awareness) | Yes | Yes | +| Vuln scanning (Trivy / Nessus / Greenbone) | Yes | Yes | +| Vigil's own vulnerability matcher (OSV / KEV / EPSS, offline bundle), evidence and fix groups | Yes | Yes | +| Remediation by fix (deploy a fix to every affected host), AI mitigations when there is no fix | Yes | Yes | +| Detection tasks (severity / cves / boost), Organization · Community · Vendor tags, Anvil SARIF import | Yes | Yes | +| Containers & stacks (Docker / Podman): lifecycle, logs, rollback, single-host stack editor and .env, adopt | Yes | Yes | +| Audit trail of stack and .env events (viewer) | — | Yes | +| **Baselines** (auto-dispatch on host approval) | Yes | Yes | +| **AI suggestions** — BYO endpoint, any OpenAI-compatible or Anthropic | Yes | Yes | +| **Status page** (public token URL, Powered-by-Vigil badge) | Yes | Yes | +| Jackil integration (alert → ticket) | Yes | Yes | +| **App & patch policies** (present / latest / pinned / absent, patching windows, deferral, approve each change) | Yes | Yes | +| Fleet view by update, fleet-wide approve / decline | Yes | Yes | +| Patch compliance numbers (% of hosts patched within SLA) | Yes | Yes | +| **Compliance report** — pass/fail per site, CSV export, branded printable report with "Licensed to" | — | Yes | | Seats | 1 admin + 1 viewer | **per-seat pricing** | | Roles | Admin, Viewer | + **Operator**, custom roles | | **Sites** (administrative boundaries; per-site scoping) | 1 (default) | **unlimited** | -| **Audit log** viewer + CSV export (recording is always on) | — | ✅ | -| **Status page branding** (logo, no badge, per-site client pages) | — | ✅ | -| **Branding** (reports, portal) | — | ✅ | +| **Audit log** viewer + CSV export (recording is always on) | — | Yes | +| **Status page branding** (logo, no badge, per-site client pages) | — | Yes | +| **Branding** (reports, portal) | — | Yes | | SSO/SAML | — | talk to us | **Seats are the only meter.** Sites are unlimited on Business because seats diff --git a/server/static/js/vigil-stacks.js b/server/static/js/vigil-stacks.js index da3b47de..d19aebca 100644 --- a/server/static/js/vigil-stacks.js +++ b/server/static/js/vigil-stacks.js @@ -216,7 +216,7 @@ function _renderStackCheck(r) { + (r.line ? `line ${r.line}` : '') + ''; } (r.missing_env || []).forEach((v) => { - html += `
⚠ ${escHtml(v)} is used but not set ` + html += `
${escHtml(v)} is used but not set ` + `
`; }); diff --git a/server/templates/pages/_settings.html b/server/templates/pages/_settings.html index eb00f4c2..188f7247 100644 --- a/server/templates/pages/_settings.html +++ b/server/templates/pages/_settings.html @@ -113,7 +113,7 @@

Add a provider

Civil SSO

Opt in to one account across every SQSY app (Vigil, Jackil, Sigil, Turnstil) via a Civil identity server. Local login always keeps working; when Civil is down, existing sessions are unaffected. Leave disabled for a standalone install.

- +
diff --git a/server/templates/pages/_vulns.html b/server/templates/pages/_vulns.html index 7a0a52ed..95eeb316 100644 --- a/server/templates/pages/_vulns.html +++ b/server/templates/pages/_vulns.html @@ -83,7 +83,6 @@